diff --git a/.gitignore b/.gitignore index a9cf842413..fdd04c9adf 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,14 @@ CMakeSettings.json #'Tags' file. tags + +# Out-of-source build/install trees (this repo builds into ./build-*/ and installs into ./install-*/) +/build-*/ +/install-*/ +# Editor / CLI artifacts +/.vscode/ +/vscode_cli.tar.gz + +# Slurm submission scripts (local scheduler jobs, not part of the source tree) +*.sbatch +*.slurm diff --git a/CMakeLists.txt b/CMakeLists.txt index fe0f4b0c16..ad2e6a358f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,11 +11,11 @@ endif() list(INSERT CMAKE_MODULE_PATH 0 "${CMAKE_CURRENT_LIST_DIR}/cmake") set(MGARD_VERSION_MAJOR "1") -set(MGARD_VERSION_MINOR "6") +set(MGARD_VERSION_MINOR "7") set(MGARD_VERSION_PATCH "0") set(MGARD_FILE_VERSION_MAJOR "1") -set(MGARD_FILE_VERSION_MINOR "0") +set(MGARD_FILE_VERSION_MINOR "7") set(MGARD_FILE_VERSION_PATCH "0") project( @@ -37,9 +37,8 @@ option(MGARD_ENABLE_CUDA "Enable CUDA support" OFF) option(MGARD_ENABLE_SERIAL "Enable SERIAL support" ON) option(MGARD_ENABLE_HIP "Enable HIP support" OFF) option(MGARD_ENABLE_SYCL "Enable SYCL support" OFF) -option(MGARD_ENABLE_LEGACY_CUDA "Enable legacy CUDA support" OFF) option(MGARD_ENABLE_AUTO_TUNING "Enable auto tuning" OFF) -option(MGARD_ENABLE_MDR "Enable MDR" OFF) +option(MGARD_ENABLE_MDR "Enable MDR" ON) option(MGARD_ENABLE_EXTERNAL_COMPRESSOR "Enable external compressor" OFF) option(MGARD_ENABLE_CLI "Build executable." OFF) @@ -140,6 +139,22 @@ if (MGARD_ENABLE_SERIAL OR target_include_directories(mdr-x PUBLIC $ $) install(TARGETS mdr-x) + add_executable(mdr-x-qoi ${CMAKE_CURRENT_SOURCE_DIR}/src/mgard-x/Executables/mdr-x-qoi.cpp) + target_link_libraries(mdr-x-qoi mgard-library ${CMAKE_DL_LIBS}) + target_include_directories(mdr-x-qoi PUBLIC $ + $) + install(TARGETS mdr-x-qoi) + find_package(MPI REQUIRED) + add_executable(pmdr-x-qoi ${CMAKE_CURRENT_SOURCE_DIR}/src/mgard-x/Executables/pmdr-x-qoi.cpp) + target_link_libraries(pmdr-x-qoi mgard-library MPI::MPI_CXX ${CMAKE_DL_LIBS}) + target_include_directories(pmdr-x-qoi PUBLIC $ + $) + install(TARGETS pmdr-x-qoi) + add_executable(pmdr-x-qoi-io ${CMAKE_CURRENT_SOURCE_DIR}/src/mgard-x/Executables/pmdr-x-qoi-io.cpp) + target_link_libraries(pmdr-x-qoi-io mgard-library MPI::MPI_CXX ${CMAKE_DL_LIBS}) + target_include_directories(pmdr-x-qoi-io PUBLIC $ + $) + install(TARGETS pmdr-x-qoi-io) endif() endif() @@ -163,7 +178,6 @@ if(MGARD_ENABLE_CUDA) set(CMAKE_CUDA_ARCHITECTURES 70) endif() - find_package(nvcomp REQUIRED) find_package(CUDAToolkit REQUIRED) target_compile_definitions(mgard-library PUBLIC MGARD_ENABLE_CUDA) @@ -175,34 +189,11 @@ if(MGARD_ENABLE_CUDA) set_source_files_properties(${MGARD_X_CUDA_SRC} PROPERTIES LANGUAGE CUDA) endif() -if (MGARD_ENABLE_LEGACY_CUDA) - enable_language(CUDA) - set(CMAKE_CUDA_STANDARD_REQUIRED TRUE) - if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) - set(CMAKE_CUDA_ARCHITECTURES 70) - endif() - - find_package(nvcomp REQUIRED) - find_package(CUDAToolkit REQUIRED) - - add_subdirectory (src/cuda) - add_subdirectory (include/cuda) - add_executable(mgard-gpu ${CMAKE_CURRENT_SOURCE_DIR}/src/cuda/MgardCudaExec.cpp) - target_link_libraries(mgard-gpu mgard-library ${CMAKE_DL_LIBS}) - target_include_directories(mgard-gpu PUBLIC $ - $) - - target_compile_definitions(mgard-library PUBLIC MGARD_ENABLE_LEGACY_CUDA) - set (CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-relaxed-constexpr -w") - - set (MGARD_X_SEPARATE_COMPILE_COMPILATION_OPTION - CUDA_SEPARABLE_COMPILATION ON) - - install(TARGETS mgard-gpu) -endif() - -if (MGARD_ENABLE_HIP) - find_package(HIP REQUIRED) +if (MGARD_ENABLE_HIP) + # __syncwarp(), used by SubGroup::sync() for wavefront-level + # synchronization, is opt-in before ROCm 7.0 and default-on from 7.0 + # onward -- require 7.0 so it needs no HIP_ENABLE_WARP_SYNC_BUILTINS define. + find_package(HIP 7.0 REQUIRED) enable_language(HIP) target_compile_definitions(mgard-library PUBLIC MGARD_ENABLE_HIP) set (CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -w") @@ -234,13 +225,7 @@ find_package(ZLIB REQUIRED) find_package(PkgConfig REQUIRED) -set( - MGARD_LIBRARY_CPP - src/compress.cpp - src/compress_internal.cpp - src/compressors.cpp - src/format.cpp -) +add_subdirectory(src/mgard) set(MAXIMUM_DIMENSION 4 CACHE STRING "Maximum supported dimension for self-describing decompression.") @@ -366,11 +351,7 @@ if(ZSTD_FOUND) endif() if(MGARD_ENABLE_CUDA) - target_link_libraries(mgard-library PUBLIC nvcomp::nvcomp CUDA::cudart) -endif() - -if(MGARD_ENABLE_LEGACY_CUDA) - target_link_libraries(mgard-library PUBLIC nvcomp::nvcomp CUDA::cudart) + target_link_libraries(mgard-library PUBLIC CUDA::cudart) endif() if(MGARD_ENABLE_OPENMP) @@ -406,7 +387,7 @@ install(DIRECTORY "include/" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/mgard") install(FILES "${MGARD_FILE_FORMAT_HPP}" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/mgard/${MGARD_FILE_FORMAT_INCLUDESUBDIR_COMPONENT}") if(MGARD_ENABLE_UNSTRUCTURED) - add_subdirectory("src/unstructured") + add_subdirectory("src/mgard/unstructured") endif() if(MGARD_ENABLE_BENCHMARKS) @@ -464,7 +445,7 @@ install(EXPORT mgard-targets NAMESPACE mgard:: ) if(MGARD_ENABLE_CLI) - add_subdirectory("src/cli") + add_subdirectory("src/mgard/cli") endif() message(STATUS "") @@ -503,12 +484,6 @@ if (MGARD_ENABLE_HIP) endif() message(STATUS "SYCL: ${MGARD_ENABLE_SYCL}") list(POP_BACK CMAKE_MESSAGE_INDENT) -message(STATUS "LEGACY CUDA: ${MGARD_ENABLE_LEGACY_CUDA}") -if (MGARD_ENABLE_LEGACY_CUDA) - list(APPEND CMAKE_MESSAGE_INDENT " ") - message(STATUS "CUDA Arch: ${CMAKE_CUDA_ARCHITECTURES}") - list(POP_BACK CMAKE_MESSAGE_INDENT) -endif() message(STATUS "auto tuning: ${MGARD_ENABLE_AUTO_TUNING}") message(STATUS "MDR: ${MGARD_ENABLE_MDR}") message(STATUS "timing: ${MGARD_ENABLE_TIMING}") diff --git a/LICENSE b/LICENSE index 0c1683e72b..70663057e5 100644 --- a/LICENSE +++ b/LICENSE @@ -175,7 +175,8 @@ END OF TERMS AND CONDITIONS - Copyright 2019 Brown University + Copyright 2019-2026 Oak Ridge National Laboratory + Copyright 2019-2026 Brown University Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index ee1a4a343d..12e0db6aba 100644 --- a/README.md +++ b/README.md @@ -13,83 +13,99 @@ We encourage you to [make a GitHub issue][issue form] if you run into any proble [format workflow badge]: https://github.com/CODARcode/MGARD/actions/workflows/format.yml/badge.svg [issue form]: https://github.com/CODARcode/MGARD/issues/new/choose -[](./doc/images/MGARD-familytree.png) +[](./doc/images/MGARD-family2.png) MGARD framework consists of the following modules. Please see the detailed instructions for each module to build and install MGARD. -## `MGARD-CPU`: MGARD implementation for CPUs -MGARD-CPU is design for running compression on CPUs. See detailed user guide in [here][mgard-cpu] +## ***MGARD-CPU***: MGARD compression implementation for CPUs +*MGARD-CPU* is designed for running compression on CPUs. See the detailed user guide [here][mgard-cpu]. In addition, *MGARD-CPU* can be configured to preserve region-of-interest ([RoI user guide][mgard-roi]) and linear quantity-of-interest ([QoI user guide][mgard-qoi]) during data compression. [mgard-cpu]: doc/MGARD-CPU.md - -## `MGARD-CUDA`: CUDA accelerated compression -MGARD-CUDA is designed for accelerating compression specifically using NVIDIA GPUs. See detailed user guide in [here][gpu instructions]. - -[gpu instructions]: doc/MGARD-GPU.md - -## `MGARD-X`: Accelerated and portable compression -MGARD-X is designed for portable compression on NVIDIA GPUs, AMD GPUs, and CPUs. See detailed user guide in [here][mgard_x instructions]. - -[mgard_x instructions]: doc/MGARD-X.md - -## `MGARD-DR`/`MGARD-XDR`: Fine-grain progressive data reconstruction -MGARD-DR and MGARD-XDR are designed for enabling fine-grain data refactoring and progressive data reconstruction. See detailed user guide in [here][mdr_x instructions]. - -[mdr_x instructions]: doc/MDR-X.md - -## `MGARD-ROI`: Preserving Region-of-Interest -MGARD-ROI is designed for preserving region-of-interest during data compression. See detailed user guide in [here][mgard-roi]. - [mgard-roi]: doc/MGARD-RoI.md +[mgard-qoi]: doc/MGARD-QoI.md -## `MGARD-QOI`: Preserving Linear Quantity-of-Interest -MGARD-QOI is designed for preserving linear quantity-of-interest during data compression. See detailed user guide in [here][mgard-qoi]. +## ***MGARD-X***: Accelerated and portable compression +*MGARD-X* is designed for portable compression on NVIDIA GPUs, AMD GPUs, and CPUs. See the detailed user guide [here][mgard_x instructions]. +In addition, *MGARD-X* can be configured to preserve region-of-interest ([ROI user guide][mgard-x-roi]) and linear quantity-of-interest ([QoI user guide][mgard-qoi]) during data compression. +[mgard_x instructions]: doc/MGARD-X.md +[mgard-x-roi]: doc/MGARD-RoI2.md [mgard-qoi]: doc/MGARD-QoI.md -## `MGARD-Lambda`: Preserving Non-Linear Quantity-of-Interest -MGARD-Lambda is designed for preserving non-linear quantity-of-interest during data compression. This is an experimental part of MGARD. Currently only support certain QoIs derived from XGC 5D data. See theory in [here][mgard-lambda-theory] and example in [here][mgard-lambda]. +## ***MGARD***$\text{-}\lambda$: Preserving Non-Linear Quantity-of-Interest +*MGARD*$\text{-}\lambda$ is specifically designed for preserving non-linear quantity-of-interest during data compression. This is an experimental part of MGARD. Currently, it only supports certain QoIs derived from XGC 5D data. See theory in [here][mgard-lambda-theory] and example in [here][mgard-lambda]. [mgard-lambda-theory]: doc/images/post-processing.pdf [mgard-lambda]: ./examples/lambda +## ***MDR/MDR-X***: Fine-grain progressive data reconstruction +*MDR* and *MDR-X* are designed to enable fine-grain data refactoring and progressive data reconstruction. See the detailed user guide [here][mdr_x instructions]. + +[mdr_x instructions]: doc/MDR-X.md + ## Self-describing format for compressed and refactored data Data produced by MGARD, MGARD-X, and MDR-X are designed to follow a unified self-describing format. See format details in [here][mgard format]. [mgard format]: doc/MGARD-format.md -## Publications +## Version history +Detailed release notes (features added, changes, and bug fixes) are linked below for each version. +* [MGARD 1.7.0](doc/release%20notes/1.7.0.md) (Sep. 2026) — Delivered ***HP-MDR***, a high-performance MDR-X refactoring and reconstruction pipeline; introduced ***BlockMGARD***, a block-based hybrid hierarchy compression pipeline with region-of-interest support; deprecated and removed the legacy standalone MGARD-CUDA backend in favor of ***MGARD-X***; added new rANS and BlockDelta lossless backends and a portable warp-cooperative LZ4 implementation; added Blackwell (sm_120) GPU build support; numerous performance improvements and bug fixes. +* [MGARD 1.6.0](doc/release%20notes/1.6.0.md) (Aug. 2025) — Redesigned the compression/decompression pipeline for higher end-to-end throughput; improved OpenMP and Huffman CPU performance and ZSTD linking; removed the prefetch option (now always enabled) and the coordinate-normalization build option; fixed issues with LZ4 compression, thread safety, MDR-X L2 error control, ADIOS2 integration, and HIP builds. +* [MGARD 1.5.2](doc/release%20notes/1.5.2.md) (Sep. 2023) — Added compression status reporting to the high-level API, an ADIOS2 operator build example, autotuning for Huffman kernels, asynchronous LZ4/Zstd compression, a pipeline optimized for compressing time-series data, and improved memory-usage estimation; fixed bugs in Huffman codebook generation, the domain decomposer, reduced-memory-footprint mode, and MDR-X reconstruction. +* [MGARD 1.5.0](doc/release%20notes/1.5.0.md) (Apr. 2023) — Added the ***MGARD***$\text{-}\lambda$ pipeline for preserving non-linear QoIs in XGC data and the ***MGARD-RoI*** pipeline for region-of-interest preservation; added a GPU pipeline for out-of-core, large-scale compression; added Apple Silicon (ARM) support; fixed issues with CUDA (older versions and 12+), the NVIDIA HPC SDK, MDR-X compilation, and linear quantization overflow. +* [MGARD 1.4.0](doc/release%20notes/1.4.0.md) (Jan. 2023) — Added multi-device support for compression/decompression, RuntimeX, and Array; added workspace pre-allocation, a new OpenMP backend, block-based domain decomposition, and high-level MDR-X APIs; modularized the compression and refactoring workflows; reduced build time via optional autotuning; fixed bugs affecting GCC 9, Huffman encoding synchronization, Xcode, and the SYCL backend. +* [MGARD 1.3.0](doc/release%20notes/1.3.0.md) (Sep. 2022) — Introduced ***MGARD-X***: portable compression for CPU (serial and multi-threaded), NVIDIA GPUs, AMD GPUs, and Intel GPUs, with a self-describing format, automatic domain decomposition, multi-GPU parallel compression, and high-/low-level APIs. Introduced ***MDR-X*** for portable multi-precision data refactoring on CPU and GPU. +* [MGARD 1.0.0](doc/release%20notes/1.0.0.md) (Sep. 2021) — Improved CPU compression/decompression speed (iterator optimizations, index precomputation, memory-access-pattern improvements); added OpenMP parallelization; added a self-describing command-line executable and high-level APIs; added support pluggable lossless compressors; fixed several multilevel-decomposition bugs. +* [MGARD 0.1.0](doc/release%20notes/0.1.0.md) (Sep. 2020) — Added initial support for unstructured data; restructured code for extensibility; added Nvidia GPU support for 2D/3D; added Huffman entropy encoding and ZSTD integration; added FP64 support; added continuous integration (Travis CI). +* [MGARD 0.0.0.2](doc/release%20notes/0.0.0.2.md) (Sep. 2019) — Initial public release. Lossy compression with preservation of $L_\infty$, $L_2$ and S-norm on primary data, and linear QoIs; added FP32 support. -### Fundamental Theory -* Xin Liang et al. [MGARD+: Optimizing Multilevel Methods for Error-bounded Scientific Data Reduction.][mgard+] *IEEE Transactions on Computers*, 2021 +## Publications +The following works either contribute to the MGARD framework and/or extend and apply MGARD for various applications, systems, and use cases. +### MGARD Foundations +* Qian Gong et al. [MGARD: A multigrid framework for high-performance, error-controlled data compression and refactoring.][mgard-softwarex] *SoftwareX*, Dec. 2023 +* Xin Liang et al. [MGARD+: Optimizing Multilevel Methods for Error-bounded Scientific Data Reduction.][mgard+] *IEEE TC*, 2021 * Mark Ainsworth et al. [Multilevel Techniques for Compression and Reduction of Scientific Data—The Unstructured Case.][unstructured] *SIAM Journal on Scientific Computing*, 42 (2), A1402–A1427, 2020. * Mark Ainsworth et al. [Multilevel Techniques for Compression and Reduction of Scientific Data—Quantitative Control of Accuracy in Derived Quantities.][quantities] *SIAM Journal on Scientific Computing* 41 (4), A2146–A2171, 2019. * Mark Ainsworth et al. [Multilevel Techniques for Compression and Reduction of Scientific Data—The Multivariate Case.][multivariate] *SIAM Journal on Scientific Computing* 41 (2), A1278–A1303, 2019. * Mark Ainsworth et al. [Multilevel Techniques for Compression and Reduction of Scientific Data—The Univariate Case.][univariate] *Computing and Visualization in Science* 19, 65–76, 2018. * Ben Whitney. [Multilevel Techniques for Compression and Reduction of Scientific Data.][thesis] PhD thesis, Brown University, 2018. -### Preserving Quantites of Interest (QoIs) -* Xuan Wu et al. [Error-controlled Progressive Retrieval of Scientific Data under Derivable Quantities of Interest.] [qoi] **the International Conference for High Performance Computing, Networking, Storage and Analysis 2024*, Nov, 2024* +### Preserving Quantities of Interest (QoIs) +* Qian Gong et al. [Physics-Aware Adaptive Checkpointing with Shadow Systems for Nonlinear PDE Simulations.][shadow-pde] *Journal of Computational Science*, Sep. 2026 +* Jaemoon Lee et al. [Error-Guaranteed Compression with Preservation of Downstream Quantities for Electron Microscopy.][em-qoi] *Microscopy and Microanalysis*, Aug. 2026 +* Qian Gong et al. [Stability-preserving Lossy Compression for Large-scale Partial Differential Equations.][stability-pde] *ACM/IEEE SC25*, Nov. 2025 +* Richard Dodson et al. [Optimising the Processing and Storage of Visibilities using lossy compression.][visibilities] *Publications of the Astronomical Society of Australia*, Jul. 2025 +* Qian Gong et al. [A General Framework for Error-controlled Unstructured Scientific Data Compression.][unstructured-framework] *2024 e-Science*, Sep. 2024 +* Tania Banerjee et al. [Fast Algorithms for Scientific Data Compression.][fast-algo] *HiPC*, Dec. 2023 +* Qian Gong et al. [Spatiotemporally adaptive compression for scientific dataset with feature preservation–a case study on simulation data with extreme climate events analysis.][climate-qoi] *2023 e-Science*, Oct. 2023 +* Tania Banerjee et al. [Online and Scalable Data Compression Pipeline with Guarantees on Quantities of Interest.][online-qoi] *IEEE e-Science*, Oct. 2023 * Tania Banerjee et al. [Scalable Hybrid Learning Techniques for Scientific Data Compression.][pp3], *Arxiv*, 2022 -* Qian Gong et al. [Region-adaptive, Error-controlled Scientific Data Compression using Multilevel Decomposition.][roi2] *the 34th International Conference on Scientific and Statistical Database Management*, Jul. 2022 -* Tania Benerjee et al. An algorithmic and software pipeline for very large scale scientific data compression with error guarantees. *International Conference on High Performance Computing, Data, and Analytics*, 2022 +* Qian Gong et al. [Region-adaptive, Error-controlled Scientific Data Compression using Multilevel Decomposition.][roi2] *ACM SSDBM*, Jul. 2022 +* Tania Benerjee et al. [An algorithmic and software pipeline for very large-scale scientific data compression with error guarantees.][qoi2] *HiPC*, 2022 * Jaemoon Lee et al. [Error-bounded learned scientific data compression with preservation of derived quantities.][pp] *Applied Sciences*, 2022 -* Qian Gong et al. [Maintaining trust in reduction: Preserving the accuracy of quantities of interest for lossy compression.][roi] *21st Smoky Mountains Computational Sciences and Engineering Conference*, Oct. 2021 +* Qian Gong et al. [Maintaining trust in reduction: Preserving the accuracy of quantities of interest for lossy compression.][roi] * Smoky Mountains Conference*, Oct. 2021 -### Pregressive Retrieval -* Jinzheng Wang et al. Improving Progressive Retrieval for HPC Scientific Data using Deep Neural Network. *IEEE International Conference on Data Engineering (ICDE)*, 2023 -* Xin Liang et al. [Error-controlled, progressive, and adaptable retrieval of scientific data with multilevel decomposition.][mdr] *the International Conference for High Performance Computing, Networking, Storage and Analysis 2021*, Nov, 2021 +### Progressive Retrieval +* Wenbo Li et al. [QProR: An Efficient Framework for Quantity-of-Interest Based Progressive Retrieval with Guaranteed Error Control.][qpror] *ACM HPDC*, Jul. 2026 +* Yanliang Li et al. [HP-MDR: High-performance and Portable Data Refactoring and Progressive Retrieval with Advanced GPUs.][hp-mdr] *ACM/IEEE SC25*, Nov 2025 +* Xuan Wu et al. [Error-controlled Progressive Retrieval of Scientific Data under Derivable Quantities of Interest.][qoi] *ACM/IEEE SC24*, Nov. 2024 +* Jinzheng Wang et al. [Improving Progressive Retrieval for HPC Scientific Data using Deep Neural Network.][progressive-dnn] *ICDE*, 2023 +* Xin Liang et al. [Error-controlled, progressive, and adaptable retrieval of scientific data with multilevel decomposition.][mdr] *ACM/IEEE SC21*, Nov. 2021 ### Parallelization and GPU Acceleration -* Jieyang Chen et al. [HPDR: High-Performance Portable Scientific Data Reduction Framework.][gpu3] 39th IEEE International Parallel and Distributed Processing Symposium, June 3-7, 2025 +* Yanliang Li et al. [BlockMGARD: Accelerating Adaptive Scientific Data Reduction with Region-of-Interest Error Control on GPUs.][gpu4] *ACM/IEEE SC26*, Nov. 2026 +* Jieyang Chen et al. [HPDR: High-Performance Portable Scientific Data Reduction Framework.][gpu3] *IEEE IPDPS*, June. 2025 * Jieyang Chen et al. [Scalable Multigrid-based Hierarchical Scientific Data Refactoring on GPUs.][gpu2] *Arxiv* -* Jieyang Chen et al. [Accelerating Multigrid-based Hierarchical Scientific Data Refactoring on GPUs.][gpu] *35th IEEE International Parallel & Distributed Processing Symposium*, May 17–21, 2021. +* Jieyang Chen et al. [Accelerating Multigrid-based Hierarchical Scientific Data Refactoring on GPUs.][gpu] *IEEE IPDPS*, May. 2021. ### System Optimizations -* Lipeng Wan et al. RAPIDS: Reconciling Availability, Accuracy, and Performance in Managing Geo-Distributed Scientific Data. *the International ACM Symposium on High-Performance Parallel and Distributed Computing*, Jun. 2023 -* Xinying Wang et al. [Unbalanced Parallel I/O: An Often-Neglected Side Effect of Lossy Scientific Data Compression.][unbalanced-io] *7th International Workshop on Data Analysis and Reduction for Big Scientific Data*, Nov. 2021 +* Vladislav Esaulov et al. [JANUS: Resilient and Adaptive Data Transmission for Enabling Timely and Efficient Cross-Facility Scientific Workflows.][janus] *ACM HPDC*, Jun. 2025 +* Lipeng Wan et al. [RAPIDS: Reconciling Availability, Accuracy, and Performance in Managing Geo-Distributed Scientific Data.][rapids] *ACM HPDC*, Jun. 2023 +* Xinying Wang et al. [Unbalanced Parallel I/O: An Often-Neglected Side Effect of Lossy Scientific Data Compression.][unbalanced-io] *DRBSD-7*, Nov. 2021 [thesis]: https://doi.org/10.26300/ya1v-hn97 +[mgard-softwarex]: https://doi.org/10.1016/j.softx.2023.101590 [univariate]: https://doi.org/10.1007/s00791-018-00303-9 [multivariate]: https://doi.org/10.1137/18M1166651 [quantities]: https://doi.org/10.1137/18M1208885 @@ -97,15 +113,29 @@ Data produced by MGARD, MGARD-X, and MDR-X are designed to follow a unified self [gpu]: https://ieeexplore.ieee.org/abstract/document/9460526/ [gpu2]: https://arxiv.org/abs/2105.12764 [gpu3]: https://ieeexplore.ieee.org/document/11078565 +[gpu4]: https://arxiv.org/abs/2609.00205 [mgard+]: https://ieeexplore.ieee.org/abstract/document/9479913/ [unbalanced-io]: https://ieeexplore.ieee.org/abstract/document/9652573/ [mdr]: https://dl.acm.org/doi/abs/10.1145/3458817.3476179 +[hp-mdr]: https://dl.acm.org/doi/10.1145/3712285.3759845 [roi]: https://link.springer.com/chapter/10.1007/978-3-030-96498-6_2 [roi2]: https://dl.acm.org/doi/abs/10.1145/3538712.3538717 [pp]: https://www.mdpi.com/1709018 [pp3]: https://arxiv.org/abs/2212.10733 [qoi]: https://ieeexplore.ieee.org/abstract/document/10793162 - +[rapids]: https://dl.acm.org/doi/10.1145/3588195.3592983 +[progressive-dnn]: https://ieeexplore.ieee.org/document/10184595/ +[qoi2]: https://ieeexplore.ieee.org/document/10106324 +[em-qoi]: https://doi.org/10.1093/mam/ozag084 +[stability-pde]: https://dl.acm.org/doi/10.1145/3712285.3759878 +[shadow-pde]: https://doi.org/10.1016/j.jocs.2026.102986 +[janus]: https://arxiv.org/abs/2506.17084 +[visibilities]: https://doi.org/10.1017/pasa.2025.29 +[qpror]: https://doi.org/10.1145/3806645.3807579 +[online-qoi]: https://ieeexplore.ieee.org/document/10254934/ +[climate-qoi]: https://arxiv.org/abs/2401.03317 +[fast-algo]: https://doi.org/10.1109/HiPC58850.2023.00030 +[unstructured-framework]: https://ieeexplore.ieee.org/document/10678699/ diff --git a/build_scripts/README.md b/build_scripts/README.md index 9364e1a661..c28adaef9b 100644 --- a/build_scripts/README.md +++ b/build_scripts/README.md @@ -18,6 +18,7 @@ The build scripts provided here are example scripts for building MGARD on system #### For AMD GPUs: * `build_mgard_hip_frontier.sh` builds MGARD for AMD MI-250X GPUs on the Frontier supercomputer at OLCF +* `build_mgard_hip_mi300.sh` builds MGARD for AMD MI-300A GPUs (e.g. the `odyssey` node on OACISS) #### For Intel GPUs: * `build_mgard_sycl_gen9.sh` builds MGARD for Intel Gen9 integrated GPUs diff --git a/build_scripts/build_mgard_adios2_cuda_summit.sh b/build_scripts/build_mgard_adios2_cuda_summit.sh index 9d90230244..e4dafcfbc7 100755 --- a/build_scripts/build_mgard_adios2_cuda_summit.sh +++ b/build_scripts/build_mgard_adios2_cuda_summit.sh @@ -29,27 +29,13 @@ export CC=gcc export CXX=g++ export CUDACXX=nvcc -#build NVCOMP -nvcomp_dir=${build_dir}/nvcomp -nvcomp_src_dir=${nvcomp_dir}/src -nvcomp_build_dir=${nvcomp_dir}/build -nvcomp_install_dir=${install_dir} -if [ ! -d "${nvcomp_src_dir}" ]; then - git clone -b v2.2.0 https://github.com/NVIDIA/nvcomp.git ${nvcomp_src_dir} -fi -mkdir -p ${nvcomp_build_dir} -cmake -S ${nvcomp_src_dir} -B ${nvcomp_build_dir}\ - -DCMAKE_INSTALL_PREFIX=${nvcomp_install_dir} -cmake --build ${nvcomp_build_dir} -j ${num_build_procs} -cmake --install ${nvcomp_build_dir} - #build ZSTD zstd_dir=${build_dir}/zstd zstd_src_dir=${zstd_dir}/src zstd_build_dir=${zstd_dir}/build zstd_install_dir=${install_dir} if [ ! -d "${zstd_src_dir}" ]; then - git clone -b v1.5.0 https://github.com/facebook/zstd.git ${zstd_src_dir} + git clone -b v1.5.6 https://github.com/facebook/zstd.git ${zstd_src_dir} fi mkdir -p ${zstd_build_dir} cmake -S ${zstd_src_dir}/build/cmake -B ${zstd_build_dir}\ @@ -65,7 +51,7 @@ protobuf_src_dir=${protobuf_dir}/src protobuf_build_dir=${protobuf_dir}/build protobuf_install_dir=${install_dir} if [ ! -d "${protobuf_src_dir}" ]; then - git clone -b v3.19.4 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} + git clone -b v3.21.12 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} fi mkdir -p ${protobuf_build_dir} cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ @@ -75,16 +61,32 @@ cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ cmake --build ${protobuf_build_dir} -j ${num_build_procs} cmake --install ${protobuf_build_dir} +#build Catch2 +catch2_dir=${build_dir}/catch2 +catch2_src_dir=${catch2_dir}/src +catch2_build_dir=${catch2_dir}/build +catch2_install_dir=${install_dir} +if [ ! -d "${catch2_src_dir}" ]; then + git clone -b v3.3.2 https://github.com/catchorg/Catch2.git ${catch2_src_dir} +fi +mkdir -p ${catch2_build_dir} +cmake -S ${catch2_src_dir} -B ${catch2_build_dir}\ + -DBUILD_TESTING=OFF\ + -DCMAKE_INSTALL_PREFIX=${catch2_install_dir} +cmake --build ${catch2_build_dir} -j ${num_build_procs} +cmake --install ${catch2_build_dir} > /dev/null 2>&1 + #build MGARD mgard_x_build_dir=${build_dir}/mgard mgard_x_install_dir=${install_dir} mkdir -p ${mgard_x_build_dir} cmake -S ${mgard_x_src_dir} -B ${mgard_x_build_dir} \ - -DCMAKE_PREFIX_PATH="${nvcomp_install_dir};${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir}"\ + -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir};${catch2_install_dir}"\ -DMGARD_ENABLE_CUDA=ON\ -DCMAKE_CUDA_ARCHITECTURES="70"\ -DMGARD_ENABLE_DOCS=OFF\ -DCMAKE_BUILD_TYPE=Release\ + -DBUILD_TESTING=ON\ -DCMAKE_INSTALL_PREFIX=${mgard_x_install_dir} cmake --build ${mgard_x_build_dir} -j ${num_build_procs} cmake --install ${mgard_x_build_dir} diff --git a/build_scripts/build_mgard_adios2_hip_frontier.sh b/build_scripts/build_mgard_adios2_hip_frontier.sh index da5cf3599f..f6397e8984 100755 --- a/build_scripts/build_mgard_adios2_hip_frontier.sh +++ b/build_scripts/build_mgard_adios2_hip_frontier.sh @@ -33,7 +33,7 @@ zstd_src_dir=${zstd_dir}/src zstd_build_dir=${zstd_dir}/build zstd_install_dir=${install_dir} if [ ! -d "${zstd_src_dir}" ]; then - git clone -b v1.5.0 https://github.com/facebook/zstd.git ${zstd_src_dir} + git clone -b v1.5.6 https://github.com/facebook/zstd.git ${zstd_src_dir} fi mkdir -p ${zstd_build_dir} cmake -S ${zstd_src_dir}/build/cmake -B ${zstd_build_dir}\ @@ -50,7 +50,7 @@ protobuf_src_dir=${protobuf_dir}/src protobuf_build_dir=${protobuf_dir}/build protobuf_install_dir=${install_dir} if [ ! -d "${protobuf_src_dir}" ]; then - git clone -b v3.19.4 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} + git clone -b v3.21.12 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} fi mkdir -p ${protobuf_build_dir} cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ @@ -61,15 +61,31 @@ cmake --build ${protobuf_build_dir} -j ${num_build_procs} cmake --install ${protobuf_build_dir} +#build Catch2 +catch2_dir=${build_dir}/catch2 +catch2_src_dir=${catch2_dir}/src +catch2_build_dir=${catch2_dir}/build +catch2_install_dir=${install_dir} +if [ ! -d "${catch2_src_dir}" ]; then + git clone -b v3.3.2 https://github.com/catchorg/Catch2.git ${catch2_src_dir} +fi +mkdir -p ${catch2_build_dir} +cmake -S ${catch2_src_dir} -B ${catch2_build_dir}\ + -DBUILD_TESTING=OFF\ + -DCMAKE_INSTALL_PREFIX=${catch2_install_dir} +cmake --build ${catch2_build_dir} -j ${num_build_procs} +cmake --install ${catch2_build_dir} > /dev/null 2>&1 + #build MGARD mgard_x_build_dir=${build_dir}/mgard mgard_x_install_dir=${install_dir} mkdir -p ${mgard_x_build_dir} cmake -S ${mgard_x_src_dir} -B ${mgard_x_build_dir} \ - -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir}"\ + -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir};${catch2_install_dir}"\ -DMGARD_ENABLE_HIP=ON\ -DCMAKE_HIP_ARCHITECTURES="gfx90a"\ -DCMAKE_BUILD_TYPE=Release\ + -DBUILD_TESTING=ON\ -DCMAKE_INSTALL_PREFIX=${mgard_x_install_dir} cmake --build ${mgard_x_build_dir} -j ${num_build_procs} cmake --install ${mgard_x_build_dir} diff --git a/build_scripts/build_mgard_apple_silicon.sh b/build_scripts/build_mgard_apple_silicon.sh index bb73d625e3..86a1202713 100755 --- a/build_scripts/build_mgard_apple_silicon.sh +++ b/build_scripts/build_mgard_apple_silicon.sh @@ -25,7 +25,7 @@ zstd_src_dir=${zstd_dir}/src zstd_build_dir=${zstd_dir}/build zstd_install_dir=${install_dir} if [ ! -d "${zstd_src_dir}" ]; then - git clone -b v1.5.0 https://github.com/facebook/zstd.git ${zstd_src_dir} + git clone -b v1.5.6 https://github.com/facebook/zstd.git ${zstd_src_dir} fi mkdir -p ${zstd_build_dir} cmake -S ${zstd_src_dir}/build/cmake -B ${zstd_build_dir}\ @@ -41,7 +41,7 @@ protobuf_src_dir=${protobuf_dir}/src protobuf_build_dir=${protobuf_dir}/build protobuf_install_dir=${install_dir} if [ ! -d "${protobuf_src_dir}" ]; then - git clone -b v3.19.4 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} + git clone -b v3.21.12 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} fi mkdir -p ${protobuf_build_dir} cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ @@ -51,19 +51,34 @@ cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ cmake --build ${protobuf_build_dir} -j ${num_build_procs} cmake --install ${protobuf_build_dir} +#build Catch2 +catch2_dir=${build_dir}/catch2 +catch2_src_dir=${catch2_dir}/src +catch2_build_dir=${catch2_dir}/build +catch2_install_dir=${install_dir} +if [ ! -d "${catch2_src_dir}" ]; then + git clone -b v3.3.2 https://github.com/catchorg/Catch2.git ${catch2_src_dir} +fi +mkdir -p ${catch2_build_dir} +cmake -S ${catch2_src_dir} -B ${catch2_build_dir}\ + -DBUILD_TESTING=OFF\ + -DCMAKE_INSTALL_PREFIX=${catch2_install_dir} +cmake --build ${catch2_build_dir} -j ${num_build_procs} +cmake --install ${catch2_build_dir} > /dev/null 2>&1 + #build MGARD mgard_x_build_dir=${build_dir}/mgard mgard_x_install_dir=${install_dir} mkdir -p ${mgard_x_build_dir} cmake -S ${mgard_x_src_dir} -B ${mgard_x_build_dir} \ - -DCMAKE_PREFIX_PATH="${nvcomp_install_dir};${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir}"\ - -DMGARD_ENABLE_SERIAL=ON\ + -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir};${catch2_install_dir}"\ -DMGARD_ENABLE_OPENMP=ON\ -DOpenMP_CXX_FLAGS="-Xclang -fopenmp -I/opt/homebrew/opt/libomp/include"\ -DOpenMP_CXX_LIB_NAMES="libomp"\ -DOpenMP_libomp_LIBRARY="omp"\ -DMGARD_ENABLE_DOCS=OFF\ -DCMAKE_BUILD_TYPE=Release\ + -DBUILD_TESTING=ON\ -DCMAKE_INSTALL_PREFIX=${mgard_x_install_dir} cmake --build ${mgard_x_build_dir} -j ${num_build_procs} cmake --install ${mgard_x_build_dir} diff --git a/build_scripts/build_mgard_cuda_ampere.sh b/build_scripts/build_mgard_cuda_ampere.sh index e042adb36d..34d396ad0a 100755 --- a/build_scripts/build_mgard_cuda_ampere.sh +++ b/build_scripts/build_mgard_cuda_ampere.sh @@ -25,27 +25,13 @@ export CC=gcc export CXX=g++ export CUDACXX=nvcc -#build NVCOMP -nvcomp_dir=${build_dir}/nvcomp -nvcomp_src_dir=${nvcomp_dir}/src -nvcomp_build_dir=${nvcomp_dir}/build -nvcomp_install_dir=${install_dir} -if [ ! -d "${nvcomp_src_dir}" ]; then - git clone -b v2.2.0 https://github.com/NVIDIA/nvcomp.git ${nvcomp_src_dir} -fi -mkdir -p ${nvcomp_build_dir} -cmake -S ${nvcomp_src_dir} -B ${nvcomp_build_dir}\ - -DCMAKE_INSTALL_PREFIX=${nvcomp_install_dir} -cmake --build ${nvcomp_build_dir} -j ${num_build_procs} -cmake --install ${nvcomp_build_dir} - #build ZSTD zstd_dir=${build_dir}/zstd zstd_src_dir=${zstd_dir}/src zstd_build_dir=${zstd_dir}/build zstd_install_dir=${install_dir} if [ ! -d "${zstd_src_dir}" ]; then - git clone -b v1.5.0 https://github.com/facebook/zstd.git ${zstd_src_dir} + git clone -b v1.5.6 https://github.com/facebook/zstd.git ${zstd_src_dir} fi mkdir -p ${zstd_build_dir} cmake -S ${zstd_src_dir}/build/cmake -B ${zstd_build_dir}\ @@ -61,7 +47,7 @@ protobuf_src_dir=${protobuf_dir}/src protobuf_build_dir=${protobuf_dir}/build protobuf_install_dir=${install_dir} if [ ! -d "${protobuf_src_dir}" ]; then - git clone -b v3.19.4 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} + git clone -b v3.21.12 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} fi mkdir -p ${protobuf_build_dir} cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ @@ -72,16 +58,34 @@ cmake --build ${protobuf_build_dir} -j ${num_build_procs} cmake --install ${protobuf_build_dir} +#build Catch2 +catch2_dir=${build_dir}/catch2 +catch2_src_dir=${catch2_dir}/src +catch2_build_dir=${catch2_dir}/build +catch2_install_dir=${install_dir} +if [ ! -d "${catch2_src_dir}" ]; then + git clone -b v3.3.2 https://github.com/catchorg/Catch2.git ${catch2_src_dir} +fi +mkdir -p ${catch2_build_dir} +cmake -S ${catch2_src_dir} -B ${catch2_build_dir}\ + -DBUILD_TESTING=OFF\ + -DCMAKE_INSTALL_PREFIX=${catch2_install_dir} +cmake --build ${catch2_build_dir} -j ${num_build_procs} +cmake --install ${catch2_build_dir} > /dev/null 2>&1 + #build MGARD mgard_x_build_dir=${build_dir}/mgard mgard_x_install_dir=${install_dir} mkdir -p ${mgard_x_build_dir} cmake -S ${mgard_x_src_dir} -B ${mgard_x_build_dir} \ - -DCMAKE_PREFIX_PATH="${nvcomp_install_dir};${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir}"\ + -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir};${catch2_install_dir}"\ -DMGARD_ENABLE_CUDA=ON\ + -DMGARD_ENABLE_OPENMP=OFF\ + -DMGARD_ENABLE_MDR=ON\ -DCMAKE_CUDA_ARCHITECTURES="80"\ -DMGARD_ENABLE_DOCS=OFF\ -DCMAKE_BUILD_TYPE=Release\ + -DBUILD_TESTING=ON\ -DCMAKE_INSTALL_PREFIX=${mgard_x_install_dir} time cmake --build ${mgard_x_build_dir} -j ${num_build_procs} cmake --install ${mgard_x_build_dir} diff --git a/build_scripts/build_mgard_cuda_andes.sh b/build_scripts/build_mgard_cuda_andes.sh index 2d01288c72..4bc31f1f20 100755 --- a/build_scripts/build_mgard_cuda_andes.sh +++ b/build_scripts/build_mgard_cuda_andes.sh @@ -29,27 +29,13 @@ export CC=gcc export CXX=g++ export CUDACXX=nvcc -#build NVCOMP -nvcomp_dir=${build_dir}/nvcomp -nvcomp_src_dir=${nvcomp_dir}/src -nvcomp_build_dir=${nvcomp_dir}/build -nvcomp_install_dir=${install_dir} -if [ ! -d "${nvcomp_src_dir}" ]; then - git clone -b v2.2.0 https://github.com/NVIDIA/nvcomp.git ${nvcomp_src_dir} -fi -mkdir -p ${nvcomp_build_dir} -cmake -S ${nvcomp_src_dir} -B ${nvcomp_build_dir}\ - -DCMAKE_INSTALL_PREFIX=${nvcomp_install_dir} -cmake --build ${nvcomp_build_dir} -j ${num_build_procs} -cmake --install ${nvcomp_build_dir} - #build ZSTD zstd_dir=${build_dir}/zstd zstd_src_dir=${zstd_dir}/src zstd_build_dir=${zstd_dir}/build zstd_install_dir=${install_dir} if [ ! -d "${zstd_src_dir}" ]; then - git clone -b v1.5.0 https://github.com/facebook/zstd.git ${zstd_src_dir} + git clone -b v1.5.6 https://github.com/facebook/zstd.git ${zstd_src_dir} fi mkdir -p ${zstd_build_dir} cmake -S ${zstd_src_dir}/build/cmake -B ${zstd_build_dir}\ @@ -65,7 +51,7 @@ protobuf_src_dir=${protobuf_dir}/src protobuf_build_dir=${protobuf_dir}/build protobuf_install_dir=${install_dir} if [ ! -d "${protobuf_src_dir}" ]; then - git clone -b v3.19.4 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} + git clone -b v3.21.12 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} fi mkdir -p ${protobuf_build_dir} cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ @@ -75,16 +61,34 @@ cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ cmake --build ${protobuf_build_dir} -j ${num_build_procs} cmake --install ${protobuf_build_dir} +#build Catch2 +catch2_dir=${build_dir}/catch2 +catch2_src_dir=${catch2_dir}/src +catch2_build_dir=${catch2_dir}/build +catch2_install_dir=${install_dir} +if [ ! -d "${catch2_src_dir}" ]; then + git clone -b v3.3.2 https://github.com/catchorg/Catch2.git ${catch2_src_dir} +fi +mkdir -p ${catch2_build_dir} +cmake -S ${catch2_src_dir} -B ${catch2_build_dir}\ + -DBUILD_TESTING=OFF\ + -DCMAKE_INSTALL_PREFIX=${catch2_install_dir} +cmake --build ${catch2_build_dir} -j ${num_build_procs} +cmake --install ${catch2_build_dir} > /dev/null 2>&1 + #build MGARD mgard_x_build_dir=${build_dir}/mgard mgard_x_install_dir=${install_dir} mkdir -p ${mgard_x_build_dir} cmake -S ${mgard_x_src_dir} -B ${mgard_x_build_dir} \ - -DCMAKE_PREFIX_PATH="${nvcomp_install_dir};${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir}"\ + -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir};${catch2_install_dir}"\ -DMGARD_ENABLE_CUDA=ON\ + -DMGARD_ENABLE_OPENMP=OFF\ + -DMGARD_ENABLE_MDR=ON\ -DCMAKE_CUDA_ARCHITECTURES="37"\ -DMGARD_ENABLE_DOCS=OFF\ -DCMAKE_BUILD_TYPE=Release\ + -DBUILD_TESTING=ON\ -DCMAKE_INSTALL_PREFIX=${mgard_x_install_dir} cmake --build ${mgard_x_build_dir} -j ${num_build_procs} cmake --install ${mgard_x_build_dir} diff --git a/build_scripts/build_mgard_cuda_blackwell.sh b/build_scripts/build_mgard_cuda_blackwell.sh new file mode 100755 index 0000000000..1ec46f760e --- /dev/null +++ b/build_scripts/build_mgard_cuda_blackwell.sh @@ -0,0 +1,98 @@ +#!/bin/sh + +# Copyright 2021, Oak Ridge National Laboratory. +# MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs +# Author: Jieyang Chen (chenj3@ornl.gov) +# Date: April 2, 2021 +# Script for building MGARD-X + +set -e +#set -x + +######## User Configurations ######## +# Source directory +mgard_x_src_dir=. +# Build directory +build_dir=./build-cuda-blackwell +# Number of processors used for building +num_build_procs=$1 +# Installtaion directory +install_dir=./install-cuda-blackwell + +export PATH=/usr/local/cuda/bin:$PATH +export LD_LIBRARY_PATH=$(pwd)/${install_dir}/lib:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$(pwd)/${install_dir}/lib64:$LD_LIBRARY_PATH +export CC=gcc +export CXX=g++ +export CUDACXX=nvcc + +# NVCOMP is not built here: mgard-x's CUDA backend (MGARD_ENABLE_CUDA) +# does not link against it. + +#build ZSTD +zstd_dir=${build_dir}/zstd +zstd_src_dir=${zstd_dir}/src +zstd_build_dir=${zstd_dir}/build +zstd_install_dir=${install_dir} +if [ ! -d "${zstd_src_dir}" ]; then + git clone -b v1.5.6 https://github.com/facebook/zstd.git ${zstd_src_dir} +fi +mkdir -p ${zstd_build_dir} +cmake -S ${zstd_src_dir}/build/cmake -B ${zstd_build_dir}\ + -DZSTD_MULTITHREAD_SUPPORT=ON\ + -DCMAKE_INSTALL_LIBDIR=lib\ + -DCMAKE_INSTALL_PREFIX=${zstd_install_dir} +cmake --build ${zstd_build_dir} -j ${num_build_procs} +cmake --install ${zstd_build_dir} + +#build Protobuf +protobuf_dir=${build_dir}/protobuf +protobuf_src_dir=${protobuf_dir}/src +protobuf_build_dir=${protobuf_dir}/build +protobuf_install_dir=${install_dir} +if [ ! -d "${protobuf_src_dir}" ]; then + git clone -b v3.21.12 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} +fi +mkdir -p ${protobuf_build_dir} +cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ + -Dprotobuf_BUILD_SHARED_LIBS=ON\ + -Dprotobuf_BUILD_TESTS=OFF\ + -DCMAKE_INSTALL_PREFIX=${protobuf_install_dir} +cmake --build ${protobuf_build_dir} -j ${num_build_procs} +cmake --install ${protobuf_build_dir} > /dev/null 2>&1 + + +#build Catch2 +catch2_dir=${build_dir}/catch2 +catch2_src_dir=${catch2_dir}/src +catch2_build_dir=${catch2_dir}/build +catch2_install_dir=${install_dir} +if [ ! -d "${catch2_src_dir}" ]; then + git clone -b v3.3.2 https://github.com/catchorg/Catch2.git ${catch2_src_dir} +fi +mkdir -p ${catch2_build_dir} +cmake -S ${catch2_src_dir} -B ${catch2_build_dir}\ + -DBUILD_TESTING=OFF\ + -DCMAKE_INSTALL_PREFIX=${catch2_install_dir} +cmake --build ${catch2_build_dir} -j ${num_build_procs} +cmake --install ${catch2_build_dir} > /dev/null 2>&1 + +#build MGARD +# MDR is off: it pulls in find_package(MPI REQUIRED) and this node has no MPI installed. +mgard_x_build_dir=${build_dir}/mgard +mgard_x_install_dir=${install_dir} +mkdir -p ${mgard_x_build_dir} +cmake -S ${mgard_x_src_dir} -B ${mgard_x_build_dir} \ + -DCMAKE_PREFIX_PATH="${zstd_install_dir};${protobuf_install_dir};${catch2_install_dir}"\ + -DMGARD_ENABLE_CUDA=ON\ + -DMGARD_ENABLE_OPENMP=OFF\ + -DMGARD_ENABLE_MDR=OFF\ + -DCMAKE_CUDA_ARCHITECTURES="120"\ + -DMGARD_ENABLE_AUTO_TUNING=OFF\ + -DMGARD_ENABLE_EXTERNAL_COMPRESSOR=OFF\ + -DMGARD_ENABLE_DOCS=OFF\ + -DCMAKE_BUILD_TYPE=Release\ + -DBUILD_TESTING=ON\ + -DCMAKE_INSTALL_PREFIX=${mgard_x_install_dir} +time cmake --build ${mgard_x_build_dir} -j ${num_build_procs} +cmake --install ${mgard_x_build_dir} > /dev/null 2>&1 diff --git a/build_scripts/build_mgard_cuda_legacy.sh b/build_scripts/build_mgard_cuda_hopper.sh similarity index 59% rename from build_scripts/build_mgard_cuda_legacy.sh rename to build_scripts/build_mgard_cuda_hopper.sh index f4e9767b77..4f076b9851 100755 --- a/build_scripts/build_mgard_cuda_legacy.sh +++ b/build_scripts/build_mgard_cuda_hopper.sh @@ -7,17 +7,17 @@ # Script for building MGARD-X set -e -set -x +#set -x ######## User Configurations ######## # Source directory mgard_x_src_dir=. # Build directory -build_dir=./build-cuda-turing +build_dir=./build-cuda-hopper # Number of processors used for building num_build_procs=$1 # Installtaion directory -install_dir=./install-cuda-turing +install_dir=./install-cuda-hopper export LD_LIBRARY_PATH=$(pwd)/${install_dir}/lib:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$(pwd)/${install_dir}/lib64:$LD_LIBRARY_PATH @@ -25,27 +25,13 @@ export CC=gcc export CXX=g++ export CUDACXX=nvcc -#build NVCOMP -nvcomp_dir=${build_dir}/nvcomp -nvcomp_src_dir=${nvcomp_dir}/src -nvcomp_build_dir=${nvcomp_dir}/build -nvcomp_install_dir=${install_dir} -if [ ! -d "${nvcomp_src_dir}" ]; then - git clone -b v2.2.0 https://github.com/NVIDIA/nvcomp.git ${nvcomp_src_dir} -fi -mkdir -p ${nvcomp_build_dir} -cmake -S ${nvcomp_src_dir} -B ${nvcomp_build_dir}\ - -DCMAKE_INSTALL_PREFIX=${nvcomp_install_dir} -cmake --build ${nvcomp_build_dir} -j ${num_build_procs} -cmake --install ${nvcomp_build_dir} - #build ZSTD zstd_dir=${build_dir}/zstd zstd_src_dir=${zstd_dir}/src zstd_build_dir=${zstd_dir}/build zstd_install_dir=${install_dir} if [ ! -d "${zstd_src_dir}" ]; then - git clone -b v1.5.0 https://github.com/facebook/zstd.git ${zstd_src_dir} + git clone -b v1.5.6 https://github.com/facebook/zstd.git ${zstd_src_dir} fi mkdir -p ${zstd_build_dir} cmake -S ${zstd_src_dir}/build/cmake -B ${zstd_build_dir}\ @@ -61,7 +47,7 @@ protobuf_src_dir=${protobuf_dir}/src protobuf_build_dir=${protobuf_dir}/build protobuf_install_dir=${install_dir} if [ ! -d "${protobuf_src_dir}" ]; then - git clone -b v3.19.4 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} + git clone -b v3.21.12 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} fi mkdir -p ${protobuf_build_dir} cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ @@ -69,20 +55,39 @@ cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ -Dprotobuf_BUILD_TESTS=OFF\ -DCMAKE_INSTALL_PREFIX=${protobuf_install_dir} cmake --build ${protobuf_build_dir} -j ${num_build_procs} -cmake --install ${protobuf_build_dir} +cmake --install ${protobuf_build_dir} > /dev/null 2>&1 +#build Catch2 +catch2_dir=${build_dir}/catch2 +catch2_src_dir=${catch2_dir}/src +catch2_build_dir=${catch2_dir}/build +catch2_install_dir=${install_dir} +if [ ! -d "${catch2_src_dir}" ]; then + git clone -b v3.3.2 https://github.com/catchorg/Catch2.git ${catch2_src_dir} +fi +mkdir -p ${catch2_build_dir} +cmake -S ${catch2_src_dir} -B ${catch2_build_dir}\ + -DBUILD_TESTING=OFF\ + -DCMAKE_INSTALL_PREFIX=${catch2_install_dir} +cmake --build ${catch2_build_dir} -j ${num_build_procs} +cmake --install ${catch2_build_dir} > /dev/null 2>&1 + #build MGARD mgard_x_build_dir=${build_dir}/mgard mgard_x_install_dir=${install_dir} mkdir -p ${mgard_x_build_dir} cmake -S ${mgard_x_src_dir} -B ${mgard_x_build_dir} \ - -DCMAKE_PREFIX_PATH="${nvcomp_install_dir};${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir}"\ - -DMGARD_ENABLE_SERIAL=OFF\ - -DMGARD_ENABLE_LEGACY_CUDA=ON\ - -DCMAKE_CUDA_ARCHITECTURES="75"\ + -DCMAKE_PREFIX_PATH="${zstd_install_dir};${protobuf_install_dir};${catch2_install_dir}"\ + -DMGARD_ENABLE_CUDA=ON\ + -DMGARD_ENABLE_OPENMP=OFF\ + -DMGARD_ENABLE_MDR=ON\ + -DCMAKE_CUDA_ARCHITECTURES="90"\ + -DMGARD_ENABLE_AUTO_TUNING=OFF\ + -DMGARD_ENABLE_EXTERNAL_COMPRESSOR=OFF\ -DMGARD_ENABLE_DOCS=OFF\ -DCMAKE_BUILD_TYPE=Release\ + -DBUILD_TESTING=ON\ -DCMAKE_INSTALL_PREFIX=${mgard_x_install_dir} -cmake --build ${mgard_x_build_dir} -j ${num_build_procs} -cmake --install ${mgard_x_build_dir} +time cmake --build ${mgard_x_build_dir} -j ${num_build_procs} +cmake --install ${mgard_x_build_dir} > /dev/null 2>&1 diff --git a/build_scripts/build_mgard_cuda_summit.sh b/build_scripts/build_mgard_cuda_summit.sh index e67d76070e..645b7971e6 100755 --- a/build_scripts/build_mgard_cuda_summit.sh +++ b/build_scripts/build_mgard_cuda_summit.sh @@ -29,27 +29,13 @@ export CC=gcc export CXX=g++ export CUDACXX=nvcc -#build NVCOMP -nvcomp_dir=${build_dir}/nvcomp -nvcomp_src_dir=${nvcomp_dir}/src -nvcomp_build_dir=${nvcomp_dir}/build -nvcomp_install_dir=${install_dir} -if [ ! -d "${nvcomp_src_dir}" ]; then - git clone -b v2.2.0 https://github.com/NVIDIA/nvcomp.git ${nvcomp_src_dir} -fi -mkdir -p ${nvcomp_build_dir} -cmake -S ${nvcomp_src_dir} -B ${nvcomp_build_dir}\ - -DCMAKE_INSTALL_PREFIX=${nvcomp_install_dir} -cmake --build ${nvcomp_build_dir} -j ${num_build_procs} -cmake --install ${nvcomp_build_dir} - #build ZSTD zstd_dir=${build_dir}/zstd zstd_src_dir=${zstd_dir}/src zstd_build_dir=${zstd_dir}/build zstd_install_dir=${install_dir} if [ ! -d "${zstd_src_dir}" ]; then - git clone -b v1.5.0 https://github.com/facebook/zstd.git ${zstd_src_dir} + git clone -b v1.5.6 https://github.com/facebook/zstd.git ${zstd_src_dir} fi mkdir -p ${zstd_build_dir} cmake -S ${zstd_src_dir}/build/cmake -B ${zstd_build_dir}\ @@ -65,7 +51,7 @@ protobuf_src_dir=${protobuf_dir}/src protobuf_build_dir=${protobuf_dir}/build protobuf_install_dir=${install_dir} if [ ! -d "${protobuf_src_dir}" ]; then - git clone -b v3.19.4 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} + git clone -b v3.21.12 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} fi mkdir -p ${protobuf_build_dir} cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ @@ -75,16 +61,34 @@ cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ cmake --build ${protobuf_build_dir} -j ${num_build_procs} cmake --install ${protobuf_build_dir} +#build Catch2 +catch2_dir=${build_dir}/catch2 +catch2_src_dir=${catch2_dir}/src +catch2_build_dir=${catch2_dir}/build +catch2_install_dir=${install_dir} +if [ ! -d "${catch2_src_dir}" ]; then + git clone -b v3.3.2 https://github.com/catchorg/Catch2.git ${catch2_src_dir} +fi +mkdir -p ${catch2_build_dir} +cmake -S ${catch2_src_dir} -B ${catch2_build_dir}\ + -DBUILD_TESTING=OFF\ + -DCMAKE_INSTALL_PREFIX=${catch2_install_dir} +cmake --build ${catch2_build_dir} -j ${num_build_procs} +cmake --install ${catch2_build_dir} > /dev/null 2>&1 + #build MGARD mgard_x_build_dir=${build_dir}/mgard mgard_x_install_dir=${install_dir} mkdir -p ${mgard_x_build_dir} cmake -S ${mgard_x_src_dir} -B ${mgard_x_build_dir} \ - -DCMAKE_PREFIX_PATH="${nvcomp_install_dir};${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir}"\ + -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir};${catch2_install_dir}"\ -DMGARD_ENABLE_CUDA=ON\ + -DMGARD_ENABLE_OPENMP=OFF\ + -DMGARD_ENABLE_MDR=ON\ -DCMAKE_CUDA_ARCHITECTURES="70"\ -DMGARD_ENABLE_DOCS=OFF\ -DCMAKE_BUILD_TYPE=Release\ + -DBUILD_TESTING=ON\ -DCMAKE_INSTALL_PREFIX=${mgard_x_install_dir} cmake --build ${mgard_x_build_dir} -j ${num_build_procs} cmake --install ${mgard_x_build_dir} diff --git a/build_scripts/build_mgard_cuda_turing.sh b/build_scripts/build_mgard_cuda_turing.sh index eeb1c10ded..91bca25b3f 100755 --- a/build_scripts/build_mgard_cuda_turing.sh +++ b/build_scripts/build_mgard_cuda_turing.sh @@ -25,27 +25,13 @@ export CC=gcc export CXX=g++ export CUDACXX=nvcc -#build NVCOMP -nvcomp_dir=${build_dir}/nvcomp -nvcomp_src_dir=${nvcomp_dir}/src -nvcomp_build_dir=${nvcomp_dir}/build -nvcomp_install_dir=${install_dir} -if [ ! -d "${nvcomp_src_dir}" ]; then - git clone -b v2.2.0 https://github.com/NVIDIA/nvcomp.git ${nvcomp_src_dir} -fi -mkdir -p ${nvcomp_build_dir} -cmake -S ${nvcomp_src_dir} -B ${nvcomp_build_dir}\ - -DCMAKE_INSTALL_PREFIX=${nvcomp_install_dir} -cmake --build ${nvcomp_build_dir} -j ${num_build_procs} -cmake --install ${nvcomp_build_dir} - #build ZSTD zstd_dir=${build_dir}/zstd zstd_src_dir=${zstd_dir}/src zstd_build_dir=${zstd_dir}/build zstd_install_dir=${install_dir} if [ ! -d "${zstd_src_dir}" ]; then - git clone -b v1.5.0 https://github.com/facebook/zstd.git ${zstd_src_dir} + git clone -b v1.5.6 https://github.com/facebook/zstd.git ${zstd_src_dir} fi mkdir -p ${zstd_build_dir} cmake -S ${zstd_src_dir}/build/cmake -B ${zstd_build_dir}\ @@ -61,7 +47,7 @@ protobuf_src_dir=${protobuf_dir}/src protobuf_build_dir=${protobuf_dir}/build protobuf_install_dir=${install_dir} if [ ! -d "${protobuf_src_dir}" ]; then - git clone -b v3.19.4 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} + git clone -b v3.21.12 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} fi mkdir -p ${protobuf_build_dir} cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ @@ -72,16 +58,34 @@ cmake --build ${protobuf_build_dir} -j ${num_build_procs} cmake --install ${protobuf_build_dir} +#build Catch2 +catch2_dir=${build_dir}/catch2 +catch2_src_dir=${catch2_dir}/src +catch2_build_dir=${catch2_dir}/build +catch2_install_dir=${install_dir} +if [ ! -d "${catch2_src_dir}" ]; then + git clone -b v3.3.2 https://github.com/catchorg/Catch2.git ${catch2_src_dir} +fi +mkdir -p ${catch2_build_dir} +cmake -S ${catch2_src_dir} -B ${catch2_build_dir}\ + -DBUILD_TESTING=OFF\ + -DCMAKE_INSTALL_PREFIX=${catch2_install_dir} +cmake --build ${catch2_build_dir} -j ${num_build_procs} +cmake --install ${catch2_build_dir} > /dev/null 2>&1 + #build MGARD mgard_x_build_dir=${build_dir}/mgard mgard_x_install_dir=${install_dir} mkdir -p ${mgard_x_build_dir} cmake -S ${mgard_x_src_dir} -B ${mgard_x_build_dir} \ - -DCMAKE_PREFIX_PATH="${nvcomp_install_dir};${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir}"\ + -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir};${catch2_install_dir}"\ -DMGARD_ENABLE_CUDA=ON\ + -DMGARD_ENABLE_OPENMP=OFF\ + -DMGARD_ENABLE_MDR=ON\ -DCMAKE_CUDA_ARCHITECTURES="75"\ -DMGARD_ENABLE_DOCS=OFF\ -DCMAKE_BUILD_TYPE=Release\ + -DBUILD_TESTING=ON\ -DCMAKE_INSTALL_PREFIX=${mgard_x_install_dir} time cmake --build ${mgard_x_build_dir} -j ${num_build_procs} cmake --install ${mgard_x_build_dir} diff --git a/build_scripts/build_mgard_hip_frontier.sh b/build_scripts/build_mgard_hip_frontier.sh index 4cb0e87ebb..cd2ca11ec2 100755 --- a/build_scripts/build_mgard_hip_frontier.sh +++ b/build_scripts/build_mgard_hip_frontier.sh @@ -9,7 +9,7 @@ set -e set -x -module load rocm/6.3.1 +module load rocm/7.2.0 module load cmake ######## User Configurations ######## @@ -33,7 +33,7 @@ zstd_src_dir=${zstd_dir}/src zstd_build_dir=${zstd_dir}/build zstd_install_dir=${install_dir} if [ ! -d "${zstd_src_dir}" ]; then - git clone -b v1.5.0 https://github.com/facebook/zstd.git ${zstd_src_dir} + git clone -b v1.5.6 https://github.com/facebook/zstd.git ${zstd_src_dir} fi mkdir -p ${zstd_build_dir} cmake -S ${zstd_src_dir}/build/cmake -B ${zstd_build_dir}\ @@ -50,7 +50,7 @@ protobuf_src_dir=${protobuf_dir}/src protobuf_build_dir=${protobuf_dir}/build protobuf_install_dir=${install_dir} if [ ! -d "${protobuf_src_dir}" ]; then - git clone -b v3.19.4 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} + git clone -b v3.21.12 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} fi mkdir -p ${protobuf_build_dir} cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ @@ -61,15 +61,33 @@ cmake --build ${protobuf_build_dir} -j ${num_build_procs} cmake --install ${protobuf_build_dir} +#build Catch2 +catch2_dir=${build_dir}/catch2 +catch2_src_dir=${catch2_dir}/src +catch2_build_dir=${catch2_dir}/build +catch2_install_dir=${install_dir} +if [ ! -d "${catch2_src_dir}" ]; then + git clone -b v3.3.2 https://github.com/catchorg/Catch2.git ${catch2_src_dir} +fi +mkdir -p ${catch2_build_dir} +cmake -S ${catch2_src_dir} -B ${catch2_build_dir}\ + -DBUILD_TESTING=OFF\ + -DCMAKE_INSTALL_PREFIX=${catch2_install_dir} +cmake --build ${catch2_build_dir} -j ${num_build_procs} +cmake --install ${catch2_build_dir} > /dev/null 2>&1 + #build MGARD mgard_x_build_dir=${build_dir}/mgard mgard_x_install_dir=${install_dir} mkdir -p ${mgard_x_build_dir} cmake -S ${mgard_x_src_dir} -B ${mgard_x_build_dir} \ - -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir}"\ + -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir};${catch2_install_dir}"\ -DMGARD_ENABLE_HIP=ON\ + -DMGARD_ENABLE_OPENMP=OFF\ + -DMGARD_ENABLE_MDR=ON\ -DCMAKE_HIP_ARCHITECTURES="gfx90a"\ -DCMAKE_BUILD_TYPE=Release\ + -DBUILD_TESTING=ON\ -DCMAKE_INSTALL_PREFIX=${mgard_x_install_dir} cmake --build ${mgard_x_build_dir} -j ${num_build_procs} cmake --install ${mgard_x_build_dir} diff --git a/build_scripts/build_mgard_hip_mi300.sh b/build_scripts/build_mgard_hip_mi300.sh new file mode 100755 index 0000000000..b52b29d22a --- /dev/null +++ b/build_scripts/build_mgard_hip_mi300.sh @@ -0,0 +1,106 @@ +#!/bin/sh + +# Copyright 2021, Oak Ridge National Laboratory. +# MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs +# Author: Jieyang Chen (chenj3@ornl.gov) +# Date: April 2, 2021 +# Script for building MGARD-X +# +# Historical note for multi-GPU MI300 nodes (e.g. odyssey, 4x MI300A): this +# used to require HIP_VISIBLE_DEVICES=0 to avoid an illegal-memory-access +# segfault in the Huffman/norm reduction stages when more than one GPU was +# visible. That was actually an MGARD-X bug, not a ROCm/driver issue: +# DeviceSpecification's and DeviceQueues's per-device query/setup loops in +# DeviceAdapterHip.h called hipSetDevice(d) for every device without +# restoring the originally-active one afterward, leaving the real active +# HIP device out of sync with the library's own bookkeeping on any node +# with more than one GPU. Fixed; HIP_VISIBLE_DEVICES should no longer be +# necessary (verified with repeated runs of the affected tests on odyssey +# with all 4 GPUs visible). + +set -e +set -x + +module load amd/7.2.0 +module load cmake/3.31.1 +module load openmpi + +######## User Configurations ######## +# Source directory +mgard_x_src_dir=. +# Build directory +build_dir=./build-hip-mi300 +# Number of processors used for building +num_build_procs=$1 +# Installtaion directory +install_dir=./install-hip-mi300 + +export LD_LIBRARY_PATH=$(pwd)/${install_dir}/lib:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$(pwd)/${install_dir}/lib64:$LD_LIBRARY_PATH +export CC=amdclang +export CXX=amdclang++ + +#build ZSTD +zstd_dir=${build_dir}/zstd +zstd_src_dir=${zstd_dir}/src +zstd_build_dir=${zstd_dir}/build +zstd_install_dir=${install_dir} +if [ ! -d "${zstd_src_dir}" ]; then + git clone -b v1.5.6 https://github.com/facebook/zstd.git ${zstd_src_dir} +fi +mkdir -p ${zstd_build_dir} +cmake -S ${zstd_src_dir}/build/cmake -B ${zstd_build_dir}\ + -DZSTD_MULTITHREAD_SUPPORT=ON\ + -DCMAKE_INSTALL_LIBDIR=lib\ + -DCMAKE_INSTALL_PREFIX=${zstd_install_dir} +cmake --build ${zstd_build_dir} -j ${num_build_procs} +cmake --install ${zstd_build_dir} + + +#build Protobuf +protobuf_dir=${build_dir}/protobuf +protobuf_src_dir=${protobuf_dir}/src +protobuf_build_dir=${protobuf_dir}/build +protobuf_install_dir=${install_dir} +if [ ! -d "${protobuf_src_dir}" ]; then + git clone -b v3.21.12 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} +fi +mkdir -p ${protobuf_build_dir} +cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ + -Dprotobuf_BUILD_SHARED_LIBS=ON\ + -Dprotobuf_BUILD_TESTS=OFF\ + -DCMAKE_INSTALL_PREFIX=${protobuf_install_dir} +cmake --build ${protobuf_build_dir} -j ${num_build_procs} +cmake --install ${protobuf_build_dir} + + +#build Catch2 +catch2_dir=${build_dir}/catch2 +catch2_src_dir=${catch2_dir}/src +catch2_build_dir=${catch2_dir}/build +catch2_install_dir=${install_dir} +if [ ! -d "${catch2_src_dir}" ]; then + git clone -b v3.3.2 https://github.com/catchorg/Catch2.git ${catch2_src_dir} +fi +mkdir -p ${catch2_build_dir} +cmake -S ${catch2_src_dir} -B ${catch2_build_dir}\ + -DBUILD_TESTING=OFF\ + -DCMAKE_INSTALL_PREFIX=${catch2_install_dir} +cmake --build ${catch2_build_dir} -j ${num_build_procs} +cmake --install ${catch2_build_dir} > /dev/null 2>&1 + +#build MGARD +mgard_x_build_dir=${build_dir}/mgard +mgard_x_install_dir=${install_dir} +mkdir -p ${mgard_x_build_dir} +cmake -S ${mgard_x_src_dir} -B ${mgard_x_build_dir} \ + -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir};${catch2_install_dir}"\ + -DMGARD_ENABLE_HIP=ON\ + -DMGARD_ENABLE_OPENMP=OFF\ + -DMGARD_ENABLE_MDR=ON\ + -DCMAKE_HIP_ARCHITECTURES="gfx942"\ + -DCMAKE_BUILD_TYPE=Release\ + -DBUILD_TESTING=ON\ + -DCMAKE_INSTALL_PREFIX=${mgard_x_install_dir} +cmake --build ${mgard_x_build_dir} -j ${num_build_procs} +cmake --install ${mgard_x_build_dir} diff --git a/build_scripts/build_mgard_openmp_cpu.sh b/build_scripts/build_mgard_openmp_cpu.sh index c368e758a5..ac381114f5 100755 --- a/build_scripts/build_mgard_openmp_cpu.sh +++ b/build_scripts/build_mgard_openmp_cpu.sh @@ -25,7 +25,7 @@ zstd_src_dir=${zstd_dir}/src zstd_build_dir=${zstd_dir}/build zstd_install_dir=${install_dir} if [ ! -d "${zstd_src_dir}" ]; then - git clone -b v1.5.0 https://github.com/facebook/zstd.git ${zstd_src_dir} + git clone -b v1.5.6 https://github.com/facebook/zstd.git ${zstd_src_dir} fi mkdir -p ${zstd_build_dir} cmake -S ${zstd_src_dir}/build/cmake -B ${zstd_build_dir}\ @@ -41,7 +41,7 @@ protobuf_src_dir=${protobuf_dir}/src protobuf_build_dir=${protobuf_dir}/build protobuf_install_dir=${install_dir} if [ ! -d "${protobuf_src_dir}" ]; then - git clone -b v3.19.4 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} + git clone -b v3.21.12 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} fi mkdir -p ${protobuf_build_dir} cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ @@ -51,16 +51,31 @@ cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ cmake --build ${protobuf_build_dir} -j ${num_build_procs} cmake --install ${protobuf_build_dir} +#build Catch2 +catch2_dir=${build_dir}/catch2 +catch2_src_dir=${catch2_dir}/src +catch2_build_dir=${catch2_dir}/build +catch2_install_dir=${install_dir} +if [ ! -d "${catch2_src_dir}" ]; then + git clone -b v3.3.2 https://github.com/catchorg/Catch2.git ${catch2_src_dir} +fi +mkdir -p ${catch2_build_dir} +cmake -S ${catch2_src_dir} -B ${catch2_build_dir}\ + -DBUILD_TESTING=OFF\ + -DCMAKE_INSTALL_PREFIX=${catch2_install_dir} +cmake --build ${catch2_build_dir} -j ${num_build_procs} +cmake --install ${catch2_build_dir} > /dev/null 2>&1 + #build MGARD mgard_x_build_dir=${build_dir}/mgard mgard_x_install_dir=${install_dir} mkdir -p ${mgard_x_build_dir} cmake -S ${mgard_x_src_dir} -B ${mgard_x_build_dir} \ - -DCMAKE_PREFIX_PATH="${nvcomp_install_dir};${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir}"\ - -DMGARD_ENABLE_SERIAL=ON\ + -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir};${catch2_install_dir}"\ -DMGARD_ENABLE_OPENMP=ON\ -DMGARD_ENABLE_DOCS=OFF\ -DCMAKE_BUILD_TYPE=Release\ + -DBUILD_TESTING=ON\ -DCMAKE_INSTALL_PREFIX=${mgard_x_install_dir} cmake --build ${mgard_x_build_dir} -j ${num_build_procs} cmake --install ${mgard_x_build_dir} diff --git a/build_scripts/build_mgard_serial.sh b/build_scripts/build_mgard_serial.sh index 420b75bb96..cea62908e8 100755 --- a/build_scripts/build_mgard_serial.sh +++ b/build_scripts/build_mgard_serial.sh @@ -25,7 +25,7 @@ zstd_src_dir=${zstd_dir}/src zstd_build_dir=${zstd_dir}/build zstd_install_dir=${install_dir} if [ ! -d "${zstd_src_dir}" ]; then - git clone -b v1.5.0 https://github.com/facebook/zstd.git ${zstd_src_dir} + git clone -b v1.5.6 https://github.com/facebook/zstd.git ${zstd_src_dir} fi mkdir -p ${zstd_build_dir} cmake -S ${zstd_src_dir}/build/cmake -B ${zstd_build_dir}\ @@ -41,7 +41,7 @@ protobuf_src_dir=${protobuf_dir}/src protobuf_build_dir=${protobuf_dir}/build protobuf_install_dir=${install_dir} if [ ! -d "${protobuf_src_dir}" ]; then - git clone -b v3.19.4 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} + git clone -b v3.21.12 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} fi mkdir -p ${protobuf_build_dir} cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ @@ -51,15 +51,30 @@ cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ cmake --build ${protobuf_build_dir} -j ${num_build_procs} cmake --install ${protobuf_build_dir} +#build Catch2 +catch2_dir=${build_dir}/catch2 +catch2_src_dir=${catch2_dir}/src +catch2_build_dir=${catch2_dir}/build +catch2_install_dir=${install_dir} +if [ ! -d "${catch2_src_dir}" ]; then + git clone -b v3.3.2 https://github.com/catchorg/Catch2.git ${catch2_src_dir} +fi +mkdir -p ${catch2_build_dir} +cmake -S ${catch2_src_dir} -B ${catch2_build_dir}\ + -DBUILD_TESTING=OFF\ + -DCMAKE_INSTALL_PREFIX=${catch2_install_dir} +cmake --build ${catch2_build_dir} -j ${num_build_procs} +cmake --install ${catch2_build_dir} > /dev/null 2>&1 + #build MGARD mgard_x_build_dir=${build_dir}/mgard mgard_x_install_dir=${install_dir} mkdir -p ${mgard_x_build_dir} cmake -S ${mgard_x_src_dir} -B ${mgard_x_build_dir} \ - -DCMAKE_PREFIX_PATH="${nvcomp_install_dir};${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir}"\ - -DMGARD_ENABLE_SERIAL=ON\ + -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir};${catch2_install_dir}"\ -DMGARD_ENABLE_DOCS=OFF\ -DCMAKE_BUILD_TYPE=Release\ + -DBUILD_TESTING=ON\ -DCMAKE_INSTALL_PREFIX=${mgard_x_install_dir} cmake --build ${mgard_x_build_dir} -j ${num_build_procs} cmake --install ${mgard_x_build_dir} diff --git a/build_scripts/build_mgard_sycl_gen9.sh b/build_scripts/build_mgard_sycl_gen9.sh index 2f83106b9b..867ce72844 100755 --- a/build_scripts/build_mgard_sycl_gen9.sh +++ b/build_scripts/build_mgard_sycl_gen9.sh @@ -30,7 +30,7 @@ zstd_src_dir=${zstd_dir}/src zstd_build_dir=${zstd_dir}/build zstd_install_dir=${install_dir} if [ ! -d "${zstd_src_dir}" ]; then - git clone -b v1.5.0 https://github.com/facebook/zstd.git ${zstd_src_dir} + git clone -b v1.5.6 https://github.com/facebook/zstd.git ${zstd_src_dir} fi mkdir -p ${zstd_build_dir} cmake -S ${zstd_src_dir}/build/cmake -B ${zstd_build_dir}\ @@ -46,7 +46,7 @@ protobuf_src_dir=${protobuf_dir}/src protobuf_build_dir=${protobuf_dir}/build protobuf_install_dir=${install_dir} if [ ! -d "${protobuf_src_dir}" ]; then - git clone -b v3.19.4 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} + git clone -b v3.21.12 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} fi mkdir -p ${protobuf_build_dir} cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ @@ -57,17 +57,32 @@ cmake --build ${protobuf_build_dir} -j ${num_build_procs} cmake --install ${protobuf_build_dir} +#build Catch2 +catch2_dir=${build_dir}/catch2 +catch2_src_dir=${catch2_dir}/src +catch2_build_dir=${catch2_dir}/build +catch2_install_dir=${install_dir} +if [ ! -d "${catch2_src_dir}" ]; then + git clone -b v3.3.2 https://github.com/catchorg/Catch2.git ${catch2_src_dir} +fi +mkdir -p ${catch2_build_dir} +cmake -S ${catch2_src_dir} -B ${catch2_build_dir}\ + -DBUILD_TESTING=OFF\ + -DCMAKE_INSTALL_PREFIX=${catch2_install_dir} +cmake --build ${catch2_build_dir} -j ${num_build_procs} +cmake --install ${catch2_build_dir} > /dev/null 2>&1 + #build MGARD mgard_x_build_dir=${build_dir}/mgard mgard_x_install_dir=${install_dir} mkdir -p ${mgard_x_build_dir} cmake -S ${mgard_x_src_dir} -B ${mgard_x_build_dir} \ - -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir}"\ - -DMGARD_ENABLE_SERIAL=OFF\ + -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir};${catch2_install_dir}"\ -DMGARD_ENABLE_SYCL=ON\ -DCMAKE_CXX_FLAGS="-O2 -fsycl -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device gen9\""\ -DMGARD_ENABLE_DOCS=OFF\ -DCMAKE_BUILD_TYPE=Release\ + -DBUILD_TESTING=ON\ -DCMAKE_INSTALL_PREFIX=${mgard_x_install_dir} cmake --build ${mgard_x_build_dir} -j ${num_build_procs} cmake --install ${mgard_x_build_dir} diff --git a/build_scripts/build_mgard_sycl_x86.sh b/build_scripts/build_mgard_sycl_x86.sh index 7837f5cc0d..599d39e6fd 100755 --- a/build_scripts/build_mgard_sycl_x86.sh +++ b/build_scripts/build_mgard_sycl_x86.sh @@ -26,7 +26,7 @@ zstd_src_dir=${zstd_dir}/src zstd_build_dir=${zstd_dir}/build zstd_install_dir=${install_dir} if [ ! -d "${zstd_src_dir}" ]; then - git clone -b v1.5.0 https://github.com/facebook/zstd.git ${zstd_src_dir} + git clone -b v1.5.6 https://github.com/facebook/zstd.git ${zstd_src_dir} fi mkdir -p ${zstd_build_dir} cmake -S ${zstd_src_dir}/build/cmake -B ${zstd_build_dir}\ @@ -44,7 +44,7 @@ protobuf_src_dir=${protobuf_dir}/src protobuf_build_dir=${protobuf_dir}/build protobuf_install_dir=${install_dir} if [ ! -d "${protobuf_src_dir}" ]; then - git clone -b v3.19.4 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} + git clone -b v3.21.12 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} fi mkdir -p ${protobuf_build_dir} cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ @@ -57,19 +57,34 @@ cmake --build ${protobuf_build_dir} -j ${num_build_procs} cmake --install ${protobuf_build_dir} +#build Catch2 +catch2_dir=${build_dir}/catch2 +catch2_src_dir=${catch2_dir}/src +catch2_build_dir=${catch2_dir}/build +catch2_install_dir=${install_dir} +if [ ! -d "${catch2_src_dir}" ]; then + git clone -b v3.3.2 https://github.com/catchorg/Catch2.git ${catch2_src_dir} +fi +mkdir -p ${catch2_build_dir} +cmake -S ${catch2_src_dir} -B ${catch2_build_dir}\ + -DBUILD_TESTING=OFF\ + -DCMAKE_INSTALL_PREFIX=${catch2_install_dir} +cmake --build ${catch2_build_dir} -j ${num_build_procs} +cmake --install ${catch2_build_dir} > /dev/null 2>&1 + #build MGARD mgard_x_build_dir=${build_dir}/mgard mgard_x_install_dir=${install_dir} mkdir -p ${mgard_x_build_dir} cmake -S ${mgard_x_src_dir} -B ${mgard_x_build_dir} \ - -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir}"\ - -DMGARD_ENABLE_SERIAL=OFF\ + -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir};${catch2_install_dir}"\ -DMGARD_ENABLE_SYCL=ON\ -DCMAKE_CXX_COMPILER=icpx\ -DCMAKE_C_COMPILER=icx\ -DCMAKE_CXX_FLAGS="-fsycl -fsycl-targets=x86_64"\ -DMGARD_ENABLE_DOCS=OFF\ -DCMAKE_BUILD_TYPE=Release\ + -DBUILD_TESTING=ON\ -DCMAKE_INSTALL_PREFIX=${mgard_x_install_dir} cmake --build ${mgard_x_build_dir} -j ${num_build_procs} cmake --install ${mgard_x_build_dir} diff --git a/build_scripts/build_mgard_sycl_xehp.sh b/build_scripts/build_mgard_sycl_xehp.sh index 85d8d25f5c..5a0efe893c 100755 --- a/build_scripts/build_mgard_sycl_xehp.sh +++ b/build_scripts/build_mgard_sycl_xehp.sh @@ -30,7 +30,7 @@ zstd_src_dir=${zstd_dir}/src zstd_build_dir=${zstd_dir}/build zstd_install_dir=${install_dir} if [ ! -d "${zstd_src_dir}" ]; then - git clone -b v1.5.0 https://github.com/facebook/zstd.git ${zstd_src_dir} + git clone -b v1.5.6 https://github.com/facebook/zstd.git ${zstd_src_dir} fi mkdir -p ${zstd_build_dir} cmake -S ${zstd_src_dir}/build/cmake -B ${zstd_build_dir}\ @@ -46,7 +46,7 @@ protobuf_src_dir=${protobuf_dir}/src protobuf_build_dir=${protobuf_dir}/build protobuf_install_dir=${install_dir} if [ ! -d "${protobuf_src_dir}" ]; then - git clone -b v3.19.4 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} + git clone -b v3.21.12 --recurse-submodules https://github.com/protocolbuffers/protobuf.git ${protobuf_src_dir} fi mkdir -p ${protobuf_build_dir} cmake -S ${protobuf_src_dir}/cmake -B ${protobuf_build_dir}\ @@ -57,17 +57,32 @@ cmake --build ${protobuf_build_dir} -j ${num_build_procs} cmake --install ${protobuf_build_dir} +#build Catch2 +catch2_dir=${build_dir}/catch2 +catch2_src_dir=${catch2_dir}/src +catch2_build_dir=${catch2_dir}/build +catch2_install_dir=${install_dir} +if [ ! -d "${catch2_src_dir}" ]; then + git clone -b v3.3.2 https://github.com/catchorg/Catch2.git ${catch2_src_dir} +fi +mkdir -p ${catch2_build_dir} +cmake -S ${catch2_src_dir} -B ${catch2_build_dir}\ + -DBUILD_TESTING=OFF\ + -DCMAKE_INSTALL_PREFIX=${catch2_install_dir} +cmake --build ${catch2_build_dir} -j ${num_build_procs} +cmake --install ${catch2_build_dir} > /dev/null 2>&1 + #build MGARD mgard_x_build_dir=${build_dir}/mgard mgard_x_install_dir=${install_dir} mkdir -p ${mgard_x_build_dir} cmake -S ${mgard_x_src_dir} -B ${mgard_x_build_dir} \ - -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir}"\ - -DMGARD_ENABLE_SERIAL=OFF\ + -DCMAKE_PREFIX_PATH="${zstd_install_dir}/lib/cmake/zstd;${protobuf_install_dir};${catch2_install_dir}"\ -DMGARD_ENABLE_SYCL=ON\ -DCMAKE_CXX_FLAGS="-O2 -fsycl -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device xehp\""\ -DMGARD_ENABLE_DOCS=OFF\ -DCMAKE_BUILD_TYPE=Release\ + -DBUILD_TESTING=ON\ -DCMAKE_INSTALL_PREFIX=${mgard_x_install_dir} cmake --build ${mgard_x_build_dir} -j ${num_build_procs} cmake --install ${mgard_x_build_dir} diff --git a/cmake/mgard-config.cmake.in b/cmake/mgard-config.cmake.in index c836219eaa..fe62f7a403 100644 --- a/cmake/mgard-config.cmake.in +++ b/cmake/mgard-config.cmake.in @@ -67,7 +67,6 @@ set(MGARD_ENABLE_CUDA @MGARD_ENABLE_CUDA@) set(MGARD_ENABLE_HIP @MGARD_ENABLE_HIP@) set(MGARD_ENABLE_SYCL @MGARD_ENABLE_SYCL@) set(MGARD_ENABLE_OPENMP @MGARD_ENABLE_OPENMP@) -set(MGARD_ENABLE_LEGACY_CUDA @MGARD_ENABLE_LEGACY_CUDA@) set(MGARD_ENABLE_MDR @MGARD_ENABLE_MDR@) if(NOT TARGET mgard::mgard) diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index d6903988ae..db7ffbd77c 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -3,8 +3,8 @@ set( MGARD_DOC_PAGES "README.md" "doc/MGARD-X.md" + "doc/MGARD-RoI2.md" "doc/MDR-X.md" - "doc/MGARD-GPU.md" "examples/README.md" "examples/linking/README.md" "examples/compression/README.md" diff --git a/doc/MGARD-GPU.md b/doc/MGARD-GPU.md deleted file mode 100644 index de1e103b3b..0000000000 --- a/doc/MGARD-GPU.md +++ /dev/null @@ -1,186 +0,0 @@ - -# MGARD-CUDA [***Deprecated***] -***MGARD-CUDA is being deprecated and will be removed in future releases of MGARD. Please use the MGARD-X portable version.*** - -MGARD-GPU is a CUDA implementation of the MGARD lossy compressor, which significantly improves MGARD's compression/decomrpession throughput via highly optimized GPU kernels. - -## Supporting features -* **Data type:** Double and single precision floating-point data -* **Dimensions:** 1D-5D -* **Error-bound type:** L\_Inf error and L\_2 error -* **Error-bound mode:** Absoluate and relative -* **Data structure:** Uniform and non-uniform spaced Cartisan gird -* **Three sets of interfaces**: - * **Command Line Inteface:** Full-featured command line executable for compression/decompression. - * **High-level APIs:** Encapsulated all GPU-related details for easiest integration with user's program. - * **Low-level APIs:** Users can fully control different steps of compression (the preprocessing step, GPU buffers allocations, GPU compression kernel invokation, GPU-CPU data copy) for more fexiable and high performance compression. - - -## Hardware and software requirements -* NVIDIA GPUs ( tested on Volta, Turing) -* CUDA 11.0+ -* CMake 3.19+ - -## Software dependencies -* [NVCOMP v2.2.0][nvcomp] -* [ZSTD v1.5.0][zstd] -* [Protobuf v3.19.4][protobuf] - -[nvcomp]: https://github.com/NVIDIA/nvcomp.git -[zstd]: https://github.com/facebook/zstd.git -[protobuf]: https://github.com/protocolbuffers/protobuf.git - -## Configure and build -+ **Step 1:** configure and build dependency libraries -+ **Step 2:** configure MGARD as follows: - - cmake -S -B - -DMGARD_ENABLE_LEGACY_CUDA=ON - -DCMAKE_CUDA_ARCHITECTURES= - -DCMAKE_PREFIX_PATH= - -+ **Step 3:** build MGARD: ```cmake --build -j8``` -[build_scripts]:[build_scrtips] - -## Using command line interface (CLI) -* An executable ```mgard-gpu``` will be built when building the MGARD-GPU library. -* To use the ```mgard-gpu``` CLI, here are the options: - - + ```-z```: compress data - + ```-i ``` path to data file to be compressed - + ```-c ``` path to compressed file - + ```-t ``` data type (s: single; d:double) - + ```-n ``` total number of dimensions - + `````` slowest dimention - + `````` 2nd slowest dimention - + ... - + `````` fastest dimention - + ```-u ``` path to coordinate file (non-uniform only) - + ```-m ``` error bound mode (abs: abolute; rel: relative) - + ```-e ``` error bound - + ```-s ``` smoothness parameter - + ```-l <1|2|3>``` choose lossless compressor (0:CPU 1:Huffman@GPU 2:Huffman@GPU+LZ4@GPU) - + ```-x```: decompress data - + ```-c ``` path to compressed file - + ```-d ``` path to decompressed file - + ```-v``` enable verbose (show timing and statistics) - -## For Using both the high-level APIs and low-level API -* **Include the header file.** MGARD-GPU APIs are included in ```mgard/compress_cuda.hpp```. -* **Configure using ```mgard_cuda::Config```** Both high-level APIs and low-level APIs have an optional parameter for users to configure the compression/decomrpession process via ```mgard_cuda::Config``` class. To configure, create a ```mgard_cuda::Config``` object and configure its fields: - + ```Config.dev_id```: sepcifying a specific GPU to use in multi-GPU systems. - + ```Config.timing```: timing each steps of compression and printing them out. - + ```Config.lossless```: control the lossless compression used: - + ```mgard_cuda::lossless_type::CPU_Lossless```: CPU lossless (ZLIB/ZSTD) - + ```mgard_cuda::lossless_type::GPU_Huffman```: GPU Huffman compression - + ```mgard_cuda::lossless_type::GPU_Huffman_LZ4```: GPU Huffman and LZ4 compression - + *Note:* there will be no effect configuring the lossless comrpessor for decompression as MGARD has to use the same lossless compressor that was used for compression. -## Using high-level APIs -* **For compression:** ```void mgard_cuda::compress(mgard_cuda::DIM D, mgard_cuda::data_type dtype, std::vector shape, double tol, double s, enum error_bound_type mode, const void *original_data, void *&compressed_data, size_t &compressed_size, mgard_cuda::Config config)``` - + ```[In] shape:``` Shape of the Dataset to be compressed (from slowest to fastest). - + ```[In] data_type:``` mgard_cuda::data_type::Float or mgard_cuda::data_type::Double. - + ```[In] type:``` mgard_cuda::error_bound_type::REL or mgard_cuda::error_bound_type::ABS. - + ```[In] tol:``` Error tolerance. - + ```[In] s:``` Smoothness parameter. - + ```[In] compressed_data:``` Dataset to be compressed. - + ```[Out] compressed_size:``` Size of comrpessed data. - + ```[In][Optional] coords```: The coordinates in each dimension (from slowest to fastest). - + ```[in][Optional] config:``` For configuring the compression process (optional). - -* **For decompression:** ```void decompress(const void *compressed_data, size_t compressed_size, void *&decompressed_data, Config config)``` - + ```[In] compressed_data:``` Compressed data. - + ```[In] compressed_size:``` Size of comrpessed data. - + ```[Out] decompressed_data:``` Decompressed data. - + ```[In][Optional] config:``` For configuring the decompression process (optional). - -## Using low-level APIs -* **Step 2: Initialize mgard_cuda::Handle.** -An object ```mgard_cuda::Handle``` needs to be created and initialized. This initializes the necessary environment for efficient compression on the GPU. It only needs to be created once if the input shape is not changed. For example, compressing the same variable on different timesteps only needs the handle to be created once. Also, the same handle can be shared in between compression and decompression APIs. - + ```mgard_cuda::Handle(std::vector shape, std::vector coords, mgard_cuda::Config config)```. - + ```[In] D_type```: Input data type (float or double). - + ```[In] N_dims```: Total number of dimensions (<=4) - + ```[In] shape```: Stores the size in each dimension (from slowest to fastest). - + ```[In][Optional] coords```: The coordinates in each dimension (from slowest to fastest). - + ```[In][Optional] config```: For configuring compression/decomrpession. -* **Step 3: Use mgard_cuda::Array.** ```mgard_cuda::Array``` is used for holding a managed array on GPU. - + For ***creating*** an array. ```mgard_cuda::Array::Array(std::vector shape)``` creates an manged array on GPU with ```shape```. - + For ***loading data*** into an array. ```void mgard_cuda::Array::loadData(D_type *data, size_t ld = 0)``` copies ```data``` into the the managed array on GPU. ```data``` can be on either on CPU or GPU. An optional ```ld``` can be provided for specifying the size of the leading dimension. - + For ***accessing data from CPU*** ```D_type * mgard_cuda::Array::getDataHost()``` returns a CPU pointer of the array. - + For ***accessing data from GPU***```D_type * mgard_cuda::Array::getDataDevice(size_t &ld)``` returns a GPU pointer of the array with the leading dimension. - + For ***getting the shape*** of an array. ```std::vector mgard_cuda::Array::getShape()``` returns the shape of the managed array. - - ***Note:*** ```mgard_cuda::Array``` will automatically release its internal CPU/GPU array when it goes out of scope. - -* **Step 4: Query specifications of original data from compressed data** In case the data type/structure/shape are unknown when decompression, the following APIs can be use to infer those information - - + For **infering data type**: ```enum mgard_cuda::data_type mgard_cuda::infer_data_type(const void *compressed_data, size_t compressed_size)``` - + ```[In] compressed_data:``` Compressed data. - + ```[In] compressed_size:``` Size of comrpessed data. - + ```[Return] Data type``` - + For **infering data shape**: ```std::vector mgard_cuda::infer_shape(const void *compressed_data, size_t compressed_size)``` - + ```[In] compressed_data:``` Compressed data. - + ```[In] compressed_size:``` Size of comrpessed data. - + ```[Return] Data shape``` - + For **infering data structure**: ```enum mgard_cuda::data_structure infer_data_structure(const void *compressed_data, size_t compressed_size)``` - + ```[In] compressed_data:``` Compressed data. - + ```[In] compressed_size:``` Size of comrpessed data. - + ```[Return] Data structure``` - + For **infering data structure**: ```std::vector infer_coords(const void *compressed_data, size_t compressed_size)``` - + ```[In] compressed_data:``` Compressed data. - + ```[In] compressed_size:``` Size of comrpessed data. - + ```[Return] Coordinates``` -* **Step 4: Invoke compression/decompression.**: - + For ***compression***: ``` - mgard_cuda::Array<1, unsigned char> mgard_cuda::compress(mgard_cuda::Handle &handle, mgard_cuda::Array in_array, mgard_cuda::error_bound_type type, D_type tol, D_type s)``` - - ```[In] in_array ```: Input data to be compressed (its value will be altered during compression). - - ```[In] type ```: Error bound type. ```mgard_cuda::REL``` for relative error bound or ```mgard_cuda::ABS``` for absolute error bound. - - ```[In] tol```: Error bound. - - ```[In] s```: Smoothness parameter. - - ```[Return]```: Compressed data. - + For ***decompression***: ```mgard_cuda::Array mgard_cuda::decompress(mgard_cuda::Handle &handle, mgard_cuda::Array<1, unsigned char> compressed_data)``` - - ```[In] compressed_data ```: Compressed data. - - ```[Return]```: Decompressed data. - -## Performance optimization -* **Optimize for fast CPU-GPU data transfer:** It is recommanded to use pinned memory on CPU for loading data into ```mgard_cuda::Array``` such that it can enable fast CPU-GPU data transfer. - + To allocate pinned memory on CPU: ```mgard_cuda::cudaMallocHostHelper(void ** data_ptr, size_t size)```. - + To free pinned memory on CPU: ```mgard_cuda::cudaFreeHostHelper(void * data_ptr)``` - -## A simple example -The following code shows how to compress/decompress a 3D dataset with the low-level APIs. - - #include - #include - #include "mgard/compress.hpp" - int main() - { - mgard_cuda::SIZE n1 = 10; - mgard_cuda::SIZE n2 = 20; - mgard_cuda::SIZE n3 = 30; - - //prepare - std::cout << "Preparing data..."; - double * in_array_cpu; - mgard_cuda::cudaMallocHostHelper((void **)&in_array_cpu, sizeof(double)*n1*n2*n3); - //... load data into in_array_cpu - std::vector shape{ n1, n2, n3 }; - mgard_cuda::Handle<3, double> handle(shape); - mgard_cuda::Array<3, double> in_array(shape); - in_array.loadData(in_array_cpu); - std::cout << "Done\n"; - - std::cout << "Compressing with MGARD-GPU..."; - double tol = 0.01, s = 0; - mgard_cuda::Array<1, unsigned char> compressed_array = mgard_cuda::compress(handle, in_array, mgard_cuda::REL, tol, s); - mgard_cuda::SIZE compressed_size = compressed_array.getShape()[0]; //compressed size in number of bytes. - unsigned char * compressed_array_cpu = compressed_array.getDataHost(); - std::cout << "Done\n"; - - std::cout << "Decompressing with MGARD-GPU..."; - // decompression - mgard_cuda::Array<3, double> decompressed_array = mgard_cuda::decompress(handle, compressed_array); - double * decompressed_array_cpu = decompressed_array.getDataHost(); - std::cout << "Done\n"; - - mgard_cuda::cudaFreeHostHelper(in_array_cpu); - } diff --git a/doc/MGARD-Lambda.md b/doc/MGARD-Lambda.md index dcb479494f..47875af128 100644 --- a/doc/MGARD-Lambda.md +++ b/doc/MGARD-Lambda.md @@ -1,6 +1,6 @@ -# MGARD-Lambda +# MGARD-$\lambda$ -MGARD-Lambda is designed to preserve non-linear quantities of interests during compression. +MGARD-$\lambda$ is designed to preserve non-linear quantities of interests during compression. *This is an experimental part of MGARD. Currently only support certain QoIs derived from XGC 5D data* #### The following shows the theory behind MGARD-Lambda. diff --git a/doc/MGARD-RoI2.md b/doc/MGARD-RoI2.md new file mode 100644 index 0000000000..fc8e56520e --- /dev/null +++ b/doc/MGARD-RoI2.md @@ -0,0 +1,89 @@ +# MGARD-X Region-of-Interest (ROI) Compression + +MGARD-X supports a different L∞ error tolerance for each block of the hybrid hierarchy. This allows important regions to use a tighter error bound while the background uses a looser error bound and achieves a higher compression ratio. + +## Supporting features + +* **Dimensions:** 1D-3D +* **Error-bound type:** L∞ +* **Error-bound mode:** Absolute and relative +* **Data structure:** Uniform Cartesian grid +* **Interfaces:** Command line interface and high-level APIs + +The ROI feature is not currently exposed through the public low-level API. Compression must use the hybrid hierarchy, and the input must be processed as one subdomain. Domain decomposition does not currently apply a global ROI map correctly. + +## ROI tolerance map + +The input map contains one tolerance for every block at the finest block-local level. MGARD-X currently uses a block edge length of 8, so data with shape + +```text +N0 N1 ... ND-1 +``` + +requires a map with + +```text +ceil(N0 / 8) * ceil(N1 / 8) * ... * ceil(ND-1 / 8) +``` + +entries. The file format is a headerless binary array of native-endian IEEE 754 ```double``` values. Entries use row-major order: the last dimension is the fastest-varying dimension. For example, a 3D block coordinate ```(b0, b1, b2)``` is stored at + +```text +(b0 * B1 + b1) * B2 + b2 +``` + +where ```Bd = ceil(Nd / 8)```. In relative mode, each value is a relative tolerance with respect to the L∞ norm of the complete dataset. In absolute mode, each value is an absolute tolerance. + +Only the finest-level map is provided by the user. When multiple local levels are used, MGARD-X derives the coarser maps by propagating the minimum tolerance of the contributing fine blocks. The global stage also uses the minimum propagated tolerance. + +## Generate a tolerance map + +The repository provides a zero-dependency [ROI map generator][roi-generator]. It initializes every block with a background tolerance and then applies one or more voxel-space ROI boxes. Coordinates follow the same slowest-to-fastest dimension order as the MGARD-X shape, and every ```START END``` interval is half-open: ```START``` is included and ```END``` is excluded. + +For example, the following command creates a map for a ```512 x 512 x 512``` dataset. The background tolerance is ```1e-2```, while voxels in ```[200, 300) x [200, 300) x [200, 300)``` use ```1e-5```: + +```console +$ python3 scripts/generate_mgard_x_roi_map.py \ + -o roi.bin -dim 512 512 512 -bg 1e-2 \ + -roi 1e-5 200 300 200 300 200 300 +``` + +The ```-roi``` option may be repeated. Each ROI is expanded to cover every intersecting block whose edge length is 8, and overlapping regions use the smallest requested tolerance. + +## Command line interface + +Compress with the hybrid hierarchy, ROI mode, and ```s = inf```: + +```console +$ mgard-x -z -i input.f32 -o compressed.mgard \ + -dt s -dim 3 512 512 512 \ + -em rel -r roi.bin -roi -s inf \ + -l huffman -d cuda -hh -ll 1 -gl 2 +``` + +The ROI map, hierarchy levels, and projection mode are stored in the compressed-data metadata. Decompression therefore does not require the original map or the hybrid compression options: + +```console +$ mgard-x -x -i compressed.mgard -o reconstructed.f32 -d cuda +``` + +Do not set a restrictive maximum-memory option that causes domain decomposition. The current ROI implementation expects the complete map to describe one subdomain. + +## High-level APIs + +Enable the hybrid hierarchy and copy the finest-level tolerance map into ```Config``` before compression: + +```cpp +mgard_x::Config config; +config.decomposition = mgard_x::decomposition_type::Hybrid; +config.enable_roi = true; +config.roi_tolerance_map = tolerance_map; // std::vector +config.num_local_refactoring_level = 1; +config.num_global_refactoring_level = 2; +config.hybrid_projection_mode = + mgard_x::hybrid_projection_mode_type::Orthogonal; +``` + +The vector length and ordering must follow the binary map format described above. Pass this ```Config``` to the regular high-level ```mgard_x::compress``` API. The regular high-level ```mgard_x::decompress``` API restores the ROI configuration from metadata. + +[roi-generator]: ../scripts/generate_mgard_x_roi_map.py diff --git a/doc/MGARD-X.md b/doc/MGARD-X.md index 11b3936044..dee9fa35f6 100644 --- a/doc/MGARD-X.md +++ b/doc/MGARD-X.md @@ -7,7 +7,7 @@ MGARD-X is a portable implementation of the MGARD lossy compressor supporting va ## Supporting features * **Data type:** Double and single precision floating-point data -* **Dimensions:** 1D-5D +* **Dimensions:** 1D-5D (the hybrid hierarchy supports 1D-3D only) * **Error-bound type:** L\_Inf error and L\_2 error * **Error-bound mode:** Absoluate and relative * **Data structure:** Uniform and non-uniform spaced Cartisan gird @@ -18,13 +18,11 @@ MGARD-X is a portable implementation of the MGARD lossy compressor supporting va |x86 CPUs|Yes|Intel CPUs, AMD CPUs| |Power CPUs|Yes|IMB Power9 CPUs| |ARM CPUs|Yes|Apple M2| -|NVIDIA GPUs|Yes*|V100, RTX2080 Ti, RTX 3090| -|AMD GPUs|Yes|MI-100, MI-250X| +|NVIDIA GPUs|Yes|V100, A100, H100, B6000, RTX2080 Ti, RTX 3090| +|AMD GPUs|Yes|MI100, MI250X, MI300| |Intel GPUs|Yes|Gen9| |Integrated GPUs|Yes|Gen9| -*LZ4 lossless compressor is only avaialble to choose on NVIDIA GPUs. Portable version is under development. - ## Interfaces * **Command Line Inteface:** Full-featured command line executable for compression/decompression. @@ -42,7 +40,7 @@ MGARD-X is a portable implementation of the MGARD lossy compressor supporting va ### For Multi-core CPUs -* GCC 7.5.0+ (x86, Power) +* GCC 9+ (x86, Power) * Clang 14+ (Apple silicon) * OpenMP 3.1+ * CMake 3.19+ @@ -51,10 +49,11 @@ MGARD-X is a portable implementation of the MGARD lossy compressor supporting va * CUDA 11.0+ or NVIDIA HPC 22.0+ * CMake 3.19+ +* ***Known issue:*** On Blackwell GPUs (sm\_120, e.g. RTX PRO 6000 Blackwell) built with CUDA 13.3/CCCL 3.x, the histogram-based lossless stages (Rans, Huffman+Zstd, Huffman+LZ4) fail with an "invalid argument" CUDA error. This is specific to the CUDA 13.3/CCCL 3.x toolchain on Blackwell; Until resolved, prefer CUDA 12.8 for Blackwell GPUs. ### For AMD GPUs -* ROCm 4.5.0+ +* ROCm 7.0.0+ * CMake 3.21+ ### For Intel GPUs @@ -64,13 +63,13 @@ MGARD-X is a portable implementation of the MGARD lossy compressor supporting va ## Software dependencies -* [NVCOMP v2.2.0][nvcomp] (for NVIDIA GPUs only) -* [ZSTD v1.5.0][zstd] -* [Protobuf v3.19.4][protobuf] +* [ZSTD v1.5.6][zstd] +* [Protobuf v3.21.12][protobuf] +* [Catch2 v3.3.2][catch2] (only needed for building tests) -[nvcomp]: https://github.com/NVIDIA/nvcomp.git [zstd]: https://github.com/facebook/zstd.git [protobuf]: https://github.com/protocolbuffers/protobuf.git +[catch2]: https://github.com/catchorg/Catch2.git ## Configure and build * **Option 1:** One-step configure and build scripts are available [here][build_scripts]. * **Option 2:** Manually confiugre and build with CMake @@ -84,8 +83,6 @@ MGARD-X is a portable implementation of the MGARD lossy compressor supporting va |MGARD\_ENABLE\_CUDA|ON/OFF|OFF|Enable portable GPU compression/decompression with CUDA on NVIDIA GPUs| |MGARD\_ENABLE\_HIP|ON/OFF|OFF|Enable portable GPU compression/decompression with HIP on AMD GPUs| |MGARD\_ENABLE\_SYCL|ON/OFF|OFF|Enable portable GPU compression/decompression with SYCL on Intel GPUs| -|MGARD\_ENABLE\_MULTI_DEVICE|ON/OFF|OFF|Enable multi-device (GPUs) compression/decompression| -|MGARD\_ENABLE\_COORDINATE\_NORMALIZATION|ON/OFF|ON|Enable coordinate normalization| |MGARD\_ENABLE\_AUTO\_TUNING|ON/OFF|OFF|Enable auto tuning feature| |MGARD\_ENABLE\_MDR|ON/OFF|OFF|Enable building MDR and MDR-X| @@ -101,7 +98,6 @@ MGARD can bound error `tol` in two ways (using L∞ norm as an ex * ***Absolute*** error mode can guarantee ***| u - u' |∞ < tol*** * ***Relative*** error mode can guarantee ***| u - u' |∞ < tol * | u |∞*** - ## Using command line interface (CLI) An executable ```mgard-x``` will be built after building the MGARD-X library. To use the ```mgard-x``` CLI, here are the options: @@ -121,7 +117,23 @@ An executable ```mgard-x``` will be built after building the MGARD-X library. To + ```-r <0|1>``` internal data layout (0: Higher throughput | 1: Higher compression ratio) + ```-b <0|1>``` domain decomposition type (0: 1D max dimension (default) | 1: N-D block) + ```-f ``` maximum memory footprint in bytes (if not specify: no limit) - + ```-l <0|1|2>``` choose lossless compressor (0:Huffman | 1:Huffman+LZ4 (NVIDIA GPU only) | 2:Huffman@ZSTD) + + ```-l / --lossless ``` choose lossless compressor: + + ```huffman```: Huffman coding + + ```huffman-lz4```: Huffman coding followed by LZ4 + + ```lz4```: LZ4 applied directly to the quantized integer stream + + ```huffman-zstd```: Huffman coding followed by ZSTD + + ```blockdelta``` or ```blockdelta-delta```: BlockDelta with delta encoding + + ```blockdelta-fixed```: BlockDelta with fixed-length encoding + + ```blockdelta-outlier```: BlockDelta with delta encoding and outlier handling + + ```zerorle-rans```: zero-run-length encoding followed by byte-alphabet rANS + + ```symbol-rans```: symbol-alphabet rANS + + ```-hh / --hybrid``` use the hybrid hierarchy (1D-3D only) + + ```-ll / --local-levels ``` number of block-local refactoring levels (default: 1) + + ```-gl / --global-levels ``` number of global refactoring levels (default: 0) + + ```-hp / --hybrid-projection ``` select the projection mode used by the hybrid hierarchy (default: orthogonal) + + ```orthogonal```: retain the L2 projection (default) + + ```hierarchical```: remove the L2 projection (L∞ only) + + ```-nkf / --no-kernel-fusion``` disable fused local decomposition and quantization kernels for the hybrid hierarchy + ```-x```: enable decompression mode + ```-c ``` path to compressed file + ```-o ``` path to decompressed file @@ -135,6 +147,7 @@ An executable ```mgard-x``` will be built after building the MGARD-X library. To * **Include the header file.** + Use ```mgard/compress_x.hpp``` for ***high-level*** compression/decompression APIs + Use ```mgard/compress_x_lowlevel.hpp``` for ***low-level*** compression/decompression APIs + + *Note:* The hybrid hierarchy is currently supported through the command line interface and high-level APIs. It supports 1D-3D data and is not exposed through the public low-level API. * **Configure using ```mgard_x::Config```** Both high-level APIs and low-level APIs have an optional parameter for users to configure the compression/decomrpession process via ```mgard_x::Config``` class. To configure, create a ```mgard_x::Config``` object and configure its fields: + ```dev_type```: sepcifying the processor for compression/decompression: + ```mgard_x::device_type::Auto```: Auto detect the best processor (***Default***) @@ -146,7 +159,6 @@ An executable ```mgard-x``` will be built after building the MGARD-X library. To + ```dev_id```: sepcifying a specific GPU to use in multi-GPU systems (***Default: 0***) + ```num_dev```: sepcifying the number of GPU to use in multi-GPU systems (***Default: 1***) - + ```reorder```: sepcifying an internal data layout (0: Higher throughput | 1: Higher compression ratio) (***Default: 0***) + ```lossless```: control the lossless compression used: + ```mgard_x::lossless_type::Huffman```: Huffman compression (***Default***) + ```mgard_x::lossless_type::Huffman_LZ4```: Huffman and LZ4 compression @@ -163,8 +175,15 @@ An executable ```mgard-x``` will be built after building the MGARD-X library. To + ```decomposition```: controls multi-level decomposition: + ```mgard_x::decomposition_type::MultiDim```: N-D decomposition (***Default***) + ```mgard_x::decomposition_type::SingleDim```: 1D-at-a-time decomposition + + ```mgard_x::decomposition_type::Hybrid```: hybrid hierarchy with block-local and global decomposition (1D-3D only; high-level APIs only) + + ```num_local_refactoring_level```: controls the number of block-local refactoring levels in the hybrid hierarchy (***Default: 1***) + + ```num_global_refactoring_level```: controls the number of global refactoring levels in the hybrid hierarchy (***Default: 0***) + + ```hybrid_projection_mode```: controls the projection mode used by the hybrid hierarchy: + + ```mgard_x::hybrid_projection_mode_type::Orthogonal```: retain the L2 projection (***Default***) + + ```mgard_x::hybrid_projection_mode_type::Hierarchical```: remove the L2 projection (L∞ only) + + ```fuse_decompose_quantize```: controls fused local decomposition and quantization for the hybrid hierarchy (***Default: true***) + + ```fuse_dequantize_recompose```: controls fused local dequantization and recomposition for the hybrid hierarchy (***Default: true***) + ```max_larget_level```: controls max level of multi-level decomposition (***Default: 0 (no limit)***) - + ```prefetch```(for high-level APIs only): controls whether or not to enable prefetch pipeline optimization (***Default: true***) + ```max_memory_footprint```(for high-level APIs only): controls maximum memory footprint in bytes (***Default: inf (no limit)***) + ```adjust_shape```(for high-level APIs only): controls whether or not to enable shape adjustment for better decomposition performance (***Default: false***) ## Using high-level APIs @@ -273,14 +292,14 @@ For achieving the best performance: + **Tune for a specific backend:** run ```mgard-x-autotuner -d ``` + **Tune for a specific shape of data on a specific backend :** run ```mgard-x-autotuner -d -n [dim1] [dim2] ... [dimN]```. + ***Note:*** MGARD-X needs to be recompiled after auto tuning to make it effective. -* **Use large data and turn on prefetch**: Using larger input data will help MGARD-X achieve more efficient compression and decompression pipeline. The high-level API allows users to provide data larger than the available memory size on the GPU. In this case, MGARD-X can achieve much better performance with pipeline optimization. Please be sure to set ```Config.prefetch = true```. -* **Use decompsition operation**: MGARD-X can be configured to reornagize the input data to imporve the performance of its multilevel decompsition process. To enable such optimization, set ```Config.prefetch = adjust_shape```. +* **Use large data**: Using larger input data will help MGARD-X achieve more efficient compression and decompression pipeline. The high-level API allows users to provide data larger than the available memory size on the GPU. In this case, MGARD-X can achieve much better performance with pipeline optimization (always enabled). +* **Use decompsition operation**: MGARD-X can be configured to reornagize the input data to imporve the performance of its multilevel decompsition process. To enable such optimization, set ```Config.adjust_shape = true```. ## Performance The figures below show the compression and decompression throughput of MGARD-X on single CPU (multi-core) and GPU. XGC fusion simulation data is used for compression and decompression. [](images/MGARD-X-CPU.png) -[](images/MGARD-X-GPU.png) +[](images/MGARD-X-GPU-2.png) The figures below show the compression and decompression throughput of MGARD-X on multiple NVIDIA V100 and AMD MI-250X GPUs. XGC fusion simulation data is used for compression and decompression. @@ -294,4 +313,3 @@ The figures below show the compression and decompression throughput of MGARD-X o [high-level-example]:../examples/mgard-x/HighLevelAPIs [low-level-example]:../examples/mgard-x/LowLevelAPIs - diff --git a/doc/images/MGARD-X-GPU-2.png b/doc/images/MGARD-X-GPU-2.png new file mode 100644 index 0000000000..96349c087f Binary files /dev/null and b/doc/images/MGARD-X-GPU-2.png differ diff --git a/doc/images/MGARD-family2.png b/doc/images/MGARD-family2.png new file mode 100644 index 0000000000..1bc5229f15 Binary files /dev/null and b/doc/images/MGARD-family2.png differ diff --git a/doc/release notes/0.0.0.2.md b/doc/release notes/0.0.0.2.md new file mode 100644 index 0000000000..728db00cca --- /dev/null +++ b/doc/release notes/0.0.0.2.md @@ -0,0 +1,7 @@ +# MGARD 0.0.0.2 Release Notes + +**Released:** September 30, 2019 + +## Added +* Initial public release, with support for lossy compression while preserving quantities of interest (QoIs). +* Single-precision (FP32) support. diff --git a/doc/release notes/0.1.0.md b/doc/release notes/0.1.0.md new file mode 100644 index 0000000000..89094ed0e4 --- /dev/null +++ b/doc/release notes/0.1.0.md @@ -0,0 +1,13 @@ +# MGARD 0.1.0 Release Notes + +**Released:** September 29, 2020 + +## Added +* Initial support for unstructured data. +* GPU support for 2D and 3D. +* Entropy encoding using Huffman. +* Integration with ZSTD. +* Continuous integration using Travis CI. + +## Changed +* Restructured code for expandability. diff --git a/doc/release notes/1.0.0.md b/doc/release notes/1.0.0.md new file mode 100644 index 0000000000..8e6ae5ebd0 --- /dev/null +++ b/doc/release notes/1.0.0.md @@ -0,0 +1,22 @@ +# MGARD 1.0.0 Release Notes + +**Released:** September 30, 2021 + +## Added +* Self-describing command-line executable. +* Support for 'flat' datasets. +* A command line interface. +* High-level APIs for easier integration with user programs and I/O libraries. +* Support for choosing different lossless compressors. +* Self-describing decompression API. +* Examples. + +## Changed +* Migrated the continuous integration setup. +* Improved CPU compression and decompression speed (iterator optimizations, index precomputation, shuffled values for better memory access patterns). +* Parallelized operator application with OpenMP. +* Improved quantization speed. +* Improved the performance of the multilevel decomposition. + +## Fixed +* Fixed several bugs in the multilevel decomposition. diff --git a/doc/release notes/1.3.0.md b/doc/release notes/1.3.0.md new file mode 100644 index 0000000000..bd7ba02e5b --- /dev/null +++ b/doc/release notes/1.3.0.md @@ -0,0 +1,15 @@ +# MGARD 1.3.0 Release Notes + +**Released:** September 30, 2022 + +## Added +* ***MGARD-X***: portable compression for CPU and GPU. + * Support for CPU (serial), CPU (multi-threaded), NVIDIA GPUs, AMD GPUs, and Intel GPUs. + * Self-describing format. + * Automatic domain decomposition for handling large datasets. + * Automatic parallel compression across multiple GPUs. + * High-level compression API for automatic metadata management and domain decomposition. + * Low-level compression API for full control over the compression process on devices. +* ***MDR-X***: portable multi-precision data refactoring on CPU and GPU. + * Modular design allowing flexible control over the data refactoring and reconstruction process. + * Same portability as MGARD-X. diff --git a/doc/release notes/1.4.0.md b/doc/release notes/1.4.0.md new file mode 100644 index 0000000000..c689ef9bca --- /dev/null +++ b/doc/release notes/1.4.0.md @@ -0,0 +1,28 @@ +# MGARD 1.4.0 Release Notes + +**Released:** January 10, 2023 + +## Added +* Multi-device support for high-level compression/decompression. +* Multi-device support in RuntimeX and Array. +* Workspace pre-allocation functionality for low-level compression/decompression. +* Logging functions. +* A build option to enable/disable uniform coordinate normalization. +* A new OpenMP backend. +* A new DeviceLauncher. +* High-level APIs for MDR-X. +* Block-based domain decomposition method. +* Ability to disable autotuning, greatly reducing build time. +* Updated build scripts for the Ampere architecture and integration with ADIOS2. + +## Changed +* Modularized the compression workflow and the multi-precision refactoring workflow. +* Improved quantization performance. + +## Fixed +* Fixed an issue with GCC 9. +* Fixed a synchronization issue with Huffman encoding. +* Fixed ambiguous overloads for Xcode. +* Fixed a build issue when multiple backends are enabled. +* Fixed a bug in the 3D correction calculation. +* Fixed an issue with collective operations in the SYCL backend. diff --git a/doc/release notes/1.5.0.md b/doc/release notes/1.5.0.md new file mode 100644 index 0000000000..a4fa10d878 --- /dev/null +++ b/doc/release notes/1.5.0.md @@ -0,0 +1,17 @@ +# MGARD 1.5.0 Release Notes + +**Released:** April 1, 2023 + +## Added +* New ***MGARD-$\lambda$*** compression pipeline for preserving non-linear QoIs in XGC data. +* New ***MGARD-RoI*** compression pipeline for preserving Region-of-Interest (RoI). +* New GPU compression pipeline for out-of-core, large-scale data compression. +* Support for Apple Silicon (ARM) architecture. + +## Fixed +* Fixed an issue with older CUDA versions. +* Fixed an issue with CUDA 12+. +* Fixed an issue with the NVIDIA HPC SDK. +* Fixed a compilation bug in MDR-X. +* Fixed an issue with linear quantization overflow. +* Fixed an issue with using device buffers as input for compression. diff --git a/doc/release notes/1.5.2.md b/doc/release notes/1.5.2.md new file mode 100644 index 0000000000..7fabd76636 --- /dev/null +++ b/doc/release notes/1.5.2.md @@ -0,0 +1,25 @@ +# MGARD 1.5.2 Release Notes + +**Released:** September 29, 2023 + +## Added +* Compression status return for the high-level API. +* Build example with ADIOS2 operators. +* Autotuning for kernels in Huffman encoding. +* Asynchronous LZ4 and Zstd compression. +* Pipeline optimization for compressing time-series data. +* Compressor cache optimization. +* Functions to explicitly initialize and finalize RuntimeX. +* A function to estimate the max output size of MDR-X. +* A README for the build scripts. + +## Changed +* Improved memory usage estimation. + +## Fixed +* Fixed a bug in Huffman codebook generation. +* Fixed a bug when using LZ4 compression. +* Fixed a bug in the domain decomposer. +* Fixed a bug when using reduced memory footprint mode. +* Fixed a bug in MDR-X reconstruction. +* Fixed a bug when calculating the sign bit using GCC. diff --git a/doc/release notes/1.6.0.md b/doc/release notes/1.6.0.md new file mode 100644 index 0000000000..d6458e57c0 --- /dev/null +++ b/doc/release notes/1.6.0.md @@ -0,0 +1,24 @@ +# MGARD 1.6.0 Release Notes + +**Released:** August 27, 2025 + +## Added +* Redesigned compression/decompression pipeline for higher end-to-end throughput. +* Improved *mgard-x* and *mdr-x* executable options. + +## Changed +* Improved OpenMP performance. +* Improved Huffman's performance on CPUs. +* Improved the linking method with ZSTD. +* Updated the GitHub testing workflow. + +## Removed +* Removed the prefetch option (always enabled now). +* Removed the coordinate-normalization option from the build system. + +## Fixed +* Fixed an issue with LZ4 compression. +* Fixed a thread safety issue. +* Fixed an MDR-X L2 error control issue. +* Fixed an integration issue with ADIOS2. +* Fixed a build issue with HIP. diff --git a/doc/release notes/1.7.0.md b/doc/release notes/1.7.0.md new file mode 100644 index 0000000000..9f62f6b3ad --- /dev/null +++ b/doc/release notes/1.7.0.md @@ -0,0 +1,32 @@ +# MGARD 1.7.0 Release Notes + +**Released:** Unreleased + +## Added +* ***HP-MDR***: a high-performance MDR-X refactoring and reconstruction pipeline, including batched bitplane (BP) encoders (v1b/v2a/OptV1), negabinary encoding, run-length encoding, adaptive and hybrid lossless compression for MDR, improved domain decomposition, better output-size estimation, and a QoI-guided progressive retrieval pipeline. +* ***BlockMGARD***: a block-based hybrid hierarchy compression pipeline with in-cache block refactoring (1D/2D/3D), region-of-interest (RoI) support, L2/L-infinity error bound control, and kernel-fusion optimizations. +* A symbol-alphabet rANS lossless backend for mgard-x, beating Huffman's compression ratio. +* A BlockDelta (FLE-based) lossless backend for mgard-x, as a Huffman alternative. +* A portable, warp-cooperative LZ4 implementation for mgard-x, removing the nvcomp dependency. +* Blackwell (sm_120) GPU build support. +* Additional unit tests: bitplane encoders, DirectInterleaver, MGARDDecomposer, HybridLevelCompressor, and ComposedRefactor/Reconstructor; unit tests are now discoverable by CTest. + +## Changed +* Deprecated and removed the legacy standalone MGARD-CUDA backend in favor of ***MGARD-X***. +* Restructured the original MGARD CPU source tree under a dedicated `mgard` directory, and reorganized unit tests. +* Replaced `exit()` calls with C++ exceptions for internal errors across mgard-x. +* Optimized the Huffman compress/decompress pipeline (roughly 1.9x faster compression). +* Bumped the bundled ZSTD to v1.5.6 and protobuf to v3.21.12; stopped building nvcomp. +* Raised the minimum ROCm requirement to 7.0 for HIP builds. +* Updated documentation: the MGARD-X guide for the hybrid hierarchy pipeline, a hybrid RoI user guide, the MGARD family tree, performance figures, and the publications list. + +## Fixed +* Fixed an MDR-X sign-buffer heap overflow in bitplane decoding. +* Fixed uninitialized padding in ComposedRefactor level buffers. +* Fixed MaxErrorEstimatorHB missing a negabinary error margin. +* Fixed a stale active-device bug in the CUDA/HIP DeviceSpecification and DeviceQueues. +* Fixed stale leading dimensions on Array shrink. +* Fixed GreedyBasedSizeInterpreter fetching more bitplanes than necessary. +* Fixed a floating-point exception in Zstd::Decompress. +* Fixed CPUPipelines so OpenMP-only builds compile again. +* Fixed clang-format violations flagged by CI. diff --git a/examples/ADIOS2/adios2-test-external-compressor.cpp b/examples/ADIOS2/adios2-test-external-compressor.cpp index 0a24996b37..e12d52a1b8 100644 --- a/examples/ADIOS2/adios2-test-external-compressor.cpp +++ b/examples/ADIOS2/adios2-test-external-compressor.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include diff --git a/examples/ADIOS2/adios2-test.cpp b/examples/ADIOS2/adios2-test.cpp index c97259c8ed..d97f4bba97 100644 --- a/examples/ADIOS2/adios2-test.cpp +++ b/examples/ADIOS2/adios2-test.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include diff --git a/examples/lambda/README.md b/examples/lambda/README.md index ed382c3123..c8e6581764 100644 --- a/examples/lambda/README.md +++ b/examples/lambda/README.md @@ -1,8 +1,8 @@ -# MGARD-Lambda +# *MGARD*$\text{-}\lambda$ Data compression while preserving non-linear quantities of interest. -*MGARD-Lambda is an experimental part of MGARD. Currently only supporting certain QoIs derived from XGC 5D data.* +*MGARD*$\text{-}\lambda$ is an experimental part of MGARD. Currently supports only certain QoIs derived from XGC 5D data. #### Prerequisites: @@ -11,7 +11,7 @@ Data compression while preserving non-linear quantities of interest. * CMake: version > 19.0 * ADIOS2: any version -First, build and install MGARD. Then, build the MGARD- example as follows: +First, build and install *MGARD*. Then, build the *MGARD*$\text{-}\lambda$ example as follows: ``` $ build_script.sh diff --git a/examples/lambda/TestXGCDecompress.cpp b/examples/lambda/TestXGCDecompress.cpp index f472115748..69b7c1ec83 100644 --- a/examples/lambda/TestXGCDecompress.cpp +++ b/examples/lambda/TestXGCDecompress.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2021, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include diff --git a/examples/lambda/TestXGCPostProcessing.cpp b/examples/lambda/TestXGCPostProcessing.cpp index eb63b9a373..6e716f967e 100644 --- a/examples/lambda/TestXGCPostProcessing.cpp +++ b/examples/lambda/TestXGCPostProcessing.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2021, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include diff --git a/examples/mgard-x/AdaptiveResolution/FlyingEdges.hpp b/examples/mgard-x/AdaptiveResolution/FlyingEdges.hpp index 10ca069d9a..2cdba07ee6 100644 --- a/examples/mgard-x/AdaptiveResolution/FlyingEdges.hpp +++ b/examples/mgard-x/AdaptiveResolution/FlyingEdges.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_FLYING_EDGES_HPP diff --git a/examples/mgard-x/AdaptiveResolution/FlyingEdgesBatched.hpp b/examples/mgard-x/AdaptiveResolution/FlyingEdgesBatched.hpp index dce2c3d1e1..998d7fb0ec 100644 --- a/examples/mgard-x/AdaptiveResolution/FlyingEdgesBatched.hpp +++ b/examples/mgard-x/AdaptiveResolution/FlyingEdgesBatched.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_FLYING_EDGES_BATCHED_HPP diff --git a/examples/mgard-x/AdaptiveResolution/SparseFlyingCells.hpp b/examples/mgard-x/AdaptiveResolution/SparseFlyingCells.hpp index 2b6e57adac..0b7562f7db 100644 --- a/examples/mgard-x/AdaptiveResolution/SparseFlyingCells.hpp +++ b/examples/mgard-x/AdaptiveResolution/SparseFlyingCells.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_SPARSE_FLYING_CELLS_HPP diff --git a/examples/mgard-x/AdaptiveResolution/SparseFlyingEdges.hpp b/examples/mgard-x/AdaptiveResolution/SparseFlyingEdges.hpp index c75e98cf8f..7d0e0dc0c9 100644 --- a/examples/mgard-x/AdaptiveResolution/SparseFlyingEdges.hpp +++ b/examples/mgard-x/AdaptiveResolution/SparseFlyingEdges.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_SPARSE_FLYING_EDGES_HPP diff --git a/examples/mgard-x/BatchTests/BatchTests.cpp b/examples/mgard-x/BatchTests/BatchTests.cpp index be63bc47e8..0606355c33 100644 --- a/examples/mgard-x/BatchTests/BatchTests.cpp +++ b/examples/mgard-x/BatchTests/BatchTests.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2021, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: September 27, 2021 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include @@ -16,10 +16,8 @@ #include #include "mgard/compress.hpp" -#include "mgard/compress_cuda.hpp" #include "mgard/compress_x.hpp" #include "mgard/mgard-x/Utilities/ErrorCalculator.h" -// #include "compress_cuda.hpp" #define ANSI_RED "\x1b[31m" #define ANSI_GREEN "\x1b[32m" @@ -27,7 +25,7 @@ using namespace std::chrono; -enum device { CPU, CUDA, X_CUDA, X_HIP, X_Serial }; +enum device { CPU, X_CUDA, X_HIP, X_Serial }; enum data_type { SINGLE, DOUBLE }; enum error_type { ABS, REL }; @@ -101,32 +99,6 @@ void compression(std::vector shape, enum device dev, T tol, T s, compressed_size = tmp_str.length(); compressed_data = (void *)malloc(compressed_size); memcpy(compressed_data, tmp_str.c_str(), compressed_size); - } else if (dev == CUDA) { -#if MGARD_ENABLE_LEGACY_CUDA - mgard_cuda::Config config; - config.lossless = mgard_cuda::lossless_type::GPU_Huffman_LZ4; - config.sync_and_check_all_kernels = true; - config.uniform_coord_mode = 1; - - mgard_cuda::data_type dtype; - if (std::is_same::value) { - dtype = mgard_cuda::data_type::Double; - } else if (std::is_same::value) { - dtype = mgard_cuda::data_type::Float; - } - - mgard_cuda::error_bound_type ebtype; - if (mode == error_type::ABS) { - ebtype = mgard_cuda::error_bound_type::ABS; - } else if (mode == error_type::REL) { - ebtype = mgard_cuda::error_bound_type::REL; - } - - mgard_cuda::compress(D, dtype, shape, tol, s, ebtype, original_data, - compressed_data, compressed_size, config); -#else - std::cout << "MGARD legacy CUDA was not built.\n"; -#endif } else { mgard_x::Config config; config.lossless = mgard_x::lossless_type::Huffman_Zstd; @@ -180,24 +152,6 @@ void decompression(std::vector shape, enum device dev, T tol, const void *decompressed_data_void = new_data_.data.get(); memcpy(decompressed_data, decompressed_data_void, original_size * sizeof(T)); - } else if (dev == CUDA) { -#if MGARD_ENABLE_LEGACY_CUDA - mgard_cuda::Config config; - config.lossless = mgard_cuda::lossless_type::GPU_Huffman; - config.sync_and_check_all_kernels = true; - config.uniform_coord_mode = 1; - mgard_cuda::data_type dtype; - if (std::is_same::value) { - dtype = mgard_cuda::data_type::Double; - } else if (std::is_same::value) { - dtype = mgard_cuda::data_type::Float; - } - - mgard_cuda::decompress(compressed_data, compressed_size, decompressed_data, - config); -#else - std::cout << "MGARD legacy CUDA was not built.\n"; -#endif } else { mgard_x::Config config; config.lossless = mgard_x::lossless_type::Huffman; @@ -371,11 +325,6 @@ int main(int argc, char *argv[]) { std::cout << "CPU\n"; } - if (strcmp(dev1, "cuda") == 0) { - device_type1 = device::CUDA; - std::cout << "LEGACY_CUDA\n"; - } - std::cout << "Device2: "; if (strcmp(dev2, "x-serial") == 0) { dev_type = mgard_x::device_type::SERIAL; @@ -396,11 +345,6 @@ int main(int argc, char *argv[]) { std::cout << "CPU\n"; } - if (strcmp(dev2, "cuda") == 0) { - device_type2 = device::CUDA; - std::cout << "LEGACY_CUDA\n"; - } - std::vector> shapes; // shapes.push_back({1000000}); diff --git a/examples/mgard-x/BatchTests/CMakeLists.txt b/examples/mgard-x/BatchTests/CMakeLists.txt index 84395ddefd..e22ebc6b74 100644 --- a/examples/mgard-x/BatchTests/CMakeLists.txt +++ b/examples/mgard-x/BatchTests/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.19) -project(mgard_cuda_cpu_compare CXX C) +project(mgard_x_cpu_compare CXX C) find_package(mgard) add_executable(BatchTests BatchTests.cpp) diff --git a/examples/mgard-x/CompressXgcData/TestXGCAbsoluteError.cpp b/examples/mgard-x/CompressXgcData/TestXGCAbsoluteError.cpp index 41432ebf69..476bdaf1d5 100644 --- a/examples/mgard-x/CompressXgcData/TestXGCAbsoluteError.cpp +++ b/examples/mgard-x/CompressXgcData/TestXGCAbsoluteError.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2021, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include diff --git a/examples/mgard-x/FlyingEdges/FlyingEdges.hpp b/examples/mgard-x/FlyingEdges/FlyingEdges.hpp index be44ed7aba..d01b636005 100644 --- a/examples/mgard-x/FlyingEdges/FlyingEdges.hpp +++ b/examples/mgard-x/FlyingEdges/FlyingEdges.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_FLYING_EDGES_HPP diff --git a/examples/mgard-x/HighLevelAPIsGPUBuffer/README.md b/examples/mgard-x/HighLevelAPIsGPUBuffer/README.md index 10a22a1bd2..6cfe2ef8a8 100644 --- a/examples/mgard-x/HighLevelAPIsGPUBuffer/README.md +++ b/examples/mgard-x/HighLevelAPIsGPUBuffer/README.md @@ -1,7 +1,7 @@ -# Compressing with MGARD-X High-level APIs +# Compressing with MGARD-X High-level APIs (GPU-resident buffers) First, build and install MGARD-X. -Then, run the following in `examples/mgard-x/HighLevelAPIs`. +Then, run the following in `examples/mgard-x/HighLevelAPIsGPUBuffer`. Build with CMake as follows or use the 'build_scripts.sh'. ```console @@ -11,5 +11,5 @@ $ build/Example ``` -`build/main` creates a dataset, compresses it with MGARD-X on NVIDIA GPU, and decomrpess it on CPU. -Read `Example.cpp` to see how the high-level compression API is used. +`build/Example` creates a dataset, compresses it with MGARD-X on NVIDIA GPU, and decomrpess it on CPU, passing GPU-resident buffers directly to the high-level API instead of host buffers. +Read `Example.cu` to see how the high-level compression API is used with GPU buffers. diff --git a/examples/mgard-x/LowLevelAPIs/CUDA/Example.cu b/examples/mgard-x/LowLevelAPIs/CUDA/Example.cu index b13af534ef..b35723e5cb 100644 --- a/examples/mgard-x/LowLevelAPIs/CUDA/Example.cu +++ b/examples/mgard-x/LowLevelAPIs/CUDA/Example.cu @@ -14,7 +14,9 @@ int main() { //... load data into in_array_cpu std::vector shape{n1, n2, n3}; mgard_x::Config config; - config.lossless = mgard_x::lossless_type::Huffman_LZ4; + // Huffman_LZ4 currently has a known buffer-sizing bug in the library + // (tracked separately); use Huffman_Zstd here until that is fixed. + config.lossless = mgard_x::lossless_type::Huffman_Zstd; mgard_x::Hierarchy<3, double, mgard_x::CUDA> hierarchy(shape, config); mgard_x::Array<3, double, mgard_x::CUDA> in_array(shape); in_array.load(in_array_cpu); diff --git a/examples/mgard-x/LowLevelAPIs/SERIAL/Example.cpp b/examples/mgard-x/LowLevelAPIs/SERIAL/Example.cpp index 6433893069..7f468ae04b 100644 --- a/examples/mgard-x/LowLevelAPIs/SERIAL/Example.cpp +++ b/examples/mgard-x/LowLevelAPIs/SERIAL/Example.cpp @@ -14,7 +14,9 @@ int main() { //... load data into in_array_cpu std::vector shape{n1, n2, n3}; mgard_x::Config config; - config.lossless = mgard_x::lossless_type::Huffman_LZ4; + // Huffman_LZ4 currently has a known buffer-sizing bug in the library + // (tracked separately); use Huffman_Zstd here until that is fixed. + config.lossless = mgard_x::lossless_type::Huffman_Zstd; mgard_x::Hierarchy<3, double, mgard_x::SERIAL> hierarchy(shape, config); mgard_x::Array<3, double, mgard_x::SERIAL> in_array(shape); in_array.load(in_array_cpu); diff --git a/examples/mgard-x/MDR-X/CUDA/reconstructor.cu b/examples/mgard-x/MDR-X/CUDA/reconstructor.cu deleted file mode 100644 index f8dcc7d6c4..0000000000 --- a/examples/mgard-x/MDR-X/CUDA/reconstructor.cu +++ /dev/null @@ -1,204 +0,0 @@ -#include "mgard/mdr_x.hpp" -#include "mgard/mgard-x/Utilities/ErrorCalculator.h" -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace std; - -template -std::vector readfile(const char *file, size_t &num) { - std::ifstream fin(file, std::ios::binary); - if (!fin) { - std::cout << " Error, Couldn't find the file" - << "\n"; - return std::vector(); - } - fin.seekg(0, std::ios::end); - const size_t num_elements = fin.tellg() / sizeof(Type); - fin.seekg(0, std::ios::beg); - auto data = std::vector(num_elements); - fin.read(reinterpret_cast(&data[0]), num_elements * sizeof(Type)); - fin.close(); - num = num_elements; - return data; -} - -template -void print_statistics(const T_data *data_ori, const T_data *data_dec, - size_t data_size) { - double max_val = data_ori[0]; - double min_val = data_ori[0]; - double max_abs = fabs(data_ori[0]); - for (int i = 0; i < data_size; i++) { - if (data_ori[i] > max_val) - max_val = data_ori[i]; - if (data_ori[i] < min_val) - min_val = data_ori[i]; - if (fabs(data_ori[i]) > max_abs) - max_abs = fabs(data_ori[i]); - } - double max_err = 0; - int pos = 0; - double mse = 0; - for (int i = 0; i < data_size; i++) { - double err = data_ori[i] - data_dec[i]; - mse += err * err; - if (fabs(err) > max_err) { - pos = i; - max_err = fabs(err); - } - } - mse /= data_size; - double psnr = 20 * log10((max_val - min_val) / sqrt(mse)); - cout << "Max value = " << max_val << ", min value = " << min_val << endl; - cout << "Max error = " << max_err << ", pos = " << pos << endl; - cout << "MSE = " << mse << ", PSNR = " << psnr << endl; - cout << "L2 error = " - << mgard_x::L_2_error({(mgard_x::SIZE)data_size}, data_ori, data_dec, - mgard_x::error_bound_type::ABS, 0) - << endl; - cout << "L_inf error = " - << mgard_x::L_inf_error(data_size, data_ori, data_dec, - mgard_x::error_bound_type::ABS) - << endl; -} - -template -void test(string filename, const vector &tolerance, - mgard_x::Hierarchy &hierarchy, - std::string metadata_file, std::vector files, double s) { - auto reconstructor = - mgard_x::MDR::ComposedReconstructor( - hierarchy, metadata_file, files); - cout << "loading metadata" << endl; - reconstructor.load_metadata(); - - size_t num_elements = 0; - auto data = readfile(filename.c_str(), num_elements); - for (int i = 0; i < tolerance.size(); i++) { - mgard_x::log::level |= mgard_x::log::TIME; - // mgard_x::Timer timer; - // timer.start(); - mgard_x::Array reconstructed_data; - reconstructor.progressive_reconstruct(tolerance[i], s, reconstructed_data); - // timer.end(); - // timer.print("Reconstruct"); - auto dims = reconstructor.get_dimensions(); - size_t size = 1; - for (int i = 0; i < dims.size(); i++) { - size *= dims[i]; - } - print_statistics(data.data(), reconstructed_data.hostCopy(), size); - } -} - -int main(int argc, char **argv) { - - int argv_id = 1; - string filename = string(argv[argv_id++]); - // int error_mode = atoi(argv[argv_id++]); - int num_tolerance = atoi(argv[argv_id++]); - vector tolerance(num_tolerance, 0); - for (int i = 0; i < num_tolerance; i++) { - tolerance[i] = atof(argv[argv_id++]); - } - double s = atof(argv[argv_id++]); - - string metadata_file = "refactored_data/metadata.bin"; - int num_levels = 0; - int num_dims = 0; - vector dims; - { - // metadata interpreter, otherwise information needs to be provided - size_t num_bytes = 0; - auto metadata = readfile(metadata_file.c_str(), num_bytes); - assert(num_bytes > num_dims * sizeof(mgard_x::SIZE) + 2); - num_dims = metadata[0]; - mgard_x::SIZE *dim = (mgard_x::SIZE *)&(metadata[1]); - printf("dim: "); - for (int i = 0; i < num_dims; i++) { - dims.push_back(dim[i]); - printf("%lu ", dim[i]); - } - printf("\n"); - num_levels = metadata[num_dims * sizeof(mgard_x::SIZE) + 1]; - cout << "number of dimension = " << num_dims - << ", number of levels = " << num_levels << endl; - } - vector files; - for (int i = 0; i < num_levels; i++) { - string filename = "refactored_data/level_" + to_string(i) + ".bin"; - files.push_back(filename); - } - - using T_data = float; - using T_stream = uint32_t; - using T_error = double; - using DeviceType = mgard_x::CUDA; - - const mgard_x::DIM D = 3; - mgard_x::Config config; - config.max_larget_level = num_levels - 1; - mgard_x::Hierarchy hierarchy(dims, config); - - auto decomposer = - mgard_x::MDR::MGARDOrthoganalDecomposer(hierarchy); - auto interleaver = - mgard_x::MDR::DirectInterleaver(hierarchy); - auto encoder = - mgard_x::MDR::GroupedBPEncoder( - hierarchy); - // auto encoder = - // mgard_x::MDR::GroupedWarpBPEncoder(hierarchy); - auto compressor = mgard_x::MDR::DefaultLevelCompressor( - hierarchy.total_num_elems(), 8192, 20480, 1.0); - auto retriever = mgard_x::MDR::ConcatLevelFileRetriever(metadata_file, files); - - // switch (error_mode) { - // case 1: - if (s != std::numeric_limits::infinity()) { - auto estimator = - mgard_x::MDR::SNormErrorEstimator(num_dims, num_levels - 1, s); - // auto interpreter = - // mgard_x::MDR::SignExcludeGreedyBasedSizeInterpreter>(estimator); - // auto interpreter = - // mgard_x::MDR::NegaBinaryGreedyBasedSizeInterpreter>(estimator); - - auto interpreter = mgard_x::MDR::RoundRobinSizeInterpreter< - mgard_x::MDR::SNormErrorEstimator>(estimator); - // auto interpreter = - // mgard_x::MDR::InorderSizeInterpreter>(estimator); - // auto estimator = mgard_x::MDR::L2ErrorEstimator_HB(num_dims, - // num_levels - 1); auto interpreter = - // mgard_x::MDR::SignExcludeGreedyBasedSizeInterpreter>(estimator); - test(filename, tolerance, hierarchy, - metadata_file, files, s); - // break; - // } - // default: - } else { - auto estimator = mgard_x::MDR::MaxErrorEstimatorOB(num_dims); - auto interpreter = mgard_x::MDR::SignExcludeGreedyBasedSizeInterpreter< - mgard_x::MDR::MaxErrorEstimatorOB>(estimator); - // auto interpreter = - // MDR::RoundRobinSizeInterpreter>(estimator); - // auto interpreter = - // MDR::InorderSizeInterpreter>(estimator); - // auto estimator = MDR::MaxErrorEstimatorHB(); - // auto interpreter = - // MDR::SignExcludeGreedyBasedSizeInterpreter>(estimator); - test(filename, tolerance, hierarchy, - metadata_file, files, s); - } - - // } - - return 0; -} \ No newline at end of file diff --git a/examples/mgard-x/MDR-X/CUDA/refactor.cu b/examples/mgard-x/MDR-X/CUDA/refactor.cu index 5782345c24..d946c53674 100644 --- a/examples/mgard-x/MDR-X/CUDA/refactor.cu +++ b/examples/mgard-x/MDR-X/CUDA/refactor.cu @@ -80,6 +80,8 @@ void test(string filename, int num_bitplanes, auto refactor = mgard_x::MDR::ComposedRefactor( hierarchy, config); refactor.Refactor(input_array, mdr_metadata, mdr_data, 0); + refactor.Compress(mdr_metadata, mdr_data, 0); + refactor.StoreMetadata(mdr_metadata, mdr_data, 0); } { @@ -97,12 +99,12 @@ void test(string filename, int num_bitplanes, mgard_x::log::level |= mgard_x::log::TIME; // mgard_x::Timer timer; // timer.start(); - reconstructor.GenerateRequest(mdr_metadata, tolerance[i], s); + mdr_metadata.requested_tol = tolerance[i]; + mdr_metadata.requested_s = s; + reconstructor.GenerateRequest(mdr_metadata); mdr_metadata.PrintStatus(); mdr_metadata.DoneLoadingBitplans(); - // reconstructor.progressive_reconstruct(tolerance[i], s, - // reconstructed_data); - reconstructor.ProgressiveReconstruct(mdr_metadata, mdr_data, + reconstructor.ProgressiveReconstruct(mdr_metadata, mdr_data, false, reconstructed_data, 0); // timer.end(); // timer.print("Reconstruct"); @@ -167,57 +169,7 @@ int main(int argc, char **argv) { config.max_larget_level = target_level; mgard_x::Hierarchy hierarchy(dims, config); - // if (false) { - auto decomposer = - mgard_x::MDR::MGARDOrthoganalDecomposer(hierarchy); - - auto interleaver = - mgard_x::MDR::DirectInterleaver(hierarchy); - // auto interleaver = mgard_x::MDR::SFCInterleaver(); - // auto interleaver = mgard_x::MDR::BlockedInterleaver(); - - auto encoder = - mgard_x::MDR::GroupedBPEncoder( - hierarchy); - // auto encoder = - // mgard_x::MDR::GroupedWarpBPEncoder( - // hierarchy); - - auto compressor = mgard_x::MDR::DefaultLevelCompressor( - hierarchy.total_num_elems(), config); - // auto compressor = mgard_x::MDR::AdaptiveLevelCompressor(32); - // auto compressor = mgard_x::MDR::NullLevelCompressor(); - - // auto collector = mgard_x::MDR::SquaredErrorCollector(); - auto collector = mgard_x::MDR::MaxErrorCollector(); - - auto writer = mgard_x::MDR::ConcatLevelFileWriter(metadata_file, files); - // auto writer = mgard_x::MDR::HPSSFileWriter(metadata_file, files, 2048, - // 512 * 1024 * 1024); - test(filename, num_bitplanes, hierarchy, metadata_file, files, tolerance, s); - - // test2(filename, dims, target_level, num_bitplanes, decomposer, - // interleaver, encoder, compressor, collector, writer); - // } - - // if (true) { - // std::vector> level_signs; - - // auto decomposer = mgard_x::MDR::MGARDOrthoganalDecomposer(hierarchy); - // auto interleaver = mgard_x::MDR::DirectInterleaver(hierarchy); - // auto encoder = mgard_x::MDR::GroupedBPEncoder(); - // auto encoder = - // mgard_m::MDR::GroupedWarpBPEncoder(); - // auto compressor = - // mgard_m::MDR::DefaultLevelCompressor(); - // auto collector = mgard_x::MDR::SquaredErrorCollector(); - // auto writer = mgard_x::MDR::ConcatLevelFileWriter(metadata_file, files); - // test2( - // filename, dims, target_level, num_bitplanes, hierarchy, decomposer, - // interleaver, encoder, compressor, collector, writer); - // } return 0; } \ No newline at end of file diff --git a/examples/mgard-x/MDR-X/HIP/CMakeLists.txt b/examples/mgard-x/MDR-X/HIP/CMakeLists.txt index bc86ab9446..fef4b7c556 100644 --- a/examples/mgard-x/MDR-X/HIP/CMakeLists.txt +++ b/examples/mgard-x/MDR-X/HIP/CMakeLists.txt @@ -6,8 +6,5 @@ find_package(mgard) set (CMAKE_CXX_STANDARD 17) set (CMAKE_CXX_STANDARD_REQUIRED ON) set_source_files_properties(refactor.cpp PROPERTIES LANGUAGE HIP) -set_source_files_properties(reconstructor.cpp PROPERTIES LANGUAGE HIP) add_executable(refactor refactor.cpp) target_link_libraries(refactor mgard::mgard) -add_executable(reconstructor reconstructor.cpp) -target_link_libraries(reconstructor mgard::mgard) diff --git a/examples/mgard-x/MDR-X/HIP/reconstructor.cpp b/examples/mgard-x/MDR-X/HIP/reconstructor.cpp deleted file mode 100644 index 7612932ed1..0000000000 --- a/examples/mgard-x/MDR-X/HIP/reconstructor.cpp +++ /dev/null @@ -1,203 +0,0 @@ -#include "mgard/mdr_x.hpp" -#include "mgard/mgard-x/Utilities/ErrorCalculator.h" -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace std; - -template -std::vector readfile(const char *file, size_t &num) { - std::ifstream fin(file, std::ios::binary); - if (!fin) { - std::cout << " Error, Couldn't find the file" - << "\n"; - return std::vector(); - } - fin.seekg(0, std::ios::end); - const size_t num_elements = fin.tellg() / sizeof(Type); - fin.seekg(0, std::ios::beg); - auto data = std::vector(num_elements); - fin.read(reinterpret_cast(&data[0]), num_elements * sizeof(Type)); - fin.close(); - num = num_elements; - return data; -} - -template -void print_statistics(const T_data *data_ori, const T_data *data_dec, - size_t data_size) { - double max_val = data_ori[0]; - double min_val = data_ori[0]; - double max_abs = fabs(data_ori[0]); - for (int i = 0; i < data_size; i++) { - if (data_ori[i] > max_val) - max_val = data_ori[i]; - if (data_ori[i] < min_val) - min_val = data_ori[i]; - if (fabs(data_ori[i]) > max_abs) - max_abs = fabs(data_ori[i]); - } - double max_err = 0; - int pos = 0; - double mse = 0; - for (int i = 0; i < data_size; i++) { - double err = data_ori[i] - data_dec[i]; - mse += err * err; - if (fabs(err) > max_err) { - pos = i; - max_err = fabs(err); - } - } - mse /= data_size; - double psnr = 20 * log10((max_val - min_val) / sqrt(mse)); - cout << "Max value = " << max_val << ", min value = " << min_val << endl; - cout << "Max error = " << max_err << ", pos = " << pos << endl; - cout << "MSE = " << mse << ", PSNR = " << psnr << endl; - cout << "L2 error = " - << mgard_x::L_2_error({(mgard_x::SIZE)data_size}, data_ori, data_dec, - mgard_x::error_bound_type::ABS, 0) - << endl; - cout << "L_inf error = " - << mgard_x::L_inf_error(data_size, data_ori, data_dec, - mgard_x::error_bound_type::ABS) - << endl; -} - -template -void test(string filename, const vector &tolerance, - mgard_x::Hierarchy &hierarchy, - Decomposer decomposer, Interleaver interleaver, Encoder encoder, - Compressor compressor, ErrorEstimator estimator, - SizeInterpreter interpreter, Retriever retriever) { - auto reconstructor = mgard_x::MDR::ComposedReconstructor< - D, T_data, T_stream, Decomposer, Interleaver, Encoder, Compressor, - SizeInterpreter, ErrorEstimator, Retriever, DeviceType>( - hierarchy, decomposer, interleaver, encoder, compressor, interpreter, - retriever); - cout << "loading metadata" << endl; - reconstructor.load_metadata(); - - size_t num_elements = 0; - auto data = readfile(filename.c_str(), num_elements); - for (int i = 0; i < tolerance.size(); i++) { - cout << "Start reconstruction" << endl; - mgard_x::Array reconstructed_data = - reconstructor.progressive_reconstruct(tolerance[i]); - auto dims = reconstructor.get_dimensions(); - size_t size = 1; - for (int i = 0; i < dims.size(); i++) { - size *= dims[i]; - } - print_statistics(data.data(), reconstructed_data.hostCopy(), size); - } -} - -int main(int argc, char **argv) { - - int argv_id = 1; - string filename = string(argv[argv_id++]); - int error_mode = atoi(argv[argv_id++]); - int num_tolerance = atoi(argv[argv_id++]); - vector tolerance(num_tolerance, 0); - for (int i = 0; i < num_tolerance; i++) { - tolerance[i] = atof(argv[argv_id++]); - } - double s = atof(argv[argv_id++]); - - string metadata_file = "refactored_data/metadata.bin"; - int num_levels = 0; - int num_dims = 0; - vector dims; - { - // metadata interpreter, otherwise information needs to be provided - size_t num_bytes = 0; - auto metadata = readfile(metadata_file.c_str(), num_bytes); - assert(num_bytes > num_dims * sizeof(mgard_x::SIZE) + 2); - num_dims = metadata[0]; - mgard_x::SIZE *dim = (mgard_x::SIZE *)&(metadata[1]); - printf("dim: "); - for (int i = 0; i < num_dims; i++) { - dims.push_back(dim[i]); - printf("%lu ", dim[i]); - } - printf("\n"); - num_levels = metadata[num_dims * sizeof(mgard_x::SIZE) + 1]; - cout << "number of dimension = " << num_dims - << ", number of levels = " << num_levels << endl; - } - vector files; - for (int i = 0; i < num_levels; i++) { - string filename = "refactored_data/level_" + to_string(i) + ".bin"; - files.push_back(filename); - } - - using T_data = float; - using T_stream = uint32_t; - using T_error = double; - using DeviceType = mgard_x::HIP; - - const mgard_x::DIM D = 3; - mgard_x::Config config; - mgard_x::Hierarchy hierarchy(dims, config, - num_levels - 1); - - auto decomposer = - mgard_x::MDR::MGARDOrthoganalDecomposer(hierarchy); - auto interleaver = - mgard_x::MDR::DirectInterleaver(hierarchy); - // auto encoder = mgard_x::MDR::GroupedBPEncoder(); - auto encoder = mgard_x::MDR::GroupedWarpBPEncoder(); - auto compressor = - mgard_x::MDR::DefaultLevelCompressor(); - auto retriever = mgard_x::MDR::ConcatLevelFileRetriever(metadata_file, files); - - switch (error_mode) { - case 1: { - auto estimator = - mgard_x::MDR::SNormErrorEstimator(num_dims, num_levels - 1, s); - // auto interpreter = - // mgard_x::MDR::SignExcludeGreedyBasedSizeInterpreter>(estimator); - // auto interpreter = - // mgard_x::MDR::NegaBinaryGreedyBasedSizeInterpreter>(estimator); - - auto interpreter = mgard_x::MDR::RoundRobinSizeInterpreter< - mgard_x::MDR::SNormErrorEstimator>(estimator); - // auto interpreter = - // mgard_x::MDR::InorderSizeInterpreter>(estimator); - // auto estimator = mgard_x::MDR::L2ErrorEstimator_HB(num_dims, - // num_levels - 1); auto interpreter = - // mgard_x::MDR::SignExcludeGreedyBasedSizeInterpreter>(estimator); - test( - filename, tolerance, hierarchy, decomposer, interleaver, encoder, - compressor, estimator, interpreter, retriever); - break; - } - default: { - auto estimator = mgard_x::MDR::MaxErrorEstimatorOB(num_dims); - auto interpreter = mgard_x::MDR::SignExcludeGreedyBasedSizeInterpreter< - mgard_x::MDR::MaxErrorEstimatorOB>(estimator); - // auto interpreter = - // MDR::RoundRobinSizeInterpreter>(estimator); - // auto interpreter = - // MDR::InorderSizeInterpreter>(estimator); - // auto estimator = MDR::MaxErrorEstimatorHB(); - // auto interpreter = - // MDR::SignExcludeGreedyBasedSizeInterpreter>(estimator); - test( - filename, tolerance, hierarchy, decomposer, interleaver, encoder, - compressor, estimator, interpreter, retriever); - } - } - - return 0; -} \ No newline at end of file diff --git a/examples/mgard-x/MDR-X/HIP/refactor.cpp b/examples/mgard-x/MDR-X/HIP/refactor.cpp index 4b7f30d2db..2ac338f756 100644 --- a/examples/mgard-x/MDR-X/HIP/refactor.cpp +++ b/examples/mgard-x/MDR-X/HIP/refactor.cpp @@ -1,4 +1,5 @@ -#include "mgard/mdr_x.hpp" +#include "mgard/mdr_x_lowlevel.hpp" +#include "mgard/mgard-x/Utilities/ErrorCalculator.h" #include #include #include @@ -8,37 +9,105 @@ #include using namespace std; -template -void test(string filename, const vector &dims, int target_level, - int num_bitplanes, +template +void print_statistics(const T_data *data_ori, const T_data *data_dec, + size_t data_size) { + double max_val = data_ori[0]; + double min_val = data_ori[0]; + double max_abs = fabs(data_ori[0]); + for (int i = 0; i < data_size; i++) { + if (data_ori[i] > max_val) + max_val = data_ori[i]; + if (data_ori[i] < min_val) + min_val = data_ori[i]; + if (fabs(data_ori[i]) > max_abs) + max_abs = fabs(data_ori[i]); + } + double max_err = 0; + int pos = 0; + double mse = 0; + for (int i = 0; i < data_size; i++) { + double err = data_ori[i] - data_dec[i]; + mse += err * err; + if (fabs(err) > max_err) { + pos = i; + max_err = fabs(err); + } + } + mse /= data_size; + double psnr = 20 * log10((max_val - min_val) / sqrt(mse)); + cout << "Max value = " << max_val << ", min value = " << min_val << endl; + cout << "Max error = " << max_err << ", pos = " << pos << endl; + cout << "MSE = " << mse << ", PSNR = " << psnr << endl; + cout << "L2 error = " + << mgard_x::L_2_error({(mgard_x::SIZE)data_size}, data_ori, data_dec, + mgard_x::error_bound_type::ABS, 0) + << endl; + cout << "L_inf error = " + << mgard_x::L_inf_error(data_size, data_ori, data_dec, + mgard_x::error_bound_type::ABS) + << endl; +} + +template +void test(string filename, int num_bitplanes, mgard_x::Hierarchy &hierarchy, - Decomposer decomposer, Interleaver interleaver, Encoder encoder, - Compressor compressor, ErrorCollector collector, Writer writer) { + std::string metadata_file, std::vector files, + const vector &tolerance, double s) { - auto refactor = - mgard_x::MDR::ComposedRefactor( - hierarchy, decomposer, interleaver, encoder, compressor, collector, - writer); size_t num_elements = 1; printf("loading file\n"); FILE *pFile; pFile = fopen(filename.c_str(), "rb"); - for (int d = 0; d < dims.size(); d++) - num_elements *= dims[d]; - vector data( - num_elements); // MGARD::readfile(filename.c_str(), num_elements); + for (int d = 0; d < D; d++) + num_elements *= hierarchy.level_shape(hierarchy.l_target(), d); + vector data(num_elements); fread(data.data(), 1, num_elements * sizeof(T_data), pFile); fclose(pFile); printf("done loading file\n"); - mgard_x::Array input_array(dims); + mgard_x::Array input_array( + hierarchy.level_shape(hierarchy.l_target())); input_array.load(data.data()); - refactor.refactor(input_array, dims, target_level, num_bitplanes); - // evaluate(data, dims, target_level, num_bitplanes, refactor); + mgard_x::log::level |= mgard_x::log::TIME; + + mgard_x::Config config; + mgard_x::MDR::MDRData mdr_data; + mgard_x::MDR::MDRMetadata mdr_metadata; + { + auto refactor = mgard_x::MDR::ComposedRefactor( + hierarchy, config); + refactor.Refactor(input_array, mdr_metadata, mdr_data, 0); + refactor.Compress(mdr_metadata, mdr_data, 0); + refactor.StoreMetadata(mdr_metadata, mdr_data, 0); + } + + { + auto reconstructor = + mgard_x::MDR::ComposedReconstructor(hierarchy, + config); + + mdr_metadata.InitializeForReconstruction(); + mgard_x::Array reconstructed_data( + hierarchy.level_shape(hierarchy.l_target())); + reconstructed_data.memset(0); + for (int i = 0; i < tolerance.size(); i++) { + mgard_x::log::level |= mgard_x::log::TIME; + mdr_metadata.requested_tol = tolerance[i]; + mdr_metadata.requested_s = s; + reconstructor.GenerateRequest(mdr_metadata); + mdr_metadata.PrintStatus(); + mdr_metadata.DoneLoadingBitplans(); + reconstructor.ProgressiveReconstruct(mdr_metadata, mdr_data, false, + reconstructed_data, 0); + auto dims = reconstructor.get_dimensions(); + size_t size = 1; + for (int d = 0; d < D; d++) { + size *= hierarchy.level_shape(hierarchy.l_target(), d); + } + print_statistics(data.data(), reconstructed_data.hostCopy(), size); + } + } } int main(int argc, char **argv) { @@ -58,6 +127,13 @@ int main(int argc, char **argv) { dims[i] = atoi(argv[argv_id++]); } + int num_tolerance = atoi(argv[argv_id++]); + vector tolerance(num_tolerance, 0); + for (int i = 0; i < num_tolerance; i++) { + tolerance[i] = atof(argv[argv_id++]); + } + double s = atof(argv[argv_id++]); + string metadata_file = "refactored_data/metadata.bin"; vector files; for (int i = 0; i <= target_level; i++) { @@ -65,8 +141,6 @@ int main(int argc, char **argv) { files.push_back(filename); } using T = float; - using T_stream = uint32_t; - using T_error = double; using DeviceType = mgard_x::HIP; if (num_bitplanes > 32) { num_bitplanes = 32; @@ -75,60 +149,14 @@ int main(int argc, char **argv) { << std::endl; } const mgard_x::DIM D = 3; - printf("dims: %lu %lu %lu\n", dims[2], dims[1], dims[0]); + printf("dims: %lu %lu %lu\n", dims[0], dims[1], dims[2]); + mgard_x::log::level |= mgard_x::log::INFO; mgard_x::Config config; - mgard_x::Hierarchy hierarchy(dims, config, target_level); + config.max_larget_level = target_level; + mgard_x::Hierarchy hierarchy(dims, config); - // if (false) { - auto decomposer = - mgard_x::MDR::MGARDOrthoganalDecomposer(hierarchy); - - auto interleaver = - mgard_x::MDR::DirectInterleaver(hierarchy); - // auto interleaver = mgard_x::MDR::SFCInterleaver(); - // auto interleaver = mgard_x::MDR::BlockedInterleaver(); - - // auto encoder = mgard_x::MDR::GroupedBPEncoder(); - auto encoder = - mgard_x::MDR::GroupedWarpBPEncoder(); - - auto compressor = - mgard_x::MDR::DefaultLevelCompressor(); - // auto compressor = mgard_x::MDR::AdaptiveLevelCompressor(32); - // auto compressor = mgard_x::MDR::NullLevelCompressor(); - - // auto collector = mgard_x::MDR::SquaredErrorCollector(); - auto collector = mgard_x::MDR::MaxErrorCollector(); - - auto writer = mgard_x::MDR::ConcatLevelFileWriter(metadata_file, files); - // auto writer = mgard_x::MDR::HPSSFileWriter(metadata_file, files, 2048, - // 512 * 1024 * 1024); - - test( - filename, dims, target_level, num_bitplanes, hierarchy, decomposer, - interleaver, encoder, compressor, collector, writer); - - // test2(filename, dims, target_level, num_bitplanes, decomposer, - // interleaver, encoder, compressor, collector, writer); - // } - - // if (true) { - // std::vector> level_signs; - - // auto decomposer = mgard_x::MDR::MGARDOrthoganalDecomposer(hierarchy); - // auto interleaver = mgard_x::MDR::DirectInterleaver(hierarchy); - // auto encoder = mgard_x::MDR::GroupedBPEncoder(); - // auto encoder = - // mgard_m::MDR::GroupedWarpBPEncoder(); - // auto compressor = - // mgard_m::MDR::DefaultLevelCompressor(); - // auto collector = mgard_x::MDR::SquaredErrorCollector(); - // auto writer = mgard_x::MDR::ConcatLevelFileWriter(metadata_file, files); - // test2( - // filename, dims, target_level, num_bitplanes, hierarchy, decomposer, - // interleaver, encoder, compressor, collector, writer); - // } + test(filename, num_bitplanes, hierarchy, metadata_file, + files, tolerance, s); return 0; -} \ No newline at end of file +} diff --git a/examples/mgard-x/MDR-X/README.md b/examples/mgard-x/MDR-X/README.md index c8143f98ff..dec03c49e1 100644 --- a/examples/mgard-x/MDR-X/README.md +++ b/examples/mgard-x/MDR-X/README.md @@ -7,23 +7,19 @@ Build with CMake as follows or use the 'build_scripts.sh'. ```console $ cmake -S . -B build $ cmake --build build -$ build/Example +$ build/refactor ``` +`build/refactor` reads in a dataset, refactors it with MDR-X on GPU or CPU, and then progressively reconstructs it in-memory at each of the given error bounds, printing the reconstruction error against the original data. +Read `refactor.cpp/refactor.cu` to see how the MDR-X API (`mgard_x::MDR::ComposedRefactor`/`ComposedReconstructor`) is used. -`build/main` read in a dataset, refactor it with MDR-X on GPU or CPU, and reconstruct it according to the given error bounds. -Read `refactor.cpp/refactor.cu` and `reconstructor.cpp/reconstructor.cu` to see how the MDR-X API is used. +The `refactor` executable takes: -The exectuables `refactor` and `reconstructor` can be used as follows: +* `` +* `` +* `` +* ` .. ` (currently 3D only) +* ` ... `: L-infinity error bounds to reconstruct at, one after another +* ``: smoothness parameter (use `0` for L2-style error control) -* `refactor` - - `` - - `` - - `` - - ` .. ` - -* `reconstructor` - - `` - - ``: 0 for L\_inf error; 1 for L\_2 error - - ` ... ` - - `` \ No newline at end of file +Example: `build/refactor data.bin 3 32 3 64 64 64 3 0.1 0.01 0.001 0` diff --git a/examples/mgard-x/MDR-X/SERIAL/CMakeLists.txt b/examples/mgard-x/MDR-X/SERIAL/CMakeLists.txt index 4dad58b66d..ca374ec737 100644 --- a/examples/mgard-x/MDR-X/SERIAL/CMakeLists.txt +++ b/examples/mgard-x/MDR-X/SERIAL/CMakeLists.txt @@ -7,5 +7,3 @@ set (CMAKE_CXX_STANDARD 17) set (CMAKE_CXX_STANDARD_REQUIRED ON) add_executable(refactor refactor.cpp) target_link_libraries(refactor mgard::mgard) -add_executable(reconstructor reconstructor.cpp) -target_link_libraries(reconstructor mgard::mgard) diff --git a/examples/mgard-x/MDR-X/SERIAL/reconstructor.cpp b/examples/mgard-x/MDR-X/SERIAL/reconstructor.cpp deleted file mode 100644 index 4e2fadb5ad..0000000000 --- a/examples/mgard-x/MDR-X/SERIAL/reconstructor.cpp +++ /dev/null @@ -1,204 +0,0 @@ -#include "mgard/mdr_x.hpp" -#include "mgard/mgard-x/Utilities/ErrorCalculator.h" -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace std; - -template -std::vector readfile(const char *file, size_t &num) { - std::ifstream fin(file, std::ios::binary); - if (!fin) { - std::cout << " Error, Couldn't find the file" - << "\n"; - return std::vector(); - } - fin.seekg(0, std::ios::end); - const size_t num_elements = fin.tellg() / sizeof(Type); - fin.seekg(0, std::ios::beg); - auto data = std::vector(num_elements); - fin.read(reinterpret_cast(&data[0]), num_elements * sizeof(Type)); - fin.close(); - num = num_elements; - return data; -} - -template -void print_statistics(const T_data *data_ori, const T_data *data_dec, - size_t data_size) { - double max_val = data_ori[0]; - double min_val = data_ori[0]; - double max_abs = fabs(data_ori[0]); - for (int i = 0; i < data_size; i++) { - if (data_ori[i] > max_val) - max_val = data_ori[i]; - if (data_ori[i] < min_val) - min_val = data_ori[i]; - if (fabs(data_ori[i]) > max_abs) - max_abs = fabs(data_ori[i]); - } - double max_err = 0; - int pos = 0; - double mse = 0; - for (int i = 0; i < data_size; i++) { - double err = data_ori[i] - data_dec[i]; - mse += err * err; - if (fabs(err) > max_err) { - pos = i; - max_err = fabs(err); - } - } - mse /= data_size; - double psnr = 20 * log10((max_val - min_val) / sqrt(mse)); - cout << "Max value = " << max_val << ", min value = " << min_val << endl; - cout << "Max error = " << max_err << ", pos = " << pos << endl; - cout << "MSE = " << mse << ", PSNR = " << psnr << endl; - cout << "L2 error = " - << mgard_x::L_2_error({(mgard_x::SIZE)data_size}, data_ori, data_dec, - mgard_x::error_bound_type::ABS, 0) - << endl; - cout << "L_inf error = " - << mgard_x::L_inf_error(data_size, data_ori, data_dec, - mgard_x::error_bound_type::ABS) - << endl; -} - -template -void test(string filename, const vector &tolerance, - mgard_x::Hierarchy &hierarchy, - Decomposer decomposer, Interleaver interleaver, Encoder encoder, - Compressor compressor, ErrorEstimator estimator, - SizeInterpreter interpreter, Retriever retriever) { - auto reconstructor = mgard_x::MDR::ComposedReconstructor< - D, T_data, T_stream, Decomposer, Interleaver, Encoder, Compressor, - SizeInterpreter, ErrorEstimator, Retriever, DeviceType>( - hierarchy, decomposer, interleaver, encoder, compressor, interpreter, - retriever); - cout << "loading metadata" << endl; - reconstructor.load_metadata(); - - size_t num_elements = 0; - auto data = readfile(filename.c_str(), num_elements); - for (int i = 0; i < tolerance.size(); i++) { - cout << "Start reconstruction" << endl; - mgard_x::Array reconstructed_data = - reconstructor.progressive_reconstruct(tolerance[i]); - auto dims = reconstructor.get_dimensions(); - size_t size = 1; - for (int i = 0; i < dims.size(); i++) { - size *= dims[i]; - } - print_statistics(data.data(), reconstructed_data.hostCopy(), size); - } -} - -int main(int argc, char **argv) { - - int argv_id = 1; - string filename = string(argv[argv_id++]); - int error_mode = atoi(argv[argv_id++]); - int num_tolerance = atoi(argv[argv_id++]); - vector tolerance(num_tolerance, 0); - for (int i = 0; i < num_tolerance; i++) { - tolerance[i] = atof(argv[argv_id++]); - } - double s = atof(argv[argv_id++]); - - string metadata_file = "refactored_data/metadata.bin"; - int num_levels = 0; - int num_dims = 0; - vector dims; - { - // metadata interpreter, otherwise information needs to be provided - size_t num_bytes = 0; - auto metadata = readfile(metadata_file.c_str(), num_bytes); - assert(num_bytes > num_dims * sizeof(mgard_x::SIZE) + 2); - num_dims = metadata[0]; - mgard_x::SIZE *dim = (mgard_x::SIZE *)&(metadata[1]); - printf("dim: "); - for (int i = 0; i < num_dims; i++) { - dims.push_back(dim[i]); - printf("%lu ", dim[i]); - } - printf("\n"); - num_levels = metadata[num_dims * sizeof(mgard_x::SIZE) + 1]; - cout << "number of dimension = " << num_dims - << ", number of levels = " << num_levels << endl; - } - vector files; - for (int i = 0; i < num_levels; i++) { - string filename = "refactored_data/level_" + to_string(i) + ".bin"; - files.push_back(filename); - } - - using T_data = float; - using T_stream = uint32_t; - using T_error = double; - using DeviceType = mgard_x::SERIAL; - - const mgard_x::DIM D = 3; - mgard_x::Config config; - mgard_x::Hierarchy hierarchy(dims, config, - num_levels - 1); - - auto decomposer = - mgard_x::MDR::MGARDOrthoganalDecomposer(hierarchy); - auto interleaver = - mgard_x::MDR::DirectInterleaver(hierarchy); - auto encoder = - mgard_x::MDR::GroupedBPEncoder(); - // auto encoder = - // mgard_x::MDR::GroupedWarpBPEncoder(); - auto compressor = - mgard_x::MDR::DefaultLevelCompressor(); - auto retriever = mgard_x::MDR::ConcatLevelFileRetriever(metadata_file, files); - - switch (error_mode) { - case 1: { - auto estimator = - mgard_x::MDR::SNormErrorEstimator(num_dims, num_levels - 1, s); - // auto interpreter = - // mgard_x::MDR::SignExcludeGreedyBasedSizeInterpreter>(estimator); - // auto interpreter = - // mgard_x::MDR::NegaBinaryGreedyBasedSizeInterpreter>(estimator); - - auto interpreter = mgard_x::MDR::RoundRobinSizeInterpreter< - mgard_x::MDR::SNormErrorEstimator>(estimator); - // auto interpreter = - // mgard_x::MDR::InorderSizeInterpreter>(estimator); - // auto estimator = mgard_x::MDR::L2ErrorEstimator_HB(num_dims, - // num_levels - 1); auto interpreter = - // mgard_x::MDR::SignExcludeGreedyBasedSizeInterpreter>(estimator); - test( - filename, tolerance, hierarchy, decomposer, interleaver, encoder, - compressor, estimator, interpreter, retriever); - break; - } - default: { - auto estimator = mgard_x::MDR::MaxErrorEstimatorOB(num_dims); - auto interpreter = mgard_x::MDR::SignExcludeGreedyBasedSizeInterpreter< - mgard_x::MDR::MaxErrorEstimatorOB>(estimator); - // auto interpreter = - // MDR::RoundRobinSizeInterpreter>(estimator); - // auto interpreter = - // MDR::InorderSizeInterpreter>(estimator); - // auto estimator = MDR::MaxErrorEstimatorHB(); - // auto interpreter = - // MDR::SignExcludeGreedyBasedSizeInterpreter>(estimator); - test( - filename, tolerance, hierarchy, decomposer, interleaver, encoder, - compressor, estimator, interpreter, retriever); - } - } - - return 0; -} \ No newline at end of file diff --git a/examples/mgard-x/MDR-X/SERIAL/refactor.cpp b/examples/mgard-x/MDR-X/SERIAL/refactor.cpp index ba1676c4fd..63a828f637 100644 --- a/examples/mgard-x/MDR-X/SERIAL/refactor.cpp +++ b/examples/mgard-x/MDR-X/SERIAL/refactor.cpp @@ -1,4 +1,5 @@ -#include "mgard/mdr_x.hpp" +#include "mgard/mdr_x_lowlevel.hpp" +#include "mgard/mgard-x/Utilities/ErrorCalculator.h" #include #include #include @@ -8,37 +9,105 @@ #include using namespace std; -template -void test(string filename, const vector &dims, int target_level, - int num_bitplanes, +template +void print_statistics(const T_data *data_ori, const T_data *data_dec, + size_t data_size) { + double max_val = data_ori[0]; + double min_val = data_ori[0]; + double max_abs = fabs(data_ori[0]); + for (int i = 0; i < data_size; i++) { + if (data_ori[i] > max_val) + max_val = data_ori[i]; + if (data_ori[i] < min_val) + min_val = data_ori[i]; + if (fabs(data_ori[i]) > max_abs) + max_abs = fabs(data_ori[i]); + } + double max_err = 0; + int pos = 0; + double mse = 0; + for (int i = 0; i < data_size; i++) { + double err = data_ori[i] - data_dec[i]; + mse += err * err; + if (fabs(err) > max_err) { + pos = i; + max_err = fabs(err); + } + } + mse /= data_size; + double psnr = 20 * log10((max_val - min_val) / sqrt(mse)); + cout << "Max value = " << max_val << ", min value = " << min_val << endl; + cout << "Max error = " << max_err << ", pos = " << pos << endl; + cout << "MSE = " << mse << ", PSNR = " << psnr << endl; + cout << "L2 error = " + << mgard_x::L_2_error({(mgard_x::SIZE)data_size}, data_ori, data_dec, + mgard_x::error_bound_type::ABS, 0) + << endl; + cout << "L_inf error = " + << mgard_x::L_inf_error(data_size, data_ori, data_dec, + mgard_x::error_bound_type::ABS) + << endl; +} + +template +void test(string filename, int num_bitplanes, mgard_x::Hierarchy &hierarchy, - Decomposer decomposer, Interleaver interleaver, Encoder encoder, - Compressor compressor, ErrorCollector collector, Writer writer) { + std::string metadata_file, std::vector files, + const vector &tolerance, double s) { - auto refactor = - mgard_x::MDR::ComposedRefactor( - hierarchy, decomposer, interleaver, encoder, compressor, collector, - writer); size_t num_elements = 1; printf("loading file\n"); FILE *pFile; pFile = fopen(filename.c_str(), "rb"); - for (int d = 0; d < dims.size(); d++) - num_elements *= dims[d]; - vector data( - num_elements); // MGARD::readfile(filename.c_str(), num_elements); + for (int d = 0; d < D; d++) + num_elements *= hierarchy.level_shape(hierarchy.l_target(), d); + vector data(num_elements); fread(data.data(), 1, num_elements * sizeof(T_data), pFile); fclose(pFile); printf("done loading file\n"); - mgard_x::Array input_array(dims); + mgard_x::Array input_array( + hierarchy.level_shape(hierarchy.l_target())); input_array.load(data.data()); - refactor.refactor(input_array, dims, target_level, num_bitplanes); - // evaluate(data, dims, target_level, num_bitplanes, refactor); + mgard_x::log::level |= mgard_x::log::TIME; + + mgard_x::Config config; + mgard_x::MDR::MDRData mdr_data; + mgard_x::MDR::MDRMetadata mdr_metadata; + { + auto refactor = mgard_x::MDR::ComposedRefactor( + hierarchy, config); + refactor.Refactor(input_array, mdr_metadata, mdr_data, 0); + refactor.Compress(mdr_metadata, mdr_data, 0); + refactor.StoreMetadata(mdr_metadata, mdr_data, 0); + } + + { + auto reconstructor = + mgard_x::MDR::ComposedReconstructor(hierarchy, + config); + + mdr_metadata.InitializeForReconstruction(); + mgard_x::Array reconstructed_data( + hierarchy.level_shape(hierarchy.l_target())); + reconstructed_data.memset(0); + for (int i = 0; i < tolerance.size(); i++) { + mgard_x::log::level |= mgard_x::log::TIME; + mdr_metadata.requested_tol = tolerance[i]; + mdr_metadata.requested_s = s; + reconstructor.GenerateRequest(mdr_metadata); + mdr_metadata.PrintStatus(); + mdr_metadata.DoneLoadingBitplans(); + reconstructor.ProgressiveReconstruct(mdr_metadata, mdr_data, false, + reconstructed_data, 0); + auto dims = reconstructor.get_dimensions(); + size_t size = 1; + for (int d = 0; d < D; d++) { + size *= hierarchy.level_shape(hierarchy.l_target(), d); + } + print_statistics(data.data(), reconstructed_data.hostCopy(), size); + } + } } int main(int argc, char **argv) { @@ -58,6 +127,13 @@ int main(int argc, char **argv) { dims[i] = atoi(argv[argv_id++]); } + int num_tolerance = atoi(argv[argv_id++]); + vector tolerance(num_tolerance, 0); + for (int i = 0; i < num_tolerance; i++) { + tolerance[i] = atof(argv[argv_id++]); + } + double s = atof(argv[argv_id++]); + string metadata_file = "refactored_data/metadata.bin"; vector files; for (int i = 0; i <= target_level; i++) { @@ -65,8 +141,6 @@ int main(int argc, char **argv) { files.push_back(filename); } using T = float; - using T_stream = uint32_t; - using T_error = double; using DeviceType = mgard_x::SERIAL; if (num_bitplanes > 32) { num_bitplanes = 32; @@ -75,60 +149,14 @@ int main(int argc, char **argv) { << std::endl; } const mgard_x::DIM D = 3; - printf("dims: %lu %lu %lu\n", dims[2], dims[1], dims[0]); + printf("dims: %lu %lu %lu\n", dims[0], dims[1], dims[2]); + mgard_x::log::level |= mgard_x::log::INFO; mgard_x::Config config; - mgard_x::Hierarchy hierarchy(dims, config, target_level); + config.max_larget_level = target_level; + mgard_x::Hierarchy hierarchy(dims, config); - // if (false) { - auto decomposer = - mgard_x::MDR::MGARDOrthoganalDecomposer(hierarchy); - - auto interleaver = - mgard_x::MDR::DirectInterleaver(hierarchy); - // auto interleaver = mgard_x::MDR::SFCInterleaver(); - // auto interleaver = mgard_x::MDR::BlockedInterleaver(); - - auto encoder = - mgard_x::MDR::GroupedBPEncoder(); - // auto encoder = - // mgard_x::MDR::GroupedWarpBPEncoder(); - - auto compressor = - mgard_x::MDR::DefaultLevelCompressor(); - // auto compressor = mgard_x::MDR::AdaptiveLevelCompressor(32); - // auto compressor = mgard_x::MDR::NullLevelCompressor(); - - // auto collector = mgard_x::MDR::SquaredErrorCollector(); - auto collector = mgard_x::MDR::MaxErrorCollector(); - - auto writer = mgard_x::MDR::ConcatLevelFileWriter(metadata_file, files); - // auto writer = mgard_x::MDR::HPSSFileWriter(metadata_file, files, 2048, - // 512 * 1024 * 1024); - - test( - filename, dims, target_level, num_bitplanes, hierarchy, decomposer, - interleaver, encoder, compressor, collector, writer); - - // test2(filename, dims, target_level, num_bitplanes, decomposer, - // interleaver, encoder, compressor, collector, writer); - // } - - // if (true) { - // std::vector> level_signs; - - // auto decomposer = mgard_x::MDR::MGARDOrthoganalDecomposer(hierarchy); - // auto interleaver = mgard_x::MDR::DirectInterleaver(hierarchy); - // auto encoder = mgard_x::MDR::GroupedBPEncoder(); - // auto encoder = - // mgard_m::MDR::GroupedWarpBPEncoder(); - // auto compressor = - // mgard_m::MDR::DefaultLevelCompressor(); - // auto collector = mgard_x::MDR::SquaredErrorCollector(); - // auto writer = mgard_x::MDR::ConcatLevelFileWriter(metadata_file, files); - // test2( - // filename, dims, target_level, num_bitplanes, hierarchy, decomposer, - // interleaver, encoder, compressor, collector, writer); - // } + test(filename, num_bitplanes, hierarchy, metadata_file, + files, tolerance, s); return 0; -} \ No newline at end of file +} diff --git a/include/MGARDConfig.hpp.in b/include/MGARDConfig.hpp.in index 4f273f8f5d..aa64b35cc4 100644 --- a/include/MGARDConfig.hpp.in +++ b/include/MGARDConfig.hpp.in @@ -1,6 +1,8 @@ #ifndef MGARDCONFIG_HPP #define MGARDCONFIG_HPP +#include + /* MGARD Version Information */ #define MGARD_VERSION_STR "@MGARD_VERSION@" #define MGARD_VERSION_MAJOR @MGARD_VERSION_MAJOR@ @@ -15,4 +17,9 @@ #define MGARD_FILE_VERSION_PATCH @MGARD_FILE_VERSION_PATCH@ #cmakedefine MGARD_FILE_VERSION_TWEAK @MGARD_FILE_VERSION_TWEAK@ +namespace mgard { +//! Magic bytes for MGARD file format ('MGARD' in ASCII). +inline constexpr std::array SIGNATURE{0x4d, 0x47, 0x41, 0x52, 0x44}; +} // namespace mgard + #endif /* MGARDCONFIG_HPP */ diff --git a/include/MGARDXConfig.h.in b/include/MGARDXConfig.h.in index b2491dedfa..eb43950dbf 100644 --- a/include/MGARDXConfig.h.in +++ b/include/MGARDXConfig.h.in @@ -18,8 +18,6 @@ #cmakedefine01 MGARD_ENABLE_SYCL -#cmakedefine01 MGARD_ENABLE_LEGACY_CUDA - #cmakedefine01 MGARD_ENABLE_AUTO_TUNING #cmakedefine01 MGARD_ENABLE_EXTERNAL_COMPRESSOR diff --git a/include/compress.hpp b/include/compress.hpp index bf14bc51fb..2fec44038d 100644 --- a/include/compress.hpp +++ b/include/compress.hpp @@ -9,17 +9,17 @@ //!\file //!\brief Compression and decompression API. -#include "CompressedDataset.hpp" -#include "TensorMeshHierarchy.hpp" -#include "utilities.hpp" +#include "mgard/CompressedDataset.hpp" +#include "mgard/TensorMeshHierarchy.hpp" +#include "mgard/utilities.hpp" #ifdef __NVCC__ #error "Please include `compress_x.hpp` instead of `compress.hpp` when "\ "compiling with NVCC." #endif -#include "adaptive_roi.hpp" #include "compress_x.hpp" +#include "mgard/adaptive_roi.hpp" //! Implementation of the MGARD compression and decompression algorithms. namespace mgard { diff --git a/include/compress.tpp b/include/compress.tpp index 7a1e565337..fb7c457815 100644 --- a/include/compress.tpp +++ b/include/compress.tpp @@ -18,14 +18,14 @@ #include #include "MGARDConfig.hpp" -#include "TensorMultilevelCoefficientQuantizer.hpp" -#include "TensorNorms.hpp" -#include "adaptive_roi.hpp" -#include "compressors.hpp" -#include "decompose.hpp" -#include "format.hpp" -#include "quantize.hpp" -#include "shuffle.hpp" +#include "mgard/TensorMultilevelCoefficientQuantizer.hpp" +#include "mgard/TensorNorms.hpp" +#include "mgard/adaptive_roi.hpp" +#include "mgard/compressors.hpp" +#include "mgard/decompose.hpp" +#include "mgard/format.hpp" +#include "mgard/quantize.hpp" +#include "mgard/shuffle.hpp" namespace mgard { diff --git a/include/compress_cuda.hpp b/include/compress_cuda.hpp deleted file mode 100644 index b69e8d52e0..0000000000 --- a/include/compress_cuda.hpp +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: September 27, 2021 - */ - -#include "cuda/Common.h" -#include "cuda/CompressionWorkflow.h" -#include "cuda/MemoryManagement.h" -#include - -#warning("The legacy MGARD-GPU dadicated for Nvidia GPUs will be deprecated in future releases of MGARD. Please use to the MGARD-X portable version.") - -#ifndef MGARD_CUDA_API_H -#define MGARD_CUDA_API_H - -namespace mgard_cuda { - -//!\file -//!\brief High level compression and decompression API. - -//! Compress a function on an N-D tensor product grid with uniform spacing -//! -//!\param[in] D Dimension. -//!\param[in] dtype Data type Float or Double -//!\param[in] shape Shape of the Dataset to be compressed -//!\param[in] tol Error tolerance. -//!\param[in] s Smoothness parameter. -//!\param[in] mode Error bound type REL or ABS. -//!\param[in] original_data Dataset to be compressed. -//!\param[out] compressed_data Compressed data. -//!\param[out] compressed_size Size of compressed data. -void compress(DIM D, data_type dtype, std::vector shape, double tol, - double s, enum error_bound_type mode, const void *original_data, - void *&compressed_data, size_t &compressed_size); - -//!\file -//!\brief High level compression and decompression API. - -//! Compress a function on an N-D tensor product grid with uniform spacing -//! -//!\param[in] D Dimension. -//!\param[in] dtype Data type Float or Double -//!\param[in] shape Shape of the Dataset to be compressed -//!\param[in] tol Error tolerance. -//!\param[in] s Smoothness parameter. -//!\param[in] mode Error bound type REL or ABS. -//!\param[in] original_data Dataset to be compressed. -//!\param[out] compressed_data Compressed data. -//!\param[out] compressed_size Size of compressed data. -//!\param[in] config For configuring the compression process. -void compress(DIM D, data_type dtype, std::vector shape, double tol, - double s, enum error_bound_type mode, const void *original_data, - void *&compressed_data, size_t &compressed_size, Config config); - -//! Compress a function on an N-D tensor product grid with non-uniform spacing -//! -//!\param[in] D Dimension. -//!\param[in] dtype Data type Float or Double -//!\param[in] shape Shape of the Dataset to be compressed -//!\param[in] tol Error tolerance. -//!\param[in] s Smoothness parameter. -//!\param[in] mode Error bound type REL or ABS. -//!\param[in] original_data Dataset to be compressed. -//!\param[out] compressed_data Compressed data. -//!\param[out] compressed_size Size of comrpessed data. -//!\param[in] coords Coordinates data. -void compress(DIM D, data_type dtype, std::vector shape, double tol, - double s, enum error_bound_type mode, const void *original_data, - void *&compressed_data, size_t &compressed_size, - std::vector coords); - -//! Compress a function on an N-D tensor product grid with non-uniform spacing -//! -//!\param[in] D Dimension. -//!\param[in] dtype Data type Float or Double -//!\param[in] shape Shape of the Dataset to be compressed -//!\param[in] tol Error tolerance. -//!\param[in] s Smoothness parameter. -//!\param[in] mode Error bound type REL or ABS. -//!\param[in] original_data Dataset to be compressed. -//!\param[out] compressed_data Compressed data. -//!\param[out] compressed_size Size of comrpessed data. -//!\param[in] coords Coordinates data. -//!\param[in] config For configuring the compression process. -void compress(DIM D, data_type dtype, std::vector shape, double tol, - double s, enum error_bound_type mode, const void *original_data, - void *&compressed_data, size_t &compressed_size, - std::vector coords, Config config); - -//! Decompress a function on an N-D tensor product grid -//! -//!\param[in] compressed_data Compressed data. -//!\param[in] compressed_size Size of comrpessed data. -//!\param[out] decompressed_data Decompressed data. -void decompress(const void *compressed_data, size_t compressed_size, - void *&decompressed_data); - -//! Decompress a function on an N-D tensor product grid -//! -//!\param[in] compressed_data Compressed data. -//!\param[in] compressed_size Size of comrpessed data. -//!\param[out] decompressed_data Decompressed data. -//!\param[in] config For configuring the decompression process. -void decompress(const void *compressed_data, size_t compressed_size, - void *&decompressed_data, Config config); - -//! Verify the compressed data -bool verify(const void *compressed_data, size_t compressed_size); - -//! Query the data type of original data -enum data_type infer_data_type(const void *compressed_data, - size_t compressed_size); - -//! Query the shape of original data -std::vector infer_shape(const void *compressed_data, - size_t compressed_size); - -//! Query the data structure of original data -enum data_structure_type infer_data_structure(const void *compressed_data, - size_t compressed_size); - -//! Query the file used to store the coordinates data -std::string infer_nonuniform_coords_file(const void *compressed_data, - size_t compressed_size); - -//! Query the coordinates -template -std::vector infer_coords(const void *compressed_data, - size_t compressed_size); - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/compress_x.hpp b/include/compress_x.hpp index e3766d29c1..9088640507 100644 --- a/include/compress_x.hpp +++ b/include/compress_x.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "mgard-x/Config/Config.h" diff --git a/include/compress_x_lowlevel.hpp b/include/compress_x_lowlevel.hpp index b9980a43d5..62856712e1 100644 --- a/include/compress_x_lowlevel.hpp +++ b/include/compress_x_lowlevel.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "mgard-x/CompressionLowLevel/Compressor.hpp" diff --git a/include/cuda/Array.h b/include/cuda/Array.h deleted file mode 100644 index d6f90b7e96..0000000000 --- a/include/cuda/Array.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGARD_CUDA_ARRAY -#define MGARD_CUDA_ARRAY -#include "Common.h" -#include - -namespace mgard_cuda { - -template class Array { -public: - Array(); - Array(std::vector shape); - Array(const Array &array); - Array(Array &array); - Array &operator=(const Array &array); - Array(Array &&array); - ~Array(); - void loadData(const T *data, SIZE ld = 0); - T *getDataHost(); - T *getDataDevice(SIZE &ld); - std::vector getShape(); - T *get_dv(); - std::vector get_ldvs_h(); - SIZE *get_ldvs_d(); - -private: - DIM D_padded; - T *dv; - T *hv; - bool device_allocated; - bool host_allocated; - std::vector ldvs_h; - SIZE *ldvs_d; - std::vector shape; - SIZE linearized_depth; -}; - -} // namespace mgard_cuda -#endif \ No newline at end of file diff --git a/include/cuda/CMakeLists.txt b/include/cuda/CMakeLists.txt deleted file mode 100644 index 425d7ba771..0000000000 --- a/include/cuda/CMakeLists.txt +++ /dev/null @@ -1,48 +0,0 @@ -set(MGARD_CUDA_HEADER - ${CMAKE_CURRENT_SOURCE_DIR}/../compress_cuda.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/Array.h - ${CMAKE_CURRENT_SOURCE_DIR}/SubArray.h - ${CMAKE_CURRENT_SOURCE_DIR}/Message.h - ${CMAKE_CURRENT_SOURCE_DIR}/Handle.h - ${CMAKE_CURRENT_SOURCE_DIR}/MemoryManagement.h - ${CMAKE_CURRENT_SOURCE_DIR}/Metadata.h - ${CMAKE_CURRENT_SOURCE_DIR}/Common.h - ${CMAKE_CURRENT_SOURCE_DIR}/CommonInternal.h - ${CMAKE_CURRENT_SOURCE_DIR}/PrecomputeKernels.h - ${CMAKE_CURRENT_SOURCE_DIR}/DataRefactoring.h - ${CMAKE_CURRENT_SOURCE_DIR}/GPKFunctor.h - ${CMAKE_CURRENT_SOURCE_DIR}/GridProcessingKernel.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/GridProcessingKernel.h - ${CMAKE_CURRENT_SOURCE_DIR}/GridProcessingKernel3D.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/GridProcessingKernel3D.h - ${CMAKE_CURRENT_SOURCE_DIR}/LPKFunctor.h - ${CMAKE_CURRENT_SOURCE_DIR}/LinearProcessingKernel.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/LinearProcessingKernel.h - ${CMAKE_CURRENT_SOURCE_DIR}/LinearProcessingKernel3D.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/LinearProcessingKernel3D.h - ${CMAKE_CURRENT_SOURCE_DIR}/IPKFunctor.h - ${CMAKE_CURRENT_SOURCE_DIR}/IterativeProcessingKernel.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/IterativeProcessingKernel.h - ${CMAKE_CURRENT_SOURCE_DIR}/IterativeProcessingKernel3D.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/IterativeProcessingKernel3D.h - ${CMAKE_CURRENT_SOURCE_DIR}/LevelwiseProcessingKernel.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/LevelwiseProcessingKernel.h - ${CMAKE_CURRENT_SOURCE_DIR}/LinearQuantization.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/LinearQuantization.h - ${CMAKE_CURRENT_SOURCE_DIR}/LosslessCompression.h - ${CMAKE_CURRENT_SOURCE_DIR}/ErrorCalculator.h - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/canonical.cuh - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/constants.hh - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/cuda_error_handling.cuh - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/cuda_mem.cuh - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/dbg_gpu_printing.cuh - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/format.hh - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/histogram.cuh - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/huffman_codec.cuh - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/huffman_workflow.cuh - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/io.hh - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/par_huffman.cuh - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/par_merge.cuh - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/timer.hh - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/types.hh - PARENT_SCOPE) \ No newline at end of file diff --git a/include/cuda/Common.h b/include/cuda/Common.h deleted file mode 100644 index 79b1ae3f91..0000000000 --- a/include/cuda/Common.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: September 27, 2021 - */ - -#ifndef MGRAD_CUDA_COMMON -#define MGRAD_CUDA_COMMON - -#include - -namespace mgard_cuda { - -enum class processor_type : uint8_t { CPU, GPU_CUDA }; - -enum class error_bound_type : uint8_t { REL, ABS }; -enum class norm_type : uint8_t { L_Inf, L_2 }; -enum class lossless_type : uint8_t { - CPU_Lossless, - GPU_Huffman, - GPU_Huffman_LZ4 -}; - -enum class data_type : uint8_t { Float, Double }; -enum class data_structure_type : uint8_t { - Cartesian_Grid_Uniform, - Cartesian_Grid_Non_Uniform -}; - -enum class endiness_type : uint8_t { Little_Endian, Big_Endian }; - -enum class coordinate_location : uint8_t { Embedded, External }; - -class Device {}; -class CUDA : public Device {}; -class HIP : public Device {}; -class DPCxx : public Device {}; -class OpenMp : public Device {}; -class Kokkos : public Device {}; - -using IDX = unsigned long long int; -using LENGTH = unsigned long long int; -using SIZE = unsigned int; -// using SIZE = int; -using DIM = uint32_t; -using QUANTIZED_INT = int; -using SERIALIZED_TYPE = unsigned char; -using Byte = unsigned char; -using OPTION = int8_t; -} // namespace mgard_cuda - -#include -#include -#include -#include - -#include "Array.h" -#include "ErrorCalculator.h" -#include "Handle.h" -#include "MemoryManagement.h" -#include "Message.h" - -#endif diff --git a/include/cuda/CommonInternal.h b/include/cuda/CommonInternal.h deleted file mode 100644 index d50ac12858..0000000000 --- a/include/cuda/CommonInternal.h +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: September 27, 2021 - */ -#include - -#include -#include - -#include -#include - -#ifndef MGRAD_CUDA_COMMON_INTERNAL -#define MGRAD_CUDA_COMMON_INTERNAL - -#define MGARDm_CONT __host__ __inline__ -#define MGARDm_KERL __global__ -#define MGARDm_EXEC __device__ __forceinline__ -#define MGARDm_CONT_EXEC __host__ __device__ __forceinline__ -#define MGARDm_COMPILE_EXEC __CUDACC__ - -#include "Common.h" -#include "Metadata.h" -#include "SubArray.h" - -#define MAX_GRID_X 2147483647 -#define MAX_GRID_Y 65536 -#define MAX_GRID_Z 65536 -#define MGARDm_WARP_SIZE 32 -#define MGARDm_MAX_NUM_WARPS_PER_TB 32 -#define MGARDm_NUM_SMs 68 - -#define SIZE_MAX_VALUE 4294967295 - -#define COPY 0 -#define ADD 1 -#define SUBTRACT 2 - -// reduction operations -#define SUM 0 -#define MAX 1 - -// #define WARP_SIZE 32 -// #define ROUND_UP_WARP(TID) ((TID) + WARP_SIZE - 1) / WARP_SIZE - -#define gpuErrchk(ans) \ - { mgard_cuda::gpuAssert((ans), __FILE__, __LINE__); } - -namespace mgard_cuda { - -inline void gpuAssert(cudaError_t code, const char *file, int line, - bool abort = true) { - if (code != cudaSuccess) { - fprintf(stderr, "GPUassert: %s %s %d\n", cudaGetErrorString(code), file, - line); - if (abort) - exit(code); - } -} - -template struct SharedMemory { - __device__ inline operator T *() { - extern __shared__ int __smem[]; - return (T *)__smem; - } - - __device__ inline operator const T *() const { - extern __shared__ int __smem[]; - return (T *)__smem; - } -}; - -template int check_shape(std::vector shape); - -bool is_2kplus1_cuda(double num); - -// __device__ int get_idx(const int ld, const int i, const int j); - -// __device__ int get_idx(const int ld1, const int ld2, const int i, const int -// j, -// const int k); - -// __forceinline__ __device__ int get_idx(const int ld, const int i, const int -// j) { -// return ld * i + j; -// } - -// ld2 = nrow -// ld1 = pitch -// for 1-3D -__host__ __forceinline__ __device__ LENGTH get_idx(const SIZE ld1, - const SIZE ld2, const SIZE z, - const SIZE y, const SIZE x) { - return ld2 * ld1 * z + ld1 * y + x; -} - -// for 3D+ -__host__ __forceinline__ __device__ LENGTH get_idx(const LENGTH ld1, - const LENGTH ld2, - const SIZE z, const SIZE y, - const SIZE x) { - return ld2 * ld1 * z + ld1 * y + x; -} - -// leading dimension first -__host__ inline LENGTH get_idx(std::vector lds, std::vector idx) { - LENGTH curr_stride = 1; - LENGTH ret_idx = 0; - for (DIM i = 0; i < idx.size(); i++) { - ret_idx += idx[i] * curr_stride; - curr_stride *= lds[i]; - } - return ret_idx; -} - -template -__forceinline__ __device__ LENGTH get_idx(SIZE *lds, SIZE *idx) { - LENGTH curr_stride = 1; - LENGTH ret_idx = 0; - for (DIM i = 0; i < D; i++) { - ret_idx += idx[i] * curr_stride; - curr_stride *= lds[i]; - } - return ret_idx; -} - -__host__ inline std::vector gen_idx(DIM D, DIM curr_dim_r, DIM curr_dim_c, - DIM curr_dim_f, SIZE idx_r, - SIZE idx_c, SIZE idx_f) { - std::vector idx(D, 0); - idx[curr_dim_r] = idx_r; - idx[curr_dim_c] = idx_c; - idx[curr_dim_f] = idx_f; - return idx; -} - -__host__ __forceinline__ __device__ int div_roundup(SIZE a, SIZE b) { - return (a - 1) / b + 1; -} - -// template -// __host__ inline void kernel_config(thrust::device_vector &shape, int -// &tbx, -// int &tby, int &tbz, int &gridx, int -// &gridy, int &gridz, -// thrust::device_vector &assigned_dimx, -// thrust::device_vector &assigned_dimy, -// thrust::device_vector &assigned_dimz) -// { - -// tbx = F; -// tby = C; -// tbz = R; -// gridx = ceil((double)shape[0] / F); -// gridy = ceil((double)shape[1] / C); -// gridz = ceil((double)shape[2] / R); -// assigned_dimx.push_back(0); -// assigned_dimy.push_back(1); -// assigned_dimz.push_back(2); - -// int d = 3; -// while (d < D) { -// if (gridx * shape[d] < MAX_GRID_X) { -// gridx *= shape[d]; -// assigned_dimx.push_back(d); -// d++; -// } else { -// break; -// } -// } - -// while (d < D) { -// if (gridy * shape[d] < MAX_GRID_Y) { -// gridy *= shape[d]; -// assigned_dimy.push_back(d); -// d++; -// } else { -// break; -// } -// } - -// while (d < D) { -// if (gridz * shape[d] < MAX_GRID_Z) { -// gridz *= shape[d]; -// assigned_dimz.push_back(d); -// d++; -// } else { -// break; -// } -// } -// } - -// template -// __forceinline__ __device__ void -// get_idx(int *shape, int assigned_nx, int *assigned_dimx, int assigned_ny, -// int *assigned_dimy, int assigned_nz, int *assigned_dimz, int *idx) { -// int bidx = blockIdx.x; -// int bidy = blockIdx.y; -// int bidz = blockIdx.z; -// idx[0] = (bidx % shape[0]) * F + threadIdx.x; -// idx[1] = (bidy % shape[1]) * C + threadIdx.y; -// idx[2] = (bidz % shape[2]) * R + threadIdx.z; -// if (idx[0] < 0) { -// printf("neg %d %d %d %d\n", bidx, shape[0], F, threadIdx.x); -// } -// if (idx[1] < 0) { -// printf("neg %d %d %d %d\n", bidy, shape[1], C, threadIdx.y); -// } -// if (idx[2] < 0) { -// printf("neg %d %d %d %d\n", bidz, shape[2], R, threadIdx.z); -// } -// // bidx /= shape[0]; -// // bidy /= shape[1]; -// // bidz /= shape[2]; -// // for (int i = 1; i < assigned_nx; i++) { -// // int d = assigned_dimx[i]; -// // idx[d] = bidx%shape[d]; -// // bidx /= shape[d]; -// // } -// // for (int i = 1; i < assigned_ny; i++) { -// // int d = assigned_dimy[i]; -// // idx[d] = bidy%shape[d]; -// // bidy /= shape[d]; -// // } -// // for (int i = 1; i < assigned_nz; i++) { -// // int d = assigned_dimz[i]; -// // idx[d] = bidz%shape[d]; -// // bidz /= shape[d]; -// // } -// } - -// template T max_norm_cuda(const T *v, size_t size); - -template __device__ T _get_dist(T *coords, int i, int j); - -// // __host__ __device__ int get_lindex_cuda(const int n, const int no, const -// int i); - -// template -// __device__ inline T tridiag_forward(T prev, T bm, T curr) { - -// #ifdef MGARD_CUDA_FMA -// if (sizeof(T) == sizeof(double)) { -// return fma(prev, bm, curr); -// } else if (sizeof(T) == sizeof(float)) { -// return fmaf(prev, bm, curr); -// } -// #else -// return curr - prev * bm; -// #endif -// } - -// template -// __device__ inline T tridiag_backward(T prev, T dist, T am, T curr) { - -// #ifdef MGARD_CUDA_FMA -// if (sizeof(T) == sizeof(double)) { -// return fma(-1 * dist, prev, curr) * am; -// } else if (sizeof(T) == sizeof(float)) { -// return fmaf(-1 * dist, prev, curr) * am; -// } -// #else -// return (curr - dist * prev) / am; -// #endif -// } - -} // namespace mgard_cuda - -#endif diff --git a/include/cuda/CompressionWorkflow.h b/include/cuda/CompressionWorkflow.h deleted file mode 100644 index e3369766ec..0000000000 --- a/include/cuda/CompressionWorkflow.h +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: September 27, 2021 - */ - -#ifndef MGRAD_CUDA_COMPRESSION_WORKFLOW -#define MGRAD_CUDA_COMPRESSION_WORKFLOW - -#include "Common.h" - -namespace mgard_cuda { - -template -Array<1, unsigned char> compress(Handle &handle, Array &in_array, - enum error_bound_type type, T tol, T s); - -template -Array decompress(Handle &handle, - Array<1, unsigned char> &compressed_array); - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/DataRefactoring.h b/include/cuda/DataRefactoring.h deleted file mode 100644 index a3106f7a6a..0000000000 --- a/include/cuda/DataRefactoring.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_DATA_REFACTORING -#define MGRAD_CUDA_DATA_REFACTORING - -#include "Common.h" - -namespace mgard_cuda { - -// template -// void calc_coeff_pointers(Handle &handle, DIM curr_dims[3], DIM l, -// SubArray doutput, -// SubArray &dcoarse, -// SubArray &dcoeff_f, -// SubArray &dcoeff_c, -// SubArray &dcoeff_r, -// SubArray &dcoeff_cf, -// SubArray &dcoeff_rf, -// SubArray &dcoeff_rc, -// SubArray &dcoeff_rcf); - -// template -// void calc_coefficients_3d(Handle &handle, SubArray dinput, -// SubArray &doutput, SIZE l, int queue_idx); - -// template -// void coefficients_restore_3d(Handle &handle, SubArray dinput, -// SubArray &doutput, SIZE l, int queue_idx); - -// template -// void calc_correction_3d(Handle &handle, SubArray dcoeff, -// SubArray &dcorrection, SIZE l, int queue_idx); - -// template -// void calc_coefficients_nd(Handle &handle, SubArray dinput1, -// SubArray dinput2, -// SubArray &doutput, SIZE l, int queue_idx); - -// template -// void coefficients_restore_nd(Handle &handle, SubArray dinput1, -// SubArray dinput2, -// SubArray &doutput, SIZE l, int queue_idx); - -// template -// void calc_correction_nd(Handle &handle, SubArray dcoeff, -// SubArray &dcorrection, SIZE l, int queue_idx); - -template -void decompose(Handle &handle, T *dv, std::vector ldvs_h, - SIZE *ldvs_d, SIZE l_target, int queue_idx); - -template -void recompose(Handle &handle, T *dv, std::vector ldvs_h, - SIZE *ldvs_d, SIZE l_target, int queue_idx); - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/DataRefactoring.hpp b/include/cuda/DataRefactoring.hpp deleted file mode 100644 index a6f74a1393..0000000000 --- a/include/cuda/DataRefactoring.hpp +++ /dev/null @@ -1,1940 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include -#include - -#include "cuda/CommonInternal.h" -#include "cuda/SubArray.h" - -#include "cuda/GridProcessingKernel.h" -#include "cuda/GridProcessingKernel3D.h" -#include "cuda/GridProcessingKernel3D.hpp" -#include "cuda/IterativeProcessingKernel.h" -#include "cuda/IterativeProcessingKernel3D.h" -#include "cuda/LevelwiseProcessingKernel.h" -#include "cuda/LinearProcessingKernel.h" -#include "cuda/LinearProcessingKernel3D.h" - -#include "cuda/DataRefactoring.h" - -// #include "cuda/Testing/ReorderToolsGPU.hpp" - -#include - -#include -namespace mgard_cuda { - -static bool store = false; -static bool verify = false; -static bool debug_print = false; - -template -void calc_coeff_pointers(Handle &handle, DIM curr_dims[3], DIM l, - SubArray doutput, SubArray &dcoarse, - SubArray &dcoeff_f, SubArray &dcoeff_c, - SubArray &dcoeff_r, SubArray &dcoeff_cf, - SubArray &dcoeff_rf, SubArray &dcoeff_rc, - SubArray &dcoeff_rcf) { - - SIZE n[3]; - SIZE nn[3]; - for (DIM d = 0; d < 3; d++) { - n[d] = handle.dofs[curr_dims[d]][l]; - nn[d] = handle.dofs[curr_dims[d]][l + 1]; - } - - dcoarse = doutput; - dcoarse.resize(curr_dims[0], nn[0]); - dcoarse.resize(curr_dims[1], nn[1]); - dcoarse.resize(curr_dims[2], nn[2]); - - dcoeff_f = doutput; - dcoeff_f.offset(curr_dims[0], nn[0]); - dcoeff_f.resize(curr_dims[0], n[0] - nn[0]); - dcoeff_f.resize(curr_dims[1], nn[1]); - dcoeff_f.resize(curr_dims[2], nn[2]); - - dcoeff_c = doutput; - dcoeff_c.offset(curr_dims[1], nn[1]); - dcoeff_c.resize(curr_dims[0], nn[0]); - dcoeff_c.resize(curr_dims[1], n[1] - nn[1]); - dcoeff_c.resize(curr_dims[2], nn[2]); - - dcoeff_r = doutput; - dcoeff_r.offset(curr_dims[2], nn[2]); - dcoeff_r.resize(curr_dims[0], nn[0]); - dcoeff_r.resize(curr_dims[1], nn[1]); - dcoeff_r.resize(curr_dims[2], n[2] - nn[2]); - - dcoeff_cf = doutput; - dcoeff_cf.offset(curr_dims[0], nn[0]); - dcoeff_cf.offset(curr_dims[1], nn[1]); - dcoeff_cf.resize(curr_dims[0], n[0] - nn[0]); - dcoeff_cf.resize(curr_dims[1], n[1] - nn[1]); - dcoeff_cf.resize(curr_dims[2], nn[2]); - - dcoeff_rf = doutput; - dcoeff_rf.offset(curr_dims[0], nn[0]); - dcoeff_rf.offset(curr_dims[2], nn[2]); - dcoeff_rf.resize(curr_dims[0], n[0] - nn[0]); - dcoeff_rf.resize(curr_dims[1], nn[1]); - dcoeff_rf.resize(curr_dims[2], n[2] - nn[2]); - - dcoeff_rc = doutput; - dcoeff_rc.offset(curr_dims[1], nn[1]); - dcoeff_rc.offset(curr_dims[2], nn[2]); - dcoeff_rc.resize(curr_dims[0], nn[0]); - dcoeff_rc.resize(curr_dims[1], n[1] - nn[1]); - dcoeff_rc.resize(curr_dims[2], n[2] - nn[2]); - - dcoeff_rcf = doutput; - dcoeff_rcf.offset(curr_dims[0], nn[0]); - dcoeff_rcf.offset(curr_dims[1], nn[1]); - dcoeff_rcf.offset(curr_dims[2], nn[2]); - dcoeff_rcf.resize(curr_dims[0], n[0] - nn[0]); - dcoeff_rcf.resize(curr_dims[1], n[1] - nn[1]); - dcoeff_rcf.resize(curr_dims[2], n[2] - nn[2]); -} - -template -void calc_coefficients_3d(Handle &handle, SubArray dinput, - SubArray &doutput, SIZE l, int queue_idx) { - - int range_l = std::min(6, (int)std::log2(handle.dofs[0][l]) - 1); - int range_lp1 = std::min(6, (int)std::log2(handle.dofs[0][l + 1]) - 1); - - std::string prefix = "decomp_"; - if (sizeof(T) == sizeof(double)) - prefix += "d_"; - if (sizeof(T) == sizeof(float)) - prefix += "f_"; - for (int d = 0; d < D; d++) - prefix += std::to_string(handle.shapes_h[0][d]) + "_"; - - dinput.project(0, 1, 2); - doutput.project(0, 1, 2); - - SIZE f = handle.dofs[0][l]; - SIZE c = handle.dofs[1][l]; - SIZE r = handle.dofs[2][l]; - SIZE ff = handle.dofs[0][l + 1]; - SIZE cc = handle.dofs[1][l + 1]; - SIZE rr = handle.dofs[2][l + 1]; - - SubArray dcoarse = doutput; - dcoarse.resize({ff, cc, rr}); - SubArray dcoeff_f = doutput; - dcoeff_f.offset({ff, 0, 0}); - dcoeff_f.resize({f - ff, cc, rr}); - SubArray dcoeff_c = doutput; - dcoeff_c.offset({0, cc, 0}); - dcoeff_c.resize({ff, c - cc, rr}); - SubArray dcoeff_r = doutput; - dcoeff_r.offset({0, 0, rr}); - dcoeff_r.resize({ff, cc, r - rr}); - SubArray dcoeff_cf = doutput; - dcoeff_cf.offset({ff, cc, 0}); - dcoeff_cf.resize({f - ff, c - cc, rr}); - SubArray dcoeff_rf = doutput; - dcoeff_rf.offset({ff, 0, rr}); - dcoeff_rf.resize({f - ff, cc, r - rr}); - SubArray dcoeff_rc = doutput; - dcoeff_rc.offset({0, cc, rr}); - dcoeff_rc.resize({ff, c - cc, r - rr}); - SubArray dcoeff_rcf = doutput; - dcoeff_rcf.offset({ff, cc, rr}); - dcoeff_rcf.resize({f - ff, c - cc, r - rr}); - - SubArray<1, T> ratio_r({handle.dofs[2][l]}, handle.ratio[2][l]); - SubArray<1, T> ratio_c({handle.dofs[1][l]}, handle.ratio[1][l]); - SubArray<1, T> ratio_f({handle.dofs[0][l]}, handle.ratio[0][l]); - - T *null = NULL; - // // GpkReo3D, D, T, CUDA>(handle).Execute( - // // handle.dofs[2][l], handle.dofs[1][l], handle.dofs[0][l], - // // handle.dofs[2][l+1], handle.dofs[1][l+1], handle.dofs[0][l+1], - // // ratio_r, ratio_c, ratio_f, - // // dinput, dcoarse, - // // dcoeff_f, dcoeff_c, dcoeff_r, - // // dcoeff_cf, dcoeff_rf, dcoeff_rc, - // // dcoeff_rcf, - // // queue_idx); - // // handle.sync_all(); - // if (debug_print) { - // printf("after pi_Ql_reo\n"); - // print_matrix_cuda(handle.dofs[2][l], handle.dofs[1][l], - // handle.dofs[0][l], - // doutput.dv, doutput.ldvs_h[0], doutput.ldvs_h[1], - // doutput.ldvs_h[0]); - // } - - gpk_reo_3d( - handle, handle.dofs[2][l], handle.dofs[1][l], handle.dofs[0][l], - handle.ratio[2][l], handle.ratio[1][l], handle.ratio[0][l], dinput.dv, - dinput.lddv1, dinput.lddv2, dcoarse.dv, dcoarse.lddv1, dcoarse.lddv2, - // null, ldvs_h[0], ldvs_h[1], - dcoeff_f.dv, dcoeff_f.lddv1, dcoeff_f.lddv2, - // null, ldvs_h[0], ldvs_h[1], - dcoeff_c.dv, dcoeff_c.lddv1, dcoeff_c.lddv2, - // null, ldvs_h[0], ldvs_h[1], - dcoeff_r.dv, dcoeff_r.lddv1, dcoeff_r.lddv2, - // null, ldvs_h[0], ldvs_h[1], - dcoeff_cf.dv, dcoeff_cf.lddv1, dcoeff_cf.lddv2, - // null, ldvs_h[0], ldvs_h[1], - dcoeff_rf.dv, dcoeff_rf.lddv1, dcoeff_rf.lddv2, - // null, ldvs_h[0], ldvs_h[1], - dcoeff_rc.dv, dcoeff_rc.lddv1, dcoeff_rc.lddv2, - // null, ldvs_h[0], ldvs_h[1], - dcoeff_rcf.dv, dcoeff_rcf.lddv1, dcoeff_rcf.lddv2, - // null, ldvs_h[0], ldvs_h[1], - queue_idx, handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - // handle.sync_all(); - verify_matrix_cuda( - handle.dofs[2][l], handle.dofs[1][l], handle.dofs[0][l], doutput.dv, - doutput.ldvs_h[0], doutput.ldvs_h[1], doutput.ldvs_h[0], - prefix + "gpk_reo_3d" + "_level_" + std::to_string(l), store, verify); - - if (debug_print) { - printf("after pi_Ql_reo\n"); - print_matrix_cuda(handle.dofs[2][l], handle.dofs[1][l], handle.dofs[0][l], - doutput.dv, doutput.ldvs_h[0], doutput.ldvs_h[1], - doutput.ldvs_h[0]); - } -} - -template -void coefficients_restore_3d(Handle &handle, SubArray dinput, - SubArray &doutput, SIZE l, int queue_idx) { - - int range_l = std::min(6, (int)std::log2(handle.dofs[0][l]) - 1); - int range_lp1 = std::min(6, (int)std::log2(handle.dofs[0][l + 1]) - 1); - - std::string prefix = "decomp_"; - if (sizeof(T) == sizeof(double)) - prefix += "d_"; - if (sizeof(T) == sizeof(float)) - prefix += "f_"; - for (int d = 0; d < D; d++) - prefix += std::to_string(handle.shapes_h[0][d]) + "_"; - - dinput.project(0, 1, 2); - doutput.project(0, 1, 2); - - SIZE f = handle.dofs[0][l]; - SIZE c = handle.dofs[1][l]; - SIZE r = handle.dofs[2][l]; - SIZE ff = handle.dofs[0][l + 1]; - SIZE cc = handle.dofs[1][l + 1]; - SIZE rr = handle.dofs[2][l + 1]; - - SubArray dcoarse = dinput; - dcoarse.resize({ff, cc, rr}); - SubArray dcoeff_f = dinput; - dcoeff_f.offset({ff, 0, 0}); - dcoeff_f.resize({f - ff, cc, rr}); - SubArray dcoeff_c = dinput; - dcoeff_c.offset({0, cc, 0}); - dcoeff_c.resize({ff, c - cc, rr}); - SubArray dcoeff_r = dinput; - dcoeff_r.offset({0, 0, rr}); - dcoeff_r.resize({ff, cc, r - rr}); - SubArray dcoeff_cf = dinput; - dcoeff_cf.offset({ff, cc, 0}); - dcoeff_cf.resize({f - ff, c - cc, rr}); - SubArray dcoeff_rf = dinput; - dcoeff_rf.offset({ff, 0, rr}); - dcoeff_rf.resize({f - ff, cc, r - rr}); - SubArray dcoeff_rc = dinput; - dcoeff_rc.offset({0, cc, rr}); - dcoeff_rc.resize({ff, c - cc, r - rr}); - SubArray dcoeff_rcf = dinput; - dcoeff_rcf.offset({ff, cc, rr}); - dcoeff_rcf.resize({f - ff, c - cc, r - rr}); - - SubArray<1, T> ratio_r({handle.dofs[2][l]}, handle.ratio[2][l]); - SubArray<1, T> ratio_c({handle.dofs[1][l]}, handle.ratio[1][l]); - SubArray<1, T> ratio_f({handle.dofs[0][l]}, handle.ratio[0][l]); - - // GpkRev3D, D, T, CUDA>(handle).Execute( - // handle.dofs[2][l], handle.dofs[1][l], handle.dofs[0][l], - // handle.dofs[2][l+1], handle.dofs[1][l+1], handle.dofs[0][l+1], - // ratio_r, ratio_c, ratio_f, - // doutput, dcoarse, - // dcoeff_f, dcoeff_c, dcoeff_r, - // dcoeff_cf, dcoeff_rf, dcoeff_rc, - // dcoeff_rcf, - // 0, 0, 0, - // handle.dofs[2][l], handle.dofs[1][l], handle.dofs[0][l], - // queue_idx); - - T *null = NULL; - gpk_rev_3d( - handle, handle.dofs[2][l], handle.dofs[1][l], handle.dofs[0][l], - handle.ratio[2][l], handle.ratio[1][l], handle.ratio[0][l], doutput.dv, - doutput.lddv1, doutput.lddv2, dcoarse.dv, dcoarse.lddv1, dcoarse.lddv2, - // null, ldvs_h[0], ldvs_h[1], - dcoeff_f.dv, dcoeff_f.lddv1, dcoeff_f.lddv2, - // null, ldvs_h[0], ldvs_h[1], - dcoeff_c.dv, dcoeff_c.lddv1, dcoeff_c.lddv2, - // null, ldvs_h[0], ldvs_h[1], - dcoeff_r.dv, dcoeff_r.lddv1, dcoeff_r.lddv2, - // null, ldvs_h[0], ldvs_h[1], - dcoeff_cf.dv, dcoeff_cf.lddv1, dcoeff_cf.lddv2, - // null, ldvs_h[0], ldvs_h[1], - dcoeff_rf.dv, dcoeff_rf.lddv1, dcoeff_rf.lddv2, - // null, ldvs_h[0], ldvs_h[1], - dcoeff_rc.dv, dcoeff_rc.lddv1, dcoeff_rc.lddv2, - // null, ldvs_h[0], ldvs_h[1], - dcoeff_rcf.dv, dcoeff_rcf.lddv1, dcoeff_rcf.lddv2, - // null, ldvs_h[0], ldvs_h[1], - 0, 0, 0, handle.dofs[2][l], handle.dofs[1][l], handle.dofs[0][l], - queue_idx, handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - - // handle.sync(0); - verify_matrix_cuda( - handle.dofs[2][l], handle.dofs[1][l], handle.dofs[0][l], doutput.dv, - doutput.ldvs_h[0], doutput.ldvs_h[1], doutput.ldvs_h[0], - prefix + "gpk_rev_3d" + "_level_" + std::to_string(l), store, verify); - - // gpk_rev(handle, - // shape, shape_c, handle.ldws_h, ldvs_h, unprocessed_dims, - // 2, 1, 0, - // handle.ratio[2][l], handle.ratio[1][l], handle.ratio[0][l], - // handle.dw, handle.ldws_h[0], handle.ldws_h[1], - // dv, ldvs_h[0], ldvs_h[1], - // dv+get_idx(ldvs_h[0], ldvs_h[1], 0, 0, handle.dofs[0][l+1]), - // ldvs_h[0], ldvs_h[1], - // // null, ldvs_h[0], ldvs_h[1], - // dv+get_idx(ldvs_h[0], ldvs_h[1], 0, handle.dofs[1][l+1], 0), - // ldvs_h[0], ldvs_h[1], - // // null, ldvs_h[0], ldvs_h[1], - // dv+get_idx(ldvs_h[0], ldvs_h[1], handle.dofs[2][l+1], 0, 0), - // ldvs_h[0], ldvs_h[1], - // // null, ldvs_h[0], ldvs_h[1], - // dv+get_idx(ldvs_h[0], ldvs_h[1], 0, handle.dofs[1][l+1], - // handle.dofs[0][l+1]), ldvs_h[0], ldvs_h[1], - // // null, ldvs_h[0], ldvs_h[1], - // dv+get_idx(ldvs_h[0], ldvs_h[1], handle.dofs[2][l+1], 0, - // handle.dofs[0][l+1]), ldvs_h[0], ldvs_h[1], - // // null, ldvs_h[0], ldvs_h[1], - // dv+get_idx(ldvs_h[0], ldvs_h[1], handle.dofs[2][l+1], - // handle.dofs[1][l+1], 0), ldvs_h[0], ldvs_h[1], - // // null,ldvs_h[0], ldvs_h[1], - // dv+get_idx(ldvs_h[0], ldvs_h[1], handle.dofs[2][l+1], - // handle.dofs[1][l+1], handle.dofs[0][l+1]), ldvs_h[0], - // ldvs_h[1], - // // null, ldvs_h[0], ldvs_h[1], - // 0, 0, 0, handle.dofs[2][l], handle.dofs[1][l], - // handle.dofs[0][l], 0, - // handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - - // print_matrix_cuda(handle.dofs[2][l], handle.dofs[1][l], - // handle.dofs[0][l], doutput.dv, doutput.ldvs_h[0], doutput.ldvs_h[1], - // doutput.ldvs_h[0],); - - // gpk_rev(handle, - // shape, shape_c, handle.ldws_h, ldvs_h, unprocessed_dims, - // 2, 1, 0, - // handle.ratio[2][l], handle.ratio[1][l], handle.ratio[0][l], - // handle.dw, handle.ldws_h[0], handle.ldws_h[1], - // dv, ldvs_h[0], ldvs_h[1], - // dv+get_idx(ldvs_h[0], ldvs_h[1], 0, 0, handle.dofs[0][l+1]), - // ldvs_h[0], ldvs_h[1], - // // null, ldvs_h[0], ldvs_h[1], - // dv+get_idx(ldvs_h[0], ldvs_h[1], 0, handle.dofs[1][l+1], 0), - // ldvs_h[0], ldvs_h[1], - // // null, ldvs_h[0], ldvs_h[1], - // dv+get_idx(ldvs_h[0], ldvs_h[1], handle.dofs[2][l+1], 0, 0), - // ldvs_h[0], ldvs_h[1], - // // null, ldvs_h[0], ldvs_h[1], - // dv+get_idx(ldvs_h[0], ldvs_h[1], 0, handle.dofs[1][l+1], - // handle.dofs[0][l+1]), ldvs_h[0], ldvs_h[1], - // // null, ldvs_h[0], ldvs_h[1], - // dv+get_idx(ldvs_h[0], ldvs_h[1], handle.dofs[2][l+1], 0, - // handle.dofs[0][l+1]), ldvs_h[0], ldvs_h[1], - // // null, ldvs_h[0], ldvs_h[1], - // dv+get_idx(ldvs_h[0], ldvs_h[1], handle.dofs[2][l+1], - // handle.dofs[1][l+1], 0), ldvs_h[0], ldvs_h[1], - // // null,ldvs_h[0], ldvs_h[1], - // dv+get_idx(ldvs_h[0], ldvs_h[1], handle.dofs[2][l+1], - // handle.dofs[1][l+1], handle.dofs[0][l+1]), ldvs_h[0], - // ldvs_h[1], - // // null, ldvs_h[0], ldvs_h[1], - // 0, 0, 0, handle.dofs[2][l], handle.dofs[1][l], - // handle.dofs[0][l], 0, - // handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - - if (debug_print) { - printf("after coeff-restore\n"); - print_matrix_cuda(handle.dofs[2][l], handle.dofs[1][l], handle.dofs[0][l], - doutput.dv, doutput.ldvs_h[0], doutput.ldvs_h[1], - doutput.ldvs_h[0]); - } -} - -template -void calc_correction_3d(Handle &handle, SubArray dcoeff, - SubArray &dcorrection, SIZE l, int queue_idx) { - - int range_l = std::min(6, (int)std::log2(handle.dofs[0][l]) - 1); - int range_lp1 = std::min(6, (int)std::log2(handle.dofs[0][l + 1]) - 1); - - std::string prefix = "decomp_"; - if (sizeof(T) == sizeof(double)) - prefix += "d_"; - if (sizeof(T) == sizeof(float)) - prefix += "f_"; - for (int d = 0; d < D; d++) - prefix += std::to_string(handle.shapes_h[0][d]) + "_"; - - SubArray dw_in1, dw_in2, dw_out; - if (D >= 1) { - dw_in1 = dcoeff; - dw_in1.resize( - {handle.dofs[0][l + 1], handle.dofs[1][l], handle.dofs[2][l]}); - dw_in2 = dcoeff; - dw_in2.offset({handle.dofs[0][l + 1], 0, 0}); - dw_in2.resize({handle.dofs[0][l] - handle.dofs[0][l + 1], handle.dofs[1][l], - handle.dofs[2][l]}); - dw_out = dcorrection; - dw_out.resize( - {handle.dofs[0][l + 1], handle.dofs[1][l], handle.dofs[2][l]}); - - lpk_reo_1_3d( - handle, handle.dofs[2][l], handle.dofs[1][l], handle.dofs[0][l], - handle.dofs[0][l + 1], handle.dofs[2][l + 1], handle.dofs[1][l + 1], - handle.dofs[0][l + 1], handle.dist[0][l], handle.ratio[0][l], dw_in1.dv, - dw_in1.ldvs_h[0], dw_in1.ldvs_h[1], dw_in2.dv, dw_in2.ldvs_h[0], - dw_in2.ldvs_h[1], dw_out.dv, dw_out.ldvs_h[0], dw_out.ldvs_h[1], - queue_idx, - handle.auto_tuning_mr1[handle.arch][handle.precision][range_lp1]); - - verify_matrix_cuda( - handle.dofs[2][l], handle.dofs[1][l], handle.dofs[0][l + 1], dw_out.dv, - dw_out.ldvs_h[0], dw_out.ldvs_h[1], dw_out.ldvs_h[0], - prefix + "lpk_reo_1_3d" + "_level_" + std::to_string(l), store, verify); - - if (debug_print) { - printf("after mass_trans_multiply_1_cpt:\n"); - print_matrix_cuda(handle.dofs[2][l], handle.dofs[1][l], - handle.dofs[0][l + 1], dw_out.dv, dw_out.ldvs_h[0], - dw_out.ldvs_h[1], dw_out.ldvs_h[0]); - } - - // PrintSubarray("after mass_trans_multiply_1_cpt::dw_in1", dw_in1); - // PrintSubarray("after mass_trans_multiply_1_cpt::dw_in2", dw_in2); - // PrintSubarray("after mass_trans_multiply_1_cpt::dw_out", dw_out); - } - - if (D >= 2) { - dw_in1 = dw_out; - dw_in1.resize( - {handle.dofs[0][l + 1], handle.dofs[1][l + 1], handle.dofs[2][l]}); - dw_in2 = dw_out; - dw_in2.offset({0, handle.dofs[1][l + 1], 0}); - dw_in2.resize({handle.dofs[0][l + 1], - handle.dofs[1][l] - handle.dofs[1][l + 1], - handle.dofs[2][l]}); - dw_out.offset({handle.dofs[0][l + 1], 0, 0}); - dw_out.resize( - {handle.dofs[0][l + 1], handle.dofs[1][l + 1], handle.dofs[2][l]}); - - lpk_reo_2_3d( - handle, handle.dofs[2][l], handle.dofs[1][l], handle.dofs[0][l + 1], - handle.dofs[1][l + 1], handle.dist[1][l], handle.ratio[1][l], dw_in1.dv, - dw_in1.ldvs_h[0], dw_in1.ldvs_h[1], dw_in2.dv, dw_in2.ldvs_h[0], - dw_in2.ldvs_h[1], dw_out.dv, dw_out.ldvs_h[0], dw_out.ldvs_h[1], - queue_idx, - handle.auto_tuning_mr1[handle.arch][handle.precision][range_lp1]); - - // handle.sync(0); - verify_matrix_cuda( - handle.dofs[2][l], handle.dofs[1][l + 1], handle.dofs[0][l + 1], - dw_out.dv, dw_out.ldvs_h[0], dw_out.ldvs_h[1], dw_out.ldvs_h[0], - prefix + "lpk_reo_2_3d" + "_level_" + std::to_string(l), store, verify); - - if (debug_print) { - printf("after mass_trans_multiply_2_cpt\n"); - print_matrix_cuda(handle.dofs[2][l], handle.dofs[1][l + 1], - handle.dofs[0][l + 1], dw_out.dv, dw_out.ldvs_h[0], - dw_out.ldvs_h[1], dw_out.ldvs_h[0]); - } - } - - if (D == 3) { - dw_in1 = dw_out; - dw_in1.resize( - {handle.dofs[0][l + 1], handle.dofs[1][l + 1], handle.dofs[2][l + 1]}); - dw_in2 = dw_out; - dw_in2.offset({0, 0, handle.dofs[2][l + 1]}); - dw_in2.resize({handle.dofs[0][l + 1], handle.dofs[1][l + 1], - handle.dofs[2][l] - handle.dofs[2][l + 1]}); - dw_out.offset({handle.dofs[0][l + 1], handle.dofs[1][l + 1], 0}); - dw_out.resize( - {handle.dofs[0][l + 1], handle.dofs[1][l + 1], handle.dofs[2][l + 1]}); - - lpk_reo_3_3d( - handle, handle.dofs[2][l], handle.dofs[1][l + 1], handle.dofs[0][l + 1], - handle.dofs[2][l + 1], handle.dist[2][l], handle.ratio[2][l], dw_in1.dv, - dw_in1.ldvs_h[0], dw_in1.ldvs_h[1], dw_in2.dv, dw_in2.ldvs_h[0], - dw_in2.ldvs_h[1], dw_out.dv, dw_out.ldvs_h[0], dw_out.ldvs_h[1], - queue_idx, - handle.auto_tuning_mr1[handle.arch][handle.precision][range_lp1]); - - // handle.sync(0); - verify_matrix_cuda( - handle.dofs[2][l + 1], handle.dofs[1][l + 1], handle.dofs[0][l + 1], - dw_out.dv, dw_out.ldvs_h[0], dw_out.ldvs_h[1], dw_out.ldvs_h[0], - prefix + "lpk_reo_3_3d" + "_level_" + std::to_string(l), store, verify); - - if (debug_print) { - printf("after mass_trans_multiply_3_cpt\n"); - print_matrix_cuda(handle.dofs[2][l + 1], handle.dofs[1][l + 1], - handle.dofs[0][l + 1], dw_out.dv, dw_out.ldvs_h[0], - dw_out.ldvs_h[1], dw_out.ldvs_h[0]); - } - } - - if (D >= 1) { - ipk_1_3d(handle, handle.dofs[2][l + 1], handle.dofs[1][l + 1], - handle.dofs[0][l + 1], handle.am[0][l + 1], handle.bm[0][l + 1], - handle.dist[0][l + 1], dw_out.dv, dw_out.ldvs_h[0], - dw_out.ldvs_h[1], queue_idx, - handle.auto_tuning_ts1[handle.arch][handle.precision][range_lp1]); - - // //handle.sync(0); - verify_matrix_cuda( - handle.dofs[2][l + 1], handle.dofs[1][l + 1], handle.dofs[0][l + 1], - dw_out.dv, dw_out.ldvs_h[0], dw_out.ldvs_h[1], dw_out.ldvs_h[0], - prefix + "ipk_1_3d" + "_level_" + std::to_string(l), store, verify); - - if (debug_print) { - printf("after solve_tridiag_1_cpt\n"); - print_matrix_cuda(handle.dofs[2][l + 1], handle.dofs[1][l + 1], - handle.dofs[0][l + 1], dw_out.dv, dw_out.ldvs_h[0], - dw_out.ldvs_h[1], dw_out.ldvs_h[0]); - } - } - if (D >= 2) { - ipk_2_3d(handle, handle.dofs[2][l + 1], handle.dofs[1][l + 1], - handle.dofs[0][l + 1], handle.am[1][l + 1], handle.bm[1][l + 1], - handle.dist[1][l + 1], dw_out.dv, dw_out.ldvs_h[0], - dw_out.ldvs_h[1], queue_idx, - handle.auto_tuning_ts1[handle.arch][handle.precision][range_lp1]); - - // handle.sync(0); - verify_matrix_cuda( - handle.dofs[2][l + 1], handle.dofs[1][l + 1], handle.dofs[0][l + 1], - dw_out.dv, dw_out.ldvs_h[0], dw_out.ldvs_h[1], dw_out.ldvs_h[0], - prefix + "ipk_2_3d" + "_level_" + std::to_string(l), store, verify); - - if (debug_print) { - printf("after solve_tridiag_2_cpt\n"); - print_matrix_cuda(handle.dofs[2][l + 1], handle.dofs[1][l + 1], - handle.dofs[0][l + 1], dw_out.dv, dw_out.ldvs_h[0], - dw_out.ldvs_h[1], dw_out.ldvs_h[0]); - } - } - - if (D == 3) { - ipk_3_3d(handle, handle.dofs[2][l + 1], handle.dofs[1][l + 1], - handle.dofs[0][l + 1], handle.am[2][l + 1], handle.bm[2][l + 1], - handle.dist[2][l + 1], dw_out.dv, dw_out.ldvs_h[0], - dw_out.ldvs_h[1], queue_idx, - handle.auto_tuning_ts3[handle.arch][handle.precision][range_lp1]); - - // handle.sync(0); - verify_matrix_cuda( - handle.dofs[2][l + 1], handle.dofs[1][l + 1], handle.dofs[0][l + 1], - dw_out.dv, dw_out.ldvs_h[0], dw_out.ldvs_h[1], dw_out.ldvs_h[0], - prefix + "ipk_3_3d" + "_level_" + std::to_string(l), store, verify); - - if (debug_print) { - printf("after solve_tridiag_3_cpt\n"); - print_matrix_cuda(handle.dofs[2][l + 1], handle.dofs[1][l + 1], - handle.dofs[0][l + 1], dw_out.dv, dw_out.ldvs_h[0], - dw_out.ldvs_h[1], dw_out.ldvs_h[0]); - } - } - - // final correction output - dcorrection = dw_out; -} - -template -void calc_coefficients_nd(Handle &handle, SubArray dinput1, - SubArray dinput2, SubArray &doutput, - SIZE l, int queue_idx) { - - int range_l = std::min(6, (int)std::log2(handle.dofs[0][l]) - 1); - int range_lp1 = std::min(6, (int)std::log2(handle.dofs[0][l + 1]) - 1); - - std::string prefix = "decomp_"; - if (sizeof(T) == sizeof(double)) - prefix += "d_"; - if (sizeof(T) == sizeof(float)) - prefix += "f_"; - for (int d = 0; d < D; d++) - prefix += std::to_string(handle.shapes_h[0][d]) + "_"; - // printf("interpolate 1-3D\n"); - - SubArray dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, - dcoeff_rc, dcoeff_rcf; - - DIM curr_dims[3]; - - int unprocessed_idx = 0; - curr_dims[0] = 0; - curr_dims[1] = 1; - curr_dims[2] = 2; - dinput1.project(curr_dims[0], curr_dims[1], curr_dims[2]); - doutput.project(curr_dims[0], curr_dims[1], curr_dims[2]); - calc_coeff_pointers(handle, curr_dims, l, doutput, dcoarse, dcoeff_f, - dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, dcoeff_rc, - dcoeff_rcf); - - gpk_reo( - handle, handle.shapes_h[l], handle.shapes_d[l], handle.shapes_d[l + 1], - dinput1.ldvs_d, doutput.ldvs_d, handle.unprocessed_n[unprocessed_idx], - handle.unprocessed_dims_d[unprocessed_idx], curr_dims[2], curr_dims[1], - curr_dims[0], handle.ratio[curr_dims[2]][l], - handle.ratio[curr_dims[1]][l], handle.ratio[curr_dims[0]][l], dinput1.dv, - dinput1.lddv1, dinput1.lddv2, dcoarse.dv, dcoarse.lddv1, dcoarse.lddv2, - // null, lddv1, lddv2, - dcoeff_f.dv, dcoeff_f.lddv1, dcoeff_f.lddv2, - // null, lddv1, lddv2, - dcoeff_c.dv, dcoeff_c.lddv1, dcoeff_c.lddv2, - // null, lddv1, lddv2, - dcoeff_r.dv, dcoeff_r.lddv1, dcoeff_r.lddv2, - // null, lddv1, lddv2, - dcoeff_cf.dv, dcoeff_cf.lddv1, dcoeff_cf.lddv2, - // null, lddv1, lddv2, - dcoeff_rf.dv, dcoeff_rf.lddv1, dcoeff_rf.lddv2, - // null, lddv1, lddv2, - dcoeff_rc.dv, dcoeff_rc.lddv1, dcoeff_rc.lddv2, - // null, lddv1, lddv2, - dcoeff_rcf.dv, dcoeff_rcf.lddv1, dcoeff_rcf.lddv2, - // null, lddv1, lddv2, - queue_idx, handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - - for (DIM d = 3; d < D; d += 2) { - // copy back to input1 for interpolation again - lwpk(handle, handle.shapes_h[l], handle.shapes_d[l], doutput.dv, - doutput.ldvs_d, dinput1.dv, dinput1.ldvs_d, queue_idx); - - // printf("interpolate %u-%uD\n", d+1, d+2); - curr_dims[0] = 0; - curr_dims[1] = d; - curr_dims[2] = d + 1; - dinput1.project(curr_dims[0], curr_dims[1], curr_dims[2]); - doutput.project(curr_dims[0], curr_dims[1], curr_dims[2]); - calc_coeff_pointers(handle, curr_dims, l, doutput, dcoarse, dcoeff_f, - dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, dcoeff_rc, - dcoeff_rcf); - - // printf("lddv1(%d), lddv2(%d), lddw1(%d), lddw2(%d)\n", lddv1, lddv2, - // lddw1, lddw2); - if (D - d == 1) { - unprocessed_idx += 1; - gpk_reo( - handle, handle.shapes_h[l], handle.shapes_d[l], - handle.shapes_d[l + 1], dinput1.ldvs_d, doutput.ldvs_d, - handle.unprocessed_n[unprocessed_idx], - handle.unprocessed_dims_d[unprocessed_idx], curr_dims[2], - curr_dims[1], curr_dims[0], handle.ratio[curr_dims[2]][l], - handle.ratio[curr_dims[1]][l], handle.ratio[curr_dims[0]][l], - dinput1.dv, dinput1.lddv1, dinput1.lddv2, dcoarse.dv, dcoarse.lddv1, - dcoarse.lddv2, - // null, lddv1, lddv2, - dcoeff_f.dv, dcoeff_f.lddv1, dcoeff_f.lddv2, - // null, lddv1, lddv2, - dcoeff_c.dv, dcoeff_c.lddv1, dcoeff_c.lddv2, - // null, lddv1, lddv2, - dcoeff_r.dv, dcoeff_r.lddv1, dcoeff_r.lddv2, - // null, lddv1, lddv2, - dcoeff_cf.dv, dcoeff_cf.lddv1, dcoeff_cf.lddv2, - // null, lddv1, lddv2, - dcoeff_rf.dv, dcoeff_rf.lddv1, dcoeff_rf.lddv2, - // null, lddv1, lddv2, - dcoeff_rc.dv, dcoeff_rc.lddv1, dcoeff_rc.lddv2, - // null, lddv1, lddv2, - dcoeff_rcf.dv, dcoeff_rcf.lddv1, dcoeff_rcf.lddv2, - // null, lddv1, lddv2, - queue_idx, - handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - } else { // D - d >= 2 - unprocessed_idx += 2; - gpk_reo( - handle, handle.shapes_h[l], handle.shapes_d[l], - handle.shapes_d[l + 1], dinput1.ldvs_d, doutput.ldvs_d, - handle.unprocessed_n[unprocessed_idx], - handle.unprocessed_dims_d[unprocessed_idx], curr_dims[2], - curr_dims[1], curr_dims[0], handle.ratio[curr_dims[2]][l], - handle.ratio[curr_dims[1]][l], handle.ratio[curr_dims[0]][l], - dinput1.dv, dinput1.lddv1, dinput1.lddv2, dcoarse.dv, dcoarse.lddv1, - dcoarse.lddv2, - // null, lddv1, lddv2, - dcoeff_f.dv, dcoeff_f.lddv1, dcoeff_f.lddv2, - // null, lddv1, lddv2, - dcoeff_c.dv, dcoeff_c.lddv1, dcoeff_c.lddv2, - // null, lddv1, lddv2, - dcoeff_r.dv, dcoeff_r.lddv1, dcoeff_r.lddv2, - // null, lddv1, lddv2, - dcoeff_cf.dv, dcoeff_cf.lddv1, dcoeff_cf.lddv2, - // null, lddv1, lddv2, - dcoeff_rf.dv, dcoeff_rf.lddv1, dcoeff_rf.lddv2, - // null, lddv1, lddv2, - dcoeff_rc.dv, dcoeff_rc.lddv1, dcoeff_rc.lddv2, - // null, lddv1, lddv2, - dcoeff_rcf.dv, dcoeff_rcf.lddv1, dcoeff_rcf.lddv2, - // null, lddv1, lddv2, - queue_idx, - handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - } - } - - if (debug_print) { // debug - printf(" after interpolation\n"); - for (int k = 0; k < doutput.shape[4]; k++) { - for (int j = 0; j < doutput.shape[3]; j++) { - printf("i,j = %d,%d\n", k, j); - print_matrix_cuda( - doutput.shape[2], doutput.shape[1], doutput.shape[0], - doutput.dv + - k * doutput.ldvs_h[0] * doutput.ldvs_h[1] * doutput.ldvs_h[2] * - doutput.ldvs_h[3] + - j * doutput.ldvs_h[0] * doutput.ldvs_h[1] * doutput.ldvs_h[2], - doutput.ldvs_h[0], doutput.ldvs_h[1], doutput.ldvs_h[0]); - } - } - } // debug - - unprocessed_idx = 0; - // printf("reorder 1-3D\n"); - curr_dims[0] = 0; - curr_dims[1] = 1; - curr_dims[2] = 2; - dinput2.project(curr_dims[0], curr_dims[1], curr_dims[2]); - dinput1.project(curr_dims[0], curr_dims[1], - curr_dims[2]); // reuse input1 as temp output - calc_coeff_pointers(handle, curr_dims, l, dinput1, dcoarse, dcoeff_f, - dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, dcoeff_rc, - dcoeff_rcf); - - gpk_reo( - handle, handle.shapes_h[l], handle.shapes_d[l], handle.shapes_d[l + 1], - dinput2.ldvs_d, dinput1.ldvs_d, handle.unprocessed_n[unprocessed_idx], - handle.unprocessed_dims_d[unprocessed_idx], curr_dims[2], curr_dims[1], - curr_dims[0], handle.ratio[curr_dims[2]][l], - handle.ratio[curr_dims[1]][l], handle.ratio[curr_dims[0]][l], dinput2.dv, - dinput2.lddv1, dinput2.lddv2, dcoarse.dv, dcoarse.lddv1, dcoarse.lddv2, - // null, lddv1, lddv2, - dcoeff_f.dv, dcoeff_f.lddv1, dcoeff_f.lddv2, - // null, lddv1, lddv2, - dcoeff_c.dv, dcoeff_c.lddv1, dcoeff_c.lddv2, - // null, lddv1, lddv2, - dcoeff_r.dv, dcoeff_r.lddv1, dcoeff_r.lddv2, - // null, lddv1, lddv2, - dcoeff_cf.dv, dcoeff_cf.lddv1, dcoeff_cf.lddv2, - // null, lddv1, lddv2, - dcoeff_rf.dv, dcoeff_rf.lddv1, dcoeff_rf.lddv2, - // null, lddv1, lddv2, - dcoeff_rc.dv, dcoeff_rc.lddv1, dcoeff_rc.lddv2, - // null, lddv1, lddv2, - dcoeff_rcf.dv, dcoeff_rcf.lddv1, dcoeff_rcf.lddv2, - // null, lddv1, lddv2, - queue_idx, handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - - DIM D_reduced = D % 2 == 0 ? D - 1 : D - 2; - for (DIM d = 3; d < D_reduced; d += 2) { - // copy back to input2 for reordering again - lwpk(handle, handle.shapes_h[l], handle.shapes_d[l], dinput1.dv, - dinput1.ldvs_d, dinput2.dv, dinput2.ldvs_d, queue_idx); - - // printf("reorder %u-%uD\n", d+1, d+2); - curr_dims[0] = 0; - curr_dims[1] = d; - curr_dims[2] = d + 1; - dinput2.project(curr_dims[0], curr_dims[1], curr_dims[2]); - dinput1.project(curr_dims[0], curr_dims[1], - curr_dims[2]); // reuse input1 as temp output - calc_coeff_pointers(handle, curr_dims, l, dinput1, dcoarse, dcoeff_f, - dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, dcoeff_rc, - dcoeff_rcf); - unprocessed_idx += 2; - gpk_reo( - handle, handle.shapes_h[l], handle.shapes_d[l], handle.shapes_d[l + 1], - dinput2.ldvs_d, dinput1.ldvs_d, handle.unprocessed_n[unprocessed_idx], - handle.unprocessed_dims_d[unprocessed_idx], curr_dims[2], curr_dims[1], - curr_dims[0], handle.ratio[curr_dims[2]][l], - handle.ratio[curr_dims[1]][l], handle.ratio[curr_dims[0]][l], - dinput2.dv, dinput2.lddv1, dinput2.lddv2, dcoarse.dv, dcoarse.lddv1, - dcoarse.lddv2, - // null, lddv1, lddv2, - dcoeff_f.dv, dcoeff_f.lddv1, dcoeff_f.lddv2, - // null, lddv1, lddv2, - dcoeff_c.dv, dcoeff_c.lddv1, dcoeff_c.lddv2, - // null, lddv1, lddv2, - dcoeff_r.dv, dcoeff_r.lddv1, dcoeff_r.lddv2, - // null, lddv1, lddv2, - dcoeff_cf.dv, dcoeff_cf.lddv1, dcoeff_cf.lddv2, - // null, lddv1, lddv2, - dcoeff_rf.dv, dcoeff_rf.lddv1, dcoeff_rf.lddv2, - // null, lddv1, lddv2, - dcoeff_rc.dv, dcoeff_rc.lddv1, dcoeff_rc.lddv2, - // null, lddv1, lddv2, - dcoeff_rcf.dv, dcoeff_rcf.lddv1, dcoeff_rcf.lddv2, - // null, lddv1, lddv2, - queue_idx, - handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - } - - // printf("calc coeff %u-%dD\n", D_reduced+1, D_reduced+2); - curr_dims[0] = 0; - curr_dims[1] = D_reduced; - curr_dims[2] = D_reduced + 1; - dinput1.project(curr_dims[0], curr_dims[1], curr_dims[2]); - doutput.project(curr_dims[0], curr_dims[1], - curr_dims[2]); // reuse input1 as temp output - calc_coeff_pointers(handle, curr_dims, l, doutput, dcoarse, dcoeff_f, - dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, dcoeff_rc, - dcoeff_rcf); - if (D - D_reduced == 1) { - // unprocessed_dims.pop_back(); - unprocessed_idx += 1; - gpk_reo( - handle, handle.shapes_h[l], handle.shapes_d[l], handle.shapes_d[l + 1], - dinput1.ldvs_d, doutput.ldvs_d, handle.unprocessed_n[unprocessed_idx], - handle.unprocessed_dims_d[unprocessed_idx], curr_dims[2], curr_dims[1], - curr_dims[0], handle.ratio[curr_dims[2]][l], - handle.ratio[curr_dims[1]][l], handle.ratio[curr_dims[0]][l], - dinput1.dv, dinput1.lddv1, dinput1.lddv2, dcoarse.dv, dcoarse.lddv1, - dcoarse.lddv2, - // null, lddv1, lddv2, - dcoeff_f.dv, dcoeff_f.lddv1, dcoeff_f.lddv2, - // null, lddv1, lddv2, - dcoeff_c.dv, dcoeff_c.lddv1, dcoeff_c.lddv2, - // null, lddv1, lddv2, - dcoeff_r.dv, dcoeff_r.lddv1, dcoeff_r.lddv2, - // null, lddv1, lddv2, - dcoeff_cf.dv, dcoeff_cf.lddv1, dcoeff_cf.lddv2, - // null, lddv1, lddv2, - dcoeff_rf.dv, dcoeff_rf.lddv1, dcoeff_rf.lddv2, - // null, lddv1, lddv2, - dcoeff_rc.dv, dcoeff_rc.lddv1, dcoeff_rc.lddv2, - // null, lddv1, lddv2, - dcoeff_rcf.dv, dcoeff_rcf.lddv1, dcoeff_rcf.lddv2, - // null, lddv1, lddv2, - queue_idx, - handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - - } else { // D-D_reduced == 2 - unprocessed_idx += 2; - gpk_reo( - handle, handle.shapes_h[l], handle.shapes_d[l], handle.shapes_d[l + 1], - dinput1.ldvs_d, doutput.ldvs_d, handle.unprocessed_n[unprocessed_idx], - handle.unprocessed_dims_d[unprocessed_idx], curr_dims[2], curr_dims[1], - curr_dims[0], handle.ratio[curr_dims[2]][l], - handle.ratio[curr_dims[1]][l], handle.ratio[curr_dims[0]][l], - dinput1.dv, dinput1.lddv1, dinput1.lddv2, dcoarse.dv, dcoarse.lddv1, - dcoarse.lddv2, - // null, lddv1, lddv2, - dcoeff_f.dv, dcoeff_f.lddv1, dcoeff_f.lddv2, - // null, lddv1, lddv2, - dcoeff_c.dv, dcoeff_c.lddv1, dcoeff_c.lddv2, - // null, lddv1, lddv2, - dcoeff_r.dv, dcoeff_r.lddv1, dcoeff_r.lddv2, - // null, lddv1, lddv2, - dcoeff_cf.dv, dcoeff_cf.lddv1, dcoeff_cf.lddv2, - // null, lddv1, lddv2, - dcoeff_rf.dv, dcoeff_rf.lddv1, dcoeff_rf.lddv2, - // null, lddv1, lddv2, - dcoeff_rc.dv, dcoeff_rc.lddv1, dcoeff_rc.lddv2, - // null, lddv1, lddv2, - dcoeff_rcf.dv, dcoeff_rcf.lddv1, dcoeff_rcf.lddv2, - // null, lddv1, lddv2, - queue_idx, - handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - } - - if (debug_print) { // debug - printf(" after calc coeff\n"); - for (int k = 0; k < doutput.shape[4]; k++) { - for (int j = 0; j < doutput.shape[3]; j++) { - printf("i,j = %d,%d\n", k, j); - print_matrix_cuda( - doutput.shape[2], doutput.shape[1], doutput.shape[0], - doutput.dv + - k * doutput.ldvs_h[0] * doutput.ldvs_h[1] * doutput.ldvs_h[2] * - doutput.ldvs_h[3] + - j * doutput.ldvs_h[0] * doutput.ldvs_h[1] * doutput.ldvs_h[2], - doutput.ldvs_h[0], doutput.ldvs_h[1], doutput.ldvs_h[0]); - } - } - } // debug -} - -template -void coefficients_restore_nd(Handle &handle, SubArray dinput1, - SubArray dinput2, SubArray &doutput, - SIZE l, int queue_idx) { - - int range_l = std::min(6, (int)std::log2(handle.dofs[0][l]) - 1); - int range_lp1 = std::min(6, (int)std::log2(handle.dofs[0][l + 1]) - 1); - - std::string prefix = "decomp_"; - if (sizeof(T) == sizeof(double)) - prefix += "d_"; - if (sizeof(T) == sizeof(float)) - prefix += "f_"; - for (int d = 0; d < D; d++) - prefix += std::to_string(handle.shapes_h[0][d]) + "_"; - - SubArray dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, - dcoeff_rc, dcoeff_rcf; - - DIM curr_dims[3]; - int unprocessed_idx = 0; - - // printf("interpolate-restore 1-3D\n"); - curr_dims[0] = 0; - curr_dims[1] = 1; - curr_dims[2] = 2; - dinput1.project(curr_dims[0], curr_dims[1], curr_dims[2]); - doutput.project(curr_dims[0], curr_dims[1], curr_dims[2]); - calc_coeff_pointers(handle, curr_dims, l, dinput1, dcoarse, dcoeff_f, - dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, dcoeff_rc, - dcoeff_rcf); - - gpk_rev( - handle, handle.shapes_h[l], handle.shapes_d[l], handle.shapes_d[l + 1], - doutput.ldvs_d, dinput1.ldvs_d, handle.unprocessed_n[unprocessed_idx], - handle.unprocessed_dims_d[unprocessed_idx], curr_dims[2], curr_dims[1], - curr_dims[0], handle.ratio[curr_dims[2]][l], - handle.ratio[curr_dims[1]][l], handle.ratio[curr_dims[0]][l], doutput.dv, - doutput.lddv1, doutput.lddv2, dcoarse.dv, dcoarse.lddv1, dcoarse.lddv2, - // null, lddv1, lddv2, - dcoeff_f.dv, dcoeff_f.lddv1, dcoeff_f.lddv2, - // null, lddv1, lddv2, - dcoeff_c.dv, dcoeff_c.lddv1, dcoeff_c.lddv2, - // null, lddv1, lddv2, - dcoeff_r.dv, dcoeff_r.lddv1, dcoeff_r.lddv2, - // null, lddv1, lddv2, - dcoeff_cf.dv, dcoeff_cf.lddv1, dcoeff_cf.lddv2, - // null, lddv1, lddv2, - dcoeff_rf.dv, dcoeff_rf.lddv1, dcoeff_rf.lddv2, - // null, lddv1, lddv2, - dcoeff_rc.dv, dcoeff_rc.lddv1, dcoeff_rc.lddv2, - // null, lddv1, lddv2, - dcoeff_rcf.dv, dcoeff_rcf.lddv1, dcoeff_rcf.lddv2, - // null, lddv1, lddv2, - 0, 0, 0, handle.dofs[curr_dims[2]][l], handle.dofs[curr_dims[1]][l], - handle.dofs[curr_dims[0]][l], queue_idx, - handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - - for (DIM d = 3; d < D; d += 2) { - lwpk(handle, handle.shapes_h[l], handle.shapes_d[l], doutput.dv, - doutput.ldvs_d, dinput1.dv, dinput1.ldvs_d, queue_idx); - - // printf("interpolate-restore %u-%uD\n", d+1, d+2); - curr_dims[0] = 0; - curr_dims[1] = d; - curr_dims[2] = d + 1; - dinput1.project(curr_dims[0], curr_dims[1], curr_dims[2]); - doutput.project(curr_dims[0], curr_dims[1], curr_dims[2]); - calc_coeff_pointers(handle, curr_dims, l, dinput1, dcoarse, dcoeff_f, - dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, dcoeff_rc, - dcoeff_rcf); - - if (D - d == 1) { - unprocessed_idx += 1; - gpk_rev( - handle, handle.shapes_h[l], handle.shapes_d[l], - handle.shapes_d[l + 1], doutput.ldvs_d, dinput1.ldvs_d, - handle.unprocessed_n[unprocessed_idx], - handle.unprocessed_dims_d[unprocessed_idx], curr_dims[2], - curr_dims[1], curr_dims[0], handle.ratio[curr_dims[2]][l], - handle.ratio[curr_dims[1]][l], handle.ratio[curr_dims[0]][l], - doutput.dv, doutput.lddv1, doutput.lddv2, dcoarse.dv, dcoarse.lddv1, - dcoarse.lddv2, - // null, lddv1, lddv2, - dcoeff_f.dv, dcoeff_f.lddv1, dcoeff_f.lddv2, - // null, lddv1, lddv2, - dcoeff_c.dv, dcoeff_c.lddv1, dcoeff_c.lddv2, - // null, lddv1, lddv2, - dcoeff_r.dv, dcoeff_r.lddv1, dcoeff_r.lddv2, - // null, lddv1, lddv2, - dcoeff_cf.dv, dcoeff_cf.lddv1, dcoeff_cf.lddv2, - // null, lddv1, lddv2, - dcoeff_rf.dv, dcoeff_rf.lddv1, dcoeff_rf.lddv2, - // null, lddv1, lddv2, - dcoeff_rc.dv, dcoeff_rc.lddv1, dcoeff_rc.lddv2, - // null, lddv1, lddv2, - dcoeff_rcf.dv, dcoeff_rcf.lddv1, dcoeff_rcf.lddv2, - // null, lddv1, lddv2, - 0, 0, 0, handle.dofs[curr_dims[2]][l], handle.dofs[curr_dims[1]][l], - handle.dofs[curr_dims[0]][l], queue_idx, - handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - } else { // D - d >= 2 - unprocessed_idx += 2; - gpk_rev( - handle, handle.shapes_h[l], handle.shapes_d[l], - handle.shapes_d[l + 1], doutput.ldvs_d, dinput1.ldvs_d, - handle.unprocessed_n[unprocessed_idx], - handle.unprocessed_dims_d[unprocessed_idx], curr_dims[2], - curr_dims[1], curr_dims[0], handle.ratio[curr_dims[2]][l], - handle.ratio[curr_dims[1]][l], handle.ratio[curr_dims[0]][l], - doutput.dv, doutput.lddv1, doutput.lddv2, dcoarse.dv, dcoarse.lddv1, - dcoarse.lddv2, - // null, lddv1, lddv2, - dcoeff_f.dv, dcoeff_f.lddv1, dcoeff_f.lddv2, - // null, lddv1, lddv2, - dcoeff_c.dv, dcoeff_c.lddv1, dcoeff_c.lddv2, - // null, lddv1, lddv2, - dcoeff_r.dv, dcoeff_r.lddv1, dcoeff_r.lddv2, - // null, lddv1, lddv2, - dcoeff_cf.dv, dcoeff_cf.lddv1, dcoeff_cf.lddv2, - // null, lddv1, lddv2, - dcoeff_rf.dv, dcoeff_rf.lddv1, dcoeff_rf.lddv2, - // null, lddv1, lddv2, - dcoeff_rc.dv, dcoeff_rc.lddv1, dcoeff_rc.lddv2, - // null, lddv1, lddv2, - dcoeff_rcf.dv, dcoeff_rcf.lddv1, dcoeff_rcf.lddv2, - // null, lddv1, lddv2, - 0, 0, 0, handle.dofs[curr_dims[2]][l], handle.dofs[curr_dims[1]][l], - handle.dofs[curr_dims[0]][l], queue_idx, - handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - } - } - // Done interpolation-restore on doutput - - if (debug_print) { // debug - printf("After interpolation reverse-reorder\n"); - for (int k = 0; k < doutput.shape[4]; k++) { - for (int j = 0; j < doutput.shape[3]; j++) { - printf("i,j = %d,%d\n", k, j); - print_matrix_cuda( - doutput.shape[2], doutput.shape[1], doutput.shape[0], - doutput.dv + - k * doutput.ldvs_h[0] * doutput.ldvs_h[1] * doutput.ldvs_h[2] * - doutput.ldvs_h[3] + - j * doutput.ldvs_h[0] * doutput.ldvs_h[1] * doutput.ldvs_h[2], - doutput.ldvs_h[0], doutput.ldvs_h[1], doutput.ldvs_h[0]); - } - } - } // debug - - unprocessed_idx = 0; - - // printf("reorder-restore 1-3D\n"); - curr_dims[0] = 0; - curr_dims[1] = 1; - curr_dims[2] = 2; - dinput2.project(curr_dims[0], curr_dims[1], curr_dims[2]); - dinput1.project(curr_dims[0], curr_dims[1], - curr_dims[2]); // reuse input1 as temp space - calc_coeff_pointers(handle, curr_dims, l, dinput2, dcoarse, dcoeff_f, - dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, dcoeff_rc, - dcoeff_rcf); - - gpk_rev( - handle, handle.shapes_h[l], handle.shapes_d[l], handle.shapes_d[l + 1], - dinput1.ldvs_d, dinput2.ldvs_d, handle.unprocessed_n[unprocessed_idx], - handle.unprocessed_dims_d[unprocessed_idx], curr_dims[2], curr_dims[1], - curr_dims[0], handle.ratio[curr_dims[2]][l], - handle.ratio[curr_dims[1]][l], handle.ratio[curr_dims[0]][l], dinput1.dv, - dinput1.lddv1, dinput1.lddv2, dcoarse.dv, dcoarse.lddv1, dcoarse.lddv2, - // null, lddv1, lddv2, - dcoeff_f.dv, dcoeff_f.lddv1, dcoeff_f.lddv2, - // null, lddv1, lddv2, - dcoeff_c.dv, dcoeff_c.lddv1, dcoeff_c.lddv2, - // null, lddv1, lddv2, - dcoeff_r.dv, dcoeff_r.lddv1, dcoeff_r.lddv2, - // null, lddv1, lddv2, - dcoeff_cf.dv, dcoeff_cf.lddv1, dcoeff_cf.lddv2, - // null, lddv1, lddv2, - dcoeff_rf.dv, dcoeff_rf.lddv1, dcoeff_rf.lddv2, - // null, lddv1, lddv2, - dcoeff_rc.dv, dcoeff_rc.lddv1, dcoeff_rc.lddv2, - // null, lddv1, lddv2, - dcoeff_rcf.dv, dcoeff_rcf.lddv1, dcoeff_rcf.lddv2, - // null, lddv1, lddv2, - 0, 0, 0, handle.dofs[curr_dims[2]][l], handle.dofs[curr_dims[1]][l], - handle.dofs[curr_dims[0]][l], queue_idx, - handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - - DIM D_reduced = D % 2 == 0 ? D - 1 : D - 2; - for (DIM d = 3; d < D_reduced; d += 2) { - // printf("reorder-reverse\n"); - // copy back to input2 for reordering again - lwpk(handle, handle.shapes_h[l], handle.shapes_d[l], dinput1.dv, - dinput1.ldvs_d, dinput2.dv, dinput2.ldvs_d, queue_idx); - // printf("reorder-restore %u-%uD\n", d+1, d+2); - curr_dims[0] = 0; - curr_dims[1] = d; - curr_dims[2] = d + 1; - dinput2.project(curr_dims[0], curr_dims[1], curr_dims[2]); - dinput1.project(curr_dims[0], curr_dims[1], - curr_dims[2]); // reuse input1 as temp output - calc_coeff_pointers(handle, curr_dims, l, dinput2, dcoarse, dcoeff_f, - dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, dcoeff_rc, - dcoeff_rcf); - - unprocessed_idx += 2; - gpk_rev( - handle, handle.shapes_h[l], handle.shapes_d[l], handle.shapes_d[l + 1], - dinput1.ldvs_d, dinput2.ldvs_d, handle.unprocessed_n[unprocessed_idx], - handle.unprocessed_dims_d[unprocessed_idx], curr_dims[2], curr_dims[1], - curr_dims[0], handle.ratio[curr_dims[2]][l], - handle.ratio[curr_dims[1]][l], handle.ratio[curr_dims[0]][l], - dinput1.dv, dinput1.lddv1, dinput1.lddv2, dcoarse.dv, dcoarse.lddv1, - dcoarse.lddv2, - // null, lddv1, lddv2, - dcoeff_f.dv, dcoeff_f.lddv1, dcoeff_f.lddv2, - // null, lddv1, lddv2, - dcoeff_c.dv, dcoeff_c.lddv1, dcoeff_c.lddv2, - // null, lddv1, lddv2, - dcoeff_r.dv, dcoeff_r.lddv1, dcoeff_r.lddv2, - // null, lddv1, lddv2, - dcoeff_cf.dv, dcoeff_cf.lddv1, dcoeff_cf.lddv2, - // null, lddv1, lddv2, - dcoeff_rf.dv, dcoeff_rf.lddv1, dcoeff_rf.lddv2, - // null, lddv1, lddv2, - dcoeff_rc.dv, dcoeff_rc.lddv1, dcoeff_rc.lddv2, - // null, lddv1, lddv2, - dcoeff_rcf.dv, dcoeff_rcf.lddv1, dcoeff_rcf.lddv2, - // null, lddv1, lddv2, - 0, 0, 0, handle.dofs[curr_dims[2]][l], handle.dofs[curr_dims[1]][l], - handle.dofs[curr_dims[0]][l], queue_idx, - handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - } - - // printf("coeff-restore %u-%dD\n", D_reduced+1, D_reduced+2); - curr_dims[0] = 0; - curr_dims[1] = D_reduced; - curr_dims[2] = D_reduced + 1; - dinput1.project(curr_dims[0], curr_dims[1], curr_dims[2]); - doutput.project(curr_dims[0], curr_dims[1], curr_dims[2]); - calc_coeff_pointers(handle, curr_dims, l, dinput1, dcoarse, dcoeff_f, - dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, dcoeff_rc, - dcoeff_rcf); - - if (D - D_reduced == 1) { - // printf("coeff-restore %u-%dD\n", D_reduced+1, D_reduced+1); - unprocessed_idx += 1; - gpk_rev( - handle, handle.shapes_h[l], handle.shapes_d[l], handle.shapes_d[l + 1], - doutput.ldvs_d, dinput1.ldvs_d, handle.unprocessed_n[unprocessed_idx], - handle.unprocessed_dims_d[unprocessed_idx], curr_dims[2], curr_dims[1], - curr_dims[0], handle.ratio[curr_dims[2]][l], - handle.ratio[curr_dims[1]][l], handle.ratio[curr_dims[0]][l], - doutput.dv, doutput.lddv1, doutput.lddv2, dcoarse.dv, dcoarse.lddv1, - dcoarse.lddv2, - // null, lddv1, lddv2, - dcoeff_f.dv, dcoeff_f.lddv1, dcoeff_f.lddv2, - // null, lddv1, lddv2, - dcoeff_c.dv, dcoeff_c.lddv1, dcoeff_c.lddv2, - // null, lddv1, lddv2, - dcoeff_r.dv, dcoeff_r.lddv1, dcoeff_r.lddv2, - // null, lddv1, lddv2, - dcoeff_cf.dv, dcoeff_cf.lddv1, dcoeff_cf.lddv2, - // null, lddv1, lddv2, - dcoeff_rf.dv, dcoeff_rf.lddv1, dcoeff_rf.lddv2, - // null, lddv1, lddv2, - dcoeff_rc.dv, dcoeff_rc.lddv1, dcoeff_rc.lddv2, - // null, lddv1, lddv2, - dcoeff_rcf.dv, dcoeff_rcf.lddv1, dcoeff_rcf.lddv2, - // null, lddv1, lddv2, - 0, 0, 0, handle.dofs[curr_dims[2]][l], handle.dofs[curr_dims[1]][l], - handle.dofs[curr_dims[0]][l], queue_idx, - handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - } else { // D - D_reduced >= 2 - // printf("coeff-restore %u-%dD\n", D_reduced+1, D_reduced+2); - unprocessed_idx += 2; - gpk_rev( - handle, handle.shapes_h[l], handle.shapes_d[l], handle.shapes_d[l + 1], - doutput.ldvs_d, dinput1.ldvs_d, handle.unprocessed_n[unprocessed_idx], - handle.unprocessed_dims_d[unprocessed_idx], curr_dims[2], curr_dims[1], - curr_dims[0], handle.ratio[curr_dims[2]][l], - handle.ratio[curr_dims[1]][l], handle.ratio[curr_dims[0]][l], - doutput.dv, doutput.lddv1, doutput.lddv2, dcoarse.dv, dcoarse.lddv1, - dcoarse.lddv2, - // null, lddv1, lddv2, - dcoeff_f.dv, dcoeff_f.lddv1, dcoeff_f.lddv2, - // null, lddv1, lddv2, - dcoeff_c.dv, dcoeff_c.lddv1, dcoeff_c.lddv2, - // null, lddv1, lddv2, - dcoeff_r.dv, dcoeff_r.lddv1, dcoeff_r.lddv2, - // null, lddv1, lddv2, - dcoeff_cf.dv, dcoeff_cf.lddv1, dcoeff_cf.lddv2, - // null, lddv1, lddv2, - dcoeff_rf.dv, dcoeff_rf.lddv1, dcoeff_rf.lddv2, - // null, lddv1, lddv2, - dcoeff_rc.dv, dcoeff_rc.lddv1, dcoeff_rc.lddv2, - // null, lddv1, lddv2, - dcoeff_rcf.dv, dcoeff_rcf.lddv1, dcoeff_rcf.lddv2, - // null, lddv1, lddv2, - 0, 0, 0, handle.dofs[curr_dims[2]][l], handle.dofs[curr_dims[1]][l], - handle.dofs[curr_dims[0]][l], queue_idx, - handle.auto_tuning_cc[handle.arch][handle.precision][range_l]); - } - - if (debug_print) { // debug - printf("After coeff restore\n"); - for (int k = 0; k < doutput.shape[4]; k++) { - for (int j = 0; j < doutput.shape[3]; j++) { - printf("i,j = %d,%d\n", k, j); - print_matrix_cuda( - doutput.shape[2], doutput.shape[1], doutput.shape[0], - doutput.dv + - k * doutput.ldvs_h[0] * doutput.ldvs_h[1] * doutput.ldvs_h[2] * - doutput.ldvs_h[3] + - j * doutput.ldvs_h[0] * doutput.ldvs_h[1] * doutput.ldvs_h[2], - doutput.ldvs_h[0], doutput.ldvs_h[1], doutput.ldvs_h[0]); - } - } - } // debug -} - -template -void calc_correction_nd(Handle &handle, SubArray dcoeff, - SubArray &dcorrection, SIZE l, int queue_idx) { - int range_l = std::min(6, (int)std::log2(handle.dofs[0][l]) - 1); - int range_lp1 = std::min(6, (int)std::log2(handle.dofs[0][l + 1]) - 1); - - std::string prefix = "decomp_"; - if (sizeof(T) == sizeof(double)) - prefix += "d_"; - if (sizeof(T) == sizeof(float)) - prefix += "f_"; - for (int d = 0; d < D; d++) - prefix += std::to_string(handle.shapes_h[0][d]) + "_"; - - SubArray dw_in1 = dcoeff; - SubArray dw_in2 = dcoeff; - SubArray dw_out = dcorrection; - - // start correction calculation - int prev_dim_r, prev_dim_c, prev_dim_f; - int curr_dim_f = 0, curr_dim_c = 1, curr_dim_r = 2; - - dw_in1.resize(curr_dim_f, handle.dofs[curr_dim_f][l + 1]); - dw_in2.offset(curr_dim_f, handle.dofs[curr_dim_f][l + 1]); - dw_in2.resize(curr_dim_f, - handle.dofs[curr_dim_f][l] - handle.dofs[curr_dim_f][l + 1]); - dw_out.resize(curr_dim_f, handle.dofs[curr_dim_f][l + 1]); - - dw_in1.project(curr_dim_f, curr_dim_c, curr_dim_r); - dw_in2.project(curr_dim_f, curr_dim_c, curr_dim_r); - dw_out.project(curr_dim_f, curr_dim_c, curr_dim_r); - - // printf("mass trans 1D\n"); - lpk_reo_1( - handle, handle.shapes_h[l], handle.shapes_h[l + 1], handle.shapes_d[l], - handle.shapes_d[l + 1], dw_in1.ldvs_d, dw_out.ldvs_d, - handle.processed_n[0], handle.processed_dims_h[0], - handle.processed_dims_d[0], curr_dim_r, curr_dim_c, curr_dim_f, - handle.dist[curr_dim_f][l], handle.ratio[curr_dim_f][l], dw_in1.dv, - dw_in1.lddv1, dw_in1.lddv2, dw_in2.dv, dw_in2.lddv1, dw_in2.lddv2, - dw_out.dv, dw_out.lddv1, dw_out.lddv2, queue_idx, - handle.auto_tuning_mr1[handle.arch][handle.precision][range_lp1]); - - if (debug_print) { // debug - printf("decomposition: after MR-1D[%d]\n", l); - for (int i = 0; i < dw_out.shape[3]; i++) { - printf("i = %d\n", i); - print_matrix_cuda(dw_out.shape[2], dw_out.shape[1], dw_out.shape[0], - dw_out.dv + i * dw_out.ldvs_h[0] * dw_out.ldvs_h[1] * - dw_out.ldvs_h[2], - dw_out.ldvs_h[0], dw_out.ldvs_h[1], dw_out.ldvs_h[0]); - } - } - - // mass trans 2D - prev_dim_f = curr_dim_f; - prev_dim_c = curr_dim_c; - prev_dim_r = curr_dim_r; - curr_dim_f = 0, curr_dim_c = 1, curr_dim_r = 2; - - dw_in1 = dw_out; - dw_in2 = dw_out; - dw_in1.resize(curr_dim_c, handle.dofs[curr_dim_c][l + 1]); - dw_in2.offset(curr_dim_c, handle.dofs[curr_dim_c][l + 1]); - dw_in2.resize(curr_dim_c, - handle.dofs[curr_dim_c][l] - handle.dofs[curr_dim_c][l + 1]); - dw_out.offset(prev_dim_f, handle.dofs[curr_dim_f][l + 1]); - dw_out.resize(curr_dim_c, handle.dofs[curr_dim_c][l + 1]); - - dw_in1.project(curr_dim_f, curr_dim_c, curr_dim_r); - dw_in2.project(curr_dim_f, curr_dim_c, curr_dim_r); - dw_out.project(curr_dim_f, curr_dim_c, curr_dim_r); - - // printf("mass trans 2D\n"); - lpk_reo_2( - handle, handle.shapes_h[l], handle.shapes_h[l + 1], handle.shapes_d[l], - handle.shapes_d[l + 1], dw_in1.ldvs_d, dw_out.ldvs_d, - handle.processed_n[1], handle.processed_dims_h[1], - handle.processed_dims_d[1], curr_dim_r, curr_dim_c, curr_dim_f, - handle.dist[curr_dim_c][l], handle.ratio[curr_dim_c][l], dw_in1.dv, - dw_in1.lddv1, dw_in1.lddv2, dw_in2.dv, dw_in2.lddv1, dw_in2.lddv2, - dw_out.dv, dw_out.lddv1, dw_out.lddv2, queue_idx, - handle.auto_tuning_mr1[handle.arch][handle.precision][range_lp1]); - - if (debug_print) { // debug - printf("decomposition: after MR-2D[%d]\n", l); - for (int i = 0; i < dw_out.shape[3]; i++) { - printf("i = %d\n", i); - print_matrix_cuda(dw_out.shape[2], dw_out.shape[1], dw_out.shape[0], - dw_out.dv + i * dw_out.ldvs_h[0] * dw_out.ldvs_h[1] * - dw_out.ldvs_h[2], - dw_out.ldvs_h[0], dw_out.ldvs_h[1], dw_out.ldvs_h[0]); - } - } - - // mass trans 3D - - prev_dim_f = curr_dim_f; - prev_dim_c = curr_dim_c; - prev_dim_r = curr_dim_r; - curr_dim_f = 0, curr_dim_c = 1, curr_dim_r = 2; - - dw_in1 = dw_out; - dw_in2 = dw_out; - dw_in1.resize(curr_dim_r, handle.dofs[curr_dim_r][l + 1]); - dw_in2.offset(curr_dim_r, handle.dofs[curr_dim_r][l + 1]); - dw_in2.resize(curr_dim_r, - handle.dofs[curr_dim_r][l] - handle.dofs[curr_dim_r][l + 1]); - dw_out.offset(prev_dim_c, handle.dofs[curr_dim_c][l + 1]); - dw_out.resize(curr_dim_r, handle.dofs[curr_dim_r][l + 1]); - - dw_in1.project(curr_dim_f, curr_dim_c, curr_dim_r); - dw_in2.project(curr_dim_f, curr_dim_c, curr_dim_r); - dw_out.project(curr_dim_f, curr_dim_c, curr_dim_r); - - // printf("mass trans 3D\n"); - lpk_reo_3( - handle, handle.shapes_h[l], handle.shapes_h[l + 1], handle.shapes_d[l], - handle.shapes_d[l + 1], dw_in1.ldvs_d, dw_out.ldvs_d, - handle.processed_n[2], handle.processed_dims_h[2], - handle.processed_dims_d[2], curr_dim_r, curr_dim_c, curr_dim_f, - handle.dist[curr_dim_r][l], handle.ratio[curr_dim_r][l], dw_in1.dv, - dw_in1.lddv1, dw_in1.lddv2, dw_in2.dv, dw_in2.lddv1, dw_in2.lddv2, - dw_out.dv, dw_out.lddv1, dw_out.lddv2, queue_idx, - handle.auto_tuning_mr1[handle.arch][handle.precision][range_lp1]); - - if (debug_print) { // debug - printf("decomposition: after MR-3D[%d]\n", l); - for (int i = 0; i < dw_out.shape[3]; i++) { - printf("i = %d\n", i); - print_matrix_cuda(dw_out.shape[2], dw_out.shape[1], dw_out.shape[0], - dw_out.dv + i * dw_out.ldvs_h[0] * dw_out.ldvs_h[1] * - dw_out.ldvs_h[2], - dw_out.ldvs_h[0], dw_out.ldvs_h[1], dw_out.ldvs_h[0]); - } - } - - // mass trans 4D+ - for (int i = 3; i < D; i++) { - prev_dim_f = curr_dim_f; - prev_dim_c = curr_dim_c; - prev_dim_r = curr_dim_r; - curr_dim_f = 0, curr_dim_c = 1, curr_dim_r = i; - dw_in1 = dw_out; - dw_in2 = dw_out; - dw_in1.resize(curr_dim_r, handle.dofs[curr_dim_r][l + 1]); - dw_in2.offset(curr_dim_r, handle.dofs[curr_dim_r][l + 1]); - dw_in2.resize(curr_dim_r, - handle.dofs[curr_dim_r][l] - handle.dofs[curr_dim_r][l + 1]); - dw_out.offset(prev_dim_r, handle.dofs[prev_dim_r][l + 1]); - dw_out.resize(curr_dim_r, handle.dofs[curr_dim_r][l + 1]); - - dw_in1.project(curr_dim_f, curr_dim_c, curr_dim_r); - dw_in2.project(curr_dim_f, curr_dim_c, curr_dim_r); - dw_out.project(curr_dim_f, curr_dim_c, curr_dim_r); - - // printf("mass trans %dD\n", i+1); - lpk_reo_3( - handle, handle.shapes_h[l], handle.shapes_h[l + 1], handle.shapes_d[l], - handle.shapes_d[l + 1], dw_in1.ldvs_d, dw_out.ldvs_d, - handle.processed_n[i], handle.processed_dims_h[i], - handle.processed_dims_d[i], curr_dim_r, curr_dim_c, curr_dim_f, - handle.dist[curr_dim_r][l], handle.ratio[curr_dim_r][l], dw_in1.dv, - dw_in1.lddv1, dw_in1.lddv2, dw_in2.dv, dw_in2.lddv1, dw_in2.lddv2, - dw_out.dv, dw_out.lddv1, dw_out.lddv2, queue_idx, - handle.auto_tuning_mr1[handle.arch][handle.precision][range_lp1]); - - if (debug_print) { // debug - printf("decomposition: after MR-%dD[%d]\n", i + 1, l); - for (int k = 0; k < dw_out.shape[4]; k++) { - for (int j = 0; j < dw_out.shape[3]; j++) { - printf("i,j = %d,%d\n", k, j); - print_matrix_cuda( - dw_out.shape[2], dw_out.shape[1], dw_out.shape[0], - dw_out.dv + - k * dw_out.ldvs_h[0] * dw_out.ldvs_h[1] * dw_out.ldvs_h[2] * - dw_out.ldvs_h[3] + - j * dw_out.ldvs_h[0] * dw_out.ldvs_h[1] * dw_out.ldvs_h[2], - dw_out.ldvs_h[0], dw_out.ldvs_h[1], dw_out.ldvs_h[0]); - } - } - } - } - - curr_dim_f = 0, curr_dim_c = 1, curr_dim_r = 2; - dw_in1.project(curr_dim_f, curr_dim_c, curr_dim_r); - dw_in2.project(curr_dim_f, curr_dim_c, curr_dim_r); - dw_out.project(curr_dim_f, curr_dim_c, curr_dim_r); - - // printf("solve tridiag 1D\n"); - ipk_1(handle, handle.shapes_h[l], handle.shapes_h[l + 1], - handle.shapes_d[l], handle.shapes_d[l + 1], dw_out.ldvs_d, - dw_out.ldvs_d, handle.processed_n[0], handle.processed_dims_h[0], - handle.processed_dims_d[0], curr_dim_r, curr_dim_c, curr_dim_f, - handle.am[curr_dim_f][l + 1], handle.bm[curr_dim_f][l + 1], - handle.dist[curr_dim_f][l + 1], dw_out.dv, dw_out.lddv1, - dw_out.lddv2, queue_idx, - handle.auto_tuning_ts1[handle.arch][handle.precision][range_lp1]); - - if (debug_print) { // debug - printf("decomposition: after TR-1D[%d]\n", l); - for (int k = 0; k < dw_out.shape[4]; k++) { - for (int j = 0; j < dw_out.shape[3]; j++) { - printf("i,j = %d,%d\n", k, j); - print_matrix_cuda(dw_out.shape[2], dw_out.shape[1], dw_out.shape[0], - dw_out.dv + - k * dw_out.ldvs_h[0] * dw_out.ldvs_h[1] * - dw_out.ldvs_h[2] * dw_out.ldvs_h[3] + - j * dw_out.ldvs_h[0] * dw_out.ldvs_h[1] * - dw_out.ldvs_h[2], - dw_out.ldvs_h[0], dw_out.ldvs_h[1], dw_out.ldvs_h[0]); - } - } - } // debug - - curr_dim_f = 0, curr_dim_c = 1, curr_dim_r = 2; - dw_in1.project(curr_dim_f, curr_dim_c, curr_dim_r); - dw_in2.project(curr_dim_f, curr_dim_c, curr_dim_r); - dw_out.project(curr_dim_f, curr_dim_c, curr_dim_r); - - // printf("solve tridiag 2D\n"); - ipk_2(handle, handle.shapes_h[l], handle.shapes_h[l + 1], - handle.shapes_d[l], handle.shapes_d[l + 1], dw_out.ldvs_d, - dw_out.ldvs_d, handle.processed_n[1], handle.processed_dims_h[1], - handle.processed_dims_d[1], curr_dim_r, curr_dim_c, curr_dim_f, - handle.am[curr_dim_c][l + 1], handle.bm[curr_dim_c][l + 1], - handle.dist[curr_dim_c][l + 1], dw_out.dv, dw_out.lddv1, - dw_out.lddv2, queue_idx, - handle.auto_tuning_ts1[handle.arch][handle.precision][range_lp1]); - - if (debug_print) { // debug - printf("decomposition: after TR-2D[%d]\n", l); - for (int k = 0; k < dw_out.shape[4]; k++) { - for (int j = 0; j < dw_out.shape[3]; j++) { - printf("i,j = %d,%d\n", k, j); - print_matrix_cuda(dw_out.shape[2], dw_out.shape[1], dw_out.shape[0], - dw_out.dv + - k * dw_out.ldvs_h[0] * dw_out.ldvs_h[1] * - dw_out.ldvs_h[2] * dw_out.ldvs_h[3] + - j * dw_out.ldvs_h[0] * dw_out.ldvs_h[1] * - dw_out.ldvs_h[2], - dw_out.ldvs_h[0], dw_out.ldvs_h[1], dw_out.ldvs_h[0]); - } - } - } // debug - - curr_dim_f = 0, curr_dim_c = 1, curr_dim_r = 2; - dw_in1.project(curr_dim_f, curr_dim_c, curr_dim_r); - dw_in2.project(curr_dim_f, curr_dim_c, curr_dim_r); - dw_out.project(curr_dim_f, curr_dim_c, curr_dim_r); - - // printf("solve tridiag 3D\n"); - ipk_3(handle, handle.shapes_h[l], handle.shapes_h[l + 1], - handle.shapes_d[l], handle.shapes_d[l + 1], dw_out.ldvs_d, - dw_out.ldvs_d, handle.processed_n[2], handle.processed_dims_h[2], - handle.processed_dims_d[2], curr_dim_r, curr_dim_c, curr_dim_f, - handle.am[curr_dim_r][l + 1], handle.bm[curr_dim_r][l + 1], - handle.dist[curr_dim_r][l + 1], dw_out.dv, dw_out.lddv1, - dw_out.lddv2, queue_idx, - handle.auto_tuning_ts1[handle.arch][handle.precision][range_lp1]); - - if (debug_print) { // debug - printf("decomposition: after TR-3D[%d]\n", l); - for (int k = 0; k < dw_out.shape[4]; k++) { - for (int j = 0; j < dw_out.shape[3]; j++) { - printf("i,j = %d,%d\n", k, j); - print_matrix_cuda(dw_out.shape[2], dw_out.shape[1], dw_out.shape[0], - dw_out.dv + - k * dw_out.ldvs_h[0] * dw_out.ldvs_h[1] * - dw_out.ldvs_h[2] * dw_out.ldvs_h[3] + - j * dw_out.ldvs_h[0] * dw_out.ldvs_h[1] * - dw_out.ldvs_h[2], - dw_out.ldvs_h[0], dw_out.ldvs_h[1], dw_out.ldvs_h[0]); - } - } - } // debug - - // mass trans 4D+ - for (int i = 3; i < D; i++) { - curr_dim_f = 0, curr_dim_c = 1, curr_dim_r = i; - dw_in1.project(curr_dim_f, curr_dim_c, curr_dim_r); - dw_in2.project(curr_dim_f, curr_dim_c, curr_dim_r); - dw_out.project(curr_dim_f, curr_dim_c, curr_dim_r); - // printf("solve tridiag %dD\n", i+1); - ipk_3( - handle, handle.shapes_h[l], handle.shapes_h[l + 1], handle.shapes_d[l], - handle.shapes_d[l + 1], dw_out.ldvs_d, dw_out.ldvs_d, - handle.processed_n[i], handle.processed_dims_h[i], - handle.processed_dims_d[i], curr_dim_r, curr_dim_c, curr_dim_f, - handle.am[curr_dim_r][l + 1], handle.bm[curr_dim_r][l + 1], - handle.dist[curr_dim_r][l + 1], dw_out.dv, dw_out.lddv1, dw_out.lddv2, - queue_idx, - handle.auto_tuning_ts1[handle.arch][handle.precision][range_lp1]); - if (debug_print) { // debug - printf("decomposition: after TR-%dD[%d]\n", i + 1, l); - for (int k = 0; k < dw_out.shape[4]; k++) { - for (int j = 0; j < dw_out.shape[3]; j++) { - printf("i,j = %d,%d\n", k, j); - print_matrix_cuda( - dw_out.shape[2], dw_out.shape[1], dw_out.shape[0], - dw_out.dv + - k * dw_out.ldvs_h[0] * dw_out.ldvs_h[1] * dw_out.ldvs_h[2] * - dw_out.ldvs_h[3] + - j * dw_out.ldvs_h[0] * dw_out.ldvs_h[1] * dw_out.ldvs_h[2], - dw_out.ldvs_h[0], dw_out.ldvs_h[1], dw_out.ldvs_h[0]); - } - } - } // debug - } - - dcorrection = dw_out; - - // { // debug - // printf("decomposition: after TR[%d]\n", l); - // for (int k = 0; k < dw_out.shape[4]; k++) { - // for (int j = 0; j < dw_out.shape[3]; j++) { - // printf("i,j = %d,%d\n", k,j); - // print_matrix_cuda(dw_out.shape[2], dw_out.shape[1], - // dw_out.shape[0], - // dw_out.dv+k*dw_out.ldvs_h[0]*dw_out.ldvs_h[1]*dw_out.ldvs_h[2]*dw_out.ldvs_h[3]+j*dw_out.ldvs_h[0]*dw_out.ldvs_h[1]*dw_out.ldvs_h[2], - // dw_out.ldvs_h[0], dw_out.ldvs_h[1], - // dw_out.ldvs_h[0]); - // } - // } - // } //debug -} - -template -void decompose(Handle &handle, T *dv, std::vector ldvs_h, - SIZE *ldvs_d, SIZE l_target, int queue_idx) { - - std::string prefix = "decomp_"; - if (sizeof(T) == sizeof(double)) - prefix += "d_"; - if (sizeof(T) == sizeof(float)) - prefix += "f_"; - for (int d = 0; d < D; d++) - prefix += std::to_string(handle.shapes_h[0][d]) + "_"; - // std::cout << prefix << std::endl; - - if (D <= 3) { - for (int l = 0; l < l_target; ++l) { - // printf("[gpu] l = %d\n", l); - int stride = std::pow(2, l); - int Cstride = stride * 2; - int range_l = std::min(6, (int)std::log2(handle.dofs[0][l]) - 1); - int range_lp1 = std::min(6, (int)std::log2(handle.dofs[0][l + 1]) - 1); - - // for calculate corrections - T *dw_out = NULL; - T *dw_in1 = NULL; - T *dw_in2 = NULL; - - // printf("range_l: %d, range_lp1: %d\n", range_l, range_lp1); - - if (debug_print) { - printf("input v\n"); - print_matrix_cuda(handle.dofs[2][l], handle.dofs[1][l], - handle.dofs[0][l], dv, ldvs_h[0], ldvs_h[1], - ldvs_h[0]); - } - - // verify_matrix_cuda(handle.dofs[2][l], handle.dofs[1][l], - // handle.dofs[0][l], dv, ldvs_h[0], ldvs_h[1], - // ldvs_h[0], prefix + "begin" + "_level_" + - // std::to_string(l), store, verify); - lwpk(handle, handle.shapes_h[l], handle.shapes_d[l], dv, - ldvs_d, handle.dw, handle.ldws_d, queue_idx); - - SubArray dinput( - {handle.dofs[0][l], handle.dofs[1][l], handle.dofs[2][l]}, handle.dw, - handle.ldws_h, handle.ldws_d); - SubArray doutput( - {handle.dofs[0][l], handle.dofs[1][l], handle.dofs[2][l]}, dv, ldvs_h, - ldvs_d); - - calc_coefficients_3d(handle, dinput, doutput, l, 0); - - SubArray dcoeff( - {handle.dofs[0][l], handle.dofs[1][l], handle.dofs[2][l]}, dv, ldvs_h, - ldvs_d); - SubArray dcorrection( - {handle.dofs[0][l] + 1, handle.dofs[1][l] + 1, handle.dofs[2][l] + 1}, - handle.dw, handle.ldws_h, handle.ldws_d); - - calc_correction_3d(handle, dcoeff, dcorrection, l, 0); - - lwpk(handle, handle.shapes_h[l + 1], handle.shapes_d[l + 1], - dcorrection.dv, dcorrection.ldvs_d, dv, ldvs_d, - queue_idx); - - if (debug_print) { - printf("after add\n"); - print_matrix_cuda(handle.dofs[2][l], handle.dofs[1][l], - handle.dofs[0][l], dv, ldvs_h[0], ldvs_h[1], - ldvs_h[0]); - } - - } // end of loop - - if (debug_print) { - printf("output of decomposition\n"); - print_matrix_cuda(handle.dofs[2][0], handle.dofs[1][0], handle.dofs[0][0], - dv, ldvs_h[0], ldvs_h[1], ldvs_h[0]); - } - } - - if (D > 3) { - - for (int l = 0; l < l_target; ++l) { - // printf("[gpu] l = %d\n", l); - int stride = std::pow(2, l); - int Cstride = stride * 2; - int range_l = std::min(6, (int)std::log2(handle.dofs[0][l]) - 1); - int range_lp1 = std::min(6, (int)std::log2(handle.dofs[0][l + 1]) - 1); - bool f_padding = handle.dofs[0][l] % 2 == 0; - bool c_padding = handle.dofs[1][l] % 2 == 0; - bool r_padding = handle.dofs[2][l] % 2 == 0; - - DIM curr_dim_r, curr_dim_c, curr_dim_f; - LENGTH lddv1, lddv2; - LENGTH lddw1, lddw2; - LENGTH lddb1, lddb2; - - int unprocessed_idx = 0; - - if (debug_print) { // debug - printf("decomposition: before coeff\n"); - for (int i = 0; i < handle.dofs[3][0]; i++) { - printf("i = %d\n", i); - print_matrix_cuda(handle.dofs[2][0], handle.dofs[1][0], - handle.dofs[0][0], - dv + i * ldvs_h[0] * ldvs_h[1] * ldvs_h[2], - ldvs_h[0], ldvs_h[1], ldvs_h[0]); - } - } - - lwpk(handle, handle.shapes_h[l], handle.shapes_d[l], dv, - ldvs_d, handle.dw, handle.ldws_d, queue_idx); - lwpk(handle, handle.shapes_h[l], handle.shapes_d[l], dv, - ldvs_d, handle.db, handle.ldbs_d, queue_idx); - - std::vector shape(handle.D_padded); - for (DIM d = 0; d < handle.D_padded; d++) - shape[d] = handle.shapes_h[l][d]; - - SubArray dinput1(shape, handle.dw, handle.ldws_h, handle.ldws_d); - SubArray dinput2(shape, handle.db, handle.ldbs_h, handle.ldbs_d); - SubArray doutput(shape, dv, ldvs_h, ldvs_d); - - calc_coefficients_nd(handle, dinput1, dinput2, doutput, l, queue_idx); - - // printf ("cjy3113\n"); - - if (debug_print) { // debug - printf("decomposition: after coeff[%d]\n", l); - for (int k = 0; k < doutput.shape[4]; k++) { - for (int j = 0; j < doutput.shape[3]; j++) { - printf("i,j = %d,%d\n", k, j); - print_matrix_cuda( - doutput.shape[2], doutput.shape[1], doutput.shape[0], - doutput.dv + - k * doutput.ldvs_h[0] * doutput.ldvs_h[1] * - doutput.ldvs_h[2] * doutput.ldvs_h[3] + - j * doutput.ldvs_h[0] * doutput.ldvs_h[1] * - doutput.ldvs_h[2], - doutput.ldvs_h[0], doutput.ldvs_h[1], doutput.ldvs_h[0]); - } - } - } // debug - - SubArray dcoeff(shape, dv, ldvs_h, ldvs_d); - SubArray dcorrection(shape, handle.dw, handle.ldws_h, - handle.ldws_d); - - calc_correction_nd(handle, dcoeff, dcorrection, l, 0); - - lwpk(handle, handle.shapes_h[l + 1], handle.shapes_d[l + 1], - dcorrection.dv, dcorrection.ldvs_d, dv, ldvs_d, - queue_idx); - if (debug_print) { // debug - printf("decomposition: after apply correction[%d]\n", l); - for (int k = 0; k < doutput.shape[4]; k++) { - for (int j = 0; j < doutput.shape[3]; j++) { - printf("i,j = %d,%d\n", k, j); - print_matrix_cuda( - doutput.shape[2], doutput.shape[1], doutput.shape[0], - doutput.dv + - k * doutput.ldvs_h[0] * doutput.ldvs_h[1] * - doutput.ldvs_h[2] * doutput.ldvs_h[3] + - j * doutput.ldvs_h[0] * doutput.ldvs_h[1] * - doutput.ldvs_h[2], - doutput.ldvs_h[0], doutput.ldvs_h[1], doutput.ldvs_h[0]); - } - } - } // debug - } - - // { // debug - // lwpk(handle, handle.shapes_h[0], handle.shapes_d[0], dv, - // ldvs_d, handle.db, handle.ldbs_d, queue_idx); - // std::vector shape(D); - // for (DIM d = 0; d < D; d++) shape[d] = handle.shapes_h[0][d]; - // SubArray dcoeff(shape, handle.db, handle.ldbs_h, - // handle.ldbs_d); SubArray doutput(shape, handle.dw, - // handle.ldws_h, handle.ldws_d); ReverseReorderGPU(handle, dcoeff, - // doutput, 0); - - // printf("decomposition: after applying correction\n"); - // for (int i = 0; i < handle.dofs[3][0]; i++) { - // printf("i = %d\n", i); - // print_matrix_cuda(handle.dofs[2][0], handle.dofs[1][0], - // handle.dofs[0][0], - // dv+i*ldvs_h[0]*ldvs_h[1]*ldvs_h[2], ldvs_h[0], - // ldvs_h[1], ldvs_h[0]); - // } - - // printf("after coeff reverse\n"); - // for (int i = 0; i < handle.dofs[3][0]; i++) { - // printf("i = %d\n", i); - // print_matrix_cuda(handle.dofs[2][0], handle.dofs[1][0], - // handle.dofs[0][0], - // doutput.dv+i*doutput.ldvs_h[0]*doutput.ldvs_h[1]*doutput.ldvs_h[2], - // doutput.ldvs_h[0], doutput.ldvs_h[1], - // doutput.ldvs_h[0]); - // } - // } - } -} - -template -void recompose(Handle &handle, T *dv, std::vector ldvs_h, - SIZE *ldvs_d, SIZE l_target, int queue_idx) { - - if (D <= 3) { - - if (debug_print) { - printf("input of recomposition\n"); - print_matrix_cuda(handle.dofs[2][0], handle.dofs[1][0], handle.dofs[0][0], - dv, ldvs_h[0], ldvs_h[1], ldvs_h[0]); - } - - std::string prefix = "recomp_"; - if (sizeof(T) == sizeof(double)) - prefix += "d_"; - if (sizeof(T) == sizeof(float)) - prefix += "f_"; - for (int d = 0; d < D; d++) - prefix += std::to_string(handle.shapes_h[0][d]) + "_"; - // std::cout << prefix << std::endl; - - for (int l = l_target - 1; l >= 0; l--) { - // printf("[gpu] l = %d\n", l); - int range_l = std::min(6, (int)std::log2(handle.dofs[0][l]) - 1); - int range_lp1 = std::min(6, (int)std::log2(handle.dofs[0][l + 1]) - 1); - - bool f_padding = handle.dofs[0][l] % 2 == 0; - bool c_padding = handle.dofs[1][l] % 2 == 0; - bool r_padding = handle.dofs[0][l] % 2 == 0; - - // printf("input v\n"); - // print_matrix_cuda(handle.dofs[2][l], handle.dofs[1][l], - // handle.dofs[0][l], - // dv, ldvs_h[0], ldvs_h[1], ldvs_h[0]); - - SubArray dcoeff( - {handle.dofs[0][l], handle.dofs[1][l], handle.dofs[2][l]}, dv, ldvs_h, - ldvs_d); - SubArray dcorrection( - {handle.dofs[0][l] + 1, handle.dofs[1][l] + 1, handle.dofs[2][l] + 1}, - handle.dw, handle.ldws_h, handle.ldws_d); - - calc_correction_3d(handle, dcoeff, dcorrection, l, 0); - - lwpk(handle, handle.shapes_h[l + 1], - handle.shapes_d[l + 1], dcorrection.dv, - dcorrection.ldvs_d, dv, ldvs_d, queue_idx); - - SubArray dinput( - {handle.dofs[0][l], handle.dofs[1][l], handle.dofs[2][l]}, dv, ldvs_h, - ldvs_d); - - SubArray doutput( - {handle.dofs[0][l], handle.dofs[1][l], handle.dofs[2][l]}, handle.dw, - handle.ldws_h, handle.ldws_d); - - coefficients_restore_3d(handle, dinput, doutput, l, 0); - - lwpk(handle, handle.shapes_h[l], handle.shapes_d[l], - handle.dw, handle.ldws_d, dv, ldvs_d, queue_idx); - - if (debug_print) { - printf("output of recomposition:\n"); - print_matrix_cuda(handle.dofs[2][l], handle.dofs[1][l], - handle.dofs[0][l], dv, ldvs_h[0], ldvs_h[1], - ldvs_h[0]); - } - } - } - if (D > 3) { - for (int l = l_target - 1; l >= 0; l--) { - // printf("[gpu] l = %d\n", l); - int range_l = std::min(6, (int)std::log2(handle.dofs[0][l]) - 1); - int range_lp1 = std::min(6, (int)std::log2(handle.dofs[0][l + 1]) - 1); - bool f_padding = handle.dofs[0][l] % 2 == 0; - bool c_padding = handle.dofs[1][l] % 2 == 0; - bool r_padding = handle.dofs[0][l] % 2 == 0; - - if (debug_print) { // debug - printf("recomposition: before corection\n"); - for (int i = 0; i < handle.dofs[3][0]; i++) { - printf("i = %d\n", i); - print_matrix_cuda(handle.dofs[2][0], handle.dofs[1][0], - handle.dofs[0][0], - dv + i * ldvs_h[0] * ldvs_h[1] * ldvs_h[2], - ldvs_h[0], ldvs_h[1], ldvs_h[0]); - } - } - - int curr_dim_r, curr_dim_c, curr_dim_f; - int lddv1, lddv2; - int lddw1, lddw2; - int lddb1, lddb2; - // un-apply correction - std::vector shape(handle.D_padded); - for (DIM d = 0; d < handle.D_padded; d++) - shape[d] = handle.shapes_h[l][d]; - - SubArray dcoeff(shape, dv, ldvs_h, ldvs_d); - SubArray dcorrection(shape, handle.dw, handle.ldws_h, - handle.ldws_d); - - if (debug_print) { // debug - printf("before subtract correction [%d]\n", l); - for (int k = 0; k < dcoeff.shape[4]; k++) { - for (int j = 0; j < dcoeff.shape[3]; j++) { - printf("i,j = %d,%d\n", k, j); - print_matrix_cuda( - dcoeff.shape[2], dcoeff.shape[1], dcoeff.shape[0], - dcoeff.dv + - k * dcoeff.ldvs_h[0] * dcoeff.ldvs_h[1] * dcoeff.ldvs_h[2] * - dcoeff.ldvs_h[3] + - j * dcoeff.ldvs_h[0] * dcoeff.ldvs_h[1] * dcoeff.ldvs_h[2], - dcoeff.ldvs_h[0], dcoeff.ldvs_h[1], dcoeff.ldvs_h[0]); - } - } - } // deb - - calc_correction_nd(handle, dcoeff, dcorrection, l, 0); - - lwpk(handle, handle.shapes_h[l + 1], - handle.shapes_d[l + 1], dcorrection.dv, - dcorrection.ldvs_d, dv, ldvs_d, queue_idx); - - if (debug_print) { // debug - printf("after subtract correction [%d]\n", l); - for (int k = 0; k < dcoeff.shape[4]; k++) { - for (int j = 0; j < dcoeff.shape[3]; j++) { - printf("i,j = %d,%d\n", k, j); - print_matrix_cuda( - dcoeff.shape[2], dcoeff.shape[1], dcoeff.shape[0], - dcoeff.dv + - k * dcoeff.ldvs_h[0] * dcoeff.ldvs_h[1] * dcoeff.ldvs_h[2] * - dcoeff.ldvs_h[3] + - j * dcoeff.ldvs_h[0] * dcoeff.ldvs_h[1] * dcoeff.ldvs_h[2], - dcoeff.ldvs_h[0], dcoeff.ldvs_h[1], dcoeff.ldvs_h[0]); - } - } - } // deb - - lwpk(handle, handle.shapes_h[l], handle.shapes_d[l], dv, - ldvs_d, handle.db, handle.ldbs_d, queue_idx); - lwpk(handle, handle.shapes_h[l], handle.shapes_d[l], dv, - ldvs_d, handle.dw, handle.ldws_d, queue_idx); - - SubArray dinput1(shape, handle.dw, handle.ldws_h, handle.ldws_d); - SubArray dinput2(shape, handle.db, handle.ldbs_h, handle.ldbs_d); - SubArray doutput(shape, dv, ldvs_h, ldvs_d); - - coefficients_restore_nd(handle, dinput1, dinput2, doutput, l, queue_idx); - - } // loop levels - - if (debug_print) { // debug - std::vector shape(handle.D_padded); - for (DIM d = 0; d < handle.D_padded; d++) - shape[d] = handle.shapes_h[0][d]; - SubArray dcoeff(shape, dv, ldvs_h, ldvs_d); - printf("final output\n"); - for (int k = 0; k < dcoeff.shape[4]; k++) { - for (int j = 0; j < dcoeff.shape[3]; j++) { - printf("i,j = %d,%d\n", k, j); - print_matrix_cuda( - dcoeff.shape[2], dcoeff.shape[1], dcoeff.shape[0], - dcoeff.dv + - k * dcoeff.ldvs_h[0] * dcoeff.ldvs_h[1] * dcoeff.ldvs_h[2] * - dcoeff.ldvs_h[3] + - j * dcoeff.ldvs_h[0] * dcoeff.ldvs_h[1] * dcoeff.ldvs_h[2], - dcoeff.ldvs_h[0], dcoeff.ldvs_h[1], dcoeff.ldvs_h[0]); - } - } - } // deb - - // { // debug - // lwpk(handle, handle.shapes_h[0], handle.shapes_d[0], dv, - // ldvs_d, handle.db, handle.ldbs_d, queue_idx); - // std::vector shape(D); - // for (DIM d = 0; d < D; d++) shape[d] = handle.shapes_h[0][d]; - // SubArray dcoeff(shape, handle.db, handle.ldbs_h, - // handle.ldbs_d); SubArray doutput(shape, handle.dw, - // handle.ldws_h, handle.ldws_d); ReverseReorderGPU(handle, dcoeff, - // doutput, 0); - - // printf("recomposition: done\n"); - // for (int i = 0; i < handle.dofs[3][0]; i++) { - // printf("i = %d\n", i); - // print_matrix_cuda(handle.dofs[2][0], handle.dofs[1][0], - // handle.dofs[0][0], - // dv+i*ldvs_h[0]*ldvs_h[1]*ldvs_h[2], ldvs_h[0], - // ldvs_h[1], ldvs_h[0]); - // } - - // // printf("after coeff reverse\n"); - // // for (int i = 0; i < handle.dofs[3][0]; i++) { - // // printf("i = %d\n", i); - // // print_matrix_cuda(handle.dofs[2][0], handle.dofs[1][0], - // // handle.dofs[0][0], - // // - // doutput.dv+i*doutput.ldvs_h[0]*doutput.ldvs_h[1]*doutput.ldvs_h[2], - // doutput.ldvs_h[0], doutput.ldvs_h[1], - // // doutput.ldvs_h[0]); - // } - - } // D > 3 -} - -} // namespace mgard_cuda \ No newline at end of file diff --git a/include/cuda/ErrorCalculator.h b/include/cuda/ErrorCalculator.h deleted file mode 100644 index 2a07c23bb4..0000000000 --- a/include/cuda/ErrorCalculator.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_ERROR_CALCULATOR -#define MGRAD_CUDA_ERROR_CALCULATOR - -namespace mgard_cuda { - -template T L_inf_norm(size_t n, T *data); - -template T L_2_norm(size_t n, T *data); - -template -T L_inf_error(size_t n, T *original_data, T *decompressed_data, - enum error_bound_type mode); - -template -T L_2_error(size_t n, T *original_data, T *decompressed_data, - enum error_bound_type mode); - -template T MSE(size_t n, T *original_data, T *decompressed_data); - -template T PSNR(size_t n, T *original_data, T *decompressed_data); - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/GPKFunctor.h b/include/cuda/GPKFunctor.h deleted file mode 100644 index 20ce200a26..0000000000 --- a/include/cuda/GPKFunctor.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_GPK_FUNCTOR -#define MGRAD_CUDA_GPK_FUNCTOR - -namespace mgard_cuda { - -template __device__ inline T lerp(T v0, T v1, T t) { -#ifdef MGARD_CUDA_FMA - if (sizeof(T) == sizeof(double)) { - return fma(t, v1, fma(-t, v0, v0)); - } else if (sizeof(T) == sizeof(float)) { - return fmaf(t, v1, fmaf(-t, v0, v0)); - } -#else - T r = v0 + v0 * t * -1; - r = r + t * v1; - return r; -#endif -} - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/GridProcessingKernel.h b/include/cuda/GridProcessingKernel.h deleted file mode 100644 index f5faead046..0000000000 --- a/include/cuda/GridProcessingKernel.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_GRID_PROCESSING_KERNEL -#define MGRAD_CUDA_GRID_PROCESSING_KERNEL - -#include "Common.h" - -namespace mgard_cuda { - -template -void gpk_reo(Handle &handle, SIZE *shape_h, SIZE *shape_d, - SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, DIM unprocessed_n, - DIM *unprocessed_dims, DIM curr_dim_r, DIM curr_dim_c, - DIM curr_dim_f, T *dratio_r, T *dratio_c, T *dratio_f, T *dv, - LENGTH lddv1, LENGTH lddv2, T *dw, LENGTH lddw1, LENGTH lddw2, - T *dwf, LENGTH lddwf1, LENGTH lddwf2, T *dwc, LENGTH lddwc1, - LENGTH lddwc2, T *dwr, LENGTH lddwr1, LENGTH lddwr2, T *dwcf, - LENGTH lddwcf1, LENGTH lddwcf2, T *dwrf, LENGTH lddwrf1, - LENGTH lddwrf2, T *dwrc, LENGTH lddwrc1, LENGTH lddwrc2, T *dwrcf, - LENGTH lddwrcf1, LENGTH lddwrcf2, int queue_idx, int config); - -template -void gpk_rev(Handle &handle, SIZE *shape_h, SIZE *shape_d, - SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, DIM unprocessed_n, - DIM *unprocessed_dims, DIM curr_dim_r, DIM curr_dim_c, - DIM curr_dim_f, T *dratio_r, T *dratio_c, T *dratio_f, T *dv, - LENGTH lddv1, LENGTH lddv2, T *dw, LENGTH lddw1, LENGTH lddw2, - T *dwf, LENGTH lddwf1, LENGTH lddwf2, T *dwc, LENGTH lddwc1, - LENGTH lddwc2, T *dwr, LENGTH lddwr1, LENGTH lddwr2, T *dwcf, - LENGTH lddwcf1, LENGTH lddwcf2, T *dwrf, LENGTH lddwrf1, - LENGTH lddwrf2, T *dwrc, LENGTH lddwrc1, LENGTH lddwrc2, T *dwrcf, - LENGTH lddwrcf1, LENGTH lddwrcf2, SIZE svr, SIZE svc, SIZE svf, - SIZE nvr, SIZE nvc, SIZE nvf, int queue_idx, int config); - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/GridProcessingKernel.hpp b/include/cuda/GridProcessingKernel.hpp deleted file mode 100644 index c6871e1b0c..0000000000 --- a/include/cuda/GridProcessingKernel.hpp +++ /dev/null @@ -1,4309 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_GRID_PROCESSING_KERNEL_TEMPLATE -#define MGRAD_CUDA_GRID_PROCESSING_KERNEL_TEMPLATE - -#include "CommonInternal.h" -#include "GPKFunctor.h" -#include "GridProcessingKernel.h" - -namespace mgard_cuda { - -template -__global__ void -_gpk_reo(SIZE *shape, SIZE *shape_c, SIZE *ldvs, SIZE *ldws, DIM unprocessed_n, - DIM *unprocessed_dims, DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, - T *dratio_r, T *dratio_c, T *dratio_f, T *dv, LENGTH lddv1, - LENGTH lddv2, T *dw, LENGTH lddw1, LENGTH lddw2, T *dwf, LENGTH lddwf1, - LENGTH lddwf2, T *dwc, LENGTH lddwc1, LENGTH lddwc2, T *dwr, - LENGTH lddwr1, LENGTH lddwr2, T *dwcf, LENGTH lddwcf1, LENGTH lddwcf2, - T *dwrf, LENGTH lddwrf1, LENGTH lddwrf2, T *dwrc, LENGTH lddwrc1, - LENGTH lddwrc2, T *dwrcf, LENGTH lddwrcf1, LENGTH lddwrcf2) { - - // bool debug = false; - // if (blockIdx.x == 0 && blockIdx.y ==0 && blockIdx.z == 0 && - // threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) debug = - // false; - - // volatile clock_t start = 0; - // volatile clock_t end = 0; - // volatile unsigned long long sum_time = 0; - - LENGTH threadId = (threadIdx.z * (blockDim.x * blockDim.y)) + - (threadIdx.y * blockDim.x) + threadIdx.x; - - SIZE nr, nc, nf; - SIZE nr_c, nc_c, nf_c; - SIZE r, c, f; - SIZE rest_r, rest_c, rest_f; - SIZE nr_p, nc_p, nf_p; - SIZE rest_r_p, rest_c_p, rest_f_p; - SIZE r_sm, c_sm, f_sm; - SIZE r_sm_ex, c_sm_ex, f_sm_ex; - SIZE r_gl, c_gl, f_gl; - SIZE r_gl_ex, c_gl_ex, f_gl_ex; - T res; - bool in_next = true; - - T *sm = SharedMemory(); - SIZE ldsm1 = F * 2 + 1; - SIZE ldsm2 = C * 2 + 1; - - T *v_sm = sm; - sm += (F * 2 + 1) * (C * 2 + 1) * (R * 2 + 1); - T *ratio_f_sm = sm; - sm += F * 2; - T *ratio_c_sm = sm; - sm += C * 2; - T *ratio_r_sm = sm; - sm += R * 2; - - SIZE *sm_size = (SIZE *)sm; - SIZE *shape_sm = sm_size; - sm_size += D_GLOBAL; - SIZE *shape_c_sm = sm_size; - sm_size += D_GLOBAL; - SIZE *ldvs_sm = sm_size; - sm_size += D_GLOBAL; - SIZE *ldws_sm = sm_size; - sm_size += D_GLOBAL; - sm = (T *)sm_size; - - DIM *sm_dim = (DIM *)sm; - DIM *unprocessed_dims_sm = sm_dim; - sm_dim += D_GLOBAL; - sm = (T *)sm_dim; - - SIZE idx[D_GLOBAL]; - if (threadId < D_GLOBAL) { - shape_sm[threadId] = shape[threadId]; - shape_c_sm[threadId] = shape_c[threadId]; - ldvs_sm[threadId] = ldvs[threadId]; - ldws_sm[threadId] = ldws[threadId]; - } - - if (threadId < unprocessed_n) { - unprocessed_dims_sm[threadId] = unprocessed_dims[threadId]; - } - __syncthreads(); - - for (DIM d = 0; d < D_GLOBAL; d++) - idx[d] = 0; - - nr = shape_sm[curr_dim_r]; - nc = shape_sm[curr_dim_c]; - nf = shape_sm[curr_dim_f]; - - nr_c = shape_c_sm[curr_dim_r]; - nc_c = shape_c_sm[curr_dim_c]; - nf_c = shape_c_sm[curr_dim_f]; - - if (D_LOCAL < 3) { - nr = 1; - nr_c = 1; - } - if (D_LOCAL < 2) { - nc = 1; - nc_c = 1; - } - - r = blockIdx.z * blockDim.z; - c = blockIdx.y * blockDim.y; - SIZE bidx = blockIdx.x; - SIZE firstD = div_roundup(shape_sm[0] - 1, blockDim.x); - f = (bidx % firstD) * blockDim.x; - - bidx /= firstD; - - // if (debug) printf("n: %d %d %d rcf: %d %d %d\n", nr, nc, nf, r, c, f); - rest_r = nr - r; - rest_c = nc - c; - rest_f = nf - f; - - nr_p = nr; - nc_p = nc; - nf_p = nf; - - rest_r_p = rest_r; - rest_c_p = rest_c; - rest_f_p = rest_f; - - if (nr % 2 == 0) { - nr_p = nr + 1; - rest_r_p = nr_p - r; - } - if (nc % 2 == 0) { - nc_p = nc + 1; - rest_c_p = nc_p - c; - } - if (nf % 2 == 0) { - nf_p = nf + 1; - rest_f_p = nf_p - f; - } - - for (DIM d = 0; d < D_GLOBAL; d++) { - if (D_LOCAL == 3 && d != curr_dim_r && d != curr_dim_c && d != curr_dim_f) { - idx[d] = bidx % shape_sm[d]; - bidx /= shape_sm[d]; - if (idx[d] >= shape_c_sm[d]) - in_next = false; - } - if (D_LOCAL == 2 && d != curr_dim_c && d != curr_dim_f) { - idx[d] = bidx % shape_sm[d]; - bidx /= shape_sm[d]; - if (idx[d] >= shape_c_sm[d]) - in_next = false; - } - } - - int skip = 0; -#pragma unroll 1 - for (DIM t = 0; t < D_GLOBAL; t++) { - for (DIM k = 0; k < unprocessed_n; k++) { - if (t == unprocessed_dims_sm[k] && - (shape_sm[t] % 2 == 1 && idx[t] % 2 == 1 || - shape_sm[t] % 2 == 0 && idx[t] % 2 == 1 && - idx[t] != shape_sm[t] - 1)) { - skip = 1; - } - } - } - - // if (blockIdx.x == 0 && blockIdx.y == 0 && blockIdx.z == 0) { - // if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) { - // printf("total_idx_sm: %d %d %d %d (skip: %d)\n", idx[3], idx[2], idx[1], - // idx[0], skip); - // } - // } - - LENGTH other_offset_v = get_idx(ldvs_sm, idx); - LENGTH other_offset_w = get_idx(ldws_sm, idx); - - dv = dv + other_offset_v; - dw = dw + other_offset_w; - dwr = dwr + other_offset_w; - dwc = dwc + other_offset_w; - dwf = dwf + other_offset_w; - dwrf = dwrf + other_offset_w; - dwrc = dwrc + other_offset_w; - dwcf = dwcf + other_offset_w; - dwrcf = dwrcf + other_offset_w; - - if (TYPE == 2) { - dwf = dw; - dwcf = dwc; - dwrf = dwr; - dwrcf = dwrc; - } - __syncthreads(); - // if (!skip) - { - r_sm = threadIdx.z; - c_sm = threadIdx.y; - f_sm = threadIdx.x; - - r_sm_ex = R * 2; - c_sm_ex = C * 2; - f_sm_ex = F * 2; - - r_gl = r + r_sm; - r_gl_ex = r + R * 2; - c_gl = c + c_sm; - c_gl_ex = c + C * 2; - f_gl = f + f_sm; - f_gl_ex = f + F * 2; - - // __syncthreads(); - // if (r_sm == 0 && c_sm == 0 && f_sm == 0) { - // //printf("setting zeros\n"); - // for (int i = 0; i < R * 2 + 1; i++) { - // for (int j = 0; j < C * 2 + 1; j++) { - // for (int k = 0; k < F * 2 + 1; k++) { - // v_sm[get_idx(ldsm1, ldsm2, i, j, k)] = 0.0; - // } - // } - // } - // //printf("done zeros\n"); - // } - // __syncthreads(); - /* Load v */ - // loading extra rules - // case 1: input = odd (non-padding required) - // case 1.a: block size < rest (need to load extra); - // case 1.b: block size > rest (NO need to load extra); - // case 2: input = even (padding requried) - // case 2.a: block size < rest (need to load extra); - // case 2.b: block size >= rest (NO need to load extra, but need - // padding); - - // Load from dv - if (r_sm < rest_r && c_sm < rest_c && f_sm < rest_f) { - - // load cubic - // asm volatile("membar.cta;"); - // start = clock64(); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl)]; - // if (blockIdx.x==0 && blockIdx.y==0&&blockIdx.z==0) { - // printf("load (%d %d %d) %f <- %d+(%d %d %d) (ld: %d %d)\n", - // r_sm, c_sm, f_sm, - // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl)], - // other_offset_v+r_gl, c_gl, f_gl, lddv1, lddv2); - // } - if (r_sm == 0) { - if (rest_r > R * 2) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm)] = - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl)]; - } - } - if (c_sm == 0) { - if (rest_c > C * 2) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm)] = - dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl)]; - } - } - if (f_sm == 0) { - if (rest_f > F * 2) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm_ex)] = - dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl_ex)]; - } - } - if (c_sm == 0 && f_sm == 0) { - if (rest_c > C * 2 && rest_f > F * 2) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm_ex)] = - dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)]; - } - } - if (r_sm == 0 && f_sm == 0) { - if (rest_r > R * 2 && rest_f > F * 2) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm_ex)] = - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)]; - } - } - if (r_sm == 0 && c_sm == 0) { - if (rest_r > R * 2 && rest_c > C * 2) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm)] = - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)]; - } - } - if (r_sm == 0 && c_sm == 0 && f_sm == 0) { - if (rest_r > R * 2 && rest_c > C * 2 && rest_f > F * 2) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm_ex)] = - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)]; - } - } - } - - __syncthreads(); - - // apply padding is necessary - if (r_sm < rest_r && c_sm < rest_c && f_sm < rest_f) { - - // printf("load main[%d %d %d]:%f --> [%d %d %d] (%d %d %d)\n", r_gl, - // c_gl, f_gl, - // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl)], r_sm, c_sm, f_sm, nr, - // nc, nf); - - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[load main] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - - // load extra surface - - if (r_sm == 0) { - if (rest_r > R * 2) { - // v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm)] = - // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl)]; - // printf("load-r[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl, f_gl, - // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl)], r_sm_ex, c_sm, - // f_sm); - } else if (nr % 2 == 0) { - // if (r == 16 && c == 0 && f == 0) { - // printf("padding (%d %d %d) %f <- (%f %f %f)\n", rest_r_p - 1, - // c_sm, f_sm, - // v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm)], rest_r - // - 1, c_sm, f_sm); - // padded = true; - // aa = v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm)]; - // bb = v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm)]; - // } - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm)]; - } - } - - if (c_sm == 0) { - if (rest_c > C * 2) { - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm)] = - // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl)]; - // printf("load-c[%d %d %d]:%f --> [%d %d %d]\n", r_gl, c_gl_ex, f_gl, - // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl)], r_sm, c_sm_ex, - // f_sm); - } else if (nc % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, f_sm)]; - } - } - - if (f_sm == 0) { - if (rest_f > F * 2) { - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm_ex)] = - // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl_ex)]; - // printf("load-f[%d %d %d]:%f --> [%d %d %d]\n", r_gl, c_gl, f_gl_ex, - // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl_ex)], r_sm, c_sm, - // f_sm_ex); - } else if (nf % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f - 1)]; - } - } - - // load extra edges - if (c_sm == 0 && f_sm == 0) { - if (rest_c > C * 2 && rest_f > F * 2) { - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm_ex)] = - // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)]; - // printf("load-cf[%d %d %d]:%f --> [%d %d %d]\n", r_gl, c_gl_ex, - // f_gl_ex, dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)], r_sm, - // c_sm_ex, f_sm_ex); - } else if (rest_c <= C * 2 && rest_f <= F * 2 && nc % 2 == 0 && - nf % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, rest_f - 1)]; - } else if (rest_c > C * 2 && rest_f <= F * 2 && nf % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, rest_f - 1)]; - } else if (rest_c <= C * 2 && rest_f > F * 2 && nc % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, f_sm_ex)]; - } - } - - if (r_sm == 0 && f_sm == 0) { - if (rest_r > R * 2 && rest_f > F * 2) { - // v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm_ex)] = - // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)]; - // printf("load-rf[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl, - // f_gl_ex, dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)], - // r_sm_ex, c_sm, f_sm_ex); - } else if (rest_r <= R * 2 && rest_f <= F * 2 && nr % 2 == 0 && - nf % 2 == 0) { - // printf("padding (%d %d %d) <- (%d %d %d)\n", rest_r_p - 1, c_sm, - // rest_f_p - 1, rest_r - 1, c_sm, rest_f - 1); - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, rest_f - 1)]; - } else if (rest_r > R * 2 && rest_f <= F * 2 && nf % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, rest_f - 1)]; - } else if (rest_r <= R * 2 && rest_f > F * 2 && nr % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm_ex)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm_ex)]; - } - } - - if (r_sm == 0 && c_sm == 0) { - if (rest_r > R * 2 && rest_c > C * 2) { - // v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm)] = - // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)]; - // printf("load-rc[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl_ex, - // f_gl, dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)], r_sm_ex, - // c_sm_ex, f_sm); - } else if (rest_r <= R * 2 && rest_c <= C * 2 && nr % 2 == 0 && - nc % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm)]; - // printf("padding (%d %d %d) <- (%d %d %d): %f\n", rest_r_p - 1, - // rest_c_p - 1, f_sm, rest_r - 1, rest_c - 1, f_sm, - // v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm)]); - } else if (rest_r > R * 2 && rest_c <= C * 2 && nc % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c - 1, f_sm)]; - } else if (rest_r <= R * 2 && rest_c > C * 2 && nr % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm_ex, f_sm)]; - } - } - // load extra vertex - - if (r_sm == 0 && c_sm == 0 && f_sm == 0) { - if (rest_r > R * 2 && rest_c > C * 2 && rest_f > F * 2) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm_ex)] = - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)]; - // printf("load-rcf[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl_ex, - // f_gl_ex, dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)], - // r_sm_ex, c_sm_ex, f_sm_ex); - } else if (rest_r <= R * 2 && rest_c <= C * 2 && rest_f <= F * 2 && - nr % 2 == 0 && nc % 2 == 0 && nf % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, - rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, rest_f - 1)]; - } else if (rest_r > R * 2 && rest_c > C * 2 && rest_f <= F * 2 && - nf % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, rest_f - 1)]; - } else if (rest_r > R * 2 && rest_c <= C * 2 && rest_f > F * 2 && - nc % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c - 1, f_sm_ex)]; - } else if (rest_r > R * 2 && rest_c <= C * 2 && rest_f <= F * 2 && - nc % 2 == 0 && nf % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c - 1, rest_f - 1)]; - } else if (rest_r <= R * 2 && rest_c > C * 2 && rest_f > F * 2 && - nr % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm_ex)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm_ex, f_sm_ex)]; - } else if (rest_r <= R * 2 && rest_c > C * 2 && rest_f <= F * 2 && - nr % 2 == 0 && nf % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm_ex, rest_f - 1)]; - } else if (rest_r <= R * 2 && rest_c <= C * 2 && rest_f > F * 2 && - nr % 2 == 0 && nc % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm_ex)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm_ex)]; - } - } - - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[load extra] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - - // load dist - if (c_sm == 0 && f_sm == 0 && r_sm < rest_r_p - 2) { - // printf("%d/%d load %f\n", r_sm, rest_r - 2, dratio_r[r + r_sm]); - ratio_r_sm[r_sm] = dratio_r[r + r_sm]; - // if (nr % 2 == 0 && R * 2 + 1 >= rest_r_p && r_sm == 0) { - // ratio_r_sm[rest_r_p - 3] = 0.5; - // } - } - if (r_sm == 0 && f_sm == 0 && c_sm < rest_c_p - 2) { - ratio_c_sm[c_sm] = dratio_c[c + c_sm]; - // if (nc % 2 == 0 && C * 2 + 1 >= rest_c_p && c_sm == 0) { - // ratio_c_sm[rest_c_p - 3] = 0.5; - // } - } - if (c_sm == 0 && r_sm == 0 && f_sm < rest_f_p - 2) { - ratio_f_sm[f_sm] = dratio_f[f + f_sm]; - // if (nf % 2 == 0 && F * 2 + 1 >= rest_f_p && f_sm == 0) { - // ratio_f_sm[rest_f_p - 3] = 0.5; - // } - } - - // if (r == 0 && c == 0 && f == 0 && r_sm == 0 && c_sm == 0 && f_sm == 0) - // { - // printf("ratio:"); - // for (int i = 0; i < R * 2 + 1; i++) { - // printf("%2.2f ", ratio_r_sm[i]); - // } - // printf("\n"); - // } - - } // restrict boundary - - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[load ratio] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - - // __syncthreads(); - // // debug print - // if (debug) { - // printf("in config: %d %d %d (%d %d %d)\n", R, C, F, r,c,f); - // printf("rest_p: %d %d %d\n", rest_r_p, rest_c_p, rest_f_p); - // bool print = false; - // for (int i = 0; i < R * 2 + 1; i++) { - // for (int j = 0; j < C * 2 + 1; j++) { - // for (int k = 0; k < F * 2 + 1; k++) { - // // if (abs(v_sm[get_idx(ldsm1, ldsm2, i, j, k)]) > 10000) { - // // print = true; - // // printf("(block %d %d %d) %2.2f \n", r,c,f, - // v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); - // // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); - // // } - // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); - // } - // printf("\n"); - // } - // printf("\n"); - // } - // } - __syncthreads(); - - if (dw && threadId < R * C * F) { - r_sm = (threadId / (C * F)) * 2; - c_sm = ((threadId % (C * F)) / F) * 2; - f_sm = ((threadId % (C * F)) % F) * 2; - r_gl = r / 2 + threadId / (C * F); - c_gl = c / 2 + threadId % (C * F) / F; - f_gl = f / 2 + threadId % (C * F) % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - ; - } - if (CALC_COEFF) { - if (in_next && f_gl < nf_c) { - ; - } else { - res -= dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - } - } - } - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } - } - - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[store coarse] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - int base = 0; - // printf("TYPE =%d \n", TYPE); - // printf("%d == %d && %llu >= %d && %llu < %d\n", r + R * 2, nr_p - 1, - // threadId, base, threadId, base + C * F); - - if (dw && r + R * 2 == nr_p - 1 && threadId >= base && - threadId < base + C * F) { - r_sm = R * 2; - c_sm = ((threadId - base) / F) * 2; - f_sm = ((threadId - base) % F) * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + (threadId - base) / F; - f_gl = f / 2 + (threadId - base) % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - ; - } - if (CALC_COEFF) { - if (in_next && f_gl < nf_c) { - ; - } else { - res -= dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - } - } - } - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } - } - - base += C * F; // ROUND_UP_WARP(C * F) * WARP_SIZE; - if (dw && c + C * 2 == nc_p - 1 && threadId >= base && - threadId < base + R * F) { - r_sm = ((threadId - base) / F) * 2; - c_sm = C * 2; - f_sm = ((threadId - base) % F) * 2; - r_gl = r / 2 + (threadId - base) / F; - c_gl = c / 2 + C; - f_gl = f / 2 + (threadId - base) % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - ; - } - if (CALC_COEFF) { - if (in_next && f_gl < nf_c) { - ; - } else { - res -= dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - } - } - } - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } - // printf("(%d %d %d) (%d %d %d) %f\n", - // r_sm, c_sm, f_sm, r_gl, c_gl, f_gl, dwork[get_idx(lddv1, lddv2, - // r_gl, c_gl, f_gl)]); - } - - base += R * F; // ROUND_UP_WARP(R * F) * WARP_SIZE; - // printf("%d %d\n", base, threadId); - if (dw && f + F * 2 == nf_p - 1 && threadId >= base && - threadId < base + R * C) { - r_sm = ((threadId - base) / C) * 2; - c_sm = ((threadId - base) % C) * 2; - f_sm = F * 2; - r_gl = r / 2 + (threadId - base) / C; - c_gl = c / 2 + (threadId - base) % C; - f_gl = f / 2 + F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - ; - } - if (CALC_COEFF) { - if (in_next && f_gl < nf_c) { - ; - } else { - res -= dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - } - } - } - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } - } - - base += R * C; // ROUND_UP_WARP(R * C) * WARP_SIZE; - // load extra edges - if (dw && c + C * 2 == nc_p - 1 && f + F * 2 == nf_p - 1 && - threadId >= base && threadId < base + R) { - r_sm = (threadId - base) * 2; - c_sm = C * 2; - f_sm = F * 2; - r_gl = r / 2 + threadId - base; - c_gl = c / 2 + C; - f_gl = f / 2 + F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - ; - } - if (CALC_COEFF) { - if (in_next && f_gl < nf_c) { - ; - } else { - res -= dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - } - } - } - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } - } - - base += R; // ROUND_UP_WARP(R) * WARP_SIZE; - // if (TYPE == 2) printf("%d %d, %d, %llu, %d\n",dw == NULL, f + F * 2, nf_p - // - 1, threadId, C); - if (dw && r + R * 2 == nr_p - 1 && f + F * 2 == nf_p - 1 && - threadId >= base && threadId < base + C) { - r_sm = R * 2; - c_sm = (threadId - base) * 2; - f_sm = F * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId - base; - f_gl = f / 2 + F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - ; - } - if (CALC_COEFF) { - if (in_next && f_gl < nf_c) { - ; - } else { - res -= dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - } - } - } - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } - // printf("store[%d %d %d]: %f\n", r_sm, c_sm, f_sm, v_sm[get_idx(ldsm1, - // ldsm2, r_sm, c_sm, f_sm)]); - } - - base += C; // ROUND_UP_WARP(C) * WARP_SIZE; - if (dw && r + R * 2 == nr_p - 1 && c + C * 2 == nc_p - 1 && - threadId >= base && threadId < base + F) { - r_sm = R * 2; - c_sm = C * 2; - f_sm = (threadId - base) * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + C; - f_gl = f / 2 + threadId - base; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - ; - } - if (CALC_COEFF) { - if (in_next && f_gl < nf_c) { - ; - } else { - res -= dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - } - } - } - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } - } - base += F; // ROUND_UP_WARP(F) * WARP_SIZE; - // // load extra vertex - if (dw && r + R * 2 == nr_p - 1 && c + C * 2 == nc_p - 1 && - f + F * 2 == nf_p - 1 && threadId >= base && threadId < base + 1) { - r_sm = R * 2; - c_sm = C * 2; - f_sm = F * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + C; - f_gl = f / 2 + F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - ; - } - if (CALC_COEFF) { - if (in_next && f_gl < nf_c) { - ; - } else { - res -= dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - } - } - } - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } - } - - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[store extra] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - - // start = clock64(); - - if (dwf && threadId >= R * C * F && threadId < R * C * F * 2) { - r_sm = ((threadId - R * C * F) / (C * F)) * 2; - c_sm = (((threadId - R * C * F) % (C * F)) / F) * 2; - f_sm = (((threadId - R * C * F) % (C * F)) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F) % (C * F)) % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { // fused - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { // calc_coeff only - res -= dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - } - } - dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl = 2 * f_gl + 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - ; - } - if (CALC_COEFF) { - if (in_next && f_gl < nf_c) { - ; - } else { - res -= dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - } - } - } - dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)] = res; - } - } - - // if (nr == 70) printf("f-store: (%d %d %d) <- %f (%d %d %d)\n", r_gl, - // c_gl, f_gl, v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], r_sm, c_sm, - // f_sm); - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[F-store] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - } - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[F-store] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - - // if (r_sm % 2 == 0 && c_sm % 2 != 0 && f_sm % 2 == 0) { - - if (dwc && threadId >= R * C * F * 2 && threadId < R * C * F * 3) { - r_sm = ((threadId - R * C * F * 2) / (C * F)) * 2; - c_sm = (((threadId - R * C * F * 2) % (C * F)) / F) * 2 + 1; - f_sm = (((threadId - R * C * F * 2) % (C * F)) % F) * 2; - r_gl = r / 2 + (threadId - R * C * F * 2) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 2) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 2) % (C * F)) % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - } - } - dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - if (CALC_COEFF) { // no need to test in_next - res -= dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - } - } - dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)] = res; - } - // if (nr == 70) printf("c-store: (%d %d %d) <- %f (%d %d %d)\n", r_gl, - // c_gl, f_gl, v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], r_sm, - // c_sm, f_sm); - } - } - - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[C-store] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - - // if (r_sm % 2 != 0 && c_sm % 2 == 0 && f_sm % 2 == 0) { - if (dwr && threadId >= R * C * F * 3 && threadId < R * C * F * 4) { - r_sm = ((threadId - R * C * F * 3) / (C * F)) * 2 + 1; - c_sm = (((threadId - R * C * F * 3) % (C * F)) / F) * 2; - f_sm = (((threadId - R * C * F * 3) % (C * F)) % F) * 2; - r_gl = r / 2 + (threadId - R * C * F * 3) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 3) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 3) % (C * F)) % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - } - } - dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - if (CALC_COEFF) { // no need to test if in_next - res -= dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - } - } - dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)] = res; - } - } - } - - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[R-store] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - __syncthreads(); - if (dwcf && threadId >= R * C * F * 4 && threadId < R * C * F * 5) { - r_sm = ((threadId - R * C * F * 4) / (C * F)) * 2; - c_sm = (((threadId - R * C * F * 4) % (C * F)) / F) * 2 + 1; - f_sm = (((threadId - R * C * F * 4) % (C * F)) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F * 4) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 4) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 4) % (C * F)) % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { - if (!skip) { - if (INTERPOLATION) { - T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T tmp = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - res = lerp(f1, f2, ratio_c_sm[c_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwcf[get_idx(lddwcf1, lddwcf2, r_gl, c_gl, f_gl)]; - } - } - dwcf[get_idx(lddwcf1, lddwcf2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl = 2 * f_gl + 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - if (CALC_COEFF) { // not need to test if in_next - res -= dwcf[get_idx(lddwcf1, lddwcf2, r_gl, c_gl, f_gl)]; - } - } - dwcf[get_idx(lddwcf1, lddwcf2, r_gl, c_gl, f_gl)] = res; - } - } - } - - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[CF-store] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - - if (dwrf && threadId >= R * C * F * 5 && threadId < R * C * F * 6) { - r_sm = ((threadId - R * C * F * 5) / (C * F)) * 2 + 1; - c_sm = (((threadId - R * C * F * 5) % (C * F)) / F) * 2; - f_sm = (((threadId - R * C * F * 5) % (C * F)) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F * 5) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 5) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 5) % (C * F)) % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - if (!skip) { - if (INTERPOLATION) { - T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res = lerp(f1, f2, ratio_r_sm[r_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwrf[get_idx(lddwrf1, lddwrf2, r_gl, c_gl, f_gl)]; - } - } - dwrf[get_idx(lddwrf1, lddwrf2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl = 2 * f_gl + 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - if (CALC_COEFF) { // no need to test if in_next - res -= dwrf[get_idx(lddwrf1, lddwrf2, r_gl, c_gl, f_gl)]; - } - } - dwrf[get_idx(lddwrf1, lddwrf2, r_gl, c_gl, f_gl)] = res; - } - } - } - - if (dwrc && threadId >= R * C * F * 6 && threadId < R * C * F * 7) { - r_sm = ((threadId - R * C * F * 6) / (C * F)) * 2 + 1; - c_sm = (((threadId - R * C * F * 6) % (C * F)) / F) * 2 + 1; - f_sm = (((threadId - R * C * F * 6) % (C * F)) % F) * 2; - r_gl = r / 2 + (threadId - R * C * F * 6) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 6) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 6) % (C * F)) % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - if (!skip) { - if (INTERPOLATION) { - T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwrc[get_idx(lddwrc1, lddwrc2, r_gl, c_gl, f_gl)]; - } - } - dwrc[get_idx(lddwrc1, lddwrc2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); - } - if (CALC_COEFF) { // no need to test if in_next - res -= dwrc[get_idx(lddwrc1, lddwrc2, r_gl, c_gl, f_gl)]; - } - } - dwrc[get_idx(lddwrc1, lddwrc2, r_gl, c_gl, f_gl)] = res; - } - } - } - - if (dwrcf && threadId >= R * C * F * 7 && threadId < R * C * F * 8) { - r_sm = ((threadId - R * C * F * 7) / (C * F)) * 2 + 1; - c_sm = (((threadId - R * C * F * 7) % (C * F)) / F) * 2 + 1; - f_sm = (((threadId - R * C * F * 7) % (C * F)) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F * 7) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 7) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 7) % (C * F)) % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { - if (!skip) { - if (INTERPOLATION) { - T f1 = lerp( - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp( - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f3 = lerp( - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f4 = lerp( - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - - T fc1 = lerp(f1, f2, ratio_c_sm[c_sm - 1]); - T fc2 = lerp(f3, f4, ratio_c_sm[c_sm - 1]); - - res = lerp(fc1, fc2, ratio_r_sm[r_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwrcf[get_idx(lddwrcf1, lddwrcf2, r_gl, c_gl, f_gl)]; - } - } - dwrcf[get_idx(lddwrcf1, lddwrcf2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl = 2 * f_gl + 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); - } - if (CALC_COEFF) { // no need to test if in_next - res -= dwrcf[get_idx(lddwrcf1, lddwrcf2, r_gl, c_gl, f_gl)]; - } - } - dwrcf[get_idx(lddwrcf1, lddwrcf2, r_gl, c_gl, f_gl)] = res; - } - } - } - // end = clock64(); - - // asm volatile("membar.cta;"); - // if (threadId < 256 && blockIdx.z == 0 && blockIdx.y == 0 && blockIdx.x == - // 0) printf("threadId %d elapsed %lu\n", threadId, end-start); - if (r + R * 2 == nr_p - 1) { - // printf("test\n"); - if (threadId < C * F) { - // printf("test1\n"); - if (dwf) { - // printf("test2\n"); - r_sm = R * 2; - c_sm = (threadId / F) * 2; - f_sm = (threadId % F) * 2 + 1; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId / F; - f_gl = f / 2 + threadId % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - // printf("test3\n"); - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - } - } - // printf("dwf (%d %d %d): %f\n", r_gl, c_gl, f_gl, res); - dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl = 2 * f_gl + 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - ; - } - if (CALC_COEFF) { // need to test if in_next - if (in_next && f_gl < nf_c) { - ; - } // in_next - else { - res -= dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - } - } - } - dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)] = res; - } - } - } - - if (dwc) { - r_sm = R * 2; - c_sm = (threadId / F) * 2 + 1; - f_sm = (threadId % F) * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId / F; - f_gl = f / 2 + threadId % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - } - } - dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - if (CALC_COEFF) { // no need to test if in_next - res -= dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - } - } - dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)] = res; - } - } - } - - // printf("(%d %d %d) (%d %d %d) %f\n", - // r_sm, c_sm, f_sm, r_gl, c_gl, f_gl, v_sm[get_idx(ldsm1, - // ldsm2, r_sm, c_sm, f_sm)]); - if (dwcf) { - r_sm = R * 2; - c_sm = (threadId / F) * 2 + 1; - f_sm = (threadId % F) * 2 + 1; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId / F; - f_gl = f / 2 + threadId % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { - if (!skip) { - if (INTERPOLATION) { - T f1 = lerp( - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp( - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res = lerp(f1, f2, ratio_c_sm[c_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwcf[get_idx(lddwcf1, lddwcf2, r_gl, c_gl, f_gl)]; - } - } - dwcf[get_idx(lddwcf1, lddwcf2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl = 2 * f_gl + 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - if (CALC_COEFF) { - res -= dwcf[get_idx(lddwcf1, lddwcf2, r_gl, c_gl, f_gl)]; - } - } - dwcf[get_idx(lddwcf1, lddwcf2, r_gl, c_gl, f_gl)] = res; - } - } - } - } - } - - if (c + C * 2 == nc_p - 1) { - if (threadId >= R * C * F && threadId < R * C * F + R * F) { - if (dwf) { - r_sm = ((threadId - R * C * F) / F) * 2; - c_sm = C * 2; - f_sm = ((threadId - R * C * F) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F) / F; - c_gl = c / 2 + C; - f_gl = f / 2 + (threadId - R * C * F) % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - } - } - dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl = 2 * f_gl + 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - ; - } - if (CALC_COEFF) { // need to test if in_next - if (in_next && f_gl < nf_c) { - ; - } // in_next - else { - res -= dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - } - } - } - dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)] = res; - } - } - } - - if (dwr) { - r_sm = ((threadId - R * C * F) / F) * 2 + 1; - c_sm = C * 2; - f_sm = ((threadId - R * C * F) % F) * 2; - r_gl = r / 2 + (threadId - R * C * F) / F; - c_gl = c / 2 + C; - f_gl = f / 2 + (threadId - R * C * F) % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - } - } - dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - if (CALC_COEFF) { - res -= dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - } - } - dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)] = res; - } - } - } - - if (dwrf) { - r_sm = ((threadId - R * C * F) / F) * 2 + 1; - c_sm = C * 2; - f_sm = ((threadId - R * C * F) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F) / F; - c_gl = c / 2 + C; - f_gl = f / 2 + (threadId - R * C * F) % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - if (!skip) { - if (INTERPOLATION) { - T f1 = lerp( - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp( - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res = lerp(f1, f2, ratio_r_sm[r_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwrf[get_idx(lddwrf1, lddwrf2, r_gl, c_gl, f_gl)]; - } - } - dwrf[get_idx(lddwrf1, lddwrf2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl = 2 * f_gl + 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - if (CALC_COEFF) { // no need to test if in_next - res -= dwrf[get_idx(lddwrf1, lddwrf2, r_gl, c_gl, f_gl)]; - } - } - dwrf[get_idx(lddwrf1, lddwrf2, r_gl, c_gl, f_gl)] = res; - } - } - } - } - } - - if (f + F * 2 == nf_p - 1) { - if (threadId >= R * C * F * 2 && threadId < R * C * F * 2 + R * C) { - if (dwc) { - r_sm = ((threadId - R * C * F * 2) / C) * 2; - c_sm = ((threadId - R * C * F * 2) % C) * 2 + 1; - f_sm = F * 2; - r_gl = r / 2 + (threadId - R * C * F * 2) / C; - c_gl = c / 2 + (threadId - R * C * F * 2) % C; - f_gl = f / 2 + F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - } - } - dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - if (CALC_COEFF) { - res -= dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - } - } - dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)] = res; - } - } - } - - if (dwr) { - r_sm = ((threadId - R * C * F * 2) / C) * 2 + 1; - c_sm = ((threadId - R * C * F * 2) % C) * 2; - f_sm = F * 2; - r_gl = r / 2 + (threadId - R * C * F * 2) / C; - c_gl = c / 2 + (threadId - R * C * F * 2) % C; - f_gl = f / 2 + F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - } - } - dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - if (CALC_COEFF) { - res -= dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - } - } - dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)] = res; - } - } - } - - if (dwrc) { - r_sm = ((threadId - R * C * F * 2) / C) * 2 + 1; - c_sm = ((threadId - R * C * F * 2) % C) * 2 + 1; - f_sm = F * 2; - r_gl = r / 2 + (threadId - R * C * F * 2) / C; - c_gl = c / 2 + (threadId - R * C * F * 2) % C; - f_gl = f / 2 + F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - if (!skip) { - if (INTERPOLATION) { - T c1 = lerp( - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - T c2 = lerp( - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwrc[get_idx(lddwrc1, lddwrc2, r_gl, c_gl, f_gl)]; - } - } - dwrc[get_idx(lddwrc1, lddwrc2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - T c1 = lerp( - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - T c2 = lerp( - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); - } - if (CALC_COEFF) { - res -= dwrc[get_idx(lddwrc1, lddwrc2, r_gl, c_gl, f_gl)]; - } - } - dwrc[get_idx(lddwrc1, lddwrc2, r_gl, c_gl, f_gl)] = res; - } - } - } - } - } - - if (dwr && c + C * 2 == nc_p - 1 && f + F * 2 == nf_p - 1) { - if (threadId >= R * C * F * 3 && threadId < R * C * F * 3 + R) { - r_sm = (threadId - R * C * F * 3) * 2 + 1; - c_sm = C * 2; - f_sm = F * 2; - r_gl = r / 2 + threadId - R * C * F * 3; - c_gl = c / 2 + C; - f_gl = f / 2 + F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - } - } - dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - if (CALC_COEFF) { - res -= dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - } - } - dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)] = res; - } - } - } - } - - if (dwc && r + R * 2 == nr_p - 1 && f + F * 2 == nf_p - 1) { - if (threadId >= R * C * F * 4 && threadId < R * C * F * 4 + C) { - r_sm = R * 2; - c_sm = (threadId - R * C * F * 4) * 2 + 1; - f_sm = F * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId - R * C * F * 4; - f_gl = f / 2 + F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - } - } - dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - if (CALC_COEFF) { - res -= dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - } - } - dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)] = res; - } - } - } - } - - // printf("test1\n"); - if (dwf && r + R * 2 == nr_p - 1 && c + C * 2 == nc_p - 1) { - // printf("test2\n"); - if (threadId >= R * C * F * 5 && threadId < R * C * F * 5 + F) { - // printf("test3\n"); - r_sm = R * 2; - c_sm = C * 2; - f_sm = (threadId - R * C * F * 5) * 2 + 1; - r_gl = r / 2 + R; - c_gl = c / 2 + C; - f_gl = f / 2 + threadId - R * C * F * 5; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - // printf("test4\n"); - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - } - if (INTERPOLATION && CALC_COEFF) { - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - } - if (!INTERPOLATION && CALC_COEFF) { - res -= dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - } - } - dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)] = res; - // printf("dwf(%d %d %d): %f\n", r_gl, c_gl, f_gl, - // dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]); - } - } else if (TYPE == 2) { - f_gl = 2 * f_gl + 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - if (!skip) { - if (INTERPOLATION) { - ; - } - if (CALC_COEFF) { // do need to test in_next - if (in_next && f_gl < nf_c) { - ; - } // in_next - else { - res -= dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - } - } - } - dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)] = res; - } - } - } - } - - } // skip - - // if (r == 0 && c == 0 && f == 0 && threadId == 0) { - // printf("out config: %d %d %d (%d %d %d)\n", R, C, F, r,c,f); - // for (int i = 0; i < R * 2 + 1; i++) { - // for (int j = 0; j < C * 2 + 1; j++) { - // for (int k = 0; k < F * 2 + 1; k++) { - // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); - // } - // printf("\n"); - // } - // printf("\n"); - // } - // } -} - -template -void gpk_reo_adaptive_launcher( - Handle &handle, SIZE *shape_h, SIZE *shape_d, SIZE *shape_c_d, - SIZE *ldvs, SIZE *ldws, DIM unprocessed_n, DIM *unprocessed_dims, - DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, T *dratio_r, T *dratio_c, - T *dratio_f, T *dv, LENGTH lddv1, LENGTH lddv2, T *dw, LENGTH lddw1, - LENGTH lddw2, T *dwf, LENGTH lddwf1, LENGTH lddwf2, T *dwc, LENGTH lddwc1, - LENGTH lddwc2, T *dwr, LENGTH lddwr1, LENGTH lddwr2, T *dwcf, - LENGTH lddwcf1, LENGTH lddwcf2, T *dwrf, LENGTH lddwrf1, LENGTH lddwrf2, - T *dwrc, LENGTH lddwrc1, LENGTH lddwrc2, T *dwrcf, LENGTH lddwrcf1, - LENGTH lddwrcf2, int queue_idx) { - - SIZE nr = shape_h[curr_dim_r]; - SIZE nc = shape_h[curr_dim_c]; - SIZE nf = shape_h[curr_dim_f]; - if (D_LOCAL == 2) { - nr = 1; - } - SIZE total_thread_z = std::max(nr - 1, (SIZE)1); - SIZE total_thread_y = std::max(nc - 1, (SIZE)1); - SIZE total_thread_x = std::max(nf - 1, (SIZE)1); - - SIZE tbx, tby, tbz, gridx, gridy, gridz; - dim3 threadsPerBlock, blockPerGrid; - SIZE sm_size; - // const int R = 4; - // const int C = 4; - // const int F = 16; - // tbz = std::min(R, total_thread_z); - // tby = std::min(C, total_thread_y); - // tbx = std::min(F, total_thread_x); - tbz = R; - tby = C; - tbx = F; - sm_size = ((R + 1) * (C + 1) * (F + 1) + R + C + F) * sizeof(T); - sm_size += (D_GLOBAL * 4) * sizeof(SIZE); - sm_size += (D_GLOBAL * 1) * sizeof(DIM); - - gridz = ceil((double)total_thread_z / tbz); - gridy = ceil((double)total_thread_y / tby); - gridx = ceil((double)total_thread_x / tbx); - for (DIM d = 0; d < D_GLOBAL; d++) { - if (D_LOCAL == 3 && d != curr_dim_f && d != curr_dim_c && d != curr_dim_r) { - gridx *= shape_h[d]; - } - if (D_LOCAL == 2 && d != curr_dim_f && d != curr_dim_c) { - gridx *= shape_h[d]; - } - } - threadsPerBlock = dim3(tbx, tby, tbz); - blockPerGrid = dim3(gridx, gridy, gridz); - - // printf("_gpk_reo exec config (%d %d %d) (%d %d %d)\n", tbx, tby, tbz, - // gridx, gridy, gridz); - - // high_resolution_clock::time_point t1 = high_resolution_clock::now(); - _gpk_reo<<>>( - shape_d, shape_c_d, ldvs, ldws, unprocessed_n, unprocessed_dims, - curr_dim_r, curr_dim_c, curr_dim_f, dratio_r, dratio_c, dratio_f, dv, - lddv1, lddv2, dw, lddw1, lddw2, dwf, lddwf1, lddwf2, dwc, lddwc1, lddwc2, - dwr, lddwr1, lddwr2, dwcf, lddwcf1, lddwcf2, dwrf, lddwrf1, lddwrf2, dwrc, - lddwrc1, lddwrc2, dwrcf, lddwrcf1, lddwrcf2); - - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } -} - -template -void gpk_reo(Handle &handle, SIZE *shape_h, SIZE *shape_d, - SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, DIM unprocessed_n, - DIM *unprocessed_dims, DIM curr_dim_r, DIM curr_dim_c, - DIM curr_dim_f, T *dratio_r, T *dratio_c, T *dratio_f, T *dv, - LENGTH lddv1, LENGTH lddv2, T *dw, LENGTH lddw1, LENGTH lddw2, - T *dwf, LENGTH lddwf1, LENGTH lddwf2, T *dwc, LENGTH lddwc1, - LENGTH lddwc2, T *dwr, LENGTH lddwr1, LENGTH lddwr2, T *dwcf, - LENGTH lddwcf1, LENGTH lddwcf2, T *dwrf, LENGTH lddwrf1, - LENGTH lddwrf2, T *dwrc, LENGTH lddwrc1, LENGTH lddwrc2, T *dwrcf, - LENGTH lddwrcf1, LENGTH lddwrcf2, int queue_idx, int config) { - -#define GPK(R, C, F) \ - { \ - gpk_reo_adaptive_launcher( \ - handle, shape_h, shape_d, shape_c_d, ldvs, ldws, unprocessed_n, \ - unprocessed_dims, curr_dim_r, curr_dim_c, curr_dim_f, dratio_r, \ - dratio_c, dratio_f, dv, lddv1, lddv2, dw, lddw1, lddw2, dwf, lddwf1, \ - lddwf2, dwc, lddwc1, lddwc2, dwr, lddwr1, lddwr2, dwcf, lddwcf1, \ - lddwcf2, dwrf, lddwrf1, lddwrf2, dwrc, lddwrc1, lddwrc2, dwrcf, \ - lddwrcf1, lddwrcf2, queue_idx); \ - } - bool profile = false; - if (handle.profile_kernels) { - profile = true; - } - if (D_LOCAL == 3) { - if (profile || config == 6) { - GPK(2, 2, 128) - } - if (profile || config == 5) { - GPK(2, 2, 64) - } - if (profile || config == 4) { - GPK(4, 4, 32) - } - if (profile || config == 3) { - GPK(4, 4, 16) - } - if (profile || config == 2) { - GPK(4, 4, 8) - } - if (profile || config == 1) { - GPK(4, 4, 4) - } - if (profile || config == 0) { - GPK(2, 2, 2) - } - // GPK(T, 4, 4, 4) - } else if (D_LOCAL == 2) { - if (profile || config == 6) { - GPK(1, 2, 128) - } - if (profile || config == 5) { - GPK(1, 2, 64) - } - if (profile || config == 4) { - GPK(1, 4, 32) - } - if (profile || config == 3) { - GPK(1, 4, 16) - } - if (profile || config == 2) { - GPK(1, 4, 8) - } - if (profile || config == 1) { - GPK(1, 4, 4) - } - if (profile || config == 0) { - GPK(1, 2, 2) - } - // GPK(T, 1, 4, 4) - } else if (D_LOCAL == 1) { - if (profile || config == 6) { - GPK(1, 1, 128) - } - if (profile || config == 5) { - GPK(1, 1, 64) - } - if (profile || config == 4) { - GPK(1, 1, 32) - } - if (profile || config == 3) { - GPK(1, 1, 16) - } - if (profile || config == 2) { - GPK(1, 1, 8) - } - if (profile || config == 1) { - GPK(1, 1, 4) - } - if (profile || config == 0) { - GPK(1, 1, 2) - } - } -#undef GPK -} - -template -__global__ void -_gpk_rev(SIZE *shape, SIZE *shape_c, SIZE *ldvs, SIZE *ldws, DIM unprocessed_n, - DIM *unprocessed_dims, DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, - T *dratio_r, T *dratio_c, T *dratio_f, T *dv, LENGTH lddv1, - LENGTH lddv2, T *dw, LENGTH lddw1, LENGTH lddw2, T *dwf, LENGTH lddwf1, - LENGTH lddwf2, T *dwc, LENGTH lddwc1, LENGTH lddwc2, T *dwr, - LENGTH lddwr1, LENGTH lddwr2, T *dwcf, LENGTH lddwcf1, LENGTH lddwcf2, - T *dwrf, LENGTH lddwrf1, LENGTH lddwrf2, T *dwrc, LENGTH lddwrc1, - LENGTH lddwrc2, T *dwrcf, LENGTH lddwrcf1, LENGTH lddwrcf2, SIZE svr, - SIZE svc, SIZE svf, SIZE nvr, SIZE nvc, SIZE nvf) { - - // bool debug = false; - // if (blockIdx.x == 0 && blockIdx.y == 0 && blockIdx.z == 0 && - // threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) - // debug = false; - - // bool debug2 = false; - // if (blockIdx.x == 0 && blockIdx.y == 0 && blockIdx.z == 0) - // debug2 = false; - - LENGTH threadId = (threadIdx.z * (blockDim.x * blockDim.y)) + - (threadIdx.y * blockDim.x) + threadIdx.x; - - SIZE nr, nc, nf; - SIZE nr_c, nc_c, nf_c; - SIZE r, c, f; - SIZE rest_r, rest_c, rest_f; - SIZE nr_p, nc_p, nf_p; - SIZE rest_r_p, rest_c_p, rest_f_p; - SIZE r_sm, c_sm, f_sm; - SIZE r_sm_ex, c_sm_ex, f_sm_ex; - SIZE r_gl, c_gl, f_gl; - SIZE r_gl_ex, c_gl_ex, f_gl_ex; - T res; - bool in_next = true; - - T *sm = SharedMemory(); - SIZE ldsm1 = F * 2 + 1; - SIZE ldsm2 = C * 2 + 1; - - T *v_sm = sm; - sm += (F * 2 + 1) * (C * 2 + 1) * (R * 2 + 1); - T *ratio_f_sm = sm; - sm += F * 2; - T *ratio_c_sm = sm; - sm += C * 2; - T *ratio_r_sm = sm; - sm += R * 2; - - SIZE *sm_size = (SIZE *)sm; - SIZE *shape_sm = sm_size; - sm_size += D_GLOBAL; - SIZE *shape_c_sm = sm_size; - sm_size += D_GLOBAL; - SIZE *ldvs_sm = sm_size; - sm_size += D_GLOBAL; - SIZE *ldws_sm = sm_size; - sm_size += D_GLOBAL; - sm = (T *)sm_size; - - DIM *sm_dim = (DIM *)sm; - DIM *unprocessed_dims_sm = sm_dim; - sm_dim += D_GLOBAL; - sm = (T *)sm_dim; - - SIZE idx[D_GLOBAL]; - if (threadId < D_GLOBAL) { - shape_sm[threadId] = shape[threadId]; - shape_c_sm[threadId] = shape_c[threadId]; - ldvs_sm[threadId] = ldvs[threadId]; - ldws_sm[threadId] = ldws[threadId]; - } - - if (threadId < unprocessed_n) { - unprocessed_dims_sm[threadId] = unprocessed_dims[threadId]; - } - __syncthreads(); - for (DIM d = 0; d < D_GLOBAL; d++) - idx[d] = 0; - - nr = shape_sm[curr_dim_r]; - nc = shape_sm[curr_dim_c]; - nf = shape_sm[curr_dim_f]; - - nr_c = shape_c_sm[curr_dim_r]; - nc_c = shape_c_sm[curr_dim_c]; - nf_c = shape_c_sm[curr_dim_f]; - - if (D_LOCAL < 3) { - nr = 1; - nr_c = 1; - } - if (D_LOCAL < 2) { - nc = 1; - nc_c = 1; - } - - r = blockIdx.z * blockDim.z; - c = blockIdx.y * blockDim.y; - SIZE bidx = blockIdx.x; - SIZE firstD = div_roundup(shape_sm[0] - 1, blockDim.x); - f = (bidx % firstD) * blockDim.x; - - bidx /= firstD; - - rest_r = nr - r; - rest_c = nc - c; - rest_f = nf - f; - - nr_p = nr; - nc_p = nc; - nf_p = nf; - - rest_r_p = rest_r; - rest_c_p = rest_c; - rest_f_p = rest_f; - - if (nr % 2 == 0) { - nr_p = nr + 1; - rest_r_p = nr_p - r; - } - if (nc % 2 == 0) { - nc_p = nc + 1; - rest_c_p = nc_p - c; - } - if (nf % 2 == 0) { - nf_p = nf + 1; - rest_f_p = nf_p - f; - } - - for (int d = 0; d < D_GLOBAL; d++) { - if (D_LOCAL == 3 && d != curr_dim_r && d != curr_dim_c && d != curr_dim_f) { - idx[d] = bidx % shape_sm[d]; - bidx /= shape_sm[d]; - if ((shape_sm[d] % 2 == 1 && idx[d] % 2 != 0) || - shape_sm[d] % 2 == 0 && - (idx[d] % 2 != 0 && idx[d] != shape_sm[d] - 1)) - in_next = false; - } - if (D_LOCAL == 2 && d != curr_dim_c && d != curr_dim_f) { - idx[d] = bidx % shape_sm[d]; - bidx /= shape_sm[d]; - if ((shape_sm[d] % 2 == 1 && idx[d] % 2 != 0) || - shape_sm[d] % 2 == 0 && - (idx[d] % 2 != 0 && idx[d] != shape_sm[d] - 1)) - in_next = false; - } - } - - int skip = 0; -#pragma unroll 1 - for (DIM t = 0; t < D_GLOBAL; t++) { - for (DIM k = 0; k < unprocessed_n; k++) { - if (t == unprocessed_dims_sm[k] && idx[t] >= shape_c_sm[t]) { - skip = 1; - } - } - } - - // if (blockIdx.x == 0 && blockIdx.y == 0 && blockIdx.z == 0) { - // if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) { - // printf("TYPE %d total_idx_sm: %d %d %d %d (skip: %d)\n", TYPE, idx[3], - // idx[2], idx[1], idx[0], skip); - // } - // } - - LENGTH other_offset_v = get_idx(ldvs_sm, idx); - LENGTH other_offset_w = get_idx(ldws_sm, idx); - - dv = dv + other_offset_v; - dw = dw + other_offset_w; - dwr = dwr + other_offset_w; - dwc = dwc + other_offset_w; - dwf = dwf + other_offset_w; - dwrf = dwrf + other_offset_w; - dwrc = dwrc + other_offset_w; - dwcf = dwcf + other_offset_w; - dwrcf = dwrcf + other_offset_w; - - if (TYPE == 2) { - dwf = dw; - dwcf = dwc; - dwrf = dwr; - dwrcf = dwrc; - } - __syncthreads(); - - r_sm = threadIdx.z; - c_sm = threadIdx.y; - f_sm = threadIdx.x; - - r_sm_ex = R * 2; - c_sm_ex = C * 2; - f_sm_ex = F * 2; - - r_gl = r + r_sm; - r_gl_ex = r + R * 2; - c_gl = c + c_sm; - c_gl_ex = c + C * 2; - f_gl = f + f_sm; - f_gl_ex = f + F * 2; - - // load dist - if (c_sm == 0 && f_sm == 0 && r_sm < rest_r - 2) { - ratio_r_sm[r_sm] = dratio_r[r + r_sm]; - if (nr % 2 == 0 && R * 2 + 1 >= rest_r_p && r_sm == 0) { - ratio_r_sm[rest_r_p - 3] = 0.5; - } - } - if (r_sm == 0 && f_sm == 0 && c_sm < rest_c - 2) { - ratio_c_sm[c_sm] = dratio_c[c + c_sm]; - if (nc % 2 == 0 && C * 2 + 1 >= rest_c_p && c_sm == 0) { - ratio_c_sm[rest_c_p - 3] = 0.5; - } - } - if (c_sm == 0 && r_sm == 0 && f_sm < rest_f - 2) { - ratio_f_sm[f_sm] = dratio_f[f + f_sm]; - if (nf % 2 == 0 && F * 2 + 1 >= rest_f_p && f_sm == 0) { - ratio_f_sm[rest_f_p - 3] = 0.5; - } - } - - if (threadIdx.z == 0 && threadIdx.y == 0 && threadIdx.x == 0) { - for (int i = 0; i < R * 2 + 1; i++) { - for (int j = 0; j < C * 2 + 1; j++) { - for (int k = 0; k < F * 2 + 1; k++) { - v_sm[get_idx(ldsm1, ldsm2, i, j, k)] = 0.0; - } - } - } - } - - __syncthreads(); - - if (dw && threadId < R * C * F) { - r_sm = (threadId / (C * F)) * 2; - c_sm = ((threadId % (C * F)) / F) * 2; - f_sm = ((threadId % (C * F)) % F) * 2; - r_gl = r / 2 + threadId / (C * F); - c_gl = c / 2 + threadId % (C * F) / F; - f_gl = f / 2 + threadId % (C * F) % F; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - if (!INTERPOLATION && COEFF_RESTORE) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - } - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - // if (debug2) - // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, - // dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], r_gl, c_gl, - // f_gl); - if (!skip) { - if (INTERPOLATION) { - ; - } - if (COEFF_RESTORE) { - bool f_in_next = (nf % 2 == 1 && f_gl % 2 == 0) || - (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); - if (in_next && f_in_next) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - ; - } - } - } - } - - f_gl += 1; - f_sm += 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - // if (debug2) - // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, - // dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], r_gl, c_gl, - // f_gl); - if (!skip) { - if (INTERPOLATION) { - ; - } - if (COEFF_RESTORE) { - bool f_in_next = (nf % 2 == 1 && f_gl % 2 == 0) || - (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); - if (in_next && f_in_next) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - ; - } - } - } - } - } - } - - int base = 0; - if (dw && threadId >= base && threadId < base + C * F) { - r_sm = R * 2; - c_sm = ((threadId - base) / F) * 2; - f_sm = ((threadId - base) % F) * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + (threadId - base) / F; - f_gl = f / 2 + (threadId - base) % F; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - if (!INTERPOLATION && COEFF_RESTORE) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - } - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - ; - } - if (COEFF_RESTORE) { - bool f_in_next = (nf % 2 == 1 && f_gl % 2 == 0) || - (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); - // printf("nf: %d, f_gl: %d, in_next: %d, f_in_next: %d\n", nf, - // f_gl, in_next, f_in_next); - if (in_next && f_in_next) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - ; - } - } - } - } - - f_gl += 1; - f_sm += 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - // if (debug2) - // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, - // dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], r_gl, c_gl, - // f_gl); - if (!skip) { - if (INTERPOLATION) { - ; - } - if (COEFF_RESTORE) { - bool f_in_next = (nf % 2 == 1 && f_gl % 2 == 0) || - (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); - // printf("nf: %d, f_gl: %d, in_next: %d, f_in_next: %d\n", nf, - // f_gl, in_next, f_in_next); - if (in_next && f_in_next) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - ; - } - } - } - } - } - } - base += C * F; // ROUND_UP_WARP(C * F) * WARP_SIZE; - if (dw && threadId >= base && threadId < base + R * F) { - r_sm = ((threadId - base) / F) * 2; - c_sm = C * 2; - f_sm = ((threadId - base) % F) * 2; - r_gl = r / 2 + (threadId - base) / F; - c_gl = c / 2 + C; - f_gl = f / 2 + (threadId - base) % F; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - if (!INTERPOLATION && COEFF_RESTORE) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - } - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - ; - } - if (COEFF_RESTORE) { - bool f_in_next = (nf % 2 == 1 && f_gl % 2 == 0) || - (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); - if (in_next && f_in_next) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - ; - } - } - } - } - - f_gl += 1; - f_sm += 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - // if (debug2) - // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, - // dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], r_gl, c_gl, - // f_gl); - if (!skip) { - if (INTERPOLATION) { - ; - } - if (COEFF_RESTORE) { - bool f_in_next = (nf % 2 == 1 && f_gl % 2 == 0) || - (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); - if (in_next && f_in_next) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - ; - } - } - } - } - } - } - base += R * F; // ROUND_UP_WARP(R * F) * WARP_SIZE; - if (dw && threadId >= base && threadId < base + R * C) { - r_sm = ((threadId - base) / C) * 2; - c_sm = ((threadId - base) % C) * 2; - f_sm = F * 2; - r_gl = r / 2 + (threadId - base) / C; - c_gl = c / 2 + (threadId - base) % C; - f_gl = f / 2 + F; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - if (!INTERPOLATION && COEFF_RESTORE) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - } - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - ; - } - if (COEFF_RESTORE) { - bool f_in_next = (nf % 2 == 1 && f_gl % 2 == 0) || - (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); - if (in_next && f_in_next) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - ; - } - } - } - } - - f_gl += 1; - f_sm += 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - // if (debug2) - // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, - // dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], r_gl, c_gl, - // f_gl); - if (!skip) { - if (INTERPOLATION) { - ; - } - if (COEFF_RESTORE) { - bool f_in_next = (nf % 2 == 1 && f_gl % 2 == 0) || - (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); - if (in_next && f_in_next) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - ; - } - } - } - } - } - } - base += R * C; // ROUND_UP_WARP(R * C) * WARP_SIZE; - // load extra edges - if (dw && threadId >= base && threadId < base + R) { - r_sm = (threadId - base) * 2; - c_sm = C * 2; - f_sm = F * 2; - r_gl = r / 2 + threadId - base; - c_gl = c / 2 + C; - f_gl = f / 2 + F; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - if (!INTERPOLATION && COEFF_RESTORE) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - } - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - ; - } - if (COEFF_RESTORE) { - bool f_in_next = (nf % 2 == 1 && f_gl % 2 == 0) || - (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); - if (in_next && f_in_next) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - ; - } - } - } - } - - f_gl += 1; - f_sm += 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - // if (debug2) - // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, - // dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], r_gl, c_gl, - // f_gl); - if (!skip) { - if (INTERPOLATION) { - ; - } - if (COEFF_RESTORE) { - bool f_in_next = (nf % 2 == 1 && f_gl % 2 == 0) || - (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); - if (in_next && f_in_next) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - ; - } - } - } - } - } - } - base += R; // ROUND_UP_WARP(R) * WARP_SIZE; - if (dw && threadId >= base && threadId < base + C) { - r_sm = R * 2; - c_sm = (threadId - base) * 2; - f_sm = F * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId - base; - f_gl = f / 2 + F; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - if (!INTERPOLATION && COEFF_RESTORE) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - } - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - ; - } - if (COEFF_RESTORE) { - bool f_in_next = (nf % 2 == 1 && f_gl % 2 == 0) || - (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); - if (in_next && f_in_next) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - ; - } - } - } - } - - f_gl += 1; - f_sm += 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - // if (debug2) - // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, - // dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], r_gl, c_gl, - // f_gl); - if (!skip) { - if (INTERPOLATION) { - ; - } - if (COEFF_RESTORE) { - bool f_in_next = (nf % 2 == 1 && f_gl % 2 == 0) || - (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); - if (in_next && f_in_next) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - ; - } - } - } - } - } - } - base += C; // ROUND_UP_WARP(C) * WARP_SIZE; - if (dw && threadId >= base && threadId < base + F) { - r_sm = R * 2; - c_sm = C * 2; - f_sm = (threadId - base) * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + C; - f_gl = f / 2 + threadId - base; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - if (!INTERPOLATION && COEFF_RESTORE) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - } - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - ; - } - if (COEFF_RESTORE) { - bool f_in_next = (nf % 2 == 1 && f_gl % 2 == 0) || - (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); - if (in_next && f_in_next) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - ; - } - } - } - } - - f_gl += 1; - f_sm += 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - // if (debug2) - // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, - // dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], r_gl, c_gl, - // f_gl); - if (!skip) { - if (INTERPOLATION) { - ; - } - if (COEFF_RESTORE) { - bool f_in_next = (nf % 2 == 1 && f_gl % 2 == 0) || - (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); - if (in_next && f_in_next) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - ; - } - } - } - } - } - } - base += F; // ROUND_UP_WARP(F) * WARP_SIZE; - // // load extra vertex - if (dw && threadId >= base && threadId < base + 1) { - r_sm = R * 2; - c_sm = C * 2; - f_sm = F * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + C; - f_gl = f / 2 + F; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - if (!INTERPOLATION && COEFF_RESTORE) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - } - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - ; - } - if (COEFF_RESTORE) { - bool f_in_next = (nf % 2 == 1 && f_gl % 2 == 0) || - (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); - if (in_next && f_in_next) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - ; - } - } - } - } - - f_gl += 1; - f_sm += 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - // if (debug2) - // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, - // dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], r_gl, c_gl, - // f_gl); - if (!skip) { - if (INTERPOLATION) { - ; - } - if (COEFF_RESTORE) { - bool f_in_next = (nf % 2 == 1 && f_gl % 2 == 0) || - (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); - if (in_next && f_in_next) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; - } else { - ; - } - } - } - } - } - } - - __syncthreads(); - - // __syncthreads(); - // if (debug) { - // printf("TYPE: %d %d %d %d\n", TYPE, min(rest_r_p, R * 2 + 1), - // min(rest_c_p, C * 2 + 1), min(rest_f_p, F * 2 + 1)); - // for (int i = 0; i < min(rest_r_p, R * 2 + 1); i++) { - // for (int j = 0; j < min(rest_c_p, C * 2 + 1); j++) { - // for (int k = 0; k < min(rest_f_p, F * 2 + 1); k++) { - // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); - // } - // printf("\n"); - // } - // printf("\n"); - // } - // } - // __syncthreads(); - - if (dwf && threadId >= R * C * F && threadId < R * C * F * 2) { - - r_sm = ((threadId - R * C * F) / (C * F)) * 2; - c_sm = (((threadId - R * C * F) % (C * F)) / F) * 2; - f_sm = (((threadId - R * C * F) % (C * F)) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F) % (C * F)) % F; - - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - - res = dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { // fused - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - f_gl = 2 * f_gl + 1; - // res = dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - ; - } - } - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - - if (dwc && threadId >= R * C * F * 2 && threadId < R * C * F * 3) { - r_sm = ((threadId - R * C * F * 2) / (C * F)) * 2; - c_sm = (((threadId - R * C * F * 2) % (C * F)) / F) * 2 + 1; - f_sm = (((threadId - R * C * F * 2) % (C * F)) % F) * 2; - r_gl = r / 2 + (threadId - R * C * F * 2) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 2) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 2) % (C * F)) % F; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - res = dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { - res = dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - - if (dwr && threadId >= R * C * F * 3 && threadId < R * C * F * 4) { - r_sm = ((threadId - R * C * F * 3) / (C * F)) * 2 + 1; - c_sm = (((threadId - R * C * F * 3) % (C * F)) / F) * 2; - f_sm = (((threadId - R * C * F * 3) % (C * F)) % F) * 2; - r_gl = r / 2 + (threadId - R * C * F * 3) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 3) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 3) % (C * F)) % F; - - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { - res = dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { - res = dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - - if (dwcf && threadId >= R * C * F * 4 && threadId < R * C * F * 5) { - r_sm = ((threadId - R * C * F * 4) / (C * F)) * 2; - c_sm = (((threadId - R * C * F * 4) % (C * F)) / F) * 2 + 1; - f_sm = (((threadId - R * C * F * 4) % (C * F)) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F * 4) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 4) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 4) % (C * F)) % F; - - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { - res = dwcf[get_idx(lddwcf1, lddwcf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res += lerp(f1, f2, ratio_c_sm[c_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res = lerp(f1, f2, ratio_c_sm[c_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl = 2 * f_gl + 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { - res = dwcf[get_idx(lddwrf1, lddwrf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - - if (dwrf && threadId >= R * C * F * 5 && threadId < R * C * F * 6) { - r_sm = ((threadId - R * C * F * 5) / (C * F)) * 2 + 1; - c_sm = (((threadId - R * C * F * 5) % (C * F)) / F) * 2; - f_sm = (((threadId - R * C * F * 5) % (C * F)) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F * 5) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 5) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 5) % (C * F)) % F; - - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - - res = dwrf[get_idx(lddwrf1, lddwrf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - - res += lerp(f1, f2, ratio_r_sm[r_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - - res = lerp(f1, f2, ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl = 2 * f_gl + 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { - res = dwrf[get_idx(lddwrf1, lddwrf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - - if (dwrc && threadId >= R * C * F * 6 && threadId < R * C * F * 7) { - r_sm = ((threadId - R * C * F * 6) / (C * F)) * 2 + 1; - c_sm = (((threadId - R * C * F * 6) % (C * F)) / F) * 2 + 1; - f_sm = (((threadId - R * C * F * 6) % (C * F)) % F) * 2; - r_gl = r / 2 + (threadId - R * C * F * 6) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 6) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 6) % (C * F)) % F; - - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - res = dwrc[get_idx(lddwrc1, lddwrc2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res += lerp(c1, c2, ratio_r_sm[r_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf) { - res = dwrc[get_idx(lddwrc1, lddwrc2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - - if (dwrcf && threadId >= R * C * F * 7 && threadId < R * C * F * 8) { - r_sm = ((threadId - R * C * F * 7) / (C * F)) * 2 + 1; - c_sm = (((threadId - R * C * F * 7) % (C * F)) / F) * 2 + 1; - f_sm = (((threadId - R * C * F * 7) % (C * F)) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F * 7) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 7) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 7) % (C * F)) % F; - - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { - res = dwrcf[get_idx(lddwrcf1, lddwrcf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - T f1 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f3 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f4 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - - T fc1 = lerp(f1, f2, ratio_c_sm[c_sm - 1]); - T fc2 = lerp(f3, f4, ratio_c_sm[c_sm - 1]); - - res += lerp(fc1, fc2, ratio_r_sm[r_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - T f1 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f3 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f4 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - - T fc1 = lerp(f1, f2, ratio_c_sm[c_sm - 1]); - T fc2 = lerp(f3, f4, ratio_c_sm[c_sm - 1]); - - res = lerp(fc1, fc2, ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl = 2 * f_gl + 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf) { - res = dwrcf[get_idx(lddwrcf1, lddwrcf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - - if (r + R * 2 == nr_p - 1) { - if (threadId < C * F) { - if (dwf) { - r_sm = R * 2; - c_sm = (threadId / F) * 2; - f_sm = (threadId % F) * 2 + 1; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId / F; - f_gl = f / 2 + threadId % F; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - res = dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl = 2 * f_gl + 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - // res = dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - ; - } - } - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - - if (dwc) { - r_sm = R * 2; - c_sm = (threadId / F) * 2 + 1; - f_sm = (threadId % F) * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId / F; - f_gl = f / 2 + threadId % F; - - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - res = dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { - res = dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - if (dwcf) { - r_sm = R * 2; - c_sm = (threadId / F) * 2 + 1; - f_sm = (threadId % F) * 2 + 1; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId / F; - f_gl = f / 2 + threadId % F; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { - res = dwcf[get_idx(lddwcf1, lddwcf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - T f1 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res += lerp(f1, f2, ratio_c_sm[c_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - T f1 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res = lerp(f1, f2, ratio_c_sm[c_sm - 1]); - } - } - - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl = 2 * f_gl + 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { - res = dwcf[get_idx(lddwcf1, lddwcf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - // if (idx[1] ==0 && idx[2] == 0) { - // printf("%f(%d %d %d) %f(%d %d %d) -> %f\n", - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - // r_sm, c_sm - 1, f_sm, v_sm[get_idx(ldsm1, ldsm2, - // r_sm, c_sm + 1, f_sm)], r_sm, c_sm + 1, f_sm, res); - // } - } - } - - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - } - } - - if (c + C * 2 == nc_p - 1) { - if (threadId >= R * C * F && threadId < R * C * F + R * F) { - if (dwf) { - r_sm = ((threadId - R * C * F) / F) * 2; - c_sm = C * 2; - f_sm = ((threadId - R * C * F) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F) / F; - c_gl = c / 2 + C; - f_gl = f / 2 + (threadId - R * C * F) % F; - - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - res = dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl = 2 * f_gl + 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - // res = dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - ; - } - } - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - if (dwr) { - r_sm = ((threadId - R * C * F) / F) * 2 + 1; - c_sm = C * 2; - f_sm = ((threadId - R * C * F) % F) * 2; - r_gl = r / 2 + (threadId - R * C * F) / F; - c_gl = c / 2 + C; - f_gl = f / 2 + (threadId - R * C * F) % F; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { - res = dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { - res = dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - if (dwrf) { - r_sm = ((threadId - R * C * F) / F) * 2 + 1; - c_sm = C * 2; - f_sm = ((threadId - R * C * F) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F) / F; - c_gl = c / 2 + C; - f_gl = f / 2 + (threadId - R * C * F) % F; - - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - res = dwrf[get_idx(lddwrf1, lddwrf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - T f1 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res += lerp(f1, f2, ratio_r_sm[r_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - T f1 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res = lerp(f1, f2, ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl = 2 * f_gl + 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { - res = dwrf[get_idx(lddwrf1, lddwrf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - } - } - - if (f + F * 2 == nf_p - 1) { - if (threadId >= R * C * F * 2 && threadId < R * C * F * 2 + R * C) { - if (dwc) { - r_sm = ((threadId - R * C * F * 2) / C) * 2; - c_sm = ((threadId - R * C * F * 2) % C) * 2 + 1; - f_sm = F * 2; - r_gl = r / 2 + (threadId - R * C * F * 2) / C; - c_gl = c / 2 + (threadId - R * C * F * 2) % C; - f_gl = f / 2 + F; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - res = dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { - res = dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - - if (dwr) { - r_sm = ((threadId - R * C * F * 2) / C) * 2 + 1; - c_sm = ((threadId - R * C * F * 2) % C) * 2; - f_sm = F * 2; - r_gl = r / 2 + (threadId - R * C * F * 2) / C; - c_gl = c / 2 + (threadId - R * C * F * 2) % C; - f_gl = f / 2 + F; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { - res = dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { - res = dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - - if (dwrc) { - r_sm = ((threadId - R * C * F * 2) / C) * 2 + 1; - c_sm = ((threadId - R * C * F * 2) % C) * 2 + 1; - f_sm = F * 2; - r_gl = r / 2 + (threadId - R * C * F * 2) / C; - c_gl = c / 2 + (threadId - R * C * F * 2) % C; - f_gl = f / 2 + F; - - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - res = dwrc[get_idx(lddwrc1, lddwrc2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - T c1 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - T c2 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res += lerp(c1, c2, ratio_r_sm[r_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - T c1 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - T c2 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf) { - res = dwrc[get_idx(lddwrc1, lddwrc2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - T c1 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - T c2 = - lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - } - } - - if (c + C * 2 == nc_p - 1 && f + F * 2 == nf_p - 1) { - if (threadId >= R * C * F * 3 && threadId < R * C * F * 3 + R) { - if (dwr) { - r_sm = (threadId - R * C * F * 3) * 2 + 1; - c_sm = C * 2; - f_sm = F * 2; - r_gl = r / 2 + threadId - R * C * F * 3; - c_gl = c / 2 + C; - f_gl = f / 2 + F; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { - res = dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { - res = dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - } - } - - if (r + R * 2 == nr_p - 1 && f + F * 2 == nf_p - 1) { - if (threadId >= R * C * F * 4 && threadId < R * C * F * 4 + C) { - if (dwc) { - r_sm = R * 2; - c_sm = (threadId - R * C * F * 4) * 2 + 1; - f_sm = F * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId - R * C * F * 4; - f_gl = f / 2 + F; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - res = dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl *= 2; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { - res = dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - } - } - - if (r + R * 2 == nr_p - 1 && c + C * 2 == nc_p - 1) { - if (threadId >= R * C * F * 5 && threadId < R * C * F * 5 + F) { - if (dwf) { - r_sm = R * 2; - c_sm = C * 2; - f_sm = (threadId - R * C * F * 5) * 2 + 1; - r_gl = r / 2 + R; - c_gl = c / 2 + C; - f_gl = f / 2 + threadId - R * C * F * 5; - if (TYPE == 1) { - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - res = dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION && COEFF_RESTORE) { - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - } else if (INTERPOLATION && !COEFF_RESTORE) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - } - } - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } else if (TYPE == 2) { - f_gl = 2 * f_gl + 1; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf) { - // res = dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - if (!skip) { - if (INTERPOLATION) { - ; - } - } - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - } - } - - // __syncthreads(); - // if (debug) { - // printf("TYPE: %d %d %d %d\n", TYPE, min(rest_r_p, R * 2 + 1), - // min(rest_c_p, C * 2 + 1), min(rest_f_p, F * 2 + 1)); - // for (int i = 0; i < min(rest_r_p, R * 2 + 1); i++) { - // for (int j = 0; j < min(rest_c_p, C * 2 + 1); j++) { - // for (int k = 0; k < min(rest_f_p, F * 2 + 1); k++) { - // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); - // } - // printf("\n"); - // } - // printf("\n"); - // } - // } - // __syncthreads(); - - __syncthreads(); - - r_sm = threadIdx.z; - c_sm = threadIdx.y; - f_sm = threadIdx.x; - - r_sm_ex = blockDim.z; - c_sm_ex = blockDim.y; - f_sm_ex = blockDim.x; - - r_gl = r + r_sm; - c_gl = c + c_sm; - f_gl = f + f_sm; - - // r_gl_ex = r + R * 2; - // c_gl_ex = c + C * 2; - // f_gl_ex = f + F * 2; - - r_gl_ex = r + rest_r - 1; - c_gl_ex = c + rest_c - 1; - f_gl_ex = f + rest_f - 1; - - int unpadding_r = rest_r; - int unpadding_c = rest_c; - int unpadding_f = rest_f; - if (nr % 2 == 0) - unpadding_r -= 1; - if (nc % 2 == 0) - unpadding_c -= 1; - if (TYPE == 1 && nf % 2 == 0) - unpadding_f -= 1; - - if (r_sm < unpadding_r && c_sm < unpadding_c && f_sm < unpadding_f) { - - // store extra rules - // case 1: input = odd (non-padding required) - // case 1.a: block size + 1 == rest (need to store extra); - // case 1.b: block size + 1 != rest (No need to store extra); - // case 2: input = even (un-padding requried) - // case 2.a: block size + 1 >= rest (No need to store extra, but need - // un-padding first); case 2.b: block size + 1 < rest (No need to store - // extra); - - if (D_LOCAL >= 3 && r_sm == 0) { - if (nr % 2 != 0 && R * 2 + 1 == rest_r) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl)] += - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm)]; - } - } - if (nr % 2 == 0 && R * 2 + 1 >= rest_r_p) { - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm)]; - } - } - - if (D_LOCAL >= 2 && c_sm == 0) { - if (nc % 2 != 0 && C * 2 + 1 == rest_c) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl)] += - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm)]; - } - } - if (nc % 2 == 0 && C * 2 + 1 >= rest_c_p) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, f_sm)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm)]; - } - } - - if (D_LOCAL >= 1 && f_sm == 0) { - if (nf % 2 != 0 && F * 2 + 1 == rest_f) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl_ex)] += - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm_ex)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm_ex)]; - } - } - if (nf % 2 == 0 && F * 2 + 1 >= rest_f_p && TYPE == 1) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f - 1)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f_p - 1)]; - } - } - - // load extra edges - if (D_LOCAL >= 2 && c_sm == 0 && f_sm == 0) { - if (nc % 2 != 0 && C * 2 + 1 == rest_c && nf % 2 != 0 && - F * 2 + 1 == rest_f) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)] += - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm_ex)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm_ex)]; - } - } - if (nc % 2 == 0 && nf % 2 == 0 && C * 2 + 1 >= rest_c_p && - F * 2 + 1 >= rest_f_p && TYPE == 1) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, rest_f - 1)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, rest_f_p - 1)]; - } - if (nc % 2 == 0 && nf % 2 != 0 && C * 2 + 1 >= rest_c_p && - F * 2 + 1 == rest_f) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)] += - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm_ex)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm_ex)]; - } - } - if (nc % 2 != 0 && nf % 2 == 0 && C * 2 + 1 == rest_c && - F * 2 + 1 >= rest_f_p && TYPE == 1) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)] += - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, rest_f_p - 1)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, rest_f_p - 1)]; - // printf("(%d %d %d): %f <- (%d %d %d)\n", - // r_gl, c_gl_ex, f_gl_ex, - // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)], - // r_sm, c_sm_ex, f_gl_ex); - } - } - } - - if (D_LOCAL >= 3 && r_sm == 0 && f_sm == 0) { - if (nr % 2 != 0 && R * 2 + 1 == rest_r && nf % 2 != 0 && - F * 2 + 1 == rest_f) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)] += - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm_ex)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm_ex)]; - } - } - if (nr % 2 == 0 && nf % 2 == 0 && R * 2 + 1 >= rest_r_p && - F * 2 + 1 >= rest_f_p && TYPE == 1) { - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, rest_f - 1)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, rest_f_p - 1)]; - } - if (nr % 2 == 0 && nf % 2 != 0 && R * 2 + 1 >= rest_r_p && - F * 2 + 1 == rest_f) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)] += - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm_ex)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm_ex)]; - } - } - if (nr % 2 != 0 && nf % 2 == 0 && R * 2 + 1 == rest_r && - F * 2 + 1 >= rest_f_p && TYPE == 1) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)] += - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, rest_f_p - 1)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, rest_f_p - 1)]; - // printf("(%d %d %d): %f <- (%d %d %d)\n", - // r_gl_ex, c_gl, rest_f-1, - // dv[get_idx(lddv1, lddv2, r_gl_ex-1, c_gl, f_gl_ex)], - // r_sm_ex, c_sm, rest_f_p-1); - } - } - } - - if (D_LOCAL >= 3 && r_sm == 0 && c_sm == 0) { - if (nr % 2 != 0 && R * 2 + 1 == rest_r && nc % 2 != 0 && - C * 2 + 1 == rest_c) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)] += - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm)]; - } - } - if (nr % 2 == 0 && nc % 2 == 0 && R * 2 + 1 >= rest_r_p && - C * 2 + 1 >= rest_c_p) { - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm)]; - } - if (nr % 2 == 0 && nc % 2 != 0 && R * 2 + 1 >= rest_r_p && - C * 2 + 1 == rest_c) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)] += - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm)]; - } - } - if (nr % 2 != 0 && nc % 2 == 0 && R * 2 + 1 == rest_r && - C * 2 + 1 >= rest_c_p) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)] += - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm)]; - } - } - } - // load extra vertex - - if (D_LOCAL >= 3 && r_sm == 0 && c_sm == 0 && f_sm == 0) { - if (nr % 2 != 0 && R * 2 + 1 == rest_r && nc % 2 != 0 && - C * 2 + 1 == rest_c && nf % 2 != 0 && F * 2 + 1 == rest_f) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] += - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm_ex)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm_ex)]; - } - } - - if (nr % 2 == 0 && nc % 2 == 0 && nf % 2 == 0 && R * 2 + 1 >= rest_r_p && - C * 2 + 1 >= rest_c_p && F * 2 + 1 >= rest_f_p && TYPE == 1) { - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, rest_f - 1)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, - rest_f_p - 1)]; - } - if (nr % 2 == 0 && nc % 2 == 0 && nf % 2 != 0 && R * 2 + 1 >= rest_r_p && - C * 2 + 1 >= rest_c_p && F * 2 + 1 == rest_f) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] += - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm_ex)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm_ex)]; - } - } - if (nr % 2 == 0 && nc % 2 != 0 && nf % 2 == 0 && R * 2 + 1 >= rest_r_p && - C * 2 + 1 == rest_c && F * 2 + 1 >= rest_f_p && TYPE == 1) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] += - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, rest_f_p - 1)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, rest_f_p - 1)]; - } - } - if (nr % 2 != 0 && nc % 2 == 0 && nf % 2 == 0 && R * 2 + 1 == rest_r && - C * 2 + 1 >= rest_c_p && F * 2 + 1 >= rest_f_p && TYPE == 1) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] += - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, rest_f_p - 1)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, rest_f_p - 1)]; - } - } - if (nr % 2 == 0 && nc % 2 != 0 && nf % 2 != 0 && R * 2 + 1 >= rest_r_p && - C * 2 + 1 == rest_c && F * 2 + 1 == rest_f) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] += - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm_ex)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm_ex)]; - } - } - if (nr % 2 != 0 && nc % 2 == 0 && nf % 2 != 0 && R * 2 + 1 == rest_r && - C * 2 + 1 >= rest_c_p && F * 2 + 1 == rest_f) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] += - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm_ex)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm_ex)]; - } - } - if (nr % 2 != 0 && nc % 2 != 0 && nf % 2 == 0 && R * 2 + 1 == rest_r && - C * 2 + 1 == rest_c && F * 2 + 1 >= rest_f_p && TYPE == 1) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] += - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, rest_f_p - 1)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, rest_f_p - 1)]; - } - } - } - } - - __syncthreads(); - - if (r_sm < rest_r && c_sm < rest_c && f_sm < rest_f) { - if (r_gl >= svr && r_gl < svr + nvr && c_gl >= svc && c_gl < svc + nvc && - f_gl >= svf && f_gl < svf + nvf) { - if (!INTERPOLATION && COEFF_RESTORE) { - dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl)] += - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - } else { - dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - } - } - } -} - -template -void gpk_rev_adaptive_launcher( - Handle &handle, SIZE *shape_h, SIZE *shape_d, SIZE *shape_c_d, - SIZE *ldvs, SIZE *ldws, DIM unprocessed_n, DIM *unprocessed_dims, - DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, T *dratio_r, T *dratio_c, - T *dratio_f, T *dv, LENGTH lddv1, LENGTH lddv2, T *dw, LENGTH lddw1, - LENGTH lddw2, T *dwf, LENGTH lddwf1, LENGTH lddwf2, T *dwc, LENGTH lddwc1, - LENGTH lddwc2, T *dwr, LENGTH lddwr1, LENGTH lddwr2, T *dwcf, - LENGTH lddwcf1, LENGTH lddwcf2, T *dwrf, LENGTH lddwrf1, LENGTH lddwrf2, - T *dwrc, LENGTH lddwrc1, LENGTH lddwrc2, T *dwrcf, LENGTH lddwrcf1, - LENGTH lddwrcf2, SIZE svr, SIZE svc, SIZE svf, SIZE nvr, SIZE nvc, SIZE nvf, - int queue_idx) { - - SIZE nr = shape_h[curr_dim_r]; - SIZE nc = shape_h[curr_dim_c]; - SIZE nf = shape_h[curr_dim_f]; - if (D_LOCAL == 2) { - nr = 1; - } - SIZE total_thread_z = std::max(nr - 1, (SIZE)1); - SIZE total_thread_y = std::max(nc - 1, (SIZE)1); - SIZE total_thread_x = std::max(nf - 1, (SIZE)1); - - SIZE tbx, tby, tbz, gridx, gridy, gridz; - dim3 threadsPerBlock, blockPerGrid; - size_t sm_size; - - // tbz = std::min(R, total_thread_z); - // tby = std::min(C, total_thread_y); - // tbx = std::min(F, total_thread_x); - tbz = R; - tby = C; - tbx = F; - sm_size = ((R + 1) * (C + 1) * (F + 1) + R + C + F) * sizeof(T); - sm_size += (D_GLOBAL * 4) * sizeof(SIZE); - sm_size += (D_GLOBAL * 1) * sizeof(DIM); - - gridz = ceil((double)total_thread_z / tbz); - gridy = ceil((double)total_thread_y / tby); - gridx = ceil((double)total_thread_x / tbx); - for (DIM d = 0; d < D_GLOBAL; d++) { - if (D_LOCAL == 3 && d != curr_dim_f && d != curr_dim_c && d != curr_dim_r) { - gridx *= shape_h[d]; - } - if (D_LOCAL == 2 && d != curr_dim_f && d != curr_dim_c) { - gridx *= shape_h[d]; - } - } - - threadsPerBlock = dim3(tbx, tby, tbz); - blockPerGrid = dim3(gridx, gridy, gridz); - - // printf("gpk_rev exec: %d %d %d %d %d %d\n", tbx, tby, tbz, gridx, gridy, - // gridz); - _gpk_rev<<>>( - shape_d, shape_c_d, ldvs, ldws, unprocessed_n, unprocessed_dims, - curr_dim_r, curr_dim_c, curr_dim_f, dratio_r, dratio_c, dratio_f, dv, - lddv1, lddv2, dw, lddw1, lddw2, dwf, lddwf1, lddwf2, dwc, lddwc1, lddwc2, - dwr, lddwr1, lddwr2, dwcf, lddwcf1, lddwcf2, dwrf, lddwrf1, lddwrf2, dwrc, - lddwrc1, lddwrc2, dwrcf, lddwrcf1, lddwrcf2, svr, svc, svf, nvr, nvc, - nvf); - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } -} - -template -void gpk_rev(Handle &handle, SIZE *shape_h, SIZE *shape_d, - SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, DIM unprocessed_n, - DIM *unprocessed_dims, DIM curr_dim_r, DIM curr_dim_c, - DIM curr_dim_f, T *dratio_r, T *dratio_c, T *dratio_f, T *dv, - LENGTH lddv1, LENGTH lddv2, T *dw, LENGTH lddw1, LENGTH lddw2, - T *dwf, LENGTH lddwf1, LENGTH lddwf2, T *dwc, LENGTH lddwc1, - LENGTH lddwc2, T *dwr, LENGTH lddwr1, LENGTH lddwr2, T *dwcf, - LENGTH lddwcf1, LENGTH lddwcf2, T *dwrf, LENGTH lddwrf1, - LENGTH lddwrf2, T *dwrc, LENGTH lddwrc1, LENGTH lddwrc2, T *dwrcf, - LENGTH lddwrcf1, LENGTH lddwrcf2, SIZE svr, SIZE svc, SIZE svf, - SIZE nvr, SIZE nvc, SIZE nvf, int queue_idx, int config) { - -#define GPK(R, C, F) \ - { \ - gpk_rev_adaptive_launcher( \ - handle, shape_h, shape_d, shape_c_d, ldvs, ldws, unprocessed_n, \ - unprocessed_dims, curr_dim_r, curr_dim_c, curr_dim_f, dratio_r, \ - dratio_c, dratio_f, dv, lddv1, lddv2, dw, lddw1, lddw2, dwf, lddwf1, \ - lddwf2, dwc, lddwc1, lddwc2, dwr, lddwr1, lddwr2, dwcf, lddwcf1, \ - lddwcf2, dwrf, lddwrf1, lddwrf2, dwrc, lddwrc1, lddwrc2, dwrcf, \ - lddwrcf1, lddwrcf2, svr, svc, svf, nvr, nvc, nvf, queue_idx); \ - } - bool profile = false; - if (handle.profile_kernels) { - profile = true; - } - if (D_LOCAL == 3) { - // if (profile || config == 6) { - // GPK(2, 2, 128) - // } - // if (profile || config == 5) { - // GPK(2, 2, 64) - // } - // if (profile || config == 4) { - // GPK(4, 4, 32) - // } - // if (profile || config == 3) { - // GPK(4, 4, 16) - // } - // if (profile || config == 2) { - // GPK(4, 4, 8) - // } - // if (profile || config == 1) { - GPK(4, 4, 4) - // } - // if (profile || config == 0) { - // GPK(4, 4, 4) - // } - } else if (D_LOCAL == 2) { - if (profile || config == 6) { - GPK(1, 2, 128) - } - if (profile || config == 5) { - GPK(1, 2, 64) - } - if (profile || config == 4) { - GPK(1, 4, 32) - } - if (profile || config == 3) { - GPK(1, 4, 16) - } - if (profile || config == 2) { - GPK(1, 4, 8) - } - if (profile || config == 1) { - GPK(1, 4, 4) - } - if (profile || config == 0) { - GPK(1, 2, 4) - } - } else if (D_LOCAL == 1) { - if (profile || config == 6) { - GPK(1, 1, 128) - } - if (profile || config == 5) { - GPK(1, 1, 64) - } - if (profile || config == 4) { - GPK(1, 1, 32) - } - if (profile || config == 3) { - GPK(1, 1, 16) - } - if (profile || config == 2) { - GPK(1, 1, 8) - } - if (profile || config == 1) { - GPK(1, 1, 8) - } - if (profile || config == 0) { - GPK(1, 1, 8) - } - } -#undef GPK -} - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/GridProcessingKernel3D.h b/include/cuda/GridProcessingKernel3D.h deleted file mode 100644 index 7bc10b7a16..0000000000 --- a/include/cuda/GridProcessingKernel3D.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_GRID_PROCESSING_KERNEL_3D -#define MGRAD_CUDA_GRID_PROCESSING_KERNEL_3D - -#include "Common.h" - -namespace mgard_cuda { - -template -void gpk_reo_3d(Handle &handle, SIZE nr, SIZE nc, SIZE nf, T *dratio_r, - T *dratio_c, T *dratio_f, T *dv, SIZE lddv1, SIZE lddv2, T *dw, - SIZE lddw1, SIZE lddw2, T *dwf, SIZE lddwf1, SIZE lddwf2, - T *dwc, SIZE lddwc1, SIZE lddwc2, T *dwr, SIZE lddwr1, - SIZE lddwr2, T *dwcf, SIZE lddwcf1, SIZE lddwcf2, T *dwrf, - SIZE lddwrf1, SIZE lddwrf2, T *dwrc, SIZE lddwrc1, SIZE lddwrc2, - T *dwrcf, SIZE lddwrcf1, SIZE lddwrcf2, int queue_idx, - int config); - -template -void gpk_rev_3d(Handle &handle, SIZE nr, SIZE nc, SIZE nf, T *dratio_r, - T *dratio_c, T *dratio_f, T *dv, SIZE lddv1, SIZE lddv2, T *dw, - SIZE lddw1, SIZE lddw2, T *dwf, SIZE lddwf1, SIZE lddwf2, - T *dwc, SIZE lddwc1, SIZE lddwc2, T *dwr, SIZE lddwr1, - SIZE lddwr2, T *dwcf, SIZE lddwcf1, SIZE lddwcf2, T *dwrf, - SIZE lddwrf1, SIZE lddwrf2, T *dwrc, SIZE lddwrc1, SIZE lddwrc2, - T *dwrcf, SIZE lddwrcf1, SIZE lddwrcf2, SIZE svr, SIZE svc, - SIZE svf, SIZE nvr, SIZE nvc, SIZE nvf, int queue_idx, - int config); - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/GridProcessingKernel3D.hpp b/include/cuda/GridProcessingKernel3D.hpp deleted file mode 100644 index 81971a0391..0000000000 --- a/include/cuda/GridProcessingKernel3D.hpp +++ /dev/null @@ -1,2379 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_GRID_PROCESSING_KERNEL_3D_TEMPLATE -#define MGRAD_CUDA_GRID_PROCESSING_KERNEL_3D_TEMPLATE - -#include "CommonInternal.h" -#include "GPKFunctor.h" -#include "GridProcessingKernel3D.h" - -namespace mgard_cuda { - -template -MGARDm_EXEC void -__gpk_reo_3d(IDX ngridz, IDX ngridy, IDX ngridx, IDX nblockz, IDX nblocky, - IDX nblockx, IDX blockz, IDX blocky, IDX blockx, IDX threadz, - IDX thready, IDX threadx, SIZE nr, SIZE nc, SIZE nf, SIZE nr_c, - SIZE nc_c, SIZE nf_c, T *dratio_r, T *dratio_c, T *dratio_f, T *dv, - SIZE lddv1, SIZE lddv2, T *dw, SIZE lddw1, SIZE lddw2, T *dwf, - SIZE lddwf1, SIZE lddwf2, T *dwc, SIZE lddwc1, SIZE lddwc2, T *dwr, - SIZE lddwr1, SIZE lddwr2, T *dwcf, SIZE lddwcf1, SIZE lddwcf2, - T *dwrf, SIZE lddwrf1, SIZE lddwrf2, T *dwrc, SIZE lddwrc1, - SIZE lddwrc2, T *dwrcf, SIZE lddwrcf1, SIZE lddwrcf2) { - - // // to be removed - int TYPE = 1; - bool INTERPOLATION = true; - bool CALC_COEFF = true; - bool in_next = false; - bool skip = false; - - SIZE r, c, f; - SIZE rest_r, rest_c, rest_f; - SIZE nr_p, nc_p, nf_p; - SIZE rest_r_p, rest_c_p, rest_f_p; - SIZE r_sm, c_sm, f_sm; - SIZE r_sm_ex, c_sm_ex, f_sm_ex; - SIZE r_gl, c_gl, f_gl; - SIZE r_gl_ex, c_gl_ex, f_gl_ex; - LENGTH threadId; - - T res; - - // r = blockIdx.z * blockDim.z; - // c = blockIdx.y * blockDim.y; - // f = blockIdx.x * blockDim.x; - - r = blockz * nblockz; - c = blocky * nblocky; - f = blockx * nblockx; - - rest_r = nr - r; - rest_c = nc - c; - rest_f = nf - f; - - nr_p = nr; - nc_p = nc; - nf_p = nf; - - rest_r_p = rest_r; - rest_c_p = rest_c; - rest_f_p = rest_f; - - if (nr % 2 == 0) { - nr_p = nr + 1; - rest_r_p = nr_p - r; - } - if (nc % 2 == 0) { - nc_p = nc + 1; - rest_c_p = nc_p - c; - } - if (nf % 2 == 0) { - nf_p = nf + 1; - rest_f_p = nf_p - f; - } - - // r_sm = threadIdx.z; - // c_sm = threadIdx.y; - // f_sm = threadIdx.x; - - r_sm = threadz; - c_sm = thready; - f_sm = threadx; - - r_sm_ex = R * 2; - c_sm_ex = C * 2; - f_sm_ex = F * 2; - - // threadId = (threadIdx.z * (blockDim.x * blockDim.y)) + - // (threadIdx.y * blockDim.x) + threadIdx.x; - - threadId = (threadz * (nblockx * nblocky)) + (thready * nblockx) + threadx; - - T *sm = SharedMemory(); - SIZE ldsm1 = F * 2 + 1; - SIZE ldsm2 = C * 2 + 1; - T *v_sm = sm; - T *ratio_f_sm = sm + (F * 2 + 1) * (C * 2 + 1) * (R * 2 + 1); - T *ratio_c_sm = ratio_f_sm + F * 2; - T *ratio_r_sm = ratio_c_sm + C * 2; - - r_gl = r + r_sm; - r_gl_ex = r + R * 2; - c_gl = c + c_sm; - c_gl_ex = c + C * 2; - f_gl = f + f_sm; - f_gl_ex = f + F * 2; - - // __syncthreads(); - // if (r_sm == 0 && c_sm == 0 && f_sm == 0) { - // //printf("setting zeros\n"); - // for (int i = 0; i < R * 2 + 1; i++) { - // for (int j = 0; j < C * 2 + 1; j++) { - // for (int k = 0; k < F * 2 + 1; k++) { - // v_sm[get_idx(ldsm1, ldsm2, i, j, k)] = 0.0; - // } - // } - // } - // //printf("done zeros\n"); - // } - // __syncthreads(); - /* Load v */ - // loading extra rules - // case 1: input = odd (non-padding required) - // case 1.a: block size < rest (need to load extra); - // case 1.b: block size > rest (NO need to load extra); - // case 2: input = even (padding requried) - // case 2.a: block size < rest (need to load extra); - // case 2.b: block size >= rest (NO need to load extra, but need - // padding); - - // Load from dv - if (r_sm < rest_r && c_sm < rest_c && f_sm < rest_f) { - - // load cubic - // asm volatile("membar.cta;"); - // start = clock64(); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl)]; - // if (blockIdx.x==0 && blockIdx.y==0&&blockIdx.z==0) { - // printf("load (%d %d %d) %f <- %d+(%d %d %d) (ld: %d %d)\n", - // r_sm, c_sm, f_sm, - // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl)], - // other_offset_v+r_gl, c_gl, f_gl, lddv1, lddv2); - // } - if (r_sm == 0) { - if (rest_r > R * 2) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm)] = - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl)]; - } - } - if (c_sm == 0) { - if (rest_c > C * 2) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm)] = - dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl)]; - } - } - if (f_sm == 0) { - if (rest_f > F * 2) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm_ex)] = - dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl_ex)]; - } - } - if (c_sm == 0 && f_sm == 0) { - if (rest_c > C * 2 && rest_f > F * 2) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm_ex)] = - dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)]; - } - } - if (r_sm == 0 && f_sm == 0) { - if (rest_r > R * 2 && rest_f > F * 2) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm_ex)] = - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)]; - } - } - if (r_sm == 0 && c_sm == 0) { - if (rest_r > R * 2 && rest_c > C * 2) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm)] = - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)]; - } - } - if (r_sm == 0 && c_sm == 0 && f_sm == 0) { - if (rest_r > R * 2 && rest_c > C * 2 && rest_f > F * 2) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm_ex)] = - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)]; - } - } - } - - __syncthreads(); - - // apply padding is necessary - if (r_sm < rest_r && c_sm < rest_c && f_sm < rest_f) { - - // printf("load main[%d %d %d]:%f --> [%d %d %d] (%d %d %d)\n", r_gl, - // c_gl, f_gl, - // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl)], r_sm, c_sm, f_sm, nr, - // nc, nf); - - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[load main] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - - // load extra surface - - if (r_sm == 0) { - if (rest_r > R * 2) { - // v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm)] = - // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl)]; - // printf("load-r[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl, f_gl, - // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl)], r_sm_ex, c_sm, - // f_sm); - } else if (nr % 2 == 0) { - // if (r == 16 && c == 0 && f == 0) { - // printf("padding (%d %d %d) %f <- (%f %f %f)\n", rest_r_p - 1, - // c_sm, f_sm, - // v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm)], rest_r - // - 1, c_sm, f_sm); - // padded = true; - // aa = v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm)]; - // bb = v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm)]; - // } - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm)]; - } - } - - if (c_sm == 0) { - if (rest_c > C * 2) { - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm)] = - // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl)]; - // printf("load-c[%d %d %d]:%f --> [%d %d %d]\n", r_gl, c_gl_ex, f_gl, - // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl)], r_sm, c_sm_ex, - // f_sm); - } else if (nc % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, f_sm)]; - } - } - - if (f_sm == 0) { - if (rest_f > F * 2) { - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm_ex)] = - // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl_ex)]; - // printf("load-f[%d %d %d]:%f --> [%d %d %d]\n", r_gl, c_gl, f_gl_ex, - // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl_ex)], r_sm, c_sm, - // f_sm_ex); - } else if (nf % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f - 1)]; - } - } - - // load extra edges - if (c_sm == 0 && f_sm == 0) { - if (rest_c > C * 2 && rest_f > F * 2) { - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm_ex)] = - // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)]; - // printf("load-cf[%d %d %d]:%f --> [%d %d %d]\n", r_gl, c_gl_ex, - // f_gl_ex, dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)], r_sm, - // c_sm_ex, f_sm_ex); - } else if (rest_c <= C * 2 && rest_f <= F * 2 && nc % 2 == 0 && - nf % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, rest_f - 1)]; - } else if (rest_c > C * 2 && rest_f <= F * 2 && nf % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, rest_f - 1)]; - } else if (rest_c <= C * 2 && rest_f > F * 2 && nc % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, f_sm_ex)]; - } - } - - if (r_sm == 0 && f_sm == 0) { - if (rest_r > R * 2 && rest_f > F * 2) { - // v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm_ex)] = - // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)]; - // printf("load-rf[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl, - // f_gl_ex, dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)], - // r_sm_ex, c_sm, f_sm_ex); - } else if (rest_r <= R * 2 && rest_f <= F * 2 && nr % 2 == 0 && - nf % 2 == 0) { - // printf("padding (%d %d %d) <- (%d %d %d)\n", rest_r_p - 1, c_sm, - // rest_f_p - 1, rest_r - 1, c_sm, rest_f - 1); - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, rest_f - 1)]; - } else if (rest_r > R * 2 && rest_f <= F * 2 && nf % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, rest_f - 1)]; - } else if (rest_r <= R * 2 && rest_f > F * 2 && nr % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm_ex)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm_ex)]; - } - } - - if (r_sm == 0 && c_sm == 0) { - if (rest_r > R * 2 && rest_c > C * 2) { - // v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm)] = - // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)]; - // printf("load-rc[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl_ex, - // f_gl, dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)], r_sm_ex, - // c_sm_ex, f_sm); - } else if (rest_r <= R * 2 && rest_c <= C * 2 && nr % 2 == 0 && - nc % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm)]; - // printf("padding (%d %d %d) <- (%d %d %d): %f\n", rest_r_p - 1, - // rest_c_p - 1, f_sm, rest_r - 1, rest_c - 1, f_sm, - // v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm)]); - } else if (rest_r > R * 2 && rest_c <= C * 2 && nc % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c - 1, f_sm)]; - } else if (rest_r <= R * 2 && rest_c > C * 2 && nr % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm_ex, f_sm)]; - } - } - // load extra vertex - - if (r_sm == 0 && c_sm == 0 && f_sm == 0) { - if (rest_r > R * 2 && rest_c > C * 2 && rest_f > F * 2) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm_ex)] = - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)]; - // printf("load-rcf[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl_ex, - // f_gl_ex, dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)], - // r_sm_ex, c_sm_ex, f_sm_ex); - } else if (rest_r <= R * 2 && rest_c <= C * 2 && rest_f <= F * 2 && - nr % 2 == 0 && nc % 2 == 0 && nf % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, rest_f - 1)]; - } else if (rest_r > R * 2 && rest_c > C * 2 && rest_f <= F * 2 && - nf % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, rest_f - 1)]; - } else if (rest_r > R * 2 && rest_c <= C * 2 && rest_f > F * 2 && - nc % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c - 1, f_sm_ex)]; - } else if (rest_r > R * 2 && rest_c <= C * 2 && rest_f <= F * 2 && - nc % 2 == 0 && nf % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c - 1, rest_f - 1)]; - } else if (rest_r <= R * 2 && rest_c > C * 2 && rest_f > F * 2 && - nr % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm_ex)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm_ex, f_sm_ex)]; - } else if (rest_r <= R * 2 && rest_c > C * 2 && rest_f <= F * 2 && - nr % 2 == 0 && nf % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, rest_f_p - 1)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm_ex, rest_f - 1)]; - } else if (rest_r <= R * 2 && rest_c <= C * 2 && rest_f > F * 2 && - nr % 2 == 0 && nc % 2 == 0) { - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm_ex)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm_ex)]; - } - } - - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[load extra] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - - // load dist - if (c_sm == 0 && f_sm == 0 && r_sm < rest_r_p - 2) { - // printf("%d/%d load %f\n", r_sm, rest_r - 2, dratio_r[r + r_sm]); - ratio_r_sm[r_sm] = dratio_r[r + r_sm]; - // if (nr % 2 == 0 && R * 2 + 1 >= rest_r_p && r_sm == 0) { - // ratio_r_sm[rest_r_p - 3] = 0.5; - // } - } - if (r_sm == 0 && f_sm == 0 && c_sm < rest_c_p - 2) { - ratio_c_sm[c_sm] = dratio_c[c + c_sm]; - // if (nc % 2 == 0 && C * 2 + 1 >= rest_c_p && c_sm == 0) { - // ratio_c_sm[rest_c_p - 3] = 0.5; - // } - } - if (c_sm == 0 && r_sm == 0 && f_sm < rest_f_p - 2) { - ratio_f_sm[f_sm] = dratio_f[f + f_sm]; - // if (nf % 2 == 0 && F * 2 + 1 >= rest_f_p && f_sm == 0) { - // ratio_f_sm[rest_f_p - 3] = 0.5; - // } - } - - // if (r == 0 && c == 0 && f == 0 && r_sm == 0 && c_sm == 0 && f_sm == 0) - // { - // printf("ratio:"); - // for (int i = 0; i < R * 2 + 1; i++) { - // printf("%2.2f ", ratio_r_sm[i]); - // } - // printf("\n"); - // } - - } // restrict boundary - - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[load ratio] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - - // __syncthreads(); - // // debug print - // if (debug) { - // printf("in config: %d %d %d (%d %d %d)\n", R, C, F, r,c,f); - // printf("rest_p: %d %d %d\n", rest_r_p, rest_c_p, rest_f_p); - // bool print = false; - // for (int i = 0; i < R * 2 + 1; i++) { - // for (int j = 0; j < C * 2 + 1; j++) { - // for (int k = 0; k < F * 2 + 1; k++) { - // // if (abs(v_sm[get_idx(ldsm1, ldsm2, i, j, k)]) > 10000) { - // // print = true; - // // printf("(block %d %d %d) %2.2f \n", r,c,f, - // v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); - // // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); - // // } - // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); - // } - // printf("\n"); - // } - // printf("\n"); - // } - // } - __syncthreads(); - - if (dw && threadId < R * C * F) { - r_sm = (threadId / (C * F)) * 2; - c_sm = ((threadId % (C * F)) / F) * 2; - f_sm = ((threadId % (C * F)) % F) * 2; - r_gl = r / 2 + threadId / (C * F); - c_gl = c / 2 + threadId % (C * F) / F; - f_gl = f / 2 + threadId % (C * F) % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf_c) { - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } - - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[store coarse] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - int base = 0; - // printf("TYPE =%d \n", TYPE); - // printf("%d == %d && %llu >= %d && %llu < %d\n", r + R * 2, nr_p - 1, - // threadId, base, threadId, base + C * F); - - if (dw && r + R * 2 == nr_p - 1 && threadId >= base && - threadId < base + C * F) { - r_sm = R * 2; - c_sm = ((threadId - base) / F) * 2; - f_sm = ((threadId - base) % F) * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + (threadId - base) / F; - f_gl = f / 2 + (threadId - base) % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf_c) { - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } - - base += C * F; // ROUND_UP_WARP(C * F) * WARP_SIZE; - if (dw && c + C * 2 == nc_p - 1 && threadId >= base && - threadId < base + R * F) { - r_sm = ((threadId - base) / F) * 2; - c_sm = C * 2; - f_sm = ((threadId - base) % F) * 2; - r_gl = r / 2 + (threadId - base) / F; - c_gl = c / 2 + C; - f_gl = f / 2 + (threadId - base) % F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf_c) { - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - // printf("(%d %d %d) (%d %d %d) %f\n", - // r_sm, c_sm, f_sm, r_gl, c_gl, f_gl, dwork[get_idx(lddv1, lddv2, - // r_gl, c_gl, f_gl)]); - } - - base += R * F; // ROUND_UP_WARP(R * F) * WARP_SIZE; - // printf("%d %d\n", base, threadId); - if (dw && f + F * 2 == nf_p - 1 && threadId >= base && - threadId < base + R * C) { - r_sm = ((threadId - base) / C) * 2; - c_sm = ((threadId - base) % C) * 2; - f_sm = F * 2; - r_gl = r / 2 + (threadId - base) / C; - c_gl = c / 2 + (threadId - base) % C; - f_gl = f / 2 + F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf_c) { - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } - - base += R * C; // ROUND_UP_WARP(R * C) * WARP_SIZE; - // load extra edges - if (dw && c + C * 2 == nc_p - 1 && f + F * 2 == nf_p - 1 && - threadId >= base && threadId < base + R) { - r_sm = (threadId - base) * 2; - c_sm = C * 2; - f_sm = F * 2; - r_gl = r / 2 + threadId - base; - c_gl = c / 2 + C; - f_gl = f / 2 + F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf_c) { - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } - - base += R; // ROUND_UP_WARP(R) * WARP_SIZE; - // if (TYPE == 2) printf("%d %d, %d, %llu, %d\n",dw == NULL, f + F * 2, nf_p - // - 1, threadId, C); - if (dw && r + R * 2 == nr_p - 1 && f + F * 2 == nf_p - 1 && - threadId >= base && threadId < base + C) { - r_sm = R * 2; - c_sm = (threadId - base) * 2; - f_sm = F * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId - base; - f_gl = f / 2 + F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf_c) { - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - // printf("store[%d %d %d]: %f\n", r_sm, c_sm, f_sm, v_sm[get_idx(ldsm1, - // ldsm2, r_sm, c_sm, f_sm)]); - } - - base += C; // ROUND_UP_WARP(C) * WARP_SIZE; - if (dw && r + R * 2 == nr_p - 1 && c + C * 2 == nc_p - 1 && - threadId >= base && threadId < base + F) { - r_sm = R * 2; - c_sm = C * 2; - f_sm = (threadId - base) * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + C; - f_gl = f / 2 + threadId - base; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf_c) { - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } - base += F; // ROUND_UP_WARP(F) * WARP_SIZE; - // // load extra vertex - if (dw && r + R * 2 == nr_p - 1 && c + C * 2 == nc_p - 1 && - f + F * 2 == nf_p - 1 && threadId >= base && threadId < base + 1) { - r_sm = R * 2; - c_sm = C * 2; - f_sm = F * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + C; - f_gl = f / 2 + F; - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf_c) { - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = res; - // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, - // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], - // r_sm, c_sm, f_sm); - } - } - - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[store extra] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - - // start = clock64(); - - if (dwf && threadId >= R * C * F && threadId < R * C * F * 2) { - r_sm = ((threadId - R * C * F) / (C * F)) * 2; - c_sm = (((threadId - R * C * F) % (C * F)) / F) * 2; - f_sm = (((threadId - R * C * F) % (C * F)) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F) % (C * F)) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf - nf_c) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)] = res; - } - - // if (nr == 70) - // printf("f-store: (%d %d %d) <- %f (%d %d %d)\n", r_gl, - // c_gl, f_gl, v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], r_sm, c_sm, - // f_sm); - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[F-store] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - } - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[F-store] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - - // if (r_sm % 2 == 0 && c_sm % 2 != 0 && f_sm % 2 == 0) { - - if (dwc && threadId >= R * C * F * 2 && threadId < R * C * F * 3) { - r_sm = ((threadId - R * C * F * 2) / (C * F)) * 2; - c_sm = (((threadId - R * C * F * 2) % (C * F)) / F) * 2 + 1; - f_sm = (((threadId - R * C * F * 2) % (C * F)) % F) * 2; - r_gl = r / 2 + (threadId - R * C * F * 2) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 2) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 2) % (C * F)) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc - nc_c && f_gl < nf_c) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)] = res; - } - } - - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[C-store] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - - // if (r_sm % 2 != 0 && c_sm % 2 == 0 && f_sm % 2 == 0) { - if (dwr && threadId >= R * C * F * 3 && threadId < R * C * F * 4) { - r_sm = ((threadId - R * C * F * 3) / (C * F)) * 2 + 1; - c_sm = (((threadId - R * C * F * 3) % (C * F)) / F) * 2; - f_sm = (((threadId - R * C * F * 3) % (C * F)) % F) * 2; - r_gl = r / 2 + (threadId - R * C * F * 3) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 3) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 3) % (C * F)) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)] = res; - } - } - - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[R-store] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - __syncthreads(); - if (dwcf && threadId >= R * C * F * 4 && threadId < R * C * F * 5) { - r_sm = ((threadId - R * C * F * 4) / (C * F)) * 2; - c_sm = (((threadId - R * C * F * 4) % (C * F)) / F) * 2 + 1; - f_sm = (((threadId - R * C * F * 4) % (C * F)) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F * 4) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 4) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 4) % (C * F)) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc - nc_c && f_gl < nf - nf_c) { - T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res = lerp(f1, f2, ratio_c_sm[c_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwcf[get_idx(lddwcf1, lddwcf2, r_gl, c_gl, f_gl)] = res; - } - } - - // asm volatile("membar.cta;"); - // start = clock64() - start; - // printf("[CF-store] block id %d,%d,%d elapsed %lu\n", blockIdx.z, - // blockIdx.y, blockIdx.x, start); start = clock64(); - - if (dwrf && threadId >= R * C * F * 5 && threadId < R * C * F * 6) { - r_sm = ((threadId - R * C * F * 5) / (C * F)) * 2 + 1; - c_sm = (((threadId - R * C * F * 5) % (C * F)) / F) * 2; - f_sm = (((threadId - R * C * F * 5) % (C * F)) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F * 5) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 5) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 5) % (C * F)) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res = lerp(f1, f2, ratio_r_sm[r_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwrf[get_idx(lddwrf1, lddwrf2, r_gl, c_gl, f_gl)] = res; - } - } - - if (dwrc && threadId >= R * C * F * 6 && threadId < R * C * F * 7) { - r_sm = ((threadId - R * C * F * 6) / (C * F)) * 2 + 1; - c_sm = (((threadId - R * C * F * 6) % (C * F)) / F) * 2 + 1; - f_sm = (((threadId - R * C * F * 6) % (C * F)) % F) * 2; - r_gl = r / 2 + (threadId - R * C * F * 6) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 6) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 6) % (C * F)) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwrc[get_idx(lddwrc1, lddwrc2, r_gl, c_gl, f_gl)] = res; - } - } - - if (dwrcf && threadId >= R * C * F * 7 && threadId < R * C * F * 8) { - r_sm = ((threadId - R * C * F * 7) / (C * F)) * 2 + 1; - c_sm = (((threadId - R * C * F * 7) % (C * F)) / F) * 2 + 1; - f_sm = (((threadId - R * C * F * 7) % (C * F)) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F * 7) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 7) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 7) % (C * F)) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { - T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f3 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f4 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - - T fc1 = lerp(f1, f2, ratio_c_sm[c_sm - 1]); - T fc2 = lerp(f3, f4, ratio_c_sm[c_sm - 1]); - - res = lerp(fc1, fc2, ratio_r_sm[r_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwrcf[get_idx(lddwrcf1, lddwrcf2, r_gl, c_gl, f_gl)] = res; - } - } - // end = clock64(); - - // asm volatile("membar.cta;"); - // if (threadId < 256 && blockIdx.z == 0 && blockIdx.y == 0 && blockIdx.x == - // 0) printf("threadId %d elapsed %lu\n", threadId, end-start); - if (r + R * 2 == nr_p - 1) { - // printf("test\n"); - if (threadId < C * F) { - // printf("test1\n"); - if (dwf) { - // printf("test2\n"); - r_sm = R * 2; - c_sm = (threadId / F) * 2; - f_sm = (threadId % F) * 2 + 1; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId / F; - f_gl = f / 2 + threadId % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - // printf("dwf (%d %d %d): %f<-(%f %f %f)\n", r_gl, c_gl, f_gl, res, - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - // ratio_f_sm[f_sm - 1]); - dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)] = res; - } - } - - if (dwc) { - r_sm = R * 2; - c_sm = (threadId / F) * 2 + 1; - f_sm = (threadId % F) * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId / F; - f_gl = f / 2 + threadId % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)] = res; - } - } - - if (dwcf) { - r_sm = R * 2; - c_sm = (threadId / F) * 2 + 1; - f_sm = (threadId % F) * 2 + 1; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId / F; - f_gl = f / 2 + threadId % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { - T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res = lerp(f1, f2, ratio_c_sm[c_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwcf[get_idx(lddwcf1, lddwcf2, r_gl, c_gl, f_gl)] = res; - } - } - } - } - - if (c + C * 2 == nc_p - 1) { - if (threadId >= R * C * F && threadId < R * C * F + R * F) { - if (dwf) { - r_sm = ((threadId - R * C * F) / F) * 2; - c_sm = C * 2; - f_sm = ((threadId - R * C * F) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F) / F; - c_gl = c / 2 + C; - f_gl = f / 2 + (threadId - R * C * F) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)] = res; - } - } - - if (dwr) { - r_sm = ((threadId - R * C * F) / F) * 2 + 1; - c_sm = C * 2; - f_sm = ((threadId - R * C * F) % F) * 2; - r_gl = r / 2 + (threadId - R * C * F) / F; - c_gl = c / 2 + C; - f_gl = f / 2 + (threadId - R * C * F) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)] = res; - } - } - - if (dwrf) { - r_sm = ((threadId - R * C * F) / F) * 2 + 1; - c_sm = C * 2; - f_sm = ((threadId - R * C * F) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F) / F; - c_gl = c / 2 + C; - f_gl = f / 2 + (threadId - R * C * F) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res = lerp(f1, f2, ratio_r_sm[r_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwrf[get_idx(lddwrf1, lddwrf2, r_gl, c_gl, f_gl)] = res; - } - } - } - } - - if (f + F * 2 == nf_p - 1) { - if (threadId >= R * C * F * 2 && threadId < R * C * F * 2 + R * C) { - if (dwc) { - r_sm = ((threadId - R * C * F * 2) / C) * 2; - c_sm = ((threadId - R * C * F * 2) % C) * 2 + 1; - f_sm = F * 2; - r_gl = r / 2 + (threadId - R * C * F * 2) / C; - c_gl = c / 2 + (threadId - R * C * F * 2) % C; - f_gl = f / 2 + F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)] = res; - } - } - - if (dwr) { - r_sm = ((threadId - R * C * F * 2) / C) * 2 + 1; - c_sm = ((threadId - R * C * F * 2) % C) * 2; - f_sm = F * 2; - r_gl = r / 2 + (threadId - R * C * F * 2) / C; - c_gl = c / 2 + (threadId - R * C * F * 2) % C; - f_gl = f / 2 + F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)] = res; - } - } - - if (dwrc) { - r_sm = ((threadId - R * C * F * 2) / C) * 2 + 1; - c_sm = ((threadId - R * C * F * 2) % C) * 2 + 1; - f_sm = F * 2; - r_gl = r / 2 + (threadId - R * C * F * 2) / C; - c_gl = c / 2 + (threadId - R * C * F * 2) % C; - f_gl = f / 2 + F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwrc[get_idx(lddwrc1, lddwrc2, r_gl, c_gl, f_gl)] = res; - } - } - } - } - - if (dwr && c + C * 2 == nc_p - 1 && f + F * 2 == nf_p - 1) { - if (threadId >= R * C * F * 3 && threadId < R * C * F * 3 + R) { - r_sm = (threadId - R * C * F * 3) * 2 + 1; - c_sm = C * 2; - f_sm = F * 2; - r_gl = r / 2 + threadId - R * C * F * 3; - c_gl = c / 2 + C; - f_gl = f / 2 + F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)] = res; - } - } - } - - if (dwc && r + R * 2 == nr_p - 1 && f + F * 2 == nf_p - 1) { - if (threadId >= R * C * F * 4 && threadId < R * C * F * 4 + C) { - r_sm = R * 2; - c_sm = (threadId - R * C * F * 4) * 2 + 1; - f_sm = F * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId - R * C * F * 4; - f_gl = f / 2 + F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)] = res; - } - } - } - - if (dwf && r + R * 2 == nr_p - 1 && c + C * 2 == nc_p - 1) { - if (threadId >= R * C * F * 5 && threadId < R * C * F * 5 + F) { - r_sm = R * 2; - c_sm = C * 2; - f_sm = (threadId - R * C * F * 5) * 2 + 1; - r_gl = r / 2 + R; - c_gl = c / 2 + C; - f_gl = f / 2 + threadId - R * C * F * 5; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; - dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)] = res; - // printf("dwf(%d %d %d): %f\n", r_gl, c_gl, f_gl, - // dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]); - } - } - } - - // if (r == 0 && c == 0 && f == 0 && threadId == 0) { - // printf("out config: %d %d %d (%d %d %d)\n", R, C, F, r,c,f); - // for (int i = 0; i < R * 2 + 1; i++) { - // for (int j = 0; j < C * 2 + 1; j++) { - // for (int k = 0; k < F * 2 + 1; k++) { - // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); - // } - // printf("\n"); - // } - // printf("\n"); - // } - // } -} - -template -MGARDm_KERL void -_gpk_reo_3d(SIZE nr, SIZE nc, SIZE nf, SIZE nr_c, SIZE nc_c, SIZE nf_c, - T *dratio_r, T *dratio_c, T *dratio_f, T *dv, SIZE lddv1, - SIZE lddv2, T *dw, SIZE lddw1, SIZE lddw2, T *dwf, SIZE lddwf1, - SIZE lddwf2, T *dwc, SIZE lddwc1, SIZE lddwc2, T *dwr, SIZE lddwr1, - SIZE lddwr2, T *dwcf, SIZE lddwcf1, SIZE lddwcf2, T *dwrf, - SIZE lddwrf1, SIZE lddwrf2, T *dwrc, SIZE lddwrc1, SIZE lddwrc2, - T *dwrcf, SIZE lddwrcf1, SIZE lddwrcf2) { - - __gpk_reo_3d( - gridDim.z, gridDim.y, gridDim.x, blockDim.z, blockDim.y, blockDim.x, - blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, - nr, nc, nf, nr_c, nc_c, nf_c, dratio_r, dratio_c, dratio_f, dv, lddv1, - lddv2, dw, lddw1, lddw2, dwf, lddwf1, lddwf2, dwc, lddwc1, lddwc2, dwr, - lddwr1, lddwr2, dwcf, lddwcf1, lddwcf2, dwrf, lddwrf1, lddwrf2, dwrc, - lddwrc1, lddwrc2, dwrcf, lddwrcf1, lddwrcf2); -} - -template -void gpk_reo_3d_adaptive_launcher( - Handle &handle, SIZE nr, SIZE nc, SIZE nf, T *dratio_r, T *dratio_c, - T *dratio_f, T *dv, SIZE lddv1, SIZE lddv2, T *dw, SIZE lddw1, SIZE lddw2, - T *dwf, SIZE lddwf1, SIZE lddwf2, T *dwc, SIZE lddwc1, SIZE lddwc2, T *dwr, - SIZE lddwr1, SIZE lddwr2, T *dwcf, SIZE lddwcf1, SIZE lddwcf2, T *dwrf, - SIZE lddwrf1, SIZE lddwrf2, T *dwrc, SIZE lddwrc1, SIZE lddwrc2, T *dwrcf, - SIZE lddwrcf1, SIZE lddwrcf2, int queue_idx) { - - SIZE nr_c = nr / 2 + 1; - SIZE nc_c = nc / 2 + 1; - SIZE nf_c = nf / 2 + 1; - SIZE total_thread_z = std::max(nr - 1, (SIZE)1); - SIZE total_thread_y = std::max(nc - 1, (SIZE)1); - SIZE total_thread_x = std::max(nf - 1, (SIZE)1); - - SIZE tbx, tby, tbz, gridx, gridy, gridz; - dim3 threadsPerBlock, blockPerGrid; - size_t sm_size; - // const int R = 4; - // const int C = 4; - // const int F = 16; - // tbz = std::min(R, total_thread_z); - // tby = std::min(C, total_thread_y); - // tbx = std::min(F, total_thread_x); - tbz = R; - tby = C; - tbx = F; - sm_size = ((R + 1) * (C + 1) * (F + 1) + R + C + F) * sizeof(T); - gridz = ceil((double)total_thread_z / tbz); - gridy = ceil((double)total_thread_y / tby); - gridx = ceil((double)total_thread_x / tbx); - threadsPerBlock = dim3(tbx, tby, tbz); - blockPerGrid = dim3(gridx, gridy, gridz); - // printf("exec config (%d %d %d) (%d %d %d)\n", tbx, tby, tbz, gridx, gridy, - // gridz); - _gpk_reo_3d - <<>>( - nr, nc, nf, nr_c, nc_c, nf_c, dratio_r, dratio_c, dratio_f, dv, lddv1, - lddv2, dw, lddw1, lddw2, dwf, lddwf1, lddwf2, dwc, lddwc1, lddwc2, - dwr, lddwr1, lddwr2, dwcf, lddwcf1, lddwcf2, dwrf, lddwrf1, lddwrf2, - dwrc, lddwrc1, lddwrc2, dwrcf, lddwrcf1, lddwrcf2); - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } -} - -template -void gpk_reo_3d(Handle &handle, SIZE nr, SIZE nc, SIZE nf, T *dratio_r, - T *dratio_c, T *dratio_f, T *dv, SIZE lddv1, SIZE lddv2, T *dw, - SIZE lddw1, SIZE lddw2, T *dwf, SIZE lddwf1, SIZE lddwf2, - T *dwc, SIZE lddwc1, SIZE lddwc2, T *dwr, SIZE lddwr1, - SIZE lddwr2, T *dwcf, SIZE lddwcf1, SIZE lddwcf2, T *dwrf, - SIZE lddwrf1, SIZE lddwrf2, T *dwrc, SIZE lddwrc1, SIZE lddwrc2, - T *dwrcf, SIZE lddwrcf1, SIZE lddwrcf2, int queue_idx, - int config) { - -#define GPK(R, C, F) \ - { \ - gpk_reo_3d_adaptive_launcher( \ - handle, nr, nc, nf, dratio_r, dratio_c, dratio_f, dv, lddv1, lddv2, \ - dw, lddw1, lddw2, dwf, lddwf1, lddwf2, dwc, lddwc1, lddwc2, dwr, \ - lddwr1, lddwr2, dwcf, lddwcf1, lddwcf2, dwrf, lddwrf1, lddwrf2, dwrc, \ - lddwrc1, lddwrc2, dwrcf, lddwrcf1, lddwrcf2, queue_idx); \ - } - bool profile = false; - if (handle.profile_kernels) { - profile = true; - } - if (D == 3) { - if (profile || config == 6) { - GPK(2, 2, 128) - } - if (profile || config == 5) { - GPK(2, 2, 64) - } - if (profile || config == 4) { - GPK(4, 4, 32) - } - if (profile || config == 3) { - GPK(4, 4, 16) - } - if (profile || config == 2) { - GPK(4, 4, 8) - } - if (profile || config == 1) { - GPK(4, 4, 4) - } - if (profile || config == 0) { - GPK(2, 2, 2) - } - // PI_QL(T, 4, 4, 4) - } else if (D == 2) { - if (profile || config == 6) { - GPK(1, 2, 128) - } - if (profile || config == 5) { - GPK(1, 2, 64) - } - if (profile || config == 4) { - GPK(1, 4, 32) - } - if (profile || config == 3) { - GPK(1, 4, 16) - } - if (profile || config == 2) { - GPK(1, 4, 8) - } - if (profile || config == 1) { - GPK(1, 4, 4) - } - if (profile || config == 0) { - GPK(1, 2, 4) - } - // PI_QL(T, 1, 4, 4) - } else if (D == 1) { - if (profile || config == 6) { - GPK(1, 1, 128) - } - if (profile || config == 5) { - GPK(1, 1, 64) - } - if (profile || config == 4) { - GPK(1, 1, 32) - } - if (profile || config == 3) { - GPK(1, 1, 16) - } - if (profile || config == 2) { - GPK(1, 1, 8) - } - if (profile || config == 1) { - GPK(1, 1, 8) - } - if (profile || config == 0) { - GPK(1, 1, 8) - } - } -#undef GPK -} - -template -__global__ void -_gpk_rev_3d(SIZE nr, SIZE nc, SIZE nf, SIZE nr_c, SIZE nc_c, SIZE nf_c, - T *dratio_r, T *dratio_c, T *dratio_f, T *dv, SIZE lddv1, - SIZE lddv2, T *dw, SIZE lddw1, SIZE lddw2, T *dwf, SIZE lddwf1, - SIZE lddwf2, T *dwc, SIZE lddwc1, SIZE lddwc2, T *dwr, SIZE lddwr1, - SIZE lddwr2, T *dwcf, SIZE lddwcf1, SIZE lddwcf2, T *dwrf, - SIZE lddwrf1, SIZE lddwrf2, T *dwrc, SIZE lddwrc1, SIZE lddwrc2, - T *dwrcf, SIZE lddwrcf1, SIZE lddwrcf2, SIZE svr, SIZE svc, - SIZE svf, SIZE nvr, SIZE nvc, SIZE nvf) { - - // to be removed - // int TYPE = 1; - // bool INTERPOLATION = true; - // bool COEFF_RESTORE = true; - // int in_next = false; - // int skip = false; - - SIZE r = blockIdx.z * blockDim.z; - SIZE c = blockIdx.y * blockDim.y; - SIZE f = blockIdx.x * blockDim.x; - - SIZE r_sm = threadIdx.z; - SIZE c_sm = threadIdx.y; - SIZE f_sm = threadIdx.x; - - SIZE r_sm_ex = R * 2; - SIZE c_sm_ex = C * 2; - SIZE f_sm_ex = F * 2; - - SIZE r_gl; - SIZE c_gl; - SIZE f_gl; - - SIZE r_gl_ex; - SIZE c_gl_ex; - SIZE f_gl_ex; - - T res; - - LENGTH threadId; - - T *sm = SharedMemory(); - - // extern __shared__ double sm[]; // size: (blockDim.x + 1) * (blockDim.y + 1) - // * (blockDim.z + 1) - SIZE ldsm1 = F * 2 + 1; - SIZE ldsm2 = C * 2 + 1; - T *v_sm = sm; - T *ratio_f_sm = sm + (F * 2 + 1) * (C * 2 + 1) * (R * 2 + 1); - T *ratio_c_sm = ratio_f_sm + F * 2; - T *ratio_r_sm = ratio_c_sm + C * 2; - - SIZE rest_r = nr - r; - SIZE rest_c = nc - c; - SIZE rest_f = nf - f; - - SIZE nr_p = nr; - SIZE nc_p = nc; - SIZE nf_p = nf; - - SIZE rest_r_p; - SIZE rest_c_p; - SIZE rest_f_p; - - rest_r_p = rest_r; - rest_c_p = rest_c; - rest_f_p = rest_f; - - threadId = (threadIdx.z * (blockDim.x * blockDim.y)) + - (threadIdx.y * blockDim.x) + threadIdx.x; - - // extern __shared__ __align__(sizeof(T)) unsigned char smem[]; - // T * sm = reinterpret_cast(smem); - - // // load dist - // if (c_sm == 0 && f_sm == 0 && r + r_sm < nr) { - // ratio_r_sm[r_sm] = dratio_r[r + r_sm]; - // } - // if (r_sm == 0 && f_sm == 0 && c + c_sm < nc) { - // ratio_c_sm[c_sm] = dratio_c[c + c_sm]; - // } - // if (c_sm == 0 && r_sm == 0 && f + f_sm < nf) { - // ratio_f_sm[f_sm] = dratio_f[f + f_sm]; - // } - - if (nr % 2 == 0) { - nr_p = nr + 1; - rest_r_p = nr_p - r; - } - if (nc % 2 == 0) { - nc_p = nc + 1; - rest_c_p = nc_p - c; - } - if (nf % 2 == 0) { - nf_p = nf + 1; - rest_f_p = nf_p - f; - } - - // load dist - if (c_sm == 0 && f_sm == 0 && r_sm < rest_r - 2) { - ratio_r_sm[r_sm] = dratio_r[r + r_sm]; - if (nr % 2 == 0 && R * 2 + 1 >= rest_r_p && r_sm == 0) { - ratio_r_sm[rest_r_p - 3] = 0.5; - } - } - if (r_sm == 0 && f_sm == 0 && c_sm < rest_c - 2) { - ratio_c_sm[c_sm] = dratio_c[c + c_sm]; - if (nc % 2 == 0 && C * 2 + 1 >= rest_c_p && c_sm == 0) { - ratio_c_sm[rest_c_p - 3] = 0.5; - } - } - if (c_sm == 0 && r_sm == 0 && f_sm < rest_f - 2) { - ratio_f_sm[f_sm] = dratio_f[f + f_sm]; - if (nf % 2 == 0 && F * 2 + 1 >= rest_f_p && f_sm == 0) { - ratio_f_sm[rest_f_p - 3] = 0.5; - } - } - - // if (threadIdx.z == 0 && threadIdx.y == 0 && threadIdx.x == 0) { - // for (int i = 0; i < R * 2 + 1; i++) { - // for (int j = 0; j < C * 2 + 1; j++) { - // for (int k = 0; k < F * 2 + 1; k++) { - // v_sm[get_idx(ldsm1, ldsm2, i, j, k)] = 71177117; - // } - // } - // } - // } - - __syncthreads(); - - if (dw && threadId < R * C * F) { - r_sm = (threadId / (C * F)) * 2; - c_sm = ((threadId % (C * F)) / F) * 2; - f_sm = ((threadId % (C * F)) % F) * 2; - r_gl = r / 2 + threadId / (C * F); - c_gl = c / 2 + threadId % (C * F) / F; - f_gl = f / 2 + threadId % (C * F) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf_c) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - // if (c_gl == nc_c - 1 && f_gl == nf_c-1) - // printf("block: (%d %d %d) thread: (%d %d %d) load0 (%d %d %d): %f - // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, - // threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], - // r_gl, c_gl, f_gl); - } - } - - int base = 0; - if (dw && threadId >= base && threadId < base + C * F) { - r_sm = R * 2; - c_sm = ((threadId - base) / F) * 2; - f_sm = ((threadId - base) % F) * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + (threadId - base) / F; - f_gl = f / 2 + (threadId - base) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf_c) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - // if (c_gl == nc_c - 1 && f_gl == nf_c-1) - // printf("block: (%d %d %d) thread: (%d %d %d) load1 (%d %d %d): %f - // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, - // threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], - // r_gl, c_gl, f_gl); - } - } - base += C * F; // ROUND_UP_WARP(C * F) * WARP_SIZE; - if (dw && threadId >= base && threadId < base + R * F) { - r_sm = ((threadId - base) / F) * 2; - c_sm = C * 2; - f_sm = ((threadId - base) % F) * 2; - r_gl = r / 2 + (threadId - base) / F; - c_gl = c / 2 + C; - f_gl = f / 2 + (threadId - base) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf_c) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - // if (c_gl == nc_c - 1 && f_gl == nf_c-1) - // printf("block: (%d %d %d) thread: (%d %d %d) load2 (%d %d %d): %f - // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, - // threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], - // r_gl, c_gl, f_gl); - } - } - base += R * F; // ROUND_UP_WARP(R * F) * WARP_SIZE; - if (dw && threadId >= base && threadId < base + R * C) { - r_sm = ((threadId - base) / C) * 2; - c_sm = ((threadId - base) % C) * 2; - f_sm = F * 2; - r_gl = r / 2 + (threadId - base) / C; - c_gl = c / 2 + (threadId - base) % C; - f_gl = f / 2 + F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf_c) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - // if (c_gl == nc_c - 1 && f_gl == nf_c-1) - // printf("block: (%d %d %d) thread: (%d %d %d) load3 (%d %d %d): %f - // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, - // threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], - // r_gl, c_gl, f_gl); - } - } - base += R * C; // ROUND_UP_WARP(R * C) * WARP_SIZE; - // load extra edges - if (dw && threadId >= base && threadId < base + R) { - r_sm = (threadId - base) * 2; - c_sm = C * 2; - f_sm = F * 2; - r_gl = r / 2 + threadId - base; - c_gl = c / 2 + C; - f_gl = f / 2 + F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf_c) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - // if (c_gl == nc_c - 1 && f_gl == nf_c-1) - // printf("block: (%d %d %d) thread: (%d %d %d) load4 (%d %d %d): %f - // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, - // threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], - // r_gl, c_gl, f_gl); - } - } - base += R; // ROUND_UP_WARP(R) * WARP_SIZE; - if (dw && threadId >= base && threadId < base + C) { - r_sm = R * 2; - c_sm = (threadId - base) * 2; - f_sm = F * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId - base; - f_gl = f / 2 + F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf_c) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - // if (c_gl == nc_c - 1 && f_gl == nf_c-1) - // printf("block: (%d %d %d) thread: (%d %d %d) load5 (%d %d %d): %f - // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, - // threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], - // r_gl, c_gl, f_gl); - } - } - base += C; // ROUND_UP_WARP(C) * WARP_SIZE; - if (dw && threadId >= base && threadId < base + F) { - r_sm = R * 2; - c_sm = C * 2; - f_sm = (threadId - base) * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + C; - f_gl = f / 2 + threadId - base; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf_c) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - // if (c_gl == nc_c - 1 && f_gl == nf_c-1) - // printf("block: (%d %d %d) thread: (%d %d %d) load6 (%d %d %d): %f - // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, - // threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], - // r_gl, c_gl, f_gl); - } - } - base += F; // ROUND_UP_WARP(F) * WARP_SIZE; - // // load extra vertex - if (dw && threadId >= base && threadId < base + 1) { - r_sm = R * 2; - c_sm = C * 2; - f_sm = F * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + C; - f_gl = f / 2 + F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf_c) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)]; - // if (c_gl == nc_c - 1 && f_gl == nf_c-1) - // printf("block: (%d %d %d) thread: (%d %d %d) load7 (%d %d %d): %f - // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, - // threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], - // r_gl, c_gl, f_gl); - } - } - - __syncthreads(); - - // __syncthreads(); - // if (threadIdx.x == 0 && threadIdx.y == 0&& threadIdx.z == 0) { - // printf("rest_p: %u %u %u RCF\n", rest_r_p, rest_c_p, rest_f_p, R, C, F); - // for (int i = 0; i < min(rest_r_p, R * 2 + 1); i++) { - // for (int j = 0; j < min(rest_c_p, C * 2 + 1); j++) { - // for (int k = 0; k < min(rest_f_p, F * 2 + 1); k++) { - // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); - // } - // printf("\n"); - // } - // printf("\n"); - // } - // } - // __syncthreads(); - - // __syncthreads(); - // if (debug) { - // printf("TYPE: %d %d %d %d\n", TYPE, - // min(rest_r_p, R * 2 + 1), - // min(rest_c_p, C * 2 + 1), - // min(rest_f_p, F * 2 + 1)); - // for (int i = 0; i < min(rest_r_p, R * 2 + 1); i++) { - // for (int j = 0; j < min(rest_c_p, C * 2 + 1); j++) { - // for (int k = 0; k < min(rest_f_p, F * 2 + 1); k++) { - // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); - // } - // printf("\n"); - // } - // printf("\n"); - // } - // } - // __syncthreads(); - - if (dwf && threadId >= R * C * F && threadId < R * C * F * 2) { - - r_sm = ((threadId - R * C * F) / (C * F)) * 2; - c_sm = (((threadId - R * C * F) % (C * F)) / F) * 2; - f_sm = (((threadId - R * C * F) % (C * F)) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F) % (C * F)) % F; - - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc_c && f_gl < nf - nf_c) { - - res = dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - - if (dwc && threadId >= R * C * F * 2 && threadId < R * C * F * 3) { - r_sm = ((threadId - R * C * F * 2) / (C * F)) * 2; - c_sm = (((threadId - R * C * F * 2) % (C * F)) / F) * 2 + 1; - f_sm = (((threadId - R * C * F * 2) % (C * F)) % F) * 2; - r_gl = r / 2 + (threadId - R * C * F * 2) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 2) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 2) % (C * F)) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc - nc_c && f_gl < nf_c) { - res = dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - - if (dwr && threadId >= R * C * F * 3 && threadId < R * C * F * 4) { - r_sm = ((threadId - R * C * F * 3) / (C * F)) * 2 + 1; - c_sm = (((threadId - R * C * F * 3) % (C * F)) / F) * 2; - f_sm = (((threadId - R * C * F * 3) % (C * F)) % F) * 2; - r_gl = r / 2 + (threadId - R * C * F * 3) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 3) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 3) % (C * F)) % F; - - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { - res = dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - // if (c_gl == nc_c-1 && f_gl == nf_c - 1) - // printf("block: (%d %d %d) thread: (%d %d %d) calc_coeff0 (%d - // %d %d): %f <- %f %f\n", blockIdx.z, blockIdx.y, blockIdx.x, - // threadIdx.z, threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, - // res, v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, - // f_sm)], - // v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - - if (dwcf && threadId >= R * C * F * 4 && threadId < R * C * F * 5) { - r_sm = ((threadId - R * C * F * 4) / (C * F)) * 2; - c_sm = (((threadId - R * C * F * 4) % (C * F)) / F) * 2 + 1; - f_sm = (((threadId - R * C * F * 4) % (C * F)) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F * 4) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 4) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 4) % (C * F)) % F; - - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && r_gl < nr_c && - c_gl < nc - nc_c && f_gl < nf - nf_c) { - res = dwcf[get_idx(lddwcf1, lddwcf2, r_gl, c_gl, f_gl)]; - T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res += lerp(f1, f2, ratio_c_sm[c_sm - 1]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - - if (dwrf && threadId >= R * C * F * 5 && threadId < R * C * F * 6) { - r_sm = ((threadId - R * C * F * 5) / (C * F)) * 2 + 1; - c_sm = (((threadId - R * C * F * 5) % (C * F)) / F) * 2; - f_sm = (((threadId - R * C * F * 5) % (C * F)) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F * 5) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 5) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 5) % (C * F)) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - res = dwrf[get_idx(lddwrf1, lddwrf2, r_gl, c_gl, f_gl)]; - T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res += lerp(f1, f2, ratio_r_sm[r_sm - 1]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - - if (dwrc && threadId >= R * C * F * 6 && threadId < R * C * F * 7) { - r_sm = ((threadId - R * C * F * 6) / (C * F)) * 2 + 1; - c_sm = (((threadId - R * C * F * 6) % (C * F)) / F) * 2 + 1; - f_sm = (((threadId - R * C * F * 6) % (C * F)) % F) * 2; - r_gl = r / 2 + (threadId - R * C * F * 6) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 6) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 6) % (C * F)) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - res = dwrc[get_idx(lddwrc1, lddwrc2, r_gl, c_gl, f_gl)]; - T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res += lerp(c1, c2, ratio_r_sm[r_sm - 1]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - - if (dwrcf && threadId >= R * C * F * 7 && threadId < R * C * F * 8) { - r_sm = ((threadId - R * C * F * 7) / (C * F)) * 2 + 1; - c_sm = (((threadId - R * C * F * 7) % (C * F)) / F) * 2 + 1; - f_sm = (((threadId - R * C * F * 7) % (C * F)) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F * 7) / (C * F); - c_gl = c / 2 + ((threadId - R * C * F * 7) % (C * F)) / F; - f_gl = f / 2 + ((threadId - R * C * F * 7) % (C * F)) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { - res = dwrcf[get_idx(lddwrcf1, lddwrcf2, r_gl, c_gl, f_gl)]; - T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f3 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f4 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - - T fc1 = lerp(f1, f2, ratio_c_sm[c_sm - 1]); - T fc2 = lerp(f3, f4, ratio_c_sm[c_sm - 1]); - - res += lerp(fc1, fc2, ratio_r_sm[r_sm - 1]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - - if (r + R * 2 == nr_p - 1) { - if (threadId < C * F) { - if (dwf) { - r_sm = R * 2; - c_sm = (threadId / F) * 2; - f_sm = (threadId % F) * 2 + 1; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId / F; - f_gl = f / 2 + threadId % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - res = dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - - if (dwc) { - r_sm = R * 2; - c_sm = (threadId / F) * 2 + 1; - f_sm = (threadId % F) * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId / F; - f_gl = f / 2 + threadId % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - res = dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - if (dwcf) { - r_sm = R * 2; - c_sm = (threadId / F) * 2 + 1; - f_sm = (threadId % F) * 2 + 1; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId / F; - f_gl = f / 2 + threadId % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { - res = dwcf[get_idx(lddwcf1, lddwcf2, r_gl, c_gl, f_gl)]; - T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res += lerp(f1, f2, ratio_c_sm[c_sm - 1]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - } - - if (c + C * 2 == nc_p - 1) { - if (threadId >= R * C * F && threadId < R * C * F + R * F) { - if (dwf) { - r_sm = ((threadId - R * C * F) / F) * 2; - c_sm = C * 2; - f_sm = ((threadId - R * C * F) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F) / F; - c_gl = c / 2 + C; - f_gl = f / 2 + (threadId - R * C * F) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - res = dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - if (dwr) { - r_sm = ((threadId - R * C * F) / F) * 2 + 1; - c_sm = C * 2; - f_sm = ((threadId - R * C * F) % F) * 2; - r_gl = r / 2 + (threadId - R * C * F) / F; - c_gl = c / 2 + C; - f_gl = f / 2 + (threadId - R * C * F) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { - res = dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - // if (c_gl == nc_c-1 && f_gl == nf_c - 1) - // printf("block: (%d %d %d) thread: (%d %d %d) calc_coeff1 (%d - // %d %d): %f <- %f %f\n", blockIdx.z, blockIdx.y, blockIdx.x, - // threadIdx.z, threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, - // res, v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, - // f_sm)], - // v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - if (dwrf) { - r_sm = ((threadId - R * C * F) / F) * 2 + 1; - c_sm = C * 2; - f_sm = ((threadId - R * C * F) % F) * 2 + 1; - r_gl = r / 2 + (threadId - R * C * F) / F; - c_gl = c / 2 + C; - f_gl = f / 2 + (threadId - R * C * F) % F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - res = dwrf[get_idx(lddwrf1, lddwrf2, r_gl, c_gl, f_gl)]; - T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - res += lerp(f1, f2, ratio_r_sm[r_sm - 1]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - } - - if (f + F * 2 == nf_p - 1) { - if (threadId >= R * C * F * 2 && threadId < R * C * F * 2 + R * C) { - if (dwc) { - r_sm = ((threadId - R * C * F * 2) / C) * 2; - c_sm = ((threadId - R * C * F * 2) % C) * 2 + 1; - f_sm = F * 2; - r_gl = r / 2 + (threadId - R * C * F * 2) / C; - c_gl = c / 2 + (threadId - R * C * F * 2) % C; - f_gl = f / 2 + F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - res = dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - - if (dwr) { - r_sm = ((threadId - R * C * F * 2) / C) * 2 + 1; - c_sm = ((threadId - R * C * F * 2) % C) * 2; - f_sm = F * 2; - r_gl = r / 2 + (threadId - R * C * F * 2) / C; - c_gl = c / 2 + (threadId - R * C * F * 2) % C; - f_gl = f / 2 + F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { - res = dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - // if (c_gl == nc_c-1 && f_gl == nf_c - 1) - // printf("block: (%d %d %d) thread: (%d %d %d) calc_coeff2 (%d - // %d %d): %f <- %f %f\n", blockIdx.z, blockIdx.y, blockIdx.x, - // threadIdx.z, threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, - // res, v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, - // f_sm)], - // v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - - if (dwrc) { - r_sm = ((threadId - R * C * F * 2) / C) * 2 + 1; - c_sm = ((threadId - R * C * F * 2) % C) * 2 + 1; - f_sm = F * 2; - r_gl = r / 2 + (threadId - R * C * F * 2) / C; - c_gl = c / 2 + (threadId - R * C * F * 2) % C; - f_gl = f / 2 + F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - res = dwrc[get_idx(lddwrc1, lddwrc2, r_gl, c_gl, f_gl)]; - T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - res += lerp(c1, c2, ratio_r_sm[r_sm - 1]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - } - - if (c + C * 2 == nc_p - 1 && f + F * 2 == nf_p - 1) { - if (threadId >= R * C * F * 3 && threadId < R * C * F * 3 + R) { - if (dwr) { - r_sm = (threadId - R * C * F * 3) * 2 + 1; - c_sm = C * 2; - f_sm = F * 2; - r_gl = r / 2 + threadId - R * C * F * 3; - c_gl = c / 2 + C; - f_gl = f / 2 + F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { - res = dwr[get_idx(lddwr1, lddwr2, r_gl, c_gl, f_gl)]; - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], - ratio_r_sm[r_sm - 1]); - // if (c_gl == nc_c-1 && f_gl == nf_c - 1) - // printf("block: (%d %d %d) thread: (%d %d %d) calc_coeff3 (%d - // %d %d): %f <- %f %f\n", blockIdx.z, blockIdx.y, blockIdx.x, - // threadIdx.z, threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, - // res, v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, - // f_sm)], - // v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - } - - if (r + R * 2 == nr_p - 1 && f + F * 2 == nf_p - 1) { - if (threadId >= R * C * F * 4 && threadId < R * C * F * 4 + C) { - if (dwc) { - r_sm = R * 2; - c_sm = (threadId - R * C * F * 4) * 2 + 1; - f_sm = F * 2; - r_gl = r / 2 + R; - c_gl = c / 2 + threadId - R * C * F * 4; - f_gl = f / 2 + F; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { - res = dwc[get_idx(lddwc1, lddwc2, r_gl, c_gl, f_gl)]; - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], - ratio_c_sm[c_sm - 1]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - } - - if (r + R * 2 == nr_p - 1 && c + C * 2 == nc_p - 1) { - if (threadId >= R * C * F * 5 && threadId < R * C * F * 5 + F) { - if (dwf) { - r_sm = R * 2; - c_sm = C * 2; - f_sm = (threadId - R * C * F * 5) * 2 + 1; - r_gl = r / 2 + R; - c_gl = c / 2 + C; - f_gl = f / 2 + threadId - R * C * F * 5; - if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && - r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { - res = dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]; - res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], - ratio_f_sm[f_sm - 1]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; - } - } - } - } - - // __syncthreads(); - // if (debug) { - // printf("TYPE: %d %d %d %d\n", TYPE, - // min(rest_r_p, R * 2 + 1), - // min(rest_c_p, C * 2 + 1), - // min(rest_f_p, F * 2 + 1)); - // for (int i = 0; i < min(rest_r_p, R * 2 + 1); i++) { - // for (int j = 0; j < min(rest_c_p, C * 2 + 1); j++) { - // for (int k = 0; k < min(rest_f_p, F * 2 + 1); k++) { - // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); - // } - // printf("\n"); - // } - // printf("\n"); - // } - // } - // __syncthreads(); - - __syncthreads(); - - r_sm = threadIdx.z; - c_sm = threadIdx.y; - f_sm = threadIdx.x; - - r_sm_ex = blockDim.z; - c_sm_ex = blockDim.y; - f_sm_ex = blockDim.x; - - r_gl = r + r_sm; - c_gl = c + c_sm; - f_gl = f + f_sm; - - // r_gl_ex = r + R * 2; - // c_gl_ex = c + C * 2; - // f_gl_ex = f + F * 2; - - r_gl_ex = r + rest_r - 1; - c_gl_ex = c + rest_c - 1; - f_gl_ex = f + rest_f - 1; - - int unpadding_r = rest_r; - int unpadding_c = rest_c; - int unpadding_f = rest_f; - if (nr % 2 == 0) - unpadding_r -= 1; - if (nc % 2 == 0) - unpadding_c -= 1; - if (nf % 2 == 0) - unpadding_f -= 1; - - if (r_sm < unpadding_r && c_sm < unpadding_c && f_sm < unpadding_f) { - - // store extra rules - // case 1: input = odd (non-padding required) - // case 1.a: block size + 1 == rest (need to store extra); - // case 1.b: block size + 1 != rest (No need to store extra); - // case 2: input = even (un-padding requried) - // case 2.a: block size + 1 >= rest (No need to store extra, but need - // un-padding first); case 2.b: block size + 1 < rest (No need to store - // extra); - - if (D >= 3 && r_sm == 0) { - if (nr % 2 != 0 && R * 2 + 1 == rest_r) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm)]; - } - if (nr % 2 == 0 && R * 2 + 1 >= rest_r_p) { - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm)]; - // if ( v_sm[get_idx(ldsm1, ldsm2, rest_r_p-1, c_sm, f_sm)] == 71177117) - // printf("un-padding0 error block: (%d %d %d) thread: (%d %d %d) - // un-padding (%d %d %d) %f (%d %d %d)\n", blockIdx.z, blockIdx.y, - // blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, - // rest_r-1, c_sm, f_sm, - // v_sm[get_idx(ldsm1, ldsm2, rest_r_p-1, c_sm, f_sm)], rest_r_p-1, - // c_sm, f_sm); - } - } - - if (D >= 2 && c_sm == 0) { - if (nc % 2 != 0 && C * 2 + 1 == rest_c) { - dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm)]; - } - if (nc % 2 == 0 && C * 2 + 1 >= rest_c_p) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, f_sm)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm)]; - // if (v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm)] == - // 71177117) - // printf("un-padding1 error block: (%d %d %d) thread: (%d %d %d) " - // "un-padding (%d %d %d) %f (%d %d %d)\n", - // blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, - // threadIdx.y, threadIdx.x, r_sm, rest_c - 1, f_sm, - // v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm)], r_sm, - // rest_c_p - 1, f_sm); - } - } - - if (D >= 1 && f_sm == 0) { - if (nf % 2 != 0 && F * 2 + 1 == rest_f) { - dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm_ex)]; - } - if (nf % 2 == 0 && F * 2 + 1 >= rest_f_p) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f - 1)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f_p - 1)]; - // if ( v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f_p-1)] == 71177117) - // printf("un-padding2 error block: (%d %d %d) thread: (%d %d %d) - // un-padding (%d %d %d) %f (%d %d %d)\n", blockIdx.z, blockIdx.y, - // blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, - // r_sm, c_sm, rest_f-1, - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f_p-1)], r_sm, c_sm, - // rest_f_p-1); - } - } - - // load extra edges - if (D >= 2 && c_sm == 0 && f_sm == 0) { - if (nc % 2 != 0 && C * 2 + 1 == rest_c && nf % 2 != 0 && - F * 2 + 1 == rest_f) { - dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm_ex)]; - } - if (nc % 2 == 0 && nf % 2 == 0 && C * 2 + 1 >= rest_c_p && - F * 2 + 1 >= rest_f_p) { - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, rest_f - 1)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, rest_f_p - 1)]; - // printf("block: (%d %d %d) thread: (%d %d %d) un-padding (%d %d %d) %f - // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, - // threadIdx.y, threadIdx.x, r_sm, rest_c-1, rest_f-1, - // v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c-1, rest_f-1)], r_sm, - // rest_c_p-1, rest_f_p-1); - } - if (nc % 2 == 0 && nf % 2 != 0 && C * 2 + 1 >= rest_c_p && - F * 2 + 1 == rest_f) { - dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm_ex)]; - } - if (nc % 2 != 0 && nf % 2 == 0 && C * 2 + 1 == rest_c && - F * 2 + 1 >= rest_f_p) { - dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, rest_f_p - 1)]; - // printf("(%d %d %d): %f <- (%d %d %d)\n", - // r_gl, c_gl_ex, f_gl_ex, - // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)], - // r_sm, c_sm_ex, f_gl_ex); - } - } - - if (D >= 3 && r_sm == 0 && f_sm == 0) { - if (nr % 2 != 0 && R * 2 + 1 == rest_r && nf % 2 != 0 && - F * 2 + 1 == rest_f) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm_ex)]; - } - if (nr % 2 == 0 && nf % 2 == 0 && R * 2 + 1 >= rest_r_p && - F * 2 + 1 >= rest_f_p) { - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, rest_f - 1)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, rest_f_p - 1)]; - // if ( v_sm[get_idx(ldsm1, ldsm2, rest_r_p-1, c_sm, rest_f_p-1)] == - // 71177117) printf("un-padding3 error block: (%d %d %d) thread: (%d %d - // %d) un-padding (%d %d %d) %f (%d %d %d)\n", blockIdx.z, blockIdx.y, - // blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, - // rest_r-1, c_sm, rest_f-1, - // v_sm[get_idx(ldsm1, ldsm2, rest_r_p-1, c_sm, rest_f_p-1)], - // rest_r_p-1, c_sm, rest_f_p-1); - } - if (nr % 2 == 0 && nf % 2 != 0 && R * 2 + 1 >= rest_r_p && - F * 2 + 1 == rest_f) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm_ex)]; - } - if (nr % 2 != 0 && nf % 2 == 0 && R * 2 + 1 == rest_r && - F * 2 + 1 >= rest_f_p) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, rest_f_p - 1)]; - // printf("(%d %d %d): %f <- (%d %d %d)\n", - // r_gl_ex, c_gl, rest_f-1, - // dv[get_idx(lddv1, lddv2, r_gl_ex-1, c_gl, f_gl_ex)], - // r_sm_ex, c_sm, rest_f_p-1); - } - } - - if (D >= 3 && r_sm == 0 && c_sm == 0) { - if (nr % 2 != 0 && R * 2 + 1 == rest_r && nc % 2 != 0 && - C * 2 + 1 == rest_c) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm)]; - } - if (nr % 2 == 0 && nc % 2 == 0 && R * 2 + 1 >= rest_r_p && - C * 2 + 1 >= rest_c_p) { - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm)]; - // if ( v_sm[get_idx(ldsm1, ldsm2, rest_r_p-1, rest_c_p-1, f_sm)] == - // 71177117) printf("un-padding4 error block: (%d %d %d) thread: (%d %d - // %d) un-padding (%d %d %d) %f (%d %d %d)\n", blockIdx.z, blockIdx.y, - // blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, - // rest_r-1, rest_c-1, f_sm, - // v_sm[get_idx(ldsm1, ldsm2, rest_r_p-1, rest_c_p-1, f_sm)], - // rest_r_p-1, rest_c_p-1, f_sm); - } - if (nr % 2 == 0 && nc % 2 != 0 && R * 2 + 1 >= rest_r_p && - C * 2 + 1 == rest_c) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm)]; - } - if (nr % 2 != 0 && nc % 2 == 0 && R * 2 + 1 == rest_r && - C * 2 + 1 >= rest_c_p) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm)]; - } - } - // load extra vertex - - if (D >= 3 && r_sm == 0 && c_sm == 0 && f_sm == 0) { - if (nr % 2 != 0 && R * 2 + 1 == rest_r && nc % 2 != 0 && - C * 2 + 1 == rest_c && nf % 2 != 0 && F * 2 + 1 == rest_f) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm_ex)]; - } - - if (nr % 2 == 0 && nc % 2 == 0 && nf % 2 == 0 && R * 2 + 1 >= rest_r_p && - C * 2 + 1 >= rest_c_p && F * 2 + 1 >= rest_f_p) { - v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, rest_f - 1)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, - rest_f_p - 1)]; - - // printf("block: (%d %d %d) thread: (%d %d %d) un-padding (%d %d %d) %f - // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, - // threadIdx.y, threadIdx.x, rest_r-1, rest_c-1, rest_f-1, - // v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c-1, rest_f-1)], - // rest_r_p-1, rest_c_p-1, rest_f_p-1); - } - if (nr % 2 == 0 && nc % 2 == 0 && nf % 2 != 0 && R * 2 + 1 >= rest_r_p && - C * 2 + 1 >= rest_c_p && F * 2 + 1 == rest_f) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm_ex)]; - } - if (nr % 2 == 0 && nc % 2 != 0 && nf % 2 == 0 && R * 2 + 1 >= rest_r_p && - C * 2 + 1 == rest_c && F * 2 + 1 >= rest_f_p) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, rest_f_p - 1)]; - } - if (nr % 2 != 0 && nc % 2 == 0 && nf % 2 == 0 && R * 2 + 1 == rest_r && - C * 2 + 1 >= rest_c_p && F * 2 + 1 >= rest_f_p) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, rest_f_p - 1)]; - } - if (nr % 2 == 0 && nc % 2 != 0 && nf % 2 != 0 && R * 2 + 1 >= rest_r_p && - C * 2 + 1 == rest_c && F * 2 + 1 == rest_f) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm_ex)]; - } - if (nr % 2 != 0 && nc % 2 == 0 && nf % 2 != 0 && R * 2 + 1 == rest_r && - C * 2 + 1 >= rest_c_p && F * 2 + 1 == rest_f) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm_ex)]; - } - if (nr % 2 != 0 && nc % 2 != 0 && nf % 2 == 0 && R * 2 + 1 == rest_r && - C * 2 + 1 == rest_c && F * 2 + 1 >= rest_f_p) { - dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, rest_f_p - 1)]; - } - } - } - - __syncthreads(); - - if (r_sm < rest_r && c_sm < rest_c && f_sm < rest_f) { - if (r_gl >= svr && r_gl < svr + nvr && c_gl >= svc && c_gl < svc + nvc && - f_gl >= svf && f_gl < svf + nvf) { - dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl)] = - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; - - // if (c_gl == nc - 1 && f_gl == nf - 1) { - // printf("block: (%d %d %d) thread: (%d %d %d) store (%d %d %d) %f - // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, - // threadIdx.y, threadIdx.x, r_gl, c_gl, f_gl, - // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], r_sm, c_sm, f_sm); - // } - } - } -} - -template -void gpk_rev_3d_adaptive_launcher( - Handle &handle, SIZE nr, SIZE nc, SIZE nf, T *dratio_r, T *dratio_c, - T *dratio_f, T *dv, SIZE lddv1, SIZE lddv2, T *dw, SIZE lddw1, SIZE lddw2, - T *dwf, SIZE lddwf1, SIZE lddwf2, T *dwc, SIZE lddwc1, SIZE lddwc2, T *dwr, - SIZE lddwr1, SIZE lddwr2, T *dwcf, SIZE lddwcf1, SIZE lddwcf2, T *dwrf, - SIZE lddwrf1, SIZE lddwrf2, T *dwrc, SIZE lddwrc1, SIZE lddwrc2, T *dwrcf, - SIZE lddwrcf1, SIZE lddwrcf2, SIZE svr, SIZE svc, SIZE svf, SIZE nvr, - SIZE nvc, SIZE nvf, int queue_idx) { - cudaDeviceSetSharedMemConfig(cudaSharedMemBankSizeEightByte); - cudaDeviceSetCacheConfig(cudaFuncCachePreferShared); - SIZE nr_c = nr / 2 + 1; - SIZE nc_c = nc / 2 + 1; - SIZE nf_c = nf / 2 + 1; - SIZE total_thread_z = std::max(nr - 1, (SIZE)1); - SIZE total_thread_y = std::max(nc - 1, (SIZE)1); - SIZE total_thread_x = std::max(nf - 1, (SIZE)1); - - SIZE tbx, tby, tbz, gridx, gridy, gridz; - dim3 threadsPerBlock, blockPerGrid; - size_t sm_size; - - // tbz = std::min(R, total_thread_z); - // tby = std::min(C, total_thread_y); - // tbx = std::min(F, total_thread_x); - tbz = R; - tby = C; - tbx = F; - sm_size = ((R + 1) * (C + 1) * (F + 1) + R + C + F) * sizeof(T); - gridz = ceil((double)total_thread_z / tbz); - gridy = ceil((double)total_thread_y / tby); - gridx = ceil((double)total_thread_x / tbx); - threadsPerBlock = dim3(tbx, tby, tbz); - blockPerGrid = dim3(gridx, gridy, gridz); - // printf("prolongate exec: %d %d %d %d %d %d\n", tbx, tby, tbz, gridx, gridy, - // gridz); - _gpk_rev_3d - <<>>( - nr, nc, nf, nr_c, nc_c, nf_c, dratio_r, dratio_c, dratio_f, dv, lddv1, - lddv2, dw, lddw1, lddw2, dwf, lddwf1, lddwf2, dwc, lddwc1, lddwc2, - dwr, lddwr1, lddwr2, dwcf, lddwcf1, lddwcf2, dwrf, lddwrf1, lddwrf2, - dwrc, lddwrc1, lddwrc2, dwrcf, lddwrcf1, lddwrcf2, svr, svc, svf, nvr, - nvc, nvf); - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } -} - -template -void gpk_rev_3d(Handle &handle, SIZE nr, SIZE nc, SIZE nf, T *dratio_r, - T *dratio_c, T *dratio_f, T *dv, SIZE lddv1, SIZE lddv2, T *dw, - SIZE lddw1, SIZE lddw2, T *dwf, SIZE lddwf1, SIZE lddwf2, - T *dwc, SIZE lddwc1, SIZE lddwc2, T *dwr, SIZE lddwr1, - SIZE lddwr2, T *dwcf, SIZE lddwcf1, SIZE lddwcf2, T *dwrf, - SIZE lddwrf1, SIZE lddwrf2, T *dwrc, SIZE lddwrc1, SIZE lddwrc2, - T *dwrcf, SIZE lddwrcf1, SIZE lddwrcf2, SIZE svr, SIZE svc, - SIZE svf, SIZE nvr, SIZE nvc, SIZE nvf, int queue_idx, - int config) { - -#define GPK(R, C, F) \ - { \ - gpk_rev_3d_adaptive_launcher( \ - handle, nr, nc, nf, dratio_r, dratio_c, dratio_f, dv, lddv1, lddv2, \ - dw, lddw1, lddw2, dwf, lddwf1, lddwf2,\ - dwc, \ - lddwc1, lddwc2, dwr, lddwr1, lddwr2, dwcf, lddwcf1, lddwcf2,\ - dwrf, \ - lddwrf1, lddwrf2, dwrc, lddwrc1, lddwrc2, dwrcf, lddwrcf1, lddwrcf2, \ - svr, svc, svf, nvr, nvc, nvf, queue_idx); \ - } - bool profile = false; - if (handle.profile_kernels) { - profile = true; - } - if (D == 3) { - if (profile || config == 6) { - GPK(2, 2, 128) - } - if (profile || config == 5) { - GPK(2, 2, 64) - } - if (profile || config == 4) { - GPK(4, 4, 32) - } - if (profile || config == 3) { - GPK(4, 4, 16) - } - if (profile || config == 2) { - GPK(4, 4, 8) - } - if (profile || config == 1) { - GPK(4, 4, 4) - } - if (profile || config == 0) { - GPK(2, 2, 2) - } - } else if (D == 2) { - if (profile || config == 6) { - GPK(1, 2, 128) - } - if (profile || config == 5) { - GPK(1, 2, 64) - } - if (profile || config == 4) { - GPK(1, 4, 32) - } - if (profile || config == 3) { - GPK(1, 4, 16) - } - if (profile || config == 2) { - GPK(1, 4, 8) - } - if (profile || config == 1) { - GPK(1, 4, 4) - } - if (profile || config == 0) { - GPK(1, 2, 4) - } - } else if (D == 1) { - if (profile || config == 6) { - GPK(1, 1, 128) - } - if (profile || config == 5) { - GPK(1, 1, 64) - } - if (profile || config == 4) { - GPK(1, 1, 32) - } - if (profile || config == 3) { - GPK(1, 1, 16) - } - if (profile || config == 2) { - GPK(1, 1, 8) - } - if (profile || config == 1) { - GPK(1, 1, 8) - } - if (profile || config == 0) { - GPK(1, 1, 8) - } - } -#undef GPK -} - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/Handle.h b/include/cuda/Handle.h deleted file mode 100644 index 282aae2f33..0000000000 --- a/include/cuda/Handle.h +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: September 27, 2021 - */ - -#ifndef MGRAD_CUDA_HANDLE -#define MGRAD_CUDA_HANDLE - -#include "Common.h" - -namespace mgard_cuda { - -struct Config { - int dev_id; - SIZE l_target; - SIZE huff_dict_size; - SIZE huff_block_size; - SIZE lz4_block_size; - bool reduce_memory_footprint; - bool profile_kernels; - bool sync_and_check_all_kernels; - bool timing; - int uniform_coord_mode; - enum lossless_type lossless; - - Config() { - dev_id = 0; - l_target = -1; // no limit - huff_dict_size = 8192; - // #ifdef MGARD_CUDA_OPTIMIZE_TURING - // huff_block_size = 1024 * 30; - // #endif - // #ifdef MGARD_CUDA_OPTIMIZE_VOLTA - huff_block_size = 1024 * 20; - // #endif - lz4_block_size = 1 << 15; - reduce_memory_footprint = false; - profile_kernels = false; - sync_and_check_all_kernels = false; - timing = false; - uniform_coord_mode = 0; - lossless = lossless_type::GPU_Huffman; - } -}; - -template struct Handle { - - /* for Internal use only */ - Handle(); - - /* for general users */ - Handle(std::vector shape); - Handle(std::vector shape, std::vector coords); - Handle(std::vector shape, Config config); - Handle(std::vector shape, std::vector coords, Config config); - ~Handle(); - - void allocate_workspace(); - void free_workspace(); - void *get(int i); - void sync(int i); - void sync_all(); - - /* CUDA env */ - void *queues; - int num_of_queues; - int dev_id = 0; - - /* Refactoring env */ - SIZE l_target; - DIM D_padded; - std::vector shape; - std::vector> dofs; - std::vector shapes_h; - std::vector shapes_d; - SIZE *ranges_h; - SIZE *ranges_d; - std::vector coords_h; - std::vector coords_d; - std::vector> dist; - std::vector> ratio; - T *volumes; - SIZE ldvolumes; - std::vector> am; - std::vector> bm; - LENGTH linearized_depth; - LENGTH padded_linearized_depth; - - enum data_structure_type dstype; - T *quantizers; - SIZE huff_dict_size; - SIZE huff_block_size; - SIZE lz4_block_size; - enum lossless_type lossless; - - bool reduce_memory_footprint; - bool profile_kernels; - bool sync_and_check_all_kernels; - bool timing; - - DIM *processed_n; - DIM **processed_dims_h; - DIM **processed_dims_d; - - DIM *unprocessed_n; - DIM **unprocessed_dims_h; - DIM **unprocessed_dims_d; - - T *dw; - SIZE lddw1, lddw2; - std::vector ldws_h; - SIZE *ldws_d; - - T *db; - SIZE lddb1, lddb2; - std::vector ldbs_h; - SIZE *ldbs_d; - - int ***auto_tuning_cc; - int ***auto_tuning_mr1, ***auto_tuning_ts1; - int ***auto_tuning_mr2, ***auto_tuning_ts2; - int ***auto_tuning_mr3, ***auto_tuning_ts3; - int arch, precision; - -private: - void padding_dimensions(std::vector &shape, std::vector &coords); - - void create_queues(); - void destroy_queues(); - - std::vector create_uniform_coords(std::vector shape, int mode); - bool uniform_coords_created = false; - - int num_arch = 3; - int num_precision = 2; - int num_range = 9; - void init_auto_tuning_table(); - void destroy_auto_tuning_table(); - bool auto_tuning_table_created = false; - - void coord_to_dist(SIZE dof, T *coord, T *dist); - void dist_to_ratio(SIZE dof, T *dist, T *ratio); - void reduce_dist(SIZE dof, T *dist, T *dist2); - void calc_am_bm(SIZE dof, T *dist, T *am, T *bm); - void calc_volume(SIZE dof, T *dist, T *volume); - void init(std::vector shape, std::vector coords, Config config); - - void destroy(); - bool initialized = false; -}; - -} // namespace mgard_cuda - -#endif diff --git a/include/cuda/IPKFunctor.h b/include/cuda/IPKFunctor.h deleted file mode 100644 index ba766fec0f..0000000000 --- a/include/cuda/IPKFunctor.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_IPK_FUNCTOR -#define MGRAD_CUDA_IPK_FUNCTOR - -namespace mgard_cuda { - -template -__device__ inline T tridiag_forward2(T prev, T am, T bm, T curr) { - -#ifdef MGARD_CUDA_FMA - if (sizeof(T) == sizeof(double)) { - return fma(prev, am * bm, curr); - } else if (sizeof(T) == sizeof(float)) { - return fmaf(prev, am * bm, curr); - } -#else - // printf("forward: %f < %f %f %f %f\n", curr - prev * (am / bm), - // curr, prev, am , bm); - return curr - prev * (am / bm); -#endif -} - -template -__device__ inline T tridiag_backward2(T prev, T am, T bm, T curr) { - -#ifdef MGARD_CUDA_FMA - if (sizeof(T) == sizeof(double)) { - return fma(am, prev, curr) * bm; - } else if (sizeof(T) == sizeof(float)) { - return fmaf(am, prev, curr) * bm; - } -#else - // printf("backward: %f < %f %f %f %f\n", (curr - am * prev) / bm, - // curr, prev, am , bm); - return (curr - am * prev) / bm; -#endif -} - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/IterativeProcessingKernel.h b/include/cuda/IterativeProcessingKernel.h deleted file mode 100644 index 7a8af5aa59..0000000000 --- a/include/cuda/IterativeProcessingKernel.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_ITERATIVE_PROCESSING_KERNEL -#define MGRAD_CUDA_ITERATIVE_PROCESSING_KERNEL - -#include "Common.h" - -namespace mgard_cuda { - -template -void ipk_1(Handle &handle, SIZE *shape_h, SIZE *shape_c_h, SIZE *shape_d, - SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, DIM processed_n, - DIM *processed_dims_h, DIM *processed_dims_d, DIM curr_dim_r, - DIM curr_dim_c, DIM curr_dim_f, T *am, T *bm, T *ddist_f, T *dv, - LENGTH lddv1, LENGTH lddv2, int queue_idx, int config); - -template -void ipk_2(Handle &handle, SIZE *shape_h, SIZE *shape_c_h, SIZE *shape_d, - SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, DIM processed_n, - DIM *processed_dims_h, DIM *processed_dims_d, DIM curr_dim_r, - DIM curr_dim_c, DIM curr_dim_f, T *am, T *bm, T *ddist_c, T *dv, - LENGTH lddv1, LENGTH lddv2, int queue_idx, int config); - -template -void ipk_3(Handle &handle, SIZE *shape_h, SIZE *shape_c_h, SIZE *shape_d, - SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, DIM processed_n, - DIM *processed_dims_h, DIM *processed_dims_d, DIM curr_dim_r, - DIM curr_dim_c, DIM curr_dim_f, T *am, T *bm, T *ddist_r, T *dv, - LENGTH lddv1, LENGTH lddv2, int queue_idx, int config); -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/IterativeProcessingKernel.hpp b/include/cuda/IterativeProcessingKernel.hpp deleted file mode 100644 index 1a8317ecbf..0000000000 --- a/include/cuda/IterativeProcessingKernel.hpp +++ /dev/null @@ -1,2067 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_ITERATIVE_PROCESSING_KERNEL_TEMPLATE -#define MGRAD_CUDA_ITERATIVE_PROCESSING_KERNEL_TEMPLATE - -#include "CommonInternal.h" -#include "IPKFunctor.h" -#include "IterativeProcessingKernel.h" -namespace mgard_cuda { - -template -__global__ void _ipk_1(SIZE *shape, SIZE *shape_c, SIZE *ldvs, SIZE *ldws, - DIM processed_n, DIM *processed_dims, DIM curr_dim_r, - DIM curr_dim_c, DIM curr_dim_f, T *am, T *bm, T *dist_f, - T *v, LENGTH ldv1, LENGTH ldv2) { - - // bool debug = false; - // if (blockIdx.z == 0 && blockIdx.y == 0 && blockIdx.x == 0 && - // threadIdx.z == 0 && threadIdx.y == 0) - // debug = false; - - // bool debug2 = false; - // if (threadIdx.z == 0 && threadIdx.y == 0 && threadIdx.x == 0) - // debug2 = false; - - LENGTH threadId = (threadIdx.z * (blockDim.x * blockDim.y)) + - (threadIdx.y * blockDim.x) + threadIdx.x; - - T *sm = SharedMemory(); - SIZE ldsm1 = F + G; - SIZE ldsm2 = C; - T *vec_sm = sm; - sm += R * ldsm1 * ldsm2; - T *am_sm = sm; - sm += ldsm1; - T *bm_sm = sm; - sm += ldsm1; - - SIZE *sm_size = (SIZE *)sm; - SIZE *shape_sm = sm_size; - sm_size += D; - SIZE *shape_c_sm = sm_size; - sm_size += D; - SIZE *ldvs_sm = sm_size; - sm_size += D; - SIZE *ldws_sm = sm_size; - sm_size += D; - sm = (T *)sm_size; - - DIM *sm_dim = (DIM *)sm; - DIM *processed_dims_sm = sm_dim; - sm_dim += D; - sm = (T *)sm_dim; - - SIZE idx[D]; - - for (LENGTH i = threadId; i < D; i += blockDim.x * blockDim.y * blockDim.z) { - shape_sm[i] = shape[i]; - shape_c_sm[i] = shape_c[i]; - ldvs_sm[i] = ldvs[i]; - ldws_sm[i] = ldws[i]; - } - for (LENGTH i = threadId; i < processed_n; - i += blockDim.x * blockDim.y * blockDim.z) { - processed_dims_sm[i] = processed_dims[i]; - } - __syncthreads(); - - for (DIM d = 0; d < D; d++) - idx[d] = 0; - - SIZE nr = shape_c_sm[curr_dim_r]; - SIZE nc = shape_c_sm[curr_dim_c]; - SIZE nf_c = shape_c_sm[curr_dim_f]; - - if (D < 3) - nr = 1; - if (D < 2) - nc = 1; - - SIZE bidx = blockIdx.x; - SIZE firstD = div_roundup(nc, C); - SIZE blockId = bidx % firstD; - // if (debug2) { - // printf("blockIdx.x %u nc %u blockDim.x %u firstD: %u blockId %u\n", - // blockIdx.x, nc, blockDim.x, firstD, blockId); - // } - bidx /= firstD; - - for (DIM d = 0; d < D; d++) { - if (d != curr_dim_r && d != curr_dim_c && d != curr_dim_f) { - SIZE t = shape_sm[d]; - // for (DIM k = 0; k < processed_n; k++) { - // if (d == processed_dims[k]) { - t = shape_c_sm[d]; - // } - //} - // if (debug2) { - // printf("%u mod %u = %u, %u / %u = %u (shape_c: %u %u %u %u %u)\n", - // bidx, t, bidx % t, bidx, t, bidx/t, shape_c_sm[4], - // shape_c_sm[3],shape_c_sm[2],shape_c_sm[1],shape_c_sm[0]); - // } - idx[d] = bidx % t; - bidx /= t; - } - } - - size_t other_offset_v = get_idx(ldvs_sm, idx); - v = v + other_offset_v; - - // if (debug2) { - // printf("ipk1 idx: %u %u %u %u %u ld: %u %u %u %u %u\n", idx[4], idx[3], - // idx[2], idx[1], idx[0], ldvs_sm[4], ldvs_sm[3], ldvs_sm[2], ldvs_sm[1], - // ldvs_sm[0]); printf("ipk1 other_offset_v: %llu\n", other_offset_v); - - // LENGTH curr_stride = 1; - // LENGTH ret_idx = 0; - // for (DIM i = 0; i < D; i++) { - // ret_idx += idx[i] * curr_stride; - // printf("%llu * %llu = %llu\n", curr_stride, ldvs_sm[i], - // curr_stride*ldvs_sm[i]); curr_stride *= ldvs_sm[i]; - - // } - // } - - SIZE c_gl = blockId * C; - SIZE r_gl = blockIdx.y * R; - SIZE f_gl = threadIdx.x; - - SIZE c_sm = threadIdx.x; - SIZE r_sm = threadIdx.y; - SIZE f_sm = threadIdx.x; - - // if (idx[3] == 0 && idx[4] == 1 && r_gl+r_sm == 0 && c_gl+c_sm == 0) { - // printf("offset = %llu\n", other_offset_v); - // debug2 = false; - // } - - // if (debug2) { - // printf("ld: (%d %d %d %d %d) (shape_c: %u %u %u %u %u)\n", - // ldvs_sm[4], ldvs_sm[3],ldvs_sm[2],ldvs_sm[1],ldvs_sm[0], - // shape_c_sm[4], - // shape_c_sm[3],shape_c_sm[2],shape_c_sm[1],shape_c_sm[0]); - // } - - T *vec = v + get_idx(ldv1, ldv2, r_gl, c_gl, 0); - - T prev_vec_sm = 0.0; - - SIZE c_rest = min(C, nc - blockId * C); - SIZE r_rest = min(R, nr - blockIdx.y * R); - - SIZE f_rest = nf_c; - SIZE f_ghost = min(nf_c, G); - SIZE f_main = F; - - // printf("r_sm: %d, r_rest: %d, c_sm: %d, c_rest: %d f_sm: %d, f_rest %d , - // nf_c: %d\n", r_sm, r_rest, c_sm, c_rest, f_sm, f_rest, nf_c); - - // printf("test %f", vec_sm[get_idx(ldsm1, ldsm2, 0, 1, 0)]); - /* Load first ghost */ - if (r_sm < r_rest && f_sm < f_ghost) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - vec[get_idx(ldv1, ldv2, r_sm, i, f_gl)]; - // if (r_sm == 0) printf("r0_stride = %d, vec_sm[%d] = %f\n", r0_stride, - // i, vec_sm[i * ldsm + c_sm]); - } - } - - if (r_sm == 0 && f_sm < f_ghost) { - am_sm[f_sm] = am[f_gl]; - bm_sm[f_sm] = bm[f_gl]; - } - - f_rest -= f_ghost; - __syncthreads(); - - while (f_rest > F - f_ghost) { - // if (c_gl == 0 && c_sm == 0 && r_gl == 0 && r_sm == 0) printf("%d %d\n", - // f_rest, F - f_ghost); - f_main = min(F, f_rest); - if (r_sm < r_rest && f_sm < f_main) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + f_ghost)] = - vec[get_idx(ldv1, ldv2, r_sm, i, f_gl + f_ghost)]; - } - } - if (r_sm == 0 && f_sm < f_main) { - am_sm[f_sm + f_ghost] = am[f_gl + f_ghost]; - bm_sm[f_sm + f_ghost] = bm[f_gl + f_ghost]; - } - - __syncthreads(); - - /* Computation of v in parallel*/ - if (r_sm < r_rest && c_sm < c_rest) { - // if (debug) printf("forward %f <- %f %f %f %f\n", - // tridiag_forward2( - // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, r_sm, - // c_sm, 0)]), - // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, - // ldsm2, r_sm, c_sm, 0)]); - - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); - - //#pragma unroll 32 - for (SIZE i = 1; i < F; i++) { - // if (debug) printf("forward %f <- %f %f %f %f\n", - // tridiag_forward2( - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], - // bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]), - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], - // bm_sm[i], - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); - - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_forward2( - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); - } - - /* Store last v */ - prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, F - 1)]; - } - __syncthreads(); - - /* flush results to v */ - if (r_sm < r_rest && f_sm < F) { - for (SIZE i = 0; i < c_rest; i++) { - // if (blockIdx.x == 0 && blockIdx.y == 0 && r_sm == 0 && i == 1) { - // printf("store [%d %d %d] %f<-%f [%d %d %d]\n", - // r_sm, i, f_gl, vec[get_idx(ldv1, ldv2, r_sm, i, f_gl)], - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)], r_sm, i, f_sm); - // } - vec[get_idx(ldv1, ldv2, r_sm, i, f_gl)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; - // if (blockIdx.x == 0 && blockIdx.y == 0 && r_sm == 0 && i == 1) { - // printf("store [%d %d %d] %f<-%f [%d %d %d]\n", - // r_sm, i, f_gl, vec[get_idx(ldv1, ldv2, r_sm, i, f_gl)], - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)], r_sm, i, f_sm); - // } - } - } - __syncthreads(); - - /* Update unloaded col */ - f_rest -= f_main; - - /* Advance c */ - f_gl += F; - - /* Copy next ghost to main */ - f_ghost = min(G, f_main - (F - G)); - if (r_sm < r_rest && f_sm < f_ghost) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + F)]; - } - } - if (r_sm == 0 && f_sm < f_ghost) { - am_sm[f_sm] = am_sm[f_sm + blockDim.x]; - bm_sm[f_sm] = bm_sm[f_sm + blockDim.x]; - } - __syncthreads(); - } // end of while - - /* Load all rest col */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + f_ghost)] = - vec[get_idx(ldv1, ldv2, r_sm, i, f_gl + f_ghost)]; - } - // if (r_sm == 0) { - // bm_sm[f_sm + f_ghost] = bm[f_gl + f_ghost]; - // } - } - if (r_sm == 0 && f_sm < f_rest) { - am_sm[f_sm + f_ghost] = am[f_gl + f_ghost]; - bm_sm[f_sm + f_ghost] = bm[f_gl + f_ghost]; - } - - __syncthreads(); - - /* Only 1 col remain */ - if (f_ghost + f_rest == 1) { - if (r_sm < r_rest && c_sm < c_rest) { - // if (debug) printf("forward %f <- %f %f %f %f\n", - // tridiag_forward2( - // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, r_sm, - // c_sm, 0)]), - // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, - // ldsm2, r_sm, c_sm, 0)]); - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); - // printf ("prev_vec_sm = %f\n", prev_vec_sm ); - // printf ("vec_sm[r_sm * ldsm + 0] = %f\n", vec_sm[r_sm * ldsm + 0] ); - } - //__syncthreads(); - - } else { - if (r_sm < r_rest && c_sm < c_rest) { - // if (debug) printf("forward %f <- %f %f %f %f\n", - // tridiag_forward2( - // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, r_sm, - // c_sm, 0)]), - // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, - // ldsm2, r_sm, c_sm, 0)]); - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); - for (SIZE i = 1; i < f_ghost + f_rest; i++) { - // if (debug) printf("forward %f <- %f %f %f %f\n", - // tridiag_forward2( - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], - // bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]), - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], - // bm_sm[i], - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_forward2( - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); - } - } - } - __syncthreads(); - /* flush results to v */ - if (r_sm < r_rest && f_sm < f_ghost + f_rest) { - for (SIZE i = 0; i < c_rest; i++) { - vec[get_idx(ldv1, ldv2, r_sm, i, f_gl)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; - // printf("c_stride = %d, c_sm = %d, vec_sm = %f, vec[%d] = - // %f\n",c_stride, c_sm, vec_sm[r_sm * ldsm + 0],i * row_stride * lddv + - // c_stride, vec[i * row_stride * lddv + c_stride]); - } - } - __syncthreads(); - - /* backward */ - f_rest = nf_c; - f_ghost = min(nf_c, G); - f_main = F; - f_gl = threadIdx.x; - prev_vec_sm = 0.0; - - /* Load first ghost */ - if (r_sm < r_rest && f_sm < f_ghost) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - vec[get_idx(ldv1, ldv2, r_sm, i, (nf_c - 1) - f_gl)]; - // if (r_sm == 0) printf("r0_stride = %d, vec_sm[%d] = %f\n", r0_stride, - // i, vec_sm[i * ldsm + c_sm]); - } - } - if (r_sm == 0 && f_sm < f_ghost) { - am_sm[f_sm] = am[nf_c - f_gl]; - bm_sm[f_sm] = bm[nf_c - f_gl]; - } - f_rest -= f_ghost; - __syncthreads(); - - while (f_rest > F - f_ghost) { - f_main = min(F, f_rest); - if (r_sm < r_rest && f_sm < f_main) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + f_ghost)] = - vec[get_idx(ldv1, ldv2, r_sm, i, (nf_c - 1) - f_gl - f_ghost)]; - } - } - if (r_sm == 0 && f_sm < f_main) { - am_sm[f_sm + f_ghost] = am[nf_c - f_gl - f_ghost]; - bm_sm[f_sm + f_ghost] = bm[nf_c - f_gl - f_ghost]; - } - __syncthreads(); - - /* Computation of v in parallel*/ - if (r_sm < r_rest && c_sm < c_rest) { - // if (debug) printf("backward %f <- %f %f %f %f\n", - // tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]), - // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, - // ldsm2, r_sm, c_sm, 0)]); - - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); - //#pragma unroll 32 - for (SIZE i = 1; i < F; i++) { - - // if (debug) printf("backward %f <- %f %f %f %f\n", - // tridiag_backward2( - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], - // am_sm[i], bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, - // i)]), - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], - // am_sm[i], bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, - // i)]); - - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_backward2( - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); - } - /* Store last v */ - prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, blockDim.x - 1)]; - } - __syncthreads(); - - /* flush results to v */ - if (r_sm < r_rest && f_sm < F) { - for (SIZE i = 0; i < c_rest; i++) { - vec[get_idx(ldv1, ldv2, r_sm, i, (nf_c - 1) - f_gl)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; - } - } - __syncthreads(); - - /* Update unloaded col */ - f_rest -= f_main; - - /* Advance c */ - f_gl += F; - - /* Copy next ghost to main */ - f_ghost = min(G, f_main - (F - G)); - if (r_sm < r_rest && f_sm < f_ghost) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + F)]; - } - if (r_sm == 0) { - am_sm[f_sm] = am_sm[f_sm + F]; - bm_sm[f_sm] = bm_sm[f_sm + F]; - } - } - __syncthreads(); - } // end of while - - /* Load all rest col */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + f_ghost)] = - vec[get_idx(ldv1, ldv2, r_sm, i, (nf_c - 1) - f_gl - f_ghost)]; - } - } - if (r_sm == 0 && f_gl + f_ghost <= nf_c) { - am_sm[f_sm + f_ghost] = am[nf_c - f_gl - f_ghost]; - bm_sm[f_sm + f_ghost] = bm[nf_c - f_gl - f_ghost]; - } - __syncthreads(); - - /* Only 1 col remain */ - if (f_ghost + f_rest == 1) { - if (r_sm < r_rest && c_sm < c_rest) { - // if (debug) printf("backward %f <- %f %f %f %f\n", - // tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]), - // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, - // ldsm2, r_sm, c_sm, 0)]); - - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); - // printf ("prev_vec_sm = %f\n", prev_vec_sm ); - // printf ("vec_sm[r_sm * ldsm + 0] = %f\n", vec_sm[r_sm * ldsm + 0] ); - } - //__syncthreads(); - - } else { - if (r_sm < r_rest && c_sm < c_rest) { - // if (debug) printf("backward %f <- %f %f %f %f\n", - // tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]), - // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, - // ldsm2, r_sm, c_sm, 0)]); - - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); - for (SIZE i = 1; i < f_ghost + f_rest; i++) { - - // if (debug) printf("backward %f <- %f %f %f %f\n", - // tridiag_backward2( - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], - // am_sm[i], bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, - // i)]), - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], - // am_sm[i], bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, - // i)]); - - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_backward2( - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); - } - } - } - __syncthreads(); - /* flush results to v */ - if (r_sm < r_rest && f_sm < f_ghost + f_rest) { - for (SIZE i = 0; i < c_rest; i++) { - vec[get_idx(ldv1, ldv2, r_sm, i, (nf_c - 1) - f_gl)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; - // printf("c_stride = %d, c_sm = %d, vec_sm = %f, vec[%d] = - // %f\n",c_stride, c_sm, vec_sm[r_sm * ldsm + 0],i * row_stride * lddv + - // c_stride, vec[i * row_stride * lddv + c_stride]); - } - } - __syncthreads(); -} - -template -void ipk_1_adaptive_launcher(Handle &handle, SIZE *shape_h, - SIZE *shape_c_h, SIZE *shape_d, SIZE *shape_c_d, - SIZE *ldvs, SIZE *ldws, DIM processed_n, - DIM *processed_dims_h, DIM *processed_dims_d, - DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, - T *am, T *bm, T *ddist_f, T *dv, LENGTH lddv1, - LENGTH lddv2, int queue_idx) { - - SIZE nr = shape_c_h[curr_dim_r]; - SIZE nc = shape_c_h[curr_dim_c]; - SIZE nf_c = shape_c_h[curr_dim_f]; - - SIZE total_thread_x = nc; - SIZE total_thread_y = nr; - SIZE total_thread_z = 1; - SIZE tbx, tby, tbz, gridx, gridy, gridz; - dim3 threadsPerBlock, blockPerGrid; - size_t sm_size; - - // tbx = std::max(C, std::min(C, total_thread_x)); - // tby = std::max(R, std::min(R, total_thread_y)); - tbx = C; - tby = R; - tbz = 1; - sm_size = (R * C + 2) * (F + G) * sizeof(T); - sm_size += (D * 4) * sizeof(SIZE); - sm_size += (D * 1) * sizeof(DIM); - gridx = ceil((double)total_thread_x / tbx); - gridy = ceil((double)total_thread_y / tby); - gridz = 1; - // printf("ipk 1 total_thread_x %d tbx %d\n", total_thread_x, tbx); - - for (DIM d = 0; d < D; d++) { - if (d != curr_dim_f && d != curr_dim_c && d != curr_dim_r) { - SIZE t = shape_h[d]; - // for (DIM k = 0; k < processed_n; k++) { - // if (d == processed_dims_h[k]) { - t = shape_c_h[d]; - // } - //} - gridx *= t; - } - } - // printf("ipk_1 exec config (%d %d %d) (%d %d %d)\n", F, tby, tbz, gridx, - // gridy, gridz); - threadsPerBlock = dim3(F, tby, tbz); - blockPerGrid = dim3(gridx, gridy, gridz); - - // printf("ipk_1 exec config (%d %d %d) (%d %d %d)\n", tbx, tby, tbz, gridx, - // gridy, gridz); - _ipk_1<<>>( - shape_d, shape_c_d, ldvs, ldws, processed_n, processed_dims_d, curr_dim_r, - curr_dim_c, curr_dim_f, am, bm, ddist_f, dv, lddv1, lddv2); - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } - // std::cout << "test\n"; -} - -template -void ipk_1(Handle &handle, SIZE *shape_h, SIZE *shape_c_h, SIZE *shape_d, - SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, DIM processed_n, - DIM *processed_dims_h, DIM *processed_dims_d, DIM curr_dim_r, - DIM curr_dim_c, DIM curr_dim_f, T *am, T *bm, T *ddist_f, T *dv, - LENGTH lddv1, LENGTH lddv2, int queue_idx, int config) { - -#define IPK(R, C, F, G) \ - { \ - ipk_1_adaptive_launcher( \ - handle, shape_h, shape_c_h, shape_d, shape_c_d, ldvs, ldws, \ - processed_n, processed_dims_h, processed_dims_d,\ - curr_dim_r, \ - curr_dim_c, curr_dim_f, am, bm, ddist_f, dv, lddv1, lddv2, queue_idx); \ - } - bool profile = false; - if (handle.profile_kernels) { - profile = true; - } - if (D >= 3) { - if (profile || config == 6) { - IPK(2, 2, 128, 2) - } - if (profile || config == 5) { - IPK(2, 2, 64, 2) - } - if (profile || config == 4) { - IPK(2, 2, 32, 2) - } - if (profile || config == 3) { - IPK(4, 4, 16, 4) - } - if (profile || config == 2) { - IPK(8, 8, 8, 4) - } - if (profile || config == 1) { - IPK(4, 4, 4, 4) - } - if (profile || config == 0) { - IPK(2, 2, 2, 2) - } - } else if (D == 2) { - if (profile || config == 6) { - IPK(1, 2, 128, 2) - } - if (profile || config == 5) { - IPK(1, 2, 64, 2) - } - if (profile || config == 4) { - IPK(1, 2, 32, 2) - } - if (profile || config == 3) { - IPK(1, 4, 16, 4) - } - if (profile || config == 2) { - IPK(1, 8, 8, 4) - } - if (profile || config == 1) { - IPK(1, 4, 4, 4) - } - if (profile || config == 0) { - IPK(1, 2, 4, 2) - } - } else if (D == 1) { - if (profile || config == 6) { - IPK(1, 1, 128, 2) - } - if (profile || config == 5) { - IPK(1, 1, 64, 2) - } - if (profile || config == 4) { - IPK(1, 1, 32, 2) - } - if (profile || config == 3) { - IPK(1, 1, 16, 4) - } - if (profile || config == 2) { - IPK(1, 1, 8, 4) - } - if (profile || config == 1) { - IPK(1, 1, 8, 4) - } - if (profile || config == 0) { - IPK(1, 1, 8, 2) - } - } -#undef IPK -} - -template -__global__ void _ipk_2(SIZE *shape, SIZE *shape_c, SIZE *ldvs, SIZE *ldws, - DIM processed_n, DIM *processed_dims, DIM curr_dim_r, - DIM curr_dim_c, DIM curr_dim_f, T *am, T *bm, T *dist_c, - T *v, LENGTH ldv1, LENGTH ldv2) { - - // bool debug = false; - // if (blockIdx.z == 0 && blockIdx.y == 0 && blockIdx.x == 0 && - // threadIdx.z == 0 && threadIdx.y == 0) - // debug = false; - - // bool debug2 = false; - // if (threadIdx.z == 0 && threadIdx.y == 0 && threadIdx.x == 0) - // debug2 = false; - - LENGTH threadId = (threadIdx.z * (blockDim.x * blockDim.y)) + - (threadIdx.y * blockDim.x) + threadIdx.x; - - T *sm = SharedMemory(); - SIZE ldsm1 = F; - SIZE ldsm2 = C + G; - T *vec_sm = sm; - sm += R * ldsm1 * ldsm2; - T *am_sm = sm; - sm += ldsm2; - T *bm_sm = sm; - sm += ldsm2; - - SIZE *sm_size = (SIZE *)sm; - SIZE *shape_sm = sm_size; - sm_size += D; - SIZE *shape_c_sm = sm_size; - sm_size += D; - SIZE *ldvs_sm = sm_size; - sm_size += D; - SIZE *ldws_sm = sm_size; - sm_size += D; - sm = (T *)sm_size; - - DIM *sm_dim = (DIM *)sm; - DIM *processed_dims_sm = sm_dim; - sm_dim += D; - sm = (T *)sm_dim; - - SIZE idx[D]; - - for (LENGTH i = threadId; i < D; i += blockDim.x * blockDim.y * blockDim.z) { - shape_sm[i] = shape[i]; - shape_c_sm[i] = shape_c[i]; - ldvs_sm[i] = ldvs[i]; - ldws_sm[i] = ldws[i]; - } - for (LENGTH i = threadId; i < processed_n; - i += blockDim.x * blockDim.y * blockDim.z) { - processed_dims_sm[i] = processed_dims[i]; - } - __syncthreads(); - - for (DIM d = 0; d < D; d++) - idx[d] = 0; - - SIZE nr = shape_c_sm[curr_dim_r]; - SIZE nc_c = shape_c_sm[curr_dim_c]; - SIZE nf_c = shape_c_sm[curr_dim_f]; - - if (D < 3) - nr = 1; - - SIZE bidx = blockIdx.x; - SIZE firstD = div_roundup(nf_c, blockDim.x); - SIZE blockId = bidx % firstD; - - bidx /= firstD; - - for (DIM d = 0; d < D; d++) { - if (d != curr_dim_r && d != curr_dim_c && d != curr_dim_f) { - SIZE t = shape_sm[d]; - // for (DIM k = 0; k < processed_n; k++) { - // if (d == processed_dims[k]) { - t = shape_c_sm[d]; - // } - //} - idx[d] = bidx % t; - bidx /= t; - } - } - - size_t other_offset_v = get_idx(ldvs_sm, idx); - v = v + other_offset_v; - - SIZE f_gl = blockId * F; - SIZE r_gl = blockIdx.y * R; - SIZE c_gl = 0; - - SIZE f_sm = threadIdx.x; - SIZE r_sm = threadIdx.y; - SIZE c_sm = threadIdx.x; - - T *vec = v + get_idx(ldv1, ldv2, r_gl, 0, f_gl); - - T prev_vec_sm = 0.0; - - SIZE f_rest = min(F, nf_c - blockId * F); - SIZE r_rest = min(R, nr - blockIdx.y * R); - - // if (blockIdx.x == 1 && blockIdx.y == 0 && f_sm == 0 && r_sm == 0) { - // prSIZEf("f_rest: %d r_rest: %d\n", f_rest, r_rest); - // } - - SIZE c_rest = nc_c; - SIZE c_ghost = min(nc_c, G); - SIZE c_main = C; - - /* Load first ghost */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_ghost; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - vec[get_idx(ldv1, ldv2, r_sm, c_gl + i, f_sm)]; - // if (r_sm == 0) printf("r0_stride = %d, vec_sm[%d] = %f\n", r0_stride, - // i, vec_sm[i * ldsm + c_sm]); - } - } - if (r_sm == 0 && c_sm < c_ghost) { - am_sm[c_sm] = am[c_gl + c_sm]; - bm_sm[c_sm] = bm[c_gl + c_sm]; - } - c_rest -= c_ghost; - __syncthreads(); - - while (c_rest > C - c_ghost) { - // printf("%d %d %d\n", c_rest, C, c_ghost); - c_main = min(C, c_rest); - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_main; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)] = - vec[get_idx(ldv1, ldv2, r_sm, c_gl + i + c_ghost, f_sm)]; - } - } - if (r_sm == 0 && c_sm < c_main) { - am_sm[c_sm + c_ghost] = am[c_gl + c_sm + c_ghost]; - bm_sm[c_sm + c_ghost] = bm[c_gl + c_sm + c_ghost]; - } - __syncthreads(); - - /* Computation of v in parallel*/ - if (r_sm < r_rest && f_sm < f_rest) { - - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = - // __fma_rn(prev_vec_sm, bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, - // r_sm, 0, f_sm)]); - // #else - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] -= prev_vec_sm * - // bm_sm[0]; - // #endif - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); - - for (SIZE i = 1; i < C; i++) { - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - // __fma_rn(vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], - // bm_sm[i], - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); - // #else - // // if (blockIdx.x == 1 && blockIdx.y == 0 && f_sm == 0 && r_sm - // == 0) { - // // printf("calc: %f %f %f -> %f \n", vec_sm[get_idx(ldsm1, - // ldsm2, r_sm, i, f_sm)], - // // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], - // bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] - - // // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)] * - // bm_sm[i]); - // // } - - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] -= - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)] * bm_sm[i]; - // #endif - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = tridiag_forward2( - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); - } - /* Store last v */ - prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, r_sm, C - 1, f_sm)]; - } - __syncthreads(); - - /* flush results to v */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < C; i++) { - // if (blockIdx.x == 1 && blockIdx.y == 0 && f_sm == 0 && r_sm == 0) { - // printf("store: %f\n", vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, - // f_sm)]); - // } - vec[get_idx(ldv1, ldv2, r_sm, c_gl + i, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; - } - } - __syncthreads(); - - /* Update unloaded col */ - c_rest -= c_main; - - /* Advance c */ - c_gl += C; - - /* Copy next ghost to main */ - c_ghost = min(G, c_main - (C - G)); - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_ghost; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + C, f_sm)]; - } - } - if (r_sm == 0 && c_sm < c_ghost) { - am_sm[c_sm] = am_sm[c_sm + C]; - bm_sm[c_sm] = bm_sm[c_sm + C]; - } - __syncthreads(); - - } // end of while - - /* Load all rest col */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)] = - vec[get_idx(ldv1, ldv2, r_sm, c_gl + i + c_ghost, f_sm)]; - } - } - if (r_sm == 0 && c_sm < c_rest) { - am_sm[c_sm + c_ghost] = am[c_gl + c_sm + c_ghost]; - bm_sm[c_sm + c_ghost] = bm[c_gl + c_sm + c_ghost]; - } - __syncthreads(); - - /* Only 1 col remain */ - if (c_ghost + c_rest == 1) { - if (r_sm < r_rest && f_sm < f_rest) { - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] -= prev_vec_sm * bm_sm[0]; - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = - // __fma_rn(prev_vec_sm, bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, - // r_sm, 0, f_sm)]); - // #else - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] -= prev_vec_sm * - // bm_sm[0]; - // #endif - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); - // printf ("prev_vec_sm = %f\n", prev_vec_sm ); - // printf ("vec_sm[r_sm * ldsm + 0] = %f\n", vec_sm[r_sm * ldsm + 0] ); - } - //__syncthreads(); - - } else { - if (r_sm < r_rest && f_sm < f_rest) { - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = - // __fma_rn(prev_vec_sm, bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, - // r_sm, 0, f_sm)]); - // #else - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] -= prev_vec_sm * - // bm_sm[0]; - // #endif - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); - for (SIZE i = 1; i < c_ghost + c_rest; i++) { - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - // __fma_rn(vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], - // bm_sm[i], - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); - // #else - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] -= - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)] * bm_sm[i]; - // #endif - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = tridiag_forward2( - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); - } - } - } - __syncthreads(); - /* flush results to v */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_ghost + c_rest; i++) { - vec[get_idx(ldv1, ldv2, r_sm, c_gl + i, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; - // printf("c_stride = %d, c_sm = %d, vec_sm = %f, vec[%d] = - // %f\n",c_stride, c_sm, vec_sm[r_sm * ldsm + 0],i * row_stride * lddv + - // c_stride, vec[i * row_stride * lddv + c_stride]); - } - } - __syncthreads(); - - /* backward */ - c_rest = nc_c; - c_ghost = min(nc_c, G); - c_main = C; - c_gl = 0; - prev_vec_sm = 0.0; - - // if (f_gl + f_sm == 0 && r_gl + r_sm == 0 && idx[3] == 0) - // debug = false; - // if (debug) - // printf("block id: (%d %d %d) thread id: (%d %d %d)\n", blockIdx.x, - // blockIdx.y, blockIdx.z, threadIdx.x, threadIdx.y, threadIdx.z); - - /* Load first ghost */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_ghost; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - vec[get_idx(ldv1, ldv2, r_sm, (nc_c - 1) - (c_gl + i), f_sm)]; - // if (debug) - // printf("load vec_sm[%d] = %f\n", get_idx(ldsm1, ldsm2, r_sm, i, - // f_sm), - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); - } - } - if (r_sm == 0 && c_sm < c_ghost) { - am_sm[c_sm] = am[nc_c - (c_gl + c_sm)]; - bm_sm[c_sm] = bm[nc_c - (c_gl + c_sm)]; - } - c_rest -= c_ghost; - __syncthreads(); - - while (c_rest > C - c_ghost) { - // printf("%d %d %d\n", c_rest, C, c_ghost); - c_main = min(C, c_rest); - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_main; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)] = vec[get_idx( - ldv1, ldv2, r_sm, (nc_c - 1) - (c_gl + i + c_ghost), f_sm)]; - // if (debug) - // printf("load vec_sm[%d] = %f\n", - // get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm), - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)]); - } - } - if (r_sm == 0 && c_sm < c_main) { - am_sm[c_sm + c_ghost] = am[nc_c - (c_gl + c_sm + c_ghost)]; - bm_sm[c_sm + c_ghost] = bm[nc_c - (c_gl + c_sm + c_ghost)]; - } - __syncthreads(); - - // if (r_gl == 0 && f_gl == 0 && r_sm == 0 && f_sm == 0) - // printf("*****test\n"); - /* Computation of v in parallel*/ - if (r_sm < r_rest && f_sm < f_rest) { - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = - // __fma_rn(dist_sm[0], prev_vec_sm, vec_sm[get_idx(ldsm1, ldsm2, - // r_sm, 0, f_sm)]) * am_sm[0]; - // #else - // // if (r_gl == 0 && f_gl == 0 && r_sm == 0 && f_sm == 0) - // // printf("(%f + %f * %f) * %f -> %f\n", - // // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)], - // // dist_sm[0], prev_vec_sm, am_sm[0], - // // (vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] - - // dist_sm[0] * prev_vec_sm) / am_sm[0]); vec_sm[get_idx(ldsm1, - // ldsm2, r_sm, 0, f_sm)] = (vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, - // f_sm)] - dist_sm[0] * prev_vec_sm) / am_sm[0]; - // #endif - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)]); - // if (debug) - // printf("calc vec_sm[%d] = %f\n", get_idx(ldsm1, ldsm2, r_sm, 0, - // f_sm), - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); - - for (SIZE i = 1; i < C; i++) { - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - // __fma_rn(dist_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, - // f_sm)], - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)]) * am_sm[i]; - // #else - // // if (r_gl == 0 && f_gl == 0 && r_sm == 0 && f_sm == 0) - // // printf("(%f + %f * %f) * %f -> %f\n", - // // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)], - // // dist_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, - // i-1, f_sm)], am_sm[i], - // // (vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] - - // // dist_sm[i] * vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, - // f_sm)]) / am_sm[i]); - - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - // (vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] - - // dist_sm[i] * vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, - // f_sm)]) / am_sm[i]; - - // #endif - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = tridiag_backward2( - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); - - // if (debug) - // printf("calc vec_sm[%d] = %f\n", get_idx(ldsm1, ldsm2, r_sm, i, - // f_sm), - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); - } - - /* Store last v */ - prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, r_sm, C - 1, f_sm)]; - } - __syncthreads(); - - /* flush results to v */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < C; i++) { - vec[get_idx(ldv1, ldv2, r_sm, (nc_c - 1) - (c_gl + i), f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; - } - } - __syncthreads(); - - /* Update unloaded col */ - c_rest -= c_main; - - /* Advance c */ - c_gl += C; - - /* Copy next ghost to main */ - c_ghost = min(G, c_main - (C - G)); - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_ghost; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + C, f_sm)]; - } - } - if (r_sm == 0 && c_sm < c_ghost) { - am_sm[c_sm] = am_sm[c_sm + C]; - bm_sm[c_sm] = bm_sm[c_sm + C]; - } - __syncthreads(); - - } // end of while - - // Load all rest col - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)] = vec[get_idx( - ldv1, ldv2, r_sm, (nc_c - 1) - (c_gl + i + c_ghost), f_sm)]; - - // if (debug) - // printf("load ec_sm[%d] = %f\n", - // get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm), - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)]); - } - } - if (r_sm == 0 && c_sm < c_rest) { - am_sm[c_sm + c_ghost] = am[nc_c - (c_gl + c_sm + c_ghost)]; - bm_sm[c_sm + c_ghost] = bm[nc_c - (c_gl + c_sm + c_ghost)]; - } - __syncthreads(); - - /* Only 1 col remain */ - if (c_ghost + c_rest == 1) { - if (r_sm < r_rest && f_sm < f_rest) { - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = - // __fma_rn(dist_sm[0], prev_vec_sm, vec_sm[get_idx(ldsm1, ldsm2, - // r_sm, 0, f_sm)]) * am_sm[0]; - // #else - // // if (r_gl == 0 && f_gl == 0 && r_sm == 0 && f_sm == 0) - // // printf("(%f + %f * %f) * %f -> %f\n", - // // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)], - // // dist_sm[0], prev_vec_sm, am_sm[0], - // // (vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] - - // dist_sm[0] * prev_vec_sm) / am_sm[0]); vec_sm[get_idx(ldsm1, - // ldsm2, r_sm, 0, f_sm)] = (vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, - // f_sm)] - dist_sm[0] * prev_vec_sm) / am_sm[0]; - // #endif - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)]); - // if (debug) - // printf("calc vec_sm[%d] = %f\n", get_idx(ldsm1, ldsm2, r_sm, 0, - // f_sm), - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); - // printf ("prev_vec_sm = %f\n", prev_vec_sm ); - // printf ("vec_sm[r_sm * ldsm + 0] = %f\n", vec_sm[r_sm * ldsm + 0] ); - } - //__syncthreads(); - - } else { - if (r_sm < r_rest && f_sm < f_rest) { - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = - // __fma_rn(dist_sm[0], prev_vec_sm, vec_sm[get_idx(ldsm1, ldsm2, - // r_sm, 0, f_sm)]) * am_sm[0]; - // #else - // // if (r_gl == 0 && f_gl == 0 && r_sm == 0 && f_sm == 0) - // // printf("(%f + %f * %f) * %f -> %f\n", - // // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)], - // // dist_sm[0], prev_vec_sm, am_sm[0], - // // (vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] - - // dist_sm[0] * prev_vec_sm) / am_sm[0]); vec_sm[get_idx(ldsm1, - // ldsm2, r_sm, 0, f_sm)] = (vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, - // f_sm)] - dist_sm[0] * prev_vec_sm) / am_sm[0]; - // #endif - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)]); - // if (debug) - // printf("calc vec_sm[%d] = %f\n", get_idx(ldsm1, ldsm2, r_sm, 0, - // f_sm), - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); - for (SIZE i = 1; i < c_ghost + c_rest; i++) { - - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - // __fma_rn(dist_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, - // f_sm)], - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)]) * am_sm[i]; - // #else - // // if (r_gl == 0 && f_gl == 0 && r_sm == 0 && f_sm == 0) - // // printf("(%f + %f * %f) * %f -> %f\n", - // // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)], - // // dist_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, - // i-1, f_sm)], am_sm[i], - // // (vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] - - // // dist_sm[i] * vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, - // f_sm)]) / am_sm[i]); - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - // (vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] - - // dist_sm[i] * vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, - // f_sm)]) / am_sm[i]; - // #endif - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = tridiag_backward2( - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); - // if (debug) - // printf("calc vec_sm[%d] = %f\n", get_idx(ldsm1, ldsm2, r_sm, i, - // f_sm), - // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); - } - } - } - __syncthreads(); - /* flush results to v */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_ghost + c_rest; i++) { - vec[get_idx(ldv1, ldv2, r_sm, (nc_c - 1) - (c_gl + i), f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; - // printf("c_stride = %d, c_sm = %d, vec_sm = %f, vec[%d] = - // %f\n",c_stride, c_sm, vec_sm[r_sm * ldsm + 0],i * row_stride * lddv + - // c_stride, vec[i * row_stride * lddv + c_stride]); - } - } - __syncthreads(); -} - -template -void ipk_2_adaptive_launcher(Handle &handle, SIZE *shape_h, - SIZE *shape_c_h, SIZE *shape_d, SIZE *shape_c_d, - SIZE *ldvs, SIZE *ldws, DIM processed_n, - DIM *processed_dims_h, DIM *processed_dims_d, - DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, - T *am, T *bm, T *ddist_c, T *dv, LENGTH lddv1, - LENGTH lddv2, int queue_idx) { - SIZE nr = shape_c_h[curr_dim_r]; - SIZE nc_c = shape_c_h[curr_dim_c]; - SIZE nf_c = shape_c_h[curr_dim_f]; - - SIZE total_thread_x = nf_c; - SIZE total_thread_y = nr; - SIZE total_thread_z = 1; - SIZE tbx, tby, tbz, gridx, gridy, gridz; - dim3 threadsPerBlock, blockPerGrid; - size_t sm_size; - - tbx = F; // std::max(F, std::min(F, total_thread_x)); - tby = R; // std::max(R, std::min(R, total_thread_y)); - tbz = 1; - sm_size = (R * F + 2) * (C + G) * sizeof(T); - sm_size += (D * 4) * sizeof(SIZE); - sm_size += (D * 1) * sizeof(DIM); - - gridx = ceil((double)total_thread_x / tbx); - gridy = ceil((double)total_thread_y / tby); - gridz = 1; - for (DIM d = 0; d < D; d++) { - if (d != curr_dim_f && d != curr_dim_c && d != curr_dim_r) { - SIZE t = shape_h[d]; - // for (DIM k = 0; k < processed_n; k++) { - // if (d == processed_dims_h[k]) { - t = shape_c_h[d]; - // } - // } - gridx *= t; - } - } - - threadsPerBlock = dim3(tbx, tby, tbz); - blockPerGrid = dim3(gridx, gridy, gridz); - _ipk_2<<>>( - shape_d, shape_c_d, ldvs, ldws, processed_n, processed_dims_d, curr_dim_r, - curr_dim_c, curr_dim_f, am, bm, ddist_c, dv, lddv1, lddv2); - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } -} - -template -void ipk_2(Handle &handle, SIZE *shape_h, SIZE *shape_c_h, SIZE *shape_d, - SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, DIM processed_n, - DIM *processed_dims_h, DIM *processed_dims_d, DIM curr_dim_r, - DIM curr_dim_c, DIM curr_dim_f, T *am, T *bm, T *ddist_c, T *dv, - LENGTH lddv1, LENGTH lddv2, int queue_idx, int config) { - -#define IPK(R, C, F, G) \ - { \ - ipk_2_adaptive_launcher( \ - handle, shape_h, shape_c_h, shape_d, shape_c_d, ldvs, ldws, \ - processed_n, processed_dims_h, processed_dims_d,\ - curr_dim_r, \ - curr_dim_c, curr_dim_f, am, bm, ddist_c, dv, lddv1, lddv2, queue_idx); \ - } - bool profile = false; - if (handle.profile_kernels) { - profile = true; - } - if (D >= 3) { - if (profile || config == 6) { - IPK(2, 2, 128, 2) - } - if (profile || config == 5) { - IPK(2, 2, 64, 2) - } - if (profile || config == 4) { - IPK(2, 2, 32, 2) - } - if (profile || config == 3) { - IPK(4, 4, 16, 4) - } - if (profile || config == 2) { - IPK(8, 8, 8, 4) - } - if (profile || config == 1) { - IPK(4, 4, 4, 4) - } - if (profile || config == 0) { - IPK(2, 2, 2, 2) - } - } else if (D == 2) { - if (profile || config == 6) { - IPK(1, 2, 128, 2) - } - if (profile || config == 5) { - IPK(1, 2, 64, 2) - } - if (profile || config == 4) { - IPK(1, 2, 32, 2) - } - if (profile || config == 3) { - IPK(1, 4, 16, 4) - } - if (profile || config == 2) { - IPK(1, 8, 8, 4) - } - if (profile || config == 1) { - IPK(1, 4, 4, 4) - } - if (profile || config == 0) { - IPK(1, 2, 4, 2) - } - } else { - printf("Error: solve_tridiag_2_cpt is only for 3D and 2D data\n"); - } -#undef IPK -} - -template -__global__ void _ipk_3(SIZE *shape, SIZE *shape_c, SIZE *ldvs, SIZE *ldws, - DIM processed_n, DIM *processed_dims, DIM curr_dim_r, - DIM curr_dim_c, DIM curr_dim_f, T *am, T *bm, T *dist_r, - T *v, LENGTH ldv1, LENGTH ldv2) { - - // bool debug = false; - // if (blockIdx.z == 0 && blockIdx.y == 0 && blockIdx.x == 0 && - // threadIdx.z == 0 && threadIdx.y == 0 ) debug = false; - - // bool debug2 = false; - // if (threadIdx.z == 0 && threadIdx.y == 0 && threadIdx.x == 0 ) debug2 = - // false; - - LENGTH threadId = (threadIdx.z * (blockDim.x * blockDim.y)) + - (threadIdx.y * blockDim.x) + threadIdx.x; - - T *sm = SharedMemory(); - SIZE ldsm1 = F; - SIZE ldsm2 = C; - - T *vec_sm = sm; - sm += (R + G) * ldsm1 * ldsm2; - T *am_sm = sm; - sm += (R + G); - T *bm_sm = sm; - sm += (R + G); - - SIZE *sm_size = (SIZE *)sm; - SIZE *shape_sm = sm_size; - sm_size += D; - SIZE *shape_c_sm = sm_size; - sm_size += D; - SIZE *ldvs_sm = sm_size; - sm_size += D; - SIZE *ldws_sm = sm_size; - sm_size += D; - sm = (T *)sm_size; - - DIM *sm_dim = (DIM *)sm; - DIM *processed_dims_sm = sm_dim; - sm_dim += D; - sm = (T *)sm_dim; - - SIZE idx[D]; - for (LENGTH i = threadId; i < D; i += blockDim.x * blockDim.y * blockDim.z) { - shape_sm[i] = shape[i]; - shape_c_sm[i] = shape_c[i]; - ldvs_sm[i] = ldvs[i]; - ldws_sm[i] = ldws[i]; - } - for (LENGTH i = threadId; i < processed_n; - i += blockDim.x * blockDim.y * blockDim.z) { - processed_dims_sm[i] = processed_dims[i]; - } - __syncthreads(); - - for (DIM d = 0; d < D; d++) - idx[d] = 0; - - SIZE nr_c = shape_c_sm[curr_dim_r]; - SIZE nc_c = shape_c_sm[curr_dim_c]; - SIZE nf_c = shape_c_sm[curr_dim_f]; - - SIZE bidx = blockIdx.x; - SIZE firstD = div_roundup(nf_c, blockDim.x); - SIZE blockId = bidx % firstD; - - bidx /= firstD; - - for (DIM d = 0; d < D; d++) { - if (d != curr_dim_r && d != curr_dim_c && d != curr_dim_f) { - SIZE t = shape_sm[d]; - // for (DIM k = 0; k < processed_n; k++) { - // if (d == processed_dims[k]) { - t = shape_c_sm[d]; - // } - // } - idx[d] = bidx % t; - bidx /= t; - } - } - - size_t other_offset_v = get_idx(ldvs_sm, idx); - v = v + other_offset_v; - - SIZE f_gl = blockId * F; - SIZE c_gl = blockIdx.y * C; - SIZE r_gl = 0; - - SIZE f_sm = threadIdx.x; - SIZE c_sm = threadIdx.y; - SIZE r_sm = threadIdx.x; - - T *vec = v + get_idx(ldv1, ldv2, 0, c_gl, f_gl); - - T prev_vec_sm = 0.0; - - SIZE f_rest = min(F, nf_c - blockId * F); - SIZE c_rest = min(C, nc_c - blockIdx.y * C); - - SIZE r_rest = nr_c; - SIZE r_ghost = min(nr_c, G); - SIZE r_main = R; - - // if (f_gl + f_sm == 32 && c_gl + c_sm == 1 ) debug = false; - - /* Load first ghost */ - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_ghost; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = - vec[get_idx(ldv1, ldv2, r_gl + i, c_sm, f_sm)]; - // if (debug) printf("load first sm[%d] %f [%d]\n", i, - // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)], r_gl + i); - } - } - - if (c_sm == 0 && r_sm < r_ghost) { - am_sm[r_sm] = am[r_gl + r_sm]; - bm_sm[r_sm] = bm[r_gl + r_sm]; - } - r_rest -= r_ghost; - __syncthreads(); - - while (r_rest > R - r_ghost) { - r_main = min(R, r_rest); - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_main; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)] = - vec[get_idx(ldv1, ldv2, r_gl + i + r_ghost, c_sm, f_sm)]; - // if (debug) printf("load ghost sm[%d] %f [%d]\n", i + r_ghost, - // vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)], - // r_gl + i + r_ghost); - } - } - if (c_sm == 0 && r_sm < r_main) { - am_sm[r_sm + r_ghost] = am[r_gl + r_sm + r_ghost]; - bm_sm[r_sm + r_ghost] = bm[r_gl + r_sm + r_ghost]; - } - __syncthreads(); - - /* Computation of v in parallel*/ - if (c_sm < c_rest && f_sm < f_rest) { - - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - // __fma_rn(prev_vec_sm, bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, 0, - // c_sm, f_sm)]); - // #else - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] -= prev_vec_sm * - // bm_sm[0]; - // #endif - // if (debug) printf("compute sm[%d] %f <- %f %f %f\n", 0, - // tridiag_forward(prev_vec_sm, bm_sm[0], - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]), - // prev_vec_sm, bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, 0, - // c_sm, f_sm)]); - - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); - - for (SIZE i = 1; i < R; i++) { - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = - // __fma_rn(vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], - // bm_sm[i], - // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); - // #else - // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] -= - // vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)] * bm_sm[i]; - // #endif - - // if (debug) printf("compute sm[%d] %f <- %f %f %f\n", i, - // tridiag_forward(vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, - // f_sm)], - // bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]), - // vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], - // bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); - - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = tridiag_forward2( - vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); - } - - /* Store last v */ - prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, R - 1, c_sm, f_sm)]; - } - __syncthreads(); - - /* flush results to v */ - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < R; i++) { - vec[get_idx(ldv1, ldv2, r_gl + i, c_sm, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]; - // if (debug) printf("store[%d] %f [%d]\n", r_gl + i, - // vec[get_idx(ldv1, ldv2, r_gl + i, c_sm, f_sm)], i); - } - } - __syncthreads(); - - // /* Update unloaded col */ - r_rest -= r_main; - - /* Advance c */ - r_gl += R; - - /* Copy next ghost to main */ - r_ghost = min(G, r_main - (R - G)); - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_ghost; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, i + R, c_sm, f_sm)]; - - // if (debug) printf("copy next ghost[%d] %f [%d]\n", i, - // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)], i+R); - } - } - if (c_sm == 0 && r_sm < r_ghost) { - am_sm[r_sm] = am_sm[r_sm + R]; - bm_sm[r_sm] = bm_sm[r_sm + R]; - } - __syncthreads(); - - } // end of while - - /* Load all rest col */ - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)] = - vec[get_idx(ldv1, ldv2, r_gl + i + r_ghost, c_sm, f_sm)]; - - // if (debug) printf("load ghost-rest sm[%d] %f [%d]\n", i + r_ghost, - // vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)], - // r_gl + i + r_ghost); - } - } - - if (c_sm == 0 && r_sm < r_rest) { - am_sm[r_sm + r_ghost] = am[r_gl + r_sm + r_ghost]; - bm_sm[r_sm + r_ghost] = bm[r_gl + r_sm + r_ghost]; - } - - __syncthreads(); - - /* Only 1 col remain */ - if (r_ghost + r_rest == 1) { - if (c_sm < c_rest && f_sm < f_rest) { - - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - // __fma_rn(prev_vec_sm, bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, 0, - // c_sm, f_sm)]); - // #else - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] -= prev_vec_sm * - // bm_sm[0]; - // #endif - // if (debug) printf("compute-rest sm[%d] %f <- %f %f %f\n", 0, - // tridiag_forward(prev_vec_sm, bm_sm[0], - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]), - // prev_vec_sm, bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, 0, - // c_sm, f_sm)]); - - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); - // printf ("prev_vec_sm = %f\n", prev_vec_sm ); - // printf ("vec_sm[r_sm * ldsm + 0] = %f\n", vec_sm[r_sm * ldsm + 0] ); - } - //__syncthreads(); - - } else { - if (c_sm < c_rest && f_sm < f_rest) { - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - // __fma_rn(prev_vec_sm, bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, 0, - // c_sm, f_sm)]); - // #else - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] -= prev_vec_sm * - // bm_sm[0]; - // #endif - - // if (debug) printf("compute-rest sm[%d] %f <- %f %f %f\n", 0, - // tridiag_forward(prev_vec_sm, bm_sm[0], - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]), - // prev_vec_sm, bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, 0, - // c_sm, f_sm)]); - - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); - for (SIZE i = 1; i < r_ghost + r_rest; i++) { - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = - // __fma_rn(vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], - // bm_sm[i], - // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); - // #else - // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] -= - // vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)] * bm_sm[i]; - // #endif - // if (debug) printf("compute-rest sm[%d] %f <- %f %f %f\n", i, - // tridiag_forward(vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, - // f_sm)], - // bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]), - // vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], - // bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); - - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = tridiag_forward2( - vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); - } - } - } - __syncthreads(); - /* flush results to v */ - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_ghost + r_rest; i++) { - - vec[get_idx(ldv1, ldv2, r_gl + i, c_sm, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]; - - // if (debug) printf("store-rest[%d] %f [%d]\n", r_gl + i, - // vec[get_idx(ldv1, ldv2, r_gl + i, c_sm, f_sm)], i); - // printf("c_stride = %d, c_sm = %d, vec_sm = %f, vec[%d] = - // %f\n",c_stride, c_sm, vec_sm[r_sm * ldsm + 0],i * row_stride * lddv + - // c_stride, vec[i * row_stride * lddv + c_stride]); - } - } - __syncthreads(); - - /* backward */ - r_rest = nr_c; - r_ghost = min(nr_c, G); - r_main = R; - r_gl = 0; - prev_vec_sm = 0.0; - - /* Load first ghost */ - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_ghost; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = - vec[get_idx(ldv1, ldv2, (nr_c - 1) - (r_gl + i), c_sm, f_sm)]; - - // if (debug) printf("load first sm[%d] %f [%d]\n", i, - // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)], (nr_c - 1) - - // (r_gl + i)); - - // if (r_sm == 0) printf("r0_stride = %d, vec_sm[%d] = %f\n", r0_stride, - // i, vec_sm[i * ldsm + c_sm]); - } - } - - if (c_sm == 0 && r_sm < r_ghost) { - am_sm[r_sm] = am[nr_c - (r_gl + r_sm)]; - bm_sm[r_sm] = bm[nr_c - (r_gl + r_sm)]; - } - r_rest -= r_ghost; - __syncthreads(); - - while (r_rest > R - r_ghost) { - r_main = min(R, r_rest); - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_main; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)] = vec[get_idx( - ldv1, ldv2, (nr_c - 1) - (r_gl + i + r_ghost), c_sm, f_sm)]; - // if (debug) printf("load ghost sm[%d] %f [%d]\n", i + r_ghost, - // vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)], - // (nr_c - 1) - (r_gl + i + r_ghost)); - } - } - if (c_sm == 0 && r_sm < r_main) { - am_sm[r_sm + r_ghost] = am[nr_c - (r_gl + r_sm + r_ghost)]; - bm_sm[r_sm + r_ghost] = bm[nr_c - (r_gl + r_sm + r_ghost)]; - } - __syncthreads(); - - /* Computation of v in parallel*/ - if (c_sm < c_rest && f_sm < f_rest) { - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - // __fma_rn(dist_sm[0], prev_vec_sm, vec_sm[get_idx(ldsm1, ldsm2, 0, - // c_sm, f_sm)]) * am_sm[0]; - // #else - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - // (vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] - dist_sm[0] * - // prev_vec_sm) / am_sm[0]; - // #endif - - // if (debug) printf("compute sm[%d] %f <- %f %f %f %f\n", 0, - // tridiag_backward(prev_vec_sm, dist_sm[0], am_sm[0], - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]), - // prev_vec_sm, dist_sm[0], am_sm[0], - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); - - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); - for (SIZE i = 1; i < R; i++) { - - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = - // __fma_rn(dist_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, - // f_sm)], - // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]) * am_sm[i]; - // #else - // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = - // (vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] - - // dist_sm[i] * vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, - // f_sm)]) / am_sm[i]; - // #endif - - // if (debug) printf("compute sm[%d] %f <- %f %f %f %f\n", i, - // tridiag_backward(vec_sm[get_idx(ldsm1, ldsm2, i - 1, - // c_sm, f_sm)], - // dist_sm[i], am_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]), - // vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], - // dist_sm[i], am_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); - - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = tridiag_backward2( - vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); - } - - /* Store last v */ - prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, R - 1, c_sm, f_sm)]; - } - __syncthreads(); - - /* flush results to v */ - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < R; i++) { - // if (blockIdx.x == 0 && blockIdx.y == 0 && threadIdx.x == 0 && - // threadIdx.y == 0) { - // printf("%d %d %d (%f) <- %d %d %d\n", (nr - 1) - (r_gl + i), c_sm, - // f_sm, - // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)], i, c_sm, - // f_sm); - // } - vec[get_idx(ldv1, ldv2, (nr_c - 1) - (r_gl + i), c_sm, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]; - - // if (debug) printf("store[%d] %f [%d]\n", (nr_c - 1) - (r_gl + i), - // vec[get_idx(ldv1, ldv2, (nr_c - 1) - (r_gl + i), c_sm, f_sm)], i); - } - } - __syncthreads(); - - // /* Update unloaded col */ - r_rest -= r_main; - - /* Advance c */ - r_gl += R; - - /* Copy next ghost to main */ - r_ghost = min(G, r_main - (R - G)); - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_ghost; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, i + R, c_sm, f_sm)]; - - // if (debug) printf("copy next ghost[%d] %f [%d]\n", i, - // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)], i+R); - } - } - if (c_sm == 0 && r_sm < r_ghost) { - am_sm[r_sm] = am_sm[r_sm + R]; - bm_sm[r_sm] = bm_sm[r_sm + R]; - } - __syncthreads(); - - } // end of while - - /* Load all rest col */ - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)] = vec[get_idx( - ldv1, ldv2, (nr_c - 1) - (r_gl + i + r_ghost), c_sm, f_sm)]; - - // if (debug) printf("load ghost-rest sm[%d] %f [%d]\n", i + r_ghost, - // vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)], - // (nr_c - 1) - (r_gl + i + r_ghost)); - } - } - if (c_sm == 0 && r_sm < r_rest) { - am_sm[r_sm + r_ghost] = am[nr_c - (r_gl + r_sm + r_ghost)]; - bm_sm[r_sm + r_ghost] = bm[nr_c - (r_gl + r_sm + r_ghost)]; - } - __syncthreads(); - - /* Only 1 col remain */ - if (r_ghost + r_rest == 1) { - if (c_sm < c_rest && f_sm < f_rest) { - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - // __fma_rn(dist_sm[0], prev_vec_sm, vec_sm[get_idx(ldsm1, ldsm2, 0, - // c_sm, f_sm)]) * am_sm[0]; - // #else - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - // (vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] - dist_sm[0] * - // prev_vec_sm) / am_sm[0]; - // #endif - // if (blockIdx.x == 0 && blockIdx.y == 0 && threadIdx.x == 0 && - // threadIdx.y == 0) { - // printf("backward 1 (%f) %f %f %f %f\n", tridiag_backward(prev_vec_sm, - // dist_sm[0], am_sm[0], - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]), prev_vec_sm, - // dist_sm[0], am_sm[0], vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, - // f_sm)]); - - // } - // if (debug) printf("compute sm[%d] %f <- %f %f %f %f\n", 0, - // tridiag_backward(prev_vec_sm, dist_sm[0], am_sm[0], - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]), - // prev_vec_sm, dist_sm[0], am_sm[0], - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); - - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); - // printf ("prev_vec_sm = %f\n", prev_vec_sm ); - // printf ("vec_sm[r_sm * ldsm + 0] = %f\n", vec_sm[r_sm * ldsm + 0] ); - } - //__syncthreads(); - - } else { - if (c_sm < c_rest && f_sm < f_rest) { - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - // __fma_rn(dist_sm[0], prev_vec_sm, vec_sm[get_idx(ldsm1, ldsm2, 0, - // c_sm, f_sm)]) * am_sm[0]; - // #else - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - // (vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] - dist_sm[0] * - // prev_vec_sm) / am_sm[0]; - // #endif - // if (blockIdx.x == 0 && blockIdx.y == 0 && threadIdx.x == 0 && - // threadIdx.y == 0) { - // printf("backward 1 (%f) %f %f %f %f\n", tridiag_backward(prev_vec_sm, - // dist_sm[0], am_sm[0], - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]), prev_vec_sm, - // dist_sm[0], am_sm[0], vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, - // f_sm)]); - - // } - - // if (debug) printf("compute sm[%d] %f <- %f %f %f %f\n", 0, - // tridiag_backward(prev_vec_sm, dist_sm[0], am_sm[0], - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]), - // prev_vec_sm, dist_sm[0], am_sm[0], - // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); - - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); - for (SIZE i = 1; i < r_ghost + r_rest; i++) { - - // #ifdef MGARD_CUDA_FMA - // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = - // __fma_rn(dist_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, - // f_sm)], - // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]) * am_sm[i]; - // #else - // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = - // (vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] - - // dist_sm[i] * vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, - // f_sm)]) / am_sm[i]; - // #endif - // if (blockIdx.x == 0 && blockIdx.y == 0 && threadIdx.x == 0 && - // threadIdx.y == 0) { printf("backward R=%d (%f) %f %f %f %f\n", i, - // tridiag_backward(vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], - // dist_sm[i], am_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, - // f_sm)]), vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], - // dist_sm[i], am_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, - // f_sm)]); - - // } - - // if (debug) printf("compute sm[%d] %f <- %f %f %f %f\n", i, - // tridiag_backward(vec_sm[get_idx(ldsm1, ldsm2, i - 1, - // c_sm, f_sm)], - // dist_sm[i], am_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]), - // vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], - // dist_sm[i], am_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); - - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = tridiag_backward2( - vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); - } - } - } - __syncthreads(); - /* flush results to v */ - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_ghost + r_rest; i++) { - vec[get_idx(ldv1, ldv2, (nr_c - 1) - (r_gl + i), c_sm, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]; - - // if (debug) printf("store-rest[%d] %f [%d]\n", (nr_c - 1) - (r_gl + i), - // vec[get_idx(ldv1, ldv2, (nr_c - 1) - (r_gl + i), c_sm, f_sm)], i); - - // printf("c_stride = %d, c_sm = %d, vec_sm = %f, vec[%d] = - // %f\n",c_stride, c_sm, vec_sm[r_sm * ldsm + 0],i * row_stride * lddv + - // c_stride, vec[i * row_stride * lddv + c_stride]); - } - } - __syncthreads(); -} - -template -void ipk_3_adaptive_launcher(Handle &handle, SIZE *shape_h, - SIZE *shape_c_h, SIZE *shape_d, SIZE *shape_c_d, - SIZE *ldvs, SIZE *ldws, DIM processed_n, - DIM *processed_dims_h, DIM *processed_dims_d, - DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, - T *am, T *bm, T *ddist_r, T *dv, LENGTH lddv1, - LENGTH lddv2, int queue_idx) { - - // printf("am: "); - // print_matrix_cuda(1, nr, am, nr); - // printf("bm: "); - // print_matrix_cuda(1, nr, bm, nr); - - SIZE nr_c = shape_c_h[curr_dim_r]; - SIZE nc_c = shape_c_h[curr_dim_c]; - SIZE nf_c = shape_c_h[curr_dim_f]; - - SIZE total_thread_x = nf_c; - SIZE total_thread_y = nc_c; - SIZE total_thread_z = 1; - SIZE tbx, tby, tbz, gridx, gridy, gridz; - dim3 threadsPerBlock, blockPerGrid; - size_t sm_size; - - tbx = F; // std::max(F, std::min(F, total_thread_x)); - tby = C; // std::max(C, std::min(C, total_thread_y)); - tbz = 1; - sm_size = (C * F + 2) * (R + G) * sizeof(T); - sm_size += (D * 4) * sizeof(SIZE); - sm_size += (D * 1) * sizeof(DIM); - - gridx = ceil((double)total_thread_x / tbx); - gridy = ceil((double)total_thread_y / tby); - gridz = 1; - for (DIM d = 0; d < D; d++) { - if (d != curr_dim_f && d != curr_dim_c && d != curr_dim_r) { - SIZE t = shape_h[d]; - // for (DIM k = 0; k < processed_n; k++) { - // if (d == processed_dims_h[k]) { - t = shape_c_h[d]; - // // } - // } - gridx *= t; - } - } - threadsPerBlock = dim3(tbx, tby, tbz); - blockPerGrid = dim3(gridx, gridy, gridz); - // printf("ipk_1 exec config (%d %d %d) (%d %d %d)\n", tbx, tby, tbz, gridx, - // gridy, gridz); - _ipk_3<<>>( - shape_d, shape_c_d, ldvs, ldws, processed_n, processed_dims_d, curr_dim_r, - curr_dim_c, curr_dim_f, am, bm, ddist_r, dv, lddv1, lddv2); - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } -} - -template -void ipk_3(Handle &handle, SIZE *shape_h, SIZE *shape_c_h, SIZE *shape_d, - SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, DIM processed_n, - DIM *processed_dims_h, DIM *processed_dims_d, DIM curr_dim_r, - DIM curr_dim_c, DIM curr_dim_f, T *am, T *bm, T *ddist_r, T *dv, - LENGTH lddv1, LENGTH lddv2, int queue_idx, int config) { - -#define IPK(R, C, F, G) \ - { \ - ipk_3_adaptive_launcher( \ - handle, shape_h, shape_c_h, shape_d, shape_c_d, ldvs, ldws, \ - processed_n, processed_dims_h, processed_dims_d,\ - curr_dim_r, \ - curr_dim_c, curr_dim_f, am, bm, ddist_r, dv, lddv1, lddv2, queue_idx); \ - } - - bool profile = false; - if (handle.profile_kernels) { - profile = true; - } - if (D >= 3) { - if (profile || config == 6) { - IPK(2, 2, 128, 2) - } - if (profile || config == 5) { - IPK(2, 2, 64, 2) - } - if (profile || config == 4) { - IPK(2, 2, 32, 2) - } - if (profile || config == 3) { - IPK(2, 2, 16, 2) - } - if (profile || config == 2) { - IPK(8, 8, 8, 4) - } - if (profile || config == 1) { - IPK(4, 4, 4, 4) - } - if (profile || config == 0) { - IPK(2, 2, 2, 2) - } - // IPK(2, 2, 64, 2) - } else { - printf("Error: solve_tridiag_3_cpt is only for 3D data\n"); - } -#undef IPK -} - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/IterativeProcessingKernel3D.h b/include/cuda/IterativeProcessingKernel3D.h deleted file mode 100644 index 40e9364045..0000000000 --- a/include/cuda/IterativeProcessingKernel3D.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_ITERATIVE_PROCESSING_KERNEL_3D -#define MGRAD_CUDA_ITERATIVE_PROCESSING_KERNEL_3D - -#include "Common.h" - -namespace mgard_cuda { - -template -void ipk_1_3d(Handle &handle, SIZE nr, SIZE nc, SIZE nf_c, T *am, T *bm, - T *ddist_f, T *dv, SIZE lddv1, SIZE lddv2, int queue_idx, - int config); - -template -void ipk_2_3d(Handle &handle, SIZE nr, SIZE nc_c, SIZE nf_c, T *am, T *bm, - T *ddist_c, T *dv, SIZE lddv1, SIZE lddv2, int queue_idx, - int config); - -template -void ipk_3_3d(Handle &handle, SIZE nr_c, SIZE nc_c, SIZE nf_c, T *am, - T *bm, T *ddist_r, T *dv, SIZE lddv1, SIZE lddv2, int queue_idx, - int config); - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/IterativeProcessingKernel3D.hpp b/include/cuda/IterativeProcessingKernel3D.hpp deleted file mode 100644 index 6537ac0871..0000000000 --- a/include/cuda/IterativeProcessingKernel3D.hpp +++ /dev/null @@ -1,1150 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_ITERATIVE_PROCESSING_KERNEL_3D_TEMPLATE -#define MGRAD_CUDA_ITERATIVE_PROCESSING_KERNEL_3D_TEMPLATE - -#include "CommonInternal.h" -#include "IPKFunctor.h" -#include "IterativeProcessingKernel3D.h" -namespace mgard_cuda { - -template -__global__ void _ipk_1_3d(SIZE nr, SIZE nc, SIZE nf_c, T *am, T *bm, T *dist_f, - T *v, SIZE ldv1, SIZE ldv2) { - - SIZE c_gl = blockIdx.x * C; - SIZE r_gl = blockIdx.y * R; - SIZE f_gl = threadIdx.x; - - SIZE c_sm = threadIdx.x; - SIZE r_sm = threadIdx.y; - SIZE f_sm = threadIdx.x; - - T *vec = v + get_idx(ldv1, ldv2, r_gl, c_gl, 0); - T *sm = SharedMemory(); - SIZE ldsm1 = F + G; - SIZE ldsm2 = C; - T *vec_sm = sm; - T *am_sm = sm + R * ldsm1 * ldsm2; - T *bm_sm = am_sm + ldsm1; - - T prev_vec_sm = 0.0; - - SIZE c_rest = min(C, nc - blockIdx.x * C); - SIZE r_rest = min(R, nr - blockIdx.y * R); - - SIZE f_rest = nf_c; - SIZE f_ghost = min(nf_c, G); - SIZE f_main = F; - - /* Load first ghost */ - if (r_sm < r_rest && f_sm < f_ghost) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - vec[get_idx(ldv1, ldv2, r_sm, i, f_gl)]; - } - if (r_sm == 0) { - am_sm[f_sm] = am[f_gl]; - bm_sm[f_sm] = bm[f_gl]; - } - } - - f_rest -= f_ghost; - __syncthreads(); - - while (f_rest > F - f_ghost) { - f_main = min(F, f_rest); - if (r_sm < r_rest && f_sm < f_main) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + f_ghost)] = - vec[get_idx(ldv1, ldv2, r_sm, i, f_gl + f_ghost)]; - } - if (r_sm == 0) - am_sm[f_sm + f_ghost] = am[f_gl + f_ghost]; - bm_sm[f_sm + f_ghost] = bm[f_gl + f_ghost]; - } - - __syncthreads(); - - /* Computation of v in parallel*/ - if (r_sm < r_rest && c_sm < c_rest) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); - - //#pragma unroll 32 - for (SIZE i = 1; i < F; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_forward2( - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); - } - - /* Store last v */ - prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, F - 1)]; - } - __syncthreads(); - - /* flush results to v */ - if (r_sm < r_rest && f_sm < F) { - for (SIZE i = 0; i < c_rest; i++) { - vec[get_idx(ldv1, ldv2, r_sm, i, f_gl)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; - } - } - __syncthreads(); - - /* Update unloaded col */ - f_rest -= f_main; - - /* Advance c */ - f_gl += F; - - /* Copy next ghost to main */ - f_ghost = min(G, f_main - (F - G)); - if (r_sm < r_rest && f_sm < f_ghost) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + F)]; - } - if (r_sm == 0) { - am_sm[f_sm] = am_sm[f_sm + blockDim.x]; - bm_sm[f_sm] = bm_sm[f_sm + blockDim.x]; - } - } - __syncthreads(); - } // end of while - - /* Load all rest col */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + f_ghost)] = - vec[get_idx(ldv1, ldv2, r_sm, i, f_gl + f_ghost)]; - } - if (r_sm == 0) { - am_sm[f_sm + f_ghost] = am[f_gl + f_ghost]; - bm_sm[f_sm + f_ghost] = bm[f_gl + f_ghost]; - } - } - - __syncthreads(); - - /* Only 1 col remain */ - if (f_ghost + f_rest == 1) { - if (r_sm < r_rest && c_sm < c_rest) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); - } - //__syncthreads(); - - } else { - if (r_sm < r_rest && c_sm < c_rest) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); - for (SIZE i = 1; i < f_ghost + f_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_forward2( - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); - } - } - } - __syncthreads(); - /* flush results to v */ - if (r_sm < r_rest && f_sm < f_ghost + f_rest) { - for (SIZE i = 0; i < c_rest; i++) { - vec[get_idx(ldv1, ldv2, r_sm, i, f_gl)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; - } - } - __syncthreads(); - - /* backward */ - f_rest = nf_c; - f_ghost = min(nf_c, G); - f_main = F; - f_gl = threadIdx.x; - prev_vec_sm = 0.0; - - /* Load first ghost */ - if (r_sm < r_rest && f_sm < f_ghost) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - vec[get_idx(ldv1, ldv2, r_sm, i, (nf_c - 1) - f_gl)]; - } - } - if (r_sm == 0 && f_gl <= nf_c) { - am_sm[f_sm] = am[nf_c - f_gl]; - bm_sm[f_sm] = bm[nf_c - f_gl]; // * -1; - } - f_rest -= f_ghost; - __syncthreads(); - - while (f_rest > F - f_ghost) { - f_main = min(F, f_rest); - if (r_sm < r_rest && f_sm < f_main) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + f_ghost)] = - vec[get_idx(ldv1, ldv2, r_sm, i, (nf_c - 1) - f_gl - f_ghost)]; - } - } - if (r_sm == 0 && f_gl + f_ghost <= nf_c) { - am_sm[f_sm + f_ghost] = am[nf_c - f_gl - f_ghost]; - bm_sm[f_sm + f_ghost] = bm[nf_c - f_gl - f_ghost]; // * -1; - } - __syncthreads(); - - /* Computation of v in parallel*/ - if (r_sm < r_rest && c_sm < c_rest) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); - //#pragma unroll 32 - for (SIZE i = 1; i < F; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_backward2( - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); - } - /* Store last v */ - prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, blockDim.x - 1)]; - } - __syncthreads(); - - /* flush results to v */ - if (r_sm < r_rest && f_sm < F) { - for (SIZE i = 0; i < c_rest; i++) { - vec[get_idx(ldv1, ldv2, r_sm, i, (nf_c - 1) - f_gl)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; - } - } - __syncthreads(); - - /* Update unloaded col */ - f_rest -= f_main; - - /* Advance c */ - f_gl += F; - - /* Copy next ghost to main */ - f_ghost = min(G, f_main - (F - G)); - if (r_sm < r_rest && f_sm < f_ghost) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + F)]; - } - if (r_sm == 0) { - am_sm[f_sm] = am_sm[f_sm + F]; - bm_sm[f_sm] = bm_sm[f_sm + F]; - } - } - __syncthreads(); - } // end of while - - /* Load all rest col */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + f_ghost)] = - vec[get_idx(ldv1, ldv2, r_sm, i, (nf_c - 1) - f_gl - f_ghost)]; - } - } - if (r_sm == 0 && f_gl + f_ghost <= nf_c) { - am_sm[f_sm + f_ghost] = am[nf_c - f_gl - f_ghost]; - bm_sm[f_sm + f_ghost] = bm[nf_c - f_gl - f_ghost]; - // printf("%u %u\n", f_gl, f_ghost); - } - __syncthreads(); - - /* Only 1 col remain */ - if (f_ghost + f_rest == 1) { - if (r_sm < r_rest && c_sm < c_rest) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); - } - //__syncthreads(); - - } else { - if (r_sm < r_rest && c_sm < c_rest) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); - for (SIZE i = 1; i < f_ghost + f_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_backward2( - vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); - } - } - } - __syncthreads(); - /* flush results to v */ - if (r_sm < r_rest && f_sm < f_ghost + f_rest) { - for (SIZE i = 0; i < c_rest; i++) { - vec[get_idx(ldv1, ldv2, r_sm, i, (nf_c - 1) - f_gl)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; - } - } - __syncthreads(); -} - -template -void ipk_1_3d_adaptive_launcher(Handle &handle, SIZE nr, SIZE nc, - SIZE nf_c, T *am, T *bm, T *ddist_f, T *dv, - SIZE lddv1, SIZE lddv2, int queue_idx) { - // std::cout << "test\n"; - - SIZE total_thread_x = nc; - SIZE total_thread_y = nr; - SIZE total_thread_z = 1; - SIZE tbx, tby, tbz, gridx, gridy, gridz; - dim3 threadsPerBlock, blockPerGrid; - SIZE sm_size; - - tbx = C; // std::max(C, std::min(C, total_thread_x)); - tby = R; // std::max(R, std::min(R, total_thread_y)); - tbz = 1; - sm_size = (R * C + 2) * (F + G) * sizeof(T); - gridx = ceil((double)total_thread_x / tbx); - gridy = ceil((double)total_thread_y / tby); - gridz = 1; - threadsPerBlock = dim3(F, tby, tbz); - blockPerGrid = dim3(gridx, gridy, gridz); - - _ipk_1_3d<<>>( - nr, nc, nf_c, am, bm, ddist_f, dv, lddv1, lddv2); - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } - // std::cout << "test\n"; -} - -template -void ipk_1_3d(Handle &handle, SIZE nr, SIZE nc, SIZE nf_c, T *am, T *bm, - T *ddist_f, T *dv, SIZE lddv1, SIZE lddv2, int queue_idx, - int config) { - -#define IPK(R, C, F, G) \ - { \ - ipk_1_3d_adaptive_launcher( \ - handle, nr, nc, nf_c, am, bm, ddist_f, dv, lddv1, lddv2, queue_idx); \ - } - bool profile = false; - if (handle.profile_kernels) { - profile = true; - } - if (D == 3) { - if (profile || config == 6) { - IPK(2, 2, 128, 2) - } - if (profile || config == 5) { - IPK(2, 2, 64, 2) - } - if (profile || config == 4) { - IPK(2, 2, 32, 2) - } - if (profile || config == 3) { - IPK(4, 4, 16, 4) - } - if (profile || config == 2) { - IPK(8, 8, 8, 4) - } - if (profile || config == 1) { - IPK(4, 4, 4, 4) - } - if (profile || config == 0) { - IPK(2, 2, 2, 2) - } - } else if (D == 2) { - if (profile || config == 6) { - IPK(1, 2, 128, 2) - } - if (profile || config == 5) { - IPK(1, 2, 64, 2) - } - if (profile || config == 4) { - IPK(1, 2, 32, 2) - } - if (profile || config == 3) { - IPK(1, 4, 16, 4) - } - if (profile || config == 2) { - IPK(1, 8, 8, 4) - } - if (profile || config == 1) { - IPK(1, 4, 4, 4) - } - if (profile || config == 0) { - IPK(1, 2, 4, 2) - } - } else if (D == 1) { - if (profile || config == 6) { - IPK(1, 1, 128, 2) - } - if (profile || config == 5) { - IPK(1, 1, 64, 2) - } - if (profile || config == 4) { - IPK(1, 1, 32, 2) - } - if (profile || config == 3) { - IPK(1, 1, 16, 4) - } - if (profile || config == 2) { - IPK(1, 1, 8, 4) - } - if (profile || config == 1) { - IPK(1, 1, 8, 4) - } - if (profile || config == 0) { - IPK(1, 1, 8, 2) - } - } -#undef IPK -} - -template -__global__ void _ipk_2_3d(SIZE nr, SIZE nc_c, SIZE nf_c, T *am, T *bm, - T *dist_c, T *v, SIZE ldv1, SIZE ldv2) { - - SIZE f_gl = blockIdx.x * F; - SIZE r_gl = blockIdx.y * R; - SIZE c_gl = 0; - - SIZE f_sm = threadIdx.x; - SIZE r_sm = threadIdx.y; - SIZE c_sm = threadIdx.x; - - T *vec = v + get_idx(ldv1, ldv2, r_gl, 0, f_gl); - T *sm = SharedMemory(); - SIZE ldsm1 = F; - SIZE ldsm2 = C + G; - T *vec_sm = sm; - T *am_sm = sm + R * ldsm1 * ldsm2; - T *bm_sm = am_sm + ldsm2; - - T prev_vec_sm = 0.0; - - SIZE f_rest = min(F, nf_c - blockIdx.x * F); - SIZE r_rest = min(R, nr - blockIdx.y * R); - - // if (blockIdx.x == 1 && blockIdx.y == 0 && f_sm == 0 && r_sm == 0) { - // prSIZEf("f_rest: %d r_rest: %d\n", f_rest, r_rest); - // } - - SIZE c_rest = nc_c; - SIZE c_ghost = min(nc_c, G); - SIZE c_main = C; - - /* Load first ghost */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_ghost; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - vec[get_idx(ldv1, ldv2, r_sm, c_gl + i, f_sm)]; - // if (r_sm == 0) printf("r0_stride = %d, vec_sm[%d] = %f\n", r0_stride, - // i, vec_sm[i * ldsm + c_sm]); - } - } - if (r_sm == 0 && c_sm < c_ghost) { - am_sm[c_sm] = am[c_gl + c_sm]; - bm_sm[c_sm] = bm[c_gl + c_sm]; - } - c_rest -= c_ghost; - __syncthreads(); - - while (c_rest > C - c_ghost) { - // printf("%d %d %d\n", c_rest, C, c_ghost); - c_main = min(C, c_rest); - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_main; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)] = - vec[get_idx(ldv1, ldv2, r_sm, c_gl + i + c_ghost, f_sm)]; - } - } - if (r_sm == 0 && c_sm < c_main) { - am_sm[c_sm + c_ghost] = am[c_gl + c_sm + c_ghost]; - bm_sm[c_sm + c_ghost] = bm[c_gl + c_sm + c_ghost]; - } - __syncthreads(); - - /* Computation of v in parallel*/ - if (r_sm < r_rest && f_sm < f_rest) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); - - for (SIZE i = 1; i < C; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = tridiag_forward2( - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); - } - /* Store last v */ - prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, r_sm, C - 1, f_sm)]; - } - __syncthreads(); - - /* flush results to v */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < C; i++) { - vec[get_idx(ldv1, ldv2, r_sm, c_gl + i, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; - } - } - __syncthreads(); - - /* Update unloaded col */ - c_rest -= c_main; - - /* Advance c */ - c_gl += C; - - /* Copy next ghost to main */ - c_ghost = min(G, c_main - (C - G)); - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_ghost; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + C, f_sm)]; - } - } - if (r_sm == 0 && c_sm < c_ghost) { - am_sm[c_sm] = am_sm[c_sm + C]; - bm_sm[c_sm] = bm_sm[c_sm + C]; - } - __syncthreads(); - - } // end of while - - /* Load all rest col */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)] = - vec[get_idx(ldv1, ldv2, r_sm, c_gl + i + c_ghost, f_sm)]; - } - } - if (r_sm == 0 && c_sm < c_rest) { - am_sm[c_sm + c_ghost] = am[c_gl + c_sm + c_ghost]; - bm_sm[c_sm + c_ghost] = bm[c_gl + c_sm + c_ghost]; - } - __syncthreads(); - - /* Only 1 col remain */ - if (c_ghost + c_rest == 1) { - if (r_sm < r_rest && f_sm < f_rest) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); - } - //__syncthreads(); - - } else { - if (r_sm < r_rest && f_sm < f_rest) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); - for (SIZE i = 1; i < c_ghost + c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = tridiag_forward2( - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); - } - } - } - __syncthreads(); - /* flush results to v */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_ghost + c_rest; i++) { - vec[get_idx(ldv1, ldv2, r_sm, c_gl + i, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; - } - } - __syncthreads(); - - /* backward */ - c_rest = nc_c; - c_ghost = min(nc_c, G); - c_main = C; - c_gl = 0; - prev_vec_sm = 0.0; - - /* Load first ghost */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_ghost; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - vec[get_idx(ldv1, ldv2, r_sm, (nc_c - 1) - (c_gl + i), f_sm)]; - } - } - if (r_sm == 0 && c_sm < c_ghost) { - am_sm[c_sm] = am[nc_c - (c_gl + c_sm)]; - bm_sm[c_sm] = bm[nc_c - (c_gl + c_sm)]; - } - c_rest -= c_ghost; - __syncthreads(); - - while (c_rest > C - c_ghost) { - // printf("%d %d %d\n", c_rest, C, c_ghost); - c_main = min(C, c_rest); - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_main; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)] = vec[get_idx( - ldv1, ldv2, r_sm, (nc_c - 1) - (c_gl + i + c_ghost), f_sm)]; - } - } - if (r_sm == 0 && c_sm < c_main) { - am_sm[c_sm + c_ghost] = am[nc_c - (c_gl + c_sm + c_ghost)]; - bm_sm[c_sm + c_ghost] = bm[nc_c - (c_gl + c_sm + c_ghost)]; - } - __syncthreads(); - - /* Computation of v in parallel*/ - if (r_sm < r_rest && f_sm < f_rest) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)]); - - for (SIZE i = 1; i < C; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = tridiag_backward2( - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); - } - - /* Store last v */ - prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, r_sm, C - 1, f_sm)]; - } - __syncthreads(); - - /* flush results to v */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < C; i++) { - vec[get_idx(ldv1, ldv2, r_sm, (nc_c - 1) - (c_gl + i), f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; - } - } - __syncthreads(); - - /* Update unloaded col */ - c_rest -= c_main; - - /* Advance c */ - c_gl += C; - - /* Copy next ghost to main */ - c_ghost = min(G, c_main - (C - G)); - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_ghost; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + C, f_sm)]; - } - } - if (r_sm == 0 && c_sm < c_ghost) { - am_sm[c_sm] = am_sm[c_sm + C]; - bm_sm[c_sm] = bm_sm[c_sm + C]; - } - __syncthreads(); - - } // end of while - - // Load all rest col - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)] = vec[get_idx( - ldv1, ldv2, r_sm, (nc_c - 1) - (c_gl + i + c_ghost), f_sm)]; - } - } - if (r_sm == 0 && c_sm < c_rest) { - am_sm[c_sm + c_ghost] = am[nc_c - (c_gl + c_sm + c_ghost)]; - bm_sm[c_sm + c_ghost] = bm[nc_c - (c_gl + c_sm + c_ghost)]; - } - __syncthreads(); - - /* Only 1 col remain */ - if (c_ghost + c_rest == 1) { - if (r_sm < r_rest && f_sm < f_rest) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)]); - } - //__syncthreads(); - - } else { - if (r_sm < r_rest && f_sm < f_rest) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)]); - for (SIZE i = 1; i < c_ghost + c_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = tridiag_backward2( - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); - } - } - } - __syncthreads(); - /* flush results to v */ - if (r_sm < r_rest && f_sm < f_rest) { - for (SIZE i = 0; i < c_ghost + c_rest; i++) { - vec[get_idx(ldv1, ldv2, r_sm, (nc_c - 1) - (c_gl + i), f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; - // printf("c_stride = %d, c_sm = %d, vec_sm = %f, vec[%d] = - // %f\n",c_stride, c_sm, vec_sm[r_sm * ldsm + 0],i * row_stride * lddv + - // c_stride, vec[i * row_stride * lddv + c_stride]); - } - } - __syncthreads(); -} - -template -void ipk_2_3d_adaptive_launcher(Handle &handle, SIZE nr, SIZE nc_c, - SIZE nf_c, T *am, T *bm, T *ddist_c, T *dv, - SIZE lddv1, SIZE lddv2, int queue_idx) { - SIZE total_thread_x = nf_c; - SIZE total_thread_y = nr; - SIZE total_thread_z = 1; - SIZE tbx, tby, tbz, gridx, gridy, gridz; - dim3 threadsPerBlock, blockPerGrid; - size_t sm_size; - - tbx = F; // std::max(F, std::min(F, total_thread_x)); - tby = R; // std::max(R, std::min(R, total_thread_y)); - tbz = 1; - sm_size = (R * F + 2) * (C + G) * sizeof(T); - gridx = ceil((double)total_thread_x / tbx); - gridy = ceil((double)total_thread_y / tby); - gridz = 1; - threadsPerBlock = dim3(tbx, tby, tbz); - blockPerGrid = dim3(gridx, gridy, gridz); - _ipk_2_3d<<>>( - nr, nc_c, nf_c, am, bm, ddist_c, dv, lddv1, lddv2); - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } -} - -template -void ipk_2_3d(Handle &handle, SIZE nr, SIZE nc_c, SIZE nf_c, T *am, T *bm, - T *ddist_c, T *dv, SIZE lddv1, SIZE lddv2, int queue_idx, - int config) { - -#define IPK(R, C, F, G) \ - { \ - ipk_2_3d_adaptive_launcher( \ - handle, nr, nc_c, nf_c, am, bm, ddist_c, dv, lddv1, lddv2, queue_idx); \ - } - bool profile = false; - if (handle.profile_kernels) { - profile = true; - } - if (D == 3) { - if (profile || config == 6) { - IPK(2, 2, 128, 2) - } - if (profile || config == 5) { - IPK(2, 2, 64, 2) - } - if (profile || config == 4) { - IPK(2, 2, 32, 2) - } - if (profile || config == 3) { - IPK(4, 4, 16, 4) - } - if (profile || config == 2) { - IPK(8, 8, 8, 4) - } - if (profile || config == 1) { - IPK(4, 4, 4, 4) - } - if (profile || config == 0) { - IPK(2, 2, 2, 2) - } - } else if (D == 2) { - if (profile || config == 6) { - IPK(1, 2, 128, 2) - } - if (profile || config == 5) { - IPK(1, 2, 64, 2) - } - if (profile || config == 4) { - IPK(1, 2, 32, 2) - } - if (profile || config == 3) { - IPK(1, 4, 16, 4) - } - if (profile || config == 2) { - IPK(1, 8, 8, 4) - } - if (profile || config == 1) { - IPK(1, 4, 4, 4) - } - if (profile || config == 0) { - IPK(1, 2, 4, 2) - } - } else { - printf("Error: ipk_2_3d is only for 3D and 2D data\n"); - } -#undef IPK -} - -template -__global__ void _ipk_3_3d(SIZE nr_c, SIZE nc_c, SIZE nf_c, T *am, T *bm, - T *dist_r, T *v, SIZE ldv1, SIZE ldv2) { - - SIZE f_gl = blockIdx.x * F; - SIZE c_gl = blockIdx.y * C; - SIZE r_gl = 0; - - SIZE f_sm = threadIdx.x; - SIZE c_sm = threadIdx.y; - SIZE r_sm = threadIdx.x; - - T *vec = v + get_idx(ldv1, ldv2, 0, c_gl, f_gl); - T *sm = SharedMemory(); - SIZE ldsm1 = F; - SIZE ldsm2 = C; - T *vec_sm = sm; - T *am_sm = sm + (R + G) * ldsm1 * ldsm2; - T *bm_sm = am_sm + (R + G); - - T prev_vec_sm = 0.0; - - SIZE f_rest = min(F, nf_c - blockIdx.x * F); - SIZE c_rest = min(C, nc_c - blockIdx.y * C); - - SIZE r_rest = nr_c; - SIZE r_ghost = min(nr_c, G); - SIZE r_main = R; - - /* Load first ghost */ - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_ghost; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = - vec[get_idx(ldv1, ldv2, r_gl + i, c_sm, f_sm)]; - } - } - - if (c_sm == 0 && r_sm < r_ghost) { - am_sm[r_sm] = am[r_gl + r_sm]; - bm_sm[r_sm] = bm[r_gl + r_sm]; - } - r_rest -= r_ghost; - __syncthreads(); - - while (r_rest > R - r_ghost) { - r_main = min(R, r_rest); - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_main; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)] = - vec[get_idx(ldv1, ldv2, r_gl + i + r_ghost, c_sm, f_sm)]; - } - } - if (c_sm == 0 && r_sm < r_main) { - am_sm[r_sm + r_ghost] = am[r_gl + r_sm + r_ghost]; - bm_sm[r_sm + r_ghost] = bm[r_gl + r_sm + r_ghost]; - } - __syncthreads(); - - /* Computation of v in parallel*/ - if (c_sm < c_rest && f_sm < f_rest) { - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); - for (SIZE i = 1; i < R; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = tridiag_forward2( - vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); - } - - /* Store last v */ - prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, R - 1, c_sm, f_sm)]; - } - __syncthreads(); - - /* flush results to v */ - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < R; i++) { - vec[get_idx(ldv1, ldv2, r_gl + i, c_sm, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]; - } - } - __syncthreads(); - - // /* Update unloaded col */ - r_rest -= r_main; - - /* Advance c */ - r_gl += R; - - /* Copy next ghost to main */ - r_ghost = min(G, r_main - (R - G)); - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_ghost; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, i + R, c_sm, f_sm)]; - } - } - if (c_sm == 0 && r_sm < r_ghost) { - am_sm[r_sm] = am_sm[r_sm + R]; - bm_sm[r_sm] = bm_sm[r_sm + R]; - } - __syncthreads(); - - } // end of while - - /* Load all rest col */ - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)] = - vec[get_idx(ldv1, ldv2, r_gl + i + r_ghost, c_sm, f_sm)]; - } - } - - if (c_sm == 0 && r_sm < r_rest) { - am_sm[r_sm + r_ghost] = am[r_gl + r_sm + r_ghost]; - bm_sm[r_sm + r_ghost] = bm[r_gl + r_sm + r_ghost]; - } - __syncthreads(); - - /* Only 1 col remain */ - if (r_ghost + r_rest == 1) { - if (c_sm < c_rest && f_sm < f_rest) { - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); - } - //__syncthreads(); - - } else { - if (c_sm < c_rest && f_sm < f_rest) { - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); - for (SIZE i = 1; i < r_ghost + r_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = tridiag_forward2( - vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); - } - } - } - __syncthreads(); - /* flush results to v */ - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_ghost + r_rest; i++) { - vec[get_idx(ldv1, ldv2, r_gl + i, c_sm, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]; - } - } - __syncthreads(); - - /* backward */ - r_rest = nr_c; - r_ghost = min(nr_c, G); - r_main = R; - r_gl = 0; - prev_vec_sm = 0.0; - - /* Load first ghost */ - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_ghost; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = - vec[get_idx(ldv1, ldv2, (nr_c - 1) - (r_gl + i), c_sm, f_sm)]; - } - } - - if (c_sm == 0 && r_sm < r_ghost) { - am_sm[r_sm] = am[nr_c - (r_gl + r_sm)]; - bm_sm[r_sm] = bm[nr_c - (r_gl + r_sm)]; - } - r_rest -= r_ghost; - __syncthreads(); - - while (r_rest > R - r_ghost) { - r_main = min(R, r_rest); - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_main; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)] = vec[get_idx( - ldv1, ldv2, (nr_c - 1) - (r_gl + i + r_ghost), c_sm, f_sm)]; - } - } - if (c_sm == 0 && r_sm < r_main) { - am_sm[r_sm + r_ghost] = am[nr_c - (r_gl + r_sm + r_ghost)]; - bm_sm[r_sm + r_ghost] = bm[nr_c - (r_gl + r_sm + r_ghost)]; - } - __syncthreads(); - - /* Computation of v in parallel*/ - if (c_sm < c_rest && f_sm < f_rest) { - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); - for (SIZE i = 1; i < R; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = tridiag_backward2( - vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); - } - - /* Store last v */ - prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, R - 1, c_sm, f_sm)]; - } - __syncthreads(); - - /* flush results to v */ - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < R; i++) { - vec[get_idx(ldv1, ldv2, (nr_c - 1) - (r_gl + i), c_sm, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]; - } - } - __syncthreads(); - - // /* Update unloaded col */ - r_rest -= r_main; - - /* Advance c */ - r_gl += R; - - /* Copy next ghost to main */ - r_ghost = min(G, r_main - (R - G)); - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_ghost; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, i + R, c_sm, f_sm)]; - } - } - if (c_sm == 0 && r_sm < r_ghost) { - am_sm[r_sm] = am_sm[r_sm + R]; - bm_sm[r_sm] = bm_sm[r_sm + R]; - } - __syncthreads(); - - } // end of while - - /* Load all rest col */ - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)] = vec[get_idx( - ldv1, ldv2, (nr_c - 1) - (r_gl + i + r_ghost), c_sm, f_sm)]; - } - } - if (c_sm == 0 && r_sm < r_rest) { - am_sm[r_sm + r_ghost] = am[nr_c - (r_gl + r_sm + r_ghost)]; - bm_sm[r_sm + r_ghost] = bm[nr_c - (r_gl + r_sm + r_ghost)]; - } - __syncthreads(); - - /* Only 1 col remain */ - if (r_ghost + r_rest == 1) { - if (c_sm < c_rest && f_sm < f_rest) { - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); - } - //__syncthreads(); - - } else { - if (c_sm < c_rest && f_sm < f_rest) { - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], - vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); - for (SIZE i = 1; i < r_ghost + r_rest; i++) { - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = tridiag_backward2( - vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], am_sm[i], - bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); - } - } - } - __syncthreads(); - /* flush results to v */ - if (c_sm < c_rest && f_sm < f_rest) { - for (SIZE i = 0; i < r_ghost + r_rest; i++) { - vec[get_idx(ldv1, ldv2, (nr_c - 1) - (r_gl + i), c_sm, f_sm)] = - vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]; - } - } - __syncthreads(); -} - -template -void ipk_3_3d_adaptive_launcher(Handle &handle, SIZE nr_c, SIZE nc_c, - SIZE nf_c, T *am, T *bm, T *ddist_r, T *dv, - SIZE lddv1, SIZE lddv2, int queue_idx) { - - // printf("am: "); - // print_matrix_cuda(1, nr, am, nr); - // printf("bm: "); - // print_matrix_cuda(1, nr, bm, nr); - - SIZE total_thread_x = nf_c; - SIZE total_thread_y = nc_c; - SIZE total_thread_z = 1; - SIZE tbx, tby, tbz, gridx, gridy, gridz; - dim3 threadsPerBlock, blockPerGrid; - size_t sm_size; - - tbx = F; // std::max(F, std::min(F, total_thread_x)); - tby = C; // std::max(C, std::min(C, total_thread_y)); - tbz = 1; - sm_size = (C * F + 2) * (R + G) * sizeof(T); - gridx = ceil((double)total_thread_x / tbx); - gridy = ceil((double)total_thread_y / tby); - gridz = 1; - threadsPerBlock = dim3(tbx, tby, tbz); - blockPerGrid = dim3(gridx, gridy, gridz); - _ipk_3_3d<<>>( - nr_c, nc_c, nf_c, am, bm, ddist_r, dv, lddv1, lddv2); - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } -} - -template -void ipk_3_3d(Handle &handle, SIZE nr_c, SIZE nc_c, SIZE nf_c, T *am, - T *bm, T *ddist_r, T *dv, SIZE lddv1, SIZE lddv2, int queue_idx, - int config) { - -#define IPK(R, C, F, G) \ - { \ - ipk_3_3d_adaptive_launcher(handle, nr_c, nc_c, nf_c, am, \ - bm, ddist_r, dv, lddv1, \ - lddv2, queue_idx); \ - } - - bool profile = false; - if (handle.profile_kernels) { - profile = true; - } - if (D == 3) { - if (profile || config == 6) { - IPK(2, 2, 128, 2) - } - if (profile || config == 5) { - IPK(2, 2, 64, 2) - } - if (profile || config == 4) { - IPK(2, 2, 32, 2) - } - if (profile || config == 3) { - IPK(2, 2, 16, 2) - } - if (profile || config == 2) { - IPK(8, 8, 8, 4) - } - if (profile || config == 1) { - IPK(4, 4, 4, 4) - } - if (profile || config == 0) { - IPK(2, 2, 2, 2) - } - } else { - printf("Error: ipk_3_3d is only for 3D data\n"); - } -#undef IPK -} - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/LPKFunctor.h b/include/cuda/LPKFunctor.h deleted file mode 100644 index b179d47625..0000000000 --- a/include/cuda/LPKFunctor.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_LPK_FUNCTOR -#define MGRAD_CUDA_LPK_FUNCTOR - -namespace mgard_cuda { - -template -__device__ inline T mass_trans(T a, T b, T c, T d, T e, T h1, T h2, T h3, T h4, - T r1, T r2, T r3, T r4) { - T tb, tc, td, tb1, tb2, tc1, tc2, td1, td2; -#ifdef MGARD_CUDA_FMA - if (sizeof(T) == sizeof(double)) { - tb1 = fma(c, h2 / 6, a * h1 / 6); - tb2 = fma(b, h2 / 6, b * h1 / 6); - - tc1 = fma(d, h3 / 6, b * h2 / 6); - tc2 = fma(c, h3 / 6, c * h2 / 6); - - td1 = fma(c, h4 / 6, e * h3 / 6); - td2 = fma(d, h4 / 6, d * h3 / 6); - - tb = fma(2, tb2, tb1); - tc = fma(2, tc2, tc1); - td = fma(2, td2, td1); - return fma(td, r4, fma(tb, r1, tc)); - } else if (sizeof(T) == sizeof(float)) { - tb1 = fmaf(c, h2 / 6, a * h1 / 6); - tb2 = fmaf(b, h2 / 6, b * h1 / 6); - - tc1 = fmaf(d, h3 / 6, b * h2 / 6); - tc2 = fmaf(c, h3 / 6, c * h2 / 6); - - td1 = fmaf(c, h4 / 6, e * h3 / 6); - td2 = fmaf(d, h4 / 6, d * h3 / 6); - - tb = fmaf(2, tb2, tb1); - tc = fmaf(2, tc2, tc1); - td = fmaf(2, td2, td1); - return fmaf(td, r4, fmaf(tb, r1, tc)); - } -#else - - if (h1 + h2 != 0) { - r1 = h1 / (h1 + h2); - } else { - r1 = 0.0; - } - if (h3 + h4 != 0) { - r4 = h4 / (h3 + h4); - } else { - r4 = 0.0; - } - - // printf("%f %f %f %f %f (%f %f %f %f)\n", a, b, c, d, e, h1, h2, h3, h4); - tb = a * (h1 / 6) + b * ((h1 + h2) / 3) + c * (h2 / 6); - tc = b * (h2 / 6) + c * ((h2 + h3) / 3) + d * (h3 / 6); - td = c * (h3 / 6) + d * ((h3 + h4) / 3) + e * (h4 / 6); - tc += tb * r1 + td * r4; - return tc; -#endif -} - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/LevelwiseProcessingKernel.h b/include/cuda/LevelwiseProcessingKernel.h deleted file mode 100644 index fa05ca961d..0000000000 --- a/include/cuda/LevelwiseProcessingKernel.h +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_LEVELWISE_PROCESSING_KERNEL -#define MGRAD_CUDA_LEVELWISE_PROCESSING_KERNEL - -#include "Common.h" - -namespace mgard_cuda { - -template -void lwpk(Handle &handle, thrust::device_vector shape, T *dv, - thrust::device_vector ldvs, T *dwork, - thrust::device_vector ldws, int queue_idx); - -template -void lwpk(Handle &handle, SIZE *shape_h, SIZE *shape_d, T *dv, SIZE *ldvs, - T *dwork, SIZE *ldws, int queue_idx); - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/LevelwiseProcessingKernel.hpp b/include/cuda/LevelwiseProcessingKernel.hpp deleted file mode 100644 index 65fffea093..0000000000 --- a/include/cuda/LevelwiseProcessingKernel.hpp +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_LEVELWISE_PROCESSING_KERNEL_TEMPLATE -#define MGRAD_CUDA_LEVELWISE_PROCESSING_KERNEL_TEMPLATE - -#include "CommonInternal.h" - -#include "LevelwiseProcessingKernel.h" -namespace mgard_cuda { - -template -__global__ void _lwpk(SIZE *shape, T *dv, SIZE *ldvs, T *dwork, SIZE *ldws) { - - size_t threadId = (threadIdx.z * (blockDim.x * blockDim.y)) + - (threadIdx.y * blockDim.x) + threadIdx.x; - SIZE *sm = SharedMemory(); - SIZE *shape_sm = sm; - SIZE *ldvs_sm = shape_sm + D; - SIZE *ldws_sm = ldvs_sm + D; - - if (threadId < D) { - shape_sm[threadId] = shape[threadId]; - ldvs_sm[threadId] = ldvs[threadId]; - ldws_sm[threadId] = ldws[threadId]; - } - __syncthreads(); - - SIZE idx[D]; - SIZE firstD = div_roundup(shape_sm[0], F); - - SIZE bidx = blockIdx.x; - idx[0] = (bidx % firstD) * F + threadIdx.x; - - // printf("firstD %d idx[0] %d\n", firstD, idx[0]); - - bidx /= firstD; - if (D >= 2) - idx[1] = blockIdx.y * blockDim.y + threadIdx.y; - if (D >= 3) - idx[2] = blockIdx.z * blockDim.z + threadIdx.z; - - for (DIM d = 3; d < D; d++) { - idx[d] = bidx % shape_sm[d]; - bidx /= shape_sm[d]; - } - // int z = blockIdx.z * blockDim.z + threadIdx.z; - // int y = blockIdx.y * blockDim.y + threadIdx.y; - // int x = blockIdx.z * blockDim.z + threadIdx.z; - bool in_range = true; - for (DIM d = 0; d < D; d++) { - if (idx[d] >= shape_sm[d]) - in_range = false; - } - if (in_range) { - // printf("%d %d %d %d\n", idx[3], idx[2], idx[1], idx[0]); - if (OP == COPY) - dwork[get_idx(ldws, idx)] = dv[get_idx(ldvs, idx)]; - if (OP == ADD) - dwork[get_idx(ldws, idx)] += dv[get_idx(ldvs, idx)]; - if (OP == SUBTRACT) - dwork[get_idx(ldws, idx)] -= dv[get_idx(ldvs, idx)]; - } -} - -template -void lwpk_adaptive_launcher(Handle &handle, SIZE *shape_h, SIZE *shape_d, - T *dv, SIZE *ldvs, T *dwork, SIZE *ldws, - int queue_idx) { - - SIZE total_thread_z = shape_h[2]; - SIZE total_thread_y = shape_h[1]; - SIZE total_thread_x = shape_h[0]; - // linearize other dimensions - SIZE tbz = R; - SIZE tby = C; - SIZE tbx = F; - SIZE gridz = ceil((double)total_thread_z / tbz); - SIZE gridy = ceil((double)total_thread_y / tby); - SIZE gridx = ceil((double)total_thread_x / tbx); - for (DIM d = 3; d < D; d++) { - gridx *= shape_h[d]; - } - - // printf("exec: %d %d %d %d %d %d\n", tbx, tby, tbz, gridx, gridy, gridz); - dim3 threadsPerBlock(tbx, tby, tbz); - dim3 blockPerGrid(gridx, gridy, gridz); - size_t sm_size = (D * 3) * sizeof(SIZE); - _lwpk<<>>( - shape_d, dv, ldvs, dwork, ldws); - - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } -} - -template -void lwpk(Handle &handle, SIZE *shape_h, SIZE *shape_d, T *dv, SIZE *ldvs, - T *dwork, SIZE *ldws, int queue_idx) { -#define COPYLEVEL(R, C, F) \ - { \ - lwpk_adaptive_launcher(handle, shape_h, shape_d, dv, \ - ldvs, dwork, ldws, queue_idx); \ - } - if (D >= 3) { - COPYLEVEL(4, 4, 4) - } - if (D == 2) { - COPYLEVEL(1, 4, 4) - } - if (D == 1) { - COPYLEVEL(1, 1, 8) - } - -#undef COPYLEVEL -} - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/LinearProcessingKernel.h b/include/cuda/LinearProcessingKernel.h deleted file mode 100644 index 1f17c696db..0000000000 --- a/include/cuda/LinearProcessingKernel.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_LINEAR_PROCESSING_KERNEL -#define MGRAD_CUDA_LINEAR_PROCESSING_KERNEL - -#include "Common.h" - -namespace mgard_cuda { - -template -void lpk_reo_1(Handle &handle, SIZE *shape_h, SIZE *shape_c_h, - SIZE *shape_d, SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, - DIM processed_n, DIM *processed_dims_h, DIM *processed_dims_d, - DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, T *ddist_f, - T *dratio_f, T *dv1, LENGTH lddv11, LENGTH lddv12, T *dv2, - LENGTH lddv21, LENGTH lddv22, T *dw, LENGTH lddw1, LENGTH lddw2, - int queue_idx, int config); - -template -void lpk_reo_2(Handle &handle, SIZE *shape_h, SIZE *shape_c_h, - SIZE *shape_d, SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, - DIM processed_n, DIM *processed_dims_h, DIM *processed_dims_d, - DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, T *ddist_c, - T *dratio_c, T *dv1, LENGTH lddv11, LENGTH lddv12, T *dv2, - LENGTH lddv21, LENGTH lddv22, T *dw, LENGTH lddw1, LENGTH lddw2, - int queue_idx, int config); - -template -void lpk_reo_3(Handle &handle, SIZE *shape_h, SIZE *shape_c_h, - SIZE *shape_d, SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, - DIM processed_n, DIM *processed_dims_h, DIM *processed_dims_d, - DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, T *ddist_r, - T *dratio_r, T *dv1, LENGTH lddv11, LENGTH lddv12, T *dv2, - LENGTH lddv21, LENGTH lddv22, T *dw, LENGTH lddw1, LENGTH lddw2, - int queue_idx, int config); - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/LinearProcessingKernel.hpp b/include/cuda/LinearProcessingKernel.hpp deleted file mode 100644 index 6a0da11fae..0000000000 --- a/include/cuda/LinearProcessingKernel.hpp +++ /dev/null @@ -1,1422 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_LINEAR_PROCESSING_KERNEL_TEMPLATE -#define MGRAD_CUDA_LINEAR_PROCESSING_KERNEL_TEMPLATE - -#include "CommonInternal.h" -#include "LPKFunctor.h" -#include "LinearProcessingKernel.h" -namespace mgard_cuda { -template -__global__ void _lpk_reo_1(SIZE *shape, SIZE *shape_c, SIZE *ldvs, SIZE *ldws, - DIM processed_n, DIM *processed_dims, DIM curr_dim_r, - DIM curr_dim_c, DIM curr_dim_f, T *ddist_f, - T *dratio_f, T *dv1, LENGTH lddv11, LENGTH lddv12, - T *dv2, LENGTH lddv21, LENGTH lddv22, T *dw, - LENGTH lddw1, LENGTH lddw2) { - - // bool debug = false; - // if (blockIdx.z == 0 && blockIdx.y == 1 && blockIdx.x == 1 && - // threadIdx.z == 0 && threadIdx.y == 0 ) debug = false; - - // bool debug = false; - // if (threadIdx.z == 0 && threadIdx.y == 0 && threadIdx.x == 0 ) debug = - // true; - - LENGTH threadId = (threadIdx.z * (blockDim.x * blockDim.y)) + - (threadIdx.y * blockDim.x) + threadIdx.x; - - T *sm = SharedMemory(); - SIZE ldsm1 = F * 2 + 3; - SIZE ldsm2 = C; - T *v_sm = sm; - sm += ldsm1 * ldsm2 * R; - - T *dist_f_sm = sm; - sm += ldsm1; - T *ratio_f_sm = sm; - sm += ldsm1; - - SIZE *sm_size = (SIZE *)sm; - SIZE *shape_sm = sm_size; - sm_size += D; - SIZE *shape_c_sm = sm_size; - sm_size += D; - SIZE *ldvs_sm = sm_size; - sm_size += D; - SIZE *ldws_sm = sm_size; - sm_size += D; - sm = (T *)sm_size; - - DIM *sm_dim = (DIM *)sm; - DIM *processed_dims_sm = sm_dim; - sm_dim += D; - sm = (T *)sm_dim; - - SIZE idx[D]; - if (threadId < D) { - shape_sm[threadId] = shape[threadId]; - shape_c_sm[threadId] = shape_c[threadId]; - ldvs_sm[threadId] = ldvs[threadId]; - ldws_sm[threadId] = ldws[threadId]; - } - if (threadId < processed_n) { - processed_dims_sm[threadId] = processed_dims[threadId]; - } - __syncthreads(); - - for (DIM d = 0; d < D; d++) - idx[d] = 0; - - SIZE nr = shape_sm[curr_dim_r]; - SIZE nc = shape_sm[curr_dim_c]; - SIZE nf = shape_sm[curr_dim_f]; - SIZE nf_c = shape_c_sm[curr_dim_f]; - - bool zero_other = true; - bool PADDING = (nf % 2 == 0); - - SIZE bidx = blockIdx.x; - SIZE firstD; - if (nf_c % 2 == 1) { - firstD = div_roundup(nf_c, blockDim.x); - } else { - firstD = div_roundup(nf_c, blockDim.x); - } - SIZE blockId = bidx % firstD; - bidx /= firstD; - - for (DIM d = 0; d < D; d++) { - if (d != curr_dim_r && d != curr_dim_c && d != curr_dim_f) { - SIZE t = shape_sm[d]; - for (DIM k = 0; k < processed_n; k++) { - if (d == processed_dims[k]) { - t = shape_c_sm[d]; - } - } - idx[d] = bidx % t; - bidx /= t; - if (idx[d] >= shape_c_sm[d]) - zero_other = false; - } - } - - SIZE zero_r = shape_c_sm[curr_dim_r]; - SIZE zero_c = shape_c_sm[curr_dim_c]; - SIZE zero_f = shape_c_sm[curr_dim_f]; - - if (D < 3) { - nr = 1; - zero_r = 1; - } - if (D < 2) { - nc = 1; - zero_c = 1; - } - - LENGTH other_offset_v = get_idx(ldvs_sm, idx); - LENGTH other_offset_w = get_idx(ldws_sm, idx); - - dv1 = dv1 + other_offset_v; - dv2 = dv2 + other_offset_v; - dw = dw + other_offset_w; - - // if (debug2) { - // printf("idx: %d %d %d %d\n", idx[3], idx[2], idx[1], idx[0]); - // printf("other_offset_v: %llu\n", other_offset_v); - // printf("other_offset_w: %llu\n", other_offset_w); - // } - SIZE r_gl = blockIdx.z * blockDim.z + threadIdx.z; - SIZE c_gl = blockIdx.y * blockDim.y + threadIdx.y; - SIZE f_gl = blockId * blockDim.x + threadIdx.x; - - SIZE r_sm = threadIdx.z; - SIZE c_sm = threadIdx.y; - SIZE f_sm = threadIdx.x; - - SIZE actual_F = F; - if (nf_c - blockId * blockDim.x < F) { - actual_F = nf_c - blockId * blockDim.x; - } - - // if (nf_c % 2 == 1){ - // if(nf_c-1 - blockId * blockDim.x < F) { actual_F = nf_c - 1 - blockId * - // blockDim.x; } - // } else { - // if(nf_c - blockId * blockDim.x < F) { actual_F = nf_c - blockId * - // blockDim.x; } - // } - - // if (debug) printf("actual_F %d\n", actual_F); - - if (r_gl < nr && c_gl < nc && f_gl < nf_c) { - if (zero_other && r_gl < zero_r && c_gl < zero_c && f_gl < zero_f) { - // if (debug) printf("load left vsm[%d]: 0.0\n", f_sm * 2 + 2); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 2)] = 0.0; - } else { - // if (debug) printf("load left vsm[%d]<-dv1[%d, %d, %d]: %f\n", f_sm * 2 - // + 2, r_gl, c_gl, f_gl, dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 2)] = - dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)]; - } - - if (f_sm == actual_F - 1) { - if (zero_other && r_gl < zero_r && c_gl < zero_c && f_gl < zero_f) { - // if (debug) printf("load left+1 vsm[%d]: 0.0\n", actual_F * 2 + 2); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 2)] = 0.0; - } else { - if (f_gl + 1 < nf_c) { - // if (debug) printf("load left+1 vsm[%d]: %f\n", actual_F * 2 + 2, - // dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl + 1)]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 2)] = - dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl + 1)]; - } else { - // if (debug) printf("load left+1 vsm[%d]: 0.0\n", actual_F * 2 + 2); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 2)] = 0.0; - } - } - } - - if (f_sm == 0) { - // left - if (zero_other && r_gl < zero_r && c_gl < zero_c && f_gl < zero_f) { - // coarse (-1) - // if (debug) printf("load left-1 vsm[0]: 0.0\n"); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = 0.0; - } else { - if (f_gl >= 1) { - // other (-1) - // if (debug) printf("load left-1 vsm[0]: %f\n", dv1[get_idx(lddv11, - // lddv12, r_gl, c_gl, f_gl-1)]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = - dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl - 1)]; - } else { - // other (-1) - // if (debug) printf("load left-1 vsm[0]: 0.0\n"); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = 0.0; - } - } - } - - // right - if (!PADDING) { // other = nf_c - 1 - if (nf_c % 2 != 0) { - if (f_gl >= 1 && f_gl < nf_c) { // shift for better memory access - // pattern - // if (debug) printf("load right vsm[%d]: %f <- %d %d %d\n", f_sm * 2 - // + 1, dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - 1)], r_gl, - // c_gl, f_gl - 1); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - 1)]; - } else { - // if (debug) printf("load right vsm[%d]: 0\n", f_sm * 2 + 1); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)] = 0.0; - } - } else { // nf_c % 2 == 0, do not shift - if (f_gl < nf_c - 1) { - // if (debug) printf("load right vsm[%d]: %f <- %d %d %d\n", f_sm * 2 - // + 3, dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)], r_gl, c_gl, - // f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)]; - } else { - // if (debug) printf("load right vsm[%d]: 0\n", f_sm * 2 + 3); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)] = 0.0; - } - } - } else { // PADDING other = nf_c - 2 - if (nf_c % 2 != 0) { - if (f_gl >= 1 && - f_gl < nf_c - 1) { // shift for better memory access pattern - // if (debug) printf("load right vsm[%d]: %f <- %d %d %d\n", f_sm * 2 - // + 1, dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - 1)], r_gl, - // c_gl, f_gl - 1); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - 1)]; - } else { - // if (debug) printf("load right vsm[%d]: 0\n", f_sm * 2 + 1); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)] = 0.0; - } - } else { // nf_c % 2 == 0 - if (f_gl < nf_c - 2) { // do not shift - // if (debug) printf("load right vsm[%d]: %f <- %d %d %d\n", f_sm * 2 - // + 3, dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)], r_gl, c_gl, - // f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)]; - } else { - // if (debug) printf("load right vsm[%d]: 0\n", f_sm * 2 + 3); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)] = 0.0; - } - } - } - - if (f_sm == actual_F - 1) { - // right (+1) - if (!PADDING) { - if (nf_c % 2 != 0) { - if (f_gl < nf_c - 1) { - // if (debug) printf("load right+1 vsm[%d]: %f <- %d %d %d\n", - // actual_F * 2 + 1, dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)], - // r_gl, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 1)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)]; - } else { - // if (debug) printf("load right+1 vsm[%d]: 0.0\n", actual_F * 2 + - // 1); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 1)] = 0.0; - } - } else { // nf_c % 2 == 0 - if (f_gl >= actual_F) { - // if (debug) printf("load right-1 vsm[1]: %f <- %d %d %d\n", - // dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - actual_F)], r_gl, - // c_gl, f_gl - actual_F); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 1)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - actual_F)]; - } else { - // if (debug) printf("load right-1 vsm[1]: 0.0\n"); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 1)] = 0.0; - } - } - } else { - if (nf_c % 2 != 0) { - if (f_gl < nf_c - 2) { - // if (debug) printf("actual_F(%d), load right+1 vsm[%d]: %f <- %d - // %d %d\n", actual_F, actual_F * 2 + 1, dv2[get_idx(lddv21, lddv22, - // r_gl, c_gl, f_gl)], r_gl, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 1)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)]; - } else { - // if (debug) printf("load right+1 vsm[%d]: 0.0\n", actual_F * 2 + - // 1); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 1)] = 0.0; - } - } else { // nf_c % 2 == 0 - if (f_gl >= actual_F && f_gl - actual_F < nf_c - 2) { - // if (debug) printf("load right-1 vsm[1]: %f <- %d %d %d\n", - // dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - actual_F)], r_gl, - // c_gl, f_gl - actual_F); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 1)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - actual_F)]; - } else { - // if (debug) printf("load right-1 vsm[1]: 0.0\n"); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 1)] = 0.0; - } - } - } - } - } - - // if (debug) printf("actual_F: %d\n", actual_F); - if (r_sm == 0 && c_sm == 0 && f_sm < actual_F) { - // if (debug) printf("blockId * F * 2 + f_sm = %d\n", blockId * F * 2 + - // f_sm); - if (blockId * F * 2 + f_sm < - nf) { // padding: num of dist == nf, non-padding: non of dist == nf - 1 - // if (debug) printf("load dist/ratio1[%d]: %f <- %d\n", 2 + f_sm, - // ddist_f[blockId * F * 2 + f_sm], blockId * F * 2 + f_sm); - dist_f_sm[2 + f_sm] = ddist_f[blockId * F * 2 + f_sm]; - ratio_f_sm[2 + f_sm] = dratio_f[blockId * F * 2 + f_sm]; - } else { - // if (debug) printf("load dist/ratio1[%d]: 0.0\n", 2 + f_sm); - dist_f_sm[2 + f_sm] = 0.0; - ratio_f_sm[2 + f_sm] = 0.0; - } - - if (blockId * F * 2 + actual_F + f_sm < nf) { - // if (debug) printf("load dist/ratio2[%d]: %f <- %d\n", 2 + actual_F + - // f_sm, ddist_f[blockId * F * 2 + actual_F + f_sm], blockId * F * 2 + - // actual_F + f_sm); - dist_f_sm[2 + actual_F + f_sm] = - ddist_f[blockId * F * 2 + actual_F + f_sm]; - ratio_f_sm[2 + actual_F + f_sm] = - dratio_f[blockId * F * 2 + actual_F + f_sm]; - } else { - // if (debug) printf("load dist/ratio2[%d]: 0.0\n", 2 + actual_F + f_sm); - dist_f_sm[2 + actual_F + f_sm] = 0.0; - ratio_f_sm[2 + actual_F + f_sm] = 0.0; - } - } - - if (blockId > 0) { - if (f_sm < 2) { - // dist_f_sm[f_sm] = ddist_f[f_gl - 2]; - // ratio_f_sm[f_sm] = dratio_f[f_gl - 2]; - // if (debug) printf("load dist/ratio-1[%d]: %f <- %d\n", f_sm, - // ddist_f[blockId * F * 2 + f_sm - 2], blockId * F * 2 + f_sm - 2); - dist_f_sm[f_sm] = ddist_f[blockId * F * 2 + f_sm - 2]; - ratio_f_sm[f_sm] = dratio_f[blockId * F * 2 + f_sm - 2]; - } - } else { - if (f_sm < 2) { - // if (debug) printf("load dist/ratio-1[%d]: 0.0 <- %d\n", f_sm); - dist_f_sm[f_sm] = 0.0; - ratio_f_sm[f_sm] = 0.0; - } - } - - __syncthreads(); - - if (r_gl < nr && c_gl < nc && f_gl < nf_c) { - T h1 = dist_f_sm[f_sm * 2]; - T h2 = dist_f_sm[f_sm * 2 + 1]; - T h3 = dist_f_sm[f_sm * 2 + 2]; - T h4 = dist_f_sm[f_sm * 2 + 3]; - T r1 = ratio_f_sm[f_sm * 2]; - T r2 = ratio_f_sm[f_sm * 2 + 1]; - T r3 = ratio_f_sm[f_sm * 2 + 2]; - T r4 = 1 - r3; - T a = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2)]; - T b = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)]; - T c = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 2)]; - T d = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)]; - T e = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 4)]; - - // bool debug = false; - // if (idx[3] == 0) debug = false; - // if (debug) { - // printf("f_sm(%d) %f %f %f %f %f f_sm_h %f %f %f %f f_sm_r %f %f %f %f, - // out: %f\n",f_sm, a,b,c,d,e, h1,h2,h3,h4,r1,r2,r3,r4, mass_trans(a, b, - // c, d, e, h1, h2, h3, h4, r1, r2, r3, r4)); - // } - - // T tb = a * h1/6 + b * (h1+h2)/3 + c * h2/6; - // T tc = b * h2/6 + c * (h2+h3)/3 + d * h3/6; - // T td = c * h3/6 + d * (h3+h4)/3 + e * h4/6; - - // if (debug) printf("f_sm(%d) tb tc td tc: %f %f %f %f\n", f_sm, tb, tc, - // td, tc+tb * r1 + td * r4); - - // tc += tb * r1 + td * r4; - - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = - mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4); - - // if (debug) printf("store[%d %d %d] %f \n", r_gl, c_gl, f_gl, - // mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4)); - - // printf("test block %d F %d nf %d\n", blockId, F, nf); - // if (f_gl+1 == nf_c-1) { - - // // T te = h4 * d + 2 * h4 * e; - // //printf("f_sm(%d) mm-e: %f\n", f_sm, te); - // // te += td * r3; - // dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl+1)] = - // mass_trans(c, d, e, (T)0.0, (T)0.0, h1, h2, (T)0.0, (T)0.0, r1, r2, - // (T)0.0, (T)0.0); - // } - } -} - -template -void lpk_reo_1_adaptive_launcher( - Handle &handle, SIZE *shape_h, SIZE *shape_c_h, SIZE *shape_d, - SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, DIM processed_n, - DIM *processed_dims_h, DIM *processed_dims_d, DIM curr_dim_r, - DIM curr_dim_c, DIM curr_dim_f, T *ddist_f, T *dratio_f, T *dv1, - LENGTH lddv11, LENGTH lddv12, T *dv2, LENGTH lddv21, LENGTH lddv22, T *dw, - LENGTH lddw1, LENGTH lddw2, int queue_idx) { - SIZE nr = shape_h[curr_dim_r]; - SIZE nc = shape_h[curr_dim_c]; - SIZE nf = shape_h[curr_dim_f]; - SIZE nf_c = shape_c_h[curr_dim_f]; - - SIZE total_thread_z = nr; - SIZE total_thread_y = nc; - SIZE total_thread_x = nf_c; - // if (nf_c % 2 == 1) { total_thread_x = nf_c - 1; } - // else { total_thread_x = nf_c; } - SIZE tbx, tby, tbz, gridx, gridy, gridz; - dim3 threadsPerBlock, blockPerGrid; - size_t sm_size; - - tbz = R; - tby = C; - tbx = F; - sm_size = (R * C * (F * 2 + 3) + (F * 2 + 3) * 2) * sizeof(T); - sm_size += (D * 4) * sizeof(SIZE); - sm_size += (D * 1) * sizeof(DIM); - gridz = ceil((double)total_thread_z / tbz); - gridy = ceil((double)total_thread_y / tby); - gridx = ceil((double)total_thread_x / tbx); - for (DIM d = 0; d < D; d++) { - if (d != curr_dim_f && d != curr_dim_c && d != curr_dim_r) { - SIZE t = shape_h[d]; - for (DIM k = 0; k < processed_n; k++) { - if (d == processed_dims_h[k]) { - t = shape_c_h[d]; - } - } - gridx *= t; - } - } - threadsPerBlock = dim3(tbx, tby, tbz); - blockPerGrid = dim3(gridx, gridy, gridz); - - // printf("lpk_reo_1 exec config (%d %d %d) (%d %d %d)\n", tbx, tby, tbz, - // gridx, gridy, gridz); - _lpk_reo_1<<>>( - shape_d, shape_c_d, ldvs, ldws, processed_n, processed_dims_d, curr_dim_r, - curr_dim_c, curr_dim_f, ddist_f, dratio_f, dv1, lddv11, lddv12, dv2, - lddv21, lddv22, dw, lddw1, lddw2); - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } -} - -template -void lpk_reo_1(Handle &handle, SIZE *shape_h, SIZE *shape_c_h, - SIZE *shape_d, SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, - DIM processed_n, DIM *processed_dims_h, DIM *processed_dims_d, - DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, T *ddist_f, - T *dratio_f, T *dv1, LENGTH lddv11, LENGTH lddv12, T *dv2, - LENGTH lddv21, LENGTH lddv22, T *dw, LENGTH lddw1, LENGTH lddw2, - int queue_idx, int config) { -#define LPK(R, C, F) \ - { \ - lpk_reo_1_adaptive_launcher( \ - handle, shape_h, shape_c_h, shape_d, shape_c_d, ldvs, ldws, \ - processed_n, processed_dims_h, processed_dims_d, curr_dim_r, \ - curr_dim_c, curr_dim_f, ddist_f, dratio_f, dv1, lddv11, lddv12, dv2, \ - lddv21, lddv22, dw, lddw1, lddw2, queue_idx); \ - } - - bool profile = false; - if (handle.profile_kernels) { - profile = true; - } - if (D >= 3) { - if (profile || config == 6) { - LPK(2, 2, 128) - } - if (profile || config == 5) { - LPK(2, 2, 64) - } - if (profile || config == 4) { - LPK(2, 2, 32) - } - if (profile || config == 3) { - LPK(4, 4, 16) - } - if (profile || config == 2) { - LPK(8, 8, 8) - } - if (profile || config == 1) { - LPK(4, 4, 4) - } - if (profile || config == 0) { - LPK(2, 2, 2) - } - } else if (D == 2) { - if (profile || config == 6) { - LPK(1, 2, 128) - } - if (profile || config == 5) { - LPK(1, 2, 64) - } - if (profile || config == 4) { - LPK(1, 2, 32) - } - if (profile || config == 3) { - LPK(1, 4, 16) - } - if (profile || config == 2) { - LPK(1, 8, 8) - } - if (profile || config == 1) { - LPK(1, 4, 4) - } - if (profile || config == 0) { - LPK(1, 2, 4) - } - } else if (D == 1) { - if (profile || config == 6) { - LPK(1, 1, 128) - } - if (profile || config == 5) { - LPK(1, 1, 64) - } - if (profile || config == 4) { - LPK(1, 1, 32) - } - if (profile || config == 3) { - LPK(1, 1, 16) - } - if (profile || config == 2) { - LPK(1, 1, 8) - } - if (profile || config == 1) { - LPK(1, 1, 8) - } - if (profile || config == 0) { - LPK(1, 1, 8) - } - } - -#undef LPK -} - -template -__global__ void _lpk_reo_2(SIZE *shape, SIZE *shape_c, SIZE *ldvs, SIZE *ldws, - DIM processed_n, DIM *processed_dims, DIM curr_dim_r, - DIM curr_dim_c, DIM curr_dim_f, T *ddist_c, - T *dratio_c, T *dv1, LENGTH lddv11, LENGTH lddv12, - T *dv2, LENGTH lddv21, LENGTH lddv22, T *dw, - LENGTH lddw1, LENGTH lddw2) { - - // bool debug = false; - // if (blockIdx.z == 0 && blockIdx.y == 0 && blockIdx.x == 0 && - // threadIdx.z == 0 && threadIdx.x == 0 ) debug = false; - - // bool debug2 = false; - // if (threadIdx.z == 0 && threadIdx.y == 0 && threadIdx.x == 0 ) debug2 = - // false; - - LENGTH threadId = (threadIdx.z * (blockDim.x * blockDim.y)) + - (threadIdx.y * blockDim.x) + threadIdx.x; - - T *sm = SharedMemory(); - SIZE ldsm1 = F; - SIZE ldsm2 = C * 2 + 3; - T *v_sm = sm; - sm += ldsm1 * ldsm2 * R; - - T *dist_c_sm = sm; - sm += ldsm2; - T *ratio_c_sm = sm; - sm += ldsm2; - - SIZE *sm_size = (SIZE *)sm; - SIZE *shape_sm = sm_size; - sm_size += D; - SIZE *shape_c_sm = sm_size; - sm_size += D; - SIZE *ldvs_sm = sm_size; - sm_size += D; - SIZE *ldws_sm = sm_size; - sm_size += D; - sm = (T *)sm_size; - - DIM *sm_dim = (DIM *)sm; - DIM *processed_dims_sm = sm_dim; - sm_dim += D; - sm = (T *)sm_dim; - - SIZE idx[D]; - if (threadId < D) { - shape_sm[threadId] = shape[threadId]; - shape_c_sm[threadId] = shape_c[threadId]; - ldvs_sm[threadId] = ldvs[threadId]; - ldws_sm[threadId] = ldws[threadId]; - } - if (threadId < processed_n) { - processed_dims_sm[threadId] = processed_dims[threadId]; - } - __syncthreads(); - - for (DIM d = 0; d < D; d++) - idx[d] = 0; - - SIZE nr = shape_sm[curr_dim_r]; - SIZE nc = shape_sm[curr_dim_c]; - SIZE nf_c = shape_c_sm[curr_dim_f]; - SIZE nc_c = shape_c_sm[curr_dim_c]; - bool PADDING = (nc % 2 == 0); - - if (D < 3) { - nr = 1; - } - - SIZE bidx = blockIdx.x; - SIZE firstD = div_roundup(nf_c, blockDim.x); - SIZE blockId_f = bidx % firstD; - bidx /= firstD; - - for (DIM d = 0; d < D; d++) { - if (d != curr_dim_r && d != curr_dim_c && d != curr_dim_f) { - SIZE t = shape_sm[d]; - for (DIM k = 0; k < processed_n; k++) { - if (d == processed_dims[k]) { - t = shape_c_sm[d]; - } - } - idx[d] = bidx % t; - bidx /= t; - } - } - - LENGTH other_offset_v = get_idx(ldvs_sm, idx); - LENGTH other_offset_w = get_idx(ldws_sm, idx); - - dv1 = dv1 + other_offset_v; - dv2 = dv2 + other_offset_v; - dw = dw + other_offset_w; - - // if (debug2) { - // printf("idx: %d %d %d %d\n", idx[3], idx[2], idx[1], idx[0]); - // printf("other_offset_v: %llu\n", other_offset_v); - // printf("other_offset_w: %llu\n", other_offset_w); - // } - - SIZE r_gl = blockIdx.z * blockDim.z + threadIdx.z; - SIZE c_gl = blockIdx.y * blockDim.y + threadIdx.y; - SIZE f_gl = blockId_f * blockDim.x + threadIdx.x; - - SIZE blockId = blockIdx.y; - - SIZE r_sm = threadIdx.z; - SIZE c_sm = threadIdx.y; - SIZE f_sm = threadIdx.x; - - SIZE actual_C = C; - if (nc_c - blockIdx.y * blockDim.y < C) { - actual_C = nc_c - blockIdx.y * blockDim.y; - } - - // if (nc_c % 2 == 1){ - // if(nc_c-1 - blockIdx.y * blockDim.y < C) { actual_C = nc_c - 1 - - // blockIdx.y * blockDim.y; } - // } else { - // if(nc_c - blockIdx.y * blockDim.y < C) { actual_C = nc_c - blockIdx.y * - // blockDim.y; } - // } - - // bool debug = false; - // if (idx[3] == 0 && r_gl == 0 ) debug = false; - - // if (debug) printf("actual_C %d\n", actual_C); - - if (r_gl < nr && c_gl < nc_c && f_gl < nf_c) { - // if (debug) printf("load up vsm[%d]: %f <- %d %d %d\n", c_sm * 2 + 2, - // dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 2, f_sm)] = - dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)]; - - if (c_sm == actual_C - 1) { - if (c_gl + 1 < nc_c) { - // if (debug) printf("load up+1 vsm[%d]: %f <- %d %d %d\n", actual_C * 2 - // + 2, dv1[get_idx(lddv11, lddv12, r_gl, blockId * C + actual_C, - // f_gl)], r_gl, blockId * C + actual_C, f_gl); - // c_gl+1 == blockId * C + C - v_sm[get_idx(ldsm1, ldsm2, r_sm, actual_C * 2 + 2, f_sm)] = - dv1[get_idx(lddv11, lddv12, r_gl, c_gl + 1, f_gl)]; - } else { - // if (debug) printf("load up+1 vsm[%d]: 0.0\n", actual_C * 2 + 2); - v_sm[get_idx(ldsm1, ldsm2, r_sm, actual_C * 2 + 2, f_sm)] = 0.0; - } - } - - if (c_sm == 0) { - if (c_gl >= 1) { - // if (debug) printf("load up-1 vsm[0]: %f <- %d %d %d\n", - // dv1[get_idx(lddv11, lddv12, r_gl, c_gl-1, f_gl)], r_gl, c_gl-1, - // f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = - dv1[get_idx(lddv11, lddv12, r_gl, c_gl - 1, f_gl)]; - } else { - // if (debug) printf("load up-1 vsm[0]: 0.0\n"); - v_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = 0.0; - } - } - - if (!PADDING) { - if (c_gl < nc_c - 1) { - // if (debug) printf("load down vsm[%d]: %f <- %d %d %d\n", c_sm * 2 + - // 3, dv2[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)]; - } else { - // if (debug) printf("load down vsm[%d]: 0.0\n", c_sm * 2 + 3); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)] = 0.0; - } - } else { - if (c_gl < nc_c - 2) { - // if (debug) printf("load down vsm[%d]: %f <- %d %d %d\n", c_sm * 2 + - // 3, dv2[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)]; - } else { - // if (debug) printf("load down vsm[%d]: 0.0\n", c_sm * 2 + 3); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)] = 0.0; - } - } - - if (c_gl >= 1 && - (PADDING && c_gl - 1 < nc_c - 2 || !PADDING && c_gl - 1 < nc_c - 1)) { - if (c_sm == 0) { - // if (debug) printf("load down-1 vsm[1]: %f <- %d %d %d\n", - // dv2[get_idx(lddv11, lddv12, r_gl, c_gl-1, f_gl)], r_gl, c_gl-1, - // f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, 1, f_sm)] = - dv2[get_idx(lddv11, lddv12, r_gl, c_gl - 1, f_gl)]; - } - } else { - if (c_sm == 0) { - // if (debug) printf("load down-1 vsm[1]: 0.0\n"); - v_sm[get_idx(ldsm1, ldsm2, r_sm, 1, f_sm)] = 0.0; - } - } - } - - // load dist/ratio using f_sm for better performance - // assumption F >= C - if (r_sm == 0 && c_sm == 0 && f_sm < actual_C) { - if (blockId * C * 2 + f_sm < nc) { - dist_c_sm[2 + f_sm] = ddist_c[blockId * C * 2 + f_sm]; - ratio_c_sm[2 + f_sm] = dratio_c[blockId * C * 2 + f_sm]; - } else { - dist_c_sm[2 + f_sm] = 0.0; - ratio_c_sm[2 + f_sm] = 0.0; - } - - if (blockId * C * 2 + actual_C + f_sm < nc) { - dist_c_sm[2 + actual_C + f_sm] = - ddist_c[blockId * C * 2 + actual_C + f_sm]; - ratio_c_sm[2 + actual_C + f_sm] = - dratio_c[blockId * C * 2 + actual_C + f_sm]; - } else { - dist_c_sm[2 + actual_C + f_sm] = 0.0; - ratio_c_sm[2 + actual_C + f_sm] = 0.0; - } - } - - if (blockId > 0) { - if (f_sm < 2) { - dist_c_sm[f_sm] = ddist_c[blockId * C * 2 - 2 + f_sm]; - ratio_c_sm[f_sm] = dratio_c[blockId * C * 2 - 2 + f_sm]; - } - } else { - if (f_sm < 2) { - dist_c_sm[f_sm] = 0.0; - ratio_c_sm[f_sm] = 0.0; - } - } - - __syncthreads(); - - if (r_gl < nr && c_gl < nc_c && f_gl < nf_c) { - T h1 = dist_c_sm[c_sm * 2]; - T h2 = dist_c_sm[c_sm * 2 + 1]; - T h3 = dist_c_sm[c_sm * 2 + 2]; - T h4 = dist_c_sm[c_sm * 2 + 3]; - T r1 = ratio_c_sm[c_sm * 2]; - T r2 = ratio_c_sm[c_sm * 2 + 1]; - T r3 = ratio_c_sm[c_sm * 2 + 2]; - T r4 = 1 - r3; - T a = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2, f_sm)]; - T b = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 1, f_sm)]; - T c = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 2, f_sm)]; - T d = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)]; - T e = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 4, f_sm)]; - - // if (debug) { - // printf("c_sm(%d) %f %f %f %f %f\n",c_sm, a,b,c,d,e); - // printf("c_sm_h(%d) %f %f %f %f\n",c_sm, h1,h2,h3,h4); - // printf("c_sm_r(%d) %f %f %f %f\n",c_sm, r1,r2,r3,r4); - // } - - // T tb = a * h1 + b * 2 * (h1+h2) + c * h2; - // T tc = b * h2 + c * 2 * (h2+h3) + d * h3; - // T td = c * h3 + d * 2 * (h3+h4) + e * h4; - - // if (debug) printf("c_sm(%d) tb tc td tc: %f %f %f %f\n", f_sm, tb, tc, - // td, tc+tb * r1 + td * r4); - - // tc += tb * r1 + td * r4; - - // if (r_gl == 0 && f_gl == 0 && r_sm == 0 && f_sm == 0) { - // printf("mr2(%d) mm2: %f -> (%d %d %d)\n", c_sm, tc, r_gl, c_gl, f_gl); - // // printf("f_sm(%d) b c d: %f %f %f\n", f_sm, tb, tc, td); - // } - - // if (debug) { - // printf("f_sm(%d) %f %f %f %f %f f_sm_h %f %f %f %f f_sm_r %f %f %f %f, - // out: %f\n",f_sm, a,b,c,d,e, h1,h2,h3,h4,r1,r2,r3,r4, mass_trans(a, b, - // c, d, e, h1, h2, h3, h4, r1, r2, r3, r4)); - // } - - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = - mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4); - - // if (debug) printf("store[%d %d %d] %f \n", r_gl, c_gl, f_gl, - // mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4)); - - // printf("%d %d %d\n", r_gl, c_gl, f_gl); - // if (blockId * C + C == nc-1) { - // if (c_gl + 1 == nc_c - 1) { - // // T te = h4 * d + 2 * h4 * e; - // // te += td * r3; - // dw[get_idx(lddw1, lddw2, r_gl, blockId * C + actual_C, f_gl)] = - // mass_trans(c, d, e, (T)0.0, (T)0.0, - // h1, h2, (T)0.0, (T)0.0, r1, r2, (T)0.0, (T)0.0); - // } - // } - } -} - -template -void lpk_reo_2_adaptive_launcher( - Handle &handle, SIZE *shape_h, SIZE *shape_c_h, SIZE *shape_d, - SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, DIM processed_n, - DIM *processed_dims_h, DIM *processed_dims_d, DIM curr_dim_r, - DIM curr_dim_c, DIM curr_dim_f, T *ddist_c, T *dratio_c, T *dv1, - LENGTH lddv11, LENGTH lddv12, T *dv2, LENGTH lddv21, LENGTH lddv22, T *dw, - LENGTH lddw1, LENGTH lddw2, int queue_idx) { - - SIZE nr = shape_h[curr_dim_r]; - SIZE nc = shape_h[curr_dim_c]; - SIZE nf = shape_h[curr_dim_f]; - SIZE nc_c = shape_c_h[curr_dim_c]; - SIZE nf_c = shape_c_h[curr_dim_f]; - - SIZE total_thread_z = nr; - SIZE total_thread_y = nc_c; - // if (nc_c % 2 == 1) { total_thread_y = nc_c - 1; } - // else { total_thread_y = nc_c; } - SIZE total_thread_x = nf_c; - SIZE tbx, tby, tbz, gridx, gridy, gridz; - dim3 threadsPerBlock, blockPerGrid; - size_t sm_size; - - tbz = R; - tby = C; - tbx = F; - sm_size = (R * (C * 2 + 3) * F + (C * 2 + 3) * 2) * sizeof(T); - sm_size += (D * 4) * sizeof(SIZE); - sm_size += (D * 1) * sizeof(DIM); - gridz = ceil((double)total_thread_z / tbz); - gridy = ceil((double)total_thread_y / tby); - gridx = ceil((double)total_thread_x / tbx); - for (DIM d = 0; d < D; d++) { - if (d != curr_dim_f && d != curr_dim_c && d != curr_dim_r) { - SIZE t = shape_h[d]; - for (DIM k = 0; k < processed_n; k++) { - if (d == processed_dims_h[k]) { - t = shape_c_h[d]; - } - } - gridx *= t; - } - } - threadsPerBlock = dim3(tbx, tby, tbz); - blockPerGrid = dim3(gridx, gridy, gridz); - // printf("lpk_reo_2 exec config (%d %d %d) (%d %d %d)\n", tbx, tby, tbz, - // gridx, gridy, gridz); - - _lpk_reo_2<<>>( - shape_d, shape_c_d, ldvs, ldws, processed_n, processed_dims_d, curr_dim_r, - curr_dim_c, curr_dim_f, ddist_c, dratio_c, dv1, lddv11, lddv12, dv2, - lddv21, lddv22, dw, lddw1, lddw2); - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } -} - -template -void lpk_reo_2(Handle &handle, SIZE *shape_h, SIZE *shape_c_h, - SIZE *shape_d, SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, - DIM processed_n, DIM *processed_dims_h, DIM *processed_dims_d, - DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, T *ddist_c, - T *dratio_c, T *dv1, LENGTH lddv11, LENGTH lddv12, T *dv2, - LENGTH lddv21, LENGTH lddv22, T *dw, LENGTH lddw1, LENGTH lddw2, - int queue_idx, int config) { - -#define LPK(R, C, F) \ - { \ - lpk_reo_2_adaptive_launcher( \ - handle, shape_h, shape_c_h, shape_d, shape_c_d, ldvs, ldws, \ - processed_n, processed_dims_h, processed_dims_d,\ - curr_dim_r, \ - curr_dim_c, curr_dim_f, ddist_c, dratio_c, dv1, lddv11, lddv12, dv2, \ - lddv21, lddv22, dw, lddw1, lddw2, queue_idx); \ - } - - bool profile = false; - if (handle.profile_kernels) { - profile = true; - } - if (D >= 3) { - if (profile || config == 6) { - LPK(2, 2, 128) - } - if (profile || config == 5) { - LPK(2, 2, 64) - } - if (profile || config == 4) { - LPK(2, 2, 32) - } - if (profile || config == 3) { - LPK(4, 4, 16) - } - if (profile || config == 2) { - LPK(8, 8, 8) - } - if (profile || config == 1) { - LPK(4, 4, 4) - } - if (profile || config == 0) { - LPK(2, 2, 2) - } - } else if (D == 2) { - if (profile || config == 6) { - LPK(1, 2, 128) - } - if (profile || config == 5) { - LPK(1, 2, 64) - } - if (profile || config == 4) { - LPK(1, 2, 32) - } - if (profile || config == 3) { - LPK(1, 4, 16) - } - if (profile || config == 2) { - LPK(1, 8, 8) - } - if (profile || config == 1) { - LPK(1, 4, 4) - } - if (profile || config == 0) { - LPK(1, 2, 4) - } - } else { - printf("Error: mass_trans_multiply_2_cpt is only for 3D and 2D data\n"); - } -#undef LPK -} - -template -__global__ void _lpk_reo_3(SIZE *shape, SIZE *shape_c, SIZE *ldvs, SIZE *ldws, - DIM processed_n, DIM *processed_dims, DIM curr_dim_r, - DIM curr_dim_c, DIM curr_dim_f, T *ddist_r, - T *dratio_r, T *dv1, LENGTH lddv11, LENGTH lddv12, - T *dv2, LENGTH lddv21, LENGTH lddv22, T *dw, - LENGTH lddw1, LENGTH lddw2) { - - // bool debug = false; - // if (blockIdx.z == gridDim.z-1 && blockIdx.y == 0 && blockIdx.x == 0 && - // threadIdx.y == 0 && threadIdx.x == 0 ) debug = false; - - // bool debug2 = false; - // if (blockIdx.z == gridDim.z-1 && blockIdx.y == 1 && blockIdx.x == 16) - // debug2 = false; - - LENGTH threadId = (threadIdx.z * (blockDim.x * blockDim.y)) + - (threadIdx.y * blockDim.x) + threadIdx.x; - - T *sm = SharedMemory(); - SIZE ldsm1 = F; - SIZE ldsm2 = C; - T *v_sm = sm; - sm += ldsm1 * ldsm2 * (R * 2 + 3); - - T *dist_r_sm = sm; - sm += (R * 2 + 3); - T *ratio_r_sm = sm; - sm += (R * 2 + 3); - - SIZE *sm_size = (SIZE *)sm; - SIZE *shape_sm = sm_size; - sm_size += D; - SIZE *shape_c_sm = sm_size; - sm_size += D; - SIZE *ldvs_sm = sm_size; - sm_size += D; - SIZE *ldws_sm = sm_size; - sm_size += D; - sm = (T *)sm_size; - - DIM *sm_dim = (DIM *)sm; - DIM *processed_dims_sm = sm_dim; - sm_dim += D; - sm = (T *)sm_dim; - - SIZE idx[D]; - if (threadId < D) { - shape_sm[threadId] = shape[threadId]; - shape_c_sm[threadId] = shape_c[threadId]; - ldvs_sm[threadId] = ldvs[threadId]; - ldws_sm[threadId] = ldws[threadId]; - } - if (threadId < processed_n) { - processed_dims_sm[threadId] = processed_dims[threadId]; - } - __syncthreads(); - - for (DIM d = 0; d < D; d++) - idx[d] = 0; - - SIZE nr = shape_sm[curr_dim_r]; - SIZE nf_c = shape_c_sm[curr_dim_f]; - SIZE nc_c = shape_c_sm[curr_dim_c]; - SIZE nr_c = shape_c_sm[curr_dim_r]; - bool PADDING = (nr % 2 == 0); - - SIZE bidx = blockIdx.x; - SIZE firstD = div_roundup(nf_c, blockDim.x); - SIZE blockId_f = bidx % firstD; - bidx /= firstD; - - for (DIM d = 0; d < D; d++) { - if (d != curr_dim_r && d != curr_dim_c && d != curr_dim_f) { - SIZE t = shape_sm[d]; - for (DIM k = 0; k < processed_n; k++) { - if (d == processed_dims[k]) { - t = shape_c_sm[d]; - } - } - idx[d] = bidx % t; - bidx /= t; - } - } - - LENGTH other_offset_v = get_idx(ldvs_sm, idx); - LENGTH other_offset_w = get_idx(ldws_sm, idx); - - dv1 = dv1 + other_offset_v; - dv2 = dv2 + other_offset_v; - dw = dw + other_offset_w; - - // if (debug2) { - // printf("idx: %d %d %d %d\n", idx[3], idx[2], idx[1], idx[0]); - // printf("other_offset_v: %llu\n", other_offset_v); - // printf("other_offset_w: %llu\n", other_offset_w); - // } - - SIZE r_gl = blockIdx.z * blockDim.z + threadIdx.z; - SIZE c_gl = blockIdx.y * blockDim.y + threadIdx.y; - SIZE f_gl = blockId_f * blockDim.x + threadIdx.x; - - SIZE blockId = blockIdx.z; - - SIZE r_sm = threadIdx.z; - SIZE c_sm = threadIdx.y; - SIZE f_sm = threadIdx.x; - - SIZE actual_R = R; - if (nr_c - blockIdx.z * blockDim.z < R) { - actual_R = nr_c - blockIdx.z * blockDim.z; - } - // if (nr_c % 2 == 1){ - // if(nr_c-1 - blockIdx.z * blockDim.z < R) { actual_R = nr_c - 1 - - // blockIdx.z * blockDim.z; } - // } else { - // if(nr_c - blockIdx.z * blockDim.z < R) { actual_R = nr_c - blockIdx.z * - // blockDim.z; } - // } - - // if (debug) printf("actual_R %d\n", actual_R); - - // bool debug = false; - // if (idx[3] == 0 && idx[2] == 0 && f_gl == 2 && c_gl == 1) debug = false; - - // if (debug) printf("RCF: %d %d %d\n", R, C, F); - if (r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - // if (debug) printf("load front vsm[%d]: %f <- %d %d %d\n", r_sm * 2 + 2, - // dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 2, c_sm, f_sm)] = - dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)]; - - if (r_sm == actual_R - 1) { - if (r_gl + 1 < nr_c) { - // if (debug) printf("load front+1 vsm[%d]: %f <- %d %d %d\n", actual_R - // * 2 + 2, dv1[get_idx(lddv11, lddv12, blockId * R + actual_R, c_gl, - // f_gl)], blockId * R + actual_R, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, actual_R * 2 + 2, c_sm, f_sm)] = - dv1[get_idx(lddv11, lddv12, r_gl + 1, c_gl, f_gl)]; - } else { - // if (debug) printf("load front+1 vsm[%d]: 0.0\n", actual_R * 2 + 2); - v_sm[get_idx(ldsm1, ldsm2, actual_R * 2 + 2, c_sm, f_sm)] = 0.0; - } - } - - if (r_sm == 0) { - if (r_gl >= 1) { - // if (debug) printf("load front-1 vsm[0]: %f <- %d %d %d\n", - // dv1[get_idx(lddv11, lddv12, r_gl-1, c_gl, f_gl)], r_gl-1, c_gl, - // f_gl); - v_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - dv1[get_idx(lddv11, lddv12, r_gl - 1, c_gl, f_gl)]; - } else { - // if (debug) printf("load front-1 vsm[0]: 0.0\n"); - v_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = 0.0; - } - } - - if (!PADDING) { - if (r_gl < nr_c - 1) { - // if (debug) printf("load back vsm[%d]: 0.0\n", r_sm * 2 + 3); - v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)]; - } else { - // if (debug) printf("load back vsm[%d]: %f <- %d %d %d\n", r_sm * 2 + - // 3, dv2[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)] = 0.0; - } - } else { - if (r_gl < nr_c - 2) { - // if (debug) printf("load back vsm[%d]: %f <- %d %d %d\n", r_sm * 2 + - // 3, dv2[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)]; - } else { - // if (debug) printf("load back vsm[%d]: 0.0\n", r_sm * 2 + 3); - v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)] = 0.0; - } - } - - if (r_gl >= 1 && - (PADDING && r_gl - 1 < nr_c - 2 || !PADDING && r_gl - 1 < nr_c - 1)) { - // if (blockId > 0) { - if (r_sm == 0) { - // if (debug) printf("load back-1 vsm[1]: %f <- %d %d %d\n", - // dv2[get_idx(lddv11, lddv12, r_gl-1, c_gl, f_gl)], r_gl-1, c_gl, - // f_gl); - v_sm[get_idx(ldsm1, ldsm2, 1, c_sm, f_sm)] = - dv2[get_idx(lddv11, lddv12, r_gl - 1, c_gl, f_gl)]; - } - } else { - if (r_sm == 0) { - // if (debug) printf("load back-1 vsm[1]: 0.0\n"); - v_sm[get_idx(ldsm1, ldsm2, 1, c_sm, f_sm)] = 0.0; - } - } - } - - // load dist/ratio using f_sm for better performance - // assumption F >= R - if (r_sm == 0 && c_sm == 0 && f_sm < actual_R) { - if (blockId * R * 2 + f_sm < nr) { - dist_r_sm[2 + f_sm] = ddist_r[blockId * R * 2 + f_sm]; - // if (debug2 ) printf("load dist 1 [%d]: %f [%d]\n", 2 + f_sm, - // dist_r_sm[2 + f_sm], blockId * R * 2 + f_sm); - ratio_r_sm[2 + f_sm] = dratio_r[blockId * R * 2 + f_sm]; - // if (debug2 )printf("load ratio 1 [%d]: %f [%d]\n", 2 + f_sm, - // ratio_r_sm[2 + f_sm], blockId * R * 2 + f_sm); - } else { - dist_r_sm[2 + f_sm] = 0.0; - ratio_r_sm[2 + f_sm] = 0.0; - } - if (blockId * R * 2 + actual_R + f_sm < nr) { - dist_r_sm[2 + actual_R + f_sm] = - ddist_r[blockId * R * 2 + actual_R + f_sm]; - // if (debug2 )printf("load dist 2 [%d]: %f [%d]\n", 2 + actual_R + f_sm, - // dist_r_sm[2 + actual_R + f_sm], blockId * R * 2 + actual_R + f_sm); - ratio_r_sm[2 + actual_R + f_sm] = - dratio_r[blockId * R * 2 + actual_R + f_sm]; - // if (debug2 )printf("load ratio 2 [%d]: %f [%d]\n", 2 + actual_R + f_sm, - // ratio_r_sm[2 + actual_R + f_sm], blockId * R * 2 + actual_R + f_sm); - } else { - dist_r_sm[2 + actual_R + f_sm] = 0.0; - ratio_r_sm[2 + actual_R + f_sm] = 0.0; - } - } - - if (blockId > 0) { - if (f_sm < 2) { - dist_r_sm[f_sm] = ddist_r[blockId * R * 2 - 2 + f_sm]; - // if (debug2 )printf("load dist -1 [%d]: %f [%d]\n", f_sm, - // dist_r_sm[f_sm], blockId * R * 2 - 2 + f_sm); - ratio_r_sm[f_sm] = dratio_r[blockId * R * 2 - 2 + f_sm]; - // if (debug2 )printf("load ratio -1 [%d]: %f [%d]\n", f_sm, - // ratio_r_sm[f_sm], blockId * R * 2 - 2 + f_sm); - } - } else { - if (f_sm < 2) { - dist_r_sm[f_sm] = 0.0; - ratio_r_sm[f_sm] = 0.0; - } - } - - __syncthreads(); - - int adjusted_nr_c = nr_c; - if (r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - T h1 = dist_r_sm[r_sm * 2]; - T h2 = dist_r_sm[r_sm * 2 + 1]; - T h3 = dist_r_sm[r_sm * 2 + 2]; - T h4 = dist_r_sm[r_sm * 2 + 3]; - T r1 = ratio_r_sm[r_sm * 2]; - T r2 = ratio_r_sm[r_sm * 2 + 1]; - T r3 = ratio_r_sm[r_sm * 2 + 2]; - T r4 = 1 - r3; - T a = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2, c_sm, f_sm)]; - T b = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 1, c_sm, f_sm)]; - T c = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 2, c_sm, f_sm)]; - T d = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)]; - T e = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 4, c_sm, f_sm)]; - - // __syncthreads(); - // if (debug) { - // printf("r_sm(%d) %f %f %f %f %f\n",r_sm, a,b,c,d,e); - // printf("r_sm_h(%d) %f %f %f %f\n",r_sm, h1,h2,h3,h4); - // printf("r_sm_r(%d) %f %f %f %f\n",r_sm, r1,r2,r3,r4); - // } - // __syncthreads(); - - // T tb = a * h1 + b * 2 * (h1+h2) + c * h2; - // T tc = b * h2 + c * 2 * (h2+h3) + d * h3; - // T td = c * h3 + d * 2 * (h3+h4) + e * h4; - - // if (debug) printf("f_sm(%d) tb tc td tc: %f %f %f %f\n", f_sm, tb, tc, - // td, tc+tb * r1 + td * r4); - - // tc += tb * r1 + td * r4; - - // if (debug) { - // printf("f_sm(%d) %f %f %f %f %f f_sm_h %f %f %f %f f_sm_r %f %f %f %f, - // out: %f\n",f_sm, a,b,c,d,e, h1,h2,h3,h4,r1,r2,r3,r4, mass_trans(a, b, - // c, d, e, h1, h2, h3, h4, r1, r2, r3, r4)); - // } - - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = - mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4); - - // if (debug) printf("store[%d %d %d] %f (%f)\n", r_gl, c_gl, f_gl, - // mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4), - // mass_trans(a, b, c, (T)0.0, (T)0.0, h1, (T)0.0, (T)0.0, - // h4, r1, r2, (T)0.0, (T)0.0)); - // // printf("%d %d %d\n", r_gl, c_gl, f_gl); - // if (blockId * R + R == nr-1) { - // if (r_gl+1 == nr_c - 1) { - // if (r_gl+1 == nr_c - 1) { - // // T te = h4 * d + 2 * h4 * e; - // // te += td * r3; - // dw[get_idx(lddw1, lddw2, blockId * R + actual_R, c_gl, f_gl)] = - // mass_trans(c, d, e, (T)0.0, (T)0.0, - // h1, h2, (T)0.0, (T)0.0, r1, r2, (T)0.0, (T)0.0); - - // if (debug) printf("store-last[%d %d %d] %f\n", blockId * R + actual_R, - // c_gl, f_gl, - // mass_trans(c, d, e, (T)0.0, (T)0.0, - // h1, h2, (T)0.0, (T)0.0, r1, r2, (T)0.0, (T)0.0)); - // } - //} - } -} - -template -void lpk_reo_3_adaptive_launcher( - Handle &handle, SIZE *shape_h, SIZE *shape_c_h, SIZE *shape_d, - SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, DIM processed_n, - DIM *processed_dims_h, DIM *processed_dims_d, DIM curr_dim_r, - DIM curr_dim_c, DIM curr_dim_f, T *ddist_r, T *dratio_r, T *dv1, - LENGTH lddv11, LENGTH lddv12, T *dv2, LENGTH lddv21, LENGTH lddv22, T *dw, - LENGTH lddw1, LENGTH lddw2, int queue_idx) { - - SIZE nr = shape_h[curr_dim_r]; - SIZE nc = shape_h[curr_dim_c]; - SIZE nf = shape_h[curr_dim_f]; - SIZE nr_c = shape_c_h[curr_dim_r]; - SIZE nc_c = shape_c_h[curr_dim_c]; - SIZE nf_c = shape_c_h[curr_dim_f]; - - SIZE total_thread_z = nr_c; - // if (nr_c % 2 == 1){ total_thread_z = nr_c - 1; } - // else { total_thread_z = nr_c; } - SIZE total_thread_y = nc_c; - SIZE total_thread_x = nf_c; - - SIZE tbx, tby, tbz, gridx, gridy, gridz; - dim3 threadsPerBlock, blockPerGrid; - size_t sm_size; - - tbz = R; - tby = C; - tbx = F; - sm_size = ((R * 2 + 3) * C * F + (R * 2 + 3) * 2) * sizeof(T); - sm_size += (D * 4) * sizeof(SIZE); - sm_size += (D * 1) * sizeof(DIM); - gridz = ceil((double)total_thread_z / tbz); - gridy = ceil((double)total_thread_y / tby); - gridx = ceil((double)total_thread_x / tbx); - for (DIM d = 0; d < D; d++) { - if (d != curr_dim_f && d != curr_dim_c && d != curr_dim_r) { - SIZE t = shape_h[d]; - for (DIM k = 0; k < processed_n; k++) { - if (d == processed_dims_h[k]) { - t = shape_c_h[d]; - } - } - gridx *= t; - } - } - threadsPerBlock = dim3(tbx, tby, tbz); - blockPerGrid = dim3(gridx, gridy, gridz); - // printf("lpk_reo_3 exec config (%d %d %d) (%d %d %d)\n", tbx, tby, tbz, - // gridx, gridy, gridz); - - _lpk_reo_3<<>>( - shape_d, shape_c_d, ldvs, ldws, processed_n, processed_dims_d, curr_dim_r, - curr_dim_c, curr_dim_f, ddist_r, dratio_r, dv1, lddv11, lddv12, dv2, - lddv21, lddv22, dw, lddw1, lddw2); - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } -} - -template -void lpk_reo_3(Handle &handle, SIZE *shape_h, SIZE *shape_c_h, - SIZE *shape_d, SIZE *shape_c_d, SIZE *ldvs, SIZE *ldws, - DIM processed_n, DIM *processed_dims_h, DIM *processed_dims_d, - DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, T *ddist_r, - T *dratio_r, T *dv1, LENGTH lddv11, LENGTH lddv12, T *dv2, - LENGTH lddv21, LENGTH lddv22, T *dw, LENGTH lddw1, LENGTH lddw2, - int queue_idx, int config) { - -#define LPK(R, C, F) \ - { \ - lpk_reo_3_adaptive_launcher( \ - handle, shape_h, shape_c_h, shape_d, shape_c_d, ldvs, ldws, \ - processed_n, processed_dims_h, processed_dims_d,\ - curr_dim_r, \ - curr_dim_c, curr_dim_f, ddist_r, dratio_r, dv1, lddv11, lddv12, dv2, \ - lddv21, lddv22, dw, lddw1, lddw2, queue_idx); \ - } - bool profile = false; - if (handle.profile_kernels) { - profile = true; - } - if (D >= 3) { - if (profile || config == 6) { - LPK(2, 2, 128) - } - if (profile || config == 5) { - LPK(2, 2, 64) - } - if (profile || config == 4) { - LPK(2, 2, 32) - } - if (profile || config == 3) { - LPK(4, 4, 16) - } - if (profile || config == 2) { - LPK(8, 8, 8) - } - if (profile || config == 1) { - LPK(4, 4, 4) - } - if (profile || config == 0) { - LPK(2, 2, 2) - } - } else { - printf("Error: mass_trans_multiply_3_cpt is only for 3D data\n"); - } - -#undef LPK -} - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/LinearProcessingKernel3D.h b/include/cuda/LinearProcessingKernel3D.h deleted file mode 100644 index eb90472911..0000000000 --- a/include/cuda/LinearProcessingKernel3D.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_LINEAR_PROCESSING_KERNEL_3D -#define MGRAD_CUDA_LINEAR_PROCESSING_KERNEL_3D - -#include "Common.h" - -namespace mgard_cuda { - -template -void lpk_reo_1_3d(Handle &handle, SIZE nr, SIZE nc, SIZE nf, SIZE nf_c, - SIZE zero_r, SIZE zero_c, SIZE zero_f, T *ddist_f, - T *dratio_f, T *dv1, SIZE lddv11, SIZE lddv12, T *dv2, - SIZE lddv21, SIZE lddv22, T *dw, SIZE lddw1, SIZE lddw2, - int queue_idx, int config); - -template -void lpk_reo_2_3d(Handle &handle, SIZE nr, SIZE nc, SIZE nf_c, SIZE nc_c, - T *ddist_c, T *dratio_c, T *dv1, SIZE lddv11, SIZE lddv12, - T *dv2, SIZE lddv21, SIZE lddv22, T *dw, SIZE lddw1, - SIZE lddw2, int queue_idx, int config); - -template -void lpk_reo_3_3d(Handle &handle, SIZE nr, SIZE nc_c, SIZE nf_c, - SIZE nr_c, T *ddist_r, T *dratio_r, T *dv1, SIZE lddv11, - SIZE lddv12, T *dv2, SIZE lddv21, SIZE lddv22, T *dw, - SIZE lddw1, SIZE lddw2, int queue_idx, int config); - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/LinearProcessingKernel3D.hpp b/include/cuda/LinearProcessingKernel3D.hpp deleted file mode 100644 index 86334abd97..0000000000 --- a/include/cuda/LinearProcessingKernel3D.hpp +++ /dev/null @@ -1,1115 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_LINEAR_PROCESSSING_KERNEL_3D_TEMPLATE -#define MGRAD_CUDA_LINEAR_PROCESSSING_KERNEL_3D_TEMPLATE - -#include "CommonInternal.h" -#include "LPKFunctor.h" -#include "LinearProcessingKernel.h" -namespace mgard_cuda { - -template -__global__ void _lpk_reo_1_3d(SIZE nr, SIZE nc, SIZE nf, SIZE nf_c, SIZE zero_r, - SIZE zero_c, SIZE zero_f, T *ddist_f, T *dratio_f, - T *dv1, SIZE lddv11, SIZE lddv12, T *dv2, - SIZE lddv21, SIZE lddv22, T *dw, SIZE lddw1, - SIZE lddw2) { - - // bool debug = false; - // if (blockIdx.z == 0 && blockIdx.y == 0 && blockIdx.x == 1 && - // threadIdx.y == 0 && threadIdx.z == 0 ) debug = false; - - // bool debug2 = false; - // if (blockIdx.z == gridDim.z-1 && blockIdx.y == 1 && blockIdx.x == 16) - // debug2 = false; - - bool PADDING = (nf % 2 == 0); - - T *sm = SharedMemory(); - // extern __shared__ double sm[]; // size: (blockDim.x + 1) * (blockDim.y + 1) - // * (blockDim.z + 1) - SIZE ldsm1 = F * 2 + 3; - SIZE ldsm2 = C; - T *v_sm = sm; - T *dist_f_sm = sm + ldsm1 * ldsm2 * R; - T *ratio_f_sm = dist_f_sm + ldsm1; - - bool debug = false; - // if (blockIdx.z == 0 && blockIdx.y == 0 && blockIdx.x == 0 && - // threadIdx.z == 0 && threadIdx.y == 0 ) debug = true; - - SIZE r_gl = blockIdx.z * blockDim.z + threadIdx.z; - SIZE c_gl = blockIdx.y * blockDim.y + threadIdx.y; - SIZE f_gl = blockIdx.x * blockDim.x + threadIdx.x; - - SIZE blockId = blockIdx.x; - - SIZE r_sm = threadIdx.z; - SIZE c_sm = threadIdx.y; - SIZE f_sm = threadIdx.x; - - SIZE actual_F = F; - if (nf_c - blockId * blockDim.x < F) { - actual_F = nf_c - blockId * blockDim.x; - } - - // if (nf_c % 2 == 1){ - // if(nf_c-1 - blockId * blockDim.x < F) { actual_F = nf_c - 1 - blockId * - // blockDim.x; } - // } else { - // if(nf_c - blockId * blockDim.x < F) { actual_F = nf_c - blockId * - // blockDim.x; } - // } - - // if (debug) printf("actual_F %d\n", actual_F); - - if (r_gl < nr && c_gl < nc && f_gl < nf_c) { - if (r_gl < zero_r && c_gl < zero_c && f_gl < zero_f) { - // if (debug) printf("load left vsm[%d]: 0.0\n", f_sm * 2 + 2); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 2)] = 0.0; - } else { - // if (debug) printf("load left vsm[%d]<-dv1[%d, %d, %d]: %f\n", f_sm * 2 - // + 2, r_gl, c_gl, f_gl, dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 2)] = - dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)]; - } - - if (f_sm == actual_F - 1) { - if (r_gl < zero_r && c_gl < zero_c && f_gl < zero_f) { - // if (debug) printf("load left+1 vsm[%d]: 0.0\n", actual_F * 2 + 2); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 2)] = 0.0; - } else { - if (f_gl + 1 < nf_c) { - // if (debug) printf("load left+1 vsm[%d]: %f\n", actual_F * 2 + 2, - // dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl + 1)]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 2)] = - dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl + 1)]; - } else { - // if (debug) printf("load left+1 vsm[%d]: 0.0\n", actual_F * 2 + 2); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 2)] = 0.0; - } - } - } - - if (f_sm == 0) { - // left - if (r_gl < zero_r && c_gl < zero_c && f_gl < zero_f) { - // coarse (-1) - // if (debug) printf("load left-1 vsm[0]: 0.0\n"); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = 0.0; - } else { - if (f_gl >= 1) { - // other (-1) - // if (debug) printf("load left-1 vsm[0]: %f\n", dv1[get_idx(lddv11, - // lddv12, r_gl, c_gl, f_gl-1)]); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = - dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl - 1)]; - } else { - // other (-1) - // if (debug) printf("load left-1 vsm[0]: 0.0\n"); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = 0.0; - } - } - } - - // right - if (!PADDING) { - if (nf_c % 2 != 0) { - if (f_gl >= 1 && f_gl < nf_c) { - // if (debug) printf("load right vsm[%d]: %f <- %d %d %d\n", f_sm * 2 - // + 1, dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - 1)], r_gl, - // c_gl, f_gl - 1); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - 1)]; - } else { - // if (debug) printf("load right vsm[%d]: 0\n", f_sm * 2 + 1); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)] = 0.0; - } - } else { // nf_c % 2 == 0 - if (f_gl < nf_c - 1) { - // if (debug) printf("load right vsm[%d]: %f <- %d %d %d\n", f_sm * 2 - // + 3, dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)], r_gl, c_gl, - // f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)]; - } else { - // if (debug) printf("load right vsm[%d]: 0\n", f_sm * 2 + 3); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)] = 0.0; - } - } - } else { // PADDING - if (nf_c % 2 != 0) { - if (f_gl >= 1 && f_gl < nf_c - 1) { - // if (debug) printf("load right vsm[%d]: %f <- %d %d %d\n", f_sm * 2 - // + 1, dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - 1)], r_gl, - // c_gl, f_gl - 1); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - 1)]; - } else { - // if (debug) printf("load right vsm[%d]: 0\n", f_sm * 2 + 1); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)] = 0.0; - } - } else { // nf_c % 2 == 0 - if (f_gl < nf_c - 2) { - // if (debug) printf("load right vsm[%d]: %f <- %d %d %d\n", f_sm * 2 - // + 3, dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)], r_gl, c_gl, - // f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)]; - } else { - // if (debug) printf("load right vsm[%d]: 0\n", f_sm * 2 + 3); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)] = 0.0; - } - } - } - - if (f_sm == actual_F - 1) { - // right (+1) - if (!PADDING) { - if (nf_c % 2 != 0) { - if (f_gl < nf_c - 1) { - // if (debug) printf("load right+1 vsm[%d]: %f <- %d %d %d\n", - // actual_F * 2 + 1, dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)], - // r_gl, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 1)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)]; - } else { - // if (debug) printf("load right+1 vsm[%d]: 0.0\n", actual_F * 2 + - // 1); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 1)] = 0.0; - } - } else { // nf_c % 2 == 0 - if (f_gl >= actual_F) { - // if (debug) printf("load right-1 vsm[1]: %f <- %d %d %d\n", - // dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - actual_F)], r_gl, - // c_gl, f_gl - actual_F); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 1)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - actual_F)]; - } else { - // if (debug) printf("load right-1 vsm[1]: 0.0\n"); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 1)] = 0.0; - } - } - } else { - if (nf_c % 2 != 0) { - if (f_gl < nf_c - 2) { - // if (debug) printf("actual_F(%d), load right+1 vsm[%d]: %f <- %d - // %d %d\n", actual_F, actual_F * 2 + 1, dv2[get_idx(lddv21, lddv22, - // r_gl, c_gl, f_gl)], r_gl, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 1)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)]; - } else { - // if (debug) printf("load right+1 vsm[%d]: 0.0\n", actual_F * 2 + - // 1); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 1)] = 0.0; - } - } else { // nf_c % 2 == 0 - if (f_gl >= actual_F && f_gl - actual_F < nf_c - 2) { - // if (debug) printf("load right-1 vsm[1]: %f <- %d %d %d\n", - // dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - actual_F)], r_gl, - // c_gl, f_gl - actual_F); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 1)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - actual_F)]; - } else { - // if (debug) printf("load right-1 vsm[1]: 0.0\n"); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 1)] = 0.0; - } - } - } - } - } - - // bool debug = false; - // if (r_gl == 0 && c_gl == 0) debug = true; - - if (r_sm == 0 && c_sm == 0 && f_sm < actual_F) { - if (blockId * F * 2 + f_sm < nf) { - dist_f_sm[2 + f_sm] = ddist_f[blockId * F * 2 + f_sm]; - ratio_f_sm[2 + f_sm] = dratio_f[blockId * F * 2 + f_sm]; - // if (debug) - // printf("load dist[%d] -> sm[%d]: %f\n", blockId * F * 2 + f_sm, - // 2 + f_sm, ddist_f[blockId * F * 2 + f_sm]); - } else { - dist_f_sm[2 + f_sm] = 0.0; - ratio_f_sm[2 + f_sm] = 0.0; - } - - if (blockId * F * 2 + actual_F + f_sm < nf) { - dist_f_sm[2 + actual_F + f_sm] = - ddist_f[blockId * F * 2 + actual_F + f_sm]; - ratio_f_sm[2 + actual_F + f_sm] = - dratio_f[blockId * F * 2 + actual_F + f_sm]; - // if (debug) - // printf("load dist[%d] -> sm[%d]: %f\n", - // blockId * F * 2 + actual_F + f_sm, 2 + actual_F + f_sm, - // ddist_f[blockId * F * 2 + actual_F + f_sm]); - } else { - dist_f_sm[2 + actual_F + f_sm] = 0.0; - ratio_f_sm[2 + actual_F + f_sm] = 0.0; - } - // dist_f_sm[2 + f_sm] = ddist_f[f_gl]; - // dist_f_sm[2 + actual_F + f_sm] = ddist_f[actual_F + f_gl]; - // ratio_f_sm[2 + f_sm] = dratio_f[f_gl]; - // ratio_f_sm[2 + actual_F + f_sm] = dratio_f[actual_F + f_gl]; - } - - if (blockId > 0) { - if (f_sm < 2) { - // dist_f_sm[f_sm] = ddist_f[f_gl - 2]; - // ratio_f_sm[f_sm] = dratio_f[f_gl - 2]; - dist_f_sm[f_sm] = ddist_f[blockId * F * 2 + f_sm - 2]; - ratio_f_sm[f_sm] = dratio_f[blockId * F * 2 + f_sm - 2]; - } - } else { - if (f_sm < 2) { - dist_f_sm[f_sm] = 0.0; - ratio_f_sm[f_sm] = 0.0; - } - } - - __syncthreads(); - - if (r_gl < nr && c_gl < nc && f_gl < nf_c) { - T h1 = dist_f_sm[f_sm * 2]; - T h2 = dist_f_sm[f_sm * 2 + 1]; - T h3 = dist_f_sm[f_sm * 2 + 2]; - T h4 = dist_f_sm[f_sm * 2 + 3]; - T r1 = ratio_f_sm[f_sm * 2]; - T r2 = ratio_f_sm[f_sm * 2 + 1]; - T r3 = ratio_f_sm[f_sm * 2 + 2]; - T r4 = 1 - r3; - T a = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2)]; - T b = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)]; - T c = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 2)]; - T d = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)]; - T e = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 4)]; - - // if (f_gl == 0 && c_gl == 2 && r_gl == 0) { - // printf("f_sm(%d) %f %f %f %f %f\n",f_sm, a,b,c,d,e); - // printf("f_sm_h(%d) %f %f %f %f\n",f_sm, h1,h2,h3,h4); - // printf("f_sm_r(%d) %f %f %f %f\n",f_sm, r1,r2,r3,r4); - // } - - // T tb = a * h1/6 + b * (h1+h2)/3 + c * h2/6; - // T tc = b * h2/6 + c * (h2+h3)/3 + d * h3/6; - // T td = c * h3/6 + d * (h3+h4)/3 + e * h4/6; - - // if (f_gl == 0 && c_gl == 2 && r_gl == 0) { - // printf("tb tc td %f %f %f\n", tb, tc, td); - // } - - // if (debug) printf("f_sm(%d) tb tc td tc: %f %f %f %f\n", f_sm, tb, tc, - // td, tc+tb * r1 + td * r4); - - // tc += tb * r1 + td * r4; - - // if (f_gl == 0 && c_gl == 2 && r_gl == 0) { - // printf("tc %f\n", tc, td); - // } - - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = - mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4); - - // if (f_gl == 0 && c_gl == 2 && r_gl == 0) { - // // if (debug) - // printf("store[%d %d %d] %f \n", r_gl, c_gl, f_gl, - // mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4)); - // } - - // printf("test block %d F %d nf %d\n", blockId, F, nf); - // if (f_gl+1 == nf_c-1) { - - // // T te = h4 * d + 2 * h4 * e; - // //printf("f_sm(%d) mm-e: %f\n", f_sm, te); - // // te += td * r3; - // dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl+1)] = - // mass_trans(c, d, e, (T)0.0, (T)0.0, h1, h2, (T)0.0, (T)0.0, r1, r2, - // (T)0.0, (T)0.0); - // } - } -} - -template -void lpk_reo_1_3d_adaptive_launcher(Handle &handle, SIZE nr, SIZE nc, - SIZE nf, SIZE nf_c, SIZE zero_r, - SIZE zero_c, SIZE zero_f, T *ddist_f, - T *dratio_f, T *dv1, SIZE lddv11, - SIZE lddv12, T *dv2, SIZE lddv21, - SIZE lddv22, T *dw, SIZE lddw1, SIZE lddw2, - int queue_idx) { - // printf("dratio_f: "); - // print_matrix_cuda(1, (nf-1)*2, dratio_f, (nf-1)*2); - SIZE total_thread_z = nr; - SIZE total_thread_y = nc; - SIZE total_thread_x = nf_c; - // if (nf_c % 2 == 1) { total_thread_x = nf_c - 1; } - // else { total_thread_x = nf; } - SIZE tbx, tby, tbz, gridx, gridy, gridz; - dim3 threadsPerBlock, blockPerGrid; - size_t sm_size; - - tbz = R; - tby = C; - tbx = F; - sm_size = (R * C * (F * 2 + 3) + (F * 2 + 3) * 2) * sizeof(T); - gridz = ceil((double)total_thread_z / tbz); - gridy = ceil((double)total_thread_y / tby); - gridx = ceil((double)total_thread_x / tbx); - threadsPerBlock = dim3(tbx, tby, tbz); - blockPerGrid = dim3(gridx, gridy, gridz); - - // printf("nr: %d nc: %d, nf: %d, nf_c: %d\n", nr, nc, nf, nf_c); - // printf("tb: %d %d %d, grid: %d %d %d\n", tbx, tby, tbz, gridx, gridy, - // gridz); - - _lpk_reo_1_3d<<>>( - nr, nc, nf, nf_c, zero_r, zero_c, zero_f, ddist_f, dratio_f, dv1, lddv11, - lddv12, dv2, lddv21, lddv22, dw, lddw1, lddw2); - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } -} - -template -void lpk_reo_1_3d(Handle &handle, SIZE nr, SIZE nc, SIZE nf, SIZE nf_c, - SIZE zero_r, SIZE zero_c, SIZE zero_f, T *ddist_f, - T *dratio_f, T *dv1, SIZE lddv11, SIZE lddv12, T *dv2, - SIZE lddv21, SIZE lddv22, T *dw, SIZE lddw1, SIZE lddw2, - int queue_idx, int config) { - -#define LPK(R, C, F) \ - { \ - lpk_reo_1_3d_adaptive_launcher( \ - handle, nr, nc, nf, nf_c, zero_r, zero_c, zero_f, ddist_f, dratio_f, \ - dv1, lddv11, lddv12, dv2, lddv21, lddv22, dw, lddw1, lddw2, \ - queue_idx); \ - } - - bool profile = false; - if (handle.profile_kernels) { - profile = true; - } - if (D == 3) { - if (profile || config == 6) { - LPK(2, 2, 128) - } - if (profile || config == 5) { - LPK(2, 2, 64) - } - if (profile || config == 4) { - LPK(2, 2, 32) - } - if (profile || config == 3) { - LPK(4, 4, 16) - } - if (profile || config == 2) { - LPK(8, 8, 8) - } - if (profile || config == 1) { - LPK(4, 4, 4) - } - if (profile || config == 0) { - LPK(2, 2, 2) - } - } else if (D == 2) { - if (profile || config == 6) { - LPK(1, 2, 128) - } - if (profile || config == 5) { - LPK(1, 2, 64) - } - if (profile || config == 4) { - LPK(1, 2, 32) - } - if (profile || config == 3) { - LPK(1, 4, 16) - } - if (profile || config == 2) { - LPK(1, 8, 8) - } - if (profile || config == 1) { - LPK(1, 4, 4) - } - if (profile || config == 0) { - LPK(1, 2, 4) - } - } else if (D == 1) { - if (profile || config == 6) { - LPK(1, 1, 128) - } - if (profile || config == 5) { - LPK(1, 1, 64) - } - if (profile || config == 4) { - LPK(1, 1, 32) - } - if (profile || config == 3) { - LPK(1, 1, 16) - } - if (profile || config == 2) { - LPK(1, 1, 8) - } - if (profile || config == 1) { - LPK(1, 1, 8) - } - if (profile || config == 0) { - LPK(1, 1, 8) - } - } -#undef LPK -} - -template -__global__ void _lpk_reo_2_3d(SIZE nr, SIZE nc, SIZE nf_c, SIZE nc_c, - T *ddist_c, T *dratio_c, T *dv1, SIZE lddv11, - SIZE lddv12, T *dv2, SIZE lddv21, SIZE lddv22, - T *dw, SIZE lddw1, SIZE lddw2) { - - // bool debug = false; - // if (blockIdx.y == gridDim.y-1 && blockIdx.x == 0 && - // threadIdx.x == 0 ) debug = false; - - // bool debug2 = false; - // if (blockIdx.z == gridDim.z-1 && blockIdx.y == 1 && blockIdx.x == 16) - // debug2 = false; - - bool PADDING = (nc % 2 == 0); - - T *sm = SharedMemory(); - - // extern __shared__ double sm[]; // size: (blockDim.x + 1) * (blockDim.y + 1) - // * (blockDim.z + 1) - SIZE ldsm1 = F; - SIZE ldsm2 = C * 2 + 3; - T *v_sm = sm; - T *dist_c_sm = sm + ldsm1 * ldsm2 * R; - T *ratio_c_sm = dist_c_sm + ldsm2; - - // bool debug = false; - // if (blockIdx.z == 0 && blockIdx.y == 0 && blockIdx.x == 0 && - // threadIdx.z == 0 && threadIdx.x == 0 ) debug = false; - - SIZE r_gl = blockIdx.z * blockDim.z + threadIdx.z; - SIZE c_gl = blockIdx.y * blockDim.y + threadIdx.y; - SIZE f_gl = blockIdx.x * blockDim.x + threadIdx.x; - - SIZE blockId = blockIdx.y; - - SIZE r_sm = threadIdx.z; - SIZE c_sm = threadIdx.y; - SIZE f_sm = threadIdx.x; - - SIZE actual_C = C; - if (nc_c - blockIdx.y * blockDim.y < C) { - actual_C = nc_c - blockIdx.y * blockDim.y; - } - - // if (nc_c % 2 == 1){ - // if(nc_c-1 - blockIdx.y * blockDim.y < C) { actual_C = nc_c - 1 - - // blockIdx.y * blockDim.y; } - // } else { - // if(nc_c - blockIdx.y * blockDim.y < C) { actual_C = nc_c - blockIdx.y * - // blockDim.y; } - // } - - // if (debug) printf("actual_C %d\n", actual_C); - - if (r_gl < nr && c_gl < nc_c && f_gl < nf_c) { - // if (debug) printf("load up vsm[%d]: %f <- %d %d %d\n", c_sm * 2 + 2, - // dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 2, f_sm)] = - dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)]; - - if (c_sm == actual_C - 1) { - if (c_gl + 1 < nc_c) { - // if (debug) printf("load up+1 vsm[%d]: %f <- %d %d %d\n", actual_C * 2 - // + 2, dv1[get_idx(lddv11, lddv12, r_gl, blockId * C + actual_C, - // f_gl)], r_gl, blockId * C + actual_C, f_gl); - // c_gl+1 == blockId * C + C - v_sm[get_idx(ldsm1, ldsm2, r_sm, actual_C * 2 + 2, f_sm)] = - dv1[get_idx(lddv11, lddv12, r_gl, c_gl + 1, f_gl)]; - } else { - // if (debug) printf("load up+1 vsm[%d]: 0.0\n", actual_C * 2 + 2); - v_sm[get_idx(ldsm1, ldsm2, r_sm, actual_C * 2 + 2, f_sm)] = 0.0; - } - } - - if (c_sm == 0) { - if (c_gl >= 1) { - // if (debug) printf("load up-1 vsm[0]: %f <- %d %d %d\n", - // dv1[get_idx(lddv11, lddv12, r_gl, c_gl-1, f_gl)], r_gl, c_gl-1, - // f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = - dv1[get_idx(lddv11, lddv12, r_gl, c_gl - 1, f_gl)]; - } else { - // if (debug) printf("load up-1 vsm[0]: 0.0\n"); - v_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = 0.0; - } - } - - if (!PADDING) { - if (c_gl < nc_c - 1) { - // if (debug) printf("load down vsm[%d]: %f <- %d %d %d\n", c_sm * 2 + - // 3, dv2[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)]; - } else { - // if (debug) printf("load down vsm[%d]: 0.0\n", c_sm * 2 + 3); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)] = 0.0; - } - } else { - if (c_gl < nc_c - 2) { - // if (debug) printf("load down vsm[%d]: %f <- %d %d %d\n", c_sm * 2 + - // 3, dv2[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)]; - } else { - // if (debug) printf("load down vsm[%d]: 0.0\n", c_sm * 2 + 3); - v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)] = 0.0; - } - } - - if (c_gl >= 1 && - (PADDING && c_gl - 1 < nc_c - 2 || !PADDING && c_gl - 1 < nc_c - 1)) { - if (c_sm == 0) { - // if (debug) printf("PADDING: %d, c_gl-1: %d nc_c-2: %d\n", PADDING, - // c_gl-1, nc_c - 2); if (debug) printf("load down-1 vsm[1]: %f <- %d %d - // %d\n", dv2[get_idx(lddv11, lddv12, r_gl, c_gl-1, f_gl)], r_gl, - // c_gl-1, f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm, 1, f_sm)] = - dv2[get_idx(lddv11, lddv12, r_gl, c_gl - 1, f_gl)]; - } - } else { - if (c_sm == 0) { - // if (debug) printf("load down-1 vsm[1]: 0.0\n"); - v_sm[get_idx(ldsm1, ldsm2, r_sm, 1, f_sm)] = 0.0; - } - } - } - - // load dist/ratio using f_sm for better performance - // assumption F >= C - if (r_sm == 0 && c_sm == 0 && f_sm < actual_C) { - if (blockId * C * 2 + f_sm < nc) { - dist_c_sm[2 + f_sm] = ddist_c[blockId * C * 2 + f_sm]; - ratio_c_sm[2 + f_sm] = dratio_c[blockId * C * 2 + f_sm]; - } else { - dist_c_sm[2 + f_sm] = 0.0; - ratio_c_sm[2 + f_sm] = 0.0; - } - - if (blockId * C * 2 + actual_C + f_sm < nc) { - dist_c_sm[2 + actual_C + f_sm] = - ddist_c[blockId * C * 2 + actual_C + f_sm]; - ratio_c_sm[2 + actual_C + f_sm] = - dratio_c[blockId * C * 2 + actual_C + f_sm]; - } else { - dist_c_sm[2 + actual_C + f_sm] = 0.0; - ratio_c_sm[2 + actual_C + f_sm] = 0.0; - } - } - - if (blockId > 0) { - if (f_sm < 2) { - dist_c_sm[f_sm] = ddist_c[blockId * C * 2 - 2 + f_sm]; - ratio_c_sm[f_sm] = dratio_c[blockId * C * 2 - 2 + f_sm]; - } - } else { - if (f_sm < 2) { - dist_c_sm[f_sm] = 0.0; - ratio_c_sm[f_sm] = 0.0; - } - } - - __syncthreads(); - - if (r_gl < nr && c_gl < nc_c && f_gl < nf_c) { - T h1 = dist_c_sm[c_sm * 2]; - T h2 = dist_c_sm[c_sm * 2 + 1]; - T h3 = dist_c_sm[c_sm * 2 + 2]; - T h4 = dist_c_sm[c_sm * 2 + 3]; - T r1 = ratio_c_sm[c_sm * 2]; - T r2 = ratio_c_sm[c_sm * 2 + 1]; - T r3 = ratio_c_sm[c_sm * 2 + 2]; - T r4 = 1 - r3; - T a = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2, f_sm)]; - T b = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 1, f_sm)]; - T c = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 2, f_sm)]; - T d = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)]; - T e = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 4, f_sm)]; - - // if (debug) { - // printf("c_sm(%d) %f %f %f %f %f\n",c_sm, a,b,c,d,e); - // printf("c_sm_h(%d) %f %f %f %f\n",c_sm, h1,h2,h3,h4); - // printf("c_sm_r(%d) %f %f %f %f\n",c_sm, r1,r2,r3,r4); - // } - - // T tb = a * h1 + b * 2 * (h1+h2) + c * h2; - // T tc = b * h2 + c * 2 * (h2+h3) + d * h3; - // T td = c * h3 + d * 2 * (h3+h4) + e * h4; - - // if (debug) printf("c_sm(%d) tb tc td tc: %f %f %f %f\n", f_sm, tb, tc, - // td, tc+tb * r1 + td * r4); - - // tc += tb * r1 + td * r4; - - // if (r_gl == 0 && f_gl == 0 && r_sm == 0 && f_sm == 0) { - // printf("mr2(%d) mm2: %f -> (%d %d %d)\n", c_sm, tc, r_gl, c_gl, f_gl); - // // printf("f_sm(%d) b c d: %f %f %f\n", f_sm, tb, tc, td); - // } - - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = - mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4); - - // if (debug) printf("store[%d %d %d] %f \n", r_gl, c_gl, f_gl, - // mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4)); - - // printf("%d %d %d\n", r_gl, c_gl, f_gl); - // if (blockId * C + C == nc-1) { - // if (c_gl + 1 == nc_c - 1) { - // // T te = h4 * d + 2 * h4 * e; - // // te += td * r3; - // dw[get_idx(lddw1, lddw2, r_gl, blockId * C + actual_C, f_gl)] = - // mass_trans(c, d, e, (T)0.0, (T)0.0, - // h1, h2, (T)0.0, (T)0.0, r1, r2, (T)0.0, (T)0.0); - // } - // } - } -} - -template -void lpk_reo_2_3d_adaptive_launcher(Handle &handle, SIZE nr, SIZE nc, - SIZE nf_c, SIZE nc_c, T *ddist_c, - T *dratio_c, T *dv1, SIZE lddv11, - SIZE lddv12, T *dv2, SIZE lddv21, - SIZE lddv22, T *dw, SIZE lddw1, SIZE lddw2, - int queue_idx) { - cudaDeviceSetSharedMemConfig(cudaSharedMemBankSizeEightByte); - cudaDeviceSetCacheConfig(cudaFuncCachePreferShared); - SIZE total_thread_z = nr; - SIZE total_thread_y = nc_c; - // if (nc_c % 2 == 1) { total_thread_y = nc_c - 1; } - // else { total_thread_y = nc_c; } - SIZE total_thread_x = nf_c; - SIZE tbx, tby, tbz, gridx, gridy, gridz; - dim3 threadsPerBlock, blockPerGrid; - size_t sm_size; - - tbz = R; - tby = C; - tbx = F; - sm_size = (R * (C * 2 + 3) * F + (C * 2 + 3) * 2) * sizeof(T); - gridz = ceil((double)total_thread_z / tbz); - gridy = ceil((double)total_thread_y / tby); - gridx = ceil((double)total_thread_x / tbx); - threadsPerBlock = dim3(tbx, tby, tbz); - blockPerGrid = dim3(gridx, gridy, gridz); - - // printf("nr: %d nc: %d, nf_c: %d, nc_c: %d\n", nr, nc, nf_c, nc_c); - // printf("tb: %d %d %d, grid: %d %d %d\n", tbx, tby, tbz, gridx, gridy, - // gridz); - - _lpk_reo_2_3d<<>>( - nr, nc, nf_c, nc_c, ddist_c, dratio_c, dv1, lddv11, lddv12, dv2, lddv21, - lddv22, dw, lddw1, lddw2); - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } -} - -template -void lpk_reo_2_3d(Handle &handle, SIZE nr, SIZE nc, SIZE nf_c, SIZE nc_c, - T *ddist_c, T *dratio_c, T *dv1, SIZE lddv11, SIZE lddv12, - T *dv2, SIZE lddv21, SIZE lddv22, T *dw, SIZE lddw1, - SIZE lddw2, int queue_idx, int config) { - -#define LPK(R, C, F) \ - { \ - lpk_reo_2_3d_adaptive_launcher( \ - handle, nr, nc, nf_c, nc_c, ddist_c, dratio_c, dv1, lddv11, lddv12, \ - dv2, lddv21, lddv22, dw, lddw1, lddw2, queue_idx); \ - } - - bool profile = false; - if (handle.profile_kernels) { - profile = true; - } - if (D == 3) { - if (profile || config == 6) { - LPK(2, 2, 128) - } - if (profile || config == 5) { - LPK(2, 2, 64) - } - if (profile || config == 4) { - LPK(2, 2, 32) - } - if (profile || config == 3) { - LPK(4, 4, 16) - } - if (profile || config == 2) { - LPK(8, 8, 8) - } - if (profile || config == 1) { - LPK(4, 4, 4) - } - if (profile || config == 0) { - LPK(2, 2, 2) - } - } else if (D == 2) { - if (profile || config == 6) { - LPK(1, 2, 128) - } - if (profile || config == 5) { - LPK(1, 2, 64) - } - if (profile || config == 4) { - LPK(1, 2, 32) - } - if (profile || config == 3) { - LPK(1, 4, 16) - } - if (profile || config == 2) { - LPK(1, 8, 8) - } - if (profile || config == 1) { - LPK(1, 4, 4) - } - if (profile || config == 0) { - LPK(1, 2, 4) - } - } else { - printf("Error: mass_trans_multiply_2_cpt is only for 3D and 2D data\n"); - } -#undef LPK -} - -template -__global__ void _lpk_reo_3_3d(SIZE nr, SIZE nc_c, SIZE nf_c, SIZE nr_c, - T *ddist_r, T *dratio_r, T *dv1, SIZE lddv11, - SIZE lddv12, T *dv2, SIZE lddv21, SIZE lddv22, - T *dw, SIZE lddw1, SIZE lddw2) { - - // bool debug = false; - // if (blockIdx.z == 0 && blockIdx.y == 0 && blockIdx.x == 0 && - // threadIdx.y == 0 && threadIdx.x == 0 ) debug = true; - - // bool debug2 = false; - // if (blockIdx.z == 0 && blockIdx.y == 0 && blockIdx.x == 0) - // debug2 = true; - - bool PADDING = (nr % 2 == 0); - T *sm = SharedMemory(); - SIZE ldsm1 = F; - SIZE ldsm2 = C; - T *v_sm = sm; - T *dist_r_sm = sm + ldsm1 * ldsm2 * (R * 2 + 3); - T *ratio_r_sm = dist_r_sm + (R * 2 + 3); - - SIZE r_gl = blockIdx.z * blockDim.z + threadIdx.z; - SIZE c_gl = blockIdx.y * blockDim.y + threadIdx.y; - SIZE f_gl = blockIdx.x * blockDim.x + threadIdx.x; - - // if (debug) printf("debugging gl: %d %d %d\n", r_gl, c_gl, f_gl); - - SIZE blockId = blockIdx.z; - - SIZE r_sm = threadIdx.z; - SIZE c_sm = threadIdx.y; - SIZE f_sm = threadIdx.x; - - SIZE actual_R = R; - if (nr_c - blockIdx.z * blockDim.z < R) { - actual_R = nr_c - blockIdx.z * blockDim.z; - } - // if (nr_c % 2 == 1){ - // if(nr_c-1 - blockIdx.z * blockDim.z < R) { actual_R = nr_c - 1 - - // blockIdx.z * blockDim.z; } - // } else { - // if(nr_c - blockIdx.z * blockDim.z < R) { actual_R = nr_c - blockIdx.z * - // blockDim.z; } - // } - - // if (debug) printf("actual_R %d\n", actual_R); - - // if (debug) printf("RCF: %d %d %d\n", R, C, F); - if (r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - // if (debug) printf("load front vsm[%d]: %f <- %d %d %d\n", r_sm * 2 + 2, - // dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 2, c_sm, f_sm)] = - dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)]; - - if (r_sm == actual_R - 1) { - if (r_gl + 1 < nr_c) { - // if (debug) printf("load front+1 vsm[%d]: %f <- %d %d %d\n", actual_R - // * 2 + 2, dv1[get_idx(lddv11, lddv12, blockId * R + actual_R, c_gl, - // f_gl)], blockId * R + actual_R, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, actual_R * 2 + 2, c_sm, f_sm)] = - dv1[get_idx(lddv11, lddv12, r_gl + 1, c_gl, f_gl)]; - } else { - // if (debug) printf("load front+1 vsm[%d]: 0.0\n", actual_R * 2 + 2); - v_sm[get_idx(ldsm1, ldsm2, actual_R * 2 + 2, c_sm, f_sm)] = 0.0; - } - } - - if (r_sm == 0) { - if (r_gl >= 1) { - // if (debug) printf("load front-1 vsm[0]: %f <- %d %d %d\n", - // dv1[get_idx(lddv11, lddv12, r_gl-1, c_gl, f_gl)], r_gl-1, c_gl, - // f_gl); - v_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = - dv1[get_idx(lddv11, lddv12, r_gl - 1, c_gl, f_gl)]; - } else { - // if (debug) printf("load front-1 vsm[0]: 0.0\n"); - v_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = 0.0; - } - } - - if (!PADDING) { - if (r_gl < nr_c - 1) { - // if (debug) printf("load back vsm[%d]: %f <- %d %d %d\n", r_sm * 2 + - // 3, dv2[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)]; - } else { - // if (debug) printf("load back vsm[%d]: 0.0\n", r_sm * 2 + 3); - v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)] = 0.0; - } - } else { - if (r_gl < nr_c - 2) { - // if (debug) printf("load back vsm[%d]: %f <- %d %d %d\n", r_sm * 2 + - // 3, dv2[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, f_gl); - v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)] = - dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)]; - } else { - // if (debug) printf("load back vsm[%d]: 0.0\n", r_sm * 2 + 3); - v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)] = 0.0; - } - } - - if (r_gl >= 1 && - (PADDING && r_gl - 1 < nr_c - 2 || !PADDING && r_gl < nr_c)) { - // if (blockId > 0) { - if (r_sm == 0) { - // if (debug) printf("load back-1 vsm[1]: %f <- %d %d %d\n", - // dv2[get_idx(lddv11, lddv12, r_gl-1, c_gl, f_gl)], r_gl-1, c_gl, - // f_gl); - v_sm[get_idx(ldsm1, ldsm2, 1, c_sm, f_sm)] = - dv2[get_idx(lddv11, lddv12, r_gl - 1, c_gl, f_gl)]; - } - } else { - if (r_sm == 0) { - // if (debug) printf("load back-1 vsm[1]: 0.0\n"); - v_sm[get_idx(ldsm1, ldsm2, 1, c_sm, f_sm)] = 0.0; - } - } - } - - // load dist/ratio using f_sm for better performance - // assumption F >= R - // if (debug2) printf("actual_R: %u\n", actual_R); - if (r_sm == 0 && c_sm == 0 && f_sm < actual_R) { - // if (debug2) printf(" RCF (%u %u %u)blockid(%u) fsm(%u) nr(%u)\n", R, C, - // F, blockId, blockId * R * 2 + f_sm, nr); - if (blockId * R * 2 + f_sm < nr) { - - dist_r_sm[2 + f_sm] = ddist_r[blockId * R * 2 + f_sm]; - // if (debug2 ) printf("load dist 1 [%d]: %f [%d]\n", 2 + f_sm, - // dist_r_sm[2 + f_sm], blockId * R * 2 + f_sm); - ratio_r_sm[2 + f_sm] = dratio_r[blockId * R * 2 + f_sm]; - // if (debug2 )printf("load ratio 1 [%d]: %f [%d]\n", 2 + f_sm, - // ratio_r_sm[2 + f_sm], blockId * R * 2 + f_sm); - } else { - dist_r_sm[2 + f_sm] = 0.0; - ratio_r_sm[2 + f_sm] = 0.0; - } - if (blockId * R * 2 + actual_R + f_sm < nr) { - dist_r_sm[2 + actual_R + f_sm] = - ddist_r[blockId * R * 2 + actual_R + f_sm]; - // if (debug2 )printf("load dist 2 [%d]: %f [%d]\n", 2 + actual_R + f_sm, - // dist_r_sm[2 + actual_R + f_sm], blockId * R * 2 + actual_R + f_sm); - ratio_r_sm[2 + actual_R + f_sm] = - dratio_r[blockId * R * 2 + actual_R + f_sm]; - // if (debug2 )printf("load ratio 2 [%d]: %f [%d]\n", 2 + actual_R + f_sm, - // ratio_r_sm[2 + actual_R + f_sm], blockId * R * 2 + actual_R + f_sm); - } else { - dist_r_sm[2 + actual_R + f_sm] = 0.0; - ratio_r_sm[2 + actual_R + f_sm] = 0.0; - } - } - - if (blockId > 0) { - if (f_sm < 2) { - dist_r_sm[f_sm] = ddist_r[blockId * R * 2 - 2 + f_sm]; - // if (debug2 )printf("load dist -1 [%d]: %f [%d]\n", f_sm, - // dist_r_sm[f_sm], blockId * R * 2 - 2 + f_sm); - ratio_r_sm[f_sm] = dratio_r[blockId * R * 2 - 2 + f_sm]; - // if (debug2 )printf("load ratio -1 [%d]: %f [%d]\n", f_sm, - // ratio_r_sm[f_sm], blockId * R * 2 - 2 + f_sm); - } - } else { - if (f_sm < 2) { - dist_r_sm[f_sm] = 0.0; - ratio_r_sm[f_sm] = 0.0; - } - } - - __syncthreads(); - - int adjusted_nr_c = nr_c; - if (r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { - T h1 = dist_r_sm[r_sm * 2]; - T h2 = dist_r_sm[r_sm * 2 + 1]; - T h3 = dist_r_sm[r_sm * 2 + 2]; - T h4 = dist_r_sm[r_sm * 2 + 3]; - T r1 = ratio_r_sm[r_sm * 2]; - T r2 = ratio_r_sm[r_sm * 2 + 1]; - T r3 = ratio_r_sm[r_sm * 2 + 2]; - T r4 = 1 - r3; - T a = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2, c_sm, f_sm)]; - T b = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 1, c_sm, f_sm)]; - T c = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 2, c_sm, f_sm)]; - T d = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)]; - T e = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 4, c_sm, f_sm)]; - - // __syncthreads(); - // if (debug) { - // printf("r_sm(%d) %f %f %f %f %f\n",r_sm, a,b,c,d,e); - // printf("r_sm_h(%d) %f %f %f %f\n",r_sm, h1,h2,h3,h4); - // printf("r_sm_r(%d) %f %f %f %f\n",r_sm, r1,r2,r3,r4); - // } - // __syncthreads(); - - // T tb = a * h1/6 + b * 2 * (h1+h2)/6 + c * h2/6; - // T tc = b * h2/6 + c * 2 * (h2+h3)/6 + d * h3/6; - // T td = c * h3/6 + d * 2 * (h3+h4)/6 + e * h4/6; - - // if (debug) printf("f_sm(%d) tb tc td tc: %f %f %f %f\n", f_sm, tb, tc, - // td, tc+tb * r1 + td * r4); - - // tc += tb * r1 + td * r4; - - dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)] = - mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4); - - // if (debug) printf("store[%d %d %d] %f (%f)\n", r_gl, c_gl, f_gl, - // mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4), - // mass_trans(a, b, c, (T)0.0, (T)0.0, h1, (T)0.0, (T)0.0, - // h4, r1, r2, (T)0.0, (T)0.0)); - // // printf("%d %d %d\n", r_gl, c_gl, f_gl); - // if (blockId * R + R == nr-1) { - // if (r_gl+1 == nr_c - 1) { - // if (r_gl+1 == nr_c - 1) { - // // T te = h4 * d + 2 * h4 * e; - // // te += td * r3; - // dw[get_idx(lddw1, lddw2, blockId * R + actual_R, c_gl, f_gl)] = - // mass_trans(c, d, e, (T)0.0, (T)0.0, - // h1, h2, (T)0.0, (T)0.0, r1, r2, (T)0.0, (T)0.0); - - // if (debug) printf("store-last[%d %d %d] %f\n", blockId * R + actual_R, - // c_gl, f_gl, - // mass_trans(c, d, e, (T)0.0, (T)0.0, - // h1, h2, (T)0.0, (T)0.0, r1, r2, (T)0.0, (T)0.0)); - // } - //} - } -} - -template -void lpk_reo_3_3d_adaptive_launcher(Handle &handle, SIZE nr, SIZE nc_c, - SIZE nf_c, SIZE nr_c, T *ddist_r, - T *dratio_r, T *dv1, SIZE lddv11, - SIZE lddv12, T *dv2, SIZE lddv21, - SIZE lddv22, T *dw, SIZE lddw1, SIZE lddw2, - int queue_idx) { - - SIZE total_thread_z = nr_c; - // if (nr_c % 2 == 1){ total_thread_z = nr_c - 1; } - // else { total_thread_z = nr_c; } - SIZE total_thread_y = nc_c; - SIZE total_thread_x = nf_c; - - SIZE tbx, tby, tbz, gridx, gridy, gridz; - dim3 threadsPerBlock, blockPerGrid; - size_t sm_size; - - tbz = R; - tby = C; - tbx = F; - sm_size = ((R * 2 + 3) * C * F + (R * 2 + 3) * 2) * sizeof(T); - gridz = ceil((double)total_thread_z / tbz); - gridy = ceil((double)total_thread_y / tby); - gridx = ceil((double)total_thread_x / tbx); - threadsPerBlock = dim3(tbx, tby, tbz); - blockPerGrid = dim3(gridx, gridy, gridz); - - // printf("nr: %d nc_c: %d, nf_c: %d, nr_c: %d\n", nr, nc_c, nf_c, nr_c); - // printf("tb: %d %d %d, grid: %d %d %d\n", tbx, tby, tbz, gridx, gridy, - // gridz); - _lpk_reo_3_3d<<>>( - nr, nc_c, nf_c, nr_c, ddist_r, dratio_r, dv1, lddv11, lddv12, dv2, lddv21, - lddv22, dw, lddw1, lddw2); - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } -} - -template -void lpk_reo_3_3d(Handle &handle, SIZE nr, SIZE nc_c, SIZE nf_c, - SIZE nr_c, T *ddist_r, T *dratio_r, T *dv1, SIZE lddv11, - SIZE lddv12, T *dv2, SIZE lddv21, SIZE lddv22, T *dw, - SIZE lddw1, SIZE lddw2, int queue_idx, int config) { - -#define LPK(R, C, F) \ - { \ - lpk_reo_3_3d_adaptive_launcher( \ - handle, nr, nc_c, nf_c, nr_c, ddist_r, dratio_r, dv1, lddv11, lddv12, \ - dv2, lddv21, lddv22, dw, lddw1, lddw2, queue_idx); \ - } - bool profile = false; - if (handle.profile_kernels) { - profile = true; - } - if (D == 3) { - if (profile || config == 6) { - LPK(2, 2, 128) - } - if (profile || config == 5) { - LPK(2, 2, 64) - } - if (profile || config == 4) { - LPK(2, 2, 32) - } - if (profile || config == 3) { - LPK(4, 4, 16) - } - if (profile || config == 2) { - LPK(8, 8, 8) - } - if (profile || config == 1) { - LPK(4, 4, 4) - } - if (profile || config == 0) { - LPK(2, 2, 2) - } - } else { - printf("Error: mass_trans_multiply_3_cpt is only for 3D data\n"); - } - -#undef LPK -} - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/LinearQuantization.h b/include/cuda/LinearQuantization.h deleted file mode 100644 index 24c79e97a7..0000000000 --- a/include/cuda/LinearQuantization.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGRAD_CUDA_LINEAR_QUANTIZATION -#define MGRAD_CUDA_LINEAR_QUANTIZATION - -#include "Common.h" - -namespace mgard_cuda { - -template -void levelwise_linear_quantize(Handle &handle, SIZE *shapes, - SIZE l_target, T *volumes, SIZE ldvolumes, - Metadata &m, T *dv, SIZE *ldvs, - QUANTIZED_INT *dwork, SIZE *ldws, - bool prep_huffmam, SIZE *shape, - LENGTH *outlier_count, LENGTH *outlier_idx, - QUANTIZED_INT *outliers, int queue_idx); - -template -void levelwise_linear_dequantize(Handle &handle, SIZE *shapes, - SIZE l_target, T *volumes, SIZE ldvolumes, - Metadata &m, QUANTIZED_INT *dv, SIZE *ldvs, - T *dwork, SIZE *ldws, bool prep_huffmam, - LENGTH outlier_count, LENGTH *outlier_idx, - QUANTIZED_INT *outliers, int queue_idx); - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/LinearQuantization.hpp b/include/cuda/LinearQuantization.hpp deleted file mode 100644 index 7be9393dbb..0000000000 --- a/include/cuda/LinearQuantization.hpp +++ /dev/null @@ -1,923 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: September 27, 2021 - */ - -#ifndef MGRAD_CUDA_LINEAR_QUANTIZATION_TEMPLATE -#define MGRAD_CUDA_LINEAR_QUANTIZATION_TEMPLATE - -#include "CommonInternal.h" - -#include "LinearQuantization.h" - -namespace mgard_cuda { - -template -void calc_quantizers(Handle &handle, T *quantizers, Metadata &m, - bool reciprocal) { - - double abs_tol = m.tol; - if (m.ebtype == error_bound_type::REL) { - abs_tol *= m.norm; - } - - // printf("tol %f, l_target %d, D %d\n", tol, l_target, D); - - abs_tol *= 2; - - // original - // tol /= l_target + 2; - // for (int l = 0; l < l_target+1; l++) { - // quantizers[l] = tol; - // } - // printf("l_target %d\n", l_target); - - // levelwise - // tol *= 2; - // T C2 = 1 + 3*std::sqrt(3)/4; - // T c = std::sqrt(std::pow(2, D)); - // T cc = (1 - c) / (1 - std::pow(c, l_target+1)); - // T level_eb = cc * tol / C2; - - // for (int l = 0; l < l_target+1; l++) { - // quantizers[l] = level_eb; - // level_eb *= c; - // } - - // s = 0; - - // levelwise with s - // tol *= 2; - // T C2 = 1 + 3 * std::sqrt(3) / 4; - // T c = std::sqrt(std::pow(2, D - 2 * s)); - // T cc = (1 - c) / (1 - std::pow(c, l_target + 1)); - // T level_eb = cc * tol / C2; - - // for (int l = 0; l < l_target + 1; l++) { - // quantizers[l] = level_eb; - // // T c = std::sqrt(std::pow(2, 2*s*l + D * (l_target - l))); - // level_eb *= c; - // if (reciprocal) - // quantizers[l] = 1.0f / quantizers[l]; - // } - - if (m.ntype == norm_type::L_Inf) { - - // printf("quantizers: "); - for (int l = 0; l < m.l_target + 1; l++) { - // ben - quantizers[l] = (abs_tol) / ((m.l_target + 1) * (1 + std::pow(3, D))); - // xin - // quantizers[l] = (tol) / ((l_target + 1) * (1 + 3 * std::sqrt(3) / 4)); - - // printf("%f ", quantizers[l]); - if (reciprocal) - quantizers[l] = 1.0f / quantizers[l]; - } - // printf("\n"); - - } else if (m.ntype == norm_type::L_2) { // s != inf - // xin - uniform - // T C2 = 1 + 3 * std::sqrt(3) / 4; - // T c = std::sqrt(std::pow(2, D - 2 * s)); - // T cc = (1 - c) / (1 - std::pow(c, l_target + 1)); - // T level_eb = cc * tol / C2; - // for (int l = 0; l < l_target + 1; l++) { - // quantizers[l] = level_eb; - // // T c = std::sqrt(std::pow(2, 2*s*l + D * (l_target - l))); - // level_eb *= c; - // if (reciprocal) - // quantizers[l] = 1.0f / quantizers[l]; - // } - - // ben - uniform - // printf("quantizers: "); - - size_t dof = 1; - for (int d = 0; d < D; d++) - dof *= handle.dofs[d][0]; - // printf("tol: %f, dof: %llu\n", tol, dof); - // printf ("dof = %llu\n", dof); - for (int l = 0; l < m.l_target + 1; l++) { - - quantizers[l] = (abs_tol) / (std::exp2(m.s * l) * std::sqrt(dof)); - - // printf("l %d, vol: %f quantizer: %f \n", l, std::pow(2, (l_target - l) - // * D), quantizers[l]); - - // printf("tol: %f quant: %e \n", tol, quantizers[l]); - if (reciprocal) - quantizers[l] = 1.0f / quantizers[l]; - } - // printf("\n"); - } - - // print quantizers - // printf("quantizers: "); - // for (int l = 0; l < l_target+1; l++) { - // printf("%f ", 1.0f/quantizers[l]); - // } - // printf("\n"); -} - -template -__global__ void -_levelwise_linear_quantize(SIZE *shapes, SIZE l_target, T *quantizers, - T *volumes, SIZE ldvolumes, T *dv, SIZE *ldvs, - QUANTIZED_INT *dwork, SIZE *ldws, bool prep_huffmam, - SIZE dict_size, SIZE *shape, LENGTH *outlier_count, - LENGTH *outlier_idx, QUANTIZED_INT *outliers) { - - size_t threadId = (threadIdx.z * (blockDim.x * blockDim.y)) + - (threadIdx.y * blockDim.x) + threadIdx.x; - T *smT = SharedMemory(); - T *quantizers_sm = smT; - smT += l_target + 1; - - T *volumes_0 = smT; - if (CALC_VOL) - smT += blockDim.x * (l_target + 1); - T *volumes_1 = smT; - if (CALC_VOL) - smT += blockDim.y * (l_target + 1); - T *volumes_2 = smT; - if (CALC_VOL) - smT += blockDim.z * (l_target + 1); - T *volumes_3_plus = smT; - if (CALC_VOL && D > 3) - smT += (D - 3) * (l_target + 1); - - SIZE *smInt = (SIZE *)smT; - SIZE *ldvs_sm = smInt; - smInt += D; - SIZE *ldws_sm = smInt; - smInt += D; - SIZE *shape_sm = smInt; - smInt += D; - SIZE *shapes_sm = smInt; - smInt += D * (l_target + 2); - - if (threadId < l_target + 1) { - quantizers_sm[threadId] = quantizers[threadId]; - } - if (threadId < D) { - ldvs_sm[threadId] = ldvs[threadId]; - ldws_sm[threadId] = ldws[threadId]; - shape_sm[threadId] = shape[threadId]; - } - if (threadId < D * (l_target + 2)) { - shapes_sm[threadId] = shapes[threadId]; - // printf ("D: %d l_target+2: %d load shapes[%llu]: %d\n", D, l_target+2, - // threadId, shapes_sm[threadId]); - } - - __syncthreads(); - - // determine global idx - SIZE idx[D]; // thread global idx - SIZE idx0[D]; // block global idx - - SIZE firstD = div_roundup(shapes_sm[l_target + 1], F); - - SIZE bidx = blockIdx.x; - idx[0] = (bidx % firstD) * F + threadIdx.x; - idx0[0] = (bidx % firstD) * F; - - // printf("shapes_sm[l_target+1]: %d firstD %d idx[0] %d\n", - // shapes_sm[l_target+1], firstD, idx[0]); - - bidx /= firstD; - if (D >= 2) { - idx[1] = blockIdx.y * blockDim.y + threadIdx.y; - idx0[1] = blockIdx.y * blockDim.y; - } - if (D >= 3) { - idx[2] = blockIdx.z * blockDim.z + threadIdx.z; - idx0[2] = blockIdx.z * blockDim.z; - } - - for (int d = 3; d < D; d++) { - idx[d] = bidx % shapes_sm[(l_target + 2) * d + l_target + 1]; - idx0[d] = idx[d]; - bidx /= shapes_sm[(l_target + 2) * d + l_target + 1]; - } - - if (CALC_VOL) { - // cache volumes - for (int l = 0; l < l_target + 1; l++) { - // volumes 0 - if (threadId < blockDim.x && - idx0[0] + threadId < shapes_sm[(l_target + 2) * 0 + l_target + 1]) { - volumes_0[l * blockDim.x + threadId] = - volumes[(0 * (l_target + 1) + l) * ldvolumes + idx0[0] + threadId]; - // printf("load %f\n", volumes[(0 * (l_target + 1) + l) * ldvolumes + - // idx0[0] + threadId]); - } - if (D >= 2) { - // volumes 1 - if (threadId < blockDim.y && - idx0[1] + threadId < shapes_sm[(l_target + 2) * 1 + l_target + 1]) { - volumes_1[l * blockDim.y + threadId] = - volumes[(1 * (l_target + 1) + l) * ldvolumes + idx0[1] + - threadId]; - } - } - if (D >= 3) { - // volumes 2 - if (threadId < blockDim.z && - idx0[2] + threadId < shapes_sm[(l_target + 2) * 2 + l_target + 1]) { - volumes_2[l * blockDim.z + threadId] = - volumes[(2 * (l_target + 1) + l) * ldvolumes + idx0[2] + - threadId]; - } - } - } - - if (D >= 4) { - if (threadId < 1) { - for (int d = 3; d < D; d++) { - for (int l = 0; l < l_target + 1; l++) { - volumes_3_plus[(d - 3) * (l_target + 1) + l] = - volumes[(d * (l_target + 1) + l) * ldvolumes + idx[d]]; - } - } - } - } - } - - // if (blockIdx.y == 0 && blockIdx.x == 0 && blockIdx.z == 0 && threadId == 0) - // { - // printf("volumes_0: "); - // for (int l = 0; l < l_target+1; l++) { - // printf("l = %d\n", l); - // for (int i = 0; i < min(blockDim.x, shapes_sm[(l_target + 2) * 0 + - // l_target + 1]) ; i++) { - // printf("%f ", volumes_0[l * blockDim.x + i]); - // } - // printf("\n"); - // } - // printf("\n"); - // if (D >= 2) { - // printf("volumes_1: "); - // for (int l = 0; l < l_target+1; l++) { - // printf("l = %d\n", l); - // for (int i = 0; i < min(blockDim.y, shapes_sm[(l_target + 2) * 1 + - // l_target + 1]); i++) { - // printf("%f ", volumes_1[l * blockDim.y + i]); - // } - // printf("\n"); - // } - - // printf("\n"); - // } - // if (D >= 3) { - // printf("volumes_2: "); - // for (int l = 0; l < l_target+1; l++) { - // printf("l = %d\n", l); - // for (int i = 0; i < min(blockDim.z, shapes_sm[(l_target + 2) * 2 + - // l_target + 1]); i++) { - // printf("%f ", volumes_2[l * blockDim.y + i]); - // } - // printf("\n"); - // } - // } - // } - - __syncthreads(); - - int level = 0; - for (DIM d = 0; d < D; d++) { - long long unsigned int l_bit = 0l; - for (SIZE l = 0; l < l_target + 1; l++) { - int bit = (idx[d] >= shapes_sm[(l_target + 2) * d + l]) && - (idx[d] < shapes_sm[(l_target + 2) * d + l + 1]); - l_bit += bit << l; - // printf("idx: %d %d d: %d l_bit: %llu\n", idx[1], idx[0], d, l_bit); - } - level = max(level, __ffsll(l_bit)); - } - level = level - 1; - - bool in_range = true; - for (DIM d = 0; d < D; d++) { - if (idx[d] >= shapes_sm[(l_target + 2) * d + l_target + 1]) - in_range = false; - } - - // printf("idx %llu, level: %d, in_range: %d idx[0]: shape_sm: %d\n", - // get_idx(shape_sm, idx), level, in_range, shapes_sm[(l_target+2) * 0 + - // l_target+1]); - - if (level >= 0 && level <= l_target && in_range) { - T t = dv[get_idx(ldvs, idx)]; - T volume = 1; - if (CALC_VOL) { - volume *= volumes_0[level * blockDim.x + threadIdx.x]; - if (D >= 2) { - volume *= volumes_1[level * blockDim.y + threadIdx.y]; - } - if (D >= 3) { - volume *= volumes_2[level * blockDim.z + threadIdx.z]; - } - if (D >= 4) { - for (int d = 3; d < D; d++) { - volume *= volumes_3_plus[(d - 3) * (l_target + 1) + level]; - } - } - if (sizeof(T) == sizeof(double)) - volume = sqrt(volume); - else if (sizeof(T) == sizeof(float)) - volume = sqrtf(volume); - } - // printf("l: %d, vol %f(%f*%f*%f), quantizers_sm: %f, quantizers: %f, - // before: %f, quantized: %d\n", level, volume, - // volumes_0[level * blockDim.x + threadIdx.x], volumes_1[level * - // blockDim.y + threadIdx.y], volumes_2[level * blockDim.z + threadIdx.z], - // quantizers_sm[level], - // (quantizers_sm[level] / volume), t, (int)copysign(0.5 + fabs(t /( - // quantizers_sm[level] / volume)), t)); - - QUANTIZED_INT quantized_data = - copysign(0.5 + fabs(t / (quantizers_sm[level] * volume)), t); - // QUANTIZED_INT quantized_data = copysign(0.5 + fabs(t / - // (quantizers_sm[level] / volume) ), t); printf("dv[%llu] %f - // quantizers[%d]%f -> dw[%llu]%d \n", - // get_idx(ldvs, idx), t, - // level, quantizers_sm[level], - // get_idx(ldws, idx), quantized_data+dict_size / 2); - - if (prep_huffmam) { - quantized_data += dict_size / 2; - if (quantized_data >= 0 && quantized_data < dict_size) { - // do nothing - } else { - LENGTH i = atomicAdd(outlier_count, (LENGTH)1); - outlier_idx[i] = get_idx(shape_sm, idx); - outliers[i] = quantized_data; - quantized_data = 0; - } - // if (get_idx(shape_sm, idx) < quant_meta_size_ratio) { - // size_t i = atomicAdd((unsigned long long int*)outlier_count, - // (unsigned long long int)1); outlier_idx[i] = get_idx(shape_sm, - // idx); - // } - } - - dwork[get_idx(ldws, idx)] = quantized_data; - } -} - -template -void levelwise_linear_quantize_adaptive_launcher( - Handle &handle, SIZE *shapes, SIZE l_target, T *volumes, - SIZE ldvolumes, Metadata &m, T *dv, SIZE *ldvs, QUANTIZED_INT *dwork, - SIZE *ldws, bool prep_huffmam, SIZE *shape, LENGTH *outlier_count, - LENGTH *outlier_idx, QUANTIZED_INT *outliers, int queue_idx) { - - T *quantizers = new T[l_target + 1]; - calc_quantizers(handle, quantizers, m, false); - cudaMemcpyAsyncHelper(handle, handle.quantizers, quantizers, - sizeof(T) * (l_target + 1), H2D, queue_idx); - // printf("norm: %f, tol: %f, s: %f, dict_size: %d\n", m.norm, m.tol, m.s, - // m.dict_size); - int total_thread_z = handle.dofs[2][0]; - int total_thread_y = handle.dofs[1][0]; - int total_thread_x = handle.dofs[0][0]; - // linearize other dimensions - int tbz = R; - int tby = C; - int tbx = F; - int gridz = ceil((double)total_thread_z / tbz); - int gridy = ceil((double)total_thread_y / tby); - int gridx = ceil((double)total_thread_x / tbx); - for (int d = 3; d < D; d++) { - gridx *= handle.dofs[d][0]; - } - - // printf("exec: %d %d %d %d %d %d\n", tbx, tby, tbz, gridx, gridy, gridz); - dim3 threadsPerBlock(tbx, tby, tbz); - dim3 blockPerGrid(gridx, gridy, gridz); - // ldvs + ldws + shape - size_t sm_size = (D * 3) * sizeof(SIZE); - // quantizer - sm_size += (l_target + 1) * sizeof(T); - // ranges - sm_size += (l_target + 2) * D * sizeof(SIZE); - // volumes - sm_size += tbx * (l_target + 1) * sizeof(T); - sm_size += tby * (l_target + 1) * sizeof(T); - sm_size += tbz * (l_target + 1) * sizeof(T); - if (D > 3) - sm_size += (D - 3) * (l_target + 1) * sizeof(T); - // printf("sm_size: %llu\n", sm_size); - if (m.ntype == norm_type::L_Inf) { - _levelwise_linear_quantize - <<>>( - shapes, l_target, handle.quantizers, volumes, ldvolumes, dv, ldvs, - dwork, ldws, prep_huffmam, m.dict_size, shape, outlier_count, - outlier_idx, outliers); - } else if (m.ntype == norm_type::L_2) { - _levelwise_linear_quantize - <<>>( - shapes, l_target, handle.quantizers, volumes, ldvolumes, dv, ldvs, - dwork, ldws, prep_huffmam, m.dict_size, shape, outlier_count, - outlier_idx, outliers); - } else { - std::cout << log::log_err << "unsupported norm type!\n"; - exit(-1); - } - - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } -} - -template -void levelwise_linear_quantize(Handle &handle, SIZE *shapes, - SIZE l_target, T *volumes, SIZE ldvolumes, - Metadata &m, T *dv, SIZE *ldvs, - QUANTIZED_INT *dwork, SIZE *ldws, - bool prep_huffmam, SIZE *shape, - LENGTH *outlier_count, LENGTH *outlier_idx, - QUANTIZED_INT *outliers, int queue_idx) { -#define QUANTIZE(R, C, F) \ - { \ - levelwise_linear_quantize_adaptive_launcher( \ - handle, shapes, l_target, volumes, ldvolumes, m, dv, ldvs, dwork, \ - ldws, prep_huffmam, shape, outlier_count, outlier_idx, outliers, \ - queue_idx); \ - } - - if (D >= 3) { - QUANTIZE(4, 4, 16) - } - if (D == 2) { - QUANTIZE(1, 4, 32) - } - if (D == 1) { - QUANTIZE(1, 1, 64) - } - -#undef QUANTIZE -} - -template -__global__ void _levelwise_linear_dequantize( - SIZE *shapes, SIZE l_target, T *quantizers, T *volumes, SIZE ldvolumes, - QUANTIZED_INT *dv, SIZE *ldvs, T *dwork, SIZE *ldws, bool prep_huffmam, - SIZE dict_size, LENGTH outlier_count, LENGTH *outlier_idx, - QUANTIZED_INT *outliers) { - - LENGTH threadId = (threadIdx.z * (blockDim.x * blockDim.y)) + - (threadIdx.y * blockDim.x) + threadIdx.x; - LENGTH blockId = (blockIdx.z * (gridDim.x * gridDim.y)) + - (blockIdx.y * gridDim.x) + blockIdx.x; - LENGTH gloablId = blockId * blockDim.x * blockDim.y * blockDim.z + threadId; - - T *smT = SharedMemory(); - T *quantizers_sm = smT; - smT += l_target + 1; - T *volumes_0 = smT; - if (CALC_VOL) - smT += blockDim.x * (l_target + 1); - T *volumes_1 = smT; - if (CALC_VOL) - smT += blockDim.y * (l_target + 1); - T *volumes_2 = smT; - if (CALC_VOL) - smT += blockDim.z * (l_target + 1); - T *volumes_3_plus = smT; - if (CALC_VOL && D > 3) - smT += (D - 3) * (l_target + 1); - - SIZE *smInt = (SIZE *)smT; - SIZE *ldvs_sm = smInt; - smInt += D; - SIZE *ldws_sm = smInt; - smInt += D; - SIZE *shape_sm = smInt; - smInt += D; - SIZE *shapes_sm = smInt; - smInt += D * (l_target + 2); - - if (threadId < l_target + 1) { - quantizers_sm[threadId] = quantizers[threadId]; - } - if (threadId < D) { - ldvs_sm[threadId] = ldvs[threadId]; - ldws_sm[threadId] = ldws[threadId]; - } - if (threadId < D * (l_target + 2)) { - shapes_sm[threadId] = shapes[threadId]; - } - - __syncthreads(); - - // bool debug = false; - // if (blockIdx.x == 0 && blockIdx.y == 0 && blockIdx.z == 0 && - // threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) { - // debug = true; - // for (int d = 0; d < D; d ++) { - // printf("shapes_sm[%d]\n", d); - // for (int l = 0; l < l_target + 1; l++) { - // printf("%d ", shapes_sm[(l_target+1) * d + l]); - // } - // printf("\n"); - // } - // } - // __syncthreads(); - - // determine global idx - SIZE idx[D]; // thread global idx - SIZE idx0[D]; // block global idx - - SIZE firstD = div_roundup(shapes_sm[l_target + 1], F); - - SIZE bidx = blockIdx.x; - idx[0] = (bidx % firstD) * F + threadIdx.x; - idx0[0] = (bidx % firstD) * F; - - // printf("shapes_sm[l_target+1]: %d firstD %d idx[0] %d\n", - // shapes_sm[l_target+1], firstD, idx[0]); - - bidx /= firstD; - if (D >= 2) { - idx[1] = blockIdx.y * blockDim.y + threadIdx.y; - idx0[1] = blockIdx.y * blockDim.y; - } - if (D >= 3) { - idx[2] = blockIdx.z * blockDim.z + threadIdx.z; - idx0[2] = blockIdx.z * blockDim.z; - } - - for (DIM d = 3; d < D; d++) { - idx[d] = bidx % shapes_sm[(l_target + 2) * d + l_target + 1]; - idx0[d] = idx[d]; - bidx /= shapes_sm[(l_target + 2) * d + l_target + 1]; - } - - if (CALC_VOL) { - // cache volumes - for (SIZE l = 0; l < l_target + 1; l++) { - // volumes 0 - if (threadId < blockDim.x && - idx0[0] + threadId < shapes_sm[(l_target + 2) * 0 + l_target + 1]) { - // printf("%d < %d[%d, %d, %d]\n", idx0[0] + (int)threadId, - // shapes_sm[(l_target + 2) * 0 + l_target + 1], - // l_target, (l_target + 2) * 0 + l_target + 1, l_target + 2); - volumes_0[l * blockDim.x + threadId] = - volumes[(0 * (l_target + 1) + l) * ldvolumes + idx0[0] + threadId]; - // printf("load %f\n", volumes_0[l * blockDim.x + threadId]); - } - if (D >= 2) { - // volumes 1 - if (threadId < blockDim.y && - idx0[1] + threadId < shapes_sm[(l_target + 2) * 1 + l_target + 1]) { - volumes_1[l * blockDim.y + threadId] = - volumes[(1 * (l_target + 1) + l) * ldvolumes + idx0[1] + - threadId]; - } - } - if (D >= 3) { - // volumes 2 - if (threadId < blockDim.z && - idx0[2] + threadId < shapes_sm[(l_target + 2) * 2 + l_target + 1]) { - volumes_2[l * blockDim.z + threadId] = - volumes[(2 * (l_target + 1) + l) * ldvolumes + idx0[2] + - threadId]; - } - } - } - - if (D >= 4) { - if (threadId < 1) { - for (DIM d = 3; d < D; d++) { - for (SIZE l = 0; l < l_target + 1; l++) { - volumes_3_plus[(d - 3) * (l_target + 1) + l] = - volumes[(d * (l_target + 1) + l) * ldvolumes + idx[d]]; - } - } - } - } - } - - // if (blockIdx.y == 0 && blockIdx.x == 0 && threadId == 0) { - // printf("volumes_0: "); - // for (int l = 0; l < l_target+1; l++) { - // printf("l = %d\n", l); - // for (int i = 0; i < min(blockDim.x, shapes_sm[(l_target + 2) * 0 + - // l_target + 1]) ; i++) { - // printf("%f ", volumes_0[l * blockDim.x + i]); - // } - // printf("\n"); - // } - // printf("\n"); - // printf("volumes_1: "); - // for (int l = 0; l < l_target+1; l++) { - // printf("l = %d\n", l); - // for (int i = 0; i < min(blockDim.y, shapes_sm[(l_target + 2) * 1 + - // l_target + 1]); i++) { - // printf("%f ", volumes_1[l * blockDim.y + i]); - // } - // printf("\n"); - // } - - // } - - __syncthreads(); - - int level = 0; - for (DIM d = 0; d < D; d++) { - long long unsigned int l_bit = 0l; - for (SIZE l = 0; l < l_target + 1; l++) { - int bit = (idx[d] >= shapes_sm[(l_target + 2) * d + l]) && - (idx[d] < shapes_sm[(l_target + 2) * d + l + 1]); - l_bit += bit << l; - // printf("idx: %d %d d: %d l_bit: %llu\n", idx[1], idx[0], d, l_bit); - } - level = max(level, __ffsll(l_bit)); - } - - bool in_range = true; - for (DIM d = 0; d < D; d++) { - if (idx[d] >= shapes_sm[(l_target + 2) * d + l_target + 1]) - in_range = false; - } - - level = level - 1; - if (level >= 0 && level <= l_target && in_range) { - // printf("%d %d %d %d\n", idx[3], idx[2], idx[1], idx[0]); - // printf("idx: %d %d l: %d\n", idx[1], idx[0], level); - QUANTIZED_INT quantized_data = dv[get_idx(ldvs, idx)]; - T volume = 1; - if (CALC_VOL) { - volume *= volumes_0[level * blockDim.x + threadIdx.x]; - if (D >= 2) - volume *= volumes_1[level * blockDim.y + threadIdx.y]; - if (D >= 3) - volume *= volumes_2[level * blockDim.z + threadIdx.z]; - if (D >= 4) { - for (int d = 3; d < D; d++) { - volume *= volumes_3_plus[(d - 3) * (l_target + 1) + level]; - } - } - if (sizeof(T) == sizeof(double)) - volume = sqrt(volume); - else if (sizeof(T) == sizeof(float)) - volume = sqrtf(volume); - } - - if (prep_huffmam) { - quantized_data -= dict_size / 2; - } - - // printf("%d %d %d %d %d %d vol %f (%f * %f * %f), dequantizers: %f, - // before: %d, dequantized: %f\n", blockIdx.z, blockIdx.y, blockIdx.x, - // threadIdx.z, threadIdx.y, threadIdx.x, volume, - // volumes_0[level * blockDim.x + threadIdx.x], volumes_1[level * - // blockDim.y + threadIdx.y], volumes_2[level * blockDim.z + threadIdx.z], - // quantizers_sm[level] / volume, quantized_data, (quantizers_sm[level] / - // volume) * (T)quantized_data); - dwork[get_idx(ldws, idx)] = - (quantizers_sm[level] * volume) * (T)quantized_data; - // dwork[get_idx(ldws, idx)] = (quantizers_sm[level] / volume) * - // (T)quantized_data; dwork[get_idx(ldws, idx)] = (T)dv[get_idx(ldvs, - // idx)]; - - // printf("dw[%llu] %d dequantizers[%d]%f -> dw[%llu]%f \n", - // get_idx(ldvs, idx), - // quantized_data, level, quantizers_sm[level], get_idx(ldws, idx), - // quantizers_sm[level] * (T)quantized_data); - } - - // //outliers - // if (gloablId < outlier_count) { - // size_t linerized_idx = outlier_idx[gloablId]; - // for (int d = 0; d < D; d++) { - // idx[d] = linerized_idx % shapes_sm[(l_target+2) * d+l_target+1]; - // linerized_idx /= shapes_sm[(l_target+2) * d+l_target+1]; - // } - // int outliter = outliers[gloablId]; - // outliter -= dict_size / 2; - - // level = 0; - // for (int d = 0; d < D; d++) { - // long long unsigned int l_bit = 0l; - // for (int l = 0; l < l_target+1; l++) { - // int bit = (idx[d] >= shapes_sm[(l_target+2) * d + l]) && (idx[d] < - // shapes_sm[(l_target+2) * d + l+1]); l_bit += bit << l; - // // printf("idx: %d %d d: %d l_bit: %llu\n", idx[1], idx[0], d, - // l_bit); - // } - // level = max(level, __ffsll(l_bit)); - // } - // level = level - 1; - - // dwork[get_idx(ldws, idx)] = quantizers_sm[level] * (T)outliter; - - // // printf("outliter: dw[%llu] %d dequantizers[%d]%f -> dw[%llu]%f \n", - // get_idx(ldvs, idx), - // // outliter, level, quantizers_sm[level], get_idx(ldws, idx), - // quantizers_sm[level] * (T)outliter); - - // } -} - -template -__global__ void _levelwise_linear_dequantize_outliers( - SIZE *shapes, SIZE l_target, T *quantizers, T *volumes, SIZE ldvolumes, - QUANTIZED_INT *dv, SIZE *ldvs, T *dwork, SIZE *ldws, SIZE dict_size, - LENGTH outlier_count, LENGTH *outlier_idx, QUANTIZED_INT *outliers) { - - size_t threadId = (threadIdx.z * (blockDim.x * blockDim.y)) + - (threadIdx.y * blockDim.x) + threadIdx.x; - size_t blockId = (blockIdx.z * (gridDim.x * gridDim.y)) + - (blockIdx.y * gridDim.x) + blockIdx.x; - size_t gloablId = blockId * blockDim.x * blockDim.y * blockDim.z + threadId; - - T *sm = SharedMemory(); - T *quantizers_sm = sm; - sm += l_target + 1; - - SIZE *sm_size = (SIZE *)sm; - SIZE *ldvs_sm = sm_size; - sm_size += D; - SIZE *ldws_sm = sm_size; - sm_size += D; - SIZE *shapes_sm = sm_size; - sm_size += D * (l_target + 2); - - if (threadId < l_target + 1) { - quantizers_sm[threadId] = quantizers[threadId]; - } - if (threadId < D) { - ldvs_sm[threadId] = ldvs[threadId]; - ldws_sm[threadId] = ldws[threadId]; - } - if (threadId < D * (l_target + 2)) { - shapes_sm[threadId] = shapes[threadId]; - } - - __syncthreads(); - SIZE idx[D]; // thread global idx - - // outliers - if (gloablId < outlier_count) { - size_t linerized_idx = outlier_idx[gloablId]; - for (DIM d = 0; d < D; d++) { - idx[d] = linerized_idx % shapes_sm[(l_target + 2) * d + l_target + 1]; - linerized_idx /= shapes_sm[(l_target + 2) * d + l_target + 1]; - } - QUANTIZED_INT outliter = outliers[gloablId]; - outliter -= dict_size / 2; - - int level = 0; - for (DIM d = 0; d < D; d++) { - long long unsigned int l_bit = 0l; - for (SIZE l = 0; l < l_target + 1; l++) { - int bit = (idx[d] >= shapes_sm[(l_target + 2) * d + l]) && - (idx[d] < shapes_sm[(l_target + 2) * d + l + 1]); - l_bit += bit << l; - // printf("idx: %d %d d: %d l_bit: %llu\n", idx[1], idx[0], d, l_bit); - } - level = max(level, __ffsll(l_bit)); - } - level = level - 1; - - T volume = 1; - - if (CALC_VOL) { - for (DIM d = 0; d < D; d++) { - volume *= volumes[(d * (l_target + 1) + level) * ldvolumes + idx[d]]; - } - if (sizeof(T) == sizeof(double)) - volume = sqrt(volume); - else if (sizeof(T) == sizeof(float)) - volume = sqrtf(volume); - } - dwork[get_idx(ldws, idx)] = - (quantizers_sm[level] * volume) * (T)outliter; - // dwork[get_idx(ldws, idx)] = (quantizers_sm[level] / volume) * - // (T)outliter; - - // printf("outliter: dw[%llu] %d dequantizers[%d]%f -> dw[%llu]%f \n", - // get_idx(ldvs, idx), - // outliter, level, quantizers_sm[level], get_idx(ldws, idx), - // quantizers_sm[level] * (T)outliter); - } -} - -template -void levelwise_linear_dequantize_adaptive_launcher( - Handle &handle, SIZE *shapes, SIZE l_target, T *volumes, - SIZE ldvolumes, Metadata &m, QUANTIZED_INT *dv, SIZE *ldvs, T *dwork, - SIZE *ldws, bool prep_huffman, LENGTH outlier_count, LENGTH *outlier_idx, - QUANTIZED_INT *outliers, int queue_idx) { - - // printf("norm: %f, tol: %f, s: %f, dict_size: %d\n", m.norm, m.tol, m.s, - // m.dict_size); - - T *quantizers = new T[l_target + 1]; - calc_quantizers(handle, quantizers, m, false); - cudaMemcpyAsyncHelper(handle, handle.quantizers, quantizers, - sizeof(T) * (l_target + 1), H2D, queue_idx); - - SIZE total_thread_z = handle.dofs[2][0]; - SIZE total_thread_y = handle.dofs[1][0]; - SIZE total_thread_x = handle.dofs[0][0]; - // linearize other dimensions - SIZE tbz = R; - SIZE tby = C; - SIZE tbx = F; - SIZE gridz = ceil((double)total_thread_z / tbz); - SIZE gridy = ceil((double)total_thread_y / tby); - SIZE gridx = ceil((double)total_thread_x / tbx); - for (DIM d = 3; d < D; d++) { - gridx *= handle.dofs[d][0]; - } - - // printf("exec: %d %d %d %d %d %d\n", tbx, tby, tbz, gridx, gridy, gridz); - dim3 threadsPerBlock(tbx, tby, tbz); - dim3 blockPerGrid(gridx, gridy, gridz); - size_t sm_size = (D * 3) * sizeof(SIZE); - sm_size += (l_target + 1) * sizeof(T); - sm_size += (l_target + 2) * D * sizeof(SIZE); - sm_size += tbx * (l_target + 1) * sizeof(T); - sm_size += tby * (l_target + 1) * sizeof(T); - sm_size += tbz * (l_target + 1) * sizeof(T); - if (D > 3) - sm_size += (D - 3) * (l_target + 1) * sizeof(T); - - if (m.ntype == norm_type::L_Inf) { - _levelwise_linear_dequantize - <<>>( - shapes, l_target, handle.quantizers, volumes, ldvolumes, dv, ldvs, - dwork, ldws, prep_huffman, m.dict_size, outlier_count, outlier_idx, - outliers); - if (prep_huffman) { - _levelwise_linear_dequantize_outliers - <<>>( - shapes, l_target, handle.quantizers, volumes, ldvolumes, dv, ldvs, - dwork, ldws, m.dict_size, outlier_count, outlier_idx, outliers); - } - } else if (m.ntype == norm_type::L_2) { - _levelwise_linear_dequantize - <<>>( - shapes, l_target, handle.quantizers, volumes, ldvolumes, dv, ldvs, - dwork, ldws, prep_huffman, m.dict_size, outlier_count, outlier_idx, - outliers); - if (prep_huffman) { - _levelwise_linear_dequantize_outliers - <<>>( - shapes, l_target, handle.quantizers, volumes, ldvolumes, dv, ldvs, - dwork, ldws, m.dict_size, outlier_count, outlier_idx, outliers); - } - } else { - std::cout << log::log_err << "unsupported norm type!\n"; - exit(-1); - } - gpuErrchk(cudaGetLastError()); - if (handle.sync_and_check_all_kernels) { - gpuErrchk(cudaDeviceSynchronize()); - } -} - -template -void levelwise_linear_dequantize(Handle &handle, SIZE *shapes, - SIZE l_target, T *volumes, SIZE ldvolumes, - Metadata &m, QUANTIZED_INT *dv, SIZE *ldvs, - T *dwork, SIZE *ldws, bool prep_huffmam, - LENGTH outlier_count, LENGTH *outlier_idx, - QUANTIZED_INT *outliers, int queue_idx) { -#define DEQUANTIZE(R, C, F) \ - { \ - levelwise_linear_dequantize_adaptive_launcher( \ - handle, shapes, l_target, volumes, ldvolumes, m, dv, ldvs, dwork, \ - ldws, prep_huffmam, outlier_count, outlier_idx, outliers, queue_idx); \ - } - - if (D >= 3) { - DEQUANTIZE(4, 4, 16) - } - if (D == 2) { - DEQUANTIZE(1, 4, 32) - } - if (D == 1) { - DEQUANTIZE(1, 1, 64) - } - -#undef DEQUANTIZE -} - -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/LosslessCompression.h b/include/cuda/LosslessCompression.h deleted file mode 100644 index 45f70f0ed6..0000000000 --- a/include/cuda/LosslessCompression.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGARD_CUDA_LOSSLESS -#define MGARD_CUDA_LOSSLESS - -#include "Common.h" -#include "nvcomp.hpp" -#include "nvcomp/cascaded.h" -#include "nvcomp/cascaded.hpp" -#include "nvcomp/lz4.hpp" -#include "nvcomp/nvcompManagerFactory.hpp" - -namespace mgard_cuda { - -template -void cascaded_compress(Handle &handle, C *input_data, size_t input_count, - void *&output_data, size_t &output_size, int n_rle, - int n_de, bool bitpack, int queue_idx); - -template -void cascaded_decompress(Handle &handle, void *input_data, - size_t input_size, C *&output_data, int queue_idx); - -template -void lz4_compress(Handle &handle, C *input_data, size_t input_count, - void *&output_data, size_t &output_size, size_t chunk_size, - int queue_idx); - -template -void lz4_decompress(Handle &handle, void *input_data, size_t input_size, - C *&output_data, size_t &output_size, int queue_idx); - -template -void SeparateOutlierAndPrimary(Handle &handle, S *dqv, size_t n, - size_t *outlier_idx, size_t outlier_count, - size_t primary_count, S *doutlier, Q *dprimary, - int queue_idx); -template -void CombineOutlierAndPrimary(Handle &handle, S *dqv, size_t n, - size_t *outlier_idx, size_t outlier_count, - size_t primary_count, S *doutlier, Q *dprimary, - int queue_idx); - -template -void huffman_compress(Handle &handle, S *input_data, size_t input_count, - std::vector &outlier_idx, H *&out_meta, - size_t &out_meta_size, H *&out_data, - size_t &out_data_size, int chunk_size, int dict_size, - int queue_idx); -template -void huffman_decompress(Handle &handle, H *in_meta, size_t in_meta_size, - H *in_data, size_t in_data_size, S *&output_data, - size_t &output_count, int queue_idx); - -template -void cpu_lossless_compression(Handle &handle, S *input_data, - size_t input_count, H *&out_data, - size_t &out_data_size); -template -void cpu_lossless_decompression(Handle &handle, H *input_data, - size_t input_count, S *&out_data, - size_t output_count); -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/MemoryManagement.h b/include/cuda/MemoryManagement.h deleted file mode 100644 index 00ed15aae3..0000000000 --- a/include/cuda/MemoryManagement.h +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: September 27, 2021 - */ - -#ifndef MGARD_CUDA_MEMORY_MANAGEMENT -#define MGARD_CUDA_MEMORY_MANAGEMENT - -#include "Handle.h" - -#include - -namespace mgard_cuda { - -enum copy_type { H2D, D2H, D2D, H2H, AUTO }; - -enum endiness_type CheckEndianess(); - -template -void PrintSubarray(std::string name, SubArrayType subArray); - -template void print_matrix(SIZE nrow, SIZE ncol, T *v, SIZE ldv); -template -void print_matrix_cuda(SIZE nrow, SIZE ncol, T *dv, SIZE lddv); - -template -void print_matrix(SIZE nrow, SIZE ncol, SIZE nfib, T *v, SIZE ldv1, SIZE ldv2); -template -void print_matrix_cuda(SIZE nrow, SIZE ncol, SIZE nfib, T *dv, SIZE lddv1, - SIZE lddv2, SIZE sizex); - -template -bool compare_matrix(SIZE nrow, SIZE ncol, T *v1, SIZE ldv1, T *v2, SIZE ldv2); -template -bool compare_matrix_cuda(SIZE nrow, SIZE ncol, T *dv1, SIZE lddv1, T *dv2, - SIZE lddv2); -template -bool compare_matrix(SIZE nrow, SIZE ncol, SIZE nfib, T *v1, SIZE ldv11, - SIZE ldv12, T *v2, SIZE ldv21, SIZE ldv22, - bool print_matrix); -template -bool compare_matrix_cuda(SIZE nrow, SIZE ncol, SIZE nfib, T *dv1, SIZE lddv11, - SIZE lddv12, SIZE sizex1, T *dv2, SIZE lddv21, - SIZE lddv22, SIZE sizex2, bool print_matrix); - -template -void verify_matrix(SIZE nrow, SIZE ncol, SIZE nfib, T *v, SIZE ldv1, SIZE ldv2, - std::string file_prefix, bool store, bool verify); -template -void verify_matrix_cuda(SIZE nrow, SIZE ncol, SIZE nfib, T *dv, SIZE lddv1, - SIZE lddv2, SIZE sizex, std::string file_prefix, - bool store, bool verify); - -template -void cudaMallocHelper(Handle &handle, void **devPtr, size_t size_t); - -template -void cudaMallocPitchHelper(Handle &handle, void **devPtr, size_t *pitch, - size_t width, size_t height); - -template -void cudaMalloc3DHelper(Handle &handle, void **devPtr, size_t *pitch, - size_t width, size_t height, size_t depth); -void cudaMallocHostHelper(void **ptr, size_t size_t); - -template -void cudaMemcpyAsyncHelper(Handle &handle, void *dst, const void *src, - size_t count, enum copy_type kind, int queue_idx); - -template -void cudaMemcpy2DAsyncHelper(Handle &handle, void *dst, size_t dpitch, - void *src, size_t spitch, size_t width, - size_t height, enum copy_type kind, int queue_idx); - -template -void cudaMemcpy3DAsyncHelper(Handle &handle, void *dst, size_t dpitch, - size_t dwidth, size_t dheight, const void *src, - size_t spitch, size_t swidth, size_t sheight, - size_t width, size_t height, size_t depth, - enum copy_type kind, int queue_idx); - -void cudaFreeHelper(void *devPtr); -void cudaFreeHostHelper(void *ptr); -void cudaMemsetHelper(void *devPtr, int value, size_t count); -void cudaMemset2DHelper(void *devPtr, size_t pitch, int value, size_t width, - size_t height); -void cudaMemset3DHelper(void *devPtr, size_t pitch, size_t dwidth, - size_t dheight, int value, size_t width, size_t height, - size_t depth); -void cudaSetDeviceHelper(int dev_id); - -template -void cudaMemcpyPeerAsyncHelper(Handle &handle, void *dst, int dst_dev, - const void *src, int src_dev, size_t count, - int queue_idx); - -template -void cudaMemcpy3DPeerAsyncHelper(Handle &handle, void *dst, int dst_dev, - size_t dpitch, size_t dwidth, size_t dheight, - const void *src, int src_dev, size_t spitch, - size_t swidth, size_t sheight, size_t width, - size_t height, size_t depth, int queue_idx); -bool isGPUPointer(const void *ptr); -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/Message.h b/include/cuda/Message.h deleted file mode 100644 index bc125f19c8..0000000000 --- a/include/cuda/Message.h +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef MESSGAE_HH -#define MESSGAE_HH - -#include -#include -#include - -using std::string; - -namespace mgard_cuda { -namespace log { - -extern const string log_null; -extern const string log_err; -extern const string log_dbg; -extern const string log_info; -extern const string log_warn; -extern const string log_time; - -// https://stackoverflow.com/a/26080768/8740097 -template void build(std::ostream &o, T t); - -template -void build(std::ostream &o, T t, Args... args); - -template void print(string log_head, Args... args); - -} // namespace log - -} // namespace mgard_cuda - -#endif // FORMAT_HH diff --git a/include/cuda/Metadata.h b/include/cuda/Metadata.h deleted file mode 100644 index 30db316bbe..0000000000 --- a/include/cuda/Metadata.h +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: September 27, 2021 - */ - -#ifndef MGRAD_CUDA_METADATA -#define MGRAD_CUDA_METADATA - -#include - -#define MAGIC_WORD "MGARD" -#define MAGIC_WORD_SIZE 5 - -namespace mgard_cuda { - -struct Metadata { - // about MGARD software - char magic_word[MAGIC_WORD_SIZE + 1] = MAGIC_WORD; - uint8_t software_version[3]; - uint8_t file_version[3]; - uint32_t metadata_size; - uint32_t metadata_crc32; - enum processor_type ptype; - - // about compression - enum error_bound_type ebtype; - double norm; // optional - double tol; - enum norm_type ntype; - double s; // optional - uint32_t l_target; - enum lossless_type ltype; - uint32_t dict_size; // optional (for GPU_Huffman) - - // about data - enum data_type dtype; - enum endiness_type etype; - enum data_structure_type dstype; - uint8_t total_dims = 0; - uint64_t *shape; - enum coordinate_location cltype; - char *nonuniform_coords_file; - std::vector coords; - -public: - SERIALIZED_TYPE *Serialize(uint32_t &total_size); - void Deserialize(SERIALIZED_TYPE *serialized_data, uint32_t &total_size); - size_t metadata_size_offset() { - size_t offset = 0; - offset += strlen(magic_word); - offset += sizeof(software_version); - offset += sizeof(file_version); - return offset; - } - ~Metadata() { - if (self_initialized) { - delete[] shape; - if (dstype == data_structure_type::Cartesian_Grid_Non_Uniform) { - for (size_t d = 0; d < total_dims; d++) { - delete[] coords[d]; - } - } - } - } - -private: - template void Serialize(T &item, SERIALIZED_TYPE *&p) { - std::memcpy(p, &item, sizeof(item)); - p += sizeof(item); - } - void Serialize(char *item, SERIALIZED_TYPE *&p) { - std::memcpy(p, item, strlen(item)); - p += strlen(item); - } - template - void Serialize(T *&item, N n, SERIALIZED_TYPE *&p) { - std::memcpy(p, item, sizeof(T) * n); - p += sizeof(T) * n; - } - - void Serialize(std::vector &coords, uint64_t *shape, - enum data_type dtype, SERIALIZED_TYPE *&p) { - for (size_t i = 0; i < coords.size(); i++) { - if (dtype == data_type::Float) { - Serialize(coords[i], shape[i] * sizeof(float), p); - } else if (dtype == data_type::Double) { - Serialize(coords[i], shape[i] * sizeof(double), p); - } - } - } - - template void Deserialize(T &item, SERIALIZED_TYPE *&p) { - std::memcpy(&item, p, sizeof(item)); - p += sizeof(item); - } - void Deserialize(char *item, SERIALIZED_TYPE *&p) { - std::memcpy(item, p, strlen(item)); - p += strlen(item); - } - template - void Deserialize(T *&item, N n, SERIALIZED_TYPE *&p) { - std::memcpy(item, p, sizeof(T) * n); - p += sizeof(T) * n; - } - - void Deserialize(std::vector &coords, uint64_t *shape, - enum data_type dtype, SERIALIZED_TYPE *&p) { - for (size_t i = 0; i < coords.size(); i++) { - if (dtype == data_type::Float) { - coords[i] = (Byte *)std::malloc(shape[i] * sizeof(float)); - Deserialize(coords[i], shape[i] * sizeof(float), p); - } else if (dtype == data_type::Double) { - coords[i] = (Byte *)std::malloc(shape[i] * sizeof(double)); - Deserialize(coords[i], shape[i] * sizeof(double), p); - } - } - } - - bool self_initialized; -}; -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/ParallelHuffman/canonical.cuh b/include/cuda/ParallelHuffman/canonical.cuh deleted file mode 100644 index c7592e1c4a..0000000000 --- a/include/cuda/ParallelHuffman/canonical.cuh +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef CANONICAL_CUH -#define CANONICAL_CUH - -#include - -namespace GPU { - -//__device__ int max_bw; - -template -__global__ void GetCanonicalCode(uint8_t *singleton, int DICT_SIZE); - -} // namespace GPU -#endif diff --git a/include/cuda/ParallelHuffman/constants.hh b/include/cuda/ParallelHuffman/constants.hh deleted file mode 100644 index 65a5471b6a..0000000000 --- a/include/cuda/ParallelHuffman/constants.hh +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef INTERNAL_CONST_HH -#define INTERNAL_CONST_HH - -#include - -// dims_L16 -extern const size_t DIM0; -extern const size_t DIM1; -extern const size_t DIM2; -extern const size_t DIM3; -extern const size_t nBLK0; -extern const size_t nBLK1; -extern const size_t nBLK2; -extern const size_t nBLK3; -extern const size_t nDIM; -extern const size_t LEN; -extern const size_t CAP; -extern const size_t RADIUS; - -// ebs_L4 -extern const size_t EB; -extern const size_t EBr; -extern const size_t EBx2; -extern const size_t EBx2_r; - -extern const int B_1d; -extern const int B_2d; -extern const int B_3d; - -#endif diff --git a/include/cuda/ParallelHuffman/cuda_error_handling.cuh b/include/cuda/ParallelHuffman/cuda_error_handling.cuh deleted file mode 100644 index fe20b7923a..0000000000 --- a/include/cuda/ParallelHuffman/cuda_error_handling.cuh +++ /dev/null @@ -1,209 +0,0 @@ -#ifndef CUDA_ERROR_HANDLING -#define CUDA_ERROR_HANDLING - -#include -#include - -#include - -// back compatibility start -static void HandleError(cudaError_t err, const char *file, int line) { - if (err != cudaSuccess) { - printf("%s in %s at line %d\n", cudaGetErrorString(err), file, line); - exit(EXIT_FAILURE); - } -} -#define HANDLE_ERROR(err) (HandleError(err, __FILE__, __LINE__)) -// back compatibility end - -static void check_cuda_error(cudaError_t status, const char *file, int line) { - if (cudaSuccess != status) { - /* - printf("\nCUDA error/status reference (as of CUDA 11):\n"); - printf("cudaSuccess -> %d\n", cudaSuccess); - printf("cudaErrorInvalidValue -> %d\n", -cudaErrorInvalidValue); printf("cudaErrorMemoryAllocation -> %d\n", -cudaErrorMemoryAllocation); printf("cudaErrorInitializationError -> -%d\n", cudaErrorInitializationError); printf("cudaErrorCudartUnloading -> %d\n", -cudaErrorCudartUnloading); printf("cudaErrorProfilerDisabled -> %d\n", -cudaErrorProfilerDisabled); printf("cudaErrorProfilerNotInitialized -(Deprecated)-> %d\n", cudaErrorProfilerNotInitialized); - printf("cudaErrorProfilerAlreadyStarted (Deprecated)-> %d\n", -cudaErrorProfilerAlreadyStarted); printf("cudaErrorProfilerAlreadyStopped -(Deprecated)-> %d\n", cudaErrorProfilerAlreadyStopped); - printf("cudaErrorInvalidConfiguration -> %d\n", -cudaErrorInvalidConfiguration); printf("cudaErrorInvalidPitchValue -> -%d\n", cudaErrorInvalidPitchValue); printf("cudaErrorInvalidSymbol -> %d\n", -cudaErrorInvalidSymbol); printf("cudaErrorInvalidHostPointer (Deprecated)-> -%d\n", cudaErrorInvalidHostPointer); printf("cudaErrorInvalidDevicePointer -(Deprecated)-> %d\n", cudaErrorInvalidDevicePointer); - printf("cudaErrorInvalidTexture -> %d\n", -cudaErrorInvalidTexture); printf("cudaErrorInvalidTextureBinding -> %d\n", -cudaErrorInvalidTextureBinding); printf("cudaErrorInvalidChannelDescriptor -> -%d\n", cudaErrorInvalidChannelDescriptor); - printf("cudaErrorInvalidMemcpyDirection -> %d\n", -cudaErrorInvalidMemcpyDirection); printf("cudaErrorAddressOfConstant -(Deprecated)-> %d\n", cudaErrorAddressOfConstant); - printf("cudaErrorTextureFetchFailed (Deprecated)-> %d\n", -cudaErrorTextureFetchFailed); printf("cudaErrorTextureNotBound (Deprecated)-> -%d\n", cudaErrorTextureNotBound); printf("cudaErrorSynchronizationError -(Deprecated)-> %d\n", cudaErrorSynchronizationError); - printf("cudaErrorInvalidFilterSetting -> %d\n", -cudaErrorInvalidFilterSetting); printf("cudaErrorInvalidNormSetting -> -%d\n", cudaErrorInvalidNormSetting); printf("cudaErrorMixedDeviceExecution -(Deprecated)-> %d\n", cudaErrorMixedDeviceExecution); - printf("cudaErrorNotYetImplemented (Deprecated)-> %d\n", -cudaErrorNotYetImplemented); printf("cudaErrorMemoryValueTooLarge (Deprecated)-> -%d\n", cudaErrorMemoryValueTooLarge); printf("cudaErrorInsufficientDriver -> -%d\n", cudaErrorInsufficientDriver); printf("cudaErrorInvalidSurface -> %d\n", -cudaErrorInvalidSurface); printf("cudaErrorDuplicateVariableName -> %d\n", -cudaErrorDuplicateVariableName); printf("cudaErrorDuplicateTextureName -> -%d\n", cudaErrorDuplicateTextureName); printf("cudaErrorDuplicateSurfaceName -> -%d\n", cudaErrorDuplicateSurfaceName); printf("cudaErrorDevicesUnavailable -> -%d\n", cudaErrorDevicesUnavailable); printf("cudaErrorIncompatibleDriverContext --> %d\n", cudaErrorIncompatibleDriverContext); - printf("cudaErrorMissingConfiguration -> %d\n", -cudaErrorMissingConfiguration); printf("cudaErrorPriorLaunchFailure -(Deprecated)-> %d\n", cudaErrorPriorLaunchFailure); - printf("cudaErrorLaunchMaxDepthExceeded -> %d\n", -cudaErrorLaunchMaxDepthExceeded); printf("cudaErrorLaunchFileScopedTex -> -%d\n", cudaErrorLaunchFileScopedTex); printf("cudaErrorLaunchFileScopedSurf -> -%d\n", cudaErrorLaunchFileScopedSurf); printf("cudaErrorSyncDepthExceeded -> -%d\n", cudaErrorSyncDepthExceeded); printf("cudaErrorLaunchPendingCountExceeded --> %d\n", cudaErrorLaunchPendingCountExceeded); - printf("cudaErrorInvalidDeviceFunction -> %d\n", -cudaErrorInvalidDeviceFunction); printf("cudaErrorNoDevice -> -%d\n", cudaErrorNoDevice); printf("cudaErrorInvalidDevice -> %d\n", -cudaErrorInvalidDevice); printf("cudaErrorStartupFailure -> %d\n", -cudaErrorStartupFailure); printf("cudaErrorInvalidKernelImage -> %d\n", -cudaErrorInvalidKernelImage); #if (CUDART_VERSION == 1100) - printf("cudaErrorDeviceUninitialized -> %d\n", -cudaErrorDeviceUninitialized); #endif printf("cudaErrorMapBufferObjectFailed -> -%d\n", cudaErrorMapBufferObjectFailed); printf("cudaErrorUnmapBufferObjectFailed --> %d\n", cudaErrorUnmapBufferObjectFailed); #if (CUDART_VERSION == 1010) - printf("cudaErrorArrayIsMapped -> %d\n", -cudaErrorArrayIsMapped); printf("cudaErrorAlreadyMapped -> %d\n", -cudaErrorAlreadyMapped); #endif printf("cudaErrorNoKernelImageForDevice -> -%d\n", cudaErrorNoKernelImageForDevice); #if (CUDART_VERSION == 1010) - printf("cudaErrorAlreadyAcquired -> %d\n", -cudaErrorAlreadyAcquired); printf("cudaErrorNotMapped -> %d\n", -cudaErrorNotMapped); printf("cudaErrorNotMappedAsArray -> %d\n", -cudaErrorNotMappedAsArray); printf("cudaErrorNotMappedAsPointer -> -%d\n", cudaErrorNotMappedAsPointer); #endif printf("cudaErrorECCUncorrectable -> -%d\n", cudaErrorECCUncorrectable); printf("cudaErrorUnsupportedLimit -> %d\n", -cudaErrorUnsupportedLimit); printf("cudaErrorDeviceAlreadyInUse -> -%d\n", cudaErrorDeviceAlreadyInUse); printf("cudaErrorPeerAccessUnsupported -> -%d\n", cudaErrorPeerAccessUnsupported); printf("cudaErrorInvalidPtx -> %d\n", -cudaErrorInvalidPtx); printf("cudaErrorInvalidGraphicsContext -> %d\n", -cudaErrorInvalidGraphicsContext); printf("cudaErrorNvlinkUncorrectable -> -%d\n", cudaErrorNvlinkUncorrectable); printf("cudaErrorJitCompilerNotFound -> -%d\n", cudaErrorJitCompilerNotFound); #if (CUDART_VERSION == 1010) - printf("cudaErrorInvalidSource -> %d\n", -cudaErrorInvalidSource); printf("cudaErrorFileNotFound -> %d\n", -cudaErrorFileNotFound); #endif printf("cudaErrorSharedObjectSymbolNotFound -> -%d\n", cudaErrorSharedObjectSymbolNotFound); - printf("cudaErrorSharedObjectInitFailed -> %d\n", -cudaErrorSharedObjectInitFailed); printf("cudaErrorOperatingSystem -> -%d\n", cudaErrorOperatingSystem); printf("cudaErrorInvalidResourceHandle -> -%d\n", cudaErrorInvalidResourceHandle); #if (CUDART_VERSION == 1010) - printf("cudaErrorIllegalState -> %d\n", -cudaErrorIllegalState); printf("cudaErrorSymbolNotFound -> %d\n", -cudaErrorSymbolNotFound); #endif printf("cudaErrorNotReady -> -%d\n", cudaErrorNotReady); printf("cudaErrorIllegalAddress -> %d\n", -cudaErrorIllegalAddress); printf("cudaErrorLaunchOutOfResources -> %d\n", -cudaErrorLaunchOutOfResources); printf("cudaErrorLaunchTimeout -> -%d\n", cudaErrorLaunchTimeout); #if (CUDART_VERSION == 1010) - printf("cudaErrorLaunchIncompatibleTexturing-> %d\n", -cudaErrorLaunchIncompatibleTexturing); #endif - printf("cudaErrorPeerAccessAlreadyEnabled -> %d\n", -cudaErrorPeerAccessAlreadyEnabled); printf("cudaErrorPeerAccessNotEnabled -> -%d\n", cudaErrorPeerAccessNotEnabled); printf("cudaErrorSetOnActiveProcess -> -%d\n", cudaErrorSetOnActiveProcess); #if (CUDART_VERSION == 1010) - printf("cudaErrorContextIsDestroyed -> %d\n", -cudaErrorContextIsDestroyed); #endif printf("cudaErrorAssert -> %d\n", -cudaErrorAssert); printf("cudaErrorTooManyPeers -> %d\n", -cudaErrorTooManyPeers); printf("cudaErrorHostMemoryAlreadyRegistered-> %d\n", -cudaErrorHostMemoryAlreadyRegistered); printf("cudaErrorHostMemoryNotRegistered --> %d\n", cudaErrorHostMemoryNotRegistered); printf("cudaErrorHardwareStackError --> %d\n", cudaErrorHardwareStackError); printf("cudaErrorIllegalInstruction -> -%d\n", cudaErrorIllegalInstruction); printf("cudaErrorMisalignedAddress -> -%d\n", cudaErrorMisalignedAddress); printf("cudaErrorInvalidAddressSpace -> -%d\n", cudaErrorInvalidAddressSpace); printf("cudaErrorInvalidPc -> %d\n", -cudaErrorInvalidPc); printf("cudaErrorLaunchFailure -> %d\n", -cudaErrorLaunchFailure); printf("cudaErrorCooperativeLaunchTooLarge -> %d\n", -cudaErrorCooperativeLaunchTooLarge); printf("cudaErrorNotPermitted -> %d\n", -cudaErrorNotPermitted); printf("cudaErrorNotSupported -> %d\n", -cudaErrorNotSupported); #if (CUDART_VERSION == 1010) - printf("cudaErrorSystemNotReady -> %d\n", -cudaErrorSystemNotReady); printf("cudaErrorSystemDriverMismatch -> %d\n", -cudaErrorSystemDriverMismatch); printf("cudaErrorCompatNotSupportedOnDevice -> -%d\n", cudaErrorCompatNotSupportedOnDevice); - printf("cudaErrorStreamCaptureUnsupported -> %d\n", -cudaErrorStreamCaptureUnsupported); printf("cudaErrorStreamCaptureInvalidated -> -%d\n", cudaErrorStreamCaptureInvalidated); printf("cudaErrorStreamCaptureMerge --> %d\n", cudaErrorStreamCaptureMerge); printf("cudaErrorStreamCaptureUnmatched --> %d\n", cudaErrorStreamCaptureUnmatched); - printf("cudaErrorStreamCaptureUnjoined -> %d\n", -cudaErrorStreamCaptureUnjoined); printf("cudaErrorStreamCaptureIsolation -> -%d\n", cudaErrorStreamCaptureIsolation); printf("cudaErrorStreamCaptureImplicit --> %d\n", cudaErrorStreamCaptureImplicit); printf("cudaErrorCapturedEvent -> -%d\n", cudaErrorCapturedEvent); printf("cudaErrorStreamCaptureWrongThread -> -%d\n", cudaErrorStreamCaptureWrongThread); #endif #if (CUDART_VERSION == 1100) - printf("cudaErrorTimeout -> %d\n", cudaErrorTimeout); - printf("cudaErrorGraphExecUpdateFailure -> %d\n", -cudaErrorGraphExecUpdateFailure); #endif printf("cudaErrorUnknown -> %d\n", -cudaErrorUnknown); printf("cudaErrorApiFailureBase (Deprecated)-> %d\n", -cudaErrorApiFailureBase); - */ - printf("\n"); - printf("CUDA API failed at \e[31m\e[1m%s:%d\e[0m with error: %s (%d)\n", // - file, line, cudaGetErrorString(status), status); - exit(EXIT_FAILURE); - } -} - -static void check_cusparse_error(cusparseStatus_t status, const char *file, - int line) { - if (CUSPARSE_STATUS_SUCCESS != status) { - printf("\nCUSPARSE status reference (as of CUDA 11):\n"); - printf("CUSPARSE_STATUS_SUCCESS -> %d\n", - CUSPARSE_STATUS_SUCCESS); - printf("CUSPARSE_STATUS_NOT_INITIALIZED -> %d\n", - CUSPARSE_STATUS_NOT_INITIALIZED); - printf("CUSPARSE_STATUS_ALLOC_FAILED -> %d\n", - CUSPARSE_STATUS_ALLOC_FAILED); - printf("CUSPARSE_STATUS_INVALID_VALUE -> %d\n", - CUSPARSE_STATUS_INVALID_VALUE); - printf("CUSPARSE_STATUS_ARCH_MISMATCH -> %d\n", - CUSPARSE_STATUS_ARCH_MISMATCH); - printf("CUSPARSE_STATUS_EXECUTION_FAILED -> %d\n", - CUSPARSE_STATUS_EXECUTION_FAILED); - printf("CUSPARSE_STATUS_INTERNAL_ERROR -> %d\n", - CUSPARSE_STATUS_INTERNAL_ERROR); - printf("CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED -> %d\n", - CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED); -#if (CUDART_VERSION == 1010) - printf("CUSPARSE_STATUS_NOT_SUPPORTED -> %d\n", - CUSPARSE_STATUS_NOT_SUPPORTED); -#endif -#if (CUDART_VERSION == 1100) - printf("CUSPARSE_STATUS_INSUFFICIENT_RESOURCES -> %d\n", - CUSPARSE_STATUS_INSUFFICIENT_RESOURCES); -#endif -#if (CUDART_VERSION == 1100) - printf("CUSPARSE_STATUS_INSUFFICIENT_RESOURCES -> %d\n", - CUSPARSE_STATUS_INSUFFICIENT_RESOURCES); -#endif - printf("\n"); -#if (CUDART_VERSION == 1010) - printf("CUSPARSE API failed at \e[31m\e[1m%s:%d\e[0m with error: %s (%d)\n", - file, line, cusparseGetErrorString(status), status); -#endif - exit(EXIT_FAILURE); - } -} - -#define CHECK_CUDA(err) (check_cuda_error(err, __FILE__, __LINE__)) -#define CHECK_CUSPARSE(err) (check_cusparse_error(err, __FILE__, __LINE__)) - -#endif diff --git a/include/cuda/ParallelHuffman/cuda_mem.cuh b/include/cuda/ParallelHuffman/cuda_mem.cuh deleted file mode 100644 index 3d036f9e65..0000000000 --- a/include/cuda/ParallelHuffman/cuda_mem.cuh +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef CUDA_MEM_CUH -#define CUDA_MEM_CUH - -#include -#include - -namespace mem { - -enum MemcpyDirection { h2d, d2h }; - -template T *CreateCUDASpace(size_t l, uint8_t i = 0); - -template -void CopyBetweenSpaces(T *src, T *dst, MemcpyDirection direct); - -template T *CreateDeviceSpaceAndMemcpyFromHost(T *var, size_t l); - -template T *CreateHostSpaceAndMemcpyFromDevice(T *d_var, size_t l); -} // namespace mem - -#endif diff --git a/include/cuda/ParallelHuffman/dbg_gpu_printing.cuh b/include/cuda/ParallelHuffman/dbg_gpu_printing.cuh deleted file mode 100644 index 493c311f0a..0000000000 --- a/include/cuda/ParallelHuffman/dbg_gpu_printing.cuh +++ /dev/null @@ -1,127 +0,0 @@ -#ifndef __GPU_PRINTING -#define __GPU_PRINTING - -template -__global__ void print_deflated(Q *coded, size_t gid) { - if (blockIdx.x * blockDim.x + threadIdx.x != gid) - return; - printf("print after deflating\n"); - // for_each(coded, coded + PART_SIZE, [](Q& i) { print_by_type(i, '_', - // '\n'); }); - for (size_t i = 0; i < PART_SIZE; i++) { - print_by_type(*(coded + i), '_', '\n'); - } - printf("\n"); -} - -template -__global__ void print_histogram(T *freq, size_t size, size_t radius = 20) { - const int DICT_SIZE = size; /* Dynamic sizing */ - if (blockIdx.x * blockDim.x + threadIdx.x == 0) { - for (size_t i = DICT_SIZE / 2 - radius; i < DICT_SIZE / 2 + radius; i++) { - if (i % 10 == 0) - printf("\n"); - printf("%4lu: %-12lu", i, static_cast(freq[i])); - } - printf("\n"); - } -} - -template -__device__ __host__ void print_by_type(T num, char sep = '_', - char ending = '\n') { - for (size_t j = 0; j < sizeof(T) * CHAR_BIT; j++) { - printf("%u", (num >> ((sizeof(T) * CHAR_BIT - 1) - j)) & 0x01u); - if (j != 0 and j != sizeof(T) * CHAR_BIT - 1 and j % 8 == 7) - printf("%c", sep); - } - printf("%c", ending); -} - -// MSB to LSB -template -__device__ __host__ void print_code_only(T num, size_t bitwidth, char sep = '_', - char ending = '\n') { - for (size_t j = 0; j < bitwidth; j++) { - printf("%u", (num >> ((bitwidth - 1) - j)) & 0x01u); - if (j != 0 and j != bitwidth - 1 and j % 8 == 7) - printf("%c", sep); - } - printf("%c", ending); -} - -template -__device__ __host__ void snippet_print_bitset_full(T num) { - print_by_type(num, '_', '\t'); - size_t bitwidth = *((uint8_t *)&num + sizeof(T) - 1); - // size_t code_bitwidth = ((static_cast(0xffu) << (sizeof(T) * 8 - 8)) & - // num) >> (sizeof(T) * 8 - 8); - printf("len: %3lu\tcode: ", bitwidth); - print_code_only(num, bitwidth, '\0', '\n'); -} - -template __global__ void print_codebook(T *codebook, size_t len) { - if (blockIdx.x * blockDim.x + threadIdx.x != 0) - return; - printf("---------------------------------------------------------------------" - "-----------\n"); - printf("printing codebook\n"); - printf("---------------------------------------------------------------------" - "-----------\n"); - __shared__ T buffer; - for (size_t i = 0; i < len; i++) { - buffer = codebook[i]; - if (buffer == ~((T)0x0)) - continue; - printf("%5lu\t", i); - snippet_print_bitset_full(buffer); - } - printf("---------------------------------------------------------------------" - "-----------\n"); - printf("done printing codebook\n"); - printf("---------------------------------------------------------------------" - "-----------\n"); -} - -template __global__ void get_entropy(T *freq) {} - -// TODO real GPU version -template -__global__ void -get_theoretical_dense_Huffman_coded_length(T *codebook, Q *freq, - size_t codebook_len) {} - -// template -//__global__ void print_Huffman_coded_before_deflating(T* coded, size_t len=200) -//{ -// if (blockIdx.x * blockDim.x + threadIdx.x != 0) return; -// printf("print Huffman coded before it is deflated\n"); -// for (size_t i = 0; i < 200; i++) { -// if (coded[i] == ~((T)0x0)) continue; -// printf("%5lu\t", i); -// snippet_print_bitset_full(coded[i]); -// } -// printf("\n"); -//} - -template -__global__ void print_Huffman_coded_before_deflating(T *coded, size_t len) { - if (blockIdx.x != 0) - return; - size_t gid = blockDim.x * blockIdx.x + threadIdx.x; - if (coded[gid] == ~((T)0x0)) - return; - printf("%5lu\t", gid); - snippet_print_bitset_full(coded[gid]); - - // if (coded[i] == ~((T)0x0)) continue; - // printf("print Huffman coded before it is deflated\n"); - // for (size_t i = 0; i < 200; i++) { - // if (coded[i] == ~((T)0x0)) continue; - // printf("%5lu\t", i); - // snippet_print_bitset_full(coded[i]); - // } - // printf("\n"); -} - -#endif \ No newline at end of file diff --git a/include/cuda/ParallelHuffman/format.hh b/include/cuda/ParallelHuffman/format.hh deleted file mode 100644 index ca24c451b7..0000000000 --- a/include/cuda/ParallelHuffman/format.hh +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef FORMAT_HH -#define FORMAT_HH - -#include -#include -#include - -using std::string; - -const string log_null = " "; -const string log_err = "\e[31m[ERR]\e[0m "; -const string log_dbg = "\e[34m[dbg]\e[0m "; -const string log_info = "\e[32m[info]\e[0m "; -const string log_warn = "\e[31m[WARN]\e[0m "; - -namespace huffman_gpu { -namespace log { - -// https://stackoverflow.com/a/26080768/8740097 -template void build(std::ostream &o, T t); - -template -void build(std::ostream &o, T t, Args... args); - -template void print(string log_head, Args... args); - -} // namespace log - -} // namespace huffman_gpu - -#endif // FORMAT_HH diff --git a/include/cuda/ParallelHuffman/histogram.cuh b/include/cuda/ParallelHuffman/histogram.cuh deleted file mode 100644 index 08f2d0d6d4..0000000000 --- a/include/cuda/ParallelHuffman/histogram.cuh +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef HISTOGRAM_CUH -#define HISTOGRAM_CUH - -#include -#include - -__global__ void naiveHistogram(int input_data[], int output[], int N, - int symbols_per_thread); - -const static unsigned int WARP_SIZE = 32; -#define MIN(a, b) ((a) < (b)) ? (a) : (b) - -// Optimized 2013 -/* Copied from J. Gomez-Luna et al */ -template -__global__ void p2013Histogram(T *input_data, Q *output, size_t N, int bins, - int R); - -#endif diff --git a/include/cuda/ParallelHuffman/huffman.cuh b/include/cuda/ParallelHuffman/huffman.cuh deleted file mode 100644 index 13737cda83..0000000000 --- a/include/cuda/ParallelHuffman/huffman.cuh +++ /dev/null @@ -1,108 +0,0 @@ -#ifndef HUFFMAN_CUH -#define HUFFMAN_CUH - -#include -#include -#include -#include - -using namespace std; -namespace prototype { -template -__global__ void GPU_Histogram(T *input_data, Q *output, size_t N, - int symbols_per_thread); - -template -__global__ void EncodeFixedLen(T *data, Q *hcoded, size_t data_len, - Q *codebook); - -} // namespace prototype - -struct alignas(8) node_t { - struct node_t *left, *right; - size_t freq; - char t; // in_node:0; otherwise:1 - uint32_t c; -}; - -typedef struct node_t *node_list; - -typedef struct alignas(8) HuffmanTree { - uint32_t stateNum; - uint32_t allNodes; - struct node_t *pool; - node_list *qqq, *qq; // the root node of the HuffmanTree is qq[1] - int n_nodes; // n_nodes is for compression - int qend; - uint64_t **code; - uint8_t *cout; - int n_inode; // n_inode is for decompression -} HuffmanTree; - -HuffmanTree *createHuffmanTree(int stateNum); - -__host__ __device__ node_list new_node(HuffmanTree *huffmanTree, size_t freq, - uint32_t c, node_list a, node_list b); -__host__ __device__ void qinsert(HuffmanTree *ht, node_list n); -__host__ __device__ node_list qremove(HuffmanTree *ht); -__host__ __device__ void build_code(HuffmanTree *ht, node_list n, int len, - uint64_t out1, uint64_t out2); - -// auxiliary functions done -__host__ HuffmanTree *createHuffmanTreeCPU(int stateNum); - -__device__ HuffmanTree *createHuffmanTreeGPU(int stateNum); - -__host__ __device__ node_list new_node(HuffmanTree *huffmanTree, size_t freq, - uint32_t c, node_list a, node_list b); - -/* priority queue */ -__host__ __device__ void qinsert(HuffmanTree *ht, node_list n); - -__host__ __device__ node_list qremove(HuffmanTree *ht); - -__host__ __device__ void build_code(HuffmanTree *ht, node_list n, int len, - uint64_t out1, uint64_t out2); - -//////////////////////////////////////////////////////////////////////////////// -// internal functions -//////////////////////////////////////////////////////////////////////////////// - -const int MAX_DEPTH = 32; -//#define MAX_DEPTH 32 - -typedef struct alignas(8) Stack { - node_list _a[MAX_DEPTH]; - uint64_t saved_path[MAX_DEPTH]; - uint64_t saved_length[MAX_DEPTH]; - uint64_t depth = 0; -} internal_stack_t; - -__device__ __forceinline__ bool isEmpty(internal_stack_t *s); - -__device__ __forceinline__ node_list top(internal_stack_t *s); - -template -__device__ __forceinline__ void push_v2(internal_stack_t *s, node_list n, - T path, T len); - -// TODO check with typing -template -__device__ __forceinline__ node_list pop_v2(internal_stack_t *s, - T *path_to_restore, - T *length_to_restore); - -template -__device__ void InOrderTraverse_v2(HuffmanTree *ht, Q *codebook); - -//////////////////////////////////////////////////////////////////////////////// -// global functions -//////////////////////////////////////////////////////////////////////////////// - -//__device__ HuffmanTree* global_gpuTree; - -template -__global__ void InitHuffTreeAndGetCodebook(int stateNum, unsigned int *freq, - H *codebook); - -#endif diff --git a/include/cuda/ParallelHuffman/huffman_codec.cuh b/include/cuda/ParallelHuffman/huffman_codec.cuh deleted file mode 100644 index 68d450c132..0000000000 --- a/include/cuda/ParallelHuffman/huffman_codec.cuh +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef DEFLATE_CUH -#define DEFLATE_CUH - -#include - -template -__global__ void EncodeFixedLen(Q *, H *, size_t, H *); - -template __global__ void Deflate(Q *, size_t, size_t *, int); - -template -__device__ void InflateChunkwise(H *, T *, size_t, uint8_t *); - -template -__global__ void Decode(H *, size_t *, Q *, size_t, int, int, uint8_t *, size_t); - -#endif diff --git a/include/cuda/ParallelHuffman/huffman_workflow.cuh b/include/cuda/ParallelHuffman/huffman_workflow.cuh deleted file mode 100644 index 0ce5cb1795..0000000000 --- a/include/cuda/ParallelHuffman/huffman_workflow.cuh +++ /dev/null @@ -1,65 +0,0 @@ -#ifndef HUFFMAN_WORKFLOW -#define HUFFMAN_WORKFLOW - -#include -//#include - -#include -#include -#include - -#include "../Common.h" -#include "../CommonInternal.h" - -using std::string; - -// const int GB_unit = 1073741824; // 1024^3 - -const int tBLK_ENCODE = 256; -const int tBLK_DEFLATE = 128; -const int tBLK_CANONICAL = 128; - -// https://stackoverflow.com/questions/12774207/fastest-way-to-check-if-a-file-exist-using-standard-c-c11-c -// inline bool exists_test2(const std::string& name) { -// return (access(name.c_str(), F_OK) != -1); -//} - -namespace wrapper { - -template -void GetFrequency(Q *d_bcode, size_t len, unsigned int *d_freq, int dict_size); - -template -void SetUpHuffmanTree(unsigned int *d_freq, H *d_codebook, int dict_size); - -template -void MakeCanonical(H *d_plain_cb, uint8_t *d_singleton, size_t total_bytes, - int dict_size); - -template -void EncodeByMemcpy(Q *d_bcode, size_t len, H *d_hcode, H *d_canonical_cb); - -template -void Deflate(H *d_hcode, size_t len, int chunk_size, int n_chunk, - size_t *d_dH_bit_meta); - -} // namespace wrapper - -template -void PrintChunkHuffmanCoding(size_t *dH_bit_meta, size_t *dH_uInt_meta, - size_t len, int chunk_size, size_t total_bits, - size_t total_uInts); - -typedef std::tuple tuple3ul; - -template -void HuffmanEncode(mgard_cuda::Handle &handle, S *dqv, size_t n, - std::vector &outlier_idx, H *&dmeta, - size_t &dmeta_size, H *&ddata, size_t &ddata_size, - int chunk_size, int dict_size); - -template -void HuffmanDecode(mgard_cuda::Handle &handle, S *&dqv, size_t &n, - H *dmeta, size_t dmeta_size, H *ddata, size_t ddata_size); - -#endif \ No newline at end of file diff --git a/include/cuda/ParallelHuffman/io.hh b/include/cuda/ParallelHuffman/io.hh deleted file mode 100644 index 2664846194..0000000000 --- a/include/cuda/ParallelHuffman/io.hh +++ /dev/null @@ -1,96 +0,0 @@ -#ifndef IO_HH -#define IO_HH - -#include -#include - -namespace io { - -/** - * @deprecated to replace with ReadBinaryToNewArray - */ -template -T *ReadBinaryFile(const std::string &__name, size_t __len) { - std::ifstream ifs(__name.c_str(), std::ios::binary | std::ios::in); - if (not ifs.is_open()) { - std::cerr << "fail to open " << __name << std::endl; - exit(1); - // return; - } - auto __a = new T[__len](); - ifs.read(reinterpret_cast(__a), std::streamsize(__len * sizeof(T))); - ifs.close(); - return __a; -} - -template -T *ReadBinaryToNewArray(const std::string &__name, size_t __len) { - std::ifstream ifs(__name.c_str(), std::ios::binary | std::ios::in); - if (not ifs.is_open()) { - std::cerr << "fail to open " << __name << std::endl; - exit(1); - // return; - } - auto __a = new T[__len](); - ifs.read(reinterpret_cast(__a), std::streamsize(__len * sizeof(T))); - ifs.close(); - return __a; -} - -/** - * @deprecated to replace with ReadBinaryToArray - */ -template -T *ReadBinaryFile(const std::string &__name, T *__a, size_t __len) { - std::ifstream ifs(__name.c_str(), std::ios::binary | std::ios::in); - if (not ifs.is_open()) { - std::cerr << "fail to open " << __name << std::endl; - exit(1); - // return; - } - // auto __a = new T[__len](); - ifs.read(reinterpret_cast(__a), std::streamsize(__len * sizeof(T))); - ifs.close(); - return __a; -} - -template -void ReadBinaryToArray(const std::string &__name, T *__a, size_t __len) { - std::ifstream ifs(__name.c_str(), std::ios::binary | std::ios::in); - if (not ifs.is_open()) { - std::cerr << "fail to open " << __name << std::endl; - exit(1); - // return; - } - // auto __a = new T[__len](); - ifs.read(reinterpret_cast(__a), std::streamsize(__len * sizeof(T))); - ifs.close(); -} - -/** - * @deprecated to replace with WriteArrayToBinary - */ -// template -// void WriteBinaryFile(T* const __a, size_t const __len, std::string const* -// const __name) -// { -// std::ofstream ofs(__name->c_str(), std::ios::binary | std::ios::out); -// if (not ofs.is_open()) return; -// ofs.write(reinterpret_cast(__a), std::streamsize(__len * -// sizeof(T))); ofs.close(); -// } - -template -void WriteArrayToBinary(const std::string &__name, T *const __a, - size_t const __len) { - std::ofstream ofs(__name.c_str(), std::ios::binary | std::ios::out); - if (not ofs.is_open()) - return; - ofs.write(reinterpret_cast(__a), - std::streamsize(__len * sizeof(T))); - ofs.close(); -} - -} // namespace io - -#endif // IO_HH diff --git a/include/cuda/ParallelHuffman/par_huffman.cuh b/include/cuda/ParallelHuffman/par_huffman.cuh deleted file mode 100644 index 126d40fcda..0000000000 --- a/include/cuda/ParallelHuffman/par_huffman.cuh +++ /dev/null @@ -1,78 +0,0 @@ -#ifndef PAR_HUFFMAN_CUH -#define PAR_HUFFMAN_CUH - -#include - -#include -#include -#include -#include - -using namespace std; -using namespace cooperative_groups; - -// Helper kernels -template -__global__ void GPU_FillArraySequence(T *array, unsigned int size); -template -__global__ void GPU_GetFirstNonzeroIndex(T *array, unsigned int size, - unsigned int *result); -template -__global__ void GPU_ReorderByIndex(T *array, Q *index, unsigned int size); -template -__global__ void GPU_ReverseArray(T *array, unsigned int size); - -// Parallel huffman global memory and kernels -namespace parHuff { -// GenerateCL Locals -// __device__ int iNodesFront = 0; -// __device__ int iNodesRear = 0; -// __device__ int lNodesCur = 0; - -// __device__ int iNodesSize = 0; -// __device__ int curLeavesNum; - -// __device__ int minFreq; - -// __device__ int tempLength; - -// __device__ int mergeFront; -// __device__ int mergeRear; - -// __device__ int lNodesIndex; - -// // GenerateCW Locals -// __device__ int CCL; -// __device__ int CDPI; -// __device__ int newCDPI; - -// // Profiling -// __device__ long long int s[10]; -// __device__ long long int st[10]; - -// Codeword length -// clang-format off -template -__global__ void GPU_GenerateCL( - F* histogram, F* CL, int size, - /* Global Arrays */ - F* lNodesFreq, int* lNodesLeader, - F* iNodesFreq, int* iNodesLeader, - F* tempFreq, int* tempIsLeaf, int* tempIndex, - F* copyFreq, int* copyIsLeaf, int* copyIndex, - uint32_t* diagonal_path_intersections, int mblocks, int mthreads); -// clang-format on - -// Forward Codebook -template -__global__ void GPU_GenerateCW(F *CL, H *CW, H *first, H *entry, int size); -} // namespace parHuff - -// Thrust sort functionality implemented in separate file -template void SortByFreq(K *freq, V *qcode, int size); - -template -void ParGetCodebook(int stateNum, unsigned int *freq, H *codebook, - uint8_t *meta); - -#endif diff --git a/include/cuda/ParallelHuffman/par_merge.cuh b/include/cuda/ParallelHuffman/par_merge.cuh deleted file mode 100644 index ed53402502..0000000000 --- a/include/cuda/ParallelHuffman/par_merge.cuh +++ /dev/null @@ -1,58 +0,0 @@ -#ifndef PAR_MERGE_CUH -#define PAR_MERGE_CUH - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -using namespace cooperative_groups; - -// Partition array -template -__device__ void -cudaWorkloadDiagonals(F *copyFreq, int *copyIndex, int *copyIsLeaf, int cStart, - int cEnd, F *iNodesFreq, int iStart, int iEnd, - int iNodesCap, uint32_t *diagonal_path_intersections, - /* Shared Memory */ - int32_t &x_top, int32_t &y_top, int32_t &x_bottom, - int32_t &y_bottom, int32_t &found, int32_t *oneorzero); - -// Merge partitions -template -__device__ void -cudaMergeSinglePath(F *copyFreq, int *copyIndex, int *copyIsLeaf, int cStart, - int cEnd, F *iNodesFreq, int iStart, int iEnd, - int iNodesCap, uint32_t *diagonal_path_intersections, - F *tempFreq, int *tempIndex, int *tempIsLeaf, - int tempLength); - -template -__device__ void -parMerge(F *copyFreq, int *copyIndex, int *copyIsLeaf, int cStart, int cEnd, - F *iNodesFreq, int iStart, int iEnd, int iNodesCap, F *tempFreq, - int *tempIndex, int *tempIsLeaf, int &tempLength, - uint32_t *diagonal_path_intersections, int blocks, int threads, - /* Shared Memory */ - int32_t &x_top, int32_t &y_top, int32_t &x_bottom, int32_t &y_bottom, - int32_t &found, int32_t *oneorzero); - -template -__device__ void merge(F *copyFreq, int *copyIndex, int *copyIsLeaf, int cStart, - int cEnd, F *iNodesFreq, int iStart, int iEnd, - int iNodesCap, F *tempFreq, int *tempIndex, - int *tempIsLeaf, int &tempLength); - -#endif diff --git a/include/cuda/ParallelHuffman/timer.hh b/include/cuda/ParallelHuffman/timer.hh deleted file mode 100644 index 046a06ab4f..0000000000 --- a/include/cuda/ParallelHuffman/timer.hh +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef TIMER_HH -#define TIMER_HH - -#include -#include - -using std::cerr; -using std::cout; -using std::endl; - -using hires = std::chrono::high_resolution_clock; -typedef std::chrono::duration duration_t; -typedef std::chrono::time_point - hires_clock_t; - -#endif // TIMER_HH diff --git a/include/cuda/ParallelHuffman/types.hh b/include/cuda/ParallelHuffman/types.hh deleted file mode 100644 index c6bf6e552c..0000000000 --- a/include/cuda/ParallelHuffman/types.hh +++ /dev/null @@ -1,79 +0,0 @@ -#ifndef TYPES_HH -#define TYPES_HH - -#include -#include // for FP32 bit representation -#include // size_t -#include -#include -#include -#include -#include -#include - -#include "format.hh" -#include "io.hh" -#include "timer.hh" - -using namespace std; - -template double GetDatumValueRange(string fname, size_t l); - -size_t *InitializeDims(size_t cap, size_t n_dims, size_t dim0, size_t dim1 = 1, - size_t dim2 = 1, size_t dim3 = 1); - -void SetDims(size_t *dims_L16, size_t new_dims[4]); - -typedef struct ErrorBoundConfigurator { - int capacity, radius; - double base, exp_base2, exp_base10; - double eb_base2, eb_base10, eb_final; - std::string mode; - - void ChangeToRelativeMode(double value_range); - - void ChangeToTightBase2(); - - ErrorBoundConfigurator(int _capacity = 32768, double _precision = 1, - double _exponent = -3, int _base = 10); - - void debug() const; - -} config_t; - -// typedef struct DimensionInfo dim_t; -// typedef struct ErrorBoundConfigurator config_t; - -double *InitializeErrorBoundFamily(struct ErrorBoundConfigurator *eb_config); - -/* -2^-1 2^-2 2^-3 -10^-1 -2^-4 2^-5 2^-6 -10^-2 -2^-7 2^-8 2^-9 -10^-3 -2^-10 2^-11 2^-12 2^-13 -10^-4 -2^-14 2^-15 2^-16 -10^-5 -2^-17 2^-18 2^-19 -10^-6 -2^-20 2^-21 2^-22 2^-23 -10^-7 -2^-24 2^-25 2^-26 -10^-8 -2^-27 2^-28 2^-29 -10^-9 -2^-30 2^-31 2^-32 2^-33 -10^-10 -2^-34 - */ - -// static std::unordered_map exp_dec2bin = {{-1, -4}, {-2, -7}, -// {-3, -10}, {-4, -14}, {-5, -17}, -// {-6, -20}, {-7, -24}, -// {-8, -27}, {-9, -30}, -// {-10, -34}}; - -#endif /* TYPES_HH */ diff --git a/include/cuda/PrecomputeKernels.h b/include/cuda/PrecomputeKernels.h deleted file mode 100644 index 4f92e6cde8..0000000000 --- a/include/cuda/PrecomputeKernels.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#ifndef MGARD_CUDA_PRECOMPUTE_KERNELS -#define MGARD_CUDA_PRECOMPUTE_KERNELS - -#include "Common.h" - -namespace mgard_cuda { -template -void calc_cpt_dist(Handle &handle, int n, T *dcoord, T *ddist, - int queue_idx); - -template -void reduce_two_dist(Handle &handle, int n, T *ddist, T *ddist_reduced, - int queue_idx); - -template -void dist_to_ratio(Handle &handle, int n, T *ddist, T *dratio, - int queue_idx); - -template -void dist_to_volume(Handle &handle, int n, T *ddist, T *dvolume, - int queue_idx); - -template -void calc_am_bm(Handle &handle, int n, T *ddist, T *am, T *bm, - int queue_idx); -} // namespace mgard_cuda - -#endif \ No newline at end of file diff --git a/include/cuda/SubArray.h b/include/cuda/SubArray.h deleted file mode 100644 index 8a4ab3d318..0000000000 --- a/include/cuda/SubArray.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: Jul 20, 2021 - */ - -#ifndef MGARD_CUDA_SUBARRAY -#define MGARD_CUDA_SUBARRAY -#include "Common.h" -#include - -namespace mgard_cuda { - -template class SubArray { -public: - SubArray(); - SubArray(Array &array); - SubArray(std::vector shape, T *dv, std::vector ldvs_h, - SIZE *ldvs_d); - SubArray(std::vector shape, T *dv); - SubArray(SubArray &subArray); - SubArray(const SubArray &subArray); - SubArray &operator=(const SubArray &subArray); - void offset(std::vector idx); - void resize(std::vector shape); - void offset(DIM dim, SIZE offset_value); - void resize(DIM dim, SIZE new_size); - void project(DIM dim0, DIM dim1, DIM dim2); - - MGARDm_CONT_EXEC T *operator()(SIZE *idx) { - LENGTH curr_stride = 1; - LENGTH offset = 0; - for (DIM i = 0; i < D; i++) { - offset += idx[i] * curr_stride; -#ifdef MGARDm_COMPILE_EXEC - curr_stride *= ldvs_d[i]; -#else - curr_stride *= ldvs_h[i]; -#endif - } - return dv + offset; - } - - MGARDm_CONT_EXEC T *operator()(IDX z, IDX y, IDX x) { - return dv + lddv2 * lddv1 * z + lddv1 * y + x; - } - MGARDm_CONT_EXEC T *operator()(IDX y, IDX x) { return dv + lddv1 * y + x; } - MGARDm_CONT_EXEC T *operator()(IDX x) { return dv + x; } - - MGARDm_CONT_EXEC bool isNull() { return dv == NULL; } - MGARDm_CONT_EXEC T *data() { return dv; } - - ~SubArray(); - - T *dv; - std::vector ldvs_h; - SIZE *ldvs_d; - std::vector shape; - DIM projected_dim0; - DIM projected_dim1; - DIM projected_dim2; - SIZE lddv1; - SIZE lddv2; - using DataType = T; - static const DIM NumDims = D; -}; - -} // namespace mgard_cuda -#endif \ No newline at end of file diff --git a/include/mdr.hpp b/include/mdr.hpp index ecc36f440f..6728a99ae5 100644 --- a/include/mdr.hpp +++ b/include/mdr.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "mgard-x/MDR/Reconstructor/Reconstructor.hpp" diff --git a/include/mdr_x.hpp b/include/mdr_x.hpp index 1551032504..072946c415 100644 --- a/include/mdr_x.hpp +++ b/include/mdr_x.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "mgard-x/Config/Config.h" diff --git a/include/mdr_x_lowlevel.hpp b/include/mdr_x_lowlevel.hpp index 27385d2a8b..9d57cd7845 100644 --- a/include/mdr_x_lowlevel.hpp +++ b/include/mdr_x_lowlevel.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "mgard-x/Hierarchy/Hierarchy.hpp" diff --git a/include/mgard-x/CompressionHighLevel/CPUPipelines.hpp b/include/mgard-x/CompressionHighLevel/CPUPipelines.hpp index dde81af30a..9677ece15f 100644 --- a/include/mgard-x/CompressionHighLevel/CPUPipelines.hpp +++ b/include/mgard-x/CompressionHighLevel/CPUPipelines.hpp @@ -54,7 +54,8 @@ enum compress_status_type compress_pipeline_cpu( } Timer timer_profile; - std::vector h2d, d2h, comp, size; + std::vector h2d, d2h, comp; + std::vector size; bool profile = false; bool profile_e2e = false; @@ -218,11 +219,8 @@ enum compress_status_type compress_pipeline_cpu( device_compressed_buffer[curr_subdomain_id].data(), compressed_size[curr_subdomain_id], byte_offset, 0); - if (profile) { - size.push_back( - compressor[curr_subdomain_id].hierarchy->total_num_elems() * - sizeof(T) / 1.0e9); - } + size.push_back(compressor[curr_subdomain_id].hierarchy->total_num_elems() * + sizeof(T)); } if (profile) { @@ -231,6 +229,10 @@ enum compress_status_type compress_pipeline_cpu( d2h.push_back(timer_profile.get()); } + SIZE total_size = 0; + for (auto s : size) + total_size += s; + if (profile) { std::cout << "comp: " << "\n"; @@ -252,9 +254,6 @@ enum compress_status_type compress_pipeline_cpu( std::cout << "size: " << "\n"; - float total_size = 0; - for (auto s : size) - total_size += s; std::cout << total_size << "\n"; std::cout << "comp_speed: " @@ -266,7 +265,7 @@ enum compress_status_type compress_pipeline_cpu( DeviceRuntime::SyncDevice(); if (log::level & log::TIME) { timer_series.end(); - timer_series.print("Compress subdomains series"); + timer_series.print("Compress pipeline", total_size); timer_series.clear(); } return compress_status_type::Success; @@ -326,7 +325,8 @@ enum compress_status_type decompress_pipeline_cpu( } Timer timer_profile; - std::vector h2d, d2h, comp, size; + std::vector h2d, d2h, comp; + std::vector size; bool profile = false; bool profile_e2e = false; @@ -358,11 +358,8 @@ enum compress_status_type decompress_pipeline_cpu( MemoryManager::Copy1D( device_compressed_buffer[curr_subdomain_id].data(), compressed_data, compressed_size[curr_subdomain_id], 0); - if (profile || profile_e2e) { - size.push_back( - compressor[curr_subdomain_id].hierarchy->total_num_elems() * - sizeof(T) / 1.0e9); - } + size.push_back(compressor[curr_subdomain_id].hierarchy->total_num_elems() * + sizeof(T)); } if (profile) { @@ -401,11 +398,9 @@ enum compress_status_type decompress_pipeline_cpu( if (CR > 1.0) { compressor[curr_subdomain_id].LosslessDecompress( device_compressed_buffer[curr_subdomain_id], 0); - compressor[curr_subdomain_id].Dequantize( + compressor[curr_subdomain_id].DequantizeRecompose( device_subdomain_buffer[curr_subdomain_id], local_ebtype, local_tol, s, norm, 0); - compressor[curr_subdomain_id].Recompose( - device_subdomain_buffer[curr_subdomain_id], 0); } else { log::info("Skipping decompression as original data was saved instead"); device_subdomain_buffer[curr_subdomain_id].resize( @@ -450,11 +445,9 @@ enum compress_status_type decompress_pipeline_cpu( if (CR > 1.0) { compressor[curr_subdomain_id].LosslessDecompress( device_compressed_buffer[curr_subdomain_id], 0); - compressor[curr_subdomain_id].Dequantize( + compressor[curr_subdomain_id].DequantizeRecompose( device_subdomain_buffer[curr_subdomain_id], local_ebtype, local_tol, s, norm, 0); - compressor[curr_subdomain_id].Recompose( - device_subdomain_buffer[curr_subdomain_id], 0); } else { log::info("Skipping decompression as original data was saved instead"); device_subdomain_buffer[curr_subdomain_id].resize( @@ -499,6 +492,10 @@ enum compress_status_type decompress_pipeline_cpu( d2h.push_back(timer_profile.get()); } + SIZE total_size = 0; + for (auto s : size) + total_size += s; + if (profile) { std::cout << "comp: " << "\n"; @@ -520,9 +517,6 @@ enum compress_status_type decompress_pipeline_cpu( std::cout << "size: " << "\n"; - float total_size = 0; - for (auto s : size) - total_size += s; std::cout << total_size << "\n"; std::cout << "comp_speed: " @@ -533,7 +527,7 @@ enum compress_status_type decompress_pipeline_cpu( DeviceRuntime::SyncDevice(); if (log::level & log::TIME) { timer_series.end(); - timer_series.print("Decompress subdomains series with prefetch"); + timer_series.print("Decompress pipeline", total_size); timer_series.clear(); } return compress_status_type::Success; diff --git a/include/mgard-x/CompressionHighLevel/CompressionHighLevel.h b/include/mgard-x/CompressionHighLevel/CompressionHighLevel.h index ebbcfc52c2..48159fa86d 100644 --- a/include/mgard-x/CompressionHighLevel/CompressionHighLevel.h +++ b/include/mgard-x/CompressionHighLevel/CompressionHighLevel.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include diff --git a/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp b/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp index c1da7ee858..9ea2438b29 100644 --- a/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp +++ b/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include @@ -44,13 +44,50 @@ namespace mgard_x { +// The hybrid hierarchy can only be given a theoretical s-norm bound when the +// block-local stage is a single level and there is no global stage. +// +// With one local level every block is transformed independently, so the total +// squared error is a plain sum over blocks and the budget divides evenly in +// quadrature -- tol / sqrt(num_blocks) per block -- exactly as +// calc_local_abs_tol does across subdomains for domain decomposition. A second +// local level mixes the coarse outputs of neighbouring blocks, and a global +// stage decomposes across blocks entirely; in both cases the blocks stop being +// independent and the sum no longer bounds the error. +// +// So for s != inf we fall back to that one configuration and say so. L-inf is +// untouched: its bound composes over levels and does not need independence. +template +void restrict_hybrid_config_for_s_norm(Config &config, T s) { + if (config.decomposition != decomposition_type::Hybrid || + s == std::numeric_limits::infinity()) { + return; + } + if (config.num_local_refactoring_level != 1) { + log::warn("hybrid with an s-norm bound supports one block-local level " + "only; falling back from " + + std::to_string(config.num_local_refactoring_level) + " to 1", + true); + config.num_local_refactoring_level = 1; + } + if (config.num_global_refactoring_level != 0) { + log::warn("hybrid with an s-norm bound cannot use a global stage (it " + "decomposes across blocks, so the per-block error budget no " + "longer composes); falling back from " + + std::to_string(config.num_global_refactoring_level) + + " global levels to 0", + true); + config.num_global_refactoring_level = 0; + } +} + template enum compress_status_type -general_compress(std::vector shape, T tol, T s, - enum error_bound_type ebtype, const void *original_data, - void *&compressed_data, size_t &compressed_size, Config config, - bool uniform, std::vector coords, - bool output_pre_allocated) { +general_compress_pipeline(std::vector shape, T tol, T s, + enum error_bound_type ebtype, + const void *original_data, void *&compressed_data, + size_t &compressed_size, Config config, bool uniform, + std::vector coords, bool output_pre_allocated) { DeviceRuntime::Initialize(); size_t total_num_elem = 1; @@ -59,6 +96,20 @@ general_compress(std::vector shape, T tol, T s, config.apply(); + // Before anything reads the level counts: the DomainDecomposer sizes + // subdomains from them, and they are recorded in the file header for the + // decompressor, so the adjustment has to happen here to stay consistent. + restrict_hybrid_config_for_s_norm(config, s); + + // Resolve the BlockMGARD basis before DomainDecomposer and compressor-cache + // construction. The resolved value is also what gets written to metadata. + // The plain (non-hybrid) path resolves Auto/explicit the same way, but + // purely from s -- which round-trips through metadata on its own -- so it + // needs no equivalent pre-resolution or persistence here. + if (config.decomposition == decomposition_type::Hybrid) { + config.projection_mode = resolve_projection_mode(config.projection_mode, s); + } + log::info("adjust_shape: " + std::to_string(config.adjust_shape)); if (config.adjust_shape) { adjust_shape(shape, config); @@ -68,14 +119,6 @@ general_compress(std::vector shape, T tol, T s, if (log::level & log::TIME) timer_total.start(); - bool reduce_memory_footprint_original = - MemoryManager::ReduceMemoryFootprint; - if (MemoryManager::ReduceMemoryFootprint) { - log::info("Original ReduceMemoryFootprint: 1"); - } else { - log::info("Original ReduceMemoryFootprint: 0"); - } - DomainDecomposer domain_decomposer; if (uniform) { domain_decomposer = @@ -138,7 +181,8 @@ general_compress(std::vector shape, T tol, T s, local_ebtype = error_bound_type::ABS; if (log::level & log::TIME) { timer_each.end(); - timer_each.print("Calculate norm of decomposed domain"); + timer_each.print("Calculate norm of decomposed domain", + total_num_elem * sizeof(T)); timer_each.clear(); } } @@ -192,17 +236,62 @@ general_compress(std::vector shape, T tol, T s, // Estimate metadata size Metadata m; + // FillForCompression does not carry the BlockDelta block size; set it here so + // it is persisted in (and restored from) the metadata header. + m.block_delta_block_size = config.block_delta_block_size; + // Same for the hybrid (BlockMGARD) parameters: the decompressor cannot + // reconstruct the buffer layout or the per-block quantization steps without + // them. + if (config.decomposition == decomposition_type::Hybrid) { + m.hybrid_num_local_levels = (uint64_t)config.num_local_refactoring_level; + m.hybrid_num_global_levels = (uint64_t)config.num_global_refactoring_level; + m.hybrid_local_block_size = MGARDX_HYBRID_LOCAL_BLOCK_SIZE; + m.hybrid_projection_mode = config.projection_mode; + m.hybrid_enable_roi = config.enable_roi; + if (config.enable_roi) { + // Level-0 block grid: the local refactor pads each dimension up to a + // multiple of the block size, so the block count per dimension is just + // the ceiling division of the original shape. + m.hybrid_roi_block_dimensions.resize(shape.size()); + size_t expected_blocks = 1; + for (DIM d = 0; d < shape.size(); d++) { + m.hybrid_roi_block_dimensions[d] = + (shape[d] + MGARDX_HYBRID_LOCAL_BLOCK_SIZE - 1) / + MGARDX_HYBRID_LOCAL_BLOCK_SIZE; + expected_blocks *= m.hybrid_roi_block_dimensions[d]; + } + if (config.roi_tolerance_map.size() != expected_blocks) { + throw ProcessingException( + "ROI tolerance map holds " + + std::to_string(config.roi_tolerance_map.size()) + + " entries but the level-0 block grid of this shape needs " + + std::to_string(expected_blocks) + "."); + } + m.hybrid_roi_tolerance_map = config.roi_tolerance_map; + // Pre-existing limitation, surfaced here rather than left silent: the + // tolerance map is a single flat array indexed by each subdomain's own + // block id, so with more than one subdomain every subdomain re-reads the + // map from offset 0 and blocks get the wrong tolerances. The map is + // still recorded verbatim, so this warns rather than refuses -- the file + // round-trips exactly as it was compressed. + if (domain_decomposer.domain_decomposed()) { + log::warn("ROI tolerance map is applied per subdomain from offset 0, " + "so ROI results are not correct under domain decomposition; " + "compress without domain decomposition for correct ROI."); + } + } + } if (uniform) { m.FillForCompression( - ebtype, tol, s, norm, config.decomposition, config.reorder, - config.lossless, config.huff_dict_size, config.huff_block_size, shape, + ebtype, tol, s, norm, config.decomposition, config.lossless, + config.huff_dict_size, config.huff_block_size, shape, domain_decomposer.domain_decomposed(), config.domain_decomposition, domain_decomposer.domain_decomposed_dim(), domain_decomposer.domain_decomposed_size()); } else { m.FillForCompression( - ebtype, tol, s, norm, config.decomposition, config.reorder, - config.lossless, config.huff_dict_size, config.huff_block_size, shape, + ebtype, tol, s, norm, config.decomposition, config.lossless, + config.huff_dict_size, config.huff_block_size, shape, domain_decomposer.domain_decomposed(), config.domain_decomposition, domain_decomposer.domain_decomposed_dim(), domain_decomposer.domain_decomposed_size(), coords); @@ -223,8 +312,6 @@ general_compress(std::vector shape, T tol, T s, } enum compress_status_type compress_status; - if (log::level & log::TIME) - timer_each.start(); DeviceRuntime::SelectDevice(config.dev_id); if constexpr (std::is_same::value || std::is_same::value || @@ -238,29 +325,20 @@ general_compress(std::vector shape, T tol, T s, domain_decomposer, local_tol, s, norm, local_ebtype, config, compressed_subdomain_data, compressed_subdomain_size); } - if (log::level & log::TIME) { - timer_each.end(); - timer_each.print("Aggregated low-level compression"); - log::time("Aggregated low-level compression throughput: " + - std::to_string((double)(total_num_elem * sizeof(T)) / - timer_each.get() / 1e9) + - " GB/s"); - timer_each.clear(); - } if (log::level & log::TIME) timer_each.start(); if (uniform) { m.FillForCompression( - ebtype, tol, s, norm, config.decomposition, config.reorder, - config.lossless, config.huff_dict_size, config.huff_block_size, shape, + ebtype, tol, s, norm, config.decomposition, config.lossless, + config.huff_dict_size, config.huff_block_size, shape, domain_decomposer.domain_decomposed(), config.domain_decomposition, domain_decomposer.domain_decomposed_dim(), domain_decomposer.domain_decomposed_size()); } else { m.FillForCompression( - ebtype, tol, s, norm, config.decomposition, config.reorder, - config.lossless, config.huff_dict_size, config.huff_block_size, shape, + ebtype, tol, s, norm, config.decomposition, config.lossless, + config.huff_dict_size, config.huff_block_size, shape, domain_decomposer.domain_decomposed(), config.domain_decomposition, domain_decomposer.domain_decomposed_dim(), domain_decomposer.domain_decomposed_size(), coords); @@ -289,30 +367,41 @@ general_compress(std::vector shape, T tol, T s, Cache::cache.SafeRelease(); DeviceRuntime::Finalize(); - MemoryManager::ReduceMemoryFootprint = - reduce_memory_footprint_original; - if (MemoryManager::ReduceMemoryFootprint) { - log::info("ReduceMemoryFootprint restored to 1"); - } else { - log::info("ReduceMemoryFootprint restored to 0"); - } - if (log::level & log::TIME) { timer_each.end(); timer_each.print("Serialization"); timer_each.clear(); timer_total.end(); - timer_total.print("High-level compression"); - log::time("High-level compression throughput: " + - std::to_string((double)(total_num_elem * sizeof(T)) / - timer_total.get() / 1e9) + - " GB/s"); + timer_total.print("High-level compression", total_num_elem * sizeof(T)); timer_total.clear(); } return compress_status; } +// Wraps the compression pipeline and translates any exception thrown by the +// internal (de)compression steps into a compress_status_type failure code, so +// that the library never terminates the calling application via exit(). +template +enum compress_status_type +general_compress(std::vector shape, T tol, T s, + enum error_bound_type ebtype, const void *original_data, + void *&compressed_data, size_t &compressed_size, Config config, + bool uniform, std::vector coords, + bool output_pre_allocated) { + try { + return general_compress_pipeline( + shape, tol, s, ebtype, original_data, compressed_data, compressed_size, + config, uniform, coords, output_pre_allocated); + } catch (const Exception &e) { + log::err(std::string("compression failed: ") + e.what()); + return e.status(); + } catch (const std::exception &e) { + log::err(std::string("compression failed: ") + e.what()); + return compress_status_type::Failure; + } +} + template enum compress_status_type compress(std::vector shape, T tol, T s, enum error_bound_type ebtype, @@ -376,9 +465,10 @@ compress(std::vector shape, T tol, T s, enum error_bound_type ebtype, template enum compress_status_type -general_decompress(std::vector shape, const void *compressed_data, - size_t compressed_size, void *&decompressed_data, - Config config, bool output_pre_allocated) { +general_decompress_pipeline(std::vector shape, + const void *compressed_data, size_t compressed_size, + void *&decompressed_data, Config config, + bool output_pre_allocated) { DeviceRuntime::Initialize(); size_t total_num_elem = 1; for (int i = 0; i < D; i++) @@ -392,14 +482,6 @@ general_decompress(std::vector shape, const void *compressed_data, if (log::level & log::TIME) timer_each.start(); - bool reduce_memory_footprint_original = - MemoryManager::ReduceMemoryFootprint; - if (MemoryManager::ReduceMemoryFootprint) { - log::info("Original ReduceMemoryFootprint: 1"); - } else { - log::info("Original ReduceMemoryFootprint: 0"); - } - // Use consistance memory space between input and output data if (!output_pre_allocated) { if (MemoryManager::IsDevicePointer(compressed_data)) { @@ -530,8 +612,6 @@ general_decompress(std::vector shape, const void *compressed_data, timer_each.clear(); } enum compress_status_type decompress_status; - if (log::level & log::TIME) - timer_each.start(); DeviceRuntime::SelectDevice(config.dev_id); if constexpr (std::is_same::value || @@ -546,15 +626,6 @@ general_decompress(std::vector shape, const void *compressed_data, domain_decomposer, local_tol, (T)m.s, (T)m.norm, local_ebtype, config, compressed_subdomain_data); } - if (log::level & log::TIME) { - timer_each.end(); - timer_each.print("Aggregated low-level decompression"); - log::time("Aggregated low-level decompression throughput: " + - std::to_string((double)(total_num_elem * sizeof(T)) / - timer_each.get() / 1e9) + - " GB/s"); - timer_each.clear(); - } if (!input_previously_pinned && config.auto_pin_host_buffers) { MemoryManager::HostUnregister((void *)compressed_data); @@ -572,27 +643,36 @@ general_decompress(std::vector shape, const void *compressed_data, Cache::cache.SafeRelease(); DeviceRuntime::Finalize(); - MemoryManager::ReduceMemoryFootprint = - reduce_memory_footprint_original; - if (MemoryManager::ReduceMemoryFootprint) { - log::info("ReduceMemoryFootprint restored to 1"); - } else { - log::info("ReduceMemoryFootprint restored to 0"); - } - if (log::level & log::TIME) { timer_total.end(); - timer_total.print("High-level decompression"); - log::time("High-level decompression throughput: " + - std::to_string((double)(total_num_elem * sizeof(T)) / - timer_total.get() / 1e9) + - " GB/s"); + timer_total.print("High-level decompression", total_num_elem * sizeof(T)); timer_total.clear(); } return decompress_status; } +// Wraps the decompression pipeline and translates any exception thrown by the +// internal steps (e.g. malformed metadata, lossless backend errors) into a +// compress_status_type failure code instead of terminating via exit(). +template +enum compress_status_type +general_decompress(std::vector shape, const void *compressed_data, + size_t compressed_size, void *&decompressed_data, + Config config, bool output_pre_allocated) { + try { + return general_decompress_pipeline( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } catch (const Exception &e) { + log::err(std::string("decompression failed: ") + e.what()); + return e.status(); + } catch (const std::exception &e) { + log::err(std::string("decompression failed: ") + e.what()); + return compress_status_type::Failure; + } +} + template enum compress_status_type decompress(std::vector shape, const void *compressed_data, @@ -970,4 +1050,4 @@ template void unpin_memory(void *ptr) { } // namespace mgard_x -#endif \ No newline at end of file +#endif diff --git a/include/mgard-x/CompressionHighLevel/ErrorToleranceCalculator.hpp b/include/mgard-x/CompressionHighLevel/ErrorToleranceCalculator.hpp index 7f80d0e45a..a213f33c86 100644 --- a/include/mgard-x/CompressionHighLevel/ErrorToleranceCalculator.hpp +++ b/include/mgard-x/CompressionHighLevel/ErrorToleranceCalculator.hpp @@ -3,9 +3,6 @@ template T calc_subdomain_norm_series_w_prefetch( DomainDecomposer &domain_decomposer, T s) { - Timer timer_series; - if (log::level & log::TIME) - timer_series.start(); DeviceRuntime::SyncQueue(0); Array<1, T, DeviceType> norm_array({1}); @@ -56,11 +53,6 @@ T calc_subdomain_norm_series_w_prefetch( current_buffer = next_buffer; DeviceRuntime::SyncQueue(1); } - if (log::level & log::TIME) { - timer_series.end(); - timer_series.print("Calculate subdomains norm series"); - timer_series.clear(); - } DeviceRuntime::SyncDevice(); return norm; diff --git a/include/mgard-x/CompressionHighLevel/GPUPipelines.hpp b/include/mgard-x/CompressionHighLevel/GPUPipelines.hpp index 67272a99e9..36ae915d4e 100644 --- a/include/mgard-x/CompressionHighLevel/GPUPipelines.hpp +++ b/include/mgard-x/CompressionHighLevel/GPUPipelines.hpp @@ -59,7 +59,8 @@ enum compress_status_type compress_pipeline_gpu( } Timer timer_profile; - std::vector h2d, d2h, comp, size; + std::vector h2d, d2h, comp; + std::vector size; bool profile = false; bool profile_e2e = false; @@ -160,10 +161,7 @@ enum compress_status_type compress_pipeline_gpu( comp.push_back(timer_profile.get()); } - if (profile || profile_e2e) { - size.push_back(compressor.hierarchy->total_num_elems() * sizeof(T) / - 1.0e9); - } + size.push_back(compressor.hierarchy->total_num_elems() * sizeof(T)); // Check if we have enough space if (compressed_size > @@ -206,25 +204,19 @@ enum compress_status_type compress_pipeline_gpu( current_queue = next_queue; } + SIZE total_size = 0; + for (auto t : size) + total_size += t; + if (profile_e2e) { DeviceRuntime::SyncDevice(); timer_profile.end(); timer_profile.print("end to end"); - float s = 0; - for (float t : size) - s += t; - timer_profile.print_throughput("end to end", s * 1e9); + + timer_profile.print_throughput("end to end", total_size * 1e9); } if (profile) { - // double total_size = domain_decomposer.shape[0] * - // domain_decomposer.shape[1] * domain_decomposer.shape[2] * sizeof(T) / - // 1e9; std::cout << "comp: " << comp / domain_decomposer.num_subdomains() - // << "(" << total_size / comp << " GB/s)"<< "\n"; std::cout << "h2d: " << - // h2d / domain_decomposer.num_subdomains() << "(" << total_size / h2d << " - // GB/s)"<< "\n"; std::cout << "d2h: " << d2h / - // domain_decomposer.num_subdomains() << "(" << byte_offset/ 1e9 / d2h << " - // GB/s)"<< "\n"; std::cout << "comp: " << "\n"; for (float t : comp) @@ -245,7 +237,7 @@ enum compress_status_type compress_pipeline_gpu( std::cout << "size: " << "\n"; - for (float t : size) + for (SIZE t : size) std::cout << t << ", "; std::cout << "\n"; @@ -260,7 +252,7 @@ enum compress_status_type compress_pipeline_gpu( DeviceRuntime::SyncDevice(); if (log::level & log::TIME) { timer_series.end(); - timer_series.print("Compress subdomains series with prefetch"); + timer_series.print("Compress pipeline", total_size); timer_series.clear(); } return compress_status_type::Success; @@ -327,7 +319,9 @@ enum compress_status_type decompress_pipeline_gpu( } Timer timer_profile; - std::vector h2d, d2h, comp, size; + Timer timer_decompress_kernel; + std::vector h2d, d2h, comp; + std::vector size; bool profile = false; bool profile_e2e = false; @@ -412,7 +406,7 @@ enum compress_status_type decompress_pipeline_gpu( } double CR = (double)compressor.hierarchy->total_num_elems() * sizeof(T) / - compressed_size; + device_compressed_buffer[current_buffer].shape(0); log::info("Subdomain CR: " + std::to_string(CR)); if (CR > 1.0) { std::stringstream ss; @@ -456,14 +450,26 @@ enum compress_status_type decompress_pipeline_gpu( timer_profile.start(); } if (CR > 1.0) { + if (log::level & log::TIME) { + DeviceRuntime::SyncDevice(); + timer_decompress_kernel.clear(); + timer_decompress_kernel.start(); + } compressor.LosslessDecompress(device_compressed_buffer[current_buffer], current_queue); - compressor.Dequantize(device_subdomain_buffer[current_buffer], - local_ebtype, local_tol, s, norm, current_queue); - compressor.Recompose(device_subdomain_buffer[current_buffer], - current_queue); + compressor.DequantizeRecompose(device_subdomain_buffer[current_buffer], + local_ebtype, local_tol, s, norm, + current_queue); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(0); + timer_decompress_kernel.end(); + timer_decompress_kernel.print("Decompression Kernel"); + timer_decompress_kernel.clear(); + } } else { log::info("Skipping decompression as original data was saved instead"); + // Print info here + // log::info("The compression ration is "+std::to_string(CR)); device_subdomain_buffer[current_buffer].resize( {compressor.hierarchy->level_shape( compressor.hierarchy->l_target())}); @@ -479,16 +485,19 @@ enum compress_status_type decompress_pipeline_gpu( linearized_width, current_queue); } + // compressor.Dequantize(device_subdomain_buffer[current_buffer], + // local_ebtype, local_tol, s, norm, current_queue); + + // compressor.Recompose(device_subdomain_buffer[current_buffer], + // current_queue); + if (profile) { DeviceRuntime::SyncDevice(); timer_profile.end(); comp.push_back(timer_profile.get()); } - if (profile || profile_e2e) { - size.push_back(compressor.hierarchy->total_num_elems() * sizeof(T) / - 1.0e9); - } + size.push_back(compressor.hierarchy->total_num_elems() * sizeof(T)); // Need to ensure decompession is complete without blocking other operations DeviceRuntime::SyncQueue(current_queue); @@ -506,10 +515,17 @@ enum compress_status_type decompress_pipeline_gpu( int previous_buffer = std::abs((current_buffer - 1) % 2); int previous_queue = previous_buffer; SIZE prev_subdomain_id = domain_decomposer.num_subdomains() - 1; + // Add resizing for device_subdomain_buffer + device_subdomain_buffer[previous_buffer].resize( + {compressor.hierarchy->level_shape(compressor.hierarchy->l_target())}); domain_decomposer.copy_subdomain( device_subdomain_buffer[previous_buffer], prev_subdomain_id, subdomain_copy_direction::SubdomainToOriginal, previous_queue); + SIZE total_size = 0; + for (auto t : size) + total_size += t; + if (profile) { DeviceRuntime::SyncDevice(); timer_profile.end(); @@ -520,10 +536,7 @@ enum compress_status_type decompress_pipeline_gpu( DeviceRuntime::SyncDevice(); timer_profile.end(); timer_profile.print("end to end"); - float s = 0; - for (float t : size) - s += t; - timer_profile.print_throughput("end to end", s * 1e9); + timer_profile.print_throughput("end to end", total_size * 1e9); } if (profile) { @@ -555,7 +568,7 @@ enum compress_status_type decompress_pipeline_gpu( std::cout << "size: " << "\n"; - for (float t : size) + for (SIZE t : size) std::cout << t << ", "; std::cout << "\n"; @@ -569,9 +582,9 @@ enum compress_status_type decompress_pipeline_gpu( DeviceRuntime::SyncDevice(); if (log::level & log::TIME) { timer_series.end(); - timer_series.print("Decompress subdomains series with prefetch"); + timer_series.print("Decompress pipeline", total_size); timer_series.clear(); } return compress_status_type::Success; } -} // namespace mgard_x +} // namespace mgard_x \ No newline at end of file diff --git a/include/mgard-x/CompressionLowLevel/CMakeLists.txt b/include/mgard-x/CompressionLowLevel/CMakeLists.txt index 7f2505204d..9872e092a3 100644 --- a/include/mgard-x/CompressionLowLevel/CMakeLists.txt +++ b/include/mgard-x/CompressionLowLevel/CMakeLists.txt @@ -3,6 +3,7 @@ list(APPEND MGARD_X_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/Compressor.hpp ${CMAKE_CURRENT_SOURCE_DIR}/HyBridHierarchyCompressor.h ${CMAKE_CURRENT_SOURCE_DIR}/HyBridHierarchyCompressor.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/LossyCompressorInterface.hpp ${CMAKE_CURRENT_SOURCE_DIR}/NormCalculator.hpp ) set(MGARD_X_HEADER ${MGARD_X_HEADER} PARENT_SCOPE) \ No newline at end of file diff --git a/include/mgard-x/CompressionLowLevel/Compressor.h b/include/mgard-x/CompressionLowLevel/Compressor.h index c7213732b3..c050ddf0a1 100644 --- a/include/mgard-x/CompressionLowLevel/Compressor.h +++ b/include/mgard-x/CompressionLowLevel/Compressor.h @@ -1,13 +1,15 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_COMPRESSOR_H #define MGARD_X_COMPRESSOR_H +#include + #include "../RuntimeX/RuntimeXPublic.h" #include "../DataRefactoring/DataRefactor.hpp" @@ -20,19 +22,39 @@ #include "../Lossless/Lossless.hpp" #include "../Quantization/LinearQuantization.hpp" +#include "../Utilities/ProjectionMode.h" #include "LossyCompressorInterface.hpp" namespace mgard_x { +// D-aware wrapper around the shared resolve_projection_mode/ +// infer_orthogonal_projection (see Utilities/ProjectionMode.h): the +// hierarchical basis (no mass-matrix correction) is only implemented for +// D <= 3, because the multi-dimensional decompose/recompose kernels only +// honor the flag there -- higher dimensions always apply the correction, so +// Auto silently stays on the orthogonal basis and an explicit Hierarchical +// request throws instead of being silently ignored. +template +inline bool infer_orthogonal_projection(compression_projection_mode_type mode, + T s) { + if (D > 3) { + if (mode == compression_projection_mode_type::Hierarchical) { + throw ProcessingException( + "the hierarchical basis is only implemented for 1D/2D/3D data"); + } + return true; + } + return infer_orthogonal_projection(mode, s); +} + template class Compressor : public LossyCompressorInterface { public: using HierarchyType = Hierarchy; using DataRefactorType = data_refactoring::DataRefactor; using LosslessCompressorType = - ComposedLosslessCompressor; + ComposedLosslessCompressor; using LinearQuantizerType = LinearQuantizer; public: @@ -48,7 +70,8 @@ class Compressor : public LossyCompressorInterface { void CalculateNorm(Array &original_data, enum error_bound_type ebtype, T s, T &norm, int queue_idx); - void Decompose(Array &original_data, int queue_idx); + void Decompose(Array &original_data, + bool orthogonal_projection, int queue_idx); void Quantize(Array &original_data, enum error_bound_type ebtype, T tol, T s, T norm, @@ -61,12 +84,23 @@ class Compressor : public LossyCompressorInterface { void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx); - void Recompose(Array &decompressed_data, int queue_idx); + void Recompose(Array &decompressed_data, + bool orthogonal_projection, int queue_idx); void Dequantize(Array &decompressed_data, enum error_bound_type ebtype, T tol, T s, T norm, int queue_idx); + // Dequantize + recompose as one step of the decompression pipelines. Here it + // is simply the two calls in sequence; a compressor that can do better (see + // HybridHierarchyCompressor, which fuses them into one pass over the local + // levels) overrides this and decides for itself. The pipelines call this + // rather than the two methods so they do not have to know which compressor + // they are driving. + void DequantizeRecompose(Array &decompressed_data, + enum error_bound_type ebtype, T tol, T s, T norm, + int queue_idx); + void LosslessDecompress(Array<1, Byte, DeviceType> &compressed_data, int queue_idx); @@ -80,6 +114,12 @@ class Compressor : public LossyCompressorInterface { bool initialized; Hierarchy *hierarchy; Config config; + // Whether the last (de)compose should use orthogonal projection. Derived + // from config.projection_mode and s (see infer_orthogonal_projection) + // during Compress/Decompress/(De)quantize and consumed by Recompose, which + // does not receive s. Defaults to true so the orthogonal path is used + // unless the config/s combination resolves to the hierarchical fast path. + bool orthogonal_projection = true; Array<1, T, DeviceType> norm_tmp_array; Array<1, T, DeviceType> norm_array; Array quantized_array; diff --git a/include/mgard-x/CompressionLowLevel/Compressor.hpp b/include/mgard-x/CompressionLowLevel/Compressor.hpp index 8e7d8921c2..59bedd16db 100644 --- a/include/mgard-x/CompressionLowLevel/Compressor.hpp +++ b/include/mgard-x/CompressionLowLevel/Compressor.hpp @@ -1,13 +1,14 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include #include #include +#include #include #include @@ -51,7 +52,7 @@ Compressor::Compressor(Hierarchy &hierarchy, } else { // norm_tmp_array = Array<1, T, DeviceType>({hierarchy.total_num_elems()}); quantized_array = Array( - hierarchy.level_shape(hierarchy.l_target()), false, false); + hierarchy.level_shape(hierarchy.l_target())); } } @@ -90,20 +91,20 @@ Compressor::EstimateMemoryFootprint(std::vector shape, hierarchy.EstimateMemoryFootprint(shape); size_t size = 0; size += DataRefactorType::EstimateMemoryFootprint(shape); - log::info( + log::dbg( "Data refactor space: " + std::to_string( (double)(DataRefactorType::EstimateMemoryFootprint(shape)) / 1e9) + " GB"); size += LinearQuantizerType::EstimateMemoryFootprint(shape); - log::info( + log::dbg( "Quantizer space: " + std::to_string( (double)(LinearQuantizerType::EstimateMemoryFootprint(shape)) / 1e9) + " GB"); size += LosslessCompressorType::EstimateMemoryFootprint( hierarchy.total_num_elems(), config); - log::info( + log::dbg( "Lossless space: " + std::to_string((double)(LosslessCompressorType::EstimateMemoryFootprint( hierarchy.total_num_elems(), config)) / @@ -130,24 +131,26 @@ void Compressor::CalculateNorm( template void Compressor::Decompose( - Array &original_data, int queue_idx) { - refactor.Decompose(SubArray(original_data), queue_idx); + Array &original_data, bool orthogonal_projection, + int queue_idx) { + refactor.Decompose(SubArray(original_data), orthogonal_projection, queue_idx); } template void Compressor::Quantize( Array &original_data, enum error_bound_type ebtype, T tol, T s, T norm, int queue_idx) { + orthogonal_projection = + infer_orthogonal_projection(config.projection_mode, s); quantizer.Quantize(original_data, ebtype, tol, s, norm, quantized_array, - lossless_compressor, queue_idx); + lossless_compressor, queue_idx, orthogonal_projection); } template void Compressor::LosslessCompress( Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { - Array<1, QUANTIZED_UNSIGNED_INT, DeviceType> quantized_liearized_array( - {hierarchy->total_num_elems()}, - (QUANTIZED_UNSIGNED_INT *)quantized_array.data()); + Array<1, QUANTIZED_INT, DeviceType> quantized_liearized_array( + {hierarchy->total_num_elems()}, (QUANTIZED_INT *)quantized_array.data()); lossless_compressor.Compress(quantized_liearized_array, compressed_data, queue_idx); } @@ -166,25 +169,43 @@ void Compressor::Deserialize( template void Compressor::Recompose( - Array &decompressed_data, int queue_idx) { - refactor.Recompose(SubArray(decompressed_data), queue_idx); + Array &decompressed_data, bool orthogonal_projection, + int queue_idx) { + // The generic decompress pipeline always passes true here, but the basis was + // decided from s during Dequantize/Decompress and stored in + // this->orthogonal_projection. Prefer the stored decision so the hierarchical + // L-infinity fast path stays consistent between decompose and recompose; the + // caller's argument is only honored when it requests the (default) orthogonal + // path, so explicit orthogonal callers are never overridden. + refactor.Recompose(SubArray(decompressed_data), + orthogonal_projection && this->orthogonal_projection, + queue_idx); +} + +template +void Compressor::DequantizeRecompose( + Array &decompressed_data, enum error_bound_type ebtype, + T tol, T s, T norm, int queue_idx) { + Dequantize(decompressed_data, ebtype, tol, s, norm, queue_idx); + Recompose(decompressed_data, true, queue_idx); } template void Compressor::Dequantize( Array &decompressed_data, enum error_bound_type ebtype, T tol, T s, T norm, int queue_idx) { + orthogonal_projection = + infer_orthogonal_projection(config.projection_mode, s); decompressed_data.resize(hierarchy->level_shape(hierarchy->l_target())); quantizer.Dequantize(decompressed_data, ebtype, tol, s, norm, quantized_array, - lossless_compressor, queue_idx); + lossless_compressor, queue_idx, orthogonal_projection); } template void Compressor::LosslessDecompress( Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { - Array<1, QUANTIZED_UNSIGNED_INT, DeviceType> quantized_liearized_data( - {hierarchy->total_num_elems()}, - (QUANTIZED_UNSIGNED_INT *)quantized_array.data()); + Array<1, QUANTIZED_INT, DeviceType> quantized_liearized_data( + {hierarchy->total_num_elems()}, (QUANTIZED_INT *)quantized_array.data()); lossless_compressor.Decompress(compressed_data, quantized_liearized_data, queue_idx); } @@ -213,25 +234,27 @@ void Compressor::Compress( timer_total.start(); } + // Resolve config.projection_mode against s (Auto defaults to the cheap + // hierarchical basis under L-infinity, orthogonal otherwise). Remember the + // decision so Recompose (which has no s) reconstructs consistently. + orthogonal_projection = + infer_orthogonal_projection(config.projection_mode, s); + CalculateNorm(original_data, ebtype, s, norm, queue_idx); - Decompose(original_data, queue_idx); + Decompose(original_data, orthogonal_projection, queue_idx); Quantize(original_data, ebtype, tol, s, norm, queue_idx); LosslessCompress(compressed_data, queue_idx); Serialize(compressed_data, queue_idx); if (config.compress_with_dryrun) { Dequantize(original_data, ebtype, tol, s, norm, queue_idx); - Recompose(original_data, queue_idx); + Recompose(original_data, orthogonal_projection, queue_idx); } if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); timer_total.end(); - timer_total.print("Low-level compression"); - log::time( - "Low-level compression throughput: " + - std::to_string((double)(hierarchy->total_num_elems() * sizeof(T)) / - timer_total.get() / 1e9) + - " GB/s"); + timer_total.print("Low-level compression", + hierarchy->total_num_elems() * sizeof(T)); timer_total.clear(); } } @@ -252,21 +275,22 @@ void Compressor::Decompress( timer_total.start(); } + // Must mirror the orthogonal-projection decision made during Compress so the + // hierarchical fast path (s == inf) reconstructs consistently. + orthogonal_projection = + infer_orthogonal_projection(config.projection_mode, s); + decompressed_data.resize(hierarchy->level_shape(hierarchy->l_target())); Deserialize(compressed_data, queue_idx); LosslessDecompress(compressed_data, queue_idx); Dequantize(decompressed_data, ebtype, tol, s, norm, queue_idx); - Recompose(decompressed_data, queue_idx); + Recompose(decompressed_data, orthogonal_projection, queue_idx); if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); timer_total.end(); - timer_total.print("Low-level decompression"); - log::time( - "Low-level decompression throughput: " + - std::to_string((double)(hierarchy->total_num_elems() * sizeof(T)) / - timer_total.get() / 1e9) + - " GB/s"); + timer_total.print("Low-level decompression", + hierarchy->total_num_elems() * sizeof(T)); timer_total.clear(); } } diff --git a/include/mgard-x/CompressionLowLevel/CompressorCache.hpp b/include/mgard-x/CompressionLowLevel/CompressorCache.hpp index 55900ab4d1..9a335f0873 100644 --- a/include/mgard-x/CompressionLowLevel/CompressorCache.hpp +++ b/include/mgard-x/CompressionLowLevel/CompressorCache.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include diff --git a/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.h b/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.h index f275eeeb77..d64affba28 100644 --- a/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.h +++ b/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.h @@ -1,29 +1,26 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_HYBRID_HIERARCHY_COMPRESSOR_H #define MGARD_X_HYBRID_HIERARCHY_COMPRESSOR_H -#include "../RuntimeX/RuntimeXPublic.h" - -#include "../DataRefactoring/DataRefactor.hpp" +#include "../DataRefactoring/BlockLocalHierarchyDataRefactor.hpp" #include "../DataRefactoring/HybridHierarchyDataRefactor.hpp" +#include "../RuntimeX/RuntimeXPublic.h" // #include "CompressionLowLevelWorkspace.hpp" -#include "NormCalculator.hpp" - #include "../Hierarchy/Hierarchy.h" - #include "../Lossless/Lossless.hpp" #include "../Quantization/HybridHierarchyLinearQuantization.hpp" -#include "../Quantization/LinearQuantization.hpp" - +#include "../Quantization/LocalQuantization.hpp" +#include "../Utilities/ProjectionMode.h" #include "LossyCompressorInterface.hpp" +#include "NormCalculator.hpp" namespace mgard_x { @@ -32,15 +29,16 @@ class HybridHierarchyCompressor : public LossyCompressorInterface { public: using HierarchyType = Hierarchy; - using DataRefactorType = data_refactoring::DataRefactor; + // using BlockLocalHierarchyDataRefactorType = + // data_refactoring::BlockLocalHierarchyDataRefactor; using HybridHierarchyDataRefactorType = data_refactoring::HybridHierarchyDataRefactor; using LosslessCompressorType = - ComposedLosslessCompressor; - using LinearQuantizerType = LinearQuantizer; - using HybridHierarchyLinearQuantizerType = - HybridHierarchyLinearQuantizer; + ComposedLosslessCompressor; + // using LocalQuantizerType = LocalQuantizer; + using HybridQuantizerType = + HybridHierarchyQuantizer; HybridHierarchyCompressor(); @@ -61,6 +59,13 @@ class HybridHierarchyCompressor enum error_bound_type ebtype, T tol, T s, T norm, int queue_idx); + // Fused Decompose+Quantize (single pass over the local levels); used by + // Compress() instead of Decompose()+Quantize() when + // hybrid_quantizer.CanFuseQuantize(s) holds. + void DecomposeQuantize(Array &original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + int queue_idx); + void LosslessCompress(Array<1, Byte, DeviceType> &compressed_data, int queue_idx); @@ -68,12 +73,29 @@ class HybridHierarchyCompressor void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx); - void Recompose(Array &decompressed_data, int queue_idx); + void Recompose(Array &decompressed_data, + bool orthogonal_projection, int queue_idx); void Dequantize(Array &decompressed_data, enum error_bound_type ebtype, T tol, T s, T norm, int queue_idx); + // Dequantize + recompose. Overrides the plain two-call version: picks the + // fused single-pass implementation below when the configuration allows it, + // and falls back to Dequantize() + Recompose() otherwise. This is the only + // place that choice is made, for both the pipelines and Decompress(). + void DequantizeRecompose(Array &decompressed_data, + enum error_bound_type ebtype, T tol, T s, T norm, + int queue_idx); + + // Fused Dequantize+Recompose: one pass over the local levels, coefficients + // never round-trip through global memory as T. Requires + // hybrid_quantizer.CanFuseQuantize(s); throws otherwise. Call + // DequantizeRecompose() instead unless you specifically want the fused path. + void DequantizeRecomposeFused(Array &decompressed_data, + enum error_bound_type ebtype, T tol, T s, + T norm, int queue_idx); + void LosslessDecompress(Array<1, Byte, DeviceType> &compressed_data, int queue_idx); @@ -84,21 +106,26 @@ class HybridHierarchyCompressor enum error_bound_type ebtype, T tol, T s, T &norm, Array &decompressed_data, int queue_idx); + static SIZE calculate_padded_size(Hierarchy &hierarchy, + Config config); + bool initialized; Hierarchy *hierarchy; Config config; + bool orthogonal_projection = true; Array<1, T, DeviceType> norm_tmp_array; Array<1, T, DeviceType> norm_array; - Array<1, T, DeviceType> decomposed_array; - Array quantized_array; + // Array<1, T, DeviceType> local_decomposed_array; + // Array<1, QUANTIZED_INT, DeviceType> local_quantized_array; + Array<1, T, DeviceType> hybrid_decomposed_array; Array<1, QUANTIZED_INT, DeviceType> hybrid_quantized_array; - DataRefactorType refactor; + // BlockLocalHierarchyDataRefactorType local_refactor; + // LocalQuantizerType local_quantizer; HybridHierarchyDataRefactorType hybrid_refactor; - LinearQuantizerType quantizer; - HybridHierarchyLinearQuantizerType hybrid_quantizer; + HybridQuantizerType hybrid_quantizer; LosslessCompressorType lossless_compressor; }; } // namespace mgard_x -#endif \ No newline at end of file +#endif diff --git a/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.hpp b/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.hpp index d3f2387540..43cb42a01f 100644 --- a/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.hpp +++ b/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.hpp @@ -1,21 +1,21 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include +#include #include #include #include #include -#include "../Utilities/Types.h" - #include "../Config/Config.h" #include "../Hierarchy/Hierarchy.h" #include "../RuntimeX/RuntimeX.h" +#include "../Utilities/Types.h" #include "CompressorCache.hpp" #include "HybridHierarchyCompressor.h" @@ -34,30 +34,38 @@ template HybridHierarchyCompressor::HybridHierarchyCompressor( Hierarchy &hierarchy, Config config) : initialized(true), hierarchy(&hierarchy), config(config), - refactor(hierarchy, config), hybrid_refactor(hierarchy, config), - lossless_compressor(hierarchy.total_num_elems(), config), - quantizer(hierarchy, config), hybrid_quantizer(hierarchy, config) { - + hybrid_refactor(hierarchy, config), + lossless_compressor(calculate_padded_size(hierarchy, config), config), + hybrid_quantizer(hierarchy, hybrid_refactor.global_hierarchy, config) { norm_array = Array<1, T, DeviceType>({1}); + hybrid_decomposed_array = + Array<1, T, DeviceType>({hybrid_refactor.DecomposedDataSize()}); + // Reuse workspace. Warning: if (sizeof(QUANTIZED_INT) <= sizeof(T)) { - // Reuse workspace if possible - norm_tmp_array = Array<1, T, DeviceType>({hierarchy.total_num_elems()}, - (T *)refactor.w_array.data()); - quantized_array = Array( - hierarchy.level_shape(hierarchy.l_target()), - (QUANTIZED_INT *)refactor.w_array.data()); + if (config.num_local_refactoring_level > 0) { + norm_tmp_array = Array<1, T, DeviceType>( + {hierarchy.total_num_elems()}, + (T *)hybrid_refactor.local_refactor.coarse_buffers[0].data()); + // hybrid_quantized_array = Array<1, QUANTIZED_INT, DeviceType>( + // {hybrid_refactor.DecomposedDataSize()}, + // (QUANTIZED_INT*)hybrid_refactor.local_refactor.coarse_buffers[0] + // .data()); + } else { + // Reuse space from global refactor + norm_tmp_array = Array<1, T, DeviceType>( + {hierarchy.total_num_elems()}, + (T *)hybrid_refactor.global_refactor.w_array.data()); + // hybrid_quantized_array = Array<1, QUANTIZED_INT, DeviceType>( + // {hybrid_refactor.DecomposedDataSize()}, + // (QUANTIZED_INT*)hybrid_refactor.global_refactor.w_array.data()); + } } else { + // if space is not enough norm_tmp_array = Array<1, T, DeviceType>({hierarchy.total_num_elems()}); - quantized_array = Array( - hierarchy.level_shape(hierarchy.l_target()), false, false); + // hybrid_quantized_array = Array<1, QUANTIZED_INT, DeviceType>( + // {hybrid_refactor.DecomposedDataSize()}); } - std::vector shape = hierarchy.level_shape(hierarchy.l_target()); - SIZE decomposed_size = hybrid_refactor.DecomposedDataSize(); - - decomposed_array = Array<1, T, DeviceType>({decomposed_size}); - hybrid_quantized_array = - Array<1, QUANTIZED_INT, DeviceType>({decomposed_size}); } template @@ -66,46 +74,81 @@ void HybridHierarchyCompressor::Adapt( this->initialized = true; this->hierarchy = &hierarchy; this->config = config; - refactor.Adapt(hierarchy, config, queue_idx); hybrid_refactor.Adapt(hierarchy, config, queue_idx); - lossless_compressor.Adapt(hierarchy.total_num_elems(), config, queue_idx); - quantizer.Adapt(hierarchy, config, queue_idx); - hybrid_quantizer.Adapt(hierarchy, config, queue_idx); + lossless_compressor.Adapt(calculate_padded_size(hierarchy, config), config, + queue_idx); + hybrid_quantizer.Adapt(hierarchy, hybrid_refactor.global_hierarchy, config, + queue_idx); norm_array.resize({1}, queue_idx); - // Reuse workspace. Warning: + hybrid_decomposed_array.resize({hybrid_refactor.DecomposedDataSize()}, + queue_idx); + hybrid_quantized_array = Array<1, QUANTIZED_INT, DeviceType>( + {hybrid_refactor.DecomposedDataSize()}); + + // Reuse workspace if (sizeof(QUANTIZED_INT) <= sizeof(T)) { - // Reuse workspace if possible - norm_tmp_array = Array<1, T, DeviceType>({hierarchy.total_num_elems()}, - (T *)refactor.w_array.data()); - quantized_array = Array( - hierarchy.level_shape(hierarchy.l_target()), - (QUANTIZED_INT *)refactor.w_array.data()); + if (config.num_local_refactoring_level > 0) { + norm_tmp_array = Array<1, T, DeviceType>( + {hierarchy.total_num_elems()}, + (T *)hybrid_refactor.local_refactor.coarse_buffers[0].data()); + // hybrid_quantized_array = Array<1, QUANTIZED_INT, DeviceType>( + // {hybrid_refactor.DecomposedDataSize()}, + // (QUANTIZED_INT*)hybrid_refactor.local_refactor.coarse_buffers[0] + // .data()); + } else { + // Reuse space from global refactor + norm_tmp_array = Array<1, T, DeviceType>( + {hierarchy.total_num_elems()}, + (T *)hybrid_refactor.global_refactor.w_array.data()); + // hybrid_quantized_array = Array<1, QUANTIZED_INT, DeviceType>( + // {hybrid_refactor.DecomposedDataSize()}, + // (QUANTIZED_INT*)hybrid_refactor.global_refactor.w_array.data()); + } } else { norm_tmp_array.resize({hierarchy.total_num_elems()}, queue_idx); - quantized_array.resize(hierarchy.level_shape(hierarchy.l_target()), - queue_idx); + // hybrid_quantized_array.resize({hybrid_refactor.DecomposedDataSize()}, + // queue_idx); } - std::vector shape = hierarchy.level_shape(hierarchy.l_target()); - SIZE decomposed_size = hybrid_refactor.DecomposedDataSize(); - - decomposed_array.resize({decomposed_size}, queue_idx); - hybrid_quantized_array.resize({decomposed_size}, queue_idx); } +// May not be accurate template size_t HybridHierarchyCompressor::EstimateMemoryFootprint( std::vector shape, Config config) { Hierarchy hierarchy; hierarchy.EstimateMemoryFootprint(shape); size_t size = 0; - size += DataRefactorType::EstimateMemoryFootprint(shape); - size += LinearQuantizerType::EstimateMemoryFootprint(shape); + // size += + // BlockLocalHierarchyDataRefactorType::EstimateMemoryFootprint(shape); + size += + HybridHierarchyDataRefactorType::EstimateMemoryFootprint(shape, config); + // log::info( + // "Data refactor space: " + + // std::to_string( + // (double)(BlockLocalHierarchyDataRefactorType::EstimateMemoryFootprint( + // shape)) / + // 1e9) + + // " GB"); + // size += LocalQuantizerType::EstimateMemoryFootprint(shape); + size += HybridQuantizerType::EstimateMemoryFootprint(shape); + // log::info( + // "Quantizer space: " + + // std::to_string( + // (double)(LocalQuantizerType::EstimateMemoryFootprint(shape)) / 1e9) + // + + // " GB"); size += LosslessCompressorType::EstimateMemoryFootprint( - hierarchy.total_num_elems(), config); + calculate_padded_size(hierarchy, config), config); + // log::info( + // "Lossless space: " + + // std::to_string((double)(LosslessCompressorType::EstimateMemoryFootprint( + // hierarchy.total_num_elems(), config)) / + // 1e9) + + // " GB"); size += sizeof(T); if (sizeof(QUANTIZED_INT) > sizeof(T)) { - size += sizeof(T) * hierarchy.total_num_elems(); - size += sizeof(QUANTIZED_INT) * hierarchy.total_num_elems(); + size += sizeof(T) * calculate_padded_size(hierarchy, config); + size += sizeof(QUANTIZED_INT) * calculate_padded_size(hierarchy, config); } return size; } @@ -124,68 +167,52 @@ void HybridHierarchyCompressor::CalculateNorm( template void HybridHierarchyCompressor::Decompose( Array &original_data, int queue_idx) { - hybrid_refactor.Decompose(original_data, decomposed_array, queue_idx); + // DumpSubArray("/home/leonli/TestInCacheBlock/org.txt",SubArray(original_data)); + // PrintSubarray("Original before decompose", SubArray(original_data)); + // SubArray temp({3,3,3}, original_data.data()); + // PrintSubarray("Orginal 8x8x8 before decompose", temp); + // hybrid_refactor.Decompose(original_data, decomposed_array, queue_idx); + hybrid_refactor.Decompose(SubArray(original_data), + SubArray(hybrid_decomposed_array), queue_idx, + orthogonal_projection); + // PrintSubarray("Decomposed after decompose", + // SubArray(local_decomposed_array)); } template void HybridHierarchyCompressor::Quantize( Array &original_data, enum error_bound_type ebtype, T tol, T s, T norm, int queue_idx) { - hybrid_quantizer.Quantize(decomposed_array, ebtype, tol, s, norm, + // Only called from Compress(), which has already resolved + // orthogonal_projection and set it on hybrid_quantizer. + SIZE total_num_elems_1D = hybrid_refactor.DecomposedDataSize(); + + SubArray<1, T, DeviceType> data_subarray({total_num_elems_1D}, + hybrid_decomposed_array.data()); + hybrid_quantizer.Quantize(data_subarray, ebtype, tol, s, norm, hybrid_quantized_array, lossless_compressor, queue_idx); } +template +void HybridHierarchyCompressor::DecomposeQuantize( + Array &original_data, enum error_bound_type ebtype, T tol, + T s, T norm, int queue_idx) { + // Only called from Compress(), which has already resolved + // orthogonal_projection and set it on hybrid_quantizer. + SubArray<1, T, DeviceType> decomposed_subarray(hybrid_decomposed_array); + SubArray<1, QUANTIZED_INT, DeviceType> quantized_subarray( + hybrid_quantized_array); + hybrid_quantizer.DecomposeQuantize( + hybrid_refactor, SubArray(original_data), decomposed_subarray, + quantized_subarray, ebtype, tol, s, norm, lossless_compressor, queue_idx); +} + template void HybridHierarchyCompressor::LosslessCompress( Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { - Array<1, QUANTIZED_UNSIGNED_INT, DeviceType> hybrid_quantized_liearized_array( - {hierarchy->total_num_elems()}, - (QUANTIZED_UNSIGNED_INT *)hybrid_quantized_array.data()); - - // SIZE ori_size = 512*512*512; - // SIZE coarse_size = 320*320*320; - - // SIZE ori_size = 64*64*64; - // SIZE coarse_size = 40*40*40; - // SIZE coeff_size = ori_size-coarse_size; - - // Array<1, QUANTIZED_UNSIGNED_INT, DeviceType> - // hybrid_quantized_liearized_array1( - // {coeff_size}, - // (QUANTIZED_UNSIGNED_INT *)hybrid_quantized_array.data()+coarse_size); - - // Array<1, QUANTIZED_UNSIGNED_INT, DeviceType> - // hybrid_quantized_liearized_array2( - // {coarse_size}, - // (QUANTIZED_UNSIGNED_INT *)hybrid_quantized_array.data()); - - // PrintSubarray("coeff", SubArray(hybrid_quantized_liearized_array1)); - // PrintSubarray("coarse", SubArray(hybrid_quantized_liearized_array2)); - // PrintSubarray("hh", SubArray(hybrid_quantized_liearized_array)); - - // DeviceRuntime::SyncQueue(queue_idx); - // DumpSubArray("quantized_hh.dat", - // SubArray(hybrid_quantized_liearized_array)); - // DumpSubArray("quantized_coarse.dat", - // SubArray(hybrid_quantized_liearized_array2)); - // DumpSubArray("quantized_coeff.dat", - // SubArray(hybrid_quantized_liearized_array1)); - // DeviceRuntime::SyncQueue(queue_idx); - // std::cout << "done dumping....\n"; - - // LosslessCompressorType lossless_compressor1(coeff_size, config); - - // lossless_compressor1.Compress(hybrid_quantized_liearized_array1, - // compressed_data, queue_idx); - - // LosslessCompressorType lossless_compressor2(coarse_size, config); - - // lossless_compressor2.Compress(hybrid_quantized_liearized_array2, - // compressed_data, queue_idx); - - lossless_compressor.Compress(hybrid_quantized_liearized_array, - compressed_data, queue_idx); + lossless_compressor.Compress(hybrid_quantized_array, compressed_data, + queue_idx); } template @@ -202,25 +229,83 @@ void HybridHierarchyCompressor::Deserialize( template void HybridHierarchyCompressor::Recompose( - Array &decompressed_data, int queue_idx) { - refactor.Recompose(decompressed_data, queue_idx); + Array &decompressed_data, bool orthogonal_projection, + int queue_idx) { + this->orthogonal_projection = orthogonal_projection; + // PrintSubarray("Decomposed before recompose", + // SubArray(local_decomposed_array)); + hybrid_refactor.Recompose(SubArray(decompressed_data), + SubArray(hybrid_decomposed_array), queue_idx, + this->orthogonal_projection); + + // SubArray temp({3,3,3}, decompressed_data.data()); + // PrintSubarray("Orginal 8x8x8 after decompose", temp); + // PrintSubarray("Decompressed after recompose", SubArray(decompressed_data)); + // DumpSubArray("/home/leonli/TestInCacheBlock/decomp.txt",SubArray(decompressed_data)); } template void HybridHierarchyCompressor::Dequantize( Array &decompressed_data, enum error_bound_type ebtype, T tol, T s, T norm, int queue_idx) { - quantizer.Dequantize(decompressed_data, ebtype, tol, s, norm, quantized_array, - lossless_compressor, queue_idx); + // Only called from DequantizeRecompose(), which has already resolved + // orthogonal_projection and set it on hybrid_quantizer. + SIZE total_num_elems_1D = hybrid_refactor.DecomposedDataSize(); + SubArray<1, T, DeviceType> decompressed_data_subarray( + {total_num_elems_1D}, hybrid_decomposed_array.data()); + // Direct calculation + hybrid_quantizer.Dequantize(decompressed_data_subarray, ebtype, tol, s, norm, + hybrid_quantized_array, lossless_compressor, + queue_idx); +} + +template +void HybridHierarchyCompressor::DequantizeRecompose( + Array &decompressed_data, enum error_bound_type ebtype, + T tol, T s, T norm, int queue_idx) { + // The decompress-side resolution point: the domain-decomposition pipelines + // (GPUPipelines/CPUPipelines) call this directly, bypassing Decompress(), so + // it has to resolve for itself rather than relying on a caller having done + // it already. + orthogonal_projection = + infer_orthogonal_projection(config.projection_mode, s); + hybrid_quantizer.SetOrthogonalProjection(orthogonal_projection); + if (config.fuse_dequantize_recompose && hybrid_quantizer.CanFuseQuantize(s)) { + log::info("Local dequantize+recompose kernels: fused"); + DequantizeRecomposeFused(decompressed_data, ebtype, tol, s, norm, + queue_idx); + } else { + log::info("Local dequantize+recompose kernels: separate (" + + (config.fuse_dequantize_recompose + ? hybrid_quantizer.WhyCannotFuseQuantize(s) + : std::string("kernel fusion disabled")) + + ")"); + Dequantize(decompressed_data, ebtype, tol, s, norm, queue_idx); + Recompose(decompressed_data, orthogonal_projection, queue_idx); + } +} + +template +void HybridHierarchyCompressor::DequantizeRecomposeFused( + Array &decompressed_data, enum error_bound_type ebtype, + T tol, T s, T norm, int queue_idx) { + SubArray<1, T, DeviceType> decomposed_subarray(hybrid_decomposed_array); + SubArray<1, QUANTIZED_INT, DeviceType> quantized_subarray( + hybrid_quantized_array); + // Build the output view from the hierarchy shape (like the unfused + // Recompose does for its final copy) rather than the array metadata, which + // the pipelines may not have resized yet. + SubArray data_subarray( + hierarchy->level_shape(hierarchy->l_target()), decompressed_data.data()); + hybrid_quantizer.DequantizeRecompose( + hybrid_refactor, data_subarray, decomposed_subarray, quantized_subarray, + ebtype, tol, s, norm, lossless_compressor, queue_idx); } template void HybridHierarchyCompressor::LosslessDecompress( Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { - Array<1, QUANTIZED_UNSIGNED_INT, DeviceType> quantized_liearized_data( - {hierarchy->total_num_elems()}, - (QUANTIZED_UNSIGNED_INT *)quantized_array.data()); - lossless_compressor.Decompress(compressed_data, quantized_liearized_data, + lossless_compressor.Decompress(compressed_data, hybrid_quantized_array, queue_idx); } @@ -228,12 +313,12 @@ template void HybridHierarchyCompressor::Compress( Array &original_data, enum error_bound_type ebtype, T tol, T s, T &norm, Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { - config.apply(); DeviceRuntime::SelectDevice(config.dev_id); log::info("Select device: " + DeviceRuntime::GetDeviceName()); Timer timer_total; + Timer timer_compress_kernel; for (int d = D - 1; d >= 0; d--) { if (hierarchy->level_shape(hierarchy->l_target(), d) != original_data.shape(d)) { @@ -243,27 +328,69 @@ void HybridHierarchyCompressor::Compress( } } - if (log::level & log::TIME) + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); timer_total.start(); + } + + // The compress-side resolution point: Decompose()/DecomposeQuantize()/ + // Quantize() below all just consume the result. + orthogonal_projection = + infer_orthogonal_projection(config.projection_mode, s); + hybrid_quantizer.SetOrthogonalProjection(orthogonal_projection); CalculateNorm(original_data, ebtype, s, norm, queue_idx); - Decompose(original_data, queue_idx); - Quantize(original_data, ebtype, tol, s, norm, queue_idx); + // log::info(std::to_string(original_data.totalNumElems())); + // PrintSubarray("Original before decompose", SubArray(original_data)); + // log::info("Before decompose()"); + + if (log::level & log::TIME) + timer_compress_kernel.start(); + if (config.fuse_decompose_quantize && hybrid_quantizer.CanFuseQuantize(s)) { + log::info("Local decompose+quantize kernels: fused"); + DecomposeQuantize(original_data, ebtype, tol, s, norm, queue_idx); + } else { + log::info("Local decompose+quantize kernels: separate (" + + (config.fuse_decompose_quantize + ? hybrid_quantizer.WhyCannotFuseQuantize(s) + : std::string("kernel fusion disabled")) + + ")"); + Decompose(original_data, queue_idx); + // log::info("After decompose()"); + // log::info(std::to_string(original_data.totalNumElems())); + // PrintSubarray("Original after decompose", SubArray(original_data)); + // // PrintSubarray("Decomposed", SubArray(decomposed_array)); + // log::info("Before quantize"); + Quantize(original_data, ebtype, tol, s, norm, queue_idx); + } + // log::info("After quantize"); + // log::info("Num of Original data after quantization:"); + // log::info(std::to_string(original_data.totalNumElems())); + // PrintSubarray("Quantized", SubArray(local_quantized_array)); + // PrintSubarray("Compressed data before lossless",SubArray(compressed_data)); + // log::info("Before lossless"); LosslessCompress(compressed_data, queue_idx); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(0); + timer_compress_kernel.end(); + timer_compress_kernel.print("Compression Kernel"); + timer_compress_kernel.clear(); + } + Serialize(compressed_data, queue_idx); + // log::info("After lossless"); + // PrintSubarray("Quantized data before + // lossless",SubArray(local_quantized_array)); PrintSubarray("Compressed data + // after lossless",SubArray(compressed_data)); From printing result, we found + // lossless didn't do anything to compressed_data if (config.compress_with_dryrun) { - Dequantize(original_data, ebtype, tol, s, norm, queue_idx); - Recompose(original_data, queue_idx); + DequantizeRecompose(original_data, ebtype, tol, s, norm, queue_idx); } if (log::level & log::TIME) { DeviceRuntime::SyncQueue(0); timer_total.end(); - timer_total.print("Low-level compression"); - log::time( - "Low-level compression throughput: " + - std::to_string((double)(hierarchy->total_num_elems() * sizeof(T)) / - timer_total.get() / 1e9) + - " GB/s"); + timer_total.print("Low-level compression", + hierarchy->total_num_elems() * sizeof(T)); timer_total.clear(); } } @@ -283,23 +410,49 @@ void HybridHierarchyCompressor::Decompress( timer_total.start(); decompressed_data.resize(hierarchy->level_shape(hierarchy->l_target())); + Deserialize(compressed_data, queue_idx); LosslessDecompress(compressed_data, queue_idx); - Dequantize(decompressed_data, ebtype, tol, s, norm, queue_idx); - Recompose(decompressed_data, queue_idx); + // DequantizeRecompose resolves orthogonal_projection itself. + DequantizeRecompose(decompressed_data, ebtype, tol, s, norm, queue_idx); if (log::level & log::TIME) { DeviceRuntime::SyncQueue(0); timer_total.end(); - timer_total.print("Low-level decompression"); - log::time( - "Low-level decompression throughput: " + - std::to_string((double)(hierarchy->total_num_elems() * sizeof(T)) / - timer_total.get() / 1e9) + - " GB/s"); + timer_total.print("Low-level decompression", + hierarchy->total_num_elems() * sizeof(T)); timer_total.clear(); } } +// Only calculating padding to 8x8x8 for once +template +SIZE HybridHierarchyCompressor::calculate_padded_size( + Hierarchy &hierarchy, Config config) { + int L = config.num_local_refactoring_level; + SIZE total_num_elems_1D = 1; + if (L > 0) { + std::vector coarse_shape = + hierarchy.level_shape(hierarchy.l_target()); + for (int l = 0; l < L; l++) { + SIZE last_level_size = 1, curr_level_size = 1; + for (DIM d = 0; d < D; d++) { + coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 8; + last_level_size *= coarse_shape[d]; + coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 5; + curr_level_size *= coarse_shape[d]; + } + total_num_elems_1D += (last_level_size - curr_level_size); + if (l == L - 1) { + total_num_elems_1D += curr_level_size; + } + } + } else { + total_num_elems_1D = hierarchy.total_num_elems(); + } + + return total_num_elems_1D; +} + } // namespace mgard_x -#endif \ No newline at end of file +#endif diff --git a/include/mgard-x/CompressionLowLevel/LossyCompressorInterface.hpp b/include/mgard-x/CompressionLowLevel/LossyCompressorInterface.hpp index 9aee88df07..8b35c36fe2 100644 --- a/include/mgard-x/CompressionLowLevel/LossyCompressorInterface.hpp +++ b/include/mgard-x/CompressionLowLevel/LossyCompressorInterface.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_LOSSY_COMPRESSOR_INTERFACE_HPP diff --git a/include/mgard-x/CompressionLowLevel/NormCalculator.hpp b/include/mgard-x/CompressionLowLevel/NormCalculator.hpp index aa9c5ad512..3012190a8b 100644 --- a/include/mgard-x/CompressionLowLevel/NormCalculator.hpp +++ b/include/mgard-x/CompressionLowLevel/NormCalculator.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_NORM_CALCULATOR_HPP @@ -24,7 +24,7 @@ T norm_calculator(Array &original_array, T norm = 0; SubArray<1, T, DeviceType> temp_subarray; if (!original_array.isPitched()) { // zero copy - log::info("Use zero copy when calculating norm"); + log::dbg("Use zero copy when calculating norm"); temp_subarray = SubArray<1, T, DeviceType>({total_elems}, original_array.data()); } else { // need to linearized @@ -73,7 +73,7 @@ T norm_calculator(Array &original_array, } if (log::level & log::TIME) { timer.end(); - timer.print("Calculate norm"); + timer.print("Calculate norm", total_elems * sizeof(T)); timer.clear(); } return norm; diff --git a/include/mgard-x/Config/Config.h b/include/mgard-x/Config/Config.h index 8fca51bea4..113c3dd0e7 100644 --- a/include/mgard-x/Config/Config.h +++ b/include/mgard-x/Config/Config.h @@ -16,13 +16,13 @@ struct Config { double estimate_outlier_ratio; SIZE huff_dict_size; SIZE huff_block_size; + SIZE block_delta_block_size; + enum block_delta_mode_type block_delta_mode; SIZE lz4_block_size; int zstd_compress_level; bool normalize_coordinates; enum lossless_type lossless; - int reorder; int log_level; - bool prefetch; bool auto_pin_host_buffers; SIZE max_larget_level; SIZE max_memory_footprint; @@ -34,8 +34,27 @@ struct Config { bool adjust_shape; bool compress_with_dryrun; int num_local_refactoring_level; + int num_global_refactoring_level; bool auto_cache_release; cpu_parallelization_mode cpu_mode; + bool mdr_qoi_mode; + int mdr_qoi_num_variables; + std::vector roi_tolerance_map; + bool enable_roi; + // Transform basis policy, shared by the plain Compressor and the hybrid + // (BlockMGARD) HybridHierarchyCompressor. Auto (default) picks Hierarchical + // under an L-infinity bound and Orthogonal otherwise; Orthogonal/ + // Hierarchical force a specific basis (Hierarchical still requires an + // L-infinity bound). See compression_projection_mode_type in Types.h. + enum compression_projection_mode_type projection_mode; + // The hybrid (BlockMGARD) local stage fuses its decompose/recompose kernels + // with quantization/dequantization so coefficients never round-trip through + // global memory as T. These select the older separate-pass implementation, + // which the test suite still exercises. Purely a performance choice: both + // paths reconstruct identically, so a file compressed either way decompresses + // either way and nothing about the choice is recorded in the file header. + bool fuse_decompose_quantize; // compression + bool fuse_dequantize_recompose; // decompression Config(); void apply(); @@ -43,4 +62,4 @@ struct Config { } // namespace mgard_x -#endif \ No newline at end of file +#endif diff --git a/include/mgard-x/DataRefactoring/BlockLocalHierarchyDataRefactor.hpp b/include/mgard-x/DataRefactoring/BlockLocalHierarchyDataRefactor.hpp new file mode 100644 index 0000000000..38d434c201 --- /dev/null +++ b/include/mgard-x/DataRefactoring/BlockLocalHierarchyDataRefactor.hpp @@ -0,0 +1,484 @@ +#include "InCacheBlock/DataRefactoring.h" +#include "MultiDimension/DataRefactoring.h" + +#ifndef MGARD_X_BLOCK_LOCAL_HIERARCHY_DATA_REFACTOR_HPP +#define MGARD_X_BLOCK_LOCAL_HIERARCHY_DATA_REFACTOR_HPP + +namespace mgard_x { + +namespace data_refactoring { + +template +class BlockLocalHierarchyDataRefactor { +public: + BlockLocalHierarchyDataRefactor() : initialized(false) {} + + // Removing all checks for L since processed in HybridHierarchyDataRefactor + BlockLocalHierarchyDataRefactor(Hierarchy &hierarchy, + Config config) + : initialized(true), hierarchy(&hierarchy), config(config) { + this->L = config.num_local_refactoring_level; + compute_local_ranges(); + w_array = Array<1, T, DeviceType>({DecomposedCoeffSize()}); + temp_coarest = Array<1, T, DeviceType>({coarse_num_elems[this->L - 1]}); + coarse_buffers.resize(2); + coarse_buffers[0] = Array(fine_shapes[0]); + coarse_buffers[1] = Array(fine_shapes[0]); + } + + void Adapt(Hierarchy &hierarchy, Config config, + int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + this->config = config; + this->L = config.num_local_refactoring_level; + compute_local_ranges(); + + w_array.resize({DecomposedCoeffSize()}, queue_idx); + temp_coarest.resize({coarse_num_elems[this->L - 1]}, queue_idx); + coarse_buffers.resize(2); + coarse_buffers[0].resize(fine_shapes[0], queue_idx); + coarse_buffers[1].resize(fine_shapes[0], queue_idx); + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + // We have 2 arrays for shape switch and another one for output coeff and + // coarest + size_t size = 3; + for (DIM d = 0; d < shape.size(); d++) { + int dim8 = ((shape[d] - 1) / 8 + 1) * 8; + size *= dim8; + } + return size * sizeof(T); + } + + size_t DecomposedDataSize() { + SIZE decomposed_size = coarse_num_elems[this->L - 1]; + for (SIZE l = 0; l < this->L; l++) { + decomposed_size += local_coeff_size[l]; + } + return decomposed_size; + } + + size_t DecomposedCoeffSize() { + SIZE decomposed_coeff_size = 0; + for (SIZE l = 0; l < this->L; l++) { + decomposed_coeff_size += local_coeff_size[l]; + } + return decomposed_coeff_size; + } + + void compute_local_ranges() { + // Get original shape from hierarchy + coarse_shape = hierarchy->level_shape(hierarchy->l_target()); + fine_num_elems.clear(); + coarse_num_elems.clear(); + local_coeff_size.clear(); + coarse_shapes.clear(); + fine_shapes.clear(); + + for (int l = 0; l < this->L; ++l) { + SIZE last_level_size = 1, curr_level_size = 1; + std::vector fine_shape(D); + for (DIM d = 0; d < D; ++d) { + coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 8; + last_level_size *= coarse_shape[d]; + fine_shape[d] = coarse_shape[d]; + coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 5; + curr_level_size *= coarse_shape[d]; + } + fine_num_elems.push_back(last_level_size); + coarse_num_elems.push_back(curr_level_size); + local_coeff_size.push_back(last_level_size - curr_level_size); + coarse_shapes.push_back(coarse_shape); + fine_shapes.push_back(fine_shape); + } + } + + void Decompose(SubArray data, + SubArray<1, T, DeviceType> output_decomposed, int queue_idx, + bool orthogonal_projection = true) { + SubArray fine(coarse_buffers[1]); + SubArray coarse; + // Zero the level-0 fine buffer when the input needs padding up to the + // next multiple of 8: CopyND below only writes the original extent, and + // 8x8x8 blocks straddling the boundary would otherwise mix uninitialized + // values into their coefficients. Deeper levels are unaffected (their + // fine buffer is a coarse buffer that is fully memset before use). + bool needs_padding = false; + for (DIM d = 0; d < D; d++) { + if (data.shape(d) != fine_shapes[0][d]) { + needs_padding = true; + break; + } + } + if (needs_padding) { + coarse_buffers[1].memset(0, queue_idx); + } + // CopyND follows the shape of 1st param + multi_dimension::CopyND(data, fine, queue_idx); + SubArray<1, T, DeviceType> decomposed_coeff(w_array); + + // Times the transform kernels only; the surrounding copies are covered by + // the Hybrid Decomposition timer in HybridHierarchyDataRefactor. + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + // Will be reused between decompose and recompose + accumulated_local_coeff_size = 0; + for (SIZE l = 0; l < this->L; l++) { + accumulated_local_coeff_size += local_coeff_size[l]; + // Think about a way to change this local_coeff + SubArray<1, T, DeviceType> local_coeff( + {local_coeff_size[l]}, + decomposed_coeff(decomposed_coeff.shape(0) - + accumulated_local_coeff_size)); + + int buffer_idx = l % 2; + coarse_buffers[buffer_idx].memset(0, queue_idx); + coarse = SubArray(coarse_shapes[l], + coarse_buffers[buffer_idx].data()); + for (DIM d = 0; d < D; d++) { + coarse.setLd(d, fine_shapes[0][d]); + } + coarse.project(D - 3, D - 2, D - 1); + + in_cache_block::decompose( + fine, coarse, local_coeff, orthogonal_projection, queue_idx); + if (l < this->L - 1) { + fine = SubArray(fine_shapes[l + 1], + coarse_buffers[buffer_idx].data()); + for (DIM d = 0; d < D; d++) { + fine.setLd(d, fine_shapes[0][d]); + } + fine.project(D - 3, D - 2, D - 1); + } + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Local Decomposition", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + + int final_buffer_id = (this->L - 1) % 2; + SubArray coarsest(coarse_shapes[this->L - 1], + coarse_buffers[final_buffer_id].data()); + for (DIM d = 0; d < D; d++) { + coarsest.setLd(d, fine_shapes[0][d]); + } + coarsest.project(D - 3, D - 2, D - 1); + // log::info("Find read buffer idx: " + std::to_string(final_buffer_id)); + + // Write the coarsest level directly into output_decomposed instead of + // staging through temp_coarest: temp_coarest is unused between here and + // the next Recompose() call, which repopulates it independently from + // input_decomposed (see below), so the staging copy was pure overhead. + SubArray coarsest_out(coarse_shapes[this->L - 1], + output_decomposed.data()); + for (DIM d = 0; d < D; d++) { + coarsest_out.setLd(d, coarse_shapes[this->L - 1][d]); + } + coarsest_out.project(D - 3, D - 2, D - 1); + multi_dimension::CopyND(coarsest, coarsest_out, queue_idx); + + SubArray<1, T, DeviceType> data_coeff({DecomposedCoeffSize()}, + output_decomposed.data() + + coarse_num_elems[this->L - 1]); + multi_dimension::CopyND(decomposed_coeff, data_coeff, queue_idx); + + // PrintSubarray("Temp in decompose:",SubArray(temp_coarest)); + } + + // Fused decompose+quantize. Runs the same per-level 8x8x8 decomposition as + // Decompose(), but each level's coefficients are quantized in-kernel and + // written directly to their final location in output_quantized, so the + // T-typed coefficient staging (w_array) and the copies into + // output_decomposed disappear. Level 0 reads straight from the (possibly + // unpadded) input, and deeper levels read the previous coarse buffer at its + // true extent — the fused kernel zero-fills out-of-range reads, replacing + // the padding memsets. Only the coarsest level is emitted in T, compacted + // at the front of output_decomposed for the global stage / coarsest + // quantization. + // + // Level l's quantizer: level_quantizers[l] (reciprocal), or per-block + // level_block_quantizers[l] in ROI mode (level_quantizers empty). + template + void DecomposeQuantize( + SubArray data, + SubArray<1, T, DeviceType> output_decomposed, + SubArray<1, Q, DeviceType> output_quantized, + const std::vector &level_quantizers, + const std::vector> &level_block_quantizers, + bool prep_huffman, SIZE dict_size, int queue_idx, + bool orthogonal_projection = true) { + bool use_block_quantizers = level_quantizers.empty(); + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SubArray fine = data; + SIZE accumulated = 0; + for (SIZE l = 0; l < this->L; l++) { + accumulated += local_coeff_size[l]; + SubArray<1, Q, DeviceType> level_quantized( + {local_coeff_size[l]}, + output_quantized(output_quantized.shape(0) - accumulated)); + + int buffer_idx = l % 2; + SubArray coarse; + if (l == this->L - 1) { + // Last level: write the coarsest data compactly to its final + // location instead of staging it in a padded buffer and copying. + coarse = SubArray(coarse_shapes[l], + output_decomposed.data()); + for (DIM d = 0; d < D; d++) { + coarse.setLd(d, coarse_shapes[l][d]); + } + } else { + coarse = SubArray(coarse_shapes[l], + coarse_buffers[buffer_idx].data()); + for (DIM d = 0; d < D; d++) { + coarse.setLd(d, fine_shapes[0][d]); + } + } + coarse.project(D - 3, D - 2, D - 1); + + in_cache_block::decompose_quantize( + fine, coarse, level_quantized, + use_block_quantizers ? (T)0 : level_quantizers[l], + use_block_quantizers ? level_block_quantizers[l] + : SubArray<1, T, DeviceType>(), + use_block_quantizers, prep_huffman, dict_size, orthogonal_projection, + queue_idx); + + if (l < this->L - 1) { + // Next level reads the coarse output at its true extent; the fused + // kernel's boundary handling supplies the zero padding. + fine = coarse; + } + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Local Decomposition+Quantization (fused)", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + // Fused dequantize+recompose. Runs the same per-level 8x8x8 recomposition + // as Recompose(), but each level's coefficients are read from their final + // location in input_quantized and dequantized in-kernel, so the T-typed + // coefficient region of the decomposed array is never materialized. Only + // the coarsest level is consumed in T from the front of input_decomposed + // (produced by the global stage / coarsest dequantization). The staging + // copies of the unfused path also disappear: the coarsest level is read + // compactly in place (no temp_coarest restore), the ping-pong buffers are + // not memset (every coarse value read at level l was written by level l+1, + // or comes from input_decomposed), and the final level writes directly to + // the unpadded output (the fused kernel bounds-checks its stores). + // + // Level l's dequantizer: level_dequantizers[l] (non-reciprocal), or + // per-block level_block_dequantizers[l] in ROI mode (level_dequantizers + // empty). Indexing matches DecomposeQuantize (level 0 = finest). + template + void RecomposeDequantize( + SubArray data, + SubArray<1, T, DeviceType> input_decomposed, + SubArray<1, Q, DeviceType> input_quantized, + const std::vector &level_dequantizers, + const std::vector> &level_block_dequantizers, + bool prep_huffman, SIZE dict_size, int queue_idx, + bool orthogonal_projection = true) { + bool use_block_quantizers = level_dequantizers.empty(); + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + // Coarsest level, read compactly in place from the front of + // input_decomposed instead of staging through temp_coarest. + SubArray coarse(coarse_shapes[this->L - 1], + input_decomposed.data()); + for (DIM d = 0; d < D; d++) { + coarse.setLd(d, coarse_shapes[this->L - 1][d]); + } + coarse.project(D - 3, D - 2, D - 1); + + SIZE accumulated = DecomposedCoeffSize(); + for (SIZE l = 0; l < this->L; l++) { + SIZE level_idx = this->L - l - 1; + + SubArray<1, Q, DeviceType> level_quantized( + {local_coeff_size[level_idx]}, + input_quantized(input_quantized.shape(0) - accumulated)); + + int buffer_idx = l % 2; + SubArray fine; + if (level_idx == 0) { + // Last level: write the reconstructed data directly to the unpadded + // output instead of staging it in a padded buffer and copying. + fine = data; + } else { + fine = SubArray(fine_shapes[level_idx], + coarse_buffers[buffer_idx].data()); + for (DIM d = 0; d < D; d++) { + fine.setLd(d, fine_shapes[0][d]); + } + fine.project(D - 3, D - 2, D - 1); + } + + in_cache_block::recompose_dequantize( + fine, coarse, level_quantized, + use_block_quantizers ? (T)0 : level_dequantizers[level_idx], + use_block_quantizers ? level_block_dequantizers[level_idx] + : SubArray<1, T, DeviceType>(), + use_block_quantizers, prep_huffman, dict_size, orthogonal_projection, + queue_idx); + + if (l < this->L - 1) { + coarse = SubArray(coarse_shapes[level_idx - 1], + coarse_buffers[buffer_idx].data()); + for (DIM d = 0; d < D; d++) { + coarse.setLd(d, fine_shapes[0][d]); + } + coarse.project(D - 3, D - 2, D - 1); + } + accumulated -= local_coeff_size[level_idx]; + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Local Recomposition+Dequantization (fused)", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + void Recompose(SubArray data, + SubArray<1, T, DeviceType> input_decomposed, int queue_idx, + bool orthogonal_projection = true) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + // Initialize accumulated_local_coeff_size so that Recompose works correctly + // regardless of whether Decompose was called first (e.g., standalone + // decompress). + accumulated_local_coeff_size = DecomposedCoeffSize(); + + // Restore temp_coarest from input_decomposed (the first + // coarse_num_elems[L-1] elements). This is critical for standalone + // decompression where Decompose was never called and temp_coarest was never + // populated. After global Recompose, + // input_decomposed[0..coarse_num_elems[L-1]-1] holds the correctly + // reconstructed coarsest values, which we must use here instead of + // stale/zero temp_coarest. + multi_dimension::CopyND( + SubArray<1, T, DeviceType>({coarse_num_elems[this->L - 1]}, + input_decomposed.data()), + SubArray(temp_coarest), queue_idx); + + coarse_buffers[0].memset(0, queue_idx); + coarse_buffers[1].memset(0, queue_idx); + + SubArray coarse(coarse_shapes[this->L - 1], + temp_coarest.data()); + + for (SIZE l = 0; l < this->L; l++) { + SIZE level_idx = this->L - l - 1; + + SubArray<1, T, DeviceType> local_coeff( + {local_coeff_size[level_idx]}, + input_decomposed(input_decomposed.shape(0) - + accumulated_local_coeff_size)); + + int buffer_idx = l % 2; + + SubArray fine(fine_shapes[level_idx], + coarse_buffers[buffer_idx].data()); + for (DIM d = 0; d < D; d++) { + fine.setLd(d, fine_shapes[0][d]); + } + fine.project(D - 3, D - 2, D - 1); + // log::info("Buffer idx for fine buffer: " + + // std::to_string(buffer_idx)); + + in_cache_block::recompose( + fine, coarse, local_coeff, orthogonal_projection, queue_idx); + + if (l < this->L - 1) { + coarse = SubArray(coarse_shapes[level_idx - 1], + coarse_buffers[buffer_idx].data()); + for (DIM d = 0; d < D; d++) { + coarse.setLd(d, fine_shapes[0][d]); + } + coarse.project(D - 3, D - 2, D - 1); + } + accumulated_local_coeff_size -= local_coeff_size[level_idx]; + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Local Recomposition", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + + // copy back, using ND + SubArray src( + hierarchy->level_shape(hierarchy->l_target()), + coarse_buffers[(this->L - 1) % 2].data()); + + for (DIM d = 0; d < D; d++) { + src.setLd(d, fine_shapes[0][d]); + } + src.project(D - 3, D - 2, D - 1); + + SubArray dst( + hierarchy->level_shape(hierarchy->l_target()), data.data()); + + multi_dimension::CopyND(src, dst, queue_idx); + } + + std::vector coarse_shape; + SIZE accumulated_local_coeff_size = 0; + bool initialized; + SIZE L; + Hierarchy *hierarchy; + Config config; + + std::vector fine_num_elems; + std::vector coarse_num_elems; + std::vector local_coeff_size; + std::vector> coarse_shapes; + std::vector> fine_shapes; + std::vector original_input_shape; + std::vector padded_input_shape; + + std::vector> coarse_buffers; + + Array<1, T, DeviceType> w_array; + Array<1, T, DeviceType> temp_coarest; +}; + +} // namespace data_refactoring +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/DataRefactoring/BlockLocalHierarchyDataRefactorInterface.hpp b/include/mgard-x/DataRefactoring/BlockLocalHierarchyDataRefactorInterface.hpp new file mode 100644 index 0000000000..957103993d --- /dev/null +++ b/include/mgard-x/DataRefactoring/BlockLocalHierarchyDataRefactorInterface.hpp @@ -0,0 +1,17 @@ +#ifndef MGARD_X_BLOCK_LOCAL_HIERARCHY_DATA_REFACTOR_INTERFACE_HPP +#define MGARD_X_BLOCK_LOCAL_HIERARCHY_DATA_REFACTOR_INTERFACE_HPP +namespace mgard_x { + +namespace data_refactoring { + +template +class BlockLocalHierarchyDataRefactor { + virtual void Decompose(SubArray data, int queue_idx) = 0; + virtual void Recompose(SubArray data, int queue_idx) = 0; +}; + +} // namespace data_refactoring + +} // namespace mgard_x + +#endif \ No newline at end of file diff --git a/include/mgard-x/DataRefactoring/CMakeLists.txt b/include/mgard-x/DataRefactoring/CMakeLists.txt index 3778089761..53ba8a46a4 100644 --- a/include/mgard-x/DataRefactoring/CMakeLists.txt +++ b/include/mgard-x/DataRefactoring/CMakeLists.txt @@ -5,5 +5,7 @@ list(APPEND MGARD_X_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/DataRefactorInterface.hpp ${CMAKE_CURRENT_SOURCE_DIR}/HybridHierarchyDataRefactor.hpp ${CMAKE_CURRENT_SOURCE_DIR}/HybridHierarchyDataRefactorInterface.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/BlockLocalHierarchyDataRefactor.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/BlockLocalHierarchyDataRefactorInterface.hpp ) set(MGARD_X_HEADER ${MGARD_X_HEADER} PARENT_SCOPE) \ No newline at end of file diff --git a/include/mgard-x/DataRefactoring/DataRefactor.hpp b/include/mgard-x/DataRefactoring/DataRefactor.hpp index 7abaf03af3..ca2a7d71f0 100644 --- a/include/mgard-x/DataRefactoring/DataRefactor.hpp +++ b/include/mgard-x/DataRefactoring/DataRefactor.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "DataRefactorInterface.hpp" @@ -48,7 +48,6 @@ class DataRefactor : public DataRefactorInterface { } static size_t EstimateMemoryFootprint(std::vector shape) { - Array<1, T, DeviceType> array_with_pitch({1}); size_t pitch_size = array_with_pitch.ld(0) * sizeof(T); @@ -71,7 +70,7 @@ class DataRefactor : public DataRefactorInterface { } void Decompose(SubArray data, int start_level, - int stop_level, int queue_idx) { + int stop_level, bool orthogonal_projection, int queue_idx) { Timer timer; if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); @@ -85,9 +84,9 @@ class DataRefactor : public DataRefactorInterface { if (config.decomposition == decomposition_type::MultiDim || config.decomposition == decomposition_type::Hybrid) { - multi_dimension::decompose(*hierarchy, data, w_subarray, - b_subarray, start_level, - stop_level, queue_idx); + multi_dimension::decompose( + *hierarchy, data, w_subarray, b_subarray, start_level, stop_level, + orthogonal_projection, queue_idx); } else if (config.decomposition == decomposition_type::SingleDim) { single_dimension::decompose( *hierarchy, data, start_level, stop_level, queue_idx); @@ -95,17 +94,13 @@ class DataRefactor : public DataRefactorInterface { if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); timer.end(); - timer.print("Decomposition"); - log::time( - "Decomposition throughput: " + - std::to_string((double)(hierarchy->total_num_elems() * sizeof(T)) / - timer.get() / 1e9) + - " GB/s"); + timer.print("Global Decomposition", + hierarchy->total_num_elems() * sizeof(T)); timer.clear(); } } void Recompose(SubArray data, int start_level, - int stop_level, int queue_idx) { + int stop_level, bool orthogonal_projection, int queue_idx) { Timer timer; if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); @@ -116,9 +111,9 @@ class DataRefactor : public DataRefactorInterface { if (D > 3) b_subarray = SubArray(b_array); if (config.decomposition == decomposition_type::MultiDim) { - multi_dimension::recompose(*hierarchy, data, w_subarray, - b_subarray, start_level, - stop_level, queue_idx); + multi_dimension::recompose( + *hierarchy, data, w_subarray, b_subarray, start_level, stop_level, + orthogonal_projection, queue_idx); } else if (config.decomposition == decomposition_type::SingleDim) { single_dimension::recompose( *hierarchy, data, start_level, stop_level, queue_idx); @@ -126,22 +121,20 @@ class DataRefactor : public DataRefactorInterface { if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); timer.end(); - timer.print("Recomposition"); - log::time( - "Recomposition throughput: " + - std::to_string((double)(hierarchy->total_num_elems() * sizeof(T)) / - timer.get() / 1e9) + - " GB/s"); + timer.print("Global Recomposition", + hierarchy->total_num_elems() * sizeof(T)); timer.clear(); } } - void Decompose(SubArray data, int queue_idx) { - Decompose(data, hierarchy->l_target(), 0, queue_idx); + void Decompose(SubArray data, bool orthogonal_projection, + int queue_idx) { + Decompose(data, hierarchy->l_target(), 0, orthogonal_projection, queue_idx); } - void Recompose(SubArray data, int queue_idx) { - Recompose(data, 0, hierarchy->l_target(), queue_idx); + void Recompose(SubArray data, bool orthogonal_projection, + int queue_idx) { + Recompose(data, 0, hierarchy->l_target(), orthogonal_projection, queue_idx); } bool initialized; diff --git a/include/mgard-x/DataRefactoring/DataRefactorInterface.hpp b/include/mgard-x/DataRefactoring/DataRefactorInterface.hpp index a1d83e9363..33d0912324 100644 --- a/include/mgard-x/DataRefactoring/DataRefactorInterface.hpp +++ b/include/mgard-x/DataRefactoring/DataRefactorInterface.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_DATA_REFACTOR_INTERFACE_HPP @@ -13,11 +13,15 @@ namespace data_refactoring { template class DataRefactorInterface { virtual void Decompose(SubArray data, int start_level, - int stop_level, int queue_idx) = 0; + int stop_level, bool orthogonal_projection, + int queue_idx) = 0; virtual void Recompose(SubArray data, int start_level, - int stop_level, int queue_idx) = 0; - virtual void Decompose(SubArray data, int queue_idx) = 0; - virtual void Recompose(SubArray data, int queue_idx) = 0; + int stop_level, bool orthogonal_projection, + int queue_idx) = 0; + virtual void Decompose(SubArray data, + bool orthogonal_projection, int queue_idx) = 0; + virtual void Recompose(SubArray data, + bool orthogonal_projection, int queue_idx) = 0; }; } // namespace data_refactoring diff --git a/include/mgard-x/DataRefactoring/HybridHierarchyDataRefactor.hpp b/include/mgard-x/DataRefactoring/HybridHierarchyDataRefactor.hpp index 049bce6360..7d8a2939c6 100644 --- a/include/mgard-x/DataRefactoring/HybridHierarchyDataRefactor.hpp +++ b/include/mgard-x/DataRefactoring/HybridHierarchyDataRefactor.hpp @@ -1,14 +1,14 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ +#include "../RuntimeX/Utilities/Exceptions.h" +#include "BlockLocalHierarchyDataRefactor.hpp" #include "DataRefactor.hpp" #include "HybridHierarchyDataRefactorInterface.hpp" -// #include "DataRefactoringWorkspace.hpp" -#include "../Linearization/LevelLinearizer.hpp" #include "InCacheBlock/DataRefactoring.h" #include "MultiDimension/DataRefactoring.h" #include "SingleDimension/DataRefactoring.h" @@ -26,38 +26,9 @@ class HybridHierarchyDataRefactor HybridHierarchyDataRefactor() : initialized(false) {} HybridHierarchyDataRefactor(Hierarchy &hierarchy, Config config) - : initialized(true), hierarchy(&hierarchy), config(config), - global_refactor(hierarchy, config) { - - coarse_shape = hierarchy.level_shape(hierarchy.l_target()); - // If we do at least one level of local refactoring - if (config.num_local_refactoring_level > 0) { - for (int l = 0; l < config.num_local_refactoring_level; l++) { - SIZE last_level_size = 1, curr_level_size = 1; - - // std::cout << coarse_shape[0] << " " << coarse_shape[1] << " " - // << coarse_shape[2] << "\n"; - for (DIM d = 0; d < D; d++) { - coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 8; - last_level_size *= coarse_shape[d]; - coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 5; - curr_level_size *= coarse_shape[d]; - } - - // std::cout << coarse_shape[0] << " " << coarse_shape[1] << " " - // << coarse_shape[2] << "\n"; - coarse_shapes.push_back(coarse_shape); - coarse_num_elems.push_back(last_level_size); - if (l == 0) { - coarse_array = Array(coarse_shape); - } - local_coeff_size.push_back(last_level_size - curr_level_size); - // std::cout << local_coeff_size[local_coeff_size.size() - 1] << "\n"; - } - } - - global_hierarchy = Hierarchy(coarse_shape, config); - global_refactor = DataRefactor(global_hierarchy, config); + : initialized(true), hierarchy(&hierarchy), config(config) { + this->L = config.num_local_refactoring_level; + this->M = config.num_global_refactoring_level; } void Adapt(Hierarchy &hierarchy, Config config, @@ -65,141 +36,218 @@ class HybridHierarchyDataRefactor this->initialized = true; this->hierarchy = &hierarchy; this->config = config; - coarse_shape = hierarchy.level_shape(hierarchy.l_target()); - coarse_shapes.clear(); - coarse_num_elems.clear(); - local_coeff_size.clear(); - // If we do at least one level of local refactoring - if (config.num_local_refactoring_level > 0) { - for (int l = 0; l < config.num_local_refactoring_level; l++) { - SIZE last_level_size = 1, curr_level_size = 1; - - // std::cout << coarse_shape[0] << " " << coarse_shape[1] << " " - // << coarse_shape[2] << "\n"; - for (DIM d = 0; d < D; d++) { - coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 8; - last_level_size *= coarse_shape[d]; - coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 5; - curr_level_size *= coarse_shape[d]; - } + this->L = config.num_local_refactoring_level; + this->M = config.num_global_refactoring_level; - // std::cout << coarse_shape[0] << " " << coarse_shape[1] << " " - // << coarse_shape[2] << "\n"; - coarse_shapes.push_back(coarse_shape); - coarse_num_elems.push_back(last_level_size); - if (l == 0) { - coarse_array.resize(coarse_shape, queue_idx); - } - local_coeff_size.push_back(last_level_size - curr_level_size); - // std::cout << local_coeff_size[local_coeff_size.size() - 1] << "\n"; - } + // Adaptive intialization for local and global + if (this->L == 0 && this->M == 0) { + throw ProcessingException("Both L and M cannot be zero"); + } + + if (this->L > 0) { + local_refactor.Adapt(hierarchy, config, queue_idx); } - global_hierarchy = Hierarchy(coarse_shape, config); - global_refactor = DataRefactor(global_hierarchy, config); + if (this->M > 0) { + if (this->L > 0) { + // With local, global adapt from the output shape of local + std::vector global_hierarchy_shape = + local_refactor.coarse_shapes[this->L - 1]; + Config global_config; + global_config.max_larget_level = this->M; + this->global_hierarchy = + Hierarchy(global_hierarchy_shape, global_config); + global_refactor.Adapt(this->global_hierarchy, global_config, queue_idx); + } else { + // Without local, global directly adapt to original shape + Config global_config; + global_config.max_larget_level = this->M; + + this->global_hierarchy = Hierarchy( + hierarchy.level_shape(hierarchy.l_target()), global_config); + global_refactor.Adapt(global_hierarchy, global_config, queue_idx); + } + } } - static size_t EstimateMemoryFootprint(std::vector shape) { + static size_t EstimateMemoryFootprint(std::vector shape, + Config config) { size_t size = 0; + + SIZE L = config.num_local_refactoring_level; + SIZE M = config.num_global_refactoring_level; + + if (L > 0) { + size += BlockLocalHierarchyDataRefactor< + D, T, DeviceType>::EstimateMemoryFootprint(shape); + if (M > 0) { + // Calculate Coarest shape from local + std::vector coarest_shape = shape; + for (int l = 0; l < config.num_local_refactoring_level; l++) { + for (DIM d = 0; d < D; d++) { + coarest_shape[d] = ((coarest_shape[d] - 1) / 8 + 1) * 5; + } + } + size += DataRefactor::EstimateMemoryFootprint( + coarest_shape); + } + } else { + size += DataRefactor::EstimateMemoryFootprint(shape); + } return size; } size_t DecomposedDataSize() { - size_t coeff_size = 0; - // local - for (int l = 0; l < config.num_local_refactoring_level; l++) { - coeff_size += local_coeff_size[l]; + if (this->L > 0) { + return local_refactor.DecomposedDataSize(); } - // global - coeff_size += global_hierarchy.total_num_elems(); - return coeff_size; + + return hierarchy->total_num_elems(); } - void Decompose(SubArray data, - SubArray<1, T, DeviceType> decomposed_data, int queue_idx) { - - // PrintSubarray("data", data); - - if (config.num_local_refactoring_level > 0) { - Timer timer; - SubArray coarse_data(coarse_array); - SIZE accumulated_local_coeff_size = 0; - for (int l = 0; l < config.num_local_refactoring_level; l++) { - if (log::level & log::TIME) - timer.start(); - accumulated_local_coeff_size += local_coeff_size[l]; - SubArray<1, T, DeviceType> local_coeff( - {local_coeff_size[l]}, - decomposed_data(decomposed_data.shape(0) - - accumulated_local_coeff_size)); - // std::cout << "accumulated_local_coeff_size: " - // << accumulated_local_coeff_size << "\n"; - in_cache_block::decompose(data, coarse_data, - local_coeff, queue_idx); - - // DeviceRuntime::SyncQueue(queue_idx); - // PrintSubarray("local_coeff_subarray", local_coeff_subarray); - // PrintSubarray("coarse_subarray", coarse_subarray); - SubArray tmp = coarse_data; - if (l + 1 < config.num_local_refactoring_level) { - coarse_data = - SubArray(coarse_shapes[l + 1], data.data()); - } - data = tmp; - if (log::level & log::TIME) { - DeviceRuntime::SyncQueue(queue_idx); - timer.end(); - timer.print("Local Decomposition"); - log::time("Decomposition throughput: " + - std::to_string((double)(coarse_num_elems[l] * sizeof(T)) / - timer.get() / 1e9) + - " GB/s"); - timer.clear(); - } + // Global-stage decomposition over the coarsest region at the front of + // decomposed_data (in-place). Factored out so the fused + // decompose+quantize path can run it separately from the local stage. + void DecomposeGlobal(SubArray<1, T, DeviceType> decomposed_data, + int queue_idx, bool orthogonal_projection = true) { + std::vector global_shape = + (this->L > 0) ? local_refactor.coarse_shapes[this->L - 1] + : hierarchy->level_shape(hierarchy->l_target()); + SubArray global_input_data(global_shape, + decomposed_data.data()); + for (DIM d = 0; d < D; d++) { + global_input_data.setLd(d, global_shape[d]); + } + global_input_data.project(D - 3, D - 2, D - 1); - // bool check = true; - // VerifySubArray("coarse", coarse_data, !check, check); - } + global_refactor.Decompose(global_input_data, orthogonal_projection, + queue_idx); + } + + // Global-stage recomposition over the coarsest region at the front of + // decomposed_data (in-place). Factored out so the fused + // dequantize+recompose path can run it separately from the local stage. + void RecomposeGlobal(SubArray<1, T, DeviceType> decomposed_data, + int queue_idx, bool orthogonal_projection = true) { + std::vector global_shape = + (this->L > 0) ? local_refactor.coarse_shapes[this->L - 1] + : hierarchy->level_shape(hierarchy->l_target()); + SubArray global_input_data(global_shape, + decomposed_data.data()); + for (DIM d = 0; d < D; d++) { + global_input_data.setLd(d, global_shape[d]); } + global_input_data.project(D - 3, D - 2, D - 1); + + global_refactor.Recompose(global_input_data, orthogonal_projection, + queue_idx); + } - // DeviceRuntime::SyncQueue(queue_idx); - // PrintSubarray("before data", data); + // Need revise further to exclude copy time + void Decompose(SubArray data, + SubArray<1, T, DeviceType> decomposed_data, int queue_idx, + bool orthogonal_projection = true) { + if (this->L == 0 && this->M == 0) { + throw ProcessingException("Both L and M cannot be zero"); + } + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + if (this->L == 0) { + // Pure Global (In-Place) + std::vector original_shape = + hierarchy->level_shape(hierarchy->l_target()); + SubArray global_input_data(original_shape, + decomposed_data.data()); + for (DIM d = 0; d < D; d++) { + global_input_data.setLd(d, original_shape[d]); + } + global_input_data.project(D - 3, D - 2, D - 1); - // PrintSubarray("coarse_data", SubArray<1, T, DeviceType>({10}, - // data.data())); + multi_dimension::CopyND(data, global_input_data, queue_idx); - // Array global_data(coarse_shape, coarse_array.data()); - SubArray global_coeff_subarray( - {global_hierarchy.level_shape(global_hierarchy.l_target())}, - decomposed_data((IDX)0)); - global_refactor.Decompose(data, queue_idx); + global_refactor.Decompose(global_input_data, orthogonal_projection, + queue_idx); + } else if (this->M == 0) { + // Pure Local + local_refactor.Decompose(data, decomposed_data, queue_idx, + orthogonal_projection); + } else { + // Local decomposition + local_refactor.Decompose(data, decomposed_data, queue_idx, + orthogonal_projection); - // DeviceRuntime::SyncQueue(queue_idx); - // PrintSubarray("after data", data); + // Global decomposition + DecomposeGlobal(decomposed_data, queue_idx, orthogonal_projection); + } - multi_dimension::CopyND(data, global_coeff_subarray, queue_idx); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Hybrid Decomposition", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } } + + // Need revise further to exclude copy time void Recompose(SubArray data, - SubArray<1, T, DeviceType> decomposed_data, int queue_idx) { + SubArray<1, T, DeviceType> decomposed_data, int queue_idx, + bool orthogonal_projection = true) { + if (this->L == 0 && this->M == 0) { + throw ProcessingException("Both L and M cannot be zero"); + } Timer timer; - if (log::level & log::TIME) + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); timer.start(); - SubArray data_subarray(data); - SubArray w_subarray(coarse_array); - SubArray<1, T, DeviceType> decomposed_data_subarray(decomposed_data); + } + if (this->L == 0) { + // Pure Global (In-Place) + std::vector original_shape = + hierarchy->level_shape(hierarchy->l_target()); + SubArray global_input_data(original_shape, + decomposed_data.data()); + for (DIM d = 0; d < D; d++) { + global_input_data.setLd(d, original_shape[d]); + } + global_input_data.project(D - 3, D - 2, D - 1); - in_cache_block::recompose( - data_subarray, w_subarray, decomposed_data_subarray, queue_idx); + global_refactor.Recompose(global_input_data, orthogonal_projection, + queue_idx); + + // Copy back to data + multi_dimension::CopyND(global_input_data, data, queue_idx); + } else if (this->M == 0) { + // Pure Local + local_refactor.Recompose(data, decomposed_data, queue_idx, + orthogonal_projection); + } else { + std::vector local_coarest_shape = + local_refactor.coarse_shapes[this->L - 1]; + SubArray global_input_data({local_coarest_shape}, + decomposed_data.data()); + for (DIM d = 0; d < D; d++) { + global_input_data.setLd(d, local_coarest_shape[d]); + } + global_input_data.project(D - 3, D - 2, D - 1); + + // Global recomposition + global_refactor.Recompose(global_input_data, orthogonal_projection, + queue_idx); + + // Local recomposition + local_refactor.Recompose(data, decomposed_data, queue_idx, + orthogonal_projection); + } if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); timer.end(); - timer.print("Recomposition"); - log::time( - "Recomposition throughput: " + - std::to_string((double)(hierarchy->total_num_elems() * sizeof(T)) / - timer.get() / 1e9) + - " GB/s"); + timer.print("Hybrid Recomposition", + hierarchy->total_num_elems() * sizeof(T)); timer.clear(); } } @@ -208,16 +256,16 @@ class HybridHierarchyDataRefactor Hierarchy *hierarchy; Hierarchy global_hierarchy; Config config; - std::vector coarse_shape; - std::vector coarse_num_elems; + + SIZE L; // Number of local levels + SIZE M; // Number of global levels + + BlockLocalHierarchyDataRefactor local_refactor; DataRefactor global_refactor; - Array coarse_array; - std::vector> coarse_shapes; - std::vector local_coeff_size; }; } // namespace data_refactoring } // namespace mgard_x -#endif \ No newline at end of file +#endif diff --git a/include/mgard-x/DataRefactoring/HybridHierarchyDataRefactorInterface.hpp b/include/mgard-x/DataRefactoring/HybridHierarchyDataRefactorInterface.hpp index f8d28888b0..6155a3597b 100644 --- a/include/mgard-x/DataRefactoring/HybridHierarchyDataRefactorInterface.hpp +++ b/include/mgard-x/DataRefactoring/HybridHierarchyDataRefactorInterface.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_HYBRID_HIERARCHY_DATA_REFACTOR_INTERFACE_HPP @@ -15,14 +15,14 @@ template class HybridHierarchyDataRefactorInterface { virtual void Decompose(SubArray data, SubArray<1, T, DeviceType> decomposed_data, - int queue_idx) = 0; + int queue_idx, bool orthogonal_projection = true) = 0; virtual void Recompose(SubArray data, SubArray<1, T, DeviceType> decomposed_data, - int queue_idx) = 0; + int queue_idx, bool orthogonal_projection = true) = 0; }; } // namespace data_refactoring } // namespace mgard_x -#endif \ No newline at end of file +#endif diff --git a/include/mgard-x/DataRefactoring/InCacheBlock/Autocorrelation8x8x8.hpp b/include/mgard-x/DataRefactoring/InCacheBlock/Autocorrelation8x8x8.hpp index 3f0d399137..a7049fb7f2 100644 --- a/include/mgard-x/DataRefactoring/InCacheBlock/Autocorrelation8x8x8.hpp +++ b/include/mgard-x/DataRefactoring/InCacheBlock/Autocorrelation8x8x8.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2023, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: Jan. 15, 2023 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_AUTOCORRELATION_8x8x8_KERNEL_TEMPLATE diff --git a/include/mgard-x/DataRefactoring/InCacheBlock/DataRefactoring.h b/include/mgard-x/DataRefactoring/InCacheBlock/DataRefactoring.h index 5687d12073..b8e52def64 100644 --- a/include/mgard-x/DataRefactoring/InCacheBlock/DataRefactoring.h +++ b/include/mgard-x/DataRefactoring/InCacheBlock/DataRefactoring.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_IN_CACHE_DATA_REFACTORING @@ -18,13 +18,49 @@ namespace data_refactoring { namespace in_cache_block { +// Block-local (in-cache) refactoring for D = 1, 2 and 3. Every dimension of a +// block holds 8 fine nodes that coarsen to 5, so one block emits 5^D coarse +// values and 8^D - 5^D coefficients. Higher D is a no-op: the hybrid +// hierarchy has no block-local stage there. + template void decompose(SubArray v, SubArray coarse, - SubArray<1, T, DeviceType> coeff, int queue_idx); + SubArray<1, T, DeviceType> coeff, bool orthogonal_projection, + int queue_idx); + +// Fused decompose+quantize: same block decomposition as decompose(), but the +// coefficients are quantized in-kernel and written as Q symbols. Reads v with +// bounds checks, so v does not need to be padded to a multiple of 8. Uses the +// per-block quantizers (indexed by the block's row-major index) when +// use_block_quantizers is set (ROI mode), the scalar quantizer otherwise. +template +void decompose_quantize(SubArray v, + SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, + bool use_block_quantizers, bool prep_huffman, + SIZE dict_size, bool orthogonal_projection, + int queue_idx); template void recompose(SubArray v, SubArray coarse, - SubArray<1, T, DeviceType> coeff, int queue_idx); + SubArray<1, T, DeviceType> coeff, bool orthogonal_projection, + int queue_idx); + +// Fused dequantize+recompose: same block recomposition as recompose(), but the +// coefficients are read as Q symbols and dequantized in-kernel. Writes v with +// bounds checks, so v does not need to be padded to a multiple of 8. Uses the +// per-block quantizers (indexed by the block's row-major index) when +// use_block_quantizers is set (ROI mode), the scalar quantizer otherwise. +template +void recompose_dequantize(SubArray v, + SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, + T quantizer, + SubArray<1, T, DeviceType> block_quantizers, + bool use_block_quantizers, bool prep_huffman, + SIZE dict_size, bool orthogonal_projection, + int queue_idx); } // namespace in_cache_block @@ -32,4 +68,4 @@ void recompose(SubArray v, SubArray coarse, } // namespace mgard_x -#endif \ No newline at end of file +#endif diff --git a/include/mgard-x/DataRefactoring/InCacheBlock/DataRefactoring.hpp b/include/mgard-x/DataRefactoring/InCacheBlock/DataRefactoring.hpp index 5fb910df97..1018b51f3e 100644 --- a/include/mgard-x/DataRefactoring/InCacheBlock/DataRefactoring.hpp +++ b/include/mgard-x/DataRefactoring/InCacheBlock/DataRefactoring.hpp @@ -1,16 +1,22 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ +#include + #include "../../Hierarchy/Hierarchy.h" #include "../../RuntimeX/RuntimeX.h" - #include "Autocorrelation8x8x8.hpp" #include "DataRefactoring.h" -#include "MultiDimension8x8x8.hpp" +#include "Decompose8.hpp" +#include "Decompose8x8.hpp" +#include "Decompose8x8x8.hpp" +#include "Recompose8.hpp" +#include "Recompose8x8.hpp" +#include "Recompose8x8x8.hpp" #include @@ -25,39 +31,113 @@ namespace in_cache_block { template void decompose(SubArray v, SubArray coarse, - SubArray<1, T, DeviceType> coeff, int queue_idx) { - if constexpr (D <= 3) { + SubArray<1, T, DeviceType> coeff, bool orthogonal_projection, + int queue_idx) { + // One kernel per dimensionality: the block geometry (8 fine nodes to 5 + // coarse ones per dimension) is shared, but the coefficient layout and the + // number of transform passes are not. + if constexpr (D == 1) { DeviceLauncher::Execute( - MultiDimension8x8x8Kernel(v, coarse, - coeff), + Decompose8Kernel(v, coarse, coeff, + orthogonal_projection), queue_idx); + } else if constexpr (D == 2) { + DeviceLauncher::Execute( + Decompose8x8Kernel(v, coarse, coeff, + orthogonal_projection), + queue_idx); + } else if constexpr (D == 3) { + DeviceLauncher::Execute( + Decompose8x8x8Kernel(v, coarse, coeff, + orthogonal_projection), + queue_idx); + } +} - // Array ac_x({(v.shape(0)-1)/8+1, (v.shape(1)-1)/8+1, - // (v.shape(2)-1)/8+1}, false, false); Array - // ac_y({(v.shape(0)-1)/8+1, (v.shape(1)-1)/8+1, (v.shape(2)-1)/8+1}, false, - // false); Array ac_z({(v.shape(0)-1)/8+1, - // (v.shape(1)-1)/8+1, (v.shape(2)-1)/8+1}, false, false); - - // DeviceLauncher::Execute( - // Autocorrelation8x8x8Kernel(v, - // SubArray(ac_x), SubArray(ac_y), - // SubArray(ac_z), - // 1), - // queue_idx); - - // PrintSubarray("ac_x", SubArray<2, T, DeviceType>({ac_x.shape(0), - // ac_x.shape(1)}, ac_x.data())); PrintSubarray("ac_y", SubArray<2, T, - // DeviceType>({ac_x.shape(0), ac_y.shape(1)}, ac_y.data())); - // PrintSubarray("ac_z", SubArray<2, T, DeviceType>({ac_z.shape(0), - // ac_z.shape(1)}, ac_z.data())); +template +void decompose_quantize(SubArray v, + SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, + bool use_block_quantizers, bool prep_huffman, + SIZE dict_size, bool orthogonal_projection, + int queue_idx) { + if constexpr (D == 1) { + DeviceLauncher::Execute( + DecomposeQuantize8Kernel( + v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection), + queue_idx); + } else if constexpr (D == 2) { + DeviceLauncher::Execute( + DecomposeQuantize8x8Kernel( + v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection), + queue_idx); + } else if constexpr (D == 3) { + DeviceLauncher::Execute( + DecomposeQuantize8x8x8Kernel( + v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection), + queue_idx); } } template void recompose(SubArray v, SubArray coarse, - SubArray<1, T, DeviceType> coeff, int queue_idx) { + SubArray<1, T, DeviceType> coeff, bool orthogonal_projection, + int queue_idx) { + if constexpr (D == 1) { + DeviceLauncher::Execute( + Recompose8Kernel(v, coarse, coeff, + orthogonal_projection), + queue_idx); + } else if constexpr (D == 2) { + DeviceLauncher::Execute( + Recompose8x8Kernel(v, coarse, coeff, + orthogonal_projection), + queue_idx); + } else if constexpr (D == 3) { + DeviceLauncher::Execute( + Recompose8x8x8Kernel(v, coarse, coeff, + orthogonal_projection), + queue_idx); + } +} - if constexpr (D <= 3) { +template +void recompose_dequantize(SubArray v, + SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, + T quantizer, + SubArray<1, T, DeviceType> block_quantizers, + bool use_block_quantizers, bool prep_huffman, + SIZE dict_size, bool orthogonal_projection, + int queue_idx) { + if constexpr (D == 1) { + DeviceLauncher::Execute( + RecomposeDequantize8Kernel( + v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection), + queue_idx); + } else if constexpr (D == 2) { + DeviceLauncher::Execute( + RecomposeDequantize8x8Kernel( + v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection), + queue_idx); + } else if constexpr (D == 3) { + DeviceLauncher::Execute( + RecomposeDequantize8x8x8Kernel( + v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection), + queue_idx); } } @@ -67,4 +147,4 @@ void recompose(SubArray v, SubArray coarse, } // namespace mgard_x -#endif \ No newline at end of file +#endif diff --git a/include/mgard-x/DataRefactoring/InCacheBlock/Decompose8.hpp b/include/mgard-x/DataRefactoring/InCacheBlock/Decompose8.hpp new file mode 100644 index 0000000000..aaca800324 --- /dev/null +++ b/include/mgard-x/DataRefactoring/InCacheBlock/Decompose8.hpp @@ -0,0 +1,331 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_DECOMPOSE_8_KERNEL_TEMPLATE +#define MGARD_X_DECOMPOSE_8_KERNEL_TEMPLATE + +#include "../../RuntimeX/RuntimeX.h" + +#include "IndexTableLowDim.hpp" + +namespace mgard_x { + +namespace data_refactoring { + +namespace in_cache_block { + +/* +1D counterpart of Decompose8x8x8Functor: + +v x c total +8 5(cor) 8-5(3) 13 per tile + +A single 8-element tile is far too little work for a thread block, so each +block owns TILES tiles side by side: threadIdx.y selects the tile and +threadIdx.x the element within it. Every shared-memory table entry is a +within-tile offset, to which the kernel adds the tile's base. + +The 3 coefficients per tile are written to coeff[tile * 3 ...] and the 5 +coarse values to coarse[tile * 5 ...], where `tile` is the tile's index in +row-major order over the whole array -- the same convention the 3D block uses +for its linearized thread-block id. +*/ +template +class Decompose8Functor : public Functor { +public: + MGARDX_CONT Decompose8Functor() {} + MGARDX_CONT Decompose8Functor(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) { + Functor(); + } + + MGARDX_EXEC void initialize_sm_8() { + sm_v = (T *)FunctorBase::GetSharedMemory(); + sm_x = sm_v + TILES * SMV_STRIDE_8; + } + + MGARDX_EXEC void init_indices() { + item = FunctorBase::GetThreadIdX(); + tile = FunctorBase::GetThreadIdY(); + bid = FunctorBase::GetBlockIdX() * TILES + tile; + num_tiles = (v.shape(D - 1) + X - 1) / X; + active = bid < (int)num_tiles; + base_v = tile * SMV_STRIDE_8; + base_x = tile * SMX_STRIDE_8; + x_gl = bid * X + item; + if (item == 0) + sm_v[base_v + SMV_ZERO_8] = (T)0; + } + + // Load + MGARDX_EXEC void Operation1() { + initialize_sm_8(); + init_indices(); + if (active) { + sm_v[base_v + item] = *v(x_gl); + } + } + + // Interpolation + MGARDX_EXEC void Operation2() { + if (active && item < NumCoeff_8) { + int const *index = Coeff1D_Offset_8(item); + T middle = sm_v[base_v + index[0]]; + T left = sm_v[base_v + index[1]]; + T right = sm_v[base_v + index[2]]; + sm_v[base_v + index[0]] = middle - (left + right) * (T)0.5; + } + } + + // MassTransX + MGARDX_EXEC void Operation3() { + if (!orthogonal_projection) + return; + if (active && item < NumMassTransX_8) { + int const *index = MassTrans_X_Offset_8(item); + T a = sm_v[base_v + index[0]]; + T b = sm_v[base_v + index[1]]; + T c = sm_v[base_v + index[2]]; + T d = sm_v[base_v + index[3]]; + T e = sm_v[base_v + index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_x[base_x + index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + } + + MGARDX_EXEC void solve_tridiag() { + T a = sm_x[base_x + 0]; + T b = sm_x[base_x + 1]; + T c = sm_x[base_x + 2]; + T d = sm_x[base_x + 3]; + T e = sm_x[base_x + 4]; + + a += 0 * amxbm_8x8x8(0); + b += a * amxbm_8x8x8(1); + c += b * amxbm_8x8x8(2); + d += c * amxbm_8x8x8(3); + e += d * amxbm_8x8x8(4); + + e = (e + am_8x8x8(5) * 0) * bm_8x8x8(5); + d = (d + am_8x8x8(4) * e) * bm_8x8x8(4); + c = (c + am_8x8x8(3) * d) * bm_8x8x8(3); + b = (b + am_8x8x8(2) * c) * bm_8x8x8(2); + a = (a + am_8x8x8(1) * b) * bm_8x8x8(1); + + sm_x[base_x + 0] = a; + sm_x[base_x + 1] = b; + sm_x[base_x + 2] = c; + sm_x[base_x + 3] = d; + sm_x[base_x + 4] = e; + } + + // TridiagX + MGARDX_EXEC void Operation4() { + if (!orthogonal_projection) + return; + if (active && item == 0) { + solve_tridiag(); + } + } + + // Apply correction and write out + MGARDX_EXEC void Operation5() { + if (!active) + return; + if (item < NumCoarse_8) { + if (orthogonal_projection) + sm_v[base_v + Coarse_Offset_8(item)] += sm_x[base_x + item]; + *coarse(bid * LowDim_Coarse + item) = + sm_v[base_v + Coarse_Offset_8(item)]; + } else { + int op_tid = item - NumCoarse_8; + *coeff(bid * NumCoeff_8 + op_tid) = sm_v[base_v + Coeff_Offset_8(op_tid)]; + } + } + + MGARDX_CONT size_t shared_memory_size() { + return (size_t)TILES * (SMV_STRIDE_8 + SMX_STRIDE_8) * sizeof(T); + } + +protected: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; + T *sm_v, *sm_x; + int item, tile, bid, x_gl; + int base_v, base_x; + SIZE num_tiles; + bool active; +}; + +// Fused decompose+quantize variant. See DecomposeQuantize8x8x8Functor: the +// input is read with bounds checks so unpadded inputs can be consumed +// directly, and the 3 per-tile coefficients are quantized on write-out. +template +class DecomposeQuantize8Functor + : public Decompose8Functor { + using Base = Decompose8Functor; + +public: + MGARDX_CONT DecomposeQuantize8Functor() {} + MGARDX_CONT DecomposeQuantize8Functor( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : Base(v, coarse, SubArray<1, T, DeviceType>(), orthogonal_projection), + quantized_coeff(quantized_coeff), quantizer(quantizer), + block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size) {} + + MGARDX_EXEC void Operation1() { + this->initialize_sm_8(); + this->init_indices(); + if (this->active) { + this->sm_v[this->base_v + this->item] = (T)0; + if (this->x_gl < (int)this->v.shape(D - 1)) { + this->sm_v[this->base_v + this->item] = *this->v(this->x_gl); + } + } + } + + MGARDX_EXEC void Operation5() { + if (!this->active) + return; + if (this->item < NumCoarse_8) { + if (this->orthogonal_projection) + this->sm_v[this->base_v + Coarse_Offset_8(this->item)] += + this->sm_x[this->base_x + this->item]; + *this->coarse(this->bid * LowDim_Coarse + this->item) = + this->sm_v[this->base_v + Coarse_Offset_8(this->item)]; + } else { + int op_tid = this->item - NumCoarse_8; + T t = this->sm_v[this->base_v + Coeff_Offset_8(op_tid)]; + T q = use_block_quantizers ? *block_quantizers(this->bid) : quantizer; + // Must stay bit-identical to QuantizeLocalLevelFunctor (volume == 1). + Q quantized_data; + if constexpr (sizeof(T) == sizeof(double)) { + quantized_data = copysign((T)0.5 + fabs(t * q), t); + } else if constexpr (sizeof(T) == sizeof(float)) { + quantized_data = copysign((T)0.5 + fabsf(t * q), t); + } + if (prep_huffman) { + quantized_data += dict_size / 2; + } + *quantized_coeff(this->bid * NumCoeff_8 + op_tid) = quantized_data; + } + } + +protected: + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; +}; + +template +class Decompose8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_1d"; + MGARDX_CONT + Decompose8Kernel(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Decompose8Functor; + FunctorType functor(v, coarse, coeff, orthogonal_projection); + + SIZE num_tiles = (v.shape(D - 1) + 7) / 8; + size_t sm_size = functor.shared_memory_size(); + SIZE tbz = 1, tby = LowDim_Tiles_1D, tbx = 8; + SIZE gridz = 1, gridy = 1; + SIZE gridx = ceil((double)num_tiles / LowDim_Tiles_1D); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; +}; + +template +class DecomposeQuantize8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_1d_fq"; + MGARDX_CONT + DecomposeQuantize8Kernel(SubArray v, + SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, + T quantizer, + SubArray<1, T, DeviceType> block_quantizers, + bool use_block_quantizers, bool prep_huffman, + SIZE dict_size, bool orthogonal_projection = true) + : v(v), coarse(coarse), quantized_coeff(quantized_coeff), + quantizer(quantizer), block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size), orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT + Task> + GenTask(int queue_idx) { + using FunctorType = + DecomposeQuantize8Functor; + FunctorType functor(v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection); + + // Same launch geometry as Decompose8Kernel; v may be unpadded here but + // ceil(shape / 8) matches the padded tile count exactly. + SIZE num_tiles = (v.shape(D - 1) + 7) / 8; + size_t sm_size = functor.shared_memory_size(); + SIZE tbz = 1, tby = LowDim_Tiles_1D, tbx = 8; + SIZE gridz = 1, gridy = 1; + SIZE gridx = ceil((double)num_tiles / LowDim_Tiles_1D); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; + bool orthogonal_projection; +}; + +} // namespace in_cache_block + +} // namespace data_refactoring + +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/DataRefactoring/InCacheBlock/Decompose8x8.hpp b/include/mgard-x/DataRefactoring/InCacheBlock/Decompose8x8.hpp new file mode 100644 index 0000000000..726dbb0be5 --- /dev/null +++ b/include/mgard-x/DataRefactoring/InCacheBlock/Decompose8x8.hpp @@ -0,0 +1,368 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_DECOMPOSE_8x8_KERNEL_TEMPLATE +#define MGARD_X_DECOMPOSE_8x8_KERNEL_TEMPLATE + +#include "../../RuntimeX/RuntimeX.h" + +#include "IndexTableLowDim.hpp" + +namespace mgard_x { + +namespace data_refactoring { + +namespace in_cache_block { + +/* +2D counterpart of Decompose8x8x8Functor. One thread block owns one 8x8 tile +and 64 threads; the stages are the same as the 3D block's, minus the z pass: + +v x y c total +8*8(64) 5*8(40) 5*5(25) 0 129 + 8*8-5*5(39) + +The 39 coefficients per tile are written to coeff[bid * 39 ...] in the +row-major non-coarse order of Coeff_Offset_8x8, and the 25 coarse values to +coarse[y_tb * 5 + jy][x_tb * 5 + jx]. +*/ +template +class Decompose8x8Functor : public Functor { +public: + MGARDX_CONT Decompose8x8Functor() {} + MGARDX_CONT Decompose8x8Functor(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) { + Functor(); + } + + MGARDX_EXEC void initialize_sm_8x8() { + sm_v = (T *)FunctorBase::GetSharedMemory(); + sm_x = sm_v + SMV_SIZE_8x8; // sm_v is padded for bank-conflict avoidance + sm_y = sm_x + SMX_SIZE_8x8; + } + + MGARDX_EXEC void init_indices() { + x = FunctorBase::GetThreadIdX(); + y = FunctorBase::GetThreadIdY(); + x_tb = FunctorBase::GetBlockIdX(); + y_tb = FunctorBase::GetBlockIdY(); + x_gl = X * x_tb + x; + y_gl = Y * y_tb + y; + tid = y * X + x; + bid = y_tb * FunctorBase::GetGridDimX() + x_tb; + if (tid == 0) + sm_v[ZERO_V_8x8] = (T)0; + offset = offset8x8(y, x); + } + + // Load + MGARDX_EXEC void Operation1() { + initialize_sm_8x8(); + init_indices(); + sm_v[offset] = *v(y_gl, x_gl); + } + + // Interpolation. The 1D and 2D coefficient updates are independent: a 1D + // update writes a node with exactly one odd coordinate, while the 2D update + // reads only nodes with two even coordinates. + MGARDX_EXEC void Operation2() { + if (tid < NumCoeff1D_8x8) { + int const *index = Coeff1D_Offset_8x8(tid); + T middle = sm_v[index[0]]; + T left = sm_v[index[1]]; + T right = sm_v[index[2]]; + sm_v[index[0]] = middle - (left + right) * (T)0.5; + } else if (tid >= 32 && tid < 32 + NumCoeff2D_8x8) { + int const *index = Coeff2D_Offset_8x8(tid - 32); + T c11 = sm_v[index[0]]; + T c00 = sm_v[index[1]]; + T c02 = sm_v[index[2]]; + T c20 = sm_v[index[3]]; + T c22 = sm_v[index[4]]; + sm_v[index[0]] = c11 - (c00 + c02 + c20 + c22) / 4; + } + } + + // MassTransX + MGARDX_EXEC void Operation3() { + if (!orthogonal_projection) + return; + if (tid < NumMassTransX_8x8) { + int const *index = MassTrans_X_Offset_8x8(tid); + T a = sm_v[index[0]]; + T b = sm_v[index[1]]; + T c = sm_v[index[2]]; + T d = sm_v[index[3]]; + T e = sm_v[index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_x[index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + } + + // MassTransY + MGARDX_EXEC void Operation4() { + if (!orthogonal_projection) + return; + if (tid < NumMassTransY_8x8) { + int const *index = MassTrans_Y_Offset_8x8(tid); + T a = sm_x[index[0]]; + T b = sm_x[index[1]]; + T c = sm_x[index[2]]; + T d = sm_x[index[3]]; + T e = sm_x[index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_y[index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + } + + MGARDX_EXEC void solve_tridiag(int const *index) { + T a = sm_y[index[0]]; + T b = sm_y[index[1]]; + T c = sm_y[index[2]]; + T d = sm_y[index[3]]; + T e = sm_y[index[4]]; + + a += 0 * amxbm_8x8x8(0); + b += a * amxbm_8x8x8(1); + c += b * amxbm_8x8x8(2); + d += c * amxbm_8x8x8(3); + e += d * amxbm_8x8x8(4); + + e = (e + am_8x8x8(5) * 0) * bm_8x8x8(5); + d = (d + am_8x8x8(4) * e) * bm_8x8x8(4); + c = (c + am_8x8x8(3) * d) * bm_8x8x8(3); + b = (b + am_8x8x8(2) * c) * bm_8x8x8(2); + a = (a + am_8x8x8(1) * b) * bm_8x8x8(1); + + sm_y[index[0]] = a; + sm_y[index[1]] = b; + sm_y[index[2]] = c; + sm_y[index[3]] = d; + sm_y[index[4]] = e; + } + + // TridiagX + MGARDX_EXEC void Operation5() { + if (!orthogonal_projection) + return; + if (tid < LowDim_Coarse) { + solve_tridiag(TriDiag_X_Offset_8x8(tid)); + } + } + + // TridiagY + MGARDX_EXEC void Operation6() { + if (!orthogonal_projection) + return; + if (tid < LowDim_Coarse) { + solve_tridiag(TriDiag_Y_Offset_8x8(tid)); + } + } + + // Apply correction and write out + MGARDX_EXEC void Operation7() { + if (tid < NumCoarse_8x8) { + if (orthogonal_projection) + sm_v[Coarse_Offset_8x8(tid)] += sm_y[tid]; + *coarse(y_tb * LowDim_Coarse + tid / LowDim_Coarse, + x_tb * LowDim_Coarse + tid % LowDim_Coarse) = + sm_v[Coarse_Offset_8x8(tid)]; + } else { + int op_tid = tid - NumCoarse_8x8; + *coeff(bid * NumCoeff_8x8 + op_tid) = sm_v[Coeff_Offset_8x8(op_tid)]; + } + } + + MGARDX_CONT size_t shared_memory_size() { + return (size_t)SM_SIZE_8x8 * sizeof(T); + } + +protected: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; + T *sm_v, *sm_x, *sm_y; + int y, x, y_tb, x_tb, y_gl, x_gl; + int tid, bid; + int offset; +}; + +// Fused decompose+quantize variant: identical transform pipeline, but +// (1) the input is read with bounds checks so unpadded inputs can be consumed +// directly (out-of-range positions contribute zeros, matching the +// memset+copy padding of the unfused path bit for bit), and +// (2) the 39 per-tile coefficients are quantized on write-out while still in +// shared memory, emitting Q symbols (with the Huffman dictionary shift +// folded in, mirroring QuantizeLocalLevelFunctor) instead of T values. +// The 5x5 coarse output stays in T: it is the next level's input. +template +class DecomposeQuantize8x8Functor + : public Decompose8x8Functor { + using Base = Decompose8x8Functor; + +public: + MGARDX_CONT DecomposeQuantize8x8Functor() {} + MGARDX_CONT DecomposeQuantize8x8Functor( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : Base(v, coarse, SubArray<1, T, DeviceType>(), orthogonal_projection), + quantized_coeff(quantized_coeff), quantizer(quantizer), + block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size) {} + + MGARDX_EXEC void Operation1() { + this->initialize_sm_8x8(); + this->init_indices(); + this->sm_v[this->offset] = (T)0; + // Unlike the unfused functor, keep the bounds check: the input here is + // not pre-padded to a multiple of 8, so edge tiles must zero-fill. + if (this->y_gl < (int)this->v.shape(D - 2) && + this->x_gl < (int)this->v.shape(D - 1)) { + this->sm_v[this->offset] = *this->v(this->y_gl, this->x_gl); + } + } + + MGARDX_EXEC void Operation7() { + if (this->tid < NumCoarse_8x8) { + if (this->orthogonal_projection) + this->sm_v[Coarse_Offset_8x8(this->tid)] += this->sm_y[this->tid]; + *this->coarse(this->y_tb * LowDim_Coarse + this->tid / LowDim_Coarse, + this->x_tb * LowDim_Coarse + this->tid % LowDim_Coarse) = + this->sm_v[Coarse_Offset_8x8(this->tid)]; + } else { + int op_tid = this->tid - NumCoarse_8x8; + T t = this->sm_v[Coeff_Offset_8x8(op_tid)]; + T q = use_block_quantizers ? *block_quantizers(this->bid) : quantizer; + // Must stay bit-identical to QuantizeLocalLevelFunctor (volume == 1). + Q quantized_data; + if constexpr (sizeof(T) == sizeof(double)) { + quantized_data = copysign((T)0.5 + fabs(t * q), t); + } else if constexpr (sizeof(T) == sizeof(float)) { + quantized_data = copysign((T)0.5 + fabsf(t * q), t); + } + if (prep_huffman) { + quantized_data += dict_size / 2; + } + *quantized_coeff(this->bid * NumCoeff_8x8 + op_tid) = quantized_data; + } + } + +protected: + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; +}; + +template +class Decompose8x8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_2d"; + MGARDX_CONT + Decompose8x8Kernel(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Decompose8x8Functor; + FunctorType functor(v, coarse, coeff, orthogonal_projection); + + SIZE total_thread_y = v.shape(D - 2); + SIZE total_thread_x = v.shape(D - 1); + + size_t sm_size = functor.shared_memory_size(); + SIZE tbz = 1, tby = 8, tbx = 8; + SIZE gridz = 1; + SIZE gridy = ceil((double)total_thread_y / tby); + SIZE gridx = ceil((double)total_thread_x / tbx); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; +}; + +template +class DecomposeQuantize8x8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_2d_fq"; + MGARDX_CONT + DecomposeQuantize8x8Kernel(SubArray v, + SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, + T quantizer, + SubArray<1, T, DeviceType> block_quantizers, + bool use_block_quantizers, bool prep_huffman, + SIZE dict_size, bool orthogonal_projection = true) + : v(v), coarse(coarse), quantized_coeff(quantized_coeff), + quantizer(quantizer), block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size), orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = DecomposeQuantize8x8Functor; + FunctorType functor(v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection); + + // Same launch geometry as Decompose8x8Kernel; v may be unpadded here but + // ceil(shape / 8) matches the padded grid exactly. + SIZE total_thread_y = v.shape(D - 2); + SIZE total_thread_x = v.shape(D - 1); + + size_t sm_size = functor.shared_memory_size(); + SIZE tbz = 1, tby = 8, tbx = 8; + SIZE gridz = 1; + SIZE gridy = ceil((double)total_thread_y / tby); + SIZE gridx = ceil((double)total_thread_x / tbx); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; + bool orthogonal_projection; +}; + +} // namespace in_cache_block + +} // namespace data_refactoring + +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/DataRefactoring/InCacheBlock/MultiDimension8x8x8.hpp b/include/mgard-x/DataRefactoring/InCacheBlock/Decompose8x8x8.hpp similarity index 61% rename from include/mgard-x/DataRefactoring/InCacheBlock/MultiDimension8x8x8.hpp rename to include/mgard-x/DataRefactoring/InCacheBlock/Decompose8x8x8.hpp index 0158bc4f60..46666ccfe2 100644 --- a/include/mgard-x/DataRefactoring/InCacheBlock/MultiDimension8x8x8.hpp +++ b/include/mgard-x/DataRefactoring/InCacheBlock/Decompose8x8x8.hpp @@ -1,12 +1,12 @@ /* - * Copyright 2023, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: Jan. 15, 2023 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ -#ifndef MGARD_X_MULTI_DIMENSION_8x8x8_KERNEL_TEMPLATE -#define MGARD_X_MULTI_DIMENSION_8x8x8_KERNEL_TEMPLATE +#ifndef MGARD_X_DECOMPOSE_8x8x8_KERNEL_TEMPLATE +#define MGARD_X_DECOMPOSE_8x8x8_KERNEL_TEMPLATE #include "../../RuntimeX/RuntimeX.h" @@ -18,9 +18,6 @@ #include "IndexTable5x5x5.hpp" #include "IndexTable8x8x8.hpp" -#define DECOMPOSE 0 -#define RECOMPOSE 1 - namespace mgard_x { namespace data_refactoring { @@ -40,21 +37,22 @@ c8(512) c5( 98) x( 18) y( 12) z( 8) c8(512) c5( 98) c3(19) c2( 8) */ -template -class MultiDimension8x8x8Functor : public Functor { +template +class Decompose8x8x8Functor : public Functor { public: - MGARDX_CONT MultiDimension8x8x8Functor() {} - MGARDX_CONT MultiDimension8x8x8Functor(SubArray v, - SubArray coarse, - SubArray<1, T, DeviceType> coeff) - : v(v), coarse(coarse), coeff(coeff) { + MGARDX_CONT Decompose8x8x8Functor() {} + MGARDX_CONT Decompose8x8x8Functor(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) { Functor(); } MGARDX_EXEC void initialize_sm_8x8x8() { sm_v = (T *)FunctorBase::GetSharedMemory(); - sm_x = sm_v + 8 * 8 * 8; + sm_x = sm_v + SMV_SIZE_8x8x8; // sm_v is padded for bank-conflict avoidance sm_y = sm_x + 5 * 8 * 8; sm_z = sm_y + 5 * 5 * 8; } @@ -103,13 +101,14 @@ class MultiDimension8x8x8Functor : public Functor { if (z == 0 && y == 0 && x == 0) sm_v[zero_const_offset] = (T)0; - offset = get_idx(ld1, ld2, z, y, x); + offset = offset8x8x8(z, y, x); // padded sm_v layout sm_v[offset] = 0.0; // Removing this check can speed up - if (z_gl < v.shape(D - 3) && y_gl < v.shape(D - 2) && - x_gl < v.shape(D - 1)) { - sm_v[offset] = *v(z_gl, y_gl, x_gl); - } + // if (z_gl < v.shape(D - 3) && y_gl < v.shape(D - 2) && + // x_gl < v.shape(D - 1)) { + sm_v[offset] = *v(z_gl, y_gl, x_gl); + // printf("v[%d, %d, %d] = %f\n", z_gl, y_gl, x_gl, sm_v[offset]); + // } } MGARDX_EXEC void Operation2() { @@ -152,6 +151,8 @@ class MultiDimension8x8x8Functor : public Functor { // MassTransX MGARDX_EXEC void Operation3() { + if (!orthogonal_projection) + return; // #ifdef MGARDX_COMPILE_CUDA // __syncthreads(); @@ -204,6 +205,8 @@ class MultiDimension8x8x8Functor : public Functor { // MassTransY MGARDX_EXEC void Operation4() { + if (!orthogonal_projection) + return; if (tid < 200) { int const *index = MassTrans_Y_Offset_8x8x8(tid); T a = sm_x[index[0]]; @@ -237,6 +240,8 @@ class MultiDimension8x8x8Functor : public Functor { // MassTransZ MGARDX_EXEC void Operation5() { + if (!orthogonal_projection) + return; if (tid < 125) { int const *index = MassTrans_Z_Offset_8x8x8(tid); T a = sm_y[index[0]]; @@ -270,6 +275,8 @@ class MultiDimension8x8x8Functor : public Functor { // TriadiagX MGARDX_EXEC void Operation6() { + if (!orthogonal_projection) + return; if (tid < 25) { int const *index = TriDiag_X_Offset_8x8x8(tid); T a = sm_z[index[0]]; @@ -318,6 +325,8 @@ class MultiDimension8x8x8Functor : public Functor { // TriadiagY MGARDX_EXEC void Operation7() { + if (!orthogonal_projection) + return; if (tid < 25) { int const *index = TriDiag_Y_Offset_8x8x8(tid); T a = sm_z[index[0]]; @@ -365,6 +374,8 @@ class MultiDimension8x8x8Functor : public Functor { // TriadiagZ MGARDX_EXEC void Operation8() { + if (!orthogonal_projection) + return; if (tid < 25) { int const *index = TriDiag_Z_Offset_8x8x8(tid); T a = sm_z[index[0]]; @@ -413,7 +424,8 @@ class MultiDimension8x8x8Functor : public Functor { // Apply Correction MGARDX_EXEC void Operation9() { if (tid < 125) { - sm_v[Coarse_Offset_8x8x8(tid)] = sm_z[tid]; + if (orthogonal_projection) + sm_v[Coarse_Offset_8x8x8(tid)] += sm_z[tid]; int const *index = Coarse_Reorder_8x8x8(tid); *coarse(z_tb * 5 + index[0], y_tb * 5 + index[1], x_tb * 5 + index[2]) = sm_v[Coarse_Offset_8x8x8(tid)]; @@ -433,36 +445,20 @@ class MultiDimension8x8x8Functor : public Functor { // if (tid < 125) { // sm_v[tid] = coarse + correction; // } - - // #ifdef MGARDX_COMPILE_CUDA - // __syncthreads(); - // if (bid == 0 && tid == 0) { - // for (int i = 0; i < 8; i++) { - // printf("sm[i = %d]\n", i); - // for (int j = 0; j < 8; j++) { - // for (int k = 0; k < 8; k++) { - // printf("%10.2f ", sm_v[get_idx(8, 8, i, j, k)]); - // } - // printf("\n"); - // } - // printf("\n"); - // } - // } - // __syncthreads(); - // #endif } MGARDX_CONT size_t shared_memory_size() { - size_t size = (Z * Y * X) + Z * Y * (X / 2 + 1) + + size_t size = SMV_SIZE_8x8x8 + Z * Y * (X / 2 + 1) + Z * (Y / 2 + 1) * (X / 2 + 1) + (Z / 2 + 1) * (Y / 2 + 1) * (X / 2 + 1) + 1; return size * sizeof(T); } -private: +protected: SubArray v; SubArray coarse; SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; T *sm_v, *sm_x, *sm_y, *sm_z, *sm_c8, *sm_c5, *sm_c3, *sm_c2; int ld1 = X; int ld2 = Y; @@ -470,7 +466,7 @@ class MultiDimension8x8x8Functor : public Functor { int tid, bid, op_tid; T left, right, middle; int offset; - int zero_const_offset = (Z * Y * X) + Z * Y * (X / 2 + 1) + + int zero_const_offset = SMV_SIZE_8x8x8 + Z * Y * (X / 2 + 1) + Z * (Y / 2 + 1) * (X / 2 + 1) + (Z / 2 + 1) * (Y / 2 + 1) * (X / 2 + 1); // #ifdef MGARDX_COMPILE_CUDA @@ -478,23 +474,167 @@ class MultiDimension8x8x8Functor : public Functor { // #endif }; -template -class MultiDimension8x8x8Kernel : public Kernel { +// Fused decompose+quantize variant: identical transform pipeline, but +// (1) the input is read with bounds checks so unpadded inputs can be consumed +// directly (out-of-range positions contribute zeros, matching the +// memset+copy padding of the unfused path bit for bit), and +// (2) the 387 per-block coefficients are quantized on write-out while still +// in shared memory, emitting Q symbols (with the Huffman dictionary shift +// folded in, mirroring QuantizeLocalLevelFunctor) instead of T values. +// The 5x5x5 coarse output stays in T: it is the next level's input. +template +class DecomposeQuantize8x8x8Functor + : public Decompose8x8x8Functor { + using Base = Decompose8x8x8Functor; + +public: + MGARDX_CONT DecomposeQuantize8x8x8Functor() {} + MGARDX_CONT DecomposeQuantize8x8x8Functor( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : Base(v, coarse, SubArray<1, T, DeviceType>(), orthogonal_projection), + quantized_coeff(quantized_coeff), quantizer(quantizer), + block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size) {} + + MGARDX_EXEC void Operation1() { + this->initialize_sm_8x8x8(); + this->x = FunctorBase::GetThreadIdX(); + this->y = FunctorBase::GetThreadIdY(); + this->z = FunctorBase::GetThreadIdZ(); + this->x_tb = FunctorBase::GetBlockIdX(); + this->y_tb = FunctorBase::GetBlockIdY(); + this->z_tb = FunctorBase::GetBlockIdZ(); + this->x_gl = X * this->x_tb + this->x; + this->y_gl = Y * this->y_tb + this->y; + this->z_gl = Z * this->z_tb + this->z; + + this->tid = this->z * X * Y + this->y * X + this->x; + this->bid = this->z_tb * FunctorBase::GetGridDimX() * + FunctorBase::GetGridDimY() + + this->y_tb * FunctorBase::GetGridDimX() + + this->x_tb; + if (this->z == 0 && this->y == 0 && this->x == 0) + this->sm_v[this->zero_const_offset] = (T)0; + + this->offset = offset8x8x8(this->z, this->y, this->x); // padded sm_v layout + this->sm_v[this->offset] = 0.0; + // Unlike the unfused functor, keep the bounds check: the input here is + // not pre-padded to a multiple of 8, so edge blocks must zero-fill. + if (this->z_gl < (int)this->v.shape(D - 3) && + this->y_gl < (int)this->v.shape(D - 2) && + this->x_gl < (int)this->v.shape(D - 1)) { + this->sm_v[this->offset] = *this->v(this->z_gl, this->y_gl, this->x_gl); + } + } + + MGARDX_EXEC void Operation9() { + if (this->tid < 125) { + if (this->orthogonal_projection) + this->sm_v[Coarse_Offset_8x8x8(this->tid)] += this->sm_z[this->tid]; + int const *index = Coarse_Reorder_8x8x8(this->tid); + *this->coarse(this->z_tb * 5 + index[0], this->y_tb * 5 + index[1], + this->x_tb * 5 + index[2]) = + this->sm_v[Coarse_Offset_8x8x8(this->tid)]; + } else { + int op_tid = this->tid - 125; + T t = this->sm_v[Coeff_Offset_8x8x8(op_tid)]; + T q = use_block_quantizers ? *block_quantizers(this->bid) : quantizer; + // Must stay bit-identical to QuantizeLocalLevelFunctor (volume == 1). + Q quantized_data; + if constexpr (sizeof(T) == sizeof(double)) { + quantized_data = copysign((T)0.5 + fabs(t * q), t); + } else if constexpr (sizeof(T) == sizeof(float)) { + quantized_data = copysign((T)0.5 + fabsf(t * q), t); + } + if (prep_huffman) { + quantized_data += dict_size / 2; + } + *quantized_coeff(this->bid * 387 + op_tid) = quantized_data; + } + } + +protected: + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; +}; + +template +class Decompose8x8x8Kernel : public Kernel { public: constexpr static bool EnableAutoTuning() { return false; } constexpr static std::string_view Name = "lwpk"; MGARDX_CONT - MultiDimension8x8x8Kernel(SubArray v, - SubArray coarse, - SubArray<1, T, DeviceType> coeff) - : v(v), coarse(coarse), coeff(coeff) {} + Decompose8x8x8Kernel(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Decompose8x8x8Functor; + FunctorType functor(v, coarse, coeff, orthogonal_projection); + + SIZE total_thread_z = v.shape(D - 3); + SIZE total_thread_y = v.shape(D - 2); + SIZE total_thread_x = v.shape(D - 1); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 8; + tby = 8; + tbx = 8; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; +}; - MGARDX_CONT Task> +template +class DecomposeQuantize8x8x8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_fq"; + MGARDX_CONT + DecomposeQuantize8x8x8Kernel( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : v(v), coarse(coarse), quantized_coeff(quantized_coeff), + quantizer(quantizer), block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size), orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> GenTask(int queue_idx) { using FunctorType = - MultiDimension8x8x8Functor; - FunctorType functor(v, coarse, coeff); + DecomposeQuantize8x8x8Functor; + FunctorType functor(v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection); + // Same launch geometry as Decompose8x8x8Kernel; v may be unpadded here + // but ceil(shape / 8) matches the padded grid exactly. SIZE total_thread_z = v.shape(D - 3); SIZE total_thread_y = v.shape(D - 2); SIZE total_thread_x = v.shape(D - 1); @@ -515,7 +655,13 @@ class MultiDimension8x8x8Kernel : public Kernel { private: SubArray v; SubArray coarse; - SubArray<1, T, DeviceType> coeff; + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; + bool orthogonal_projection; }; } // namespace in_cache_block @@ -524,4 +670,4 @@ class MultiDimension8x8x8Kernel : public Kernel { } // namespace mgard_x -#endif \ No newline at end of file +#endif diff --git a/include/mgard-x/DataRefactoring/InCacheBlock/IndexTable3x3x3.hpp b/include/mgard-x/DataRefactoring/InCacheBlock/IndexTable3x3x3.hpp index 7a8eee1d1e..5300ce661e 100644 --- a/include/mgard-x/DataRefactoring/InCacheBlock/IndexTable3x3x3.hpp +++ b/include/mgard-x/DataRefactoring/InCacheBlock/IndexTable3x3x3.hpp @@ -1,10 +1,13 @@ /* - * Copyright 2023, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: Jan. 15, 2023 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ +#ifndef MGARD_X_INDEX_TABLE_3X3X3_HPP +#define MGARD_X_INDEX_TABLE_3X3X3_HPP + namespace mgard_x { // clang-format off @@ -363,4 +366,6 @@ MGARDX_EXEC int Coarse_Offset_3x3x3(SIZE i) { } // clang-format on -} // namespace mgard_x \ No newline at end of file +} // namespace mgard_x + +#endif \ No newline at end of file diff --git a/include/mgard-x/DataRefactoring/InCacheBlock/IndexTable5x5x5.hpp b/include/mgard-x/DataRefactoring/InCacheBlock/IndexTable5x5x5.hpp index 45e79d2d89..fbe2f6506b 100644 --- a/include/mgard-x/DataRefactoring/InCacheBlock/IndexTable5x5x5.hpp +++ b/include/mgard-x/DataRefactoring/InCacheBlock/IndexTable5x5x5.hpp @@ -1,9 +1,11 @@ /* - * Copyright 2023, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: Jan. 15, 2023 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ +#ifndef MGARD_X_INDEX_TABLE_5X5X5_HPP +#define MGARD_X_INDEX_TABLE_5X5X5_HPP namespace mgard_x { // clang-format off @@ -783,4 +785,6 @@ MGARDX_EXEC int Coarse_Offset_5x5x5(SIZE i) { } // clang-format on -} // namespace mgard_x \ No newline at end of file +} // namespace mgard_x + +#endif \ No newline at end of file diff --git a/include/mgard-x/DataRefactoring/InCacheBlock/IndexTable8x8x8.hpp b/include/mgard-x/DataRefactoring/InCacheBlock/IndexTable8x8x8.hpp index e0f38bbbbe..80e306697a 100644 --- a/include/mgard-x/DataRefactoring/InCacheBlock/IndexTable8x8x8.hpp +++ b/include/mgard-x/DataRefactoring/InCacheBlock/IndexTable8x8x8.hpp @@ -1,18 +1,37 @@ /* - * Copyright 2023, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: Jan. 15, 2023 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ +#ifndef MGARD_X_INDEX_TABLE_8X8X8_HPP +#define MGARD_X_INDEX_TABLE_8X8X8_HPP + namespace mgard_x { // clang-format off +// --- Padded shared-memory layout for the 8x8x8 input plane (sm_v) --- +// The transform stores the 8^3 block in shared memory (sm_v) and then reads it +// with power-of-two strides (coefficient stencils differ by 2/4/8 in y and z), +// which collide on the 32 shared-memory banks (ncu measured avg 2.8-way load +// conflicts, ~65% of shared-load wavefronts wasted). Pad sm_v's y-stride to 9 +// and z-stride to 71 (both coprime with 32) so those accesses scatter across +// banks. Only sm_v is padded; sm_x/sm_y/sm_z keep their layouts. Every sm_v +// access routes through offset8x8x8, so the constexpr offset tables below +// regenerate for the padded layout automatically. +static constexpr int SMV_LDY_8x8x8 = 9; // physical x-row length (was 8) +static constexpr int SMV_LDZ_8x8x8 = 71; // physical z-plane stride (was 64) +static constexpr int SMV_SIZE_8x8x8 = 568; // 7*71 + 7*9 + 7 + 1 (was 512) + MGARDX_EXEC constexpr int offset8x8x8(SIZE z, SIZE y, SIZE x) { - return z * 8 * 8 + y * 8 + x; + return z * SMV_LDZ_8x8x8 + y * SMV_LDY_8x8x8 + x; } MGARDX_EXEC constexpr int offset8x8x8(SIZE z, SIZE y, SIZE x, SIZE ld1, SIZE ld2) { + // sm_v is the only array with an 8x8 leading layout; use the padded strides. + if (ld1 == 8 && ld2 == 8) + return z * SMV_LDZ_8x8x8 + y * SMV_LDY_8x8x8 + x; return z * ld1 * ld2 + y * ld1 + x; } @@ -1949,7 +1968,7 @@ MGARDX_EXEC T const *MassTrans_Weights_8x8x8(SIZE i) { } MGARDX_EXEC int const *MassTrans_X_Offset_8x8x8(SIZE i) { - static constexpr int zero_offset = 8*8*8 + 8*8*5 + 8*5*5 + 5*5*5; + static constexpr int zero_offset = SMV_SIZE_8x8x8 + 8*8*5 + 8*5*5 + 5*5*5; #define OFFSET1(Z, Y) \ { \ zero_offset, \ @@ -2498,4 +2517,6 @@ MGARDX_EXEC int Coeff_Offset_8x8x8(SIZE i) { } // clang-format on -} // namespace mgard_x \ No newline at end of file +} // namespace mgard_x + +#endif \ No newline at end of file diff --git a/include/mgard-x/DataRefactoring/InCacheBlock/IndexTableLowDim.hpp b/include/mgard-x/DataRefactoring/InCacheBlock/IndexTableLowDim.hpp new file mode 100644 index 0000000000..7946db8896 --- /dev/null +++ b/include/mgard-x/DataRefactoring/InCacheBlock/IndexTableLowDim.hpp @@ -0,0 +1,373 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_INDEX_TABLE_LOW_DIM_HPP +#define MGARD_X_INDEX_TABLE_LOW_DIM_HPP + +#include "IndexTable8x8x8.hpp" + +namespace mgard_x { + +// --------------------------------------------------------------------------- +// Index tables for the 1D (8) and 2D (8x8) in-cache block transforms. +// +// The block geometry is the same in every dimension as the 3D block: an +// 8-wide block coarsens to the 5 nodes {0, 2, 4, 6, 7}, and the trailing cell +// [6, 7] is treated as two half cells around a phantom node at 6.5. So the +// coarse-grid spacing is {2, 2, 2, 1} and the mass-transform weight cases are +// identical to the 3D block's -- am_8x8x8 / bm_8x8x8 / +// MassTrans_Weights_8x8x8 are reused here verbatim rather than duplicated. +// +// IndexTable8x8x8.hpp spells out every table entry by hand. The low-dimension +// tables are two orders of magnitude smaller, so they are built with constexpr +// loops instead: the geometry rules are stated once below and the compiler +// expands them into the same kind of `static constexpr` lookup tables. +// --------------------------------------------------------------------------- + +// Minimal constexpr table containers. Not std::array: these are read from +// device code, where std::array's constexpr accessors are not portably usable. +template struct LowDimTable { int v[N][K]; }; + +static constexpr int LowDim_Block = 8; // fine nodes per block per dim +static constexpr int LowDim_Coarse = 5; // coarse nodes per block per dim +static constexpr int LowDim_Coeff1D = 3; // coefficient nodes per dim + +static constexpr int lowdim_coarse_idx[LowDim_Coarse] = {0, 2, 4, 6, 7}; +static constexpr int lowdim_coeff_idx[LowDim_Coeff1D] = {1, 3, 5}; + +MGARDX_CONT_EXEC constexpr bool lowdim_is_coarse(int i) { + return i == 0 || i == 2 || i == 4 || i == 6 || i == 7; +} + +// Fine-grid position feeding stencil slot `slot` (0..4, i.e. -2..+2 half-cell +// steps) of coarse node `j`, or -1 when that slot has no node -- either +// outside the block or the phantom node at 6.5. +// +// `masked` additionally drops fine nodes that are themselves coarse nodes. +// The correction is the mass matrix applied to the *coefficient* field, which +// is zero at coarse nodes; along a line whose other coordinates are all +// coarse, the fine nodes at coarse positions therefore contribute nothing. +// This mirrors the OFFSET1 (masked) / OFFSET2 (full) split of +// MassTrans_X_Offset_8x8x8. +MGARDX_CONT_EXEC constexpr int lowdim_mt_pos(int j, int slot, bool masked) { + int p = -1; + if (j == 3) { + // Coarse node 6: slot 3 is the phantom node at 6.5, slot 4 is node 7. + const int tail[5] = {4, 5, 6, -1, 7}; + p = tail[slot]; + } else if (j == 4) { + // Coarse node 7: slot 1 is the phantom node at 6.5, nothing to the right. + const int tail[5] = {6, -1, 7, -1, -1}; + p = tail[slot]; + } else { + p = lowdim_coarse_idx[j] + slot - 2; + if (p < 0 || p >= LowDim_Block) { + p = -1; + } + } + if (masked && p >= 0 && lowdim_is_coarse(p)) { + p = -1; + } + return p; +} + +// --------------------------------------------------------------------------- +// 2D: 8x8 block +// +// Shared memory, in one allocation, offsets below relative to sm_v: +// sm_v [8][9] the block itself; the x-row length is padded from 8 to 9 so +// the power-of-two coefficient strides scatter across the 32 +// banks, same rationale as SMV_LDY_8x8x8 +// sm_x [8][5] after the x mass transform +// sm_y [5][5] after the y mass transform, then the tridiagonal solves +// one trailing slot holding a constant zero, read by stencil slots that +// have no node. +// --------------------------------------------------------------------------- +static constexpr int SMV_LDY_8x8 = 9; +static constexpr int SMV_SIZE_8x8 = 7 * SMV_LDY_8x8 + 7 + 1; // 71 +static constexpr int SMX_SIZE_8x8 = LowDim_Block * LowDim_Coarse; +static constexpr int SMY_SIZE_8x8 = LowDim_Coarse * LowDim_Coarse; +static constexpr int SM_SIZE_8x8 = + SMV_SIZE_8x8 + SMX_SIZE_8x8 + SMY_SIZE_8x8 + 1; +// The one zero slot, addressed relative to sm_v and to sm_x respectively. +static constexpr int ZERO_V_8x8 = SMV_SIZE_8x8 + SMX_SIZE_8x8 + SMY_SIZE_8x8; +static constexpr int ZERO_X_8x8 = SMX_SIZE_8x8 + SMY_SIZE_8x8; + +static constexpr int NumCoarse_8x8 = LowDim_Coarse * LowDim_Coarse; // 25 +static constexpr int NumCoeff_8x8 = + LowDim_Block * LowDim_Block - NumCoarse_8x8; // 39 +static constexpr int NumCoeff1D_8x8 = 2 * LowDim_Coarse * LowDim_Coeff1D; // 30 +static constexpr int NumCoeff2D_8x8 = LowDim_Coeff1D * LowDim_Coeff1D; // 9 +static constexpr int NumMassTransX_8x8 = LowDim_Block * LowDim_Coarse; // 40 +static constexpr int NumMassTransY_8x8 = LowDim_Coarse * LowDim_Coarse; // 25 + +MGARDX_CONT_EXEC constexpr int offset8x8(int y, int x) { + return y * SMV_LDY_8x8 + x; +} + +// Interpolation, 1D coefficients: {middle, left, right} sm_v offsets. +// First the 15 x-direction coefficients (on coarse rows), then the 15 +// y-direction ones (on coarse columns). +MGARDX_CONT_EXEC constexpr LowDimTable make_coeff1d_8x8() { + LowDimTable t{}; + int n = 0; + for (int iy = 0; iy < LowDim_Coarse; iy++) { + for (int j = 0; j < LowDim_Coeff1D; j++) { + int y = lowdim_coarse_idx[iy]; + int x = lowdim_coeff_idx[j]; + t.v[n][0] = offset8x8(y, x); + t.v[n][1] = offset8x8(y, x - 1); + t.v[n][2] = offset8x8(y, x + 1); + n++; + } + } + for (int j = 0; j < LowDim_Coeff1D; j++) { + for (int ix = 0; ix < LowDim_Coarse; ix++) { + int y = lowdim_coeff_idx[j]; + int x = lowdim_coarse_idx[ix]; + t.v[n][0] = offset8x8(y, x); + t.v[n][1] = offset8x8(y - 1, x); + t.v[n][2] = offset8x8(y + 1, x); + n++; + } + } + return t; +} + +MGARDX_EXEC int const *Coeff1D_Offset_8x8(SIZE i) { + static constexpr LowDimTable t = make_coeff1d_8x8(); + return t.v[i]; +} + +// Interpolation, 2D coefficients: {middle, and the four surrounding corners}. +MGARDX_CONT_EXEC constexpr LowDimTable make_coeff2d_8x8() { + LowDimTable t{}; + int n = 0; + for (int jy = 0; jy < LowDim_Coeff1D; jy++) { + for (int jx = 0; jx < LowDim_Coeff1D; jx++) { + int y = lowdim_coeff_idx[jy]; + int x = lowdim_coeff_idx[jx]; + t.v[n][0] = offset8x8(y, x); + t.v[n][1] = offset8x8(y - 1, x - 1); + t.v[n][2] = offset8x8(y - 1, x + 1); + t.v[n][3] = offset8x8(y + 1, x - 1); + t.v[n][4] = offset8x8(y + 1, x + 1); + n++; + } + } + return t; +} + +MGARDX_EXEC int const *Coeff2D_Offset_8x8(SIZE i) { + static constexpr LowDimTable t = make_coeff2d_8x8(); + return t.v[i]; +} + +// X mass transform: {5 sm_v inputs, sm_x output, weight case}, one row per +// (fine y, coarse x). Rows on coarse y are masked, as explained above. +MGARDX_CONT_EXEC constexpr LowDimTable +make_masstrans_x_8x8() { + LowDimTable t{}; + for (int y = 0; y < LowDim_Block; y++) { + bool masked = lowdim_is_coarse(y); + for (int j = 0; j < LowDim_Coarse; j++) { + int n = y * LowDim_Coarse + j; + for (int s = 0; s < 5; s++) { + int p = lowdim_mt_pos(j, s, masked); + t.v[n][s] = (p < 0) ? ZERO_V_8x8 : offset8x8(y, p); + } + t.v[n][5] = y * LowDim_Coarse + j; + t.v[n][6] = j; + } + } + return t; +} + +MGARDX_EXEC int const *MassTrans_X_Offset_8x8(SIZE i) { + static constexpr LowDimTable t = make_masstrans_x_8x8(); + return t.v[i]; +} + +// Y mass transform: {5 sm_x inputs, sm_y output, weight case}, one row per +// (coarse y, coarse x). Never masked: the x pass already zeroed the coarse +// nodes' contribution, and the remaining passes are plain 1D mass transforms. +MGARDX_CONT_EXEC constexpr LowDimTable +make_masstrans_y_8x8() { + LowDimTable t{}; + for (int jy = 0; jy < LowDim_Coarse; jy++) { + for (int jx = 0; jx < LowDim_Coarse; jx++) { + int n = jy * LowDim_Coarse + jx; + for (int s = 0; s < 5; s++) { + int p = lowdim_mt_pos(jy, s, false); + t.v[n][s] = (p < 0) ? ZERO_X_8x8 : (p * LowDim_Coarse + jx); + } + t.v[n][5] = jy * LowDim_Coarse + jx; + t.v[n][6] = jy; + } + } + return t; +} + +MGARDX_EXEC int const *MassTrans_Y_Offset_8x8(SIZE i) { + static constexpr LowDimTable t = make_masstrans_y_8x8(); + return t.v[i]; +} + +// Tridiagonal solves over sm_y: one line per row (x solve) or column (y solve). +MGARDX_CONT_EXEC constexpr LowDimTable +make_tridiag_x_8x8() { + LowDimTable t{}; + for (int jy = 0; jy < LowDim_Coarse; jy++) { + for (int k = 0; k < LowDim_Coarse; k++) { + t.v[jy][k] = jy * LowDim_Coarse + k; + } + } + return t; +} + +MGARDX_EXEC int const *TriDiag_X_Offset_8x8(SIZE i) { + static constexpr LowDimTable t = + make_tridiag_x_8x8(); + return t.v[i]; +} + +MGARDX_CONT_EXEC constexpr LowDimTable +make_tridiag_y_8x8() { + LowDimTable t{}; + for (int jx = 0; jx < LowDim_Coarse; jx++) { + for (int k = 0; k < LowDim_Coarse; k++) { + t.v[jx][k] = k * LowDim_Coarse + jx; + } + } + return t; +} + +MGARDX_EXEC int const *TriDiag_Y_Offset_8x8(SIZE i) { + static constexpr LowDimTable t = + make_tridiag_y_8x8(); + return t.v[i]; +} + +// sm_v offsets of the 25 coarse nodes, in the same row-major order as sm_y, +// so the correction at sm_y[i] belongs to Coarse_Offset_8x8(i). +MGARDX_CONT_EXEC constexpr LowDimTable make_coarse_8x8() { + LowDimTable t{}; + for (int jy = 0; jy < LowDim_Coarse; jy++) { + for (int jx = 0; jx < LowDim_Coarse; jx++) { + t.v[jy * LowDim_Coarse + jx][0] = + offset8x8(lowdim_coarse_idx[jy], lowdim_coarse_idx[jx]); + } + } + return t; +} + +MGARDX_EXEC int Coarse_Offset_8x8(SIZE i) { + static constexpr LowDimTable t = make_coarse_8x8(); + return t.v[i][0]; +} + +// sm_v offsets of the 39 coefficient nodes, enumerated row-major over the +// non-coarse positions -- the 2D analogue of Coeff_Offset_8x8x8's ordering. +// This ordering is part of the compressed layout: it is what the ROI +// per-block quantizer indexing and the recompose kernel both assume. +MGARDX_CONT_EXEC constexpr LowDimTable make_coeff_8x8() { + LowDimTable t{}; + int n = 0; + for (int y = 0; y < LowDim_Block; y++) { + for (int x = 0; x < LowDim_Block; x++) { + if (lowdim_is_coarse(y) && lowdim_is_coarse(x)) { + continue; + } + t.v[n][0] = offset8x8(y, x); + n++; + } + } + return t; +} + +MGARDX_EXEC int Coeff_Offset_8x8(SIZE i) { + static constexpr LowDimTable t = make_coeff_8x8(); + return t.v[i][0]; +} + +// --------------------------------------------------------------------------- +// 1D: 8-element block +// +// A single 8-element block is far too little work for a thread block, so the +// 1D kernels give each thread block LowDim_Tiles_1D independent tiles, laid +// out side by side in shared memory: +// sm_v [tiles][9] 8 values plus a per-tile zero slot at index 8 +// sm_x [tiles][5] after the mass transform, then the tridiagonal solve +// A per-tile zero slot (rather than one shared slot) keeps every table entry +// a pure within-tile offset, so the kernels can add tile * stride uniformly. +// --------------------------------------------------------------------------- +static constexpr SIZE LowDim_Tiles_1D = 32; +static constexpr int SMV_STRIDE_8 = LowDim_Block + 1; // 9 +static constexpr int SMV_ZERO_8 = LowDim_Block; // per-tile zero slot +static constexpr int SMX_STRIDE_8 = LowDim_Coarse; // 5 +static constexpr SIZE SM_SIZE_8 = + LowDim_Tiles_1D * (SMV_STRIDE_8 + SMX_STRIDE_8); + +static constexpr int NumCoarse_8 = LowDim_Coarse; // 5 +static constexpr int NumCoeff_8 = LowDim_Block - LowDim_Coarse; // 3 +static constexpr int NumMassTransX_8 = LowDim_Coarse; // 5 + +// Interpolation: {middle, left, right} within-tile sm_v offsets. +MGARDX_CONT_EXEC constexpr LowDimTable make_coeff1d_8() { + LowDimTable t{}; + for (int j = 0; j < NumCoeff_8; j++) { + int x = lowdim_coeff_idx[j]; + t.v[j][0] = x; + t.v[j][1] = x - 1; + t.v[j][2] = x + 1; + } + return t; +} + +MGARDX_EXEC int const *Coeff1D_Offset_8(SIZE i) { + static constexpr LowDimTable t = make_coeff1d_8(); + return t.v[i]; +} + +// Mass transform: {5 sm_v inputs, sm_x output, weight case}. Always masked -- +// in 1D every line is a line of coarse nodes. +MGARDX_CONT_EXEC constexpr LowDimTable +make_masstrans_x_8() { + LowDimTable t{}; + for (int j = 0; j < LowDim_Coarse; j++) { + for (int s = 0; s < 5; s++) { + int p = lowdim_mt_pos(j, s, true); + t.v[j][s] = (p < 0) ? SMV_ZERO_8 : p; + } + t.v[j][5] = j; + t.v[j][6] = j; + } + return t; +} + +MGARDX_EXEC int const *MassTrans_X_Offset_8(SIZE i) { + static constexpr LowDimTable t = make_masstrans_x_8(); + return t.v[i]; +} + +MGARDX_EXEC int Coarse_Offset_8(SIZE i) { + static constexpr LowDimTable t = {{{lowdim_coarse_idx[0]}, + {lowdim_coarse_idx[1]}, + {lowdim_coarse_idx[2]}, + {lowdim_coarse_idx[3]}, + {lowdim_coarse_idx[4]}}}; + return t.v[i][0]; +} + +MGARDX_EXEC int Coeff_Offset_8(SIZE i) { + static constexpr LowDimTable t = { + {{lowdim_coeff_idx[0]}, {lowdim_coeff_idx[1]}, {lowdim_coeff_idx[2]}}}; + return t.v[i][0]; +} + +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/DataRefactoring/InCacheBlock/Recompose8.hpp b/include/mgard-x/DataRefactoring/InCacheBlock/Recompose8.hpp new file mode 100644 index 0000000000..9987c5fa8d --- /dev/null +++ b/include/mgard-x/DataRefactoring/InCacheBlock/Recompose8.hpp @@ -0,0 +1,310 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_RECOMPOSE_8_KERNEL_TEMPLATE +#define MGARD_X_RECOMPOSE_8_KERNEL_TEMPLATE + +#include "../../RuntimeX/RuntimeX.h" + +#include "IndexTableLowDim.hpp" + +namespace mgard_x { + +namespace data_refactoring { + +namespace in_cache_block { + +// Exact inverse of Decompose8Functor: same stages in reverse, with the +// correction subtracted instead of added and the interpolation added back. +template +class Recompose8Functor : public Functor { +public: + MGARDX_CONT Recompose8Functor() {} + MGARDX_CONT Recompose8Functor(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) { + Functor(); + } + + MGARDX_EXEC void initialize_sm_8() { + sm_v = (T *)FunctorBase::GetSharedMemory(); + sm_x = sm_v + TILES * SMV_STRIDE_8; + } + + MGARDX_EXEC void init_indices() { + item = FunctorBase::GetThreadIdX(); + tile = FunctorBase::GetThreadIdY(); + bid = FunctorBase::GetBlockIdX() * TILES + tile; + num_tiles = (v.shape(D - 1) + X - 1) / X; + active = bid < (int)num_tiles; + base_v = tile * SMV_STRIDE_8; + base_x = tile * SMX_STRIDE_8; + x_gl = bid * X + item; + if (item == 0) + sm_v[base_v + SMV_ZERO_8] = (T)0; + } + + // Load + MGARDX_EXEC void Operation1() { + initialize_sm_8(); + init_indices(); + if (!active) + return; + if (item < NumCoarse_8) { + sm_v[base_v + Coarse_Offset_8(item)] = + *coarse(bid * LowDim_Coarse + item); + } else { + int op_tid = item - NumCoarse_8; + sm_v[base_v + Coeff_Offset_8(op_tid)] = *coeff(bid * NumCoeff_8 + op_tid); + } + } + + // MassTransX + MGARDX_EXEC void Operation2() { + if (!orthogonal_projection) + return; + if (active && item < NumMassTransX_8) { + int const *index = MassTrans_X_Offset_8(item); + T a = sm_v[base_v + index[0]]; + T b = sm_v[base_v + index[1]]; + T c = sm_v[base_v + index[2]]; + T d = sm_v[base_v + index[3]]; + T e = sm_v[base_v + index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_x[base_x + index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + } + + MGARDX_EXEC void solve_tridiag() { + T a = sm_x[base_x + 0]; + T b = sm_x[base_x + 1]; + T c = sm_x[base_x + 2]; + T d = sm_x[base_x + 3]; + T e = sm_x[base_x + 4]; + + a += 0 * amxbm_8x8x8(0); + b += a * amxbm_8x8x8(1); + c += b * amxbm_8x8x8(2); + d += c * amxbm_8x8x8(3); + e += d * amxbm_8x8x8(4); + + e = (e + am_8x8x8(5) * 0) * bm_8x8x8(5); + d = (d + am_8x8x8(4) * e) * bm_8x8x8(4); + c = (c + am_8x8x8(3) * d) * bm_8x8x8(3); + b = (b + am_8x8x8(2) * c) * bm_8x8x8(2); + a = (a + am_8x8x8(1) * b) * bm_8x8x8(1); + + sm_x[base_x + 0] = a; + sm_x[base_x + 1] = b; + sm_x[base_x + 2] = c; + sm_x[base_x + 3] = d; + sm_x[base_x + 4] = e; + } + + // TridiagX + MGARDX_EXEC void Operation3() { + if (!orthogonal_projection) + return; + if (active && item == 0) { + solve_tridiag(); + } + } + + // Subtract correction + MGARDX_EXEC void Operation4() { + if (orthogonal_projection && active && item < NumCoarse_8) { + sm_v[base_v + Coarse_Offset_8(item)] -= sm_x[base_x + item]; + } + } + + // Undo interpolation + MGARDX_EXEC void Operation5() { + if (active && item < NumCoeff_8) { + int const *index = Coeff1D_Offset_8(item); + T middle = sm_v[base_v + index[0]]; + T left = sm_v[base_v + index[1]]; + T right = sm_v[base_v + index[2]]; + sm_v[base_v + index[0]] = middle + (left + right) * (T)0.5; + } + } + + // Store + MGARDX_EXEC void Operation6() { + if (active) { + *v(x_gl) = sm_v[base_v + item]; + } + } + + MGARDX_CONT size_t shared_memory_size() { + return (size_t)TILES * (SMV_STRIDE_8 + SMX_STRIDE_8) * sizeof(T); + } + +protected: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; + T *sm_v, *sm_x; + int item, tile, bid, x_gl; + int base_v, base_x; + SIZE num_tiles; + bool active; +}; + +// Fused dequantize+recompose variant. See RecomposeDequantize8x8x8Functor: +// the 3 per-tile coefficients are read as Q symbols and dequantized while +// being staged into shared memory, and the store is bounds checked so an +// unpadded destination can be filled directly. +template +class RecomposeDequantize8Functor + : public Recompose8Functor { + using Base = Recompose8Functor; + +public: + MGARDX_CONT RecomposeDequantize8Functor() {} + MGARDX_CONT RecomposeDequantize8Functor( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : Base(v, coarse, SubArray<1, T, DeviceType>(), orthogonal_projection), + quantized_coeff(quantized_coeff), quantizer(quantizer), + block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size) {} + + MGARDX_EXEC void Operation1() { + this->initialize_sm_8(); + this->init_indices(); + if (!this->active) + return; + if (this->item < NumCoarse_8) { + this->sm_v[this->base_v + Coarse_Offset_8(this->item)] = + *this->coarse(this->bid * LowDim_Coarse + this->item); + } else { + int op_tid = this->item - NumCoarse_8; + Q quantized_data = *quantized_coeff(this->bid * NumCoeff_8 + op_tid); + if (prep_huffman) { + quantized_data -= dict_size / 2; + } + T q = use_block_quantizers ? *block_quantizers(this->bid) : quantizer; + // Must stay bit-identical to QuantizeLocalLevelFunctor (volume == 1, + // non-reciprocal quantizer). + this->sm_v[this->base_v + Coeff_Offset_8(op_tid)] = q * (T)quantized_data; + } + } + + MGARDX_EXEC void Operation6() { + if (this->active && this->x_gl < (int)this->v.shape(D - 1)) { + *this->v(this->x_gl) = this->sm_v[this->base_v + this->item]; + } + } + +protected: + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; +}; + +template +class Recompose8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_1d"; + MGARDX_CONT + Recompose8Kernel(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Recompose8Functor; + FunctorType functor(v, coarse, coeff, orthogonal_projection); + + SIZE num_tiles = (v.shape(D - 1) + 7) / 8; + size_t sm_size = functor.shared_memory_size(); + SIZE tbz = 1, tby = LowDim_Tiles_1D, tbx = 8; + SIZE gridz = 1, gridy = 1; + SIZE gridx = ceil((double)num_tiles / LowDim_Tiles_1D); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; +}; + +template +class RecomposeDequantize8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_1d_fq"; + MGARDX_CONT + RecomposeDequantize8Kernel(SubArray v, + SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, + T quantizer, + SubArray<1, T, DeviceType> block_quantizers, + bool use_block_quantizers, bool prep_huffman, + SIZE dict_size, bool orthogonal_projection = true) + : v(v), coarse(coarse), quantized_coeff(quantized_coeff), + quantizer(quantizer), block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size), orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT + Task> + GenTask(int queue_idx) { + using FunctorType = + RecomposeDequantize8Functor; + FunctorType functor(v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection); + + SIZE num_tiles = (v.shape(D - 1) + 7) / 8; + size_t sm_size = functor.shared_memory_size(); + SIZE tbz = 1, tby = LowDim_Tiles_1D, tbx = 8; + SIZE gridz = 1, gridy = 1; + SIZE gridx = ceil((double)num_tiles / LowDim_Tiles_1D); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; + bool orthogonal_projection; +}; + +} // namespace in_cache_block + +} // namespace data_refactoring + +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/DataRefactoring/InCacheBlock/Recompose8x8.hpp b/include/mgard-x/DataRefactoring/InCacheBlock/Recompose8x8.hpp new file mode 100644 index 0000000000..e56bebbb66 --- /dev/null +++ b/include/mgard-x/DataRefactoring/InCacheBlock/Recompose8x8.hpp @@ -0,0 +1,345 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_RECOMPOSE_8x8_KERNEL_TEMPLATE +#define MGARD_X_RECOMPOSE_8x8_KERNEL_TEMPLATE + +#include "../../RuntimeX/RuntimeX.h" + +#include "IndexTableLowDim.hpp" + +namespace mgard_x { + +namespace data_refactoring { + +namespace in_cache_block { + +// Exact inverse of Decompose8x8Functor: same stages in reverse, with the +// correction subtracted instead of added and the interpolation added back. +template +class Recompose8x8Functor : public Functor { +public: + MGARDX_CONT Recompose8x8Functor() {} + MGARDX_CONT Recompose8x8Functor(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) { + Functor(); + } + + MGARDX_EXEC void initialize_sm_8x8() { + sm_v = (T *)FunctorBase::GetSharedMemory(); + sm_x = sm_v + SMV_SIZE_8x8; // sm_v is padded for bank-conflict avoidance + sm_y = sm_x + SMX_SIZE_8x8; + } + + MGARDX_EXEC void init_indices() { + x = FunctorBase::GetThreadIdX(); + y = FunctorBase::GetThreadIdY(); + x_tb = FunctorBase::GetBlockIdX(); + y_tb = FunctorBase::GetBlockIdY(); + x_gl = X * x_tb + x; + y_gl = Y * y_tb + y; + tid = y * X + x; + bid = y_tb * FunctorBase::GetGridDimX() + x_tb; + if (tid == 0) + sm_v[ZERO_V_8x8] = (T)0; + offset = offset8x8(y, x); + } + + // Load + MGARDX_EXEC void Operation1() { + initialize_sm_8x8(); + init_indices(); + if (tid < NumCoarse_8x8) { + sm_v[Coarse_Offset_8x8(tid)] = + *coarse(y_tb * LowDim_Coarse + tid / LowDim_Coarse, + x_tb * LowDim_Coarse + tid % LowDim_Coarse); + } else { + int op_tid = tid - NumCoarse_8x8; + sm_v[Coeff_Offset_8x8(op_tid)] = *coeff(bid * NumCoeff_8x8 + op_tid); + } + } + + // MassTransX + MGARDX_EXEC void Operation2() { + if (!orthogonal_projection) + return; + if (tid < NumMassTransX_8x8) { + int const *index = MassTrans_X_Offset_8x8(tid); + T a = sm_v[index[0]]; + T b = sm_v[index[1]]; + T c = sm_v[index[2]]; + T d = sm_v[index[3]]; + T e = sm_v[index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_x[index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + } + + // MassTransY + MGARDX_EXEC void Operation3() { + if (!orthogonal_projection) + return; + if (tid < NumMassTransY_8x8) { + int const *index = MassTrans_Y_Offset_8x8(tid); + T a = sm_x[index[0]]; + T b = sm_x[index[1]]; + T c = sm_x[index[2]]; + T d = sm_x[index[3]]; + T e = sm_x[index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_y[index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + } + + MGARDX_EXEC void solve_tridiag(int const *index) { + T a = sm_y[index[0]]; + T b = sm_y[index[1]]; + T c = sm_y[index[2]]; + T d = sm_y[index[3]]; + T e = sm_y[index[4]]; + + a += 0 * amxbm_8x8x8(0); + b += a * amxbm_8x8x8(1); + c += b * amxbm_8x8x8(2); + d += c * amxbm_8x8x8(3); + e += d * amxbm_8x8x8(4); + + e = (e + am_8x8x8(5) * 0) * bm_8x8x8(5); + d = (d + am_8x8x8(4) * e) * bm_8x8x8(4); + c = (c + am_8x8x8(3) * d) * bm_8x8x8(3); + b = (b + am_8x8x8(2) * c) * bm_8x8x8(2); + a = (a + am_8x8x8(1) * b) * bm_8x8x8(1); + + sm_y[index[0]] = a; + sm_y[index[1]] = b; + sm_y[index[2]] = c; + sm_y[index[3]] = d; + sm_y[index[4]] = e; + } + + // TridiagX + MGARDX_EXEC void Operation4() { + if (!orthogonal_projection) + return; + if (tid < LowDim_Coarse) { + solve_tridiag(TriDiag_X_Offset_8x8(tid)); + } + } + + // TridiagY + MGARDX_EXEC void Operation5() { + if (!orthogonal_projection) + return; + if (tid < LowDim_Coarse) { + solve_tridiag(TriDiag_Y_Offset_8x8(tid)); + } + } + + // Subtract correction + MGARDX_EXEC void Operation6() { + if (orthogonal_projection && tid < NumCoarse_8x8) { + sm_v[Coarse_Offset_8x8(tid)] -= sm_y[tid]; + } + } + + // Undo interpolation + MGARDX_EXEC void Operation7() { + if (tid < NumCoeff1D_8x8) { + int const *index = Coeff1D_Offset_8x8(tid); + T middle = sm_v[index[0]]; + T left = sm_v[index[1]]; + T right = sm_v[index[2]]; + sm_v[index[0]] = middle + (left + right) * (T)0.5; + } else if (tid >= 32 && tid < 32 + NumCoeff2D_8x8) { + int const *index = Coeff2D_Offset_8x8(tid - 32); + T c11 = sm_v[index[0]]; + T c00 = sm_v[index[1]]; + T c02 = sm_v[index[2]]; + T c20 = sm_v[index[3]]; + T c22 = sm_v[index[4]]; + sm_v[index[0]] = c11 + (c00 + c02 + c20 + c22) / 4; + } + } + + // Store + MGARDX_EXEC void Operation8() { *v(y_gl, x_gl) = sm_v[offset]; } + + MGARDX_CONT size_t shared_memory_size() { + return (size_t)SM_SIZE_8x8 * sizeof(T); + } + +protected: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; + T *sm_v, *sm_x, *sm_y; + int y, x, y_tb, x_tb, y_gl, x_gl; + int tid, bid; + int offset; +}; + +// Fused dequantize+recompose variant (inverse of DecomposeQuantize8x8): +// identical transform pipeline, but +// (1) the 39 per-tile coefficients are read as Q symbols and dequantized +// while being staged into shared memory, and +// (2) the output is written with bounds checks so an unpadded destination can +// be filled directly. +template +class RecomposeDequantize8x8Functor + : public Recompose8x8Functor { + using Base = Recompose8x8Functor; + +public: + MGARDX_CONT RecomposeDequantize8x8Functor() {} + MGARDX_CONT RecomposeDequantize8x8Functor( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : Base(v, coarse, SubArray<1, T, DeviceType>(), orthogonal_projection), + quantized_coeff(quantized_coeff), quantizer(quantizer), + block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size) {} + + MGARDX_EXEC void Operation1() { + this->initialize_sm_8x8(); + this->init_indices(); + if (this->tid < NumCoarse_8x8) { + this->sm_v[Coarse_Offset_8x8(this->tid)] = + *this->coarse(this->y_tb * LowDim_Coarse + this->tid / LowDim_Coarse, + this->x_tb * LowDim_Coarse + this->tid % LowDim_Coarse); + } else { + int op_tid = this->tid - NumCoarse_8x8; + Q quantized_data = *quantized_coeff(this->bid * NumCoeff_8x8 + op_tid); + if (prep_huffman) { + quantized_data -= dict_size / 2; + } + T q = use_block_quantizers ? *block_quantizers(this->bid) : quantizer; + // Must stay bit-identical to QuantizeLocalLevelFunctor (volume == 1). + this->sm_v[Coeff_Offset_8x8(op_tid)] = q * (T)quantized_data; + } + } + + MGARDX_EXEC void Operation8() { + // The destination is not padded to a multiple of 8, so edge tiles must + // drop their out-of-range values. + if (this->y_gl < (int)this->v.shape(D - 2) && + this->x_gl < (int)this->v.shape(D - 1)) { + *this->v(this->y_gl, this->x_gl) = this->sm_v[this->offset]; + } + } + +protected: + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; +}; + +template +class Recompose8x8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_2d"; + MGARDX_CONT + Recompose8x8Kernel(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Recompose8x8Functor; + FunctorType functor(v, coarse, coeff, orthogonal_projection); + + SIZE total_thread_y = v.shape(D - 2); + SIZE total_thread_x = v.shape(D - 1); + + size_t sm_size = functor.shared_memory_size(); + SIZE tbz = 1, tby = 8, tbx = 8; + SIZE gridz = 1; + SIZE gridy = ceil((double)total_thread_y / tby); + SIZE gridx = ceil((double)total_thread_x / tbx); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; +}; + +template +class RecomposeDequantize8x8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_2d_fq"; + MGARDX_CONT + RecomposeDequantize8x8Kernel( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : v(v), coarse(coarse), quantized_coeff(quantized_coeff), + quantizer(quantizer), block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size), orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = + RecomposeDequantize8x8Functor; + FunctorType functor(v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection); + + SIZE total_thread_y = v.shape(D - 2); + SIZE total_thread_x = v.shape(D - 1); + + size_t sm_size = functor.shared_memory_size(); + SIZE tbz = 1, tby = 8, tbx = 8; + SIZE gridz = 1; + SIZE gridy = ceil((double)total_thread_y / tby); + SIZE gridx = ceil((double)total_thread_x / tbx); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; + bool orthogonal_projection; +}; + +} // namespace in_cache_block + +} // namespace data_refactoring + +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/DataRefactoring/InCacheBlock/Recompose8x8x8.hpp b/include/mgard-x/DataRefactoring/InCacheBlock/Recompose8x8x8.hpp new file mode 100644 index 0000000000..28c191397d --- /dev/null +++ b/include/mgard-x/DataRefactoring/InCacheBlock/Recompose8x8x8.hpp @@ -0,0 +1,663 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_RECOMPOSE_8x8x8_KERNEL_TEMPLATE +#define MGARD_X_RECOMPOSE_8x8x8_KERNEL_TEMPLATE + +#include "../../RuntimeX/RuntimeX.h" + +#include "../MultiDimension/Correction/LPKFunctor.h" + +#include "../MultiDimension/Correction/IPKFunctor.h" + +#include "IndexTable3x3x3.hpp" +#include "IndexTable5x5x5.hpp" +#include "IndexTable8x8x8.hpp" + +namespace mgard_x { + +namespace data_refactoring { + +namespace in_cache_block { + +/* + +v x y z c total +8*8*8(512) 5*8*8(320) 5*5*8(200) 5*5*5(125) 0 1157 +5*5*5(125) 3*5*5(75) 3*3*5(45) 3*3*3(27) 8*8*8-5*5*5(387) 659 +3*3*3(27) 2*3*3(18) 2*2*3(12) 2*2*2(8) 8*8*8-3*3*3(485) 550 + + v(512) x(320) y(200) z(125) +c8(512) v(125) x( 75) y( 45) z(27) +c8(512) c5( 98) x( 18) y( 12) z( 8) +c8(512) c5( 98) c3(19) c2( 8) +*/ + +template +class Recompose8x8x8Functor : public Functor { +public: + MGARDX_CONT Recompose8x8x8Functor() {} + MGARDX_CONT Recompose8x8x8Functor(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) { + Functor(); + } + + MGARDX_EXEC void initialize_sm_8x8x8() { + sm_v = (T *)FunctorBase::GetSharedMemory(); + sm_x = sm_v + SMV_SIZE_8x8x8; // sm_v is padded for bank-conflict avoidance + sm_y = sm_x + 5 * 8 * 8; + sm_z = sm_y + 5 * 5 * 8; + } + + MGARDX_EXEC void initialize_sm_5x5x5() { + sm_c8 = (T *)FunctorBase::GetSharedMemory(); + sm_v = sm_c8 + 8 * 8 * 8; + sm_x = sm_v + 5 * 5 * 5; + sm_y = sm_x + 5 * 5 * 3; + sm_z = sm_y + 5 * 3 * 3; + } + + MGARDX_EXEC void initialize_sm_3x3x3() { + sm_c8 = (T *)FunctorBase::GetSharedMemory(); + sm_c5 = sm_c8 + 8 * 8 * 8; + sm_v = sm_c5 + 5 * 5 * 5; + sm_x = sm_v + 3 * 3 * 3; + sm_y = sm_x + 3 * 3 * 2; + sm_z = sm_y + 3 * 2 * 2; + } + + MGARDX_EXEC void initialize_sm_2x2x2() { + sm_c8 = (T *)FunctorBase::GetSharedMemory(); + sm_c5 = sm_c8 + 8 * 8 * 8; + sm_c3 = sm_c5 + 5 * 5 * 5; + sm_c2 = sm_c3 + 3 * 3 * 3; + } + + // Load data + MGARDX_EXEC void Operation1() { + initialize_sm_8x8x8(); + x = FunctorBase::GetThreadIdX(); + y = FunctorBase::GetThreadIdY(); + z = FunctorBase::GetThreadIdZ(); + x_tb = FunctorBase::GetBlockIdX(); + y_tb = FunctorBase::GetBlockIdY(); + z_tb = FunctorBase::GetBlockIdZ(); + x_gl = X * x_tb + x; + y_gl = Y * y_tb + y; + z_gl = Z * z_tb + z; + + tid = z * X * Y + y * X + x; + bid = z_tb * FunctorBase::GetGridDimX() * + FunctorBase::GetGridDimY() + + y_tb * FunctorBase::GetGridDimX() + x_tb; + if (z == 0 && y == 0 && x == 0) + sm_v[zero_const_offset] = (T)0; + + if (tid < 125) { + int const *index = Coarse_Reorder_8x8x8(tid); + sm_v[Coarse_Offset_8x8x8(tid)] = *coarse( + z_tb * 5 + index[0], y_tb * 5 + index[1], x_tb * 5 + index[2]); + } else { + int op_tid = tid - 125; + sm_v[Coeff_Offset_8x8x8(op_tid)] = *coeff(bid * 387 + op_tid); + // if (bid == 0 && op_tid == 0) { + // for (int i = 0; i < 387; i++) { + // printf("%f ", sm_v[Coeff_Offset_8x8x8(i)]); + // } + // printf("\n"); + // } + } + } + + // MassTransX + MGARDX_EXEC void Operation2() { + if (!orthogonal_projection) + return; + + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 0) { + // printf("int\n"); + // for (int i = 0; i < 8; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 8; j++) { + // for (int k = 0; k < 8; k++) { + // printf("%.6f ", sm_v[get_idx(8, 8, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + + if (tid < 320) { + int const *index = MassTrans_X_Offset_8x8x8(tid); + T a = sm_v[index[0]]; + T b = sm_v[index[1]]; + T c = sm_v[index[2]]; + T d = sm_v[index[3]]; + T e = sm_v[index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_x[index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 5) { + // printf("tra - x\n"); + // for (int i = 0; i < 8; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 8; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%.6f ", sm_x[get_idx(5, 8, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // MassTransY + MGARDX_EXEC void Operation3() { + if (!orthogonal_projection) + return; + if (tid < 200) { + int const *index = MassTrans_Y_Offset_8x8x8(tid); + T a = sm_x[index[0]]; + T b = sm_x[index[1]]; + T c = sm_x[index[2]]; + T d = sm_x[index[3]]; + T e = sm_x[index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_y[index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 0) { + // printf("tra - y\n"); + // for (int i = 0; i < 8; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 5; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%.6f ", sm_y[get_idx(5, 5, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // MassTransZ + MGARDX_EXEC void Operation4() { + if (!orthogonal_projection) + return; + if (tid < 125) { + int const *index = MassTrans_Z_Offset_8x8x8(tid); + T a = sm_y[index[0]]; + T b = sm_y[index[1]]; + T c = sm_y[index[2]]; + T d = sm_y[index[3]]; + T e = sm_y[index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_z[index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 0) { + // printf("tra - z\n"); + // for (int i = 0; i < 5; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 5; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%.6f ", sm_z[get_idx(5, 5, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // TriadiagX + MGARDX_EXEC void Operation5() { + if (!orthogonal_projection) + return; + if (tid < 25) { + int const *index = TriDiag_X_Offset_8x8x8(tid); + T a = sm_z[index[0]]; + T b = sm_z[index[1]]; + T c = sm_z[index[2]]; + T d = sm_z[index[3]]; + T e = sm_z[index[4]]; + + a += 0 * amxbm_8x8x8(0); + b += a * amxbm_8x8x8(1); + c += b * amxbm_8x8x8(2); + d += c * amxbm_8x8x8(3); + e += d * amxbm_8x8x8(4); + + e = (e + am_8x8x8(5) * 0) * bm_8x8x8(5); + d = (d + am_8x8x8(4) * e) * bm_8x8x8(4); + c = (c + am_8x8x8(3) * d) * bm_8x8x8(3); + b = (b + am_8x8x8(2) * c) * bm_8x8x8(2); + a = (a + am_8x8x8(1) * b) * bm_8x8x8(1); + + sm_z[index[0]] = a; + sm_z[index[1]] = b; + sm_z[index[2]] = c; + sm_z[index[3]] = d; + sm_z[index[4]] = e; + } + + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 0) { + // printf("tri - x\n"); + // for (int i = 0; i < 5; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 5; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%.6f ", sm_z[get_idx(5, 5, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // TriadiagY + MGARDX_EXEC void Operation6() { + if (!orthogonal_projection) + return; + if (tid < 25) { + int const *index = TriDiag_Y_Offset_8x8x8(tid); + T a = sm_z[index[0]]; + T b = sm_z[index[1]]; + T c = sm_z[index[2]]; + T d = sm_z[index[3]]; + T e = sm_z[index[4]]; + + a += 0 * amxbm_8x8x8(0); + b += a * amxbm_8x8x8(1); + c += b * amxbm_8x8x8(2); + d += c * amxbm_8x8x8(3); + e += d * amxbm_8x8x8(4); + + e = (e + am_8x8x8(5) * 0) * bm_8x8x8(5); + d = (d + am_8x8x8(4) * e) * bm_8x8x8(4); + c = (c + am_8x8x8(3) * d) * bm_8x8x8(3); + b = (b + am_8x8x8(2) * c) * bm_8x8x8(2); + a = (a + am_8x8x8(1) * b) * bm_8x8x8(1); + + sm_z[index[0]] = a; + sm_z[index[1]] = b; + sm_z[index[2]] = c; + sm_z[index[3]] = d; + sm_z[index[4]] = e; + } + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 0) { + // printf("tri - y\n"); + // for (int i = 0; i < 5; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 5; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%.6f ", sm_z[get_idx(5, 5, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // TriadiagZ + MGARDX_EXEC void Operation7() { + if (!orthogonal_projection) + return; + if (tid < 25) { + int const *index = TriDiag_Z_Offset_8x8x8(tid); + T a = sm_z[index[0]]; + T b = sm_z[index[1]]; + T c = sm_z[index[2]]; + T d = sm_z[index[3]]; + T e = sm_z[index[4]]; + + a += 0 * amxbm_8x8x8(0); + b += a * amxbm_8x8x8(1); + c += b * amxbm_8x8x8(2); + d += c * amxbm_8x8x8(3); + e += d * amxbm_8x8x8(4); + + e = (e + am_8x8x8(5) * 0) * bm_8x8x8(5); + d = (d + am_8x8x8(4) * e) * bm_8x8x8(4); + c = (c + am_8x8x8(3) * d) * bm_8x8x8(3); + b = (b + am_8x8x8(2) * c) * bm_8x8x8(2); + a = (a + am_8x8x8(1) * b) * bm_8x8x8(1); + + sm_z[index[0]] = a; + sm_z[index[1]] = b; + sm_z[index[2]] = c; + sm_z[index[3]] = d; + sm_z[index[4]] = e; + } + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (bid == 0 && tid == 0) { + // printf("tri - z\n"); + // for (int i = 0; i < 5; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 5; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%10.2f ", sm_z[get_idx(5, 5, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // Subtract correction + MGARDX_EXEC void Operation8() { + if (orthogonal_projection && tid < 125) { + sm_v[Coarse_Offset_8x8x8(tid)] -= sm_z[tid]; + } + } + + MGARDX_EXEC void Operation9() { + // #ifdef MGARDX_COMPILE_CUDA + // start = clock(); + // #endif + + op_tid = tid; + if (tid < 225) { + left = sm_v[Coeff1D_L_Offset_8x8x8(op_tid)]; + right = sm_v[Coeff1D_R_Offset_8x8x8(op_tid)]; + middle = sm_v[Coeff1D_M_Offset_8x8x8(op_tid)]; + // printf("l %f, r %f, m %f\n", left, right, middle); + middle = middle + (left + right) * (T)0.5; + sm_v[Coeff1D_M_Offset_8x8x8(op_tid)] = middle; + } else if (tid >= 256 && tid < 256 + 135) { + op_tid -= 256; + T c00 = sm_v[Coeff2D_LL_Offset_8x8x8(op_tid)]; + T c02 = sm_v[Coeff2D_LR_Offset_8x8x8(op_tid)]; + T c20 = sm_v[Coeff2D_RL_Offset_8x8x8(op_tid)]; + T c22 = sm_v[Coeff2D_RR_Offset_8x8x8(op_tid)]; + T c11 = sm_v[Coeff2D_MM_Offset_8x8x8(op_tid)]; + c11 += (c00 + c02 + c20 + c22) / 4; + sm_v[Coeff2D_MM_Offset_8x8x8(op_tid)] = c11; + } else if (tid >= 416 && tid < 416 + 27) { + op_tid -= 416; + T c000 = sm_v[Coeff3D_LLL_Offset_8x8x8(op_tid)]; + T c002 = sm_v[Coeff3D_LLR_Offset_8x8x8(op_tid)]; + T c020 = sm_v[Coeff3D_LRL_Offset_8x8x8(op_tid)]; + T c022 = sm_v[Coeff3D_LRR_Offset_8x8x8(op_tid)]; + T c200 = sm_v[Coeff3D_RLL_Offset_8x8x8(op_tid)]; + T c202 = sm_v[Coeff3D_RLR_Offset_8x8x8(op_tid)]; + T c220 = sm_v[Coeff3D_RRL_Offset_8x8x8(op_tid)]; + T c222 = sm_v[Coeff3D_RRR_Offset_8x8x8(op_tid)]; + T c111 = sm_v[Coeff3D_MMM_Offset_8x8x8(op_tid)]; + c111 += (c000 + c002 + c020 + c022 + c200 + c202 + c220 + c222) / 8; + sm_v[Coeff3D_MMM_Offset_8x8x8(op_tid)] = c111; + } + } + + // store data + MGARDX_EXEC void Operation10() { + offset = offset8x8x8(z, y, x); // padded sm_v layout + *v(z_gl, y_gl, x_gl) = sm_v[offset]; + // printf("v[%d, %d, %d] = %f\n", z_gl, y_gl, x_gl, sm_v[offset]); + // } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = SMV_SIZE_8x8x8 + Z * Y * (X / 2 + 1) + + Z * (Y / 2 + 1) * (X / 2 + 1) + + (Z / 2 + 1) * (Y / 2 + 1) * (X / 2 + 1) + 1; + return size * sizeof(T); + } + +protected: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; + T *sm_v, *sm_x, *sm_y, *sm_z, *sm_c8, *sm_c5, *sm_c3, *sm_c2; + int ld1 = X; + int ld2 = Y; + int z, y, x, z_tb, y_tb, x_tb, z_gl, y_gl, x_gl; + int tid, bid, op_tid; + T left, right, middle; + int offset; + int zero_const_offset = SMV_SIZE_8x8x8 + Z * Y * (X / 2 + 1) + + Z * (Y / 2 + 1) * (X / 2 + 1) + + (Z / 2 + 1) * (Y / 2 + 1) * (X / 2 + 1); + // #ifdef MGARDX_COMPILE_CUDA + // clock_t start, end; + // #endif +}; + +// Fused dequantize+recompose variant (inverse of DecomposeQuantize8x8x8): +// identical transform pipeline, but +// (1) the 387 per-block coefficients are read as Q symbols and dequantized +// while being staged into shared memory (with the Huffman dictionary +// shift removed, mirroring QuantizeLocalLevelFunctor's DEQUANTIZE op), +// so the T-typed coefficient array never exists in global memory, and +// (2) the reconstructed values are stored with bounds checks so the final +// level can write directly into the unpadded output array. +// The 5x5x5 coarse input stays in T: it is the previous level's output. +template +class RecomposeDequantize8x8x8Functor + : public Recompose8x8x8Functor { + using Base = Recompose8x8x8Functor; + +public: + MGARDX_CONT RecomposeDequantize8x8x8Functor() {} + MGARDX_CONT RecomposeDequantize8x8x8Functor( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : Base(v, coarse, SubArray<1, T, DeviceType>(), orthogonal_projection), + quantized_coeff(quantized_coeff), quantizer(quantizer), + block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size) {} + + MGARDX_EXEC void Operation1() { + this->initialize_sm_8x8x8(); + this->x = FunctorBase::GetThreadIdX(); + this->y = FunctorBase::GetThreadIdY(); + this->z = FunctorBase::GetThreadIdZ(); + this->x_tb = FunctorBase::GetBlockIdX(); + this->y_tb = FunctorBase::GetBlockIdY(); + this->z_tb = FunctorBase::GetBlockIdZ(); + this->x_gl = X * this->x_tb + this->x; + this->y_gl = Y * this->y_tb + this->y; + this->z_gl = Z * this->z_tb + this->z; + + this->tid = this->z * X * Y + this->y * X + this->x; + this->bid = this->z_tb * FunctorBase::GetGridDimX() * + FunctorBase::GetGridDimY() + + this->y_tb * FunctorBase::GetGridDimX() + + this->x_tb; + if (this->z == 0 && this->y == 0 && this->x == 0) + this->sm_v[this->zero_const_offset] = (T)0; + + if (this->tid < 125) { + int const *index = Coarse_Reorder_8x8x8(this->tid); + this->sm_v[Coarse_Offset_8x8x8(this->tid)] = + *this->coarse(this->z_tb * 5 + index[0], this->y_tb * 5 + index[1], + this->x_tb * 5 + index[2]); + } else { + int op_tid = this->tid - 125; + Q quantized_data = *quantized_coeff(this->bid * 387 + op_tid); + if (prep_huffman) { + quantized_data -= dict_size / 2; + } + T q = use_block_quantizers ? *block_quantizers(this->bid) : quantizer; + // Must stay bit-identical to QuantizeLocalLevelFunctor (volume == 1, + // non-reciprocal quantizer). + this->sm_v[Coeff_Offset_8x8x8(op_tid)] = q * (T)quantized_data; + } + } + + MGARDX_EXEC void Operation10() { + // Unlike the unfused functor, keep a bounds check on the store: the + // final level writes directly into the unpadded output array, so edge + // blocks must drop out-of-range results. + if (this->z_gl < (int)this->v.shape(D - 3) && + this->y_gl < (int)this->v.shape(D - 2) && + this->x_gl < (int)this->v.shape(D - 1)) { + this->offset = offset8x8x8(this->z, this->y, this->x); + *this->v(this->z_gl, this->y_gl, this->x_gl) = this->sm_v[this->offset]; + } + } + +protected: + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; +}; + +template +class Recompose8x8x8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk"; + MGARDX_CONT + Recompose8x8x8Kernel(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Recompose8x8x8Functor; + FunctorType functor(v, coarse, coeff, orthogonal_projection); + + SIZE total_thread_z = v.shape(D - 3); + SIZE total_thread_y = v.shape(D - 2); + SIZE total_thread_x = v.shape(D - 1); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 8; + tby = 8; + tbx = 8; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; +}; + +template +class RecomposeDequantize8x8x8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_fdq"; + MGARDX_CONT + RecomposeDequantize8x8x8Kernel( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : v(v), coarse(coarse), quantized_coeff(quantized_coeff), + quantizer(quantizer), block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size), orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT + Task> + GenTask(int queue_idx) { + using FunctorType = + RecomposeDequantize8x8x8Functor; + FunctorType functor(v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection); + + // Same launch geometry as Recompose8x8x8Kernel; v may be unpadded here + // but ceil(shape / 8) matches the padded grid exactly. + SIZE total_thread_z = v.shape(D - 3); + SIZE total_thread_y = v.shape(D - 2); + SIZE total_thread_x = v.shape(D - 1); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 8; + tby = 8; + tbx = 8; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; + bool orthogonal_projection; +}; + +} // namespace in_cache_block + +} // namespace data_refactoring + +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CalcCoefficients3D.hpp b/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CalcCoefficients3D.hpp index b6fb7c780f..bd6b9592f1 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CalcCoefficients3D.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CalcCoefficients3D.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../../../RuntimeX/RuntimeX.h" diff --git a/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsND.hpp b/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsND.hpp index 72ec731622..297f8a10ce 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsND.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsND.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../../../Hierarchy/Hierarchy.h" diff --git a/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsPointers.hpp b/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsPointers.hpp index bf31445018..ce6ea75d8e 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsPointers.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsPointers.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../../../Hierarchy/Hierarchy.h" diff --git a/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestore3D.hpp b/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestore3D.hpp index bc2e73ccb1..8e0ebf6786 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestore3D.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestore3D.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../../../Hierarchy/Hierarchy.h" diff --git a/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestoreND.hpp b/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestoreND.hpp index 4627e4c3d5..d81fb5b343 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestoreND.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestoreND.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../../../Hierarchy/Hierarchy.h" diff --git a/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/GPKFunctor.h b/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/GPKFunctor.h index 2844b0989b..3b2c893a68 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/GPKFunctor.h +++ b/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/GPKFunctor.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_GPK_FUNCTOR diff --git a/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/GridProcessingKernel.hpp b/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/GridProcessingKernel.hpp index 3df53346ea..9da8e62730 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/GridProcessingKernel.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/GridProcessingKernel.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_GRID_PROCESSING_KERNEL_TEMPLATE diff --git a/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/GridProcessingKernel3D.hpp b/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/GridProcessingKernel3D.hpp index 646f6ecd09..aad806276a 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/GridProcessingKernel3D.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/Coefficient/GridProcessingKernel3D.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_GRID_PROCESSING_KERNEL_3D_TEMPLATE diff --git a/include/mgard-x/DataRefactoring/MultiDimension/CopyND/AddND.hpp b/include/mgard-x/DataRefactoring/MultiDimension/CopyND/AddND.hpp index 188c9317c1..ed9a77a356 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/CopyND/AddND.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/CopyND/AddND.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../../../Hierarchy/Hierarchy.h" diff --git a/include/mgard-x/DataRefactoring/MultiDimension/CopyND/Copy3D.hpp b/include/mgard-x/DataRefactoring/MultiDimension/CopyND/Copy3D.hpp index b75b659d04..c2f8c5421f 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/CopyND/Copy3D.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/CopyND/Copy3D.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../../../Hierarchy/Hierarchy.h" diff --git a/include/mgard-x/DataRefactoring/MultiDimension/CopyND/CopyND.hpp b/include/mgard-x/DataRefactoring/MultiDimension/CopyND/CopyND.hpp index dd97ddf5b9..1d89734b86 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/CopyND/CopyND.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/CopyND/CopyND.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../../../Hierarchy/Hierarchy.h" diff --git a/include/mgard-x/DataRefactoring/MultiDimension/CopyND/LevelwiseProcessingKernel.hpp b/include/mgard-x/DataRefactoring/MultiDimension/CopyND/LevelwiseProcessingKernel.hpp index bfb433b1a2..ab5281dda2 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/CopyND/LevelwiseProcessingKernel.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/CopyND/LevelwiseProcessingKernel.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_LEVELWISE_PROCESSING_KERNEL_TEMPLATE diff --git a/include/mgard-x/DataRefactoring/MultiDimension/CopyND/SubtractND.hpp b/include/mgard-x/DataRefactoring/MultiDimension/CopyND/SubtractND.hpp index 9ea9cb81eb..0a9e4d5e0d 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/CopyND/SubtractND.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/CopyND/SubtractND.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../../../Hierarchy/Hierarchy.h" diff --git a/include/mgard-x/DataRefactoring/MultiDimension/Correction/CalcCorrection3D.hpp b/include/mgard-x/DataRefactoring/MultiDimension/Correction/CalcCorrection3D.hpp index b656cd9a49..8bd18e3a6d 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/Correction/CalcCorrection3D.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/Correction/CalcCorrection3D.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../../../Hierarchy/Hierarchy.h" diff --git a/include/mgard-x/DataRefactoring/MultiDimension/Correction/CalcCorrectionND.hpp b/include/mgard-x/DataRefactoring/MultiDimension/Correction/CalcCorrectionND.hpp index 280a8bac4a..31674e519a 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/Correction/CalcCorrectionND.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/Correction/CalcCorrectionND.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../../../Hierarchy/Hierarchy.h" diff --git a/include/mgard-x/DataRefactoring/MultiDimension/Correction/IPKFunctor.h b/include/mgard-x/DataRefactoring/MultiDimension/Correction/IPKFunctor.h index cc33b00931..775ae1dd2e 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/Correction/IPKFunctor.h +++ b/include/mgard-x/DataRefactoring/MultiDimension/Correction/IPKFunctor.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_IPK_FUNCTOR diff --git a/include/mgard-x/DataRefactoring/MultiDimension/Correction/IterativeProcessingKernel.hpp b/include/mgard-x/DataRefactoring/MultiDimension/Correction/IterativeProcessingKernel.hpp index b2615cf643..3d17ce62bb 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/Correction/IterativeProcessingKernel.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/Correction/IterativeProcessingKernel.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_ITERATIVE_PROCESSING_KERNEL_TEMPLATE @@ -200,7 +200,7 @@ class Ipk1ReoFunctor : public IterFunctor { tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); - //#pragma unroll 32 + // #pragma unroll 32 for (SIZE i = 1; i < F; i++) { // if (debug) printf("forward %f <- %f %f %f %f\n", // tridiag_forward2( @@ -401,7 +401,7 @@ class Ipk1ReoFunctor : public IterFunctor { vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); - //#pragma unroll 32 + // #pragma unroll 32 for (SIZE i = 1; i < F; i++) { // if (debug) printf("backward %f <- %f %f %f %f\n", diff --git a/include/mgard-x/DataRefactoring/MultiDimension/Correction/IterativeProcessingKernel3D.hpp b/include/mgard-x/DataRefactoring/MultiDimension/Correction/IterativeProcessingKernel3D.hpp index 2b992d0832..953d9b00c9 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/Correction/IterativeProcessingKernel3D.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/Correction/IterativeProcessingKernel3D.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_ITERATIVE_PROCESSING_KERNEL_3D_TEMPLATE @@ -112,7 +112,7 @@ class Ipk1Reo3DFunctor : public IterFunctor { tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); - //#pragma unroll 32 + // #pragma unroll 32 for (SIZE i = 1; i < F; i++) { vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_forward2( vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], @@ -249,7 +249,7 @@ class Ipk1Reo3DFunctor : public IterFunctor { vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); - //#pragma unroll 32 + // #pragma unroll 32 for (SIZE i = 1; i < F; i++) { vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_backward2( vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], diff --git a/include/mgard-x/DataRefactoring/MultiDimension/Correction/LPKFunctor.h b/include/mgard-x/DataRefactoring/MultiDimension/Correction/LPKFunctor.h index 8ed80c9ffb..6cdfbd96e7 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/Correction/LPKFunctor.h +++ b/include/mgard-x/DataRefactoring/MultiDimension/Correction/LPKFunctor.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_LPK_FUNCTOR diff --git a/include/mgard-x/DataRefactoring/MultiDimension/Correction/LinearProcessingKernel.hpp b/include/mgard-x/DataRefactoring/MultiDimension/Correction/LinearProcessingKernel.hpp index eefdb7c81b..b8c1f8e0f2 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/Correction/LinearProcessingKernel.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/Correction/LinearProcessingKernel.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_LINEAR_PROCESSING_KERNEL_TEMPLATE diff --git a/include/mgard-x/DataRefactoring/MultiDimension/Correction/LinearProcessingKernel3D.hpp b/include/mgard-x/DataRefactoring/MultiDimension/Correction/LinearProcessingKernel3D.hpp index d7174d0a9f..ab4f458598 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/Correction/LinearProcessingKernel3D.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/Correction/LinearProcessingKernel3D.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_LINEAR_PROCESSSING_KERNEL_3D_TEMPLATE diff --git a/include/mgard-x/DataRefactoring/MultiDimension/DataRefactoring.h b/include/mgard-x/DataRefactoring/MultiDimension/DataRefactoring.h index 0010f8275a..2e2e385228 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/DataRefactoring.h +++ b/include/mgard-x/DataRefactoring/MultiDimension/DataRefactoring.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_DATA_REFACTORING @@ -87,13 +87,13 @@ template void decompose(Hierarchy &hierarchy, SubArray &v, SubArray w, SubArray b, int start_level, int stop_level, - int queue_idx); + bool orthogonal_projection, int queue_idx); template void recompose(Hierarchy &hierarchy, SubArray &v, SubArray w, SubArray b, int start_level, int stop_level, - int queue_idx); + bool orthogonal_projection, int queue_idx); template void decompose_adaptive_resolution( diff --git a/include/mgard-x/DataRefactoring/MultiDimension/DataRefactoring.hpp b/include/mgard-x/DataRefactoring/MultiDimension/DataRefactoring.hpp index b1111a10d8..89324976dd 100644 --- a/include/mgard-x/DataRefactoring/MultiDimension/DataRefactoring.hpp +++ b/include/mgard-x/DataRefactoring/MultiDimension/DataRefactoring.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../../Hierarchy/Hierarchy.h" @@ -25,16 +25,14 @@ template void decompose(Hierarchy &hierarchy, SubArray &v, SubArray w, SubArray b, int start_level, int stop_level, - int queue_idx) { + bool orthogonal_projection, int queue_idx) { if (start_level < 0 || start_level > hierarchy.l_target()) { - std::cout << log::log_err << "decompose: start_level out of bound.\n"; - exit(-1); + throw std::runtime_error("decompose: start_level out of bound."); } if (stop_level < 0 || stop_level > hierarchy.l_target()) { - std::cout << log::log_err << "decompose: stop_level out of bound.\n"; - exit(-1); + throw std::runtime_error("decompose: stop_level out of bound."); } std::string prefix = "decomp_"; @@ -89,16 +87,18 @@ void decompose(Hierarchy &hierarchy, CopyND(v_fine, w_fine, queue_idx); v_coeff.resize(hierarchy.level_shape(l)); + v_coarse.resize(hierarchy.level_shape(l - 1)); CalcCoefficients3D(hierarchy, w_fine, v_coeff, l, queue_idx); - w_correction.resize(hierarchy.level_shape(l)); - CalcCorrection3D(hierarchy, v_coeff, w_correction, l, queue_idx); + if (orthogonal_projection) { + w_correction.resize(hierarchy.level_shape(l)); + CalcCorrection3D(hierarchy, v_coeff, w_correction, l, queue_idx); - w_correction.resize(hierarchy.level_shape(l - 1)); - v_coarse.resize(hierarchy.level_shape(l - 1)); - AddND(w_correction, v_coarse, queue_idx); - if (multidim_refactoring_debug_print) { - PrintSubarray("after add", v); + w_correction.resize(hierarchy.level_shape(l - 1)); + AddND(w_correction, v_coarse, queue_idx); + if (multidim_refactoring_debug_print) { + PrintSubarray("after add", v); + } } // if (log::level & log::TIME) { // DeviceRuntime::SyncQueue(queue_idx); @@ -180,16 +180,14 @@ template void recompose(Hierarchy &hierarchy, SubArray &v, SubArray w, SubArray b, int start_level, int stop_level, - int queue_idx) { + bool orthogonal_projection, int queue_idx) { if (stop_level < 0 || stop_level > hierarchy.l_target()) { - std::cout << log::log_err << "recompose: stop_level out of bound.\n"; - exit(-1); + throw std::runtime_error("recompose: stop_level out of bound."); } if (start_level < 0 || start_level > hierarchy.l_target()) { - std::cout << log::log_err << "recompose: start_level out of bound.\n"; - exit(-1); + throw std::runtime_error("recompose: start_level out of bound."); } Array workspace; @@ -232,13 +230,15 @@ void recompose(Hierarchy &hierarchy, for (int l = start_level; l < stop_level; l++) { - v_coeff.resize(hierarchy.level_shape(l + 1)); - w_correction.resize(hierarchy.level_shape(l + 1)); - CalcCorrection3D(hierarchy, v_coeff, w_correction, l + 1, queue_idx); + if (orthogonal_projection) { + v_coeff.resize(hierarchy.level_shape(l + 1)); + w_correction.resize(hierarchy.level_shape(l + 1)); + CalcCorrection3D(hierarchy, v_coeff, w_correction, l + 1, queue_idx); - w_correction.resize(hierarchy.level_shape(l)); - v_coarse.resize(hierarchy.level_shape(l)); - SubtractND(w_correction, v_coarse, queue_idx); + w_correction.resize(hierarchy.level_shape(l)); + v_coarse.resize(hierarchy.level_shape(l)); + SubtractND(w_correction, v_coarse, queue_idx); + } v_coeff.resize(hierarchy.level_shape(l + 1)); w_fine.resize(hierarchy.level_shape(l + 1)); diff --git a/include/mgard-x/DataRefactoring/SingleDimension/Coefficient/CalcCoefficients.hpp b/include/mgard-x/DataRefactoring/SingleDimension/Coefficient/CalcCoefficients.hpp index f4db8e3eec..1307f3a5d2 100644 --- a/include/mgard-x/DataRefactoring/SingleDimension/Coefficient/CalcCoefficients.hpp +++ b/include/mgard-x/DataRefactoring/SingleDimension/Coefficient/CalcCoefficients.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../../../Hierarchy/Hierarchy.h" diff --git a/include/mgard-x/DataRefactoring/SingleDimension/Coefficient/CoefficientKernel.hpp b/include/mgard-x/DataRefactoring/SingleDimension/Coefficient/CoefficientKernel.hpp index c5d28912be..3f79df45ff 100644 --- a/include/mgard-x/DataRefactoring/SingleDimension/Coefficient/CoefficientKernel.hpp +++ b/include/mgard-x/DataRefactoring/SingleDimension/Coefficient/CoefficientKernel.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_SINGLE_DIMENSION_COEFFICIENT_KERNEL_TEMPLATE diff --git a/include/mgard-x/DataRefactoring/SingleDimension/Coefficient/CoefficientsRestore.hpp b/include/mgard-x/DataRefactoring/SingleDimension/Coefficient/CoefficientsRestore.hpp index 522bd30e4f..cfeb525a39 100644 --- a/include/mgard-x/DataRefactoring/SingleDimension/Coefficient/CoefficientsRestore.hpp +++ b/include/mgard-x/DataRefactoring/SingleDimension/Coefficient/CoefficientsRestore.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../../../Hierarchy/Hierarchy.h" diff --git a/include/mgard-x/DataRefactoring/SingleDimension/Correction/CalcCorrection.hpp b/include/mgard-x/DataRefactoring/SingleDimension/Correction/CalcCorrection.hpp index 909652793c..047fdb5a35 100644 --- a/include/mgard-x/DataRefactoring/SingleDimension/Correction/CalcCorrection.hpp +++ b/include/mgard-x/DataRefactoring/SingleDimension/Correction/CalcCorrection.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../../../Hierarchy/Hierarchy.h" diff --git a/include/mgard-x/DataRefactoring/SingleDimension/Correction/MassTransKernel.hpp b/include/mgard-x/DataRefactoring/SingleDimension/Correction/MassTransKernel.hpp index ccdcdc2744..28f771b5ce 100644 --- a/include/mgard-x/DataRefactoring/SingleDimension/Correction/MassTransKernel.hpp +++ b/include/mgard-x/DataRefactoring/SingleDimension/Correction/MassTransKernel.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_SINGLE_DIMENSION_MASSTRANS_KERNEL_TEMPLATE diff --git a/include/mgard-x/DataRefactoring/SingleDimension/DataRefactoring.h b/include/mgard-x/DataRefactoring/SingleDimension/DataRefactoring.h index 05b7fb894d..b2702349b7 100644 --- a/include/mgard-x/DataRefactoring/SingleDimension/DataRefactoring.h +++ b/include/mgard-x/DataRefactoring/SingleDimension/DataRefactoring.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_SINGLE_DIMENSION_DATA_REFACTORING diff --git a/include/mgard-x/DataRefactoring/SingleDimension/DataRefactoring.hpp b/include/mgard-x/DataRefactoring/SingleDimension/DataRefactoring.hpp index 3753cef879..8927ce93ef 100644 --- a/include/mgard-x/DataRefactoring/SingleDimension/DataRefactoring.hpp +++ b/include/mgard-x/DataRefactoring/SingleDimension/DataRefactoring.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../../Hierarchy/Hierarchy.h" @@ -26,8 +26,7 @@ void decompose(Hierarchy &hierarchy, int queue_idx) { if (stop_level < 0) { - std::cout << log::log_err << "decompose: stop_level out of bound.\n"; - exit(-1); + throw std::runtime_error("decompose: stop_level out of bound."); } std::vector workspace_shape = @@ -112,8 +111,7 @@ void recompose(Hierarchy &hierarchy, int queue_idx) { if (stop_level < 0 || stop_level > hierarchy.l_target()) { - std::cout << log::log_err << "recompose: stop_level out of bound.\n"; - exit(-1); + throw std::runtime_error("recompose: stop_level out of bound."); } std::vector workspace_shape = diff --git a/include/mgard-x/DomainDecomposer/DomainDecomposer.hpp b/include/mgard-x/DomainDecomposer/DomainDecomposer.hpp index 9e77227a19..164b1c6c6b 100644 --- a/include/mgard-x/DomainDecomposer/DomainDecomposer.hpp +++ b/include/mgard-x/DomainDecomposer/DomainDecomposer.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_DOMAIN_DECOMPOSER_HPP @@ -42,14 +42,14 @@ class DomainDecomposer { estimate_memory_usgae = input_space + output_space; - log::info("Input output space: " + - std::to_string((double)(input_space + output_space) / 1e9) + - " GB"); + log::dbg("Input output space: " + + std::to_string((double)(input_space + output_space) / 1e9) + + " GB"); using HierarchyType = typename OperatorType::HierarchyType; HierarchyType hierarchy; estimate_memory_usgae += hierarchy.EstimateMemoryFootprint(shape); - log::info( + log::dbg( "Hierarchy space: " + std::to_string((double)hierarchy.EstimateMemoryFootprint(shape) / 1e9) + " GB"); @@ -60,11 +60,11 @@ class DomainDecomposer { } estimate_memory_usgae += OperatorType::EstimateMemoryFootprint(shape, config); - log::info("Compressor space: " + - std::to_string( - (double)OperatorType::EstimateMemoryFootprint(shape, config) / - 1e9) + - " GB"); + log::dbg("Operation space: " + + std::to_string( + (double)OperatorType::EstimateMemoryFootprint(shape, config) / + 1e9) + + " GB"); return estimate_memory_usgae; } @@ -76,15 +76,9 @@ class DomainDecomposer { size_t aval = std::min((SIZE)DeviceRuntime::GetAvailableMemory(), config.max_memory_footprint); - log::info("Estimated memory usage: " + std::to_string((double)estm / 1e9) + - "GB, Available: " + std::to_string((double)aval / 1e9) + "GB"); + log::dbg("Estimated memory usage: " + std::to_string((double)estm / 1e9) + + "GB, Available: " + std::to_string((double)aval / 1e9) + "GB"); bool need = estm >= aval; - if (need) { - // Fast copy for domain decomposition need we disable pitched memory - // allocation - log::info("ReduceMemoryFootprint set to 1"); - MemoryManager::ReduceMemoryFootprint = true; - } return need; } @@ -123,8 +117,8 @@ class DomainDecomposer { std::vector subdomain_shape(int subdomain_id) { if (subdomain_id >= _num_subdomains) { - log::err("DomainDecomposer::subdomain_shape wrong subdomain_id."); - exit(-1); + throw std::runtime_error( + "DomainDecomposer: subdomain_shape wrong subdomain_id."); } if (!_domain_decomposed) { return shape; @@ -161,8 +155,7 @@ class DomainDecomposer { _domain_decomposed_sizes[subdomain_id]; return chunk_shape; } else { - log::err("Wrong domain decomposition type."); - exit(-1); + throw std::runtime_error("Wrong domain decomposition type."); return shape; } } @@ -296,8 +289,7 @@ class DomainDecomposer { delete[] chunk_coords[d]; return hierarchy; } else { - log::err("Wrong domain decomposition type."); - exit(-1); + throw std::runtime_error("Wrong domain decomposition type."); } } } @@ -355,8 +347,7 @@ class DomainDecomposer { std::to_string(this->_num_subdomains) + " subdomains using Variable method"); } else { - log::err("Wrong domain decomposition type."); - exit(-1); + throw std::runtime_error("Wrong domain decomposition type."); } } @@ -415,8 +406,7 @@ class DomainDecomposer { std::to_string(this->_num_subdomains) + " subdomains using Variable method"); } else { - log::err("Wrong domain decomposition type."); - exit(-1); + throw std::runtime_error("Wrong domain decomposition type."); } } @@ -438,10 +428,6 @@ class DomainDecomposer { this->_num_subdomains = 1; log::info("DomainDecomposer: no decomposition used"); } else { - // Fast copy for domain decomposition need we disable pitched memory - // allocation - log::info("ReduceMemoryFootprint set to 1"); - MemoryManager::ReduceMemoryFootprint = true; if (config.domain_decomposition == domain_decomposition_type::MaxDim) { this->_num_subdomains = (shape[this->_domain_decomposed_dim] - 1) / this->_domain_decomposed_size + @@ -474,8 +460,7 @@ class DomainDecomposer { std::to_string(this->_num_subdomains) + " subdomains using Variable method"); } else { - log::err("Wrong domain decomposition type."); - exit(-1); + throw std::runtime_error("Wrong domain decomposition type."); } } @@ -497,10 +482,6 @@ class DomainDecomposer { this->_num_subdomains = 1; log::info("DomainDecomposer: no decomposition used"); } else { - // Fast copy for domain decomposition need we disable pitched memory - // allocation - log::info("ReduceMemoryFootprint set to 1"); - MemoryManager::ReduceMemoryFootprint = true; if (config.domain_decomposition == domain_decomposition_type::MaxDim) { this->_num_subdomains = (shape[this->_domain_decomposed_dim] - 1) / this->_domain_decomposed_size + @@ -533,8 +514,7 @@ class DomainDecomposer { std::to_string(this->_num_subdomains) + " subdomains using Variable method"); } else { - log::err("Wrong domain decomposition type."); - exit(-1); + throw std::runtime_error("Wrong domain decomposition type."); } } @@ -633,8 +613,7 @@ class DomainDecomposer { return decomposed_original_data[subdomain_id]; } } else { - log::err("Wrong domain decomposition type."); - exit(-1); + throw std::runtime_error("Wrong domain decomposition type."); } } @@ -649,14 +628,14 @@ class DomainDecomposer { void copy_subdomain(Array &subdomain_data, int subdomain_id, enum subdomain_copy_direction direction, int queue_idx) { if (subdomain_id >= _num_subdomains) { - log::err("DomainDecomposer::copy_subdomain wrong subdomain_id."); - exit(-1); + throw std::runtime_error( + "DomainDecomposer::copy_subdomain wrong subdomain_id."); } if (!_domain_decomposed) { // if (keep_original_data_decomposed) { - // log::err("Do not support restoring to decomposed data when no domain - // decomposition was used."); exit(-1); + // throw std::runtime_error("Do not support restoring to decomposed data + // when no domain decomposition was used."); // } if (direction == subdomain_copy_direction::OriginalToSubdomain) { @@ -677,16 +656,12 @@ class DomainDecomposer { linearized_width, queue_idx); } } else { - // Pitched memory allocation has to be disable for the correctness of the - // following copies - assert(MemoryManager::ReduceMemoryFootprint == true); - bool pitched = false; if (config.domain_decomposition == domain_decomposition_type::MaxDim || config.domain_decomposition == domain_decomposition_type::Variable) { if (keep_original_data_decomposed) { - log::err("Do not support restoring to decomposed data when using " - "MaxDim or Variable"); - exit(-1); + throw std::runtime_error( + "Do not support restoring to decomposed data when using " + "MaxDim or Variable"); } T *data = original_data_ptr(subdomain_id); if (direction == subdomain_copy_direction::OriginalToSubdomain) { @@ -834,12 +809,11 @@ class DomainDecomposer { } } } else { - log::err("Copy subdomain does not support higher than 5D data."); - exit(-1); + throw std::runtime_error( + "Copy subdomain does not support higher than 5D data."); } } else { - log::err("Wrong domain decomposition type."); - exit(-1); + throw std::runtime_error("Wrong domain decomposition type."); } } } @@ -880,4 +854,4 @@ class DomainDecomposer { } // namespace mgard_x -#endif +#endif \ No newline at end of file diff --git a/include/mgard-x/ExternalCompressionLowLevel/ZFP/Compressor.h b/include/mgard-x/ExternalCompressionLowLevel/ZFP/Compressor.h index 33acb10004..f19102b2d6 100644 --- a/include/mgard-x/ExternalCompressionLowLevel/ZFP/Compressor.h +++ b/include/mgard-x/ExternalCompressionLowLevel/ZFP/Compressor.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_ZFP_COMPRESSOR_H @@ -62,6 +62,13 @@ class Compressor : public LossyCompressorInterface { void Recompose(Array &decompressed_data, int queue_idx); + // Same contract as Compressor::DequantizeRecompose. Spelled out here rather + // than inherited because this Recompose() takes no orthogonal_projection + // argument. + void DequantizeRecompose(Array &decompressed_data, + enum error_bound_type ebtype, T tol, T s, T norm, + int queue_idx); + void Dequantize(Array &decompressed_data, enum error_bound_type ebtype, T tol, T s, T norm, int queue_idx); diff --git a/include/mgard-x/ExternalCompressionLowLevel/ZFP/Compressor.hpp b/include/mgard-x/ExternalCompressionLowLevel/ZFP/Compressor.hpp index 067dfac680..355d4caa70 100644 --- a/include/mgard-x/ExternalCompressionLowLevel/ZFP/Compressor.hpp +++ b/include/mgard-x/ExternalCompressionLowLevel/ZFP/Compressor.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include @@ -94,6 +94,14 @@ void Compressor::Deserialize( // Do nothing } +template +void Compressor::DequantizeRecompose( + Array &decompressed_data, enum error_bound_type ebtype, + T tol, T s, T norm, int queue_idx) { + Dequantize(decompressed_data, ebtype, tol, s, norm, queue_idx); + Recompose(decompressed_data, queue_idx); +} + template void Compressor::Recompose( Array &decompressed_data, int queue_idx) { diff --git a/include/mgard-x/ExternalCompressionLowLevel/ZFP/shared.h b/include/mgard-x/ExternalCompressionLowLevel/ZFP/shared.h index c77c1fb973..383c2519a2 100644 --- a/include/mgard-x/ExternalCompressionLowLevel/ZFP/shared.h +++ b/include/mgard-x/ExternalCompressionLowLevel/ZFP/shared.h @@ -1,13 +1,13 @@ #ifndef MGARD_X_ZFP_SHARED_H #define MGARD_X_ZFP_SHARED_H -//#define CUDA_ZFP_RATE_PRINT 1 +// #define CUDA_ZFP_RATE_PRINT 1 typedef unsigned long long ZFPWord; typedef unsigned int uint; #define ZFPWsize ((uint)(CHAR_BIT * sizeof(ZFPWord))) #include "type_info.h" -//#include "zfp.h" +// #include "zfp.h" #include "constants.h" #include diff --git a/include/mgard-x/Hierarchy/Hierarchy.h b/include/mgard-x/Hierarchy/Hierarchy.h index fa6b557312..aa608cd539 100644 --- a/include/mgard-x/Hierarchy/Hierarchy.h +++ b/include/mgard-x/Hierarchy/Hierarchy.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../Config/Config.h" @@ -26,6 +26,7 @@ template struct Hierarchy { SIZE total_num_elems(); SIZE level_num_elems(SIZE level); + std::vector level_num_elems(); SIZE linearized_width(); SIZE l_target(); std::vector level_shape(SIZE level); diff --git a/include/mgard-x/Hierarchy/Hierarchy.hpp b/include/mgard-x/Hierarchy/Hierarchy.hpp index 3baf968f9d..a9b4acb646 100644 --- a/include/mgard-x/Hierarchy/Hierarchy.hpp +++ b/include/mgard-x/Hierarchy/Hierarchy.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../RuntimeX/RuntimeX.h" @@ -28,20 +28,31 @@ void Hierarchy::coord_to_dist(SIZE dof, T *coord, T *dist) { T *h_dist = new T[dof]; for (int i = 0; i < dof; i++) h_dist[i] = 0.0; - // cudaMemcpyAsyncHelper(*this, h_coord, coord, dof * sizeof(T), AUTO, 0); MemoryManager::Copy1D(h_coord, coord, dof, 0); DeviceRuntime::SyncQueue(0); - // this->sync(0); for (int i = 0; i < dof - 1; i++) { h_dist[i] = h_coord[i + 1] - h_coord[i]; } - if (dof != 2 && dof % 2 == 0) { - T last_dist = h_dist[dof - 2]; - h_dist[dof - 2] = last_dist / 2.0; - h_dist[dof - 1] = last_dist / 2.0; + // For non-uniform grids we restore the original "split the last cell in half" + // handling when the number of nodes is even. This preserves the total volume + // and gives the multigrid coarsening a well-defined boundary cell. For + // uniform grids we intentionally skip the split so that every level's spacing + // stays uniform (all interpolation ratios == 0.5), which the + // UniformMultiDimension fast path relies on. + if (dstype == data_structure_type::Cartesian_Grid_Non_Uniform) { + if (dof != 2 && dof % 2 == 0) { + T last_dist = h_dist[dof - 2]; + h_dist[dof - 2] = last_dist / 2.0; + h_dist[dof - 1] = last_dist / 2.0; + } } - // cudaMemcpyAsyncHelper(*this, dist, h_dist, dof * sizeof(T), AUTO, 0); - // this->sync(0); + + // std::cout << "dist-dof: " << dof << " = "; + // for (int i = 0; i < dof-1; i++) { + // std::cout << h_dist[i] << " "; + // } + // std::cout << "\n"; + MemoryManager::Copy1D(dist, h_dist, dof, 0); DeviceRuntime::SyncQueue(0); @@ -58,20 +69,16 @@ void Hierarchy::dist_to_ratio(SIZE dof, T *dist, T *ratio) { T *h_ratio = new T[dof]; for (int i = 0; i < dof; i++) h_ratio[i] = 0.0; - // cudaMemcpyAsyncHelper(*this, h_dist, dist, dof * sizeof(T), AUTO, 0); - // this->sync(0); MemoryManager::Copy1D(h_dist, dist, dof, 0); DeviceRuntime::SyncQueue(0); for (int i = 0; i < dof - 2; i++) { h_ratio[i] = h_dist[i] / (h_dist[i + 1] + h_dist[i]); // printf("dof: %llu ratio: %f\n", dof, h_ratio[i]); } - if (dof % 2 == 0) { - h_ratio[dof - 2] = h_dist[dof - 2] / (h_dist[dof - 1] + h_dist[dof - 2]); - // printf("dof: %llu ratio: %f\n", dof, h_ratio[dof - 2]); - } - // cudaMemcpyAsyncHelper(*this, ratio, h_ratio, dof * sizeof(T), AUTO, 0); - // this->sync(0); + // if (dof % 2 == 0) { + // h_ratio[dof - 2] = h_dist[dof - 2] / (h_dist[dof - 1] + h_dist[dof - 2]); + // printf("dof: %llu ratio: %f\n", dof, h_ratio[dof - 2]); + // } MemoryManager::Copy1D(ratio, h_ratio, dof, 0); DeviceRuntime::SyncQueue(0); delete[] h_dist; @@ -92,16 +99,39 @@ void Hierarchy::reduce_dist(SIZE dof, T *dist, T *dist2) { // this->sync(0); MemoryManager::Copy1D(h_dist, dist, dof, 0); DeviceRuntime::SyncQueue(0); - for (int i = 0; i < dof2 - 1; i++) { - h_dist2[i] = h_dist[i * 2] + h_dist[i * 2 + 1]; - } - if (dof2 != 2 && dof2 % 2 == 0) { - T last_dist = h_dist2[dof2 - 2]; - h_dist2[dof2 - 2] = last_dist / 2.0; - h_dist2[dof2 - 1] = last_dist / 2.0; + if (dstype == data_structure_type::Cartesian_Grid_Non_Uniform) { + // Coarsen by merging each pair of fine cells so the coarse spacing follows + // the true (non-uniform) node geometry, then split the last cell in half + // for even-sized coarse grids (mirrors coord_to_dist). This is the original + // pre-uniform-adjustment behavior. + for (int i = 0; i < dof2 - 1; i++) { + h_dist2[i] = h_dist[i * 2] + h_dist[i * 2 + 1]; + } + if (dof2 != 2 && dof2 % 2 == 0) { + T last_dist = h_dist2[dof2 - 2]; + h_dist2[dof2 - 2] = last_dist / 2.0; + h_dist2[dof2 - 1] = last_dist / 2.0; + } + } else { + // Uniform grid: spread the total extent evenly so every coarse cell is + // identical (spacing doubles each level for 2^k+1, stays uniform + // otherwise). Uniform spacing => all interpolation ratios collapse to 0.5. + T total_dist = 0; + for (int i = 0; i < dof - 1; i++) { + total_dist += h_dist[i]; + } + for (int i = 0; i < dof2 - 1; i++) { + h_dist2[i] = total_dist / (dof2 - 1); + } } // cudaMemcpyAsyncHelper(*this, dist2, h_dist2, dof2 * sizeof(T), AUTO, 0); // this->sync(0); + // std::cout << "dist-dof: " << dof2 << " = "; + // for (int i = 0; i < dof2-1; i++) { + // std::cout << h_dist2[i] << " "; + // } + // std::cout << "\n"; + MemoryManager::Copy1D(dist2, h_dist2, dof2, 0); DeviceRuntime::SyncQueue(0); delete[] h_dist; @@ -171,12 +201,60 @@ void Hierarchy::calc_volume(SIZE dof, T *dist, T *volume, } MemoryManager::Copy1D(h_dist, dist, dof, 0); DeviceRuntime::SyncQueue(0); - // level-wise uniform quantizer - if (dof > 1) { - for (int i = 0; i < dof; i++) { - h_volume[i] = 1.0 / (T)(dof - 1); - } + + T total_dist = 0; + for (int i = 0; i < dof - 1; i++) { + total_dist += h_dist[i]; } + for (int i = 0; i < dof; i++) { + h_volume[i] = total_dist / (dof - 1); + } + // if (dof == 2) { + // h_volume[0] = h_dist[0] / 2; + // h_volume[1] = h_dist[0] / 2; + // } else { + // int node_coeff_div = dof / 2 + 1; + // T *h_node_volume = new T[node_coeff_div]; + // T *h_coeff_volume = new T[dof-node_coeff_div]; + // h_volume[0] = h_dist[0] / 2; + // for (int i = 1; i < dof - 1; i++) { + // h_volume[i] = (h_dist[i - 1] + h_dist[i]) / 2; + // } + // h_volume[dof-1] = h_dist[dof-1] / 2; + + // int node = 0, coeff = 0; + // for (int i = 0; i < dof; i++) { + // if (i % 2 == 0 || i == dof - 1) { + // h_node_volume[node++] = h_volume[i]; + // } else { + // h_coeff_volume[coeff++] = h_volume[i]; + // } + // } + // for (int i = 0; i < node_coeff_div; i++) { + // h_volume[i] = h_node_volume[i]; + // } + // for (int i = node_coeff_div; i < dof; i++) { + // h_volume[i] = h_coeff_volume[i-node_coeff_div]; + // } + // for (int i = 1; i < dof - 1; i++) { + // if (i % 2 == 0) { // node + // h_volume[i / 2] = (h_dist[i - 1] + h_dist[i]) / 2; + // } else { // coeff + // h_volume[node_coeff_div + i / 2] = (h_dist[i - 1] + h_dist[i]) / 2; + // } + // } + // if (dof % 2 != 0) { + // h_volume[node_coeff_div - 1] = h_dist[dof - 2] / 2; + // } else { + // h_volume[node_coeff_div - 1] = h_dist[dof - 1] / 2; + // } + // } + + // std::cout << "vol-dof: " << dof << " = "; + // for (int i = 0; i < dof; i++) { + // std::cout << h_volume[i] << " "; + // } + // std::cout << "\n"; if (reciprocal) { for (int i = 0; i < dof; i++) { @@ -540,12 +618,16 @@ SIZE Hierarchy::total_num_elems() { template SIZE Hierarchy::level_num_elems(SIZE level) { if (level > _l_target + 1) { - log::err("Hierarchy::level_num_elems level out of bound."); - exit(-1); + throw std::runtime_error("Hierarchy::level_num_elems level out of bound."); } return _level_num_elems[level]; } +template +std::vector Hierarchy::level_num_elems() { + return _level_num_elems; +} + template SIZE Hierarchy::linearized_width() { return _linearized_width; @@ -559,8 +641,7 @@ SIZE Hierarchy::l_target() { template std::vector Hierarchy::level_shape(SIZE level) { if (level > _l_target + 1) { - log::err("Hierarchy::level_shape level out of bound."); - exit(-1); + throw std::runtime_error("Hierarchy::level_shape level out of bound."); } return _level_shape[level]; } @@ -568,8 +649,7 @@ std::vector Hierarchy::level_shape(SIZE level) { template SIZE Hierarchy::level_shape(SIZE level, DIM dim) { if (level > _l_target + 1) { - log::err("Hierarchy::level_shape level out of bound."); - exit(-1); + throw std::runtime_error("Hierarchy::level_shape level out of bound."); } if (dim >= D) return 1; @@ -580,8 +660,8 @@ template Array<1, SIZE, DeviceType> & Hierarchy::level_shape_array(SIZE level) { if (level > _l_target + 1) { - log::err("Hierarchy::level_shape_array level out of bound."); - exit(-1); + throw std::runtime_error( + "Hierarchy::level_shape_array level out of bound."); } return _level_shape_array[level]; } @@ -590,8 +670,7 @@ template Array<1, T, DeviceType> &Hierarchy::dist(SIZE level, DIM dim) { if (level > _l_target + 1) { - log::err("Hierarchy::dist level out of bound."); - exit(-1); + throw std::runtime_error("Hierarchy::dist level out of bound."); } if (dim >= D) return dummy_array; @@ -602,8 +681,7 @@ template Array<1, T, DeviceType> &Hierarchy::ratio(SIZE level, DIM dim) { if (level > _l_target + 1) { - log::err("Hierarchy::ratio level out of bound."); - exit(-1); + throw std::runtime_error("Hierarchy::ratio level out of bound."); } if (dim >= D) return dummy_array; @@ -613,8 +691,7 @@ Array<1, T, DeviceType> &Hierarchy::ratio(SIZE level, template Array<1, T, DeviceType> &Hierarchy::am(SIZE level, DIM dim) { if (level > _l_target + 1) { - log::err("Hierarchy::am level out of bound."); - exit(-1); + throw std::runtime_error("Hierarchy::am level out of bound."); } if (dim >= D) return dummy_array; @@ -624,8 +701,7 @@ Array<1, T, DeviceType> &Hierarchy::am(SIZE level, DIM dim) { template Array<1, T, DeviceType> &Hierarchy::bm(SIZE level, DIM dim) { if (level > _l_target + 1) { - log::err("Hierarchy::bm level out of bound."); - exit(-1); + throw std::runtime_error("Hierarchy::bm level out of bound."); } if (dim >= D) return dummy_array; @@ -636,8 +712,7 @@ template Array<1, DIM, DeviceType> & Hierarchy::processed(SIZE idx, DIM &processed_n) { if (idx >= D) { - log::err("Hierarchy::processed idx out of bound."); - exit(-1); + throw std::runtime_error("Hierarchy::processed idx out of bound."); } processed_n = _processed_n[idx]; return _processed_dims[idx]; @@ -647,8 +722,7 @@ template Array<1, DIM, DeviceType> & Hierarchy::unprocessed(SIZE idx, DIM &processed_n) { if (idx >= D) { - log::err("Hierarchy::unprocessed idx out of bound."); - exit(-1); + throw std::runtime_error("Hierarchy::unprocessed idx out of bound."); } processed_n = _unprocessed_n[idx]; return _unprocessed_dims[idx]; @@ -741,18 +815,16 @@ template Hierarchy::Hierarchy(std::vector shape, Config config) { int ret = check_shape(shape); if (ret == -1) { - log::err( + throw std::runtime_error( "Number of dimensions mismatch. mgard_x::Hierarchy not initialized!"); - exit(-1); } if (ret == -2) { - log::err("Size of any dimension cannot be smaller than 3. " - "mgard_x::Hierarchy not initialized!"); + throw std::runtime_error("Size of any dimension cannot be smaller than 3. " + "mgard_x::Hierarchy not initialized!"); std::stringstream ss; for (DIM d = 0; d < D; d++) ss << shape[d] << " "; - log::err("Input shape: " + ss.str()); - exit(-1); + throw std::runtime_error("Input shape: " + ss.str()); } dstype = data_structure_type::Cartesian_Grid_Uniform; std::vector coords = @@ -769,14 +841,12 @@ Hierarchy::Hierarchy(std::vector shape, std::vector coords, Config config) { int ret = check_shape(shape); if (ret == -1) { - log::err( + throw std::runtime_error( "Number of dimensions mismatch. mgard_x::Hierarchy not initialized!"); - exit(-1); } if (ret == -2) { - log::err("Size of any dimension cannot be smaller than 3. " - "mgard_x::Hierarchy not initialized!"); - exit(-1); + throw std::runtime_error("Size of any dimension cannot be smaller than 3. " + "mgard_x::Hierarchy not initialized!"); } dstype = data_structure_type::Cartesian_Grid_Non_Uniform; diff --git a/include/mgard-x/Linearization/CMakeLists.txt b/include/mgard-x/Linearization/CMakeLists.txt deleted file mode 100644 index 3eccbac450..0000000000 --- a/include/mgard-x/Linearization/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -list(APPEND MGARD_X_HEADER - ${CMAKE_CURRENT_SOURCE_DIR}/LevelLinearizer.hpp - ) -set(MGARD_X_HEADER ${MGARD_X_HEADER} PARENT_SCOPE) \ No newline at end of file diff --git a/include/mgard-x/Lossless/Bitcomp.hpp b/include/mgard-x/Lossless/Bitcomp.hpp deleted file mode 100644 index 75c23969ca..0000000000 --- a/include/mgard-x/Lossless/Bitcomp.hpp +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2022, Oak Ridge National Laboratory. - * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 - */ - -#include "nvcomp.hpp" -#include "nvcomp/bitcomp.hpp" - -#ifndef MGARD_X_BITCOMP_TEMPLATE_HPP -#define MGARD_X_BITCOMP_TEMPLATE_HPP - -namespace mgard_x { - -template -Array<1, Byte, DeviceType> -BitcompCompress(SubArray<1, C, DeviceType> &input_data, int algorithm_type) { - using Mem = MemoryManager; - nvcomp::BitcompCompressor compressor(nvcomp::TypeOf(), algorithm_type); - - size_t *temp_bytes; - size_t *output_bytes; - Mem::MallocHost(temp_bytes, 1, 0); - Mem::MallocHost(output_bytes, 1, 0); - DeviceRuntime::SyncQueue(0); - - size_t input_count = input_data.shape(0); - - compressor.configure(input_count * sizeof(C), temp_bytes, output_bytes); - - Array<1, Byte, DeviceType> temp_space({(SIZE)*temp_bytes}); - Array<1, Byte, DeviceType> output_data({(SIZE)*output_bytes}); - - compressor.compress_async(input_data.data(), input_count * sizeof(C), - temp_space.data(), *temp_bytes, output_data.data(), - output_bytes, - DeviceRuntime::GetQueue(0)); - DeviceRuntime::SyncQueue(0); - output_data.shape(0) = *output_bytes; - Mem::FreeHost(temp_bytes); - Mem::FreeHost(output_bytes); - return output_data; -} - -template -Array<1, C, DeviceType> -BitcompDecompress(SubArray<1, Byte, DeviceType> &input_data) { - using Mem = MemoryManager; - nvcomp::BitcompDecompressor decompressor; - - size_t *temp_bytes; - size_t *output_bytes; - Mem::MallocHost(temp_bytes, 1, 0); - Mem::MallocHost(output_bytes, 1, 0); - - decompressor.configure(input_data.data(), input_size, temp_bytes, - output_bytes, DeviceRuntime::GetQueue(0)); - - Array<1, Byte, DeviceType> temp_space({(SIZE)*temp_bytes}); - Array<1, C, DeviceType> output_data({(SIZE)*output_bytes}); - - decompressor.decompress_async(input_data.data(), input_size, - temp_space.data(), *temp_bytes, - output_data.data(), *output_bytes, - DeviceRuntime::GetQueue(0)); - DeviceRuntime::SyncQueue(0); - output_data.shape(0) = (*output_bytes) / sizeof(C); - Mem::FreeHost(temp_bytes); - Mem::FreeHost(output_bytes); - return output_data; -} -} // namespace mgard_x - -#endif \ No newline at end of file diff --git a/include/mgard-x/Lossless/BlockDelta/BlockDelta.hpp b/include/mgard-x/Lossless/BlockDelta/BlockDelta.hpp new file mode 100644 index 0000000000..7c649bb651 --- /dev/null +++ b/include/mgard-x/Lossless/BlockDelta/BlockDelta.hpp @@ -0,0 +1,494 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_BLOCK_DELTA_HPP +#define MGARD_X_BLOCK_DELTA_HPP + +#include "../../RuntimeX/Utilities/Serializer.hpp" +#include "../../Utilities/Types.h" // block_delta_mode_type +#include "../LosslessCompressorInterface.hpp" +#include "BlockDeltaFused.hpp" // CUDA/HIP single-kernel decoupled look-back +#include "BlockDeltaKernels.hpp" // pulls in RuntimeX (types + macros) first + +namespace mgard_x { + +// True for the GPU backends that get the fused single-kernel (decoupled +// look-back) implementation. Everything else uses the portable multi-kernel +// path. The branch is resolved at compile time, so only one path is +// instantiated per backend. +template struct is_gpu_device { + static constexpr bool value = std::is_same::value || + std::is_same::value; +}; + +// BlockDelta lossless backend: a non-entropy alternative to Huffman that +// operates on the same signed quantized-integer stream. See BlockDeltaKernels +// for the encoding scheme. Self-contained: Compress writes a complete buffer +// (header + per-block bit-widths + packed stream); Serialize is therefore a +// no-op and Deserialize parses the header back. +template class BlockDeltaLossless { +public: + BlockDeltaLossless() : initialized(false) {} + + BlockDeltaLossless( + SIZE max_size, int block_size, + block_delta_mode_type mode = block_delta_mode_type::Delta) { + Resize(max_size, block_size, mode, 0); + DeviceRuntime::SyncQueue(0); + } + + void Resize(SIZE max_size, int block_size, block_delta_mode_type mode, + int queue_idx) { + this->initialized = true; + this->max_size = max_size; + this->block_size = block_size; + this->mode = mode; + SIZE max_nblocks = (max_size - 1) / block_size + 1; + bitwidth_array = Array<1, Byte, DeviceType>({max_nblocks}); + bytecount_array = Array<1, size_t, DeviceType>({max_nblocks}); + byte_offset_array = Array<1, size_t, DeviceType>({max_nblocks + 1}); + // Per-block outlier counts (Outlier mode only; tiny, always allocated). + oc_array = Array<1, uint16_t, DeviceType>({max_nblocks}); + // Scan workspace for the multi-kernel path (extended exclusive scan over + // nblocks). Always sized: the GPU backend still uses the portable path for + // Outlier mode, which scans. + DeviceCollective::ScanSumExtended( + max_nblocks, SubArray<1, size_t, DeviceType>(), + SubArray<1, size_t, DeviceType>(), scan_workspace, false, queue_idx); + if constexpr (is_gpu_device::value) { + // Decoupled-look-back state for the fused path: one status word per block + // plus a global tile counter and a total-size slot. + status_array = Array<1, unsigned long long, DeviceType>({max_nblocks}); + counter_array = Array<1, unsigned int, DeviceType>({1}); + total_array = Array<1, unsigned long long, DeviceType>({1}); + } + } + + static size_t EstimateMemoryFootprint(SIZE primary_count, int block_size) { + SIZE max_nblocks = (primary_count - 1) / block_size + 1; + size_t size = max_nblocks * sizeof(Byte); // bitwidth + size += max_nblocks * sizeof(size_t); // bytecount + size += (max_nblocks + 1) * sizeof(size_t); // byte_offset + size += max_nblocks * sizeof(uint16_t); // outlier counts + return size; + } + + // ---- public LosslessCompressorInterface-style entry points -------------- + + void Compress(Array<1, T, DeviceType> &original_data, + Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + if constexpr (is_gpu_device::value) { + CompressFused(original_data, compressed_data, queue_idx); + } else { + CompressPortable(original_data, compressed_data, queue_idx); + } + } + + void Decompress(Array<1, Byte, DeviceType> &compressed_data, + Array<1, T, DeviceType> &decompressed_data, int queue_idx) { + if constexpr (is_gpu_device::value) { + DecompressFused(compressed_data, decompressed_data, queue_idx); + } else { + DecompressPortable(compressed_data, decompressed_data, queue_idx); + } + } + + // Memory-movement stage (no computation): copy the bit-width array out of the + // workspace and write the scalar header into the compressed buffer. Compress + // already wrote the packed bitstream in place; this only fills the metadata + // sections around it. Kept separate from Compress so it can overlap with the + // next subdomain's kernels in the pipeline (same split as Huffman). + void Serialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + SubArray<1, Byte, DeviceType> cd(compressed_data); + size_t n_v = n, nblocks_v = nblocks, bitwidth_bytes_v = nblocks, + oc_bytes_v = nblocks * sizeof(uint16_t), + packed_bytes_v = packed_bytes; + int block_size_v = block_size; + Byte mode_v = (Byte)mode; + SIZE byte_offset = 0; + SerializeArray(cd, signature, kSignatureLen, byte_offset, queue_idx); + SerializeArray(cd, &n_v, 1, byte_offset, queue_idx); + SerializeArray(cd, &block_size_v, 1, byte_offset, queue_idx); + SerializeArray(cd, &mode_v, 1, byte_offset, queue_idx); + SerializeArray(cd, &nblocks_v, 1, byte_offset, queue_idx); + SerializeArray(cd, &bitwidth_bytes_v, 1, byte_offset, queue_idx); + SerializeArray(cd, bitwidth_array.data(), nblocks, byte_offset, + queue_idx); + if (mode == block_delta_mode_type::Outlier) { + SerializeArray(cd, &oc_bytes_v, 1, byte_offset, queue_idx); + SerializeArray(cd, oc_array.data(), nblocks, byte_offset, + queue_idx); + } + SerializeArray(cd, &packed_bytes_v, 1, byte_offset, queue_idx); + // packed[] is already in place (written by Compress); nothing to copy. + DeviceRuntime::SyncQueue(queue_idx); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("BlockDelta serialize", compressed_data.shape(0)); + timer.clear(); + } + } + + void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + ParseHeader(compressed_data, queue_idx); + } + +private: + // Layout walker; mirrors the section order written by Serialize. Returns the + // total compressed size and, via the out-params, where the bit-width array, + // the outlier-count array (Outlier mode only), and the packed bitstream land. + // Layout: signature | n | block_size | mode | nblocks | bitwidth_bytes | + // bitwidth[] | [Outlier: oc_bytes | oc[]] | packed_bytes | packed[] + SIZE ComputeLayout(Byte mode_v, size_t nblocks, size_t packed_bytes, + SIZE &bitwidth_byte_offset, SIZE &oc_byte_offset, + SIZE &packed_byte_offset) { + SIZE off = 0; + advance_with_align(off, kSignatureLen); // signature + advance_with_align(off, 1); // n + advance_with_align(off, 1); // block_size + advance_with_align(off, 1); // mode + advance_with_align(off, 1); // nblocks + advance_with_align(off, 1); // bitwidth_bytes + align_byte_offset(off); + bitwidth_byte_offset = off; + advance_with_align(off, nblocks); // bitwidth[] + oc_byte_offset = 0; + if (mode_v == (Byte)block_delta_mode_type::Outlier) { + advance_with_align(off, 1); // oc_bytes + align_byte_offset(off); + oc_byte_offset = off; + advance_with_align(off, nblocks); // oc[] + } + advance_with_align(off, 1); // packed_bytes + align_byte_offset(off); + packed_byte_offset = off; + advance_with_align(off, packed_bytes); + return off; + } + + void CompressPortable(Array<1, T, DeviceType> &original_data, + Array<1, Byte, DeviceType> &compressed_data, + int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + n = original_data.shape(0); + nblocks = (SIZE)((n - 1) / block_size + 1); + SubArray<1, T, DeviceType> data_subarray(original_data); + SubArray<1, Byte, DeviceType> bitwidth_subarray({(SIZE)nblocks}, + bitwidth_array.data()); + SubArray<1, size_t, DeviceType> bytecount_subarray({(SIZE)nblocks}, + bytecount_array.data()); + SubArray<1, size_t, DeviceType> byte_offset_subarray( + {(SIZE)nblocks + 1}, byte_offset_array.data()); + SubArray<1, uint16_t, DeviceType> oc_subarray({(SIZE)nblocks}, + oc_array.data()); + Byte mode_v = (Byte)mode; + + // 1) per-block bit-width + byte-count (+ outlier count) + DeviceLauncher::Execute( + BlockBitwidthKernel( + data_subarray, (SIZE)n, (SIZE)block_size, (SIZE)nblocks, mode_v, + bitwidth_subarray, bytecount_subarray, oc_subarray), + queue_idx); + + // 2) exclusive scan of byte-counts -> per-block byte offsets (+ total) + DeviceCollective::ScanSumExtended( + (SIZE)nblocks, bytecount_subarray, byte_offset_subarray, scan_workspace, + true, queue_idx); + MemoryManager::Copy1D( + &packed_bytes, byte_offset_subarray.data() + nblocks, 1, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + // 3) size the output buffer. Computation only: the metadata (signature, + // scalar header, bit-width array, outlier counts) is *not* written here -- + // that memory movement is deferred to Serialize so it can overlap with the + // next subdomain's computation in the pipeline. + SIZE bitwidth_byte_offset, oc_byte_offset, packed_byte_offset; + SIZE compressed_size = + ComputeLayout(mode_v, nblocks, packed_bytes, bitwidth_byte_offset, + oc_byte_offset, packed_byte_offset); + compressed_data.resize({compressed_size}, queue_idx); + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + + // 4) pack into the packed region (a kernel writing to its final location, + // mirroring how Huffman's deflate writes its bitstream in place). + SubArray<1, Byte, DeviceType> packed_subarray( + {(SIZE)packed_bytes}, (Byte *)compressed_subarray(packed_byte_offset)); + DeviceLauncher::Execute( + BlockPackKernel(data_subarray, (SIZE)n, (SIZE)block_size, + (SIZE)nblocks, mode_v, bitwidth_subarray, + byte_offset_subarray, packed_subarray), + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + log::info("BlockDelta compress ratio: " + std::to_string(n * sizeof(T)) + + "/" + std::to_string(compressed_size) + " (" + + std::to_string((double)n * sizeof(T) / compressed_size) + ")"); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("BlockDelta compress", n * sizeof(T)); + timer.clear(); + } + } + + void ParseHeader(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + SIZE byte_offset = 0; + + Byte *sig = signature_verify; + DeserializeArray(compressed_subarray, sig, kSignatureLen, byte_offset, + false, queue_idx); + size_t *n_ptr = &n, *nblocks_ptr = &nblocks, + *bitwidth_bytes_ptr = &bitwidth_bytes, + *packed_bytes_ptr = &packed_bytes; + int *block_size_ptr = &block_size; + Byte mode_v = 0, *mode_ptr = &mode_v; + DeserializeArray(compressed_subarray, n_ptr, 1, byte_offset, false, + queue_idx); + DeserializeArray(compressed_subarray, block_size_ptr, 1, byte_offset, + false, queue_idx); + DeserializeArray(compressed_subarray, mode_ptr, 1, byte_offset, false, + queue_idx); + DeserializeArray(compressed_subarray, nblocks_ptr, 1, byte_offset, + false, queue_idx); + DeserializeArray(compressed_subarray, bitwidth_bytes_ptr, 1, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + this->mode = (block_delta_mode_type)mode_v; + for (int i = 0; i < kSignatureLen; i++) { + if (signature_verify[i] != signature[i]) { + throw std::runtime_error("BlockDelta signature mismatch."); + } + } + // zero-copy device pointers into the compressed buffer + DeserializeArray(compressed_subarray, bitwidth_ptr, bitwidth_bytes, + byte_offset, true, queue_idx); + if (this->mode == block_delta_mode_type::Outlier) { + size_t oc_bytes = 0, *oc_bytes_ptr = &oc_bytes; + DeserializeArray(compressed_subarray, oc_bytes_ptr, 1, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + DeserializeArray(compressed_subarray, oc_ptr, + oc_bytes / sizeof(uint16_t), byte_offset, true, + queue_idx); + } + DeserializeArray(compressed_subarray, packed_bytes_ptr, 1, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + DeserializeArray(compressed_subarray, packed_ptr, packed_bytes, + byte_offset, true, queue_idx); + } + + void DecompressPortable(Array<1, Byte, DeviceType> &compressed_data, + Array<1, T, DeviceType> &decompressed_data, + int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + decompressed_data.resize({(SIZE)n}, queue_idx); + SubArray<1, T, DeviceType> data_subarray(decompressed_data); + SubArray<1, Byte, DeviceType> bitwidth_subarray({(SIZE)nblocks}, + bitwidth_ptr); + SubArray<1, Byte, DeviceType> packed_subarray({(SIZE)packed_bytes}, + packed_ptr); + SubArray<1, size_t, DeviceType> bytecount_subarray({(SIZE)nblocks}, + bytecount_array.data()); + SubArray<1, size_t, DeviceType> byte_offset_subarray( + {(SIZE)nblocks + 1}, byte_offset_array.data()); + // Outlier mode reads its per-block counts from the (zero-copy) stream + // pointer; other modes don't touch it. + SubArray<1, uint16_t, DeviceType> oc_subarray( + {(SIZE)nblocks}, + mode == block_delta_mode_type::Outlier ? oc_ptr : oc_array.data()); + Byte mode_v = (Byte)mode; + + // Rebuild per-block byte offsets from the stored bit-widths (+ counts). + DeviceLauncher::Execute( + BlockBytecountKernel( + (SIZE)n, (SIZE)block_size, (SIZE)nblocks, mode_v, bitwidth_subarray, + oc_subarray, bytecount_subarray), + queue_idx); + DeviceCollective::ScanSumExtended( + (SIZE)nblocks, bytecount_subarray, byte_offset_subarray, scan_workspace, + true, queue_idx); + + DeviceLauncher::Execute( + BlockUnpackKernel( + packed_subarray, (SIZE)n, (SIZE)block_size, (SIZE)nblocks, mode_v, + bitwidth_subarray, byte_offset_subarray, data_subarray), + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("BlockDelta decompress", n * sizeof(T)); + timer.clear(); + } + } + + // -------- Fused (CUDA/HIP) path ----------------------------------------- + // Single kernel: the cross-block byte-offset scan is resolved in-kernel via + // decoupled look-back (see BlockDeltaFused.hpp). The byte layout is identical + // to the portable path, so streams are interchangeable across backends. + void CompressFused(Array<1, T, DeviceType> &original_data, + Array<1, Byte, DeviceType> &compressed_data, + int queue_idx) { +#if defined(MGARDX_COMPILE_CUDA) || defined(MGARDX_COMPILE_HIP) + // Outlier mode has variable per-block side records that don't fit the + // single-pass look-back cleanly; use the portable multi-kernel path (which + // also runs on the GPU). Fixed and Delta use the fused single kernel. + if (mode == block_delta_mode_type::Outlier) { + CompressPortable(original_data, compressed_data, queue_idx); + return; + } + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + n = original_data.shape(0); + nblocks = (SIZE)((n - 1) / block_size + 1); + + // Worst case: a block keeps full width -> packed <= n * sizeof(T). The + // bit-width and packed offsets depend only on nblocks, so they are valid + // for the real (trimmed) layout too. + size_t worst_packed = (size_t)n * sizeof(T); + SIZE bitwidth_off, oc_off, packed_off; + SIZE worst_size = ComputeLayout((Byte)mode, nblocks, worst_packed, + bitwidth_off, oc_off, packed_off); + compressed_data.resize({worst_size}, queue_idx); + Byte *base = SubArray<1, Byte, DeviceType>(compressed_data).data(); + + // Reset decoupled-look-back state. + MemoryManager::Memset1D(status_array.data(), (SIZE)nblocks, 0, + queue_idx); + MemoryManager::Memset1D(counter_array.data(), 1, 0, queue_idx); + + // Computation only: the kernel writes the bit-widths into the workspace and + // the packed bitstream into its final location. The metadata memory + // movement (bit-widths + scalar header) is deferred to Serialize. + bool use_delta = (mode != block_delta_mode_type::Fixed); + auto stream = DeviceRuntime::GetQueue(queue_idx); + block_delta_fused::launch_encode( + original_data.data(), (SIZE)n, (SIZE)block_size, (SIZE)nblocks, + use_delta, bitwidth_array.data(), base + packed_off, + status_array.data(), counter_array.data(), total_array.data(), stream); + + unsigned long long total = 0; + MemoryManager::Copy1D(&total, total_array.data(), 1, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + packed_bytes = (size_t)total; + + // Trim to the real size (in-place shrink preserves the kernel's packed + // writes). bitwidth_off is unused here -- Serialize recomputes the layout. + (void)bitwidth_off; + SIZE compressed_size = packed_off + (SIZE)packed_bytes; + compressed_data.resize({compressed_size}, queue_idx); + + log::info( + "BlockDelta(fused) compress ratio: " + std::to_string(n * sizeof(T)) + + "/" + std::to_string(compressed_size) + " (" + + std::to_string((double)n * sizeof(T) / compressed_size) + ")"); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("BlockDelta(fused) compress", n * sizeof(T)); + timer.clear(); + } +#else + CompressPortable(original_data, compressed_data, queue_idx); +#endif + } + + void DecompressFused(Array<1, Byte, DeviceType> &compressed_data, + Array<1, T, DeviceType> &decompressed_data, + int queue_idx) { +#if defined(MGARDX_COMPILE_CUDA) || defined(MGARDX_COMPILE_HIP) + // ParseHeader (via Deserialize) has set n, nblocks, block_size, mode, + // bitwidth_ptr and packed_ptr. Outlier mode uses the portable path. + if (mode == block_delta_mode_type::Outlier) { + DecompressPortable(compressed_data, decompressed_data, queue_idx); + return; + } + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + decompressed_data.resize({(SIZE)n}, queue_idx); + MemoryManager::Memset1D(status_array.data(), (SIZE)nblocks, 0, + queue_idx); + MemoryManager::Memset1D(counter_array.data(), 1, 0, queue_idx); + + bool use_delta = (mode != block_delta_mode_type::Fixed); + auto stream = DeviceRuntime::GetQueue(queue_idx); + block_delta_fused::launch_decode( + packed_ptr, (SIZE)n, (SIZE)block_size, (SIZE)nblocks, use_delta, + bitwidth_ptr, decompressed_data.data(), status_array.data(), + counter_array.data(), stream); + DeviceRuntime::SyncQueue(queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("BlockDelta(fused) decompress", n * sizeof(T)); + timer.clear(); + } +#else + DecompressPortable(compressed_data, decompressed_data, queue_idx); +#endif + } + +public: + bool initialized; + SIZE max_size; + int block_size; + block_delta_mode_type mode; + size_t n; + size_t nblocks; + size_t bitwidth_bytes; + size_t packed_bytes; + +private: + static constexpr int kSignatureLen = 8; + Byte signature[8] = {'M', 'G', 'X', 'B', 'L', 'K', 'D', '\0'}; + Byte signature_verify[8] = {0}; + // zero-copy pointers into the compressed buffer, set by ParseHeader + Byte *bitwidth_ptr = nullptr; + Byte *packed_ptr = nullptr; + uint16_t *oc_ptr = nullptr; + + Array<1, Byte, DeviceType> bitwidth_array; + Array<1, size_t, DeviceType> bytecount_array; + Array<1, size_t, DeviceType> byte_offset_array; + Array<1, uint16_t, DeviceType> oc_array; + Array<1, Byte, DeviceType> scan_workspace; + // Fused (GPU) decoupled-look-back state. + Array<1, unsigned long long, DeviceType> status_array; + Array<1, unsigned int, DeviceType> counter_array; + Array<1, unsigned long long, DeviceType> total_array; +}; + +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/Lossless/BlockDelta/BlockDeltaFused.hpp b/include/mgard-x/Lossless/BlockDelta/BlockDeltaFused.hpp new file mode 100644 index 0000000000..e86683b428 --- /dev/null +++ b/include/mgard-x/Lossless/BlockDelta/BlockDeltaFused.hpp @@ -0,0 +1,204 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_BLOCK_DELTA_FUSED_HPP +#define MGARD_X_BLOCK_DELTA_FUSED_HPP + +#include "BlockDeltaKernels.hpp" // block_delta:: helpers + RuntimeX types/macros + +// Single-kernel (decoupled look-back) implementation of BlockDelta for the GPU +// backends. The cross-block byte-offset prefix sum -- the one global dependency +// in the scheme -- is resolved inside one kernel launch via a Merrill/Garland +// style decoupled look-back, instead of a separate ScanSumExtended pass. +// +// Mapping: one thread per data block. Each thread acquires a *dynamic* tile id +// via atomicAdd so logical order matches dispatch order (this is what keeps the +// look-back deadlock-free under the GPU's non-preemptive scheduling -- see the +// design discussion). It then computes its block's byte count, publishes it, +// looks back to obtain its exclusive prefix (= its byte offset), and packs. +// +// Intra-block work is serial per thread (same as the portable path); the point +// here is the fused cross-block scan, not intra-block parallelism. The byte +// layout is identical to the portable path, so streams are cross-compatible. + +#if defined(MGARDX_COMPILE_CUDA) || defined(MGARDX_COMPILE_HIP) + +namespace mgard_x { +namespace block_delta_fused { + +// Per-tile status word: top 2 bits = state, low 62 bits = value (a byte count +// or a prefix sum, both well within 62 bits). State INVALID == 0 so a zeroed +// status array starts "not ready". +enum : unsigned long long { + ST_INVALID = 0ULL, + ST_AGGREGATE = 1ULL, + ST_PREFIX = 2ULL +}; +__device__ __forceinline__ unsigned long long +make_status(unsigned long long state, unsigned long long val) { + return (state << 62) | (val & ((1ULL << 62) - 1)); +} +__device__ __forceinline__ unsigned long long st_state(unsigned long long s) { + return s >> 62; +} +__device__ __forceinline__ unsigned long long st_val(unsigned long long s) { + return s & ((1ULL << 62) - 1); +} + +// Exclusive prefix of byte counts for tile `b` via decoupled look-back. +// `bc` is this tile's own byte count. Publishes AGGREGATE before walking back, +// then PREFIX once the exclusive prefix is known. +__device__ __forceinline__ unsigned long long +lookback_exclusive_prefix(volatile unsigned long long *status, unsigned int b, + unsigned long long bc) { + status[b] = make_status(ST_AGGREGATE, bc); + __threadfence(); + unsigned long long excl = 0; + long look = (long)b - 1; + while (look >= 0) { + unsigned long long s; + do { + s = status[look]; + } while (st_state(s) == ST_INVALID); + excl += st_val(s); + if (st_state(s) == ST_PREFIX) + break; // PREFIX value already folds in everything <= look + look--; + } + status[b] = make_status(ST_PREFIX, excl + bc); + __threadfence(); + return excl; +} + +template +__global__ void +encode_kernel(const T *__restrict__ data, SIZE n, SIZE block_size, SIZE nblocks, + bool use_delta, Byte *__restrict__ bitwidth, + Byte *__restrict__ packed, volatile unsigned long long *status, + unsigned int *counter, unsigned long long *total) { + using UT = typename std::make_unsigned::type; + unsigned int b = atomicAdd(counter, 1u); + if (b >= nblocks) + return; + SIZE start = b * block_size; + SIZE len = block_size < (n - start) ? block_size : (n - start); + + // bit-width over (delta+)zigzag + T prev = 0; + UT acc = 0; + for (SIZE i = 0; i < len; i++) { + T x = data[start + i]; + acc |= block_delta::zigzag(use_delta ? (T)(x - prev) : x); + prev = x; + } + int bw = block_delta::bit_length(acc); + bitwidth[b] = (Byte)bw; + unsigned long long bc = (unsigned long long)block_delta::block_bytes(bw, len); + + // fused cross-block scan + unsigned long long excl = lookback_exclusive_prefix(status, b, bc); + + // pack into our (disjoint) byte range + Byte *out = packed + excl; + prev = 0; + UT buf = 0; + int cnt = 0; + size_t pos = 0; + for (SIZE i = 0; i < len; i++) { + T x = data[start + i]; + UT z = block_delta::zigzag(use_delta ? (T)(x - prev) : x); + prev = x; + for (int k = 0; k < bw; k++) { + buf |= (UT)((z >> k) & 1) << cnt; + if (++cnt == 8) { + out[pos++] = (Byte)(buf & 0xff); + buf = 0; + cnt = 0; + } + } + } + if (cnt > 0) + out[pos++] = (Byte)(buf & 0xff); + + if (b == nblocks - 1) + *total = excl + bc; // last tile holds the full packed size +} + +template +__global__ void +decode_kernel(const Byte *__restrict__ packed, SIZE n, SIZE block_size, + SIZE nblocks, bool use_delta, const Byte *__restrict__ bitwidth, + T *__restrict__ data, volatile unsigned long long *status, + unsigned int *counter) { + using UT = typename std::make_unsigned::type; + unsigned int b = atomicAdd(counter, 1u); + if (b >= nblocks) + return; + SIZE start = b * block_size; + SIZE len = block_size < (n - start) ? block_size : (n - start); + int bw = (int)bitwidth[b]; + unsigned long long bc = (unsigned long long)block_delta::block_bytes(bw, len); + + unsigned long long excl = lookback_exclusive_prefix(status, b, bc); + + const Byte *in = packed + excl; + T prev = 0; + Byte cur = 0; + int cnt = 0; + size_t pos = 0; + for (SIZE i = 0; i < len; i++) { + UT z = 0; + for (int k = 0; k < bw; k++) { + if (cnt == 0) { + cur = in[pos++]; + cnt = 8; + } + z |= (UT)(cur & 1) << k; + cur >>= 1; + cnt--; + } + T d = block_delta::unzigzag(z); + prev = use_delta ? (T)(prev + d) : d; + data[start + i] = prev; + } +} + +#ifdef MGARDX_COMPILE_CUDA +using gpuStream_t = cudaStream_t; +#else +using gpuStream_t = hipStream_t; +#endif + +template +inline void launch_encode(const T *data, SIZE n, SIZE block_size, SIZE nblocks, + bool use_delta, Byte *bitwidth, Byte *packed, + unsigned long long *status, unsigned int *counter, + unsigned long long *total, gpuStream_t stream) { + SIZE tpb = 256; + SIZE grid = (nblocks - 1) / tpb + 1; + encode_kernel<<>>( + data, n, block_size, nblocks, use_delta, bitwidth, packed, + (volatile unsigned long long *)status, counter, total); +} + +template +inline void launch_decode(const Byte *packed, SIZE n, SIZE block_size, + SIZE nblocks, bool use_delta, const Byte *bitwidth, + T *data, unsigned long long *status, + unsigned int *counter, gpuStream_t stream) { + SIZE tpb = 256; + SIZE grid = (nblocks - 1) / tpb + 1; + decode_kernel<<>>( + packed, n, block_size, nblocks, use_delta, bitwidth, data, + (volatile unsigned long long *)status, counter); +} + +} // namespace block_delta_fused +} // namespace mgard_x + +#endif // CUDA || HIP + +#endif diff --git a/include/mgard-x/Lossless/BlockDelta/BlockDeltaKernels.hpp b/include/mgard-x/Lossless/BlockDelta/BlockDeltaKernels.hpp new file mode 100644 index 0000000000..ec92e21fab --- /dev/null +++ b/include/mgard-x/Lossless/BlockDelta/BlockDeltaKernels.hpp @@ -0,0 +1,544 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_BLOCK_DELTA_KERNELS_HPP +#define MGARD_X_BLOCK_DELTA_KERNELS_HPP + +#include "../../RuntimeX/RuntimeX.h" + +// Portable (multi-kernel) building blocks for the BlockDelta lossless backend. +// +// Three encoding modes (mirroring cuSZp), selected per call: +// Fixed (0): zigzag(value) -> fixed-length pack at per-block width +// Delta (1): zigzag(value-prev) -> fixed-length pack (default) +// Outlier (2): Delta, but values exceeding a per-block budget width are +// peeled into a side record list so a few large values don't +// widen the whole block. +// +// Each block is padded to a whole byte and owns a disjoint byte range of the +// output (no atomics; trivially parallel). For Fixed/Delta a block's byte range +// is just its packed bitstream. For Outlier it is laid out as: +// [outlier_count : 2B][main FLE : ceil(bw*len/8)B][records : oc * (2B pos + +// sizeof(T)B value)] +// where outlier slots in the main stream store 0 and the full zigzag value +// lives in the record. Records are written in increasing position order so +// decode can patch them with an O(len+oc) single scan. +// +// All functors use a 1-thread-per-data-block mapping (the portable correctness +// reference). The CUDA/HIP fused path implements Fixed/Delta in one kernel. + +namespace mgard_x { + +namespace block_delta { + +enum : Byte { MODE_FIXED = 0, MODE_DELTA = 1, MODE_OUTLIER = 2 }; + +// Bytes per outlier record: 2-byte intra-block position + the value. +template MGARDX_CONT_EXEC constexpr int outlier_rec_bytes() { + return 2 + (int)sizeof(T); +} + +template +MGARDX_CONT_EXEC typename std::make_unsigned::type zigzag(T v) { + using UT = typename std::make_unsigned::type; + constexpr int bits = sizeof(T) * 8; + return (UT)((v << 1) ^ (v >> (bits - 1))); +} + +template +MGARDX_CONT_EXEC T unzigzag(typename std::make_unsigned::type z) { + return (T)((z >> 1) ^ (~(z & 1) + 1)); // (z>>1) ^ -(z&1) +} + +template MGARDX_CONT_EXEC int bit_length(UT acc) { + int n = 0; + while (acc) { + n++; + acc >>= 1; + } + return n; +} + +// Bytes to hold `nvalues` values of `bits` bits each, padded to a byte. +MGARDX_CONT_EXEC size_t block_bytes(int bits, SIZE nvalues) { + return (size_t)(((size_t)bits * nvalues + 7) / 8); +} + +// Per-block sizing shared by host/device. Computes the block's fixed-length +// width `bw`, byte count `bc`, and (Outlier only) outlier count `oc`. +template +MGARDX_CONT_EXEC void size_block(const T *data, SIZE start, SIZE len, Byte mode, + int &bw, size_t &bc, int &oc) { + using UT = typename std::make_unsigned::type; + bool use_delta = (mode != MODE_FIXED); + T prev = 0; + UT acc = 0; + int hist[65]; + if (mode == MODE_OUTLIER) { + for (int k = 0; k < 65; k++) + hist[k] = 0; + } + for (SIZE i = 0; i < len; i++) { + T x = data[start + i]; + UT z = zigzag(use_delta ? (T)(x - prev) : x); + prev = x; + acc |= z; + if (mode == MODE_OUTLIER) + hist[bit_length(z)]++; + } + int bw_max = bit_length(acc); + if (mode != MODE_OUTLIER) { + bw = bw_max; + oc = 0; + bc = block_bytes(bw, len); + return; + } + // suffix sums: suf[k] = #{ bit_length(z) >= k } + int suf[66]; + suf[65] = 0; + for (int k = 64; k >= 0; k--) + suf[k] = suf[k + 1] + hist[k]; + int best_bw = bw_max, best_oc = 0; + size_t best_cost = (size_t)-1; + for (int b = 0; b <= bw_max; b++) { + int o = suf[b + 1]; // values needing > b bits + size_t cost = 2 + block_bytes(b, len) + (size_t)o * outlier_rec_bytes(); + if (cost < best_cost) { + best_cost = cost; + best_bw = b; + best_oc = o; + } + } + bw = best_bw; + oc = best_oc; + bc = 2 + block_bytes(best_bw, len) + (size_t)best_oc * outlier_rec_bytes(); +} + +} // namespace block_delta + +// --------------------------------------------------------------------------- +// Kernel 1: per-block bit-width + byte-count (+ outlier count) -- encode +// sizing. +// --------------------------------------------------------------------------- +template +class BlockBitwidthFunctor : public Functor { +public: + MGARDX_CONT BlockBitwidthFunctor() {} + MGARDX_CONT BlockBitwidthFunctor(SubArray<1, T, DeviceType> data, SIZE n, + SIZE block_size, SIZE nblocks, Byte mode, + SubArray<1, Byte, DeviceType> bitwidth, + SubArray<1, size_t, DeviceType> bytecount, + SubArray<1, uint16_t, DeviceType> oc) + : data(data), n(n), block_size(block_size), nblocks(nblocks), mode(mode), + bitwidth(bitwidth), bytecount(bytecount), oc(oc) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE b = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (b >= nblocks) + return; + SIZE start = b * block_size; + SIZE len = block_size < (n - start) ? block_size : (n - start); + int bw; + size_t bc; + int ocount; + block_delta::size_block(data.data(), start, len, mode, bw, bc, ocount); + *bitwidth(b) = (Byte)bw; + *bytecount(b) = bc; + *oc(b) = (uint16_t)ocount; + } + + MGARDX_EXEC void Operation2() {} + MGARDX_EXEC void Operation3() {} + MGARDX_EXEC void Operation4() {} + MGARDX_EXEC void Operation5() {} + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, T, DeviceType> data; + SIZE n, block_size, nblocks; + Byte mode; + SubArray<1, Byte, DeviceType> bitwidth; + SubArray<1, size_t, DeviceType> bytecount; + SubArray<1, uint16_t, DeviceType> oc; +}; + +template +class BlockBitwidthKernel : public Kernel { +public: + constexpr static DIM NumDim = 1; + constexpr static bool EnableAutoTuning() { return false; } + using DataType = T; + constexpr static std::string_view Name = "block_delta_bitwidth"; + MGARDX_CONT BlockBitwidthKernel(SubArray<1, T, DeviceType> data, SIZE n, + SIZE block_size, SIZE nblocks, Byte mode, + SubArray<1, Byte, DeviceType> bitwidth, + SubArray<1, size_t, DeviceType> bytecount, + SubArray<1, uint16_t, DeviceType> oc) + : data(data), n(n), block_size(block_size), nblocks(nblocks), mode(mode), + bitwidth(bitwidth), bytecount(bytecount), oc(oc) {} + + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = BlockBitwidthFunctor; + FunctorType functor(data, n, block_size, nblocks, mode, bitwidth, bytecount, + oc); + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (nblocks - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, tbz, tby, tbx, + functor.shared_memory_size(), queue_idx, std::string(Name)); + } + +private: + SubArray<1, T, DeviceType> data; + SIZE n, block_size, nblocks; + Byte mode; + SubArray<1, Byte, DeviceType> bitwidth; + SubArray<1, size_t, DeviceType> bytecount; + SubArray<1, uint16_t, DeviceType> oc; +}; + +// --------------------------------------------------------------------------- +// Kernel 2: pack each block into its (disjoint) byte range of the output. +// --------------------------------------------------------------------------- +template +class BlockPackFunctor : public Functor { +public: + MGARDX_CONT BlockPackFunctor() {} + MGARDX_CONT BlockPackFunctor(SubArray<1, T, DeviceType> data, SIZE n, + SIZE block_size, SIZE nblocks, Byte mode, + SubArray<1, Byte, DeviceType> bitwidth, + SubArray<1, size_t, DeviceType> byte_offset, + SubArray<1, Byte, DeviceType> packed) + : data(data), n(n), block_size(block_size), nblocks(nblocks), mode(mode), + bitwidth(bitwidth), byte_offset(byte_offset), packed(packed) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + using UT = typename std::make_unsigned::type; + SIZE b = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (b >= nblocks) + return; + int bw = (int)*bitwidth(b); + SIZE start = b * block_size; + SIZE len = block_size < (n - start) ? block_size : (n - start); + bool use_delta = (mode != block_delta::MODE_FIXED); + bool outlier = (mode == block_delta::MODE_OUTLIER); + Byte *out = packed(*byte_offset(b)); + size_t main_start = outlier ? 2 : 0; + + // main fixed-length stream + T prev = 0; + UT buf = 0; + int cnt = 0; + size_t pos = main_start; + for (SIZE i = 0; i < len; i++) { + T x = *data(start + i); + UT z = block_delta::zigzag(use_delta ? (T)(x - prev) : x); + prev = x; + UT sv = + (outlier && bw < (int)(sizeof(T) * 8) && z >= ((UT)1 << bw)) ? 0 : z; + for (int k = 0; k < bw; k++) { + buf |= (UT)((sv >> k) & 1) << cnt; + if (++cnt == 8) { + out[pos++] = (Byte)(buf & 0xff); + buf = 0; + cnt = 0; + } + } + } + if (cnt > 0) + out[pos++] = (Byte)(buf & 0xff); + + if (outlier) { + // records + header + int oc = 0; + size_t rp = main_start + block_delta::block_bytes(bw, len); + prev = 0; + for (SIZE i = 0; i < len; i++) { + T x = *data(start + i); + UT z = block_delta::zigzag((T)(x - prev)); + prev = x; + if (bw < (int)(sizeof(T) * 8) && z >= ((UT)1 << bw)) { + out[rp] = (Byte)(i & 0xff); + out[rp + 1] = (Byte)((i >> 8) & 0xff); + rp += 2; + for (int k = 0; k < (int)sizeof(T); k++) + out[rp + k] = (Byte)((z >> (8 * k)) & 0xff); + rp += sizeof(T); + oc++; + } + } + out[0] = (Byte)(oc & 0xff); + out[1] = (Byte)((oc >> 8) & 0xff); + } + } + + MGARDX_EXEC void Operation2() {} + MGARDX_EXEC void Operation3() {} + MGARDX_EXEC void Operation4() {} + MGARDX_EXEC void Operation5() {} + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, T, DeviceType> data; + SIZE n, block_size, nblocks; + Byte mode; + SubArray<1, Byte, DeviceType> bitwidth; + SubArray<1, size_t, DeviceType> byte_offset; + SubArray<1, Byte, DeviceType> packed; +}; + +template +class BlockPackKernel : public Kernel { +public: + constexpr static DIM NumDim = 1; + constexpr static bool EnableAutoTuning() { return false; } + using DataType = T; + constexpr static std::string_view Name = "block_delta_pack"; + MGARDX_CONT BlockPackKernel(SubArray<1, T, DeviceType> data, SIZE n, + SIZE block_size, SIZE nblocks, Byte mode, + SubArray<1, Byte, DeviceType> bitwidth, + SubArray<1, size_t, DeviceType> byte_offset, + SubArray<1, Byte, DeviceType> packed) + : data(data), n(n), block_size(block_size), nblocks(nblocks), mode(mode), + bitwidth(bitwidth), byte_offset(byte_offset), packed(packed) {} + + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = BlockPackFunctor; + FunctorType functor(data, n, block_size, nblocks, mode, bitwidth, + byte_offset, packed); + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (nblocks - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, tbz, tby, tbx, + functor.shared_memory_size(), queue_idx, std::string(Name)); + } + +private: + SubArray<1, T, DeviceType> data; + SIZE n, block_size, nblocks; + Byte mode; + SubArray<1, Byte, DeviceType> bitwidth; + SubArray<1, size_t, DeviceType> byte_offset; + SubArray<1, Byte, DeviceType> packed; +}; + +// --------------------------------------------------------------------------- +// Kernel 3: rebuild per-block byte-count from bit-width (+oc) -- decode sizing. +// --------------------------------------------------------------------------- +template +class BlockBytecountFunctor : public Functor { +public: + MGARDX_CONT BlockBytecountFunctor() {} + MGARDX_CONT BlockBytecountFunctor(SIZE n, SIZE block_size, SIZE nblocks, + Byte mode, + SubArray<1, Byte, DeviceType> bitwidth, + SubArray<1, uint16_t, DeviceType> oc, + SubArray<1, size_t, DeviceType> bytecount) + : n(n), block_size(block_size), nblocks(nblocks), mode(mode), + bitwidth(bitwidth), oc(oc), bytecount(bytecount) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE b = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (b >= nblocks) + return; + SIZE start = b * block_size; + SIZE len = block_size < (n - start) ? block_size : (n - start); + int bw = (int)*bitwidth(b); + if (mode == block_delta::MODE_OUTLIER) { + *bytecount(b) = 2 + block_delta::block_bytes(bw, len) + + (size_t)(*oc(b)) * block_delta::outlier_rec_bytes(); + } else { + *bytecount(b) = block_delta::block_bytes(bw, len); + } + } + + MGARDX_EXEC void Operation2() {} + MGARDX_EXEC void Operation3() {} + MGARDX_EXEC void Operation4() {} + MGARDX_EXEC void Operation5() {} + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SIZE n, block_size, nblocks; + Byte mode; + SubArray<1, Byte, DeviceType> bitwidth; + SubArray<1, uint16_t, DeviceType> oc; + SubArray<1, size_t, DeviceType> bytecount; +}; + +template +class BlockBytecountKernel : public Kernel { +public: + constexpr static DIM NumDim = 1; + constexpr static bool EnableAutoTuning() { return false; } + using DataType = Byte; + constexpr static std::string_view Name = "block_delta_bytecount"; + MGARDX_CONT BlockBytecountKernel(SIZE n, SIZE block_size, SIZE nblocks, + Byte mode, + SubArray<1, Byte, DeviceType> bitwidth, + SubArray<1, uint16_t, DeviceType> oc, + SubArray<1, size_t, DeviceType> bytecount) + : n(n), block_size(block_size), nblocks(nblocks), mode(mode), + bitwidth(bitwidth), oc(oc), bytecount(bytecount) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = BlockBytecountFunctor; + FunctorType functor(n, block_size, nblocks, mode, bitwidth, oc, bytecount); + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (nblocks - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, tbz, tby, tbx, + functor.shared_memory_size(), queue_idx, std::string(Name)); + } + +private: + SIZE n, block_size, nblocks; + Byte mode; + SubArray<1, Byte, DeviceType> bitwidth; + SubArray<1, uint16_t, DeviceType> oc; + SubArray<1, size_t, DeviceType> bytecount; +}; + +// --------------------------------------------------------------------------- +// Kernel 4: unpack each block back into the signed quantized stream. +// --------------------------------------------------------------------------- +template +class BlockUnpackFunctor : public Functor { +public: + MGARDX_CONT BlockUnpackFunctor() {} + MGARDX_CONT BlockUnpackFunctor(SubArray<1, Byte, DeviceType> packed, SIZE n, + SIZE block_size, SIZE nblocks, Byte mode, + SubArray<1, Byte, DeviceType> bitwidth, + SubArray<1, size_t, DeviceType> byte_offset, + SubArray<1, T, DeviceType> data) + : packed(packed), n(n), block_size(block_size), nblocks(nblocks), + mode(mode), bitwidth(bitwidth), byte_offset(byte_offset), data(data) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + using UT = typename std::make_unsigned::type; + SIZE b = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (b >= nblocks) + return; + int bw = (int)*bitwidth(b); + SIZE start = b * block_size; + SIZE len = block_size < (n - start) ? block_size : (n - start); + bool use_delta = (mode != block_delta::MODE_FIXED); + bool outlier = (mode == block_delta::MODE_OUTLIER); + Byte *in = packed(*byte_offset(b)); + + size_t main_start = 0; + int oc = 0; + if (outlier) { + oc = (int)in[0] | ((int)in[1] << 8); + main_start = 2; + } + size_t rec_start = main_start + block_delta::block_bytes(bw, len); + constexpr int RECSZ = 2 + (int)sizeof(T); + int cursor = 0; + long next_pos = -1; + if (outlier && oc > 0) + next_pos = (long)in[rec_start] | ((long)in[rec_start + 1] << 8); + + T prev = 0; + Byte cur = 0; + int cnt = 0; + size_t pos = main_start; + for (SIZE i = 0; i < len; i++) { + UT z = 0; + for (int k = 0; k < bw; k++) { + if (cnt == 0) { + cur = in[pos++]; + cnt = 8; + } + z |= (UT)(cur & 1) << k; + cur >>= 1; + cnt--; + } + if (outlier && (long)i == next_pos) { + size_t ro = rec_start + (size_t)cursor * RECSZ + 2; + UT v = 0; + for (int k = 0; k < (int)sizeof(T); k++) + v |= (UT)in[ro + k] << (8 * k); + z = v; + cursor++; + next_pos = + (cursor < oc) + ? ((long)in[rec_start + (size_t)cursor * RECSZ] | + ((long)in[rec_start + (size_t)cursor * RECSZ + 1] << 8)) + : -1; + } + T d = block_delta::unzigzag(z); + prev = use_delta ? (T)(prev + d) : d; + *data(start + i) = prev; + } + } + + MGARDX_EXEC void Operation2() {} + MGARDX_EXEC void Operation3() {} + MGARDX_EXEC void Operation4() {} + MGARDX_EXEC void Operation5() {} + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Byte, DeviceType> packed; + SIZE n, block_size, nblocks; + Byte mode; + SubArray<1, Byte, DeviceType> bitwidth; + SubArray<1, size_t, DeviceType> byte_offset; + SubArray<1, T, DeviceType> data; +}; + +template +class BlockUnpackKernel : public Kernel { +public: + constexpr static DIM NumDim = 1; + constexpr static bool EnableAutoTuning() { return false; } + using DataType = T; + constexpr static std::string_view Name = "block_delta_unpack"; + MGARDX_CONT BlockUnpackKernel(SubArray<1, Byte, DeviceType> packed, SIZE n, + SIZE block_size, SIZE nblocks, Byte mode, + SubArray<1, Byte, DeviceType> bitwidth, + SubArray<1, size_t, DeviceType> byte_offset, + SubArray<1, T, DeviceType> data) + : packed(packed), n(n), block_size(block_size), nblocks(nblocks), + mode(mode), bitwidth(bitwidth), byte_offset(byte_offset), data(data) {} + + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = BlockUnpackFunctor; + FunctorType functor(packed, n, block_size, nblocks, mode, bitwidth, + byte_offset, data); + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (nblocks - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, tbz, tby, tbx, + functor.shared_memory_size(), queue_idx, std::string(Name)); + } + +private: + SubArray<1, Byte, DeviceType> packed; + SIZE n, block_size, nblocks; + Byte mode; + SubArray<1, Byte, DeviceType> bitwidth; + SubArray<1, size_t, DeviceType> byte_offset; + SubArray<1, T, DeviceType> data; +}; + +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/Lossless/CPU.hpp b/include/mgard-x/Lossless/CPU.hpp index cf4f829a72..d44043c16e 100644 --- a/include/mgard-x/Lossless/CPU.hpp +++ b/include/mgard-x/Lossless/CPU.hpp @@ -1,18 +1,26 @@ #ifndef MGARD_X_CPU_LOSSLESS_TEMPLATE_HPP #define MGARD_X_CPU_LOSSLESS_TEMPLATE_HPP +#include +#include #include +#include "../RuntimeX/Utilities/Exceptions.h" + /*! CHECK - * Check that the condition holds. If it doesn't print a message and die. + * Check that the condition holds. If it doesn't, throw an exception so the + * calling application can handle the failure instead of the whole process + * being terminated. */ #define CHECK(cond, ...) \ do { \ if (!(cond)) { \ - fprintf(stderr, "%s:%d CHECK(%s) failed: ", __FILE__, __LINE__, #cond); \ - fprintf(stderr, "" __VA_ARGS__); \ - fprintf(stderr, "\n"); \ - exit(1); \ + char mgard_x_check_msg[256]; \ + std::snprintf(mgard_x_check_msg, sizeof(mgard_x_check_msg), \ + "CHECK(" #cond ") failed: " __VA_ARGS__); \ + throw mgard_x::ProcessingException(std::string(__FILE__) + ":" + \ + std::to_string(__LINE__) + " " + \ + mgard_x_check_msg); \ } \ } while (0) diff --git a/include/mgard-x/Lossless/Cascaded.hpp b/include/mgard-x/Lossless/Cascaded.hpp deleted file mode 100644 index 80feb99d9e..0000000000 --- a/include/mgard-x/Lossless/Cascaded.hpp +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2022, Oak Ridge National Laboratory. - * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 - */ - -#ifndef MGARD_X_CASCADED_TEMPLATE_HPP -#define MGARD_X_CASCADED_TEMPLATE_HPP - -#ifdef MGARDX_COMPILE_CUDA - -#include "nvcomp.hpp" -#include "nvcomp/cascaded.h" -#include "nvcomp/cascaded.hpp" -#include "nvcomp/nvcompManagerFactory.hpp" - -#endif - -namespace mgard_x { - -template -Array<1, Byte, DeviceType> -CascadedCompress(SubArray<1, C, DeviceType> &input_data, int n_rle, int n_de, - bool bitpack) { -#ifdef MGARDX_COMPILE_CUDA - using Mem = MemoryManager; - nvcompBatchedCascadedOpts_t options = nvcompBatchedCascadedDefaultOpts; - options.type = nvcomp::TypeOf(); - options.num_RLEs = n_rle; - options.num_deltas = n_de; - options.use_bp = bitpack; - nvcomp::CascadedManager nvcomp_manager{ - options, DeviceRuntime::GetQueue(0)}; - size_t input_count = input_data.shape(0); - auto comp_config = - nvcomp_manager.configure_compression(input_count * sizeof(C)); - Array<1, Byte, DeviceType> output_data( - {(SIZE)comp_config.max_compressed_buffer_size}); - nvcomp_manager.compress(input_data.data(), output_data.data(), comp_config); - output_data.shape(0) = - nvcomp_manager.get_compressed_output_size(output_data.data()); - DeviceRuntime::SyncQueue(0); - return output_data; -#else - log::err( - "Cascaded for is only available on CUDA devices. Portable version is " - "in development."); - exit(-1); -#endif -} - -template -Array<1, C, DeviceType> -CascadedDecompress(SubArray<1, Byte, DeviceType> &input_data) { -#ifdef MGARDX_COMPILE_CUDA - auto decomp_nvcomp_manager = nvcomp::create_manager( - input_data.data(), DeviceRuntime::GetQueue(0)); - size_t input_size = input_data.shape(0); - nvcomp::DecompressionConfig decomp_config = - decomp_nvcomp_manager->configure_decompression(input_data.data()); - Array<1, C, DeviceType> output_data({(SIZE)decomp_config.decomp_data_size}); - decomp_nvcomp_manager->decompress(output_data.data(), input_data.data(), - decomp_config); - output_data.shape(0) = decomp_config.decomp_data_size / sizeof(C); - DeviceRuntime::SyncQueue(0); - return output_data; -#else - log::err( - "Cascaded for is only available on CUDA devices. Portable version is " - "in development."); - exit(-1); -#endif -} -} // namespace mgard_x - -#endif \ No newline at end of file diff --git a/include/mgard-x/Lossless/LZ4.hpp b/include/mgard-x/Lossless/LZ4.hpp deleted file mode 100644 index 5b80ee9a84..0000000000 --- a/include/mgard-x/Lossless/LZ4.hpp +++ /dev/null @@ -1,447 +0,0 @@ -#ifndef MGARD_X_LZ4_TEMPLATE_HPP -#define MGARD_X_LZ4_TEMPLATE_HPP - -#ifdef MGARDX_COMPILE_CUDA - -// #include "ParallelHuffman/Condense.hpp" -#include "nvcomp.hpp" -#include "nvcomp/lz4.hpp" -#include "nvcomp/nvcompManagerFactory.hpp" - -#endif - -namespace mgard_x { - -template class LZ4 { - -public: - LZ4() {} - LZ4(SIZE n, SIZE chunk_size) { - Resize(n, chunk_size, 0); - DeviceRuntime::SyncQueue(0); - } - - void Resize(SIZE n, SIZE chunk_size, int queue_idx) { -#ifdef MGARDX_COMPILE_CUDA - this->chunk_size = chunk_size; - input_data.resize({n}, queue_idx); - nvcompType_t dtype = NVCOMP_TYPE_UCHAR; - nvcomp::LZ4Manager nvcomp_manager{ - chunk_size, dtype, DeviceRuntime::GetQueue(queue_idx)}; - size_t temp_size = nvcomp_manager.get_required_scratch_buffer_size(); - temp_data.resize({temp_size}, queue_idx); - // size_t batch_size = (n + chunk_size - 1) / chunk_size; - // size_t comp_temp_bytes; - // nvcompBatchedLZ4CompressGetTempSize( - // batch_size, chunk_size, nvcompBatchedLZ4DefaultOpts, - // &comp_temp_bytes); - // size_t decomp_temp_bytes; - // nvcompBatchedLZ4DecompressGetTempSize(batch_size, chunk_size, - // &decomp_temp_bytes); - // temp_data.resize({std::max(comp_temp_bytes, decomp_temp_bytes)}, - // queue_idx); nvcompBatchedLZ4CompressGetMaxOutputChunkSize( - // chunk_size, nvcompBatchedLZ4DefaultOpts, &max_out_bytes); - - // host_uncompressed_bytes.resize(batch_size); - // uncompressed_bytes.resize({batch_size}, queue_idx); - // host_uncompressed_ptrs.resize(batch_size); - // uncompressed_data_ptrs.resize({batch_size}, queue_idx); - // compressed_bytes.resize({batch_size}, queue_idx); - // compressed_chunck_data.resize({max_out_bytes * batch_size}, queue_idx); - // host_compressed_ptrs.resize(batch_size); - // compressed_data_ptrs.resize({batch_size}, queue_idx); - // host_compressed_bytes.resize(batch_size); - // host_compressed_write_offset.resize(batch_size); - // compressed_write_offset.resize({batch_size}, queue_idx); -#endif - } - - static size_t EstimateMemoryFootprint(SIZE n, SIZE chunk_size) { - size_t size = 0; -#ifdef MGARDX_COMPILE_CUDA - size += n; - nvcompType_t dtype = NVCOMP_TYPE_UCHAR; - nvcomp::LZ4Manager nvcomp_manager{chunk_size, dtype, - DeviceRuntime::GetQueue(0)}; - size += nvcomp_manager.get_required_scratch_buffer_size(); - // size_t batch_size = (n + chunk_size - 1) / chunk_size; - // size_t comp_temp_bytes; - // nvcompBatchedLZ4CompressGetTempSize( - // batch_size, chunk_size, nvcompBatchedLZ4DefaultOpts, - // &comp_temp_bytes); - // size_t decomp_temp_bytes; - // nvcompBatchedLZ4DecompressGetTempSize(batch_size, chunk_size, - // &decomp_temp_bytes); - // size += std::max(comp_temp_bytes, decomp_temp_bytes); - // size_t max_out_bytes; - // nvcompBatchedLZ4CompressGetMaxOutputChunkSize( - // chunk_size, nvcompBatchedLZ4DefaultOpts, &max_out_bytes); - // size += batch_size * 5 * sizeof(size_t); - // size += max_out_bytes * batch_size; -#endif - return size; - } - - /* - void Compress2(Array<1, Byte, DeviceType> &data, int queue_idx) { - #ifdef MGARDX_COMPILE_CUDA - Timer timer; - if (log::level & log::TIME) { - DeviceRuntime::SyncQueue(queue_idx); - timer.start(); - } - input_data.resize({data.shape(0)}, queue_idx); - MemoryManager::Copy1D(input_data.data(), data.data(), - data.shape(0), queue_idx); - Array<1, Byte, DeviceType> &output_data = data; - - size_t uncompressed_total_bytes = input_data.shape(0); - size_t batch_size = - (uncompressed_total_bytes + chunk_size - 1) / chunk_size; - - // Input size - // host_uncompressed_bytes.resize(batch_size); - for (size_t i = 0; i < batch_size; ++i) { - if (i + 1 < batch_size) { - host_uncompressed_bytes[i] = chunk_size; - } else { - // last chunk may be smaller - host_uncompressed_bytes[i] = - uncompressed_total_bytes - (chunk_size * i); - } - } - // uncompressed_bytes.resize({batch_size}, queue_idx); - MemoryManager::Copy1D(uncompressed_bytes.data(), - host_uncompressed_bytes.data(), - batch_size, queue_idx); - - // Input data - // host_uncompressed_ptrs.resize(batch_size); - SubArray input_data_subarray(input_data); - for (size_t i = 0; i < batch_size; ++i) { - host_uncompressed_ptrs[i] = input_data_subarray(chunk_size * i); - } - // uncompressed_data_ptrs.resize({batch_size}, queue_idx); - MemoryManager::Copy1D(uncompressed_data_ptrs.data(), - host_uncompressed_ptrs.data(), - batch_size, queue_idx); - - // get the maxmimum output size for each chunk - // size_t max_out_bytes; - // nvcompBatchedLZ4CompressGetMaxOutputChunkSize(chunk_size, - // nvcompBatchedLZ4DefaultOpts, &max_out_bytes); - - // Output size - // compressed_bytes.resize({batch_size}, queue_idx); - - // Output data - // compressed_chunck_data.resize({max_out_bytes * batch_size}, queue_idx); - SubArray compressed_chunck_data_subarray(compressed_chunck_data); - // host_compressed_ptrs.resize(batch_size); - for (size_t i = 0; i < batch_size; ++i) { - host_compressed_ptrs[i] = compressed_chunck_data_subarray(i * - chunk_size); - } - // compressed_data_ptrs.resize({batch_size}, queue_idx); - MemoryManager::Copy1D(compressed_data_ptrs.data(), - host_compressed_ptrs.data(), batch_size, - queue_idx); - - // And finally, call the API to compress the data - nvcompStatus_t comp_res = nvcompBatchedLZ4CompressAsync( - uncompressed_data_ptrs.data(), uncompressed_bytes.data(), - chunk_size, // The maximum chunk size - batch_size, temp_data.data(), temp_data.shape(0), - compressed_data_ptrs.data(), compressed_bytes.data(), - nvcompBatchedLZ4DefaultOpts, - DeviceRuntime::GetQueue(queue_idx)); - - if (comp_res != nvcompSuccess) { - std::cerr << "Failed compression!" << std::endl; - assert(comp_res == nvcompSuccess); - } - - // host_compressed_bytes.resize(batch_size); - MemoryManager::Copy1D(host_compressed_bytes.data(), - compressed_bytes.data(), batch_size, - queue_idx); - DeviceRuntime::SyncQueue(queue_idx); - - // host_compressed_write_offset.resize(batch_size); - // compressed_write_offset.resize({batch_size}, queue_idx); - host_compressed_write_offset[0] = 0; - for (int i = 1; i < batch_size; i++) { - host_compressed_write_offset[i] = - host_compressed_write_offset[i - 1] + host_compressed_bytes[i - 1]; - } - MemoryManager::Copy1D(compressed_write_offset.data(), - host_compressed_write_offset.data(), - batch_size, queue_idx); - DeviceRuntime::SyncQueue(queue_idx); - - size_t total_compressed_bytes = - host_compressed_write_offset[batch_size - 1] + - host_compressed_bytes[batch_size - 1]; - - SIZE byte_offset = 0; - SIZE compressed_size; - advance_with_align(byte_offset, 1); - advance_with_align(byte_offset, 1); - advance_with_align(byte_offset, batch_size); - advance_with_align(byte_offset, total_compressed_bytes); - compressed_size = byte_offset; - output_data.resize({(SIZE)(compressed_size)}); - SubArray output_data_subarray(output_data); - - byte_offset = 0; - SerializeArray(output_data_subarray, &uncompressed_total_bytes, 1, - byte_offset, queue_idx); - SerializeArray(output_data_subarray, &chunk_size, 1, byte_offset, - queue_idx); - SerializeArray(output_data_subarray, host_compressed_bytes.data(), - batch_size, byte_offset, queue_idx); - SubArray<1, Byte, DeviceType> output_condensed_subarray( - {compressed_size - byte_offset}, output_data_subarray(byte_offset)); - - DeviceLauncher::Execute( - CondenseKernel( - compressed_chunck_data_subarray, - SubArray(compressed_write_offset), SubArray(compressed_bytes), - output_condensed_subarray, chunk_size), queue_idx); - - log::info("LZ4 block size: " + std::to_string(chunk_size)); - - log::info( - "LZ4 compress ratio: " + std::to_string(uncompressed_total_bytes) + - "/" + std::to_string(compressed_size) + " (" + - std::to_string((double)(uncompressed_total_bytes) / compressed_size) + - ")"); - if (log::level & log::TIME) { - DeviceRuntime::SyncQueue(queue_idx); - timer.end(); - timer.print("LZ4 compress"); - timer.print_throughput("LZ4 compress", uncompressed_total_bytes); - timer.clear(); - } - #else - log::err("LZ4 for is only available on CUDA devices. Portable version is " - "in development."); - exit(-1); - #endif - } - - void Decompress2(Array<1, Byte, DeviceType> &data, int queue_idx) { - #ifdef MGARDX_COMPILE_CUDA - Timer timer; - if (log::level & log::TIME) { - DeviceRuntime::SyncQueue(queue_idx); - timer.start(); - } - input_data.resize({data.shape(0)}, queue_idx); - MemoryManager::Copy1D(input_data.data(), data.data(), - data.shape(0), queue_idx); - Array<1, Byte, DeviceType> &output_data = data; - - SubArray compressed_subarray(input_data); - SubArray uncompressed_subarray(output_data); - size_t byte_offset = 0; - size_t uncompressed_total_bytes; - size_t chunk_size; - size_t batch_size; - - size_t *uncompressed_total_bytes_ptr = &uncompressed_total_bytes; - size_t *chunk_size_ptr = &chunk_size; - DeserializeArray(compressed_subarray, - uncompressed_total_bytes_ptr, 1, byte_offset, false, queue_idx); - DeserializeArray(compressed_subarray, chunk_size_ptr, 1, - byte_offset, false, queue_idx); - DeviceRuntime::SyncQueue(queue_idx); - batch_size = (uncompressed_total_bytes + chunk_size - 1) / chunk_size; - // host_compressed_bytes.resize(batch_size); - size_t *host_compressed_bytes_ptr = host_compressed_bytes.data(); - DeserializeArray(compressed_subarray, host_compressed_bytes_ptr, - batch_size, byte_offset, false, queue_idx); - - // cudaStreamSynchronize(DeviceRuntime::GetQueue(queue_idx)); - - // compressed_bytes.resize({batch_size}, queue_idx); - MemoryManager::Copy1D(compressed_bytes.data(), - host_compressed_bytes.data(), - batch_size, queue_idx); - - // host_compressed_ptrs.resize(batch_size); - for (size_t i = 0; i < batch_size; ++i) { - Byte *ptr; - DeserializeArray(compressed_subarray, ptr, - host_compressed_bytes[i], byte_offset, true, queue_idx); - host_compressed_ptrs[i] = ptr; - } - // compressed_data_ptrs.resize({batch_size}, queue_idx); - MemoryManager::Copy1D(compressed_data_ptrs.data(), - host_compressed_ptrs.data(), batch_size, - queue_idx); - - // host_uncompressed_bytes.resize(batch_size); - for (size_t i = 0; i < batch_size; ++i) { - if (i + 1 < batch_size) { - host_uncompressed_bytes[i] = chunk_size; - } else { - // last chunk may be smaller - host_uncompressed_bytes[i] = - uncompressed_total_bytes - (chunk_size * i); - } - } - // uncompressed_bytes.resize({batch_size}, queue_idx); - MemoryManager::Copy1D(uncompressed_bytes.data(), - host_uncompressed_bytes.data(), - batch_size, queue_idx); - - // host_uncompressed_ptrs.resize(batch_size); - for (size_t i = 0; i < batch_size; ++i) { - host_uncompressed_ptrs[i] = uncompressed_subarray(chunk_size * i); - } - // uncompressed_data_ptrs.resize({batch_size}, queue_idx); - MemoryManager::Copy1D(uncompressed_data_ptrs.data(), - host_uncompressed_ptrs.data(), - batch_size, queue_idx); - - device_statuses.resize({batch_size}); - - nvcompStatus_t decomp_res = nvcompBatchedLZ4DecompressAsync( - compressed_data_ptrs.data(), compressed_bytes.data(), - uncompressed_bytes.data(), uncompressed_bytes.data(), batch_size, - temp_data.data(), temp_data.shape(0), uncompressed_data_ptrs.data(), - device_statuses.data(), - DeviceRuntime::GetQueue(queue_idx)); - - if (decomp_res != nvcompSuccess) { - std::cerr << "Failed compression!" << std::endl; - assert(decomp_res == nvcompSuccess); - } - - if (log::level & log::TIME) { - DeviceRuntime::SyncQueue(queue_idx); - timer.end(); - timer.print("LZ4 decompress"); - timer.print_throughput("LZ4 decompress", uncompressed_total_bytes); - timer.clear(); - } - #else - log::err("LZ4 for is only available on CUDA devices. Portable version is " - "in development."); - exit(-1); - #endif - } - */ - void Compress(Array<1, Byte, DeviceType> &data, int queue_idx) { -#ifdef MGARDX_COMPILE_CUDA - Timer timer; - if (log::level & log::TIME) { - DeviceRuntime::SyncQueue(queue_idx); - timer.start(); - } - // Make a copy of the input data - input_data.resize({data.shape(0)}, queue_idx); - MemoryManager::Copy1D(input_data.data(), data.data(), - data.shape(0), queue_idx); - Array<1, Byte, DeviceType> &output_data = data; - nvcompType_t dtype = NVCOMP_TYPE_UCHAR; - nvcomp::LZ4Manager nvcomp_manager{ - chunk_size, dtype, DeviceRuntime::GetQueue(queue_idx)}; - nvcomp_manager.set_scratch_buffer(temp_data.data()); - size_t input_count = input_data.shape(0); - nvcomp::CompressionConfig comp_config = - nvcomp_manager.configure_compression(input_count); - output_data.resize({(SIZE)comp_config.max_compressed_buffer_size}, - queue_idx); - nvcomp_manager.compress((uint8_t *)input_data.data(), output_data.data(), - comp_config); - output_data.shape(0) = - nvcomp_manager.get_compressed_output_size(output_data.data()); - DeviceRuntime::SyncQueue(queue_idx); - log::info("LZ4 block size: " + std::to_string(chunk_size)); - - log::info("LZ4 compress ratio: " + std::to_string(input_count) + "/" + - std::to_string(output_data.shape(0)) + " (" + - std::to_string((double)(input_count) / output_data.shape(0)) + - ")"); - if (log::level & log::TIME) { - DeviceRuntime::SyncQueue(queue_idx); - timer.end(); - timer.print("LZ4 compress"); - timer.print_throughput("LZ4 compress", input_count); - timer.clear(); - } -#else - log::err("LZ4 is only available on CUDA devices. Portable version is " - "in development."); - exit(-1); -#endif - } - - void Decompress(Array<1, Byte, DeviceType> &data, int queue_idx) { -#ifdef MGARDX_COMPILE_CUDA - Timer timer; - if (log::level & log::TIME) { - DeviceRuntime::SyncQueue(queue_idx); - timer.start(); - } - // Make a copy of the input data - input_data.resize({data.shape(0)}, queue_idx); - MemoryManager::Copy1D(input_data.data(), data.data(), - data.shape(0), queue_idx); - Array<1, Byte, DeviceType> &output_data = data; - nvcompType_t dtype = NVCOMP_TYPE_UCHAR; - nvcomp::LZ4Manager nvcomp_manager{ - chunk_size, dtype, DeviceRuntime::GetQueue(queue_idx)}; - nvcomp_manager.set_scratch_buffer(temp_data.data()); - size_t input_size = input_data.shape(0); - nvcomp::DecompressionConfig decomp_config = - nvcomp_manager.configure_decompression(input_data.data()); - output_data.resize({(SIZE)decomp_config.decomp_data_size}, queue_idx); - nvcomp_manager.decompress(output_data.data(), input_data.data(), - decomp_config); - output_data.shape(0) = decomp_config.decomp_data_size; - if (log::level & log::TIME) { - DeviceRuntime::SyncQueue(queue_idx); - timer.end(); - timer.print("LZ4 decompress"); - timer.print_throughput("LZ4 decompress", output_data.shape(0)); - timer.clear(); - } -#else - log::err("LZ4 is only available on CUDA devices. Portable version is " - "in development."); - exit(-1); -#endif - } - -#ifdef MGARDX_COMPILE_CUDA - // Workspace - size_t chunk_size; - // size_t max_out_bytes; - - Array<1, Byte, DeviceType> input_data; - Array<1, Byte, DeviceType> temp_data; - - // std::vector host_uncompressed_bytes; - // std::vector host_uncompressed_ptrs; - // Array<1, size_t, DeviceType> uncompressed_bytes; - // Array<1, void *, DeviceType> uncompressed_data_ptrs; - - // std::vector host_compressed_ptrs; - // Array<1, void *, DeviceType> compressed_data_ptrs; - // Array<1, Byte, DeviceType> compressed_chunck_data; - // std::vector host_compressed_bytes; - // Array<1, size_t, DeviceType> compressed_bytes; - - // std::vector host_compressed_write_offset; - // Array<1, size_t, DeviceType> compressed_write_offset; - - // Array<1, nvcompStatus_t, DeviceType> device_statuses; -#endif -}; - -} // namespace mgard_x - -#endif diff --git a/include/mgard-x/Lossless/LZ4/LZ4.hpp b/include/mgard-x/Lossless/LZ4/LZ4.hpp new file mode 100644 index 0000000000..85ea8eecd6 --- /dev/null +++ b/include/mgard-x/Lossless/LZ4/LZ4.hpp @@ -0,0 +1,329 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_LZ4_HPP +#define MGARD_X_LZ4_HPP + +#include "../../RuntimeX/Utilities/Serializer.hpp" +#include "LZ4Fused.hpp" // CUDA warp-per-chunk compress kernel (optional fast path) +#include "LZ4Kernels.hpp" + +namespace mgard_x { + +// Portable, nvcomp-free LZ4 (warp-cooperative on GPU via LZ4Fused, the size-1 +// SubGroupScalar functor on CPU). In-place Compress/Decompress(Array<1,Byte>&): +// the composed pipeline feeds it the Huffman output (Huffman_LZ4) or the raw +// quantized byte stream (standalone LZ4), and gets it back. The buffer is +// fully self-describing -- Compress writes the header; Decompress parses it -- +// so there is no separate Serialize stage (unlike BlockDelta, the upstream +// caller already serializes Huffman before handing us the byte stream). +// +// Container layout written by Compress: +// signature(8) | uncompressed_total:size_t | chunk_size:size_t | +// nchunks:size_t | comp_bytes[nchunks]:size_t | packed_bytes:size_t | +// packed[] +// comp_bytes[] is the per-chunk compressed length; Decompress exclusive-scans +// it to recover each chunk's offset into packed[] (same trick as BlockDelta). +template class LZ4 { +public: + LZ4() : initialized(false) {} + + // The warp-per-chunk CUDA compress kernel (LZ4Fused) replaces the portable + // 1-thread-per-chunk functor when available; it keeps its hash table in + // shared memory, so the global htable_array is not needed on that path. True + // for the GPU backend (CUDA/HIP) that has the fused warp-per-chunk path + // compiled in this TU. SERIAL/OpenMP/other always use the portable functor. + static constexpr bool fused_backend() { + bool r = false; +#if defined(MGARDX_COMPILE_CUDA) + r = r || std::is_same::value; +#endif +#if defined(MGARDX_COMPILE_HIP) + r = r || std::is_same::value; +#endif +#if defined(MGARDX_COMPILE_SYCL) + r = r || std::is_same::value; +#endif + return r; + } + + bool uses_fused() const { +#if defined(MGARDX_COMPILE_CUDA) || defined(MGARDX_COMPILE_HIP) || \ + defined(MGARDX_COMPILE_SYCL) + return fused_backend() && lz4_fused::fused_ok(chunk_size); +#else + return false; +#endif + } + + LZ4(SIZE n, SIZE chunk_size) { + Resize(n, chunk_size, 0); + DeviceRuntime::SyncQueue(0); + } + + void Resize(SIZE n, SIZE chunk_size, int queue_idx) { + this->initialized = true; + this->max_size = n; + this->chunk_size = chunk_size; + SIZE max_nchunks = (n - 1) / chunk_size + 1; + this->max_chunk_out = (SIZE)lz4::compress_bound(chunk_size); + + input_data = Array<1, Byte, DeviceType>({n}); + // Global per-chunk hash table only feeds the portable functor path; the + // fused CUDA kernel uses shared memory instead, so skip the (large) alloc. + if (!uses_fused()) + htable_array = Array<1, uint16_t, DeviceType>( + {(SIZE)((size_t)max_nchunks * lz4::HASH_SIZE)}); + scratch_array = Array<1, Byte, DeviceType>( + {(SIZE)((size_t)max_nchunks * max_chunk_out)}); + comp_bytes_array = Array<1, size_t, DeviceType>({max_nchunks}); + byte_offset_array = Array<1, size_t, DeviceType>({max_nchunks + 1}); + DeviceCollective::ScanSumExtended( + max_nchunks, SubArray<1, size_t, DeviceType>(), + SubArray<1, size_t, DeviceType>(), scan_workspace, false, queue_idx); + } + + static size_t EstimateMemoryFootprint(SIZE n, SIZE chunk_size) { + SIZE max_nchunks = (n - 1) / chunk_size + 1; + size_t max_chunk_out = lz4::compress_bound(chunk_size); + size_t size = n; // input + size += (size_t)max_nchunks * lz4::HASH_SIZE * sizeof(uint16_t); // htable + size += (size_t)max_nchunks * max_chunk_out; // scratch + size += (size_t)max_nchunks * sizeof(size_t); // comp_bytes + size += (size_t)(max_nchunks + 1) * sizeof(size_t); // offsets + return size; + } + + void Compress(Array<1, Byte, DeviceType> &data, int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + // Snapshot the input; `data` is reused as the compressed output. + SIZE input_count = data.shape(0); + n = input_count; + nchunks = (n - 1) / chunk_size + 1; + input_data.resize({n}, queue_idx); + MemoryManager::Copy1D(input_data.data(), data.data(), n, + queue_idx); + + SubArray<1, Byte, DeviceType> input_subarray({(SIZE)n}, input_data.data()); + SubArray<1, Byte, DeviceType> scratch_subarray( + {(SIZE)((size_t)nchunks * max_chunk_out)}, scratch_array.data()); + SubArray<1, size_t, DeviceType> comp_bytes_subarray( + {(SIZE)nchunks}, comp_bytes_array.data()); + SubArray<1, size_t, DeviceType> byte_offset_subarray( + {(SIZE)nchunks + 1}, byte_offset_array.data()); + + // 1) compress each chunk into its scratch slot; record per-chunk size. + // CUDA: warp-per-chunk fused kernel (shared-memory hash, parallel match + // scan). Other backends / oversized chunks: portable 1-thread functor. + bool did_fused = false; +#if defined(MGARDX_COMPILE_CUDA) || defined(MGARDX_COMPILE_HIP) || \ + defined(MGARDX_COMPILE_SYCL) + if constexpr (fused_backend()) { + if (lz4_fused::fused_ok(chunk_size)) { + auto stream = DeviceRuntime::GetQueue(queue_idx); + lz4_fused::launch_compress(input_subarray.data(), (SIZE)n, + (int)chunk_size, (SIZE)nchunks, + (SIZE)max_chunk_out, scratch_subarray.data(), + comp_bytes_subarray.data(), stream); + did_fused = true; + } + } +#endif + if (!did_fused) { + SubArray<1, uint16_t, DeviceType> htable_subarray( + {(SIZE)((size_t)nchunks * lz4::HASH_SIZE)}, htable_array.data()); + DeviceLauncher::Execute( + LZ4ChunkCompressKernel( + input_subarray, (SIZE)n, (SIZE)chunk_size, (SIZE)nchunks, + (SIZE)max_chunk_out, htable_subarray, scratch_subarray, + comp_bytes_subarray), + queue_idx); + } + + // 2) exclusive scan of per-chunk sizes -> offsets into packed[] (+ total). + DeviceCollective::ScanSumExtended( + (SIZE)nchunks, comp_bytes_subarray, byte_offset_subarray, + scan_workspace, true, queue_idx); + MemoryManager::Copy1D( + &packed_bytes, byte_offset_subarray.data() + nchunks, 1, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + // 3) size the output and 4) gather chunks into the packed region. + SIZE packed_byte_offset; + SIZE compressed_size = + ComputeLayout(nchunks, packed_bytes, packed_byte_offset); + data.resize({compressed_size}, queue_idx); + SubArray<1, Byte, DeviceType> out_subarray(data); + SubArray<1, Byte, DeviceType> packed_subarray( + {(SIZE)packed_bytes}, (Byte *)out_subarray(packed_byte_offset)); + DeviceLauncher::Execute( + LZ4CondenseKernel(scratch_subarray, (SIZE)nchunks, + (SIZE)max_chunk_out, comp_bytes_subarray, + byte_offset_subarray, packed_subarray), + queue_idx); + + // Header (everything except packed[], which the condense kernel wrote). + size_t n_v = n, chunk_v = chunk_size, nchunks_v = nchunks, + packed_v = packed_bytes; + SIZE byte_offset = 0; + SerializeArray(out_subarray, signature, kSignatureLen, byte_offset, + queue_idx); + SerializeArray(out_subarray, &n_v, 1, byte_offset, queue_idx); + SerializeArray(out_subarray, &chunk_v, 1, byte_offset, queue_idx); + SerializeArray(out_subarray, &nchunks_v, 1, byte_offset, queue_idx); + SerializeArray(out_subarray, comp_bytes_array.data(), nchunks, + byte_offset, queue_idx); + SerializeArray(out_subarray, &packed_v, 1, byte_offset, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + log::info("LZ4 compress ratio: " + std::to_string(input_count) + "/" + + std::to_string(compressed_size) + " (" + + std::to_string((double)input_count / compressed_size) + ")"); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("LZ4 compress", input_count); + timer.clear(); + } + } + + void Decompress(Array<1, Byte, DeviceType> &data, int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + // Snapshot the compressed input; `data` is reused as the output. + input_data.resize({data.shape(0)}, queue_idx); + MemoryManager::Copy1D(input_data.data(), data.data(), + data.shape(0), queue_idx); + SubArray<1, Byte, DeviceType> in_subarray(input_data); + + // Parse header. comp_bytes_ptr / packed_ptr are zero-copy into in_subarray. + SIZE byte_offset = 0; + Byte *sig = signature_verify; + DeserializeArray(in_subarray, sig, kSignatureLen, byte_offset, false, + queue_idx); + size_t *n_ptr = &n, *chunk_ptr = &chunk_size_runtime, + *nchunks_ptr = &nchunks, *packed_ptr_sz = &packed_bytes; + DeserializeArray(in_subarray, n_ptr, 1, byte_offset, false, + queue_idx); + DeserializeArray(in_subarray, chunk_ptr, 1, byte_offset, false, + queue_idx); + DeserializeArray(in_subarray, nchunks_ptr, 1, byte_offset, false, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + for (int i = 0; i < kSignatureLen; i++) + if (signature_verify[i] != signature[i]) + throw std::runtime_error("LZ4 signature mismatch."); + + size_t *comp_bytes_ptr = nullptr; + DeserializeArray(in_subarray, comp_bytes_ptr, nchunks, byte_offset, + true, queue_idx); // zero-copy device pointer + DeserializeArray(in_subarray, packed_ptr_sz, 1, byte_offset, false, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + Byte *packed_ptr = nullptr; + DeserializeArray(in_subarray, packed_ptr, packed_bytes, byte_offset, + true, queue_idx); + + SubArray<1, size_t, DeviceType> comp_bytes_subarray({(SIZE)nchunks}, + comp_bytes_ptr); + SubArray<1, size_t, DeviceType> byte_offset_subarray( + {(SIZE)nchunks + 1}, byte_offset_array.data()); + SubArray<1, Byte, DeviceType> packed_subarray({(SIZE)packed_bytes}, + packed_ptr); + + // Rebuild per-chunk offsets into packed[] from the stored sizes. + DeviceCollective::ScanSumExtended( + (SIZE)nchunks, comp_bytes_subarray, byte_offset_subarray, + scan_workspace, true, queue_idx); + + data.resize({(SIZE)n}, queue_idx); + SubArray<1, Byte, DeviceType> out_subarray(data); + // CUDA: warp-per-chunk fused decoder (cooperative literal/match copy). + // Other backends / oversized chunks: portable 1-thread decode functor. + bool did_fused = false; +#if defined(MGARDX_COMPILE_CUDA) || defined(MGARDX_COMPILE_HIP) || \ + defined(MGARDX_COMPILE_SYCL) + if constexpr (fused_backend()) { + if (lz4_fused::fused_ok(chunk_size_runtime)) { + auto stream = DeviceRuntime::GetQueue(queue_idx); + lz4_fused::launch_decompress(packed_subarray.data(), + byte_offset_subarray.data(), (SIZE)n, + (int)chunk_size_runtime, (SIZE)nchunks, + out_subarray.data(), stream); + did_fused = true; + } + } +#endif + if (!did_fused) { + DeviceLauncher::Execute( + LZ4ChunkDecompressKernel( + packed_subarray, byte_offset_subarray, (SIZE)n, + (SIZE)chunk_size_runtime, (SIZE)nchunks, out_subarray), + queue_idx); + } + DeviceRuntime::SyncQueue(queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("LZ4 decompress", n); + timer.clear(); + } + } + +private: + // Layout walker; mirrors the header order written by Compress. Returns the + // total compressed size and, via the out-param, where packed[] begins. + SIZE ComputeLayout(size_t nchunks, size_t packed_bytes, + SIZE &packed_byte_offset) { + SIZE off = 0; + advance_with_align(off, kSignatureLen); // signature + advance_with_align(off, 1); // n + advance_with_align(off, 1); // chunk_size + advance_with_align(off, 1); // nchunks + advance_with_align(off, nchunks); // comp_bytes[] + advance_with_align(off, 1); // packed_bytes + align_byte_offset(off); + packed_byte_offset = off; + advance_with_align(off, packed_bytes); // packed[] + return off; + } + +public: + bool initialized; + SIZE max_size; + SIZE chunk_size; + SIZE max_chunk_out; + size_t n = 0; + size_t nchunks = 0; + size_t packed_bytes = 0; + size_t chunk_size_runtime = 0; // chunk_size read back on Decompress + +private: + static constexpr int kSignatureLen = 8; + Byte signature[8] = {'M', 'G', 'X', 'L', 'Z', '4', 'P', '\0'}; + Byte signature_verify[8] = {0}; + + Array<1, Byte, DeviceType> input_data; + Array<1, uint16_t, DeviceType> htable_array; + Array<1, Byte, DeviceType> scratch_array; + Array<1, size_t, DeviceType> comp_bytes_array; + Array<1, size_t, DeviceType> byte_offset_array; + Array<1, Byte, DeviceType> scan_workspace; +}; + +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/Lossless/LZ4/LZ4Fused.hpp b/include/mgard-x/Lossless/LZ4/LZ4Fused.hpp new file mode 100644 index 0000000000..fa1b418219 --- /dev/null +++ b/include/mgard-x/Lossless/LZ4/LZ4Fused.hpp @@ -0,0 +1,177 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_LZ4_FUSED_HPP +#define MGARD_X_LZ4_FUSED_HPP + +#include "LZ4Kernels.hpp" // lz4_coop:: cooperative codec (SubGroup<> lives in the DeviceAdapters) + +// GPU launch wrappers for the portable LZ4 backend: one sub-group (warp on +// CUDA, wavefront on HIP) per chunk, running the SAME lz4_coop::compress_chunk +// / decompress_chunk used by every backend -- here with the GPU sub-group, so +// the hash table lives in shared memory and the match scan / byte copies run +// across lanes. The portable functor path (LZ4Kernels) runs the identical codec +// with a size-1 SubGroupScalar. Both __global__ kernels are templated so their +// definitions have weak linkage and don't clash across GPU TUs (a non-template +// __global__ in a header -> nvlink "merge_elf failed"). chunk_size must be +// <= 65535 (uint16 hash slot). The HIP path is UNTESTED (no AMD hardware here). + +#if defined(MGARDX_COMPILE_CUDA) || defined(MGARDX_COMPILE_HIP) + +namespace mgard_x { +namespace lz4_fused { + +// Per-backend sub-group type and stream type. +#if defined(MGARDX_COMPILE_CUDA) +using FusedSubGroup = SubGroup; +using gpuStream_t = cudaStream_t; +#else +using FusedSubGroup = SubGroup; +using gpuStream_t = hipStream_t; +#endif + +static constexpr int WARPS_PER_BLOCK = 4; +static constexpr int SG = FusedSubGroup::size(); + +template +__global__ void compress_kernel(const Byte *__restrict__ input, SIZE n, + int chunk_size, SIZE nchunks, + SIZE max_chunk_out, Byte *__restrict__ scratch, + size_t *__restrict__ comp_bytes) { + constexpr int HS = 1 << lz4::HASH_LOG; + __shared__ uint16_t s_ht[WPB][HS]; + const int warp = threadIdx.x / FusedSubGroup::size(); + const SIZE c = (SIZE)blockIdx.x * WPB + warp; + if (c >= nchunks) + return; + const SIZE start = c * (SIZE)chunk_size; + const int len = + (int)((SIZE)chunk_size < (n - start) ? (SIZE)chunk_size : (n - start)); + FusedSubGroup sg; + int cb = lz4_coop::compress_chunk( + sg, input + start, len, scratch + (size_t)c * max_chunk_out, s_ht[warp]); + if (sg.lane() == 0) + comp_bytes[c] = (size_t)cb; +} + +template +__global__ void decompress_kernel(const Byte *__restrict__ packed, + const size_t *__restrict__ byte_offset, + SIZE n, int chunk_size, SIZE nchunks, + Byte *__restrict__ output) { + const int warp = threadIdx.x / FusedSubGroup::size(); + const SIZE c = (SIZE)blockIdx.x * WPB + warp; + if (c >= nchunks) + return; + const SIZE start = c * (SIZE)chunk_size; + const int outLen = + (int)((SIZE)chunk_size < (n - start) ? (SIZE)chunk_size : (n - start)); + FusedSubGroup sg; + lz4_coop::decompress_chunk(sg, packed + byte_offset[c], output + start, + outLen); +} + +inline void launch_compress(const Byte *input, SIZE n, int chunk_size, + SIZE nchunks, SIZE max_chunk_out, Byte *scratch, + size_t *comp_bytes, gpuStream_t stream) { + dim3 block(WARPS_PER_BLOCK * SG); + dim3 grid((unsigned)((nchunks + WARPS_PER_BLOCK - 1) / WARPS_PER_BLOCK)); + compress_kernel<<>>( + input, n, chunk_size, nchunks, max_chunk_out, scratch, comp_bytes); +} + +inline void launch_decompress(const Byte *packed, const size_t *byte_offset, + SIZE n, int chunk_size, SIZE nchunks, + Byte *output, gpuStream_t stream) { + dim3 block(WARPS_PER_BLOCK * SG); + dim3 grid((unsigned)((nchunks + WARPS_PER_BLOCK - 1) / WARPS_PER_BLOCK)); + decompress_kernel<<>>( + packed, byte_offset, n, chunk_size, nchunks, output); +} + +// Fused chunks are valid only when a position fits the uint16 hash slot. +inline bool fused_ok(SIZE chunk_size) { return chunk_size <= 65535; } + +} // namespace lz4_fused +} // namespace mgard_x + +#endif // CUDA || HIP + +// --------------------------------------------------------------------------- +// SYCL path (oneAPI/DPC++). UNTESTED -- no Intel GPU / SYCL toolchain here. +// SYCL can't use <<<>>>; it submits an nd_range parallel_for with a local +// accessor for the shared hash table. One sub-group per chunk runs the SAME +// lz4_coop codec via SubGroup. reqd_sub_group_size pins the width to 32. +// --------------------------------------------------------------------------- +#if defined(MGARDX_COMPILE_SYCL) +namespace mgard_x { +namespace lz4_fused { + +static constexpr int WARPS_PER_BLOCK = 4; +static constexpr int SYCL_SG = 32; + +inline bool fused_ok(SIZE chunk_size) { return chunk_size <= 65535; } + +inline void launch_compress(const Byte *input, SIZE n, int chunk_size, + SIZE nchunks, SIZE max_chunk_out, Byte *scratch, + size_t *comp_bytes, sycl::queue q) { + constexpr int HS = 1 << lz4::HASH_LOG; + constexpr int WPB = WARPS_PER_BLOCK; + size_t groups = (nchunks + WPB - 1) / WPB; + sycl::range<1> global(groups * WPB * SYCL_SG), local(WPB * SYCL_SG); + q.submit([&](sycl::handler &h) { + sycl::local_accessor ht(sycl::range<1>(WPB * HS), h); + h.parallel_for( + sycl::nd_range<1>(global, local), + [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(SYCL_SG)]] { + sycl::sub_group nsg = it.get_sub_group(); + int warp = (int)nsg.get_group_id()[0]; + SIZE c = (SIZE)it.get_group(0) * WPB + warp; + if (c >= nchunks) + return; + SIZE start = c * (SIZE)chunk_size; + int len = (int)((SIZE)chunk_size < (n - start) ? (SIZE)chunk_size + : (n - start)); + SubGroup sg(nsg); + uint16_t *htp = &ht[(size_t)warp * HS]; + int cb = lz4_coop::compress_chunk( + sg, input + start, len, scratch + (size_t)c * max_chunk_out, htp); + if (sg.lane() == 0) + comp_bytes[c] = (size_t)cb; + }); + }); +} + +inline void launch_decompress(const Byte *packed, const size_t *byte_offset, + SIZE n, int chunk_size, SIZE nchunks, + Byte *output, sycl::queue q) { + constexpr int WPB = WARPS_PER_BLOCK; + size_t groups = (nchunks + WPB - 1) / WPB; + sycl::range<1> global(groups * WPB * SYCL_SG), local(WPB * SYCL_SG); + q.submit([&](sycl::handler &h) { + h.parallel_for( + sycl::nd_range<1>(global, local), + [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(SYCL_SG)]] { + sycl::sub_group nsg = it.get_sub_group(); + int warp = (int)nsg.get_group_id()[0]; + SIZE c = (SIZE)it.get_group(0) * WPB + warp; + if (c >= nchunks) + return; + SIZE start = c * (SIZE)chunk_size; + int outLen = (int)((SIZE)chunk_size < (n - start) ? (SIZE)chunk_size + : (n - start)); + SubGroup sg(nsg); + lz4_coop::decompress_chunk(sg, packed + byte_offset[c], + output + start, outLen); + }); + }); +} + +} // namespace lz4_fused +} // namespace mgard_x +#endif // SYCL + +#endif diff --git a/include/mgard-x/Lossless/LZ4/LZ4Kernels.hpp b/include/mgard-x/Lossless/LZ4/LZ4Kernels.hpp new file mode 100644 index 0000000000..949663b828 --- /dev/null +++ b/include/mgard-x/Lossless/LZ4/LZ4Kernels.hpp @@ -0,0 +1,505 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_LZ4_KERNELS_HPP +#define MGARD_X_LZ4_KERNELS_HPP + +#include "../../RuntimeX/RuntimeX.h" + +// Portable (multi-kernel) building blocks for a self-contained LZ4 backend -- +// a byte-oriented LZ77 dictionary codec, no external nvcomp dependency. +// +// The input byte stream is split into independent chunks of `chunk_size`. Each +// chunk is compressed/decompressed in isolation (its back-references never +// cross a chunk boundary), which makes the whole thing trivially parallel: one +// thread per chunk, no atomics, no cross-chunk state. This mirrors how nvcomp +// batches LZ4 and how BlockDelta gives each data-block a disjoint byte range. +// +// Compression is two phases because a chunk's compressed size is not known +// until it is actually compressed: +// 1) LZ4ChunkCompress : compress chunk c into a worst-case-sized scratch slot +// (scratch + c*max_chunk_out) and record comp_bytes[c]. +// 2) (host) exclusive scan comp_bytes -> byte_offset (final contiguous +// layout) 3) LZ4Condense : copy each chunk's comp_bytes[c] from its +// scratch slot +// to packed[byte_offset[c]] (the gather/compaction +// pass). +// Decompression is single-phase: rebuild byte_offset from the stored +// comp_bytes, then one thread per chunk parses tokens and copies. +// +// NOTE (perf): 1 thread per chunk is the portable correctness reference. With +// large chunks it is low-occupancy and keeps a per-thread hash table in global +// scratch. The optimization is one thread *block* per chunk (cooperative +// match-find + a shared-memory hash table), and/or a fused decoupled-look-back +// pass like BlockDeltaFused. Use a small chunk_size (4-8 KB) for this path. + +namespace mgard_x { + +namespace lz4 { + +// ---- format / tuning constants ------------------------------------------- +enum : int { + MIN_MATCH = 4, // a back-reference must cover >= 4 bytes to pay off + LAST_LITERALS = 5, // final 5 bytes of a chunk are always emitted literally + MFLIMIT = 12, // last match must start >= 12 bytes before chunk end + MAX_DISTANCE = 65535, // 16-bit offset -> 64 KB window + HASH_LOG = 12, // per-chunk hash table: 1<> (32 - HASH_LOG); +} + +// Write an LZ4 length in the 255-continuation form. Returns the new write pos. +MGARDX_EXEC int write_len(Byte *out, int op, int length) { + while (length >= 255) { + out[op++] = (Byte)255; + length -= 255; + } + out[op++] = (Byte)length; + return op; +} + +} // namespace lz4 + +// --------------------------------------------------------------------------- +// ONE cooperative LZ4 codec, written against the portable SubGroup abstraction +// (mgard_x::SubGroup / SubGroupScalar, defined in the DeviceAdapter +// headers). A "sub-group" is a set of lanes running in lockstep that can shfl / +// ballot / ffs / sync. At size 1 (CPU) every collective is an identity and this +// exact code degenerates into the greedy serial encoder/decoder. Warp-size is +// handled via SG::size()/full_mask()/the mask typedef, so it is correct for any +// width -- the match scan examines size() positions per step (a wider sub-group +// gives slightly different, still-valid matches => CR can vary marginally with +// width; decode output is width-independent). +// --------------------------------------------------------------------------- +namespace lz4_coop { + +// Cooperative LZ4 encode of one chunk. `ht` is sub-group-shared scratch of +// lz4::HASH_SIZE uint16 slots (0 = empty, else position+1; needs chunk_size +// <= 65535). All lanes of `sg` call this together; returns the byte count. +// At size 1 this is exactly the greedy serial encoder. +template +MGARDX_EXEC int compress_chunk(SG sg, const Byte *in, int len, Byte *out, + uint16_t *ht) { + const int W = SG::size(); + const int lane = sg.lane(); + const int MF = lz4::MFLIMIT, LL = lz4::LAST_LITERALS, MM = lz4::MIN_MATCH; + const int MAXD = lz4::MAX_DISTANCE; + const int HS = (int)lz4::HASH_SIZE; + + for (int i = lane; i < HS; i += W) + ht[i] = 0; + sg.sync(); + + int ip = 0, anchor = 0, op = 0; + while (ip < len - MF) { + const int pos = ip + lane; + // Phase 1: does this lane's position have a valid 4-byte match? (no extend) + int rref = -1; + if (pos < len - MF) { + uint32_t seq = lz4::read4(in + pos); + uint16_t slot = ht[lz4::hash4(seq)]; + if (slot != 0) { + int r = (int)slot - 1; + if (pos - r <= MAXD && lz4::read4(in + r) == seq) + rref = r; + } + } + typename SG::mask_t ballot = sg.ballot(rref >= 0); + + if (ballot == 0) { + if (pos < len - 3) + ht[lz4::hash4(lz4::read4(in + pos))] = (uint16_t)(pos + 1); + sg.sync(); + ip += W; + continue; + } + + const int j = sg.ffs(ballot) - 1; // first (left-most) matching lane + const int mpos = ip + j; + const int r_j = sg.shfl(rref, j); + const int off_j = mpos - r_j; + + // Phase 2: cooperative match-length extension (W bytes per step). + int ml = MM; + { + int base = MM; + while (true) { + int q = base + lane, a = mpos + q, b = r_j + q; + int eq = (a < len - LL && in[b] == in[a]) ? 1 : 0; + typename SG::mask_t good = sg.ballot(eq); + if (good == sg.full_mask()) + base += W; + else { + ml = base + (sg.ffs(~good & sg.full_mask()) - 1); + break; + } + } + } + + // Record literal positions we passed [ip, mpos] for later windows. + if (lane <= j && pos < len - 3) + ht[lz4::hash4(lz4::read4(in + pos))] = (uint16_t)(pos + 1); + sg.sync(); + + // Emit: token | litlen-ext | literals | offset | matchlen-ext. + const int litlen = mpos - anchor; + const int numLitExt = litlen < 15 ? 0 : ((litlen - 15) / 255 + 1); + const int litStart = op + 1 + numLitExt; + const int mlf = ml - MM; + const int numMlExt = mlf < 15 ? 0 : ((mlf - 15) / 255 + 1); + const int after = litStart + litlen + 2 + numMlExt; + if (lane == 0) { + int tHi = litlen < 15 ? litlen : 15, tLo = mlf < 15 ? mlf : 15; + out[op] = (Byte)((tHi << 4) | tLo); + if (litlen >= 15) + lz4::write_len(out, op + 1, litlen - 15); + int offPos = litStart + litlen; + out[offPos] = (Byte)(off_j & 0xff); + out[offPos + 1] = (Byte)((off_j >> 8) & 0xff); + if (mlf >= 15) + lz4::write_len(out, offPos + 2, mlf - 15); + } + for (int k = lane; k < litlen; k += W) // cooperative literal copy + out[litStart + k] = in[anchor + k]; + sg.sync(); + + op = after; + ip = mpos + ml; + anchor = ip; + } + + // Terminal literal-only sequence [anchor, len). + const int litlen = len - anchor; + const int numLitExt = litlen < 15 ? 0 : ((litlen - 15) / 255 + 1); + const int litStart = op + 1 + numLitExt; + if (lane == 0) { + int tHi = litlen < 15 ? litlen : 15; + out[op] = (Byte)(tHi << 4); + if (litlen >= 15) + lz4::write_len(out, op + 1, litlen - 15); + } + for (int k = lane; k < litlen; k += W) + out[litStart + k] = in[anchor + k]; + sg.sync(); + return litStart + litlen; +} + +// Cooperative decode of one chunk. Lane 0 parses the serial token stream; the +// sub-group cooperatively copies literals and the match. Overlap-safe via the +// periodic seed out[op+k] = out[op-offset + (k mod offset)] (reads only the +// already-decoded seed). Driven by output length. Size 1 = serial decoder. +template +MGARDX_EXEC void decompress_chunk(SG sg, const Byte *in, Byte *out, + int outLen) { + const int W = SG::size(); + const int lane = sg.lane(); + const int MM = lz4::MIN_MATCH; + int ip = 0, op = 0; + while (op < outLen) { + int litlen = 0, lit_in = 0, off = 0, mlen = 0, is_term = 0; + if (lane == 0) { + Byte token = in[ip++]; + litlen = token >> 4; + if (litlen == 15) { + Byte b; + do { + b = in[ip++]; + litlen += b; + } while (b == 255); + } + lit_in = ip; + ip += litlen; + if (op + litlen >= outLen) { + is_term = 1; + } else { + off = (int)in[ip] | ((int)in[ip + 1] << 8); + ip += 2; + mlen = (token & 0xf) + MM; + if ((token & 0xf) == 15) { + Byte b; + do { + b = in[ip++]; + mlen += b; + } while (b == 255); + } + } + } + litlen = sg.shfl(litlen, 0); + lit_in = sg.shfl(lit_in, 0); + is_term = sg.shfl(is_term, 0); + off = sg.shfl(off, 0); + mlen = sg.shfl(mlen, 0); + + for (int k = lane; k < litlen; k += W) + out[op + k] = in[lit_in + k]; + op += litlen; + sg.sync(); + if (is_term) + break; + + const int seed = op - off; + if (off >= mlen) { + for (int k = lane; k < mlen; k += W) + out[op + k] = out[seed + k]; + } else { + for (int k = lane; k < mlen; k += W) + out[op + k] = out[seed + (k % off)]; + } + op += mlen; + sg.sync(); + } +} + +} // namespace lz4_coop + +// --------------------------------------------------------------------------- +// Kernel 1: compress each chunk into its scratch slot; record comp_bytes[c]. +// --------------------------------------------------------------------------- +template +class LZ4ChunkCompressFunctor : public Functor { +public: + MGARDX_CONT LZ4ChunkCompressFunctor() {} + MGARDX_CONT + LZ4ChunkCompressFunctor(SubArray<1, Byte, DeviceType> input, SIZE n, + SIZE chunk_size, SIZE nchunks, SIZE max_chunk_out, + SubArray<1, uint16_t, DeviceType> htable, + SubArray<1, Byte, DeviceType> scratch, + SubArray<1, size_t, DeviceType> comp_bytes) + : input(input), n(n), chunk_size(chunk_size), nchunks(nchunks), + max_chunk_out(max_chunk_out), htable(htable), scratch(scratch), + comp_bytes(comp_bytes) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE c = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (c >= nchunks) + return; + SIZE start = c * chunk_size; + int len = (int)(chunk_size < (n - start) ? chunk_size : (n - start)); + Byte *out = scratch((size_t)c * max_chunk_out); + uint16_t *ht = htable((size_t)c * lz4::HASH_SIZE); + int outlen = + lz4_coop::compress_chunk(SubGroupScalar{}, input(start), len, out, ht); + *comp_bytes(c) = (size_t)outlen; + } + + MGARDX_EXEC void Operation2() {} + MGARDX_EXEC void Operation3() {} + MGARDX_EXEC void Operation4() {} + MGARDX_EXEC void Operation5() {} + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Byte, DeviceType> input; + SIZE n, chunk_size, nchunks, max_chunk_out; + SubArray<1, uint16_t, DeviceType> htable; + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, size_t, DeviceType> comp_bytes; +}; + +template class LZ4ChunkCompressKernel : public Kernel { +public: + constexpr static DIM NumDim = 1; + constexpr static bool EnableAutoTuning() { return false; } + using DataType = Byte; + constexpr static std::string_view Name = "lz4_chunk_compress"; + MGARDX_CONT LZ4ChunkCompressKernel(SubArray<1, Byte, DeviceType> input, + SIZE n, SIZE chunk_size, SIZE nchunks, + SIZE max_chunk_out, + SubArray<1, uint16_t, DeviceType> htable, + SubArray<1, Byte, DeviceType> scratch, + SubArray<1, size_t, DeviceType> comp_bytes) + : input(input), n(n), chunk_size(chunk_size), nchunks(nchunks), + max_chunk_out(max_chunk_out), htable(htable), scratch(scratch), + comp_bytes(comp_bytes) {} + + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = LZ4ChunkCompressFunctor; + FunctorType functor(input, n, chunk_size, nchunks, max_chunk_out, htable, + scratch, comp_bytes); + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (nchunks - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, tbz, tby, tbx, + functor.shared_memory_size(), queue_idx, std::string(Name)); + } + +private: + SubArray<1, Byte, DeviceType> input; + SIZE n, chunk_size, nchunks, max_chunk_out; + SubArray<1, uint16_t, DeviceType> htable; + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, size_t, DeviceType> comp_bytes; +}; + +// --------------------------------------------------------------------------- +// Kernel 2: gather each chunk's compressed bytes into the contiguous output. +// --------------------------------------------------------------------------- +template +class LZ4CondenseFunctor : public Functor { +public: + MGARDX_CONT LZ4CondenseFunctor() {} + MGARDX_CONT LZ4CondenseFunctor(SubArray<1, Byte, DeviceType> scratch, + SIZE nchunks, SIZE max_chunk_out, + SubArray<1, size_t, DeviceType> comp_bytes, + SubArray<1, size_t, DeviceType> byte_offset, + SubArray<1, Byte, DeviceType> packed) + : scratch(scratch), nchunks(nchunks), max_chunk_out(max_chunk_out), + comp_bytes(comp_bytes), byte_offset(byte_offset), packed(packed) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE c = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (c >= nchunks) + return; + Byte *src = scratch((size_t)c * max_chunk_out); + Byte *dst = packed(*byte_offset(c)); + size_t cb = *comp_bytes(c); + // TUNING: serial per-thread copy. The cooperative (block-per-chunk) version + // copies this with the whole block striding over cb. + for (size_t k = 0; k < cb; k++) + dst[k] = src[k]; + } + + MGARDX_EXEC void Operation2() {} + MGARDX_EXEC void Operation3() {} + MGARDX_EXEC void Operation4() {} + MGARDX_EXEC void Operation5() {} + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Byte, DeviceType> scratch; + SIZE nchunks, max_chunk_out; + SubArray<1, size_t, DeviceType> comp_bytes; + SubArray<1, size_t, DeviceType> byte_offset; + SubArray<1, Byte, DeviceType> packed; +}; + +template class LZ4CondenseKernel : public Kernel { +public: + constexpr static DIM NumDim = 1; + constexpr static bool EnableAutoTuning() { return false; } + using DataType = Byte; + constexpr static std::string_view Name = "lz4_condense"; + MGARDX_CONT LZ4CondenseKernel(SubArray<1, Byte, DeviceType> scratch, + SIZE nchunks, SIZE max_chunk_out, + SubArray<1, size_t, DeviceType> comp_bytes, + SubArray<1, size_t, DeviceType> byte_offset, + SubArray<1, Byte, DeviceType> packed) + : scratch(scratch), nchunks(nchunks), max_chunk_out(max_chunk_out), + comp_bytes(comp_bytes), byte_offset(byte_offset), packed(packed) {} + + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = LZ4CondenseFunctor; + FunctorType functor(scratch, nchunks, max_chunk_out, comp_bytes, + byte_offset, packed); + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (nchunks - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, tbz, tby, tbx, + functor.shared_memory_size(), queue_idx, std::string(Name)); + } + +private: + SubArray<1, Byte, DeviceType> scratch; + SIZE nchunks, max_chunk_out; + SubArray<1, size_t, DeviceType> comp_bytes; + SubArray<1, size_t, DeviceType> byte_offset; + SubArray<1, Byte, DeviceType> packed; +}; + +// --------------------------------------------------------------------------- +// Kernel 3: decode each chunk back into the contiguous output stream. +// --------------------------------------------------------------------------- +template +class LZ4ChunkDecompressFunctor : public Functor { +public: + MGARDX_CONT LZ4ChunkDecompressFunctor() {} + MGARDX_CONT + LZ4ChunkDecompressFunctor(SubArray<1, Byte, DeviceType> packed, + SubArray<1, size_t, DeviceType> byte_offset, SIZE n, + SIZE chunk_size, SIZE nchunks, + SubArray<1, Byte, DeviceType> output) + : packed(packed), byte_offset(byte_offset), n(n), chunk_size(chunk_size), + nchunks(nchunks), output(output) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE c = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (c >= nchunks) + return; + SIZE start = c * chunk_size; + int len = (int)(chunk_size < (n - start) ? chunk_size : (n - start)); + lz4_coop::decompress_chunk(SubGroupScalar{}, packed(*byte_offset(c)), + output(start), len); + } + + MGARDX_EXEC void Operation2() {} + MGARDX_EXEC void Operation3() {} + MGARDX_EXEC void Operation4() {} + MGARDX_EXEC void Operation5() {} + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Byte, DeviceType> packed; + SubArray<1, size_t, DeviceType> byte_offset; + SIZE n, chunk_size, nchunks; + SubArray<1, Byte, DeviceType> output; +}; + +template class LZ4ChunkDecompressKernel : public Kernel { +public: + constexpr static DIM NumDim = 1; + constexpr static bool EnableAutoTuning() { return false; } + using DataType = Byte; + constexpr static std::string_view Name = "lz4_chunk_decompress"; + MGARDX_CONT + LZ4ChunkDecompressKernel(SubArray<1, Byte, DeviceType> packed, + SubArray<1, size_t, DeviceType> byte_offset, SIZE n, + SIZE chunk_size, SIZE nchunks, + SubArray<1, Byte, DeviceType> output) + : packed(packed), byte_offset(byte_offset), n(n), chunk_size(chunk_size), + nchunks(nchunks), output(output) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = LZ4ChunkDecompressFunctor; + FunctorType functor(packed, byte_offset, n, chunk_size, nchunks, output); + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (nchunks - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, tbz, tby, tbx, + functor.shared_memory_size(), queue_idx, std::string(Name)); + } + +private: + SubArray<1, Byte, DeviceType> packed; + SubArray<1, size_t, DeviceType> byte_offset; + SIZE n, chunk_size, nchunks; + SubArray<1, Byte, DeviceType> output; +}; + +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/Lossless/Lossless.hpp b/include/mgard-x/Lossless/Lossless.hpp index 4f0070d6f8..140d630f42 100644 --- a/include/mgard-x/Lossless/Lossless.hpp +++ b/include/mgard-x/Lossless/Lossless.hpp @@ -1,16 +1,19 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ +#include "BlockDelta/BlockDelta.hpp" #include "CPU.hpp" -#include "Cascaded.hpp" -#include "LZ4.hpp" +#include "LZ4/LZ4.hpp" #include "LosslessCompressorInterface.hpp" #include "ParallelHuffman/Huffman.hpp" +#include "ParallelRLE/ZeroRunLengthEncoding.hpp" +#include "SymbolRans/SymbolRans.hpp" #include "Zstd.hpp" +#include "rANS/Rans.hpp" #ifndef MGARD_X_LOSSLESS_TEMPLATE_HPP #define MGARD_X_LOSSLESS_TEMPLATE_HPP @@ -26,18 +29,53 @@ class ComposedLosslessCompressor ComposedLosslessCompressor() : initialized(false) {} + // Whether the configured lossless path actually uses the (workspace-heavy) + // Huffman backend. BlockDelta and standalone LZ4 have their own + // (de)compressors and never touch the Huffman workspace; every other type is + // Huffman / Huffman+LZ4 / Huffman+Zstd (or CPU_Lossless falling through to + // Huffman in Compress()). + static bool uses_huffman(enum lossless_type lossless) { + return lossless != lossless_type::BlockDelta && + lossless != lossless_type::LZ4 && + lossless != lossless_type::ZeroRLE_Rans && + lossless != lossless_type::SymbolRans; + } + + // Worst-case byte size of the RLE0 blob fed to rANS: a (count, symbol) pair + // per element when nothing repeats (uint32 count + T symbol), plus slack. + static SIZE rle_rans_bound(SIZE n) { + return n * (sizeof(uint32_t) + sizeof(T)) + 64; + } + ComposedLosslessCompressor(SIZE n, Config config) - : initialized(true), n(n), config(config), - huffman(n, config.huff_dict_size, config.huff_block_size, - config.estimate_outlier_ratio) { + : initialized(true), n(n), config(config) { static_assert(!std::is_floating_point::value, "ComposedLosslessCompressor: Type of T must be integer."); + if (uses_huffman(config.lossless)) { + huffman.Resize(n, config.huff_dict_size, config.huff_block_size, + config.estimate_outlier_ratio, 0); + } if (config.lossless == lossless_type::Huffman_LZ4) { lz4.Resize(n * sizeof(H), config.lz4_block_size, 0); } + if (config.lossless == lossless_type::LZ4) { + lz4.Resize(n * sizeof(T), config.lz4_block_size, 0); + } if (config.lossless == lossless_type::Huffman_Zstd) { zstd.Resize(n * sizeof(H), config.zstd_compress_level, 0); } + if (config.lossless == lossless_type::BlockDelta) { + blockdelta.Resize(n, config.block_delta_block_size, + config.block_delta_mode, 0); + } + if (config.lossless == lossless_type::ZeroRLE_Rans) { + zerorle.Resize(n, 0); + rans.Resize(rle_rans_bound(n), 256, 0); + } + if (config.lossless == lossless_type::SymbolRans) { + symbolrans.Resize(n, config.huff_dict_size, config.estimate_outlier_ratio, + 0); + } DeviceRuntime::SyncQueue(0); } @@ -45,35 +83,103 @@ class ComposedLosslessCompressor this->initialized = true; this->n = n; this->config = config; - huffman.Resize(n, config.huff_dict_size, config.huff_block_size, - config.estimate_outlier_ratio, queue_idx); + if (uses_huffman(config.lossless)) { + huffman.Resize(n, config.huff_dict_size, config.huff_block_size, + config.estimate_outlier_ratio, queue_idx); + } if (config.lossless == lossless_type::Huffman_LZ4) { lz4.Resize(n * sizeof(H), config.lz4_block_size, queue_idx); } + if (config.lossless == lossless_type::LZ4) { + lz4.Resize(n * sizeof(T), config.lz4_block_size, queue_idx); + } if (config.lossless == lossless_type::Huffman_Zstd) { zstd.Resize(n * sizeof(H), config.zstd_compress_level, queue_idx); } + if (config.lossless == lossless_type::BlockDelta) { + blockdelta.Resize(n, config.block_delta_block_size, + config.block_delta_mode, queue_idx); + } + if (config.lossless == lossless_type::ZeroRLE_Rans) { + zerorle.Resize(n, queue_idx); + rans.Resize(rle_rans_bound(n), 256, queue_idx); + } + if (config.lossless == lossless_type::SymbolRans) { + symbolrans.Resize(n, config.huff_dict_size, config.estimate_outlier_ratio, + queue_idx); + } } static size_t EstimateMemoryFootprint(SIZE primary_count, Config config) { - size_t size = Huffman::EstimateMemoryFootprint( - primary_count, config.huff_dict_size, config.huff_block_size, - config.estimate_outlier_ratio); + size_t size = 0; + if (uses_huffman(config.lossless)) { + size += Huffman::EstimateMemoryFootprint( + primary_count, config.huff_dict_size, config.huff_block_size, + config.estimate_outlier_ratio); + } if (config.lossless == lossless_type::Huffman_LZ4) { size += LZ4::EstimateMemoryFootprint( primary_count * sizeof(H), config.lz4_block_size); } + if (config.lossless == lossless_type::LZ4) { + size += LZ4::EstimateMemoryFootprint( + primary_count * sizeof(T), config.lz4_block_size); + } if (config.lossless == lossless_type::Huffman_Zstd) { size += Zstd::EstimateMemoryFootprint(primary_count * sizeof(H)); } + if (config.lossless == lossless_type::BlockDelta) { + size += BlockDeltaLossless::EstimateMemoryFootprint( + primary_count, config.block_delta_block_size); + } + if (config.lossless == lossless_type::ZeroRLE_Rans) { + // RLE0 blob + rANS scratch (~2x the blob) dominate. + size += rle_rans_bound(primary_count) * 3; + } + if (config.lossless == lossless_type::SymbolRans) { + size += SymbolRans::EstimateMemoryFootprint( + primary_count, config.huff_dict_size, config.estimate_outlier_ratio); + } return size; } void Compress(Array<1, T, DeviceType> &original_data, Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { - huffman.CompressPrimary(original_data, compressed_data, queue_idx); + if (config.lossless == lossless_type::BlockDelta) { + blockdelta.Compress(original_data, compressed_data, queue_idx); + return; + } + + if (config.lossless == lossless_type::LZ4) { + // LZ4 directly on the raw quantized stream (no Huffman). View the T array + // as bytes into compressed_data, then compress it in place. + SIZE nbytes = original_data.shape(0) * sizeof(T); + compressed_data.resize({nbytes}, queue_idx); + MemoryManager::Copy1D(compressed_data.data(), + (Byte *)original_data.data(), nbytes, + queue_idx); + lz4.Compress(compressed_data, queue_idx); + return; + } + + if (config.lossless == lossless_type::ZeroRLE_Rans) { + // Zero-RLE the quantized stream into a (counts, symbols) byte blob, then + // entropy-code that blob with rANS. Self-contained: the rANS output is + // the final compressed stream (Serialize/Deserialize are no-ops here). + zerorle.Compress(original_data, rle_bytes, 0.0, queue_idx); + rans.Compress(rle_bytes, compressed_data, queue_idx); + return; + } + + if (config.lossless == lossless_type::SymbolRans) { + // Outlier separation + symbol-alphabet rANS over the dict_size primary. + symbolrans.Compress(original_data, compressed_data, queue_idx); + return; + } + + huffman.Compress(original_data, compressed_data, 0.0, queue_idx); if (config.lossless == lossless_type::Huffman_LZ4) { huffman.Serialize(compressed_data, queue_idx); @@ -90,17 +196,57 @@ class ComposedLosslessCompressor if (config.lossless == lossless_type::Huffman) { huffman.Serialize(compressed_data, queue_idx); } + if (config.lossless == lossless_type::BlockDelta) { + blockdelta.Serialize(compressed_data, queue_idx); + } } void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { if (config.lossless == lossless_type::Huffman) { huffman.Deserialize(compressed_data, queue_idx); } + if (config.lossless == lossless_type::BlockDelta) { + blockdelta.Deserialize(compressed_data, queue_idx); + } } void Decompress(Array<1, Byte, DeviceType> &compressed_data, Array<1, T, DeviceType> &decompressed_data, int queue_idx) { + if (config.lossless == lossless_type::BlockDelta) { + // Deserialize (the memory-movement stage) was already run separately by + // the pipeline; Decompress is computation only. + blockdelta.Decompress(compressed_data, decompressed_data, queue_idx); + return; + } + + if (config.lossless == lossless_type::LZ4) { + // Inverse of the standalone LZ4 path: decompress to the raw byte stream, + // then reinterpret it back into the quantized T array. + lz4.Decompress(compressed_data, queue_idx); + SIZE nbytes = compressed_data.shape(0); + decompressed_data.resize({(SIZE)(nbytes / sizeof(T))}, queue_idx); + MemoryManager::Copy1D((Byte *)decompressed_data.data(), + compressed_data.data(), nbytes, + queue_idx); + return; + } + + if (config.lossless == lossless_type::ZeroRLE_Rans) { + // Inverse of the RLE0 -> rANS path: rANS-decode back to the RLE0 blob, + // then expand it to the quantized stream. + rans.Deserialize(compressed_data, queue_idx); + rans.Decompress(compressed_data, rle_bytes, queue_idx); + zerorle.Deserialize(rle_bytes, queue_idx); + zerorle.Decompress(rle_bytes, decompressed_data, queue_idx); + return; + } + + if (config.lossless == lossless_type::SymbolRans) { + symbolrans.Decompress(compressed_data, decompressed_data, queue_idx); + return; + } + if (config.lossless == lossless_type::Huffman_LZ4) { lz4.Decompress(compressed_data, queue_idx); huffman.Deserialize(compressed_data, queue_idx); @@ -111,7 +257,7 @@ class ComposedLosslessCompressor huffman.Deserialize(compressed_data, queue_idx); } - huffman.DecompressPrimary(compressed_data, decompressed_data, queue_idx); + huffman.Decompress(compressed_data, decompressed_data, queue_idx); } bool initialized; @@ -120,8 +266,14 @@ class ComposedLosslessCompressor Huffman huffman; LZ4 lz4; Zstd zstd; + BlockDeltaLossless blockdelta; + parallel_rle::ZeroRunLengthEncoding + zerorle; + rans::Rans rans; + Array<1, Byte, DeviceType> rle_bytes; + SymbolRans symbolrans; }; } // namespace mgard_x -#endif \ No newline at end of file +#endif diff --git a/include/mgard-x/Lossless/LosslessCompressorInterface.hpp b/include/mgard-x/Lossless/LosslessCompressorInterface.hpp index 5ebe710a08..be8c754632 100644 --- a/include/mgard-x/Lossless/LosslessCompressorInterface.hpp +++ b/include/mgard-x/Lossless/LosslessCompressorInterface.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_LOSSLESS_COMPRESSOR_INTERFACE_HPP diff --git a/include/mgard-x/Lossless/ParallelHuffman/Condense.hpp b/include/mgard-x/Lossless/ParallelHuffman/Condense.hpp index 3ea28c1644..3de4fb3542 100644 --- a/include/mgard-x/Lossless/ParallelHuffman/Condense.hpp +++ b/include/mgard-x/Lossless/ParallelHuffman/Condense.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_CONDENSE_TEMPLATE_HPP @@ -56,9 +56,10 @@ class CondenseKernel : public Kernel { CondenseKernel(SubArray<1, H, DeviceType> v, SubArray<1, size_t, DeviceType> write_offsets, SubArray<1, size_t, DeviceType> actual_lengths, - SubArray<1, H, DeviceType> condensed_v, SIZE chunck_size) + SubArray<1, H, DeviceType> condensed_v, SIZE chunck_size, + SIZE nchunk) : v(v), write_offsets(write_offsets), actual_lengths(actual_lengths), - condensed_v(condensed_v), chunck_size(chunck_size) {} + condensed_v(condensed_v), chunck_size(chunck_size), nchunk(nchunk) {} MGARDX_CONT Task> GenTask(int queue_idx) { @@ -72,7 +73,7 @@ class CondenseKernel : public Kernel { tbx = 256; gridz = 1; gridy = 1; - gridx = write_offsets.shape(0); + gridx = nchunk; return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, std::string(Name)); } @@ -83,6 +84,7 @@ class CondenseKernel : public Kernel { SubArray<1, size_t, DeviceType> actual_lengths; SubArray<1, H, DeviceType> condensed_v; SIZE chunck_size; + SIZE nchunk; }; } // namespace mgard_x diff --git a/include/mgard-x/Lossless/ParallelHuffman/Decode.hpp b/include/mgard-x/Lossless/ParallelHuffman/Decode.hpp index 5feae217cd..3ed366c4d5 100644 --- a/include/mgard-x/Lossless/ParallelHuffman/Decode.hpp +++ b/include/mgard-x/Lossless/ParallelHuffman/Decode.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_DECODE_TEMPLATE_HPP @@ -30,9 +30,20 @@ class DecodeFunctor : public Functor { MGARDX_EXEC void Operation1() { if (CACHE_SINGLETION) { + // Opt 3: cooperatively stage only the small, hottest decodebook tables + // (first[] and entry[], together sizeof(H)*2*word_bw bytes ~ 1KB) into + // shared memory. These are touched on every decoded bit. The large keys[] + // table is left in global memory so caching never inflates the per-block + // shared-memory footprint and hurts occupancy (which is what made caching + // the whole decodebook a net loss for large dictionaries). All threads + // participate; the framework inserts a block sync before Operation2. _s_singleton = (uint8_t *)FunctorBase::GetSharedMemory(); - if (FunctorBase::GetThreadIdX() == 0) { - memcpy(_s_singleton, singleton((IDX)0), singleton_size); + size_t fe_bytes = sizeof(H) * (2 * sizeof(H) * 8); + uint8_t *src = singleton((IDX)0); + SIZE tid = FunctorBase::GetThreadIdX(); + SIZE nthreads = FunctorBase::GetBlockDimX(); + for (SIZE b = tid; b < (SIZE)fe_bytes; b += nthreads) { + _s_singleton[b] = src[b]; } } else { _s_singleton = singleton((IDX)0); @@ -43,7 +54,6 @@ class DecodeFunctor : public Functor { size_t chunk_id = FunctorBase::GetBlockIdX() * FunctorBase::GetBlockDimX() + FunctorBase::GetThreadIdX(); - // if (chunk_id == 0) printf("n_chunk: %lu\n", n_chunk); if (chunk_id >= n_chunk) return; @@ -51,66 +61,48 @@ class DecodeFunctor : public Functor { SIZE bcode_offset = chunk_size * chunk_id; size_t total_bw = *dH_meta(chunk_id); - uint8_t next_bit; - size_t idx_bit; - size_t idx_byte = 0; - size_t idx_bcoded = 0; + const size_t word_bw = sizeof(H) * 8; + // first[]/entry[] come from _s_singleton (shared when cached, else global); + // keys[] always stays in global memory. auto first = reinterpret_cast(_s_singleton); auto entry = first + sizeof(H) * 8; - auto keys = - reinterpret_cast(_s_singleton + sizeof(H) * (2 * sizeof(H) * 8)); - H v = (*densely(densely_offset + idx_byte) >> (sizeof(H) * 8 - 1)) & - 0x1; // get the first bit + auto keys = reinterpret_cast(singleton((IDX)0) + + sizeof(H) * (2 * sizeof(H) * 8)); + + // Opt 2: hold the current densely word in a register and refetch from + // global memory only when the bit cursor crosses into the next word, + // instead of re-loading the same word once per bit (up to word_bw redundant + // loads). + size_t cached_word_idx = 0; + H cached_word = *densely(densely_offset); + + H v = (cached_word >> (word_bw - 1)) & 0x1; // get the first bit size_t l = 1; size_t i = 0; + size_t idx_bcoded = 0; while (i < total_bw) { while (v < first[l]) { // append next i_cb bit ++i; - idx_byte = i / (sizeof(H) * 8); - idx_bit = i % (sizeof(H) * 8); - next_bit = ((*densely(densely_offset + idx_byte) >> - (sizeof(H) * 8 - 1 - idx_bit)) & - 0x1); + size_t idx_word = i / word_bw; + if (idx_word != cached_word_idx) { + cached_word = *densely(densely_offset + idx_word); + cached_word_idx = idx_word; + } + H next_bit = (cached_word >> (word_bw - 1 - (i % word_bw))) & 0x1; v = (v << 1) | next_bit; ++l; } - - // debug - start - // if (!chunk_id) { - // // if ((entry[l] + v - first[l])*sizeof(Q) + sizeof(H) * (2 * sizeof(H) - // * 8) >= 1280) { - // printf("out of range: %llu\n", (entry[l] + v - first[l])*sizeof(Q) + - // sizeof(H) * (2 * sizeof(H) * 8)); printf("l: %llu\n", l); - // printf("entry[l]: %llu\n", entry[l]); - // printf("v: %llu\n", v); - // printf("first[l]: %llu\n", first[l]); - // printf("entry:"); - // for (int i = 0; i < 64; i++) { - // printf("%llu ", entry[i]); - // } - // printf("\n"); - // printf("first:"); - // for (int i = 0; i < 64; i++) { - // printf("%llu ", first[i]); - // } - // printf("\n"); - // } - // debug - end - // if (entry[l] + v - first[l] > 100000) { - // printf("offset: %llu + %llu i: %llu l: %llu (%llu, %llu, %llu)\n", - // sizeof(H) * (2 * sizeof(H) * 8), entry[l] + v - first[l], i, l, - // entry[l], v, first[l]); - // } *bcode(bcode_offset + idx_bcoded) = keys[entry[l] + v - first[l]]; idx_bcoded++; { ++i; - idx_byte = i / (sizeof(H) * 8); - idx_bit = i % (sizeof(H) * 8); - next_bit = ((*densely(densely_offset + idx_byte) >> - (sizeof(H) * 8 - 1 - idx_bit)) & - 0x1); - v = 0x0 | next_bit; + size_t idx_word = i / word_bw; + if (idx_word != cached_word_idx) { + cached_word = *densely(densely_offset + idx_word); + cached_word_idx = idx_word; + } + H next_bit = (cached_word >> (word_bw - 1 - (i % word_bw))) & 0x1; + v = next_bit; } l = 1; } @@ -118,7 +110,8 @@ class DecodeFunctor : public Functor { MGARDX_CONT size_t shared_memory_size() { if (CACHE_SINGLETION) { - return singleton_size; + // Only first[]/entry[] are cached (see Operation1), not the full table. + return sizeof(H) * (2 * sizeof(H) * 8); } else { return 0; } @@ -153,6 +146,12 @@ class DecodeKernel : public Kernel { chunk_size(chunk_size), n_chunk(n_chunk), singleton(singleton), singleton_size(singleton_size) {} + // NOTE: decode is one independent thread per chunk and is dominated by warp + // straggler divergence (each thread loops until its own chunk's bits are + // exhausted, and the warp runs until its slowest chunk finishes). Larger + // (full-warp) blocks raise occupancy but put 32 chunks under one straggler + // group instead of 16, which measured ~15% SLOWER on NYX 512^3. The autotuned + // (half-warp) default is intentionally left in place. template MGARDX_CONT Task> GenTask(int queue_idx) { @@ -190,29 +189,17 @@ void Decode(SubArray<1, H, DeviceType> densely, SubArray<1, Q, DeviceType> bcode, SIZE len, int chunk_size, int n_chunk, SubArray<1, uint8_t, DeviceType> singleton, size_t singleton_size, int queue_idx) { - int maxbytes = DeviceRuntime::GetMaxSharedMemorySize(); - // Shared memory is disabled as it does not provide better performance - // if (singleton_size <= maxbytes) { - // if (DeviceRuntime::PrintKernelConfig) { - // std::cout << log::log_info - // << "Decode: using share memory to cache decodebook\n"; - // } - // DeviceLauncher::Execute( - // DecodeKernel(densely, dH_meta, bcode, len, - // chunk_size, n_chunk, singleton, - // singleton_size), - // queue_idx); - // } else { + // Opt 3 caches only first[]/entry[] (~1KB) in shared memory, which always + // fits and never hurts occupancy, so the cached path is used unconditionally. if (DeviceRuntime::PrintKernelConfig) { std::cout << log::log_info - << "Decode: not using share memory to cache decodebook\n"; + << "Decode: caching first[]/entry[] in shared memory\n"; } DeviceLauncher::Execute( - DecodeKernel(densely, dH_meta, bcode, len, - chunk_size, n_chunk, singleton, - singleton_size), + DecodeKernel(densely, dH_meta, bcode, len, + chunk_size, n_chunk, singleton, + singleton_size), queue_idx); - // } } } // namespace mgard_x diff --git a/include/mgard-x/Lossless/ParallelHuffman/Deflate.hpp b/include/mgard-x/Lossless/ParallelHuffman/Deflate.hpp index 069d8544ca..dd4414a693 100644 --- a/include/mgard-x/Lossless/ParallelHuffman/Deflate.hpp +++ b/include/mgard-x/Lossless/ParallelHuffman/Deflate.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_DEFLATE_TEMPLATE_HPP @@ -42,20 +42,25 @@ class DeflateFunctor : public Functor { : hcoded.shape(0) - gid * PART_SIZE; // if ((gid + 1) * PART_SIZE > hcoded.shape(0)) printf("\n\ngid // %lu\tending %lu\n\n", gid, ending); - H msb_bw_word_lsb, _1, _2; + H msb_bw_word_lsb, _1, _2, buffer; H *current = hcoded(gid * PART_SIZE); for (size_t i = 0; i < ending; i++) { + msb_bw_word_lsb = *hcoded(gid * PART_SIZE + i); bitwidth = *((uint8_t *)&msb_bw_word_lsb + (sizeof(H) - 1)); *((uint8_t *)&msb_bw_word_lsb + sizeof(H) - 1) = 0x0; - if (densely_coded_lsb_pos == sizeof(H) * 8) - *current = 0x0; // a new unit of data type + if (densely_coded_lsb_pos == sizeof(H) * 8) { + // *current = 0x0; // a new unit of data type + buffer = 0x0; + } if (bitwidth <= densely_coded_lsb_pos) { densely_coded_lsb_pos -= bitwidth; - *current |= msb_bw_word_lsb << densely_coded_lsb_pos; + // *current |= msb_bw_word_lsb << densely_coded_lsb_pos; + buffer |= msb_bw_word_lsb << densely_coded_lsb_pos; if (densely_coded_lsb_pos == 0) { densely_coded_lsb_pos = sizeof(H) * 8; + *current = buffer; ++current; } } else { @@ -66,14 +71,23 @@ class DeflateFunctor : public Functor { _1 = msb_bw_word_lsb >> (bitwidth - densely_coded_lsb_pos); _2 = msb_bw_word_lsb << (sizeof(H) * 8 - (bitwidth - densely_coded_lsb_pos)); - *current |= _1; - *(++current) = 0x0; - *current |= _2; + buffer |= _1; + *current = buffer; + // *current |= _1; + // *(++current) = 0x0; + current++; + // *current |= _2; + buffer = 0x0; + buffer = _2; densely_coded_lsb_pos = sizeof(H) * 8 - (bitwidth - densely_coded_lsb_pos); } total_bitwidth += bitwidth; } + // if (densely_coded_lsb_pos != sizeof(H) * 8) { + *current = buffer; + // ++current; + // } *densely_meta(gid) = total_bitwidth; } diff --git a/include/mgard-x/Lossless/ParallelHuffman/DictionaryShift.hpp b/include/mgard-x/Lossless/ParallelHuffman/DictionaryShift.hpp index 391ec0e5c0..fb0b4768a0 100644 --- a/include/mgard-x/Lossless/ParallelHuffman/DictionaryShift.hpp +++ b/include/mgard-x/Lossless/ParallelHuffman/DictionaryShift.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_DICTIONARY_SHIFT_TEMPLATE_HPP diff --git a/include/mgard-x/Lossless/ParallelHuffman/EncodeFixedLen.hpp b/include/mgard-x/Lossless/ParallelHuffman/EncodeFixedLen.hpp index e361b59c64..ef9e786728 100644 --- a/include/mgard-x/Lossless/ParallelHuffman/EncodeFixedLen.hpp +++ b/include/mgard-x/Lossless/ParallelHuffman/EncodeFixedLen.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_ENCODE_FIXED_LEN_TEMPLATE_HPP diff --git a/include/mgard-x/Lossless/ParallelHuffman/EntropyCalculator.hpp b/include/mgard-x/Lossless/ParallelHuffman/EntropyCalculator.hpp index e57d657c25..9b0a84856e 100644 --- a/include/mgard-x/Lossless/ParallelHuffman/EntropyCalculator.hpp +++ b/include/mgard-x/Lossless/ParallelHuffman/EntropyCalculator.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_ENTROPY_CALCULAOR_HPP @@ -20,6 +20,14 @@ double CalculateLC(SIZE total_num_elems, SIZE dict_size, MemoryManager::Copy1D(_cl, CL.data(), dict_size, queue_idx); DeviceRuntime::SyncQueue(queue_idx); double LC = 0; + // for (SIZE i = 0; i < dict_size; i++) { + // std::cout << _freq[i] << " "; + // } + // std::cout << "\n"; + // for (SIZE i = 0; i < dict_size; i++) { + // std::cout << _cl[i] << " "; + // } + // std::cout << "\n"; for (SIZE i = 0; i < dict_size; i++) { LC += (double)_freq[i] / total_num_elems * _cl[i]; } diff --git a/include/mgard-x/Lossless/ParallelHuffman/FillArraySequence.hpp b/include/mgard-x/Lossless/ParallelHuffman/FillArraySequence.hpp index 6926deb7d6..aec1a91db5 100644 --- a/include/mgard-x/Lossless/ParallelHuffman/FillArraySequence.hpp +++ b/include/mgard-x/Lossless/ParallelHuffman/FillArraySequence.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_FILL_ARRAY_SEQUENCE_TEMPLATE_HPP diff --git a/include/mgard-x/Lossless/ParallelHuffman/GenerateCL.hpp b/include/mgard-x/Lossless/ParallelHuffman/GenerateCL.hpp index 617a3fa1c5..24b971ed9e 100644 --- a/include/mgard-x/Lossless/ParallelHuffman/GenerateCL.hpp +++ b/include/mgard-x/Lossless/ParallelHuffman/GenerateCL.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2021, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-GPU: MultiGrid Adaptive Reduction oT Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: December 1, 2021 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_GENERATE_CL_TEMPLATE_HPP @@ -10,6 +10,8 @@ #include "../../RuntimeX/RuntimeX.h" +#include + namespace mgard_x { #define MAX(X, Y) (((X) > (Y)) ? (X) : (Y)) @@ -45,14 +47,25 @@ class GenerateCLFunctor : public HuffmanCLCustomizedFunctor { SubArray<1, int, DeviceType> copyIsLeaf, SubArray<1, int, DeviceType> copyIndex, SubArray<1, uint32_t, DeviceType> diagonal_path_intersections, - SubArray<1, int, DeviceType> status) + SubArray<1, int, DeviceType, false, true> status, + // Width of the per-block diagonal merge-path search (Operations 6-9) + // and of the thread block that runs it. Read from the device's real + // warp/wavefront size (DeviceRuntime::GetWarpSize(), host-side only, + // so it is threaded through as a plain functor member rather than + // read from device code) instead of assuming the CUDA-oriented + // MGARDX_WARP_SIZE=32: the merge-path search here uses shared memory + // with an explicit sync between operations (not warp-implicit + // shuffle/ballot lockstep), so it is safe to size to whatever the + // hardware actually reports, letting each block search a full + // wavefront's worth of the diagonal per step. + SIZE warp_size) : histogram(histogram), CL(CL), dict_size(dict_size), lNodesFreq(lNodesFreq), lNodesLeader(lNodesLeader), iNodesFreq(iNodesFreq), iNodesLeader(iNodesLeader), tempFreq(tempFreq), tempIsLeaf(tempIsLeaf), tempIndex(tempIndex), copyFreq(copyFreq), copyIsLeaf(copyIsLeaf), copyIndex(copyIndex), diagonal_path_intersections(diagonal_path_intersections), - status(status) { + status(status), warp_size(warp_size) { HuffmanCLCustomizedFunctor(); } @@ -215,8 +228,8 @@ class GenerateCLFunctor : public HuffmanCLCustomizedFunctor { // printf("update leader of leaf %d to just created internal node // %d\n", *status((IDX)_lNodesCur), // *lNodesLeader((IDX)(*status((IDX)_lNodesCur)))); - ++(*CL((IDX)(*status((IDX)_lNodesCur)))), - ++(*status((IDX)_lNodesCur)); + ++(*CL((IDX)(*status((IDX)_lNodesCur)))); + ++(*status((IDX)_lNodesCur)); // printf("remove mid[0] from unprocessed leaf node list. _lNodesCur = // %d\n", *status((IDX)_lNodesCur)); printf("update CL(%d) = %u\n", // *status((IDX)_lNodesCur-1), @@ -246,8 +259,8 @@ class GenerateCLFunctor : public HuffmanCLCustomizedFunctor { // %d\n", // *status((IDX)_lNodesCur), // *lNodesLeader((IDX)(*status((IDX)_lNodesCur)))); - ++(*CL((IDX)(*status((IDX)_lNodesCur)))), - ++(*status((IDX)_lNodesCur)); + ++(*CL((IDX)(*status((IDX)_lNodesCur)))); + ++(*status((IDX)_lNodesCur)); // printf("remove mid[1] from unprocessed leaf node list. _lNodesCur = // %d\n", *status((IDX)_lNodesCur)); printf("update CL(%d) = %u\n", // *status((IDX)_lNodesCur-1), @@ -312,6 +325,9 @@ class GenerateCLFunctor : public HuffmanCLCustomizedFunctor { } MGARDX_EXEC void Operation4() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); // Copy all leaf nodes to be parallel merged in a temp buffer if (i - (*status((IDX)_lNodesCur)) < (*status((IDX)_curLeavesNum))) { *copyFreq((IDX)i - (*status((IDX)_lNodesCur))) = *lNodesFreq((IDX)i); @@ -447,10 +463,9 @@ class GenerateCLFunctor : public HuffmanCLCustomizedFunctor { // combinedIndex: %d\n", A_length, B_length, (*status((IDX)_tempLength)), // combinedIndex); // } - threadOffset = - FunctorBase::GetThreadIdX() - MGARDX_WARP_SIZE / 2; + threadOffset = FunctorBase::GetThreadIdX() - warp_size / 2; - if (FunctorBase::GetThreadIdX() < MGARDX_WARP_SIZE) { + if (FunctorBase::GetThreadIdX() < warp_size) { // Figure out the coordinates of our diagonal if (A_length >= B_length) { *x_top = MIN(combinedIndex, A_length); @@ -485,7 +500,7 @@ class GenerateCLFunctor : public HuffmanCLCustomizedFunctor { // dict_size); getfrom_y = (*status((IDX)_mergeFront)) + current_y; - if (FunctorBase::GetThreadIdX() < MGARDX_WARP_SIZE) { + if (FunctorBase::GetThreadIdX() < warp_size) { if (getfrom_y >= dict_size) getfrom_y -= dict_size; @@ -515,8 +530,8 @@ class GenerateCLFunctor : public HuffmanCLCustomizedFunctor { // oneorzero[28], oneorzero[29], oneorzero[30], oneorzero[31]); // If we find the meeting of the '1's and '0's, we found the // intersection of the path and diagonal - if (FunctorBase::GetThreadIdX() > 0 and // - FunctorBase::GetThreadIdX() < MGARDX_WARP_SIZE and // + if (FunctorBase::GetThreadIdX() > 0 and // + FunctorBase::GetThreadIdX() < warp_size and // (oneorzero[FunctorBase::GetThreadIdX()] != oneorzero[FunctorBase::GetThreadIdX() - 1]) // ) { @@ -533,8 +548,8 @@ class GenerateCLFunctor : public HuffmanCLCustomizedFunctor { MGARDX_EXEC void Operation9() { // Adjust the search window on the diagonal - if (FunctorBase::GetThreadIdX() == MGARDX_WARP_SIZE / 2) { - if (oneorzero[MGARDX_WARP_SIZE - 1] != 0) { + if (FunctorBase::GetThreadIdX() == warp_size / 2) { + if (oneorzero[warp_size - 1] != 0) { *x_bottom = current_x; *y_bottom = current_y; } else { @@ -740,7 +755,7 @@ class GenerateCLFunctor : public HuffmanCLCustomizedFunctor { MGARDX_CONT size_t shared_memory_size() { size_t sm_size = 0; sm_size += 5 * sizeof(int32_t); - sm_size += DeviceRuntime::GetWarpSize() * sizeof(int32_t); + sm_size += warp_size * sizeof(int32_t); return sm_size; } @@ -760,7 +775,8 @@ class GenerateCLFunctor : public HuffmanCLCustomizedFunctor { SubArray<1, int, DeviceType> copyIsLeaf; SubArray<1, int, DeviceType> copyIndex; SubArray<1, uint32_t, DeviceType> diagonal_path_intersections; - SubArray<1, int, DeviceType> status; + SubArray<1, int, DeviceType, false, true> status; + SIZE warp_size; int32_t *x_top; int32_t *y_top; @@ -802,7 +818,7 @@ class GenerateCLKernel : public Kernel { SubArray<1, int, DeviceType> copyIsLeaf, SubArray<1, int, DeviceType> copyIndex, SubArray<1, uint32_t, DeviceType> diagonal_path_intersections, - SubArray<1, int, DeviceType> status) + SubArray<1, int, DeviceType, false, true> status) : histogram(histogram), CL(CL), dict_size(dict_size), lNodesFreq(lNodesFreq), lNodesLeader(lNodesLeader), iNodesFreq(iNodesFreq), iNodesLeader(iNodesLeader), tempFreq(tempFreq), @@ -811,19 +827,59 @@ class GenerateCLKernel : public Kernel { diagonal_path_intersections(diagonal_path_intersections), status(status) {} + // Block width of GenerateCL's merge-path search, and the number of uint32 + // entries diagonal_path_intersections must hold for it. + // + // These have to agree with GenTask below and with whoever allocates the + // array (HuffmanWorkspace), so they live here as the single definition. + // The kernel writes indices up to 2 * gridDim.x + 1 (see Operation7/8), and + // GenTask launches ceil(dict_size / block width) blocks -- sizing the array + // from an occupancy estimate instead overflows it whenever the dictionary + // needs more blocks than the device is estimated to run at once. + MGARDX_CONT static SIZE BlockWidth() { + if constexpr (std::is_same::value) { + // Empirically tuned on MI300: the merge-path search here uses shared + // memory + explicit inter-Operation sync barriers rather than raw + // warp-shuffle/ballot, so its "width" is really just a block size and + // isn't tied to the hardware wavefront. A direct sweep (64/128/256/ + // 512/1024) on the real 64-wide CDNA wavefront found throughput + // climbing well past 64 -- 64 (the true wavefront size) measures + // ~86 GB/s, 512 measures 110-114 GB/s, a ~30% further gain. 1024 (the + // hardware max block width) also completes safely here (unlike + // OutlierSeparator's tbx, which hangs at 1024) but measures ~111 GB/s, + // matching 512 within noise, so 512 is used as the smallest width that + // captures the full gain. Revisit with a fresh sweep if this kernel's + // algorithm changes. + return 512; + } else { + return DeviceRuntime::GetWarpSize(); + } + } + + MGARDX_CONT static SIZE DiagonalPathIntersectionsSize(SIZE dict_size) { + SIZE gridx = (dict_size - 1) / BlockWidth() + 1; + // Keep the old occupancy-derived size as a floor so no device shrinks. + SIZE occupancy_blocks = + (DeviceRuntime::GetMaxNumThreadsPerTB() / + DeviceRuntime::GetWarpSize()) * + DeviceRuntime::GetNumSMs(); + return 2 * (std::max(gridx, occupancy_blocks) + 1); + } + MGARDX_CONT Task> GenTask(int queue_idx) { using FunctorType = GenerateCLFunctor; + SIZE warp_size = BlockWidth(); FunctorType Functor(histogram, CL, dict_size, lNodesFreq, lNodesLeader, iNodesFreq, iNodesLeader, tempFreq, tempIsLeaf, tempIndex, copyFreq, copyIsLeaf, copyIndex, - diagonal_path_intersections, status); + diagonal_path_intersections, status, warp_size); SIZE tbx, tby, tbz, gridx, gridy, gridz; size_t sm_size = Functor.shared_memory_size(); tbz = 1; tby = 1; - tbx = DeviceRuntime::GetWarpSize(); + tbx = warp_size; int cg_blocks_sm = DeviceRuntime::GetOccupancyMaxActiveBlocksPerSM( @@ -868,7 +924,7 @@ class GenerateCLKernel : public Kernel { SubArray<1, int, DeviceType> copyIsLeaf; SubArray<1, int, DeviceType> copyIndex; SubArray<1, uint32_t, DeviceType> diagonal_path_intersections; - SubArray<1, int, DeviceType> status; + SubArray<1, int, DeviceType, false, true> status; }; #undef MOD diff --git a/include/mgard-x/Lossless/ParallelHuffman/GenerateCW.hpp b/include/mgard-x/Lossless/ParallelHuffman/GenerateCW.hpp index 6fbcf8e2d9..82b9e5ff8e 100644 --- a/include/mgard-x/Lossless/ParallelHuffman/GenerateCW.hpp +++ b/include/mgard-x/Lossless/ParallelHuffman/GenerateCW.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_GENERATE_CW_TEMPLATE_HPP @@ -25,12 +25,10 @@ template class GenerateCWFunctor : public HuffmanCWCustomizedFunctor { public: MGARDX_CONT GenerateCWFunctor() {} - MGARDX_CONT GenerateCWFunctor(SubArray<1, T, DeviceType> CL, - SubArray<1, H, DeviceType> CW, - SubArray<1, H, DeviceType> first, - SubArray<1, H, DeviceType> entry, - SIZE dict_size, - SubArray<1, int, DeviceType> status) + MGARDX_CONT GenerateCWFunctor( + SubArray<1, T, DeviceType> CL, SubArray<1, H, DeviceType> CW, + SubArray<1, H, DeviceType> first, SubArray<1, H, DeviceType> entry, + SIZE dict_size, SubArray<1, int, DeviceType, false, true> status) : CL(CL), CW(CW), first(first), entry(entry), dict_size(dict_size), status(status) { HuffmanCWCustomizedFunctor(); @@ -243,7 +241,7 @@ class GenerateCWFunctor : public HuffmanCWCustomizedFunctor { SubArray<1, H, DeviceType> CW; SubArray<1, H, DeviceType> first; SubArray<1, H, DeviceType> entry; - SubArray<1, int, DeviceType> status; + SubArray<1, int, DeviceType, false, true> status; SIZE dict_size; // unsigned int thread; @@ -264,7 +262,7 @@ class GenerateCWKernel : public Kernel { GenerateCWKernel(SubArray<1, T, DeviceType> CL, SubArray<1, H, DeviceType> CW, SubArray<1, H, DeviceType> first, SubArray<1, H, DeviceType> entry, SIZE dict_size, - SubArray<1, int, DeviceType> status) + SubArray<1, int, DeviceType, false, true> status) : CL(CL), CW(CW), first(first), entry(entry), dict_size(dict_size), status(status) {} @@ -312,7 +310,7 @@ class GenerateCWKernel : public Kernel { SubArray<1, H, DeviceType> CW; SubArray<1, H, DeviceType> first; SubArray<1, H, DeviceType> entry; - SubArray<1, int, DeviceType> status; + SubArray<1, int, DeviceType, false, true> status; SIZE dict_size; }; diff --git a/include/mgard-x/Lossless/ParallelHuffman/GetCodebook.hpp b/include/mgard-x/Lossless/ParallelHuffman/GetCodebook.hpp index fc3343d377..9ce10d7c55 100644 --- a/include/mgard-x/Lossless/ParallelHuffman/GetCodebook.hpp +++ b/include/mgard-x/Lossless/ParallelHuffman/GetCodebook.hpp @@ -1,10 +1,11 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ +#include "../../RuntimeX/Utilities/Exceptions.h" #include "EntropyCalculator.hpp" #include "FillArraySequence.hpp" #include "GenerateCL.hpp" @@ -23,7 +24,7 @@ namespace mgard_x { // Parallel codebook generation wrapper template -void GetCodebook(int dict_size, +void GetCodebook(int dict_size, size_t primary_count, SubArray<1, unsigned int, DeviceType> _d_freq_subarray, SubArray<1, H, DeviceType> _d_codebook_subarray, SubArray<1, uint8_t, DeviceType> _d_decode_meta_subarray, @@ -101,14 +102,12 @@ void GetCodebook(int dict_size, if (log::level & log::INFO) { // PrintSubarray("GenerateCL::CL_subarray", workspace.CL_subarray); // std::cout << "GenerateCL: max_CL: " << max_CL << std::endl; - double LC = - CalculateLC(workspace.huff_array.shape(0), nz_dict_size, - _nz_d_freq_subarray, workspace.CL_subarray, queue_idx); - double entropy = - CalculateEntropy(workspace.huff_array.shape(0), nz_dict_size, - _nz_d_freq_subarray, queue_idx); - log::info("LC: " + std::to_string(LC)); - log::info("Entropy: " + std::to_string(entropy)); + double LC = CalculateLC(primary_count, nz_dict_size, _nz_d_freq_subarray, + workspace.CL_subarray, queue_idx); + double entropy = CalculateEntropy(primary_count, nz_dict_size, + _nz_d_freq_subarray, queue_idx); + log::dbg("LC: " + std::to_string(LC)); + log::dbg("Entropy: " + std::to_string(entropy)); } // DumpSubArray("cl_"+std::to_string(workspace.huff_array.shape(0))+".dat", @@ -116,22 +115,17 @@ void GetCodebook(int dict_size, int max_CW_bits = (sizeof(H) * 8) - 8; if (max_CL > max_CW_bits) { - std::cout << log::log_err << "Cannot store all Huffman codewords in " - << max_CW_bits + 8 << "-bit representation" << std::endl; - std::cout << log::log_err - << "Huffman codeword representation requires at least " - << max_CL + 8 << " bits (longest codeword: " << max_CL << " bits)" - << std::endl; // Throw (instead of exit) so callers can catch and fall back to another // lossless backend (e.g. raw Zstd) or retry with a smaller huff_dict_size. // A longer dictionary makes the tree deeper, so a degenerate/low-entropy // input can produce codewords exceeding the H-type budget (sizeof(H)*8 - // 8). - throw std::runtime_error( - "MGARD-X Huffman: longest codeword (" + std::to_string(max_CL) + - " bits) exceeds the " + std::to_string(max_CW_bits) + - "-bit budget of the H code type; retry with a smaller huff_dict_size " - "or a different lossless backend"); + throw ProcessingException( + "Cannot store all Huffman codewords in " + + std::to_string(max_CW_bits + 8) + + "-bit representation; representation requires at least " + + std::to_string(max_CL + 8) + + " bits (longest codeword: " + std::to_string(max_CL) + " bits)"); } DeviceLauncher::Execute( diff --git a/include/mgard-x/Lossless/ParallelHuffman/GetFirstNonzeroIndex.hpp b/include/mgard-x/Lossless/ParallelHuffman/GetFirstNonzeroIndex.hpp index bc8ea40f33..5373903c70 100644 --- a/include/mgard-x/Lossless/ParallelHuffman/GetFirstNonzeroIndex.hpp +++ b/include/mgard-x/Lossless/ParallelHuffman/GetFirstNonzeroIndex.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_GET_FIRST_NONZERO_INDEX_TEMPLATE_HPP diff --git a/include/mgard-x/Lossless/ParallelHuffman/Histogram.hpp b/include/mgard-x/Lossless/ParallelHuffman/Histogram.hpp index a309dd4c3c..38cd83808a 100644 --- a/include/mgard-x/Lossless/ParallelHuffman/Histogram.hpp +++ b/include/mgard-x/Lossless/ParallelHuffman/Histogram.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_HUFFMAN_HISTOGRAM_TEMPLATE_HPP @@ -15,12 +15,14 @@ template class HistogramFunctor : public Functor { public: MGARDX_CONT HistogramFunctor() {} - MGARDX_CONT HistogramFunctor(SubArray<1, T, DeviceType> input_data, - SubArray<1, int, DeviceType> local_histogram, - SubArray<1, Q, DeviceType> output, SIZE N, - int bins, int RPerBlock) + MGARDX_CONT + HistogramFunctor(SubArray<1, T, DeviceType> input_data, + SubArray<1, int, DeviceType, false, true> local_histogram, + SubArray<1, Q, DeviceType> output, SIZE N, int bins, + int RPerBlock, SIZE warp_size) : input_data(input_data), local_histogram(local_histogram), - output(output), N(N), bins(bins), RPerBlock(RPerBlock) { + output(output), N(N), bins(bins), RPerBlock(RPerBlock), + warp_size(warp_size) { Functor(); } @@ -32,16 +34,23 @@ class HistogramFunctor : public Functor { bins); } - warpid = (int)(FunctorBase::GetThreadIdX() / MGARDX_WARP_SIZE); - lane = FunctorBase::GetThreadIdX() % MGARDX_WARP_SIZE; - warps_block = FunctorBase::GetBlockDimX() / MGARDX_WARP_SIZE; + // Grid-stride partitioning granularity: this is arithmetic work + // partitioning (which "warp group" a thread belongs to for load + // balancing), not a hardware lockstep/shuffle assumption, so it is safe + // to size to the real warp/wavefront (DeviceRuntime::GetWarpSize(), + // threaded through as warp_size) rather than the CUDA-oriented + // MGARDX_WARP_SIZE=32 constant. Matching the real wavefront also keeps + // each warp's global-memory accesses (input_data(i) below) coalesced. + warpid = (int)(FunctorBase::GetThreadIdX() / warp_size); + lane = FunctorBase::GetThreadIdX() % warp_size; + warps_block = FunctorBase::GetBlockDimX() / warp_size; off_rep = (bins) * (FunctorBase::GetThreadIdX() % RPerBlock); begin = (N / warps_block) * warpid + - MGARDX_WARP_SIZE * FunctorBase::GetBlockIdX() + lane; + warp_size * FunctorBase::GetBlockIdX() + lane; end = (N / warps_block) * (warpid + 1); - step = MGARDX_WARP_SIZE * FunctorBase::GetGridDimX(); + step = warp_size * FunctorBase::GetGridDimX(); // final warp handles data outside of the warps_block partitions if (warpid >= warps_block - 1) @@ -97,11 +106,12 @@ class HistogramFunctor : public Functor { private: SubArray<1, T, DeviceType> input_data; - SubArray<1, int, DeviceType> local_histogram; + SubArray<1, int, DeviceType, false, true> local_histogram; SubArray<1, Q, DeviceType> output; SIZE N; int bins; int RPerBlock; + SIZE warp_size; int *Hs; @@ -124,7 +134,7 @@ class HistogramKernel : public Kernel { MGARDX_CONT HistogramKernel(SubArray<1, T, DeviceType> input_data, - SubArray<1, int, DeviceType> local_histogram, + SubArray<1, int, DeviceType, false, true> local_histogram, SubArray<1, Q, DeviceType> output, SIZE N, int bins, int RPerBlock, int threadsPerBlock, int numBlocks) : input_data(input_data), local_histogram(local_histogram), @@ -135,8 +145,8 @@ class HistogramKernel : public Kernel { GenTask(int queue_idx) { using FunctorType = HistogramFunctor; - FunctorType functor(input_data, local_histogram, output, N, bins, - RPerBlock); + FunctorType functor(input_data, local_histogram, output, N, bins, RPerBlock, + DeviceRuntime::GetWarpSize()); SIZE tbx, tby, tbz, gridx, gridy, gridz; size_t sm_size = functor.shared_memory_size(); @@ -153,7 +163,7 @@ class HistogramKernel : public Kernel { private: SubArray<1, T, DeviceType> input_data; - SubArray<1, int, DeviceType> local_histogram; + SubArray<1, int, DeviceType, false, true> local_histogram; SubArray<1, Q, DeviceType> output; SIZE N; int bins; @@ -190,7 +200,7 @@ MGARDX_CONT void Histogram(SubArray<1, T, DeviceType> input_data, SubArray<1, Q, DeviceType> output, SIZE N, int bins, int queue_idx) { int maxbytes = DeviceRuntime::GetMaxSharedMemorySize(); - SubArray<1, int, DeviceType> local_histogram; + SubArray<1, int, DeviceType, false, true> local_histogram; if (bins * sizeof(int) < maxbytes) { if (DeviceRuntime::PrintKernelConfig) { std::cout << log::log_info @@ -220,8 +230,8 @@ MGARDX_CONT void Histogram(SubArray<1, T, DeviceType> input_data, int RPerBlock = 2; int threadsPerBlock, numBlocks; ExecutionConfig(N, bins, RPerBlock, threadsPerBlock, numBlocks); - Array<1, int, DeviceType> local_histogram_array( - {(SIZE)RPerBlock * bins * numBlocks}, false, true); + Array<1, int, DeviceType, false, true> local_histogram_array( + {(SIZE)RPerBlock * bins * numBlocks}); local_histogram_array.memset(0); // TODO: can we not sync all queues? DeviceRuntime::SyncAllQueues(); diff --git a/include/mgard-x/Lossless/ParallelHuffman/Huffman.hpp b/include/mgard-x/Lossless/ParallelHuffman/Huffman.hpp index 4f2105f649..754e824716 100644 --- a/include/mgard-x/Lossless/ParallelHuffman/Huffman.hpp +++ b/include/mgard-x/Lossless/ParallelHuffman/Huffman.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_HUFFMAN_TEMPLATE_HPP @@ -22,6 +22,7 @@ static bool debug_print_huffman = false; #include "Histogram.hpp" #include "HuffmanWorkspace.hpp" #include "OutlierSeparator.hpp" +#include "ParallelDeflate.hpp" #include using namespace std::chrono; @@ -29,7 +30,7 @@ using namespace std::chrono; namespace mgard_x { template -class Huffman : public LosslessCompressorInterface { +class Huffman { public: Huffman() : initialized(false) {} @@ -58,9 +59,104 @@ class Huffman : public LosslessCompressorInterface { primary_count, dict_size, chunk_size, estimated_outlier_ratio); } - void CompressPrimary(Array<1, Q, DeviceType> &primary_data, + double EstimateCR(Array<1, Q, DeviceType> &primary_data, int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + SubArray primary_subarray(primary_data); + workspace.reset(queue_idx); + + primary_count = primary_subarray.shape(0); + + Histogram(primary_subarray, + workspace.freq_subarray, + primary_count, dict_size, queue_idx); + auto type_bw = sizeof(H) * 8; + + SubArray<1, H, DeviceType> _d_first_subarray( + {(SIZE)type_bw}, (H *)workspace.decodebook_subarray((IDX)0)); + SubArray<1, H, DeviceType> _d_entry_subarray( + {(SIZE)type_bw}, + (H *)workspace.decodebook_subarray(sizeof(H) * type_bw)); + SubArray<1, Q, DeviceType> _d_qcode_subarray( + {(SIZE)dict_size}, + (Q *)workspace.decodebook_subarray(sizeof(H) * 2 * type_bw)); + + // Sort Qcodes by frequency + DeviceLauncher::Execute( + FillArraySequenceKernel(_d_qcode_subarray), queue_idx); + + MemoryManager::Copy1D(workspace._d_freq_copy_subarray.data(), + workspace.freq_subarray.data(), dict_size, + queue_idx); + MemoryManager::Copy1D(workspace._d_qcode_copy_subarray.data(), + _d_qcode_subarray.data(), dict_size, + queue_idx); + DeviceCollective::SortByKey( + (SIZE)dict_size, workspace._d_freq_copy_subarray, + workspace._d_qcode_copy_subarray, workspace.freq_subarray, + _d_qcode_subarray, workspace.sort_by_key_workspace, true, queue_idx); + + DeviceLauncher::Execute( + GetFirstNonzeroIndexKernel( + workspace.freq_subarray, workspace.first_nonzero_index_subarray), + queue_idx); + + unsigned int first_nonzero_index; + MemoryManager().Copy1D( + &first_nonzero_index, workspace.first_nonzero_index_subarray(IDX(0)), 1, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + int nz_dict_size = dict_size - first_nonzero_index; + + SubArray<1, unsigned int, DeviceType> _nz_d_freq_subarray( + {(SIZE)nz_dict_size}, workspace.freq_subarray(first_nonzero_index)); + SubArray<1, H, DeviceType> _nz_d_codebook_subarray( + {(SIZE)nz_dict_size}, workspace.codebook_subarray(first_nonzero_index)); + + DeviceLauncher::Execute( + GenerateCLKernel( + _nz_d_freq_subarray, workspace.CL_subarray, nz_dict_size, + _nz_d_freq_subarray, workspace.lNodesLeader_subarray, + workspace.iNodesFreq_subarray, workspace.iNodesLeader_subarray, + workspace.tempFreq_subarray, workspace.tempIsLeaf_subarray, + workspace.tempIndex_subarray, workspace.copyFreq_subarray, + workspace.copyIsLeaf_subarray, workspace.copyIndex_subarray, + workspace.diagonal_path_intersections_subarray, + workspace.status_subarray), + queue_idx); + + unsigned int *_freq = new unsigned int[dict_size]; + unsigned int *_cl = new unsigned int[dict_size]; + MemoryManager::Copy1D(_freq, workspace.freq_subarray.data(), + dict_size, queue_idx); + MemoryManager::Copy1D(_cl, workspace.CL_subarray.data(), + dict_size, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + double LC = 0; + for (SIZE i = 0; i < dict_size; i++) { + LC += (double)_freq[i] * _cl[i]; + } + delete[] _freq; + delete[] _cl; + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Huffman estimate CR", primary_count * sizeof(Q)); + timer.clear(); + } + + double CR = (double)(sizeof(Q) * primary_count) / (LC / 8 + 2000); + return CR; + } + + bool CompressPrimary(Array<1, Q, DeviceType> &primary_data, Array<1, Byte, DeviceType> &compressed_data, - int queue_idx) { + float target_cr, int queue_idx) { Timer timer; if (log::level & log::TIME) { @@ -73,104 +169,160 @@ class Huffman : public LosslessCompressorInterface { primary_count = primary_subarray.shape(0); + // Per-stage timing scaffolding (disabled; uncomment the block below and the + // mark() calls to profile each stage under log::TIME). Each mark() syncs + // the queue, prints the elapsed time of the stage just finished against the + // primary input size, and restarts the stage timer. Note: this serializes + // the pipeline (one sync per stage). + // Timer timer_stage; + // auto mark = [&](const char *name) { + // if (log::level & log::TIME) { + // DeviceRuntime::SyncQueue(queue_idx); + // timer_stage.end(); + // timer_stage.print(name, primary_count * sizeof(Q)); + // timer_stage.clear(); + // timer_stage.start(); + // } + // }; + // if (log::level & log::TIME) { + // DeviceRuntime::SyncQueue(queue_idx); + // timer_stage.start(); + // } + Histogram(primary_subarray, workspace.freq_subarray, primary_count, dict_size, queue_idx); + // mark("Huffman stage: histogram"); if (debug_print_huffman) { PrintSubarray("Histogram::freq_subarray", workspace.freq_subarray); } - GetCodebook(dict_size, workspace.freq_subarray, workspace.codebook_subarray, - workspace.decodebook_subarray, workspace, queue_idx); + GetCodebook(dict_size, primary_count, workspace.freq_subarray, + workspace.codebook_subarray, workspace.decodebook_subarray, + workspace, queue_idx); + // mark("Huffman stage: codebook"); + + if (target_cr > 1.0) { + workspace.freq_array.hostCopy(false, queue_idx); + workspace.CL_array.hostCopy(false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + unsigned int *_freq = workspace.freq_array.dataHost(); + unsigned int *_cl = workspace.CL_array.dataHost(); + double LC = 0; + for (SIZE i = 0; i < dict_size; i++) { + LC += (double)_freq[i] * _cl[i]; + } + double estimated_cr = + (double)(sizeof(Q) * primary_count) / (LC / 8 + 2000); + log::info("Huffman estimated CR: " + std::to_string(estimated_cr) + + " (target: " + std::to_string(target_cr) + ")"); + if (estimated_cr < target_cr) { + return false; + } + } + if (debug_print_huffman) { PrintSubarray("GetCodebook::codebook_subarray", workspace.codebook_subarray); PrintSubarray("GetCodebook::decodebook_subarray", workspace.decodebook_subarray); } + // Encoding is fused into the deflate kernels below: instead of first + // materializing huff[i] = codebook[data[i]] into a primary_count-sized + // array and reading it back twice, GroupBits and Pack look up + // codebook[data[i]] on the fly. This removes a full pass and the huff_array + // allocation. + + // Parallel deflate sizing. The actual bit-packing into the final buffer is + // deferred to Serialize (once the output layout is known) so that each + // chunk can be written directly to its final offset, eliminating a separate + // condense/gather pass. Here we only compute the output geometry: + // per-group bit sums -> scan -> per-chunk bit lengths + word counts + // -> scan -> per-chunk word offsets (and the total ddata word count). + SIZE groups_per_chunk = + (chunk_size - 1) / GetDeflateGroupSize(chunk_size) + 1; + auto nchunk = (primary_count - 1) / chunk_size + 1; + SIZE ngroups = (SIZE)(nchunk * groups_per_chunk); DeviceLauncher::Execute( - EncodeFixedLenKernel(primary_subarray, - workspace.huff_subarray, - workspace.codebook_subarray), + DeflateGroupBitsKernel( + primary_subarray, workspace.codebook_subarray, + workspace.deflate_group_bits_subarray, primary_count, chunk_size, + groups_per_chunk, ngroups), queue_idx); - - if (debug_print_huffman) { - PrintSubarray("EncodeFixedLen::huff_subarray", workspace.huff_subarray); - } - - // deflate + // mark("Huffman stage: deflate group-bits"); + DeviceCollective::ScanSumExtended( + ngroups, workspace.deflate_group_bits_subarray, + workspace.deflate_group_offsets_subarray, + workspace.deflate_group_scan_workspace, true, queue_idx); DeviceLauncher::Execute( - DeflateKernel(workspace.huff_subarray, - workspace.huff_bitwidths_subarray, - chunk_size), + DeflateChunkMetaKernel( + workspace.deflate_group_offsets_subarray, + workspace.huff_bitwidths_subarray, + workspace.deflate_chunk_words_subarray, (SIZE)nchunk, + groups_per_chunk), queue_idx); - - if (debug_print_huffman) { - PrintSubarray("Deflate::huff_subarray", workspace.huff_subarray); - PrintSubarray("Deflate::huff_bitwidths_subarray", - workspace.huff_bitwidths_subarray); - } + DeviceCollective::ScanSumExtended( + (SIZE)nchunk, workspace.deflate_chunk_words_subarray, + workspace.deflate_chunk_word_offsets_subarray, + workspace.deflate_chunk_scan_workspace, true, queue_idx); + // Total number of H-words in the densely packed stream. + MemoryManager::Copy1D( + &ddata_size, + workspace.deflate_chunk_word_offsets_subarray.data() + nchunk, 1, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + // mark("Huffman stage: deflate sizing (scans+meta)"); + + // Pack the densely-coded bitstream directly into its final location in the + // output buffer (zero-copy): the byte offset where the packed stream lands + // is fully determined by the sizes of the preceding sections, all known + // here, so we can size the output now and have the pack kernel write + // straight to it. Serialize then only fills in the surrounding metadata. + // Each group writes MSB-first to its final intra-chunk offset; words fully + // owned by a group use plain stores while the (<=2) words shared with + // neighbouring groups use atomicOr, so the destination must be zeroed + // first. + SIZE packed_byte_offset; + SIZE compressed_size = ComputeSerializedLayout(packed_byte_offset); + compressed_data.resize({compressed_size}, queue_idx); + SubArray<1, Byte, DeviceType> compressed_data_subarray(compressed_data); + SubArray<1, H, DeviceType> packed_subarray( + {(SIZE)ddata_size}, (H *)compressed_data_subarray(packed_byte_offset)); + MemoryManager::Memset1D(packed_subarray.data(), ddata_size, 0, + queue_idx); + DeviceLauncher::Execute( + DeflatePackKernel( + primary_subarray, workspace.codebook_subarray, + workspace.deflate_group_offsets_subarray, + workspace.deflate_chunk_word_offsets_subarray, packed_subarray, + primary_count, chunk_size, groups_per_chunk, ngroups), + queue_idx); + // mark("Huffman stage: deflate pack"); // Serialize(compressed_data, queue_idx); if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); timer.end(); - timer.print("Huffman compress"); - log::time("Huffman compression throughput: " + - std::to_string((double)primary_count * sizeof(Q) / timer.get() / - 1e9) + - " GB/s"); + timer.print("Huffman compress", primary_count * sizeof(Q)); timer.clear(); } + return true; } - void Serialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { - Timer timer; - if (log::level & log::TIME) { - DeviceRuntime::SyncQueue(queue_idx); - timer.start(); - } + // Walks the serialized layout, returning the total compressed size and, via + // packed_byte_offset, the byte offset at which the densely-packed Huffman + // stream lands. Must stay in lockstep with the section order written by + // Serialize. Depends only on values known after CompressPrimary's sizing + // pass (primary_count, dict_size, chunk_size, ddata_size, outlier_count). + SIZE ComputeSerializedLayout(SIZE &packed_byte_offset) { auto nchunk = (primary_count - 1) / chunk_size + 1; - size_t *h_meta = new size_t[nchunk * 3](); - size_t *dH_uInt_meta = h_meta; - size_t *dH_bit_meta = h_meta + nchunk; - size_t *dH_uInt_entry = h_meta + nchunk * 2; - - MemoryManager().Copy1D(dH_bit_meta, - workspace.huff_bitwidths_subarray.data(), - nchunk, queue_idx); - DeviceRuntime::SyncQueue(queue_idx); - // transform in uInt - memcpy(dH_uInt_meta, dH_bit_meta, nchunk * sizeof(size_t)); - std::for_each(dH_uInt_meta, dH_uInt_meta + nchunk, - [&](size_t &i) { i = (i - 1) / (sizeof(H) * 8) + 1; }); - // make it entries - memcpy(dH_uInt_entry + 1, dH_uInt_meta, (nchunk - 1) * sizeof(size_t)); - for (auto i = 1; i < nchunk; i++) - dH_uInt_entry[i] += dH_uInt_entry[i - 1]; - - // sum bits from each chunk - auto total_bits = - std::accumulate(dH_bit_meta, dH_bit_meta + nchunk, (size_t)0); - auto total_uInts = - std::accumulate(dH_uInt_meta, dH_uInt_meta + nchunk, (size_t)0); - - // printf("huffman encode time: %.6f s\n", time_span.count()); - - // out_meta: |outlier count|outlier idx|outlier data|primary count|dict - // size|chunk size|huffmeta size|huffmeta|decodebook size|decodebook| - // out_data: |huffman data| - size_t type_bw = sizeof(H) * 8; - size_t decodebook_size = workspace.decodebook_subarray.shape(0); size_t huffmeta_size = 2 * nchunk; - - size_t ddata_size = total_uInts; - SIZE byte_offset = 0; + advance_with_align(byte_offset, 7); // signature advance_with_align(byte_offset, 1); advance_with_align(byte_offset, 1); advance_with_align(byte_offset, 1); @@ -180,16 +332,43 @@ class Huffman : public LosslessCompressorInterface { advance_with_align( byte_offset, (sizeof(H) * (2 * type_bw) + sizeof(Q) * dict_size)); advance_with_align(byte_offset, 1); + align_byte_offset(byte_offset); + packed_byte_offset = byte_offset; advance_with_align(byte_offset, ddata_size); - // outliter + // outlier advance_with_align(byte_offset, 1); advance_with_align(byte_offset, outlier_count); advance_with_align(byte_offset, outlier_count); + return byte_offset; + } + + void Serialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + auto nchunk = (primary_count - 1) / chunk_size + 1; - compressed_data.resize({(SIZE)(byte_offset)}); + // out_meta: |signature|primary count|dict size|chunk size|huffmeta size| + // huffmeta|decodebook size|decodebook|ddata size| + // out_data: |huffman data| + // huffmeta = |per-chunk bit lengths|per-chunk word offsets|, both already + // computed on the device during CompressPrimary. ddata_size (total packed + // word count) was also computed there. + + size_t type_bw = sizeof(H) * 8; + size_t decodebook_size = workspace.decodebook_subarray.shape(0); + size_t huffmeta_size = 2 * nchunk; + + // The output buffer was already sized and the packed Huffman stream already + // written into it by CompressPrimary (zero-copy). Here we only fill in the + // metadata sections around it; the packed region is skipped, not rewritten. SubArray compressed_data_subarray(compressed_data); - byte_offset = 0; + SIZE byte_offset = 0; + SerializeArray(compressed_data_subarray, signature, 7, byte_offset, + queue_idx); SerializeArray(compressed_data_subarray, &primary_count, 1, byte_offset, queue_idx); SerializeArray(compressed_data_subarray, &dict_size, 1, byte_offset, @@ -198,8 +377,15 @@ class Huffman : public LosslessCompressorInterface { queue_idx); SerializeArray(compressed_data_subarray, &huffmeta_size, 1, byte_offset, queue_idx); - SerializeArray(compressed_data_subarray, dH_bit_meta, huffmeta_size, + // huffmeta first half: per-chunk bit lengths (device). + SerializeArray(compressed_data_subarray, + workspace.huff_bitwidths_subarray.data(), nchunk, byte_offset, queue_idx); + // huffmeta second half: per-chunk word offsets (device). The extended scan + // holds nchunk+1 entries; the first nchunk are the chunk start offsets. + SerializeArray(compressed_data_subarray, + workspace.deflate_chunk_word_offsets_subarray.data(), + nchunk, byte_offset, queue_idx); SerializeArray(compressed_data_subarray, &decodebook_size, 1, byte_offset, queue_idx); SerializeArray(compressed_data_subarray, @@ -209,23 +395,9 @@ class Huffman : public LosslessCompressorInterface { SerializeArray(compressed_data_subarray, &ddata_size, 1, byte_offset, queue_idx); - align_byte_offset(byte_offset); - - MemoryManager::Copy1D( - workspace.condense_write_offsets_subarray.data(), dH_uInt_entry, nchunk, - queue_idx); - MemoryManager::Copy1D( - workspace.condense_actual_lengths_subarray.data(), dH_uInt_meta, nchunk, - queue_idx); - SubArray<1, H, DeviceType> compressed_data_cast_subarray( - {(SIZE)ddata_size}, (H *)compressed_data_subarray(byte_offset)); - DeviceLauncher::Execute( - CondenseKernel( - workspace.huff_subarray, workspace.condense_write_offsets_subarray, - workspace.condense_actual_lengths_subarray, - compressed_data_cast_subarray, chunk_size), - queue_idx); - + // The densely-packed Huffman stream is already in place (written directly + // by CompressPrimary); just advance past it. Must mirror + // ComputeSerializedLayout. advance_with_align(byte_offset, ddata_size); // outlier @@ -240,16 +412,14 @@ class Huffman : public LosslessCompressorInterface { DeviceRuntime::SyncQueue(queue_idx); - delete[] h_meta; - - log::info("Huffman block size: " + std::to_string(chunk_size)); - log::info("Huffman dictionary size: " + std::to_string(dict_size)); - log::info("Huffman compress ratio (primary): " + - std::to_string(primary_count * sizeof(Q)) + "/" + - std::to_string(ddata_size * sizeof(H)) + " (" + - std::to_string((double)primary_count * sizeof(Q) / - (ddata_size * sizeof(H))) + - ")"); + log::dbg("Huffman block size: " + std::to_string(chunk_size)); + log::dbg("Huffman dictionary size: " + std::to_string(dict_size)); + log::dbg("Huffman compress ratio (primary): " + + std::to_string(primary_count * sizeof(Q)) + "/" + + std::to_string(ddata_size * sizeof(H)) + " (" + + std::to_string((double)primary_count * sizeof(Q) / + (ddata_size * sizeof(H))) + + ")"); log::info( "Huffman compress ratio: " + std::to_string(primary_count * sizeof(Q)) + "/" + std::to_string(compressed_data.shape(0)) + " (" + @@ -259,19 +429,41 @@ class Huffman : public LosslessCompressorInterface { if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); timer.end(); - timer.print("Huffman serialize"); + timer.print("Huffman serialize", compressed_data.shape(0)); timer.clear(); } } + bool Verify(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + SubArray compressed_subarray(compressed_data); + SIZE byte_offset = 0; + // Deserialize the signature into the workspace's pinned host buffer; the + // device buffer of the Array is unused (the comparison happens on host). + Byte *signature_verify = workspace.signature_verify_array.dataHost(); + DeserializeArray(compressed_subarray, signature_verify, 7, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + for (int i = 0; i < 7; i++) { + if (signature[i] != signature_verify[i]) { + return false; + } + } + return true; + } + void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { Timer timer; if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); timer.start(); } + if (!Verify(compressed_data, queue_idx)) { + throw std::runtime_error("Huffman signature mismatch."); + } + SubArray compressed_subarray(compressed_data); + Byte *signature_ptr = nullptr; size_t *primary_count_ptr = &primary_count; int *dict_size_ptr = &dict_size; int *chunk_size_ptr = &chunk_size; @@ -281,6 +473,8 @@ class Huffman : public LosslessCompressorInterface { ATOMIC_IDX *outlier_count_ptr = &outlier_count; SIZE byte_offset = 0; + DeserializeArray(compressed_subarray, signature_ptr, 7, byte_offset, + true, queue_idx); DeserializeArray(compressed_subarray, primary_count_ptr, 1, byte_offset, false, queue_idx); DeserializeArray(compressed_subarray, dict_size_ptr, 1, byte_offset, @@ -319,7 +513,7 @@ class Huffman : public LosslessCompressorInterface { if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); timer.end(); - timer.print("Huffman deserialize"); + timer.print("Huffman deserialize", compressed_data.shape(0)); timer.clear(); } } @@ -352,17 +546,14 @@ class Huffman : public LosslessCompressorInterface { if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); timer.end(); - timer.print("Huffman decompress"); - log::time("Huffman decompression throughput: " + - std::to_string((double)primary_count * sizeof(Q) / timer.get() / - 1e9) + - " GB/s"); + timer.print("Huffman decompress", primary_count * sizeof(Q)); timer.clear(); } } - void Compress(Array<1, S, DeviceType> &original_data, - Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + bool Compress(Array<1, S, DeviceType> &original_data, + Array<1, Byte, DeviceType> &compressed_data, float target_cr, + int queue_idx) { Timer timer; if (log::level & log::TIME) { @@ -370,10 +561,13 @@ class Huffman : public LosslessCompressorInterface { timer.start(); } - DeviceLauncher::Execute( - DictionaryShiftKernel( - SubArray(original_data), dict_size), - queue_idx); + ATOMIC_IDX zero = 0; + MemoryManager::Copy1D(workspace.outlier_count_subarray.data(), + &zero, 1, queue_idx); + + // The dictionary shift (+dict_size/2) is folded into the quantizer, so the + // incoming data already lives in the non-negative dictionary range. A + // single block-aggregated pass separates the out-of-range outliers. DeviceLauncher::Execute( OutlierSeparatorKernel( SubArray(original_data), dict_size, @@ -383,38 +577,36 @@ class Huffman : public LosslessCompressorInterface { MemoryManager::Copy1D( &outlier_count, workspace.outlier_count_subarray.data(), 1, queue_idx); DeviceRuntime::SyncQueue(queue_idx); - if (outlier_count <= workspace.outlier_subarray.shape(0)) { - // outlier buffer has sufficient size - log::info( - "Outlier ratio: " + std::to_string(outlier_count) + "/" + - std::to_string(original_data.shape(0)) + " (" + - std::to_string((double)100 * outlier_count / original_data.shape(0)) + - "%)"); - } else { - log::err("Not enough workspace for outliers."); - exit(-1); + + log::info( + "Outlier ratio: " + std::to_string(outlier_count) + "/" + + std::to_string(original_data.shape(0)) + " (" + + std::to_string((double)100 * outlier_count / original_data.shape(0)) + + "%)"); + if (outlier_count > workspace.outlier_subarray.shape(0)) { + throw std::runtime_error("Not enough workspace for outliers."); } if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); timer.end(); - timer.print("Huffman outlier"); + timer.print("Huffman outlier", original_data.shape(0) * sizeof(S)); timer.clear(); } // Cast to unsigned type Array<1, Q, DeviceType> primary_data({original_data.shape(0)}, (Q *)original_data.data()); - CompressPrimary(primary_data, compressed_data, queue_idx); + return CompressPrimary(primary_data, compressed_data, target_cr, queue_idx); } void Decompress(Array<1, Byte, DeviceType> &compressed_data, Array<1, S, DeviceType> &decompressed_data, int queue_idx) { // Cast to unsigned type. - // We use temporarily use size 1 as it we be resized to the correct size. - Array<1, Q, DeviceType> primary_data({1}, (Q *)decompressed_data.data()); - + // We must use the correct size to avoid resize to new allocation + Array<1, Q, DeviceType> primary_data({decompressed_data.shape(0)}, + (Q *)decompressed_data.data()); DecompressPrimary(compressed_data, primary_data, queue_idx); Timer timer; @@ -423,22 +615,19 @@ class Huffman : public LosslessCompressorInterface { timer.start(); } + // Restore the outliers; the dictionary shift is undone later in the + // dequantizer (mirroring the quantizer), so no shift pass is needed here. DeviceLauncher::Execute( OutlierSeparatorKernel( decompressed_data, dict_size, workspace.outlier_count_subarray, workspace.outlier_idx_subarray, workspace.outlier_subarray), queue_idx); - DeviceLauncher::Execute( - DictionaryShiftKernel( - decompressed_data, dict_size), - queue_idx); - DeviceRuntime::SyncQueue(queue_idx); if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); timer.end(); - timer.print("Huffman outlier"); + timer.print("Huffman outlier", decompressed_data.shape(0) * sizeof(S)); timer.clear(); } } @@ -457,7 +646,7 @@ class Huffman : public LosslessCompressorInterface { ATOMIC_IDX *outlier_idx; S *outlier; H *ddata; - + Byte signature[7] = {'M', 'G', 'X', 'H', 'U', 'F', 'F'}; HuffmanWorkspace workspace; }; diff --git a/include/mgard-x/Lossless/ParallelHuffman/HuffmanWorkspace.hpp b/include/mgard-x/Lossless/ParallelHuffman/HuffmanWorkspace.hpp index dcefc0fd53..4b0a997b0b 100644 --- a/include/mgard-x/Lossless/ParallelHuffman/HuffmanWorkspace.hpp +++ b/include/mgard-x/Lossless/ParallelHuffman/HuffmanWorkspace.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_HUFFMAN_WORKSPACE_HPP @@ -10,6 +10,8 @@ #include "../../Hierarchy/Hierarchy.h" #include "../../RuntimeX/RuntimeXPublic.h" +#include "GenerateCL.hpp" +#include "ParallelDeflate.hpp" namespace mgard_x { @@ -29,11 +31,16 @@ class HuffmanWorkspace { freq_subarray = SubArray(freq_array); codebook_subarray = SubArray(codebook_array); decodebook_subarray = SubArray(decodebook_array); - huff_subarray = SubArray(huff_array); huff_bitwidths_subarray = SubArray(huff_bitwidths_array); condense_write_offsets_subarray = SubArray(condense_write_offsets_array); condense_actual_lengths_subarray = SubArray(condense_actual_lengths_array); + deflate_group_bits_subarray = SubArray(deflate_group_bits_array); + deflate_group_offsets_subarray = SubArray(deflate_group_offsets_array); + deflate_chunk_words_subarray = SubArray(deflate_chunk_words_array); + deflate_chunk_word_offsets_subarray = + SubArray(deflate_chunk_word_offsets_array); + // Codebook first_nonzero_index_subarray = SubArray(first_nonzero_index_array); sort_by_key_workspace_subarray = SubArray(sort_by_key_workspace); @@ -68,12 +75,31 @@ class HuffmanWorkspace { size_t type_bw = sizeof(H) * 8; size_t decodebook_size = sizeof(H) * (2 * type_bw) + sizeof(Q) * dict_size; size += decodebook_size * sizeof(uint8_t); - size += primary_count * sizeof(H); size_t nchunk = (primary_count - 1) / chunk_size + 1; size += nchunk * sizeof(size_t); size += nchunk * sizeof(size_t); size += nchunk * sizeof(size_t); + // Parallel deflate + SIZE groups_per_chunk = + (chunk_size - 1) / GetDeflateGroupSize(chunk_size) + 1; + size_t ngroups = nchunk * groups_per_chunk; + size += ngroups * sizeof(size_t); // group_bits + size += (ngroups + 1) * sizeof(size_t); // group_offsets + size += nchunk * sizeof(size_t); // chunk_words + size += (nchunk + 1) * sizeof(size_t); // chunk_word_offsets + { + Array<1, Byte, DeviceType> tmp_group_scan, tmp_chunk_scan; + DeviceCollective::ScanSumExtended( + (SIZE)ngroups, SubArray<1, size_t, DeviceType>(), + SubArray<1, size_t, DeviceType>(), tmp_group_scan, false, 0); + DeviceCollective::ScanSumExtended( + (SIZE)nchunk, SubArray<1, size_t, DeviceType>(), + SubArray<1, size_t, DeviceType>(), tmp_chunk_scan, false, 0); + size += tmp_group_scan.shape(0); + size += tmp_chunk_scan.shape(0); + } + size += sizeof(unsigned int); Array<1, Byte, DeviceType> tmp; DeviceCollective::SortByKey( @@ -87,33 +113,52 @@ class HuffmanWorkspace { size += sizeof(int) * dict_size * 6; size += sizeof(H) * dict_size; size += sizeof(int) * 16; - SIZE mblocks = (DeviceRuntime::GetMaxNumThreadsPerTB() / - DeviceRuntime::GetWarpSize()) * - DeviceRuntime::GetNumSMs(); - size += 2 * (mblocks + 1) * sizeof(uint32_t); + size += + GenerateCLKernel::DiagonalPathIntersectionsSize(dict_size) * + sizeof(uint32_t); + size += 7 * sizeof(Byte); // signature_verify_array return size; } void allocate(SIZE primary_count, SIZE dict_size, SIZE chunk_size, double estimated_outlier_ratio) { - outlier_count_array = Array<1, ATOMIC_IDX, DeviceType>({1}, false, false); + outlier_count_array = Array<1, ATOMIC_IDX, DeviceType>({1}); outlier_idx_array = Array<1, ATOMIC_IDX, DeviceType>( {(SIZE)(primary_count * estimated_outlier_ratio)}); outlier_array = Array<1, S, DeviceType>( {(SIZE)(primary_count * estimated_outlier_ratio)}); freq_array = Array<1, unsigned int, DeviceType>({dict_size}); + freq_array.hostAllocate(false); codebook_array = Array<1, H, DeviceType>({dict_size}); size_t type_bw = sizeof(H) * 8; size_t decodebook_size = sizeof(H) * (2 * type_bw) + sizeof(Q) * dict_size; decodebook_array = Array<1, uint8_t, DeviceType>({(SIZE)decodebook_size}); - huff_array = Array<1, H, DeviceType>({primary_count}); size_t nchunk = (primary_count - 1) / chunk_size + 1; huff_bitwidths_array = Array<1, size_t, DeviceType>({(SIZE)nchunk}); condense_write_offsets_array = Array<1, size_t, DeviceType>({(SIZE)nchunk}); condense_actual_lengths_array = Array<1, size_t, DeviceType>({(SIZE)nchunk}); + + // Parallel deflate + SIZE groups_per_chunk = + (chunk_size - 1) / GetDeflateGroupSize(chunk_size) + 1; + SIZE ngroups = (SIZE)(nchunk * groups_per_chunk); + deflate_group_bits_array = Array<1, size_t, DeviceType>({ngroups}); + deflate_group_offsets_array = Array<1, size_t, DeviceType>({ngroups + 1}); + deflate_chunk_words_array = Array<1, size_t, DeviceType>({(SIZE)nchunk}); + deflate_chunk_word_offsets_array = + Array<1, size_t, DeviceType>({(SIZE)nchunk + 1}); + DeviceCollective::ScanSumExtended( + ngroups, SubArray<1, size_t, DeviceType>(), + SubArray<1, size_t, DeviceType>(), deflate_group_scan_workspace, false, + 0); + DeviceCollective::ScanSumExtended( + (SIZE)nchunk, SubArray<1, size_t, DeviceType>(), + SubArray<1, size_t, DeviceType>(), deflate_chunk_scan_workspace, false, + 0); // Codebook first_nonzero_index_array = Array<1, unsigned int, DeviceType>({1}); // first_nonzero_index_array.hostCopy(); // Create host allocation @@ -125,6 +170,7 @@ class HuffmanWorkspace { _d_freq_copy_array = Array<1, unsigned int, DeviceType>({(SIZE)dict_size}); _d_qcode_copy_array = Array<1, Q, DeviceType>({(SIZE)dict_size}); CL_array = Array<1, unsigned int, DeviceType>({dict_size}); + CL_array.hostAllocate(false); lNodesLeader_array = Array<1, int, DeviceType>({dict_size}); iNodesFreq_array = Array<1, unsigned int, DeviceType>({dict_size}); iNodesLeader_array = Array<1, int, DeviceType>({dict_size}); @@ -135,14 +181,14 @@ class HuffmanWorkspace { copyIsLeaf_array = Array<1, int, DeviceType>({dict_size}); copyIndex_array = Array<1, int, DeviceType>({dict_size}); _d_codebook_array_org = Array<1, H, DeviceType>({dict_size}); - status_array = Array<1, int, DeviceType>( - {(SIZE)16}, false, - false); // non-managed: atomicMin on managed mem unreliable on ROCm - SIZE mblocks = (DeviceRuntime::GetMaxNumThreadsPerTB() / - DeviceRuntime::GetWarpSize()) * - DeviceRuntime::GetNumSMs(); - diagonal_path_intersections_array = - Array<1, uint32_t, DeviceType>({2 * (mblocks + 1)}); + status_array = Array<1, int, DeviceType, false, false>( + {(SIZE)16}); // non-managed: atomicMin on managed mem unreliable on ROCm + diagonal_path_intersections_array = Array<1, uint32_t, DeviceType>( + {GenerateCLKernel:: + DiagonalPathIntersectionsSize(dict_size)}); + + signature_verify_array = Array<1, Byte, DeviceType>({(SIZE)7}); + signature_verify_array.hostAllocate(false); // outlier_count_array.memset(0); // outlier_idx_array.memset(0); @@ -163,15 +209,32 @@ class HuffmanWorkspace { queue_idx); freq_array.resize({dict_size}, queue_idx); + freq_array.hostAllocate(false); codebook_array.resize({dict_size}, queue_idx); size_t type_bw = sizeof(H) * 8; size_t decodebook_size = sizeof(H) * (2 * type_bw) + sizeof(Q) * dict_size; decodebook_array.resize({(SIZE)decodebook_size}, queue_idx); - huff_array.resize({primary_count}, queue_idx); size_t nchunk = (primary_count - 1) / chunk_size + 1; huff_bitwidths_array.resize({(SIZE)nchunk}, queue_idx); condense_write_offsets_array.resize({(SIZE)nchunk}, queue_idx); condense_actual_lengths_array.resize({(SIZE)nchunk}, queue_idx); + + // Parallel deflate + SIZE groups_per_chunk = + (chunk_size - 1) / GetDeflateGroupSize(chunk_size) + 1; + SIZE ngroups = (SIZE)(nchunk * groups_per_chunk); + deflate_group_bits_array.resize({ngroups}, queue_idx); + deflate_group_offsets_array.resize({ngroups + 1}, queue_idx); + deflate_chunk_words_array.resize({(SIZE)nchunk}, queue_idx); + deflate_chunk_word_offsets_array.resize({(SIZE)nchunk + 1}, queue_idx); + DeviceCollective::ScanSumExtended( + ngroups, SubArray<1, size_t, DeviceType>(), + SubArray<1, size_t, DeviceType>(), deflate_group_scan_workspace, false, + queue_idx); + DeviceCollective::ScanSumExtended( + (SIZE)nchunk, SubArray<1, size_t, DeviceType>(), + SubArray<1, size_t, DeviceType>(), deflate_chunk_scan_workspace, false, + queue_idx); // Codebook first_nonzero_index_array.resize({1}, queue_idx); // first_nonzero_index_array.hostCopy(); // Create host allocation @@ -184,6 +247,7 @@ class HuffmanWorkspace { _d_freq_copy_array.resize({(SIZE)dict_size}, queue_idx); _d_qcode_copy_array.resize({(SIZE)dict_size}, queue_idx); CL_array.resize({dict_size}, queue_idx); + CL_array.hostAllocate(false); lNodesLeader_array.resize({dict_size}, queue_idx); iNodesFreq_array.resize({dict_size}, queue_idx); iNodesLeader_array.resize({dict_size}, queue_idx); @@ -195,10 +259,14 @@ class HuffmanWorkspace { copyIndex_array.resize({dict_size}, queue_idx); _d_codebook_array_org.resize({dict_size}, queue_idx); status_array.resize({(SIZE)16}, queue_idx); - SIZE mblocks = (DeviceRuntime::GetMaxNumThreadsPerTB() / - DeviceRuntime::GetWarpSize()) * - DeviceRuntime::GetNumSMs(); - diagonal_path_intersections_array.resize({2 * (mblocks + 1)}, queue_idx); + diagonal_path_intersections_array.resize( + {GenerateCLKernel:: + DiagonalPathIntersectionsSize(dict_size)}, + queue_idx); + + signature_verify_array.resize({(SIZE)7}, queue_idx); + signature_verify_array.hostAllocate(false, queue_idx); + // outlier_count_array.memset(0, queue_idx); // outlier_idx_array.memset(0, queue_idx); // outlier_array.memset(0, queue_idx); @@ -231,11 +299,21 @@ class HuffmanWorkspace { Array<1, unsigned int, DeviceType> freq_array; Array<1, H, DeviceType> codebook_array; Array<1, uint8_t, DeviceType> decodebook_array; - Array<1, H, DeviceType> huff_array; Array<1, size_t, DeviceType> huff_bitwidths_array; Array<1, size_t, DeviceType> condense_write_offsets_array; Array<1, size_t, DeviceType> condense_actual_lengths_array; + // Parallel deflate (cooperative bit-packing). group_bits holds per-group bit + // sums; group_offsets is their extended exclusive scan. chunk_words holds + // per-chunk H-word counts; chunk_word_offsets is their extended exclusive + // scan (the last element is the total ddata word count). + Array<1, size_t, DeviceType> deflate_group_bits_array; + Array<1, size_t, DeviceType> deflate_group_offsets_array; + Array<1, size_t, DeviceType> deflate_chunk_words_array; + Array<1, size_t, DeviceType> deflate_chunk_word_offsets_array; + Array<1, Byte, DeviceType> deflate_group_scan_workspace; + Array<1, Byte, DeviceType> deflate_chunk_scan_workspace; + // Codebook Array<1, unsigned int, DeviceType> first_nonzero_index_array; Array<1, Byte, DeviceType> sort_by_key_workspace; @@ -252,9 +330,14 @@ class HuffmanWorkspace { Array<1, int, DeviceType> copyIsLeaf_array; Array<1, int, DeviceType> copyIndex_array; Array<1, H, DeviceType> _d_codebook_array_org; - Array<1, int, DeviceType> status_array; + Array<1, int, DeviceType, false, true> status_array; Array<1, uint32_t, DeviceType> diagonal_path_intersections_array; + // Holds the deserialized signature during Verify. Only its pinned host copy + // is used (read back via dataHost()); the device buffer is unused but comes + // along with the Array and is freed automatically with the workspace. + Array<1, Byte, DeviceType> signature_verify_array; + SubArray<1, ATOMIC_IDX, DeviceType> outlier_count_subarray; SubArray<1, ATOMIC_IDX, DeviceType> outlier_idx_subarray; SubArray<1, S, DeviceType> outlier_subarray; @@ -262,11 +345,15 @@ class HuffmanWorkspace { SubArray<1, unsigned int, DeviceType> freq_subarray; SubArray<1, H, DeviceType> codebook_subarray; SubArray<1, uint8_t, DeviceType> decodebook_subarray; - SubArray<1, H, DeviceType> huff_subarray; SubArray<1, size_t, DeviceType> huff_bitwidths_subarray; SubArray<1, size_t, DeviceType> condense_write_offsets_subarray; SubArray<1, size_t, DeviceType> condense_actual_lengths_subarray; + SubArray<1, size_t, DeviceType> deflate_group_bits_subarray; + SubArray<1, size_t, DeviceType> deflate_group_offsets_subarray; + SubArray<1, size_t, DeviceType> deflate_chunk_words_subarray; + SubArray<1, size_t, DeviceType> deflate_chunk_word_offsets_subarray; + // Codebook SubArray<1, unsigned int, DeviceType> first_nonzero_index_subarray; SubArray<1, Byte, DeviceType> sort_by_key_workspace_subarray; @@ -283,7 +370,7 @@ class HuffmanWorkspace { SubArray<1, int, DeviceType> copyIsLeaf_subarray; SubArray<1, int, DeviceType> copyIndex_subarray; SubArray<1, H, DeviceType> _d_codebook_subarray_org; - SubArray<1, int, DeviceType> status_subarray; + SubArray<1, int, DeviceType, false, true> status_subarray; SubArray<1, uint32_t, DeviceType> diagonal_path_intersections_subarray; }; diff --git a/include/mgard-x/Lossless/ParallelHuffman/OutlierSeparator.hpp b/include/mgard-x/Lossless/ParallelHuffman/OutlierSeparator.hpp index e49ffe2f37..3ca4dbf019 100644 --- a/include/mgard-x/Lossless/ParallelHuffman/OutlierSeparator.hpp +++ b/include/mgard-x/Lossless/ParallelHuffman/OutlierSeparator.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_OUTLIER_SEPARATOR_TEMPLATE_HPP @@ -28,36 +28,83 @@ class OutlierSeparatorFunctor : public Functor { Functor(); } + // SEPARATE uses block-aggregated atomics: every outlier first reserves a slot + // in a block-local counter (shared memory, block scope), then a single thread + // reserves the whole block's contiguous output range with one device-scope + // atomic. This collapses up to blockDim global atomics on the single counter + // address into one per block, removing the contention that made this the + // bottleneck. The functor framework inserts a block barrier between + // consecutive Operation*() phases. MGARDX_EXEC void Operation1() { - SIZE id = FunctorBase::GetBlockIdX() * - FunctorBase::GetBlockDimX() + - FunctorBase::GetThreadIdX(); - if (id < v.shape(0)) { - T value = *v(id); - if constexpr (OP == MGARDX_SEPARATE_OUTLIER) { - // printf("%d %lld %d\n", value, dict_size, value < 0 || value >= - // dict_size); + id = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + if constexpr (OP == MGARDX_SEPARATE_OUTLIER) { + // sm[0]: block-local outlier count, sm[1]: block base offset + sm = (ATOMIC_IDX *)FunctorBase::GetSharedMemory(); + if (FunctorBase::GetThreadIdX() == 0) { + sm[0] = 0; + } + is_outlier = false; + if (id < v.shape(0)) { + value = *v(id); if (value < 0 || value >= dict_size) { - ATOMIC_IDX outlier_write_index = - Atomic::Add(outlier_count((IDX)0), (ATOMIC_IDX)1); - if (outlier_write_index < outlier_index.shape(0)) { - *outlier_index(outlier_write_index) = id; - *outlier_value(outlier_write_index) = value; - *v(id) = 0; - } + is_outlier = true; } - } else if constexpr (OP == MGARDX_RESTORE_OUTLIER) { - if (id < outlier_value.shape(0)) { - ATOMIC_IDX index = *outlier_index(id); - QUANTIZED_INT value = *outlier_value(id); - *v(index) = value; + } + } else if constexpr (OP == MGARDX_RESTORE_OUTLIER) { + // Grid is sized to the outlier count, so this is a sparse scatter. + if (id < outlier_value.shape(0)) { + ATOMIC_IDX index = *outlier_index(id); + T val = *outlier_value(id); + *v(index) = val; + } + } + } + + // Reserve a block-local slot for each outlier. + MGARDX_EXEC void Operation2() { + if constexpr (OP == MGARDX_SEPARATE_OUTLIER) { + if (is_outlier) { + local_offset = Atomic::Add(&sm[0], (ATOMIC_IDX)1); + } + } + } + + // One global atomic per block reserves the block's output range. + MGARDX_EXEC void Operation3() { + if constexpr (OP == MGARDX_SEPARATE_OUTLIER) { + if (FunctorBase::GetThreadIdX() == 0) { + sm[1] = Atomic::Add(outlier_count((IDX)0), sm[0]); + } + } + } + + // Scatter outliers to their global slots and zero them in the primary stream. + MGARDX_EXEC void Operation4() { + if constexpr (OP == MGARDX_SEPARATE_OUTLIER) { + if (is_outlier) { + ATOMIC_IDX outlier_write_index = sm[1] + local_offset; + if (outlier_write_index < outlier_index.shape(0)) { + *outlier_index(outlier_write_index) = id; + *outlier_value(outlier_write_index) = value; + *v(id) = 0; } } } } - MGARDX_CONT size_t shared_memory_size() { return 0; } + MGARDX_EXEC void Operation5() {} + + MGARDX_CONT size_t shared_memory_size() { + if constexpr (OP == MGARDX_SEPARATE_OUTLIER) { + return 2 * sizeof(ATOMIC_IDX); + } else { + return 0; + } + } private: SubArray<1, T, DeviceType> v; @@ -65,6 +112,13 @@ class OutlierSeparatorFunctor : public Functor { SubArray<1, ATOMIC_IDX, DeviceType> outlier_count; SubArray<1, ATOMIC_IDX, DeviceType> outlier_index; SubArray<1, T, DeviceType> outlier_value; + + // Per-thread state carried across the Operation*() phases (SEPARATE only). + SIZE id; + bool is_outlier; + T value; + ATOMIC_IDX local_offset; + ATOMIC_IDX *sm; }; template @@ -89,10 +143,32 @@ class OutlierSeparatorKernel : public Kernel { size_t sm_size = functor.shared_memory_size(); tbz = 1; tby = 1; - tbx = 256; + if constexpr (std::is_same::value) { + // Empirically tuned on MI300: a direct sweep (64/128/256/512/768/896/ + // 960/1024) found throughput climbing steeply with block size -- + // 256 (the original) measures ~224 GB/s, 960 measures ~815 GB/s, a + // >3.5x improvement with no logic change. 1024 (the hardware max) + // reliably hung the kernel launch on this system, so 960 is used as + // the largest safely-tested value rather than chasing the last ~7% + // theoretical gain from 1024 at the risk of a hang. This tuning is + // HIP/CDNA3-specific and untested on other backends, so it is guarded + // here rather than applied unconditionally. + tbx = 960; + } else { + tbx = 256; + } gridz = 1; gridy = 1; - gridx = (v.shape(0) - 1) / tbx + 1; + // RESTORE is a sparse scatter over the outliers only (outlier_value is + // sized to the outlier count after deserialization), so launch just enough + // threads instead of one per element of the full array. + SIZE launch_count; + if constexpr (OP == MGARDX_RESTORE_OUTLIER) { + launch_count = outlier_value.shape(0); + } else { + launch_count = v.shape(0); + } + gridx = (launch_count == 0) ? 1 : (launch_count - 1) / tbx + 1; return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, std::string(Name)); } diff --git a/include/mgard-x/Lossless/ParallelHuffman/ParallelDeflate.hpp b/include/mgard-x/Lossless/ParallelHuffman/ParallelDeflate.hpp new file mode 100644 index 0000000000..dbf8699877 --- /dev/null +++ b/include/mgard-x/Lossless/ParallelHuffman/ParallelDeflate.hpp @@ -0,0 +1,412 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_PARALLEL_DEFLATE_TEMPLATE_HPP +#define MGARD_X_PARALLEL_DEFLATE_TEMPLATE_HPP +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x { + +// Number of consecutive symbols packed by a single thread ("group"). Each +// Huffman chunk is split into ceil(chunk_size / group_size) groups so that +// the bit-packing of one chunk is shared by many threads instead of a single +// one. Must stay small enough to keep good parallelism but large enough that +// most output words a group produces are fully owned (written with a plain +// store) rather than shared at the boundaries (written with an atomicOr). +// +// Empirically the pack kernel is fastest when a full warp covers about one +// Huffman chunk, i.e. group_size ~= huff_block_size / 32: for the default +// huff_block_size = 1024 that measured ~3.6x faster than a fixed 256 on NYX +// 512^3 on Hopper. That "one warp per chunk" reasoning does not transfer +// directly to a 64-wide CDNA wavefront, though: group_size = chunk_size/64 +// = 16 measured *worse* on MI300 (~73 GB/s) than the original 32 (~74 +// GB/s), and a direct sweep (16/32/64/128/256/512) found the real peak at +// group_size = 128 (~86 GB/s) -- larger, not smaller, than Hopper's tuning. +// Fewer, bigger groups apparently win here because each group boundary +// costs an atomicOr-merged word instead of a plain store (see DeflatePack +// below); halving group_size doubles the boundary count for the same data, +// and on MI300 that atomic-merge overhead outweighs the extra parallelism +// from narrower groups. Revisit with a fresh sweep if huff_block_size +// changes (currently group_size = chunk_size/8, matching the measured +// optimum at the default chunk_size = 1024). +template +MGARDX_CONT SIZE GetDeflateGroupSize(SIZE chunk_size) { + SIZE group_size; + if constexpr (std::is_same::value) { + group_size = chunk_size / 8; + } else { + SIZE warp_size = DeviceRuntime::GetWarpSize(); + group_size = chunk_size / warp_size; + } + return group_size > 0 ? group_size : 1; +} + +// Helper: extract the per-symbol bitwidth, stored in the most-significant byte +// of each fixed-length Huffman codeword (codebook[symbol]). +template MGARDX_EXEC uint8_t deflate_bitwidth(H word) { + return *((uint8_t *)&word + (sizeof(H) - 1)); +} + +// Phase 1 (sizing): each thread sums the bitwidths of the symbols in one group. +// The fixed-length codeword for symbol s is codebook[s], so we look it up on +// the fly instead of reading a materialized per-symbol array (encode is fused +// here). Groups never cross chunk boundaries, so a later per-chunk reduction +// over the scanned group sums yields per-chunk bit lengths. +template +class DeflateGroupBitsFunctor : public Functor { +public: + MGARDX_CONT DeflateGroupBitsFunctor() {} + MGARDX_CONT DeflateGroupBitsFunctor( + SubArray<1, Q, DeviceType> data, SubArray<1, H, DeviceType> codebook, + SubArray<1, size_t, DeviceType> group_bits, size_t primary_count, + SIZE chunk_size, SIZE groups_per_chunk, SIZE ngroups, + SIZE deflate_group_size) + : data(data), codebook(codebook), group_bits(group_bits), + primary_count(primary_count), chunk_size(chunk_size), + groups_per_chunk(groups_per_chunk), ngroups(ngroups), + deflate_group_size(deflate_group_size) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + if (gid >= ngroups) + return; + SIZE chunk_id = gid / groups_per_chunk; + SIZE local = gid % groups_per_chunk; + size_t sym_base = + (size_t)chunk_id * chunk_size + (size_t)local * deflate_group_size; + size_t chunk_end = (size_t)(chunk_id + 1) * chunk_size; + if (chunk_end > primary_count) + chunk_end = primary_count; + size_t sym_end = sym_base + deflate_group_size; + if (sym_end > chunk_end) + sym_end = chunk_end; + size_t bits = 0; + for (size_t i = sym_base; i < sym_end; i++) { + bits += deflate_bitwidth(*codebook(*data(i))); + } + *group_bits(gid) = bits; + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Q, DeviceType> data; + SubArray<1, H, DeviceType> codebook; + SubArray<1, size_t, DeviceType> group_bits; + size_t primary_count; + SIZE chunk_size; + SIZE groups_per_chunk; + SIZE ngroups; + SIZE deflate_group_size; +}; + +template +class DeflateGroupBitsKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "deflate_group_bits"; + MGARDX_CONT + DeflateGroupBitsKernel(SubArray<1, Q, DeviceType> data, + SubArray<1, H, DeviceType> codebook, + SubArray<1, size_t, DeviceType> group_bits, + size_t primary_count, SIZE chunk_size, + SIZE groups_per_chunk, SIZE ngroups) + : data(data), codebook(codebook), group_bits(group_bits), + primary_count(primary_count), chunk_size(chunk_size), + groups_per_chunk(groups_per_chunk), ngroups(ngroups) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = DeflateGroupBitsFunctor; + SIZE deflate_group_size = GetDeflateGroupSize(chunk_size); + FunctorType functor(data, codebook, group_bits, primary_count, chunk_size, + groups_per_chunk, ngroups, deflate_group_size); + SIZE tbx; + if constexpr (std::is_same::value) { + // Empirically tuned on MI300: this kernel has no shared memory or + // cross-thread communication (each thread independently sums bitwidths + // for its own group), so block size is a pure occupancy knob. A direct + // sweep (256/512/768/960) found a non-monotonic curve -- 256 and 512 + // both measure ~113 GB/s (combined with DeflatePack below, via the + // CLI's "Huffman compress" timer), 768 measures ~120-124 GB/s, and 960 + // drops back to ~111 GB/s. 768 is the peak, likely an occupancy sweet + // spot for this kernel's register/LDS footprint rather than a value + // that keeps climbing like OutlierSeparator's. This tuning is + // HIP/CDNA3-specific and untested on other backends, so it is guarded + // here rather than applied unconditionally. Revisit with a fresh + // sweep if this kernel's per-thread work changes. + tbx = 768; + } else { + tbx = 256; + } + size_t sm_size = functor.shared_memory_size(); + SIZE gridx = (ngroups - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, 1, 1, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, Q, DeviceType> data; + SubArray<1, H, DeviceType> codebook; + SubArray<1, size_t, DeviceType> group_bits; + size_t primary_count; + SIZE chunk_size; + SIZE groups_per_chunk; + SIZE ngroups; +}; + +// Phase 2 (per-chunk meta): from the extended exclusive scan of group bit sums, +// compute each chunk's total bit length and its output length in H-words. +template +class DeflateChunkMetaFunctor : public Functor { +public: + MGARDX_CONT DeflateChunkMetaFunctor() {} + MGARDX_CONT + DeflateChunkMetaFunctor(SubArray<1, size_t, DeviceType> group_offsets, + SubArray<1, size_t, DeviceType> chunk_bits, + SubArray<1, size_t, DeviceType> chunk_words, + SIZE nchunk, SIZE groups_per_chunk) + : group_offsets(group_offsets), chunk_bits(chunk_bits), + chunk_words(chunk_words), nchunk(nchunk), + groups_per_chunk(groups_per_chunk) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE chunk_id = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + if (chunk_id >= nchunk) + return; + size_t total_bits = *group_offsets((chunk_id + 1) * groups_per_chunk) - + *group_offsets(chunk_id * groups_per_chunk); + *chunk_bits(chunk_id) = total_bits; + *chunk_words(chunk_id) = (total_bits + sizeof(H) * 8 - 1) / (sizeof(H) * 8); + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, size_t, DeviceType> group_offsets; + SubArray<1, size_t, DeviceType> chunk_bits; + SubArray<1, size_t, DeviceType> chunk_words; + SIZE nchunk; + SIZE groups_per_chunk; +}; + +template +class DeflateChunkMetaKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "deflate_chunk_meta"; + MGARDX_CONT + DeflateChunkMetaKernel(SubArray<1, size_t, DeviceType> group_offsets, + SubArray<1, size_t, DeviceType> chunk_bits, + SubArray<1, size_t, DeviceType> chunk_words, + SIZE nchunk, SIZE groups_per_chunk) + : group_offsets(group_offsets), chunk_bits(chunk_bits), + chunk_words(chunk_words), nchunk(nchunk), + groups_per_chunk(groups_per_chunk) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = DeflateChunkMetaFunctor; + FunctorType functor(group_offsets, chunk_bits, chunk_words, nchunk, + groups_per_chunk); + SIZE tbx = 256; + size_t sm_size = functor.shared_memory_size(); + SIZE gridx = (nchunk - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, 1, 1, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, size_t, DeviceType> group_offsets; + SubArray<1, size_t, DeviceType> chunk_bits; + SubArray<1, size_t, DeviceType> chunk_words; + SIZE nchunk; + SIZE groups_per_chunk; +}; + +// Phase 3 (pack): each thread densely packs one group's symbols, MSB-first, +// directly into the final compressed buffer at the chunk's word offset plus the +// group's intra-chunk bit offset. Output words fully covered by the group are +// written with a plain store; the (at most two) words shared with neighbouring +// groups are merged with atomicOr. The destination region must be zeroed first +// so the atomicOr merges only contribute new bits. +template +class DeflatePackFunctor : public Functor { +public: + MGARDX_CONT DeflatePackFunctor() {} + MGARDX_CONT + DeflatePackFunctor(SubArray<1, Q, DeviceType> data, + SubArray<1, H, DeviceType> codebook, + SubArray<1, size_t, DeviceType> group_offsets, + SubArray<1, size_t, DeviceType> chunk_word_offsets, + SubArray<1, H, DeviceType> condensed, size_t primary_count, + SIZE chunk_size, SIZE groups_per_chunk, SIZE ngroups, + SIZE deflate_group_size) + : data(data), codebook(codebook), group_offsets(group_offsets), + chunk_word_offsets(chunk_word_offsets), condensed(condensed), + primary_count(primary_count), chunk_size(chunk_size), + groups_per_chunk(groups_per_chunk), ngroups(ngroups), + deflate_group_size(deflate_group_size) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + if (gid >= ngroups) + return; + SIZE chunk_id = gid / groups_per_chunk; + SIZE local = gid % groups_per_chunk; + size_t sym_base = + (size_t)chunk_id * chunk_size + (size_t)local * deflate_group_size; + if (sym_base >= primary_count) + return; + size_t chunk_end = (size_t)(chunk_id + 1) * chunk_size; + if (chunk_end > primary_count) + chunk_end = primary_count; + size_t sym_end = sym_base + deflate_group_size; + if (sym_end > chunk_end) + sym_end = chunk_end; + + const size_t bpw = sizeof(H) * 8; + size_t group_start_bit = + *group_offsets(gid) - *group_offsets(chunk_id * groups_per_chunk); + size_t chunk_base_word = *chunk_word_offsets(chunk_id); + + size_t lsb_pos = bpw - (group_start_bit % bpw); + size_t cur = chunk_base_word + group_start_bit / bpw; + H buffer = 0; + bool first_word = true; + bool left_shared = (group_start_bit % bpw) != 0; + + for (size_t i = sym_base; i < sym_end; i++) { + H word = *codebook(*data(i)); // encode fused in: codeword for symbol + uint8_t bitwidth = deflate_bitwidth(word); + *((uint8_t *)&word + (sizeof(H) - 1)) = 0x0; // clear bitwidth byte + if (lsb_pos == bpw) + buffer = 0x0; // start of a fresh output word + if (bitwidth <= lsb_pos) { + lsb_pos -= bitwidth; + buffer |= word << lsb_pos; + if (lsb_pos == 0) { + // completed an output word + if (first_word && left_shared) { + Atomic::Or( + condensed(cur), buffer); + } else { + *condensed(cur) = buffer; + } + first_word = false; + cur++; + lsb_pos = bpw; + buffer = 0x0; + } + } else { + // code straddles two output words + H _1 = word >> (bitwidth - lsb_pos); + H _2 = word << (bpw - (bitwidth - lsb_pos)); + buffer |= _1; + if (first_word && left_shared) { + Atomic::Or( + condensed(cur), buffer); + } else { + *condensed(cur) = buffer; + } + first_word = false; + cur++; + buffer = _2; + lsb_pos = bpw - (bitwidth - lsb_pos); + } + } + // Trailing partial word is shared with the next group (or, for the last + // group in a chunk, lands in a word owned solely by this chunk). Either way + // it must be merged, not stored, since neighbours contribute the rest. + if (lsb_pos != bpw) { + Atomic::Or( + condensed(cur), buffer); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Q, DeviceType> data; + SubArray<1, H, DeviceType> codebook; + SubArray<1, size_t, DeviceType> group_offsets; + SubArray<1, size_t, DeviceType> chunk_word_offsets; + SubArray<1, H, DeviceType> condensed; + size_t primary_count; + SIZE chunk_size; + SIZE groups_per_chunk; + SIZE ngroups; + SIZE deflate_group_size; +}; + +template +class DeflatePackKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "deflate_pack"; + MGARDX_CONT + DeflatePackKernel(SubArray<1, Q, DeviceType> data, + SubArray<1, H, DeviceType> codebook, + SubArray<1, size_t, DeviceType> group_offsets, + SubArray<1, size_t, DeviceType> chunk_word_offsets, + SubArray<1, H, DeviceType> condensed, size_t primary_count, + SIZE chunk_size, SIZE groups_per_chunk, SIZE ngroups) + : data(data), codebook(codebook), group_offsets(group_offsets), + chunk_word_offsets(chunk_word_offsets), condensed(condensed), + primary_count(primary_count), chunk_size(chunk_size), + groups_per_chunk(groups_per_chunk), ngroups(ngroups) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = DeflatePackFunctor; + SIZE deflate_group_size = GetDeflateGroupSize(chunk_size); + FunctorType functor(data, codebook, group_offsets, chunk_word_offsets, + condensed, primary_count, chunk_size, groups_per_chunk, + ngroups, deflate_group_size); + // Same empirical sweep, reasoning, and HIP-only guard as + // DeflateGroupBitsKernel above -- 768 is the measured occupancy sweet + // spot on MI300 (256/512 ~113 GB/s, 768 ~120-124 GB/s, 960 back down to + // ~111 GB/s), untested on other backends. + SIZE tbx; + if constexpr (std::is_same::value) { + tbx = 768; + } else { + tbx = 256; + } + size_t sm_size = functor.shared_memory_size(); + SIZE gridx = (ngroups - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, 1, 1, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, Q, DeviceType> data; + SubArray<1, H, DeviceType> codebook; + SubArray<1, size_t, DeviceType> group_offsets; + SubArray<1, size_t, DeviceType> chunk_word_offsets; + SubArray<1, H, DeviceType> condensed; + size_t primary_count; + SIZE chunk_size; + SIZE groups_per_chunk; + SIZE ngroups; +}; + +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/Lossless/ParallelHuffman/ReorderByIndex.hpp b/include/mgard-x/Lossless/ParallelHuffman/ReorderByIndex.hpp index 171086f10f..ccfc69f07c 100644 --- a/include/mgard-x/Lossless/ParallelHuffman/ReorderByIndex.hpp +++ b/include/mgard-x/Lossless/ParallelHuffman/ReorderByIndex.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_REORDER_BY_INDEX_TEMPLATE_HPP diff --git a/include/mgard-x/Lossless/ParallelHuffman/ReverseArray.hpp b/include/mgard-x/Lossless/ParallelHuffman/ReverseArray.hpp index 961e4d21ee..5f89c06e51 100644 --- a/include/mgard-x/Lossless/ParallelHuffman/ReverseArray.hpp +++ b/include/mgard-x/Lossless/ParallelHuffman/ReverseArray.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_REVERSE_ARRAY_TEMPLATE_HPP diff --git a/include/mgard-x/Lossless/ParallelRLE/Convert.hpp b/include/mgard-x/Lossless/ParallelRLE/Convert.hpp new file mode 100644 index 0000000000..ed29d374b0 --- /dev/null +++ b/include/mgard-x/Lossless/ParallelRLE/Convert.hpp @@ -0,0 +1,86 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_RLE_CONVERT_TEMPLATE_HPP +#define MGARD_X_RLE_CONVERT_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x { +namespace parallel_rle { +template +class ConvertFunctor : public Functor { +public: + MGARDX_CONT ConvertFunctor() {} + MGARDX_CONT + ConvertFunctor(SubArray<1, C_run, DeviceType> counts_in, + SubArray<1, C_global, DeviceType> counts_out) + : counts_in(counts_in), counts_out(counts_out) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + IDX n = counts_in.shape(0); + + for (IDX i = start; i < n; i += grid_size) { + *counts_out(i) = (C_global)*counts_in(i); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, C_run, DeviceType> counts_in; + SubArray<1, C_global, DeviceType> counts_out; +}; + +template +class ConvertKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "convert"; + MGARDX_CONT + ConvertKernel(SubArray<1, C_run, DeviceType> counts_in, + SubArray<1, C_global, DeviceType> counts_out) + : counts_in(counts_in), counts_out(counts_out) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = ConvertFunctor; + FunctorType functor(counts_in, counts_out); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (counts_in.shape(0) - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, C_run, DeviceType> counts_in; + SubArray<1, C_global, DeviceType> counts_out; +}; +} // namespace parallel_rle +} // namespace mgard_x + +#endif \ No newline at end of file diff --git a/include/mgard-x/Lossless/ParallelRLE/Decode.hpp b/include/mgard-x/Lossless/ParallelRLE/Decode.hpp new file mode 100644 index 0000000000..8b62e1c058 --- /dev/null +++ b/include/mgard-x/Lossless/ParallelRLE/Decode.hpp @@ -0,0 +1,109 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_RLE_DECODE_TEMPLATE_HPP +#define MGARD_X_RLE_DECODE_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x { +namespace parallel_rle { +template +class DecodeFunctor : public Functor { +public: + MGARDX_CONT DecodeFunctor() {} + MGARDX_CONT DecodeFunctor(SubArray<1, C_run, DeviceType> counts, + SubArray<1, T_symbol, DeviceType> symbols, + SubArray<1, C_global, DeviceType> start_positions, + SubArray<1, T_symbol, DeviceType> data) + : counts(counts), symbols(symbols), start_positions(start_positions), + data(data) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + + IDX start = FunctorBase::GetBlockIdX(); + // * + // FunctorBase::GetBlockDimX() + + // FunctorBase::GetThreadIdX(); + + IDX n = counts.shape(0); + IDX grid_size = FunctorBase::GetGridDimX(); + // * + // FunctorBase::GetBlockDimX(); + + for (IDX i = start; i < n; i += grid_size) { + C_global curr_start_pos = i == 0 ? 0 : *start_positions(i - 1); + C_global next_start_pos = *start_positions(i); + T_symbol symbol = *symbols(i); + for (C_global j = + FunctorBase::GetThreadIdX() + curr_start_pos; + j < next_start_pos; j += FunctorBase::GetBlockDimX()) { + *data(j) = symbol; + } + + // for (SIZE j = curr_start_pos; j < curr_start_pos+1; j++) { + // *data(j) = symbol; + // } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, C_run, DeviceType> counts; + SubArray<1, T_symbol, DeviceType> symbols; + SubArray<1, C_global, DeviceType> start_positions; + SubArray<1, T_symbol, DeviceType> data; +}; + +template +class DecodeKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "decode"; + MGARDX_CONT + DecodeKernel(SubArray<1, C_run, DeviceType> counts, + SubArray<1, T_symbol, DeviceType> symbols, + SubArray<1, C_global, DeviceType> start_positions, + SubArray<1, T_symbol, DeviceType> data) + : counts(counts), symbols(symbols), start_positions(start_positions), + data(data) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = DecodeFunctor; + FunctorType functor(counts, symbols, start_positions, data); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 8; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = counts.shape(0); + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, C_run, DeviceType> counts; + SubArray<1, T_symbol, DeviceType> symbols; + SubArray<1, C_global, DeviceType> start_positions; + SubArray<1, T_symbol, DeviceType> data; +}; +} // namespace parallel_rle +} // namespace mgard_x + +#endif \ No newline at end of file diff --git a/include/mgard-x/Lossless/ParallelRLE/Encode.hpp b/include/mgard-x/Lossless/ParallelRLE/Encode.hpp new file mode 100644 index 0000000000..f9e477607a --- /dev/null +++ b/include/mgard-x/Lossless/ParallelRLE/Encode.hpp @@ -0,0 +1,105 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_RLE_ENCODE_TEMPLATE_HPP +#define MGARD_X_RLE_ENCODE_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x { +namespace parallel_rle { +template +class EncodeFunctor : public Functor { +public: + MGARDX_CONT EncodeFunctor() {} + MGARDX_CONT + EncodeFunctor(C_global total_run_length, + SubArray<1, T_symbol, DeviceType> data, + SubArray<1, C_global, DeviceType> start_positions, + SubArray<1, C_run, DeviceType> counts, + SubArray<1, T_symbol, DeviceType> symbols) + : total_run_length(total_run_length), data(data), + start_positions(start_positions), counts(counts), symbols(symbols) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + for (IDX i = start; i < total_run_length; i += grid_size) { + C_global curr_start_pos = *start_positions(i); + C_global next_start_pos = *start_positions(i + 1); + + *symbols(i) = *data(curr_start_pos); + *counts(i) = (C_run)(next_start_pos - curr_start_pos); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + C_global total_run_length; + SubArray<1, T_symbol, DeviceType> data; + SubArray<1, C_global, DeviceType> start_positions; + SubArray<1, C_run, DeviceType> counts; + SubArray<1, T_symbol, DeviceType> symbols; +}; + +template +class EncodeKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "encode"; + MGARDX_CONT + EncodeKernel(C_global total_run_length, + SubArray<1, T_symbol, DeviceType> data, + SubArray<1, C_global, DeviceType> start_positions, + SubArray<1, C_run, DeviceType> counts, + SubArray<1, T_symbol, DeviceType> symbols) + : total_run_length(total_run_length), data(data), + start_positions(start_positions), counts(counts), symbols(symbols) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = EncodeFunctor; + FunctorType functor(total_run_length, data, start_positions, counts, + symbols); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (total_run_length - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + C_global total_run_length; + SubArray<1, T_symbol, DeviceType> data; + SubArray<1, C_global, DeviceType> start_positions; + SubArray<1, C_run, DeviceType> counts; + SubArray<1, T_symbol, DeviceType> symbols; +}; +} // namespace parallel_rle +} // namespace mgard_x + +#endif \ No newline at end of file diff --git a/include/mgard-x/Lossless/ParallelRLE/RunLengthEncoding.hpp b/include/mgard-x/Lossless/ParallelRLE/RunLengthEncoding.hpp new file mode 100644 index 0000000000..ed8282ab7e --- /dev/null +++ b/include/mgard-x/Lossless/ParallelRLE/RunLengthEncoding.hpp @@ -0,0 +1,361 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_RUN_LENGTH_ENCODING_TEMPLATE_HPP +#define MGARD_X_RUN_LENGTH_ENCODING_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" +#include "Convert.hpp" +#include "Decode.hpp" +#include "Encode.hpp" +#include "StartMarks.hpp" +#include "StartPositions.hpp" + +namespace mgard_x { + +namespace parallel_rle { + +template +class RunLengthEncoding { +public: + RunLengthEncoding() : initialized(false) {} + + RunLengthEncoding(SIZE max_size) : initialized(true), max_size(max_size) {} + + void Resize(SIZE max_size, int queue_idx) { + this->initialized = true; + this->max_size = max_size; + start_marks.resize({max_size}, queue_idx); + scanned_start_marks.resize({max_size}, queue_idx); + start_positions.resize({max_size}, queue_idx); + MemoryManager::MallocHost(signature_verify, 7 * sizeof(char), + queue_idx); + DeviceCollective::ScanSumInclusive( + max_size, SubArray<1, C_global, DeviceType>(), + SubArray<1, C_global, DeviceType>(), this->scan_workspace, false, + queue_idx); + } + + static size_t EstimateMemoryFootprint(SIZE n) { + size_t memory_footprint = 0; + memory_footprint += n * sizeof(SIZE) * 3; + Array<1, Byte, DeviceType> tmp_workspace; + DeviceCollective::ScanSumInclusive( + n, SubArray<1, C_global, DeviceType>(), + SubArray<1, C_global, DeviceType>(), tmp_workspace, false, 0); + memory_footprint += tmp_workspace.shape(0); + return 0; + } + + double EstimateCR(Array<1, T_symbol, DeviceType> &original_data, + int queue_idx) { + Timer timer; + // Timer timer_each; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE original_length = original_data.shape(0); + + start_marks.resize({original_length}, queue_idx); + scanned_start_marks.resize({original_length}, queue_idx); + start_positions.resize({original_length}, queue_idx); + + // timer_each.start(); + + DeviceLauncher::Execute( + StartMarksKernel( + SubArray(original_data), SubArray(start_marks)), + queue_idx); + + // DeviceRuntime::SyncQueue(queue_idx); + // timer_each.end(); timer_each.print("StartMarksKernel", original_length * + // sizeof(T_symbol)); timer_each.clear(); timer_each.start(); + // DeviceRuntime::SyncQueue(queue_idx); + // PrintSubarray("StartMarksKernel", SubArray(start_marks)); + + DeviceCollective::ScanSumInclusive( + original_length, SubArray(start_marks), SubArray(scanned_start_marks), + scan_workspace, true, queue_idx); + + // DeviceRuntime::SyncQueue(queue_idx); + // timer_each.end(); timer_each.print("ScanSumInclusive", original_length * + // sizeof(T_symbol)); timer_each.clear(); timer_each.start(); + + C_global _total_run_length = 0; + MemoryManager::Copy1D( + &_total_run_length, scanned_start_marks.data() + original_length - 1, 1, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("RLE estimate CR", original_length * sizeof(T_symbol)); + timer.clear(); + } + + return (double)(original_length * sizeof(T_symbol)) / + (_total_run_length * (sizeof(T_symbol) + sizeof(C_run)) + 30); + } + + bool Compress(Array<1, T_symbol, DeviceType> &original_data, + Array<1, Byte, DeviceType> &compressed_data, float target_cr, + int queue_idx) { + Timer timer; + // Timer timer_each; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + // PrintSubarray("original_data", SubArray(original_data)); + + SIZE original_length = original_data.shape(0); + + start_marks.resize({original_length}, queue_idx); + scanned_start_marks.resize({original_length}, queue_idx); + start_positions.resize({original_length}, queue_idx); + + // timer_each.start(); + + DeviceLauncher::Execute( + StartMarksKernel( + SubArray(original_data), SubArray(start_marks)), + queue_idx); + + // DeviceRuntime::SyncQueue(queue_idx); + // timer_each.end(); timer_each.print("StartMarksKernel", original_length * + // sizeof(T_symbol)); timer_each.clear(); timer_each.start(); + // DeviceRuntime::SyncQueue(queue_idx); + // PrintSubarray("StartMarksKernel", SubArray(start_marks)); + + DeviceCollective::ScanSumInclusive( + original_length, SubArray(start_marks), SubArray(scanned_start_marks), + scan_workspace, true, queue_idx); + + // DeviceRuntime::SyncQueue(queue_idx); + // timer_each.end(); timer_each.print("ScanSumInclusive", original_length * + // sizeof(T_symbol)); timer_each.clear(); timer_each.start(); + + C_global _total_run_length = 0; + MemoryManager::Copy1D( + &_total_run_length, scanned_start_marks.data() + original_length - 1, 1, + queue_idx); + + if (target_cr > 0) { + double est_cr = + (double)(original_length * sizeof(T_symbol)) / + (_total_run_length * (sizeof(T_symbol) + sizeof(C_run)) + 30); + log::info("RLE estimated CR: " + std::to_string(est_cr) + + " (target: " + std::to_string(target_cr) + ")"); + if (est_cr < target_cr) { + return false; + } + } + + // DeviceRuntime::SyncQueue(queue_idx); + // PrintSubarray("scanned_start_marks", SubArray(scanned_start_marks)); + DeviceLauncher::Execute( + StartPositionsKernel( + SubArray(scanned_start_marks), SubArray(start_positions)), + queue_idx); + + // wait for total_run_length to be copied + total_run_length = _total_run_length; + DeviceRuntime::SyncQueue(queue_idx); + + // DeviceRuntime::SyncQueue(queue_idx); + // timer_each.end(); timer_each.print("StartPositionsKernel", + // original_length * sizeof(T_symbol)); timer_each.clear(); + // timer_each.start(); + // DeviceRuntime::SyncQueue(queue_idx); + // PrintSubarray("start_positions", SubArray(start_positions)); + + SIZE byte_offset = 0; + advance_with_align(byte_offset, 7); // signature + advance_with_align(byte_offset, 1); + advance_with_align(byte_offset, 1); + advance_with_align(byte_offset, total_run_length); + advance_with_align(byte_offset, total_run_length); + + SIZE output_size = byte_offset; + compressed_data.resize({output_size}, queue_idx); + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + + byte_offset = 0; + SerializeArray(compressed_subarray, signature, 7, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &total_run_length, 1, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &original_length, 1, byte_offset, + queue_idx); + + SubArray<1, C_run, DeviceType> counts( + {total_run_length}, (C_run *)(compressed_data.data() + byte_offset)); + advance_with_align(byte_offset, total_run_length); + SubArray<1, T_symbol, DeviceType> symbols( + {total_run_length}, (T_symbol *)(compressed_data.data() + byte_offset)); + advance_with_align(byte_offset, total_run_length); + + DeviceLauncher::Execute( + EncodeKernel( + total_run_length, SubArray(original_data), + SubArray(start_positions), counts, symbols), + queue_idx); + + // DeviceRuntime::SyncQueue(queue_idx); + // timer_each.end(); timer_each.print("EncodeKernel", original_length * + // sizeof(T_symbol)); timer_each.clear(); timer_each.start(); + + // PrintSubarray("counts", counts); + // PrintSubarray("symbols", symbols); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + log::info("RLE compression ratio: " + + std::to_string(original_length * sizeof(T_symbol)) + "/" + + std::to_string(compressed_data.shape(0)) + " (" + + std::to_string((double)original_length * sizeof(T_symbol) / + compressed_data.shape(0)) + + ")"); + timer.print("RLE compress", original_length * sizeof(T_symbol)); + timer.clear(); + } + + return true; + + // C_run * counts_host = new C_run[total_run_length]; + // MemoryManager::Copy1D(counts_host, counts.data(), + // total_run_length, queue_idx); + // DeviceRuntime::SyncQueue(queue_idx); + // C_run max_run = 0; + // for (int i = 0; i < total_run_length; i++) { + // if (counts_host[i] > max_run) { + // max_run = counts_host[i]; + // } + // } + + // std::cout << "total_run_length: " << total_run_length << std::endl; + // std::cout << "max_run: " << max_run << std::endl; + // IDX MAX_RUN = (IDX)1u << sizeof(C_run) * 8; + // std::cout << "max_run allowed: " << MAX_RUN << std::endl; + } + + void Serialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) {} + + bool Verify(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + SubArray compressed_subarray(compressed_data); + SIZE byte_offset = 0; + DeserializeArray(compressed_subarray, signature_verify, 7, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + for (int i = 0; i < 7; i++) { + if (signature[i] != signature_verify[i]) { + return false; + } + } + return true; + } + + void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + if (!Verify(compressed_data, queue_idx)) { + throw std::runtime_error("RLE signature mismatch."); + } + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + Byte *signature_ptr = nullptr; + total_run_length_ptr = &total_run_length; + original_length_ptr = &original_length; + SIZE byte_offset = 0; + DeserializeArray(compressed_subarray, signature_ptr, 7, byte_offset, + true, queue_idx); + DeserializeArray(compressed_subarray, total_run_length_ptr, 1, + byte_offset, false, queue_idx); + DeserializeArray(compressed_subarray, original_length_ptr, 1, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + DeserializeArray(compressed_subarray, counts_ptr, total_run_length, + byte_offset, true, queue_idx); + DeserializeArray(compressed_subarray, symbols_ptr, + total_run_length, byte_offset, true, queue_idx); + + DeviceRuntime::SyncQueue(queue_idx); + // PrintSubarray("counts", SubArray<1, C_run, + // DeviceType>({total_run_length}, counts_ptr)); PrintSubarray("symbols", + // SubArray<1, T_symbol, DeviceType>({total_run_length}, symbols_ptr)); + // std::cout << "total_run_length: " << total_run_length << std::endl; + // std::cout << "original_length: " << original_length << std::endl; + } + + void Decompress(Array<1, Byte, DeviceType> &compressed_data, + Array<1, T_symbol, DeviceType> &decompressed_data, + int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + SubArray<1, C_run, DeviceType> counts({total_run_length}, counts_ptr); + SubArray<1, T_symbol, DeviceType> symbols({total_run_length}, symbols_ptr); + decompressed_data.resize({(SIZE)original_length}, queue_idx); + + // reuse array + SubArray counts_SIZE(start_marks); + DeviceLauncher::Execute( + ConvertKernel(counts, + counts_SIZE), + queue_idx); + + DeviceCollective::ScanSumInclusive( + total_run_length, counts_SIZE, SubArray(start_positions), + scan_workspace, true, queue_idx); + + // DeviceRuntime::SyncQueue(queue_idx); + // PrintSubarray("start_positions", SubArray(start_positions)); + + DeviceLauncher::Execute( + DecodeKernel( + counts, symbols, SubArray(start_positions), + SubArray(decompressed_data)), + queue_idx); + + // DeviceRuntime::SyncQueue(queue_idx); + // PrintSubarray("decompressed_data", SubArray(decompressed_data)); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("RLE decompress", original_length * sizeof(T_symbol)); + timer.clear(); + } + } + + bool initialized; + SIZE max_size; + SIZE original_length = 0; + SIZE total_run_length = 0; + SIZE *total_run_length_ptr = nullptr; + SIZE *original_length_ptr = nullptr; + C_run *counts_ptr = nullptr; + T_symbol *symbols_ptr = nullptr; + Byte signature[7] = {'M', 'G', 'X', 'R', 'L', 'E', 'C'}; + Byte *signature_verify; + + Array<1, C_global, DeviceType> start_marks; + Array<1, C_global, DeviceType> scanned_start_marks; + Array<1, C_global, DeviceType> start_positions; + Array<1, Byte, DeviceType> scan_workspace; +}; + +} // namespace parallel_rle +} // namespace mgard_x +#endif \ No newline at end of file diff --git a/include/mgard-x/Lossless/ParallelRLE/StartMarks.hpp b/include/mgard-x/Lossless/ParallelRLE/StartMarks.hpp new file mode 100644 index 0000000000..fc3182076a --- /dev/null +++ b/include/mgard-x/Lossless/ParallelRLE/StartMarks.hpp @@ -0,0 +1,94 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_RLE_START_MARKS_TEMPLATE_HPP +#define MGARD_X_RLE_START_MARKS_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x { +namespace parallel_rle { +template +class StartMarksFunctor : public Functor { +public: + MGARDX_CONT StartMarksFunctor() {} + MGARDX_CONT StartMarksFunctor(SubArray<1, T_symbol, DeviceType> data, + SubArray<1, C_global, DeviceType> start_marks) + : data(data), start_marks(start_marks) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX n = data.shape(0); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + // HIP will fail if making the following line a constexpr + IDX MAX_RUN = (IDX)1 << (sizeof(C_run) * 8); + for (IDX i = start; i < n; i += grid_size) { + if (i == 0) { + *start_marks(i) = 1; + } else { + if (i % MAX_RUN == 0) { + *start_marks(i) = 1; + } else { + *start_marks(i) = (*data(i) != *data(i - 1) ? 1 : 0); + } + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, T_symbol, DeviceType> data; + SubArray<1, C_global, DeviceType> start_marks; +}; + +template +class StartMarksKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "start marks"; + MGARDX_CONT + StartMarksKernel(SubArray<1, T_symbol, DeviceType> data, + SubArray<1, C_global, DeviceType> start_marks) + : data(data), start_marks(start_marks) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = + StartMarksFunctor; + FunctorType functor(data, start_marks); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (data.shape(0) - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, T_symbol, DeviceType> data; + SubArray<1, C_global, DeviceType> start_marks; +}; +} // namespace parallel_rle +} // namespace mgard_x + +#endif \ No newline at end of file diff --git a/include/mgard-x/Lossless/ParallelRLE/StartPositions.hpp b/include/mgard-x/Lossless/ParallelRLE/StartPositions.hpp new file mode 100644 index 0000000000..85a8354ef3 --- /dev/null +++ b/include/mgard-x/Lossless/ParallelRLE/StartPositions.hpp @@ -0,0 +1,99 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_RLE_START_POSITIONS_TEMPLATE_HPP +#define MGARD_X_RLE_START_POSITIONS_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x { +namespace parallel_rle { +template +class StartPositionsFunctor : public Functor { +public: + MGARDX_CONT StartPositionsFunctor() {} + MGARDX_CONT + StartPositionsFunctor(SubArray<1, C_global, DeviceType> scanned_start_marks, + SubArray<1, C_global, DeviceType> start_positions) + : scanned_start_marks(scanned_start_marks), + start_positions(start_positions) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + IDX n = scanned_start_marks.shape(0); + + for (IDX i = start; i < n; i += grid_size) { + C_global curr_pos = *scanned_start_marks(i); + C_global prev_pos = i > 0 ? *scanned_start_marks(i - 1) : 0; + + if (i == n - 1) { + *start_positions(curr_pos) = n; + } + + if (i == 0) { + *start_positions((IDX)0) = 0; + } else if (curr_pos != prev_pos) { + *start_positions(curr_pos - 1) = i; + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, C_global, DeviceType> scanned_start_marks; + SubArray<1, C_global, DeviceType> start_positions; +}; + +template +class StartPositionsKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "start positions"; + MGARDX_CONT + StartPositionsKernel(SubArray<1, C_global, DeviceType> scanned_start_marks, + SubArray<1, C_global, DeviceType> start_positions) + : scanned_start_marks(scanned_start_marks), + start_positions(start_positions) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = + StartPositionsFunctor; + FunctorType functor(scanned_start_marks, start_positions); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (scanned_start_marks.shape(0) - 1) / tbx + 1; + gridx = std::max((SIZE)1, gridx / repeat_factor); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, C_global, DeviceType> scanned_start_marks; + SubArray<1, C_global, DeviceType> start_positions; +}; +} // namespace parallel_rle +} // namespace mgard_x + +#endif \ No newline at end of file diff --git a/include/mgard-x/Lossless/ParallelRLE/ZeroDecode.hpp b/include/mgard-x/Lossless/ParallelRLE/ZeroDecode.hpp new file mode 100644 index 0000000000..ec2b58983c --- /dev/null +++ b/include/mgard-x/Lossless/ParallelRLE/ZeroDecode.hpp @@ -0,0 +1,172 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_ZERO_RLE_DECODE_TEMPLATE_HPP +#define MGARD_X_ZERO_RLE_DECODE_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x { +namespace parallel_rle { + +// Step 1 of zero-RLE decode: widen each run count and add one, producing the +// per-symbol stride (preceding zeros + the symbol itself). The inclusive scan +// of these strides yields cum[s] = position[s] + 1, so position[s] = cum[s] - 1 +// recovers each nonzero's absolute index. +template +class ZeroStrideFunctor : public Functor { +public: + MGARDX_CONT ZeroStrideFunctor() {} + MGARDX_CONT + ZeroStrideFunctor(SubArray<1, C_run, DeviceType> counts, + SubArray<1, C_global, DeviceType> strides) + : counts(counts), strides(strides) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + IDX n = counts.shape(0); + + for (IDX i = start; i < n; i += grid_size) { + *strides(i) = (C_global)(*counts(i)) + 1; + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, C_run, DeviceType> counts; + SubArray<1, C_global, DeviceType> strides; +}; + +template +class ZeroStrideKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "zero rle stride"; + MGARDX_CONT + ZeroStrideKernel(SubArray<1, C_run, DeviceType> counts, + SubArray<1, C_global, DeviceType> strides) + : counts(counts), strides(strides) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = + ZeroStrideFunctor; + FunctorType functor(counts, strides); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (counts.shape(0) - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, C_run, DeviceType> counts; + SubArray<1, C_global, DeviceType> strides; +}; + +// Step 2 of zero-RLE decode: scatter each nonzero symbol into its recovered +// absolute position. The output buffer is pre-zeroed by the caller, so only the +// nonzeros need to be written; runs of zeros (including any trailing zeros) are +// left untouched. +template +class ZeroScatterFunctor : public Functor { +public: + MGARDX_CONT ZeroScatterFunctor() {} + MGARDX_CONT + ZeroScatterFunctor(SubArray<1, T_symbol, DeviceType> symbols, + SubArray<1, C_global, DeviceType> strides, + SubArray<1, T_symbol, DeviceType> data) + : symbols(symbols), strides(strides), data(data) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + IDX n = symbols.shape(0); + + for (IDX i = start; i < n; i += grid_size) { + // strides has been inclusive-scanned: strides[i] == position[i] + 1. + C_global pos = *strides(i) - 1; + *data(pos) = *symbols(i); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, T_symbol, DeviceType> symbols; + SubArray<1, C_global, DeviceType> strides; + SubArray<1, T_symbol, DeviceType> data; +}; + +template +class ZeroScatterKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "zero rle scatter"; + MGARDX_CONT + ZeroScatterKernel(SubArray<1, T_symbol, DeviceType> symbols, + SubArray<1, C_global, DeviceType> strides, + SubArray<1, T_symbol, DeviceType> data) + : symbols(symbols), strides(strides), data(data) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = + ZeroScatterFunctor; + FunctorType functor(symbols, strides, data); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (symbols.shape(0) - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, T_symbol, DeviceType> symbols; + SubArray<1, C_global, DeviceType> strides; + SubArray<1, T_symbol, DeviceType> data; +}; +} // namespace parallel_rle +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/Lossless/ParallelRLE/ZeroEncode.hpp b/include/mgard-x/Lossless/ParallelRLE/ZeroEncode.hpp new file mode 100644 index 0000000000..471c70a763 --- /dev/null +++ b/include/mgard-x/Lossless/ParallelRLE/ZeroEncode.hpp @@ -0,0 +1,114 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_ZERO_RLE_ENCODE_TEMPLATE_HPP +#define MGARD_X_ZERO_RLE_ENCODE_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x { +namespace parallel_rle { + +// Zero-RLE encode. start_positions holds the absolute index of each of the +// num_symbols nonzeros (computed by the shared StartPositionsKernel), so for +// symbol s: +// symbols[s] = data[start_positions[s]] +// counts[s] = number of zeros immediately preceding it +// = start_positions[s] - start_positions[s-1] - 1 (s > 0) +// = start_positions[0] (s == 0) +// Trailing zeros after the last nonzero are not stored; they are recovered on +// decode from the known original length. +template +class ZeroEncodeFunctor : public Functor { +public: + MGARDX_CONT ZeroEncodeFunctor() {} + MGARDX_CONT + ZeroEncodeFunctor(C_global num_symbols, + SubArray<1, T_symbol, DeviceType> data, + SubArray<1, C_global, DeviceType> start_positions, + SubArray<1, C_run, DeviceType> counts, + SubArray<1, T_symbol, DeviceType> symbols) + : num_symbols(num_symbols), data(data), start_positions(start_positions), + counts(counts), symbols(symbols) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + for (IDX i = start; i < num_symbols; i += grid_size) { + C_global curr_pos = *start_positions(i); + C_global prev_pos = (i == 0) ? 0 : (*start_positions(i - 1) + 1); + + *symbols(i) = *data(curr_pos); + *counts(i) = (C_run)(curr_pos - prev_pos); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + C_global num_symbols; + SubArray<1, T_symbol, DeviceType> data; + SubArray<1, C_global, DeviceType> start_positions; + SubArray<1, C_run, DeviceType> counts; + SubArray<1, T_symbol, DeviceType> symbols; +}; + +template +class ZeroEncodeKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "zero rle encode"; + MGARDX_CONT + ZeroEncodeKernel(C_global num_symbols, SubArray<1, T_symbol, DeviceType> data, + SubArray<1, C_global, DeviceType> start_positions, + SubArray<1, C_run, DeviceType> counts, + SubArray<1, T_symbol, DeviceType> symbols) + : num_symbols(num_symbols), data(data), start_positions(start_positions), + counts(counts), symbols(symbols) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = + ZeroEncodeFunctor; + FunctorType functor(num_symbols, data, start_positions, counts, symbols); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (num_symbols - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + C_global num_symbols; + SubArray<1, T_symbol, DeviceType> data; + SubArray<1, C_global, DeviceType> start_positions; + SubArray<1, C_run, DeviceType> counts; + SubArray<1, T_symbol, DeviceType> symbols; +}; +} // namespace parallel_rle +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/Lossless/ParallelRLE/ZeroRunLengthEncoding.hpp b/include/mgard-x/Lossless/ParallelRLE/ZeroRunLengthEncoding.hpp new file mode 100644 index 0000000000..6ea1a57bcd --- /dev/null +++ b/include/mgard-x/Lossless/ParallelRLE/ZeroRunLengthEncoding.hpp @@ -0,0 +1,287 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_ZERO_RUN_LENGTH_ENCODING_TEMPLATE_HPP +#define MGARD_X_ZERO_RUN_LENGTH_ENCODING_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" +#include "ZeroDecode.hpp" +#include "ZeroEncode.hpp" +#include "ZeroStartMarks.hpp" +#include "ZeroStartPositions.hpp" + +namespace mgard_x { + +namespace parallel_rle { + +// Zero-run-length encoding (RLE0). Only zeros are run-encoded; nonzero values +// are stored as literals together with the count of zeros that immediately +// precede each one. Trailing zeros are implied by the stored original length. +// This is the sparse-data front end intended to feed an entropy backend (e.g. +// Huffman/rANS) on the two output streams (counts, symbols) separately. +// +// Limitation: a single zero gap must fit in C_run (use uint32_t unless gaps can +// exceed 2^32). Unlike the full RLE, gaps are not split at C_run boundaries. +template +class ZeroRunLengthEncoding { +public: + ZeroRunLengthEncoding() : initialized(false) {} + + ZeroRunLengthEncoding(SIZE max_size) + : initialized(true), max_size(max_size) {} + + void Resize(SIZE max_size, int queue_idx) { + this->initialized = true; + this->max_size = max_size; + start_marks.resize({max_size}, queue_idx); + scanned_start_marks.resize({max_size}, queue_idx); + start_positions.resize({max_size}, queue_idx); + MemoryManager::MallocHost(signature_verify, 7 * sizeof(char), + queue_idx); + DeviceCollective::ScanSumInclusive( + max_size, SubArray<1, C_global, DeviceType>(), + SubArray<1, C_global, DeviceType>(), this->scan_workspace, false, + queue_idx); + } + + // Run the mark + scan stages and read back the number of nonzeros (= number + // of stored symbols). Leaves scanned_start_marks populated for the caller. + C_global CountSymbols(Array<1, T_symbol, DeviceType> &original_data, + int queue_idx) { + SIZE original_length = original_data.shape(0); + + start_marks.resize({original_length}, queue_idx); + scanned_start_marks.resize({original_length}, queue_idx); + start_positions.resize({original_length}, queue_idx); + + DeviceLauncher::Execute( + ZeroStartMarksKernel( + SubArray(original_data), SubArray(start_marks)), + queue_idx); + + DeviceCollective::ScanSumInclusive( + original_length, SubArray(start_marks), SubArray(scanned_start_marks), + scan_workspace, true, queue_idx); + + C_global _num_symbols = 0; + MemoryManager::Copy1D( + &_num_symbols, scanned_start_marks.data() + original_length - 1, 1, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + return _num_symbols; + } + + double EstimateCR(Array<1, T_symbol, DeviceType> &original_data, + int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE original_length = original_data.shape(0); + C_global _num_symbols = CountSymbols(original_data, queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Zero RLE estimate CR", original_length * sizeof(T_symbol)); + timer.clear(); + } + + return (double)(original_length * sizeof(T_symbol)) / + (_num_symbols * (sizeof(T_symbol) + sizeof(C_run)) + 30); + } + + bool Compress(Array<1, T_symbol, DeviceType> &original_data, + Array<1, Byte, DeviceType> &compressed_data, float target_cr, + int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE original_length = original_data.shape(0); + C_global _num_symbols = CountSymbols(original_data, queue_idx); + + if (target_cr > 0) { + double est_cr = (double)(original_length * sizeof(T_symbol)) / + (_num_symbols * (sizeof(T_symbol) + sizeof(C_run)) + 30); + log::info("Zero RLE estimated CR: " + std::to_string(est_cr) + + " (target: " + std::to_string(target_cr) + ")"); + if (est_cr < target_cr) { + return false; + } + } + + num_symbols = _num_symbols; + + // Compact nonzero positions, then derive (gap, value) pairs from them. + if (num_symbols > 0) { + DeviceLauncher::Execute( + ZeroStartPositionsKernel( + SubArray(scanned_start_marks), SubArray(start_positions)), + queue_idx); + } + + SIZE byte_offset = 0; + advance_with_align(byte_offset, 7); // signature + advance_with_align(byte_offset, 1); // num_symbols + advance_with_align(byte_offset, 1); // original_length + advance_with_align(byte_offset, num_symbols); + advance_with_align(byte_offset, num_symbols); + + SIZE output_size = byte_offset; + compressed_data.resize({output_size}, queue_idx); + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + + byte_offset = 0; + SerializeArray(compressed_subarray, signature, 7, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &num_symbols, 1, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &original_length, 1, byte_offset, + queue_idx); + + SubArray<1, C_run, DeviceType> counts( + {num_symbols}, (C_run *)(compressed_data.data() + byte_offset)); + advance_with_align(byte_offset, num_symbols); + SubArray<1, T_symbol, DeviceType> symbols( + {num_symbols}, (T_symbol *)(compressed_data.data() + byte_offset)); + advance_with_align(byte_offset, num_symbols); + + if (num_symbols > 0) { + DeviceLauncher::Execute( + ZeroEncodeKernel( + num_symbols, SubArray(original_data), SubArray(start_positions), + counts, symbols), + queue_idx); + } + + DeviceRuntime::SyncQueue(queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + log::info("Zero RLE compression ratio: " + + std::to_string(original_length * sizeof(T_symbol)) + "/" + + std::to_string(compressed_data.shape(0)) + " (" + + std::to_string((double)original_length * sizeof(T_symbol) / + compressed_data.shape(0)) + + ")"); + timer.end(); + timer.print("Zero RLE compress", original_length * sizeof(T_symbol)); + timer.clear(); + } + + return true; + } + + void Serialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) {} + + bool Verify(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + SubArray compressed_subarray(compressed_data); + SIZE byte_offset = 0; + DeserializeArray(compressed_subarray, signature_verify, 7, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + for (int i = 0; i < 7; i++) { + if (signature[i] != signature_verify[i]) { + return false; + } + } + return true; + } + + void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + if (!Verify(compressed_data, queue_idx)) { + throw std::runtime_error("Zero RLE signature mismatch."); + } + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + Byte *signature_ptr = nullptr; + num_symbols_ptr = &num_symbols; + original_length_ptr = &original_length; + SIZE byte_offset = 0; + DeserializeArray(compressed_subarray, signature_ptr, 7, byte_offset, + true, queue_idx); + DeserializeArray(compressed_subarray, num_symbols_ptr, 1, byte_offset, + false, queue_idx); + DeserializeArray(compressed_subarray, original_length_ptr, 1, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + DeserializeArray(compressed_subarray, counts_ptr, num_symbols, + byte_offset, true, queue_idx); + DeserializeArray(compressed_subarray, symbols_ptr, num_symbols, + byte_offset, true, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + } + + void Decompress(Array<1, Byte, DeviceType> &compressed_data, + Array<1, T_symbol, DeviceType> &decompressed_data, + int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SubArray<1, C_run, DeviceType> counts({(SIZE)num_symbols}, counts_ptr); + SubArray<1, T_symbol, DeviceType> symbols({(SIZE)num_symbols}, symbols_ptr); + decompressed_data.resize({(SIZE)original_length}, queue_idx); + + // Trailing/interior zeros are never written, so start from an all-zero + // buffer and scatter only the nonzeros. + MemoryManager::Memset1D(decompressed_data.data(), + original_length, 0, queue_idx); + + if (num_symbols > 0) { + // strides[i] = counts[i] + 1; inclusive scan -> position[i] + 1. + SubArray<1, C_global, DeviceType> strides(start_marks); + DeviceLauncher::Execute( + ZeroStrideKernel(counts, + strides), + queue_idx); + + DeviceCollective::ScanSumInclusive( + num_symbols, strides, SubArray(start_positions), scan_workspace, true, + queue_idx); + + DeviceLauncher::Execute( + ZeroScatterKernel( + symbols, SubArray(start_positions), SubArray(decompressed_data)), + queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Zero RLE decompress", original_length * sizeof(T_symbol)); + timer.clear(); + } + } + + bool initialized; + SIZE max_size; + SIZE original_length = 0; + SIZE num_symbols = 0; + SIZE *num_symbols_ptr = nullptr; + SIZE *original_length_ptr = nullptr; + C_run *counts_ptr = nullptr; + T_symbol *symbols_ptr = nullptr; + Byte signature[7] = {'M', 'G', 'X', 'Z', 'R', 'L', '0'}; + Byte *signature_verify; + + Array<1, C_global, DeviceType> start_marks; + Array<1, C_global, DeviceType> scanned_start_marks; + Array<1, C_global, DeviceType> start_positions; + Array<1, Byte, DeviceType> scan_workspace; +}; + +} // namespace parallel_rle +} // namespace mgard_x +#endif diff --git a/include/mgard-x/Lossless/ParallelRLE/ZeroStartMarks.hpp b/include/mgard-x/Lossless/ParallelRLE/ZeroStartMarks.hpp new file mode 100644 index 0000000000..7743d434b7 --- /dev/null +++ b/include/mgard-x/Lossless/ParallelRLE/ZeroStartMarks.hpp @@ -0,0 +1,91 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_ZERO_RLE_START_MARKS_TEMPLATE_HPP +#define MGARD_X_ZERO_RLE_START_MARKS_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x { +namespace parallel_rle { + +// Zero-RLE (RLE0) marks: a 1 at every *nonzero* element, 0 at zeros. The +// inclusive scan of these marks gives, for each nonzero, its rank among the +// nonzeros (and, in the last element, the total nonzero count = number of +// stored symbols). Unlike full RLE, zeros are never stored as symbols; they +// are recovered on decode from the gap (preceding-zero count) of each symbol. +template +class ZeroStartMarksFunctor : public Functor { +public: + MGARDX_CONT ZeroStartMarksFunctor() {} + MGARDX_CONT + ZeroStartMarksFunctor(SubArray<1, T_symbol, DeviceType> data, + SubArray<1, C_global, DeviceType> start_marks) + : data(data), start_marks(start_marks) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX n = data.shape(0); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + for (IDX i = start; i < n; i += grid_size) { + *start_marks(i) = (*data(i) != (T_symbol)0) ? 1 : 0; + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, T_symbol, DeviceType> data; + SubArray<1, C_global, DeviceType> start_marks; +}; + +template +class ZeroStartMarksKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "zero rle start marks"; + MGARDX_CONT + ZeroStartMarksKernel(SubArray<1, T_symbol, DeviceType> data, + SubArray<1, C_global, DeviceType> start_marks) + : data(data), start_marks(start_marks) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = + ZeroStartMarksFunctor; + FunctorType functor(data, start_marks); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (data.shape(0) - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, T_symbol, DeviceType> data; + SubArray<1, C_global, DeviceType> start_marks; +}; +} // namespace parallel_rle +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/Lossless/ParallelRLE/ZeroStartPositions.hpp b/include/mgard-x/Lossless/ParallelRLE/ZeroStartPositions.hpp new file mode 100644 index 0000000000..321ea64c50 --- /dev/null +++ b/include/mgard-x/Lossless/ParallelRLE/ZeroStartPositions.hpp @@ -0,0 +1,102 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_ZERO_RLE_START_POSITIONS_TEMPLATE_HPP +#define MGARD_X_ZERO_RLE_START_POSITIONS_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x { +namespace parallel_rle { + +// Compact the absolute positions of the nonzeros. scanned_start_marks is the +// inclusive scan of the nonzero marks, so scanned[i] is the count of nonzeros +// in [0, i]. Element i is a nonzero exactly when its running count differs from +// the previous one; in that case its 0-based rank is scanned[i] - 1 and its +// position is i. Unlike the full-RLE StartPositions there is no i == 0 override +// (position 0 is only a symbol when data[0] is nonzero) and no terminal entry +// (ZeroEncode derives gaps from consecutive positions, not a sentinel). +template +class ZeroStartPositionsFunctor : public Functor { +public: + MGARDX_CONT ZeroStartPositionsFunctor() {} + MGARDX_CONT ZeroStartPositionsFunctor( + SubArray<1, C_global, DeviceType> scanned_start_marks, + SubArray<1, C_global, DeviceType> start_positions) + : scanned_start_marks(scanned_start_marks), + start_positions(start_positions) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + IDX n = scanned_start_marks.shape(0); + + for (IDX i = start; i < n; i += grid_size) { + C_global curr_pos = *scanned_start_marks(i); + C_global prev_pos = i > 0 ? *scanned_start_marks(i - 1) : 0; + if (curr_pos != prev_pos) { + *start_positions(curr_pos - 1) = i; + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, C_global, DeviceType> scanned_start_marks; + SubArray<1, C_global, DeviceType> start_positions; +}; + +template +class ZeroStartPositionsKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "zero rle start positions"; + MGARDX_CONT + ZeroStartPositionsKernel( + SubArray<1, C_global, DeviceType> scanned_start_marks, + SubArray<1, C_global, DeviceType> start_positions) + : scanned_start_marks(scanned_start_marks), + start_positions(start_positions) {} + + MGARDX_CONT + Task> + GenTask(int queue_idx) { + using FunctorType = + ZeroStartPositionsFunctor; + FunctorType functor(scanned_start_marks, start_positions); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (scanned_start_marks.shape(0) - 1) / tbx + 1; + gridx = std::max((SIZE)1, gridx / repeat_factor); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, C_global, DeviceType> scanned_start_marks; + SubArray<1, C_global, DeviceType> start_positions; +}; +} // namespace parallel_rle +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/Lossless/SymbolRans/SymbolRans.hpp b/include/mgard-x/Lossless/SymbolRans/SymbolRans.hpp new file mode 100644 index 0000000000..d0434ac28f --- /dev/null +++ b/include/mgard-x/Lossless/SymbolRans/SymbolRans.hpp @@ -0,0 +1,186 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_SYMBOL_RANS_TEMPLATE_HPP +#define MGARD_X_SYMBOL_RANS_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" +#include "../ParallelHuffman/OutlierSeparator.hpp" +#include "../rANS/Rans.hpp" + +namespace mgard_x { + +// Symbol-alphabet rANS lossless backend. Reuses Huffman's outlier front-end +// (values outside [0, dict_size) stored sparsely and zeroed in place) then +// entropy-codes the primary symbol stream with rANS over the dict_size +// alphabet. On the same symbol model rANS codes fractional bits, so its output +// is always <= Huffman's (no integer-code-length rounding loss) and uncapped -- +// the architecturally correct way to beat Huffman on ratio. Q = unsigned +// quantized type, S = signed. +template class SymbolRans { +public: + SymbolRans() : initialized(false) {} + + // SYMBOL_RANS_SEGMENT can be overridden at compile time to sweep the rANS + // segment size (smaller -> more parallel streams / occupancy, but more + // per-segment overhead -> lower CR). +#ifndef SYMBOL_RANS_SEGMENT +#define SYMBOL_RANS_SEGMENT 0 +#endif + + void Resize(SIZE n, int dict_size, double outlier_ratio, int queue_idx) { + initialized = true; + this->max_size = n; + this->dict_size = dict_size; + SIZE max_outliers = (SIZE)((double)n * outlier_ratio) + 1; + outlier_count_d.resize({1}, queue_idx); + outlier_idx_d.resize({max_outliers}, queue_idx); + outlier_val_d.resize({max_outliers}, queue_idx); + // NOTE: the shared-stream interleaved layout (interleaved=true) is the + // foundation for warp-cooperative coalesced encode, but its CUDA fast-path + // (warp kernels) is not implemented yet -- the sequential interleaved + // kernels would be a severe CUDA regression (one thread per block). So the + // symbol path uses the validated per-stream layout (block-interleaved + // symbol mapping already gives coalesced reads + a decode that beats + // Huffman). Flip to true once the warp kernels land. + rans.Resize(n, dict_size, queue_idx, (SIZE)SYMBOL_RANS_SEGMENT, + /*interleaved=*/false); + MemoryManager::MallocHost(signature_verify, 7 * sizeof(char), + queue_idx); + } + + void Compress(Array<1, S, DeviceType> &original_data, + Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + SIZE n = original_data.shape(0); + + // Separate out-of-dictionary outliers (zeroed in place in original_data). + ATOMIC_IDX zero = 0; + MemoryManager::Copy1D(outlier_count_d.data(), &zero, 1, + queue_idx); + DeviceLauncher::Execute( + OutlierSeparatorKernel( + SubArray(original_data), dict_size, SubArray(outlier_count_d), + SubArray(outlier_idx_d), SubArray(outlier_val_d)), + queue_idx); + MemoryManager::Copy1D(&outlier_count, outlier_count_d.data(), 1, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + if (outlier_count > outlier_val_d.shape(0)) { + throw std::runtime_error("SymbolRans: not enough outlier workspace."); + } + + // The in-range primary stream (outliers now zero) reinterpreted as + // unsigned. + Array<1, Q, DeviceType> primary({n}, (Q *)original_data.data()); + rans.Compress(primary, rans_stream, queue_idx); + SIZE rans_bytes = rans_stream.shape(0); + + // Layout: signature | n | dict_size | outlier_count | rans_bytes | + // outlier_idx[] | outlier_val[] | rANS stream + SIZE byte_offset = 0; + advance_with_align(byte_offset, 7); + advance_with_align(byte_offset, 1); + advance_with_align(byte_offset, 1); + advance_with_align(byte_offset, 1); + advance_with_align(byte_offset, 1); + advance_with_align(byte_offset, outlier_count); + advance_with_align(byte_offset, outlier_count); + advance_with_align(byte_offset, rans_bytes); + + compressed_data.resize({byte_offset}, queue_idx); + SubArray<1, Byte, DeviceType> cs(compressed_data); + + SIZE n_s = n, dict_s = dict_size, oc_s = outlier_count, rb_s = rans_bytes; + byte_offset = 0; + SerializeArray(cs, signature, 7, byte_offset, queue_idx); + SerializeArray(cs, &n_s, 1, byte_offset, queue_idx); + SerializeArray(cs, &dict_s, 1, byte_offset, queue_idx); + SerializeArray(cs, &oc_s, 1, byte_offset, queue_idx); + SerializeArray(cs, &rb_s, 1, byte_offset, queue_idx); + SerializeArray(cs, outlier_idx_d.data(), outlier_count, + byte_offset, queue_idx); + SerializeArray(cs, outlier_val_d.data(), outlier_count, byte_offset, + queue_idx); + align_byte_offset(byte_offset); + MemoryManager::Copy1D(compressed_data.data() + byte_offset, + rans_stream.data(), rans_bytes, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + } + + void Serialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) {} + void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + } + + void Decompress(Array<1, Byte, DeviceType> &compressed_data, + Array<1, S, DeviceType> &decompressed_data, int queue_idx) { + SubArray<1, Byte, DeviceType> cs(compressed_data); + Byte *sig = nullptr; + SIZE n_s, dict_s, oc_s, rb_s; + SIZE *p_n = &n_s, *p_d = &dict_s, *p_oc = &oc_s, *p_rb = &rb_s; + SIZE byte_offset = 0; + DeserializeArray(cs, sig, 7, byte_offset, true, queue_idx); + DeserializeArray(cs, p_n, 1, byte_offset, false, queue_idx); + DeserializeArray(cs, p_d, 1, byte_offset, false, queue_idx); + DeserializeArray(cs, p_oc, 1, byte_offset, false, queue_idx); + DeserializeArray(cs, p_rb, 1, byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + SIZE n = n_s; + dict_size = dict_s; + outlier_count = oc_s; + SIZE rans_bytes = rb_s; + + ATOMIC_IDX *oidx_ptr = nullptr; + S *oval_ptr = nullptr; + DeserializeArray(cs, oidx_ptr, outlier_count, byte_offset, true, + queue_idx); + DeserializeArray(cs, oval_ptr, outlier_count, byte_offset, true, + queue_idx); + align_byte_offset(byte_offset); + Array<1, Byte, DeviceType> rans_alias({rans_bytes}, + compressed_data.data() + byte_offset); + + decompressed_data.resize({n}, queue_idx); + Array<1, Q, DeviceType> primary({n}, (Q *)decompressed_data.data()); + rans.Deserialize(rans_alias, queue_idx); + rans.Decompress(rans_alias, primary, queue_idx); + + // Scatter the outliers back into the decoded stream. + SubArray<1, ATOMIC_IDX, DeviceType> oidx({(SIZE)outlier_count}, oidx_ptr); + SubArray<1, S, DeviceType> oval({(SIZE)outlier_count}, oval_ptr); + DeviceLauncher::Execute( + OutlierSeparatorKernel( + SubArray(decompressed_data), dict_size, SubArray(outlier_count_d), + oidx, oval), + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + } + + static size_t EstimateMemoryFootprint(SIZE n, int dict_size, + double outlier_ratio) { + size_t size = 0; + size += (size_t)(n * outlier_ratio) * (sizeof(ATOMIC_IDX) + sizeof(S)); + size += (size_t)n * 2 + 16 * ((size_t)n / 2048 + 1); // rANS scratch ~2x + return size; + } + + bool initialized; + SIZE max_size = 0; + int dict_size = 0; + ATOMIC_IDX outlier_count = 0; + Byte signature[7] = {'M', 'G', 'X', 'S', 'R', 'A', 'N'}; + Byte *signature_verify; + + rans::Rans rans; + Array<1, Byte, DeviceType> rans_stream; + Array<1, ATOMIC_IDX, DeviceType> outlier_count_d; + Array<1, ATOMIC_IDX, DeviceType> outlier_idx_d; + Array<1, S, DeviceType> outlier_val_d; +}; + +} // namespace mgard_x +#endif diff --git a/include/mgard-x/Lossless/Zstd.hpp b/include/mgard-x/Lossless/Zstd.hpp index 2f44262d43..34e5e729d4 100644 --- a/include/mgard-x/Lossless/Zstd.hpp +++ b/include/mgard-x/Lossless/Zstd.hpp @@ -1,17 +1,23 @@ #ifndef MGARD_X_ZSTD_TEMPLATE_HPP #define MGARD_X_ZSTD_TEMPLATE_HPP +#include +#include #include +#include "../RuntimeX/Utilities/Exceptions.h" + namespace mgard_x { #define CHECK(cond, ...) \ do { \ if (!(cond)) { \ - fprintf(stderr, "%s:%d CHECK(%s) failed: ", __FILE__, __LINE__, #cond); \ - fprintf(stderr, "" __VA_ARGS__); \ - fprintf(stderr, "\n"); \ - exit(1); \ + char mgard_x_check_msg[256]; \ + std::snprintf(mgard_x_check_msg, sizeof(mgard_x_check_msg), \ + "CHECK(" #cond ") failed: " __VA_ARGS__); \ + throw mgard_x::ProcessingException(std::string(__FILE__) + ":" + \ + std::to_string(__LINE__) + " " + \ + mgard_x_check_msg); \ } \ } while (0) @@ -84,14 +90,13 @@ template class Zstd { actual_out_size + sizeof(size_t), queue_idx); DeviceRuntime::SyncQueue(queue_idx); - log::info("Zstd compression level: " + std::to_string(compressionLevel)); - log::info("Zstd compress ratio: " + - std::to_string((double)(input_count) / - (actual_out_size + sizeof(size_t)))); + log::dbg("Zstd compression level: " + std::to_string(compressionLevel)); + log::dbg("Zstd compress ratio: " + + std::to_string((double)(input_count) / + (actual_out_size + sizeof(size_t)))); if (log::level & log::TIME) { timer.end(); - timer.print("Zstd compress"); - timer.print_throughput("Zstd compress", input_count); + timer.print("Zstd compress", input_count); timer.clear(); } } @@ -135,8 +140,7 @@ template class Zstd { DeviceRuntime::SyncQueue(queue_idx); if (log::level & log::TIME) { timer.end(); - timer.print("Zstd decompress"); - timer.print_throughput("Zstd decompress", actual_out_count); + timer.print("Zstd decompress", actual_out_count); timer.clear(); } } diff --git a/include/mgard-x/Lossless/rANS/Rans.hpp b/include/mgard-x/Lossless/rANS/Rans.hpp new file mode 100644 index 0000000000..cae5b208ee --- /dev/null +++ b/include/mgard-x/Lossless/rANS/Rans.hpp @@ -0,0 +1,510 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_RANS_TEMPLATE_HPP +#define MGARD_X_RANS_TEMPLATE_HPP + +#include + +#include "../../RuntimeX/RuntimeX.h" +#include "../ParallelHuffman/Histogram.hpp" +#include "RansCommon.hpp" +#include "RansDecode.hpp" +#include "RansEncode.hpp" +#include "RansInterleaved.hpp" + +namespace mgard_x { +namespace rans { + +// Static rANS entropy coder over a symbol alphabet of arbitrary size (set at +// Resize): 256 for a byte stream, or dict_size for the Huffman-style primary +// quantized-symbol stream. Unlike Huffman it codes fractional bits, so the +// compressed size is the order-0 entropy of the symbol stream (no 1-bit/symbol +// floor and no integer-code-length rounding loss). Parallelism comes from +// splitting the input into independent segments, each its own rANS stream. +// +// Q is the symbol storage type (Byte, or the unsigned quantized type). Symbols +// must lie in [0, alphabet) and alphabet must be <= 65536 (slot table is +// uint16). The compressed stream is always bytes. +template class Rans { +public: + Rans() : initialized(false) {} + + // Choose M = 2^scale_bits with generous headroom over the alphabet so the + // normalized frequencies (each present symbol >= 1) keep enough precision: + // too small an M rounds the per-symbol probabilities coarsely and erases + // rANS's fractional-bit advantage over Huffman. Aim for ~16x the alphabet, + // capped at 2^16 (the max that keeps the 32-bit state valid with byte + // renormalization). + static uint32_t ScaleBitsFor(int alphabet) { + uint32_t sb = 12; + while (((uint32_t)1 << sb) < (uint32_t)alphabet * 16) { + sb++; + } + if (sb > 16) { + sb = 16; + } + return sb; + } + + void Resize(SIZE max_size, int alphabet, int queue_idx, + SIZE segment_size_override = 0, bool interleaved_mode = false) { + this->initialized = true; + this->max_size = max_size; + this->alphabet = alphabet; + this->scale_bits = ScaleBitsFor(alphabet); + this->interleaved = interleaved_mode; + this->segment_size = segment_size_override > 0 ? segment_size_override + : DEFAULT_SEGMENT_SIZE; + + // Two layouts share the same scratch/compact/offset machinery: + // - non-interleaved: one stream per lane (num_segments = blocks*NLANES), + // each stream's scratch holds up to segment_size symbols. + // - interleaved (warp-coalesced): one shared stream per block of NLANES + // lanes (num_segments = blocks), each block's scratch holds NLANES + // lanes' worth of bytes plus NLANES state flushes. + SIZE block_symbols = (SIZE)RANS_NLANES * segment_size; + SIZE max_blocks = (max_size + block_symbols - 1) / block_symbols; + SIZE max_segments; + if (interleaved) { + this->seg_capacity = + (IDX)RANS_NLANES * segment_size * 2 + (IDX)RANS_NLANES * 8 + 16; + max_segments = max_blocks; + } else { + this->seg_capacity = (IDX)segment_size * 2 + 16; + max_segments = max_blocks * (SIZE)RANS_NLANES; + } + if (max_segments == 0) { + max_segments = 1; + } + SIZE table_size = (SIZE)1 << scale_bits; + + freq32.resize({(SIZE)alphabet}, queue_idx); + freq_d.resize({(SIZE)alphabet}, queue_idx); + cum_d.resize({(SIZE)alphabet}, queue_idx); + esym_d.resize({(SIZE)alphabet}, queue_idx); + slot2sym_d.resize({table_size}, queue_idx); + seg_len_d.resize({max_segments}, queue_idx); + scratch.resize({(SIZE)(max_segments * seg_capacity)}, queue_idx); + + hcounts.resize(alphabet); + hfreq.resize(alphabet); + hcum.resize(alphabet); + hnorm.resize(alphabet); + hesym.resize(alphabet); + hslot.resize(table_size); + hseg_len.resize(max_segments); + hseg_off.resize(max_segments); + + MemoryManager::MallocHost(signature_verify, 7 * sizeof(char), + queue_idx); + } + + // Normalize the raw symbol counts so they sum to exactly M = 2^scale_bits, + // with every present symbol getting frequency >= 1, then build the cumulative + // table, the slot->symbol lookup, and the reciprocal-multiply encode tables. + void BuildTables(SIZE n, int queue_idx) { + uint32_t M = 1u << scale_bits; + + MemoryManager::Copy1D(hcounts.data(), freq32.data(), alphabet, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + uint32_t sum = 0; + int argmax = -1; + uint32_t max_freq = 0; + for (int s = 0; s < alphabet; s++) { + uint32_t f = 0; + if (hcounts[s] > 0) { + double scaled = (double)hcounts[s] * (double)M / (double)n; + f = (uint32_t)(scaled + 0.5); + if (f == 0) { + f = 1; + } + } + hnorm[s] = f; + sum += f; + if (f > max_freq) { + max_freq = f; + argmax = s; + } + } + + // Reconcile the rounded sum to exactly M by nudging the largest bins (never + // dropping a present symbol below 1). + while (sum > M) { + int best = -1; + uint32_t best_f = 1; + for (int s = 0; s < alphabet; s++) { + if (hnorm[s] > best_f) { + best_f = hnorm[s]; + best = s; + } + } + if (best < 0) { + break; + } + hnorm[best]--; + sum--; + } + while (sum < M) { + hnorm[argmax]++; + sum++; + } + + uint32_t c = 0; + for (int s = 0; s < alphabet; s++) { + hfreq[s] = hnorm[s]; + hcum[s] = c; + for (uint32_t k = 0; k < hnorm[s]; k++) { + hslot[c + k] = (uint16_t)s; + } + RansEncPacked &ep = hesym[s]; + if (hnorm[s] > 0) { + RansEncSymbol es; + RansEncSymbolInit(es, c, hnorm[s], scale_bits); + ep.x_max = es.x_max; + ep.rcp_freq = es.rcp_freq; + ep.bias = es.bias; + ep.cmpl_freq = (uint16_t)es.cmpl_freq; + ep.rcp_shift = (uint16_t)es.rcp_shift; + } else { + ep.x_max = 0; + ep.rcp_freq = 0; + ep.bias = 0; + ep.cmpl_freq = 0; + ep.rcp_shift = 0; + } + c += hnorm[s]; + } + + freq_d.load(hfreq.data(), 0, queue_idx); + cum_d.load(hcum.data(), 0, queue_idx); + slot2sym_d.load(hslot.data(), 0, queue_idx); + esym_d.load(hesym.data(), 0, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + } + + bool Compress(Array<1, Q, DeviceType> &input_data, + Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE n = input_data.shape(0); + SIZE block_symbols = (SIZE)RANS_NLANES * segment_size; + SIZE num_blocks = (n + block_symbols - 1) / block_symbols; + // The compaction unit ("segment") is one shared stream per block when + // interleaved, else one stream per lane. + SIZE num_segments = + interleaved ? num_blocks : num_blocks * (SIZE)RANS_NLANES; + if (n == 0) { + num_segments = 0; + } + + if (n > 0) { + MemoryManager::Memset1D(freq32.data(), alphabet, 0, + queue_idx); + Histogram(SubArray(input_data), SubArray(freq32), + n, alphabet, queue_idx); + BuildTables(n, queue_idx); + } + + if (num_segments > 0) { + seg_len_d.resize({num_segments}, queue_idx); + if (interleaved) { + if constexpr ((int)SubGroup::size() % (int)RANS_NLANES == + 0) { + // Warp-cooperative coalesced encode for subgroups that are a multiple + // of RANS_NLANES: a 32-lane CUDA warp handles one logical block, a + // 64-lane CDNA wavefront handles two. The bitstream is identical + // (always 32 logical lanes), so it stays cross-backend compatible. + DeviceLauncher::Execute( + InterleavedEncodeWarpKernel( + SubArray(input_data), n, segment_size, num_blocks, + SubArray(esym_d), seg_capacity, SubArray(scratch), + SubArray(seg_len_d)), + queue_idx); + } else { + // Portable sequential reference (size-1 subgroup backends). + DeviceLauncher::Execute( + InterleavedEncodeKernel( + SubArray(input_data), n, segment_size, num_blocks, + SubArray(esym_d), seg_capacity, SubArray(scratch), + SubArray(seg_len_d)), + queue_idx); + } + } else { + DeviceLauncher::Execute( + EncodeKernel(SubArray(input_data), n, segment_size, + num_segments, SubArray(esym_d), + seg_capacity, SubArray(scratch), + SubArray(seg_len_d)), + queue_idx); + } + + MemoryManager::Copy1D(hseg_len.data(), seg_len_d.data(), + num_segments, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + } + + uint64_t stream_bytes = 0; + for (SIZE p = 0; p < num_segments; p++) { + hseg_off[p] = (uint32_t)stream_bytes; + stream_bytes += hseg_len[p]; + } + + SIZE byte_offset = 0; + advance_with_align(byte_offset, 7); // signature + advance_with_align(byte_offset, 1); // scale_bits + advance_with_align(byte_offset, 1); // alphabet + advance_with_align(byte_offset, 1); // original_length + advance_with_align(byte_offset, 1); // segment_size + advance_with_align(byte_offset, 1); // interleaved + advance_with_align(byte_offset, 1); // num_segments + advance_with_align(byte_offset, 1); // stream_bytes + advance_with_align(byte_offset, alphabet); // norm freq + advance_with_align(byte_offset, num_segments); // seg offsets + advance_with_align(byte_offset, (SIZE)stream_bytes); // stream + + compressed_data.resize({byte_offset}, queue_idx); + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + + SIZE scale_bits_s = scale_bits, alphabet_s = alphabet, + original_length_s = n, segment_size_s = segment_size, + num_segments_s = num_segments, stream_bytes_s = stream_bytes, + interleaved_s = interleaved ? 1 : 0; + + byte_offset = 0; + SerializeArray(compressed_subarray, signature, 7, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &scale_bits_s, 1, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &alphabet_s, 1, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &original_length_s, 1, + byte_offset, queue_idx); + SerializeArray(compressed_subarray, &segment_size_s, 1, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &interleaved_s, 1, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &num_segments_s, 1, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &stream_bytes_s, 1, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, hnorm.data(), alphabet, + byte_offset, queue_idx); + + align_byte_offset(byte_offset); + SubArray<1, uint32_t, DeviceType> seg_offset_sub( + {num_segments}, (uint32_t *)(compressed_data.data() + byte_offset)); + SerializeArray(compressed_subarray, hseg_off.data(), num_segments, + byte_offset, queue_idx); + + align_byte_offset(byte_offset); + SubArray<1, Byte, DeviceType> stream_sub( + {(SIZE)stream_bytes}, (Byte *)(compressed_data.data() + byte_offset)); + + if (num_segments > 0) { + DeviceLauncher::Execute( + CompactKernel(SubArray(scratch), SubArray(seg_len_d), + seg_offset_sub, num_segments, seg_capacity, + stream_sub), + queue_idx); + } + DeviceRuntime::SyncQueue(queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + log::info( + "rANS compression ratio: " + std::to_string(n * sizeof(Q)) + "/" + + std::to_string(compressed_data.shape(0)) + " (" + + std::to_string((double)n * sizeof(Q) / compressed_data.shape(0)) + + ")"); + timer.end(); + timer.print("rANS compress", n * sizeof(Q)); + timer.clear(); + } + return true; + } + + void Serialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) {} + + bool Verify(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + SubArray compressed_subarray(compressed_data); + SIZE byte_offset = 0; + DeserializeArray(compressed_subarray, signature_verify, 7, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + for (int i = 0; i < 7; i++) { + if (signature[i] != signature_verify[i]) { + return false; + } + } + return true; + } + + void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + if (!Verify(compressed_data, queue_idx)) { + throw std::runtime_error("rANS signature mismatch."); + } + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + Byte *signature_ptr = nullptr; + SIZE scale_bits_s, alphabet_s, original_length_s, segment_size_s, + num_segments_s, stream_bytes_s, interleaved_s; + SIZE *p_scale = &scale_bits_s, *p_alpha = &alphabet_s, + *p_len = &original_length_s, *p_seg = &segment_size_s, + *p_num = &num_segments_s, *p_stream = &stream_bytes_s, + *p_il = &interleaved_s; + + SIZE byte_offset = 0; + DeserializeArray(compressed_subarray, signature_ptr, 7, byte_offset, + true, queue_idx); + DeserializeArray(compressed_subarray, p_scale, 1, byte_offset, false, + queue_idx); + DeserializeArray(compressed_subarray, p_alpha, 1, byte_offset, false, + queue_idx); + DeserializeArray(compressed_subarray, p_len, 1, byte_offset, false, + queue_idx); + DeserializeArray(compressed_subarray, p_seg, 1, byte_offset, false, + queue_idx); + DeserializeArray(compressed_subarray, p_il, 1, byte_offset, false, + queue_idx); + DeserializeArray(compressed_subarray, p_num, 1, byte_offset, false, + queue_idx); + DeserializeArray(compressed_subarray, p_stream, 1, byte_offset, false, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + scale_bits = scale_bits_s; + alphabet = alphabet_s; + original_length = original_length_s; + segment_size = segment_size_s; + num_segments = num_segments_s; + stream_bytes = stream_bytes_s; + interleaved = interleaved_s != 0; + + if ((int)hnorm.size() < alphabet) { + hnorm.resize(alphabet); + hfreq.resize(alphabet); + hcum.resize(alphabet); + hslot.resize((size_t)1 << scale_bits); + } + uint16_t *hnorm_ptr = hnorm.data(); + DeserializeArray(compressed_subarray, hnorm_ptr, alphabet, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + seg_offset_ptr = nullptr; + DeserializeArray(compressed_subarray, seg_offset_ptr, + num_segments, byte_offset, true, queue_idx); + stream_ptr = nullptr; + DeserializeArray(compressed_subarray, stream_ptr, stream_bytes, + byte_offset, true, queue_idx); + } + + void Decompress(Array<1, Byte, DeviceType> &compressed_data, + Array<1, Q, DeviceType> &decompressed_data, int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + decompressed_data.resize({(SIZE)original_length}, queue_idx); + + if (num_segments > 0) { + uint32_t c = 0; + for (int s = 0; s < alphabet; s++) { + hfreq[s] = hnorm[s]; + hcum[s] = c; + for (uint32_t k = 0; k < hnorm[s]; k++) { + hslot[c + k] = (uint16_t)s; + } + c += hnorm[s]; + } + if ((int)freq_d.shape(0) < alphabet) { + freq_d.resize({(SIZE)alphabet}, queue_idx); + cum_d.resize({(SIZE)alphabet}, queue_idx); + slot2sym_d.resize({(SIZE)1 << scale_bits}, queue_idx); + } + freq_d.load(hfreq.data(), 0, queue_idx); + cum_d.load(hcum.data(), 0, queue_idx); + slot2sym_d.load(hslot.data(), 0, queue_idx); + + SubArray<1, uint32_t, DeviceType> seg_offset_sub({(SIZE)num_segments}, + seg_offset_ptr); + SubArray<1, Byte, DeviceType> stream_sub({(SIZE)stream_bytes}, + stream_ptr); + + if (interleaved) { + // num_segments == num_blocks here. + DeviceLauncher::Execute( + InterleavedDecodeKernel( + stream_sub, seg_offset_sub, original_length, segment_size, + num_segments, SubArray(freq_d), SubArray(cum_d), + SubArray(slot2sym_d), scale_bits, SubArray(decompressed_data)), + queue_idx); + } else { + DeviceLauncher::Execute( + DecodeKernel( + stream_sub, seg_offset_sub, original_length, segment_size, + num_segments, SubArray(freq_d), SubArray(cum_d), + SubArray(slot2sym_d), scale_bits, SubArray(decompressed_data)), + queue_idx); + } + DeviceRuntime::SyncQueue(queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("rANS decompress", original_length * sizeof(Q)); + timer.clear(); + } + } + + static constexpr SIZE DEFAULT_SEGMENT_SIZE = 2048; + + bool initialized; + bool interleaved = false; + SIZE max_size = 0; + int alphabet = 256; + uint32_t scale_bits = 12; + SIZE segment_size = DEFAULT_SEGMENT_SIZE; + IDX seg_capacity = 0; + SIZE original_length = 0; + SIZE num_segments = 0; + SIZE stream_bytes = 0; + + uint32_t *seg_offset_ptr = nullptr; + Byte *stream_ptr = nullptr; + Byte signature[7] = {'M', 'G', 'X', 'R', 'A', 'N', 'S'}; + Byte *signature_verify; + + Array<1, uint32_t, DeviceType> freq32; + Array<1, uint32_t, DeviceType> freq_d; + Array<1, uint32_t, DeviceType> cum_d; + Array<1, RansEncPacked, DeviceType> esym_d; + Array<1, uint16_t, DeviceType> slot2sym_d; + Array<1, uint32_t, DeviceType> seg_len_d; + Array<1, Byte, DeviceType> scratch; + + std::vector hcounts; + std::vector hfreq; + std::vector hcum; + std::vector hnorm; + std::vector hesym; + std::vector hslot; + std::vector hseg_len; + std::vector hseg_off; +}; + +} // namespace rans +} // namespace mgard_x +#endif diff --git a/include/mgard-x/Lossless/rANS/RansCommon.hpp b/include/mgard-x/Lossless/rANS/RansCommon.hpp new file mode 100644 index 0000000000..6d40bc26cf --- /dev/null +++ b/include/mgard-x/Lossless/rANS/RansCommon.hpp @@ -0,0 +1,179 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_RANS_COMMON_TEMPLATE_HPP +#define MGARD_X_RANS_COMMON_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x { +namespace rans { + +// Static range-ANS (rANS) over a byte alphabet (256 symbols), 32-bit state, +// byte-wise renormalization. The normalization interval is [RANS_L, RANS_L*256) +// so the state always fits in uint32 (RANS_L*256 == 2^31). Encoding writes +// bytes back-to-front (decreasing pointer); decoding reads them front-to-back. +// These primitives follow the well-known ryg_rans construction. + +static constexpr uint32_t RANS_L = 1u << 23; // lower bound of normalization +static constexpr int RANS_ALPHABET = 256; + +// Warp-interleaving factor. Streams are grouped into blocks of RANS_NLANES +// consecutive streams that interleave over a contiguous super-segment, so that +// at each step the RANS_NLANES streams (= consecutive threads in a warp) touch +// RANS_NLANES consecutive input/output positions -> coalesced memory access, +// with no extra per-stream metadata (same stream/flush/offset count as the +// non-interleaved layout). Stream p covers symbol positions +// (p / RANS_NLANES) * (RANS_NLANES * S) + (p % RANS_NLANES) + j * RANS_NLANES +// for j = 0..count_p-1, where S = symbols-per-stream (segment_size). +static constexpr IDX RANS_NLANES = 32; + +// Number of symbol positions assigned to interleaved stream p, given total n +// and S symbols-per-stream. Clamped to S so a stream never steals the next +// block's symbols. +MGARDX_CONT_EXEC IDX RansStreamBase(IDX p, IDX S) { + return (p / RANS_NLANES) * (RANS_NLANES * S) + (p % RANS_NLANES); +} +MGARDX_CONT_EXEC IDX RansStreamCount(IDX p, IDX S, IDX n) { + IDX base = RansStreamBase(p, S); + if (base >= n) { + return 0; + } + IDX count = (n - 1 - base) / RANS_NLANES + 1; + return count > S ? S : count; +} + +// Per-symbol precomputed encode constants (Alverson reciprocal, ryg_rans). They +// turn the encode step's integer division (x / freq, x % freq) into a +// multiply-high + shift, which is far cheaper on the GPU and is the encode +// bottleneck fix. Built once per frequency table on the host (see Rans.hpp). +// x_max = ((RANS_L >> scale_bits) << 8) * freq (renorm threshold) +// rcp_freq = fixed-point reciprocal of freq +// rcp_shift = reciprocal shift +// bias = additive bias +// cmpl_freq = (1 << scale_bits) - freq +struct RansEncSymbol { + uint32_t x_max; + uint32_t rcp_freq; + uint32_t bias; + uint32_t cmpl_freq; + uint32_t rcp_shift; +}; + +// Packed 16-byte encode-table entry so the hot encode loop fetches all +// per-symbol constants in a single (uint4) load instead of 5 separate array +// reads. The encode loop is compute/latency-bound, so cutting table loads +// 5x->1x is the main encode speedup. Internal to encode only (not part of the +// bitstream). +struct alignas(16) RansEncPacked { + uint32_t x_max; + uint32_t rcp_freq; + uint32_t bias; + uint16_t cmpl_freq; + uint16_t rcp_shift; +}; + +// Host-side: fill the precomputed encode constants for one symbol. Mirrors +// ryg_rans RansEncSymbolInit; produces a bitstream identical to the division +// form, so the decoder is unchanged. +MGARDX_CONT void RansEncSymbolInit(RansEncSymbol &s, uint32_t cum, + uint32_t freq, uint32_t scale_bits) { + s.x_max = ((RANS_L >> scale_bits) << 8) * freq; + s.cmpl_freq = (1u << scale_bits) - freq; + if (freq < 2) { + s.rcp_freq = ~0u; + s.rcp_shift = 0; + s.bias = cum + (1u << scale_bits) - 1; + } else { + uint32_t shift = 0; + while (freq > (1u << shift)) { + shift++; + } + s.rcp_freq = (uint32_t)(((1ull << (shift + 31)) + freq - 1) / freq); + s.rcp_shift = shift - 1; + s.bias = cum; + } +} + +// Split form of the encode step for warp-cooperative encoding, where the +// renorm bytes of all lanes in a step must be gathered and written coalesced +// (so they cannot be written inline). RansEncCollect reduces x and returns its +// renorm bytes (LSB first) in buf; RansEncApply then folds in the symbol. +MGARDX_CONT_EXEC int RansEncCollect(uint32_t &x, uint32_t x_max, Byte buf[4]) { + int nb = 0; + while (x >= x_max) { + buf[nb] = (Byte)(x & 0xFFu); + nb++; + x >>= 8; + } + return nb; +} +MGARDX_CONT_EXEC uint32_t RansEncApply(uint32_t x, uint32_t rcp_freq, + uint32_t bias, uint32_t cmpl_freq, + uint32_t rcp_shift) { + uint32_t q = (uint32_t)(((uint64_t)x * rcp_freq) >> 32); + return x + bias + (q >> rcp_shift) * cmpl_freq; +} + +// Encode one symbol with the precomputed constants: renormalize by flushing low +// bytes (back-to-front via the decreasing cursor ptr), then advance the state +// via reciprocal multiply instead of division. +template +MGARDX_EXEC uint32_t RansEncPut(uint32_t x, uint32_t x_max, uint32_t rcp_freq, + uint32_t bias, uint32_t cmpl_freq, + uint32_t rcp_shift, + SubArray<1, Byte, DeviceType> &out, IDX &ptr) { + if (x >= x_max) { + do { + --ptr; + *out(ptr) = (Byte)(x & 0xFFu); + x >>= 8; + } while (x >= x_max); + } + uint32_t q = (uint32_t)(((uint64_t)x * rcp_freq) >> 32); + return x + bias + (q >> rcp_shift) * cmpl_freq; +} + +// Flush the final 32-bit state as 4 little-endian bytes at the front of the +// (back-to-front) segment buffer. +template +MGARDX_EXEC void RansEncFlush(uint32_t x, SubArray<1, Byte, DeviceType> &out, + IDX &ptr) { + ptr -= 4; + *out(ptr + 0) = (Byte)(x >> 0); + *out(ptr + 1) = (Byte)(x >> 8); + *out(ptr + 2) = (Byte)(x >> 16); + *out(ptr + 3) = (Byte)(x >> 24); +} + +// Read the 4-byte little-endian state that RansEncFlush wrote. +template +MGARDX_EXEC uint32_t RansDecInit(SubArray<1, Byte, DeviceType> &in, IDX &rp) { + uint32_t x = (uint32_t)(*in(rp + 0)) | ((uint32_t)(*in(rp + 1)) << 8) | + ((uint32_t)(*in(rp + 2)) << 16) | + ((uint32_t)(*in(rp + 3)) << 24); + rp += 4; + return x; +} + +// Given the current slot (= x & mask) and the decoded symbol's (freq, cum), +// advance the state and renormalize by pulling bytes front-to-back. +template +MGARDX_EXEC uint32_t RansDecAdvance(uint32_t x, uint32_t freq, uint32_t cum, + uint32_t slot, uint32_t scale_bits, + SubArray<1, Byte, DeviceType> &in, + IDX &rp) { + x = freq * (x >> scale_bits) + slot - cum; + while (x < RANS_L) { + x = (x << 8) | (uint32_t)(*in(rp)); + ++rp; + } + return x; +} + +} // namespace rans +} // namespace mgard_x +#endif diff --git a/include/mgard-x/Lossless/rANS/RansDecode.hpp b/include/mgard-x/Lossless/rANS/RansDecode.hpp new file mode 100644 index 0000000000..c6ad51f1b4 --- /dev/null +++ b/include/mgard-x/Lossless/rANS/RansDecode.hpp @@ -0,0 +1,133 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_RANS_DECODE_TEMPLATE_HPP +#define MGARD_X_RANS_DECODE_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" +#include "RansCommon.hpp" + +namespace mgard_x { +namespace rans { + +// Decode each segment independently: thread p reads its byte stream starting at +// seg_offset[p], reconstructs the state, then walks symbols forward (the mirror +// of the reverse encode), looking each up via the slot->symbol table and +// scattering into output[p*seg_size, ...). +template +class DecodeFunctor : public Functor { +public: + MGARDX_CONT DecodeFunctor() {} + MGARDX_CONT DecodeFunctor(SubArray<1, Byte, DeviceType> stream, + SubArray<1, uint32_t, DeviceType> seg_offset, + SIZE n, SIZE segment_size, SIZE num_segments, + SubArray<1, uint32_t, DeviceType> freq, + SubArray<1, uint32_t, DeviceType> cum, + SubArray<1, uint16_t, DeviceType> slot2sym, + uint32_t scale_bits, + SubArray<1, Q, DeviceType> output) + : stream(stream), seg_offset(seg_offset), n(n), + segment_size(segment_size), num_segments(num_segments), freq(freq), + cum(cum), slot2sym(slot2sym), scale_bits(scale_bits), output(output) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + uint32_t mask = (1u << scale_bits) - 1; + + for (IDX p = start; p < num_segments; p += grid_size) { + // Mirror the interleaved encode mapping: stream p produces symbols for + // positions base_pos + j*RANS_NLANES (consecutive streams -> consecutive + // output positions -> coalesced stores). + IDX base_pos = RansStreamBase(p, segment_size); + IDX count = RansStreamCount(p, segment_size, n); + + IDX rp = (IDX)(*seg_offset(p)); + uint32_t x = RansDecInit(stream, rp); + + for (IDX j = 0; j < count; j++) { + uint32_t slot = x & mask; + uint32_t s = (uint32_t)(*slot2sym(slot)); + *output(base_pos + j * RANS_NLANES) = (Q)s; + x = RansDecAdvance(x, *freq(s), *cum(s), slot, scale_bits, + stream, rp); + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Byte, DeviceType> stream; + SubArray<1, uint32_t, DeviceType> seg_offset; + SIZE n; + SIZE segment_size; + SIZE num_segments; + SubArray<1, uint32_t, DeviceType> freq; + SubArray<1, uint32_t, DeviceType> cum; + SubArray<1, uint16_t, DeviceType> slot2sym; + uint32_t scale_bits; + SubArray<1, Q, DeviceType> output; +}; + +template class DecodeKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "rans decode"; + MGARDX_CONT + DecodeKernel(SubArray<1, Byte, DeviceType> stream, + SubArray<1, uint32_t, DeviceType> seg_offset, SIZE n, + SIZE segment_size, SIZE num_segments, + SubArray<1, uint32_t, DeviceType> freq, + SubArray<1, uint32_t, DeviceType> cum, + SubArray<1, uint16_t, DeviceType> slot2sym, uint32_t scale_bits, + SubArray<1, Q, DeviceType> output) + : stream(stream), seg_offset(seg_offset), n(n), + segment_size(segment_size), num_segments(num_segments), freq(freq), + cum(cum), slot2sym(slot2sym), scale_bits(scale_bits), output(output) {} + + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = DecodeFunctor; + FunctorType functor(stream, seg_offset, n, segment_size, num_segments, freq, + cum, slot2sym, scale_bits, output); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + // One thread per segment, full grid (no repeat_factor cap) to maximize + // occupancy for the serial per-segment decode chain. + gridx = (num_segments - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), gridx); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, Byte, DeviceType> stream; + SubArray<1, uint32_t, DeviceType> seg_offset; + SIZE n; + SIZE segment_size; + SIZE num_segments; + SubArray<1, uint32_t, DeviceType> freq; + SubArray<1, uint32_t, DeviceType> cum; + SubArray<1, uint16_t, DeviceType> slot2sym; + uint32_t scale_bits; + SubArray<1, Q, DeviceType> output; +}; + +} // namespace rans +} // namespace mgard_x +#endif diff --git a/include/mgard-x/Lossless/rANS/RansEncode.hpp b/include/mgard-x/Lossless/rANS/RansEncode.hpp new file mode 100644 index 0000000000..c2ce6f4b73 --- /dev/null +++ b/include/mgard-x/Lossless/rANS/RansEncode.hpp @@ -0,0 +1,207 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_RANS_ENCODE_TEMPLATE_HPP +#define MGARD_X_RANS_ENCODE_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" +#include "RansCommon.hpp" + +namespace mgard_x { +namespace rans { + +// One independent rANS stream per segment (interleaving across threads gives +// the parallelism). Thread p encodes input[p*seg_size, ...) in reverse into a +// private scratch region [p*seg_capacity, (p+1)*seg_capacity), filling it +// back-to-front, then records the produced byte length. Bytes live at the high +// end of the region: [seg_capacity - seg_len, seg_capacity). +template +class EncodeFunctor : public Functor { +public: + MGARDX_CONT EncodeFunctor() {} + MGARDX_CONT + EncodeFunctor(SubArray<1, Q, DeviceType> input, SIZE n, SIZE segment_size, + SIZE num_segments, SubArray<1, RansEncPacked, DeviceType> esym, + IDX seg_capacity, SubArray<1, Byte, DeviceType> scratch, + SubArray<1, uint32_t, DeviceType> seg_len) + : input(input), n(n), segment_size(segment_size), + num_segments(num_segments), esym(esym), seg_capacity(seg_capacity), + scratch(scratch), seg_len(seg_len) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + for (IDX p = start; p < num_segments; p += grid_size) { + // Interleaved symbol mapping: consecutive streams (= consecutive warp + // lanes) read consecutive positions, so the input loads coalesce. + IDX base_pos = RansStreamBase(p, segment_size); + IDX count = RansStreamCount(p, segment_size, n); + + IDX base = p * seg_capacity; + IDX ptr = base + seg_capacity; // cursor, fills downward + + uint32_t x = RANS_L; + for (IDX j = count; j > 0; j--) { + IDX pos = base_pos + (j - 1) * RANS_NLANES; + uint32_t s = (uint32_t)(*input(pos)); + RansEncPacked e = *esym(s); // single 16-byte load of all constants + x = RansEncPut(x, e.x_max, e.rcp_freq, e.bias, + (uint32_t)e.cmpl_freq, (uint32_t)e.rcp_shift, + scratch, ptr); + } + RansEncFlush(x, scratch, ptr); + + *seg_len(p) = (uint32_t)(base + seg_capacity - ptr); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Q, DeviceType> input; + SIZE n; + SIZE segment_size; + SIZE num_segments; + SubArray<1, RansEncPacked, DeviceType> esym; + IDX seg_capacity; + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, uint32_t, DeviceType> seg_len; +}; + +template class EncodeKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "rans encode"; + MGARDX_CONT + EncodeKernel(SubArray<1, Q, DeviceType> input, SIZE n, SIZE segment_size, + SIZE num_segments, SubArray<1, RansEncPacked, DeviceType> esym, + IDX seg_capacity, SubArray<1, Byte, DeviceType> scratch, + SubArray<1, uint32_t, DeviceType> seg_len) + : input(input), n(n), segment_size(segment_size), + num_segments(num_segments), esym(esym), seg_capacity(seg_capacity), + scratch(scratch), seg_len(seg_len) {} + + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = EncodeFunctor; + FunctorType functor(input, n, segment_size, num_segments, esym, + seg_capacity, scratch, seg_len); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + // One thread per segment, full grid (no repeat_factor cap): the per-thread + // rANS loop is a long serial dependent chain, so we want maximum occupancy + // to hide its latency rather than fewer threads each doing more segments. + gridx = (num_segments - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), gridx); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, Q, DeviceType> input; + SIZE n; + SIZE segment_size; + SIZE num_segments; + SubArray<1, RansEncPacked, DeviceType> esym; + IDX seg_capacity; + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, uint32_t, DeviceType> seg_len; +}; + +// Copy each segment's bytes out of its scratch region (back-to-front layout) +// into a single contiguous stream at seg_offset[p]. One block per segment. +template +class CompactFunctor : public Functor { +public: + MGARDX_CONT CompactFunctor() {} + MGARDX_CONT CompactFunctor(SubArray<1, Byte, DeviceType> scratch, + SubArray<1, uint32_t, DeviceType> seg_len, + SubArray<1, uint32_t, DeviceType> seg_offset, + SIZE num_segments, IDX seg_capacity, + SubArray<1, Byte, DeviceType> stream) + : scratch(scratch), seg_len(seg_len), seg_offset(seg_offset), + num_segments(num_segments), seg_capacity(seg_capacity), stream(stream) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + for (IDX p = FunctorBase::GetBlockIdX(); p < num_segments; + p += FunctorBase::GetGridDimX()) { + uint32_t len = *seg_len(p); + IDX src = p * seg_capacity + (seg_capacity - (IDX)len); + IDX dst = (IDX)(*seg_offset(p)); + for (IDX j = FunctorBase::GetThreadIdX(); j < len; + j += FunctorBase::GetBlockDimX()) { + *stream(dst + j) = *scratch(src + j); + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, uint32_t, DeviceType> seg_len; + SubArray<1, uint32_t, DeviceType> seg_offset; + SIZE num_segments; + IDX seg_capacity; + SubArray<1, Byte, DeviceType> stream; +}; + +template class CompactKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "rans compact"; + MGARDX_CONT + CompactKernel(SubArray<1, Byte, DeviceType> scratch, + SubArray<1, uint32_t, DeviceType> seg_len, + SubArray<1, uint32_t, DeviceType> seg_offset, SIZE num_segments, + IDX seg_capacity, SubArray<1, Byte, DeviceType> stream) + : scratch(scratch), seg_len(seg_len), seg_offset(seg_offset), + num_segments(num_segments), seg_capacity(seg_capacity), stream(stream) { + } + + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = CompactFunctor; + FunctorType functor(scratch, seg_len, seg_offset, num_segments, + seg_capacity, stream); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + (SIZE)num_segments); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, uint32_t, DeviceType> seg_len; + SubArray<1, uint32_t, DeviceType> seg_offset; + SIZE num_segments; + IDX seg_capacity; + SubArray<1, Byte, DeviceType> stream; +}; + +} // namespace rans +} // namespace mgard_x +#endif diff --git a/include/mgard-x/Lossless/rANS/RansInterleaved.hpp b/include/mgard-x/Lossless/rANS/RansInterleaved.hpp new file mode 100644 index 0000000000..f7a7dc2a22 --- /dev/null +++ b/include/mgard-x/Lossless/rANS/RansInterleaved.hpp @@ -0,0 +1,448 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_RANS_INTERLEAVED_TEMPLATE_HPP +#define MGARD_X_RANS_INTERLEAVED_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" +#include "RansCommon.hpp" + +namespace mgard_x { +namespace rans { + +// Shared-stream interleaved rANS: a "block" of RANS_NLANES independent rANS +// states shares ONE byte stream, with the states' renorm bytes intermixed in +// LIFO emission order (standard interleaved rANS, generalized to NLANES +// states). This is the layout required to coalesce the encode byte writes (the +// CUDA warp-cooperative kernels below process one block per warp and write the +// per-step bytes coalesced). The SEQUENTIAL reference functors here (one thread +// per block) define the canonical byte order and are the correctness oracle / +// portable fallback; they produce a bitstream identical to the warp kernels. +// +// Block b, lane L owns symbol positions b*NLANES*S + L + j*NLANES, j < count, +// where S = symbols-per-lane (segment_size). Encode order: j high->low, then +// L = 0..NLANES-1; states flushed L = 0..NLANES-1 at the end (lowest address). +// Decode mirrors: init states L = NLANES-1..0, then j low->high, L = +// NLANES-1..0. + +template +MGARDX_EXEC IDX RansBlockLaneBase(IDX b, IDX L, IDX S) { + return b * (RANS_NLANES * S) + L; +} +template +MGARDX_EXEC IDX RansBlockLaneCount(IDX b, IDX L, IDX S, IDX n) { + IDX base = b * (RANS_NLANES * S) + L; + if (base >= n) { + return 0; + } + IDX count = (n - 1 - base) / RANS_NLANES + 1; + return count > S ? S : count; +} + +// --------------------------------------------------------------------------- +// Sequential reference encode: one thread per block. +// --------------------------------------------------------------------------- +template +class InterleavedEncodeFunctor : public Functor { +public: + MGARDX_CONT InterleavedEncodeFunctor() {} + MGARDX_CONT InterleavedEncodeFunctor( + SubArray<1, Q, DeviceType> input, SIZE n, SIZE segment_size, + SIZE num_blocks, SubArray<1, RansEncPacked, DeviceType> esym, + IDX block_capacity, SubArray<1, Byte, DeviceType> scratch, + SubArray<1, uint32_t, DeviceType> seg_len) + : input(input), n(n), segment_size(segment_size), num_blocks(num_blocks), + esym(esym), block_capacity(block_capacity), scratch(scratch), + seg_len(seg_len) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + for (IDX b = start; b < num_blocks; b += grid_size) { + uint32_t x[RANS_NLANES]; + for (IDX L = 0; L < RANS_NLANES; L++) { + x[L] = RANS_L; + } + // Lane 0 starts earliest, so it has the most symbols => maxcount. + IDX maxcount = RansBlockLaneCount(b, 0, segment_size, n); + + IDX ptr = b * block_capacity + block_capacity; // cursor, fills downward + + for (IDX jj = maxcount; jj > 0; jj--) { + IDX j = jj - 1; + for (IDX L = 0; L < RANS_NLANES; L++) { + IDX base = b * (RANS_NLANES * segment_size) + L; + IDX count = RansBlockLaneCount(b, L, segment_size, n); + if (j < count) { + IDX pos = base + j * RANS_NLANES; + uint32_t s = (uint32_t)(*input(pos)); + RansEncPacked e = *esym(s); + x[L] = RansEncPut(x[L], e.x_max, e.rcp_freq, e.bias, + (uint32_t)e.cmpl_freq, + (uint32_t)e.rcp_shift, scratch, ptr); + } + } + } + for (IDX L = 0; L < RANS_NLANES; L++) { + RansEncFlush(x[L], scratch, ptr); + } + *seg_len(b) = (uint32_t)(b * block_capacity + block_capacity - ptr); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Q, DeviceType> input; + SIZE n; + SIZE segment_size; + SIZE num_blocks; + SubArray<1, RansEncPacked, DeviceType> esym; + IDX block_capacity; + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, uint32_t, DeviceType> seg_len; +}; + +template +class InterleavedEncodeKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "rans interleaved encode"; + MGARDX_CONT InterleavedEncodeKernel( + SubArray<1, Q, DeviceType> input, SIZE n, SIZE segment_size, + SIZE num_blocks, SubArray<1, RansEncPacked, DeviceType> esym, + IDX block_capacity, SubArray<1, Byte, DeviceType> scratch, + SubArray<1, uint32_t, DeviceType> seg_len) + : input(input), n(n), segment_size(segment_size), num_blocks(num_blocks), + esym(esym), block_capacity(block_capacity), scratch(scratch), + seg_len(seg_len) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = InterleavedEncodeFunctor; + FunctorType functor(input, n, segment_size, num_blocks, esym, + block_capacity, scratch, seg_len); + SIZE tbx = 256, gridx; + gridx = (num_blocks - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), gridx); + return Task(functor, 1, 1, gridx, 1, 1, tbx, 0, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, Q, DeviceType> input; + SIZE n; + SIZE segment_size; + SIZE num_blocks; + SubArray<1, RansEncPacked, DeviceType> esym; + IDX block_capacity; + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, uint32_t, DeviceType> seg_len; +}; + +// --------------------------------------------------------------------------- +// Sequential reference decode: one thread per block. +// --------------------------------------------------------------------------- +template +class InterleavedDecodeFunctor : public Functor { +public: + MGARDX_CONT InterleavedDecodeFunctor() {} + MGARDX_CONT InterleavedDecodeFunctor( + SubArray<1, Byte, DeviceType> stream, + SubArray<1, uint32_t, DeviceType> seg_offset, SIZE n, SIZE segment_size, + SIZE num_blocks, SubArray<1, uint32_t, DeviceType> freq, + SubArray<1, uint32_t, DeviceType> cum, + SubArray<1, uint16_t, DeviceType> slot2sym, uint32_t scale_bits, + SubArray<1, Q, DeviceType> output) + : stream(stream), seg_offset(seg_offset), n(n), + segment_size(segment_size), num_blocks(num_blocks), freq(freq), + cum(cum), slot2sym(slot2sym), scale_bits(scale_bits), output(output) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + uint32_t mask = (1u << scale_bits) - 1; + + for (IDX b = start; b < num_blocks; b += grid_size) { + IDX rp = (IDX)(*seg_offset(b)); + uint32_t x[RANS_NLANES]; + // Init states in reverse of the flush order (flush wrote L=0..NLANES-1, + // so the lowest address is L=NLANES-1, read first going forward). + for (IDX LL = RANS_NLANES; LL > 0; LL--) { + x[LL - 1] = RansDecInit(stream, rp); + } + + IDX maxcount = RansBlockLaneCount(b, 0, segment_size, n); + for (IDX j = 0; j < maxcount; j++) { + for (IDX LL = RANS_NLANES; LL > 0; LL--) { + IDX L = LL - 1; + IDX count = RansBlockLaneCount(b, L, segment_size, n); + if (j < count) { + uint32_t slot = x[L] & mask; + uint32_t s = (uint32_t)(*slot2sym(slot)); + IDX pos = b * (RANS_NLANES * segment_size) + L + j * RANS_NLANES; + *output(pos) = (Q)s; + x[L] = RansDecAdvance(x[L], *freq(s), *cum(s), slot, + scale_bits, stream, rp); + } + } + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Byte, DeviceType> stream; + SubArray<1, uint32_t, DeviceType> seg_offset; + SIZE n; + SIZE segment_size; + SIZE num_blocks; + SubArray<1, uint32_t, DeviceType> freq; + SubArray<1, uint32_t, DeviceType> cum; + SubArray<1, uint16_t, DeviceType> slot2sym; + uint32_t scale_bits; + SubArray<1, Q, DeviceType> output; +}; + +template +class InterleavedDecodeKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "rans interleaved decode"; + MGARDX_CONT InterleavedDecodeKernel( + SubArray<1, Byte, DeviceType> stream, + SubArray<1, uint32_t, DeviceType> seg_offset, SIZE n, SIZE segment_size, + SIZE num_blocks, SubArray<1, uint32_t, DeviceType> freq, + SubArray<1, uint32_t, DeviceType> cum, + SubArray<1, uint16_t, DeviceType> slot2sym, uint32_t scale_bits, + SubArray<1, Q, DeviceType> output) + : stream(stream), seg_offset(seg_offset), n(n), + segment_size(segment_size), num_blocks(num_blocks), freq(freq), + cum(cum), slot2sym(slot2sym), scale_bits(scale_bits), output(output) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = InterleavedDecodeFunctor; + FunctorType functor(stream, seg_offset, n, segment_size, num_blocks, freq, + cum, slot2sym, scale_bits, output); + SIZE tbx = 256, gridx; + gridx = (num_blocks - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), gridx); + return Task(functor, 1, 1, gridx, 1, 1, tbx, 0, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, Byte, DeviceType> stream; + SubArray<1, uint32_t, DeviceType> seg_offset; + SIZE n; + SIZE segment_size; + SIZE num_blocks; + SubArray<1, uint32_t, DeviceType> freq; + SubArray<1, uint32_t, DeviceType> cum; + SubArray<1, uint16_t, DeviceType> slot2sym; + uint32_t scale_bits; + SubArray<1, Q, DeviceType> output; +}; + +// --------------------------------------------------------------------------- +// Warp-cooperative encode: one SubGroup (= RANS_NLANES lanes) per block, lane L +// owns rANS state L. Written against the portable SubGroup abstraction, so the +// per-step renorm bytes of all lanes are gathered with a subgroup prefix-sum +// and written COALESCED, in the exact byte order of the sequential reference +// above (lane 0 at the high end of each step's range). Used only where the +// subgroup size equals RANS_NLANES (CUDA warp); other backends use the +// sequential kernel. +template +class InterleavedEncodeWarpFunctor : public Functor { +public: + MGARDX_CONT InterleavedEncodeWarpFunctor() {} + MGARDX_CONT InterleavedEncodeWarpFunctor( + SubArray<1, Q, DeviceType> input, SIZE n, SIZE segment_size, + SIZE num_blocks, SubArray<1, RansEncPacked, DeviceType> esym, + IDX block_capacity, SubArray<1, Byte, DeviceType> scratch, + SubArray<1, uint32_t, DeviceType> seg_len) + : input(input), n(n), segment_size(segment_size), num_blocks(num_blocks), + esym(esym), block_capacity(block_capacity), scratch(scratch), + seg_len(seg_len) { + Functor(); + } + + // Exclusive prefix sum of v across ONE RANS_NLANES-lane segment of the + // subgroup (lanes [seg_base, seg_base+RANS_NLANES)); returns the segment + // total via the reference parameter. logical_lane is the 0..RANS_NLANES-1 + // position within the segment. A wavefront wider than RANS_NLANES holds + // several independent segments, each scanning only its own lanes. + MGARDX_EXEC int SubgroupExclScan(SubGroup &sg, int logical_lane, + int seg_base, int v, int &total) { + int incl = v; + for (int d = 1; d < (int)RANS_NLANES; d <<= 1) { + int src = (logical_lane - d < 0) ? (seg_base + logical_lane) + : (seg_base + logical_lane - d); + int t = sg.shfl(incl, src); + if (logical_lane >= d) { + incl += t; + } + } + total = sg.shfl(incl, seg_base + (int)RANS_NLANES - 1); + return incl - v; + } + + MGARDX_EXEC void Operation1() { + SubGroup sg; + constexpr int W = + SubGroup::size(); // 32 (warp) or 64 (wavefront) + constexpr int SUBBLOCKS = + W / (int)RANS_NLANES; // logical blocks per subgroup + int lane_in_sg = sg.lane(); // 0..W-1 + int seg = lane_in_sg / (int)RANS_NLANES; // which logical block + int logical_lane = lane_in_sg % (int)RANS_NLANES; + int seg_base = seg * (int)RANS_NLANES; + + IDX gtid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX subgroup_idx = gtid / (IDX)W; + IDX num_subgroups = (FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX()) / + (IDX)W; + + // The loop bound is uniform across the whole subgroup so every lane + // iterates together (required for the subgroup shuffles), even when the + // wavefront's several blocks have different lengths. + for (IDX sgi = subgroup_idx; sgi * (IDX)SUBBLOCKS < num_blocks; + sgi += num_subgroups) { + IDX b = sgi * (IDX)SUBBLOCKS + (IDX)seg; + bool block_active = (b < num_blocks); + + IDX base = b * (RANS_NLANES * segment_size) + (IDX)logical_lane; + int count = 0; + if (block_active && base < n) { + IDX c = (n - 1 - base) / RANS_NLANES + 1; + count = (int)(c > segment_size ? segment_size : c); + } + // Subgroup-wide max symbol count -> uniform inner-loop bound. + int maxcount = count; + for (int d = 1; d < W; d <<= 1) { + int t = sg.shfl(maxcount, lane_in_sg ^ d); + maxcount = t > maxcount ? t : maxcount; + } + + uint32_t x = RANS_L; + IDX ptr = b * block_capacity + block_capacity; // per-block shared cursor + + for (int jj = maxcount; jj > 0; jj--) { + int j = jj - 1; + Byte buf[4]; + int nb = 0; + RansEncPacked e; + bool active = (j < count); // false for inactive-block lanes + if (active) { + IDX pos = base + (IDX)j * RANS_NLANES; + uint32_t s = (uint32_t)(*input(pos)); + e = *esym(s); + nb = RansEncCollect(x, e.x_max, buf); + } + int total; + int excl = SubgroupExclScan(sg, logical_lane, seg_base, nb, total); + for (int k = 0; k < nb; k++) { + *scratch(ptr - (IDX)excl - 1 - (IDX)k) = buf[k]; + } + ptr -= (IDX)total; + if (active) { + x = RansEncApply(x, e.rcp_freq, e.bias, (uint32_t)e.cmpl_freq, + (uint32_t)e.rcp_shift); + } + sg.sync(); + } + + // Flush NLANES states (4 LE bytes each), logical lane 0 at the high end. + // Inactive-block lanes contribute 0 to keep the scan in lockstep, but + // write nothing. + int flush_nb = block_active ? 4 : 0; + int total; + int excl = SubgroupExclScan(sg, logical_lane, seg_base, flush_nb, total); + if (block_active) { + IDX p0 = ptr - (IDX)excl - 4; + *scratch(p0 + 0) = (Byte)(x >> 0); + *scratch(p0 + 1) = (Byte)(x >> 8); + *scratch(p0 + 2) = (Byte)(x >> 16); + *scratch(p0 + 3) = (Byte)(x >> 24); + ptr -= (IDX)total; + if (logical_lane == 0) { + *seg_len(b) = (uint32_t)(b * block_capacity + block_capacity - ptr); + } + } + sg.sync(); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Q, DeviceType> input; + SIZE n; + SIZE segment_size; + SIZE num_blocks; + SubArray<1, RansEncPacked, DeviceType> esym; + IDX block_capacity; + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, uint32_t, DeviceType> seg_len; +}; + +template +class InterleavedEncodeWarpKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "rans interleaved warp encode"; + MGARDX_CONT InterleavedEncodeWarpKernel( + SubArray<1, Q, DeviceType> input, SIZE n, SIZE segment_size, + SIZE num_blocks, SubArray<1, RansEncPacked, DeviceType> esym, + IDX block_capacity, SubArray<1, Byte, DeviceType> scratch, + SubArray<1, uint32_t, DeviceType> seg_len) + : input(input), n(n), segment_size(segment_size), num_blocks(num_blocks), + esym(esym), block_capacity(block_capacity), scratch(scratch), + seg_len(seg_len) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = InterleavedEncodeWarpFunctor; + FunctorType functor(input, n, segment_size, num_blocks, esym, + block_capacity, scratch, seg_len); + SIZE tbx = 256; // 8 subgroups (warps) per block + SIZE total_threads = num_blocks * RANS_NLANES; + SIZE gridx = (total_threads - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), gridx); + return Task(functor, 1, 1, gridx, 1, 1, tbx, 0, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, Q, DeviceType> input; + SIZE n; + SIZE segment_size; + SIZE num_blocks; + SubArray<1, RansEncPacked, DeviceType> esym; + IDX block_capacity; + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, uint32_t, DeviceType> seg_len; +}; + +} // namespace rans +} // namespace mgard_x +#endif diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/GroupedBPEncoderGPU.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/BPEncoderBaseline.hpp similarity index 85% rename from include/mgard-x/MDR-X/BitplaneEncoder/GroupedBPEncoderGPU.hpp rename to include/mgard-x/MDR-X/BitplaneEncoder/BPEncoderBaseline.hpp index 43fcd0bc76..c131ec6697 100644 --- a/include/mgard-x/MDR-X/BitplaneEncoder/GroupedBPEncoderGPU.hpp +++ b/include/mgard-x/MDR-X/BitplaneEncoder/BPEncoderBaseline.hpp @@ -1,5 +1,5 @@ -#ifndef _MDR_GROUPED_BP_ENCODER_GPU_HPP -#define _MDR_GROUPED_BP_ENCODER_GPU_HPP +#ifndef _MDR_BP_ENCODER_BASELINE_HPP +#define _MDR_BP_ENCODER_BASELINE_HPP #include "../../RuntimeX/RuntimeX.h" @@ -43,18 +43,19 @@ MGARDX_EXEC void print_bits(T v, int num_bits, bool reverse = false) { template -class GroupedEncoderFunctor : public Functor { + OPTION ErrorColectingAlgorithm, bool ControlL2, typename DeviceType> +class BPEncoderBaselineFunctor : public Functor { public: MGARDX_CONT - GroupedEncoderFunctor() {} + BPEncoderBaselineFunctor() {} MGARDX_CONT - GroupedEncoderFunctor(SIZE n, SIZE num_batches_per_TB, SIZE num_bitplanes, - SIZE exp, SubArray<1, T, DeviceType> v, - SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, - SubArray<2, T_error, DeviceType> level_errors_workspace) + BPEncoderBaselineFunctor( + SIZE n, SIZE num_batches_per_TB, int num_bitplanes, + SubArray<1, T, DeviceType> abs_max, SubArray<1, T, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) : n(n), num_bitplanes(num_bitplanes), - num_batches_per_TB(num_batches_per_TB), exp(exp), + num_batches_per_TB(num_batches_per_TB), abs_max(abs_max), encoded_bitplanes(encoded_bitplanes), v(v), level_errors_workspace(level_errors_workspace) { Functor(); @@ -137,6 +138,8 @@ class GroupedEncoderFunctor : public Functor { MGARDX_EXEC void Operation2() { // __syncthreads(); + frexp(*abs_max((IDX)0), &exp); + if (BinaryType == NEGABINARY) exp += 2; // convert to fixpoint data @@ -191,7 +194,7 @@ class GroupedEncoderFunctor : public Functor { template MGARDX_EXEC void ErrorCollect(T *v, T_error *temp, T_error *errors, - SIZE num_elems, SIZE num_bitplanes, SIZE IdX, + SIZE num_elems, int num_bitplanes, SIZE IdX, SIZE IdY) { SIZE bitplane_idx = IdY * nblockx + IdX; if (bitplane_idx < num_bitplanes) { @@ -275,10 +278,12 @@ class GroupedEncoderFunctor : public Functor { // FunctorBase::GetThreadIdX(), // FunctorBase::GetThreadIdY()); - ErrorCollect<32, 32, 1>(sm_shifted, sm_temp_errors, sm_errors, - num_elems_per_TB, num_bitplanes, - FunctorBase::GetThreadIdX(), - FunctorBase::GetThreadIdY()); + if constexpr (ControlL2) { + ErrorCollect<32, 32, 1>(sm_shifted, sm_temp_errors, sm_errors, + num_elems_per_TB, num_bitplanes, + FunctorBase::GetThreadIdX(), + FunctorBase::GetThreadIdY()); + } } // get max bit-plane length @@ -303,16 +308,18 @@ class GroupedEncoderFunctor : public Functor { } } - // error - if (local_bitplane_idx < num_bitplanes + 1) { - sm_errors[local_bitplane_idx] = - ldexp(sm_errors[local_bitplane_idx], 2 * (-(int)num_bitplanes + exp)); - } + if constexpr (ControlL2) { + // error + if (local_bitplane_idx < num_bitplanes + 1) { + sm_errors[local_bitplane_idx] = ldexp(sm_errors[local_bitplane_idx], + 2 * (-(int)num_bitplanes + exp)); + } - if (local_bitplane_idx < num_bitplanes + 1) { - *level_errors_workspace(local_bitplane_idx, - FunctorBase::GetBlockIdX()) = - sm_errors[local_bitplane_idx]; + if (local_bitplane_idx < num_bitplanes + 1) { + *level_errors_workspace(local_bitplane_idx, + FunctorBase::GetBlockIdX()) = + sm_errors[local_bitplane_idx]; + } } } @@ -393,8 +400,9 @@ class GroupedEncoderFunctor : public Functor { // parameters SIZE n; SIZE num_batches_per_TB; - SIZE num_bitplanes; - SIZE exp; + int num_bitplanes; + SubArray<1, T, DeviceType> abs_max; + int exp; SubArray<1, T, DeviceType> v; SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; SubArray<2, T_error, DeviceType> level_errors_workspace; @@ -418,18 +426,19 @@ class GroupedEncoderFunctor : public Functor { template -class GroupedEncoderKernel : public Kernel { + bool ControlL2, typename DeviceType> +class BPEncoderBaselineKernel : public Kernel { public: constexpr static bool EnableAutoTuning() { return false; } constexpr static std::string_view Name = "grouped bp encoder"; MGARDX_CONT - GroupedEncoderKernel(SIZE n, SIZE num_batches_per_TB, SIZE num_bitplanes, - SIZE exp, SubArray<1, T, DeviceType> v, - SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, - SubArray<2, T_error, DeviceType> level_errors_workspace) + BPEncoderBaselineKernel( + SIZE n, SIZE num_batches_per_TB, int num_bitplanes, + SubArray<1, T, DeviceType> abs_max, SubArray<1, T, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) : n(n), num_bitplanes(num_bitplanes), - num_batches_per_TB(num_batches_per_TB), exp(exp), + num_batches_per_TB(num_batches_per_TB), abs_max(abs_max), encoded_bitplanes(encoded_bitplanes), v(v), level_errors_workspace(level_errors_workspace) {} @@ -438,13 +447,13 @@ class GroupedEncoderKernel : public Kernel { using T_fp = typename std::conditional::value, uint64_t, uint32_t>::type; using FunctorType = - GroupedEncoderFunctor; + BPEncoderBaselineFunctor; using TaskType = Task; MGARDX_CONT TaskType GenTask(int queue_idx) { - FunctorType functor(n, num_batches_per_TB, num_bitplanes, exp, v, + FunctorType functor(n, num_batches_per_TB, num_bitplanes, abs_max, v, encoded_bitplanes, level_errors_workspace); SIZE tbx, tby, tbz, gridx, gridy, gridz; size_t sm_size = functor.shared_memory_size(); @@ -462,8 +471,8 @@ class GroupedEncoderKernel : public Kernel { private: SIZE n; SIZE num_batches_per_TB; - SIZE num_bitplanes; - SIZE exp; + int num_bitplanes; + SubArray<1, T, DeviceType> abs_max; SubArray<1, T, DeviceType> v; SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; SubArray<2, T_error, DeviceType> level_errors_workspace; @@ -471,19 +480,20 @@ class GroupedEncoderKernel : public Kernel { template -class GroupedDecoderFunctor : public Functor { +class BPDecoderBaselineFunctor : public Functor { public: MGARDX_CONT - GroupedDecoderFunctor() {} + BPDecoderBaselineFunctor() {} MGARDX_CONT - GroupedDecoderFunctor(SIZE n, SIZE num_batches_per_TB, SIZE starting_bitplane, - SIZE num_bitplanes, SIZE exp, - SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, - SubArray<1, bool, DeviceType> signs, - SubArray<1, T, DeviceType> v) + BPDecoderBaselineFunctor( + SIZE n, SIZE num_batches_per_TB, int starting_bitplane, int num_bitplanes, + SubArray<1, T, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, SubArray<1, T, DeviceType> v) : n(n), num_batches_per_TB(num_batches_per_TB), starting_bitplane(starting_bitplane), num_bitplanes(num_bitplanes), - exp(exp), encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) { + abs_max(abs_max), encoded_bitplanes(encoded_bitplanes), signs(signs), + v(v) { Functor(); if (BinaryType == BINARY) { max_length_per_TB = num_batches_per_TB * 2; @@ -530,6 +540,7 @@ class GroupedDecoderFunctor : public Functor { ending_bitplane = starting_bitplane + num_bitplanes; + frexp(*abs_max((IDX)0), &exp); if (BinaryType == NEGABINARY) exp += 2; // data @@ -728,9 +739,10 @@ class GroupedDecoderFunctor : public Functor { // parameters SIZE n; SIZE num_batches_per_TB; - SIZE starting_bitplane; - SIZE num_bitplanes; - SIZE exp; + int starting_bitplane; + int num_bitplanes; + SubArray<1, T, DeviceType> abs_max; + int exp; SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; SubArray<1, bool, DeviceType> signs; SubArray<1, T, DeviceType> v; @@ -753,33 +765,35 @@ class GroupedDecoderFunctor : public Functor { template -class GroupedDecoderKernel : public Kernel { +class BPDecoderBaselineKernel : public Kernel { public: constexpr static bool EnableAutoTuning() { return false; } constexpr static std::string_view Name = "grouped bp decoder"; MGARDX_CONT - GroupedDecoderKernel(SIZE n, SIZE num_batches_per_TB, SIZE starting_bitplane, - SIZE num_bitplanes, SIZE exp, - SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, - SubArray<1, bool, DeviceType> signs, - SubArray<1, T, DeviceType> v) + BPDecoderBaselineKernel(SIZE n, SIZE num_batches_per_TB, + int starting_bitplane, int num_bitplanes, + SubArray<1, T, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, + SubArray<1, T, DeviceType> v) : n(n), num_batches_per_TB(num_batches_per_TB), starting_bitplane(starting_bitplane), num_bitplanes(num_bitplanes), - exp(exp), encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) {} + abs_max(abs_max), encoded_bitplanes(encoded_bitplanes), signs(signs), + v(v) {} using T_sfp = typename std::conditional::value, int64_t, int32_t>::type; using T_fp = typename std::conditional::value, uint64_t, uint32_t>::type; using FunctorType = - GroupedDecoderFunctor; + BPDecoderBaselineFunctor; using TaskType = Task; MGARDX_CONT TaskType GenTask(int queue_idx) { FunctorType functor(n, num_batches_per_TB, starting_bitplane, num_bitplanes, - exp, encoded_bitplanes, signs, v); + abs_max, encoded_bitplanes, signs, v); SIZE tbx, tby, tbz, gridx, gridy, gridz; size_t sm_size = functor.shared_memory_size(); const SIZE num_elems_per_TB = sizeof(T_bitplane) * 8 * num_batches_per_TB; @@ -796,9 +810,9 @@ class GroupedDecoderKernel : public Kernel { private: SIZE n; SIZE num_batches_per_TB; - SIZE starting_bitplane; - SIZE num_bitplanes; - SIZE exp; + int starting_bitplane; + int num_bitplanes; + SubArray<1, T, DeviceType> abs_max; SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; SubArray<1, bool, DeviceType> signs; SubArray<1, T, DeviceType> v; @@ -807,11 +821,13 @@ class GroupedDecoderKernel : public Kernel { // general bitplane encoder that encodes data by block using T_stream type // buffer template + bool ControlL2, typename DeviceType> class GroupedBPEncoder : public concepts::BitplaneEncoderInterface { + ControlL2, DeviceType> { public: + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; GroupedBPEncoder() : initialized(false) { static_assert(std::is_floating_point::value, "GeneralBPEncoder: input data must be floating points."); @@ -840,9 +856,8 @@ class GroupedBPEncoder this->hierarchy = &hierarchy; SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); - SIZE max_bitplane = 64; level_errors_work_array.resize( - {max_bitplane + 1, num_blocks(max_level_num_elems)}, queue_idx); + {MAX_BITPLANES + 1, num_blocks(max_level_num_elems)}, queue_idx); DeviceCollective::Sum( num_blocks(max_level_num_elems), SubArray<1, T_error, DeviceType>(), SubArray<1, T_error, DeviceType>(), level_error_sum_work_array, false, @@ -852,68 +867,67 @@ class GroupedBPEncoder static size_t EstimateMemoryFootprint(std::vector shape) { Hierarchy hierarchy(shape, Config()); SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); - SIZE max_bitplane = 64; size_t size = 0; size += hierarchy.EstimateMemoryFootprint(shape); size += - (max_bitplane + 1) * num_blocks(max_level_num_elems) * sizeof(T_error); + (MAX_BITPLANES + 1) * num_blocks(max_level_num_elems) * sizeof(T_error); for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { size += hierarchy.level_num_elems(level_idx) * sizeof(bool); } return size; } - void encode(SIZE n, SIZE num_bitplanes, int32_t exp, + void encode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, SubArray<1, T_data, DeviceType> v, SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, - SubArray<1, T_error, DeviceType> level_errors, - std::vector &streams_sizes, int queue_idx) { + SubArray<1, T_error, DeviceType> level_errors, int queue_idx) { SubArray<2, T_error, DeviceType> level_errors_work(level_errors_work_array); DeviceLauncher::Execute( - GroupedEncoderKernel( - n, num_batches_per_TB, num_bitplanes, exp, v, encoded_bitplanes, + BPEncoderBaselineKernel< + T_data, T_bitplane, T_error, BINARY_TYPE, DATA_ENCODING_ALGORITHM, + ERROR_COLLECTING_ALGORITHM, ControlL2, DeviceType>( + n, num_batches_per_TB, num_bitplanes, abs_max, v, encoded_bitplanes, level_errors_work), queue_idx); - SIZE reduce_size = num_blocks(n); - for (int i = 0; i < num_bitplanes + 1; i++) { - SubArray<1, T_error, DeviceType> curr_errors({reduce_size}, - level_errors_work(i, 0)); - SubArray<1, T_error, DeviceType> sum_error({1}, level_errors(i)); - DeviceCollective::Sum(reduce_size, curr_errors, sum_error, - level_error_sum_work_array, true, - queue_idx); - } - for (int i = 0; i < num_bitplanes; i++) { - streams_sizes[i] = buffer_size(n) * sizeof(T_bitplane); + if constexpr (ControlL2) { + SIZE reduce_size = num_blocks(n); + for (int i = 0; i < num_bitplanes + 1; i++) { + SubArray<1, T_error, DeviceType> curr_errors({reduce_size}, + level_errors_work(i, 0)); + SubArray<1, T_error, DeviceType> sum_error({1}, level_errors(i)); + DeviceCollective::Sum(reduce_size, curr_errors, sum_error, + level_error_sum_work_array, true, + queue_idx); + } } } - void decode(SIZE n, SIZE num_bitplanes, int32_t exp, + void decode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, int level, SubArray<1, T_data, DeviceType> v, int queue_idx) {} // decode the data and record necessary information for progressiveness - void progressive_decode(SIZE n, SIZE starting_bitplane, SIZE num_bitplanes, - int32_t exp, + void progressive_decode(SIZE n, int starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, SubArray<1, bool, DeviceType> level_signs, int level, SubArray<1, T_data, DeviceType> v, int queue_idx) { if (num_bitplanes > 0) { DeviceLauncher::Execute( - GroupedDecoderKernel( - n, num_batches_per_TB, starting_bitplane, num_bitplanes, exp, + BPDecoderBaselineKernel( + n, num_batches_per_TB, starting_bitplane, num_bitplanes, abs_max, encoded_bitplanes, level_signs, v), queue_idx); } } - static SIZE buffer_size(SIZE n) { + static SIZE bitplane_length(SIZE n) { const SIZE num_elems_per_TB = sizeof(T_bitplane) * 8 * num_batches_per_TB; const SIZE bitplane_max_length_per_TB = num_batches_per_TB * 2; SIZE num_blocks = (n - 1) / num_elems_per_TB + 1; diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/BPEncoderLocalityBlock.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/BPEncoderLocalityBlock.hpp new file mode 100644 index 0000000000..c0b5d25c85 --- /dev/null +++ b/include/mgard-x/MDR-X/BitplaneEncoder/BPEncoderLocalityBlock.hpp @@ -0,0 +1,713 @@ +#ifndef _MDR_BP_ENCODER_LOCALITY_BLOCK_HPP +#define _MDR_BP_ENCODER_LOCALITY_BLOCK_HPP + +#include "../../RuntimeX/RuntimeX.h" + +#include "BitplaneEncoderInterface.hpp" +#include + +namespace mgard_x { +namespace MDR { + +template +class BPEncoderLocalityBlockFunctor : public Functor { +public: + MGARDX_CONT + BPEncoderLocalityBlockFunctor() {} + MGARDX_CONT + BPEncoderLocalityBlockFunctor( + SIZE n, int num_bitplanes, SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) { + Functor(); + } + + MGARDX_EXEC void encode_batch(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_bitplane buffer = 0; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_bitplane bit = + (v[data_idx] >> (num_bitplanes - 1 - bp_idx)) & (T_bitplane)1; + buffer += bit << BATCH_SIZE - 1 - data_idx; + } + encoded[bp_idx] = buffer; + } + } + + MGARDX_EXEC void encode_batch_with_prediction(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + T_fp data_values[BATCH_SIZE]; + for (int i = 0; i < BATCH_SIZE; i++) { + data_values[i] = 0; + } + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_bitplane buffer = 0; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_bitplane cur_bit = + (v[data_idx] >> (num_bitplanes - 1 - bp_idx)) & (T_bitplane)1; + if (bp_idx == 0) { + buffer += cur_bit << BATCH_SIZE - 1 - data_idx; + } else { + if constexpr (NegaBinary) { + data_values[data_idx] = data_values[data_idx] * (-2); + } else { + data_values[data_idx] = data_values[data_idx] * 2; + } + if (data_idx) { + T_bitplane pred_bit = 0; + T_fp d0 = abs((T_sfp)data_values[data_idx - 1] - + (T_sfp)data_values[data_idx]); + T_fp d1 = abs((T_sfp)data_values[data_idx - 1] - + (T_sfp)data_values[data_idx] - 1); + if (d0 > d1) { + pred_bit = 1; + } + if (pred_bit != cur_bit) { + buffer += (T_bitplane)1 << BATCH_SIZE - 1 - data_idx; + } + } else { + buffer += cur_bit << BATCH_SIZE - 1 - data_idx; + } + data_values[data_idx] += cur_bit; + } + } + encoded[bp_idx] = buffer; + } + } + + MGARDX_EXEC void error_collect_binary(T_data *shifted_data, T_error *errors, + int num_bitplanes, int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_error diff = (T_error)(fp_data & mask) + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void error_collect_negabinary(T_data *shifted_data, + T_error *errors, int num_bitplanes, + int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_fp ngb_data = Math::binary2negabinary((T_sfp)data); + T_error diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void EncodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_fp fp_sign[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_bitplane encoded_sign[1]; + T_error errors[MAX_BITPLANES + 1]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + // SIZE batch_idx = gid; + // if (batch_idx < num_batches) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = 0; + if (batch_idx * BATCH_SIZE + data_idx < n) { + data = *v(batch_idx * BATCH_SIZE + data_idx); + } + shifted_data[data_idx] = ldexp(data, num_bitplanes - exp); + fp_data[data_idx] = (T_fp)fabs(shifted_data[data_idx]); + fp_sign[data_idx] = (T_fp)(signbit(data) == 0 ? 0 : 1); + // fp_data[data_idx] = (T_fp)fabs(shifted_data[data_idx]); + // if (batch_idx == 0) { + // printf("fp_data[data_idx]: %llu\n", fp_data[data_idx]); + // } + // printf("%f: ", data); print_bits(fp_data[data_idx], b); + // printf("data: %f, fp_data[data_idx]: %llu, signbit(data): %lld, + // fp_sign[data_idx]: %llu \n", data, fp_data[data_idx], signbit(data), + // fp_sign[data_idx]); + } + // encode data + encode_batch(fp_data, encoded_data, num_bitplanes); + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + *encoded_bitplanes(bp_idx, batch_idx) = encoded_data[bp_idx]; + // if (batch_idx == 0) { + // printf("encoded_data: %llu\n", encoded_data[bp_idx]); + // } + // print_bits(encoded_bitplanes[bp_idx * b + batch_idx * 2], + // batch_size); + } + // encode sign + encode_batch(fp_sign, encoded_sign, 1); + + // if (batch_idx == 0) { + // printf("encoded_sign: %u\n", encoded_sign[0]); + // } + + *encoded_bitplanes(0, num_batches + batch_idx) = encoded_sign[0]; + // set rest of the bitplanes to 0 + for (int bp_idx = 1; bp_idx < num_bitplanes; bp_idx++) { + *encoded_bitplanes(bp_idx, num_batches + batch_idx) = (T_bitplane)0; + } + // // encode sign + // encode_batch(signs, encoded_sign, BATCH_SIZE, 1); + // print_bits(encoded_bitplanes[0 * b + batch_idx * 2 + 1], batch_size); + + if constexpr (ControlL2) { + // errors[] persists across grid-stride batch iterations, so it must + // be reset here -- otherwise each batch's recorded error is + // contaminated by (and, on the very first iteration, reads + // uninitialized) prior state. + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = 0; + } + error_collect_binary(shifted_data, errors, num_bitplanes, exp); + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + *level_errors_workspace(bp_idx, batch_idx) = errors[bp_idx]; + } + } + } + } + + MGARDX_EXEC void EncodeNegaBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_error errors[MAX_BITPLANES + 1]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + exp += 2; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = 0; + if (batch_idx * BATCH_SIZE + data_idx < n) { + data = *v(batch_idx * BATCH_SIZE + data_idx); + } + shifted_data[data_idx] = ldexp(data, num_bitplanes - exp); + fp_data[data_idx] = + Math::binary2negabinary((T_sfp)shifted_data[data_idx]); + // fp_data[data_idx] = (T_fp)fabs(shifted_data[data_idx]); + + // printf("%f: ", data); print_bits(fp_data[data_idx], b); + } + // encode data + encode_batch(fp_data, encoded_data, num_bitplanes); + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + *encoded_bitplanes(bp_idx, batch_idx) = encoded_data[bp_idx]; + // print_bits(encoded_bitplanes[bp_idx * b + batch_idx * 2], + // batch_size); + } + + if constexpr (ControlL2) { + // errors[] persists across grid-stride batch iterations, so it must + // be reset here -- otherwise each batch's recorded error is + // contaminated by (and, on the very first iteration, reads + // uninitialized) prior state. + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = 0; + } + error_collect_negabinary(shifted_data, errors, num_bitplanes, exp); + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + *level_errors_workspace(bp_idx, batch_idx) = errors[bp_idx]; + } + } + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + EncodeNegaBinary(); + } else { + EncodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPEncoderLocalityBlockKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp encoder"; + MGARDX_CONT + BPEncoderLocalityBlockKernel( + SIZE n, int num_bitplanes, SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) {} + + using FunctorType = + BPEncoderLocalityBlockFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + FunctorType functor(n, num_bitplanes, abs_max, v, encoded_bitplanes, + level_errors_workspace); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 16; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (n - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; +}; + +template +class BPDecoderLocalityBlockFunctor : public Functor { +public: + MGARDX_CONT + BPDecoderLocalityBlockFunctor() {} + MGARDX_CONT + BPDecoderLocalityBlockFunctor( + SIZE n, SIZE starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) { + Functor(); + } + + MGARDX_EXEC void decode_batch(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_fp buffer = 0; + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_fp bit = (encoded[bp_idx] >> (BATCH_SIZE - 1 - data_idx)) & (T_fp)1; + buffer += bit << (num_bitplanes - 1 - bp_idx); + } + v[data_idx] = buffer; + } + } + + MGARDX_EXEC void DecodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_fp fp_sign[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_bitplane encoded_sign[MAX_BITPLANES]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + int ending_bitplane = starting_bitplane + num_bitplanes; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = + *encoded_bitplanes(starting_bitplane + bp_idx, batch_idx); + // print_bits(encoded_data[bp_idx], batch_size); + } + // encode data + decode_batch(fp_data, encoded_data, num_bitplanes); + + if (starting_bitplane == 0) { + // decode sign + encoded_sign[0] = *encoded_bitplanes(0, num_batches + batch_idx); + decode_batch(fp_sign, encoded_sign, 1); + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + if (batch_idx * BATCH_SIZE + data_idx < n) { + *signs(batch_idx * BATCH_SIZE + data_idx) = fp_sign[data_idx]; + } + } + } else { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + if (batch_idx * BATCH_SIZE + data_idx < n) { + fp_sign[data_idx] = *signs(batch_idx * BATCH_SIZE + data_idx); + } + } + } + + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = ldexp((T_data)fp_data[data_idx], -ending_bitplane + exp); + if (batch_idx * BATCH_SIZE + data_idx < n) { + *v(batch_idx * BATCH_SIZE + data_idx) = + fp_sign[data_idx] ? -data : data; + } + // printf("data: %f, fp_data[data_idx]: %llu\n", *v(batch_idx * + // BATCH_SIZE + data_idx), fp_data[data_idx]); printf("%f: ", data); + // print_bits(fp_data[data_idx], b); + } + } + } + + MGARDX_EXEC void DecodeNegaBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + exp += 2; + + int ending_bitplane = starting_bitplane + num_bitplanes; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = + *encoded_bitplanes(starting_bitplane + bp_idx, batch_idx); + // print_bits(encoded_data[bp_idx], batch_size); + } + // encode data + decode_batch(fp_data, encoded_data, num_bitplanes); + + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = ldexp( + (T_data)Math::negabinary2binary(fp_data[data_idx]), + -ending_bitplane + exp); + if (batch_idx * BATCH_SIZE + data_idx < n) { + *v(batch_idx * BATCH_SIZE + data_idx) = + ending_bitplane % 2 != 0 ? -data : data; + } + // printf("%f: ", data); print_bits(fp_data[data_idx], b); + } + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + DecodeNegaBinary(); + } else { + DecodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + SIZE starting_bitplane; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPDecoderLocalityBlockKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp decoder"; + MGARDX_CONT + BPDecoderLocalityBlockKernel( + SIZE n, SIZE starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) {} + + using FunctorType = + BPDecoderLocalityBlockFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + + FunctorType functor(n, starting_bitplane, num_bitplanes, abs_max, + encoded_bitplanes, signs, v); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 8; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (n - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + SIZE starting_bitplane; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; +}; + +// general bitplane encoder that encodes data by block using T_stream type +// buffer +template +class BPEncoderLocalityBlock + : public concepts::BitplaneEncoderInterface { +public: + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + using T_sfp = typename std::conditional::value, + int64_t, int32_t>::type; + using T_fp = typename std::conditional::value, + uint64_t, uint32_t>::type; + + BPEncoderLocalityBlock() : initialized(false) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + } + BPEncoderLocalityBlock(Hierarchy &hierarchy) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + Adapt(hierarchy, 0); + DeviceRuntime::SyncQueue(0); + } + + static SIZE bitplane_length(SIZE n) { + if constexpr (!NegaBinary) { + return num_blocks(n) * 2; + } else { + return num_blocks(n); + } + } + + static SIZE num_blocks(SIZE n) { + const SIZE batch_size = sizeof(T_bitplane) * 8; + SIZE num_blocks = (n - 1) / batch_size + 1; + return num_blocks; + } + + void Adapt(Hierarchy &hierarchy, int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + SIZE max_level_num_elems = + round_up(hierarchy.level_num_elems(hierarchy.l_target()), BATCH_SIZE); + + level_errors_work_array.resize( + {MAX_BITPLANES + 1, num_blocks(max_level_num_elems)}, queue_idx); + DeviceCollective::Sum( + num_blocks(max_level_num_elems), SubArray<1, T_error, DeviceType>(), + SubArray<1, T_error, DeviceType>(), level_error_sum_work_array, false, + queue_idx); + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + Hierarchy hierarchy(shape, Config()); + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + size_t size = 0; + size += hierarchy.EstimateMemoryFootprint(shape); + size += + (MAX_BITPLANES + 1) * num_blocks(max_level_num_elems) * sizeof(T_error); + for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { + size += hierarchy.level_num_elems(level_idx) * sizeof(bool); + } + return size; + } + + void encode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, T_error, DeviceType> level_errors, int queue_idx) { + + SubArray<2, T_error, DeviceType> level_errors_work(level_errors_work_array); + + DeviceLauncher::Execute( + BPEncoderLocalityBlockKernel( + n, num_bitplanes, abs_max, v, encoded_bitplanes, level_errors_work), + queue_idx); + + if constexpr (ControlL2) { + SIZE reduce_size = num_blocks(n); + for (int i = 0; i < num_bitplanes + 1; i++) { + SubArray<1, T_error, DeviceType> curr_errors({reduce_size}, + level_errors_work(i, 0)); + SubArray<1, T_error, DeviceType> sum_error({1}, level_errors(i)); + DeviceCollective::Sum(reduce_size, curr_errors, sum_error, + level_error_sum_work_array, true, + queue_idx); + } + } + } + + void decode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) {} + + // decode the data and record necessary information for progressiveness + void progressive_decode(SIZE n, int starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> level_signs, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) { + + if (num_bitplanes > 0) { + DeviceLauncher::Execute( + BPDecoderLocalityBlockKernel( + n, starting_bitplane, num_bitplanes, abs_max, encoded_bitplanes, + level_signs, v), + queue_idx); + } + } + + void print() const { std::cout << "Grouped bitplane encoder" << std::endl; } + +private: + bool initialized; + Hierarchy *hierarchy; + Array<2, T_error, DeviceType> level_errors_work_array; + Array<1, Byte, DeviceType> level_error_sum_work_array; +}; +} // namespace MDR +} // namespace mgard_x +#endif diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/BPEncoderRegisterBlock.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/BPEncoderRegisterBlock.hpp new file mode 100644 index 0000000000..a1a2d12182 --- /dev/null +++ b/include/mgard-x/MDR-X/BitplaneEncoder/BPEncoderRegisterBlock.hpp @@ -0,0 +1,756 @@ +#ifndef _MDR_BP_ENCODER_REGISTER_BLOCK_HPP +#define _MDR_BP_ENCODER_REGISTER_BLOCK_HPP + +#include "../../RuntimeX/RuntimeX.h" + +#include "BitplaneEncoderInterface.hpp" +#include + +namespace mgard_x { +namespace MDR { + +template +class BPEncoderRegisterBlockFunctor : public Functor { +public: + MGARDX_CONT + BPEncoderRegisterBlockFunctor() {} + MGARDX_CONT + BPEncoderRegisterBlockFunctor( + SIZE n, SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), abs_max(abs_max), encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) { + Functor(); + } + + MGARDX_EXEC void encode_batch(T_fp *v, T_bitplane *encoded) { + +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + T_bitplane buffer = 0; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_bitplane bit = + (v[data_idx] >> (NUM_BITPLANES - 1 - bp_idx)) & (T_bitplane)1; + buffer |= bit << BATCH_SIZE - 1 - data_idx; + } + encoded[bp_idx] = buffer; + } + } + + MGARDX_EXEC void error_collect_binary(T_data *shifted_data, T_error *errors, + int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_error diff = (T_error)(fp_data & mask) + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[NUM_BITPLANES - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < NUM_BITPLANES + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-NUM_BITPLANES + exp)); + } + } + + MGARDX_EXEC void error_collect_negabinary(T_data *shifted_data, + T_error *errors, int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_fp ngb_data = Math::binary2negabinary((T_sfp)data); + T_error diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[NUM_BITPLANES - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < NUM_BITPLANES + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-NUM_BITPLANES + exp)); + } + } + + MGARDX_EXEC void EncodeBinary() { + SIZE batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE num_full_batches = n / BATCH_SIZE; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[NUM_BITPLANES]; + T_bitplane encoded_sign = 0; + T_error errors[NUM_BITPLANES + 1]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + if (batch_idx >= num_full_batches) { + return; + } + + if (exp > 0) { +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = *v(data_idx * num_full_batches + batch_idx); + // this can cause overflow + shifted_data[data_idx] = data * ((T_fp)1 << NUM_BITPLANES - exp); + // ldexp without constant argument is slow + // shifted_data[data_idx] = ldexp(data, NUM_BITPLANES - exp); + fp_data[data_idx] = (T_fp)fabs(shifted_data[data_idx]); + + // if (num_full_batches == 1) printf("data: %f * %d %d, shifted_data: %f + // fp_data: %llu \n", data, NUM_BITPLANES, exp, shifted_data[data_idx], + // fp_data[data_idx]); + } + } else { +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = *v(data_idx * num_full_batches + batch_idx); + shifted_data[data_idx] = data * pow(2, NUM_BITPLANES - exp); + fp_data[data_idx] = (T_fp)fabs(shifted_data[data_idx]); + } + } + + // encode sign + // Shift amount runs up to BATCH_SIZE - 1 (bits of T_bitplane), so the + // value being shifted must be T_bitplane, not T_fp: when T_bitplane is + // wider than T_fp (e.g. uint64_t bitplanes with float data, T_fp = + // uint32_t), shifting a T_fp by >= 32 is undefined behavior. + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + encoded_sign += (T_bitplane)(signbit(shifted_data[data_idx]) == 0 ? 0 : 1) + << (BATCH_SIZE - 1 - data_idx); + } + // encode data + encode_batch(fp_data, encoded_data); +// store data +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + // if (num_full_batches == 1) printf("encoded_data: %u\n", + // encoded_data[bp_idx]); + *encoded_bitplanes(bp_idx, batch_idx) = encoded_data[bp_idx]; + } + // store sign + *encoded_bitplanes(0, num_full_batches + batch_idx) = encoded_sign; +// set rest of the bitplanes to 0 +#pragma unroll + for (int bp_idx = 1; bp_idx < NUM_BITPLANES; bp_idx++) { + *encoded_bitplanes(bp_idx, num_full_batches + batch_idx) = (T_bitplane)0; + } + if constexpr (ControlL2) { + // errors[] is uninitialized stack memory; error_collect_binary + // accumulates into it with +=, so it must be zeroed first. + for (int bp_idx = 0; bp_idx < NUM_BITPLANES + 1; bp_idx++) { + errors[bp_idx] = 0; + } + error_collect_binary(shifted_data, errors, exp); + for (int bp_idx = 0; bp_idx < NUM_BITPLANES + 1; bp_idx++) { + *level_errors_workspace(bp_idx, batch_idx) = errors[bp_idx]; + } + } + } + + MGARDX_EXEC void EncodeNegaBinary() { + SIZE batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE num_full_batches = n / BATCH_SIZE; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[NUM_BITPLANES]; + T_error errors[NUM_BITPLANES + 1]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + exp += 2; + + if (batch_idx >= num_full_batches) { + return; + } + + if (exp > 0) { +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = 0; + data = *v(data_idx * num_full_batches + batch_idx); + // This can cause overflow + shifted_data[data_idx] = data * ((T_fp)1 << NUM_BITPLANES - exp); + // ldexp without constant argument is slow + // shifted_data[data_idx] = ldexp(data, NUM_BITPLANES - exp); + fp_data[data_idx] = + Math::binary2negabinary((T_sfp)shifted_data[data_idx]); + } + } else { +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = 0; + data = *v(data_idx * num_full_batches + batch_idx); + shifted_data[data_idx] = data * pow(2, NUM_BITPLANES - exp); + // ldexp without constant argument is slow + // shifted_data[data_idx] = ldexp(data, NUM_BITPLANES - exp); + fp_data[data_idx] = + Math::binary2negabinary((T_sfp)shifted_data[data_idx]); + } + } + + // encode data + encode_batch(fp_data, encoded_data); +// store data +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + *encoded_bitplanes(bp_idx, batch_idx) = encoded_data[bp_idx]; + } + + if constexpr (ControlL2) { + // errors[] is uninitialized stack memory; error_collect_negabinary + // accumulates into it with +=, so it must be zeroed first. +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES + 1; bp_idx++) { + errors[bp_idx] = 0; + } + error_collect_negabinary(shifted_data, errors, exp); +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES + 1; bp_idx++) { + *level_errors_workspace(bp_idx, batch_idx) = errors[bp_idx]; + } + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + EncodeNegaBinary(); + } else { + EncodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; +}; + +template +class BPEncoderRegisterBlockKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp encoder"; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + MGARDX_CONT + BPEncoderRegisterBlockKernel( + SIZE n, SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), abs_max(abs_max), encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) {} + + using FunctorType = + BPEncoderRegisterBlockFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + FunctorType functor(n, abs_max, v, encoded_bitplanes, + level_errors_workspace); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE total_thread = std::max((SIZE)1, n / BATCH_SIZE); + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (total_thread - 1) / tbx + 1; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; +}; + +template +class BPDecoderRegisterBlockFunctor : public Functor { +public: + MGARDX_CONT + BPDecoderRegisterBlockFunctor() {} + MGARDX_CONT + BPDecoderRegisterBlockFunctor( + SIZE n, int starting_bitplane, SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) { + Functor(); + } + + MGARDX_EXEC void decode_batch(T_fp *v, T_bitplane *encoded) { +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_fp buffer = 0; + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + T_fp bit = (encoded[bp_idx] >> (BATCH_SIZE - 1 - data_idx)) & (T_fp)1; + buffer += bit << (NUM_BITPLANES - 1 - bp_idx); + // printf("bit: %llu, buffer: %llu\n", bit, buffer); + } + v[data_idx] = buffer; + } + } + + MGARDX_EXEC void DecodeBinary() { + SIZE batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE num_full_batches = n / BATCH_SIZE; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_fp fp_sign[BATCH_SIZE]; + T_bitplane encoded_data[NUM_BITPLANES]; + T_bitplane encoded_sign; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + if (batch_idx >= num_full_batches) { + return; + } + + int ending_bitplane = starting_bitplane + NUM_BITPLANES; + +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + encoded_data[bp_idx] = + *encoded_bitplanes(starting_bitplane + bp_idx, batch_idx); + // if (num_full_batches == 1) printf("encoded_data: %u\n", + // encoded_data[bp_idx]); + } + // decode data + decode_batch(fp_data, encoded_data); + + if (starting_bitplane == 0) { + // decode sign + encoded_sign = *encoded_bitplanes(0, num_full_batches + batch_idx); +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + fp_sign[data_idx] = + (encoded_sign >> (BATCH_SIZE - 1 - data_idx)) & (T_fp)1; + *signs(data_idx * num_full_batches + batch_idx) = fp_sign[data_idx]; + } + } else { +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + fp_sign[data_idx] = *signs(data_idx * num_full_batches + batch_idx); + } + } +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + shifted_data[data_idx] = (T_data)fp_data[data_idx]; + // It is beneficial to use pow instead of ldexp + T_data data = shifted_data[data_idx] * pow(2, -ending_bitplane + exp); + // T_data data = ldexp(shifted_data[data_idx], -ending_bitplane + exp); + data = fp_sign[data_idx] ? -data : data; + *v(data_idx * num_full_batches + batch_idx) = data; + + // if (num_full_batches == 1) printf("%llu %f %f\n", fp_data[data_idx], + // shifted_data[data_idx], data); + } + } + + MGARDX_EXEC void DecodeNegaBinary() { + SIZE batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE num_full_batches = n / BATCH_SIZE; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[NUM_BITPLANES]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + exp += 2; + + if (batch_idx >= num_full_batches) { + return; + } + + int ending_bitplane = starting_bitplane + NUM_BITPLANES; + +// load bitplanes +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + encoded_data[bp_idx] = + *encoded_bitplanes(starting_bitplane + bp_idx, batch_idx); + // print_bits(encoded_data[bp_idx], batch_size); + } + // decode data + decode_batch(fp_data, encoded_data); + +// store data +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + shifted_data[data_idx] = + Math::negabinary2binary(fp_data[data_idx]); + // No noticing difference between the two + T_data data = shifted_data[data_idx] * pow(2, -ending_bitplane + exp); + // T_data data = ldexp(shifted_data[data_idx], -ending_bitplane + exp); + data = ending_bitplane % 2 != 0 ? -data : data; + *v(data_idx * num_full_batches + batch_idx) = data; + // printf("%f: ", data); print_bits(fp_data[data_idx], b); + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + DecodeNegaBinary(); + } else { + DecodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + int starting_bitplane; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPDecoderRegisterBlockKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp decoder"; + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + MGARDX_CONT + BPDecoderRegisterBlockKernel( + SIZE n, int starting_bitplane, SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) {} + + using FunctorType = + BPDecoderRegisterBlockFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + + FunctorType functor(n, starting_bitplane, abs_max, encoded_bitplanes, signs, + v); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE total_thread = std::max((SIZE)1, n / BATCH_SIZE); + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (total_thread - 1) / tbx + 1; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int starting_bitplane; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; +}; + +// general bitplane encoder that encodes data by block using T_stream type +// buffer +template +class BPEncoderRegisterBlock + : public concepts::BitplaneEncoderInterface { +public: + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + using T_sfp = typename std::conditional::value, + int64_t, int32_t>::type; + using T_fp = typename std::conditional::value, + uint64_t, uint32_t>::type; + + BPEncoderRegisterBlock() : initialized(false) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + } + BPEncoderRegisterBlock(Hierarchy &hierarchy) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + Adapt(hierarchy, 0); + DeviceRuntime::SyncQueue(0); + } + + static SIZE bitplane_length(SIZE n) { + if constexpr (!NegaBinary) { + return num_blocks(n) * 2; + } else { + return num_blocks(n); + } + } + + static SIZE num_blocks(SIZE n) { + const SIZE batch_size = sizeof(T_bitplane) * 8; + SIZE num_blocks = (n - 1) / batch_size + 1; + return num_blocks; + } + + void Adapt(Hierarchy &hierarchy, int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + SIZE max_level_num_elems = + round_up(hierarchy.level_num_elems(hierarchy.l_target()), BATCH_SIZE); + + level_errors_work_array.resize( + {MAX_BITPLANES + 1, num_blocks(max_level_num_elems)}, queue_idx); + DeviceCollective::Sum( + num_blocks(max_level_num_elems), SubArray<1, T_error, DeviceType>(), + SubArray<1, T_error, DeviceType>(), level_error_sum_work_array, false, + queue_idx); + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + Hierarchy hierarchy(shape, Config()); + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + size_t size = 0; + size += hierarchy.EstimateMemoryFootprint(shape); + size += + (MAX_BITPLANES + 1) * num_blocks(max_level_num_elems) * sizeof(T_error); + for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { + size += hierarchy.level_num_elems(level_idx) * sizeof(bool); + } + return size; + } + + // TODO: remove num_bitplanes in the future + void encode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, T_error, DeviceType> level_errors, int queue_idx) { + + if (n % BATCH_SIZE != 0) { + throw std::runtime_error( + "BPEncoderV1b: n is not a multiple of BATCH_SIZE"); + } + SubArray<2, T_error, DeviceType> level_errors_work(level_errors_work_array); + + DeviceLauncher::Execute( + BPEncoderRegisterBlockKernel( + n, abs_max, v, encoded_bitplanes, level_errors_work), + queue_idx); + + if constexpr (ControlL2) { + SIZE reduce_size = num_blocks(n); + for (int i = 0; i < MAX_BITPLANES + 1; i++) { + SubArray<1, T_error, DeviceType> curr_errors({reduce_size}, + level_errors_work(i, 0)); + SubArray<1, T_error, DeviceType> sum_error({1}, level_errors(i)); + DeviceCollective::Sum(reduce_size, curr_errors, sum_error, + level_error_sum_work_array, true, + queue_idx); + } + } + } + + void decode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) {} + + // decode the data and record necessary information for progressiveness + void progressive_decode(SIZE n, int starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> level_signs, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) { + + // if (num_bitplanes > 0) { + // DeviceLauncher::Execute( + // BPDecoderRegisterBlockKernel(n, starting_bitplane, + // num_bitplanes, + // abs_max, encoded_bitplanes, + // level_signs, v), + // queue_idx); + // } + +#define V1B_DECODE(NUM_BITPLANES) \ + if (num_bitplanes == NUM_BITPLANES) { \ + DeviceLauncher::Execute( \ + BPDecoderRegisterBlockKernel( \ + n, starting_bitplane, abs_max, encoded_bitplanes, level_signs, v), \ + queue_idx); \ + } + V1B_DECODE(1); + V1B_DECODE(2); + V1B_DECODE(3); + V1B_DECODE(4); + V1B_DECODE(5); + V1B_DECODE(6); + V1B_DECODE(7); + V1B_DECODE(8); + V1B_DECODE(9); + V1B_DECODE(10); + V1B_DECODE(11); + V1B_DECODE(12); + V1B_DECODE(13); + V1B_DECODE(14); + V1B_DECODE(15); + V1B_DECODE(16); + V1B_DECODE(17); + V1B_DECODE(18); + V1B_DECODE(19); + V1B_DECODE(20); + V1B_DECODE(21); + V1B_DECODE(22); + V1B_DECODE(23); + V1B_DECODE(24); + V1B_DECODE(25); + V1B_DECODE(26); + V1B_DECODE(27); + V1B_DECODE(28); + V1B_DECODE(29); + V1B_DECODE(30); + V1B_DECODE(31); + V1B_DECODE(32); + V1B_DECODE(33); + V1B_DECODE(34); + V1B_DECODE(35); + V1B_DECODE(36); + V1B_DECODE(37); + V1B_DECODE(38); + V1B_DECODE(39); + V1B_DECODE(40); + V1B_DECODE(41); + V1B_DECODE(42); + V1B_DECODE(43); + V1B_DECODE(44); + V1B_DECODE(45); + V1B_DECODE(46); + V1B_DECODE(47); + V1B_DECODE(48); + V1B_DECODE(49); + V1B_DECODE(50); + V1B_DECODE(51); + V1B_DECODE(52); + V1B_DECODE(53); + V1B_DECODE(54); + V1B_DECODE(55); + V1B_DECODE(56); + V1B_DECODE(57); + V1B_DECODE(58); + V1B_DECODE(59); + V1B_DECODE(60); + V1B_DECODE(61); + V1B_DECODE(62); + V1B_DECODE(63); + V1B_DECODE(64); + } + + void print() const { std::cout << "Grouped bitplane encoder" << std::endl; } + +private: + bool initialized; + Hierarchy *hierarchy; + Array<2, T_error, DeviceType> level_errors_work_array; + Array<1, Byte, DeviceType> level_error_sum_work_array; +}; +} // namespace MDR +} // namespace mgard_x +#endif diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/BitplaneEncoder.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/BitplaneEncoder.hpp index 95deb75d32..5414b91c48 100644 --- a/include/mgard-x/MDR-X/BitplaneEncoder/BitplaneEncoder.hpp +++ b/include/mgard-x/MDR-X/BitplaneEncoder/BitplaneEncoder.hpp @@ -1,11 +1,8 @@ #ifndef _MDR_ENCODER_HPP #define _MDR_ENCODER_HPP -// #include "GroupedBPEncoder.hpp" -// #include "NegaBinaryBPEncoder.hpp" -// #include "PerBitBPEncoder.hpp" -#include "GroupedBPEncoderGPU.hpp" -#include "GroupedWarpBPEncoderGPU.hpp" -// #include "PerBitBPEncoderGPU.hpp" +#include "BPEncoderBaseline.hpp" +#include "BPEncoderLocalityBlock.hpp" +#include "BPEncoderRegisterBlock.hpp" #endif diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/BitplaneEncoderInterface.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/BitplaneEncoderInterface.hpp index 315e7b8f44..eff81c1e38 100644 --- a/include/mgard-x/MDR-X/BitplaneEncoder/BitplaneEncoderInterface.hpp +++ b/include/mgard-x/MDR-X/BitplaneEncoder/BitplaneEncoderInterface.hpp @@ -31,33 +31,34 @@ namespace MDR { namespace concepts { // concept of encoder which encodes T type data into bitstreams template + bool ControlL2, typename DeviceType> class BitplaneEncoderInterface { public: virtual ~BitplaneEncoderInterface() = default; - virtual void encode(SIZE n, SIZE num_bitplanes, int32_t exp, + virtual void encode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, SubArray<1, T_data, DeviceType> v, SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, SubArray<1, T_error, DeviceType> level_errors, - std::vector &streams_sizes, int queue_idx) = 0; + int queue_idx) = 0; - virtual void decode(SIZE n, SIZE num_bitplanes, int32_t exp, + virtual void decode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, int level, SubArray<1, T_data, DeviceType> v, int queue_idx) = 0; virtual void - progressive_decode(SIZE n, SIZE starting_bitplanes, SIZE num_bitplanes, - int32_t exp, + progressive_decode(SIZE n, int starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, SubArray<1, bool, DeviceType> level_signs, int level, - SubArray<1, T_data, DeviceType> v, - - int queue_idx) = 0; + SubArray<1, T_data, DeviceType> v, int queue_idx) = 0; virtual void print() const = 0; }; + } // namespace concepts } // namespace MDR } // namespace mgard_x diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV1.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV1.hpp new file mode 100644 index 0000000000..65bc4a036e --- /dev/null +++ b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV1.hpp @@ -0,0 +1,694 @@ +#ifndef _MDR_BP_ENCODER_OPT_V1_HPP +#define _MDR_BP_ENCODER_OPT_V1_HPP + +#include "../../RuntimeX/RuntimeX.h" + +#include "BitplaneEncoderInterface.hpp" +#include + +namespace mgard_x { +namespace MDR { + +template +class BPEncoderOptV1Functor : public Functor { +public: + MGARDX_CONT + BPEncoderOptV1Functor() {} + MGARDX_CONT + BPEncoderOptV1Functor(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) { + Functor(); + } + + MGARDX_EXEC void encode_batch(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_bitplane buffer = 0; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_bitplane bit = + (v[data_idx] >> (num_bitplanes - 1 - bp_idx)) & (T_bitplane)1; + buffer += bit << BATCH_SIZE - 1 - data_idx; + } + encoded[bp_idx] = buffer; + } + } + + MGARDX_EXEC void encode_batch_with_prediction(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + T_fp data_values[BATCH_SIZE]; + for (int i = 0; i < BATCH_SIZE; i++) { + data_values[i] = 0; + } + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_bitplane buffer = 0; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_bitplane cur_bit = + (v[data_idx] >> (num_bitplanes - 1 - bp_idx)) & (T_bitplane)1; + if (bp_idx == 0) { + buffer += cur_bit << BATCH_SIZE - 1 - data_idx; + } else { + if constexpr (NegaBinary) { + data_values[data_idx] = data_values[data_idx] * (-2); + } else { + data_values[data_idx] = data_values[data_idx] * 2; + } + if (data_idx) { + T_bitplane pred_bit = 0; + T_fp d0 = abs((T_sfp)data_values[data_idx - 1] - + (T_sfp)data_values[data_idx]); + T_fp d1 = abs((T_sfp)data_values[data_idx - 1] - + (T_sfp)data_values[data_idx] - 1); + if (d0 > d1) { + pred_bit = 1; + } + if (pred_bit != cur_bit) { + buffer += (T_bitplane)1 << BATCH_SIZE - 1 - data_idx; + } + } else { + buffer += cur_bit << BATCH_SIZE - 1 - data_idx; + } + data_values[data_idx] += cur_bit; + } + } + encoded[bp_idx] = buffer; + } + } + + MGARDX_EXEC void error_collect_binary(T_data *shifted_data, T_error *errors, + int num_bitplanes, int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_error diff = (T_error)(fp_data & mask) + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void error_collect_negabinary(T_data *shifted_data, + T_error *errors, int num_bitplanes, + int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_fp ngb_data = Math::binary2negabinary((T_sfp)data); + T_error diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void EncodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_fp fp_sign[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_bitplane encoded_sign[1]; + T_error errors[MAX_BITPLANES + 1]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + // SIZE batch_idx = gid; + // if (batch_idx < num_batches) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = 0; + if (batch_idx * BATCH_SIZE + data_idx < n) { + data = *v(batch_idx * BATCH_SIZE + data_idx); + } + shifted_data[data_idx] = ldexp(data, num_bitplanes - exp); + fp_data[data_idx] = (T_fp)fabs(shifted_data[data_idx]); + fp_sign[data_idx] = (T_fp)(signbit(data) == 0 ? 0 : 1); + // fp_data[data_idx] = (T_fp)fabs(shifted_data[data_idx]); + // if (batch_idx == 0) { + // printf("fp_data[data_idx]: %llu\n", fp_data[data_idx]); + // } + // printf("%f: ", data); print_bits(fp_data[data_idx], b); + // printf("data: %f, fp_data[data_idx]: %llu, signbit(data): %lld, + // fp_sign[data_idx]: %llu \n", data, fp_data[data_idx], signbit(data), + // fp_sign[data_idx]); + } + // encode data + encode_batch(fp_data, encoded_data, num_bitplanes); + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + *encoded_bitplanes(bp_idx, batch_idx) = encoded_data[bp_idx]; + // if (batch_idx == 0) { + // printf("encoded_data: %llu\n", encoded_data[bp_idx]); + // } + // print_bits(encoded_bitplanes[bp_idx * b + batch_idx * 2], + // batch_size); + } + // encode sign + encode_batch(fp_sign, encoded_sign, 1); + + // if (batch_idx == 0) { + // printf("encoded_sign: %u\n", encoded_sign[0]); + // } + + *encoded_bitplanes(0, num_batches + batch_idx) = encoded_sign[0]; + // set rest of the bitplanes to 0 + for (int bp_idx = 1; bp_idx < num_bitplanes; bp_idx++) { + *encoded_bitplanes(bp_idx, num_batches + batch_idx) = (T_bitplane)0; + } + // // encode sign + // encode_batch(signs, encoded_sign, BATCH_SIZE, 1); + // print_bits(encoded_bitplanes[0 * b + batch_idx * 2 + 1], batch_size); + + if constexpr (CollectError) { + error_collect_binary(shifted_data, errors, num_bitplanes, exp); + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + *level_errors_workspace(bp_idx, batch_idx) = errors[bp_idx]; + } + } + } + } + + MGARDX_EXEC void EncodeNegaBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_error errors[MAX_BITPLANES + 1]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + exp += 2; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = 0; + if (batch_idx * BATCH_SIZE + data_idx < n) { + data = *v(batch_idx * BATCH_SIZE + data_idx); + } + shifted_data[data_idx] = ldexp(data, num_bitplanes - exp); + fp_data[data_idx] = + Math::binary2negabinary((T_sfp)shifted_data[data_idx]); + // fp_data[data_idx] = (T_fp)fabs(shifted_data[data_idx]); + + // printf("%f: ", data); print_bits(fp_data[data_idx], b); + } + // encode data + encode_batch(fp_data, encoded_data, num_bitplanes); + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + *encoded_bitplanes(bp_idx, batch_idx) = encoded_data[bp_idx]; + // print_bits(encoded_bitplanes[bp_idx * b + batch_idx * 2], + // batch_size); + } + + if constexpr (CollectError) { + error_collect_negabinary(shifted_data, errors, num_bitplanes, exp); + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + *level_errors_workspace(bp_idx, batch_idx) = errors[bp_idx]; + } + } + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + EncodeNegaBinary(); + } else { + EncodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPEncoderOptV1Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp encoder"; + MGARDX_CONT + BPEncoderOptV1Kernel(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) {} + + using FunctorType = + BPEncoderOptV1Functor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + FunctorType functor(n, num_bitplanes, abs_max, v, encoded_bitplanes, + level_errors_workspace); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 16; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (n - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; +}; + +template +class BPDecoderOptV1Functor : public Functor { +public: + MGARDX_CONT + BPDecoderOptV1Functor() {} + MGARDX_CONT + BPDecoderOptV1Functor(SIZE n, SIZE starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, + SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) { + Functor(); + } + + MGARDX_EXEC void decode_batch(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_fp buffer = 0; + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_fp bit = (encoded[bp_idx] >> (BATCH_SIZE - 1 - data_idx)) & (T_fp)1; + buffer += bit << (num_bitplanes - 1 - bp_idx); + } + v[data_idx] = buffer; + } + } + + MGARDX_EXEC void DecodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_fp fp_sign[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_bitplane encoded_sign[MAX_BITPLANES]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + int ending_bitplane = starting_bitplane + num_bitplanes; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = + *encoded_bitplanes(starting_bitplane + bp_idx, batch_idx); + // print_bits(encoded_data[bp_idx], batch_size); + } + // encode data + decode_batch(fp_data, encoded_data, num_bitplanes); + + if (starting_bitplane == 0) { + // decode sign + encoded_sign[0] = *encoded_bitplanes(0, num_batches + batch_idx); + decode_batch(fp_sign, encoded_sign, 1); + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + *signs(batch_idx * BATCH_SIZE + data_idx) = fp_sign[data_idx]; + } + } else { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + fp_sign[data_idx] = *signs(batch_idx * BATCH_SIZE + data_idx); + } + } + + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = ldexp((T_data)fp_data[data_idx], -ending_bitplane + exp); + if (batch_idx * BATCH_SIZE + data_idx < n) { + *v(batch_idx * BATCH_SIZE + data_idx) = + fp_sign[data_idx] ? -data : data; + } + // printf("data: %f, fp_data[data_idx]: %llu\n", *v(batch_idx * + // BATCH_SIZE + data_idx), fp_data[data_idx]); printf("%f: ", data); + // print_bits(fp_data[data_idx], b); + } + } + } + + MGARDX_EXEC void DecodeNegaBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + exp += 2; + + int ending_bitplane = starting_bitplane + num_bitplanes; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = + *encoded_bitplanes(starting_bitplane + bp_idx, batch_idx); + // print_bits(encoded_data[bp_idx], batch_size); + } + // encode data + decode_batch(fp_data, encoded_data, num_bitplanes); + + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = ldexp( + (T_data)Math::negabinary2binary(fp_data[data_idx]), + -ending_bitplane + exp); + if (batch_idx * BATCH_SIZE + data_idx < n) { + *v(batch_idx * BATCH_SIZE + data_idx) = + ending_bitplane % 2 != 0 ? -data : data; + } + // printf("%f: ", data); print_bits(fp_data[data_idx], b); + } + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + DecodeNegaBinary(); + } else { + DecodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + SIZE starting_bitplane; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPDecoderOptV1Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp decoder"; + MGARDX_CONT + BPDecoderOptV1Kernel(SIZE n, SIZE starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, + SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) {} + + using FunctorType = BPDecoderOptV1Functor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + + FunctorType functor(n, starting_bitplane, num_bitplanes, abs_max, + encoded_bitplanes, signs, v); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 8; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (n - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + SIZE starting_bitplane; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; +}; + +// general bitplane encoder that encodes data by block using T_stream type +// buffer +template +class BPEncoderOptV1 + : public concepts::BitplaneEncoderInterface { +public: + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + using T_sfp = typename std::conditional::value, + int64_t, int32_t>::type; + using T_fp = typename std::conditional::value, + uint64_t, uint32_t>::type; + + BPEncoderOptV1() : initialized(false) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + } + BPEncoderOptV1(Hierarchy &hierarchy) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + Adapt(hierarchy, 0); + DeviceRuntime::SyncQueue(0); + } + + static SIZE bitplane_length(SIZE n) { + if constexpr (!NegaBinary) { + return num_blocks(n) * 2; + } else { + return num_blocks(n); + } + } + + static SIZE num_blocks(SIZE n) { + const SIZE batch_size = sizeof(T_bitplane) * 8; + SIZE num_blocks = (n - 1) / batch_size + 1; + return num_blocks; + } + + void Adapt(Hierarchy &hierarchy, int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + SIZE max_level_num_elems = + round_up(hierarchy.level_num_elems(hierarchy.l_target()), BATCH_SIZE); + + level_errors_work_array.resize( + {MAX_BITPLANES + 1, num_blocks(max_level_num_elems)}, queue_idx); + DeviceCollective::Sum( + num_blocks(max_level_num_elems), SubArray<1, T_error, DeviceType>(), + SubArray<1, T_error, DeviceType>(), level_error_sum_work_array, false, + queue_idx); + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + Hierarchy hierarchy(shape, Config()); + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + size_t size = 0; + size += hierarchy.EstimateMemoryFootprint(shape); + size += + (MAX_BITPLANES + 1) * num_blocks(max_level_num_elems) * sizeof(T_error); + for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { + size += hierarchy.level_num_elems(level_idx) * sizeof(bool); + } + return size; + } + + void encode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, T_error, DeviceType> level_errors, int queue_idx) { + + SubArray<2, T_error, DeviceType> level_errors_work(level_errors_work_array); + + DeviceLauncher::Execute( + BPEncoderOptV1Kernel( + n, num_bitplanes, abs_max, v, encoded_bitplanes, level_errors_work), + queue_idx); + + if constexpr (CollectError) { + SIZE reduce_size = num_blocks(n); + for (int i = 0; i < num_bitplanes + 1; i++) { + SubArray<1, T_error, DeviceType> curr_errors({reduce_size}, + level_errors_work(i, 0)); + SubArray<1, T_error, DeviceType> sum_error({1}, level_errors(i)); + DeviceCollective::Sum(reduce_size, curr_errors, sum_error, + level_error_sum_work_array, true, + queue_idx); + } + } + } + + void decode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) {} + + // decode the data and record necessary information for progressiveness + void progressive_decode(SIZE n, int starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> level_signs, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) { + + if (num_bitplanes > 0) { + DeviceLauncher::Execute( + BPDecoderOptV1Kernel(n, starting_bitplane, num_bitplanes, + abs_max, encoded_bitplanes, + level_signs, v), + queue_idx); + } + } + + void print() const { std::cout << "Grouped bitplane encoder" << std::endl; } + +private: + bool initialized; + Hierarchy *hierarchy; + Array<2, T_error, DeviceType> level_errors_work_array; + Array<1, Byte, DeviceType> level_error_sum_work_array; +}; +} // namespace MDR +} // namespace mgard_x +#endif diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV1a.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV1a.hpp new file mode 100644 index 0000000000..5ead43c479 --- /dev/null +++ b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV1a.hpp @@ -0,0 +1,649 @@ +#ifndef _MDR_BP_ENCODER_OPT_V1a_HPP +#define _MDR_BP_ENCODER_OPT_V1a_HPP + +#include "../../RuntimeX/RuntimeX.h" + +#include "BitplaneEncoderInterface.hpp" +#include + +namespace mgard_x { +namespace MDR { + +template +class BPEncoderOptV1aFunctor : public Functor { +public: + MGARDX_CONT + BPEncoderOptV1aFunctor() {} + MGARDX_CONT + BPEncoderOptV1aFunctor( + SIZE n, int num_bitplanes, SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) { + Functor(); + } + + MGARDX_EXEC void encode_batch(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_bitplane buffer = 0; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_bitplane bit = (v[data_idx] >> (num_bitplanes - 1 - bp_idx)) & 1u; + buffer += bit << BATCH_SIZE - 1 - data_idx; + } + encoded[bp_idx] = buffer; + } + } + + MGARDX_EXEC void error_collect_binary(T_data *shifted_data, T_error *errors, + int num_bitplanes, int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_error diff = (T_error)(fp_data & mask) + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void error_collect_negabinary(T_data *shifted_data, + T_error *errors, int num_bitplanes, + int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_fp ngb_data = Math::binary2negabinary((T_sfp)data); + T_error diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void EncodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + SIZE warp_size = 32; + SIZE warp_id = FunctorBase::GetThreadIdX() / warp_size; + SIZE lane_id = FunctorBase::GetThreadIdX() % warp_size; + + // T_data *sm = (T_data *)FunctorBase::GetSharedMemory(); + + // T_data *sm_shifted_data_warp = sm + warp_id * warp_size * BATCH_SIZE; + + // T_data *sm_shifted_data = sm_shifted_data_warp + lane_id * BATCH_SIZE; + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_fp fp_sign[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_bitplane encoded_sign[1]; + T_error errors[MAX_BITPLANES + 1]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + // using WarpExchangeT = + // cub::WarpExchange; + + // Allocate shared memory for WarpExchange + // __shared__ typename WarpExchangeT::TempStorage temp_storage[8]; + // WarpExchangeT(temp_storage[warp_id]).StripedToBlocked(shifted_data, + // shifted_data); + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + // SIZE batch_idx = gid; + SIZE batch_idx_warp = (batch_idx / warp_size) * warp_size; + SIZE load_data_idx = batch_idx_warp * BATCH_SIZE + lane_id; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = *v(load_data_idx + data_idx * BATCH_SIZE); + shifted_data[data_idx] = ldexp(data, num_bitplanes - exp); + fp_data[data_idx] = (T_fp)fabs(shifted_data[data_idx]); + fp_sign[data_idx] = (T_fp)(signbit(data) == 0 ? 0 : 1); + } + + // if (batch_idx < num_batches) { + // encode data + encode_batch(fp_data, encoded_data, num_bitplanes); + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + *encoded_bitplanes(bp_idx, batch_idx) = encoded_data[bp_idx]; + } + // encode sign + encode_batch(fp_sign, encoded_sign, 1); + + *encoded_bitplanes(0, num_batches + batch_idx) = encoded_sign[0]; + // set rest of the bitplanes to 0 + for (int bp_idx = 1; bp_idx < num_bitplanes; bp_idx++) { + *encoded_bitplanes(bp_idx, num_batches + batch_idx) = (T_bitplane)0; + } + + if constexpr (CollectError) { + error_collect_binary(shifted_data, errors, num_bitplanes, exp); + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + *level_errors_workspace(bp_idx, batch_idx) = errors[bp_idx]; + } + } + } + } + + MGARDX_EXEC void EncodeNegaBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + SIZE warp_size = 32; + SIZE warp_id = FunctorBase::GetThreadIdX() / warp_size; + SIZE lane_id = FunctorBase::GetThreadIdX() % warp_size; + + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_error errors[MAX_BITPLANES + 1]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + exp += 2; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + SIZE batch_idx_warp = (batch_idx / warp_size) * warp_size; + SIZE load_data_idx = batch_idx_warp * BATCH_SIZE + lane_id; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = *v(load_data_idx + data_idx * BATCH_SIZE); + shifted_data[data_idx] = ldexp(data, num_bitplanes - exp); + fp_data[data_idx] = + Math::binary2negabinary((T_sfp)shifted_data[data_idx]); + } + // encode data + encode_batch(fp_data, encoded_data, num_bitplanes); + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + *encoded_bitplanes(bp_idx, batch_idx) = encoded_data[bp_idx]; + // print_bits(encoded_bitplanes[bp_idx * b + batch_idx * 2], + // batch_size); + } + + if constexpr (CollectError) { + error_collect_negabinary(shifted_data, errors, num_bitplanes, exp); + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + *level_errors_workspace(bp_idx, batch_idx) = errors[bp_idx]; + } + } + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + EncodeNegaBinary(); + } else { + EncodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + + size_t size = 0; + // size += sizeof(T_data) * BATCH_SIZE * 256; + return size; + } + +private: + // parameters + SIZE n; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPEncoderOptV1aKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static bool ConfigTask() { return false; } + constexpr static std::string_view Name = "grouped bp encoder"; + MGARDX_CONT + BPEncoderOptV1aKernel(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) {} + + using FunctorType = + BPEncoderOptV1aFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + n = std::max((n / 32) * 32, (SIZE)1); + FunctorType functor(n, num_bitplanes, abs_max, v, encoded_bitplanes, + level_errors_workspace); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 16; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (n - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; +}; + +template +class BPDecoderOptV1aFunctor : public Functor { +public: + MGARDX_CONT + BPDecoderOptV1aFunctor() {} + MGARDX_CONT + BPDecoderOptV1aFunctor(SIZE n, int starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, + SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) { + Functor(); + } + + MGARDX_EXEC void decode_batch(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_fp buffer = 0; + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_fp bit = (encoded[bp_idx] >> (BATCH_SIZE - 1 - data_idx)) & 1u; + buffer += bit << (num_bitplanes - 1 - bp_idx); + } + v[data_idx] = buffer; + } + } + + MGARDX_EXEC void DecodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_fp fp_sign[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_bitplane encoded_sign[MAX_BITPLANES]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + int ending_bitplane = starting_bitplane + num_bitplanes; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = + *encoded_bitplanes(starting_bitplane + bp_idx, batch_idx); + // print_bits(encoded_data[bp_idx], batch_size); + } + // encode data + decode_batch(fp_data, encoded_data, num_bitplanes); + + if (starting_bitplane == 0) { + // decode sign + encoded_sign[0] = *encoded_bitplanes(0, num_batches + batch_idx); + decode_batch(fp_sign, encoded_sign, 1); + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + *signs(batch_idx * BATCH_SIZE + data_idx) = fp_sign[data_idx]; + } + } else { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + fp_sign[data_idx] = *signs(batch_idx * BATCH_SIZE + data_idx); + } + } + + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = ldexp((T_data)fp_data[data_idx], -ending_bitplane + exp); + if (batch_idx * BATCH_SIZE + data_idx < n) { + *v(batch_idx * BATCH_SIZE + data_idx) = + fp_sign[data_idx] ? -data : data; + } + // printf("data: %f, fp_data[data_idx]: %llu\n", *v(batch_idx * + // BATCH_SIZE + data_idx), fp_data[data_idx]); printf("%f: ", data); + // print_bits(fp_data[data_idx], b); + } + } + } + + MGARDX_EXEC void DecodeNegaBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + exp += 2; + + int ending_bitplane = starting_bitplane + num_bitplanes; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = + *encoded_bitplanes(starting_bitplane + bp_idx, batch_idx); + // print_bits(encoded_data[bp_idx], batch_size); + } + // encode data + decode_batch(fp_data, encoded_data, num_bitplanes); + + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = ldexp( + (T_data)Math::negabinary2binary(fp_data[data_idx]), + -ending_bitplane + exp); + if (batch_idx * BATCH_SIZE + data_idx < n) { + *v(batch_idx * BATCH_SIZE + data_idx) = + ending_bitplane % 2 != 0 ? -data : data; + } + // printf("%f: ", data); print_bits(fp_data[data_idx], b); + } + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + DecodeNegaBinary(); + } else { + DecodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + int starting_bitplane; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPDecoderOptV1aKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp decoder"; + MGARDX_CONT + BPDecoderOptV1aKernel(SIZE n, SIZE starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, + SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) {} + + using FunctorType = BPDecoderOptV1aFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + + FunctorType functor(n, starting_bitplane, num_bitplanes, abs_max, + encoded_bitplanes, signs, v); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 8; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (n - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int starting_bitplane; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; +}; + +// general bitplane encoder that encodes data by block using T_stream type +// buffer +template +class BPEncoderOptV1a + : public concepts::BitplaneEncoderInterface { +public: + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + using T_sfp = typename std::conditional::value, + int64_t, int32_t>::type; + using T_fp = typename std::conditional::value, + uint64_t, uint32_t>::type; + + BPEncoderOptV1a() : initialized(false) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + } + BPEncoderOptV1a(Hierarchy &hierarchy) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + Adapt(hierarchy, 0); + DeviceRuntime::SyncQueue(0); + } + + static SIZE bitplane_length(SIZE n) { + if constexpr (!NegaBinary) { + return num_blocks(n) * 2; + } else { + return num_blocks(n); + } + } + + static SIZE num_blocks(SIZE n) { + const SIZE batch_size = sizeof(T_bitplane) * 8; + SIZE num_blocks = (n - 1) / batch_size + 1; + return num_blocks; + } + + void Adapt(Hierarchy &hierarchy, int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + + level_errors_work_array.resize( + {MAX_BITPLANES + 1, num_blocks(max_level_num_elems)}, queue_idx); + DeviceCollective::Sum( + num_blocks(max_level_num_elems), SubArray<1, T_error, DeviceType>(), + SubArray<1, T_error, DeviceType>(), level_error_sum_work_array, false, + queue_idx); + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + Hierarchy hierarchy(shape, Config()); + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + size_t size = 0; + size += hierarchy.EstimateMemoryFootprint(shape); + size += + (MAX_BITPLANES + 1) * num_blocks(max_level_num_elems) * sizeof(T_error); + for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { + size += hierarchy.level_num_elems(level_idx) * sizeof(bool); + } + return size; + } + + void encode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, T_error, DeviceType> level_errors, int queue_idx) { + + SubArray<2, T_error, DeviceType> level_errors_work(level_errors_work_array); + + DeviceLauncher::Execute( + BPEncoderOptV1aKernel( + n, num_bitplanes, abs_max, v, encoded_bitplanes, level_errors_work), + queue_idx); + + if constexpr (CollectError) { + SIZE reduce_size = num_blocks(n); + for (int i = 0; i < num_bitplanes + 1; i++) { + SubArray<1, T_error, DeviceType> curr_errors({reduce_size}, + level_errors_work(i, 0)); + SubArray<1, T_error, DeviceType> sum_error({1}, level_errors(i)); + DeviceCollective::Sum(reduce_size, curr_errors, sum_error, + level_error_sum_work_array, true, + queue_idx); + } + } + } + + void decode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) {} + + // decode the data and record necessary information for progressiveness + void progressive_decode(SIZE n, int starting_bitplanes, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> level_signs, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) { + + if (num_bitplanes > 0) { + DeviceLauncher::Execute( + BPDecoderOptV1aKernel(n, starting_bitplanes, + num_bitplanes, abs_max, + encoded_bitplanes, level_signs, v), + queue_idx); + } + } + + void print() const { std::cout << "Grouped bitplane encoder" << std::endl; } + +private: + bool initialized; + Hierarchy *hierarchy; + Array<2, T_error, DeviceType> level_errors_work_array; + Array<1, Byte, DeviceType> level_error_sum_work_array; +}; +} // namespace MDR +} // namespace mgard_x +#endif diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV1b.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV1b.hpp new file mode 100644 index 0000000000..21c1c6cc9b --- /dev/null +++ b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV1b.hpp @@ -0,0 +1,740 @@ +#ifndef _MDR_BP_ENCODER_OPT_V1b_HPP +#define _MDR_BP_ENCODER_OPT_V1b_HPP + +#include "../../RuntimeX/RuntimeX.h" + +#include "BitplaneEncoderInterface.hpp" +#include + +namespace mgard_x { +namespace MDR { + +template +class BPEncoderOptV1bFunctor : public Functor { +public: + MGARDX_CONT + BPEncoderOptV1bFunctor() {} + MGARDX_CONT + BPEncoderOptV1bFunctor( + SIZE n, SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), abs_max(abs_max), encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) { + Functor(); + } + + MGARDX_EXEC void encode_batch(T_fp *v, T_bitplane *encoded) { + +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + T_bitplane buffer = 0; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_bitplane bit = + (v[data_idx] >> (NUM_BITPLANES - 1 - bp_idx)) & (T_bitplane)1; + buffer |= bit << BATCH_SIZE - 1 - data_idx; + } + encoded[bp_idx] = buffer; + } + } + + MGARDX_EXEC void error_collect_binary(T_data *shifted_data, T_error *errors, + int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_error diff = (T_error)(fp_data & mask) + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[NUM_BITPLANES - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < NUM_BITPLANES + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-NUM_BITPLANES + exp)); + } + } + + MGARDX_EXEC void error_collect_negabinary(T_data *shifted_data, + T_error *errors, int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_fp ngb_data = Math::binary2negabinary((T_sfp)data); + T_error diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[NUM_BITPLANES - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < NUM_BITPLANES + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-NUM_BITPLANES + exp)); + } + } + + MGARDX_EXEC void EncodeBinary() { + SIZE batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE num_full_batches = n / BATCH_SIZE; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[NUM_BITPLANES]; + T_bitplane encoded_sign = 0; + T_error errors[NUM_BITPLANES + 1]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + if (batch_idx >= num_full_batches) { + return; + } + + if (exp > 0) { +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = *v(data_idx * num_full_batches + batch_idx); + // this can cause overflow + shifted_data[data_idx] = data * ((T_fp)1 << NUM_BITPLANES - exp); + // ldexp without constant argument is slow + // shifted_data[data_idx] = ldexp(data, NUM_BITPLANES - exp); + fp_data[data_idx] = (T_fp)fabs(shifted_data[data_idx]); + + // if (num_full_batches == 1) printf("data: %f * %d %d, shifted_data: %f + // fp_data: %llu \n", data, NUM_BITPLANES, exp, shifted_data[data_idx], + // fp_data[data_idx]); + } + } else { +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = *v(data_idx * num_full_batches + batch_idx); + shifted_data[data_idx] = data * pow(2, NUM_BITPLANES - exp); + fp_data[data_idx] = (T_fp)fabs(shifted_data[data_idx]); + } + } + + // encode sign + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + encoded_sign += (T_fp)(signbit(shifted_data[data_idx]) == 0 ? 0 : 1) + << (BATCH_SIZE - 1 - data_idx); + } + // encode data + encode_batch(fp_data, encoded_data); +// store data +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + // if (num_full_batches == 1) printf("encoded_data: %u\n", + // encoded_data[bp_idx]); + *encoded_bitplanes(bp_idx, batch_idx) = encoded_data[bp_idx]; + } + // store sign + *encoded_bitplanes(0, num_full_batches + batch_idx) = encoded_sign; +// set rest of the bitplanes to 0 +#pragma unroll + for (int bp_idx = 1; bp_idx < NUM_BITPLANES; bp_idx++) { + *encoded_bitplanes(bp_idx, num_full_batches + batch_idx) = (T_bitplane)0; + } + if constexpr (CollectError) { + error_collect_binary(shifted_data, errors, exp); + for (int bp_idx = 0; bp_idx < NUM_BITPLANES + 1; bp_idx++) { + *level_errors_workspace(bp_idx, batch_idx) = errors[bp_idx]; + } + } + } + + MGARDX_EXEC void EncodeNegaBinary() { + SIZE batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE num_full_batches = n / BATCH_SIZE; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[NUM_BITPLANES]; + T_error errors[NUM_BITPLANES + 1]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + exp += 2; + + if (batch_idx >= num_full_batches) { + return; + } + + if (exp > 0) { +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = 0; + data = *v(data_idx * num_full_batches + batch_idx); + // This can cause overflow + shifted_data[data_idx] = data * ((T_fp)1 << NUM_BITPLANES - exp); + // ldexp without constant argument is slow + // shifted_data[data_idx] = ldexp(data, NUM_BITPLANES - exp); + fp_data[data_idx] = + Math::binary2negabinary((T_sfp)shifted_data[data_idx]); + } + } else { +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = 0; + data = *v(data_idx * num_full_batches + batch_idx); + shifted_data[data_idx] = data * pow(2, NUM_BITPLANES - exp); + // ldexp without constant argument is slow + // shifted_data[data_idx] = ldexp(data, NUM_BITPLANES - exp); + fp_data[data_idx] = + Math::binary2negabinary((T_sfp)shifted_data[data_idx]); + } + } + + // encode data + encode_batch(fp_data, encoded_data); +// store data +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + *encoded_bitplanes(bp_idx, batch_idx) = encoded_data[bp_idx]; + } + + if constexpr (CollectError) { + error_collect_negabinary(shifted_data, errors, exp); +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES + 1; bp_idx++) { + *level_errors_workspace(bp_idx, batch_idx) = errors[bp_idx]; + } + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + EncodeNegaBinary(); + } else { + EncodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; +}; + +template +class BPEncoderOptV1bKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp encoder"; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + MGARDX_CONT + BPEncoderOptV1bKernel(SIZE n, SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), abs_max(abs_max), encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) {} + + using FunctorType = BPEncoderOptV1bFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + FunctorType functor(n, abs_max, v, encoded_bitplanes, + level_errors_workspace); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE total_thread = std::max((SIZE)1, n / BATCH_SIZE); + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (total_thread - 1) / tbx + 1; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; +}; + +template +class BPDecoderOptV1bFunctor : public Functor { +public: + MGARDX_CONT + BPDecoderOptV1bFunctor() {} + MGARDX_CONT + BPDecoderOptV1bFunctor(SIZE n, int starting_bitplane, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, + SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) { + Functor(); + } + + MGARDX_EXEC void decode_batch(T_fp *v, T_bitplane *encoded) { +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_fp buffer = 0; + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + T_fp bit = (encoded[bp_idx] >> (BATCH_SIZE - 1 - data_idx)) & (T_fp)1; + buffer += bit << (NUM_BITPLANES - 1 - bp_idx); + // printf("bit: %llu, buffer: %llu\n", bit, buffer); + } + v[data_idx] = buffer; + } + } + + MGARDX_EXEC void DecodeBinary() { + SIZE batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE num_full_batches = n / BATCH_SIZE; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_fp fp_sign[BATCH_SIZE]; + T_bitplane encoded_data[NUM_BITPLANES]; + T_bitplane encoded_sign; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + if (batch_idx >= num_full_batches) { + return; + } + + int ending_bitplane = starting_bitplane + NUM_BITPLANES; + +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + encoded_data[bp_idx] = + *encoded_bitplanes(starting_bitplane + bp_idx, batch_idx); + // if (num_full_batches == 1) printf("encoded_data: %u\n", + // encoded_data[bp_idx]); + } + // decode data + decode_batch(fp_data, encoded_data); + + if (starting_bitplane == 0) { + // decode sign + encoded_sign = *encoded_bitplanes(0, num_full_batches + batch_idx); +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + fp_sign[data_idx] = + (encoded_sign >> (BATCH_SIZE - 1 - data_idx)) & (T_fp)1; + *signs(data_idx * num_full_batches + batch_idx) = fp_sign[data_idx]; + } + } else { +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + fp_sign[data_idx] = *signs(data_idx * num_full_batches + batch_idx); + } + } +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + shifted_data[data_idx] = (T_data)fp_data[data_idx]; + // It is beneficial to use pow instead of ldexp + T_data data = shifted_data[data_idx] * pow(2, -ending_bitplane + exp); + // T_data data = ldexp(shifted_data[data_idx], -ending_bitplane + exp); + data = fp_sign[data_idx] ? -data : data; + *v(data_idx * num_full_batches + batch_idx) = data; + + // if (num_full_batches == 1) printf("%llu %f %f\n", fp_data[data_idx], + // shifted_data[data_idx], data); + } + } + + MGARDX_EXEC void DecodeNegaBinary() { + SIZE batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE num_full_batches = n / BATCH_SIZE; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[NUM_BITPLANES]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + exp += 2; + + if (batch_idx >= num_full_batches) { + return; + } + + int ending_bitplane = starting_bitplane + NUM_BITPLANES; + +// load bitplanes +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + encoded_data[bp_idx] = + *encoded_bitplanes(starting_bitplane + bp_idx, batch_idx); + // print_bits(encoded_data[bp_idx], batch_size); + } + // decode data + decode_batch(fp_data, encoded_data); + +// store data +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + shifted_data[data_idx] = + Math::negabinary2binary(fp_data[data_idx]); + // No noticing difference between the two + T_data data = shifted_data[data_idx] * pow(2, -ending_bitplane + exp); + // T_data data = ldexp(shifted_data[data_idx], -ending_bitplane + exp); + data = ending_bitplane % 2 != 0 ? -data : data; + *v(data_idx * num_full_batches + batch_idx) = data; + // printf("%f: ", data); print_bits(fp_data[data_idx], b); + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + DecodeNegaBinary(); + } else { + DecodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + int starting_bitplane; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPDecoderOptV1bKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp decoder"; + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + MGARDX_CONT + BPDecoderOptV1bKernel(SIZE n, int starting_bitplane, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, + SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) {} + + using FunctorType = + BPDecoderOptV1bFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + + FunctorType functor(n, starting_bitplane, abs_max, encoded_bitplanes, signs, + v); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE total_thread = std::max((SIZE)1, n / BATCH_SIZE); + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (total_thread - 1) / tbx + 1; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int starting_bitplane; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; +}; + +// general bitplane encoder that encodes data by block using T_stream type +// buffer +template +class BPEncoderOptV1b + : public concepts::BitplaneEncoderInterface { +public: + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + using T_sfp = typename std::conditional::value, + int64_t, int32_t>::type; + using T_fp = typename std::conditional::value, + uint64_t, uint32_t>::type; + + BPEncoderOptV1b() : initialized(false) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + } + BPEncoderOptV1b(Hierarchy &hierarchy) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + Adapt(hierarchy, 0); + DeviceRuntime::SyncQueue(0); + } + + static SIZE bitplane_length(SIZE n) { + if constexpr (!NegaBinary) { + return num_blocks(n) * 2; + } else { + return num_blocks(n); + } + } + + static SIZE num_blocks(SIZE n) { + const SIZE batch_size = sizeof(T_bitplane) * 8; + SIZE num_blocks = (n - 1) / batch_size + 1; + return num_blocks; + } + + void Adapt(Hierarchy &hierarchy, int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + SIZE max_level_num_elems = + round_up(hierarchy.level_num_elems(hierarchy.l_target()), BATCH_SIZE); + + level_errors_work_array.resize( + {MAX_BITPLANES + 1, num_blocks(max_level_num_elems)}, queue_idx); + DeviceCollective::Sum( + num_blocks(max_level_num_elems), SubArray<1, T_error, DeviceType>(), + SubArray<1, T_error, DeviceType>(), level_error_sum_work_array, false, + queue_idx); + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + Hierarchy hierarchy(shape, Config()); + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + size_t size = 0; + size += hierarchy.EstimateMemoryFootprint(shape); + size += + (MAX_BITPLANES + 1) * num_blocks(max_level_num_elems) * sizeof(T_error); + for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { + size += hierarchy.level_num_elems(level_idx) * sizeof(bool); + } + return size; + } + + // TODO: remove num_bitplanes in the future + void encode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, T_error, DeviceType> level_errors, int queue_idx) { + + if (n % BATCH_SIZE != 0) { + log::err("BPEncoderV1b: n is not a multiple of BATCH_SIZE"); + exit(-1); + } + SubArray<2, T_error, DeviceType> level_errors_work(level_errors_work_array); + + DeviceLauncher::Execute( + BPEncoderOptV1bKernel(n, abs_max, v, encoded_bitplanes, + level_errors_work), + queue_idx); + + if constexpr (CollectError) { + SIZE reduce_size = num_blocks(n); + for (int i = 0; i < MAX_BITPLANES + 1; i++) { + SubArray<1, T_error, DeviceType> curr_errors({reduce_size}, + level_errors_work(i, 0)); + SubArray<1, T_error, DeviceType> sum_error({1}, level_errors(i)); + DeviceCollective::Sum(reduce_size, curr_errors, sum_error, + level_error_sum_work_array, true, + queue_idx); + } + } + } + + void decode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) {} + + // decode the data and record necessary information for progressiveness + void progressive_decode(SIZE n, int starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> level_signs, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) { + + // if (num_bitplanes > 0) { + // DeviceLauncher::Execute( + // BPDecoderOptV1bKernel(n, starting_bitplane, + // num_bitplanes, + // abs_max, encoded_bitplanes, + // level_signs, v), + // queue_idx); + // } + +#define V1B_DECODE(NUM_BITPLANES) \ + if (num_bitplanes == NUM_BITPLANES) { \ + DeviceLauncher::Execute( \ + BPDecoderOptV1bKernel( \ + n, starting_bitplane, abs_max, encoded_bitplanes, level_signs, v), \ + queue_idx); \ + } + V1B_DECODE(1); + V1B_DECODE(2); + V1B_DECODE(3); + V1B_DECODE(4); + V1B_DECODE(5); + V1B_DECODE(6); + V1B_DECODE(7); + V1B_DECODE(8); + V1B_DECODE(9); + V1B_DECODE(10); + V1B_DECODE(11); + V1B_DECODE(12); + V1B_DECODE(13); + V1B_DECODE(14); + V1B_DECODE(15); + V1B_DECODE(16); + V1B_DECODE(17); + V1B_DECODE(18); + V1B_DECODE(19); + V1B_DECODE(20); + V1B_DECODE(21); + V1B_DECODE(22); + V1B_DECODE(23); + V1B_DECODE(24); + V1B_DECODE(25); + V1B_DECODE(26); + V1B_DECODE(27); + V1B_DECODE(28); + V1B_DECODE(29); + V1B_DECODE(30); + V1B_DECODE(31); + V1B_DECODE(32); + V1B_DECODE(33); + V1B_DECODE(34); + V1B_DECODE(35); + V1B_DECODE(36); + V1B_DECODE(37); + V1B_DECODE(38); + V1B_DECODE(39); + V1B_DECODE(40); + V1B_DECODE(41); + V1B_DECODE(42); + V1B_DECODE(43); + V1B_DECODE(44); + V1B_DECODE(45); + V1B_DECODE(46); + V1B_DECODE(47); + V1B_DECODE(48); + V1B_DECODE(49); + V1B_DECODE(50); + V1B_DECODE(51); + V1B_DECODE(52); + V1B_DECODE(53); + V1B_DECODE(54); + V1B_DECODE(55); + V1B_DECODE(56); + V1B_DECODE(57); + V1B_DECODE(58); + V1B_DECODE(59); + V1B_DECODE(60); + V1B_DECODE(61); + V1B_DECODE(62); + V1B_DECODE(63); + V1B_DECODE(64); + } + + void print() const { std::cout << "Grouped bitplane encoder" << std::endl; } + +private: + bool initialized; + Hierarchy *hierarchy; + Array<2, T_error, DeviceType> level_errors_work_array; + Array<1, Byte, DeviceType> level_error_sum_work_array; +}; +} // namespace MDR +} // namespace mgard_x +#endif diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV1b2.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV1b2.hpp new file mode 100644 index 0000000000..350273c5db --- /dev/null +++ b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV1b2.hpp @@ -0,0 +1,646 @@ +#ifndef _MDR_BP_ENCODER_OPT_V1b_HPP +#define _MDR_BP_ENCODER_OPT_V1b_HPP + +#include "../../RuntimeX/RuntimeX.h" + +#include "BitplaneEncoderInterface.hpp" +#include + +namespace mgard_x { +namespace MDR { + +template +class BPEncoderOptV1bFunctor : public Functor { +public: + MGARDX_CONT + BPEncoderOptV1bFunctor() {} + MGARDX_CONT + BPEncoderOptV1bFunctor( + SIZE n, int num_bitplanes, int exp, SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), exp(exp), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) { + Functor(); + } + + template + MGARDX_EXEC void encode_batch(T_fp *v, T_bitplane *encoded) { + for (int bp_idx = 0; bp_idx < NUN_BITPLANES; bp_idx++) { + T_bitplane buffer = 0; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_bitplane bit = (v[data_idx] >> (NUN_BITPLANES - 1 - bp_idx)) & 1u; + buffer += bit << BATCH_SIZE - 1 - data_idx; + } + encoded[bp_idx] = buffer; + } + } + + MGARDX_EXEC void error_collect_binary(T_data *shifted_data, T_error *errors, + int num_bitplanes, int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_error diff = (T_error)(fp_data & mask) + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void error_collect_negabinary(T_data *shifted_data, + T_error *errors, int num_bitplanes, + int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_fp ngb_data = Math::binary2negabinary((T_sfp)data); + T_error diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void EncodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_fp fp_sign[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_bitplane encoded_sign[1]; + T_error errors[MAX_BITPLANES + 1]; + + SIZE lane_id = FunctorBase::GetThreadIdX() % BATCH_SIZE; + + int shift_exp = num_bitplanes - exp; + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + // SIZE batch_idx = gid; + // if (batch_idx < num_batches) { + SIZE coop_batch_idx = batch_idx / BATCH_SIZE * BATCH_SIZE; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = 0; + SIZE load_idx = (data_idx + coop_batch_idx) * BATCH_SIZE + lane_id; + // SIZE load_idx = batch_idx * BATCH_SIZE + data_idx; + load_idx = load_idx < n ? load_idx : n - 1; + data = *v(load_idx); + + shifted_data[data_idx] = ldexp(data, 5); + fp_data[data_idx] = (T_fp)fabs(shifted_data[data_idx]); + fp_sign[data_idx] = (T_fp)(signbit(data) == 0 ? 0 : 1); + // if (batch_idx == 0) { + // printf("fp_data[data_idx]: %llu\n", fp_data[data_idx]); + // } + // printf("%f: ", data); print_bits(fp_data[data_idx], b); + // printf("data: %f, fp_data[data_idx]: %llu, signbit(data): %lld, + // fp_sign[data_idx]: %llu \n", data, fp_data[data_idx], signbit(data), + // fp_sign[data_idx]); + } + // encode data + encode_batch(fp_data, encoded_data); + for (int bp_idx = 0; bp_idx < MAX_BITPLANES; bp_idx++) { + *encoded_bitplanes(bp_idx, batch_idx) = encoded_data[bp_idx]; + // if (batch_idx == 0) { + // printf("encoded_data: %llu\n", encoded_data[bp_idx]); + // } + // print_bits(encoded_bitplanes[bp_idx * b + batch_idx * 2], + // batch_size); + } + // encode sign + encode_batch<1>(fp_sign, encoded_sign); + + // if (batch_idx == 0) { + // printf("encoded_sign: %u\n", encoded_sign[0]); + // } + + *encoded_bitplanes(0, num_batches + batch_idx) = encoded_sign[0]; + // set rest of the bitplanes to 0 + for (int bp_idx = 1; bp_idx < MAX_BITPLANES; bp_idx++) { + *encoded_bitplanes(bp_idx, num_batches + batch_idx) = (T_bitplane)0; + } + + if constexpr (CollectError) { + error_collect_binary(shifted_data, errors, MAX_BITPLANES, exp); + for (int bp_idx = 0; bp_idx < MAX_BITPLANES + 1; bp_idx++) { + *level_errors_workspace(bp_idx, batch_idx) = errors[bp_idx]; + } + } + } + } + + MGARDX_EXEC void EncodeNegaBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_error errors[MAX_BITPLANES + 1]; + + exp += 2; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = 0; + if (batch_idx * BATCH_SIZE + data_idx < n) { + data = *v(batch_idx * BATCH_SIZE + data_idx); + } + shifted_data[data_idx] = ldexp(data, num_bitplanes - exp); + fp_data[data_idx] = + Math::binary2negabinary((T_sfp)shifted_data[data_idx]); + // fp_data[data_idx] = (T_fp)fabs(shifted_data[data_idx]); + + // printf("%f: ", data); print_bits(fp_data[data_idx], b); + } + // encode data + encode_batch(fp_data, encoded_data, num_bitplanes); + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + *encoded_bitplanes(bp_idx, batch_idx) = encoded_data[bp_idx]; + // print_bits(encoded_bitplanes[bp_idx * b + batch_idx * 2], + // batch_size); + } + + if constexpr (CollectError) { + error_collect_negabinary(shifted_data, errors, num_bitplanes, exp); + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + *level_errors_workspace(bp_idx, batch_idx) = errors[bp_idx]; + } + } + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + EncodeNegaBinary(); + } else { + EncodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + int num_bitplanes; + int exp; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPEncoderOptV1bKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp encoder"; + MGARDX_CONT + BPEncoderOptV1bKernel(SIZE n, int num_bitplanes, int exp, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), exp(exp), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) {} + + using FunctorType = + BPEncoderOptV1bFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + FunctorType functor(n, num_bitplanes, exp, v, encoded_bitplanes, + level_errors_workspace); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 16; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (n - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int num_bitplanes; + int exp; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; +}; + +template +class BPDecoderOptV1bFunctor : public Functor { +public: + MGARDX_CONT + BPDecoderOptV1bFunctor() {} + MGARDX_CONT + BPDecoderOptV1bFunctor(SIZE n, SIZE starting_bitplane, int num_bitplanes, + int exp, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, + SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), exp(exp), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) { + Functor(); + } + + MGARDX_EXEC void decode_batch(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_fp buffer = 0; + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_fp bit = (encoded[bp_idx] >> (BATCH_SIZE - 1 - data_idx)) & 1u; + buffer += bit << (num_bitplanes - 1 - bp_idx); + } + v[data_idx] = buffer; + } + } + + MGARDX_EXEC void DecodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + SIZE lane_id = FunctorBase::GetThreadIdX() % BATCH_SIZE; + + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_fp fp_sign[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_bitplane encoded_sign[MAX_BITPLANES]; + + int ending_bitplane = starting_bitplane + num_bitplanes; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + SIZE coop_batch_idx = batch_idx / BATCH_SIZE * BATCH_SIZE; + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = + *encoded_bitplanes(starting_bitplane + bp_idx, batch_idx); + // print_bits(encoded_data[bp_idx], batch_size); + } + // encode data + decode_batch(fp_data, encoded_data, num_bitplanes); + + if (starting_bitplane == 0) { + // decode sign + encoded_sign[0] = *encoded_bitplanes(0, num_batches + batch_idx); + decode_batch(fp_sign, encoded_sign, 1); + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + SIZE store_idx = (data_idx + coop_batch_idx) * BATCH_SIZE + lane_id; + // SIZE store_idx = batch_idx * BATCH_SIZE + data_idx; + *signs(store_idx) = fp_sign[data_idx]; + } + } else { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + SIZE store_idx = (data_idx + coop_batch_idx) * BATCH_SIZE + lane_id; + // SIZE store_idx = batch_idx * BATCH_SIZE + data_idx; + fp_sign[data_idx] = *signs(store_idx); + } + } + + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = ldexp((T_data)fp_data[data_idx], -ending_bitplane + exp); + + SIZE store_idx = (data_idx + coop_batch_idx) * BATCH_SIZE + lane_id; + // SIZE store_idx = batch_idx * BATCH_SIZE + data_idx; + if (store_idx < n) { + *v(store_idx) = fp_sign[data_idx] ? -data : data; + } + // printf("data: %f, fp_data[data_idx]: %llu\n", *v(batch_idx * + // BATCH_SIZE + data_idx), fp_data[data_idx]); printf("%f: ", data); + // print_bits(fp_data[data_idx], b); + } + } + } + + MGARDX_EXEC void DecodeNegaBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + + exp += 2; + + int ending_bitplane = starting_bitplane + num_bitplanes; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = + *encoded_bitplanes(starting_bitplane + bp_idx, batch_idx); + // print_bits(encoded_data[bp_idx], batch_size); + } + // encode data + decode_batch(fp_data, encoded_data, num_bitplanes); + + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = ldexp( + (T_data)Math::negabinary2binary(fp_data[data_idx]), + -ending_bitplane + exp); + if (batch_idx * BATCH_SIZE + data_idx < n) { + *v(batch_idx * BATCH_SIZE + data_idx) = + ending_bitplane % 2 != 0 ? -data : data; + } + // printf("%f: ", data); print_bits(fp_data[data_idx], b); + } + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + DecodeNegaBinary(); + } else { + DecodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + SIZE starting_bitplane; + int num_bitplanes; + int exp; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPDecoderOptV1bKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp decoder"; + MGARDX_CONT + BPDecoderOptV1bKernel(SIZE n, SIZE starting_bitplane, int num_bitplanes, + int exp, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, + SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), exp(exp), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) {} + + using FunctorType = BPDecoderOptV1bFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + + FunctorType functor(n, starting_bitplane, num_bitplanes, exp, + encoded_bitplanes, signs, v); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 8; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (n - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + SIZE starting_bitplane; + int num_bitplanes; + int exp; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; +}; + +// general bitplane encoder that encodes data by block using T_stream type +// buffer +template +class BPEncoderOptV1b + : public concepts::BitplaneEncoderInterface { +public: + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + using T_sfp = typename std::conditional::value, + int64_t, int32_t>::type; + using T_fp = typename std::conditional::value, + uint64_t, uint32_t>::type; + + BPEncoderOptV1b() : initialized(false) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + } + BPEncoderOptV1b(Hierarchy &hierarchy) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + Adapt(hierarchy, 0); + DeviceRuntime::SyncQueue(0); + } + + static SIZE bitplane_length(SIZE n) { + if constexpr (!NegaBinary) { + return num_blocks(n) * 2; + } else { + return num_blocks(n); + } + } + + static SIZE num_blocks(SIZE n) { + const SIZE batch_size = sizeof(T_bitplane) * 8; + SIZE num_blocks = (n - 1) / batch_size + 1; + return num_blocks; + } + + void Adapt(Hierarchy &hierarchy, int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + + level_errors_work_array.resize( + {MAX_BITPLANES + 1, num_blocks(max_level_num_elems)}, queue_idx); + DeviceCollective::Sum( + num_blocks(max_level_num_elems), SubArray<1, T_error, DeviceType>(), + SubArray<1, T_error, DeviceType>(), level_error_sum_work_array, false, + queue_idx); + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + Hierarchy hierarchy(shape, Config()); + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + size_t size = 0; + size += hierarchy.EstimateMemoryFootprint(shape); + size += + (MAX_BITPLANES + 1) * num_blocks(max_level_num_elems) * sizeof(T_error); + for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { + size += hierarchy.level_num_elems(level_idx) * sizeof(bool); + } + return size; + } + + void encode(SIZE n, int num_bitplanes, int32_t exp, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, T_error, DeviceType> level_errors, int queue_idx) { + + SubArray<2, T_error, DeviceType> level_errors_work(level_errors_work_array); + + DeviceLauncher::Execute( + BPEncoderOptV1bKernel( + n, num_bitplanes, exp, v, encoded_bitplanes, level_errors_work), + queue_idx); + + if constexpr (CollectError) { + SIZE reduce_size = num_blocks(n); + for (int i = 0; i < num_bitplanes + 1; i++) { + SubArray<1, T_error, DeviceType> curr_errors({reduce_size}, + level_errors_work(i, 0)); + SubArray<1, T_error, DeviceType> sum_error({1}, level_errors(i)); + DeviceCollective::Sum(reduce_size, curr_errors, sum_error, + level_error_sum_work_array, true, + queue_idx); + } + } + } + + void decode(SIZE n, int num_bitplanes, int32_t exp, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) {} + + // decode the data and record necessary information for progressiveness + void progressive_decode(SIZE n, SIZE starting_bitplanes, int num_bitplanes, + int32_t exp, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> level_signs, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) { + + if (num_bitplanes > 0) { + DeviceLauncher::Execute( + BPDecoderOptV1bKernel(n, starting_bitplanes, + num_bitplanes, exp, + encoded_bitplanes, level_signs, v), + queue_idx); + } + } + + void print() const { std::cout << "Grouped bitplane encoder" << std::endl; } + +private: + bool initialized; + Hierarchy *hierarchy; + Array<2, T_error, DeviceType> level_errors_work_array; + Array<1, Byte, DeviceType> level_error_sum_work_array; +}; +} // namespace MDR +} // namespace mgard_x +#endif diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV2.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV2.hpp new file mode 100644 index 0000000000..af71336de8 --- /dev/null +++ b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV2.hpp @@ -0,0 +1,632 @@ +#ifndef _MDR_BP_ENCODER_OPT_V2_HPP +#define _MDR_BP_ENCODER_OPT_V2_HPP + +#include "../../RuntimeX/RuntimeX.h" + +#include "BitplaneEncoderInterface.hpp" +#include + +namespace mgard_x { +namespace MDR { + +template +class BPEncoderOptV2Functor : public Functor { +public: + MGARDX_CONT + BPEncoderOptV2Functor() {} + MGARDX_CONT + BPEncoderOptV2Functor(SIZE n, int num_bitplanes, int exp, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), exp(exp), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) { + Functor(); + } + + MGARDX_EXEC void encode_batch(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_bitplane buffer = 0; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_bitplane bit = (v[data_idx] >> (num_bitplanes - 1 - bp_idx)) & 1u; + buffer += bit << BATCH_SIZE - 1 - data_idx; + } + encoded[bp_idx] = buffer; + } + } + + MGARDX_EXEC void error_collect_binary(T_data *shifted_data, T_error *errors, + int num_bitplanes, int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_error diff = (T_error)(fp_data & mask) + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void error_collect_negabinary(T_data *shifted_data, + T_error *errors, int num_bitplanes, + int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_fp ngb_data = Math::binary2negabinary((T_sfp)data); + T_error diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void EncodeBinary() { + SIZE batch_idx = FunctorBase::GetBlockIdX(); + SIZE tid = FunctorBase::GetThreadIdX(); + + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + T_data data; + T_data shifted_data; + T_fp fp_data; + T_fp fp_sign; + T_bitplane encoded_data = 0; + T_bitplane encoded_sign; + T_error errors; + + SIZE data_idx = tid; + SIZE my_bp_idx = tid; + if (batch_idx < num_batches) { + data = 0; + if (batch_idx * BATCH_SIZE + data_idx < n) { + data = *v(batch_idx * BATCH_SIZE + data_idx); + } + shifted_data = ldexp(data, num_bitplanes - exp); + fp_data = (T_fp)fabs(shifted_data); + fp_sign = (T_fp)(signbit(data) == 0 ? 0 : 1); +#define FULL_MASK 0xffffffff + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_bitplane bit = (fp_data >> (num_bitplanes - 1 - bp_idx)) & 1u; + T_bitplane shifted_bit = bit << BATCH_SIZE - 1 - data_idx; + T_bitplane buffer = 0; + // option 1 + // for (int offset = 16; offset > 0; offset /= 2) { + // buffer |= __shfl_down_sync(FULL_MASK, shifted_bit, offset); + // } + // option 2 + buffer = __reduce_add_sync(FULL_MASK, shifted_bit); + + // option 3 + // buffer = __match_any_sync(FULL_MASK, bit); + // if (!bit) buffer ^= FULL_MASK; + + buffer = __shfl_sync(FULL_MASK, buffer, 0); + if (my_bp_idx == bp_idx) { + encoded_data = buffer; + } + } + + // if (batch_idx == 0) { + // printf("thread %llu, fp_data %u, encoded_data: %u\n", tid, fp_data, + // encoded_data); + // } + + encoded_sign = fp_sign << BATCH_SIZE - 1 - data_idx; + // option 1 + // for (int offset = 16; offset > 0; offset /= 2) { + // encoded_sign |= __shfl_down_sync(FULL_MASK, encoded_sign, offset); + // } + // option 2 + encoded_sign = __reduce_add_sync(FULL_MASK, encoded_sign); + + // if (batch_idx == 0) { + // if (my_bp_idx == 0) + // printf("thread %llu, encoded_sign %u, \n", tid, encoded_sign); + // } + + *encoded_bitplanes(my_bp_idx, batch_idx) = encoded_data; + *encoded_bitplanes(my_bp_idx, num_batches + batch_idx) = + my_bp_idx == 0 ? encoded_sign : (T_bitplane)0; + } + } + + MGARDX_EXEC void EncodeNegaBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_error errors[MAX_BITPLANES + 1]; + + exp += 2; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = 0; + if (batch_idx * BATCH_SIZE + data_idx < n) { + data = *v(batch_idx * BATCH_SIZE + data_idx); + } + shifted_data[data_idx] = ldexp(data, num_bitplanes - exp); + fp_data[data_idx] = + Math::binary2negabinary((T_sfp)shifted_data[data_idx]); + // fp_data[data_idx] = (T_fp)fabs(shifted_data[data_idx]); + + // printf("%f: ", data); print_bits(fp_data[data_idx], b); + } + // encode data + encode_batch(fp_data, encoded_data, num_bitplanes); + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + *encoded_bitplanes(bp_idx, batch_idx) = encoded_data[bp_idx]; + // print_bits(encoded_bitplanes[bp_idx * b + batch_idx * 2], + // batch_size); + } + + if constexpr (CollectError) { + error_collect_negabinary(shifted_data, errors, num_bitplanes, exp); + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + *level_errors_workspace(bp_idx, batch_idx) = errors[bp_idx]; + } + } + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + EncodeNegaBinary(); + } else { + EncodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + int num_bitplanes; + int exp; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPEncoderOptV2Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp encoder"; + MGARDX_CONT + BPEncoderOptV2Kernel(SIZE n, int num_bitplanes, int exp, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), exp(exp), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) {} + + using FunctorType = + BPEncoderOptV2Functor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + FunctorType functor(n, num_bitplanes, exp, v, encoded_bitplanes, + level_errors_workspace); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 16; + tbz = 1; + tby = 1; + tbx = 32; + gridz = 1; + gridy = 1; + gridx = (n - 1) / tbx + 1; + // gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + // gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int num_bitplanes; + int exp; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; +}; + +template +class BPDecoderOptV2Functor : public Functor { +public: + MGARDX_CONT + BPDecoderOptV2Functor() {} + MGARDX_CONT + BPDecoderOptV2Functor(SIZE n, SIZE starting_bitplane, int num_bitplanes, + int exp, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, + SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), exp(exp), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) { + Functor(); + } + + MGARDX_EXEC void decode_batch(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_fp buffer = 0; + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_fp bit = (encoded[bp_idx] >> (BATCH_SIZE - 1 - data_idx)) & 1u; + buffer += bit << (num_bitplanes - 1 - bp_idx); + } + v[data_idx] = buffer; + } + } + + MGARDX_EXEC void DecodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_fp fp_sign[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_bitplane encoded_sign[MAX_BITPLANES]; + + int ending_bitplane = starting_bitplane + num_bitplanes; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = + *encoded_bitplanes(starting_bitplane + bp_idx, batch_idx); + // print_bits(encoded_data[bp_idx], batch_size); + } + // encode data + decode_batch(fp_data, encoded_data, num_bitplanes); + + if (starting_bitplane == 0) { + // decode sign + encoded_sign[0] = *encoded_bitplanes(0, num_batches + batch_idx); + decode_batch(fp_sign, encoded_sign, 1); + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + *signs(batch_idx * BATCH_SIZE + data_idx) = fp_sign[data_idx]; + } + } else { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + fp_sign[data_idx] = *signs(batch_idx * BATCH_SIZE + data_idx); + } + } + + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = ldexp((T_data)fp_data[data_idx], -ending_bitplane + exp); + if (batch_idx * BATCH_SIZE + data_idx < n) { + *v(batch_idx * BATCH_SIZE + data_idx) = + fp_sign[data_idx] ? -data : data; + } + // printf("data: %f, fp_data[data_idx]: %llu\n", *v(batch_idx * + // BATCH_SIZE + data_idx), fp_data[data_idx]); printf("%f: ", data); + // print_bits(fp_data[data_idx], b); + } + } + } + + MGARDX_EXEC void DecodeNegaBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + + exp += 2; + + int ending_bitplane = starting_bitplane + num_bitplanes; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = + *encoded_bitplanes(starting_bitplane + bp_idx, batch_idx); + // print_bits(encoded_data[bp_idx], batch_size); + } + // encode data + decode_batch(fp_data, encoded_data, num_bitplanes); + + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = ldexp( + (T_data)Math::negabinary2binary(fp_data[data_idx]), + -ending_bitplane + exp); + if (batch_idx * BATCH_SIZE + data_idx < n) { + *v(batch_idx * BATCH_SIZE + data_idx) = + ending_bitplane % 2 != 0 ? -data : data; + } + // printf("%f: ", data); print_bits(fp_data[data_idx], b); + } + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + DecodeNegaBinary(); + } else { + DecodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + SIZE starting_bitplane; + int num_bitplanes; + int exp; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPDecoderOptV2Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp decoder"; + MGARDX_CONT + BPDecoderOptV2Kernel(SIZE n, SIZE starting_bitplane, int num_bitplanes, + int exp, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, + SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), exp(exp), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) {} + + using FunctorType = BPDecoderOptV2Functor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + + FunctorType functor(n, starting_bitplane, num_bitplanes, exp, + encoded_bitplanes, signs, v); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 8; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (n - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + SIZE starting_bitplane; + int num_bitplanes; + int exp; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; +}; + +// general bitplane encoder that encodes data by block using T_stream type +// buffer +template +class BPEncoderOptV2 + : public concepts::BitplaneEncoderInterface { +public: + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + using T_sfp = typename std::conditional::value, + int64_t, int32_t>::type; + using T_fp = typename std::conditional::value, + uint64_t, uint32_t>::type; + + BPEncoderOptV2() : initialized(false) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + } + BPEncoderOptV2(Hierarchy &hierarchy) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + Adapt(hierarchy, 0); + DeviceRuntime::SyncQueue(0); + } + + static SIZE bitplane_length(SIZE n) { + if constexpr (!NegaBinary) { + return num_blocks(n) * 2; + } else { + return num_blocks(n); + } + } + + static SIZE num_blocks(SIZE n) { + const SIZE batch_size = sizeof(T_bitplane) * 8; + SIZE num_blocks = (n - 1) / batch_size + 1; + return num_blocks; + } + + void Adapt(Hierarchy &hierarchy, int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + + level_errors_work_array.resize( + {MAX_BITPLANES + 1, num_blocks(max_level_num_elems)}, queue_idx); + DeviceCollective::Sum( + num_blocks(max_level_num_elems), SubArray<1, T_error, DeviceType>(), + SubArray<1, T_error, DeviceType>(), level_error_sum_work_array, false, + queue_idx); + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + Hierarchy hierarchy(shape, Config()); + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + size_t size = 0; + size += hierarchy.EstimateMemoryFootprint(shape); + size += + (MAX_BITPLANES + 1) * num_blocks(max_level_num_elems) * sizeof(T_error); + for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { + size += hierarchy.level_num_elems(level_idx) * sizeof(bool); + } + return size; + } + + void encode(SIZE n, int num_bitplanes, int32_t exp, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, T_error, DeviceType> level_errors, int queue_idx) { + + SubArray<2, T_error, DeviceType> level_errors_work(level_errors_work_array); + + DeviceLauncher::Execute( + BPEncoderOptV2Kernel( + n, num_bitplanes, exp, v, encoded_bitplanes, level_errors_work), + queue_idx); + + if constexpr (CollectError) { + SIZE reduce_size = num_blocks(n); + for (int i = 0; i < num_bitplanes + 1; i++) { + SubArray<1, T_error, DeviceType> curr_errors({reduce_size}, + level_errors_work(i, 0)); + SubArray<1, T_error, DeviceType> sum_error({1}, level_errors(i)); + DeviceCollective::Sum(reduce_size, curr_errors, sum_error, + level_error_sum_work_array, true, + queue_idx); + } + } + } + + void decode(SIZE n, int num_bitplanes, int32_t exp, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) {} + + // decode the data and record necessary information for progressiveness + void progressive_decode(SIZE n, SIZE starting_bitplanes, int num_bitplanes, + int32_t exp, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> level_signs, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) { + + if (num_bitplanes > 0) { + DeviceLauncher::Execute( + BPDecoderOptV2Kernel(n, starting_bitplanes, num_bitplanes, + exp, encoded_bitplanes, level_signs, + v), + queue_idx); + } + } + + void print() const { std::cout << "Grouped bitplane encoder" << std::endl; } + +private: + bool initialized; + Hierarchy *hierarchy; + Array<2, T_error, DeviceType> level_errors_work_array; + Array<1, Byte, DeviceType> level_error_sum_work_array; +}; +} // namespace MDR +} // namespace mgard_x +#endif diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV2a.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV2a.hpp new file mode 100644 index 0000000000..cfd278bd27 --- /dev/null +++ b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV2a.hpp @@ -0,0 +1,717 @@ +#ifndef _MDR_BP_ENCODER_OPT_V2a_HPP +#define _MDR_BP_ENCODER_OPT_V2a_HPP + +#include "../../RuntimeX/RuntimeX.h" + +#include "BitplaneEncoderInterface.hpp" +#include + +namespace mgard_x { +namespace MDR { + +template +class BPEncoderOptV2aFunctor : public Functor { +public: + MGARDX_CONT + BPEncoderOptV2aFunctor() {} + MGARDX_CONT + BPEncoderOptV2aFunctor( + SIZE n, int num_bitplanes, SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) { + Functor(); + } + + MGARDX_EXEC void encode_batch(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_bitplane buffer = 0; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_bitplane bit = + (v[data_idx] >> (num_bitplanes - 1 - bp_idx)) & (T_bitplane)1; + buffer += bit << BATCH_SIZE - 1 - data_idx; + } + encoded[bp_idx] = buffer; + } + } + + MGARDX_EXEC void error_collect_binary(T_data *shifted_data, T_error *errors, + int num_bitplanes, int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_error diff = (T_error)(fp_data & mask) + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void error_collect_negabinary(T_data *shifted_data, + T_error *errors, int num_bitplanes, + int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_fp ngb_data = Math::binary2negabinary((T_sfp)data); + T_error diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void EncodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE tid = FunctorBase::GetThreadIdX(); + + SIZE lane_id = tid % BATCH_SIZE; + SIZE warp_id = gid / BATCH_SIZE; + + SIZE num_batches = n / BATCH_SIZE; + + T_bitplane buffer; + T_bitplane encoded_data[NUM_BITPLANES]; + T_bitplane encoded_sign; + T_error errors; + + T_fp *sm_p = (T_fp *)FunctorBase::GetSharedMemory(); + T_fp *fp_data = sm_p + BATCH_SIZE * (tid / BATCH_SIZE); + + int exp; + frexp(*abs_max((IDX)0), &exp); + + SIZE actual_batch_per_warp = std::min(M, num_batches - warp_id * M); + +#pragma unroll + for (int i = 0; i < M; i++) { + SIZE batch_idx = warp_id * M + i; + // if (batch_idx < num_batches) { + // actual_batch_per_warp++; + T_data data = *v(batch_idx * BATCH_SIZE + lane_id); + T_data shifted_data = ldexp(data, NUM_BITPLANES - exp); + T_fp fp_data = (T_fp)fabs(shifted_data); + T_fp fp_sign = (T_fp)(signbit(data) == 0 ? 0 : 1); +#define FULL_MASK 0xffffffff + +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + T_bitplane bit = + (fp_data >> (NUM_BITPLANES - 1 - bp_idx)) & (T_bitplane)1; + + // option 1 + // T_bitplane shifted_bit = bit << BATCH_SIZE - 1 - lane_id; + // for (int offset = 16; offset > 0; offset /= 2) { + // buffer |= __shfl_down_sync(FULL_MASK, shifted_bit, offset); + // } + // buffer = __shfl_sync(FULL_MASK, buffer, 0); + + // option 2 + T_bitplane shifted_bit = bit << BATCH_SIZE - 1 - lane_id; + buffer = __reduce_add_sync(FULL_MASK, shifted_bit); + // buffer = __shfl_sync(FULL_MASK, buffer, 0); + + // option 3 + // buffer = __match_any_sync(FULL_MASK, bit); + // if (!bit) buffer ^= FULL_MASK; + // buffer = __shfl_sync(FULL_MASK, buffer, 0); + + // option 4 + // buffer = __ballot_sync(FULL_MASK, bit); + + // Save to mine registers + if (lane_id == i) { + encoded_data[bp_idx] = buffer; + } + } + + // option 5 + // fp_data[lane_id] = fp_data; + + // option 1 + // encoded_sign = fp_sign << BATCH_SIZE - 1 - data_idx; + // for (int offset = 16; offset > 0; offset /= 2) { + // encoded_sign |= __shfl_down_sync(FULL_MASK, encoded_sign, offset); + // } + // option 2 + // encoded_sign = fp_sign << BATCH_SIZE - 1 - data_idx; + // encoded_sign = __reduce_add_sync(FULL_MASK, encoded_sign); + + // option 3 + // buffer = __ballot_sync(FULL_MASK, fp_sign); + + if (lane_id == i) { + encoded_sign = buffer; + } + // } + } + + // if (lane_id < actual_batch_per_warp) { +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + *encoded_bitplanes(bp_idx, warp_id * M + lane_id) = encoded_data[bp_idx]; + } + *encoded_bitplanes(0, num_batches + warp_id * M + lane_id) = encoded_sign; +#pragma unroll + for (int bp_idx = 1; bp_idx < NUM_BITPLANES; bp_idx++) { + *encoded_bitplanes(bp_idx, num_batches + warp_id * M + lane_id) = + (T_bitplane)0; + } + // } + } + + MGARDX_EXEC void EncodeNegaBinary() { + SIZE max_batches_per_warp = 32; + SIZE global_batch_start = + FunctorBase::GetBlockIdX() * max_batches_per_warp; + SIZE tid = FunctorBase::GetThreadIdX(); + + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + SIZE num_batches_this_warp = + std::min(num_batches - global_batch_start, max_batches_per_warp); + T_data data; + T_data shifted_data; + T_fp fp_data; + T_fp fp_sign; + T_bitplane buffer; + T_bitplane encoded_data[MAX_BITPLANES]; + T_bitplane encoded_sign; + T_error errors; + + int exp; + frexp(*abs_max((IDX)0), &exp); + exp += 2; + + SIZE data_idx = tid; + SIZE my_batch_idx = tid; + + for (SIZE local_batch_idx = 0; local_batch_idx < num_batches_this_warp; + local_batch_idx++) { + SIZE global_batch_idx = global_batch_start + local_batch_idx; + data = 0; + if (global_batch_idx * BATCH_SIZE + data_idx < n) { + data = *v(global_batch_idx * BATCH_SIZE + data_idx); + } + shifted_data = ldexp(data, num_bitplanes - exp); + fp_data = Math::binary2negabinary((T_sfp)shifted_data); +#define FULL_MASK 0xffffffff + unsigned long long full_mask = 0xffffffff; + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + // T_bitplane bit = (fp_data >> (num_bitplanes - 1 - bp_idx)) & 1u; + T_bitplane bit = (T_bitplane)1; + // T_bitplane shifted_bit = bit << BATCH_SIZE - 1 - data_idx; + // option 1 + // for (int offset = 16; offset > 0; offset /= 2) { + // buffer |= __shfl_down_sync(FULL_MASK, shifted_bit, offset); + // } + // option 2 + // buffer = __reduce_add_sync(FULL_MASK, shifted_bit); + + // option 3 + // buffer = __match_any_sync(FULL_MASK, bit); + // if (!bit) buffer ^= FULL_MASK; + + // option 4 + // buffer = __ballot_sync(FULL_MASK, bit); + buffer = __ballot_sync(full_mask, bit); + + // buffer = __shfl_sync(FULL_MASK, buffer, 0); + if (my_batch_idx == local_batch_idx) { + encoded_data[bp_idx] = buffer; + } + } + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + // printf("thread %llu, encoded_data %u, \n", tid, + // encoded_data[bp_idx]); print_bits(encoded_data[bp_idx], b); + *encoded_bitplanes(bp_idx, global_batch_start + my_batch_idx) = + encoded_data[bp_idx]; + } + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + EncodeNegaBinary(); + } else { + EncodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + // size += sizeof(T_fp) * BATCH_SIZE * (256/32); + return size; + } + +private: + // parameters + SIZE n; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPEncoderOptV2aKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp encoder"; + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + MGARDX_CONT + BPEncoderOptV2aKernel(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) {} + + using FunctorType = + BPEncoderOptV2aFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + FunctorType functor(n, num_bitplanes, abs_max, v, encoded_bitplanes, + level_errors_workspace); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE num_batches = n / BATCH_SIZE; + tbz = 1; + tby = 1; + tbx = 32; + gridz = 1; + gridy = 1; + // gridx = num_batches / ((tbx/32)*32); + gridx = (num_batches - 1) / ((tbx / 32) * M) + 1; + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; +}; + +template +class BPDecoderOptV2aFunctor : public Functor { +public: + MGARDX_CONT + BPDecoderOptV2aFunctor() {} + MGARDX_CONT + BPDecoderOptV2aFunctor(SIZE n, int starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, + SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) { + Functor(); + } + + MGARDX_EXEC void decode_batch(T_fp *v, T_bitplane *encoded) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_fp buffer = 0; + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_fp bit = (encoded[bp_idx] >> (BATCH_SIZE - 1 - data_idx)) & (T_fp)1; + buffer += bit << (num_bitplanes - 1 - bp_idx); + } + v[data_idx] = buffer; + } + } + + MGARDX_EXEC void DecodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE tid = FunctorBase::GetThreadIdX(); + + uint32_t lane_id = tid % BATCH_SIZE; + SIZE warp_id = gid / BATCH_SIZE; + + SIZE num_batches = n / BATCH_SIZE; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_fp fp_sign[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_bitplane encoded_sign; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + int ending_bitplane = starting_bitplane + num_bitplanes; + + SIZE actual_batch_per_warp = std::min(M, num_batches - warp_id * M); + + if (lane_id < actual_batch_per_warp) { +#pragma unroll + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = *encoded_bitplanes(starting_bitplane + bp_idx, + warp_id * M + lane_id); + } + encoded_sign = *encoded_bitplanes(0, num_batches + warp_id * M + lane_id); + } + + if (lane_id < actual_batch_per_warp) { + // decode data + decode_batch(fp_data, encoded_data); +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + fp_sign[data_idx] = + (encoded_sign >> (BATCH_SIZE - 1 - data_idx)) & (T_fp)1; + } +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + shifted_data[data_idx] = (T_data)fp_data[data_idx]; + // It is beneficial to use pow instead of ldexp + T_data data = shifted_data[data_idx] * pow(2, -ending_bitplane + exp); + // T_data data = ldexp(shifted_data[data_idx], -ending_bitplane + exp); + data = fp_sign[data_idx] ? -data : data; + shifted_data[data_idx] = data; + } + } + + for (u_int32_t mask = 0; mask < BATCH_SIZE; mask++) { + // printf("lane_id: %d, mask: %u, lane_id^mask: %d\n", lane_id, mask, + // lane_id^mask); #define FULL_MASK 0xffffffff + unsigned long long full_mask = 0xFFFFFFFF; + T_data buffer = + __shfl_xor_sync(full_mask, shifted_data[lane_id ^ mask], mask); + shifted_data[lane_id ^ mask] = buffer; + } + + for (int i = 0; i < actual_batch_per_warp; i++) { + SIZE batch_idx = warp_id * M + i; + *v(batch_idx * BATCH_SIZE + lane_id) = shifted_data[i]; + } + } + + MGARDX_EXEC void DecodeNegaBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + + int exp; + frexp(*abs_max((IDX)0), &exp); + exp += 2; + + int ending_bitplane = starting_bitplane + num_bitplanes; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = + *encoded_bitplanes(starting_bitplane + bp_idx, batch_idx); + // print_bits(encoded_data[bp_idx], batch_size); + } + // encode data + decode_batch(fp_data, encoded_data); + + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = ldexp( + (T_data)Math::negabinary2binary(fp_data[data_idx]), + -ending_bitplane + exp); + if (batch_idx * BATCH_SIZE + data_idx < n) { + *v(batch_idx * BATCH_SIZE + data_idx) = + ending_bitplane % 2 != 0 ? -data : data; + } + // printf("%f: ", data); print_bits(fp_data[data_idx], b); + } + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + DecodeNegaBinary(); + } else { + DecodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + int starting_bitplane; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPDecoderOptV2aKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp decoder"; + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + MGARDX_CONT + BPDecoderOptV2aKernel(SIZE n, int starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, + SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) {} + + using FunctorType = BPDecoderOptV2aFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + + FunctorType functor(n, starting_bitplane, num_bitplanes, abs_max, + encoded_bitplanes, signs, v); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE num_batches = n / BATCH_SIZE; + tbz = 1; + tby = 1; + tbx = 32; + gridz = 1; + gridy = 1; + // gridx = num_batches / ((tbx/32)*32); + gridx = (num_batches - 1) / ((tbx / 32) * M) + 1; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int starting_bitplane; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; +}; + +// general bitplane encoder that encodes data by block using T_stream type +// buffer +template +class BPEncoderOptV2a + : public concepts::BitplaneEncoderInterface { +public: + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + using T_sfp = typename std::conditional::value, + int64_t, int32_t>::type; + using T_fp = typename std::conditional::value, + uint64_t, uint32_t>::type; + + BPEncoderOptV2a() : initialized(false) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + } + BPEncoderOptV2a(Hierarchy &hierarchy) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + Adapt(hierarchy, 0); + DeviceRuntime::SyncQueue(0); + } + + static SIZE bitplane_length(SIZE n) { + if constexpr (!NegaBinary) { + return num_blocks(n) * 2; + } else { + return num_blocks(n); + } + } + + static SIZE num_blocks(SIZE n) { + const SIZE batch_size = sizeof(T_bitplane) * 8; + SIZE num_blocks = (n - 1) / batch_size + 1; + return num_blocks; + } + + void Adapt(Hierarchy &hierarchy, int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + + level_errors_work_array.resize( + {MAX_BITPLANES + 1, num_blocks(max_level_num_elems)}, queue_idx); + DeviceCollective::Sum( + num_blocks(max_level_num_elems), SubArray<1, T_error, DeviceType>(), + SubArray<1, T_error, DeviceType>(), level_error_sum_work_array, false, + queue_idx); + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + Hierarchy hierarchy(shape, Config()); + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + size_t size = 0; + size += hierarchy.EstimateMemoryFootprint(shape); + size += + (MAX_BITPLANES + 1) * num_blocks(max_level_num_elems) * sizeof(T_error); + for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { + size += hierarchy.level_num_elems(level_idx) * sizeof(bool); + } + return size; + } + + void encode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, T_error, DeviceType> level_errors, int queue_idx) { + + SubArray<2, T_error, DeviceType> level_errors_work(level_errors_work_array); + + constexpr SIZE max_batch_per_warp = 8; + DeviceLauncher::Execute( + BPEncoderOptV2aKernel( + n, num_bitplanes, abs_max, v, encoded_bitplanes, level_errors_work), + queue_idx); + + if constexpr (CollectError) { + SIZE reduce_size = num_blocks(n); + for (int i = 0; i < num_bitplanes + 1; i++) { + SubArray<1, T_error, DeviceType> curr_errors({reduce_size}, + level_errors_work(i, 0)); + SubArray<1, T_error, DeviceType> sum_error({1}, level_errors(i)); + DeviceCollective::Sum(reduce_size, curr_errors, sum_error, + level_error_sum_work_array, true, + queue_idx); + } + } + } + + void decode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) {} + + // decode the data and record necessary information for progressiveness + void progressive_decode(SIZE n, int starting_bitplanes, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> level_signs, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) { + + constexpr SIZE max_batch_per_warp = 8; + if (num_bitplanes > 0) { + DeviceLauncher::Execute( + BPDecoderOptV2aKernel( + n, starting_bitplanes, num_bitplanes, abs_max, encoded_bitplanes, + level_signs, v), + queue_idx); + } + } + + void print() const { std::cout << "Grouped bitplane encoder" << std::endl; } + +private: + bool initialized; + Hierarchy *hierarchy; + Array<2, T_error, DeviceType> level_errors_work_array; + Array<1, Byte, DeviceType> level_error_sum_work_array; +}; +} // namespace MDR +} // namespace mgard_x +#endif diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV3.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV3.hpp new file mode 100644 index 0000000000..2e8a028951 --- /dev/null +++ b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderOptV3.hpp @@ -0,0 +1,645 @@ +#ifndef _MDR_BP_ENCODER_OPT_V3_HPP +#define _MDR_BP_ENCODER_OPT_V3_HPP + +#include "../../RuntimeX/RuntimeX.h" + +#include "BitplaneEncoderInterface.hpp" +#include + +namespace mgard_x { +namespace MDR { + +template +class BPEncoderOptV3Functor : public Functor { +public: + MGARDX_CONT + BPEncoderOptV3Functor() {} + MGARDX_CONT + BPEncoderOptV3Functor(SIZE n, int num_bitplanes, int exp, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), exp(exp), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) { + Functor(); + } + + MGARDX_EXEC void encode_batch(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_bitplane buffer = 0; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_bitplane bit = (v[data_idx] >> (num_bitplanes - 1 - bp_idx)) & 1u; + buffer += bit << BATCH_SIZE - 1 - data_idx; + } + encoded[bp_idx] = buffer; + } + } + + MGARDX_EXEC void error_collect_binary(T_data *shifted_data, T_error *errors, + int num_bitplanes, int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_error diff = (T_error)(fp_data & mask) + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void error_collect_negabinary(T_data *shifted_data, + T_error *errors, int num_bitplanes, + int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_fp ngb_data = Math::binary2negabinary((T_sfp)data); + T_error diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void EncodeBinary() { + + int8_t *sm_p = (int8_t *)FunctorBase::GetSharedMemory(); + + SIZE bid = FunctorBase::GetBlockIdX(); + SIZE num_warps_per_block = FunctorBase::GetBlockDimX() / 32; + + SIZE tid = FunctorBase::GetThreadIdX(); + SIZE grid_size = FunctorBase::GetGridDimX(); + SIZE warp_id = tid / 32; + SIZE lane_id = tid % 32; + + T_fp *fp_data = (T_fp *)(sm_p + BATCH_SIZE * sizeof(T_fp) * warp_id); + sm_p += BATCH_SIZE * sizeof(T_fp) * num_warps_per_block; + + T_bitplane *encoded_data = + (T_bitplane *)(sm_p + (MAX_BITPLANES * sizeof(T_bitplane) * warp_id)); + + SIZE batch_idx_start = bid * num_warps_per_block + warp_id; + SIZE batch_step_size = grid_size * num_warps_per_block; + + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + + T_data data; + T_data shifted_data; + T_fp fp_sign; + T_bitplane encoded_sign; + T_error errors; + + SIZE data_idx = lane_id; + SIZE my_bp_idx = lane_id; + for (SIZE batch_idx = batch_idx_start; batch_idx < num_batches; + batch_idx += batch_step_size) { + data = 0; + if (batch_idx * BATCH_SIZE + data_idx < n) { + data = *v(batch_idx * BATCH_SIZE + data_idx); + } + shifted_data = ldexp(data, num_bitplanes - exp); + fp_data[data_idx] = (T_fp)fabs(shifted_data); + // fp_sign = (T_fp)(signbit(data) == 0 ? 0 : 1); + __syncthreads(); + + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_bitplane bit = + (fp_data[data_idx] >> (num_bitplanes - 1 - my_bp_idx)) & 1u; + encoded_data[data_idx] |= bit << BATCH_SIZE - 1 - data_idx; + } + __syncthreads(); + + // if (batch_idx == 0) { + // printf("thread %llu, fp_data %u, encoded_data: %u\n", tid, fp_data, + // encoded_data); + // } + + // encoded_sign = fp_sign << BATCH_SIZE - 1 - data_idx; + // option 1 + // for (int offset = 16; offset > 0; offset /= 2) { + // encoded_sign |= __shfl_down_sync(FULL_MASK, encoded_sign, offset); + // } + // option 2 + // encoded_sign = __reduce_add_sync(FULL_MASK, encoded_sign); + + // if (batch_idx == 0) { + // if (my_bp_idx == 0) + // printf("thread %llu, encoded_sign %u, \n", tid, encoded_sign); + // } + + int bp_idx2 = tid / 8; + int batch_idx2 = tid % 8; + T_bitplane *encoded_data2 = + (T_bitplane *)(sm_p + (bp_idx2 * sizeof(T_bitplane) * batch_idx2)); + *encoded_bitplanes(bp_idx2, batch_idx2) = *encoded_data2; + + // *encoded_bitplanes(my_bp_idx, num_batches + batch_idx) = my_bp_idx == 0 + // ? + // encoded_sign + // : + // (T_bitplane)0; + } + } + + MGARDX_EXEC void EncodeNegaBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_error errors[MAX_BITPLANES + 1]; + + exp += 2; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = 0; + if (batch_idx * BATCH_SIZE + data_idx < n) { + data = *v(batch_idx * BATCH_SIZE + data_idx); + } + shifted_data[data_idx] = ldexp(data, num_bitplanes - exp); + fp_data[data_idx] = + Math::binary2negabinary((T_sfp)shifted_data[data_idx]); + // fp_data[data_idx] = (T_fp)fabs(shifted_data[data_idx]); + + // printf("%f: ", data); print_bits(fp_data[data_idx], b); + } + // encode data + encode_batch(fp_data, encoded_data, num_bitplanes); + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + *encoded_bitplanes(bp_idx, batch_idx) = encoded_data[bp_idx]; + // print_bits(encoded_bitplanes[bp_idx * b + batch_idx * 2], + // batch_size); + } + + if constexpr (CollectError) { + error_collect_negabinary(shifted_data, errors, num_bitplanes, exp); + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + *level_errors_workspace(bp_idx, batch_idx) = errors[bp_idx]; + } + } + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + EncodeNegaBinary(); + } else { + EncodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = sizeof(T_fp) * BATCH_SIZE * 8; + size += sizeof(T_bitplane) * MAX_BITPLANES * 8; // for encoded bitplanes + return size; + } + +private: + // parameters + SIZE n; + int num_bitplanes; + int exp; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPEncoderOptV3Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp encoder"; + MGARDX_CONT + BPEncoderOptV3Kernel(SIZE n, int num_bitplanes, int exp, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), exp(exp), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) {} + + using FunctorType = + BPEncoderOptV3Functor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + FunctorType functor(n, num_bitplanes, exp, v, encoded_bitplanes, + level_errors_workspace); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 16; + tbz = 1; + tby = 1; + tbx = 32 * 8; + gridz = 1; + gridy = 1; + gridx = (n - 1) / tbx + 1; + // gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + // gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int num_bitplanes; + int exp; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; +}; + +template +class BPDecoderOptV3Functor : public Functor { +public: + MGARDX_CONT + BPDecoderOptV3Functor() {} + MGARDX_CONT + BPDecoderOptV3Functor(SIZE n, SIZE starting_bitplane, int num_bitplanes, + int exp, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, + SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), exp(exp), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) { + Functor(); + } + + MGARDX_EXEC void decode_batch(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_fp buffer = 0; + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_fp bit = (encoded[bp_idx] >> (BATCH_SIZE - 1 - data_idx)) & 1u; + buffer += bit << (num_bitplanes - 1 - bp_idx); + } + v[data_idx] = buffer; + } + } + + MGARDX_EXEC void DecodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_fp fp_sign[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_bitplane encoded_sign[MAX_BITPLANES]; + + int ending_bitplane = starting_bitplane + num_bitplanes; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = + *encoded_bitplanes(starting_bitplane + bp_idx, batch_idx); + // print_bits(encoded_data[bp_idx], batch_size); + } + // encode data + decode_batch(fp_data, encoded_data, num_bitplanes); + + if (starting_bitplane == 0) { + // decode sign + encoded_sign[0] = *encoded_bitplanes(0, num_batches + batch_idx); + decode_batch(fp_sign, encoded_sign, 1); + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + *signs(batch_idx * BATCH_SIZE + data_idx) = fp_sign[data_idx]; + } + } else { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + fp_sign[data_idx] = *signs(batch_idx * BATCH_SIZE + data_idx); + } + } + + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = ldexp((T_data)fp_data[data_idx], -ending_bitplane + exp); + if (batch_idx * BATCH_SIZE + data_idx < n) { + *v(batch_idx * BATCH_SIZE + data_idx) = + fp_sign[data_idx] ? -data : data; + } + // printf("data: %f, fp_data[data_idx]: %llu\n", *v(batch_idx * + // BATCH_SIZE + data_idx), fp_data[data_idx]); printf("%f: ", data); + // print_bits(fp_data[data_idx], b); + } + } + } + + MGARDX_EXEC void DecodeNegaBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + SIZE grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + SIZE num_batches = (n - 1) / BATCH_SIZE + 1; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + + exp += 2; + + int ending_bitplane = starting_bitplane + num_bitplanes; + + for (SIZE batch_idx = gid; batch_idx < num_batches; + batch_idx += grid_size) { + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = + *encoded_bitplanes(starting_bitplane + bp_idx, batch_idx); + // print_bits(encoded_data[bp_idx], batch_size); + } + // encode data + decode_batch(fp_data, encoded_data, num_bitplanes); + + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = ldexp( + (T_data)Math::negabinary2binary(fp_data[data_idx]), + -ending_bitplane + exp); + if (batch_idx * BATCH_SIZE + data_idx < n) { + *v(batch_idx * BATCH_SIZE + data_idx) = + ending_bitplane % 2 != 0 ? -data : data; + } + // printf("%f: ", data); print_bits(fp_data[data_idx], b); + } + } + } + + MGARDX_EXEC void Operation1() { + if constexpr (NegaBinary) { + DecodeNegaBinary(); + } else { + DecodeBinary(); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + SIZE starting_bitplane; + int num_bitplanes; + int exp; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPDecoderOptV3Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp decoder"; + MGARDX_CONT + BPDecoderOptV3Kernel(SIZE n, SIZE starting_bitplane, int num_bitplanes, + int exp, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, + SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), exp(exp), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) {} + + using FunctorType = BPDecoderOptV3Functor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + + FunctorType functor(n, starting_bitplane, num_bitplanes, exp, + encoded_bitplanes, signs, v); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 8; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (n - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + SIZE starting_bitplane; + int num_bitplanes; + int exp; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; +}; + +// general bitplane encoder that encodes data by block using T_stream type +// buffer +template +class BPEncoderOptV3 + : public concepts::BitplaneEncoderInterface { +public: + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + using T_sfp = typename std::conditional::value, + int64_t, int32_t>::type; + using T_fp = typename std::conditional::value, + uint64_t, uint32_t>::type; + + BPEncoderOptV3() : initialized(false) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + } + BPEncoderOptV3(Hierarchy &hierarchy) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + Adapt(hierarchy, 0); + DeviceRuntime::SyncQueue(0); + } + + static SIZE bitplane_length(SIZE n) { + if constexpr (!NegaBinary) { + return num_blocks(n) * 2; + } else { + return num_blocks(n); + } + } + + static SIZE num_blocks(SIZE n) { + const SIZE batch_size = sizeof(T_bitplane) * 8; + SIZE num_blocks = (n - 1) / batch_size + 1; + return num_blocks; + } + + void Adapt(Hierarchy &hierarchy, int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + + level_errors_work_array.resize( + {MAX_BITPLANES + 1, num_blocks(max_level_num_elems)}, queue_idx); + DeviceCollective::Sum( + num_blocks(max_level_num_elems), SubArray<1, T_error, DeviceType>(), + SubArray<1, T_error, DeviceType>(), level_error_sum_work_array, false, + queue_idx); + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + Hierarchy hierarchy(shape, Config()); + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + size_t size = 0; + size += hierarchy.EstimateMemoryFootprint(shape); + size += + (MAX_BITPLANES + 1) * num_blocks(max_level_num_elems) * sizeof(T_error); + for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { + size += hierarchy.level_num_elems(level_idx) * sizeof(bool); + } + return size; + } + + void encode(SIZE n, int num_bitplanes, int32_t exp, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, T_error, DeviceType> level_errors, int queue_idx) { + + SubArray<2, T_error, DeviceType> level_errors_work(level_errors_work_array); + + DeviceLauncher::Execute( + BPEncoderOptV3Kernel( + n, num_bitplanes, exp, v, encoded_bitplanes, level_errors_work), + queue_idx); + + if constexpr (CollectError) { + SIZE reduce_size = num_blocks(n); + for (int i = 0; i < num_bitplanes + 1; i++) { + SubArray<1, T_error, DeviceType> curr_errors({reduce_size}, + level_errors_work(i, 0)); + SubArray<1, T_error, DeviceType> sum_error({1}, level_errors(i)); + DeviceCollective::Sum(reduce_size, curr_errors, sum_error, + level_error_sum_work_array, true, + queue_idx); + } + } + } + + void decode(SIZE n, int num_bitplanes, int32_t exp, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) {} + + // decode the data and record necessary information for progressiveness + void progressive_decode(SIZE n, SIZE starting_bitplanes, int num_bitplanes, + int32_t exp, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> level_signs, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) { + + if (num_bitplanes > 0) { + DeviceLauncher::Execute( + BPDecoderOptV3Kernel(n, starting_bitplanes, num_bitplanes, + exp, encoded_bitplanes, level_signs, + v), + queue_idx); + } + } + + void print() const { std::cout << "Grouped bitplane encoder" << std::endl; } + +private: + bool initialized; + Hierarchy *hierarchy; + Array<2, T_error, DeviceType> level_errors_work_array; + Array<1, Byte, DeviceType> level_error_sum_work_array; +}; +} // namespace MDR +} // namespace mgard_x +#endif diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderRegisterBallot.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderRegisterBallot.hpp new file mode 100644 index 0000000000..66c2a6c9db --- /dev/null +++ b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderRegisterBallot.hpp @@ -0,0 +1,559 @@ +#ifndef _MDR_BP_ENCODER_REGISTER_BALLOT_HPP +#define _MDR_BP_ENCODER_REGISTER_BALLOT_HPP + +#include "../../RuntimeX/RuntimeX.h" + +#include "BitplaneEncoderInterface.hpp" +#include + +namespace mgard_x { +namespace MDR { + +template +class BPEncoderRegisterBallotFunctor : public Functor { +public: + MGARDX_CONT + BPEncoderRegisterBallotFunctor() {} + MGARDX_CONT + BPEncoderRegisterBallotFunctor( + SIZE n, int num_bitplanes, SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) { + Functor(); + } + + MGARDX_EXEC void encode_batch(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_bitplane buffer = 0; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_bitplane bit = + (v[data_idx] >> (num_bitplanes - 1 - bp_idx)) & (T_bitplane)1; + buffer += bit << BATCH_SIZE - 1 - data_idx; + } + encoded[bp_idx] = buffer; + } + } + + MGARDX_EXEC void error_collect_binary(T_data *shifted_data, T_error *errors, + int num_bitplanes, int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_error diff = (T_error)(fp_data & mask) + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void error_collect_negabinary(T_data *shifted_data, + T_error *errors, int num_bitplanes, + int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_fp ngb_data = Math::binary2negabinary((T_sfp)data); + T_error diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void EncodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE tid = FunctorBase::GetThreadIdX(); + + SIZE lane_id = tid % BATCH_SIZE; + SIZE warp_id = gid / BATCH_SIZE; + + SIZE num_batches = n / BATCH_SIZE; + + T_bitplane buffer; + T_bitplane encoded_data[NUM_BITPLANES]; + T_bitplane encoded_sign; + T_error errors; + + T_fp *sm_p = (T_fp *)FunctorBase::GetSharedMemory(); + T_fp *fp_data = sm_p + BATCH_SIZE * (tid / BATCH_SIZE); + + int exp; + frexp(*abs_max((IDX)0), &exp); + + SIZE actual_batch_per_warp = std::min(M, num_batches - warp_id * M); + +#pragma unroll + for (int i = 0; i < M; i++) { + SIZE batch_idx = warp_id * M + i; + // if (batch_idx < num_batches) { + // actual_batch_per_warp++; + T_data data = *v(batch_idx * BATCH_SIZE + lane_id); + T_data shifted_data = ldexp(data, NUM_BITPLANES - exp); + T_fp fp_data = (T_fp)fabs(shifted_data); + T_fp fp_sign = (T_fp)(signbit(data) == 0 ? 0 : 1); +#define FULL_MASK 0xffffffff + +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + T_bitplane bit = + (fp_data >> (NUM_BITPLANES - 1 - bp_idx)) & (T_bitplane)1; + buffer = __ballot_sync(FULL_MASK, bit); + if (lane_id == i) { + encoded_data[bp_idx] = buffer; + } + } + buffer = __ballot_sync(FULL_MASK, fp_sign); + + if (lane_id == i) { + encoded_sign = buffer; + } + } + + // if (lane_id < actual_batch_per_warp) { +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + *encoded_bitplanes(bp_idx, warp_id * M + lane_id) = encoded_data[bp_idx]; + } + *encoded_bitplanes(0, num_batches + warp_id * M + lane_id) = encoded_sign; +#pragma unroll + for (int bp_idx = 1; bp_idx < NUM_BITPLANES; bp_idx++) { + *encoded_bitplanes(bp_idx, num_batches + warp_id * M + lane_id) = + (T_bitplane)0; + } + // } + } + + MGARDX_EXEC void Operation1() { EncodeBinary(); } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + // size += sizeof(T_fp) * BATCH_SIZE * (256/32); + return size; + } + +private: + // parameters + SIZE n; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPEncoderRegisterBallotKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp encoder"; + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + MGARDX_CONT + BPEncoderRegisterBallotKernel( + SIZE n, int num_bitplanes, SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) {} + + using FunctorType = + BPEncoderRegisterBallotFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + FunctorType functor(n, num_bitplanes, abs_max, v, encoded_bitplanes, + level_errors_workspace); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE num_batches = n / BATCH_SIZE; + tbz = 1; + tby = 1; + tbx = 32; + gridz = 1; + gridy = 1; + // gridx = num_batches / ((tbx/32)*32); + gridx = (num_batches - 1) / ((tbx / 32) * M) + 1; + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; +}; + +template +class BPDecoderRegisterBallotFunctor : public Functor { +public: + MGARDX_CONT + BPDecoderRegisterBallotFunctor() {} + MGARDX_CONT + BPDecoderRegisterBallotFunctor( + SIZE n, int starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) { + Functor(); + } + + MGARDX_EXEC void decode_batch(T_fp *v, T_bitplane *encoded) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_fp buffer = 0; + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_fp bit = (encoded[bp_idx] >> (BATCH_SIZE - 1 - data_idx)) & (T_fp)1; + buffer += bit << (num_bitplanes - 1 - bp_idx); + } + v[data_idx] = buffer; + } + } + + MGARDX_EXEC void DecodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE tid = FunctorBase::GetThreadIdX(); + + uint32_t lane_id = tid % BATCH_SIZE; + SIZE warp_id = gid / BATCH_SIZE; + + SIZE num_batches = n / BATCH_SIZE; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_fp fp_sign[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_bitplane encoded_sign; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + int ending_bitplane = starting_bitplane + num_bitplanes; + + SIZE actual_batch_per_warp = std::min(M, num_batches - warp_id * M); + + if (lane_id < actual_batch_per_warp) { +#pragma unroll + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = *encoded_bitplanes(starting_bitplane + bp_idx, + warp_id * M + lane_id); + } + encoded_sign = *encoded_bitplanes(0, num_batches + warp_id * M + lane_id); + } + + if (lane_id < actual_batch_per_warp) { + // decode data + decode_batch(fp_data, encoded_data); +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + fp_sign[data_idx] = + (encoded_sign >> (BATCH_SIZE - 1 - data_idx)) & (T_fp)1; + } +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + shifted_data[data_idx] = (T_data)fp_data[data_idx]; + // It is beneficial to use pow instead of ldexp + T_data data = shifted_data[data_idx] * pow(2, -ending_bitplane + exp); + // T_data data = ldexp(shifted_data[data_idx], -ending_bitplane + exp); + data = fp_sign[data_idx] ? -data : data; + shifted_data[data_idx] = data; + } + } + + for (u_int32_t mask = 0; mask < BATCH_SIZE; mask++) { + // printf("lane_id: %d, mask: %u, lane_id^mask: %d\n", lane_id, mask, + // lane_id^mask); #define FULL_MASK 0xffffffff + unsigned long long full_mask = 0xFFFFFFFF; + T_data buffer = + __shfl_xor_sync(full_mask, shifted_data[lane_id ^ mask], mask); + shifted_data[lane_id ^ mask] = buffer; + } + + for (int i = 0; i < actual_batch_per_warp; i++) { + SIZE batch_idx = warp_id * M + i; + *v(batch_idx * BATCH_SIZE + lane_id) = shifted_data[i]; + } + } + + MGARDX_EXEC void Operation1() { DecodeBinary(); } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + int starting_bitplane; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPDecoderRegisterBallotKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp decoder"; + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + MGARDX_CONT + BPDecoderRegisterBallotKernel( + SIZE n, int starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) {} + + using FunctorType = + BPDecoderRegisterBallotFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + + FunctorType functor(n, starting_bitplane, num_bitplanes, abs_max, + encoded_bitplanes, signs, v); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE num_batches = n / BATCH_SIZE; + tbz = 1; + tby = 1; + tbx = 32; + gridz = 1; + gridy = 1; + // gridx = num_batches / ((tbx/32)*32); + gridx = (num_batches - 1) / ((tbx / 32) * M) + 1; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int starting_bitplane; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; +}; + +// general bitplane encoder that encodes data by block using T_stream type +// buffer +template +class BPEncoderRegisterBallot + : public concepts::BitplaneEncoderInterface { +public: + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + using T_sfp = typename std::conditional::value, + int64_t, int32_t>::type; + using T_fp = typename std::conditional::value, + uint64_t, uint32_t>::type; + + BPEncoderRegisterBallot() : initialized(false) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + } + BPEncoderRegisterBallot(Hierarchy &hierarchy) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + Adapt(hierarchy, 0); + DeviceRuntime::SyncQueue(0); + } + + static SIZE bitplane_length(SIZE n) { + if constexpr (!NegaBinary) { + return num_blocks(n) * 2; + } else { + return num_blocks(n); + } + } + + static SIZE num_blocks(SIZE n) { + const SIZE batch_size = sizeof(T_bitplane) * 8; + SIZE num_blocks = (n - 1) / batch_size + 1; + return num_blocks; + } + + void Adapt(Hierarchy &hierarchy, int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + + level_errors_work_array.resize( + {MAX_BITPLANES + 1, num_blocks(max_level_num_elems)}, queue_idx); + DeviceCollective::Sum( + num_blocks(max_level_num_elems), SubArray<1, T_error, DeviceType>(), + SubArray<1, T_error, DeviceType>(), level_error_sum_work_array, false, + queue_idx); + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + Hierarchy hierarchy(shape, Config()); + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + size_t size = 0; + size += hierarchy.EstimateMemoryFootprint(shape); + size += + (MAX_BITPLANES + 1) * num_blocks(max_level_num_elems) * sizeof(T_error); + for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { + size += hierarchy.level_num_elems(level_idx) * sizeof(bool); + } + return size; + } + + void encode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, T_error, DeviceType> level_errors, int queue_idx) { + + SubArray<2, T_error, DeviceType> level_errors_work(level_errors_work_array); + + constexpr SIZE max_batch_per_warp = 8; + DeviceLauncher::Execute( + BPEncoderRegisterBallotKernel( + n, num_bitplanes, abs_max, v, encoded_bitplanes, level_errors_work), + queue_idx); + + if constexpr (CollectError) { + SIZE reduce_size = num_blocks(n); + for (int i = 0; i < num_bitplanes + 1; i++) { + SubArray<1, T_error, DeviceType> curr_errors({reduce_size}, + level_errors_work(i, 0)); + SubArray<1, T_error, DeviceType> sum_error({1}, level_errors(i)); + DeviceCollective::Sum(reduce_size, curr_errors, sum_error, + level_error_sum_work_array, true, + queue_idx); + } + } + } + + void decode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) {} + + // decode the data and record necessary information for progressiveness + void progressive_decode(SIZE n, int starting_bitplanes, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> level_signs, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) { + + constexpr SIZE max_batch_per_warp = 8; + if (num_bitplanes > 0) { + DeviceLauncher::Execute( + BPDecoderRegisterBallotKernel( + n, starting_bitplanes, num_bitplanes, abs_max, encoded_bitplanes, + level_signs, v), + queue_idx); + } + } + + void print() const { std::cout << "Grouped bitplane encoder" << std::endl; } + +private: + bool initialized; + Hierarchy *hierarchy; + Array<2, T_error, DeviceType> level_errors_work_array; + Array<1, Byte, DeviceType> level_error_sum_work_array; +}; +} // namespace MDR +} // namespace mgard_x +#endif diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderRegisterMatchAny.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderRegisterMatchAny.hpp new file mode 100644 index 0000000000..f1842e5864 --- /dev/null +++ b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderRegisterMatchAny.hpp @@ -0,0 +1,563 @@ +#ifndef _MDR_BP_ENCODER_REGISTER_MATCH_ANY_HPP +#define _MDR_BP_ENCODER_REGISTER_MATCH_ANY_HPP + +#include "../../RuntimeX/RuntimeX.h" + +#include "BitplaneEncoderInterface.hpp" +#include + +namespace mgard_x { +namespace MDR { + +template +class BPEncoderRegisterMatchAnyFunctor : public Functor { +public: + MGARDX_CONT + BPEncoderRegisterMatchAnyFunctor() {} + MGARDX_CONT + BPEncoderRegisterMatchAnyFunctor( + SIZE n, int num_bitplanes, SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) { + Functor(); + } + + MGARDX_EXEC void encode_batch(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_bitplane buffer = 0; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_bitplane bit = + (v[data_idx] >> (num_bitplanes - 1 - bp_idx)) & (T_bitplane)1; + buffer += bit << BATCH_SIZE - 1 - data_idx; + } + encoded[bp_idx] = buffer; + } + } + + MGARDX_EXEC void error_collect_binary(T_data *shifted_data, T_error *errors, + int num_bitplanes, int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_error diff = (T_error)(fp_data & mask) + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void error_collect_negabinary(T_data *shifted_data, + T_error *errors, int num_bitplanes, + int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_fp ngb_data = Math::binary2negabinary((T_sfp)data); + T_error diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void EncodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE tid = FunctorBase::GetThreadIdX(); + + SIZE lane_id = tid % BATCH_SIZE; + SIZE warp_id = gid / BATCH_SIZE; + + SIZE num_batches = n / BATCH_SIZE; + + T_bitplane buffer; + T_bitplane encoded_data[NUM_BITPLANES]; + T_bitplane encoded_sign; + T_error errors; + + T_fp *sm_p = (T_fp *)FunctorBase::GetSharedMemory(); + T_fp *fp_data = sm_p + BATCH_SIZE * (tid / BATCH_SIZE); + + int exp; + frexp(*abs_max((IDX)0), &exp); + + SIZE actual_batch_per_warp = std::min(M, num_batches - warp_id * M); + +#pragma unroll + for (int i = 0; i < M; i++) { + SIZE batch_idx = warp_id * M + i; + // if (batch_idx < num_batches) { + // actual_batch_per_warp++; + T_data data = *v(batch_idx * BATCH_SIZE + lane_id); + T_data shifted_data = ldexp(data, NUM_BITPLANES - exp); + T_fp fp_data = (T_fp)fabs(shifted_data); + T_fp fp_sign = (T_fp)(signbit(data) == 0 ? 0 : 1); +#define FULL_MASK 0xffffffff + +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + T_bitplane bit = + (fp_data >> (NUM_BITPLANES - 1 - bp_idx)) & (T_bitplane)1; + buffer = __match_any_sync(FULL_MASK, bit); + if (!bit) + buffer ^= FULL_MASK; + buffer = __shfl_sync(FULL_MASK, buffer, 0); + if (lane_id == i) { + encoded_data[bp_idx] = buffer; + } + } + + buffer = __ballot_sync(FULL_MASK, fp_sign); + + if (lane_id == i) { + encoded_sign = buffer; + } + } + + // if (lane_id < actual_batch_per_warp) { +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + *encoded_bitplanes(bp_idx, warp_id * M + lane_id) = encoded_data[bp_idx]; + } + *encoded_bitplanes(0, num_batches + warp_id * M + lane_id) = encoded_sign; +#pragma unroll + for (int bp_idx = 1; bp_idx < NUM_BITPLANES; bp_idx++) { + *encoded_bitplanes(bp_idx, num_batches + warp_id * M + lane_id) = + (T_bitplane)0; + } + // } + } + + MGARDX_EXEC void Operation1() { EncodeBinary(); } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + // size += sizeof(T_fp) * BATCH_SIZE * (256/32); + return size; + } + +private: + // parameters + SIZE n; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPEncoderRegisterMatchAnyKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp encoder"; + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + MGARDX_CONT + BPEncoderRegisterMatchAnyKernel( + SIZE n, int num_bitplanes, SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) {} + + using FunctorType = + BPEncoderRegisterMatchAnyFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + FunctorType functor(n, num_bitplanes, abs_max, v, encoded_bitplanes, + level_errors_workspace); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE num_batches = n / BATCH_SIZE; + tbz = 1; + tby = 1; + tbx = 32; + gridz = 1; + gridy = 1; + // gridx = num_batches / ((tbx/32)*32); + gridx = (num_batches - 1) / ((tbx / 32) * M) + 1; + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; +}; + +template +class BPDecoderRegisterMatchAnyFunctor : public Functor { +public: + MGARDX_CONT + BPDecoderRegisterMatchAnyFunctor() {} + MGARDX_CONT + BPDecoderRegisterMatchAnyFunctor( + SIZE n, int starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) { + Functor(); + } + + MGARDX_EXEC void decode_batch(T_fp *v, T_bitplane *encoded) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_fp buffer = 0; + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_fp bit = (encoded[bp_idx] >> (BATCH_SIZE - 1 - data_idx)) & (T_fp)1; + buffer += bit << (num_bitplanes - 1 - bp_idx); + } + v[data_idx] = buffer; + } + } + + MGARDX_EXEC void DecodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE tid = FunctorBase::GetThreadIdX(); + + uint32_t lane_id = tid % BATCH_SIZE; + SIZE warp_id = gid / BATCH_SIZE; + + SIZE num_batches = n / BATCH_SIZE; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_fp fp_sign[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_bitplane encoded_sign; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + int ending_bitplane = starting_bitplane + num_bitplanes; + + SIZE actual_batch_per_warp = std::min(M, num_batches - warp_id * M); + + if (lane_id < actual_batch_per_warp) { +#pragma unroll + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = *encoded_bitplanes(starting_bitplane + bp_idx, + warp_id * M + lane_id); + } + encoded_sign = *encoded_bitplanes(0, num_batches + warp_id * M + lane_id); + } + + if (lane_id < actual_batch_per_warp) { + // decode data + decode_batch(fp_data, encoded_data); +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + fp_sign[data_idx] = + (encoded_sign >> (BATCH_SIZE - 1 - data_idx)) & (T_fp)1; + } +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + shifted_data[data_idx] = (T_data)fp_data[data_idx]; + // It is beneficial to use pow instead of ldexp + T_data data = shifted_data[data_idx] * pow(2, -ending_bitplane + exp); + // T_data data = ldexp(shifted_data[data_idx], -ending_bitplane + exp); + data = fp_sign[data_idx] ? -data : data; + shifted_data[data_idx] = data; + } + } + + for (u_int32_t mask = 0; mask < BATCH_SIZE; mask++) { + // printf("lane_id: %d, mask: %u, lane_id^mask: %d\n", lane_id, mask, + // lane_id^mask); #define FULL_MASK 0xffffffff + unsigned long long full_mask = 0xFFFFFFFF; + T_data buffer = + __shfl_xor_sync(full_mask, shifted_data[lane_id ^ mask], mask); + shifted_data[lane_id ^ mask] = buffer; + } + + for (int i = 0; i < actual_batch_per_warp; i++) { + SIZE batch_idx = warp_id * M + i; + *v(batch_idx * BATCH_SIZE + lane_id) = shifted_data[i]; + } + } + + MGARDX_EXEC void Operation1() { DecodeBinary(); } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + int starting_bitplane; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPDecoderRegisterMatchAnyKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp decoder"; + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + MGARDX_CONT + BPDecoderRegisterMatchAnyKernel( + SIZE n, int starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) {} + + using FunctorType = + BPDecoderRegisterMatchAnyFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + + FunctorType functor(n, starting_bitplane, num_bitplanes, abs_max, + encoded_bitplanes, signs, v); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE num_batches = n / BATCH_SIZE; + tbz = 1; + tby = 1; + tbx = 32; + gridz = 1; + gridy = 1; + // gridx = num_batches / ((tbx/32)*32); + gridx = (num_batches - 1) / ((tbx / 32) * M) + 1; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int starting_bitplane; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; +}; + +// general bitplane encoder that encodes data by block using T_stream type +// buffer +template +class BPEncoderRegisterMatchAny + : public concepts::BitplaneEncoderInterface { +public: + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + using T_sfp = typename std::conditional::value, + int64_t, int32_t>::type; + using T_fp = typename std::conditional::value, + uint64_t, uint32_t>::type; + + BPEncoderRegisterMatchAny() : initialized(false) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + } + BPEncoderRegisterMatchAny(Hierarchy &hierarchy) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + Adapt(hierarchy, 0); + DeviceRuntime::SyncQueue(0); + } + + static SIZE bitplane_length(SIZE n) { + if constexpr (!NegaBinary) { + return num_blocks(n) * 2; + } else { + return num_blocks(n); + } + } + + static SIZE num_blocks(SIZE n) { + const SIZE batch_size = sizeof(T_bitplane) * 8; + SIZE num_blocks = (n - 1) / batch_size + 1; + return num_blocks; + } + + void Adapt(Hierarchy &hierarchy, int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + + level_errors_work_array.resize( + {MAX_BITPLANES + 1, num_blocks(max_level_num_elems)}, queue_idx); + DeviceCollective::Sum( + num_blocks(max_level_num_elems), SubArray<1, T_error, DeviceType>(), + SubArray<1, T_error, DeviceType>(), level_error_sum_work_array, false, + queue_idx); + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + Hierarchy hierarchy(shape, Config()); + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + size_t size = 0; + size += hierarchy.EstimateMemoryFootprint(shape); + size += + (MAX_BITPLANES + 1) * num_blocks(max_level_num_elems) * sizeof(T_error); + for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { + size += hierarchy.level_num_elems(level_idx) * sizeof(bool); + } + return size; + } + + void encode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, T_error, DeviceType> level_errors, int queue_idx) { + + SubArray<2, T_error, DeviceType> level_errors_work(level_errors_work_array); + + constexpr SIZE max_batch_per_warp = 8; + DeviceLauncher::Execute( + BPEncoderRegisterMatchAnyKernel< + T_data, T_fp, T_sfp, T_bitplane, T_error, MAX_BITPLANES, + max_batch_per_warp, NegaBinary, CollectError, DeviceType>( + n, num_bitplanes, abs_max, v, encoded_bitplanes, level_errors_work), + queue_idx); + + if constexpr (CollectError) { + SIZE reduce_size = num_blocks(n); + for (int i = 0; i < num_bitplanes + 1; i++) { + SubArray<1, T_error, DeviceType> curr_errors({reduce_size}, + level_errors_work(i, 0)); + SubArray<1, T_error, DeviceType> sum_error({1}, level_errors(i)); + DeviceCollective::Sum(reduce_size, curr_errors, sum_error, + level_error_sum_work_array, true, + queue_idx); + } + } + } + + void decode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) {} + + // decode the data and record necessary information for progressiveness + void progressive_decode(SIZE n, int starting_bitplanes, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> level_signs, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) { + + constexpr SIZE max_batch_per_warp = 8; + if (num_bitplanes > 0) { + DeviceLauncher::Execute( + BPDecoderRegisterMatchAnyKernel( + n, starting_bitplanes, num_bitplanes, abs_max, encoded_bitplanes, + level_signs, v), + queue_idx); + } + } + + void print() const { std::cout << "Grouped bitplane encoder" << std::endl; } + +private: + bool initialized; + Hierarchy *hierarchy; + Array<2, T_error, DeviceType> level_errors_work_array; + Array<1, Byte, DeviceType> level_error_sum_work_array; +}; +} // namespace MDR +} // namespace mgard_x +#endif diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderRegisterReduceAll.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderRegisterReduceAll.hpp new file mode 100644 index 0000000000..bb48c161a1 --- /dev/null +++ b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderRegisterReduceAll.hpp @@ -0,0 +1,563 @@ +#ifndef _MDR_BP_ENCODER_REGISTER_REDUCE_ALL_HPP +#define _MDR_BP_ENCODER_REGISTER_REDUCE_ALL_HPP + +#include "../../RuntimeX/RuntimeX.h" + +#include "BitplaneEncoderInterface.hpp" +#include + +namespace mgard_x { +namespace MDR { + +template +class BPEncoderRegisterReduceAllFunctor : public Functor { +public: + MGARDX_CONT + BPEncoderRegisterReduceAllFunctor() {} + MGARDX_CONT + BPEncoderRegisterReduceAllFunctor( + SIZE n, int num_bitplanes, SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) { + Functor(); + } + + MGARDX_EXEC void encode_batch(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_bitplane buffer = 0; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_bitplane bit = + (v[data_idx] >> (num_bitplanes - 1 - bp_idx)) & (T_bitplane)1; + buffer += bit << BATCH_SIZE - 1 - data_idx; + } + encoded[bp_idx] = buffer; + } + } + + MGARDX_EXEC void error_collect_binary(T_data *shifted_data, T_error *errors, + int num_bitplanes, int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_error diff = (T_error)(fp_data & mask) + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void error_collect_negabinary(T_data *shifted_data, + T_error *errors, int num_bitplanes, + int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_fp ngb_data = Math::binary2negabinary((T_sfp)data); + T_error diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void EncodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE tid = FunctorBase::GetThreadIdX(); + + SIZE lane_id = tid % BATCH_SIZE; + SIZE warp_id = gid / BATCH_SIZE; + + SIZE num_batches = n / BATCH_SIZE; + + T_bitplane buffer; + T_bitplane encoded_data[NUM_BITPLANES]; + T_bitplane encoded_sign; + T_error errors; + + T_fp *sm_p = (T_fp *)FunctorBase::GetSharedMemory(); + T_fp *fp_data = sm_p + BATCH_SIZE * (tid / BATCH_SIZE); + + int exp; + frexp(*abs_max((IDX)0), &exp); + + SIZE actual_batch_per_warp = std::min(M, num_batches - warp_id * M); + +#pragma unroll + for (int i = 0; i < M; i++) { + SIZE batch_idx = warp_id * M + i; + // if (batch_idx < num_batches) { + // actual_batch_per_warp++; + T_data data = *v(batch_idx * BATCH_SIZE + lane_id); + T_data shifted_data = ldexp(data, NUM_BITPLANES - exp); + T_fp fp_data = (T_fp)fabs(shifted_data); + T_fp fp_sign = (T_fp)(signbit(data) == 0 ? 0 : 1); +#define FULL_MASK 0xffffffff + +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + T_bitplane bit = + (fp_data >> (NUM_BITPLANES - 1 - bp_idx)) & (T_bitplane)1; + T_bitplane shifted_bit = bit << BATCH_SIZE - 1 - lane_id; + buffer = __reduce_add_sync(FULL_MASK, shifted_bit); + buffer = __shfl_sync(FULL_MASK, buffer, 0); + if (lane_id == i) { + encoded_data[bp_idx] = buffer; + } + } + + encoded_sign = fp_sign << BATCH_SIZE - 1 - lane_id; + encoded_sign = __reduce_add_sync(FULL_MASK, encoded_sign); + + if (lane_id == i) { + encoded_sign = buffer; + } + } + + // if (lane_id < actual_batch_per_warp) { +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + *encoded_bitplanes(bp_idx, warp_id * M + lane_id) = encoded_data[bp_idx]; + } + *encoded_bitplanes(0, num_batches + warp_id * M + lane_id) = encoded_sign; +#pragma unroll + for (int bp_idx = 1; bp_idx < NUM_BITPLANES; bp_idx++) { + *encoded_bitplanes(bp_idx, num_batches + warp_id * M + lane_id) = + (T_bitplane)0; + } + // } + } + + MGARDX_EXEC void Operation1() { EncodeBinary(); } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + // size += sizeof(T_fp) * BATCH_SIZE * (256/32); + return size; + } + +private: + // parameters + SIZE n; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPEncoderRegisterReduceAllKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp encoder"; + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + MGARDX_CONT + BPEncoderRegisterReduceAllKernel( + SIZE n, int num_bitplanes, SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) {} + + using FunctorType = + BPEncoderRegisterReduceAllFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + FunctorType functor(n, num_bitplanes, abs_max, v, encoded_bitplanes, + level_errors_workspace); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE num_batches = n / BATCH_SIZE; + tbz = 1; + tby = 1; + tbx = 32; + gridz = 1; + gridy = 1; + // gridx = num_batches / ((tbx/32)*32); + gridx = (num_batches - 1) / ((tbx / 32) * M) + 1; + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; +}; + +template +class BPDecoderRegisterReduceAllFunctor : public Functor { +public: + MGARDX_CONT + BPDecoderRegisterReduceAllFunctor() {} + MGARDX_CONT + BPDecoderRegisterReduceAllFunctor( + SIZE n, int starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) { + Functor(); + } + + MGARDX_EXEC void decode_batch(T_fp *v, T_bitplane *encoded) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_fp buffer = 0; + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_fp bit = (encoded[bp_idx] >> (BATCH_SIZE - 1 - data_idx)) & (T_fp)1; + buffer += bit << (num_bitplanes - 1 - bp_idx); + } + v[data_idx] = buffer; + } + } + + MGARDX_EXEC void DecodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE tid = FunctorBase::GetThreadIdX(); + + uint32_t lane_id = tid % BATCH_SIZE; + SIZE warp_id = gid / BATCH_SIZE; + + SIZE num_batches = n / BATCH_SIZE; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_fp fp_sign[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_bitplane encoded_sign; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + int ending_bitplane = starting_bitplane + num_bitplanes; + + SIZE actual_batch_per_warp = std::min(M, num_batches - warp_id * M); + + if (lane_id < actual_batch_per_warp) { +#pragma unroll + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = *encoded_bitplanes(starting_bitplane + bp_idx, + warp_id * M + lane_id); + } + encoded_sign = *encoded_bitplanes(0, num_batches + warp_id * M + lane_id); + } + + if (lane_id < actual_batch_per_warp) { + // decode data + decode_batch(fp_data, encoded_data); +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + fp_sign[data_idx] = + (encoded_sign >> (BATCH_SIZE - 1 - data_idx)) & (T_fp)1; + } +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + shifted_data[data_idx] = (T_data)fp_data[data_idx]; + // It is beneficial to use pow instead of ldexp + T_data data = shifted_data[data_idx] * pow(2, -ending_bitplane + exp); + // T_data data = ldexp(shifted_data[data_idx], -ending_bitplane + exp); + data = fp_sign[data_idx] ? -data : data; + shifted_data[data_idx] = data; + } + } + + for (u_int32_t mask = 0; mask < BATCH_SIZE; mask++) { + // printf("lane_id: %d, mask: %u, lane_id^mask: %d\n", lane_id, mask, + // lane_id^mask); #define FULL_MASK 0xffffffff + unsigned long long full_mask = 0xFFFFFFFF; + T_data buffer = + __shfl_xor_sync(full_mask, shifted_data[lane_id ^ mask], mask); + shifted_data[lane_id ^ mask] = buffer; + } + + for (int i = 0; i < actual_batch_per_warp; i++) { + SIZE batch_idx = warp_id * M + i; + *v(batch_idx * BATCH_SIZE + lane_id) = shifted_data[i]; + } + } + + MGARDX_EXEC void Operation1() { DecodeBinary(); } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + int starting_bitplane; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPDecoderRegisterReduceAllKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp decoder"; + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + MGARDX_CONT + BPDecoderRegisterReduceAllKernel( + SIZE n, int starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) {} + + using FunctorType = + BPDecoderRegisterReduceAllFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + + FunctorType functor(n, starting_bitplane, num_bitplanes, abs_max, + encoded_bitplanes, signs, v); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE num_batches = n / BATCH_SIZE; + tbz = 1; + tby = 1; + tbx = 32; + gridz = 1; + gridy = 1; + // gridx = num_batches / ((tbx/32)*32); + gridx = (num_batches - 1) / ((tbx / 32) * M) + 1; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int starting_bitplane; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; +}; + +// general bitplane encoder that encodes data by block using T_stream type +// buffer +template +class BPEncoderRegisterReduceAll + : public concepts::BitplaneEncoderInterface { +public: + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + using T_sfp = typename std::conditional::value, + int64_t, int32_t>::type; + using T_fp = typename std::conditional::value, + uint64_t, uint32_t>::type; + + BPEncoderRegisterReduceAll() : initialized(false) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + } + BPEncoderRegisterReduceAll(Hierarchy &hierarchy) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + Adapt(hierarchy, 0); + DeviceRuntime::SyncQueue(0); + } + + static SIZE bitplane_length(SIZE n) { + if constexpr (!NegaBinary) { + return num_blocks(n) * 2; + } else { + return num_blocks(n); + } + } + + static SIZE num_blocks(SIZE n) { + const SIZE batch_size = sizeof(T_bitplane) * 8; + SIZE num_blocks = (n - 1) / batch_size + 1; + return num_blocks; + } + + void Adapt(Hierarchy &hierarchy, int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + + level_errors_work_array.resize( + {MAX_BITPLANES + 1, num_blocks(max_level_num_elems)}, queue_idx); + DeviceCollective::Sum( + num_blocks(max_level_num_elems), SubArray<1, T_error, DeviceType>(), + SubArray<1, T_error, DeviceType>(), level_error_sum_work_array, false, + queue_idx); + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + Hierarchy hierarchy(shape, Config()); + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + size_t size = 0; + size += hierarchy.EstimateMemoryFootprint(shape); + size += + (MAX_BITPLANES + 1) * num_blocks(max_level_num_elems) * sizeof(T_error); + for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { + size += hierarchy.level_num_elems(level_idx) * sizeof(bool); + } + return size; + } + + void encode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, T_error, DeviceType> level_errors, int queue_idx) { + + SubArray<2, T_error, DeviceType> level_errors_work(level_errors_work_array); + + constexpr SIZE max_batch_per_warp = 8; + DeviceLauncher::Execute( + BPEncoderRegisterReduceAllKernel< + T_data, T_fp, T_sfp, T_bitplane, T_error, MAX_BITPLANES, + max_batch_per_warp, NegaBinary, CollectError, DeviceType>( + n, num_bitplanes, abs_max, v, encoded_bitplanes, level_errors_work), + queue_idx); + + if constexpr (CollectError) { + SIZE reduce_size = num_blocks(n); + for (int i = 0; i < num_bitplanes + 1; i++) { + SubArray<1, T_error, DeviceType> curr_errors({reduce_size}, + level_errors_work(i, 0)); + SubArray<1, T_error, DeviceType> sum_error({1}, level_errors(i)); + DeviceCollective::Sum(reduce_size, curr_errors, sum_error, + level_error_sum_work_array, true, + queue_idx); + } + } + } + + void decode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) {} + + // decode the data and record necessary information for progressiveness + void progressive_decode(SIZE n, int starting_bitplanes, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> level_signs, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) { + + constexpr SIZE max_batch_per_warp = 8; + if (num_bitplanes > 0) { + DeviceLauncher::Execute( + BPDecoderRegisterReduceAllKernel( + n, starting_bitplanes, num_bitplanes, abs_max, encoded_bitplanes, + level_signs, v), + queue_idx); + } + } + + void print() const { std::cout << "Grouped bitplane encoder" << std::endl; } + +private: + bool initialized; + Hierarchy *hierarchy; + Array<2, T_error, DeviceType> level_errors_work_array; + Array<1, Byte, DeviceType> level_error_sum_work_array; +}; +} // namespace MDR +} // namespace mgard_x +#endif \ No newline at end of file diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderRegisterShift.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderRegisterShift.hpp new file mode 100644 index 0000000000..db4c8a6eb9 --- /dev/null +++ b/include/mgard-x/MDR-X/BitplaneEncoder/old/BPEncoderRegisterShift.hpp @@ -0,0 +1,567 @@ +#ifndef _MDR_BP_ENCODER_REGISTER_SHIFT_HPP +#define _MDR_BP_ENCODER_REGISTER_SHIFT_HPP + +#include "../../RuntimeX/RuntimeX.h" + +#include "BitplaneEncoderInterface.hpp" +#include + +namespace mgard_x { +namespace MDR { + +template +class BPEncoderRegisterShiftFunctor : public Functor { +public: + MGARDX_CONT + BPEncoderRegisterShiftFunctor() {} + MGARDX_CONT + BPEncoderRegisterShiftFunctor( + SIZE n, int num_bitplanes, SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) { + Functor(); + } + + MGARDX_EXEC void encode_batch(T_fp *v, T_bitplane *encoded, + int num_bitplanes) { + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_bitplane buffer = 0; + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_bitplane bit = + (v[data_idx] >> (num_bitplanes - 1 - bp_idx)) & (T_bitplane)1; + buffer += bit << BATCH_SIZE - 1 - data_idx; + } + encoded[bp_idx] = buffer; + } + } + + MGARDX_EXEC void error_collect_binary(T_data *shifted_data, T_error *errors, + int num_bitplanes, int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_error diff = (T_error)(fp_data & mask) + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void error_collect_negabinary(T_data *shifted_data, + T_error *errors, int num_bitplanes, + int exp) { + + int batch_idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_error mantissa = fabs(data) - fp_data; + T_fp mask = ((T_fp)1 << bp_idx) - 1; + T_fp ngb_data = Math::binary2negabinary((T_sfp)data); + T_error diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + // if (bp_idx == 31 && batch_idx == 0) { + // printf( + // "data: %f fp_data: %llu fps_data: %lld mask: %llu diff: + // %f\n", data, fp_data, sfp_data, mask, diff); + // } + errors[num_bitplanes - bp_idx] += diff * diff; + } + } + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_data data = shifted_data[data_idx]; + errors[0] += data * data; + } + + for (int bp_idx = 0; bp_idx < num_bitplanes + 1; bp_idx++) { + errors[bp_idx] = ldexp(errors[bp_idx], 2 * (-(int)num_bitplanes + exp)); + } + } + + MGARDX_EXEC void EncodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE tid = FunctorBase::GetThreadIdX(); + + SIZE lane_id = tid % BATCH_SIZE; + SIZE warp_id = gid / BATCH_SIZE; + + SIZE num_batches = n / BATCH_SIZE; + + T_bitplane buffer; + T_bitplane encoded_data[NUM_BITPLANES]; + T_bitplane encoded_sign; + T_error errors; + + T_fp *sm_p = (T_fp *)FunctorBase::GetSharedMemory(); + T_fp *fp_data = sm_p + BATCH_SIZE * (tid / BATCH_SIZE); + + int exp; + frexp(*abs_max((IDX)0), &exp); + + SIZE actual_batch_per_warp = std::min(M, num_batches - warp_id * M); + +#pragma unroll + for (int i = 0; i < M; i++) { + SIZE batch_idx = warp_id * M + i; + // if (batch_idx < num_batches) { + // actual_batch_per_warp++; + T_data data = *v(batch_idx * BATCH_SIZE + lane_id); + T_data shifted_data = ldexp(data, NUM_BITPLANES - exp); + T_fp fp_data = (T_fp)fabs(shifted_data); + T_fp fp_sign = (T_fp)(signbit(data) == 0 ? 0 : 1); +#define FULL_MASK 0xffffffff + +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + T_bitplane bit = + (fp_data >> (NUM_BITPLANES - 1 - bp_idx)) & (T_bitplane)1; + T_bitplane shifted_bit = bit << BATCH_SIZE - 1 - lane_id; + for (int offset = 16; offset > 0; offset /= 2) { + buffer |= __shfl_down_sync(FULL_MASK, shifted_bit, offset); + } + buffer = __shfl_sync(FULL_MASK, buffer, 0); + + if (lane_id == i) { + encoded_data[bp_idx] = buffer; + } + } + encoded_sign = fp_sign << BATCH_SIZE - 1 - lane_id; + for (int offset = 16; offset > 0; offset /= 2) { + encoded_sign |= __shfl_down_sync(FULL_MASK, encoded_sign, offset); + } + + if (lane_id == i) { + encoded_sign = buffer; + } + } + + // if (lane_id < actual_batch_per_warp) { +#pragma unroll + for (int bp_idx = 0; bp_idx < NUM_BITPLANES; bp_idx++) { + *encoded_bitplanes(bp_idx, warp_id * M + lane_id) = encoded_data[bp_idx]; + } + *encoded_bitplanes(0, num_batches + warp_id * M + lane_id) = encoded_sign; +#pragma unroll + for (int bp_idx = 1; bp_idx < NUM_BITPLANES; bp_idx++) { + *encoded_bitplanes(bp_idx, num_batches + warp_id * M + lane_id) = + (T_bitplane)0; + } + // } + } + + MGARDX_EXEC void Operation1() { EncodeBinary(); } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + // size += sizeof(T_fp) * BATCH_SIZE * (256/32); + return size; + } + +private: + // parameters + SIZE n; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPEncoderRegisterShiftKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp encoder"; + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + MGARDX_CONT + BPEncoderRegisterShiftKernel( + SIZE n, int num_bitplanes, SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<2, T_error, DeviceType> level_errors_workspace) + : n(n), num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), v(v), + level_errors_workspace(level_errors_workspace) {} + + using FunctorType = + BPEncoderRegisterShiftFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + FunctorType functor(n, num_bitplanes, abs_max, v, encoded_bitplanes, + level_errors_workspace); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE num_batches = n / BATCH_SIZE; + tbz = 1; + tby = 1; + tbx = 32; + gridz = 1; + gridy = 1; + // gridx = num_batches / ((tbx/32)*32); + gridx = (num_batches - 1) / ((tbx / 32) * M) + 1; + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<1, T_data, DeviceType> v; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<2, T_error, DeviceType> level_errors_workspace; +}; + +template +class BPDecoderRegisterShiftFunctor : public Functor { +public: + MGARDX_CONT + BPDecoderRegisterShiftFunctor() {} + MGARDX_CONT + BPDecoderRegisterShiftFunctor( + SIZE n, int starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) { + Functor(); + } + + MGARDX_EXEC void decode_batch(T_fp *v, T_bitplane *encoded) { + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + T_fp buffer = 0; + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + T_fp bit = (encoded[bp_idx] >> (BATCH_SIZE - 1 - data_idx)) & (T_fp)1; + buffer += bit << (num_bitplanes - 1 - bp_idx); + } + v[data_idx] = buffer; + } + } + + MGARDX_EXEC void DecodeBinary() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + SIZE tid = FunctorBase::GetThreadIdX(); + + uint32_t lane_id = tid % BATCH_SIZE; + SIZE warp_id = gid / BATCH_SIZE; + + SIZE num_batches = n / BATCH_SIZE; + + T_data shifted_data[BATCH_SIZE]; + T_fp fp_data[BATCH_SIZE]; + T_fp fp_sign[BATCH_SIZE]; + T_bitplane encoded_data[MAX_BITPLANES]; + T_bitplane encoded_sign; + + int exp; + frexp(*abs_max((IDX)0), &exp); + + int ending_bitplane = starting_bitplane + num_bitplanes; + + SIZE actual_batch_per_warp = std::min(M, num_batches - warp_id * M); + + if (lane_id < actual_batch_per_warp) { +#pragma unroll + for (int bp_idx = 0; bp_idx < num_bitplanes; bp_idx++) { + encoded_data[bp_idx] = *encoded_bitplanes(starting_bitplane + bp_idx, + warp_id * M + lane_id); + } + encoded_sign = *encoded_bitplanes(0, num_batches + warp_id * M + lane_id); + } + + if (lane_id < actual_batch_per_warp) { + // decode data + decode_batch(fp_data, encoded_data); +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + fp_sign[data_idx] = + (encoded_sign >> (BATCH_SIZE - 1 - data_idx)) & (T_fp)1; + } +#pragma unroll + for (int data_idx = 0; data_idx < BATCH_SIZE; data_idx++) { + shifted_data[data_idx] = (T_data)fp_data[data_idx]; + // It is beneficial to use pow instead of ldexp + T_data data = shifted_data[data_idx] * pow(2, -ending_bitplane + exp); + // T_data data = ldexp(shifted_data[data_idx], -ending_bitplane + exp); + data = fp_sign[data_idx] ? -data : data; + shifted_data[data_idx] = data; + } + } + + for (u_int32_t mask = 0; mask < BATCH_SIZE; mask++) { + // printf("lane_id: %d, mask: %u, lane_id^mask: %d\n", lane_id, mask, + // lane_id^mask); #define FULL_MASK 0xffffffff + unsigned long long full_mask = 0xFFFFFFFF; + T_data buffer = + __shfl_xor_sync(full_mask, shifted_data[lane_id ^ mask], mask); + shifted_data[lane_id ^ mask] = buffer; + } + + for (int i = 0; i < actual_batch_per_warp; i++) { + SIZE batch_idx = warp_id * M + i; + *v(batch_idx * BATCH_SIZE + lane_id) = shifted_data[i]; + } + } + + MGARDX_EXEC void Operation1() { DecodeBinary(); } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + // parameters + SIZE n; + int starting_bitplane; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; +}; + +template +class BPDecoderRegisterShiftKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "grouped bp decoder"; + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + MGARDX_CONT + BPDecoderRegisterShiftKernel( + SIZE n, int starting_bitplane, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> signs, SubArray<1, T_data, DeviceType> v) + : n(n), starting_bitplane(starting_bitplane), + num_bitplanes(num_bitplanes), abs_max(abs_max), + encoded_bitplanes(encoded_bitplanes), signs(signs), v(v) {} + + using FunctorType = + BPDecoderRegisterShiftFunctor; + using TaskType = Task; + + MGARDX_CONT TaskType GenTask(int queue_idx) { + + FunctorType functor(n, starting_bitplane, num_bitplanes, abs_max, + encoded_bitplanes, signs, v); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE num_batches = n / BATCH_SIZE; + tbz = 1; + tby = 1; + tbx = 32; + gridz = 1; + gridy = 1; + // gridx = num_batches / ((tbx/32)*32); + gridx = (num_batches - 1) / ((tbx / 32) * M) + 1; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE n; + int starting_bitplane; + int num_bitplanes; + SubArray<1, T_data, DeviceType> abs_max; + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes; + SubArray<1, bool, DeviceType> signs; + SubArray<1, T_data, DeviceType> v; +}; + +// general bitplane encoder that encodes data by block using T_stream type +// buffer +template +class BPEncoderRegisterShift + : public concepts::BitplaneEncoderInterface { +public: + static constexpr int BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr int MAX_BITPLANES = sizeof(T_data) * 8; + using T_sfp = typename std::conditional::value, + int64_t, int32_t>::type; + using T_fp = typename std::conditional::value, + uint64_t, uint32_t>::type; + + BPEncoderRegisterShift() : initialized(false) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + } + BPEncoderRegisterShift(Hierarchy &hierarchy) { + static_assert(std::is_floating_point::value, + "GeneralBPEncoder: input data must be floating points."); + static_assert(!std::is_same::value, + "GeneralBPEncoder: long double is not supported."); + static_assert(std::is_unsigned::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + static_assert(std::is_integral::value, + "GroupedBPBlockEncoder: streams must be unsigned integers."); + Adapt(hierarchy, 0); + DeviceRuntime::SyncQueue(0); + } + + static SIZE bitplane_length(SIZE n) { + if constexpr (!NegaBinary) { + return num_blocks(n) * 2; + } else { + return num_blocks(n); + } + } + + static SIZE num_blocks(SIZE n) { + const SIZE batch_size = sizeof(T_bitplane) * 8; + SIZE num_blocks = (n - 1) / batch_size + 1; + return num_blocks; + } + + void Adapt(Hierarchy &hierarchy, int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + + level_errors_work_array.resize( + {MAX_BITPLANES + 1, num_blocks(max_level_num_elems)}, queue_idx); + DeviceCollective::Sum( + num_blocks(max_level_num_elems), SubArray<1, T_error, DeviceType>(), + SubArray<1, T_error, DeviceType>(), level_error_sum_work_array, false, + queue_idx); + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + Hierarchy hierarchy(shape, Config()); + SIZE max_level_num_elems = hierarchy.level_num_elems(hierarchy.l_target()); + size_t size = 0; + size += hierarchy.EstimateMemoryFootprint(shape); + size += + (MAX_BITPLANES + 1) * num_blocks(max_level_num_elems) * sizeof(T_error); + for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { + size += hierarchy.level_num_elems(level_idx) * sizeof(bool); + } + return size; + } + + void encode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<1, T_data, DeviceType> v, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, T_error, DeviceType> level_errors, int queue_idx) { + + SubArray<2, T_error, DeviceType> level_errors_work(level_errors_work_array); + + constexpr SIZE max_batch_per_warp = 8; + DeviceLauncher::Execute( + BPEncoderRegisterShiftKernel( + n, num_bitplanes, abs_max, v, encoded_bitplanes, level_errors_work), + queue_idx); + + if constexpr (CollectError) { + SIZE reduce_size = num_blocks(n); + for (int i = 0; i < num_bitplanes + 1; i++) { + SubArray<1, T_error, DeviceType> curr_errors({reduce_size}, + level_errors_work(i, 0)); + SubArray<1, T_error, DeviceType> sum_error({1}, level_errors(i)); + DeviceCollective::Sum(reduce_size, curr_errors, sum_error, + level_error_sum_work_array, true, + queue_idx); + } + } + } + + void decode(SIZE n, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) {} + + // decode the data and record necessary information for progressiveness + void progressive_decode(SIZE n, int starting_bitplanes, int num_bitplanes, + SubArray<1, T_data, DeviceType> abs_max, + SubArray<2, T_bitplane, DeviceType> encoded_bitplanes, + SubArray<1, bool, DeviceType> level_signs, int level, + SubArray<1, T_data, DeviceType> v, int queue_idx) { + + constexpr SIZE max_batch_per_warp = 8; + if (num_bitplanes > 0) { + DeviceLauncher::Execute( + BPDecoderRegisterShiftKernel( + n, starting_bitplanes, num_bitplanes, abs_max, encoded_bitplanes, + level_signs, v), + queue_idx); + } + } + + void print() const { std::cout << "Grouped bitplane encoder" << std::endl; } + +private: + bool initialized; + Hierarchy *hierarchy; + Array<2, T_error, DeviceType> level_errors_work_array; + Array<1, Byte, DeviceType> level_error_sum_work_array; +}; +} // namespace MDR +} // namespace mgard_x +#endif diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/GroupedBPEncoder.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/old/GroupedBPEncoder.hpp similarity index 100% rename from include/mgard-x/MDR-X/BitplaneEncoder/GroupedBPEncoder.hpp rename to include/mgard-x/MDR-X/BitplaneEncoder/old/GroupedBPEncoder.hpp diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/GroupedWarpBPEncoderGPU.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/old/GroupedWarpBPEncoderGPU.hpp similarity index 100% rename from include/mgard-x/MDR-X/BitplaneEncoder/GroupedWarpBPEncoderGPU.hpp rename to include/mgard-x/MDR-X/BitplaneEncoder/old/GroupedWarpBPEncoderGPU.hpp diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/NegaBinaryBPEncoder.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/old/NegaBinaryBPEncoder.hpp similarity index 100% rename from include/mgard-x/MDR-X/BitplaneEncoder/NegaBinaryBPEncoder.hpp rename to include/mgard-x/MDR-X/BitplaneEncoder/old/NegaBinaryBPEncoder.hpp diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/PerBitBPEncoder.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/old/PerBitBPEncoder.hpp similarity index 100% rename from include/mgard-x/MDR-X/BitplaneEncoder/PerBitBPEncoder.hpp rename to include/mgard-x/MDR-X/BitplaneEncoder/old/PerBitBPEncoder.hpp diff --git a/include/mgard-x/MDR-X/BitplaneEncoder/PerBitBPEncoderGPU.hpp b/include/mgard-x/MDR-X/BitplaneEncoder/old/PerBitBPEncoderGPU.hpp similarity index 100% rename from include/mgard-x/MDR-X/BitplaneEncoder/PerBitBPEncoderGPU.hpp rename to include/mgard-x/MDR-X/BitplaneEncoder/old/PerBitBPEncoderGPU.hpp diff --git a/include/mgard-x/MDR-X/Decomposer/MGARD.hpp b/include/mgard-x/MDR-X/Decomposer/MGARD.hpp index d40379fc7a..19aa778e87 100644 --- a/include/mgard-x/MDR-X/Decomposer/MGARD.hpp +++ b/include/mgard-x/MDR-X/Decomposer/MGARD.hpp @@ -8,14 +8,16 @@ namespace mgard_x { namespace MDR { -// MGARD decomposer with orthogonal basis -template -class MGARDOrthoganalDecomposer - : public concepts::DecomposerInterface { + +struct DecompsitionBasis {}; +struct Orthogonal : DecompsitionBasis {}; +struct Hierarchical : DecompsitionBasis {}; + +template +class MGARDDecomposer : public concepts::DecomposerInterface { public: - MGARDOrthoganalDecomposer() : initialized(false) {} - MGARDOrthoganalDecomposer(Hierarchy &hierarchy, - Config config) { + MGARDDecomposer() : initialized(false) {} + MGARDDecomposer(Hierarchy &hierarchy, Config config) { Adapt(hierarchy, config, 0); DeviceRuntime::SyncQueue(0); } @@ -36,15 +38,21 @@ class MGARDOrthoganalDecomposer } void decompose(Array &v, int start_level, int stop_level, int queue_idx) { - refactor.Decompose(v, start_level, stop_level, queue_idx); + if constexpr (std::is_same::value) { + refactor.Decompose(v, start_level, stop_level, true, queue_idx); + } else if constexpr (std::is_same::value) { + refactor.Decompose(v, start_level, stop_level, false, queue_idx); + } } void recompose(Array &v, int start_level, int stop_level, int queue_idx) { - refactor.Recompose(v, start_level, stop_level, queue_idx); - } - void print() const { - std::cout << "MGARD orthogonal decomposer" << std::endl; + if constexpr (std::is_same::value) { + refactor.Recompose(v, start_level, stop_level, true, queue_idx); + } else if constexpr (std::is_same::value) { + refactor.Recompose(v, start_level, stop_level, false, queue_idx); + } } + void print() const { std::cout << "MGARD decomposer" << std::endl; } private: bool initialized; diff --git a/include/mgard-x/MDR-X/ErrorEstimator/MaxErrorEstimator.hpp b/include/mgard-x/MDR-X/ErrorEstimator/MaxErrorEstimator.hpp index 8945c63762..4caa72536b 100644 --- a/include/mgard-x/MDR-X/ErrorEstimator/MaxErrorEstimator.hpp +++ b/include/mgard-x/MDR-X/ErrorEstimator/MaxErrorEstimator.hpp @@ -21,9 +21,9 @@ template class MaxErrorEstimatorOB : public MaxErrorEstimator { c = 1.0 + 21.0 * sqrt(3) / 8; break; default: - std::cerr << num_dims << "-Dimentional error estimation not implemented." - << std::endl; - exit(-1); + throw std::runtime_error( + std::to_string(num_dims) + + "-Dimentional error estimation not implemented."); } c *= 4; // 2 more bitplane for negabinary } @@ -48,22 +48,32 @@ template class MaxErrorEstimatorOB : public MaxErrorEstimator { T c = 0; }; // max error estimator for hierarchical basis -// c = 1 as all the operations are linear +// c = 1 as all the operations are linear (before the negabinary correction +// below: negabinary shifts the encoder's fixed-point exponent by 2 extra +// bits of range headroom -- see BPEncoderLocalityBlock/RegisterBlock's +// EncodeNegaBinary, `exp += 2` -- so a given bitplane count buys less +// achievable precision under negabinary than under binary. MaxErrorEstimatorOB +// already discounts for this ("2 more bitplane for negabinary"); this was +// missing here, which let the greedy interpreter under-request bitplanes for +// Hierarchical + NegaBinary and narrowly miss the requested L-infinity bound. template class MaxErrorEstimatorHB : public MaxErrorEstimator { public: - MaxErrorEstimatorHB() {} - inline T estimate_error(T error, int level) const { return error; } + MaxErrorEstimatorHB() { c *= 4; } + inline T estimate_error(T error, int level) const { return c * error; } inline T estimate_error(T data, T reconstructed_data, int level) const { - return data - reconstructed_data; + return c * (data - reconstructed_data); } inline T estimate_error_gain(T base, T current_level_err, T next_level_err, int level) const { - return current_level_err - next_level_err; + return c * (current_level_err - next_level_err); } void print() const { std::cout << "Max absolute error estimator for hierarchical basis." << std::endl; } + +private: + T c = 1; }; } // namespace MDR } // namespace mgard_x diff --git a/include/mgard-x/MDR-X/Interleaver/DirectInterleaver.hpp b/include/mgard-x/MDR-X/Interleaver/DirectInterleaver.hpp index 61388c48db..23a0cc54d3 100644 --- a/include/mgard-x/MDR-X/Interleaver/DirectInterleaver.hpp +++ b/include/mgard-x/MDR-X/Interleaver/DirectInterleaver.hpp @@ -3,7 +3,7 @@ #include "../../RuntimeX/RuntimeX.h" -#include "../../Linearization/LevelLinearizer.hpp" +#include "LevelLinearizer.hpp" #include "InterleaverInterface.hpp" @@ -70,11 +70,12 @@ class DirectInterleaver void Adapt(Hierarchy &hierarchy, int queue_idx) { this->initialized = true; this->hierarchy = &hierarchy; - if (initialized) { + if (levels_decomposed_data_device_length < hierarchy.l_target() + 1) { MemoryManager::Free(levels_decomposed_data_device, queue_idx); + MemoryManager::Malloc1D(levels_decomposed_data_device, + hierarchy.l_target() + 1, queue_idx); + levels_decomposed_data_device_length = hierarchy.l_target() + 1; } - MemoryManager::Malloc1D(levels_decomposed_data_device, - hierarchy.l_target() + 1, queue_idx); } ~DirectInterleaver() { if (initialized) { @@ -98,12 +99,13 @@ class DirectInterleaver return size; } - void interleave(SubArray decomposed_data, - SubArray<1, T, DeviceType> *levels_decomposed_data, - SIZE target_level, int queue_idx) { + void + interleave(SubArray decomposed_data, + std::vector> levels_decomposed_data, + SIZE target_level, int queue_idx) { MemoryManager::Copy1D(levels_decomposed_data_device, - levels_decomposed_data, target_level + 1, - queue_idx); + levels_decomposed_data.data(), + target_level + 1, queue_idx); DeviceLauncher::Execute( DirectInterleaverKernel( SubArray(hierarchy->level_ranges()), @@ -111,12 +113,13 @@ class DirectInterleaver levels_decomposed_data_device), queue_idx); } - void reposition(SubArray<1, T, DeviceType> *levels_decomposed_data, - SubArray decomposed_data, SIZE target_level, - int queue_idx) { + void + reposition(std::vector> levels_decomposed_data, + SubArray decomposed_data, SIZE target_level, + int queue_idx) { MemoryManager::Copy1D(levels_decomposed_data_device, - levels_decomposed_data, target_level + 1, - queue_idx); + levels_decomposed_data.data(), + target_level + 1, queue_idx); DeviceLauncher::Execute( DirectInterleaverKernel( SubArray(hierarchy->level_ranges()), @@ -130,6 +133,7 @@ class DirectInterleaver bool initialized; Hierarchy *hierarchy; SubArray<1, T, DeviceType> *levels_decomposed_data_device = nullptr; + SIZE levels_decomposed_data_device_length = 0; }; } // namespace MDR diff --git a/include/mgard-x/MDR-X/Interleaver/InterleaverInterface.hpp b/include/mgard-x/MDR-X/Interleaver/InterleaverInterface.hpp index 66269c51fd..dbfabc47d0 100644 --- a/include/mgard-x/MDR-X/Interleaver/InterleaverInterface.hpp +++ b/include/mgard-x/MDR-X/Interleaver/InterleaverInterface.hpp @@ -34,13 +34,15 @@ template class InterleaverInterface { public: virtual ~InterleaverInterface() = default; - virtual void interleave(SubArray decomposed_data, - SubArray<1, T, DeviceType> *levels_decomposed_data, - SIZE num_levels, int queue_idx) = 0; - - virtual void reposition(SubArray<1, T, DeviceType> *levels_decomposed_data, - SubArray decomposed_data, - SIZE num_levels, int queue_idx) = 0; + virtual void + interleave(SubArray decomposed_data, + std::vector> levels_decomposed_data, + SIZE num_levels, int queue_idx) = 0; + + virtual void + reposition(std::vector> levels_decomposed_data, + SubArray decomposed_data, SIZE num_levels, + int queue_idx) = 0; virtual void print() const = 0; }; diff --git a/include/mgard-x/Linearization/LevelLinearizer.hpp b/include/mgard-x/MDR-X/Interleaver/LevelLinearizer.hpp similarity index 100% rename from include/mgard-x/Linearization/LevelLinearizer.hpp rename to include/mgard-x/MDR-X/Interleaver/LevelLinearizer.hpp diff --git a/include/mgard-x/MDR-X/LosslessCompressor/DefaultLevelCompressor.hpp b/include/mgard-x/MDR-X/LosslessCompressor/DefaultLevelCompressor.hpp index 6aa9cf06a0..2877f88076 100644 --- a/include/mgard-x/MDR-X/LosslessCompressor/DefaultLevelCompressor.hpp +++ b/include/mgard-x/MDR-X/LosslessCompressor/DefaultLevelCompressor.hpp @@ -2,6 +2,7 @@ #define _MDR_DEFAULT_LEVEL_COMPRESSOR_HPP #include "../../Lossless/ParallelHuffman/Huffman.hpp" +#include "../../Lossless/ParallelRLE/RunLengthEncoding.hpp" #include "../../Lossless/Zstd.hpp" // #include "../RefactorUtils.hpp" #include "LevelCompressorInterface.hpp" @@ -10,131 +11,166 @@ namespace mgard_x { namespace MDR { +struct HUFFMAN {}; +struct RLE {}; + // interface for lossless compressor -template +template class DefaultLevelCompressor - : public concepts::LevelCompressorInterface { + : public concepts::LevelCompressorInterface { public: + using T_compress = u_int8_t; + // using T_compress = u_int16_t; + + static constexpr int byte_ratio = sizeof(T_bitplane) / sizeof(T_compress); + static constexpr int _huff_dict_size = 256; + static constexpr int _huff_block_size = 1024; + static constexpr int num_merged_bitplanes = 4; + DefaultLevelCompressor() : initialized(false) {} DefaultLevelCompressor(SIZE max_n, Config config) { - Adapt(max_n, config, 0); + this->initialized = true; + Adapt(max_n * byte_ratio, config, 0); DeviceRuntime::SyncQueue(0); } ~DefaultLevelCompressor(){}; - void Adapt(SIZE max_n, Config config, int queue_idx) { + void Adapt(SIZE max_n, SIZE max_level, SIZE max_bitplanes, Config config, + int queue_idx) { this->initialized = true; this->config = config; - huffman.Resize(max_n, config.huff_dict_size, config.huff_block_size, - config.estimate_outlier_ratio, queue_idx); - zstd.Resize(max_n * sizeof(T), config.zstd_compress_level, queue_idx); + if constexpr (std::is_same::value) { + huffman.Resize(max_n * byte_ratio * num_merged_bitplanes, _huff_dict_size, + _huff_block_size, config.estimate_outlier_ratio, + queue_idx); + } + if constexpr (std::is_same::value) { + rle.Resize(max_n * byte_ratio * num_merged_bitplanes, queue_idx); + } } static size_t EstimateMemoryFootprint(SIZE max_n, Config config) { size_t size = 0; - size += Huffman::EstimateMemoryFootprint( - max_n, config.huff_dict_size, config.huff_block_size, - config.estimate_outlier_ratio); - size += Zstd::EstimateMemoryFootprint(max_n * sizeof(T)); + if constexpr (std::is_same::value) { + size += Huffman:: + EstimateMemoryFootprint(max_n * byte_ratio * num_merged_bitplanes, + _huff_dict_size, _huff_block_size, + config.estimate_outlier_ratio); + } + if constexpr (std::is_same::value) { + size += parallel_rle::RunLengthEncoding< + T_compress, u_int32_t, u_int32_t, + DeviceType>::EstimateMemoryFootprint(max_n * byte_ratio * + num_merged_bitplanes); + } return size; } // compress level, overwrite and free original streams; rewrite streams sizes void - compress_level(std::vector &bitplane_sizes, - Array<2, T, DeviceType> &encoded_bitplanes, + compress_level(SubArray<2, T_bitplane, DeviceType> &encoded_bitplanes, std::vector> &compressed_bitplanes, - int queue_idx) { - - SubArray<2, T, DeviceType> encoded_bitplanes_subarray(encoded_bitplanes); - for (SIZE bitplane_idx = 0; - bitplane_idx < encoded_bitplanes_subarray.shape(0); bitplane_idx++) { - T *bitplane = encoded_bitplanes_subarray(bitplane_idx, 0); - // MDR::Zstd - // T *bitplane_host = new T[bitplane_sizes[bitplane_idx]]; - - // MemoryManager::Copy1D( - // bitplane_host, bitplane, bitplane_sizes[bitplane_idx] / sizeof(T), - // 0); - // DeviceRuntime::SyncQueue(0); - - // Byte *compressed_host = NULL; - // SIZE compressed_bitplane_size = - // ::MDR::ZSTD::compress((uint8_t *)bitplane_host, - // bitplane_sizes[bitplane_idx], - // &compressed_host); - // Array<1, Byte, DeviceType> compressed_bitplane( - // {compressed_bitplane_size}); - // compressed_bitplane.load(compressed_host); - // compressed_bitplanes[bitplane_idx] = compressed_bitplane; - // bitplane_sizes[bitplane_idx] = compressed_bitplane_size; - - // Huffman - // Array<1, T, DeviceType> - // encoded_bitplane({encoded_bitplanes_subarray.shape(1)}, bitplane); - // huffman.Compress(encoded_bitplane, compressed_bitplanes[bitplane_idx], - // queue_idx); bitplane_sizes[bitplane_idx] = - // compressed_bitplanes[bitplane_idx].shape(0); - - Array<1, Byte, DeviceType> compressed_bitplane( - {bitplane_sizes[bitplane_idx]}); - MemoryManager::Copy1D( - compressed_bitplane.data(), (uint8_t *)bitplane, - bitplane_sizes[bitplane_idx], queue_idx); - DeviceRuntime::SyncQueue(queue_idx); - int old_log_level = log::level; - log::level = log::ERR; - zstd.Compress(compressed_bitplane, queue_idx); - DeviceRuntime::SyncQueue(queue_idx); - log::level = old_log_level; - compressed_bitplanes[bitplane_idx] = compressed_bitplane; - bitplane_sizes[bitplane_idx] = compressed_bitplane.shape(0); + int level_idx, int queue_idx) { + + std::vector cr, time; + for (SIZE bitplane_idx = 0; bitplane_idx < encoded_bitplanes.shape(0); + bitplane_idx++) { + if (bitplane_idx % num_merged_bitplanes == 0) { + SIZE merged_bitplane_size = + encoded_bitplanes.shape(1) * byte_ratio * num_merged_bitplanes; + Timer timer; + timer.start(); + T_compress *bitplane = (T_compress *)encoded_bitplanes(bitplane_idx, 0); + + Array<1, T_compress, DeviceType> encoded_bitplane( + {merged_bitplane_size}, bitplane); + int old_log_level = log::level; + // log::level = 0; + if constexpr (std::is_same::value) { + ATOMIC_IDX zero = 0; + MemoryManager::Copy1D( + huffman.workspace.outlier_count_subarray.data(), &zero, 1, + queue_idx); + MemoryManager::Copy1D( + &huffman.outlier_count, + huffman.workspace.outlier_count_subarray.data(), 1, queue_idx); + huffman.CompressPrimary(encoded_bitplane, + compressed_bitplanes[bitplane_idx], 0.0, + queue_idx); + huffman.Serialize(compressed_bitplanes[bitplane_idx], queue_idx); + } + if constexpr (std::is_same::value) { + rle.Compress(encoded_bitplane, compressed_bitplanes[bitplane_idx], + 0.0, queue_idx); + rle.Serialize(compressed_bitplanes[bitplane_idx], queue_idx); + } + log::level = old_log_level; + cr.push_back((float)merged_bitplane_size / + compressed_bitplanes[bitplane_idx].shape(0)); + + timer.end(); + time.push_back(timer.get()); + timer.clear(); + // timer.print("Compressing bitplane", merged_bitplane_size); + // timer.clear(); + } else { + compressed_bitplanes[bitplane_idx].resize({1}, queue_idx); + } + } + std::string cr_string = ""; + for (auto x : cr) { + cr_string += std::to_string(x) + ", "; + } + log::info("CR: " + cr_string); + + std::string time_string = ""; + for (auto x : time) { + time_string += std::to_string(x) + " "; } + log::info("Time: " + time_string); } // decompress level, create new buffer and overwrite original streams; will // not change stream sizes void decompress_level( - std::vector &bitplane_sizes, std::vector> &compressed_bitplanes, - Array<2, T, DeviceType> &encoded_bitplanes, uint8_t starting_bitplane, - uint8_t num_bitplanes, int queue_idx) { - - SubArray<2, T, DeviceType> encoded_bitplanes_subarray(encoded_bitplanes); - - for (SIZE bitplane_idx = starting_bitplane; bitplane_idx < num_bitplanes; - bitplane_idx++) { - T *bitplane = encoded_bitplanes_subarray(bitplane_idx, 0); - // MDR::Zstd - // SIZE compressed_size = bitplane_sizes[starting_bitplane + - // bitplane_idx]; Byte *compressed_host = new Byte[compressed_size]; - // MemoryManager::Copy1D( - // compressed_host, - // compressed_bitplanes[starting_bitplane + bitplane_idx].data(), - // compressed_size, 0); - // DeviceRuntime::SyncQueue(0); - - // Byte *bitplane_host = NULL; - // SIZE decompressed_size = ::MDR::ZSTD::decompress( - // compressed_host, compressed_size, &bitplane_host); - - // MemoryManager::Copy1D(bitplane, (T *)bitplane_host, - // decompressed_size / sizeof(T), 0); - // DeviceRuntime::SyncQueue(0); - - // Huffman - // Array<1, T, DeviceType> - // encoded_bitplane({encoded_bitplanes_subarray.shape(1)}, bitplane); - // huffman.Decompress(compressed_bitplanes[bitplane_idx], - // encoded_bitplane, queue_idx); - // std::cout << "decompress level: " << bitplane_idx << "\n"; - int old_log_level = log::level; - log::level = log::ERR; - zstd.Decompress(compressed_bitplanes[bitplane_idx], queue_idx); - log::level = old_log_level; - MemoryManager::Copy1D( - (uint8_t *)bitplane, compressed_bitplanes[bitplane_idx].data(), - compressed_bitplanes[bitplane_idx].shape(0), queue_idx); - DeviceRuntime::SyncQueue(queue_idx); + SubArray<2, T_bitplane, DeviceType> &encoded_bitplanes, + uint8_t starting_bitplane, uint8_t num_bitplanes, int level_idx, + int queue_idx) { + + std::vector time; + for (SIZE bitplane_idx = starting_bitplane; + bitplane_idx < starting_bitplane + num_bitplanes; bitplane_idx++) { + if (bitplane_idx % num_merged_bitplanes == 0) { + Timer timer; + timer.start(); + T_compress *bitplane = (T_compress *)encoded_bitplanes(bitplane_idx, 0); + SIZE merged_bitplane_size = + encoded_bitplanes.shape(1) * byte_ratio * num_merged_bitplanes; + + Array<1, T_compress, DeviceType> encoded_bitplane( + {merged_bitplane_size}, bitplane); + int old_log_level = log::level; + // log::level = 0; + if constexpr (std::is_same::value) { + huffman.Deserialize(compressed_bitplanes[bitplane_idx], queue_idx); + huffman.DecompressPrimary(compressed_bitplanes[bitplane_idx], + encoded_bitplane, queue_idx); + } + if constexpr (std::is_same::value) { + rle.Deserialize(compressed_bitplanes[bitplane_idx], queue_idx); + rle.Decompress(compressed_bitplanes[bitplane_idx], encoded_bitplane, + queue_idx); + } + log::level = old_log_level; + timer.end(); + time.push_back(timer.get()); + timer.clear(); + } + } + std::string time_string = ""; + for (auto x : time) { + time_string += std::to_string(x) + " "; } + log::info("Time: " + time_string); } // release the buffer created @@ -142,8 +178,9 @@ class DefaultLevelCompressor void print() const {} bool initialized; - Huffman huffman; - Zstd zstd; + Huffman huffman; + parallel_rle::RunLengthEncoding + rle; Config config; }; diff --git a/include/mgard-x/MDR-X/LosslessCompressor/HybridLevelCompressor.hpp b/include/mgard-x/MDR-X/LosslessCompressor/HybridLevelCompressor.hpp new file mode 100644 index 0000000000..7e862d548c --- /dev/null +++ b/include/mgard-x/MDR-X/LosslessCompressor/HybridLevelCompressor.hpp @@ -0,0 +1,208 @@ +#ifndef _MDR_HYBRID_LEVEL_COMPRESSOR_HPP +#define _MDR_HYBRID_LEVEL_COMPRESSOR_HPP + +#include "../../Lossless/ParallelHuffman/Huffman.hpp" +#include "../../Lossless/ParallelRLE/RunLengthEncoding.hpp" +#include "../../Lossless/Zstd.hpp" +// #include "../RefactorUtils.hpp" +#include "LevelCompressorInterface.hpp" +#include "LosslessCompressor.hpp" + +namespace mgard_x { +namespace MDR { + +// interface for lossless compressor +template +class HybridLevelCompressor + : public concepts::LevelCompressorInterface { +public: + using T_compress = u_int8_t; + // using T_compress = u_int16_t; + + static constexpr int byte_ratio = sizeof(T_bitplane) / sizeof(T_compress); + static constexpr int _huff_dict_size = 256; + static constexpr int _huff_block_size = 1024; + static constexpr int num_merged_bitplanes = 4; + + SIZE size_threshold = 1e6; + float cr_threshold = 2.0; + + HybridLevelCompressor() : initialized(false) {} + HybridLevelCompressor(SIZE max_n, Config config) { + this->initialized = true; + Adapt(max_n * byte_ratio, config, 0); + DeviceRuntime::SyncQueue(0); + } + ~HybridLevelCompressor(){}; + + void Adapt(SIZE max_n, SIZE max_bitplanes, Config config, int queue_idx) { + this->initialized = true; + this->config = config; + huffman.Resize(max_n * byte_ratio * num_merged_bitplanes, _huff_dict_size, + _huff_block_size, config.estimate_outlier_ratio, queue_idx); + rle.Resize(max_n * byte_ratio * num_merged_bitplanes, queue_idx); + zstd.Resize(max_n * sizeof(T_bitplane), config.zstd_compress_level, + queue_idx); + } + static size_t EstimateMemoryFootprint(SIZE max_n, Config config) { + size_t size = 0; + size += Huffman:: + EstimateMemoryFootprint(max_n * byte_ratio * num_merged_bitplanes, + _huff_dict_size, _huff_block_size, + config.estimate_outlier_ratio); + size += parallel_rle::RunLengthEncoding< + T_compress, u_int32_t, u_int32_t, + DeviceType>::EstimateMemoryFootprint(max_n * byte_ratio * + num_merged_bitplanes); + size += + Zstd::EstimateMemoryFootprint(max_n * sizeof(T_bitplane)); + return size; + } + + void + compress_level(SubArray<2, T_bitplane, DeviceType> &encoded_bitplanes, + std::vector> &compressed_bitplanes, + int level_idx, int queue_idx) { + + std::vector cr, time; + bool huffman_success, rle_success; + for (SIZE bitplane_idx = 0; bitplane_idx < encoded_bitplanes.shape(0); + bitplane_idx++) { + if (bitplane_idx % num_merged_bitplanes == 0) { + SIZE merged_bitplane_size = + encoded_bitplanes.shape(1) * byte_ratio * num_merged_bitplanes; + Timer timer; + timer.start(); + T_compress *bitplane = (T_compress *)encoded_bitplanes(bitplane_idx, 0); + + Array<1, T_compress, DeviceType> encoded_bitplane( + {merged_bitplane_size}, bitplane); + int old_log_level = log::level; + log::level = 0; + huffman_success = false; + rle_success = false; + // cr_threshold = 2.0; + if (merged_bitplane_size > size_threshold) { + rle_success = + rle.Compress(encoded_bitplane, compressed_bitplanes[bitplane_idx], + cr_threshold, queue_idx); + if (rle_success) { + rle.Serialize(compressed_bitplanes[bitplane_idx], queue_idx); + } else { + ATOMIC_IDX zero = 0; + MemoryManager::Copy1D( + huffman.workspace.outlier_count_subarray.data(), &zero, 1, + queue_idx); + MemoryManager::Copy1D( + &huffman.outlier_count, + huffman.workspace.outlier_count_subarray.data(), 1, queue_idx); + huffman_success = huffman.CompressPrimary( + encoded_bitplane, compressed_bitplanes[bitplane_idx], + cr_threshold, queue_idx); + if (huffman_success) { + huffman.Serialize(compressed_bitplanes[bitplane_idx], queue_idx); + } + } + } + + if (huffman_success == false && rle_success == false) { + // direct copy + compressed_bitplanes[bitplane_idx].resize({merged_bitplane_size}); + MemoryManager::Copy1D( + compressed_bitplanes[bitplane_idx].data(), (Byte *)bitplane, + merged_bitplane_size, queue_idx); + } + + log::level = old_log_level; + cr.push_back((float)merged_bitplane_size / + compressed_bitplanes[bitplane_idx].shape(0)); + + timer.end(); + time.push_back(timer.get()); + timer.clear(); + // timer.print("Compressing bitplane", merged_bitplane_size); + // timer.clear(); + } else { + compressed_bitplanes[bitplane_idx].resize({0}, queue_idx); + } + } + // std::string cr_string = ""; + // for (auto x : cr) { + // cr_string += std::to_string(x) + ", "; + // } + // log::info("CR: " + cr_string); + + // std::string time_string = ""; + // for (auto x : time) { + // time_string += std::to_string(x) + " "; + // } + // log::info("Time: " + time_string); + } + + // decompress level, create new buffer and overwrite original streams; will + // not change stream sizes + void decompress_level( + std::vector> &compressed_bitplanes, + SubArray<2, T_bitplane, DeviceType> &encoded_bitplanes, + uint8_t starting_bitplane, uint8_t num_bitplanes, int level_idx, + int queue_idx) { + + std::vector time; + for (SIZE bitplane_idx = starting_bitplane; + bitplane_idx < starting_bitplane + num_bitplanes; bitplane_idx++) { + if (bitplane_idx % num_merged_bitplanes == 0) { + Timer timer; + timer.start(); + T_compress *bitplane = (T_compress *)encoded_bitplanes(bitplane_idx, 0); + SIZE merged_bitplane_size = + encoded_bitplanes.shape(1) * byte_ratio * num_merged_bitplanes; + + Array<1, T_compress, DeviceType> encoded_bitplane( + {merged_bitplane_size}, bitplane); + int old_log_level = log::level; + log::level = 0; + + // Huffman + if (huffman.Verify(compressed_bitplanes[bitplane_idx], queue_idx)) { + huffman.Deserialize(compressed_bitplanes[bitplane_idx], queue_idx); + huffman.DecompressPrimary(compressed_bitplanes[bitplane_idx], + encoded_bitplane, queue_idx); + // RLE + } else if (rle.Verify(compressed_bitplanes[bitplane_idx], queue_idx)) { + rle.Deserialize(compressed_bitplanes[bitplane_idx], queue_idx); + rle.Decompress(compressed_bitplanes[bitplane_idx], encoded_bitplane, + queue_idx); + } else { + // Direct copy + MemoryManager::Copy1D( + (uint8_t *)bitplane, compressed_bitplanes[bitplane_idx].data(), + merged_bitplane_size, queue_idx); + } + log::level = old_log_level; + timer.end(); + time.push_back(timer.get()); + timer.clear(); + } + } + // std::string time_string = ""; + // for (auto x : time) { + // time_string += std::to_string(x) + " "; + // } + // log::info("Time: " + time_string); + } + + // release the buffer created + void decompress_release() {} + + void print() const {} + bool initialized; + Huffman huffman; + parallel_rle::RunLengthEncoding + rle; + Zstd zstd; + Config config; +}; + +} // namespace MDR +} // namespace mgard_x +#endif diff --git a/include/mgard-x/MDR-X/LosslessCompressor/LevelCompressor.hpp b/include/mgard-x/MDR-X/LosslessCompressor/LevelCompressor.hpp index 258ebd7e31..ff11248372 100644 --- a/include/mgard-x/MDR-X/LosslessCompressor/LevelCompressor.hpp +++ b/include/mgard-x/MDR-X/LosslessCompressor/LevelCompressor.hpp @@ -3,6 +3,7 @@ // #include "AdaptiveLevelCompressor.hpp" #include "DefaultLevelCompressor.hpp" -#include "NullLevelCompressor.hpp" +#include "HybridLevelCompressor.hpp" +// #include "NullLevelCompressor.hpp" #endif diff --git a/include/mgard-x/MDR-X/LosslessCompressor/LevelCompressorInterface.hpp b/include/mgard-x/MDR-X/LosslessCompressor/LevelCompressorInterface.hpp index d8ee2744e6..29db1c5191 100644 --- a/include/mgard-x/MDR-X/LosslessCompressor/LevelCompressorInterface.hpp +++ b/include/mgard-x/MDR-X/LosslessCompressor/LevelCompressorInterface.hpp @@ -39,18 +39,16 @@ template class LevelCompressorInterface { // compress level, overwrite and free original streams; rewrite streams sizes virtual void - compress_level(std::vector &bitplane_sizes, - Array<2, T, DeviceType> &encoded_bitplanes, + compress_level(SubArray<2, T, DeviceType> &encoded_bitplanes, std::vector> &compressed_bitplanes, - int queue_idx) = 0; + int level_idx, int queue_idx) = 0; // decompress level, create new buffer and overwrite original streams; will // not change stream sizes virtual void decompress_level( - std::vector &bitplane_sizes, std::vector> &compressed_bitplanes, - Array<2, T, DeviceType> &encoded_bitplanes, uint8_t starting_bitplane, - uint8_t num_bitplanes, int queue_idx) = 0; + SubArray<2, T, DeviceType> &encoded_bitplanes, uint8_t starting_bitplane, + uint8_t num_bitplanes, int level_idx, int queue_idx) = 0; // release the buffer created virtual void decompress_release() = 0; diff --git a/include/mgard-x/MDR-X/LosslessCompressor/NullLevelCompressor.hpp b/include/mgard-x/MDR-X/LosslessCompressor/NullLevelCompressor.hpp index 9dafab8d1e..0f290cf9f6 100644 --- a/include/mgard-x/MDR-X/LosslessCompressor/NullLevelCompressor.hpp +++ b/include/mgard-x/MDR-X/LosslessCompressor/NullLevelCompressor.hpp @@ -24,9 +24,9 @@ namespace mgard_x { namespace MDR { // interface for lossless compressor -template +template class NullLevelCompressor - : public concepts::LevelCompressorInterface { + : public concepts::LevelCompressorInterface { public: NullLevelCompressor() : initialized(false) {} NullLevelCompressor(SIZE max_n, Config config) { @@ -42,75 +42,40 @@ class NullLevelCompressor static size_t EstimateMemoryFootprint(SIZE max_n, Config config) { size_t size = 0; - size += Huffman::EstimateMemoryFootprint( - max_n, config.huff_dict_size, config.huff_block_size, - config.estimate_outlier_ratio); + size += Huffman:: + EstimateMemoryFootprint(max_n, config.huff_dict_size, + config.huff_block_size, + config.estimate_outlier_ratio); return size; } // compress level, overwrite and free original streams; rewrite streams sizes void - compress_level(std::vector &bitplane_sizes, - Array<2, T, DeviceType> &encoded_bitplanes, + compress_level(SubArray<2, T_bitplane, DeviceType> &encoded_bitplanes, std::vector> &compressed_bitplanes, int queue_idx) { - SubArray<2, T, DeviceType> encoded_bitplanes_subarray(encoded_bitplanes); - for (SIZE bitplane_idx = 0; - bitplane_idx < encoded_bitplanes_subarray.shape(0); bitplane_idx++) { - T *bitplane = encoded_bitplanes_subarray(bitplane_idx, 0); + for (SIZE bitplane_idx = 0; bitplane_idx < encoded_bitplanes.shape(0); + bitplane_idx++) { + T_bitplane *bitplane = encoded_bitplanes(bitplane_idx, 0); - Array<1, Byte, DeviceType> compressed_bitplane( - {bitplane_sizes[bitplane_idx]}); + compressed_bitplanes[bitplane_idx].resize( + {encoded_bitplanes.shape(1) * sizeof(T_bitplane)}); MemoryManager::Copy1D( - compressed_bitplane.data(), (Byte *)bitplane, - bitplane_sizes[bitplane_idx], queue_idx); - DeviceRuntime::SyncQueue(queue_idx); - compressed_bitplanes[bitplane_idx] = compressed_bitplane; - bitplane_sizes[bitplane_idx] = bitplane_sizes[bitplane_idx]; + compressed_bitplanes[bitplane_idx].data(), (Byte *)bitplane, + encoded_bitplanes.shape(1) * sizeof(T_bitplane), queue_idx); } } // decompress level, create new buffer and overwrite original streams; will // not change stream sizes void decompress_level( - std::vector &bitplane_sizes, std::vector> &compressed_bitplanes, - Array<2, T, DeviceType> &encoded_bitplanes, uint8_t starting_bitplane, - uint8_t num_bitplanes, int queue_idx) { - - SubArray<2, T, DeviceType> encoded_bitplanes_subarray(encoded_bitplanes); + SubArray<2, T_bitplane, DeviceType> &encoded_bitplanes, + uint8_t starting_bitplane, uint8_t num_bitplanes, int queue_idx) { for (SIZE bitplane_idx = starting_bitplane; bitplane_idx < starting_bitplane + num_bitplanes; bitplane_idx++) { - // std::cout << "decompress level: " << bitplane_idx << " " << - // (int)num_bitplanes << "\n"; - T *bitplane = encoded_bitplanes_subarray(bitplane_idx, 0); - // MDR::Zstd - // SIZE compressed_size = bitplane_sizes[starting_bitplane + - // bitplane_idx]; Byte *compressed_host = new Byte[compressed_size]; - // MemoryManager::Copy1D( - // compressed_host, - // compressed_bitplanes[starting_bitplane + bitplane_idx].data(), - // compressed_size, 0); - // DeviceRuntime::SyncQueue(0); - - // Byte *bitplane_host = NULL; - // SIZE decompressed_size = ::MDR::ZSTD::decompress( - // compressed_host, compressed_size, &bitplane_host); - - // MemoryManager::Copy1D(bitplane, (T *)bitplane_host, - // decompressed_size / sizeof(T), 0); - // DeviceRuntime::SyncQueue(0); - - // Huffman - // Array<1, T, DeviceType> - // encoded_bitplane({encoded_bitplanes_subarray.shape(1)}, bitplane); - // huffman.Decompress(compressed_bitplanes[bitplane_idx], - // encoded_bitplane, queue_idx); - // int old_log_level = log::level; - // log::level = log::ERR; - // ZstdDecompress(compressed_bitplanes[bitplane_idx]); - // log::level = old_log_level; + T_bitplane *bitplane = encoded_bitplanes(bitplane_idx, 0); MemoryManager::Copy1D( (uint8_t *)bitplane, compressed_bitplanes[bitplane_idx].data(), compressed_bitplanes[bitplane_idx].shape(0), queue_idx); diff --git a/include/mgard-x/MDR-X/Reconstructor/ComposedReconstructor.hpp b/include/mgard-x/MDR-X/Reconstructor/ComposedReconstructor.hpp index db35d4650d..3d579ea5c8 100644 --- a/include/mgard-x/MDR-X/Reconstructor/ComposedReconstructor.hpp +++ b/include/mgard-x/MDR-X/Reconstructor/ComposedReconstructor.hpp @@ -14,24 +14,52 @@ #include "../Retriever/Retriever.hpp" #include "../SizeInterpreter/SizeInterpreter.hpp" #include "ReconstructorInterface.hpp" +#include // #include "../DataStructures/MDRData.hpp" namespace mgard_x { namespace MDR { // a decomposition-based scientific data reconstructor: inverse operator of // composed refactor -template +template class ComposedReconstructor : public concepts::ReconstructorInterface { public: using HierarchyType = Hierarchy; using T_bitplane = uint32_t; using T_error = double; - using Decomposer = MGARDOrthoganalDecomposer; + using Decomposer = MGARDDecomposer; using Interleaver = DirectInterleaver; - using Encoder = GroupedBPEncoder; - // using Compressor = DefaultLevelCompressor; - using Compressor = NullLevelCompressor; + + constexpr static bool ProfileBPEncoder = false; + // using Encoder = GroupedBPEncoder; + // using Encoder = BPEncoderLocalityBlock; + using Encoder = BPEncoderRegisterBlock; + // using Encoder = BPEncoderRegisterShift; + // using Encoder = BPEncoderRegisterBallot; + // using Encoder = BPEncoderRegisterReduceAll; + // using Encoder = BPEncoderRegisterMatchAny; + + // using Compressor = DefaultLevelCompressor; + // using Compressor = DefaultLevelCompressor; + using Compressor = HybridLevelCompressor; + // using Compressor = NullLevelCompressor; + + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr SIZE MAX_BITPLANES = sizeof(T_data) * 8; ComposedReconstructor() : initialized(false) {} ComposedReconstructor(Hierarchy &hierarchy, @@ -40,10 +68,7 @@ class ComposedReconstructor DeviceRuntime::SyncQueue(0); } - ~ComposedReconstructor() { - delete[] levels_array; - delete[] levels_data; - } + ~ComposedReconstructor() {} void Adapt(Hierarchy &hierarchy, Config config, int queue_idx) { @@ -52,10 +77,10 @@ class ComposedReconstructor decomposer.Adapt(hierarchy, config, queue_idx); interleaver.Adapt(hierarchy, queue_idx); encoder.Adapt(hierarchy, queue_idx); - compressor.Adapt( - Encoder::buffer_size(hierarchy.level_num_elems(hierarchy.l_target())), - config, queue_idx); - total_num_bitplanes = config.total_num_bitplanes; + // batched_encoder.Adapt(hierarchy, queue_idx); + compressor.Adapt(Encoder::bitplane_length( + hierarchy.level_num_elems(hierarchy.l_target())), + Encoder::MAX_BITPLANES, config, queue_idx); prev_reconstructed = false; partial_reconsctructed_data.resize( @@ -63,22 +88,33 @@ class ComposedReconstructor interpolation_workspace.resize(hierarchy.level_shape(hierarchy.l_target()), queue_idx); - delete[] levels_array; - delete[] levels_data; - levels_array = new Array<1, T_data, DeviceType>[hierarchy.l_target() + 1]; - levels_data = new SubArray<1, T_data, DeviceType>[hierarchy.l_target() + 1]; + level_data_array.resize(hierarchy.l_target() + 1); + level_data_subarray.resize(hierarchy.l_target() + 1); + level_num_elems.resize(hierarchy.l_target() + 1); + exp.resize(hierarchy.l_target() + 1); for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { - levels_array[level_idx].resize({hierarchy.level_num_elems(level_idx)}, - queue_idx); - levels_data[level_idx] = - SubArray<1, T_data, DeviceType>(levels_array[level_idx]); + level_data_array[level_idx].resize( + {round_up(hierarchy.level_num_elems(level_idx), BATCH_SIZE)}, + queue_idx); + level_data_subarray[level_idx] = + SubArray<1, T_data, DeviceType>(level_data_array[level_idx]); + level_num_elems[level_idx] = hierarchy.level_num_elems(level_idx); } encoded_bitplanes_array.resize(hierarchy.l_target() + 1); + encoded_bitplanes_subarray.resize(hierarchy.l_target() + 1); + level_num_bitplanes.resize(hierarchy.l_target() + 1); + level_signs_subarray.resize(hierarchy.l_target() + 1); + abs_max_array.resize(hierarchy.l_target() + 1); for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { encoded_bitplanes_array[level_idx].resize( - {(SIZE)total_num_bitplanes, - encoder.buffer_size(hierarchy.level_num_elems(level_idx))}, + {(SIZE)Encoder::MAX_BITPLANES, + encoder.bitplane_length(hierarchy.level_num_elems(level_idx))}, queue_idx); + encoded_bitplanes_subarray[level_idx] = + SubArray<2, T_bitplane, DeviceType>( + encoded_bitplanes_array[level_idx]); + abs_max_array[level_idx].resize({1}, queue_idx); + abs_max_array[level_idx].hostAllocate(false, queue_idx); } } @@ -100,19 +136,20 @@ class ComposedReconstructor } size += partial_data_size * 2; // including interpolation workspace for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { - size += hierarchy.level_num_elems(level_idx) * sizeof(T_data); + size += round_up(hierarchy.level_num_elems(level_idx), BATCH_SIZE) * + sizeof(T_data); } for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { - size += config.total_num_bitplanes * - Encoder::buffer_size(hierarchy.level_num_elems(level_idx)) * + size += Encoder::MAX_BITPLANES * + Encoder::bitplane_length(hierarchy.level_num_elems(level_idx)) * sizeof(T_bitplane); } - SIZE max_n = - Encoder::buffer_size(hierarchy.level_num_elems(hierarchy.l_target())); + SIZE max_n = Encoder::bitplane_length( + hierarchy.level_num_elems(hierarchy.l_target())); - size += (config.total_num_bitplanes + 1) * sizeof(T_error); + size += (Encoder::MAX_BITPLANES + 1) * sizeof(T_error); size += Decomposer::EstimateMemoryFootprint(shape); size += Interleaver::EstimateMemoryFootprint(shape); size += Encoder::EstimateMemoryFootprint(shape); @@ -120,6 +157,28 @@ class ComposedReconstructor return size; } + static std::vector> + EstimateMaxBitplaneSizes(Hierarchy &hierarchy) { + std::vector> estimation; + estimation.resize(hierarchy.l_target() + 1); + for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { + estimation[level_idx].resize(Encoder::MAX_BITPLANES); + for (int bitplane_idx = 0; bitplane_idx < Encoder::MAX_BITPLANES; + bitplane_idx++) { + if (bitplane_idx % Compressor::num_merged_bitplanes == 0) { + estimation[level_idx][bitplane_idx] = + Encoder::bitplane_length(hierarchy.level_num_elems(level_idx)) * + sizeof(T_bitplane) * Compressor::num_merged_bitplanes; + // For Huffman-only model (metadata storage) + estimation[level_idx][bitplane_idx] += 1e6; + } else { + estimation[level_idx][bitplane_idx] = 1; + } + } + } + return estimation; + } + void GenerateRequest(MDRMetadata &mdr_metadata) { mgard_x::Timer timer; timer.start(); @@ -141,30 +200,97 @@ class ComposedReconstructor } level_errors = level_abs_errors; - MaxErrorEstimatorOB estimator(D); + if constexpr (std::is_same::value) { + MaxErrorEstimatorOB estimator(D); + GreedyBasedSizeInterpreter interpreter(estimator); + if (mdr_metadata.segmented) { + retrieve_sizes = interpreter.interpret_retrieve_size( + mdr_metadata.level_sizes, level_errors, + mdr_metadata.requested_size, mdr_metadata.corresponding_error, + mdr_metadata.requested_level_num_bitplanes); + } else if (mdr_metadata.corresponding_error_return) { + retrieve_sizes = interpreter.interpret_retrieve_size( + mdr_metadata.level_sizes, level_errors, + mdr_metadata.requested_tol, mdr_metadata.corresponding_error, + mdr_metadata.requested_level_num_bitplanes); + } else { + retrieve_sizes = interpreter.interpret_retrieve_size( + mdr_metadata.level_sizes, level_errors, + mdr_metadata.requested_tol, + mdr_metadata.requested_level_num_bitplanes); + } + } else if constexpr (std::is_same::value) { + MaxErrorEstimatorHB estimator; + GreedyBasedSizeInterpreter interpreter(estimator); + if (mdr_metadata.segmented) { + retrieve_sizes = interpreter.interpret_retrieve_size( + mdr_metadata.level_sizes, level_errors, + mdr_metadata.requested_size, mdr_metadata.corresponding_error, + mdr_metadata.requested_level_num_bitplanes); + } else if (mdr_metadata.corresponding_error_return) { + retrieve_sizes = interpreter.interpret_retrieve_size( + mdr_metadata.level_sizes, level_errors, + mdr_metadata.requested_tol, mdr_metadata.corresponding_error, + mdr_metadata.requested_level_num_bitplanes); + } else { + retrieve_sizes = interpreter.interpret_retrieve_size( + mdr_metadata.level_sizes, level_errors, + mdr_metadata.requested_tol, + mdr_metadata.requested_level_num_bitplanes); + } + } // SignExcludeGreedyBasedSizeInterpreter interpreter(estimator); - GreedyBasedSizeInterpreter interpreter(estimator); // RoundRobinSizeInterpreter interpreter(estimator); // InorderSizeInterpreter interpreter(estimator); - retrieve_sizes = interpreter.interpret_retrieve_size( - mdr_metadata.level_sizes, level_errors, mdr_metadata.requested_tol, - mdr_metadata.requested_level_num_bitplanes); + } else { log::info("ErrorEstimator is base of SquaredErrorEstimator, using level " "squared error directly"); - SNormErrorEstimator estimator(D, hierarchy->l_target(), - mdr_metadata.requested_s); - // InorderSizeInterpreter interpreter(estimator); - GreedyBasedSizeInterpreter interpreter(estimator); + + if constexpr (std::is_same::value) { + using Estimator = SNormErrorEstimator; + Estimator estimator(D, hierarchy->l_target(), mdr_metadata.requested_s); + using BinaryInterp = GreedyBasedSizeInterpreter; + using NegaBinaryInterp = + NegaBinaryGreedyBasedSizeInterpreter; + using Interpreter = + typename std::conditional::type; + Interpreter interpreter(estimator); + retrieve_sizes = interpreter.interpret_retrieve_size( + mdr_metadata.level_sizes, level_errors, + std::pow(mdr_metadata.requested_tol, 2), + mdr_metadata.requested_level_num_bitplanes); + } else if constexpr (std::is_same::value) { + using Estimator = L2ErrorEstimator_HB; + Estimator estimator(D, hierarchy->l_target()); + using BinaryInterp = GreedyBasedSizeInterpreter; + using NegaBinaryInterp = + NegaBinaryGreedyBasedSizeInterpreter; + using Interpreter = + typename std::conditional::type; + Interpreter interpreter(estimator); + retrieve_sizes = interpreter.interpret_retrieve_size( + mdr_metadata.level_sizes, level_errors, + std::pow(mdr_metadata.requested_tol, 2), + mdr_metadata.requested_level_num_bitplanes); + } + // using BinaryInterpreter = InorderSizeInterpreter; // SignExcludeGreedyBasedSizeInterpreter interpreter(estimator); // NegaBinaryGreedyBasedSizeInterpreter interpreter(estimator); - retrieve_sizes = interpreter.interpret_retrieve_size( - mdr_metadata.level_sizes, level_errors, - std::pow(mdr_metadata.requested_tol, 2), - mdr_metadata.requested_level_num_bitplanes); + } + + for (uint8_t &n : mdr_metadata.requested_level_num_bitplanes) { + // Ensure requested bitplanes is a multiple of num_merged_bitplanes + // This ensure all each batch of merged bitplanes are used for + // Reconstruction. Otherwise, unsed bitplanes will not be guaranteed + // to be in memory in future reconstructions. + int m = Compressor::num_merged_bitplanes; + n = ((n - 1) / m + 1) * m; } timer.end(); - timer.print("Preprocessing"); + // timer.print("Preprocessing"); } void InterpolateToLevel(Array &reconstructed_data, @@ -188,6 +314,57 @@ class ComposedReconstructor timer.print("Interpolation"); } + void LoadMetadata(MDRMetadata &mdr_metadata, MDRData &mdr_data, + int queue_idx) { + for (int level_idx = 0; level_idx <= mdr_metadata.CurrFinalLevel(); + level_idx++) { + level_num_bitplanes[level_idx] = + mdr_metadata.loaded_level_num_bitplanes[level_idx] - + mdr_metadata.prev_used_level_num_bitplanes[level_idx]; + level_signs_subarray[level_idx] = + SubArray<1, bool, DeviceType>(mdr_data.level_signs[level_idx]); + + T_data abs_max = (T_data)mdr_metadata.level_error_bounds[level_idx]; + MemoryManager::Copy1D(abs_max_array[level_idx].data(), + &abs_max, 1, queue_idx); + } + } + + void Decompress(MDRMetadata &mdr_metadata, MDRData &mdr_data, + int queue_idx) { + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + SIZE decompressed_size = 0; + for (int level_idx = 0; level_idx <= mdr_metadata.CurrFinalLevel(); + level_idx++) { + // Number of bitplanes need to be retrieved in addition to previously + // already retrieved bitplanes + SIZE num_bitplanes = + mdr_metadata.loaded_level_num_bitplanes[level_idx] - + mdr_metadata.prev_used_level_num_bitplanes[level_idx]; + // Decompress bitplanes: compressed_bitplanes[level_idx] --> + // encoded_bitplanes + compressor.decompress_level( + mdr_data.compressed_bitplanes[level_idx], + encoded_bitplanes_subarray[level_idx], + mdr_metadata.prev_used_level_num_bitplanes[level_idx], + level_num_bitplanes[level_idx], level_idx, queue_idx); + decompressed_size += encoded_bitplanes_subarray[level_idx].shape(1) * + num_bitplanes * sizeof(T_bitplane); + } + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + // timer.print("Lossless", hierarchy->total_num_elems() * sizeof(T_data)); + timer.print("Lossless", decompressed_size); + timer.clear(); + timer.start(); + } + } void ProgressiveReconstruct(MDRMetadata &mdr_metadata, MDRData &mdr_data, bool adaptive_resolution, @@ -196,85 +373,122 @@ class ComposedReconstructor mdr_data.VerifyLoadedBitplans(mdr_metadata); - Timer timer; + Timer timer, timer_all; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer_all.start(); + } // Decompress and decode bitplanes of each level int prev_final_level = mdr_metadata.PrevFinalLevel(); int curr_final_level = mdr_metadata.CurrFinalLevel(); - log::info("Prev Final level: " + std::to_string(prev_final_level)); - log::info("Curr Final level: " + std::to_string(curr_final_level)); + // log::info("Prev Final level: " + std::to_string(prev_final_level)); + // log::info("Curr Final level: " + std::to_string(curr_final_level)); if (!adaptive_resolution) { curr_final_level = hierarchy->l_target(); } - for (int level_idx = 0; level_idx <= curr_final_level; level_idx++) { - timer.start(); - // Number of bitplanes need to be retrieved in addition to previously - // already retrieved bitplanes - SIZE num_bitplanes = - mdr_metadata.loaded_level_num_bitplanes[level_idx] - - mdr_metadata.prev_used_level_num_bitplanes[level_idx]; - // Decompress bitplanes: compressed_bitplanes[level_idx] --> - // encoded_bitplanes - compressor.decompress_level( - mdr_metadata.level_sizes[level_idx], - mdr_data.compressed_bitplanes[level_idx], - encoded_bitplanes_array[level_idx], - mdr_metadata.prev_used_level_num_bitplanes[level_idx], num_bitplanes, - queue_idx); - timer.end(); - timer.print("Lossless"); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); timer.start(); + } + + // for (int i = 1; i <= 32; i++) { + // std::cout << "["; - int level_exp = 0; - frexp(mdr_metadata.level_error_bounds[level_idx], &level_exp); + for (int level_idx = 0; level_idx <= curr_final_level; level_idx++) { + DeviceRuntime::SyncQueue(queue_idx); + Timer timer_iter; + if constexpr (ProfileBPEncoder) { + DeviceRuntime::SyncQueue(queue_idx); + timer_iter.start(); + } encoder.progressive_decode( - hierarchy->level_num_elems(level_idx), - mdr_metadata.prev_used_level_num_bitplanes[level_idx], num_bitplanes, - level_exp, - SubArray<2, T_bitplane, DeviceType>( - encoded_bitplanes_array[level_idx]), - SubArray(mdr_data.level_signs[level_idx]), level_idx, - levels_data[level_idx], queue_idx); - if (num_bitplanes == 0) { - levels_array[level_idx].memset(0); + level_data_subarray[level_idx].shape(0), + mdr_metadata.prev_used_level_num_bitplanes[level_idx], + level_num_bitplanes[level_idx], SubArray(abs_max_array[level_idx]), + encoded_bitplanes_subarray[level_idx], + level_signs_subarray[level_idx], level_idx, + level_data_subarray[level_idx], queue_idx); + if constexpr (ProfileBPEncoder) { + DeviceRuntime::SyncQueue(queue_idx); + timer_iter.end(); + timer_iter.print( + "Decoding level (# of coefficients: " + + std::to_string(level_data_subarray[level_idx].shape(0)) + ")", + level_data_subarray[level_idx].shape(0) * sizeof(T_data), true); } + // if (level_idx < curr_final_level) { + // printf("%.6f, ", timer_iter.get()); + // } else { + // printf("%.6f", timer_iter.get()); + // } + } + // std::cout << "],\n"; + // } + + for (int level_idx = 0; level_idx <= curr_final_level; level_idx++) { + if (level_num_bitplanes[level_idx] == 0) { + level_data_array[level_idx].memset(0, queue_idx); + } + } + + if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); - compressor.decompress_release(); timer.end(); - timer.print("Decoding"); + timer.print("Decoding", hierarchy->total_num_elems() * sizeof(T_data)); + timer.clear(); + timer.start(); } partial_reconsctructed_data.resize( hierarchy->level_shape(curr_final_level)); - timer.start(); // Put decoded coefficients back to reordered layout interleaver.reposition( - levels_data, + level_data_subarray, SubArray(partial_reconsctructed_data), curr_final_level, queue_idx); - DeviceRuntime::SyncQueue(queue_idx); - timer.end(); - timer.print("Reposition"); - timer.start(); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Reposition", hierarchy->total_num_elems() * sizeof(T_data)); + timer.clear(); + } + decomposer.recompose(partial_reconsctructed_data, 0, curr_final_level, queue_idx); - DeviceRuntime::SyncQueue(queue_idx); - timer.end(); - timer.print("Recomposing"); + if (adaptive_resolution) { // Interpolate previous reconstructed data to the same resolution InterpolateToLevel(reconstructed_data, prev_final_level, curr_final_level, queue_idx); } + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } SubArray partial_reconstructed_subarray(partial_reconsctructed_data); SubArray reconstructed_subarray(reconstructed_data); data_refactoring::multi_dimension::AddND(partial_reconstructed_subarray, reconstructed_subarray, queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("AddND", hierarchy->total_num_elems() * sizeof(T_data)); + timer.clear(); + } mdr_metadata.DoneReconstruct(); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer_all.end(); + timer_all.print("Decoding + Reposition + Recompose", + hierarchy->total_num_elems() * sizeof(T_data)); + timer_all.clear(); + } } const std::vector &get_dimensions() { return dimensions; } @@ -301,19 +515,22 @@ class ComposedReconstructor Array partial_reconsctructed_data; Array interpolation_workspace; - Array<1, T_data, DeviceType> *levels_array = nullptr; - SubArray<1, T_data, DeviceType> *levels_data = nullptr; + std::vector> level_data_array; + std::vector> level_data_subarray; std::vector> encoded_bitplanes_array; - SIZE total_num_bitplanes; + std::vector> encoded_bitplanes_subarray; + std::vector> level_signs_subarray; + std::vector> abs_max_array; bool prev_reconstructed; + std::vector level_num_elems; + std::vector exp; + std::vector data; std::vector dimensions; std::vector level_error_bounds; std::vector level_num_bitplanes; - std::vector> level_components; - std::vector> level_sizes; std::vector level_num; std::vector> level_squared_errors; }; diff --git a/include/mgard-x/MDR-X/Reconstructor/ReconstructorCache.hpp b/include/mgard-x/MDR-X/Reconstructor/ReconstructorCache.hpp index 936957d1de..f405f70fa4 100644 --- a/include/mgard-x/MDR-X/Reconstructor/ReconstructorCache.hpp +++ b/include/mgard-x/MDR-X/Reconstructor/ReconstructorCache.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_MDR_X_RECONSTRUCTOR_CACHE_HPP #define MGARD_X_MDR_X_RECONSTRUCTOR_CACHE_HPP @@ -88,18 +88,18 @@ class ReconstructorBundle { hierarchy_cache = new std::unordered_map(); } - void Initialize() { + void Initialize(int num_buffers = 2) { log::info("Initializing reconstructor cache"); hierarchy_cache = new std::unordered_map(); reconstructor = new ReconstructorType(); - device_subdomain_buffer = new Array[2]; - mdr_data = new MDRData[2]; + device_subdomain_buffer = new Array[num_buffers]; + mdr_data = new MDRData[num_buffers]; initialized = true; } - void SafeInitialize() { + void SafeInitialize(int num_buffers = 2) { if (!initialized) { - Initialize(); + Initialize(num_buffers); } } diff --git a/include/mgard-x/MDR-X/Refactor/ComposedRefactor.hpp b/include/mgard-x/MDR-X/Refactor/ComposedRefactor.hpp index e8d3a8f026..78feeaa850 100644 --- a/include/mgard-x/MDR-X/Refactor/ComposedRefactor.hpp +++ b/include/mgard-x/MDR-X/Refactor/ComposedRefactor.hpp @@ -16,18 +16,45 @@ namespace mgard_x { namespace MDR { // a decomposition-based scientific data refactor: compose a refactor using // decomposer, interleaver, encoder, and error collector -template +template class ComposedRefactor : public concepts::RefactorInterface { public: using HierarchyType = Hierarchy; using T_bitplane = uint32_t; using T_error = double; - using Decomposer = MGARDOrthoganalDecomposer; + using Decomposer = MGARDDecomposer; using Interleaver = DirectInterleaver; - using Encoder = GroupedBPEncoder; - // using Compressor = DefaultLevelCompressor; - using Compressor = NullLevelCompressor; + + constexpr static bool ProfileBPEncoder = false; + // using Encoder = GroupedBPEncoder; + // using Encoder = BPEncoderLocalityBlock; + using Encoder = BPEncoderRegisterBlock; + // using Encoder = BPEncoderRegisterShift; + // using Encoder = BPEncoderRegisterBallot; + // using Encoder = BPEncoderRegisterReduceAll; + // using Encoder = BPEncoderRegisterMatchAny; + + // using Compressor = DefaultLevelCompressor; + // using Compressor = DefaultLevelCompressor; + using Compressor = HybridLevelCompressor; + // using Compressor = NullLevelCompressor; + + static constexpr SIZE BATCH_SIZE = sizeof(T_bitplane) * 8; + static constexpr SIZE MAX_BITPLANES = sizeof(T_data) * 8; ComposedRefactor() : initialized(false) {} @@ -41,17 +68,14 @@ class ComposedRefactor hierarchy.EstimateMemoryFootprint(shape); SIZE size = 0; for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { - size += config.total_num_bitplanes * - Encoder::buffer_size(hierarchy.level_num_elems(level_idx)) * + size += Encoder::MAX_BITPLANES * + Encoder::bitplane_length(hierarchy.level_num_elems(level_idx)) * sizeof(T_bitplane); } return size; } - ~ComposedRefactor() { - delete[] levels_array; - delete[] levels_data; - } + ~ComposedRefactor() {} void Adapt(Hierarchy &hierarchy, Config config, int queue_idx) { @@ -60,34 +84,59 @@ class ComposedRefactor decomposer.Adapt(hierarchy, config, queue_idx); interleaver.Adapt(hierarchy, queue_idx); encoder.Adapt(hierarchy, queue_idx); - compressor.Adapt( - Encoder::buffer_size(hierarchy.level_num_elems(hierarchy.l_target())), - config, queue_idx); - total_num_bitplanes = config.total_num_bitplanes; - - delete[] levels_array; - delete[] levels_data; - levels_array = new Array<1, T_data, DeviceType>[hierarchy.l_target() + 1]; - levels_data = new SubArray<1, T_data, DeviceType>[hierarchy.l_target() + 1]; + // batched_encoder.Adapt(hierarchy, queue_idx); + compressor.Adapt(encoder.bitplane_length( + hierarchy.level_num_elems(hierarchy.l_target())), + Encoder::MAX_BITPLANES, config, queue_idx); + + level_data_array.resize(hierarchy.l_target() + 1); + level_data_subarray.resize(hierarchy.l_target() + 1); + abs_max_array.resize(hierarchy.l_target() + 1); for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { - levels_array[level_idx].resize({hierarchy.level_num_elems(level_idx)}, - queue_idx); - levels_data[level_idx] = - SubArray<1, T_data, DeviceType>(levels_array[level_idx]); + level_data_array[level_idx].resize( + {round_up(hierarchy.level_num_elems(level_idx), BATCH_SIZE)}, + queue_idx); + // interleave() only ever writes the level's real elements; the + // round-up padding above (needed so the encoder's batch-aligned + // kernels can run) is otherwise left as whatever cudaMalloc/pool + // memory previously held. AbsMax and encode() below both operate + // over the full padded length, so uninitialized padding pollutes + // the level-wide abs_max scale factor -- harmless while the pool + // memory happens to be zero, but corrupts every real element's + // encoding once some larger, unrelated allocation has left large + // leftover values in that memory. Zero it once here; interleave() + // never touches it again for the lifetime of this object. + level_data_array[level_idx].memset(0, queue_idx); + level_data_subarray[level_idx] = + SubArray<1, T_data, DeviceType>(level_data_array[level_idx]); + abs_max_array[level_idx].resize({1}, queue_idx); + abs_max_array[level_idx].hostAllocate(false, queue_idx); } - abs_max_result_array.resize({1}, queue_idx); + DeviceCollective::AbsMax( hierarchy.level_num_elems(hierarchy.l_target()), SubArray<1, T_data, DeviceType>(), SubArray<1, T_data, DeviceType>(), abs_max_workspace, false, 0); encoded_bitplanes_array.resize(hierarchy.l_target() + 1); + encoded_bitplanes_subarray.resize(hierarchy.l_target() + 1); + level_num_elems.resize(hierarchy.l_target() + 1); + level_errors_array.resize(hierarchy.l_target() + 1); + level_errors_subarray.resize(hierarchy.l_target() + 1); + exp.resize(hierarchy.l_target() + 1); for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { encoded_bitplanes_array[level_idx].resize( - {(SIZE)total_num_bitplanes, - encoder.buffer_size(hierarchy.level_num_elems(level_idx))}, + {(SIZE)Encoder::MAX_BITPLANES, + encoder.bitplane_length(hierarchy.level_num_elems(level_idx))}, queue_idx); + encoded_bitplanes_subarray[level_idx] = + SubArray<2, T_bitplane, DeviceType>( + encoded_bitplanes_array[level_idx]); + level_num_elems[level_idx] = hierarchy.level_num_elems(level_idx); + level_errors_array[level_idx].resize({(SIZE)Encoder::MAX_BITPLANES + 1}, + queue_idx); + level_errors_subarray[level_idx] = + SubArray<1, T_error, DeviceType>(level_errors_array[level_idx]); } - level_errors_array.resize({(SIZE)total_num_bitplanes + 1}, queue_idx); } static size_t EstimateMemoryFootprint(std::vector shape, @@ -96,7 +145,8 @@ class ComposedRefactor size_t size = 0; size += hierarchy.EstimateMemoryFootprint(shape); for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { - size += hierarchy.level_num_elems(level_idx) * sizeof(T_data); + size += round_up(hierarchy.level_num_elems(level_idx), BATCH_SIZE) * + sizeof(T_data); } size += sizeof(T_data); Array<1, Byte, DeviceType> tmp; @@ -106,15 +156,16 @@ class ComposedRefactor tmp, false, 0); size += tmp.shape(0); for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { - size += config.total_num_bitplanes * - Encoder::buffer_size(hierarchy.level_num_elems(level_idx)) * + size += Encoder::MAX_BITPLANES * + Encoder::bitplane_length(hierarchy.level_num_elems(level_idx)) * sizeof(T_bitplane); + size += sizeof(T_error) * (Encoder::MAX_BITPLANES + 1); } - SIZE max_n = - Encoder::buffer_size(hierarchy.level_num_elems(hierarchy.l_target())); + SIZE max_n = Encoder::bitplane_length( + hierarchy.level_num_elems(hierarchy.l_target())); - size += (config.total_num_bitplanes + 1) * sizeof(T_error); + size += (Encoder::MAX_BITPLANES + 1) * sizeof(T_error); size += Decomposer::EstimateMemoryFootprint(shape); size += Interleaver::EstimateMemoryFootprint(shape); size += Encoder::EstimateMemoryFootprint(shape); @@ -122,25 +173,54 @@ class ComposedRefactor return size; } + static std::vector> + EstimateMaxBitplaneSizes(Hierarchy &hierarchy) { + std::vector> estimation; + estimation.resize(hierarchy.l_target() + 1); + for (int level_idx = 0; level_idx < hierarchy.l_target() + 1; level_idx++) { + estimation[level_idx].resize(Encoder::MAX_BITPLANES); + for (int bitplane_idx = 0; bitplane_idx < Encoder::MAX_BITPLANES; + bitplane_idx++) { + if (bitplane_idx % Compressor::num_merged_bitplanes == 0) { + estimation[level_idx][bitplane_idx] = + Encoder::bitplane_length(hierarchy.level_num_elems(level_idx)) * + sizeof(T_bitplane) * Compressor::num_merged_bitplanes; + // For Huffman-only model (metadata storage) + estimation[level_idx][bitplane_idx] += 1e6; + } else { + estimation[level_idx][bitplane_idx] = 1; + } + } + } + return estimation; + } + void Refactor(Array &data_array, MDRMetadata &mdr_metadata, MDRData &mdr_data, int queue_idx) { SIZE target_level = hierarchy->l_target(); - mdr_metadata.Initialize(hierarchy->l_target() + 1, total_num_bitplanes); - mdr_data.Resize(hierarchy->l_target() + 1, total_num_bitplanes); + mdr_metadata.Initialize(hierarchy->l_target() + 1, Encoder::MAX_BITPLANES); + mdr_data.Resize(*this, *hierarchy, queue_idx); SubArray data(data_array); - Timer timer; + Timer timer, timer_all; if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); - timer.start(); + timer_all.start(); } decomposer.decompose(data_array, hierarchy->l_target(), 0, queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + interleaver.interleave(data, level_data_subarray, hierarchy->l_target(), + queue_idx); if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); timer.end(); - timer.print("Decompose"); + timer.print("Interleave", hierarchy->total_num_elems() * sizeof(T_data)); timer.clear(); } @@ -148,75 +228,193 @@ class ComposedRefactor DeviceRuntime::SyncQueue(queue_idx); timer.start(); } - interleaver.interleave(data, levels_data, hierarchy->l_target(), queue_idx); + + for (int level_idx = 0; level_idx < hierarchy->l_target() + 1; + level_idx++) { + DeviceCollective::AbsMax( + level_data_subarray[level_idx].shape(0), + level_data_subarray[level_idx], SubArray(abs_max_array[level_idx]), + abs_max_workspace, true, queue_idx); + + { + // DumpSubArray("level_"+std::to_string(level_idx), + // level_data_subarray[level_idx]); for (SIZE i = 0; i < + // level_data_subarray[level_idx].shape(0); i += 1e6) { + // // for (SIZE i = 0; i < 10; i += 10) { + // SIZE n = std::min(level_data_subarray[level_idx].shape(0) - i, + // (SIZE)1e6); SubArray<1, T_data, DeviceType> data_block({n}, + // level_data_subarray[level_idx](i)); + // // PrintSubarray("data_block", data_block); + // T_data * ddd = new T_data[n]; + // MemoryManager::Copy1D(ddd, data_block.data(), n, + // queue_idx); DeviceRuntime::SyncQueue(queue_idx); + + // T_data min = fabs(ddd[0]); + // T_data max = fabs(ddd[0]); + // for (SIZE j = 0; j < n; j++) { + // min = std::min(min, fabs(ddd[j])); + // max = std::max(max, fabs(ddd[j])); + // } + + // int c = 0; + // for (SIZE j = 0; j < n; j++) { + // if (fabs(ddd[i]) > max * 0.001) { + // c++; + // } + // } + // std::cout << "cpu: [" << n << "] " << max << " - "<< min << " c: " + // << c << std::endl; + + // DeviceCollective::AbsMax( + // n, + // data_block, SubArray(abs_max_array[level_idx]), + // abs_max_workspace, true, queue_idx); + // DeviceRuntime::SyncQueue(queue_idx); + // abs_max_array[level_idx].hostCopy(false, queue_idx); + // DeviceRuntime::SyncQueue(queue_idx); + // T_data abs_max = abs_max_array[level_idx].dataHost()[0]; + + // DeviceCollective::AbsMin( + // n, + // data_block, SubArray(abs_max_array[level_idx]), + // abs_max_workspace, true, queue_idx); + // DeviceRuntime::SyncQueue(queue_idx); + // abs_max_array[level_idx].hostCopy(false, queue_idx); + // DeviceRuntime::SyncQueue(queue_idx); + // T_data abs_min = abs_max_array[level_idx].dataHost()[0]; + + // std::cout << "abs: " << abs_max << " - "<< abs_min << std::endl; + // } + } + + encoded_bitplanes_array[level_idx].resize( + {(SIZE)Encoder::MAX_BITPLANES, + encoder.bitplane_length(hierarchy->level_num_elems(level_idx))}, + queue_idx); + encoded_bitplanes_subarray[level_idx] = + SubArray<2, T_bitplane, DeviceType>( + encoded_bitplanes_array[level_idx]); + + Timer timer_iter; + if constexpr (ProfileBPEncoder) { + DeviceRuntime::SyncQueue(queue_idx); + timer_iter.start(); + } + encoder.encode(level_data_subarray[level_idx].shape(0), + Encoder::MAX_BITPLANES, SubArray(abs_max_array[level_idx]), + level_data_subarray[level_idx], + encoded_bitplanes_subarray[level_idx], + level_errors_subarray[level_idx], queue_idx); + if constexpr (ProfileBPEncoder) { + DeviceRuntime::SyncQueue(queue_idx); + timer_iter.end(); + timer_iter.print( + "Encoding level (# of coefficients: " + + std::to_string(level_data_subarray[level_idx].shape(0)) + ")", + level_data_subarray[level_idx].shape(0) * sizeof(T_data), true); + } + } + if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); timer.end(); - timer.print("Interleave"); + timer.print("Encoding", hierarchy->total_num_elems() * sizeof(T_data)); timer.clear(); } + // if (log::level & log::TIME) { + // DeviceRuntime::SyncQueue(queue_idx); + // timer.start(); + // } + + // for (int level_idx = 0; level_idx < hierarchy->l_target() + 1; + // level_idx++) { + // compressor.compress_level(encoded_bitplanes_subarray[level_idx], + // mdr_data.compressed_bitplanes[level_idx], + // level_idx, queue_idx); + // for (int bitplane_idx = 0; bitplane_idx < Encoder::MAX_BITPLANES; + // bitplane_idx++) { + // mdr_metadata.level_sizes[level_idx][bitplane_idx] += + // mdr_data.compressed_bitplanes[level_idx][bitplane_idx].shape(0); + // } + // } + // if (log::level & log::TIME) { + // DeviceRuntime::SyncQueue(queue_idx); + // timer.end(); + // timer.print("Lossless", hierarchy->total_num_elems() * sizeof(T_data)); + // timer.clear(); + // } + + // Compress(mdr_metadata, mdr_data, queue_idx); + // StoreMetadata(mdr_metadata, mdr_data, queue_idx); + // for (int level_idx = 0; level_idx < hierarchy->l_target() + 1; + // level_idx++) { + // abs_max_array[level_idx].hostCopy(false, queue_idx); + // DeviceRuntime::SyncQueue(queue_idx); + // T_data level_max_error = abs_max_array[level_idx].dataHost()[0]; + // mdr_metadata.level_error_bounds[level_idx] = level_max_error; + // mdr_metadata.level_num_elems[level_idx] = + // hierarchy->level_num_elems(level_idx); std::vector + // squared_error(Encoder::MAX_BITPLANES + 1); + // MemoryManager::Copy1D(squared_error.data(), + // level_errors_array[level_idx].data(), + // Encoder::MAX_BITPLANES + 1, + // queue_idx); + // mdr_metadata.level_squared_errors[level_idx] = squared_error; + // // PrintSubarray("level_errors", level_errors_subarray[level_idx]); + // } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer_all.end(); + timer_all.print("Decompose + Interleave + Encoding", + hierarchy->total_num_elems() * sizeof(T_data)); + timer_all.clear(); + } + } + + void Compress(MDRMetadata &mdr_metadata, MDRData &mdr_data, + int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } for (int level_idx = 0; level_idx < hierarchy->l_target() + 1; level_idx++) { + compressor.compress_level(encoded_bitplanes_subarray[level_idx], + mdr_data.compressed_bitplanes[level_idx], + level_idx, queue_idx); + } + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Lossless", hierarchy->total_num_elems() * sizeof(T_data)); + timer.clear(); + } + } - if (log::level & log::TIME) { - DeviceRuntime::SyncQueue(queue_idx); - timer.start(); - } - SubArray<1, T_data, DeviceType> result(abs_max_result_array); - DeviceCollective::AbsMax(levels_data[level_idx].shape(0), - levels_data[level_idx], result, - abs_max_workspace, true, queue_idx); - T_data level_max_error; - MemoryManager::Copy1D(&level_max_error, result.data(), 1, - queue_idx); + void StoreMetadata(MDRMetadata &mdr_metadata, MDRData &mdr_data, + int queue_idx) { + for (int level_idx = 0; level_idx < hierarchy->l_target() + 1; + level_idx++) { + abs_max_array[level_idx].hostCopy(false, queue_idx); DeviceRuntime::SyncQueue(queue_idx); - int level_exp = 0; - frexp(level_max_error, &level_exp); - // printf("level: %d, level_max_error: %.10f, level_exp: %d\n", level_idx, - // level_max_error, level_exp); + T_data level_max_error = abs_max_array[level_idx].dataHost()[0]; mdr_metadata.level_error_bounds[level_idx] = level_max_error; mdr_metadata.level_num_elems[level_idx] = hierarchy->level_num_elems(level_idx); - if (log::level & log::TIME) { - DeviceRuntime::SyncQueue(queue_idx); - timer.end(); - timer.print("Max Error"); - timer.clear(); - timer.start(); - } - - SubArray<2, T_bitplane, DeviceType> encoded_bitplanes( - encoded_bitplanes_array[level_idx]); - SubArray<1, T_error, DeviceType> level_errors(level_errors_array); - std::vector bitplane_sizes(total_num_bitplanes); - encoder.encode(hierarchy->level_num_elems(level_idx), total_num_bitplanes, - level_exp, levels_data[level_idx], encoded_bitplanes, - level_errors, bitplane_sizes, queue_idx); - std::vector squared_error(total_num_bitplanes + 1); + std::vector squared_error(Encoder::MAX_BITPLANES + 1); MemoryManager::Copy1D(squared_error.data(), - level_errors_array.data(), - total_num_bitplanes + 1, queue_idx); + level_errors_array[level_idx].data(), + Encoder::MAX_BITPLANES + 1, queue_idx); mdr_metadata.level_squared_errors[level_idx] = squared_error; - // PrintSubarray("level_errors", level_errors); - if (log::level & log::TIME) { - DeviceRuntime::SyncQueue(queue_idx); - timer.end(); - timer.print("Encoding"); - timer.clear(); - timer.start(); - } - - compressor.compress_level( - bitplane_sizes, encoded_bitplanes_array[level_idx], - mdr_data.compressed_bitplanes[level_idx], queue_idx); - mdr_metadata.level_sizes[level_idx] = bitplane_sizes; - if (log::level & log::TIME) { - DeviceRuntime::SyncQueue(queue_idx); - timer.end(); - timer.print("Compress"); - timer.clear(); + for (int bitplane_idx = 0; bitplane_idx < Encoder::MAX_BITPLANES; + bitplane_idx++) { + mdr_metadata.level_sizes[level_idx][bitplane_idx] += + mdr_data.compressed_bitplanes[level_idx][bitplane_idx].shape(0); } + // PrintSubarray("level_errors", level_errors_subarray[level_idx]); } } @@ -238,17 +436,23 @@ class ComposedRefactor Decomposer decomposer; Interleaver interleaver; Encoder encoder; + // BatchedEncoder batched_encoder; Compressor compressor; - Array<1, T_data, DeviceType> *levels_array = nullptr; - SubArray<1, T_data, DeviceType> *levels_data = nullptr; - Array<1, T_data, DeviceType> abs_max_result_array; + std::vector> level_data_array; + std::vector> level_data_subarray; + + std::vector> abs_max_array; Array<1, Byte, DeviceType> abs_max_workspace; + std::vector> encoded_bitplanes_array; - Array<1, T_error, DeviceType> level_errors_array; + std::vector> encoded_bitplanes_subarray; + + std::vector> level_errors_array; + std::vector> level_errors_subarray; - SIZE total_num_bitplanes; - std::vector> level_components; + std::vector level_num_elems; + std::vector exp; }; } // namespace MDR } // namespace mgard_x diff --git a/include/mgard-x/MDR-X/Refactor/RefactorCache.hpp b/include/mgard-x/MDR-X/Refactor/RefactorCache.hpp index 146b39add8..c6aa733515 100644 --- a/include/mgard-x/MDR-X/Refactor/RefactorCache.hpp +++ b/include/mgard-x/MDR-X/Refactor/RefactorCache.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_MDR_X_REFACTOR_CACHE_HPP #define MGARD_X_MDR_X_REFACTOR_CACHE_HPP @@ -88,18 +88,18 @@ class RefactorBundle { hierarchy_cache = new std::unordered_map(); } - void Initialize() { + void Initialize(int num_buffers = 2) { log::info("Initializing refactor cache"); hierarchy_cache = new std::unordered_map(); refactor = new RefactorType(); - device_subdomain_buffer = new Array[2]; - mdr_data = new MDRData[2]; + device_subdomain_buffer = new Array[num_buffers]; + mdr_data = new MDRData[num_buffers]; initialized = true; } - void SafeInitialize() { + void SafeInitialize(int num_buffers = 2) { if (!initialized) { - Initialize(); + Initialize(num_buffers); } } diff --git a/include/mgard-x/MDR-X/SizeInterpreter/GreedyBasedSizeInterpreter.hpp b/include/mgard-x/MDR-X/SizeInterpreter/GreedyBasedSizeInterpreter.hpp index 24322d553e..ff0d066edc 100644 --- a/include/mgard-x/MDR-X/SizeInterpreter/GreedyBasedSizeInterpreter.hpp +++ b/include/mgard-x/MDR-X/SizeInterpreter/GreedyBasedSizeInterpreter.hpp @@ -27,6 +27,87 @@ class GreedyBasedSizeInterpreter : public concepts::SizeInterpreterInterface { interpret_retrieve_size(const std::vector> &level_sizes, const std::vector> &level_errors, double tolerance, std::vector &index) const { + int num_levels = level_sizes.size(); + std::vector retrieve_sizes(num_levels, 0); + + double accumulated_error = 0; + for (int i = 0; i < num_levels; i++) { + accumulated_error += + error_estimator.estimate_error(level_errors[i][index[i]], i); + } + std::priority_queue, + CompareUnitErrorGain> + heap; + // old before the fix + // for (int i = 0; i < num_levels; i++) { + // if (index[i] < level_sizes[i].size()) { + // double error_gain = error_estimator.estimate_error_gain( + // accumulated_error, level_errors[i][index[i]], + // level_errors[i][index[i] + 1], i); + // heap.push(UnitErrorGain(error_gain / level_sizes[i][index[i]], i)); + // } + // } + // new + double min_error = accumulated_error; + for (int i = 0; i < num_levels; i++) { + min_error -= error_estimator.estimate_error(level_errors[i][index[i]], i); + min_error += error_estimator.estimate_error(level_errors[i].back(), i); + // fetch the first component if index is 0 + if (index[i] == 0) { + retrieve_sizes[i] += level_sizes[i][index[i]]; + accumulated_error -= + error_estimator.estimate_error(level_errors[i][index[i]], i); + accumulated_error += + error_estimator.estimate_error(level_errors[i][index[i] + 1], i); + index[i]++; + // std::cout << i; + } + // push the next one + if (index[i] != level_sizes[i].size()) { + double error_gain = error_estimator.estimate_error_gain( + accumulated_error, level_errors[i][index[i]], + level_errors[i][index[i] + 1], i); + heap.push(UnitErrorGain(error_gain / level_sizes[i][index[i]], i)); + } + if (min_error < tolerance) { + // the min error of first 0~i levels meets the tolerance + num_levels = i + 1; + break; + } + } + + bool tolerance_met = false; + while ((!tolerance_met) && (!heap.empty())) { + auto unit_error_gain = heap.top(); + heap.pop(); + int i = unit_error_gain.level; + int j = index[i]; + retrieve_sizes[i] += level_sizes[i][j]; + accumulated_error -= + error_estimator.estimate_error(level_errors[i][j], i); + accumulated_error += + error_estimator.estimate_error(level_errors[i][j + 1], i); + if (accumulated_error < tolerance) { + tolerance_met = true; + } + index[i]++; + if (index[i] < level_sizes[i].size()) { + double error_gain = error_estimator.estimate_error_gain( + accumulated_error, level_errors[i][index[i]], + level_errors[i][index[i] + 1], i); + heap.push(UnitErrorGain(error_gain / level_sizes[i][index[i]], i)); + } + } + // std::cout << "Requested tolerance = " << tolerance + // << ", estimated error = " << accumulated_error << std::endl; + return retrieve_sizes; + } + // corresponding error return + std::vector + interpret_retrieve_size(const std::vector> &level_sizes, + const std::vector> &level_errors, + double tolerance, double &eb, + std::vector &index) const { const int num_levels = level_sizes.size(); std::vector retrieve_sizes(num_levels, 0); @@ -71,8 +152,60 @@ class GreedyBasedSizeInterpreter : public concepts::SizeInterpreterInterface { } // std::cout << "Requested tolerance = " << tolerance // << ", estimated error = " << accumulated_error << std::endl; + eb = accumulated_error; + return retrieve_sizes; + } + // segmented + std::vector + interpret_retrieve_size(const std::vector> &level_sizes, + const std::vector> &level_errors, + uint32_t requested_size, double &eb, + std::vector &index) const { + const int num_levels = level_sizes.size(); + std::vector retrieve_sizes(num_levels, 0); + double accumulated_error = 0; + uint32_t accumulated_size = 0; + for (int i = 0; i < num_levels; i++) { + accumulated_error += + error_estimator.estimate_error(level_errors[i][index[i]], i); + } + std::priority_queue, + CompareUnitErrorGain> + heap; + for (int i = 0; i < num_levels; i++) { + if (index[i] < level_sizes[i].size()) { + double error_gain = error_estimator.estimate_error_gain( + accumulated_error, level_errors[i][index[i]], + level_errors[i][index[i] + 1], i); + heap.push(UnitErrorGain(error_gain / level_sizes[i][index[i]], i)); + } + } + + while ((accumulated_size < requested_size) && (!heap.empty())) { + auto unit_error_gain = heap.top(); + heap.pop(); + int i = unit_error_gain.level; + int j = index[i]; + retrieve_sizes[i] += level_sizes[i][j]; + accumulated_error -= + error_estimator.estimate_error(level_errors[i][j], i); + accumulated_error += + error_estimator.estimate_error(level_errors[i][j + 1], i); + accumulated_size += level_sizes[i][j]; + index[i]++; + if (index[i] < level_sizes[i].size()) { + double error_gain = error_estimator.estimate_error_gain( + accumulated_error, level_errors[i][index[i]], + level_errors[i][index[i] + 1], i); + heap.push(UnitErrorGain(error_gain / level_sizes[i][index[i]], i)); + } + } + // std::cout << "Requested size = " << requested_size + // << ", estimated error = " << accumulated_error << std::endl; + eb = accumulated_error; return retrieve_sizes; } + void print() const { std::cout << "Greedy based size interpreter." << std::endl; } @@ -260,12 +393,12 @@ class NegaBinaryGreedyBasedSizeInterpreter heap.push(estimated_efficiency(accumulated_error, index[i], i, level_errors[i], level_sizes[i])); } - for (int k = 0; k < num; k++) - std::cout << i; + // for (int k = 0; k < num; k++) + // std::cout << i; } - std::cout << std::endl; - std::cout << "Requested tolerance = " << tolerance - << ", estimated error = " << accumulated_error << std::endl; + // std::cout << std::endl; + // std::cout << "Requested tolerance = " << tolerance + // << ", estimated error = " << accumulated_error << std::endl; return retrieve_sizes; } void print() const { diff --git a/include/mgard-x/MDR/BitplaneEncoder/GroupedBPEncoder.hpp b/include/mgard-x/MDR/BitplaneEncoder/GroupedBPEncoder.hpp index 65916e7d4b..87fd897de3 100644 --- a/include/mgard-x/MDR/BitplaneEncoder/GroupedBPEncoder.hpp +++ b/include/mgard-x/MDR/BitplaneEncoder/GroupedBPEncoder.hpp @@ -1,6 +1,7 @@ #ifndef _MDR_GROUPED_BP_ENCODER_HPP #define _MDR_GROUPED_BP_ENCODER_HPP +#include "../../RuntimeX/Utilities/Exceptions.h" #include "BitplaneEncoderInterface.hpp" namespace MDR { @@ -367,8 +368,7 @@ class GroupedBPEncoder : public concepts::BitplaneEncoderInterface { } else if (std::is_same::value) { block_size = 8; } else { - std::cerr << "Integer type not supported." << std::endl; - exit(0); + throw mgard_x::ProcessingException("Integer type not supported."); } return block_size; } diff --git a/include/mgard-x/MDR/BitplaneEncoder/NegaBinaryBPEncoder.hpp b/include/mgard-x/MDR/BitplaneEncoder/NegaBinaryBPEncoder.hpp index 0134188939..932a1810f0 100644 --- a/include/mgard-x/MDR/BitplaneEncoder/NegaBinaryBPEncoder.hpp +++ b/include/mgard-x/MDR/BitplaneEncoder/NegaBinaryBPEncoder.hpp @@ -1,6 +1,7 @@ #ifndef _MDR_NEGABINARY_BP_ENCODER_HPP #define _MDR_NEGABINARY_BP_ENCODER_HPP +#include "../../RuntimeX/Utilities/Exceptions.h" #include "BitplaneEncoderInterface.hpp" namespace MDR { @@ -248,8 +249,7 @@ class NegaBinaryBPEncoder : public concepts::BitplaneEncoderInterface { } else if (std::is_same::value) { block_size = 8; } else { - std::cerr << "Integer type not supported." << std::endl; - exit(0); + throw mgard_x::ProcessingException("Integer type not supported."); } return block_size; } diff --git a/include/mgard-x/MDR/ErrorEstimator/MaxErrorEstimator.hpp b/include/mgard-x/MDR/ErrorEstimator/MaxErrorEstimator.hpp index 91b23e006f..10022c9849 100644 --- a/include/mgard-x/MDR/ErrorEstimator/MaxErrorEstimator.hpp +++ b/include/mgard-x/MDR/ErrorEstimator/MaxErrorEstimator.hpp @@ -21,9 +21,9 @@ template class MaxErrorEstimatorOB : public MaxErrorEstimator { c = 1.0 + 21.0 * sqrt(3) / 8; break; default: - std::cerr << num_dims << "-Dimentional error estimation not implemented." - << std::endl; - exit(-1); + throw std::runtime_error( + std::to_string(num_dims) + + "-Dimentional error estimation not implemented."); } c *= 4; // 2 more bitplane for negabinary } diff --git a/include/mgard-x/MDR/Reconstructor/ComposedReconstructor.hpp b/include/mgard-x/MDR/Reconstructor/ComposedReconstructor.hpp index 505ace8925..79226565d5 100644 --- a/include/mgard-x/MDR/Reconstructor/ComposedReconstructor.hpp +++ b/include/mgard-x/MDR/Reconstructor/ComposedReconstructor.hpp @@ -1,6 +1,7 @@ #ifndef _MDR_COMPOSED_RECONSTRUCTOR_HPP #define _MDR_COMPOSED_RECONSTRUCTOR_HPP +#include "../../RuntimeX/Utilities/Exceptions.h" #include "../BitplaneEncoder/BitplaneEncoder.hpp" #include "../Decomposer/Decomposer.hpp" #include "../ErrorCollector/ErrorCollector.hpp" @@ -51,8 +52,7 @@ class ComposedReconstructor : public concepts::ReconstructorInterface { "level squared error directly" << std::endl; } else { - std::cerr << "Customized error estimator not supported yet" << std::endl; - exit(-1); + throw std::runtime_error("Customized error estimator not supported yet"); } timer.end(); timer.print("Preprocessing"); @@ -98,11 +98,11 @@ class ComposedReconstructor : public concepts::ReconstructorInterface { data[i] += cur_data[i]; } } else if (cur_data.size()) { - std::cerr << "Reconstruct size changes, not supported yet." << std::endl; - std::cerr << "Sizes before reconstruction: " << cur_data.size() - << std::endl; - std::cerr << "Sizes after reconstruction: " << data.size() << std::endl; - exit(0); + throw mgard_x::ProcessingException( + "Reconstruct size changes, not supported yet. Sizes before " + "reconstruction: " + + std::to_string(cur_data.size()) + + "; sizes after reconstruction: " + std::to_string(data.size())); } return data.data(); } diff --git a/include/mgard-x/MDRHighLevel/GenerateRequest.hpp b/include/mgard-x/MDRHighLevel/GenerateRequest.hpp new file mode 100644 index 0000000000..06077bae0d --- /dev/null +++ b/include/mgard-x/MDRHighLevel/GenerateRequest.hpp @@ -0,0 +1,30 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_MDR_GENERATE_PIPELINE_HPP +#define MGARD_X_MDR_GENERATE_PIPELINE_HPP + +namespace mgard_x { +namespace MDR { + +template +void generate_request(DomainDecomposer, + DeviceType> &domain_decomposer, + Config config, RefactoredMetadata &refactored_metadata) { + + for (int subdomain_id = 0; subdomain_id < domain_decomposer.num_subdomains(); + subdomain_id++) { + Hierarchy hierarchy = + domain_decomposer.subdomain_hierarchy(subdomain_id); + ComposedReconstructor reconstructor(hierarchy, config); + reconstructor.GenerateRequest(refactored_metadata.metadata[subdomain_id]); + } +} + +} // namespace MDR +} // namespace mgard_x +#endif \ No newline at end of file diff --git a/include/mgard-x/MDRHighLevel/MDRDataHighLevel.hpp b/include/mgard-x/MDRHighLevel/MDRDataHighLevel.hpp index 4ab4c3b6a3..bea8d71f50 100644 --- a/include/mgard-x/MDRHighLevel/MDRDataHighLevel.hpp +++ b/include/mgard-x/MDRHighLevel/MDRDataHighLevel.hpp @@ -1,13 +1,15 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_MDR_HIGH_LEVEL_DATA_HPP #define MGARD_X_MDR_HIGH_LEVEL_DATA_HPP +#include "../DomainDecomposer/DomainDecomposer.hpp" + #include "../RuntimeX/DataStructures/MDRMetadata.hpp" namespace mgard_x { @@ -29,6 +31,14 @@ class RefactoredMetadata { std::vector header; std::vector metadata; SIZE num_subdomains; + size_t total_size = 0; + double relative_eb = 0; + int decrease_method = 0; + double max_est_error = 0; + bool MPI_enabled = false; + double IO_time = 0; + double kernel_time = 0; + std::string input_path; template void Serialize(Byte *&ptr, T *data, SIZE bytes) { memcpy(ptr, (Byte *)data, bytes); @@ -80,9 +90,35 @@ class RefactoredMetadata { class RefactoredData { public: - void InitializeForRefactor(SIZE num_subdomains) { - this->num_subdomains = num_subdomains; + template + void InitializeForRefactor( + DomainDecomposer &domain_decomposer, + Config config) { + num_subdomains = domain_decomposer.num_subdomains(); data.resize(num_subdomains); + data_allocation_size.resize(num_subdomains); + for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { + Hierarchy hierarchy( + domain_decomposer.subdomain_shape(id), config); + std::vector> estimation = + RefactorType::EstimateMaxBitplaneSizes(hierarchy); + SIZE num_levels = estimation.size(); + SIZE num_bitplanes = estimation[0].size(); + data[id].resize(num_levels); + data_allocation_size[id].resize(num_levels); + for (int level_idx = 0; level_idx < num_levels; level_idx++) { + data[id][level_idx].resize(num_bitplanes); + data_allocation_size[id][level_idx].resize(num_bitplanes); + for (int bitplane_idx = 0; bitplane_idx < num_bitplanes; + bitplane_idx++) { + MemoryManager::MallocHost( + data[id][level_idx][bitplane_idx], + estimation[level_idx][bitplane_idx], 0); + data_allocation_size[id][level_idx][bitplane_idx] = + estimation[level_idx][bitplane_idx]; + } + } + } } void InitializeForReconstruction(RefactoredMetadata &refactored_metadata) { int num_subdomains = refactored_metadata.metadata.size(); @@ -102,6 +138,7 @@ class RefactoredData { } std::vector>> data; + std::vector>> data_allocation_size; std::vector> level_signs; SIZE num_subdomains; }; @@ -114,6 +151,39 @@ class ReconstructedData { shape.resize(num_subdomains); data.resize(num_subdomains); initialized = true; + qoi_in_progress = false; + } + + template + void ResizeToSingleDomain(std::vector domain_shape) { + // First time reconstruction + Initialize(1); + SIZE total_num_elem = 1; + for (int i = 0; i < D; i++) + total_num_elem *= domain_shape[i]; + MemoryManager::MallocHost(data[0], total_num_elem * sizeof(T), + 0); + // Is memset necessary? + memset(data[0], 0, total_num_elem * sizeof(T)); + offset[0] = std::vector(D, 0); + shape[0] = domain_shape; + } + + template + void ResizeToMultipleSubdomains(DomainDecomposerType &domain_decomposer) { + SIZE num_subdomains = domain_decomposer.num_subdomains(); + Initialize(num_subdomains); + for (SIZE subdomain_id = 0; subdomain_id < num_subdomains; subdomain_id++) { + SIZE total_num_elem = 1; + for (int i = 0; + i < domain_decomposer.subdomain_shape(subdomain_id).size(); i++) + total_num_elem *= domain_decomposer.subdomain_shape(subdomain_id)[i]; + MemoryManager::MallocHost(data[subdomain_id], + total_num_elem * sizeof(T), 0); + // Is memset necessary? + memset(data[subdomain_id], 0, total_num_elem * sizeof(T)); + } } bool IsInitialized() { return initialized; } std::vector> offset; @@ -121,6 +191,7 @@ class ReconstructedData { std::vector data; SIZE num_subdomains; bool initialized = false; + bool qoi_in_progress = false; }; } // namespace MDR diff --git a/include/mgard-x/MDRHighLevel/MDRHighLevel.h b/include/mgard-x/MDRHighLevel/MDRHighLevel.h index ca69bc0cd7..173e066344 100644 --- a/include/mgard-x/MDRHighLevel/MDRHighLevel.h +++ b/include/mgard-x/MDRHighLevel/MDRHighLevel.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include @@ -26,34 +26,34 @@ namespace mgard_x { namespace MDR { -template -void MDRefactor(DIM D, data_type dtype, std::vector shape, - const void *original_data, +template +void MDRefactor(std::vector shape, const void *original_data, RefactoredMetadata &refactored_metadata, RefactoredData &refactored_data, Config config, bool output_pre_allocated); -template -void MDRefactor(DIM D, data_type dtype, std::vector shape, - const void *original_data, std::vector coords, +template +void MDRefactor(std::vector shape, const void *original_data, + std::vector coords, RefactoredMetadata &refactored_metadata, RefactoredData &refactored_data, Config config, bool output_pre_allocated); -template -void MDRequest(RefactoredMetadata &refactored_metadata); +template +void MDRequest(std::vector shape, RefactoredMetadata &refactored_metadata, + Config config); -template -SIZE MDRMaxOutputDataSize(DIM D, data_type dtype, std::vector shape, - Config config); +template +SIZE MDRMaxOutputDataSize(std::vector shape, Config config); -template -void MDReconstruct(RefactoredMetadata &refactored_metadata, +template +void MDReconstruct(std::vector shape, + RefactoredMetadata &refactored_metadata, RefactoredData &refactored_data, ReconstructedData &reconstructed_data, Config config, bool output_pre_allocated); -template void release_cache(); +template void release_cache(); } // namespace MDR } // namespace mgard_x diff --git a/include/mgard-x/MDRHighLevel/MDRHighLevel.hpp b/include/mgard-x/MDRHighLevel/MDRHighLevel.hpp index 8dec1fc3fa..3a45cab191 100644 --- a/include/mgard-x/MDRHighLevel/MDRHighLevel.hpp +++ b/include/mgard-x/MDRHighLevel/MDRHighLevel.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_MDR_HIGH_LEVEL_API_HPP @@ -21,7 +21,10 @@ #include "../MDR-X/Reconstructor/ReconstructorCache.hpp" #include "../MDR-X/Refactor/RefactorCache.hpp" -#define BINSIZE 10 +#include "GenerateRequest.hpp" +#include "ReconstructPipeline.hpp" +#include "ReconstructPipelineQoI.hpp" +#include "RefactorPipeline.hpp" namespace mgard_x { namespace MDR { @@ -43,231 +46,6 @@ SIZE get_max_output_data_size( return size; } -template -void generate_request(DomainDecomposer, - DeviceType> &domain_decomposer, - Config config, RefactoredMetadata &refactored_metadata) { - for (int subdomain_id = 0; subdomain_id < domain_decomposer.num_subdomains(); - subdomain_id++) { - Hierarchy hierarchy = - domain_decomposer.subdomain_hierarchy(subdomain_id); - ComposedReconstructor reconstructor(hierarchy, config); - reconstructor.GenerateRequest(refactored_metadata.metadata[subdomain_id]); - } -} - -template -void refactor_pipeline( - DomainDecomposer &domain_decomposer, - Config &config, RefactoredMetadata &refactored_metadata, - RefactoredData &refactored_data) { - Timer timer_series; - if (log::level & log::TIME) - timer_series.start(); - - using Cache = RefactorCache; - using HierarchyType = typename RefactorType::HierarchyType; - - RefactorType &refactor = *Cache::cache.refactor; - - Array *device_subdomain_buffer = - Cache::cache.device_subdomain_buffer; - MDRData *mdr_data = Cache::cache.mdr_data; - - if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(0), - domain_decomposer.uniform)) { - Cache::cache.ClearHierarchyCache(); - } - - for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { - if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(id), - domain_decomposer.uniform)) { - Cache::cache.InsertHierarchyCache( - domain_decomposer.subdomain_hierarchy(id)); - } - } - - HierarchyType &hierarchy = - Cache::cache.GetHierarchyCache(domain_decomposer.subdomain_shape(0)); - - log::info("Adjust device buffers"); - device_subdomain_buffer[0].resize(domain_decomposer.subdomain_shape(0)); - device_subdomain_buffer[1].resize(domain_decomposer.subdomain_shape(0)); - mdr_data[0].Resize(hierarchy.l_target() + 1, config.total_num_bitplanes); - mdr_data[1].Resize(hierarchy.l_target() + 1, config.total_num_bitplanes); - DeviceRuntime::SyncDevice(); - - // Prefetch the first subdomain to one buffer - int current_buffer = 0; - int current_queue = 0; - domain_decomposer.copy_subdomain( - device_subdomain_buffer[current_buffer], 0, - subdomain_copy_direction::OriginalToSubdomain, current_queue); - - for (SIZE curr_subdomain_id = 0; - curr_subdomain_id < domain_decomposer.num_subdomains(); - curr_subdomain_id++) { - SIZE next_subdomain_id; - int next_buffer = (current_buffer + 1) % 2; - int next_queue = (current_queue + 1) % 3; - HierarchyType &hierarchy = Cache::cache.GetHierarchyCache( - domain_decomposer.subdomain_shape(curr_subdomain_id)); - log::info("Adapt Refactor to hierarchy"); - refactor.Adapt(hierarchy, config, current_queue); - - // Prefetch the next subdomain - if (curr_subdomain_id + 1 < domain_decomposer.num_subdomains()) { - next_subdomain_id = curr_subdomain_id + 1; - domain_decomposer.copy_subdomain( - device_subdomain_buffer[next_buffer], next_subdomain_id, - subdomain_copy_direction::OriginalToSubdomain, next_queue); - } - - std::stringstream ss; - for (DIM d = 0; d < D; d++) { - ss << hierarchy.level_shape(hierarchy.l_target(), d) << " "; - } - log::info("Refactoring subdomain " + std::to_string(curr_subdomain_id) + - " with shape: " + ss.str()); - - refactor.Refactor(device_subdomain_buffer[current_buffer], - refactored_metadata.metadata[curr_subdomain_id], - mdr_data[current_buffer], current_queue); - mdr_data[current_buffer].CopyToRefactoredData( - refactored_metadata.metadata[curr_subdomain_id], - refactored_data.data[curr_subdomain_id], current_queue); - - current_buffer = next_buffer; - current_queue = next_queue; - } - DeviceRuntime::SyncDevice(); - if (log::level & log::TIME) { - timer_series.end(); - timer_series.print("Refactor subdomain series with prefetch"); - timer_series.clear(); - } -} - -template -void reconstruct_pipeline( - DomainDecomposer &domain_decomposer, - Config &config, RefactoredMetadata &refactored_metadata, - RefactoredData &refactored_data, ReconstructedData &reconstructed_data) { - Timer timer_series; - if (log::level & log::TIME) - timer_series.start(); - - using Cache = ReconstructorCache; - using HierarchyType = typename ReconstructorType::HierarchyType; - - ReconstructorType &reconstructor = *Cache::cache.reconstructor; - Array *device_subdomain_buffer = - Cache::cache.device_subdomain_buffer; - MDRData *mdr_data = Cache::cache.mdr_data; - if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(0), - domain_decomposer.uniform)) { - Cache::cache.ClearHierarchyCache(); - } - for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { - if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(id), - domain_decomposer.uniform)) { - Cache::cache.InsertHierarchyCache( - domain_decomposer.subdomain_hierarchy(id)); - } - } - - log::info("Adjust device buffers"); - int current_buffer = 0; - int current_queue = 0; - // Prefetch the first subdomain - mdr_data[current_buffer].Resize(refactored_metadata.metadata[0]); - device_subdomain_buffer[current_buffer].resize( - domain_decomposer.subdomain_shape(0)); - mdr_data[current_buffer].CopyFromRefactoredData( - refactored_metadata.metadata[0], refactored_data.data[0], current_queue); - mdr_data[current_buffer].CopyFromRefactoredSigns( - refactored_metadata.metadata[0], refactored_data.level_signs[0], - current_queue); - - for (SIZE curr_subdomain_id = 0; - curr_subdomain_id < domain_decomposer.num_subdomains(); - curr_subdomain_id++) { - SIZE next_subdomain_id; - int next_buffer = (current_buffer + 1) % 2; - int next_queue = (current_queue + 1) % 3; - HierarchyType &hierarchy = Cache::cache.GetHierarchyCache( - domain_decomposer.subdomain_shape(curr_subdomain_id)); - log::info("Adapt Refactor to hierarchy"); - reconstructor.Adapt(hierarchy, config, current_queue); - - if (curr_subdomain_id + 1 < domain_decomposer.num_subdomains()) { - // Prefetch the next subdomain - next_subdomain_id = curr_subdomain_id + 1; - mdr_data[next_buffer].Resize( - refactored_metadata.metadata[next_subdomain_id]); - device_subdomain_buffer[next_buffer].resize( - domain_decomposer.subdomain_shape(next_subdomain_id)); - mdr_data[next_buffer].CopyFromRefactoredData( - refactored_metadata.metadata[next_subdomain_id], - refactored_data.data[next_subdomain_id], next_queue); - // Copy signs - mdr_data[next_buffer].CopyFromRefactoredSigns( - refactored_metadata.metadata[next_subdomain_id], - refactored_data.level_signs[next_subdomain_id], next_queue); - } - - mdr_data[current_buffer].CopyFromRefactoredData( - refactored_metadata.metadata[curr_subdomain_id], - refactored_data.data[curr_subdomain_id], current_queue); - mdr_data[current_buffer].CopyFromRefactoredSigns( - refactored_metadata.metadata[curr_subdomain_id], - refactored_data.level_signs[curr_subdomain_id], current_queue); - - std::stringstream ss; - for (DIM d = 0; d < D; d++) { - ss << hierarchy.level_shape(hierarchy.l_target(), d) << " "; - } - log::info("Reconstruct subdomain " + std::to_string(curr_subdomain_id) + - " with shape: " + ss.str()); - device_subdomain_buffer[current_buffer].resize( - hierarchy.level_shape(hierarchy.l_target())); - // Load previously reconstructred data - domain_decomposer.copy_subdomain( - device_subdomain_buffer[current_buffer], curr_subdomain_id, - subdomain_copy_direction::OriginalToSubdomain, current_queue); - // Reconstruct - reconstructor.ProgressiveReconstruct( - refactored_metadata.metadata[curr_subdomain_id], - mdr_data[current_buffer], config.mdr_adaptive_resolution, - device_subdomain_buffer[current_buffer], current_queue); - - // Update level signs for future progressive reconstruction - mdr_data[current_buffer].CopyToRefactoredSigns( - refactored_metadata.metadata[curr_subdomain_id], - refactored_data.level_signs[curr_subdomain_id], current_queue); - - // Update reconstructed data - domain_decomposer.copy_subdomain( - device_subdomain_buffer[current_buffer], curr_subdomain_id, - subdomain_copy_direction::SubdomainToOriginal, current_queue); - - if (config.mdr_adaptive_resolution) { - reconstructed_data.shape[curr_subdomain_id] = - device_subdomain_buffer[current_buffer].shape(); - reconstructed_data.offset[curr_subdomain_id] = - domain_decomposer.dim_subdomain_offset(curr_subdomain_id); - } - current_buffer = next_buffer; - current_queue = next_queue; - } - DeviceRuntime::SyncDevice(); - if (log::level & log::TIME) { - timer_series.end(); - timer_series.print("Reconstruct subdomain series with prefetch"); - timer_series.clear(); - } -} - template void load(Config &config, Metadata &metadata) { config.domain_decomposition = metadata.ddtype; @@ -275,7 +53,6 @@ void load(Config &config, Metadata &metadata) { config.lossless = metadata.ltype; config.huff_dict_size = metadata.huff_dict_size; config.huff_block_size = metadata.huff_block_size; - config.reorder = metadata.reorder; config.total_num_bitplanes = metadata.number_bitplanes; } @@ -311,14 +88,6 @@ void MDRefactor(std::vector shape, const void *original_data, using Cache = RefactorCache; Cache::cache.SafeInitialize(); - bool reduce_memory_footprint_original = - MemoryManager::ReduceMemoryFootprint; - if (MemoryManager::ReduceMemoryFootprint) { - log::info("Original ReduceMemoryFootprint: 1"); - } else { - log::info("Original ReduceMemoryFootprint: 0"); - } - DomainDecomposer, DeviceType> domain_decomposer; if (uniform) { @@ -332,10 +101,6 @@ void MDRefactor(std::vector shape, const void *original_data, } domain_decomposer.set_original_data((T *)original_data); - if (domain_decomposer.domain_decomposed()) { - MemoryManager::ReduceMemoryFootprint = true; - } - if (log::level & log::TIME) timer_each.start(); bool input_previously_pinned = @@ -347,7 +112,7 @@ void MDRefactor(std::vector shape, const void *original_data, } refactored_metadata.InitializeForRefactor(domain_decomposer.num_subdomains()); - refactored_data.InitializeForRefactor(domain_decomposer.num_subdomains()); + refactored_data.InitializeForRefactor(domain_decomposer, config); log::info("Output preallocated: " + std::to_string(output_pre_allocated)); log::info("Input previously pinned: " + @@ -359,22 +124,9 @@ void MDRefactor(std::vector shape, const void *original_data, timer_each.clear(); } - if (log::level & log::TIME) - timer_each.start(); - refactor_pipeline(domain_decomposer, config, refactored_metadata, refactored_data); - if (log::level & log::TIME) { - timer_each.end(); - timer_each.print("Aggregated low-level refactoring"); - log::time("Aggregated low-level refactoring throughput: " + - std::to_string((double)(total_num_elem * sizeof(T)) / - timer_each.get() / 1e9) + - " GB/s"); - timer_each.clear(); - } - if (log::level & log::TIME) timer_each.start(); Metadata m; @@ -408,24 +160,12 @@ void MDRefactor(std::vector shape, const void *original_data, Cache::cache.SafeRelease(); DeviceRuntime::Finalize(); - MemoryManager::ReduceMemoryFootprint = - reduce_memory_footprint_original; - if (MemoryManager::ReduceMemoryFootprint) { - log::info("ReduceMemoryFootprint restored to 1"); - } else { - log::info("ReduceMemoryFootprint restored to 0"); - } - if (log::level & log::TIME) { timer_each.end(); timer_each.print("Serialization"); timer_each.clear(); timer_total.end(); - timer_total.print("High-level refactoring"); - log::time("High-level refactoring throughput: " + - std::to_string((double)(total_num_elem * sizeof(T)) / - timer_total.get() / 1e9) + - " GB/s"); + timer_total.print("High-level refactoring", total_num_elem * sizeof(T)); timer_total.clear(); } } @@ -454,10 +194,9 @@ void MDRefactor(std::vector shape, const void *original_data, } template -void MDRequest(std::vector shape, - RefactoredMetadata &refactored_metadata) { +void MDRequest(std::vector shape, RefactoredMetadata &refactored_metadata, + Config config) { DeviceRuntime::Initialize(); - Config config; Metadata m; m.Deserialize((SERIALIZED_TYPE *)refactored_metadata.header.data()); load(config, m); @@ -514,21 +253,12 @@ void MDReconstruct(std::vector shape, if (log::level & log::TIME) timer_each.start(); - if (m.domain_decomposed) { - // Fast copy for domain decomposition need we disable pitched memory - // allocation - MemoryManager::ReduceMemoryFootprint = true; - } - using Cache = ReconstructorCache; - Cache::cache.SafeInitialize(); - bool reduce_memory_footprint_original = - MemoryManager::ReduceMemoryFootprint; - if (MemoryManager::ReduceMemoryFootprint) { - log::info("Original ReduceMemoryFootprint: 1"); - } else { - log::info("Original ReduceMemoryFootprint: 0"); + if (!config.mdr_qoi_mode) { + Cache::cache.SafeInitialize(3); + } else if (config.mdr_qoi_mode && !reconstructed_data.qoi_in_progress) { + Cache::cache.SafeInitialize(config.mdr_qoi_num_variables); } // Initialize DomainDecomposer @@ -548,29 +278,16 @@ void MDReconstruct(std::vector shape, m.domain_decomposed_size, config, coords); } if (!config.mdr_adaptive_resolution) { + // Should not re-allocate if the data is already allocated if (!reconstructed_data.IsInitialized()) { - // First time reconstruction - reconstructed_data.Initialize(1); - reconstructed_data.data[0] = (Byte *)malloc(total_num_elem * sizeof(T)); - memset(reconstructed_data.data[0], 0, total_num_elem * sizeof(T)); - reconstructed_data.offset[0] = std::vector(D, 0); - reconstructed_data.shape[0] = shape; + reconstructed_data.template ResizeToSingleDomain(shape); } domain_decomposer.set_original_data((T *)reconstructed_data.data[0]); } else { + // Should not re-allocate if the data is already allocated if (!reconstructed_data.IsInitialized()) { - // First time reconstruction - reconstructed_data.Initialize(domain_decomposer.num_subdomains()); - for (int subdomain_id = 0; - subdomain_id < domain_decomposer.num_subdomains(); subdomain_id++) { - SIZE n = 1; - for (int i = 0; - i < domain_decomposer.subdomain_shape(subdomain_id).size(); i++) { - n *= domain_decomposer.subdomain_shape(subdomain_id)[i]; - } - reconstructed_data.data[subdomain_id] = (Byte *)malloc(n * sizeof(T)); - memset(reconstructed_data.data[subdomain_id], 0, n * sizeof(T)); - } + reconstructed_data.template ResizeToMultipleSubdomains( + domain_decomposer); } std::vector decomposed_original_data( domain_decomposer.num_subdomains()); @@ -588,20 +305,12 @@ void MDReconstruct(std::vector shape, timer_each.clear(); } - if (log::level & log::TIME) - timer_each.start(); - - reconstruct_pipeline(domain_decomposer, config, refactored_metadata, - refactored_data, reconstructed_data); - - if (log::level & log::TIME) { - timer_each.end(); - timer_each.print("Aggregated low-level reconstruction"); - log::time("Aggregated low-level reconstruction throughput: " + - std::to_string((double)(total_num_elem * sizeof(T)) / - timer_each.get() / 1e9) + - " GB/s"); - timer_each.clear(); + if (config.mdr_qoi_mode) { + reconstruct_pipeline_qoi(domain_decomposer, config, refactored_metadata, + refactored_data, reconstructed_data); + } else { + reconstruct_pipeline(domain_decomposer, config, refactored_metadata, + refactored_data, reconstructed_data); } if (m.dstype == data_structure_type::Cartesian_Grid_Non_Uniform) { @@ -609,25 +318,15 @@ void MDReconstruct(std::vector shape, delete[] coords[d]; } - if (config.auto_cache_release) + if (config.auto_cache_release && + (!config.mdr_qoi_mode || !reconstructed_data.qoi_in_progress)) { Cache::cache.SafeRelease(); - DeviceRuntime::Finalize(); - - MemoryManager::ReduceMemoryFootprint = - reduce_memory_footprint_original; - if (MemoryManager::ReduceMemoryFootprint) { - log::info("ReduceMemoryFootprint restored to 1"); - } else { - log::info("ReduceMemoryFootprint restored to 0"); } + DeviceRuntime::Finalize(); if (log::level & log::TIME) { timer_total.end(); - timer_total.print("High-level reconstruction"); - log::time("High-level reconstruction throughput: " + - std::to_string((double)(total_num_elem * sizeof(T)) / - timer_total.get() / 1e9) + - " GB/s"); + timer_total.print("High-level reconstruction", total_num_elem * sizeof(T)); timer_total.clear(); } } @@ -644,221 +343,6 @@ void MDReconstruct(std::vector shape, output_pre_allocated); } -template -void MDRefactor(DIM D, data_type dtype, std::vector shape, - const void *original_data, - RefactoredMetadata &refactored_metadata, - RefactoredData &refactored_data, Config config, - bool output_pre_allocated) { - if (dtype == data_type::Float) { - if (D == 1) { - MDRefactor<1, float, DeviceType>(shape, original_data, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else if (D == 2) { - MDRefactor<2, float, DeviceType>(shape, original_data, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else if (D == 3) { - MDRefactor<3, float, DeviceType>(shape, original_data, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else if (D == 4) { - MDRefactor<4, float, DeviceType>(shape, original_data, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else if (D == 5) { - MDRefactor<5, float, DeviceType>(shape, original_data, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else { - log::err("do not support higher than five dimentions"); - exit(-1); - } - } else if (dtype == data_type::Double) { - if (D == 1) { - MDRefactor<1, double, DeviceType>(shape, original_data, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else if (D == 2) { - MDRefactor<2, double, DeviceType>(shape, original_data, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else if (D == 3) { - MDRefactor<3, double, DeviceType>(shape, original_data, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else if (D == 4) { - MDRefactor<4, double, DeviceType>(shape, original_data, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else if (D == 5) { - MDRefactor<5, double, DeviceType>(shape, original_data, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else { - log::err("do not support higher than five dimentions"); - exit(-1); - } - } else { - log::err("do not support types other than double and float!"); - exit(-1); - } -} - -template -void MDRefactor(DIM D, data_type dtype, std::vector shape, - const void *original_data, std::vector coords, - RefactoredMetadata &refactored_metadata, - RefactoredData &refactored_data, Config config, - bool output_pre_allocated) { - if (dtype == data_type::Float) { - std::vector float_coords; - for (auto &coord : coords) - float_coords.push_back((float *)coord); - if (D == 1) { - MDRefactor<1, float, DeviceType>(shape, original_data, float_coords, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else if (D == 2) { - MDRefactor<2, float, DeviceType>(shape, original_data, float_coords, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else if (D == 3) { - MDRefactor<3, float, DeviceType>(shape, original_data, float_coords, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else if (D == 4) { - MDRefactor<4, float, DeviceType>(shape, original_data, float_coords, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else if (D == 5) { - MDRefactor<5, float, DeviceType>(shape, original_data, float_coords, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else { - log::err("do not support higher than five dimentions"); - exit(-1); - } - } else if (dtype == data_type::Double) { - std::vector double_coords; - for (auto &coord : coords) - double_coords.push_back((double *)coord); - if (D == 1) { - MDRefactor<1, double, DeviceType>(shape, original_data, double_coords, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else if (D == 2) { - MDRefactor<2, double, DeviceType>(shape, original_data, double_coords, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else if (D == 3) { - MDRefactor<3, double, DeviceType>(shape, original_data, double_coords, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else if (D == 4) { - MDRefactor<4, double, DeviceType>(shape, original_data, double_coords, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else if (D == 5) { - MDRefactor<5, double, DeviceType>(shape, original_data, double_coords, - refactored_metadata, refactored_data, - config, output_pre_allocated); - } else { - log::err("do not support higher than five dimentions"); - exit(-1); - } - } else { - log::err("do not support types other than double and float!"); - exit(-1); - } -} - -template -void MDRequest(RefactoredMetadata &refactored_metadata) { - Metadata meta; - meta.Deserialize((SERIALIZED_TYPE *)refactored_metadata.header.data()); - - std::vector shape = std::vector(meta.total_dims); - for (DIM d = 0; d < shape.size(); d++) - shape[d] = (SIZE)meta.shape[d]; - data_type dtype = meta.dtype; - - if (dtype == data_type::Float) { - if (shape.size() == 1) { - MDRequest<1, float, DeviceType>(shape, refactored_metadata); - } else if (shape.size() == 2) { - MDRequest<2, float, DeviceType>(shape, refactored_metadata); - } else if (shape.size() == 3) { - MDRequest<3, float, DeviceType>(shape, refactored_metadata); - } else if (shape.size() == 4) { - MDRequest<4, float, DeviceType>(shape, refactored_metadata); - } else if (shape.size() == 5) { - MDRequest<5, float, DeviceType>(shape, refactored_metadata); - } else { - log::err("do not support higher than five dimentions"); - exit(-1); - } - } else if (dtype == data_type::Double) { - if (shape.size() == 1) { - MDRequest<1, double, DeviceType>(shape, refactored_metadata); - } else if (shape.size() == 2) { - MDRequest<2, double, DeviceType>(shape, refactored_metadata); - } else if (shape.size() == 3) { - MDRequest<3, double, DeviceType>(shape, refactored_metadata); - } else if (shape.size() == 4) { - MDRequest<4, double, DeviceType>(shape, refactored_metadata); - } else if (shape.size() == 5) { - MDRequest<5, double, DeviceType>(shape, refactored_metadata); - } else { - log::err("do not support higher than five dimentions"); - exit(-1); - } - } else { - log::err("do not support types other than double and float!"); - exit(-1); - } -} - -template -SIZE MDRMaxOutputDataSize(DIM D, data_type dtype, std::vector shape, - Config config) { - if (dtype == data_type::Float) { - if (shape.size() == 1) { - return MDRMaxOutputDataSize<1, float, DeviceType>(shape, config); - } else if (shape.size() == 2) { - return MDRMaxOutputDataSize<2, float, DeviceType>(shape, config); - } else if (shape.size() == 3) { - return MDRMaxOutputDataSize<3, float, DeviceType>(shape, config); - } else if (shape.size() == 4) { - return MDRMaxOutputDataSize<4, float, DeviceType>(shape, config); - } else if (shape.size() == 5) { - return MDRMaxOutputDataSize<5, float, DeviceType>(shape, config); - } else { - log::err("do not support higher than five dimentions"); - exit(-1); - } - } else if (dtype == data_type::Double) { - if (shape.size() == 1) { - return MDRMaxOutputDataSize<1, double, DeviceType>(shape, config); - } else if (shape.size() == 2) { - return MDRMaxOutputDataSize<2, double, DeviceType>(shape, config); - } else if (shape.size() == 3) { - return MDRMaxOutputDataSize<3, double, DeviceType>(shape, config); - } else if (shape.size() == 4) { - return MDRMaxOutputDataSize<4, double, DeviceType>(shape, config); - } else if (shape.size() == 5) { - return MDRMaxOutputDataSize<5, double, DeviceType>(shape, config); - } else { - log::err("do not support higher than five dimentions"); - exit(-1); - } - } else { - log::err("do not support types other than double and float!"); - exit(-1); - } -} - template void release_cache() { using Cache1 = RefactorCache>; @@ -868,89 +352,6 @@ template void release_cache() { Cache2::cache.SafeRelease(); } -template void release_cache() { - release_cache<1, T, DeviceType>(); - release_cache<2, T, DeviceType>(); - release_cache<3, T, DeviceType>(); - release_cache<4, T, DeviceType>(); - release_cache<5, T, DeviceType>(); -} - -template void release_cache() { - release_cache(); - release_cache(); -} - -template -void MDReconstruct(RefactoredMetadata &refactored_metadata, - RefactoredData &refactored_data, - ReconstructedData &reconstructed_data, Config config, - bool output_pre_allocated) { - - Metadata meta; - meta.Deserialize((SERIALIZED_TYPE *)refactored_metadata.header.data()); - - std::vector shape = std::vector(meta.total_dims); - for (DIM d = 0; d < shape.size(); d++) - shape[d] = (SIZE)meta.shape[d]; - data_type dtype = meta.dtype; - - if (dtype == data_type::Float) { - if (shape.size() == 1) { - MDReconstruct<1, float, DeviceType>(shape, refactored_metadata, - refactored_data, reconstructed_data, - config, output_pre_allocated); - } else if (shape.size() == 2) { - MDReconstruct<2, float, DeviceType>(shape, refactored_metadata, - refactored_data, reconstructed_data, - config, output_pre_allocated); - } else if (shape.size() == 3) { - MDReconstruct<3, float, DeviceType>(shape, refactored_metadata, - refactored_data, reconstructed_data, - config, output_pre_allocated); - } else if (shape.size() == 4) { - MDReconstruct<4, float, DeviceType>(shape, refactored_metadata, - refactored_data, reconstructed_data, - config, output_pre_allocated); - } else if (shape.size() == 5) { - MDReconstruct<5, float, DeviceType>(shape, refactored_metadata, - refactored_data, reconstructed_data, - config, output_pre_allocated); - } else { - log::err("do not support higher than five dimentions"); - exit(-1); - } - } else if (dtype == data_type::Double) { - if (shape.size() == 1) { - MDReconstruct<1, double, DeviceType>(shape, refactored_metadata, - refactored_data, reconstructed_data, - config, output_pre_allocated); - } else if (shape.size() == 2) { - MDReconstruct<2, double, DeviceType>(shape, refactored_metadata, - refactored_data, reconstructed_data, - config, output_pre_allocated); - } else if (shape.size() == 3) { - MDReconstruct<3, double, DeviceType>(shape, refactored_metadata, - refactored_data, reconstructed_data, - config, output_pre_allocated); - } else if (shape.size() == 4) { - MDReconstruct<4, double, DeviceType>(shape, refactored_metadata, - refactored_data, reconstructed_data, - config, output_pre_allocated); - } else if (shape.size() == 5) { - MDReconstruct<5, double, DeviceType>(shape, refactored_metadata, - refactored_data, reconstructed_data, - config, output_pre_allocated); - } else { - log::err("do not support higher than five dimentions"); - exit(-1); - } - } else { - log::err("do not support types other than double and float!"); - exit(-1); - } -} - } // namespace MDR } // namespace mgard_x diff --git a/include/mgard-x/MDRHighLevel/MaxAbsIndexKernel.hpp b/include/mgard-x/MDRHighLevel/MaxAbsIndexKernel.hpp new file mode 100644 index 0000000000..751ec8a18b --- /dev/null +++ b/include/mgard-x/MDRHighLevel/MaxAbsIndexKernel.hpp @@ -0,0 +1,109 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../Hierarchy/Hierarchy.h" +#include "../RuntimeX/RuntimeX.h" +#include + +#ifndef MGARD_X_MaxAbsIndexKernel +#define MGARD_X_MaxAbsIndexKernel + +namespace mgard_x { + +namespace data_refactoring { + +namespace multi_dimension { + +template +class MaxAbsIndexFunctor : public Functor { +public: + MGARDX_CONT MaxAbsIndexFunctor() {} + MGARDX_CONT MaxAbsIndexFunctor(SubArray<1, double, DeviceType> x, + SubArray<1, double, DeviceType> maxabs, + SubArray<1, uint32_t, DeviceType> index) + : x(x), maxabs(maxabs), index(index) { + Functor(); + n = x.shape(0); + } + + MGARDX_EXEC void Operation1() { + int tid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + int stride = FunctorBase::GetBlockDimX() * + FunctorBase::GetGridDimX(); + + for (int i = tid; i < n; i += stride) { + double value = (double)*x(i); + if (value == (double)(*maxabs(0))) + *index(0) = (uint32_t)i; + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + SubArray<1, T, DeviceType> x; + SubArray<1, double, DeviceType> maxabs; + SubArray<1, uint32_t, DeviceType> index; + uint32_t n; +}; + +template +class MaxAbsIndexKernel : public Kernel { +public: + constexpr static std::string_view Name = "max abs index kernel"; + constexpr static double EnableAutoTuning() { return false; } + MGARDX_CONT + MaxAbsIndexKernel(SubArray<1, double, DeviceType> x, + SubArray<1, double, DeviceType> maxabs, + SubArray<1, uint32_t, DeviceType> index) + : x(x), maxabs(maxabs), index(index) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = MaxAbsIndexFunctor; + FunctorType functor(x, maxabs, index); + SIZE total_thread_x = x.shape(0); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = ceil((double)total_thread_x / tbx); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray x; + SubArray<1, double, DeviceType> maxabs; + SubArray<1, uint32_t, DeviceType> index; +}; + +template +void Copy3D(SubArray x, + SubArray<1, double, DeviceType> maxabs, + SubArray<1, uint32_t, DeviceType> out, int queue_idx) { + + DeviceLauncher::Execute( + MaxAbsIndexKernel(x, maxabs, out), queue_idx); +} + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x + +#endif \ No newline at end of file diff --git a/include/mgard-x/MDRHighLevel/QoIKernel.hpp b/include/mgard-x/MDRHighLevel/QoIKernel.hpp new file mode 100644 index 0000000000..73edd72d4d --- /dev/null +++ b/include/mgard-x/MDRHighLevel/QoIKernel.hpp @@ -0,0 +1,148 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../Hierarchy/Hierarchy.h" +#include "../RuntimeX/RuntimeX.h" +#include + +#ifndef MGARD_X_QoIKernel +#define MGARD_X_QoIKernel + +namespace mgard_x { + +namespace data_refactoring { + +namespace multi_dimension { + +template +class QoIFunctor : public Functor { +public: + MGARDX_CONT QoIFunctor() {} + MGARDX_CONT + QoIFunctor(SubArray x, SubArray y, + SubArray z, SubArray out, + double eb_x, double eb_y, double eb_z, double tolerance) + : x(x), y(y), z(z), out(out), eb_x(eb_x), eb_y(eb_y), eb_z(eb_z), + tolerance(tolerance) { + Functor(); + n = x.shape(0) * x.shape(1) * x.shape(2); + } + + MGARDX_EXEC T compute_bound_x_square(T x, T eb) { + return 2 * fabs(x) * eb + eb * eb; + } + + MGARDX_EXEC T compute_bound_square_root_x(T x, T eb) { + if (x == 0) { + return sqrt(eb); + } + if (x > eb) { + return eb / (sqrt(x - eb) + sqrt(x)); + } else { + return eb / sqrt(x); + } + } + + MGARDX_EXEC void Operation1() { + int tid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + int stride = FunctorBase::GetBlockDimX() * + FunctorBase::GetGridDimX(); + + for (int i = tid; i < n; i += stride) { + double Vx = (double)*x(i); + double Vy = (double)*y(i); + double Vz = (double)*z(i); + // if (check_flag(tolerance_exceed_flag)) return; + double e_V_TOT_2 = compute_bound_x_square(Vx, eb_x) + + compute_bound_x_square(Vy, eb_y) + + compute_bound_x_square(Vz, eb_z); + double V_TOT_2 = Vx * Vx + Vy * Vy + Vz * Vz; + double e_V_TOT = compute_bound_square_root_x(V_TOT_2, e_V_TOT_2); + *out(i) = e_V_TOT; + // double V_TOT = sqrt(V_TOT_2); + // if (e_V_TOT > tolerance){ + // raise_flag(tolerance_exceed_flag); + // return; + // } + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + SubArray x; + SubArray y; + SubArray z; + SubArray out; + double eb_x; + double eb_y; + double eb_z; + double tolerance; + uint32_t n; +}; + +template +class QoIKernel : public Kernel { +public: + constexpr static std::string_view Name = "qoi kernel"; + constexpr static double EnableAutoTuning() { return false; } + MGARDX_CONT + QoIKernel(SubArray x, SubArray y, + SubArray z, SubArray out, + double eb_x, double eb_y, double eb_z, double tolerance) + : x(x), y(y), z(z), out(out), eb_x(eb_x), eb_y(eb_y), eb_z(eb_z), + tolerance(tolerance) {} + + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = QoIFunctor; + FunctorType functor(x, y, z, out, eb_x, eb_y, eb_z, tolerance); + SIZE total_thread_x = x.shape(0) * x.shape(1) * x.shape(2); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = ceil((double)total_thread_x / tbx); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray x; + SubArray y; + SubArray z; + SubArray out; + double eb_x; + double eb_y; + double eb_z; + double tolerance; +}; + +template +void Copy3D(SubArray x, SubArray y, + SubArray z, SubArray out, + int queue_idx) { + + DeviceLauncher::Execute(QoIKernel(x, y, z, out), + queue_idx); +} + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x + +#endif \ No newline at end of file diff --git a/include/mgard-x/MDRHighLevel/ReconstructPipeline.hpp b/include/mgard-x/MDRHighLevel/ReconstructPipeline.hpp new file mode 100644 index 0000000000..5e21d8d5c9 --- /dev/null +++ b/include/mgard-x/MDRHighLevel/ReconstructPipeline.hpp @@ -0,0 +1,189 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_MDR_RECONSTRUCT_PIPELINE_HPP +#define MGARD_X_MDR_RECONSTRUCT_PIPELINE_HPP + +namespace mgard_x { +namespace MDR { + +template +void reconstruct_pipeline( + DomainDecomposer &domain_decomposer, + Config &config, RefactoredMetadata &refactored_metadata, + RefactoredData &refactored_data, ReconstructedData &reconstructed_data) { + + using Cache = ReconstructorCache; + using HierarchyType = typename ReconstructorType::HierarchyType; + + bool timing_pipeline = true; + + ReconstructorType &reconstructor = *Cache::cache.reconstructor; + Array *device_subdomain_buffer = + Cache::cache.device_subdomain_buffer; + MDRData *mdr_data = Cache::cache.mdr_data; + if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(0), + domain_decomposer.uniform)) { + Cache::cache.ClearHierarchyCache(); + } + for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { + if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(id), + domain_decomposer.uniform)) { + Cache::cache.InsertHierarchyCache( + domain_decomposer.subdomain_hierarchy(id)); + } + } + + HierarchyType &hierarchy = + Cache::cache.GetHierarchyCache(domain_decomposer.subdomain_shape(0)); + + log::info("Adjust device buffers"); + mdr_data[0].Resize(reconstructor, hierarchy, 0); + mdr_data[1].Resize(reconstructor, hierarchy, 0); + mdr_data[2].Resize(reconstructor, hierarchy, 0); + device_subdomain_buffer[0].resize(domain_decomposer.subdomain_shape(0), 0); + device_subdomain_buffer[1].resize(domain_decomposer.subdomain_shape(0), 0); + device_subdomain_buffer[2].resize(domain_decomposer.subdomain_shape(0), 0); + + Timer timer_series; + if (timing_pipeline) + timer_series.start(); + // Prefetch the first subdomain + int current_buffer = 0; + int current_queue = 0; + mdr_data[current_buffer].Resize(refactored_metadata.metadata[0], + current_queue); + device_subdomain_buffer[current_buffer].resize( + domain_decomposer.subdomain_shape(0), current_queue); + mdr_data[current_buffer].CopyFromRefactoredData( + refactored_metadata.metadata[0], refactored_data.data[0], current_queue); + mdr_data[current_buffer].CopyFromRefactoredSigns( + refactored_metadata.metadata[0], refactored_data.level_signs[0], + current_queue); + // Load previously reconstructred data + domain_decomposer.copy_subdomain( + device_subdomain_buffer[current_buffer], 0, + subdomain_copy_direction::OriginalToSubdomain, current_queue); + + SIZE total_size = 0; + + for (SIZE curr_subdomain_id = 0; + curr_subdomain_id < domain_decomposer.num_subdomains(); + curr_subdomain_id++) { + SIZE next_subdomain_id; + int next_buffer = (current_buffer + 1) % 3; + int next_queue = (current_queue + 1) % 3; + HierarchyType &hierarchy = Cache::cache.GetHierarchyCache( + domain_decomposer.subdomain_shape(curr_subdomain_id)); + log::info("Adapt Refactor to hierarchy"); + reconstructor.Adapt(hierarchy, config, current_queue); + total_size += hierarchy.total_num_elems() * sizeof(T); + if (curr_subdomain_id + 1 < domain_decomposer.num_subdomains()) { + // Prefetch the next subdomain + next_subdomain_id = curr_subdomain_id + 1; + mdr_data[next_buffer].Resize( + refactored_metadata.metadata[next_subdomain_id], next_queue); + device_subdomain_buffer[next_buffer].resize( + domain_decomposer.subdomain_shape(next_subdomain_id), next_queue); + + mdr_data[next_buffer].CopyFromRefactoredData( + refactored_metadata.metadata[next_subdomain_id], + refactored_data.data[next_subdomain_id], next_queue); + // Copy signs + mdr_data[next_buffer].CopyFromRefactoredSigns( + refactored_metadata.metadata[next_subdomain_id], + refactored_data.level_signs[next_subdomain_id], next_queue); + // Load previously reconstructred data + domain_decomposer.copy_subdomain( + device_subdomain_buffer[next_buffer], next_subdomain_id, + subdomain_copy_direction::OriginalToSubdomain, next_queue); + } + + std::stringstream ss; + for (DIM d = 0; d < D; d++) { + ss << hierarchy.level_shape(hierarchy.l_target(), d) << " "; + } + log::info("Reconstruct subdomain " + std::to_string(curr_subdomain_id) + + " with shape: " + ss.str()); + + reconstructor.LoadMetadata(refactored_metadata.metadata[curr_subdomain_id], + mdr_data[current_buffer], current_queue); + reconstructor.Decompress(refactored_metadata.metadata[curr_subdomain_id], + mdr_data[current_buffer], current_queue); + + if (curr_subdomain_id > 0) { + // We delay D2H since since it can delay the D2H in lossless decompession + // and dequantization + int previous_buffer = std::abs((current_buffer + 3 - 1) % 3); + int previous_queue = std::abs((current_queue + 3 - 1) % 3); + SIZE prev_subdomain_id = curr_subdomain_id - 1; + // Update level signs for future progressive reconstruction + mdr_data[previous_buffer].CopyToRefactoredSigns( + refactored_metadata.metadata[prev_subdomain_id], + refactored_data.level_signs[prev_subdomain_id], previous_queue); + + // Update reconstructed data + domain_decomposer.copy_subdomain( + device_subdomain_buffer[previous_buffer], prev_subdomain_id, + subdomain_copy_direction::SubdomainToOriginal, previous_queue); + } + + // Reconstruct + reconstructor.ProgressiveReconstruct( + refactored_metadata.metadata[curr_subdomain_id], + mdr_data[current_buffer], config.mdr_adaptive_resolution, + device_subdomain_buffer[current_buffer], current_queue); + + // Need to ensure reconstruction is complete before next reconstruction + DeviceRuntime::SyncQueue(current_queue); + + // // Update level signs for future progressive reconstruction + // mdr_data[current_buffer].CopyToRefactoredSigns( + // refactored_metadata.metadata[curr_subdomain_id], + // refactored_data.level_signs[curr_subdomain_id], current_queue); + + // // Update reconstructed data + // domain_decomposer.copy_subdomain( + // device_subdomain_buffer[current_buffer], curr_subdomain_id, + // subdomain_copy_direction::SubdomainToOriginal, current_queue); + + if (config.mdr_adaptive_resolution) { + reconstructed_data.shape[curr_subdomain_id] = + device_subdomain_buffer[current_buffer].shape(); + reconstructed_data.offset[curr_subdomain_id] = + domain_decomposer.dim_subdomain_offset(curr_subdomain_id); + } + current_buffer = next_buffer; + current_queue = next_queue; + } + + // Copy the last subdomain + int previous_buffer = std::abs((current_buffer - 1) % 3); + int previous_queue = std::abs((current_queue - 1) % 3); + SIZE prev_subdomain_id = domain_decomposer.num_subdomains() - 1; + // Update level signs for future progressive reconstruction + mdr_data[previous_buffer].CopyToRefactoredSigns( + refactored_metadata.metadata[prev_subdomain_id], + refactored_data.level_signs[prev_subdomain_id], previous_queue); + + // Update reconstructed data + domain_decomposer.copy_subdomain( + device_subdomain_buffer[previous_buffer], prev_subdomain_id, + subdomain_copy_direction::SubdomainToOriginal, previous_queue); + + DeviceRuntime::SyncDevice(); + if (timing_pipeline) { + timer_series.end(); + // log::csv("time.csv", timer_series.get()); + timer_series.print("Reconstruct pipeline", total_size, true); + timer_series.clear(); + } +} + +} // namespace MDR +} // namespace mgard_x +#endif \ No newline at end of file diff --git a/include/mgard-x/MDRHighLevel/ReconstructPipelineQoI.hpp b/include/mgard-x/MDRHighLevel/ReconstructPipelineQoI.hpp new file mode 100644 index 0000000000..ad56ed2086 --- /dev/null +++ b/include/mgard-x/MDRHighLevel/ReconstructPipelineQoI.hpp @@ -0,0 +1,456 @@ +#ifndef MGARD_X_MDR_RECONSTRUCT_PIPELINE_QOI_HPP +#define MGARD_X_MDR_RECONSTRUCT_PIPELINE_QOI_HPP + +#include +#include +#include +#include +#include + +#include "mgard-x/Config/Config.h" +#include "mgard-x/MDRHighLevel/MDRDataHighLevel.hpp" +#include "mgard-x/MDRHighLevel/MDRHighLevel.hpp" +#include "mgard-x/MDRHighLevel/MaxAbsIndexKernel.hpp" +#include "mgard-x/MDRHighLevel/QoIKernel.hpp" + +namespace mgard_x { +namespace MDR { + +// f(x) = x^2 +template inline double compute_bound_x_square(T x, T eb) { + return 2 * fabs(x) * eb + eb * eb; +} + +// f(x) = sqrt(x) +template inline double compute_bound_square_root_x(T x, T eb) { + if (x == 0) { + return sqrt(eb); + } + if (x > eb) { + return eb / (sqrt(x - eb) + sqrt(x)); + } else { + return eb / sqrt(x); + } +} + +template +inline void error_bound_uniform_decrease(T vx, T vy, T vz, double tau, + double max_error, + std::vector &ebs) { + double V_TOT_2 = vx * vx + vy * vy + vz * vz; + double estimate_error = max_error; + double eb_vx = ebs[0]; + double eb_vy = ebs[1]; + double eb_vz = ebs[2]; + { + double e_V_TOT_2 = compute_bound_x_square((double)vx, eb_vx) + + compute_bound_x_square((double)vy, eb_vy) + + compute_bound_x_square((double)vz, eb_vz); + estimate_error = compute_bound_square_root_x(V_TOT_2, e_V_TOT_2); + // std::cout << "validation of max error = " << estimate_error << std::endl; + } + while (estimate_error > tau) { + eb_vx = eb_vx / 1.5; + eb_vy = eb_vy / 1.5; + eb_vz = eb_vz / 1.5; + double e_V_TOT_2 = compute_bound_x_square((double)vx, eb_vx) + + compute_bound_x_square((double)vy, eb_vy) + + compute_bound_x_square((double)vz, eb_vz); + estimate_error = compute_bound_square_root_x(V_TOT_2, e_V_TOT_2); + } + ebs[0] = eb_vx; + ebs[1] = eb_vy; + ebs[2] = eb_vz; + return; +} + +template +void reconstruct_pipeline_qoi( + DomainDecomposer &domain_decomposer, + Config &config, RefactoredMetadata &refactored_metadata, + RefactoredData &refactored_data, ReconstructedData &reconstructed_data) { + Timer timer_series, qoi_timer; + + using Cache = ReconstructorCache; + using HierarchyType = typename ReconstructorType::HierarchyType; + + ReconstructorType &reconstructor = *Cache::cache.reconstructor; + Array *device_subdomain_buffer = + Cache::cache.device_subdomain_buffer; + MDRData *mdr_data = Cache::cache.mdr_data; + + Array error_out(domain_decomposer.subdomain_shape(0)); + Array<1, double, DeviceType> error_final_out({1}); + Array<1, uint32_t, DeviceType> max_index_d({1}); + Array<1, Byte, DeviceType> workspace; + + for (int i = 0; i < 2; i++) { + error_final_out.resize({1}, i); + DeviceCollective::AbsMax( + domain_decomposer.subdomain_shape(0)[0] * + domain_decomposer.subdomain_shape(0)[1] * + domain_decomposer.subdomain_shape(0)[2], + SubArray<1, double, DeviceType>(), SubArray<1, double, DeviceType>(), + workspace, false, 0); + } + + if (config.mdr_qoi_num_variables != domain_decomposer.num_subdomains()) { + throw std::runtime_error( + "QOI mode requires the number of variables to be equal to the " + "number of subdomains"); + } + + log::info("Adjust device buffers"); + if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(0), + domain_decomposer.uniform)) { + Cache::cache.ClearHierarchyCache(); + } + for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { + if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(id), + domain_decomposer.uniform)) { + Cache::cache.InsertHierarchyCache( + domain_decomposer.subdomain_hierarchy(id)); + } + mdr_data[id].Resize(refactored_metadata.metadata[id], 0); + device_subdomain_buffer[id].resize(domain_decomposer.subdomain_shape(id), + 0); + // Reset all signs to 0 for the initial QOI reconstruction + if (!reconstructed_data.qoi_in_progress) { + mdr_data[id].ResetSigns(0); + } + } + + HierarchyType &hierarchy = + Cache::cache.GetHierarchyCache(domain_decomposer.subdomain_shape(0)); + reconstructor.Adapt(hierarchy, config, 0); + + int current_buffer = 0; + int current_queue = 0; + + DeviceRuntime::SyncDevice(); + timer_series.start(); + // Prefetch the first subdomain + mdr_data[current_buffer].CopyFromRefactoredData( + refactored_metadata.metadata[0], refactored_data.data[0], current_queue); + + SIZE total_size = 0; + uint32_t max_iter = 500; + uint32_t iter = 0; + double error_final_out_host; + int buffer_for_variable[3]; + std::vector ebs(3); + std::vector last_ebs(3); + double last_maximal_error; + double tol = refactored_metadata.metadata[0].tau; + + reconstructed_data.qoi_in_progress = true; + + while ((reconstructed_data.qoi_in_progress) && (iter < max_iter)) { + iter++; + // std::cout << "======= Iteration " << iter << " =======" << std::endl; + for (SIZE curr_subdomain_id = 0; + curr_subdomain_id < domain_decomposer.num_subdomains(); + curr_subdomain_id++) { + + SIZE next_subdomain_id; + int next_buffer = + (current_buffer + 1) % domain_decomposer.num_subdomains(); + int next_queue = (current_queue + 1) % domain_decomposer.num_subdomains(); + HierarchyType &hierarchy = Cache::cache.GetHierarchyCache( + domain_decomposer.subdomain_shape(curr_subdomain_id)); + log::info("Adapt Refactor to hierarchy"); + reconstructor.Adapt(hierarchy, config, current_queue); + if (iter == 1) + total_size += hierarchy.total_num_elems() * sizeof(T); + reconstructor.LoadMetadata( + refactored_metadata.metadata[curr_subdomain_id], + mdr_data[current_buffer], current_queue); + reconstructor.Decompress(refactored_metadata.metadata[curr_subdomain_id], + mdr_data[current_buffer], current_queue); + if (curr_subdomain_id + 1 < domain_decomposer.num_subdomains()) { + // Prefetch the next subdomain + next_subdomain_id = curr_subdomain_id + 1; + mdr_data[next_buffer].CopyFromRefactoredData( + refactored_metadata.metadata[next_subdomain_id], + refactored_data.data[next_subdomain_id], next_queue); + } + + if (curr_subdomain_id == config.mdr_qoi_num_variables - 1) { + // We are about to finish reconstructing all variables + // so, we need to fetch more data + // + // We need to update the metadata for all variables + if (refactored_metadata.decrease_method) { + ebs[0] = refactored_metadata.metadata[0].corresponding_error; + ebs[1] = refactored_metadata.metadata[1].corresponding_error; + ebs[2] = refactored_metadata.metadata[2].corresponding_error; + } else { + ebs[0] = refactored_metadata.metadata[0].requested_tol; + ebs[1] = refactored_metadata.metadata[1].requested_tol; + ebs[2] = refactored_metadata.metadata[2].requested_tol; + } + // std::cout << "current ebs : "; + // for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { + // if (refactored_metadata.decrease_method) std::cout << + // refactored_metadata.metadata[id].corresponding_error << ", "; else + // std::cout << refactored_metadata.metadata[id].requested_tol << ", + // "; + // // refactored_metadata.metadata[id].requested_size = + // usr_def_requested_size; //new tolerance + + // } + // std::cout << std::endl; + + // for (auto &metadata : refactored_metadata.metadata) { + // metadata.PrintStatus(); + // } + // size_t size_read = read_mdr(refactored_metadata, refactored_data, + // "/home/linusli037/Polaris/MGARD/build-cuda-turing/mgard/miniNYX/XYZ", + // false, config); refactored_metadata.total_size += size_read;***** + // initiate the bitplane transfer for the 1st variable which + // should coorespond to the next_buffer + // mdr_data[0].CopyFromRefactoredData( + // refactored_metadata.metadata[0], + // refactored_data.data[0], next_queue); + } + + std::stringstream ss; + for (DIM d = 0; d < D; d++) { + ss << hierarchy.level_shape(hierarchy.l_target(), d) << " "; + } + log::info("Reconstruct subdomain " + std::to_string(curr_subdomain_id) + + " with shape: " + ss.str()); + + // Reconstruct + + reconstructor.ProgressiveReconstruct( + refactored_metadata.metadata[curr_subdomain_id], + mdr_data[current_buffer], config.mdr_adaptive_resolution, + device_subdomain_buffer[current_buffer], current_queue); + + if (curr_subdomain_id == config.mdr_qoi_num_variables - 1) { + + // DeviceRuntime::SyncQueue(current_queue); + + // for (int q = 0; q < 2; q++) { + // DeviceRuntime::SyncQueue(q); + // } + + // We are done with reconstructing all variables now + // Do error estimation here + // Var0 can be accessed from device_subdomain_buffer[0].data() + // Var1 can be accessed from device_subdomain_buffer[1].data() + // Var2 can be accessed from device_subdomain_buffer[2].data() + + // if (tol NOT met) { + // need to contine reconstructing. Device buffers will NOT be + // released reconstructed_data.qoi_in_progress = true; + // } else { + // will stop reconstructing. Device buffers will be released + // reconstructed_data.qoi_in_progress = false; + // } + // we set it true for testing only + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(current_queue); + qoi_timer.start(); + } + DeviceLauncher::Execute( + mgard_x::data_refactoring::multi_dimension::QoIKernel( + SubArray(device_subdomain_buffer[0]), + SubArray(device_subdomain_buffer[1]), + SubArray(device_subdomain_buffer[2]), SubArray(error_out), + ebs[0], ebs[1], ebs[2], tol), + current_queue); + SubArray<1, double, DeviceType> out_1d( + {device_subdomain_buffer[0].shape(0) * + device_subdomain_buffer[0].shape(1) * + device_subdomain_buffer[0].shape(2)}, + error_out.data()); + // std::vector + // out_vec(refactored_metadata.metadata[0].num_elements); std::cout << + // "num_elements = " << refactored_metadata.metadata[0].num_elements << + // std::endl; std::cout << "out_vec.data() = " << out_vec.data() << + // std::endl; MemoryManager::Copy1D(out_vec.data(), + // out_1d.data(), refactored_metadata.metadata[0].num_elements, + // current_queue); + // std::cout << "max est error = " << *std::max_element(out_vec.begin(), + // out_vec.end()) << std::endl; + DeviceCollective::AbsMax( + device_subdomain_buffer[0].shape(0) * + device_subdomain_buffer[0].shape(1) * + device_subdomain_buffer[0].shape(2), + out_1d, SubArray(error_final_out), workspace, true, current_queue); + if (log::level || log::TIME) { + DeviceRuntime::SyncQueue(current_queue); + qoi_timer.end(); + qoi_timer.print("QoI error estimation: ", total_size / 3); + qoi_timer.clear(); + } + MemoryManager::Copy1D( + &error_final_out_host, error_final_out.data(), 1, current_queue); + DeviceRuntime::SyncQueue(current_queue); + // reconstructed_data.qoi_in_progress = error_final_out_host ? true : + // false; std::cout << "==== maximal est error = " << + // error_final_out_host << " ====" << std::endl; + reconstructed_data.qoi_in_progress = + (error_final_out_host > tol) ? true : false; + if (reconstructed_data.qoi_in_progress) { + // CPU version + if (refactored_metadata.decrease_method == 0) { + DeviceLauncher::Execute( + mgard_x::data_refactoring::multi_dimension::MaxAbsIndexKernel< + 1, double, DeviceType>(out_1d, SubArray(error_final_out), + SubArray(max_index_d)), + current_queue); + uint32_t max_index_h; + MemoryManager::Copy1D(&max_index_h, max_index_d.data(), + 1, current_queue); + DeviceRuntime::SyncQueue(current_queue); + std::vector new_ebs = ebs; + + T vx, vy, vz; + + T *vx_ptr = device_subdomain_buffer[0].data(); + T *vy_ptr = device_subdomain_buffer[1].data(); + T *vz_ptr = device_subdomain_buffer[2].data(); + + MemoryManager::Copy1D(&vx, &vx_ptr[max_index_h], 1, + current_queue); + MemoryManager::Copy1D(&vy, &vy_ptr[max_index_h], 1, + current_queue); + MemoryManager::Copy1D(&vz, &vz_ptr[max_index_h], 1, + current_queue); + + error_bound_uniform_decrease(vx, vy, vz, tol, + error_final_out_host, new_ebs); + + // std::cout << "new ebs : "; + for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { + refactored_metadata.metadata[id].requested_tol = new_ebs[id]; + // std::cout << refactored_metadata.metadata[id].requested_tol << + // ", "; + reconstructor.GenerateRequest(refactored_metadata.metadata[id]); + } + // std::cout << std::endl; + } else if (refactored_metadata.decrease_method == 1) { + // Segmented + for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { + reconstructor.GenerateRequest(refactored_metadata.metadata[id]); + } + } else if (refactored_metadata.decrease_method == 2) { + // Hybrid Threshold = 2 + if (error_final_out_host / tol > 2 && + (refactored_metadata.metadata[0].corresponding_error_return)) { + // std::cout << "new ebs : "; + for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { + refactored_metadata.metadata[id].requested_tol = std::max( + refactored_metadata.metadata[id].corresponding_error / 4, + tol / error_final_out_host * + refactored_metadata.metadata[id].corresponding_error); + // std::cout << refactored_metadata.metadata[id].requested_tol + // << ", "; + reconstructor.GenerateRequest(refactored_metadata.metadata[id]); + } + // std::cout << std::endl; + } else { + for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { + if (refactored_metadata.metadata[id] + .corresponding_error_return) { + // std::cout << "Switch to Segmented ..." << std::endl; + refactored_metadata.metadata[id].corresponding_error_return = + false; + refactored_metadata.metadata[id].segmented = true; + refactored_metadata.metadata[id].requested_size = 1; + } + reconstructor.GenerateRequest(refactored_metadata.metadata[id]); + } + } + } else if (refactored_metadata.decrease_method >= 3) { + // Hybrid Threshold = 10 with relative or uniform value range eb + if (error_final_out_host / tol > 10 && + (refactored_metadata.metadata[0].corresponding_error_return)) { + // std::cout << "new ebs : "; + for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { + refactored_metadata.metadata[id].requested_tol = std::max( + refactored_metadata.metadata[id].corresponding_error / 4, + tol / error_final_out_host * + refactored_metadata.metadata[id].corresponding_error); + // std::cout << refactored_metadata.metadata[id].requested_tol + // << ", "; + reconstructor.GenerateRequest(refactored_metadata.metadata[id]); + } + // std::cout << std::endl; + } else { + for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { + if (refactored_metadata.metadata[id] + .corresponding_error_return) { + // std::cout << "Switch to Segmented ..." << std::endl; + refactored_metadata.metadata[id].corresponding_error_return = + false; + refactored_metadata.metadata[id].segmented = true; + refactored_metadata.metadata[id].requested_size = 1; + } + reconstructor.GenerateRequest(refactored_metadata.metadata[id]); + } + } + // IO_timer.start(); + // size_t size_read = read_mdrx(refactored_metadata, + // refactored_data, false, config); IO_timer.end(); + // refactored_metadata.IO_time += IO_timer.get(); + } + + mdr_data[0].CopyFromRefactoredData(refactored_metadata.metadata[0], + refactored_data.data[0], + next_queue); + } + last_maximal_error = error_final_out_host; + // std::cout << "reconstructed_data.qoi_in_progress = " << + // reconstructed_data.qoi_in_progress << std::endl; + } + + DeviceRuntime::SyncQueue(current_queue); + + current_buffer = next_buffer; + current_queue = next_queue; + last_ebs[0] = ebs[0]; + last_ebs[1] = ebs[1]; + last_ebs[2] = ebs[2]; + } + } + + refactored_metadata.metadata[0].requested_tol = ebs[0]; + refactored_metadata.metadata[1].requested_tol = ebs[1]; + refactored_metadata.metadata[2].requested_tol = ebs[2]; + // Copy final data out if we are done with reconstructing + for (SIZE curr_subdomain_id = 0; + curr_subdomain_id < domain_decomposer.num_subdomains(); + curr_subdomain_id++) { + // Update reconstructed data + domain_decomposer.copy_subdomain( + device_subdomain_buffer[curr_subdomain_id], curr_subdomain_id, + subdomain_copy_direction::SubdomainToOriginal, current_queue); + } + + DeviceRuntime::SyncDevice(); + timer_series.end(); + timer_series.print("Reconstruct pipeline", total_size); + if (!refactored_metadata.MPI_enabled) { + std::cout << "Reconstruct pipeline: " << timer_series.get() << "s (" + << (double)total_size / timer_series.get() / 1e9 << " GB/s)" + << std::endl; + } else { + refactored_metadata.kernel_time = timer_series.get(); + } + timer_series.clear(); + + // std::cout << "Iterations = " << iter << std::endl; + // std::cout << "Est_max_error = " << error_final_out_host << std::endl; + refactored_metadata.max_est_error = error_final_out_host; +} + +} // namespace MDR +} // namespace mgard_x + +#endif // MGARD_X_MDR_RECONSTRUCT_PIPELINE_QOI_HPP diff --git a/include/mgard-x/MDRHighLevel/RefactorPipeline.hpp b/include/mgard-x/MDRHighLevel/RefactorPipeline.hpp new file mode 100644 index 0000000000..ae7de5a189 --- /dev/null +++ b/include/mgard-x/MDRHighLevel/RefactorPipeline.hpp @@ -0,0 +1,119 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_MDR_REFACTOR_PIPELINE_HPP +#define MGARD_X_MDR_REFACTOR_PIPELINE_HPP + +namespace mgard_x { +namespace MDR { + +template +void refactor_pipeline( + DomainDecomposer &domain_decomposer, + Config &config, RefactoredMetadata &refactored_metadata, + RefactoredData &refactored_data) { + + using Cache = RefactorCache; + using HierarchyType = typename RefactorType::HierarchyType; + + bool timing_pipeline = true; + + RefactorType &refactor = *Cache::cache.refactor; + + Array *device_subdomain_buffer = + Cache::cache.device_subdomain_buffer; + MDRData *mdr_data = Cache::cache.mdr_data; + + if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(0), + domain_decomposer.uniform)) { + Cache::cache.ClearHierarchyCache(); + } + + SIZE total_size = 0; + + for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { + if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(id), + domain_decomposer.uniform)) { + Cache::cache.InsertHierarchyCache( + domain_decomposer.subdomain_hierarchy(id)); + } + } + + HierarchyType &hierarchy = + Cache::cache.GetHierarchyCache(domain_decomposer.subdomain_shape(0)); + + log::info("Adjust device buffers"); + refactor.Adapt(hierarchy, config, 0); + device_subdomain_buffer[0].resize(domain_decomposer.subdomain_shape(0), 0); + device_subdomain_buffer[1].resize(domain_decomposer.subdomain_shape(0), 0); + mdr_data[0].Resize(refactor, hierarchy, 0); + mdr_data[1].Resize(refactor, hierarchy, 0); + DeviceRuntime::SyncDevice(); + + Timer timer_series; + if (timing_pipeline) + timer_series.start(); + // Prefetch the first subdomain to one buffer + int current_buffer = 0; + int current_queue = 0; + domain_decomposer.copy_subdomain( + device_subdomain_buffer[current_buffer], 0, + subdomain_copy_direction::OriginalToSubdomain, current_queue); + + for (SIZE curr_subdomain_id = 0; + curr_subdomain_id < domain_decomposer.num_subdomains(); + curr_subdomain_id++) { + SIZE next_subdomain_id; + int next_buffer = (current_buffer + 1) % 2; + int next_queue = (current_queue + 1) % 3; + HierarchyType &hierarchy = Cache::cache.GetHierarchyCache( + domain_decomposer.subdomain_shape(curr_subdomain_id)); + log::info("Adapt Refactor to hierarchy"); + refactor.Adapt(hierarchy, config, current_queue); + total_size += hierarchy.total_num_elems() * sizeof(T); + // Prefetch the next subdomain + if (curr_subdomain_id + 1 < domain_decomposer.num_subdomains()) { + next_subdomain_id = curr_subdomain_id + 1; + domain_decomposer.copy_subdomain( + device_subdomain_buffer[next_buffer], next_subdomain_id, + subdomain_copy_direction::OriginalToSubdomain, next_queue); + } + + std::stringstream ss; + for (DIM d = 0; d < D; d++) { + ss << hierarchy.level_shape(hierarchy.l_target(), d) << " "; + } + log::info("Refactoring subdomain " + std::to_string(curr_subdomain_id) + + " with shape: " + ss.str()); + + refactor.Refactor(device_subdomain_buffer[current_buffer], + refactored_metadata.metadata[curr_subdomain_id], + mdr_data[current_buffer], current_queue); + refactor.Compress(refactored_metadata.metadata[curr_subdomain_id], + mdr_data[current_buffer], current_queue); + refactor.StoreMetadata(refactored_metadata.metadata[curr_subdomain_id], + mdr_data[current_buffer], current_queue); + mdr_data[current_buffer].CopyToRefactoredData( + refactored_metadata.metadata[curr_subdomain_id], + refactored_data.data[curr_subdomain_id], + refactored_data.data_allocation_size[curr_subdomain_id], current_queue); + + current_buffer = next_buffer; + current_queue = next_queue; + } + DeviceRuntime::SyncDevice(); + if (timing_pipeline) { + timer_series.end(); + // log::csv("time.csv", timer_series.get()); + timer_series.print("Refactor pipeline", total_size, true); + timer_series.clear(); + } +} + +} // namespace MDR +} // namespace mgard_x +#endif \ No newline at end of file diff --git a/include/mgard-x/Metadata/Metadata.hpp b/include/mgard-x/Metadata/Metadata.hpp index 2f11d810f7..5549156a55 100644 --- a/include/mgard-x/Metadata/Metadata.hpp +++ b/include/mgard-x/Metadata/Metadata.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "../Config/Config.h" @@ -36,7 +36,24 @@ struct MetadataBase { enum decomposition_type decomposition; uint32_t l_target = 0; - uint32_t reorder = 0; + + // Hybrid (block-local + global) hierarchy, a.k.a. BlockMGARD. Only + // meaningful when `decomposition == decomposition_type::Hybrid`; these are + // the parameters the decompressor must reproduce exactly, so they travel in + // the header rather than being re-supplied by the caller. + uint64_t hybrid_num_local_levels = 0; + uint64_t hybrid_num_global_levels = 0; + // Edge length of the block-local transform block (compile-time 8 today). + uint64_t hybrid_local_block_size = 0; + bool hybrid_enable_roi = false; + // Resolved basis stored in a stream. Auto is a compression-side policy and + // is never serialized. + compression_projection_mode_type hybrid_projection_mode = + compression_projection_mode_type::Orthogonal; + // Level-0 per-block tolerances; deeper levels are re-derived on read. + std::vector hybrid_roi_tolerance_map; + // Number of level-0 blocks per dimension, used to validate the map above. + std::vector hybrid_roi_block_dimensions; bool domain_decomposed = false; enum domain_decomposition_type ddtype; @@ -57,8 +74,9 @@ struct MetadataBase { double s = 0; // optional enum lossless_type ltype; - uint32_t huff_dict_size = 0; // optional (for Huffman) - uint32_t huff_block_size = 0; // optional (for Huffman) + uint32_t huff_dict_size = 0; // optional (for Huffman) + uint32_t huff_block_size = 0; // optional (for Huffman) + uint32_t block_delta_block_size = 0; // optional (for BlockDelta) enum processor_type ptype; @@ -78,9 +96,9 @@ template struct Metadata : MetadataBase { template void FillForCompression(enum error_bound_type ebtype, T tol, T s, T norm, enum decomposition_type decomposition, - uint32_t reorder, enum lossless_type ltype, - uint32_t huff_dict_size, uint32_t huff_block_size, - std::vector shape, bool domain_decomposed, + enum lossless_type ltype, uint32_t huff_dict_size, + uint32_t huff_block_size, std::vector shape, + bool domain_decomposed, domain_decomposition_type ddtype, uint8_t domain_decomposed_dim, uint64_t domain_decomposed_size) { @@ -108,7 +126,6 @@ template struct Metadata : MetadataBase { } this->norm = norm; this->decomposition = decomposition; - this->reorder = reorder; this->ltype = ltype; this->huff_dict_size = huff_dict_size; this->huff_block_size = huff_block_size; @@ -129,13 +146,13 @@ template struct Metadata : MetadataBase { template void FillForCompression(enum error_bound_type ebtype, T tol, T s, T norm, - enum decomposition_type decomposition, uint32_t reorder, + enum decomposition_type decomposition, enum lossless_type ltype, uint32_t huff_dict_size, uint32_t huff_block_size, std::vector shape, bool domain_decomposed, domain_decomposition_type ddtype, uint8_t domain_decomposed_dim, uint64_t domain_decomposed_size, std::vector coords) { - FillForCompression(ebtype, tol, s, norm, decomposition, reorder, ltype, + FillForCompression(ebtype, tol, s, norm, decomposition, ltype, huff_dict_size, huff_block_size, shape, domain_decomposed, ddtype, domain_decomposed_dim, domain_decomposed_size); @@ -259,4 +276,4 @@ bool infer_domain_decomposed(const void *compressed_data, } // namespace mgard_x -#endif \ No newline at end of file +#endif diff --git a/include/mgard-x/Quantization/CMakeLists.txt b/include/mgard-x/Quantization/CMakeLists.txt index cc06facbb5..5b9c3b85b2 100644 --- a/include/mgard-x/Quantization/CMakeLists.txt +++ b/include/mgard-x/Quantization/CMakeLists.txt @@ -1,4 +1,6 @@ list(APPEND MGARD_X_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/LinearQuantization.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/HybridHierarchyLinearQuantization.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/LocalQuantization.hpp ) set(MGARD_X_HEADER ${MGARD_X_HEADER} PARENT_SCOPE) \ No newline at end of file diff --git a/include/mgard-x/Quantization/HybridHierarchyLinearQuantization.hpp b/include/mgard-x/Quantization/HybridHierarchyLinearQuantization.hpp index d96332bf04..dc692309bb 100644 --- a/include/mgard-x/Quantization/HybridHierarchyLinearQuantization.hpp +++ b/include/mgard-x/Quantization/HybridHierarchyLinearQuantization.hpp @@ -1,270 +1,138 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_HYBRID_HIERARCHY_LINEAR_QUANTIZATION_TEMPLATE #define MGARD_X_HYBRID_HIERARCHY_LINEAR_QUANTIZATION_TEMPLATE +#include + #include "../RuntimeX/RuntimeX.h" #include "LinearQuantization.hpp" +#include "LocalQuantization.hpp" #include "QuantizationInterface.hpp" namespace mgard_x { -template -class QuantizeLevelFunctor : public Functor { +#define MGARDX_QUANTIZE 1 +#define MGARDX_DEQUANTIZE 2 + +template +class HybridHierarchyQuantizer + : public QuantizationInterface { public: - MGARDX_CONT QuantizeLevelFunctor() {} - MGARDX_CONT - QuantizeLevelFunctor(T quantizer, SubArray<1, T, DeviceType> v, - SubArray<1, Q, DeviceType> quantized_v, - bool prep_huffman, SIZE dict_size, - ATOMIC_IDX outlier_idx_hierarchy_offset, - SubArray<1, ATOMIC_IDX, DeviceType> outlier_count, - SubArray<1, ATOMIC_IDX, DeviceType> outlier_indexes, - SubArray<1, QUANTIZED_INT, DeviceType> outliers) - : quantizer(quantizer), v(v), quantized_v(quantized_v), - prep_huffman(prep_huffman), dict_size(dict_size), - outlier_count(outlier_count), outlier_indexes(outlier_indexes), - outliers(outliers) { - Functor(); + HybridHierarchyQuantizer() : initialized(false) {} + + HybridHierarchyQuantizer(Hierarchy &hierarchy, + Hierarchy &global_hierarchy, + Config config) + : initialized(true), hierarchy(&hierarchy), + global_hierarchy(&global_hierarchy), config(config) { + this->L = config.num_local_refactoring_level; + this->M = config.num_global_refactoring_level; + + if (this->L == 0 && this->M == 0) { + throw ProcessingException("Both L and M cannot be zero"); + } } - MGARDX_EXEC void Operation1() { - idx = FunctorBase::GetBlockIdX() * - FunctorBase::GetBlockDimX() + - FunctorBase::GetThreadIdX(); - if (idx < v.shape(0)) { - T t = *v(idx); - Q quantized_data; - T volume = 1; - if constexpr (OP == MGARDX_QUANTIZE) { - if (sizeof(T) == sizeof(double)) - quantized_data = copysign((T)0.5 + fabs(t * quantizer * volume), t); - else if (sizeof(T) == sizeof(float)) - quantized_data = copysign((T)0.5 + fabsf(t * quantizer * volume), t); - if (prep_huffman) { - quantized_data += dict_size / 2; - if (quantized_data >= 0 && quantized_data < dict_size) { - // do nothing - } else { - ATOMIC_IDX outlier_write_offset = - Atomic::Add(outlier_count((IDX)0), (ATOMIC_IDX)1); - - ATOMIC_IDX outlier_idx = idx + outlier_idx_hierarchy_offset; - // Avoid out of range error - // If we have too much outlier than our allocation - // we return the true outlier_count and do quanziation again - if (outlier_write_offset < outlier_indexes.shape(0)) { - *outlier_indexes(outlier_write_offset) = outlier_idx; - *outliers(outlier_write_offset) = quantized_data; - } - quantized_data = 0; - } - } - // store quantized value - *quantized_v(idx) = quantized_data; - - } else if constexpr (OP == MGARDX_DEQUANTIZE) { - // read quantized value - quantized_data = *quantized_v(idx); - if (prep_huffman) { - quantized_data -= dict_size / 2; - } - *v(idx) = (quantizer * volume) * (T)quantized_data; + void Adapt(Hierarchy &hierarchy, + Hierarchy &global_hierarchy, Config config, + int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + this->global_hierarchy = &global_hierarchy; + this->config = config; + this->L = config.num_local_refactoring_level; + this->M = config.num_global_refactoring_level; + + if (this->L == 0 && this->M == 0) { + throw ProcessingException("Both L and M cannot be zero"); + } + + if (this->L > 0) { + local_quantizer.Adapt(hierarchy, config, queue_idx); + if (config.enable_roi) { + this->initial_block_tolerances = config.roi_tolerance_map; + ComputeLocalShapes(); + SetBlockTolerances(this->initial_block_tolerances, queue_idx); } } + + if (this->M > 0) { + global_quantizer.Adapt(global_hierarchy, config, queue_idx); + } } - MGARDX_CONT size_t shared_memory_size() { + static size_t EstimateMemoryFootprint(std::vector shape) { size_t size = 0; return size; } -private: - SIZE idx; - T quantizer; - SubArray<1, T, DeviceType> v; - SubArray<1, Q, DeviceType> quantized_v; - bool prep_huffman; - SIZE dict_size; - ATOMIC_IDX outlier_idx_hierarchy_offset; - SubArray<1, ATOMIC_IDX, DeviceType> outlier_count; - SubArray<1, ATOMIC_IDX, DeviceType> outlier_indexes; - SubArray<1, QUANTIZED_INT, DeviceType> outliers; -}; - -template -class QuantizeLevelKernel : public Kernel { -public: - constexpr static bool EnableAutoTuning() { return false; } - constexpr static std::string_view Name = "lwpk"; - MGARDX_CONT - QuantizeLevelKernel(T quantizer, SubArray<1, T, DeviceType> v, - SubArray<1, Q, DeviceType> quantized_v, bool prep_huffman, - SIZE dict_size, ATOMIC_IDX outlier_idx_hierarchy_offset, - SubArray<1, ATOMIC_IDX, DeviceType> outlier_count, - SubArray<1, ATOMIC_IDX, DeviceType> outlier_indexes, - SubArray<1, QUANTIZED_INT, DeviceType> outliers) - : quantizer(quantizer), v(v), quantized_v(quantized_v), - prep_huffman(prep_huffman), dict_size(dict_size), - outlier_count(outlier_count), outlier_indexes(outlier_indexes), - outliers(outliers) {} - - MGARDX_CONT Task> - GenTask(int queue_idx) { - using FunctorType = QuantizeLevelFunctor; - FunctorType functor(quantizer, v, quantized_v, prep_huffman, dict_size, - outlier_idx_hierarchy_offset, outlier_count, - outlier_indexes, outliers); - - SIZE total_thread_z = 1; - SIZE total_thread_y = 1; - SIZE total_thread_x = v.shape(0); - - SIZE tbx, tby, tbz, gridx, gridy, gridz; - size_t sm_size = functor.shared_memory_size(); - tbz = 1; - tby = 1; - tbx = 256; - gridz = ceil((double)total_thread_z / tbz); - gridy = ceil((double)total_thread_y / tby); - gridx = ceil((double)total_thread_x / tbx); - // printf("%u %u %u\n", shape.dataHost()[2], shape.dataHost()[1], - // shape.dataHost()[0]); PrintSubarray("shape", shape); - return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, - std::string(Name)); + void SetOrthogonalProjection(bool enabled) { + orthogonal_projection = enabled; + local_quantizer.SetOrthogonalProjection(enabled); } -private: - T quantizer; - SubArray<1, T, DeviceType> v; - SubArray<1, Q, DeviceType> quantized_v; - bool prep_huffman; - SIZE dict_size; - ATOMIC_IDX outlier_idx_hierarchy_offset; - SubArray<1, ATOMIC_IDX, DeviceType> outlier_count; - SubArray<1, ATOMIC_IDX, DeviceType> outlier_indexes; - SubArray<1, QUANTIZED_INT, DeviceType> outliers; -}; - -template -class HybridHierarchyLinearQuantizer - : public QuantizationInterface { -public: - HybridHierarchyLinearQuantizer() : initialized(false) {} - - HybridHierarchyLinearQuantizer(Hierarchy &hierarchy, - Config config) - : initialized(true), hierarchy(&hierarchy), config(config), - global_quantizer(hierarchy, config) { - - coarse_shape = hierarchy.level_shape(hierarchy.l_target()); - // If we do at least one level of local refactoring - if (config.num_local_refactoring_level > 0) { - for (int l = 0; l < config.num_local_refactoring_level; l++) { - SIZE last_level_size = 1, curr_level_size = 1; - for (DIM d = 0; d < D; d++) { - last_level_size *= coarse_shape[d]; - coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 5; - curr_level_size *= coarse_shape[d]; - } - coarse_shapes.push_back(coarse_shape); - coarse_num_elems.push_back(last_level_size); - local_coeff_size.push_back(last_level_size - curr_level_size); - } - } - - global_hierarchy = Hierarchy(coarse_shape, config); - global_quantizer = - LinearQuantizer(global_hierarchy, config); + // Set block-level tolerances according to ROI table + void SetBlockTolerances(const std::vector &initial_block_tolerances, + int queue_idx) { + BuildROIToleranceMap(initial_block_tolerances); + // Upload once here so Quantize/Dequantize don't have to re-upload the + // tolerance map (which barely changes) on every call. + device_roi_tolerance_map.resize({(SIZE)roi_tolerance_map.size()}, + queue_idx); + device_roi_tolerance_map.load(roi_tolerance_map.data(), 0, queue_idx); } - void Adapt(Hierarchy &hierarchy, Config config, - int queue_idx) { - this->initialized = true; - this->hierarchy = &hierarchy; - this->config = config; - coarse_shape = hierarchy.level_shape(hierarchy.l_target()); - coarse_shapes.clear(); - coarse_num_elems.clear(); - local_coeff_size.clear(); - // If we do at least one level of local refactoring - if (config.num_local_refactoring_level > 0) { - for (int l = 0; l < config.num_local_refactoring_level; l++) { - SIZE last_level_size = 1, curr_level_size = 1; - for (DIM d = 0; d < D; d++) { - last_level_size *= coarse_shape[d]; - coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 5; - curr_level_size *= coarse_shape[d]; - } - coarse_shapes.push_back(coarse_shape); - coarse_num_elems.push_back(last_level_size); - local_coeff_size.push_back(last_level_size - curr_level_size); - } + // An s-norm bound is only defensible when the block-local stage is a single + // level with no global stage: only then is every block transformed + // independently, so the total squared error is a sum over blocks and the + // budget divides evenly in quadrature (see restrict_hybrid_config_for_s_norm, + // which makes the public compress path fall back to exactly this). + // + // Here we can only refuse: L and M fixed the buffer layout back at Adapt(), + // so a low-level caller cannot be silently re-configured mid-flight. + void CheckSNormConfiguration(T s) { + if (s == std::numeric_limits::infinity()) { + return; + } + if (this->L != 1 || this->M != 0) { + throw ProcessingException( + "hybrid with an s-norm bound requires exactly one block-local level " + "and no global stage (got L=" + + std::to_string(this->L) + ", M=" + std::to_string(this->M) + + "); the public compress path falls back to this automatically"); + } + if (config.enable_roi) { + throw ProcessingException("ROI mode is L-inf only"); } + } - global_hierarchy = Hierarchy(coarse_shape, config); - global_quantizer.Adapt(global_hierarchy, config, queue_idx); + // Tolerance for the coarsest layer when it has to be quantized on its own + // (M == 0, so there is no global stage to cover it). In ROI mode the + // coarsest layer has no tolerance of its own -- the map only describes the + // coefficient layers' block grids -- so use the tightest tolerance among + // the coarsest blocks, which is exactly what the global stage is given when + // M > 0 (see ErrorBudgetAllocation). + T CoarsestTolerance(T tol) { + return config.enable_roi ? (T)GetMinToleranceForGlobal() : tol; } - void CalcQuantizers(size_t dof, enum error_bound_type type, T tol, T s, - T norm, SIZE global_l_target, - SIZE num_local_refactoring_level, - enum decomposition_type decomposition, bool reciprocal, - T &quantizer, T &coarse_abs_tol) { + // Called only when this->M > 0 + T ErrorBudgetAllocation(T tol) { + T global_tol = tol; - double abs_tol = tol; - if (type == error_bound_type::REL) { - abs_tol *= norm; - } - abs_tol *= 2; - SIZE total_num_levels = global_l_target + num_local_refactoring_level + 1; - // std::cout << "total_num_levels: " << total_num_levels << "\n"; - if (s == std::numeric_limits::infinity()) { - quantizer = (abs_tol) / (total_num_levels * (1 + std::pow(3, D))); - // std::cout << "quantizer: " << quantizer << "\n"; - coarse_abs_tol = - (quantizer * (global_l_target + 1) * (1 + std::pow(3, D))) / 2; - // std::cout << "coarse_abs_tol: " << coarse_abs_tol << "\n"; - if (reciprocal) - quantizer = 1.0f / quantizer; - } else { // s != inf - - log::err("s != inf not supported yet."); - exit(-1); - // xin - uniform - // T C2 = 1 + 3 * std::sqrt(3) / 4; - // T c = std::sqrt(std::pow(2, D - 2 * s)); - // T cc = (1 - c) / (1 - std::pow(c, l_target + 1)); - // T level_eb = cc * tol / C2; - // for (int l = 0; l < l_target + 1; l++) { - // quantizers[l] = level_eb; - // // T c = std::sqrt(std::pow(2, 2*s*l + D * (l_target - l))); - // level_eb *= c; - // if (reciprocal) - // quantizers[l] = 1.0f / quantizers[l]; - // } - - // ben - uniform - for (int l = 0; l < total_num_levels; l++) { - quantizer = (abs_tol) / (std::exp2(s * l) * std::sqrt(dof)); - if (reciprocal) - quantizer = 1.0f / quantizer; + if (this->L > 0) { + if (this->config.enable_roi) { + global_tol = GetMinToleranceForGlobal(); } + global_tol = global_tol / (1 << this->L); } - } - - static size_t EstimateMemoryFootprint(std::vector shape) { - Hierarchy hierarchy; - hierarchy.EstimateMemoryFootprint(shape); - size_t size = 0; - size += sizeof(T) * (hierarchy->l_target() + 1); - return size; + return global_tol; } void Quantize(SubArray original_data, @@ -275,95 +143,353 @@ class HybridHierarchyLinearQuantizer enum error_bound_type ebtype, T tol, T s, T norm, SubArray quantized_data, int queue_idx) {} + // Quantize the global (coarsest) region at the front of the decomposed + // array with the global quantizer. Shared between the unfused Quantize() + // path and the fused decompose+quantize path. Only valid when M > 0. + template + void QuantizeGlobalPart(SubArray<1, T, DeviceType> original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray<1, Q, DeviceType> quantized_data, + LosslessCompressorType &lossless, int queue_idx) { + T global_tol = ErrorBudgetAllocation(tol); + + std::vector global_shape = + global_hierarchy->level_shape(global_hierarchy->l_target()); + SubArray global_data_v(global_shape, + original_data.data()); + SubArray global_data_q(global_shape, + quantized_data.data()); + for (DIM d = 0; d < D; d++) { + global_data_v.setLd(d, global_shape[d]); + global_data_q.setLd(d, global_shape[d]); + } + global_data_v.project(D - 3, D - 2, D - 1); + global_data_q.project(D - 3, D - 2, D - 1); + global_quantizer.Quantize(global_data_v, ebtype, global_tol, s, norm, + global_data_q, lossless, queue_idx, + orthogonal_projection); + } + + // Dequantize the global (coarsest) region at the front of the decomposed + // array with the global quantizer. Shared between the unfused Dequantize() + // path and the fused dequantize+recompose path. Only valid when M > 0. + template + void DequantizeGlobalPart(SubArray<1, T, DeviceType> original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray<1, Q, DeviceType> quantized_data, + LosslessCompressorType &lossless, int queue_idx) { + T global_tol = ErrorBudgetAllocation(tol); + + std::vector global_shape = + global_hierarchy->level_shape(global_hierarchy->l_target()); + SubArray global_data_v(global_shape, + original_data.data()); + SubArray global_data_q(global_shape, + quantized_data.data()); + for (DIM d = 0; d < D; d++) { + global_data_v.setLd(d, global_shape[d]); + global_data_q.setLd(d, global_shape[d]); + } + global_data_v.project(D - 3, D - 2, D - 1); + global_data_q.project(D - 3, D - 2, D - 1); + global_quantizer.Dequantize(global_data_v, ebtype, global_tol, s, norm, + global_data_q, lossless, queue_idx, + orthogonal_projection); + } + template void Quantize(SubArray<1, T, DeviceType> original_data, enum error_bound_type ebtype, T tol, T s, T norm, SubArray<1, Q, DeviceType> quantized_data, LosslessCompressorType &lossless, int queue_idx) { - bool prep_huffman = - config.lossless != lossless_type::CPU_Lossless; // always do Huffman - - Array coarse_data(coarse_shape, original_data.data()); - Array coarse_quantized_data(coarse_shape, - quantized_data.data()); - - T quantizer, coarse_abs_tol; - CalcQuantizers(hierarchy->total_num_elems(), ebtype, tol, s, norm, - global_hierarchy.l_target(), - config.num_local_refactoring_level, config.decomposition, - true, quantizer, coarse_abs_tol); - - // Array original_data_array(coarse_shape, - // original_data.data()); Array<1, T, DeviceType> - // norm_tmp_array({coarse_num_elems[coarse_num_elems.size()-1]}); Array<1, - // T, DeviceType> norm_array({1}); - - // verify_matrix_cuda(coarse_shape[0], coarse_shape[1], coarse_shape[2], - // original_data.data(), coarse_shape[0], coarse_shape[1], - // coarse_shape[0], "coarse_data", true, false); - - // T coarse_norm = norm_calculator(original_data_array, - // SubArray(norm_tmp_array), - // SubArray(norm_array), s, - // config.normalize_coordinates); - // std::cout << "coarse_norm: " << coarse_norm << "\n"; - log::info("coarse_abs_tol: " + std::to_string(coarse_abs_tol)); - log::info("local quantizer: " + std::to_string(quantizer)); - global_quantizer.Quantize(coarse_data, error_bound_type::ABS, - coarse_abs_tol, s, norm, coarse_quantized_data, - lossless, queue_idx); - - SIZE accumulated_local_coeff_size = 0; - for (int l = 0; l < config.num_local_refactoring_level; l++) { - accumulated_local_coeff_size += local_coeff_size[l]; - SubArray<1, T, DeviceType> local_data( - {local_coeff_size[l]}, - original_data(original_data.shape(0) - accumulated_local_coeff_size)); - SubArray<1, Q, DeviceType> local_quantized_data( - {local_coeff_size[l]}, quantized_data(quantized_data.shape(0) - - accumulated_local_coeff_size)); - - Timer timer; - if (log::level & log::TIME) - timer.start(); - bool done_quantization = false; - DeviceLauncher::Execute( - QuantizeLevelKernel( - quantizer, local_data, local_quantized_data, prep_huffman, - config.huff_dict_size, global_hierarchy.total_num_elems(), - lossless.huffman.workspace.outlier_count_subarray, - lossless.huffman.workspace.outlier_idx_subarray, - lossless.huffman.workspace.outlier_subarray), - queue_idx); - - MemoryManager::Copy1D( - &lossless.huffman.outlier_count, - lossless.huffman.workspace.outlier_count_subarray.data(), 1, - queue_idx); + if (this->L == 0 && this->M == 0) { + throw ProcessingException("Both L and M cannot be zero"); + } + CheckSNormConfiguration(s); + Timer timer; + if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); - if (lossless.huffman.outlier_count <= - lossless.huffman.workspace.outlier_subarray.shape(0)) { - // outlier buffer has sufficient size - done_quantization = true; - if (log::level & log::TIME) { - timer.end(); - timer.print("Quantization"); - log::time("Quantization throughput: " + - std::to_string((double)(coarse_num_elems[l] * sizeof(T)) / - timer.get() / 1e9) + - " GB/s"); - timer.clear(); - } - log::info( - "Outlier ratio: " + std::to_string(lossless.huffman.outlier_count) + - "/" + std::to_string(hierarchy->total_num_elems()) + " (" + - std::to_string((double)100 * lossless.huffman.outlier_count / - hierarchy->total_num_elems()) + - "%)"); + timer.start(); + } + + SIZE global_q_size = 0; + + // Global quantization + if (this->M > 0) { + global_q_size = global_hierarchy->total_num_elems(); + QuantizeGlobalPart(original_data, ebtype, tol, s, norm, quantized_data, + lossless, queue_idx); + } + + // Local quantization + if (this->L > 0) { + SIZE local_length = original_data.shape(0) - global_q_size; + + SubArray<1, T, DeviceType> local_data_v({local_length}, + original_data(global_q_size)); + SubArray<1, Q, DeviceType> local_data_q({local_length}, + quantized_data(global_q_size)); + + // Switch between ROI and Non-ROI + if (config.enable_roi) { + local_quantizer.Quantize( + local_data_v, ebtype, 0.0, s, norm, local_data_q, + SubArray<1, double, DeviceType>(device_roi_tolerance_map), + level_offsets, level_block_counts, lossless, queue_idx); } else { - log::err("Not enough workspace for outliers."); - exit(-1); + local_quantizer.Quantize(local_data_v, ebtype, tol, s, norm, + local_data_q, lossless, queue_idx); + } + } + + // Coarsest layer, at the front of the decomposed array. When M > 0 the + // global stage above quantized it; when M == 0 the non-ROI local + // quantizer covers it as its layer 0, but the ROI local quantizer's + // per-block quantizers only span the L coefficient layers. Without this + // the coarsest layer -- (5/8)^D of the array -- would never be written, + // and only a same-process decompress would appear to work, by reading the + // values left in the cached decomposed buffer. + if (this->L > 0 && this->M == 0 && config.enable_roi) { + SIZE coarsest_size = local_quantizer.layer_len[0]; + SubArray<1, T, DeviceType> coarsest_v({coarsest_size}, + original_data.data()); + SubArray<1, Q, DeviceType> coarsest_q({coarsest_size}, + quantized_data.data()); + local_quantizer.QuantizeCoarsest(coarsest_v, coarsest_q, ebtype, + CoarsestTolerance(tol), s, norm, + queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Hybrid Quantization", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + // Whether the fused decompose+quantize path can be used: it covers the + // local stage only (L > 0), relies on the 3D in-cache block kernel, and has + // the same L-inf-only constraint as the local quantizer. + bool CanFuseQuantize(T s) { + // s != inf is fine here: its weighting is a per-level scalar (the node + // volume is constant within a level), which is exactly what the fused + // kernels already take. ROI is not -- its per-block quantizers have no + // s-norm formulation, so it stays L-inf only. + if (config.enable_roi && s != std::numeric_limits::infinity()) { + return false; + } + return this->L > 0 && D >= 1 && D <= 3; + } + + // Which of the conditions above ruled the fused path out, for logging. Kept + // next to CanFuseQuantize so the two cannot drift apart. Returns an empty + // string when fusing is possible. + std::string WhyCannotFuseQuantize(T s) { + if (this->L == 0) { + return "no block-local levels"; + } + if (D > 3) { + return "fused kernel supports 1D, 2D and 3D only"; + } + if (config.enable_roi && s != std::numeric_limits::infinity()) { + return "ROI mode is L-inf only"; + } + return ""; + } + + // Fused decompose+quantization driver: the local levels are decomposed and + // quantized in one kernel per level (coefficients never round-trip through + // global memory as T), writing symbols directly to their final location in + // quantized_data. The coarsest region is then handled as in the unfused + // path: global decompose + global quantize when M > 0, otherwise a single + // coarsest-layer quantization -- in ROI mode too, since the per-block + // quantizers only span the coefficient layers. + template + void DecomposeQuantize(RefactorType &refactor, + SubArray data, + SubArray<1, T, DeviceType> decomposed_data, + SubArray<1, Q, DeviceType> quantized_data, + enum error_bound_type ebtype, T tol, T s, T norm, + LosslessCompressorType &lossless, int queue_idx) { + if (!CanFuseQuantize(s)) { + throw ProcessingException("DecomposeQuantize requires L > 0, D <= 3, and " + "(non-ROI or s == inf)"); + } + CheckSNormConfiguration(s); + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + SIZE huff_dict_size = config.huff_dict_size; + + if (config.enable_roi) { + // Per-level per-block reciprocal quantizers from the device-resident + // tolerance map, same math and block ordering as the ROI Quantize path + // (the fused kernel indexes them by block id, which matches the + // idx / hybrid_local_coeff_per_block(D) mapping of the unfused ROI + // kernel). + double C = orthogonal_projection ? (1 + std::pow(3, D)) : 1.0; + double norm_factor = + (ebtype == error_bound_type::REL) ? (double)norm : 1.0; + std::vector> device_quantizers(this->L); + std::vector> block_quantizers(this->L); + for (SIZE l = 0; l < this->L; ++l) { + SIZE level_offset = level_offsets[l]; + SIZE num_blocks = level_block_counts[l]; + double denom = std::pow(2, this->L - l + 1) * C; + device_quantizers[l] = Array<1, T, DeviceType>({num_blocks}, queue_idx); + DeviceLauncher::Execute( + ComputeROIQuantizersKernel( + SubArray<1, double, DeviceType>(device_roi_tolerance_map), + level_offset, num_blocks, norm_factor, denom, + /*reciprocal=*/true, + SubArray<1, T, DeviceType>(device_quantizers[l])), + queue_idx); + block_quantizers[l] = SubArray<1, T, DeviceType>(device_quantizers[l]); + } + refactor.local_refactor.DecomposeQuantize( + data, decomposed_data, quantized_data, std::vector(), + block_quantizers, prep_huffman, huff_dict_size, queue_idx, + orthogonal_projection); + } else { + std::vector level_quantizers = + local_quantizer.DecomposeLevelQuantizers(ebtype, tol, s, norm); + refactor.local_refactor.DecomposeQuantize( + data, decomposed_data, quantized_data, level_quantizers, + std::vector>(), prep_huffman, + huff_dict_size, queue_idx, orthogonal_projection); + } + + // Coarsest region (compacted at the front of decomposed_data by the + // fused local stage). + if (this->M > 0) { + refactor.DecomposeGlobal(decomposed_data, queue_idx, + orthogonal_projection); + QuantizeGlobalPart(decomposed_data, ebtype, tol, s, norm, quantized_data, + lossless, queue_idx); + } else { + // Also in ROI mode: the per-block quantizers only span the coefficient + // layers, so the coarsest layer needs one of its own here. + SIZE coarsest_size = local_quantizer.layer_len[0]; + SubArray<1, T, DeviceType> coarsest_v({coarsest_size}, + decomposed_data.data()); + SubArray<1, Q, DeviceType> coarsest_q({coarsest_size}, + quantized_data.data()); + local_quantizer.QuantizeCoarsest(coarsest_v, coarsest_q, ebtype, + CoarsestTolerance(tol), s, norm, + queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Hybrid Decomposition+Quantization (fused)", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + // Fused dequantize+recomposition driver (inverse of DecomposeQuantize): + // the coarsest region is first reconstructed as in the unfused path + // (global dequantize + global recompose when M > 0, otherwise a single + // coarsest-layer dequantization, in ROI mode too), + // then the local levels are dequantized and recomposed in one kernel per + // level (coefficients never round-trip through global memory as T), + // writing the final level directly into the unpadded output. + template + void DequantizeRecompose(RefactorType &refactor, + SubArray data, + SubArray<1, T, DeviceType> decomposed_data, + SubArray<1, Q, DeviceType> quantized_data, + enum error_bound_type ebtype, T tol, T s, T norm, + LosslessCompressorType &lossless, int queue_idx) { + if (!CanFuseQuantize(s)) { + throw ProcessingException("DequantizeRecompose requires L > 0, D <= 3, " + "and (non-ROI or s == inf)"); + } + CheckSNormConfiguration(s); + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + SIZE huff_dict_size = config.huff_dict_size; + + // Coarsest region first: it is the input of the local recomposition. + if (this->M > 0) { + DequantizeGlobalPart(decomposed_data, ebtype, tol, s, norm, + quantized_data, lossless, queue_idx); + refactor.RecomposeGlobal(decomposed_data, queue_idx, + orthogonal_projection); + } else { + // Also in ROI mode -- see the matching branch in DecomposeQuantize. + SIZE coarsest_size = local_quantizer.layer_len[0]; + SubArray<1, T, DeviceType> coarsest_v({coarsest_size}, + decomposed_data.data()); + SubArray<1, Q, DeviceType> coarsest_q({coarsest_size}, + quantized_data.data()); + local_quantizer.DequantizeCoarsest(coarsest_v, coarsest_q, ebtype, + CoarsestTolerance(tol), s, norm, + queue_idx); + } + + if (config.enable_roi) { + // Per-level per-block dequantizers from the device-resident tolerance + // map, same math and block ordering as the ROI Dequantize path (the + // fused kernel indexes them by block id, which matches the + // idx / hybrid_local_coeff_per_block(D) mapping of the unfused ROI + // kernel). + double C = orthogonal_projection ? (1 + std::pow(3, D)) : 1.0; + double norm_factor = + (ebtype == error_bound_type::REL) ? (double)norm : 1.0; + std::vector> device_dequantizers(this->L); + std::vector> block_dequantizers(this->L); + for (SIZE l = 0; l < this->L; ++l) { + SIZE level_offset = level_offsets[l]; + SIZE num_blocks = level_block_counts[l]; + double denom = std::pow(2, this->L - l + 1) * C; + device_dequantizers[l] = + Array<1, T, DeviceType>({num_blocks}, queue_idx); + DeviceLauncher::Execute( + ComputeROIQuantizersKernel( + SubArray<1, double, DeviceType>(device_roi_tolerance_map), + level_offset, num_blocks, norm_factor, denom, + /*reciprocal=*/false, + SubArray<1, T, DeviceType>(device_dequantizers[l])), + queue_idx); + block_dequantizers[l] = + SubArray<1, T, DeviceType>(device_dequantizers[l]); } + refactor.local_refactor.RecomposeDequantize( + data, decomposed_data, quantized_data, std::vector(), + block_dequantizers, prep_huffman, huff_dict_size, queue_idx, + orthogonal_projection); + } else { + std::vector level_dequantizers = + local_quantizer.RecomposeLevelDequantizers(ebtype, tol, s, norm); + refactor.local_refactor.RecomposeDequantize( + data, decomposed_data, quantized_data, level_dequantizers, + std::vector>(), prep_huffman, + huff_dict_size, queue_idx, orthogonal_projection); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Hybrid Dequantization+Recomposition (fused)", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); } } @@ -371,19 +497,344 @@ class HybridHierarchyLinearQuantizer void Dequantize(SubArray<1, T, DeviceType> original_data, enum error_bound_type ebtype, T tol, T s, T norm, SubArray<1, Q, DeviceType> quantized_data, - LosslessCompressorType &lossless_compressor, int queue_idx) {} + LosslessCompressorType &lossless, int queue_idx) { + if (this->L == 0 && this->M == 0) { + throw ProcessingException("Both L and M cannot be zero"); + } + CheckSNormConfiguration(s); + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE global_q_size = 0; + if (this->M > 0) { + global_q_size = global_hierarchy->total_num_elems(); + // log::info("Total Elems: " + std::to_string(global_q_size)); + } + + // Global dequantization + if (this->M > 0) { + DequantizeGlobalPart(original_data, ebtype, tol, s, norm, quantized_data, + lossless, queue_idx); + } + + // Local dequantization + if (this->L > 0) { + SIZE local_length = original_data.shape(0) - global_q_size; + SubArray<1, T, DeviceType> local_data_v({local_length}, + original_data(global_q_size)); + SubArray<1, Q, DeviceType> local_data_q({local_length}, + quantized_data(global_q_size)); + // Switch between ROI and Non-ROI + if (config.enable_roi) { + local_quantizer.Dequantize( + local_data_v, ebtype, 0.0, s, norm, local_data_q, + SubArray<1, double, DeviceType>(device_roi_tolerance_map), + level_offsets, level_block_counts, lossless, queue_idx); + } else { + local_quantizer.Dequantize(local_data_v, ebtype, tol, s, norm, + local_data_q, lossless, queue_idx); + } + } + + // Coarsest layer -- inverse of the corresponding block in Quantize(). + if (this->L > 0 && this->M == 0 && config.enable_roi) { + SIZE coarsest_size = local_quantizer.layer_len[0]; + SubArray<1, T, DeviceType> coarsest_v({coarsest_size}, + original_data.data()); + SubArray<1, Q, DeviceType> coarsest_q({coarsest_size}, + quantized_data.data()); + local_quantizer.DequantizeCoarsest(coarsest_v, coarsest_q, ebtype, + CoarsestTolerance(tol), s, norm, + queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Hybrid Dequantization", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + // Compute fine and coarse shapes for all local levels + void ComputeLocalShapes() { + fine_shapes.clear(); + coarse_shapes.clear(); + + // Only compute if L > 0 + if (this->L == 0) { + return; + } + + // Get original shape from hierarchy + std::vector coarse_shape = + hierarchy->level_shape(hierarchy->l_target()); + + for (SIZE l = 0; l < this->L; ++l) { + std::vector fine_shape(D); + for (DIM d = 0; d < D; ++d) { + // Round up to multiple of 8 for fine shape + coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 8; + fine_shape[d] = coarse_shape[d]; + // Compute next coarse shape (8->5 mapping) + coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 5; + } + fine_shapes.push_back(fine_shape); + coarse_shapes.push_back(coarse_shape); + } + } + + // Build ROI tolerance map for all local levels (stored as 1D array) + void BuildROIToleranceMap(const std::vector &initial_tolerances) { + roi_tolerance_map.clear(); + level_offsets.clear(); + level_block_counts.clear(); + + // Only build ROI map if L > 0 + if (this->L == 0) { + return; + } + + // Level 0: use initial tolerances directly + level_offsets.push_back(0); + level_block_counts.push_back(initial_tolerances.size()); + roi_tolerance_map.insert(roi_tolerance_map.end(), + initial_tolerances.begin(), + initial_tolerances.end()); + + // Propagate tolerances through local levels + for (SIZE level = 1; level < this->L; ++level) { + SIZE prev_offset = level_offsets[level - 1]; + SIZE prev_count = level_block_counts[level - 1]; + + std::vector prev_level_tolerances( + roi_tolerance_map.begin() + prev_offset, + roi_tolerance_map.begin() + prev_offset + prev_count); + + std::vector next_level_tolerances = + PropagateTolerances(prev_level_tolerances, level - 1, level); + + level_offsets.push_back(roi_tolerance_map.size()); + level_block_counts.push_back(next_level_tolerances.size()); + roi_tolerance_map.insert(roi_tolerance_map.end(), + next_level_tolerances.begin(), + next_level_tolerances.end()); + } + } + + // Propagate error to next level + std::vector + PropagateTolerances(const std::vector ¤t_tolerances, + SIZE curr_level, SIZE next_level) { + // Get current and next level block dimensions from computed shapes + std::vector curr_blocks = GetBlockDimensions(curr_level); + std::vector next_blocks = GetBlockDimensions(next_level); + + SIZE next_size = 1; + for (DIM d = 0; d < D; ++d) { + next_size *= next_blocks[d]; + } + + // Initialize tolerance list for next block + std::vector next_tolerances(next_size, + std::numeric_limits::max()); + + // For each block in next level, find minimum tolerance from contributing + // blocks + for (SIZE idx = 0; idx < next_size; ++idx) { + std::vector next_coord = LinearToCoord(idx, next_blocks); + double min_tol = std::numeric_limits::max(); + + // Find all contributing blocks from current level + std::vector contributing_blocks = + GetContributingBlocks(next_coord, curr_blocks); + + for (SIZE contrib_idx : contributing_blocks) { + if (contrib_idx < current_tolerances.size()) { + min_tol = std::min(min_tol, current_tolerances[contrib_idx]); + } + } + + next_tolerances[idx] = min_tol; + } + + return next_tolerances; + } + + // Contributing tables: + // NEXT BLOCK CUR BLOCK + // 0 0,1 + // 1 1,2,3 + // 2 3,4 + // 3 4,5,6 + // 4 6,7 + // Get contributing block indices from previous level for a given next-level + // block + std::vector + GetContributingBlocks(const std::vector &next_coord, + const std::vector &curr_blocks) { + std::vector> contrib_per_dim(D); + + // For each dimension, apply the 5->8 inverse mapping + for (DIM d = 0; d < D; ++d) { + SIZE next_idx = next_coord[d]; + SIZE group = next_idx / 5; + SIZE offset = next_idx % 5; + SIZE base = group * 8; + + // Apply the propagation pattern + switch (offset) { + case 0: // min(0, 1) + contrib_per_dim[d] = {base + 0, base + 1}; + break; + case 1: // min(1, 2, 3) + contrib_per_dim[d] = {base + 1, base + 2, base + 3}; + break; + case 2: // min(3, 4) + contrib_per_dim[d] = {base + 3, base + 4}; + break; + case 3: // min(4, 5, 6) + contrib_per_dim[d] = {base + 4, base + 5, base + 6}; + break; + case 4: // min(6, 7) + contrib_per_dim[d] = {base + 6, base + 7}; + break; + } + + // Filter out-of-bounds indices + std::vector valid; + for (SIZE idx : contrib_per_dim[d]) { + if (idx < curr_blocks[d]) { + valid.push_back(idx); + } + } + contrib_per_dim[d] = valid; + } + + // Generate all combinations (Cartesian product) + return CartesianProduct(contrib_per_dim, curr_blocks); + } + + // Cartesian product of contributing indices across dimensions + std::vector + CartesianProduct(const std::vector> &indices_per_dim, + const std::vector &blocks) { + std::vector result; + std::vector coord(D); + CartesianProductHelper(indices_per_dim, blocks, 0, coord, result); + return result; + } + + void + CartesianProductHelper(const std::vector> &indices_per_dim, + const std::vector &blocks, DIM dim, + std::vector &coord, std::vector &result) { + if (dim == D) { + result.push_back(CoordToLinear(coord, blocks)); + return; + } + + for (SIZE idx : indices_per_dim[dim]) { + coord[dim] = idx; + CartesianProductHelper(indices_per_dim, blocks, dim + 1, coord, result); + } + } + + // Convert linear index to coordinate + std::vector LinearToCoord(SIZE linear_idx, + const std::vector &dims) { + std::vector coord(D); + for (int d = D - 1; d >= 0; --d) { + coord[d] = linear_idx % dims[d]; + linear_idx /= dims[d]; + } + return coord; + } + + // Convert coordinate to linear index + SIZE CoordToLinear(const std::vector &coord, + const std::vector &dims) { + SIZE linear = 0; + SIZE stride = 1; + for (int d = D - 1; d >= 0; --d) { + linear += coord[d] * stride; + stride *= dims[d]; + } + return linear; + } + + // Get block dimensions at a specific level + std::vector GetBlockDimensions(SIZE level) { + // Use the fine shape for this level (before decomposition) + const std::vector &fine_shape = fine_shapes[level]; + + // Calculate block size (8x8x8 for local decomposition) + const SIZE BLOCK_SIZE = 8; + + // Calculate number of blocks in each dimension + std::vector block_dims(D); + for (DIM d = 0; d < D; ++d) { + block_dims[d] = (fine_shape[d] + BLOCK_SIZE - 1) / BLOCK_SIZE; + } + + return block_dims; + } + + // Get minimum tolerance for global quantization + T GetMinToleranceForGlobal() { + if (!this->config.enable_roi) { + return std::numeric_limits::max(); + } + + // Get the last level + SIZE last_level_idx = level_offsets.size() - 1; + SIZE last_offset = level_offsets[last_level_idx]; + SIZE last_count = level_block_counts[last_level_idx]; + + double min_tol = std::numeric_limits::max(); + for (SIZE i = 0; i < last_count; ++i) { + min_tol = std::min(min_tol, roi_tolerance_map[last_offset + i]); + } + + return min_tol; + } bool initialized; + SIZE L; // Number of local levels + SIZE M; // Number of global levels + Hierarchy *hierarchy; - Hierarchy global_hierarchy; + Hierarchy *global_hierarchy; Config config; - std::vector coarse_shape; - std::vector coarse_num_elems; + bool orthogonal_projection = true; + + LocalQuantizer local_quantizer; LinearQuantizer global_quantizer; + + // 1D ROI tolerance map: all levels stored consecutively + std::vector roi_tolerance_map; + + // Device-resident copy of roi_tolerance_map, uploaded once in + // SetBlockTolerances and reused by every Quantize/Dequantize call. + Array<1, double, DeviceType> device_roi_tolerance_map; + + // Offset for each level in the 1D tolerance map + std::vector level_offsets; + + // Number of blocks at each level + std::vector level_block_counts; + + // Fine and coarse shapes for each local level + std::vector> fine_shapes; std::vector> coarse_shapes; - std::vector local_coeff_size; + + std::vector initial_block_tolerances; }; } // namespace mgard_x -#endif \ No newline at end of file +#endif diff --git a/include/mgard-x/Quantization/LinearQuantization.hpp b/include/mgard-x/Quantization/LinearQuantization.hpp index 740b8bdc7b..43bbef8e62 100644 --- a/include/mgard-x/Quantization/LinearQuantization.hpp +++ b/include/mgard-x/Quantization/LinearQuantization.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_LINEAR_QUANTIZATION_TEMPLATE @@ -25,126 +25,18 @@ class LevelwiseLinearQuantizerNDFunctor : public Functor { SubArray<2, SIZE, DeviceType> level_ranges, SubArray<2, int, DeviceType> level_marks, SIZE l_target, SubArray<1, T, DeviceType> quantizers, - SubArray<3, T, DeviceType> level_volumes, SubArray v, - SubArray quantized_v, - SubArray<1, QUANTIZED_INT, DeviceType> *quantized_linearized_v, - bool prep_huffman, bool calc_vol, bool level_linearize, SIZE dict_size, - SubArray<1, ATOMIC_IDX, DeviceType> outlier_count, - SubArray<1, ATOMIC_IDX, DeviceType> outlier_indexes, - SubArray<1, QUANTIZED_INT, DeviceType> outliers) + SubArray<3, T, DeviceType> level_volumes, bool calc_vol, + SubArray v, + SubArray quantized_v, bool prep_huffman, + SIZE dict_size) : level_ranges(level_ranges), level_marks(level_marks), l_target(l_target), quantizers(quantizers), - level_volumes(level_volumes), v(v), quantized_v(quantized_v), - quantized_linearized_v(quantized_linearized_v), - prep_huffman(prep_huffman), calc_vol(calc_vol), - level_linearize(level_linearize), dict_size(dict_size), - outlier_count(outlier_count), outlier_indexes(outlier_indexes), - outliers(outliers) { + level_volumes(level_volumes), calc_vol(calc_vol), v(v), + quantized_v(quantized_v), prep_huffman(prep_huffman), + dict_size(dict_size) { Functor(); } - MGARDX_EXEC SIZE calc_level_offset() { - // Use curr_region to encode region id to distinguish different regions - // curr_region of current level is always >=1, - // since curr_region=0 refers to the next coarser level - // most significant bit --> fastest dim - // least signigiciant bit --> slowest dim - for (int d = D - 1; d >= 0; d--) { - level = Math::Max(level, *level_marks(d, idx[d])); - } - - SIZE curr_region = 0; - for (int d = D - 1; d >= 0; d--) { - SIZE bit = level == *level_marks(d, idx[d]); - curr_region += bit << d; - } - - // region size - SIZE coarse_level_size[D]; - SIZE diff_level_size[D]; - for (int d = D - 1; d >= 0; d--) { - coarse_level_size[d] = *level_ranges(level, d); - diff_level_size[d] = - *level_ranges(level + 1, d) - *level_ranges(level, d); - } - - SIZE curr_region_dims[D]; - for (int d = D - 1; d >= 0; d--) { - // Use region id to decode dimension of this region - SIZE bit = (curr_region >> d) & 1u; - curr_region_dims[d] = bit ? diff_level_size[d] : coarse_level_size[d]; - } - - SIZE curr_region_size = 1; - for (int d = D - 1; d >= 0; d--) { - curr_region_size *= curr_region_dims[d]; - } - - // region offset - SIZE curr_region_offset = 0; - // prev_region start with 1 since that is the region id of the first - // region of current level - for (SIZE prev_region = 1; prev_region < curr_region; prev_region++) { - SIZE prev_region_size = 1; - for (int d = D - 1; d >= 0; d--) { - // Use region id to decode dimension of a previous region - SIZE bit = (prev_region >> d) & 1u; - // Calculate the num of elements of the previous region - prev_region_size *= bit ? diff_level_size[d] : coarse_level_size[d]; - } - curr_region_offset += prev_region_size; - } - - // printf("(%u %u): level: %u, curr_region: %u, curr_region_offset: %u\n", - // idx[0], idx[1], level, curr_region, curr_region_offset); - - // thread offset - SIZE curr_region_thread_idx[D]; - SIZE curr_thread_offset = 0; - SIZE coarse_level_offset = 0; - for (int d = D - 1; d >= 0; d--) { - SIZE bit = (curr_region >> d) & 1u; - curr_region_thread_idx[d] = bit ? idx[d] - coarse_level_size[d] : idx[d]; - } - - SIZE global_data_idx[D]; - for (int d = D - 1; d >= 0; d--) { - SIZE bit = (curr_region >> d) & 1u; - if (level == 0) { - global_data_idx[d] = curr_region_thread_idx[d]; - } else if (*level_ranges(level + 1, d) % 2 == 0 && - curr_region_thread_idx[d] == *level_ranges(level + 1, d) / 2) { - global_data_idx[d] = *level_ranges(level + 1, d) - 1; - } else { - global_data_idx[d] = curr_region_thread_idx[d] * 2 + bit; - } - } - - SIZE stride = 1; - for (int d = D - 1; d >= 0; d--) { - curr_thread_offset += global_data_idx[d] * stride; - stride *= *level_ranges(level + 1, d); - } - - stride = 1; - for (int d = D - 1; d >= 0; d--) { - if (global_data_idx[d] % 2 != 0 && - global_data_idx[d] != *level_ranges(level + 1, d) - 1) { - coarse_level_offset = 0; - } - if (global_data_idx[d]) { - coarse_level_offset += ((global_data_idx[d] - 1) / 2 + 1) * stride; - } - stride *= (*level_ranges(level + 1, d)) / 2 + 1; - } - - if (level == 0) - coarse_level_offset = 0; - - SIZE level_offset = curr_thread_offset - coarse_level_offset; - return level_offset; - } - MGARDX_EXEC void Operation1() { // determine global idx SIZE firstD = div_roundup(v.shape(D - 1), F); @@ -201,62 +93,24 @@ class LevelwiseLinearQuantizerNDFunctor : public Functor { QUANTIZED_INT quantized_data; if constexpr (OP == MGARDX_QUANTIZE) { - if (sizeof(T) == sizeof(double)) + if constexpr (sizeof(T) == sizeof(double)) { quantized_data = copysign((T)0.5 + fabs(t * quantizer * volume), t); - else if (sizeof(T) == sizeof(float)) + } else if constexpr (sizeof(T) == sizeof(float)) { quantized_data = copysign((T)0.5 + fabsf(t * quantizer * volume), t); + } + // Shift quantized values into the non-negative dictionary range the + // Huffman coder expects. Folding the shift here makes it free (the + // element is already being written) and removes a standalone full-array + // pass. Gated by prep_huffman so an alternative lossless backend that + // does not need outlier handling keeps the raw signed values. The value + // stays signed (QUANTIZED_INT); out-of-range entries are detected later + // by the outlier separation pass. if (prep_huffman) { quantized_data += dict_size / 2; - if (quantized_data >= 0 && quantized_data < dict_size) { - // do nothing - } else { - ATOMIC_IDX outlier_write_offset = - Atomic::Add(outlier_count((IDX)0), (ATOMIC_IDX)1); - - ATOMIC_IDX outlier_idx = 0; - if (!level_linearize) { - // calculate the outlier index in the non-level linearized order - ATOMIC_IDX curr_stride = 1; - for (int d = D - 1; d >= 0; d--) { - outlier_idx += idx[d] * curr_stride; - curr_stride *= v.shape(d); - } - } else { - // calculate the outlier index in the level linearized order - SIZE level_offset = calc_level_offset(); - // Assume we put it in quantized_linearized_v and calculate its - // offset - outlier_idx = quantized_linearized_v[level](level_offset) - - quantized_v.data(); - } - // Avoid out of range error - // If we have too much outlier than our allocation - // we return the true outlier_count and do quanziation again - if (outlier_write_offset < outlier_indexes.shape(0)) { - *outlier_indexes(outlier_write_offset) = outlier_idx; - *outliers(outlier_write_offset) = quantized_data; - } - quantized_data = 0; - } - } - if (!level_linearize) { - // store quantized value in non-level linearized position - quantized_v[idx] = quantized_data; - } else { - // store quantized value in level linearized position - SIZE level_offset = calc_level_offset(); - *(quantized_linearized_v[level](level_offset)) = quantized_data; } + quantized_v[idx] = quantized_data; } else if constexpr (OP == MGARDX_DEQUANTIZE) { - if (!level_linearize) { - // read quantized value in non-level linearized position - quantized_data = quantized_v[idx]; - } else { - // read quantized value in level linearized position - SIZE level_offset = calc_level_offset(); - quantized_data = *(quantized_linearized_v[level](level_offset)); - } + quantized_data = quantized_v[idx]; if (prep_huffman) { quantized_data -= dict_size / 2; } @@ -279,20 +133,10 @@ class LevelwiseLinearQuantizerNDFunctor : public Functor { SubArray<3, T, DeviceType> level_volumes; SubArray v; SubArray quantized_v; - SubArray<1, QUANTIZED_INT, DeviceType> *quantized_linearized_v; - bool prep_huffman; bool calc_vol; - bool level_linearize; + bool prep_huffman; SIZE dict_size; SubArray<1, SIZE, DeviceType> shape; - SubArray<1, ATOMIC_IDX, DeviceType> outlier_count; - SubArray<1, ATOMIC_IDX, DeviceType> outlier_indexes; - SubArray<1, QUANTIZED_INT, DeviceType> outliers; - - T *volumes_0; - T *volumes_1; - T *volumes_2; - T *volumes_3_plus; SIZE idx[D]; // thread global idx SIZE idx0[D]; // block global idx @@ -300,55 +144,6 @@ class LevelwiseLinearQuantizerNDFunctor : public Functor { int level; }; -template -class OutlierRestoreFunctor : public Functor { -public: - MGARDX_CONT OutlierRestoreFunctor() {} - MGARDX_CONT - OutlierRestoreFunctor(SubArray quantized_v, - ATOMIC_IDX outlier_count, - SubArray<1, ATOMIC_IDX, DeviceType> outlier_indexes, - SubArray<1, QUANTIZED_INT, DeviceType> outliers) - : quantized_v(quantized_v), outlier_count(outlier_count), - outlier_indexes(outlier_indexes), outliers(outliers) { - Functor(); - } - - MGARDX_EXEC void Operation1() { - threadId = (FunctorBase::GetThreadIdZ() * - (FunctorBase::GetBlockDimX() * - FunctorBase::GetBlockDimY())) + - (FunctorBase::GetThreadIdY() * - FunctorBase::GetBlockDimX()) + - FunctorBase::GetThreadIdX(); - blockId = (FunctorBase::GetBlockIdZ() * - (FunctorBase::GetGridDimX() * - FunctorBase::GetGridDimY())) + - (FunctorBase::GetBlockIdY() * - FunctorBase::GetGridDimX()) + - FunctorBase::GetBlockIdX(); - gloablId = blockId * FunctorBase::GetBlockDimX() * - FunctorBase::GetBlockDimY() * - FunctorBase::GetBlockDimZ() + - threadId; - - if (gloablId < outlier_count) { - ATOMIC_IDX linerized_idx = *outlier_indexes(gloablId); - QUANTIZED_INT outliter = *outliers(gloablId); - *quantized_v(linerized_idx) = outliter; - } - } - - MGARDX_CONT size_t shared_memory_size() { return 0; } - -private: - IDX threadId, blockId, gloablId; - SubArray quantized_v; - ATOMIC_IDX outlier_count; - SubArray<1, ATOMIC_IDX, DeviceType> outlier_indexes; - SubArray<1, QUANTIZED_INT, DeviceType> outliers; -}; - template class LevelwiseLinearQuantizerKernel : public Kernel { public: @@ -360,22 +155,15 @@ class LevelwiseLinearQuantizerKernel : public Kernel { SubArray<2, SIZE, DeviceType> level_ranges, SubArray<2, int, DeviceType> level_marks, SIZE l_target, SubArray<1, T, DeviceType> quantizers, - SubArray<3, T, DeviceType> level_volumes, T s, SIZE dict_size, + SubArray<3, T, DeviceType> level_volumes, bool calc_vol, SubArray v, SubArray quantized_v, bool prep_huffman, - bool level_linearize, - SubArray<1, QUANTIZED_INT, DeviceType> *quantized_linearized_v, - SubArray<1, ATOMIC_IDX, DeviceType> outlier_count, - SubArray<1, ATOMIC_IDX, DeviceType> outlier_indexes, - SubArray<1, QUANTIZED_INT, DeviceType> outliers) + SIZE dict_size) : level_ranges(level_ranges), level_marks(level_marks), l_target(l_target), quantizers(quantizers), - level_volumes(level_volumes), s(s), dict_size(dict_size), v(v), + level_volumes(level_volumes), calc_vol(calc_vol), v(v), quantized_v(quantized_v), prep_huffman(prep_huffman), - level_linearize(level_linearize), - quantized_linearized_v(quantized_linearized_v), - outlier_count(outlier_count), outlier_indexes(outlier_indexes), - outliers(outliers) {} + dict_size(dict_size) {} template MGARDX_CONT @@ -384,12 +172,9 @@ class LevelwiseLinearQuantizerKernel : public Kernel { using FunctorType = LevelwiseLinearQuantizerNDFunctor; - bool calc_vol = - s != std::numeric_limits::infinity(); // m.ntype == norm_type::L_2; FunctorType functor(level_ranges, level_marks, l_target, quantizers, - level_volumes, v, quantized_v, quantized_linearized_v, - prep_huffman, calc_vol, level_linearize, dict_size, - outlier_count, outlier_indexes, outliers); + level_volumes, calc_vol, v, quantized_v, prep_huffman, + dict_size); SIZE total_thread_z = v.shape(D - 3); SIZE total_thread_y = v.shape(D - 2); @@ -419,59 +204,13 @@ class LevelwiseLinearQuantizerKernel : public Kernel { SIZE l_target; SubArray<1, T, DeviceType> quantizers; SubArray<3, T, DeviceType> level_volumes; - T s; + bool calc_vol; SubArray v; SubArray quantized_v; bool prep_huffman; - bool level_linearize; - SubArray<1, QUANTIZED_INT, DeviceType> *quantized_linearized_v; SIZE dict_size; + bool level_linearize; SubArray<1, SIZE, DeviceType> shape; - SubArray<1, ATOMIC_IDX, DeviceType> outlier_count; - SubArray<1, ATOMIC_IDX, DeviceType> outlier_indexes; - SubArray<1, QUANTIZED_INT, DeviceType> outliers; -}; - -template -class OutlierRestoreKernel : public Kernel { -public: - // 1D parallelization - constexpr static DIM NumDim = 1; - using DataType = T; - constexpr static std::string_view Name = "ork"; - constexpr static bool EnableAutoTuning() { return false; } - MGARDX_CONT - OutlierRestoreKernel(SubArray quantized_v, - ATOMIC_IDX outlier_count, - SubArray<1, ATOMIC_IDX, DeviceType> outlier_indexes, - SubArray<1, QUANTIZED_INT, DeviceType> outliers) - : quantized_v(quantized_v), outlier_count(outlier_count), - outlier_indexes(outlier_indexes), outliers(outliers) {} - - MGARDX_CONT Task> - GenTask(int queue_idx) { - using FunctorType = OutlierRestoreFunctor; - FunctorType functor(quantized_v, outlier_count, outlier_indexes, outliers); - SIZE total_thread_z = 1; - SIZE total_thread_y = 1; - SIZE total_thread_x = outlier_count; - SIZE tbx, tby, tbz, gridx, gridy, gridz; - size_t sm_size = functor.shared_memory_size(); - tbz = 1; - tby = 1; - tbx = 256; - gridz = ceil((double)total_thread_z / tbz); - gridy = ceil((double)total_thread_y / tby); - gridx = ceil((double)total_thread_x / tbx); - return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, - std::string(Name)); - } - -private: - SubArray quantized_v; - ATOMIC_IDX outlier_count; - SubArray<1, ATOMIC_IDX, DeviceType> outlier_indexes; - SubArray<1, QUANTIZED_INT, DeviceType> outliers; }; template @@ -494,7 +233,8 @@ class LinearQuantizer : public QuantizationInterface { void CalcQuantizers(size_t dof, T *quantizers, enum error_bound_type type, T tol, T s, T norm, SIZE l_target, - enum decomposition_type decomposition, bool reciprocal) { + enum decomposition_type decomposition, + bool orthogonal_projection, bool reciprocal) { double abs_tol = tol; if (type == error_bound_type::REL) { @@ -507,11 +247,23 @@ class LinearQuantizer : public QuantizationInterface { for (int l = 0; l < l_target + 1; l++) { if (decomposition == decomposition_type::MultiDim || decomposition == decomposition_type::Hybrid) { - // ben - quantizers[l] = (abs_tol) / ((l_target + 1) * (1 + std::pow(3, D))); + if (!orthogonal_projection) { + // Hierarchical basis (no mass-matrix correction): reconstruction is + // multilinear prolongation, a partition of unity, so a level-l + // quantization error propagates to any node with amplification + // <= 1. The max error is then bounded by sum over (l_target+1) + // levels of (step/2), i.e. the (1+3^D) orthogonal amplification + // drops out. + quantizers[l] = (abs_tol) / (l_target + 1); + } else { + // ben + quantizers[l] = (abs_tol) / ((l_target + 1) * (1 + std::pow(3, D))); + } // xin // quantizers[l] = (tol) / ((l_target + 1) * (1 + 3 * std::sqrt(3) / // 4)); + // quantizers[l] = (0.5 * abs_tol) / std::pow(2, l_target - 1); + } else if (decomposition == decomposition_type::SingleDim) { // ken quantizers[l] = @@ -564,10 +316,16 @@ class LinearQuantizer : public QuantizationInterface { void Quantize(SubArray original_data, enum error_bound_type ebtype, T tol, T s, T norm, SubArray quantized_data, - LosslessCompressorType &lossless, int queue_idx) { - - bool prep_huffman = - config.lossless != lossless_type::CPU_Lossless; // always do Huffman + LosslessCompressorType &lossless, int queue_idx, + bool orthogonal_projection = true) { + + // Toggle controlled from outside via the configured lossless backend: when + // a backend needs a non-negative Huffman dictionary we fold the dictionary + // shift into quantization; alternative backends that handle signed values + // directly leave the quantized data untouched (no shift / no outliers). + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; SIZE total_elems = hierarchy->total_num_elems(); SubArray<2, SIZE, DeviceType> level_ranges_subarray( hierarchy->level_ranges()); @@ -577,40 +335,10 @@ class LinearQuantizer : public QuantizationInterface { SubArray<1, T, DeviceType> quantizers_subarray(quantizers_array); T *quantizers = new T[hierarchy->l_target() + 1]; CalcQuantizers(total_elems, quantizers, ebtype, tol, s, norm, - hierarchy->l_target(), config.decomposition, true); + hierarchy->l_target(), config.decomposition, + orthogonal_projection, true); MemoryManager::Copy1D(quantizers_subarray.data(), quantizers, hierarchy->l_target() + 1, queue_idx); - ATOMIC_IDX zero = 0; - MemoryManager::Copy1D( - lossless.huffman.workspace.outlier_count_subarray.data(), &zero, 1, - queue_idx); - - SubArray<1, Q, DeviceType> *quantized_linearized_v_host = nullptr; - SubArray<1, Q, DeviceType> *quantized_linearized_v = nullptr; - - if (config.reorder) { // only if we need linerization - quantized_linearized_v_host = - new SubArray<1, Q, DeviceType>[hierarchy->l_target() + 1]; - SIZE *ranges_h = level_ranges_subarray.dataHost(); - SIZE last_level_size = 0; - for (SIZE l = 0; l < hierarchy->l_target() + 1; l++) { - SIZE level_size = 1; - for (DIM d = 0; d < D; d++) { - level_size *= ranges_h[(l + 1) * D + d]; - } - quantized_linearized_v_host[l] = SubArray<1, Q, DeviceType>( - {level_size - last_level_size}, quantized_data(last_level_size)); - last_level_size = level_size; - } - - MemoryManager::Malloc1D(quantized_linearized_v, - hierarchy->l_target() + 1, queue_idx); - DeviceRuntime::SyncDevice(); - MemoryManager::Copy1D(quantized_linearized_v, - quantized_linearized_v_host, - hierarchy->l_target() + 1, queue_idx); - DeviceRuntime::SyncDevice(); - } Timer timer; if (log::level & log::TIME) { @@ -618,65 +346,20 @@ class LinearQuantizer : public QuantizationInterface { timer.start(); } - bool done_quantization = false; - while (!done_quantization) { - DeviceLauncher::Execute( - LevelwiseLinearQuantizerKernel( - level_ranges_subarray, level_marks_subarray, - hierarchy->l_target(), quantizers_subarray, - level_volumes_subarray, s, config.huff_dict_size, original_data, - quantized_data, prep_huffman, config.reorder, - quantized_linearized_v, - lossless.huffman.workspace.outlier_count_subarray, - lossless.huffman.workspace.outlier_idx_subarray, - lossless.huffman.workspace.outlier_subarray), - queue_idx); - - MemoryManager::Copy1D( - &lossless.huffman.outlier_count, - lossless.huffman.workspace.outlier_count_subarray.data(), 1, - queue_idx); + bool calc_vol = + s != std::numeric_limits::infinity(); // m.ntype == norm_type::L_2; + DeviceLauncher::Execute( + LevelwiseLinearQuantizerKernel( + level_ranges_subarray, level_marks_subarray, hierarchy->l_target(), + quantizers_subarray, level_volumes_subarray, calc_vol, + original_data, quantized_data, prep_huffman, config.huff_dict_size), + queue_idx); + + if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); - if (lossless.huffman.outlier_count <= - lossless.huffman.workspace.outlier_subarray.shape(0)) { - // outlier buffer has sufficient size - done_quantization = true; - if (log::level & log::TIME) { - DeviceRuntime::SyncQueue(queue_idx); - timer.end(); - timer.print("Quantization"); - log::time("Quantization throughput: " + - std::to_string( - (double)(hierarchy->total_num_elems() * sizeof(T)) / - timer.get() / 1e9) + - " GB/s"); - timer.clear(); - } - log::info( - "Outlier ratio: " + std::to_string(lossless.huffman.outlier_count) + - "/" + std::to_string(total_elems) + " (" + - std::to_string((double)100 * lossless.huffman.outlier_count / - total_elems) + - "%)"); - } else { - log::info("Not enough workspace for outliers. Re-allocating to " + - std::to_string(lossless.huffman.outlier_count)); - lossless.huffman.workspace.outlier_idx_array = - Array<1, ATOMIC_IDX, DeviceType>( - {(SIZE)lossless.huffman.outlier_count}); - lossless.huffman.workspace.outlier_array = - Array<1, QUANTIZED_INT, DeviceType>( - {(SIZE)lossless.huffman.outlier_count}); - lossless.huffman.workspace.outlier_idx_subarray = - SubArray(lossless.huffman.workspace.outlier_idx_array); - lossless.huffman.workspace.outlier_subarray = - SubArray(lossless.huffman.workspace.outlier_array); - lossless.huffman.workspace.outlier_count_array.memset(0); - } - } - if (config.reorder) { - delete[] quantized_linearized_v_host; - MemoryManager::Free(quantized_linearized_v); + timer.end(); + timer.print("Quantization", hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); } delete[] quantizers; @@ -686,96 +369,53 @@ class LinearQuantizer : public QuantizationInterface { void Dequantize(SubArray original_data, enum error_bound_type ebtype, T tol, T s, T norm, SubArray quantized_data, - LosslessCompressorType &lossless_compressor, int queue_idx) { + LosslessCompressorType &lossless_compressor, int queue_idx, + bool orthogonal_projection = true) { SIZE total_elems = hierarchy->total_num_elems(); - MemoryManager::Copy1D( - lossless_compressor.huffman.workspace.outlier_count_subarray.data(), - &lossless_compressor.huffman.outlier_count, 1, queue_idx); SubArray<2, SIZE, DeviceType> level_ranges_subarray( hierarchy->level_ranges()); SubArray<2, int, DeviceType> level_marks_subarray(hierarchy->level_marks()); SubArray<3, T, DeviceType> level_volumes_subarray( hierarchy->level_volumes(true)); - bool prep_huffman = config.lossless != lossless_type::CPU_Lossless; + // Must mirror the toggle used during Quantize so the dictionary shift is + // undone exactly when it was applied. + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; SubArray<1, T, DeviceType> quantizers_subarray(quantizers_array); T *quantizers = new T[hierarchy->l_target() + 1]; CalcQuantizers(total_elems, quantizers, ebtype, tol, s, norm, - hierarchy->l_target(), config.decomposition, false); + hierarchy->l_target(), config.decomposition, + orthogonal_projection, false); MemoryManager::Copy1D(quantizers_subarray.data(), quantizers, hierarchy->l_target() + 1, queue_idx); DeviceRuntime::SyncQueue(queue_idx); - SubArray<1, Q, DeviceType> *quantized_linearized_v_host = nullptr; - SubArray<1, Q, DeviceType> *quantized_linearized_v = nullptr; - if (config.reorder) { // only if we need linerization - quantized_linearized_v_host = - new SubArray<1, Q, DeviceType>[hierarchy->l_target() + 1]; - SIZE *ranges_h = level_ranges_subarray.dataHost(); - SIZE last_level_size = 0; - for (SIZE l = 0; l < hierarchy->l_target() + 1; l++) { - SIZE level_size = 1; - for (DIM d = 0; d < D; d++) { - level_size *= ranges_h[(l + 1) * D + d]; - } - quantized_linearized_v_host[l] = SubArray<1, Q, DeviceType>( - {level_size - last_level_size}, quantized_data(last_level_size)); - last_level_size = level_size; - } - - MemoryManager::Malloc1D(quantized_linearized_v, - hierarchy->l_target() + 1, queue_idx); - DeviceRuntime::SyncDevice(); - MemoryManager::Copy1D(quantized_linearized_v, - quantized_linearized_v_host, - hierarchy->l_target() + 1, queue_idx); - DeviceRuntime::SyncDevice(); - } - Timer timer; if (log::level & log::TIME) { DeviceRuntime::SyncQueue(queue_idx); timer.start(); } - if (prep_huffman && lossless_compressor.huffman.outlier_count) { - DeviceLauncher::Execute( - OutlierRestoreKernel( - quantized_data, lossless_compressor.huffman.outlier_count, - lossless_compressor.huffman.workspace.outlier_idx_subarray, - lossless_compressor.huffman.workspace.outlier_subarray), - queue_idx); - } - + bool calc_vol = + s != std::numeric_limits::infinity(); // m.ntype == norm_type::L_2; DeviceLauncher::Execute( LevelwiseLinearQuantizerKernel( level_ranges_subarray, level_marks_subarray, hierarchy->l_target(), - quantizers_subarray, level_volumes_subarray, s, - config.huff_dict_size, original_data, quantized_data, prep_huffman, - config.reorder, quantized_linearized_v, - lossless_compressor.huffman.workspace.outlier_count_subarray, - lossless_compressor.huffman.workspace.outlier_idx_subarray, - lossless_compressor.huffman.workspace.outlier_subarray), + quantizers_subarray, level_volumes_subarray, calc_vol, + original_data, quantized_data, prep_huffman, config.huff_dict_size), queue_idx); DeviceRuntime::SyncQueue(queue_idx); if (log::level & log::TIME) { timer.end(); - timer.print("Dequantization"); - log::time( - "Dequantization throughput: " + - std::to_string((double)(hierarchy->total_num_elems() * sizeof(T)) / - timer.get() / 1e9) + - " GB/s"); + timer.print("Dequantization", hierarchy->total_num_elems() * sizeof(T)); timer.clear(); } - if (config.reorder) { - delete[] quantized_linearized_v_host; - MemoryManager::Free(quantized_linearized_v); - } delete[] quantizers; } diff --git a/include/mgard-x/Quantization/LocalQuantization.hpp b/include/mgard-x/Quantization/LocalQuantization.hpp new file mode 100644 index 0000000000..8dd97e32d9 --- /dev/null +++ b/include/mgard-x/Quantization/LocalQuantization.hpp @@ -0,0 +1,786 @@ +#ifndef MGARD_X_LOCAL_QUANTIZATION_TEMPLATE +#define MGARD_X_LOCAL_QUANTIZATION_TEMPLATE + +#include "../RuntimeX/RuntimeX.h" + +#include "QuantizationInterface.hpp" +#include +#include + +namespace mgard_x { + +#define MGARDX_QUANTIZE 1 +#define MGARDX_DEQUANTIZE 2 + +// Non-ROI Version +template +class QuantizeLocalLevelFunctor : public Functor { +public: + MGARDX_CONT QuantizeLocalLevelFunctor() {} + MGARDX_CONT QuantizeLocalLevelFunctor(T quantizer, + SubArray<1, T, DeviceType> v, + SubArray<1, Q, DeviceType> quantized_v, + bool prep_huffman, SIZE dict_size) + : quantizer(quantizer), v(v), quantized_v(quantized_v), + prep_huffman(prep_huffman), dict_size(dict_size) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + if (idx < v.shape(0)) { + T t = *v(idx); + Q quantized_data; + T volume = 1; + + if constexpr (OP == MGARDX_QUANTIZE) { + if constexpr (sizeof(T) == sizeof(double)) { + quantized_data = copysign((T)0.5 + fabs(t * quantizer * volume), t); + } else if constexpr (sizeof(T) == sizeof(float)) { + quantized_data = copysign((T)0.5 + fabsf(t * quantizer * volume), t); + } + // Fold the Huffman dictionary shift into quantization (mirrors + // LevelwiseLinearQuantizerNDFunctor): the lossless stage expects + // values in the non-negative dictionary range and separates + // out-of-range entries as outliers. + if (prep_huffman) { + quantized_data += dict_size / 2; + } + *quantized_v(idx) = quantized_data; + } else if constexpr (OP == MGARDX_DEQUANTIZE) { + quantized_data = *quantized_v(idx); + if (prep_huffman) { + quantized_data -= dict_size / 2; + } + *v(idx) = (quantizer * volume) * (T)quantized_data; + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SIZE idx; + T quantizer; + SubArray<1, T, DeviceType> v; + SubArray<1, Q, DeviceType> quantized_v; + bool prep_huffman; + SIZE dict_size; +}; + +template +class QuantizeLocalLevelKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lvl_qk"; + + MGARDX_CONT + QuantizeLocalLevelKernel(T quantizer, SubArray<1, T, DeviceType> v, + SubArray<1, Q, DeviceType> quantized_v, + bool prep_huffman, SIZE dict_size) + : quantizer(quantizer), v(v), quantized_v(quantized_v), + prep_huffman(prep_huffman), dict_size(dict_size) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = QuantizeLocalLevelFunctor; + FunctorType functor(quantizer, v, quantized_v, prep_huffman, dict_size); + + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (v.shape(0) + tbx - 1) / tbx; + SIZE gridy = 1, gridz = 1; + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, 0, queue_idx, + std::string(Name)); + } + +private: + T quantizer; + SubArray<1, T, DeviceType> v; + SubArray<1, Q, DeviceType> quantized_v; + bool prep_huffman; + SIZE dict_size; +}; + +// ROI Version +// +// The coefficient array is a row-major sequence of block-local blocks, each +// contributing hybrid_local_coeff_per_block(D) coefficients, so a +// coefficient's block index -- and therefore its ROI quantizer -- is +// idx / that count. The count is dimension dependent (3 in 1D, 39 in 2D, +// 387 in 3D), hence the D template parameter. +template +class QuantizeLocalLevelROIFunctor : public Functor { +public: + MGARDX_CONT QuantizeLocalLevelROIFunctor() {} + MGARDX_CONT QuantizeLocalLevelROIFunctor( + SubArray<1, T, DeviceType> quantizers, SubArray<1, T, DeviceType> v, + SubArray<1, Q, DeviceType> quantized_v, bool prep_huffman, SIZE dict_size) + : quantizers(quantizers), v(v), quantized_v(quantized_v), + prep_huffman(prep_huffman), dict_size(dict_size) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + if (idx < v.shape(0)) { + SIZE block_idx = idx / kCoeffPerBlock; + + T quantizer = *quantizers(block_idx); + + T t = *v(idx); + Q quantized_data; + T volume = 1; + + if constexpr (OP == MGARDX_QUANTIZE) { + if constexpr (sizeof(T) == sizeof(double)) { + quantized_data = copysign((T)0.5 + fabs(t * quantizer * volume), t); + } else if constexpr (sizeof(T) == sizeof(float)) { + quantized_data = copysign((T)0.5 + fabsf(t * quantizer * volume), t); + } + if (prep_huffman) { + quantized_data += dict_size / 2; + } + *quantized_v(idx) = quantized_data; + } else if constexpr (OP == MGARDX_DEQUANTIZE) { + quantized_data = *quantized_v(idx); + if (prep_huffman) { + quantized_data -= dict_size / 2; + } + *v(idx) = (quantizer * volume) * (T)quantized_data; + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + static constexpr SIZE kCoeffPerBlock = (SIZE)hybrid_local_coeff_per_block(D); + SIZE idx; + SubArray<1, T, DeviceType> quantizers; + SubArray<1, T, DeviceType> v; + SubArray<1, Q, DeviceType> quantized_v; + bool prep_huffman; + SIZE dict_size; +}; + +template +class QuantizeLocalLevelROIKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lvl_qk_roi"; + + MGARDX_CONT + QuantizeLocalLevelROIKernel(SubArray<1, T, DeviceType> quantizers, + SubArray<1, T, DeviceType> v, + SubArray<1, Q, DeviceType> quantized_v, + bool prep_huffman, SIZE dict_size) + : quantizers(quantizers), v(v), quantized_v(quantized_v), + prep_huffman(prep_huffman), dict_size(dict_size) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = QuantizeLocalLevelROIFunctor; + FunctorType functor(quantizers, v, quantized_v, prep_huffman, dict_size); + + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (v.shape(0) + tbx - 1) / tbx; + SIZE gridy = 1, gridz = 1; + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, 0, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, T, DeviceType> quantizers; + SubArray<1, T, DeviceType> v; + SubArray<1, Q, DeviceType> quantized_v; + bool prep_huffman; + SIZE dict_size; +}; + +// Computes per-block ROI quantizers directly on device from a device-resident +// tolerance map, avoiding a host-side loop plus a per-call H2D transfer of the +// result (the tolerance map itself is uploaded once, not on every call). +template +class ComputeROIQuantizersFunctor : public Functor { +public: + MGARDX_CONT ComputeROIQuantizersFunctor() {} + MGARDX_CONT + ComputeROIQuantizersFunctor(SubArray<1, double, DeviceType> tolerance_map, + SIZE level_offset, SIZE num_blocks, + double norm_factor, double denom, bool reciprocal, + SubArray<1, T, DeviceType> quantizers) + : tolerance_map(tolerance_map), level_offset(level_offset), + num_blocks(num_blocks), norm_factor(norm_factor), denom(denom), + reciprocal(reciprocal), quantizers(quantizers) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + if (idx < num_blocks) { + double block_tol = *tolerance_map(level_offset + idx) * norm_factor * 2; + double block_quantizer = block_tol / denom; + *quantizers(idx) = + reciprocal ? (T)(1.0 / block_quantizer) : (T)block_quantizer; + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SIZE idx; + SubArray<1, double, DeviceType> tolerance_map; + SIZE level_offset; + SIZE num_blocks; + double norm_factor; + double denom; + bool reciprocal; + SubArray<1, T, DeviceType> quantizers; +}; + +template +class ComputeROIQuantizersKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "roi_qcalc"; + + MGARDX_CONT + ComputeROIQuantizersKernel(SubArray<1, double, DeviceType> tolerance_map, + SIZE level_offset, SIZE num_blocks, + double norm_factor, double denom, bool reciprocal, + SubArray<1, T, DeviceType> quantizers) + : tolerance_map(tolerance_map), level_offset(level_offset), + num_blocks(num_blocks), norm_factor(norm_factor), denom(denom), + reciprocal(reciprocal), quantizers(quantizers) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = ComputeROIQuantizersFunctor; + FunctorType functor(tolerance_map, level_offset, num_blocks, norm_factor, + denom, reciprocal, quantizers); + + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (num_blocks + tbx - 1) / tbx; + SIZE gridy = 1, gridz = 1; + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, 0, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, double, DeviceType> tolerance_map; + SIZE level_offset; + SIZE num_blocks; + double norm_factor; + double denom; + bool reciprocal; + SubArray<1, T, DeviceType> quantizers; +}; + +template +class LocalQuantizer : public QuantizationInterface { +public: + LocalQuantizer() : initialized(false) {} + LocalQuantizer(Hierarchy &hierarchy, Config config) + : initialized(true), hierarchy(&hierarchy), config(config) { + this->L = config.num_local_refactoring_level; + this->M = config.num_global_refactoring_level; + compute_local_ranges(); + prepare_layers(); + } + + // Add logic to determine if roi or not + void Adapt(Hierarchy &hierarchy, Config config, + int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + this->config = config; + this->L = config.num_local_refactoring_level; + this->M = config.num_global_refactoring_level; + compute_local_ranges(); + prepare_layers(); + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + size_t size = 0; + return size; + } + + void SetOrthogonalProjection(bool enabled) { + orthogonal_projection = enabled; + } + + void compute_local_ranges() { + coarse_shape = hierarchy->level_shape(hierarchy->l_target()); + + fine_num_elems.clear(); + coarse_num_elems.clear(); + local_coeff_size.clear(); + fine_shapes.clear(); + coarse_shapes.clear(); + + for (int l = 0; l < this->L; ++l) { + SIZE last_level_size = 1, curr_level_size = 1; + std::vector fine_shape(D); + for (DIM d = 0; d < D; ++d) { + coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 8; + fine_shape[d] = coarse_shape[d]; + last_level_size *= coarse_shape[d]; + coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 5; + curr_level_size *= coarse_shape[d]; + } + fine_num_elems.push_back(last_level_size); + coarse_num_elems.push_back(curr_level_size); + local_coeff_size.push_back(last_level_size - curr_level_size); + fine_shapes.push_back(fine_shape); + coarse_shapes.push_back(coarse_shape); + } + } + + void prepare_layers() { + if (this->L == 0) { + layer_len.clear(); + layer_off.clear(); + return; + } + + layer_len.assign(this->L + 1, 0); + layer_off.assign(this->L + 1, 0); + + // The length of coarsest layer (level 0) + layer_len[0] = coarse_num_elems[this->L - 1]; + layer_off[0] = 0; + + SIZE accum = layer_len[0]; + + for (SIZE l = 1; l <= this->L; ++l) { + layer_len[l] = local_coeff_size[this->L - l]; + layer_off[l] = accum; + accum += layer_len[l]; + } + } + + // Calculate quantizers between levels(Used in Non-ROI) + void CalcQuantizers(size_t dof, T *quantizers, enum error_bound_type type, + T tol, T s, T norm, SIZE l_target, + enum decomposition_type decomposition, bool reciprocal) { + double abs_tol = tol; + if (type == error_bound_type::REL) { + abs_tol *= norm; + } + abs_tol *= 2; + + if (s == std::numeric_limits::infinity()) { + double C = orthogonal_projection ? (1 + std::pow(3, D)) : 1.0; + + for (int l = 0; l <= l_target; l++) { + // Modified here + quantizers[l] = (abs_tol) / (std::pow(2, l + 1) * C); + + if (reciprocal) { + quantizers[l] = 1.0f / quantizers[l]; + } + } + } else { + // s-norm (L2 family), non-ROI only. + // + // Same per-level law as LinearQuantizer's s != inf branch, + // step_l = abs_tol / (2^(s*l) * sqrt(dof)), + // with l counted from the coarsest layer, matching layer_len/layer_off. + // + // LinearQuantizer additionally scales each value by its node volume, + // reading a per-node level_volumes array. We cannot: the block-local + // quantize kernels take one scalar per layer and have no volume input. + // We do not need to. Hierarchy::calc_volume fills that array with + // total_dist / (dof - 1) -- the *average* spacing, identical for every + // node of a level -- so the per-node volume is a per-level constant and + // folds into the scalar exactly. + // + // Sign convention follows the kernels: quantize multiplies by the + // reciprocal step and by the volume, dequantize multiplies by the step + // and by the reciprocal volume, so the pair round-trips. + for (int l = 0; l <= l_target; l++) { + double step = + abs_tol / (std::exp2((double)s * l) * std::sqrt((double)dof)); + double volume = LayerVolume(l); + quantizers[l] = reciprocal ? (volume / step) : (step / volume); + } + } + } + + // sqrt of the cell volume of the grid that layer `l` lives on, matching + // LevelwiseLinearQuantizerFunctor's `volume = sqrt(prod_d spacing_d)`. + // + // Layer 0 is the coarsest data (grid coarse_shapes[L-1]); layer l >= 1 holds + // the coefficients dropped by block-local level L - l, which live on that + // level's fine grid. Coordinates are treated as normalized to a unit extent + // per dimension, which is what the block-local transform already assumes -- + // its mass/tridiagonal weights are built from a fixed uniform spacing. + double LayerVolume(int l) { + if (this->L == 0 || fine_shapes.empty()) { + return 1.0; + } + const std::vector &shape = + (l == 0) ? coarse_shapes[this->L - 1] + : fine_shapes[this->L - std::min(l, this->L)]; + double volume = 1.0; + for (DIM d = 0; d < D; d++) { + SIZE n = shape[d]; + volume *= (n > 1) ? (1.0 / (double)(n - 1)) : 1.0; + } + return std::sqrt(volume); + } + + // Reciprocal quantizers indexed by decompose level (level 0 = finest + // coefficients) for the fused decompose+quantize path. Decompose level l + // corresponds to non-ROI layer L - l, i.e. quantizer index L - l. + std::vector DecomposeLevelQuantizers(enum error_bound_type ebtype, T tol, + T s, T norm) { + std::vector quantizers(this->L + 1); + CalcQuantizers(hierarchy->total_num_elems(), quantizers.data(), ebtype, tol, + s, norm, this->L, config.decomposition, true); + std::vector level_quantizers(this->L); + for (SIZE l = 0; l < this->L; l++) { + level_quantizers[l] = quantizers[this->L - l]; + } + return level_quantizers; + } + + // Non-reciprocal dequantizers indexed by decompose level (level 0 = finest + // coefficients) for the fused dequantize+recompose path. Decompose level l + // corresponds to non-ROI layer L - l, i.e. quantizer index L - l. + std::vector RecomposeLevelDequantizers(enum error_bound_type ebtype, T tol, + T s, T norm) { + std::vector quantizers(this->L + 1); + CalcQuantizers(hierarchy->total_num_elems(), quantizers.data(), ebtype, tol, + s, norm, this->L, config.decomposition, false); + std::vector level_dequantizers(this->L); + for (SIZE l = 0; l < this->L; l++) { + level_dequantizers[l] = quantizers[this->L - l]; + } + return level_dequantizers; + } + + // Dequantize only the coarsest layer (layer 0). Used by the fused + // dequantize+recompose path when there is no global stage; the coefficient + // layers are dequantized inside the recompose kernels. + void DequantizeCoarsest(SubArray<1, T, DeviceType> v, + SubArray<1, Q, DeviceType> quantized_v, + enum error_bound_type ebtype, T tol, T s, T norm, + int queue_idx) { + std::vector quantizers(this->L + 1); + CalcQuantizers(hierarchy->total_num_elems(), quantizers.data(), ebtype, tol, + s, norm, this->L, config.decomposition, false); + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + DeviceLauncher::Execute( + QuantizeLocalLevelKernel( + quantizers[0], v, quantized_v, prep_huffman, config.huff_dict_size), + queue_idx); + } + + // Quantize only the coarsest layer (layer 0). Used by the fused + // decompose+quantize path when there is no global stage; the coefficient + // layers have already been quantized inside the decompose kernels. + void QuantizeCoarsest(SubArray<1, T, DeviceType> v, + SubArray<1, Q, DeviceType> quantized_v, + enum error_bound_type ebtype, T tol, T s, T norm, + int queue_idx) { + std::vector quantizers(this->L + 1); + CalcQuantizers(hierarchy->total_num_elems(), quantizers.data(), ebtype, tol, + s, norm, this->L, config.decomposition, true); + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + DeviceLauncher::Execute( + QuantizeLocalLevelKernel( + quantizers[0], v, quantized_v, prep_huffman, config.huff_dict_size), + queue_idx); + } + + void Quantize(SubArray original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray quantized_data, int queue_idx) {} + + void Dequantize(SubArray original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray quantized_data, int queue_idx) {} + + // Non-ROI + template + void Quantize(SubArray<1, T, DeviceType> original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray<1, Q, DeviceType> quantized_data, + LosslessCompressorType &lossless, int queue_idx) { + T *host_quantizers = new T[this->L + 1]; + CalcQuantizers(hierarchy->total_num_elems(), host_quantizers, ebtype, tol, + s, norm, this->L, config.decomposition, true); + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + SIZE huff_dict_size = config.huff_dict_size; + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE start_level = (this->M > 0) ? 1 : 0; + SIZE offset_adjustment = (this->M > 0) ? layer_off[1] : 0; + + for (SIZE l = start_level; l <= this->L; ++l) { + SIZE adjusted_off = layer_off[l] - offset_adjustment; + SubArray<1, T, DeviceType> v_in({layer_len[l]}, + original_data((IDX)adjusted_off)); + SubArray<1, Q, DeviceType> qv({layer_len[l]}, + quantized_data((IDX)adjusted_off)); + // Launch + T quantizer = host_quantizers[l]; + DeviceLauncher::Execute( + QuantizeLocalLevelKernel( + quantizer, v_in, qv, prep_huffman, huff_dict_size), + queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Local Quantization", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + + delete[] host_quantizers; + } + + // Non-ROI + template + void Dequantize(SubArray<1, T, DeviceType> original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray<1, Q, DeviceType> quantized_data, + LosslessCompressorType &lossless, int queue_idx) { + T *host_quantizers = new T[this->L + 1]; + CalcQuantizers(hierarchy->total_num_elems(), host_quantizers, ebtype, tol, + s, norm, this->L, config.decomposition, false); + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + SIZE huff_dict_size = config.huff_dict_size; + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE start_level = (this->M > 0) ? 1 : 0; + SIZE offset_adjustment = (this->M > 0) ? layer_off[1] : 0; + + for (SIZE l = start_level; l <= this->L; ++l) { + SIZE adjusted_off = layer_off[l] - offset_adjustment; + SubArray<1, T, DeviceType> v_in({layer_len[l]}, + original_data((IDX)adjusted_off)); + SubArray<1, Q, DeviceType> qv({layer_len[l]}, + quantized_data((IDX)adjusted_off)); + // Launch + T quantizer = host_quantizers[l]; + DeviceLauncher::Execute( + QuantizeLocalLevelKernel( + quantizer, v_in, qv, prep_huffman, huff_dict_size), + queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Local Dequantization", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + + delete[] host_quantizers; + } + + // With ROI + template + void Quantize(SubArray<1, T, DeviceType> original_data, + enum error_bound_type ebtype, double tol, T s, T norm, + SubArray<1, Q, DeviceType> quantized_data, + SubArray<1, double, DeviceType> device_roi_tolerance_map, + const std::vector &level_offsets, + const std::vector &level_block_counts, + LosslessCompressorType &lossless, int queue_idx) { + if (s != std::numeric_limits::infinity()) { + throw ProcessingException("Only L-inf supported"); + } + + double C = orthogonal_projection ? (1 + std::pow(3, D)) : 1.0; + double norm_factor = (ebtype == error_bound_type::REL) ? (double)norm : 1.0; + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + SIZE huff_dict_size = config.huff_dict_size; + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE accumulated_coeff_size = 0; + // Process Layer 1 to Layer L with ROI tolerances + for (SIZE l = 0; l < this->L; ++l) { + SIZE roi_level = l; + SIZE level_offset = level_offsets[roi_level]; + SIZE num_blocks = level_block_counts[roi_level]; + + // l=0 is finest coefficients (laid out at the end of the data array), + // which maps to non-ROI layer L. The correct exponent is (L - l + 1). + double denom = std::pow(2, this->L - l + 1) * C; + + // Compute per-block quantizers directly on device from the + // already-uploaded tolerance map (reciprocal for quantization), instead + // of recomputing on host and re-uploading every call. + Array<1, T, DeviceType> device_quantizers({num_blocks}, queue_idx); + DeviceLauncher::Execute( + ComputeROIQuantizersKernel( + device_roi_tolerance_map, level_offset, num_blocks, norm_factor, + denom, /*reciprocal=*/true, + SubArray<1, T, DeviceType>(device_quantizers)), + queue_idx); + + accumulated_coeff_size += local_coeff_size[l]; + SubArray<1, T, DeviceType> v_in( + {local_coeff_size[l]}, + original_data(original_data.shape(0) - accumulated_coeff_size)); + SubArray<1, Q, DeviceType> qv( + {local_coeff_size[l]}, + quantized_data(quantized_data.shape(0) - accumulated_coeff_size)); + + DeviceLauncher::Execute( + QuantizeLocalLevelROIKernel( + SubArray<1, T, DeviceType>(device_quantizers), v_in, qv, + prep_huffman, huff_dict_size), + queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Local Quantization with ROI", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + // With ROI + template + void Dequantize(SubArray<1, T, DeviceType> original_data, + enum error_bound_type ebtype, double tol, T s, T norm, + SubArray<1, Q, DeviceType> quantized_data, + SubArray<1, double, DeviceType> device_roi_tolerance_map, + const std::vector &level_offsets, + const std::vector &level_block_counts, + LosslessCompressorType &lossless, int queue_idx) { + if (s != std::numeric_limits::infinity()) { + throw ProcessingException("Only L-inf supported"); + } + + double C = orthogonal_projection ? (1 + std::pow(3, D)) : 1.0; + double norm_factor = (ebtype == error_bound_type::REL) ? (double)norm : 1.0; + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + SIZE huff_dict_size = config.huff_dict_size; + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE accumulated_coeff_size = 0; + // Process Layer 1 to Layer L with ROI tolerances + for (SIZE l = 0; l < this->L; ++l) { + SIZE roi_level = l; + SIZE level_offset = level_offsets[roi_level]; + SIZE num_blocks = level_block_counts[roi_level]; + + // l=0 is finest coefficients (laid out at the end of the data array), + // which maps to non-ROI layer L. The correct exponent is (L - l + 1). + double denom = std::pow(2, this->L - l + 1) * C; + + // Compute per-block quantizers directly on device from the + // already-uploaded tolerance map (no reciprocal for dequantization). + Array<1, T, DeviceType> device_quantizers({num_blocks}, queue_idx); + DeviceLauncher::Execute( + ComputeROIQuantizersKernel( + device_roi_tolerance_map, level_offset, num_blocks, norm_factor, + denom, /*reciprocal=*/false, + SubArray<1, T, DeviceType>(device_quantizers)), + queue_idx); + + accumulated_coeff_size += local_coeff_size[l]; + SubArray<1, T, DeviceType> v_in( + {local_coeff_size[l]}, + original_data(original_data.shape(0) - accumulated_coeff_size)); + SubArray<1, Q, DeviceType> qv( + {local_coeff_size[l]}, + quantized_data(quantized_data.shape(0) - accumulated_coeff_size)); + + DeviceLauncher::Execute( + QuantizeLocalLevelROIKernel( + SubArray<1, T, DeviceType>(device_quantizers), v_in, qv, + prep_huffman, huff_dict_size), + queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Local Dequantization with ROI", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + bool initialized; + SIZE L; + SIZE M; + Hierarchy *hierarchy; + Config config; + bool orthogonal_projection = true; + + // For Non-ROI + std::vector layer_len; + std::vector layer_off; + + // For ROI + std::vector tol_table; + + std::vector fine_num_elems; + std::vector coarse_num_elems; + std::vector local_coeff_size; + std::vector coarse_shape; + // Grid each block-local level lives on, needed by the s != inf quantizer to + // weight a layer by its node spacing. + std::vector> fine_shapes; + std::vector> coarse_shapes; +}; + +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/Quantization/QuantizationInterface.hpp b/include/mgard-x/Quantization/QuantizationInterface.hpp index 28637b3508..5f2b90686c 100644 --- a/include/mgard-x/Quantization/QuantizationInterface.hpp +++ b/include/mgard-x/Quantization/QuantizationInterface.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_QUANTIZATION_INTERFACE_HPP diff --git a/include/mgard-x/RuntimeX/AutoTuners/AutoTuner.h b/include/mgard-x/RuntimeX/AutoTuners/AutoTuner.h index 39278c2766..11a9fdcee1 100644 --- a/include/mgard-x/RuntimeX/AutoTuners/AutoTuner.h +++ b/include/mgard-x/RuntimeX/AutoTuners/AutoTuner.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_AUTOTUNER_H @@ -61,8 +61,7 @@ MGARDX_CONT void FillAutoTunerTable(std::string kernel_name, int config) { } else if (std::is_same::value) { device_type_string = "Sycl"; } else { - std::cout << log::log_err << "invalid device_type in FillAutoTunerTable.\n"; - exit(-1); + throw std::runtime_error("invalid device_type in FillAutoTunerTable."); } string curr_file_path = __FILE__; diff --git a/include/mgard-x/RuntimeX/AutoTuners/AutoTunerCuda.h b/include/mgard-x/RuntimeX/AutoTuners/AutoTunerCuda.h index aba4b688ad..fad5c794fd 100644 --- a/include/mgard-x/RuntimeX/AutoTuners/AutoTunerCuda.h +++ b/include/mgard-x/RuntimeX/AutoTuners/AutoTunerCuda.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_AUTOTUNER_CUDA_H @@ -72,8 +72,8 @@ template <> class AutoTuningTable { static constexpr int lwdqzk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, {0, 0, 0, 0, 0, 0, 0, 0, 0}}; - static constexpr int llk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, - {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + static constexpr int llk[num_types][num_dims] = {{3, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; static constexpr int sdck[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, {0, 0, 0, 0, 0, 0, 0, 0, 0}}; diff --git a/include/mgard-x/RuntimeX/AutoTuners/AutoTunerHip.h b/include/mgard-x/RuntimeX/AutoTuners/AutoTunerHip.h index 0028e73114..84c55f2bd3 100644 --- a/include/mgard-x/RuntimeX/AutoTuners/AutoTunerHip.h +++ b/include/mgard-x/RuntimeX/AutoTuners/AutoTunerHip.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_AUTOTUNER_HIP_H @@ -66,20 +66,20 @@ template <> class AutoTuningTable { static constexpr int lwpk[num_types][num_dims] = {{3, 5, 5, 5, 2, 1, 2, 0, 0}, {3, 6, 4, 1, 2, 1, 1, 0, 0}}; - static constexpr int lwqzk[num_types][num_dims] = {{4, 4, 3, 3, 3, 0, 2, 0, 0}, - {2, 3, 3, 3, 3, 0, 2, 0, 0}}; + static constexpr int lwqzk[num_types][num_dims] = {{4, 6, 5, 5, 4, 0, 2, 0, 0}, + {3, 4, 6, 5, 4, 0, 2, 0, 0}}; - static constexpr int lwdqzk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, - {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + static constexpr int lwdqzk[num_types][num_dims] = {{4, 4, 3, 3, 3, 0, 2, 0, 0}, + {2, 3, 3, 3, 3, 0, 2, 0, 0}}; - static constexpr int llk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, - {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + static constexpr int llk[num_types][num_dims] = {{3, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; - static constexpr int sdck[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, - {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + static constexpr int sdck[num_types][num_dims] = {{3, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; - static constexpr int sdmtk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, - {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + static constexpr int sdmtk[num_types][num_dims] = {{3, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; static constexpr int encode[num_types][num_dims] = {{4, 4, 4, 4, 4, 4, 4, 4, 4}, {4, 4, 4, 4, 4, 4, 4, 4, 4}}; @@ -87,7 +87,7 @@ template <> class AutoTuningTable { static constexpr int deflate[num_types][num_dims] = {{1, 3, 3, 3, 3, 3, 3, 3, 3}, {3, 3, 3, 3, 3, 3, 3, 3, 3}}; - static constexpr int decode[num_types][num_dims] = {{1, 3, 3, 3, 3, 3, 3, 3, 3}, + static constexpr int decode[num_types][num_dims] = {{5, 3, 3, 3, 3, 3, 3, 3, 3}, {3, 3, 3, 3, 3, 3, 3, 3, 3}}; }; diff --git a/include/mgard-x/RuntimeX/AutoTuners/AutoTunerKokkos.h b/include/mgard-x/RuntimeX/AutoTuners/AutoTunerKokkos.h index 0f3afa176c..191bfa8f22 100644 --- a/include/mgard-x/RuntimeX/AutoTuners/AutoTunerKokkos.h +++ b/include/mgard-x/RuntimeX/AutoTuners/AutoTunerKokkos.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_AUTOTUNER_KOKKOS_H diff --git a/include/mgard-x/RuntimeX/AutoTuners/AutoTunerOpenmp.h b/include/mgard-x/RuntimeX/AutoTuners/AutoTunerOpenmp.h index 479d2ea91d..d6302d877a 100644 --- a/include/mgard-x/RuntimeX/AutoTuners/AutoTunerOpenmp.h +++ b/include/mgard-x/RuntimeX/AutoTuners/AutoTunerOpenmp.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_AUTOTUNER_OPENMP_H diff --git a/include/mgard-x/RuntimeX/AutoTuners/AutoTunerSerial.h b/include/mgard-x/RuntimeX/AutoTuners/AutoTunerSerial.h index 4edbb78f2d..d931f2c074 100644 --- a/include/mgard-x/RuntimeX/AutoTuners/AutoTunerSerial.h +++ b/include/mgard-x/RuntimeX/AutoTuners/AutoTunerSerial.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_AUTOTUNER_SERIAL_H diff --git a/include/mgard-x/RuntimeX/AutoTuners/AutoTunerSycl.h b/include/mgard-x/RuntimeX/AutoTuners/AutoTunerSycl.h index 3d6e326e49..093326a8ca 100644 --- a/include/mgard-x/RuntimeX/AutoTuners/AutoTunerSycl.h +++ b/include/mgard-x/RuntimeX/AutoTuners/AutoTunerSycl.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_AUTOTUNER_SYCL_H diff --git a/include/mgard-x/RuntimeX/DataStructures/Array.h b/include/mgard-x/RuntimeX/DataStructures/Array.h index abb2f7de68..cfb30c77c2 100644 --- a/include/mgard-x/RuntimeX/DataStructures/Array.h +++ b/include/mgard-x/RuntimeX/DataStructures/Array.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_ARRAY @@ -12,15 +12,15 @@ namespace mgard_x { -template class Array { +template +class Array { public: Array(); - Array(std::vector shape, bool pitched = true, bool managed = false, - int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + Array(std::vector shape, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); Array(std::vector shape, T *dv); void initialize(std::vector shape); - void allocate(bool pitched, bool managed, - int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + void allocate(int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); void copy(const Array &array, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); void move(Array &&array); void memset(int value, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); @@ -32,6 +32,8 @@ template class Array { ~Array(); void load(const T *data, SIZE ld = 0, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + void hostAllocate(bool keep = false, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); T *hostCopy(bool keep = false, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); T *data(SIZE &ld); SIZE &shape(DIM d); @@ -45,13 +47,11 @@ template class Array { bool hasDeviceAllocation(); bool hasHostAllocation(); int resideDevice(); - void resize(std::vector shape, + bool resize(std::vector shape, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); private: int dev_id; - bool pitched; - bool managed; bool keepHostCopy = false; T *dv = nullptr; T *hv = nullptr; diff --git a/include/mgard-x/RuntimeX/DataStructures/Array.hpp b/include/mgard-x/RuntimeX/DataStructures/Array.hpp index 5aa4c19581..2775758d78 100644 --- a/include/mgard-x/RuntimeX/DataStructures/Array.hpp +++ b/include/mgard-x/RuntimeX/DataStructures/Array.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_ARRAY_HPP @@ -19,20 +19,21 @@ namespace mgard_x { -template -Array::Array() { +template +Array::Array() { initialize(std::vector(D, 1)); } -template -Array::Array(std::vector shape, bool pitched, - bool managed, int queue_idx) { +template +Array::Array(std::vector shape, + int queue_idx) { initialize(shape); - allocate(pitched, managed, queue_idx); + allocate(queue_idx); } -template -Array::Array(std::vector shape, T *dv) { +template +Array::Array(std::vector shape, + T *dv) { initialize(shape); __shape_allocation = shape; __ldvs_allocation = shape; @@ -41,13 +42,13 @@ Array::Array(std::vector shape, T *dv) { this->dv = dv; } -template -void Array::initialize(std::vector shape) { +template +void Array::initialize( + std::vector shape) { if (shape.size() != D) { - std::cerr << log::log_err << "Number of dimensions mismatch (" - << shape.size() << "!=" << D - << "). mgard_x::Array not initialized!\n"; - exit(-1); + throw std::runtime_error( + "Number of dimensions mismatch (" + std::to_string(shape.size()) + + "!=" + std::to_string(D) + "). mgard_x::Array not initialized!"); } dev_id = DeviceRuntime::GetDevice(); __shape = shape; @@ -60,17 +61,12 @@ void Array::initialize(std::vector shape) { host_allocated = false; device_allocated = false; external_allocation = false; - pitched = false; - managed = false; } -template -void Array::allocate(bool pitched, bool managed, - int queue_idx) { - this->pitched = pitched && !MemoryManager::ReduceMemoryFootprint; - this->managed = managed; - if (this->pitched) { - if (!this->managed) { +template +void Array::allocate(int queue_idx) { + if (Pitched) { + if (!Managed) { SIZE ld = 0; MemoryManager::MallocND(dv, __shape[D - 1], linearized_width, ld, queue_idx); @@ -80,7 +76,7 @@ void Array::allocate(bool pitched, bool managed, << "Does not support managed memory in pitched mode.\n"; } } else { - if (!this->managed) { + if (!Managed) { MemoryManager::Malloc1D(dv, __shape[D - 1] * linearized_width, queue_idx); } else { @@ -94,12 +90,12 @@ void Array::allocate(bool pitched, bool managed, external_allocation = false; } -template -void Array::copy(const Array &array, - int queue_idx) { +template +void Array::copy( + const Array &array, int queue_idx) { initialize(array.__shape); if (array.device_allocated) { - allocate(array.pitched, array.managed, queue_idx); + allocate(queue_idx); MemoryManager::CopyND(dv, __ldvs[D - 1], array.dv, array.__ldvs[D - 1], array.__shape[D - 1], array.linearized_width, queue_idx); @@ -109,12 +105,11 @@ void Array::copy(const Array &array, } } -template -void Array::move(Array &&array) { +template +void Array::move( + Array &&array) { initialize(array.__shape); this->dev_id = array.dev_id; - this->pitched = array.pitched; - this->managed = array.managed; if (array.device_allocated) { this->dv = array.dv; this->__ldvs = array.__ldvs; @@ -132,9 +127,10 @@ void Array::move(Array &&array) { } } -template -void Array::memset(int value, int queue_idx) { - if (this->pitched) { +template +void Array::memset(int value, + int queue_idx) { + if (Pitched) { MemoryManager::MemsetND(dv, __ldvs[D - 1], __shape[D - 1], linearized_width, value, queue_idx); } else { @@ -143,8 +139,8 @@ void Array::memset(int value, int queue_idx) { } } -template -void Array::free(int queue_idx) { +template +void Array::free(int queue_idx) { if (device_allocated && !external_allocation) { MemoryManager::Free(dv, queue_idx); device_allocated = false; @@ -157,40 +153,45 @@ void Array::free(int queue_idx) { } } -template -Array::Array(const Array &array) { +template +Array::Array( + const Array &array) { this->copy(array); } -template -Array & -Array::operator=(const Array &array) { +template +Array & +Array::operator=( + const Array &array) { // printf("Array operator =\n"); this->copy(array); return *this; } -template -Array & -Array::operator=(Array &&array) { +template +Array & +Array::operator=( + Array &&array) { // printf("Array move = \n"); this->move(std::move(array)); return *this; } -template -Array::Array(Array &&array) { +template +Array::Array( + Array &&array) { // printf("Array move\n"); this->move(std::move(array)); } -template -Array::~Array() { +template +Array::~Array() { this->free(); } -template -void Array::load(const T *data, SIZE ld, int queue_idx) { +template +void Array::load(const T *data, SIZE ld, + int queue_idx) { log::dbg("Calling Array::load"); if (ld == 0) { ld = __shape[D - 1]; @@ -198,13 +199,27 @@ void Array::load(const T *data, SIZE ld, int queue_idx) { MemoryManager::CopyND(dv, __ldvs[D - 1], data, ld, __shape[D - 1], linearized_width, queue_idx); } +template +void Array::hostAllocate(bool keep, + int queue_idx) { + log::dbg("Calling Array::hostAllocate"); + if (!device_allocated) { + throw std::runtime_error("device buffer not initialized."); + } + if (!host_allocated) { + MemoryManager::MallocHost(hv, __shape[D - 1] * linearized_width, + queue_idx); + host_allocated = true; + } + keepHostCopy = keep; +} -template -T *Array::hostCopy(bool keep, int queue_idx) { +template +T *Array::hostCopy(bool keep, + int queue_idx) { log::dbg("Calling Array::hostCopy"); if (!device_allocated) { - std::cout << log::log_err << "device buffer not initialized.\n"; - exit(-1); + throw std::runtime_error("device buffer not initialized."); } if (!host_allocated) { MemoryManager::MallocHost(hv, __shape[D - 1] * linearized_width, @@ -218,28 +233,27 @@ T *Array::hostCopy(bool keep, int queue_idx) { return hv; } -template -T *Array::data(SIZE &ld) { +template +T *Array::data(SIZE &ld) { if (!device_allocated) { - std::cout << log::log_err << "device buffer not initialized.\n"; - exit(-1); + throw std::runtime_error("device buffer not initialized."); } ld = __ldvs[D - 1]; return dv; } -template -SIZE &Array::shape(DIM d) { +template +SIZE &Array::shape(DIM d) { return __shape[d]; } -template -std::vector &Array::shape() { +template +std::vector &Array::shape() { return __shape; } -template -SIZE Array::totalNumElems() { +template +SIZE Array::totalNumElems() { SIZE total_num_elems = 1; for (DIM d = 0; d < D; d++) { total_num_elems *= __shape[d]; @@ -247,59 +261,62 @@ SIZE Array::totalNumElems() { return total_num_elems; } -template -T *Array::data() { +template +T *Array::data() { if (!device_allocated) { - std::cout << log::log_err << "device buffer not initialized.\n"; - exit(-1); + throw std::runtime_error("device buffer not initialized."); } return dv; } -template -T *Array::dataHost() { +template +T *Array::dataHost() { if (!host_allocated) { - std::cout << log::log_err << "host buffer not initialized.\n"; - exit(-1); + throw std::runtime_error("host buffer not initialized."); } return hv; } -template -SIZE Array::ld(DIM d) { +template +SIZE Array::ld(DIM d) { return __ldvs[d]; } -template -bool Array::isPitched() { - return pitched; +template +bool Array::isPitched() { + return Pitched; } -template -bool Array::isManaged() { - return managed; +template +bool Array::isManaged() { + return Managed; } -template -int Array::resideDevice() { +template +int Array::resideDevice() { return dev_id; } -template -bool Array::hasDeviceAllocation() { +template +bool Array::hasDeviceAllocation() { return device_allocated; } -template -bool Array::hasHostAllocation() { +template +bool Array::hasHostAllocation() { return host_allocated; } -template -void Array::resize(std::vector shape, int queue_idx) { - bool inplace_resizable = false; - if (device_allocated) { - if (!isPitched()) { +template +bool Array::resize(std::vector shape, + int queue_idx) { + if (!device_allocated) { + initialize(shape); + allocate(queue_idx); + return false; + } else { + bool inplace_resizable = false; + if (Pitched) { // check total number of elements SIZE original_num_elems = 1; SIZE new_num_elems = 1; @@ -329,17 +346,27 @@ void Array::resize(std::vector shape, int queue_idx) { // We can reuse existing allocation inplace_resizable = true; __shape = shape; + // Keep the leading dimensions in step with the shape, as initialize() + // and the pitched branch above both do. A non-pitched array is dense, + // and callers rely on that: several of them derive a stride from the + // shape rather than asking the array for its ld (see the + // setLd(fine_shapes[0][d]) sites in + // BlockLocalHierarchyDataRefactor). Leaving a shrunk array with the + // wider ld of its previous shape leaves those two strides disagreeing + // over the same buffer, which silently corrupts the data. + __ldvs = __shape; linearized_width = 1; for (DIM d = 0; d < D - 1; d++) { linearized_width *= __shape[d]; } } } - } - // If cannot reuse existing allocation or there is no existing allocation - if (!inplace_resizable) { - initialize(shape); - allocate(isPitched(), isManaged(), queue_idx); + // If cannot reuse existing allocation or there is no existing allocation + if (!inplace_resizable) { + initialize(shape); + allocate(queue_idx); + } + return inplace_resizable; } } diff --git a/include/mgard-x/RuntimeX/DataStructures/MDRData.hpp b/include/mgard-x/RuntimeX/DataStructures/MDRData.hpp index b5abafa6c2..1a60ed3e7b 100644 --- a/include/mgard-x/RuntimeX/DataStructures/MDRData.hpp +++ b/include/mgard-x/RuntimeX/DataStructures/MDRData.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MDR_X_MDR_DATA_HPP @@ -28,34 +28,58 @@ template class MDRData { } } - void Resize(SIZE num_levels, SIZE num_bitplanes) { - if (compressed_bitplanes.size() != num_levels) { - compressed_bitplanes.resize(num_levels); - } - if (level_signs.size() != num_levels) { - level_signs.resize(num_levels); - } + // Resize before refactoring + template + void Resize(RefactorType &refactor, HierarchyType &hierarchy, int queue_idx) { + std::vector> estimation = + RefactorType::EstimateMaxBitplaneSizes(hierarchy); + SIZE num_levels = estimation.size(); + SIZE num_bitplanes = estimation[0].size(); + std::vector level_num_elems = hierarchy.level_num_elems(); + compressed_bitplanes.resize(num_levels); + level_signs.resize(num_levels); for (int level_idx = 0; level_idx < num_levels; level_idx++) { - if (compressed_bitplanes[level_idx].size() != num_bitplanes) { - compressed_bitplanes[level_idx].resize(num_bitplanes); + compressed_bitplanes[level_idx].resize(num_bitplanes); + // The decoder's batched bitplane kernels (e.g. + // BPDecoderRegisterBlockFunctor::DecodeBinary) write one sign per slot + // across the full BATCH_SIZE-aligned range, exactly like + // level_data_array in ComposedRefactor/ComposedReconstructor -- so this + // needs the same round_up, not the raw element count, or the decoder + // overruns it whenever a level's element count isn't already a + // multiple of the batch size. + level_signs[level_idx].resize( + {round_up(hierarchy.level_num_elems(level_idx), + RefactorType::BATCH_SIZE)}, + queue_idx); + for (int bitplane_idx = 0; bitplane_idx < num_bitplanes; bitplane_idx++) { + compressed_bitplanes[level_idx][bitplane_idx].resize( + {estimation[level_idx][bitplane_idx]}, queue_idx); } } } - void Resize(MDRMetadata &mdr_metadata) { + // Resize before reconstruction + void Resize(MDRMetadata &mdr_metadata, int queue_idx) { compressed_bitplanes.resize(mdr_metadata.num_levels); level_signs.resize(mdr_metadata.num_levels); for (int level_idx = 0; level_idx < mdr_metadata.num_levels; level_idx++) { compressed_bitplanes[level_idx].resize(mdr_metadata.num_bitplanes); - level_signs[level_idx].resize({mdr_metadata.level_num_elems[level_idx]}); + level_signs[level_idx].resize({mdr_metadata.level_num_elems[level_idx]}, + queue_idx); for (int bitplane_idx = 0; bitplane_idx < mdr_metadata.num_bitplanes; bitplane_idx++) { compressed_bitplanes[level_idx][bitplane_idx].resize( - {mdr_metadata.level_sizes[level_idx][bitplane_idx]}); + {mdr_metadata.level_sizes[level_idx][bitplane_idx]}, queue_idx); } } } + void ResetSigns(int queue_idx) { + for (int level_idx = 0; level_idx < level_signs.size(); level_idx++) { + level_signs[level_idx].memset(0, queue_idx); + } + } + void CopyFromRefactoredData(MDRMetadata &mdr_metadata, std::vector> &refactored_data, int queue_idx) { @@ -85,19 +109,29 @@ template class MDRData { void CopyToRefactoredData(MDRMetadata &mdr_metadata, std::vector> &refactored_data, + std::vector> &allocation_size, int queue_idx) { refactored_data.resize(mdr_metadata.num_levels); for (int level_idx = 0; level_idx < mdr_metadata.num_levels; level_idx++) { refactored_data[level_idx].resize(mdr_metadata.num_bitplanes); for (int bitplane_idx = 0; bitplane_idx < mdr_metadata.num_bitplanes; bitplane_idx++) { - MemoryManager::MallocHost( - refactored_data[level_idx][bitplane_idx], - mdr_metadata.level_sizes[level_idx][bitplane_idx], queue_idx); - MemoryManager::Copy1D( - refactored_data[level_idx][bitplane_idx], - compressed_bitplanes[level_idx][bitplane_idx].data(), - mdr_metadata.level_sizes[level_idx][bitplane_idx], queue_idx); + if (allocation_size[level_idx][bitplane_idx] >= + mdr_metadata.level_sizes[level_idx][bitplane_idx]) { + MemoryManager::Copy1D( + refactored_data[level_idx][bitplane_idx], + compressed_bitplanes[level_idx][bitplane_idx].data(), + mdr_metadata.level_sizes[level_idx][bitplane_idx], queue_idx); + } else { + throw std::runtime_error( + "Bitplane copy failed. level_idx(" + std::to_string(level_idx) + + ") bitplane_idx(" + std::to_string(bitplane_idx) + + ") Insufficient buffer space " + + std::to_string(allocation_size[level_idx][bitplane_idx]) + + " vs. " + + std::to_string( + mdr_metadata.level_sizes[level_idx][bitplane_idx])); + } } } } @@ -116,17 +150,20 @@ template class MDRData { // TODO: load for (int level_idx = 0; level_idx < mdr_metadata.num_levels; level_idx++) { if (!level_signs[level_idx].hasDeviceAllocation()) { - log::err("Bitplane verification failed. level_signs(" + - std::to_string(level_idx) + ") No allocation found.\n"); - exit(-1); + throw std::runtime_error("Bitplane verification failed. level_signs(" + + std::to_string(level_idx) + + ") No allocation found.\n"); } - if (level_signs[level_idx].shape(0) != + // level_signs is allocated rounded up to the decoder's batch size (see + // Resize above), so it is expected to be >= the raw element count, not + // exactly equal to it. + if (level_signs[level_idx].shape(0) < mdr_metadata.level_num_elems[level_idx]) { - log::err("Bitplane verification failed. level_signs(" + - std::to_string(level_idx) + ") Size mismatch " + - std::to_string(level_signs[level_idx].shape(0)) + " vs. " + - std::to_string(mdr_metadata.level_num_elems[level_idx])); - exit(-1); + throw std::runtime_error( + "Bitplane verification failed. level_signs(" + + std::to_string(level_idx) + ") Insufficient buffer space " + + std::to_string(level_signs[level_idx].shape(0)) + " vs. " + + std::to_string(mdr_metadata.level_num_elems[level_idx])); } for (int bitplane_idx = mdr_metadata.prev_used_level_num_bitplanes[level_idx]; @@ -134,22 +171,22 @@ template class MDRData { bitplane_idx++) { if (!compressed_bitplanes[level_idx][bitplane_idx] .hasDeviceAllocation()) { - log::err("Bitplane verification failed. level_idx(" + - std::to_string(level_idx) + ") bitplane_idx(" + - std::to_string(bitplane_idx) + ") No allocation found.\n"); - exit(-1); + throw std::runtime_error( + "Bitplane verification failed. level_idx(" + + std::to_string(level_idx) + ") bitplane_idx(" + + std::to_string(bitplane_idx) + ") No allocation found.\n"); } if (compressed_bitplanes[level_idx][bitplane_idx].shape(0) != mdr_metadata.level_sizes[level_idx][bitplane_idx]) { - log::err("Bitplane verification failed. level_idx(" + - std::to_string(level_idx) + ") bitplane_idx(" + - std::to_string(bitplane_idx) + ") Size mismatch " + - std::to_string( - compressed_bitplanes[level_idx][bitplane_idx].shape(0)) + - " vs. " + - std::to_string( - mdr_metadata.level_sizes[level_idx][bitplane_idx])); - exit(-1); + throw std::runtime_error( + "Bitplane verification failed. level_idx(" + + std::to_string(level_idx) + ") bitplane_idx(" + + std::to_string(bitplane_idx) + ") Size mismatch " + + std::to_string( + compressed_bitplanes[level_idx][bitplane_idx].shape(0)) + + " vs. " + + std::to_string( + mdr_metadata.level_sizes[level_idx][bitplane_idx])); } } } diff --git a/include/mgard-x/RuntimeX/DataStructures/MDRMetadata.hpp b/include/mgard-x/RuntimeX/DataStructures/MDRMetadata.hpp index 42bace5701..c5483d0689 100644 --- a/include/mgard-x/RuntimeX/DataStructures/MDRMetadata.hpp +++ b/include/mgard-x/RuntimeX/DataStructures/MDRMetadata.hpp @@ -1,13 +1,15 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MDR_X_MDR_METADATA_HPP #define MDR_X_MDR_METADATA_HPP +#include + namespace mgard_x { namespace MDR { @@ -40,11 +42,18 @@ class MDRMetadata { std::vector> level_squared_errors; std::vector> level_sizes; std::vector level_num_elems; + bool segmented = false; + bool corresponding_error_return = false; + size_t retrieved_size = 0; // For progressive reconstruction T_error loaded_tol, loaded_s; T_error requested_tol, requested_s; T_error prev_tol, prev_s; + T_error tau; + uint32_t requested_size; + size_t num_elements; + double corresponding_error; std::vector loaded_level_num_bitplanes; std::vector requested_level_num_bitplanes; std::vector prev_used_level_num_bitplanes; @@ -67,8 +76,24 @@ class MDRMetadata { } } + uint32_t GetLoadedBitPlaneSizes() { + uint32_t bitplanes_size = 0; + for (int level_idx = 0; level_idx < num_levels; level_idx++) { + // std::cout << "level[" << level_idx << "]" << ", loaded bitplanes: " << + // (int)loaded_level_num_bitplanes[level_idx] << ":" << std::endl; + for (int bitplane_idx = 0; + bitplane_idx < loaded_level_num_bitplanes[level_idx]; + bitplane_idx++) { + // std::cout << (int)level_sizes[level_idx][bitplane_idx] << " "; + bitplanes_size += level_sizes[level_idx][bitplane_idx]; + } + // std::cout << "\n"; + } + return bitplanes_size; + } + void PrintStatus() { - printf("Request tol: %f, s: %f\n", requested_tol, requested_s); + printf("Request size: %u, s: %f\n", requested_size, requested_s); for (int level_idx = 0; level_idx < num_levels; level_idx++) { printf("Level %d bitplanes: used [%2d] loaded [%2d] requested [%2d]\n", level_idx, prev_used_level_num_bitplanes[level_idx], @@ -136,12 +161,12 @@ class MDRMetadata { } template void Serialize(Byte *&ptr, T *data, SIZE bytes) { - memcpy(ptr, (Byte *)data, bytes); + std::memcpy(ptr, (Byte *)data, bytes); ptr += bytes; } template void Deserialize(Byte *&ptr, T *data, SIZE bytes) { - memcpy((Byte *)data, ptr, bytes); + std::memcpy((Byte *)data, ptr, bytes); ptr += bytes; } diff --git a/include/mgard-x/RuntimeX/DataStructures/SubArray.hpp b/include/mgard-x/RuntimeX/DataStructures/SubArray.hpp index 75ca749aef..de25e2dbac 100644 --- a/include/mgard-x/RuntimeX/DataStructures/SubArray.hpp +++ b/include/mgard-x/RuntimeX/DataStructures/SubArray.hpp @@ -1,7 +1,7 @@ /* * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: Jul 20, 2021 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_SUBARRAY_HPP @@ -13,13 +13,15 @@ namespace mgard_x { -template class SubArray { +template +class SubArray { public: MGARDX_CONT_EXEC SubArray(); MGARDX_CONT - SubArray(Array &array); + SubArray(Array &array); MGARDX_CONT SubArray(std::vector shape, T *dv); @@ -39,8 +41,7 @@ template class SubArray { MGARDX_CONT T *dataHost() { if (!has_host_pointer) { - std::cerr << log::log_err << "Host pointer not initialized!\n"; - exit(-1); + throw std::runtime_error("Host pointer not initialized!"); } return v; } @@ -77,10 +78,10 @@ template class SubArray { void setLd(DIM d, SIZE ld) { __ldvs[d] = ld; } MGARDX_CONT_EXEC - bool isPitched() { return this->pitched; } + bool isPitched() { return Pitched; } - MGARDX_CONT_EXEC - void setPitched(bool pitched) { this->pitched = pitched; } + // MGARDX_CONT_EXEC + // void setPitched(bool pitched) { this->pitched = pitched; } MGARDX_CONT_EXEC SIZE lddv1() const { return __lddv1; } @@ -178,9 +179,6 @@ template class SubArray { SIZE __lddv1; SIZE __lddv2; - bool pitched; - bool managed; - SIZE ptr_offset; MGARDX_CONT_EXEC @@ -195,8 +193,9 @@ template class SubArray { } }; -template -MGARDX_CONT_EXEC void SubArray::initialize() { +template +MGARDX_CONT_EXEC void +SubArray::initialize() { dv = nullptr; v = nullptr; has_host_pointer = false; @@ -211,20 +210,17 @@ MGARDX_CONT_EXEC void SubArray::initialize() { __lddv1 = 1; __lddv2 = 1; - pitched = false; - managed = false; - ptr_offset = 0; } -template -MGARDX_CONT_EXEC SubArray::SubArray() { +template +MGARDX_CONT_EXEC SubArray::SubArray() { initialize(); } -template -MGARDX_CONT -SubArray::SubArray(Array &array) { +template +MGARDX_CONT SubArray::SubArray( + Array &array) { initialize(); dv = array.data(); for (DIM d = 0; d < D; d++) { @@ -238,14 +234,13 @@ SubArray::SubArray(Array &array) { v = array.dataHost(); has_host_pointer = true; } - pitched = array.isPitched(); - managed = array.isManaged(); } // TODO: update shape -template -MGARDX_CONT SubArray::SubArray(std::vector shape, - T *dv) { +template +MGARDX_CONT +SubArray::SubArray(std::vector shape, + T *dv) { initialize(); this->dv = dv; for (DIM d = 0; d < D; d++) { @@ -257,10 +252,11 @@ MGARDX_CONT SubArray::SubArray(std::vector shape, __lddv2 = __ldvs[D - 2]; } -template -MGARDX_CONT SubArray<1, T, DeviceType> SubArray::Linearize() { +template +MGARDX_CONT SubArray<1, T, DeviceType> +SubArray::Linearize() { SubArray<1, T, DeviceType> subArray; - if (!pitched) { + if (!Pitched) { SIZE linearized_shape = 1; for (DIM d = 0; d < D; d++) linearized_shape *= this->__shape[d]; @@ -272,24 +268,20 @@ MGARDX_CONT SubArray<1, T, DeviceType> SubArray::Linearize() { if (has_host_pointer) { subArray.setDataHost(dataHost()); } - subArray.setPitched(isPitched()); } else { - std::cout << log::log_err - << "Linearized pitched SubArray not implemented!\n"; - exit(-1); + throw std::runtime_error("Linearized pitched SubArray not implemented!"); } return subArray; } -template +template MGARDX_CONT SubArray<3, T, DeviceType> -SubArray::Slice3D(DIM d2, DIM d1, DIM d0) { +SubArray::Slice3D(DIM d2, DIM d1, DIM d0) { // d2 is slowest dim. // d0 is fastest dim. if (D < 3) { - std::cout << log::log_err << "calling Slice3D on SubArray with " << D - << "D data.\n"; - exit(-1); + throw std::runtime_error("calling Slice3D on SubArray with " + + std::to_string(D) + "D data."); } SubArray<3, T, DeviceType> subArray; subArray.setShape(2, __shape[d0]); @@ -304,12 +296,12 @@ SubArray::Slice3D(DIM d2, DIM d1, DIM d0) { if (has_host_pointer) { subArray.setDataHost(v); } - subArray.setPitched(pitched); return subArray; } -template -MGARDX_CONT void SubArray::offset(std::vector idx) { +template +MGARDX_CONT void +SubArray::offset(std::vector idx) { if (idx.size() < D) { std::cerr << log::log_err << "SubArray::resize insufficient idx length.\n"; } @@ -321,8 +313,9 @@ MGARDX_CONT void SubArray::offset(std::vector idx) { dv += calc_offset(_idx); } -template -MGARDX_CONT void SubArray::resize(std::vector shape) { +template +MGARDX_CONT void +SubArray::resize(std::vector shape) { if (shape.size() < D) { std::cerr << log::log_err << "SubArray::resize insufficient shape length.\n"; @@ -334,9 +327,10 @@ MGARDX_CONT void SubArray::resize(std::vector shape) { } } -template -MGARDX_CONT void SubArray::offset_dim(DIM dim, - SIZE offset_value) { +template +MGARDX_CONT void +SubArray::offset_dim(DIM dim, + SIZE offset_value) { if (dim >= D) return; SIZE idx[D]; @@ -347,20 +341,25 @@ MGARDX_CONT void SubArray::offset_dim(DIM dim, dv += calc_offset(idx); } -template -MGARDX_CONT void SubArray::resize(DIM dim, SIZE new_size) { +template +MGARDX_CONT void +SubArray::resize(DIM dim, SIZE new_size) { if (dim >= D) return; __shape[dim] = new_size; } -template -MGARDX_CONT void SubArray::project(DIM dim_slowest, - DIM dim_medium, - DIM dim_fastest) { +template +MGARDX_CONT void SubArray::project( + DIM dim_slowest, DIM dim_medium, DIM dim_fastest) { projected_dim_slowest = dim_slowest; projected_dim_medium = dim_medium; projected_dim_fastest = dim_fastest; + // Callers routinely pass the 3D triple (0, 1, 2) or (D-3, D-2, D-1); on a + // 1D or 2D SubArray the leading dimensions wrap or fall off the end, so + // clamp all three into range before indexing __ldvs. + if (projected_dim_fastest >= D) + projected_dim_fastest = D - 1; if (projected_dim_slowest >= D) projected_dim_slowest = 0; if (projected_dim_medium >= D) diff --git a/include/mgard-x/RuntimeX/DataStructures/SubArrayCopy.hpp b/include/mgard-x/RuntimeX/DataStructures/SubArrayCopy.hpp index 2674a60ee9..158b641edb 100644 --- a/include/mgard-x/RuntimeX/DataStructures/SubArrayCopy.hpp +++ b/include/mgard-x/RuntimeX/DataStructures/SubArrayCopy.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_SUBARRAY_COPY_HPP diff --git a/include/mgard-x/RuntimeX/DataTypes.h b/include/mgard-x/RuntimeX/DataTypes.h index d5c0f0e2fa..ec54ea7cd8 100644 --- a/include/mgard-x/RuntimeX/DataTypes.h +++ b/include/mgard-x/RuntimeX/DataTypes.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_RUNTIME_X_DATA_TYPES_H diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapter.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapter.h index 9ca0870f22..8ca9a090ab 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapter.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapter.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_DEVICE_ADAPTER_H #define MGARD_X_DEVICE_ADAPTER_H @@ -46,6 +46,7 @@ struct Atomic { MGARDX_EXEC static T Min(T *result, T value); MGARDX_EXEC static T Max(T *result, T value); MGARDX_EXEC static T Add(T *result, T value); + MGARDX_EXEC static T Or(T *result, T value); }; // based on de Bruijn sequence: @@ -171,6 +172,34 @@ template class DeviceLauncher { MGARDX_CONT static void Execute(KernelType kernel); }; +// --------------------------------------------------------------------------- +// Portable sub-group abstraction. A "sub-group" is a set of lanes that run in +// lockstep and can exchange values (shfl), vote (ballot -> mask), find the +// first set lane (ffs, 1-indexed), and barrier (sync). It lets a cooperative +// kernel be written once and lowered to each backend's native primitives: +// SubGroup -> 32-lane warp SubGroup -> wavefront +// SubGroup -> sycl::sub_group SubGroup -> size 1 +// Each backend specializes SubGroup in its DeviceAdapter file. +// +// SubGroupScalar is the size-1 group: every collective is an identity, so a +// cooperative kernel run through it degenerates to the plain serial algorithm. +// It is what the SERIAL/OPENMP specializations are, and what any +// 1-thread-per-item (non-cooperative) kernel should use on ANY backend. +struct SubGroupScalar { + using mask_t = uint32_t; + static constexpr int size() { return 1; } + MGARDX_EXEC int lane() const { return 0; } + MGARDX_EXEC mask_t full_mask() const { return 1u; } + template MGARDX_EXEC T shfl(T v, int) const { return v; } + MGARDX_EXEC mask_t ballot(int pred) const { return pred ? 1u : 0u; } + MGARDX_EXEC int ffs(mask_t m) const { return m ? 1 : 0; } + MGARDX_EXEC void sync() const {} +}; + +// Primary template -- each backend provides an explicit specialization in its +// DeviceAdapter file. Left undefined so an unsupported backend fails loudly. +template struct SubGroup; + template struct KeyValueComparator { bool operator()(std::pair a, std::pair b) const { return a.first < b.first; @@ -192,6 +221,12 @@ template class DeviceCollective { Array<1, Byte, DeviceType> &workspace, bool workspace_allocated, int queue_idx); + template + MGARDX_CONT static void AbsMin(SIZE n, SubArray<1, T, DeviceType> v, + SubArray<1, T, DeviceType> result, + Array<1, Byte, DeviceType> &workspace, + bool workspace_allocated, int queue_idx); + template MGARDX_CONT static void SquareSum(SIZE n, SubArray<1, T, DeviceType> v, SubArray<1, T, DeviceType> result, @@ -280,8 +315,6 @@ template class MemoryManager { template MGARDX_CONT static void HostRegister(T *ptr, SIZE n); template MGARDX_CONT static void HostUnregister(T *ptr); - - static bool ReduceMemoryFootprint; }; template class DeviceRuntime { diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterCuda.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterCuda.h index 6d7dccd4fb..6e97f662f5 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterCuda.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterCuda.h @@ -1,20 +1,25 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ +#include "../Utilities/Exceptions.h" #include "DeviceAdapter.h" #include #include #include // #include +#if defined(CUB_VERSION) && CUB_VERSION >= 300000 +#include +#endif #include #include #include #include +#include #include // using namespace nvcuda; @@ -115,10 +120,13 @@ template inline void ErrorAsyncCheckTask(cudaError_t code, TaskType &task, bool abort = true) { if (code != cudaSuccess) { - log::err(std::string(cudaGetErrorString(code)) + " while executing " + - task.GetFunctorName().c_str() + " with CUDA (Async-check)"); + std::string msg = std::string(cudaGetErrorString(code)) + + " while executing " + task.GetFunctorName().c_str() + + " with CUDA (Async-check)"; if (abort) - exit(code); + throw ProcessingException(msg); + else + std::cerr << msg << std::endl; } } @@ -126,30 +134,39 @@ template inline void ErrorSyncCheckTask(cudaError_t code, TaskType &task, bool abort = true) { if (code != cudaSuccess) { - log::err(std::string(cudaGetErrorString(code)) + " while executing " + - task.GetFunctorName().c_str() + " with CUDA (Sync-check)"); + std::string msg = std::string(cudaGetErrorString(code)) + + " while executing " + task.GetFunctorName().c_str() + + " with CUDA (Sync-check)"; if (abort) - exit(code); + throw ProcessingException(msg); + else + std::cerr << msg << std::endl; } } inline void ErrorAsyncCheck(cudaError_t code, std::string task, bool abort = true) { if (code != cudaSuccess) { - log::err(std::string(cudaGetErrorString(code)) + " while executing " + - task.c_str() + " with CUDA (Async-check)"); + std::string msg = std::string(cudaGetErrorString(code)) + + " while executing " + task.c_str() + + " with CUDA (Async-check)"; if (abort) - exit(code); + throw ProcessingException(msg); + else + std::cerr << msg << std::endl; } } inline void ErrorSyncCheck(cudaError_t code, std::string task, bool abort = true) { if (code != cudaSuccess) { - log::err(std::string(cudaGetErrorString(code)) + " while executing " + - task.c_str() + " with CUDA (Sync-check)"); + std::string msg = std::string(cudaGetErrorString(code)) + + " while executing " + task.c_str() + + " with CUDA (Sync-check)"; if (abort) - exit(code); + throw ProcessingException(msg); + else + std::cerr << msg << std::endl; } } @@ -159,10 +176,13 @@ inline void ErrorSyncCheck(cudaError_t code, std::string task, inline void gpuAssert(cudaError_t code, const char *file, int line, bool abort = true) { if (code != cudaSuccess) { - fprintf(stderr, "GPUassert: %s %s %d\n", cudaGetErrorString(code), file, - line); if (abort) - exit(code); + throw ProcessingException(std::string("GPUassert: ") + + cudaGetErrorString(code) + " " + file + " " + + std::to_string(line)); + else + fprintf(stderr, "GPUassert: %s %s %d\n", cudaGetErrorString(code), file, + line); } } @@ -215,6 +235,41 @@ struct Atomic { return atomicAdd(result, value); #endif } + MGARDX_EXEC static T Or(T *result, T value) { + // atomicOr is only overloaded for 32-/64-bit unsigned integers, so cast to + // the matching-width unsigned type to stay portable across H = uint32/64. + if constexpr (sizeof(T) == 8) { + using U = unsigned long long int; + U *r = reinterpret_cast(result); + U v = static_cast(value); +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 600 + if constexpr (Scope == AtomicSystemScope) { + return static_cast(atomicOr_system(r, v)); + } else if constexpr (Scope == AtomicDeviceScope) { + return static_cast(atomicOr(r, v)); + } else { + return static_cast(atomicOr_block(r, v)); + } +#else + return static_cast(atomicOr(r, v)); +#endif + } else { + using U = unsigned int; + U *r = reinterpret_cast(result); + U v = static_cast(value); +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 600 + if constexpr (Scope == AtomicSystemScope) { + return static_cast(atomicOr_system(r, v)); + } else if constexpr (Scope == AtomicDeviceScope) { + return static_cast(atomicOr(r, v)); + } else { + return static_cast(atomicOr_block(r, v)); + } +#else + return static_cast(atomicOr(r, v)); +#endif + } + } }; template <> struct Math { @@ -490,6 +545,8 @@ template <> class DeviceSpecification { public: MGARDX_CONT DeviceSpecification() { + int original_device = 0; + cudaGetDevice(&original_device); cudaGetDeviceCount(&NumDevices); MaxSharedMemorySize = new int[NumDevices]; WarpSize = new int[NumDevices]; @@ -520,6 +577,15 @@ template <> class DeviceSpecification { cudaGetDeviceProperties(&prop, d); DeviceNames[d] = std::string(prop.name); } + // This loop's per-device cudaSetDevice(d) calls otherwise leave the + // active CUDA device at NumDevices-1 for the rest of the process, + // silently out of sync with DeviceRuntime::cuda_dev_id (which + // stays at its default of 0 unless explicitly changed). Streams and + // allocations made before the mismatch is next resolved end up tied to + // the wrong device's context, which can surface much later as an + // unrelated illegal memory access. Restore the device that was active + // before this query loop ran. + gpuErrchk(cudaSetDevice(original_device)); } MGARDX_CONT int GetNumDevices() { return NumDevices; } @@ -592,6 +658,8 @@ template <> class DeviceQueues { void Initialize() { if (!initialized) { log::dbg("Calling DeviceQueues::Initialize"); + int original_device = 0; + cudaGetDevice(&original_device); cudaGetDeviceCount(&NumDevices); streams = new cudaStream_t *[NumDevices]; for (int d = 0; d < NumDevices; d++) { @@ -601,6 +669,9 @@ template <> class DeviceQueues { gpuErrchk(cudaStreamCreate(&streams[d][i])); } } + // See DeviceSpecification's constructor for why this restore matters: + // otherwise the active device is left at NumDevices-1 here too. + gpuErrchk(cudaSetDevice(original_device)); initialized = true; } } @@ -609,6 +680,8 @@ template <> class DeviceQueues { void Destroy() { if (initialized) { log::dbg("Calling DeviceQueues::Destroy"); + int original_device = 0; + cudaGetDevice(&original_device); for (int d = 0; d < NumDevices; d++) { gpuErrchk(cudaSetDevice(d)); for (int i = 0; i < MGARDX_NUM_QUEUES; i++) { @@ -616,6 +689,7 @@ template <> class DeviceQueues { } delete[] streams[d]; } + gpuErrchk(cudaSetDevice(original_device)); delete[] streams; streams = nullptr; initialized = false; @@ -653,6 +727,22 @@ template <> class DeviceQueues { extern int cuda_dev_id; #pragma omp threadprivate(cuda_dev_id) +// 32-lane warp sub-group (CUDA). Stateless: reads its lane from threadIdx. +template <> struct SubGroup { + using mask_t = unsigned; + static constexpr int size() { return 32; } + __device__ int lane() const { return threadIdx.x & 31; } + __device__ mask_t full_mask() const { return 0xffffffffu; } + template __device__ T shfl(T v, int src) const { + return __shfl_sync(0xffffffffu, v, src); + } + __device__ mask_t ballot(int pred) const { + return __ballot_sync(0xffffffffu, pred); + } + __device__ int ffs(mask_t m) const { return __ffs((int)m); } + __device__ void sync() const { __syncwarp(); } +}; + template <> class DeviceRuntime { public: MGARDX_CONT @@ -753,7 +843,7 @@ template <> class DeviceRuntime { &numBlocks, CudaHuffmanCWCustomizedKernel>, blockSize, dynamicSMemSize)); } else { - log::err("GetOccupancyMaxActiveBlocksPerSM Error!"); + throw std::runtime_error("GetOccupancyMaxActiveBlocksPerSM Error!"); } return numBlocks; } @@ -782,7 +872,7 @@ template <> class DeviceRuntime { CudaHuffmanCWCustomizedKernel>, cudaFuncAttributeMaxDynamicSharedMemorySize, maxbytes)); } else { - log::err("SetPreferredSharedMemoryCarveout Error!"); + throw std::runtime_error("SetPreferredSharedMemoryCarveout Error!"); } } @@ -824,14 +914,9 @@ template <> class MemoryManager { if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { DeviceRuntime::SyncQueue(queue_idx); } - if (ReduceMemoryFootprint) { - gpuErrchk(cudaMalloc(&ptr, n1 * n2 * sizeof(T))); - ld = n1; - } else { - size_t pitch = 0; - gpuErrchk(cudaMallocPitch(&ptr, &pitch, n1 * sizeof(T), (size_t)n2)); - ld = pitch / sizeof(T); - } + size_t pitch = 0; + gpuErrchk(cudaMallocPitch(&ptr, &pitch, n1 * sizeof(T), (size_t)n2)); + ld = pitch / sizeof(T); if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { DeviceRuntime::SyncQueue(queue_idx); } @@ -1004,7 +1089,7 @@ template <> class MemoryManager { template MGARDX_CONT static void HostRegister(T *ptr, SIZE n) { log::dbg("Calling MemoryManager::HostRegister"); - if (!CheckHostRegister(ptr)) { + if (!CheckHostRegister(ptr) && n > 0) { gpuErrchk(cudaHostRegister((void *)ptr, n * sizeof(T), cudaHostRegisterPortable)); } @@ -1016,8 +1101,6 @@ template <> class MemoryManager { gpuErrchk(cudaHostUnregister((void *)ptr)); } } - - static bool ReduceMemoryFootprint; }; #define ALIGN_LEFT 0 // for encoding @@ -1078,7 +1161,12 @@ struct BlockReduce { static void Max(T intput, T &output) { __shared__ TempStorageType temp_storage; BlockReduceType blockReduce(temp_storage); +#if defined(CUB_VERSION) && CUB_VERSION >= 300000 + // cub::Max() was removed as of CCCL 3.0 (bundled with CUDA 13+). + output = blockReduce.Reduce(intput, ::cuda::maximum<>()); +#else output = blockReduce.Reduce(intput, cub::Max()); +#endif } }; @@ -2430,8 +2518,7 @@ template <> class DeviceLauncher { CUDA>(std::string(KernelType::Name), min_config); } #else - log::err("MGARD is not built with auto tuning enabled."); - exit(-1); + throw std::runtime_error("MGARD is not built with auto tuning enabled."); #endif } @@ -2468,6 +2555,13 @@ struct AbsMaxOp { } }; +struct AbsMinOp { + template + __device__ __forceinline__ T operator()(const T &a, const T &b) const { + return (fabs(b) > fabs(a)) ? fabs(a) : fabs(b); + } +}; + struct SquareOp { template __device__ __forceinline__ T operator()(const T &a) const { @@ -2522,6 +2616,28 @@ template <> class DeviceCollective { } } + template + MGARDX_CONT static void AbsMin(SIZE n, SubArray<1, T, CUDA> v, + SubArray<1, T, CUDA> result, + Array<1, Byte, CUDA> &workspace, + bool workspace_allocated, int queue_idx) { + + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + AbsMinOp absMinOp; + cudaStream_t stream = DeviceRuntime::GetQueue(queue_idx); + cub::DeviceReduce::Reduce(d_temp_storage, temp_storage_bytes, v.data(), + result.data(), n, absMinOp, + std::numeric_limits::max(), stream); + ErrorAsyncCheck(cudaGetLastError(), "DeviceCollective::AbsMin"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(cudaDeviceSynchronize(), "DeviceCollective::AbsMin"); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } + template MGARDX_CONT static void SquareSum(SIZE n, SubArray<1, T, CUDA> v, SubArray<1, T, CUDA> result, @@ -2529,8 +2645,8 @@ template <> class DeviceCollective { bool workspace_allocated, int queue_idx) { SquareOp squareOp; - cub::TransformInputIterator transformed_input_iter( - v.data(), squareOp); + auto transformed_input_iter = + thrust::make_transform_iterator(v.data(), squareOp); Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; cudaStream_t stream = DeviceRuntime::GetQueue(queue_idx); @@ -2647,8 +2763,7 @@ template <> class DeviceCollective { thrust::equal_to binary_pred; - struct ThrustBinaryOp - : public thrust::binary_function { + struct ThrustBinaryOp { MGARDX_CONT_EXEC ValueT operator()(ValueT x, ValueT y) { BinaryOpType op; diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterHip.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterHip.h index 77e40b0458..68a60dc0b0 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterHip.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterHip.h @@ -1,12 +1,15 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ +#include "../Utilities/Exceptions.h" #include "DeviceAdapter.h" +#define HIP_ENABLE_WARP_SYNC_BUILTINS +#include #include #include #include @@ -49,10 +52,13 @@ template inline void ErrorAsyncCheckTask(hipError_t code, TaskType &task, bool abort = true) { if (code != hipSuccess) { - log::err(std::string(hipGetErrorString(code)) + " while executing " + - task.GetFunctorName().c_str() + " with HIP (Async-check)"); + std::string msg = std::string(hipGetErrorString(code)) + + " while executing " + task.GetFunctorName().c_str() + + " with HIP (Async-check)"; if (abort) - exit(code); + throw ProcessingException(msg); + else + std::cerr << msg << std::endl; } } @@ -60,30 +66,39 @@ template inline void ErrorSyncCheckTask(hipError_t code, TaskType &task, bool abort = true) { if (code != hipSuccess) { - log::err(std::string(hipGetErrorString(code)) + " while executing " + - task.GetFunctorName().c_str() + " with HIP (Sync-check)"); + std::string msg = std::string(hipGetErrorString(code)) + + " while executing " + task.GetFunctorName().c_str() + + " with HIP (Sync-check)"; if (abort) - exit(code); + throw ProcessingException(msg); + else + std::cerr << msg << std::endl; } } inline void ErrorAsyncCheck(hipError_t code, std::string task, bool abort = true) { if (code != hipSuccess) { - log::err(std::string(hipGetErrorString(code)) + " while executing " + - task.c_str() + " with HIP (Async-check)"); + std::string msg = std::string(hipGetErrorString(code)) + + " while executing " + task.c_str() + + " with HIP (Async-check)"; if (abort) - exit(code); + throw ProcessingException(msg); + else + std::cerr << msg << std::endl; } } inline void ErrorSyncCheck(hipError_t code, std::string task, bool abort = true) { if (code != hipSuccess) { - log::err(std::string(hipGetErrorString(code)) + " while executing " + - task.c_str() + " with HIP (Sync-check)"); + std::string msg = std::string(hipGetErrorString(code)) + + " while executing " + task.c_str() + + " with HIP (Sync-check)"; if (abort) - exit(code); + throw ProcessingException(msg); + else + std::cerr << msg << std::endl; } } @@ -93,10 +108,13 @@ inline void ErrorSyncCheck(hipError_t code, std::string task, inline void gpuAssert(hipError_t code, const char *file, int line, bool abort = true) { if (code != hipSuccess) { - fprintf(stderr, "GPUassert: %s %s %d\n", hipGetErrorString(code), file, - line); if (abort) - exit(code); + throw ProcessingException(std::string("GPUassert: ") + + hipGetErrorString(code) + " " + file + " " + + std::to_string(line)); + else + fprintf(stderr, "GPUassert: %s %s %d\n", hipGetErrorString(code), file, + line); } } @@ -137,6 +155,19 @@ struct Atomic { return atomicAdd(result, value); } } + MGARDX_EXEC static T Or(T *result, T value) { + // Cast to the matching-width unsigned type for portability across + // H = uint32/64 (atomicOr is only defined for 32-/64-bit unsigned ints). + if constexpr (sizeof(T) == 8) { + using U = unsigned long long int; + return static_cast( + atomicOr(reinterpret_cast(result), static_cast(value))); + } else { + using U = unsigned int; + return static_cast( + atomicOr(reinterpret_cast(result), static_cast(value))); + } + } }; template <> struct Math { @@ -402,6 +433,8 @@ template <> class DeviceSpecification { public: MGARDX_CONT DeviceSpecification() { + int original_device = 0; + hipGetDevice(&original_device); hipGetDeviceCount(&NumDevices); MaxSharedMemorySize = new int[NumDevices]; WarpSize = new int[NumDevices]; @@ -429,11 +462,25 @@ template <> class DeviceSpecification { SupportCooperativeGroups[d] = true; hipDeviceProp_t prop; hipGetDeviceProperties(&prop, d); - // Setting WarpSize[d] to true value (64) can trigger a bug - WarpSize[d] = MGARDX_WARP_SIZE; // equal to 32 + // WarpSize[d] holds the real hardware wavefront size queried above + // (64 on CDNA, 32 on RDNA). Code that launches warp-width thread + // blocks or partitions work by warp (Histogram, GenerateCL) must read + // this value at runtime rather than assuming MGARDX_WARP_SIZE (a + // CUDA-oriented compile-time constant) to stay correct and efficient + // across architectures. // DeviceNames[d] = std::string(prop.name); // Not working in HIP DeviceNames[d] = std::string("AMD GPU"); } + // This loop's per-device hipSetDevice(d) calls otherwise leave the + // active HIP device at NumDevices-1 for the rest of the process, + // silently out of sync with DeviceRuntime::hip_dev_id (which + // stays at its default of 0 unless explicitly changed). Streams and + // allocations made before the mismatch is next resolved end up tied to + // the wrong device's context, which can surface much later as an + // unrelated illegal memory access. Restore the device that was active + // before this query loop ran. (Same bug as the CUDA backend's + // DeviceSpecification, confirmed and fixed there first.) + gpuErrchk(hipSetDevice(original_device)); } MGARDX_CONT int GetNumDevices() { return NumDevices; } @@ -505,6 +552,8 @@ template <> class DeviceQueues { void Initialize() { if (!initialized) { log::dbg("Calling DeviceQueues::Initialize"); + int original_device = 0; + hipGetDevice(&original_device); hipGetDeviceCount(&NumDevices); streams = new hipStream_t *[NumDevices]; for (int d = 0; d < NumDevices; d++) { @@ -514,6 +563,9 @@ template <> class DeviceQueues { gpuErrchk(hipStreamCreate(&streams[d][i])); } } + // See DeviceSpecification's constructor for why this restore matters: + // otherwise the active device is left at NumDevices-1 here too. + gpuErrchk(hipSetDevice(original_device)); initialized = true; } } @@ -522,6 +574,8 @@ template <> class DeviceQueues { void Destroy() { if (initialized) { log::dbg("Calling DeviceQueues::Destroy"); + int original_device = 0; + hipGetDevice(&original_device); for (int d = 0; d < NumDevices; d++) { gpuErrchk(hipSetDevice(d)); for (int i = 0; i < MGARDX_NUM_QUEUES; i++) { @@ -529,6 +583,7 @@ template <> class DeviceQueues { } delete[] streams[d]; } + gpuErrchk(hipSetDevice(original_device)); delete[] streams; streams = nullptr; initialized = false; @@ -566,6 +621,33 @@ template <> class DeviceQueues { extern int hip_dev_id; #pragma omp threadprivate(hip_dev_id) +// AMD wavefront sub-group (HIP). UNTESTED -- no AMD hardware available here. +// See the verification notes at the original definition: width via +// __AMDGCN_WAVEFRONT_SIZE__ (64 CDNA / 32 RDNA), 64-bit ballot mask, +// __ballot/__shfl/__ffsll/__syncwarp. If the project's "64 triggers a bug" note +// bites, force size()=32 and launch half-wavefront blocks. +template <> struct SubGroup { + using mask_t = unsigned long long; +#ifdef __AMDGCN_WAVEFRONT_SIZE__ + static constexpr int size() { return __AMDGCN_WAVEFRONT_SIZE__; } +#else + static constexpr int size() { return 64; } +#endif + __device__ int lane() const { return threadIdx.x & (size() - 1); } + __device__ mask_t full_mask() const { + return size() == 64 ? ~0ull : ((1ull << size()) - 1); + } + template __device__ T shfl(T v, int src) const { + return __shfl(v, src, size()); + } + __device__ mask_t ballot(int pred) const { return (mask_t)__ballot(pred); } + __device__ int ffs(mask_t m) const { return __ffsll((long long)m); } + // __syncwarp() is opt-in (behind HIP_ENABLE_WARP_SYNC_BUILTINS) before ROCm + // 7.0 and default-on from 7.0 onward; see the HIP find_package version + // floor in the top-level CMakeLists.txt. + __device__ void sync() const { __syncwarp(); } +}; + template <> class DeviceRuntime { public: MGARDX_CONT @@ -666,7 +748,7 @@ template <> class DeviceRuntime { &numBlocks, HipHuffmanCWCustomizedKernel>, blockSize, dynamicSMemSize)); } else { - log::err("GetOccupancyMaxActiveBlocksPerSM Error!"); + throw std::runtime_error("GetOccupancyMaxActiveBlocksPerSM Error!"); } // HIP tends to over estimate this value numBlocks /= 2; @@ -697,7 +779,7 @@ template <> class DeviceRuntime { (const void *)HipHuffmanCWCustomizedKernel>, hipFuncAttributeMaxDynamicSharedMemorySize, maxbytes)); } else { - log::err("SetPreferredSharedMemoryCarveout Error!"); + throw std::runtime_error("SetPreferredSharedMemoryCarveout Error!"); } } @@ -747,15 +829,10 @@ template <> class MemoryManager { } using converted_T = typename std::conditional::value, Byte, T>::type; - if (ReduceMemoryFootprint) { - gpuErrchk(hipMalloc((void **)&ptr, n1 * n2 * sizeof(converted_T))); - ld = n1; - } else { - size_t pitch = 0; - gpuErrchk(hipMallocPitch((void **)&ptr, &pitch, n1 * sizeof(converted_T), - (size_t)n2)); - ld = pitch / sizeof(converted_T); - } + size_t pitch = 0; + gpuErrchk(hipMallocPitch((void **)&ptr, &pitch, n1 * sizeof(converted_T), + (size_t)n2)); + ld = pitch / sizeof(converted_T); if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { DeviceRuntime::SyncQueue(queue_idx); } @@ -952,8 +1029,10 @@ template <> class MemoryManager { using converted_T = typename std::conditional::value, Byte, T>::type; // if (!CheckHostRegister(ptr)) { - gpuErrchk(hipHostRegister((void *)ptr, n * sizeof(converted_T), - hipHostRegisterPortable)); + if (n > 0) { + gpuErrchk(hipHostRegister((void *)ptr, n * sizeof(converted_T), + hipHostRegisterPortable)); + } //} } @@ -963,8 +1042,6 @@ template <> class MemoryManager { gpuErrchk(hipHostUnregister((void *)ptr)); //} } - - static bool ReduceMemoryFootprint; }; #define ALIGN_LEFT 0 // for encoding @@ -1892,7 +1969,7 @@ template void HipHuffmanCLCustomizedNoCGKernel(Task task) { stream>>>(task); DeviceRuntime::SyncQueue(task.GetQueueIdx()); - // std::cout << "calling Single_Operation4_Kernel\n"; + // std::cout << "calling Single_Operation5_Kernel\n"; Single_Operation5_Kernel<<>>(task); DeviceRuntime::SyncQueue(task.GetQueueIdx()); @@ -2261,8 +2338,7 @@ template <> class DeviceLauncher { HIP>(std::string(KernelType::Name), min_config); } #else - log::err("MGARD is not built with auto tuning enabled."); - exit(-1); + throw std::runtime_error("MGARD is not built with auto tuning enabled."); #endif } @@ -2299,6 +2375,13 @@ struct AbsMaxOp { } }; +struct AbsMinOp { + template + __device__ __forceinline__ T operator()(const T &a, const T &b) const { + return (fabs(b) > fabs(a)) ? fabs(a) : fabs(b); + } +}; + struct SquareOp { template __device__ __forceinline__ T operator()(const T &a) const { @@ -2352,6 +2435,28 @@ template <> class DeviceCollective { } } + template + MGARDX_CONT static void AbsMin(SIZE n, SubArray<1, T, HIP> v, + SubArray<1, T, HIP> result, + Array<1, Byte, HIP> &workspace, + bool workspace_allocated, int queue_idx) { + + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + AbsMinOp absMinOp; + hipStream_t stream = DeviceRuntime::GetQueue(queue_idx); + hipcub::DeviceReduce::Reduce(d_temp_storage, temp_storage_bytes, v.data(), + result.data(), n, absMinOp, + std::numeric_limits::max(), stream); + ErrorAsyncCheck(hipGetLastError(), "DeviceCollective::AbsMax"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(hipDeviceSynchronize(), "DeviceCollective::AbsMax"); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } + template MGARDX_CONT static void SquareSum(SIZE n, SubArray<1, T, HIP> v, SubArray<1, T, HIP> result, diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.h index 1cfb3ae0bc..510f12d88d 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.h @@ -1,13 +1,14 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_DEVICE_ADAPTER_KOKKOS_H #define MGARD_X_DEVICE_ADAPTER_KOKKOS_H +#include "../Utilities/Exceptions.h" #include "DeviceAdapter.h" #include "Kokkos_Core.hpp" @@ -20,10 +21,13 @@ namespace mgard_x { inline void gpuAssert(cudaError_t code, const char *file, int line, bool abort = true) { if (code != cudaSuccess) { - fprintf(stderr, "GPUassert: %s %s %d\n", cudaGetErrorString(code), file, - line); if (abort) - exit(code); + throw ProcessingException(std::string("GPUassert: ") + + cudaGetErrorString(code) + " " + file + " " + + std::to_string(line)); + else + fprintf(stderr, "GPUassert: %s %s %d\n", cudaGetErrorString(code), file, + line); } } #endif @@ -36,346 +40,343 @@ inline void gpuAssert(cudaError_t code, const char *file, int line, inline void gpuAssert(hipError_t code, const char *file, int line, bool abort = true) { if (code != hipSuccess) { - fprintf(stderr, "GPUassert: %s %s %d\n", hipGetErrorString(code), file, - line); if (abort) - exit(code); + throw ProcessingException(std::string("GPUassert: ") + + hipGetErrorString(code) + " " + file + " " + + std::to_string(line)); + else + fprintf(stderr, "GPUassert: %s %s %d\n", hipGetErrorString(code), file, + line); } +} #endif - template MGARDX_KERL void KOKKOSKernel(TaskType task) { - typedef Kokkos::TeamPolicy::member_type member_type; - typedef Kokkos::KOKKOS::scratch_memory_space ScratchSpace; - typedef Kokkos::View> - ShareMemoryView; - SIZE league_size = - task.GetGridDimZ() * task.GetGridDimY() * task.GetGridDimX(); - SIZE team_size = - task.GetBlockDimZ() * task.GetBlockDimY() * task.GetBlockDimX(); - Kokkos::TeamPolicy policy(league_size, team_size); - - Kokkos::parallel_for( - task.GetFunctorName(), - policy.set_scratch_size(0, task.GetSharedMemorySize()), - KOKKOS_LAMBDA(member_type team_member) { - IDX threadx = team_member.team_rank() % task.GetBlockDimX(); - IDX thready = team_member.team_rank() / task.GetBlockDimX(); - IDX threadz = team_member.team_rank() / - (task.GetBlockDimX() * task.GetBlockDimY()); - IDX blockx = team_member.league_rank() % task.GetGridDimX(); - IDX blocky = team_member.league_rank() / task.GetGridDimX(); - IDX blockz = team_member.league_rank() / - (task.GetGridDimX() * task.GetGridDimY()); - ShareMemoryView shared_memory(team_member.team_scratch(0), - team_member.team_size()); - task.GetFunctor().Init(task.GetGridDimZ(), task.GetGridDimY(), - task.GetGridDimX(), task.GetBlockDimZ(), - task.GetBlockDimY(), task.GetBlockDimX(), - blockz, blocky, blockx, threadz, thready, - threadx, shared_memory.data()); - task.Operations1(); - team_member.team_barrier(); - task.Operations2(); - team_member.team_barrier(); - task.Operations3(); - team_member.team_barrier(); - task.Operations4(); - team_member.team_barrier(); - task.Operations5(); - team_member.team_barrier(); - task.Operations6(); - team_member.team_barrier(); - task.Operations7(); - team_member.team_barrier(); - task.Operations8(); - team_member.team_barrier(); - task.Operations9(); - team_member.team_barrier(); - task.Operations10(); - team_member.team_barrier(); - }); - } +template MGARDX_KERL void KOKKOSKernel(TaskType task) { + typedef Kokkos::TeamPolicy::member_type member_type; + typedef Kokkos::KOKKOS::scratch_memory_space ScratchSpace; + typedef Kokkos::View> + ShareMemoryView; + SIZE league_size = + task.GetGridDimZ() * task.GetGridDimY() * task.GetGridDimX(); + SIZE team_size = + task.GetBlockDimZ() * task.GetBlockDimY() * task.GetBlockDimX(); + Kokkos::TeamPolicy policy(league_size, team_size); + + Kokkos::parallel_for( + task.GetFunctorName(), + policy.set_scratch_size(0, task.GetSharedMemorySize()), + KOKKOS_LAMBDA(member_type team_member) { + IDX threadx = team_member.team_rank() % task.GetBlockDimX(); + IDX thready = team_member.team_rank() / task.GetBlockDimX(); + IDX threadz = team_member.team_rank() / + (task.GetBlockDimX() * task.GetBlockDimY()); + IDX blockx = team_member.league_rank() % task.GetGridDimX(); + IDX blocky = team_member.league_rank() / task.GetGridDimX(); + IDX blockz = team_member.league_rank() / + (task.GetGridDimX() * task.GetGridDimY()); + ShareMemoryView shared_memory(team_member.team_scratch(0), + team_member.team_size()); + task.GetFunctor().Init(task.GetGridDimZ(), task.GetGridDimY(), + task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX(), blockz, + blocky, blockx, threadz, thready, threadx, + shared_memory.data()); + task.Operations1(); + team_member.team_barrier(); + task.Operations2(); + team_member.team_barrier(); + task.Operations3(); + team_member.team_barrier(); + task.Operations4(); + team_member.team_barrier(); + task.Operations5(); + team_member.team_barrier(); + task.Operations6(); + team_member.team_barrier(); + task.Operations7(); + team_member.team_barrier(); + task.Operations8(); + team_member.team_barrier(); + task.Operations9(); + team_member.team_barrier(); + task.Operations10(); + team_member.team_barrier(); + }); +} - template <> class DeviceSpecification { - public: - MGARDX_CONT - DeviceSpecification() { - - Kokkos::InitArguments args; - args.device_id = 0; - Kokkos::initialize(args); - - NumDevices = 1; - MaxSharedMemorySize = new int[NumDevices]; - WarpSize = new int[NumDevices]; - NumSMs = new int[NumDevices]; - ArchitectureGeneration = new int[NumDevices]; - MaxNumThreadsPerSM = new int[NumDevices]; - - for (int d = 0; d < NumDevices; d++) { - MaxSharedMemorySize[d] = 1e6; - WarpSize[d] = 32; - NumSMs[d] = 80; - MaxNumThreadsPerSM[d] = 1024; - ArchitectureGeneration[d] = 1; - } +template <> class DeviceSpecification { +public: + MGARDX_CONT + DeviceSpecification() { + + Kokkos::InitArguments args; + args.device_id = 0; + Kokkos::initialize(args); + + NumDevices = 1; + MaxSharedMemorySize = new int[NumDevices]; + WarpSize = new int[NumDevices]; + NumSMs = new int[NumDevices]; + ArchitectureGeneration = new int[NumDevices]; + MaxNumThreadsPerSM = new int[NumDevices]; + + for (int d = 0; d < NumDevices; d++) { + MaxSharedMemorySize[d] = 1e6; + WarpSize[d] = 32; + NumSMs[d] = 80; + MaxNumThreadsPerSM[d] = 1024; + ArchitectureGeneration[d] = 1; } + } - MGARDX_CONT int GetNumDevices() { return NumDevices; } + MGARDX_CONT int GetNumDevices() { return NumDevices; } - MGARDX_CONT int GetMaxSharedMemorySize(int dev_id) { - return MaxSharedMemorySize[dev_id]; - } + MGARDX_CONT int GetMaxSharedMemorySize(int dev_id) { + return MaxSharedMemorySize[dev_id]; + } - MGARDX_CONT int GetWarpSize(int dev_id) { return WarpSize[dev_id]; } + MGARDX_CONT int GetWarpSize(int dev_id) { return WarpSize[dev_id]; } - MGARDX_CONT int GetNumSMs(int dev_id) { return NumSMs[dev_id]; } + MGARDX_CONT int GetNumSMs(int dev_id) { return NumSMs[dev_id]; } - MGARDX_CONT int GetArchitectureGeneration(int dev_id) { - return ArchitectureGeneration[dev_id]; - } + MGARDX_CONT int GetArchitectureGeneration(int dev_id) { + return ArchitectureGeneration[dev_id]; + } - MGARDX_CONT int GetMaxNumThreadsPerSM(int dev_id) { - return MaxNumThreadsPerSM[dev_id]; - } + MGARDX_CONT int GetMaxNumThreadsPerSM(int dev_id) { + return MaxNumThreadsPerSM[dev_id]; + } - MGARDX_CONT - ~DeviceSpecification() { - delete[] MaxSharedMemorySize; - delete[] WarpSize; - delete[] NumSMs; - delete[] ArchitectureGeneration; - } + MGARDX_CONT + ~DeviceSpecification() { + delete[] MaxSharedMemorySize; + delete[] WarpSize; + delete[] NumSMs; + delete[] ArchitectureGeneration; + } - int NumDevices; - int *MaxSharedMemorySize; - int *WarpSize; - int *NumSMs; - int *ArchitectureGeneration; - int *MaxNumThreadsPerSM; - }; - - template <> class DeviceRuntime { - public: - MGARDX_CONT - DeviceRuntime() {} - - MGARDX_CONT static void SelectDevice(SIZE dev_id) { - // do not support for now - curr_dev_id = dev_id; - } + int NumDevices; + int *MaxSharedMemorySize; + int *WarpSize; + int *NumSMs; + int *ArchitectureGeneration; + int *MaxNumThreadsPerSM; +}; + +template <> class DeviceRuntime { +public: + MGARDX_CONT + DeviceRuntime() {} + + MGARDX_CONT static void SelectDevice(SIZE dev_id) { + // do not support for now + curr_dev_id = dev_id; + } - MGARDX_CONT static int GetQueue(SIZE queue_id) { - // do not support for now - return 0; - } + MGARDX_CONT static int GetQueue(SIZE queue_id) { + // do not support for now + return 0; + } - MGARDX_CONT static void SyncQueue(SIZE queue_id) { - // do not support for now - // queues.SyncQueue(curr_dev_id, queue_id); - } + MGARDX_CONT static void SyncQueue(SIZE queue_id) { + // do not support for now + // queues.SyncQueue(curr_dev_id, queue_id); + } - MGARDX_CONT static void SyncAllQueues() { - // do not support for now - // queues.SyncAllQueues(curr_dev_id); - } + MGARDX_CONT static void SyncAllQueues() { + // do not support for now + // queues.SyncAllQueues(curr_dev_id); + } - MGARDX_CONT static void SyncDevice() { KOKKOS::impl_static_fence(); } + MGARDX_CONT static void SyncDevice() { KOKKOS::impl_static_fence(); } - MGARDX_CONT static int GetMaxSharedMemorySize() { - return DeviceSpecs.GetMaxSharedMemorySize(curr_dev_id); - } + MGARDX_CONT static int GetMaxSharedMemorySize() { + return DeviceSpecs.GetMaxSharedMemorySize(curr_dev_id); + } - MGARDX_CONT static int GetWarpSize() { - return DeviceSpecs.GetWarpSize(curr_dev_id); - } + MGARDX_CONT static int GetWarpSize() { + return DeviceSpecs.GetWarpSize(curr_dev_id); + } - MGARDX_CONT static int GetNumSMs() { - return DeviceSpecs.GetNumSMs(curr_dev_id); - } + MGARDX_CONT static int GetNumSMs() { + return DeviceSpecs.GetNumSMs(curr_dev_id); + } - MGARDX_CONT static int GetArchitectureGeneration() { - return DeviceSpecs.GetArchitectureGeneration(curr_dev_id); - } + MGARDX_CONT static int GetArchitectureGeneration() { + return DeviceSpecs.GetArchitectureGeneration(curr_dev_id); + } - MGARDX_CONT static int GetMaxNumThreadsPerSM() { - return DeviceSpecs.GetMaxNumThreadsPerSM(curr_dev_id); - } + MGARDX_CONT static int GetMaxNumThreadsPerSM() { + return DeviceSpecs.GetMaxNumThreadsPerSM(curr_dev_id); + } - template - MGARDX_CONT static int - GetOccupancyMaxActiveBlocksPerSM(FunctorType functor, int blockSize, - size_t dynamicSMemSize) { - int numBlocks = 0; - // Task task = Task(functor, 1, 1, 1, 1, 1, - // blockSize, dynamicSMemSize, 0); if constexpr (std::is_same::value) { - // if constexpr (std::is_base_of, FunctorType>::value) { - // gpuErrchk(cudaOccupancyMaxActiveBlocksPerMultiprocessor( - // &numBlocks, Kernel>, blockSize, - // dynamicSMemSize)); - // } else if constexpr (std::is_base_of, - // FunctorType>::value) { - // gpuErrchk(cudaOccupancyMaxActiveBlocksPerMultiprocessor( - // &numBlocks, IterKernel>, blockSize, - // dynamicSMemSize)); - // } else if constexpr - // (std::is_base_of, - // FunctorType>::value) { - // gpuErrchk(cudaOccupancyMaxActiveBlocksPerMultiprocessor( - // &numBlocks, HuffmanCLCustomizedKernel>, - // blockSize, dynamicSMemSize)); - // } else if constexpr - // (std::is_base_of, - // FunctorType>::value) { - // gpuErrchk(cudaOccupancyMaxActiveBlocksPerMultiprocessor( - // &numBlocks, HuffmanCWCustomizedKernel>, - // blockSize, dynamicSMemSize)); - // } else { - // std::cout << log::log_err << "GetOccupancyMaxActiveBlocksPerSM - // Error!\n"; - // } - // } else if (std::is_same::value) { - // if constexpr (std::is_base_of, FunctorType>::value) { - // gpuErrchk(hipOccupancyMaxActiveBlocksPerMultiprocessor( - // &numBlocks, Kernel>, blockSize, - // dynamicSMemSize)); - // } else if constexpr (std::is_base_of, - // FunctorType>::value) { - // gpuErrchk(hipOccupancyMaxActiveBlocksPerMultiprocessor( - // &numBlocks, IterKernel>, blockSize, - // dynamicSMemSize)); - // } else if constexpr (std::is_base_of, - // FunctorType>::value) { - // gpuErrchk(hipOccupancyMaxActiveBlocksPerMultiprocessor( - // &numBlocks, HuffmanCLCustomizedKernel>, - // blockSize, dynamicSMemSize)); - // } else if constexpr (std::is_base_of, - // FunctorType>::value) { - // gpuErrchk(hipOccupancyMaxActiveBlocksPerMultiprocessor( - // &numBlocks, HuffmanCWCustomizedKernel>, - // blockSize, dynamicSMemSize)); - // } else { - // std::cout << log::log_err << "GetOccupancyMaxActiveBlocksPerSM - // Error!\n"; - // } - // } - - return numBlocks; - } + template + MGARDX_CONT static int + GetOccupancyMaxActiveBlocksPerSM(FunctorType functor, int blockSize, + size_t dynamicSMemSize) { + int numBlocks = 0; + // Task task = Task(functor, 1, 1, 1, 1, 1, + // blockSize, dynamicSMemSize, 0); if constexpr (std::is_same::value) { + // if constexpr (std::is_base_of, FunctorType>::value) { + // gpuErrchk(cudaOccupancyMaxActiveBlocksPerMultiprocessor( + // &numBlocks, Kernel>, blockSize, + // dynamicSMemSize)); + // } else if constexpr (std::is_base_of, + // FunctorType>::value) { + // gpuErrchk(cudaOccupancyMaxActiveBlocksPerMultiprocessor( + // &numBlocks, IterKernel>, blockSize, + // dynamicSMemSize)); + // } else if constexpr + // (std::is_base_of, + // FunctorType>::value) { + // gpuErrchk(cudaOccupancyMaxActiveBlocksPerMultiprocessor( + // &numBlocks, HuffmanCLCustomizedKernel>, + // blockSize, dynamicSMemSize)); + // } else if constexpr + // (std::is_base_of, + // FunctorType>::value) { + // gpuErrchk(cudaOccupancyMaxActiveBlocksPerMultiprocessor( + // &numBlocks, HuffmanCWCustomizedKernel>, + // blockSize, dynamicSMemSize)); + // } else { + // std::cout << log::log_err << "GetOccupancyMaxActiveBlocksPerSM + // Error!\n"; + // } + // } else if (std::is_same::value) { + // if constexpr (std::is_base_of, FunctorType>::value) { + // gpuErrchk(hipOccupancyMaxActiveBlocksPerMultiprocessor( + // &numBlocks, Kernel>, blockSize, + // dynamicSMemSize)); + // } else if constexpr (std::is_base_of, + // FunctorType>::value) { + // gpuErrchk(hipOccupancyMaxActiveBlocksPerMultiprocessor( + // &numBlocks, IterKernel>, blockSize, + // dynamicSMemSize)); + // } else if constexpr (std::is_base_of, + // FunctorType>::value) { + // gpuErrchk(hipOccupancyMaxActiveBlocksPerMultiprocessor( + // &numBlocks, HuffmanCLCustomizedKernel>, + // blockSize, dynamicSMemSize)); + // } else if constexpr (std::is_base_of, + // FunctorType>::value) { + // gpuErrchk(hipOccupancyMaxActiveBlocksPerMultiprocessor( + // &numBlocks, HuffmanCWCustomizedKernel>, + // blockSize, dynamicSMemSize)); + // } else { + // std::cout << log::log_err << "GetOccupancyMaxActiveBlocksPerSM + // Error!\n"; + // } + // } + + return numBlocks; + } - template - MGARDX_CONT static void SetMaxDynamicSharedMemorySize(FunctorType functor, - int maxbytes) { - // skip for now - } + template + MGARDX_CONT static void SetMaxDynamicSharedMemorySize(FunctorType functor, + int maxbytes) { + // skip for now + } - MGARDX_CONT - ~DeviceRuntime() {} - - static int curr_dev_id; - // static DeviceQueues queues; - static bool SyncAllKernelsAndCheckErrors; - static DeviceSpecification DeviceSpecs; - }; - - template <> class MemoryManager { - public: - MGARDX_CONT - MemoryManager(){}; - - template - MGARDX_CONT static void Malloc1D(T *&ptr, SIZE n, int queue_idx) { - using converted_T = - typename std::conditional::value, Byte, - T>::type; - ptr = (T *)Kokkos::kokkos_malloc( - n * sizeof(converted_T)); - if (ptr == NULL) { - std::cout << log::log_err << "MemoryManager::Malloc1D error.\n"; - } + MGARDX_CONT + ~DeviceRuntime() {} + + static int curr_dev_id; + // static DeviceQueues queues; + static bool SyncAllKernelsAndCheckErrors; + static DeviceSpecification DeviceSpecs; +}; + +template <> class MemoryManager { +public: + MGARDX_CONT + MemoryManager(){}; + + template + MGARDX_CONT static void Malloc1D(T *&ptr, SIZE n, int queue_idx) { + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = (T *)Kokkos::kokkos_malloc(n * + sizeof(converted_T)); + if (ptr == NULL) { + std::cout << log::log_err << "MemoryManager::Malloc1D error.\n"; } + } - template - MGARDX_CONT static void MallocND(T *&ptr, SIZE n1, SIZE n2, SIZE &ld, - int queue_idx) { - using converted_T = - typename std::conditional::value, Byte, - T>::type; - ptr = (T *)Kokkos::kokkos_malloc( - n1 * n2 * sizeof(converted_T)); - ld = n1; - if (ptr == NULL) { - std::cout << log::log_err << "MemoryManager::MallocND error.\n"; - } + template + MGARDX_CONT static void MallocND(T *&ptr, SIZE n1, SIZE n2, SIZE &ld, + int queue_idx) { + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = (T *)Kokkos::kokkos_malloc(n1 * n2 * + sizeof(converted_T)); + ld = n1; + if (ptr == NULL) { + std::cout << log::log_err << "MemoryManager::MallocND error.\n"; } + } - template MGARDX_CONT static void Free(T *ptr) { - if (ptr == NULL) - return; - Kokkos::kokkos_free(ptr); - } + template MGARDX_CONT static void Free(T *ptr) { + if (ptr == NULL) + return; + Kokkos::kokkos_free(ptr); + } - template - MGARDX_CONT static void Copy1D(T *dst_ptr, const T *src_ptr, SIZE n, - int queue_idx) { - using converted_T = - typename std::conditional::value, Byte, - T>::type; - Kokkos::Impl::DeepCopy( - dst_ptr, src_ptr, n * sizeof(converted_T)); - } + template + MGARDX_CONT static void Copy1D(T *dst_ptr, const T *src_ptr, SIZE n, + int queue_idx) { + using converted_T = + typename std::conditional::value, Byte, T>::type; + Kokkos::Impl::DeepCopy( + dst_ptr, src_ptr, n * sizeof(converted_T)); + } - template - MGARDX_CONT static void CopyND(T *dst_ptr, SIZE dst_ld, const T *src_ptr, - SIZE src_ld, SIZE n1, SIZE n2, - int queue_idx) { - using converted_T = - typename std::conditional::value, Byte, - T>::type; - Kokkos::Impl::DeepCopy( - dst_ptr, src_ptr, n1 * n2 * sizeof(converted_T)); - } + template + MGARDX_CONT static void CopyND(T *dst_ptr, SIZE dst_ld, const T *src_ptr, + SIZE src_ld, SIZE n1, SIZE n2, int queue_idx) { + using converted_T = + typename std::conditional::value, Byte, T>::type; + Kokkos::Impl::DeepCopy( + dst_ptr, src_ptr, n1 * n2 * sizeof(converted_T)); + } - template - MGARDX_CONT static void MallocHost(T *&ptr, SIZE n, int queue_idx) { - using converted_T = - typename std::conditional::value, Byte, - T>::type; - ptr = (T *)std::malloc(n * sizeof(converted_T)); - if (ptr == NULL) { - std::cout << log::log_err - << "MemoryManager::MallocHost error.\n"; - } + template + MGARDX_CONT static void MallocHost(T *&ptr, SIZE n, int queue_idx) { + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = (T *)std::malloc(n * sizeof(converted_T)); + if (ptr == NULL) { + std::cout << log::log_err << "MemoryManager::MallocHost error.\n"; } + } - template MGARDX_CONT static void FreeHost(T *ptr) { - if (ptr == NULL) - return; - std::free(ptr); - } + template MGARDX_CONT static void FreeHost(T *ptr) { + if (ptr == NULL) + return; + std::free(ptr); + } - template - MGARDX_CONT static void Memset1D(T *ptr, SIZE n, int value) { - for (SIZE i = 0; i < n; ++i) - ptr[i] = value; - } + template + MGARDX_CONT static void Memset1D(T *ptr, SIZE n, int value) { + for (SIZE i = 0; i < n; ++i) + ptr[i] = value; + } - template - MGARDX_CONT static void MemsetND(T *ptr, SIZE ld, SIZE n1, SIZE n2, - int value) { - for (SIZE i = 0; i < n1 * n2; ++i) - ptr[i] = value; - } + template + MGARDX_CONT static void MemsetND(T *ptr, SIZE ld, SIZE n1, SIZE n2, + int value) { + for (SIZE i = 0; i < n1 * n2; ++i) + ptr[i] = value; + } - template MGARDX_CONT static bool IsDevicePointer(T *ptr) { - return true; - } + template MGARDX_CONT static bool IsDevicePointer(T *ptr) { + return true; + } - static bool ReduceMemoryFootprint; - }; -} + static bool ReduceMemoryFootprint; +}; +} // namespace mgard_x #endif \ No newline at end of file diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.h index 2ba823a6ba..6fcb21a7ce 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.h @@ -56,6 +56,15 @@ struct Atomic { } return old; } + MGARDX_EXEC static T Or(T *result, T value) { + T old = 0; +#pragma omp atomic capture + { + old = *result; + *result |= value; + } + return old; + } }; template <> struct Math { @@ -237,6 +246,10 @@ template <> class DeviceQueues { extern int openmp_dev_id; #pragma omp threadprivate(openmp_dev_id) +// OpenMP parallelism is across the grid (work-groups over cores), not within a +// sub-group, so its sub-group is the size-1 scalar group. +template <> struct SubGroup : SubGroupScalar {}; + template <> class DeviceRuntime { public: MGARDX_CONT @@ -340,7 +353,7 @@ template <> class MemoryManager { typename std::conditional::value, Byte, T>::type; ptr = (T *)std::malloc(n * sizeof(converted_T)); if (ptr == nullptr) { - log::err("MemoryManager::Malloc1D error."); + throw std::runtime_error("MemoryManager::Malloc1D error."); } } @@ -353,7 +366,7 @@ template <> class MemoryManager { ptr = (T *)std::malloc(n1 * n2 * sizeof(converted_T)); ld = n1; if (ptr == nullptr) { - log::err("MemoryManager::MallocND error."); + throw std::runtime_error("MemoryManager::MallocND error."); } } @@ -365,7 +378,7 @@ template <> class MemoryManager { typename std::conditional::value, Byte, T>::type; ptr = (T *)std::malloc(n * sizeof(converted_T)); if (ptr == nullptr) { - log::err("MemoryManager::MallocManaged1D error."); + throw std::runtime_error("MemoryManager::MallocManaged1D error."); } } @@ -415,7 +428,7 @@ template <> class MemoryManager { typename std::conditional::value, Byte, T>::type; ptr = (T *)std::malloc(n * sizeof(converted_T)); if (ptr == nullptr) { - log::err("MemoryManager::MallocHost error."); + throw std::runtime_error("MemoryManager::MallocHost error."); } } @@ -468,8 +481,6 @@ template <> class MemoryManager { template MGARDX_CONT static void HostUnregister(T *ptr) { log::dbg("Calling MemoryManager::HostUnregister"); } - - static bool ReduceMemoryFootprint; }; #define ALIGN_LEFT 0 // for encoding @@ -1373,8 +1384,7 @@ template <> class DeviceLauncher { OPENMP>(std::string(KernelType::Name), min_config); } #else - log::err("MGARD is not built with auto tuning enabled."); - exit(-1); + throw std::runtime_error("MGARD is not built with auto tuning enabled."); #endif } @@ -1439,6 +1449,23 @@ template <> class DeviceCollective { } } + template + MGARDX_CONT static void AbsMin(SIZE n, SubArray<1, T, OPENMP> v, + SubArray<1, T, OPENMP> result, + Array<1, Byte, OPENMP> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + T max_result = std::numeric_limits::max(); + for (SIZE i = 0; i < n; ++i) { + max_result = std::min((T)fabs(*v(i)), max_result); + } + *result((IDX)0) = max_result; + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + template MGARDX_CONT static void SquareSum(SIZE n, SubArray<1, T, OPENMP> v, SubArray<1, T, OPENMP> result, @@ -1472,7 +1499,8 @@ template <> class DeviceCollective { workspace.resize({(SIZE)1}, queue_idx); } #else - log::err("Please recompile with GCC 9+ to use ScanSumInclusive."); + throw std::runtime_error( + "Please recompile with GCC 9+ to use ScanSumInclusive."); #endif } @@ -1491,7 +1519,8 @@ template <> class DeviceCollective { workspace.resize({(SIZE)1}, queue_idx); } #else - log::err("Please recompile with GCC 9+ to use ScanSumExclusive."); + throw std::runtime_error( + "Please recompile with GCC 9+ to use ScanSumExclusive."); #endif } @@ -1511,7 +1540,8 @@ template <> class DeviceCollective { workspace.resize({(SIZE)1}, queue_idx); } #else - log::err("Please recompile with GCC 9+ to use ScanSumExtended."); + throw std::runtime_error( + "Please recompile with GCC 9+ to use ScanSumExtended."); #endif } diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSerial.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSerial.h index 384c6ed630..91a1e62c2e 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSerial.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSerial.h @@ -42,6 +42,11 @@ struct Atomic { *result += value; return old; } + MGARDX_EXEC static T Or(T *result, T value) { + T old = *result; + *result |= value; + return old; + } }; template <> struct Math { @@ -729,6 +734,10 @@ template <> class DeviceQueues { extern int serial_dev_id; #pragma omp threadprivate(serial_dev_id) +// SERIAL has no SIMD lanes: its sub-group is the size-1 scalar group, so any +// cooperative kernel degenerates to the plain serial algorithm. +template <> struct SubGroup : SubGroupScalar {}; + template <> class DeviceRuntime { public: MGARDX_CONT @@ -832,7 +841,7 @@ template <> class MemoryManager { typename std::conditional::value, Byte, T>::type; ptr = (T *)std::malloc(n * sizeof(converted_T)); if (ptr == nullptr) { - log::err("MemoryManager::Malloc1D error."); + throw std::runtime_error("MemoryManager::Malloc1D error."); } } @@ -845,7 +854,7 @@ template <> class MemoryManager { ptr = (T *)std::malloc(n1 * n2 * sizeof(converted_T)); ld = n1; if (ptr == nullptr) { - log::err("MemoryManager::MallocND error."); + throw std::runtime_error("MemoryManager::MallocND error."); } } @@ -857,7 +866,7 @@ template <> class MemoryManager { typename std::conditional::value, Byte, T>::type; ptr = (T *)std::malloc(n * sizeof(converted_T)); if (ptr == nullptr) { - log::err("MemoryManager::MallocManaged1D error."); + throw std::runtime_error("MemoryManager::MallocManaged1D error."); } } @@ -907,7 +916,7 @@ template <> class MemoryManager { typename std::conditional::value, Byte, T>::type; ptr = (T *)std::malloc(n * sizeof(converted_T)); if (ptr == nullptr) { - log::err("MemoryManager::MallocHost error."); + throw std::runtime_error("MemoryManager::MallocHost error."); } } @@ -960,8 +969,6 @@ template <> class MemoryManager { template MGARDX_CONT static void HostUnregister(T *ptr) { log::dbg("Calling MemoryManager::HostUnregister"); } - - static bool ReduceMemoryFootprint; }; #define ALIGN_LEFT 0 // for encoding @@ -1300,8 +1307,7 @@ template <> class DeviceLauncher { SERIAL>(std::string(KernelType::Name), min_config); } #else - log::err("MGARD is not built with auto tuning enabled."); - exit(-1); + throw std::runtime_error("MGARD is not built with auto tuning enabled."); #endif } @@ -1366,6 +1372,23 @@ template <> class DeviceCollective { } } + template + MGARDX_CONT static void AbsMin(SIZE n, SubArray<1, T, SERIAL> v, + SubArray<1, T, SERIAL> result, + Array<1, Byte, SERIAL> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + T max_result = std::numeric_limits::max(); + for (SIZE i = 0; i < n; ++i) { + max_result = std::min((T)fabs(*v(i)), max_result); + } + *result((IDX)0) = max_result; + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + template MGARDX_CONT static void SquareSum(SIZE n, SubArray<1, T, SERIAL> v, SubArray<1, T, SERIAL> result, @@ -1391,16 +1414,18 @@ template <> class DeviceCollective { bool workspace_allocated, int queue_idx) { - // Need gcc 9 and c++17 -#if (__GNUC__ >= 9) if (workspace_allocated) { - std::inclusive_scan(v((IDX)0), v((IDX)n), result((IDX)0)); + // Serial inclusive scan: result[i] = sum(v[0..i]). Hand-rolled instead of + // std::inclusive_scan so the header parses under nvcc's default host + // compiler (older libstdc++ hides the C++17 algorithms). + T acc = (T)0; + for (SIZE i = 0; i < n; i++) { + acc += *v((IDX)i); + *result((IDX)i) = acc; + } } else { workspace.resize({(SIZE)1}, queue_idx); } -#else - log::err("Please recompile with GCC 9+ to use ScanSumInclusive."); -#endif } template @@ -1410,16 +1435,17 @@ template <> class DeviceCollective { bool workspace_allocated, int queue_idx) { - // Need gcc 9 and c++17 -#if (__GNUC__ >= 9) if (workspace_allocated) { - std::exclusive_scan(v((IDX)0), v((IDX)n), result((IDX)0)); + // Serial exclusive scan: result[0] = 0, result[i] = sum(v[0..i-1]). + // Hand-rolled in place of std::exclusive_scan (see ScanSumInclusive). + T acc = (T)0; + for (SIZE i = 0; i < n; i++) { + *result((IDX)i) = acc; + acc += *v((IDX)i); + } } else { workspace.resize({(SIZE)1}, queue_idx); } -#else - log::err("Please recompile with GCC 9+ to use ScanSumExclusive."); -#endif } template @@ -1428,17 +1454,21 @@ template <> class DeviceCollective { Array<1, Byte, SERIAL> &workspace, bool workspace_allocated, int queue_idx) { - // Need gcc 9 and c++17 -#if (__GNUC__ >= 9) + if (workspace_allocated) { - std::inclusive_scan(v((IDX)0), v((IDX)n), result((IDX)1)); - *result((IDX)0) = 0; + // Serial extended scan: result has n+1 entries with result[0] = 0 and + // result[i+1] = sum(v[0..i]) (exclusive prefix plus the grand total in + // the last slot). Hand-rolled in place of std::inclusive_scan into + // result+1. + T acc = (T)0; + *result((IDX)0) = (T)0; + for (SIZE i = 0; i < n; i++) { + acc += *v((IDX)i); + *result((IDX)(i + 1)) = acc; + } } else { workspace.resize({(SIZE)1}, queue_idx); } -#else - log::err("Please recompile with GCC 9+ to use ScanSumExtended."); -#endif } template diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSycl.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSycl.h index be09871a3f..239e6ad955 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSycl.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSycl.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "DeviceAdapter.h" @@ -150,6 +150,43 @@ struct Atomic { } } } + MGARDX_EXEC static T Or(T *result, T value) { + if constexpr (MemoryType == AtomicGlobalMemory) { + if constexpr (Scope == AtomicSystemScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::system, + sycl::access::address_space::global_space>; + return AtomicRef(result[0]).fetch_or(value); + } else if constexpr (Scope == AtomicDeviceScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::device, + sycl::access::address_space::global_space>; + return AtomicRef(result[0]).fetch_or(value); + } else { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::work_group, + sycl::access::address_space::global_space>; + return AtomicRef(result[0]).fetch_or(value); + } + } else { + if constexpr (Scope == AtomicSystemScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::system, + sycl::access::address_space::local_space>; + return AtomicRef(result[0]).fetch_or(value); + } else if constexpr (Scope == AtomicDeviceScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::device, + sycl::access::address_space::local_space>; + return AtomicRef(result[0]).fetch_or(value); + } else { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::work_group, + sycl::access::address_space::local_space>; + return AtomicRef(result[0]).fetch_or(value); + } + } + } }; template <> struct Math { @@ -370,6 +407,32 @@ template <> class DeviceQueues { extern int sycl_dev_id; #pragma omp threadprivate(sycl_dev_id) +// Intel sub-group (oneAPI/DPC++). UNTESTED -- no Intel GPU / SYCL toolchain +// here. Holds the native sycl::sub_group (constructed from nd_item in the +// kernel); width pinned to 32 by [[sycl::reqd_sub_group_size(32)]] at the +// launch site so the ballot mask fits a uint32. See verification notes at the +// original definition (group_ballot/extract_bits, select_from_group, +// group_barrier). +template <> struct SubGroup { + sycl::sub_group sg_; + using mask_t = uint32_t; + MGARDX_EXEC SubGroup(sycl::sub_group s) : sg_(s) {} + static constexpr int size() { return 32; } + MGARDX_EXEC int lane() const { return (int)sg_.get_local_linear_id(); } + MGARDX_EXEC mask_t full_mask() const { return 0xffffffffu; } + template MGARDX_EXEC T shfl(T v, int src) const { + return sycl::select_from_group(sg_, v, (size_t)src); + } + MGARDX_EXEC mask_t ballot(int pred) const { + auto m = sycl::ext::oneapi::group_ballot(sg_, pred != 0); + uint32_t bits = 0; + m.extract_bits(bits); + return bits; + } + MGARDX_EXEC int ffs(mask_t m) const { return __builtin_ffs((int)m); } + MGARDX_EXEC void sync() const { sycl::group_barrier(sg_); } +}; + template <> class DeviceRuntime { public: MGARDX_CONT @@ -706,8 +769,6 @@ template <> class MemoryManager { template MGARDX_CONT static void HostUnregister(T *ptr) { log::dbg("Calling MemoryManager::HostUnregister"); } - - static bool ReduceMemoryFootprint; }; template class SyclKernel { @@ -1737,8 +1798,7 @@ template <> class DeviceLauncher { SYCL>(std::string(KernelType::Name), min_config); } #else - log::err("MGARD is not built with auto tuning enabled."); - exit(-1); + throw std::runtime_error("MGARD is not built with auto tuning enabled."); #endif } @@ -1774,6 +1834,12 @@ template struct AbsMaxOp { } }; +template struct AbsMinOp { + T operator()(const T &a, const T &b) const { + return (fabs(b) > fabs(a)) ? fabs(a) : fabs(b); + } +}; + template struct SquareOp { T operator()(const T &a) const { return a * a; } }; @@ -1825,6 +1891,28 @@ template <> class DeviceCollective { } } + template + MGARDX_CONT static void AbsMin(SIZE n, SubArray<1, T, SYCL> v, + SubArray<1, T, SYCL> result, + Array<1, Byte, SYCL> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + sycl::queue q = DeviceRuntime::GetQueue(queue_idx); + q.submit([&](sycl::handler &h) { + T *res = result.data(); + T *input = v.data(); + h.parallel_for( + sycl::range{n}, + sycl::reduction(res, std::numeric_limits::max(), AbsMinOp()), + [=](sycl::id<1> i, auto &res) { res.combine(input[i]); }); + }); + DeviceRuntime::SyncDevice(); + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + template MGARDX_CONT static void SquareSum(SIZE n, SubArray<1, T, SYCL> v, SubArray<1, T, SYCL> result, diff --git a/include/mgard-x/RuntimeX/Functors/Functor.h b/include/mgard-x/RuntimeX/Functors/Functor.h index 1623d7ee6f..bd60d0b557 100644 --- a/include/mgard-x/RuntimeX/Functors/Functor.h +++ b/include/mgard-x/RuntimeX/Functors/Functor.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_FUNCTOR diff --git a/include/mgard-x/RuntimeX/Kernel/Kernel.h b/include/mgard-x/RuntimeX/Kernel/Kernel.h index 540f581f73..a2c9a31ef3 100644 --- a/include/mgard-x/RuntimeX/Kernel/Kernel.h +++ b/include/mgard-x/RuntimeX/Kernel/Kernel.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_KERNEL #define MGARD_X_KERNEL diff --git a/include/mgard-x/RuntimeX/RuntimeX.h b/include/mgard-x/RuntimeX/RuntimeX.h index f221edf6f3..9c8ab25d02 100644 --- a/include/mgard-x/RuntimeX/RuntimeX.h +++ b/include/mgard-x/RuntimeX/RuntimeX.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "DataTypes.h" @@ -55,6 +55,7 @@ #endif #include "Utilities/CheckShape.hpp" +#include "Utilities/Exceptions.h" #include "Utilities/OffsetCalculators.hpp" #include "DataStructures/Array.hpp" diff --git a/include/mgard-x/RuntimeX/RuntimeXPublic.h b/include/mgard-x/RuntimeX/RuntimeXPublic.h index 111d70c1af..e20422bf9c 100644 --- a/include/mgard-x/RuntimeX/RuntimeXPublic.h +++ b/include/mgard-x/RuntimeX/RuntimeXPublic.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "MGARDXConfig.h" diff --git a/include/mgard-x/RuntimeX/Tasks/Task.h b/include/mgard-x/RuntimeX/Tasks/Task.h index 60fe0b1f47..696d61f089 100644 --- a/include/mgard-x/RuntimeX/Tasks/Task.h +++ b/include/mgard-x/RuntimeX/Tasks/Task.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_TASK #define MGARD_X_TASK diff --git a/include/mgard-x/RuntimeX/Utilities/CheckShape.hpp b/include/mgard-x/RuntimeX/Utilities/CheckShape.hpp index e16a3b7bd5..4113b74949 100644 --- a/include/mgard-x/RuntimeX/Utilities/CheckShape.hpp +++ b/include/mgard-x/RuntimeX/Utilities/CheckShape.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_CHECK_SHAPE_HPP diff --git a/include/mgard-x/RuntimeX/Utilities/Exceptions.h b/include/mgard-x/RuntimeX/Utilities/Exceptions.h new file mode 100644 index 0000000000..fd4a47e516 --- /dev/null +++ b/include/mgard-x/RuntimeX/Utilities/Exceptions.h @@ -0,0 +1,53 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + */ + +#ifndef MGARD_X_EXCEPTIONS_H +#define MGARD_X_EXCEPTIONS_H + +#include +#include + +#include "../../Utilities/Types.h" + +namespace mgard_x { + +//! Base class for all exceptions thrown by the MGARD-X library. +//! +//! Errors that are internal to the (de)compression process are reported by +//! throwing an exception rather than calling `exit()`, so that the calling +//! application can handle the failure gracefully (clean up, report to a job +//! scheduler, keep other threads/GPUs alive, etc.). Every exception carries a +//! `compress_status_type` so the high-level API can translate a thrown +//! exception back into the status code returned by `compress()`/`decompress()`. +class Exception : public std::runtime_error { +public: + explicit Exception(const std::string &msg, compress_status_type status = + compress_status_type::Failure) + : std::runtime_error(msg), status_(status) {} + + //! Status code that the high-level API should return for this exception. + compress_status_type status() const noexcept { return status_; } + +private: + compress_status_type status_; +}; + +//! Thrown when compressed data or its metadata header is malformed, +//! corrupted, or was produced by an incompatible version of MGARD. +class InvalidDataException : public Exception { +public: + explicit InvalidDataException(const std::string &msg) : Exception(msg) {} +}; + +//! Thrown when an internal processing step fails (e.g. a lossless backend +//! error or a Huffman codebook that cannot be represented). +class ProcessingException : public Exception { +public: + explicit ProcessingException(const std::string &msg) : Exception(msg) {} +}; + +} // namespace mgard_x + +#endif // MGARD_X_EXCEPTIONS_H diff --git a/include/mgard-x/RuntimeX/Utilities/Log.h b/include/mgard-x/RuntimeX/Utilities/Log.h index b64cc192d1..391cf83e37 100644 --- a/include/mgard-x/RuntimeX/Utilities/Log.h +++ b/include/mgard-x/RuntimeX/Utilities/Log.h @@ -1,6 +1,7 @@ #ifndef MGARD_X_MESSGAE_HH #define MGARD_X_MESSGAE_HH +#include #include #include #include @@ -39,6 +40,13 @@ void build(std::ostream &o, T t, Args... args); template void print(string log_head, Args... args); +template void csv(std::string file, T v) { + std::ofstream myfile; + myfile.open(file, std::ios_base::app); + myfile << v << ", "; + myfile.close(); +} + } // namespace log } // namespace mgard_x diff --git a/include/mgard-x/RuntimeX/Utilities/OffsetCalculators.hpp b/include/mgard-x/RuntimeX/Utilities/OffsetCalculators.hpp index c0c7647d2e..acf94a9ffa 100644 --- a/include/mgard-x/RuntimeX/Utilities/OffsetCalculators.hpp +++ b/include/mgard-x/RuntimeX/Utilities/OffsetCalculators.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_OFFSET_CALCULATORS_H @@ -52,6 +52,10 @@ template MGARDX_CONT_EXEC SIZE roundup(T2 a) { return ((a - 1) / sizeof(T1) + 1) * sizeof(T1); } +template MGARDX_CONT_EXEC T round_up(T a, T b) { + return ((a - 1) / b + 1) * b; +} + } // namespace mgard_x #endif diff --git a/include/mgard-x/RuntimeX/Utilities/Serializer.hpp b/include/mgard-x/RuntimeX/Utilities/Serializer.hpp index 5c9a904b5b..78f372df81 100644 --- a/include/mgard-x/RuntimeX/Utilities/Serializer.hpp +++ b/include/mgard-x/RuntimeX/Utilities/Serializer.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_SERIALIZER_HH @@ -51,7 +51,7 @@ template void Serialize(Byte *serialize_ptr, T *data_ptr, SIZE count, SIZE &byte_offset, int queue_idx) { using Mem = MemoryManager; - // align_byte_offset(byte_offset); + align_byte_offset(byte_offset); Mem::Copy1D(serialize_ptr + byte_offset, (Byte *)data_ptr, count * sizeof(T), queue_idx); byte_offset += count * sizeof(T); diff --git a/include/mgard-x/RuntimeX/Utilities/SubArrayPrinter.hpp b/include/mgard-x/RuntimeX/Utilities/SubArrayPrinter.hpp index 01a2db5b91..26273a105d 100644 --- a/include/mgard-x/RuntimeX/Utilities/SubArrayPrinter.hpp +++ b/include/mgard-x/RuntimeX/Utilities/SubArrayPrinter.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_MEMORY_MANAGEMENT_HPP @@ -15,6 +15,7 @@ #include #include // std::stringstream #include // std::runtime_error +#include #include #include // std::pair #include @@ -70,7 +71,7 @@ void PrintSubarray(std::string name, SubArrayType subArray) { for (int j = 0; j < ncol; j++) { for (int k = 0; k < nfib; k++) { // std::cout << "[ " << j << ", " << k <<" ]: "; - if (std::is_same::value) { + if (std::is_same::value) { std::cout << std::setw(8) << (unsigned int)v[nfib * ncol * i + nfib * j + k] << " "; } else { @@ -107,8 +108,7 @@ void CompareSubarray(std::string name, SubArrayType subArray1, if (subArray1.shape(D - 1) != subArray2.shape(D - 1) || subArray1.shape(D - 2) != subArray2.shape(D - 2) || subArray1.shape(D - 3) != subArray2.shape(D - 3)) { - std::cout << log::log_err << "CompareSubarray: shape mismatch!\n"; - exit(-1); + throw std::runtime_error("CompareSubarray: shape mismatch!"); } using T = typename SubArrayType::DataType; @@ -146,7 +146,7 @@ void CompareSubarray(std::string name, SubArrayType subArray1, } else { std::cout << ANSI_GREEN; } - if (std::is_same::value) { + if (std::is_same::value) { std::cout << std::setw(8) << (unsigned int)v2[nfib * ncol * i + nfib * j + k] << ", "; } else { @@ -189,8 +189,7 @@ void CompareSubarray(std::string name, SubArrayType1 subArray1, subArray1.shape(D - 2) != subArray2.shape[1]) || (SubArrayType1::NumDims >= 3 && subArray1.shape(D - 3) != subArray2.shape[2])) { - std::cout << log::log_err << "CompareSubarray: shape mismatch!\n"; - exit(-1); + throw std::runtime_error("CompareSubarray: shape mismatch!"); } using T = typename SubArrayType1::DataType; @@ -231,7 +230,7 @@ void CompareSubarray(std::string name, SubArrayType1 subArray1, if (print) std::cout << ANSI_GREEN; } - if (std::is_same::value) { + if (std::is_same::value) { if (print) std::cout << std::setw(8) << (unsigned int)v2[nfib * ncol * i + nfib * j + k] @@ -261,16 +260,13 @@ void CompareSubarray(std::string name, SubArrayType1 subArray1, template void CompareSubarray4D(SubArrayType subArray1, SubArrayType subArray2) { if (SubArrayType::NumDims != 4) { - std::cout << log::log_err - << "CompareSubarray4D expects 4D subarray type.\n"; - exit(-1); + throw std::runtime_error("CompareSubarray4D expects 4D subarray type."); } DIM D = SubArrayType::NumDims; if (subArray1.shape(D - 4) != subArray2.shape(D - 4)) { - std::cout << log::log_err << "CompareSubarray4D mismatch 4D size.\n"; - exit(-1); + throw std::runtime_error("CompareSubarray4D mismatch 4D size."); } using T = typename SubArrayType::DataType; @@ -291,8 +287,7 @@ void CompareSubarray4D(SubArrayType subArray1, SubArrayType subArray2) { template void PrintSubarray4D(std::string name, SubArrayType subArray1) { if (SubArrayType::NumDims != 4) { - std::cout << log::log_err << "PrintSubarray4D expects 4D subarray type.\n"; - exit(-1); + throw std::runtime_error("PrintSubarray4D expects 4D subarray type."); } DIM D = SubArrayType::NumDims; @@ -361,7 +356,7 @@ void verify_matrix(SIZE nrow, SIZE ncol, SIZE nfib, T *v, SIZE ldv1, SIZE ldv2, delete[] v2; if (mismatch) - exit(-1); + throw std::runtime_error("verify_matrix: mismatch"); } } diff --git a/include/mgard-x/RuntimeX/Utilities/Timer.hpp b/include/mgard-x/RuntimeX/Utilities/Timer.hpp index b597153bea..9bb3168953 100644 --- a/include/mgard-x/RuntimeX/Utilities/Timer.hpp +++ b/include/mgard-x/RuntimeX/Utilities/Timer.hpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_TIMER_HPP @@ -22,19 +22,27 @@ class Timer { double time = (double)(end_time.tv_sec - start_time.tv_sec) + (double)(end_time.tv_nsec - start_time.tv_nsec) / (double)1000000000; - return time; + return total_time; } - double get_throughput(SIZE n) { return (double)n / get() / 1e9; } + double get_throughput(SIZE num_bytes) { + return (double)num_bytes / get() / 1e9; + } void clear() { total_time = 0; } - void print(std::string s) { - log::time(s + " time: " + std::to_string(total_time) + " s"); + void print(std::string s, SIZE num_bytes = 0, bool override = false) { + if (num_bytes == 0) { + log::time(s + ": " + std::to_string(total_time) + " s", override); + } else { + log::time(s + ": " + std::to_string(total_time) + " s (" + + std::to_string(get_throughput(num_bytes)) + " GB/s)", + override); + } } - void print_throughput(std::string s, SIZE n) { - log::time(s + " throughput: " + std::to_string(get_throughput(n)) + - " GB/s"); + void print_throughput(std::string s, SIZE n, bool override = false) { + log::time(s + " throughput: " + std::to_string(get_throughput(n)) + " GB/s", + override); } private: diff --git a/include/mgard-x/Utilities/ErrorCalculator.h b/include/mgard-x/Utilities/ErrorCalculator.h index d2a89531d1..fe5ededc32 100644 --- a/include/mgard-x/Utilities/ErrorCalculator.h +++ b/include/mgard-x/Utilities/ErrorCalculator.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_ERROR_CALCULATOR diff --git a/include/mgard-x/Utilities/ProjectionMode.h b/include/mgard-x/Utilities/ProjectionMode.h new file mode 100644 index 0000000000..94e31689cc --- /dev/null +++ b/include/mgard-x/Utilities/ProjectionMode.h @@ -0,0 +1,51 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + */ + +#ifndef MGARD_X_UTILITIES_PROJECTION_MODE_H +#define MGARD_X_UTILITIES_PROJECTION_MODE_H + +#include + +#include "../RuntimeX/Utilities/Exceptions.h" +#include "Types.h" + +namespace mgard_x { + +// Resolves Auto against the error-bound norm s, then validates the result: +// Hierarchical only supports L-infinity error control (its reconstruction is +// a partition-of-unity prolongation, so it only bounds the max error, not the +// L_2 norm). This is the single choke point both Compressor and +// HybridHierarchyCompressor call, so one Config field controls both. +template +inline compression_projection_mode_type +resolve_projection_mode(compression_projection_mode_type mode, T s) { + if (mode == compression_projection_mode_type::Auto) { + mode = s == std::numeric_limits::infinity() + ? compression_projection_mode_type::Hierarchical + : compression_projection_mode_type::Orthogonal; + } + if (mode == compression_projection_mode_type::Hierarchical && + s != std::numeric_limits::infinity()) { + throw ProcessingException( + "the hierarchical basis only supports L-infinity error control"); + } + if (mode != compression_projection_mode_type::Orthogonal && + mode != compression_projection_mode_type::Hierarchical) { + throw ProcessingException("unknown compression projection mode"); + } + return mode; +} + +// The boolean the (de)compose/quantize kernels actually consume. +template +inline bool infer_orthogonal_projection(compression_projection_mode_type mode, + T s) { + return resolve_projection_mode(mode, s) == + compression_projection_mode_type::Orthogonal; +} + +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/Utilities/Types.h b/include/mgard-x/Utilities/Types.h index a1b57a360c..e4caf284e9 100644 --- a/include/mgard-x/Utilities/Types.h +++ b/include/mgard-x/Utilities/Types.h @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #ifndef MGARD_X_UTILITIES_TYPES @@ -18,6 +18,47 @@ namespace mgard_x { enum class decomposition_type : uint8_t { MultiDim, SingleDim, Hybrid }; +// Edge length of the block-local transform block used by the hybrid +// (BlockMGARD) hierarchy. The 8 -> 5 coarsening in the block-local refactor +// and the ROI block grid are both built on this. Compile-time today; recorded +// in the file header so a future change stays readable. +constexpr uint64_t MGARDX_HYBRID_LOCAL_BLOCK_SIZE = 8; + +// Coarse edge length one block-local level produces: 8 fine nodes coarsen to +// the 5 nodes {0, 2, 4, 6, 7} per dimension. +constexpr uint64_t MGARDX_HYBRID_LOCAL_COARSE_SIZE = 5; + +// Number of coefficients one block-local block contributes per level: +// 8^D - 5^D (387 in 3D, 39 in 2D, 3 in 1D). The coefficient array is a +// row-major sequence of blocks, so `index / this` is the block index -- which +// is how the ROI per-block quantizers are addressed. +constexpr uint64_t hybrid_local_coeff_per_block(uint64_t num_dims) { + uint64_t fine = 1, coarse = 1; + for (uint64_t d = 0; d < num_dims; d++) { + fine *= MGARDX_HYBRID_LOCAL_BLOCK_SIZE; + coarse *= MGARDX_HYBRID_LOCAL_COARSE_SIZE; + } + return fine - coarse; +} + +// Transform basis used by the (de)compose step, shared by the plain +// Compressor and the hybrid (BlockMGARD) HybridHierarchyCompressor via +// Config::projection_mode -- one setting controls both. +// +// - Auto (default): resolved from the error-bound norm at (de)compress time +// -- Hierarchical under an L-infinity bound, Orthogonal otherwise. See +// resolve_projection_mode in Utilities/ProjectionMode.h. +// - Orthogonal / Hierarchical: forced explicitly. Hierarchical still only +// supports L-infinity error control; requesting it under any other bound +// throws. Unlike kernel fusion, the resolved basis changes the coefficient +// stream, so it is recorded in the compressed-file metadata for the hybrid +// path. +enum class compression_projection_mode_type : uint8_t { + Auto, + Orthogonal, + Hierarchical +}; + enum class processor_type : uint8_t { CPU, GPU_CUDA, @@ -32,11 +73,29 @@ enum class error_bound_type : uint8_t { REL, ABS }; enum class norm_type : uint8_t { L_Inf, L_2 }; enum class lossless_type : uint8_t { Huffman, - Huffman_LZ4, + Huffman_LZ4, // Huffman, then the portable LZ4 backend (Lossless/LZ4/) Huffman_Zstd, - CPU_Lossless + CPU_Lossless, + BlockDelta, + // Portable LZ4 applied directly to the (raw signed) quantized integer stream, + // with no Huffman entropy stage. Self-contained like BlockDelta. + LZ4, + // Zero-run-length encoding (RLE0) of the quantized stream, then a + // byte-alphabet rANS entropy stage on the resulting (counts, symbols) blob. + // Breaks Huffman's ~1-bit/symbol ratio floor. Self-contained like LZ4. + ZeroRLE_Rans, + // Symbol-alphabet rANS: same model as Huffman (outlier separation + dict_size + // histogram) but rANS entropy coding instead of Huffman codes -> always <= + // Huffman size (fractional bits), uncapped. Self-contained. + SymbolRans }; +// Encoding variant for the BlockDelta lossless backend (mirrors cuSZp): +// Fixed - fixed-length encoding only (no delta) +// Delta - per-block delta + zigzag + fixed-length encoding +// Outlier - Delta plus per-block outlier peeling (large values stored aside) +enum class block_delta_mode_type : uint8_t { Fixed, Delta, Outlier }; + enum class data_type : uint8_t { Float, Double }; enum class data_structure_type : uint8_t { Cartesian_Grid_Uniform, diff --git a/include/CompressedDataset.hpp b/include/mgard/CompressedDataset.hpp similarity index 100% rename from include/CompressedDataset.hpp rename to include/mgard/CompressedDataset.hpp diff --git a/include/CompressedDataset.tpp b/include/mgard/CompressedDataset.tpp similarity index 100% rename from include/CompressedDataset.tpp rename to include/mgard/CompressedDataset.tpp diff --git a/include/LinearQuantizer.hpp b/include/mgard/LinearQuantizer.hpp similarity index 100% rename from include/LinearQuantizer.hpp rename to include/mgard/LinearQuantizer.hpp diff --git a/include/LinearQuantizer.tpp b/include/mgard/LinearQuantizer.tpp similarity index 100% rename from include/LinearQuantizer.tpp rename to include/mgard/LinearQuantizer.tpp diff --git a/include/TensorLinearOperator.hpp b/include/mgard/TensorLinearOperator.hpp similarity index 100% rename from include/TensorLinearOperator.hpp rename to include/mgard/TensorLinearOperator.hpp diff --git a/include/TensorLinearOperator.tpp b/include/mgard/TensorLinearOperator.tpp similarity index 100% rename from include/TensorLinearOperator.tpp rename to include/mgard/TensorLinearOperator.tpp diff --git a/include/TensorMassMatrix.hpp b/include/mgard/TensorMassMatrix.hpp similarity index 100% rename from include/TensorMassMatrix.hpp rename to include/mgard/TensorMassMatrix.hpp diff --git a/include/TensorMassMatrix.tpp b/include/mgard/TensorMassMatrix.tpp similarity index 100% rename from include/TensorMassMatrix.tpp rename to include/mgard/TensorMassMatrix.tpp diff --git a/include/TensorMeshHierarchy.hpp b/include/mgard/TensorMeshHierarchy.hpp similarity index 100% rename from include/TensorMeshHierarchy.hpp rename to include/mgard/TensorMeshHierarchy.hpp diff --git a/include/TensorMeshHierarchy.tpp b/include/mgard/TensorMeshHierarchy.tpp similarity index 100% rename from include/TensorMeshHierarchy.tpp rename to include/mgard/TensorMeshHierarchy.tpp diff --git a/include/TensorMeshHierarchyIteration.hpp b/include/mgard/TensorMeshHierarchyIteration.hpp similarity index 100% rename from include/TensorMeshHierarchyIteration.hpp rename to include/mgard/TensorMeshHierarchyIteration.hpp diff --git a/include/TensorMeshHierarchyIteration.tpp b/include/mgard/TensorMeshHierarchyIteration.tpp similarity index 100% rename from include/TensorMeshHierarchyIteration.tpp rename to include/mgard/TensorMeshHierarchyIteration.tpp diff --git a/include/TensorMultilevelCoefficientQuantizer.hpp b/include/mgard/TensorMultilevelCoefficientQuantizer.hpp similarity index 100% rename from include/TensorMultilevelCoefficientQuantizer.hpp rename to include/mgard/TensorMultilevelCoefficientQuantizer.hpp diff --git a/include/TensorMultilevelCoefficientQuantizer.tpp b/include/mgard/TensorMultilevelCoefficientQuantizer.tpp similarity index 100% rename from include/TensorMultilevelCoefficientQuantizer.tpp rename to include/mgard/TensorMultilevelCoefficientQuantizer.tpp diff --git a/include/TensorNorms.hpp b/include/mgard/TensorNorms.hpp similarity index 100% rename from include/TensorNorms.hpp rename to include/mgard/TensorNorms.hpp diff --git a/include/TensorNorms.tpp b/include/mgard/TensorNorms.tpp similarity index 100% rename from include/TensorNorms.tpp rename to include/mgard/TensorNorms.tpp diff --git a/include/TensorProlongation.hpp b/include/mgard/TensorProlongation.hpp similarity index 100% rename from include/TensorProlongation.hpp rename to include/mgard/TensorProlongation.hpp diff --git a/include/TensorProlongation.tpp b/include/mgard/TensorProlongation.tpp similarity index 100% rename from include/TensorProlongation.tpp rename to include/mgard/TensorProlongation.tpp diff --git a/include/TensorQuantityOfInterest.hpp b/include/mgard/TensorQuantityOfInterest.hpp similarity index 100% rename from include/TensorQuantityOfInterest.hpp rename to include/mgard/TensorQuantityOfInterest.hpp diff --git a/include/TensorQuantityOfInterest.tpp b/include/mgard/TensorQuantityOfInterest.tpp similarity index 100% rename from include/TensorQuantityOfInterest.tpp rename to include/mgard/TensorQuantityOfInterest.tpp diff --git a/include/TensorRestriction.hpp b/include/mgard/TensorRestriction.hpp similarity index 100% rename from include/TensorRestriction.hpp rename to include/mgard/TensorRestriction.hpp diff --git a/include/TensorRestriction.tpp b/include/mgard/TensorRestriction.tpp similarity index 100% rename from include/TensorRestriction.tpp rename to include/mgard/TensorRestriction.tpp diff --git a/include/adaptive_roi.hpp b/include/mgard/adaptive_roi.hpp similarity index 100% rename from include/adaptive_roi.hpp rename to include/mgard/adaptive_roi.hpp diff --git a/include/adaptive_roi.tpp b/include/mgard/adaptive_roi.tpp similarity index 100% rename from include/adaptive_roi.tpp rename to include/mgard/adaptive_roi.tpp diff --git a/include/blas.hpp b/include/mgard/blas.hpp similarity index 100% rename from include/blas.hpp rename to include/mgard/blas.hpp diff --git a/include/blas.tpp b/include/mgard/blas.tpp similarity index 100% rename from include/blas.tpp rename to include/mgard/blas.tpp diff --git a/include/cli/arguments.hpp b/include/mgard/cli/arguments.hpp similarity index 99% rename from include/cli/arguments.hpp rename to include/mgard/cli/arguments.hpp index 6e7e29e6e3..39e6943277 100644 --- a/include/cli/arguments.hpp +++ b/include/mgard/cli/arguments.hpp @@ -133,5 +133,5 @@ struct DecompressionArguments { } // namespace cli -#include "cli/arguments.tpp" +#include "arguments.tpp" #endif diff --git a/include/cli/arguments.tpp b/include/mgard/cli/arguments.tpp similarity index 100% rename from include/cli/arguments.tpp rename to include/mgard/cli/arguments.tpp diff --git a/include/cli/cli_internal.hpp b/include/mgard/cli/cli_internal.hpp similarity index 96% rename from include/cli/cli_internal.hpp rename to include/mgard/cli/cli_internal.hpp index b117e288a0..ded3309dad 100644 --- a/include/cli/cli_internal.hpp +++ b/include/mgard/cli/cli_internal.hpp @@ -5,7 +5,7 @@ #include -#include "cli/arguments.hpp" +#include "arguments.hpp" namespace cli { @@ -53,5 +53,5 @@ int decompress(const DecompressionArguments &arguments); } // namespace cli -#include "cli/cli_internal.tpp" +#include "cli_internal.tpp" #endif diff --git a/include/cli/cli_internal.tpp b/include/mgard/cli/cli_internal.tpp similarity index 100% rename from include/cli/cli_internal.tpp rename to include/mgard/cli/cli_internal.tpp diff --git a/include/cli/cmdline.hpp b/include/mgard/cli/cmdline.hpp similarity index 100% rename from include/cli/cmdline.hpp rename to include/mgard/cli/cmdline.hpp diff --git a/include/cli/output.hpp b/include/mgard/cli/output.hpp similarity index 99% rename from include/cli/output.hpp rename to include/mgard/cli/output.hpp index 78ec03694d..6fc902c92e 100644 --- a/include/cli/output.hpp +++ b/include/mgard/cli/output.hpp @@ -12,7 +12,7 @@ #include #include -#include "cli/cmdline.hpp" +#include "cmdline.hpp" namespace cli { diff --git a/include/compress_internal.hpp b/include/mgard/compress_internal.hpp similarity index 100% rename from include/compress_internal.hpp rename to include/mgard/compress_internal.hpp diff --git a/include/compress_internal.tpp b/include/mgard/compress_internal.tpp similarity index 100% rename from include/compress_internal.tpp rename to include/mgard/compress_internal.tpp diff --git a/include/compressors.hpp b/include/mgard/compressors.hpp similarity index 100% rename from include/compressors.hpp rename to include/mgard/compressors.hpp diff --git a/include/decompose.hpp b/include/mgard/decompose.hpp similarity index 100% rename from include/decompose.hpp rename to include/mgard/decompose.hpp diff --git a/include/decompose.tpp b/include/mgard/decompose.tpp similarity index 100% rename from include/decompose.tpp rename to include/mgard/decompose.tpp diff --git a/include/format.hpp b/include/mgard/format.hpp similarity index 98% rename from include/format.hpp rename to include/mgard/format.hpp index e7821e64e6..36d66eaab2 100644 --- a/include/format.hpp +++ b/include/mgard/format.hpp @@ -11,6 +11,8 @@ #include "proto/mgard.pb.h" +#include "MGARDConfig.hpp" + #ifdef __NVCC__ // NVCC breaks on `utilities.hpp`. See (we think) and struct MemoryBuffer; namespace mgard { -//! Magic bytes for MGARD file format ('MGARD' in ASCII). -inline constexpr std::array SIGNATURE{0x4d, 0x47, 0x41, 0x52, 0x44}; - //! Size in bytes of the serialized header size. inline constexpr std::size_t HEADER_SIZE_SIZE = 8; diff --git a/include/format.tpp b/include/mgard/format.tpp similarity index 100% rename from include/format.tpp rename to include/mgard/format.tpp diff --git a/include/quantize.hpp b/include/mgard/quantize.hpp similarity index 100% rename from include/quantize.hpp rename to include/mgard/quantize.hpp diff --git a/include/quantize.tpp b/include/mgard/quantize.tpp similarity index 100% rename from include/quantize.tpp rename to include/mgard/quantize.tpp diff --git a/include/shuffle.hpp b/include/mgard/shuffle.hpp similarity index 100% rename from include/shuffle.hpp rename to include/mgard/shuffle.hpp diff --git a/include/shuffle.tpp b/include/mgard/shuffle.tpp similarity index 100% rename from include/shuffle.tpp rename to include/mgard/shuffle.tpp diff --git a/include/unstructured/EnumeratedMeshRange.hpp b/include/mgard/unstructured/EnumeratedMeshRange.hpp similarity index 83% rename from include/unstructured/EnumeratedMeshRange.hpp rename to include/mgard/unstructured/EnumeratedMeshRange.hpp index 5c8f7db192..48aa1b5266 100644 --- a/include/unstructured/EnumeratedMeshRange.hpp +++ b/include/mgard/unstructured/EnumeratedMeshRange.hpp @@ -5,10 +5,10 @@ #include -#include "utilities.hpp" +#include "mgard/utilities.hpp" -#include "unstructured/MeshHierarchy.hpp" -#include "unstructured/MeshLevel.hpp" +#include "MeshHierarchy.hpp" +#include "MeshLevel.hpp" namespace mgard { diff --git a/include/unstructured/IndicatorInput.hpp b/include/mgard/unstructured/IndicatorInput.hpp similarity index 97% rename from include/unstructured/IndicatorInput.hpp rename to include/mgard/unstructured/IndicatorInput.hpp index ab165cc43e..1e4949c72c 100644 --- a/include/unstructured/IndicatorInput.hpp +++ b/include/mgard/unstructured/IndicatorInput.hpp @@ -13,10 +13,10 @@ #include "moab/EntityHandle.hpp" #include "moab/Range.hpp" -#include "utilities.hpp" +#include "mgard/utilities.hpp" -#include "unstructured/MeshHierarchy.hpp" -#include "unstructured/MeshLevel.hpp" +#include "MeshHierarchy.hpp" +#include "MeshLevel.hpp" namespace mgard { diff --git a/include/unstructured/LinearOperator.hpp b/include/mgard/unstructured/LinearOperator.hpp similarity index 100% rename from include/unstructured/LinearOperator.hpp rename to include/mgard/unstructured/LinearOperator.hpp diff --git a/include/unstructured/MassMatrix.hpp b/include/mgard/unstructured/MassMatrix.hpp similarity index 97% rename from include/unstructured/MassMatrix.hpp rename to include/mgard/unstructured/MassMatrix.hpp index 7734d5e7b2..73f8ef685b 100644 --- a/include/unstructured/MassMatrix.hpp +++ b/include/mgard/unstructured/MassMatrix.hpp @@ -7,8 +7,8 @@ #include "moab/EntityHandle.hpp" -#include "unstructured/LinearOperator.hpp" -#include "unstructured/MeshLevel.hpp" +#include "LinearOperator.hpp" +#include "MeshLevel.hpp" namespace mgard { diff --git a/include/unstructured/MeshHierarchy.hpp b/include/mgard/unstructured/MeshHierarchy.hpp similarity index 99% rename from include/unstructured/MeshHierarchy.hpp rename to include/mgard/unstructured/MeshHierarchy.hpp index 56b8220794..72afad9aa5 100644 --- a/include/unstructured/MeshHierarchy.hpp +++ b/include/mgard/unstructured/MeshHierarchy.hpp @@ -12,10 +12,10 @@ #include "moab/Range.hpp" #include "moab/Types.hpp" -#include "utilities.hpp" +#include "mgard/utilities.hpp" -#include "unstructured/MeshLevel.hpp" -#include "unstructured/data.hpp" +#include "MeshLevel.hpp" +#include "data.hpp" namespace mgard { diff --git a/include/unstructured/MeshLevel.hpp b/include/mgard/unstructured/MeshLevel.hpp similarity index 99% rename from include/unstructured/MeshLevel.hpp rename to include/mgard/unstructured/MeshLevel.hpp index cbeba0b8cc..60e30ceefa 100644 --- a/include/unstructured/MeshLevel.hpp +++ b/include/mgard/unstructured/MeshLevel.hpp @@ -8,7 +8,7 @@ #include "moab/EntityType.hpp" #include "moab/Interface.hpp" -#include "utilities.hpp" +#include "mgard/utilities.hpp" namespace mgard { diff --git a/include/unstructured/MeshRefiner.hpp b/include/mgard/unstructured/MeshRefiner.hpp similarity index 94% rename from include/unstructured/MeshRefiner.hpp rename to include/mgard/unstructured/MeshRefiner.hpp index 6edc6c53f2..c258356779 100644 --- a/include/unstructured/MeshRefiner.hpp +++ b/include/mgard/unstructured/MeshRefiner.hpp @@ -3,7 +3,7 @@ //!\file //!\brief Function object which refines meshes. -#include "unstructured/MeshLevel.hpp" +#include "MeshLevel.hpp" namespace mgard { diff --git a/include/unstructured/MultilevelCoefficientQuantizer.hpp b/include/mgard/unstructured/MultilevelCoefficientQuantizer.hpp similarity index 97% rename from include/unstructured/MultilevelCoefficientQuantizer.hpp rename to include/mgard/unstructured/MultilevelCoefficientQuantizer.hpp index ebf6a6233e..e432317bb1 100644 --- a/include/unstructured/MultilevelCoefficientQuantizer.hpp +++ b/include/mgard/unstructured/MultilevelCoefficientQuantizer.hpp @@ -7,11 +7,11 @@ #include -#include "utilities.hpp" +#include "mgard/utilities.hpp" -#include "unstructured/IndicatorInput.hpp" -#include "unstructured/MeshHierarchy.hpp" -#include "unstructured/data.hpp" +#include "IndicatorInput.hpp" +#include "MeshHierarchy.hpp" +#include "data.hpp" namespace mgard { @@ -240,5 +240,5 @@ class MultilevelCoefficientDequantizer::iterator { } // namespace mgard -#include "unstructured/MultilevelCoefficientQuantizer.tpp" +#include "MultilevelCoefficientQuantizer.tpp" #endif diff --git a/include/unstructured/MultilevelCoefficientQuantizer.tpp b/include/mgard/unstructured/MultilevelCoefficientQuantizer.tpp similarity index 98% rename from include/unstructured/MultilevelCoefficientQuantizer.tpp rename to include/mgard/unstructured/MultilevelCoefficientQuantizer.tpp index dc47d323ca..c597c2b82c 100644 --- a/include/unstructured/MultilevelCoefficientQuantizer.tpp +++ b/include/mgard/unstructured/MultilevelCoefficientQuantizer.tpp @@ -1,8 +1,8 @@ #include -#include "LinearQuantizer.hpp" +#include "mgard/LinearQuantizer.hpp" -#include "unstructured/indicators.hpp" +#include "indicators.hpp" namespace mgard { diff --git a/include/unstructured/SituatedCoefficientRange.hpp b/include/mgard/unstructured/SituatedCoefficientRange.hpp similarity index 89% rename from include/unstructured/SituatedCoefficientRange.hpp rename to include/mgard/unstructured/SituatedCoefficientRange.hpp index 01065db330..e80b459707 100644 --- a/include/unstructured/SituatedCoefficientRange.hpp +++ b/include/mgard/unstructured/SituatedCoefficientRange.hpp @@ -10,9 +10,9 @@ #include "moab/EntityHandle.hpp" #include "moab/Range.hpp" -#include "utilities.hpp" +#include "mgard/utilities.hpp" -#include "unstructured/MeshHierarchy.hpp" +#include "MeshHierarchy.hpp" namespace mgard { @@ -34,5 +34,5 @@ class SituatedCoefficientRange } // namespace mgard -#include "unstructured/SituatedCoefficientRange.tpp" +#include "SituatedCoefficientRange.tpp" #endif diff --git a/include/unstructured/SituatedCoefficientRange.tpp b/include/mgard/unstructured/SituatedCoefficientRange.tpp similarity index 100% rename from include/unstructured/SituatedCoefficientRange.tpp rename to include/mgard/unstructured/SituatedCoefficientRange.tpp diff --git a/include/unstructured/UniformEdgeFamilies.hpp b/include/mgard/unstructured/UniformEdgeFamilies.hpp similarity index 97% rename from include/unstructured/UniformEdgeFamilies.hpp rename to include/mgard/unstructured/UniformEdgeFamilies.hpp index 4a5594a913..97768a7f61 100644 --- a/include/unstructured/UniformEdgeFamilies.hpp +++ b/include/mgard/unstructured/UniformEdgeFamilies.hpp @@ -11,7 +11,7 @@ #include "moab/EntityHandle.hpp" -#include "unstructured/MeshLevel.hpp" +#include "MeshLevel.hpp" namespace mgard { @@ -119,5 +119,5 @@ template class EdgeFamilyIterable { } // namespace mgard -#include "unstructured/UniformEdgeFamilies.tpp" +#include "UniformEdgeFamilies.tpp" #endif diff --git a/include/unstructured/UniformEdgeFamilies.tpp b/include/mgard/unstructured/UniformEdgeFamilies.tpp similarity index 100% rename from include/unstructured/UniformEdgeFamilies.tpp rename to include/mgard/unstructured/UniformEdgeFamilies.tpp diff --git a/include/unstructured/UniformMeshHierarchy.hpp b/include/mgard/unstructured/UniformMeshHierarchy.hpp similarity index 97% rename from include/unstructured/UniformMeshHierarchy.hpp rename to include/mgard/unstructured/UniformMeshHierarchy.hpp index 73ff2afa7b..4cdb0551bb 100644 --- a/include/unstructured/UniformMeshHierarchy.hpp +++ b/include/mgard/unstructured/UniformMeshHierarchy.hpp @@ -6,9 +6,9 @@ #include -#include "unstructured/MeshHierarchy.hpp" -#include "unstructured/MeshLevel.hpp" -#include "unstructured/UniformEdgeFamilies.hpp" +#include "MeshHierarchy.hpp" +#include "MeshLevel.hpp" +#include "UniformEdgeFamilies.hpp" namespace mgard { diff --git a/include/unstructured/UniformMeshRefiner.hpp b/include/mgard/unstructured/UniformMeshRefiner.hpp similarity index 96% rename from include/unstructured/UniformMeshRefiner.hpp rename to include/mgard/unstructured/UniformMeshRefiner.hpp index 3bbf1f1722..6443c5c45c 100644 --- a/include/unstructured/UniformMeshRefiner.hpp +++ b/include/mgard/unstructured/UniformMeshRefiner.hpp @@ -6,8 +6,8 @@ #include "moab/Range.hpp" #include "moab/Types.hpp" -#include "unstructured/MeshLevel.hpp" -#include "unstructured/MeshRefiner.hpp" +#include "MeshLevel.hpp" +#include "MeshRefiner.hpp" namespace mgard { diff --git a/include/unstructured/UniformRestriction.hpp b/include/mgard/unstructured/UniformRestriction.hpp similarity index 91% rename from include/unstructured/UniformRestriction.hpp rename to include/mgard/unstructured/UniformRestriction.hpp index 26b41ed0fe..7b6e018567 100644 --- a/include/unstructured/UniformRestriction.hpp +++ b/include/mgard/unstructured/UniformRestriction.hpp @@ -4,8 +4,8 @@ //!\brief Restriction for piecewise linears on mesh hierarchies produced by //! uniform refinement. -#include "unstructured/LinearOperator.hpp" -#include "unstructured/MeshLevel.hpp" +#include "LinearOperator.hpp" +#include "MeshLevel.hpp" namespace mgard { diff --git a/include/unstructured/data.hpp b/include/mgard/unstructured/data.hpp similarity index 98% rename from include/unstructured/data.hpp rename to include/mgard/unstructured/data.hpp index 400849ed53..83d5ed9732 100644 --- a/include/unstructured/data.hpp +++ b/include/mgard/unstructured/data.hpp @@ -77,5 +77,5 @@ bool operator!=(const MultilevelCoefficients &a, } // namespace mgard -#include "unstructured/data.tpp" +#include "data.tpp" #endif diff --git a/include/unstructured/data.tpp b/include/mgard/unstructured/data.tpp similarity index 100% rename from include/unstructured/data.tpp rename to include/mgard/unstructured/data.tpp diff --git a/include/unstructured/estimators.hpp b/include/mgard/unstructured/estimators.hpp similarity index 95% rename from include/unstructured/estimators.hpp rename to include/mgard/unstructured/estimators.hpp index aa2c06419b..75b241caf4 100644 --- a/include/unstructured/estimators.hpp +++ b/include/mgard/unstructured/estimators.hpp @@ -3,8 +3,8 @@ //!\file //!\brief Function space norm estimators. -#include "unstructured/MeshHierarchy.hpp" -#include "unstructured/data.hpp" +#include "MeshHierarchy.hpp" +#include "data.hpp" namespace mgard { diff --git a/include/unstructured/indicators.hpp b/include/mgard/unstructured/indicators.hpp similarity index 88% rename from include/unstructured/indicators.hpp rename to include/mgard/unstructured/indicators.hpp index 1e14ccd115..290de4d9f0 100644 --- a/include/unstructured/indicators.hpp +++ b/include/mgard/unstructured/indicators.hpp @@ -3,9 +3,9 @@ //!\file //!\brief Function space norm estimator indicators. -#include "unstructured/IndicatorInput.hpp" -#include "unstructured/MeshHierarchy.hpp" -#include "unstructured/estimators.hpp" +#include "IndicatorInput.hpp" +#include "MeshHierarchy.hpp" +#include "estimators.hpp" namespace mgard { diff --git a/include/unstructured/measure.hpp b/include/mgard/unstructured/measure.hpp similarity index 100% rename from include/unstructured/measure.hpp rename to include/mgard/unstructured/measure.hpp diff --git a/include/unstructured/norms.hpp b/include/mgard/unstructured/norms.hpp similarity index 90% rename from include/unstructured/norms.hpp rename to include/mgard/unstructured/norms.hpp index 83a008f5ef..e5a6ebad2c 100644 --- a/include/unstructured/norms.hpp +++ b/include/mgard/unstructured/norms.hpp @@ -3,8 +3,8 @@ //!\file //!\brief Unstructured mesh function space norms. -#include "unstructured/MeshHierarchy.hpp" -#include "unstructured/data.hpp" +#include "MeshHierarchy.hpp" +#include "data.hpp" namespace mgard { diff --git a/include/unstructured/pcg.hpp b/include/mgard/unstructured/pcg.hpp similarity index 98% rename from include/unstructured/pcg.hpp rename to include/mgard/unstructured/pcg.hpp index c47c6c0c1e..fcc9995204 100644 --- a/include/unstructured/pcg.hpp +++ b/include/mgard/unstructured/pcg.hpp @@ -6,7 +6,7 @@ #include -#include "unstructured/LinearOperator.hpp" +#include "LinearOperator.hpp" namespace mgard { diff --git a/include/utilities.hpp b/include/mgard/utilities.hpp similarity index 100% rename from include/utilities.hpp rename to include/mgard/utilities.hpp diff --git a/include/utilities.tpp b/include/mgard/utilities.tpp similarity index 100% rename from include/utilities.tpp rename to include/mgard/utilities.tpp diff --git a/scripts/generate_dimension_switch.py b/scripts/generate_dimension_switch.py index 863915989a..0dfbed9df6 100644 --- a/scripts/generate_dimension_switch.py +++ b/scripts/generate_dimension_switch.py @@ -126,7 +126,7 @@ def __str__(self) -> str: g_arguments: typing.Tuple[FunctionArgument, ...] if args.compress: - header = 'cli/cli_internal.hpp' + header = 'mgard/cli/cli_internal.hpp' namespace = 'cli' f_name = 'compress' f_ret_type = 'int' @@ -138,7 +138,7 @@ def __str__(self) -> str: g_name = 'compress_N' g_arguments = f_arguments[: 1] elif args.decompress: - header = 'compress_internal.hpp' + header = 'mgard/compress_internal.hpp' namespace = 'mgard' f_name = 'decompress' f_ret_type = 'MemoryBuffer' diff --git a/scripts/generate_mgard_x_roi_map.py b/scripts/generate_mgard_x_roi_map.py new file mode 100755 index 0000000000..342b53ac70 --- /dev/null +++ b/scripts/generate_mgard_x_roi_map.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +"""Generate a block-level tolerance map for the MGARD-X hybrid hierarchy.""" + +import argparse +import math +from array import array +from itertools import product +from pathlib import Path + + +BLOCK_SIZE = 8 + + +def positive_tolerance(value): + tolerance = float(value) + if not math.isfinite(tolerance) or tolerance <= 0: + raise argparse.ArgumentTypeError("tolerances must be finite and positive") + return tolerance + + +def linear_index(coordinate, shape): + index = 0 + for position, extent in zip(coordinate, shape): + index = index * extent + position + return index + + +def parse_roi(tokens, shape, parser): + expected = 1 + 2 * len(shape) + if len(tokens) != expected: + parser.error( + "each --roi requires TOL followed by one START END pair per " + f"dimension ({expected} values for {len(shape)}D data)" + ) + + try: + tolerance = positive_tolerance(tokens[0]) + except argparse.ArgumentTypeError as error: + parser.error(str(error)) + bounds = [] + for dimension, extent in enumerate(shape): + try: + start = int(tokens[1 + 2 * dimension]) + end = int(tokens[2 + 2 * dimension]) + except ValueError: + parser.error("ROI bounds must be integers") + if start < 0 or end <= start or end > extent: + parser.error( + f"ROI dimension {dimension} must satisfy " + f"0 <= START < END <= {extent}" + ) + bounds.append((start, end)) + return tolerance, bounds + + +def main(): + parser = argparse.ArgumentParser( + description=( + "Generate the raw float64 tolerance map consumed by MGARD-X " + "hybrid ROI compression." + ) + ) + parser.add_argument("-o", "--output", required=True, type=Path) + parser.add_argument( + "-dim", + "--shape", + required=True, + nargs="+", + type=int, + metavar="N", + help="1D-3D data shape in slowest-to-fastest dimension order", + ) + parser.add_argument( + "-bg", + "--background", + required=True, + type=positive_tolerance, + metavar="TOL", + help="tolerance assigned to blocks outside every ROI", + ) + parser.add_argument( + "-roi", + "--roi", + action="append", + default=[], + nargs="+", + metavar="VALUE", + help="TOL START0 END0 [START1 END1 ...]; may be repeated", + ) + args = parser.parse_args() + + if not 1 <= len(args.shape) <= 3: + parser.error("the hybrid hierarchy supports 1D-3D data only") + if any(extent <= 0 for extent in args.shape): + parser.error("all shape extents must be positive") + + block_shape = [ + (extent + BLOCK_SIZE - 1) // BLOCK_SIZE for extent in args.shape + ] + total_blocks = math.prod(block_shape) + tolerances = array("d", [args.background]) * total_blocks + + parsed_rois = [parse_roi(tokens, args.shape, parser) for tokens in args.roi] + for tolerance, bounds in parsed_rois: + block_ranges = [] + for start, end in bounds: + first = start // BLOCK_SIZE + past_last = (end + BLOCK_SIZE - 1) // BLOCK_SIZE + block_ranges.append(range(first, past_last)) + for coordinate in product(*block_ranges): + index = linear_index(coordinate, block_shape) + tolerances[index] = min(tolerances[index], tolerance) + + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("wb") as output: + tolerances.tofile(output) + + print("Data shape:", " x ".join(map(str, args.shape))) + print("Block shape:", " x ".join(map(str, block_shape))) + print("Block size:", BLOCK_SIZE) + print("ROI regions:", len(parsed_rois)) + print(f"Wrote {total_blocks} float64 values to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/src/cuda/API.cu b/src/cuda/API.cu deleted file mode 100644 index 05bd9d10cd..0000000000 --- a/src/cuda/API.cu +++ /dev/null @@ -1,465 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: September 27, 2021 - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "cuda/CommonInternal.h" - -// #include "cuda/CompressionWorkflow.h" -#include "compress_cuda.hpp" - -#include "cuda/MemoryManagement.h" - -#include "cuda/DataRefactoring.h" -#include "cuda/LinearQuantization.h" -#include "cuda/LosslessCompression.h" - -namespace mgard_cuda { - -bool verify(const void *compressed_data, size_t compressed_size) { - char magic_word[MAGIC_WORD_SIZE + 1]; - if (compressed_size < sizeof(magic_word)) - return false; - SIZE meta_size = *(SIZE *)compressed_data; - Metadata meta; - meta.Deserialize((SERIALIZED_TYPE *)compressed_data, meta_size); - std::memcpy(magic_word, meta.magic_word, MAGIC_WORD_SIZE); - magic_word[MAGIC_WORD_SIZE] = '\0'; - if (strcmp(magic_word, MAGIC_WORD) == 0) { - return true; - } else { - return false; - } -} - -enum data_type infer_data_type(const void *compressed_data, - size_t compressed_size) { - if (!verify(compressed_data, compressed_size)) { - std::cout << log::log_err << "cannot verify the data!\n"; - exit(-1); - } - Metadata meta; - SIZE meta_size = *(SIZE *)compressed_data + meta.metadata_size_offset(); - meta.Deserialize((SERIALIZED_TYPE *)compressed_data, meta_size); - return meta.dtype; -} - -std::vector infer_shape(const void *compressed_data, - size_t compressed_size) { - if (!verify(compressed_data, compressed_size)) { - std::cout << log::log_err << "cannot verify the data!\n"; - exit(-1); - } - - Metadata meta; - uint32_t meta_size = - *(uint32_t *)compressed_data + meta.metadata_size_offset(); - meta.Deserialize((SERIALIZED_TYPE *)compressed_data, meta_size); - std::vector shape(meta.total_dims); - for (DIM d = 0; d < meta.total_dims; d++) { - shape[d] = (SIZE)(*(meta.shape + d)); - } - return shape; -} - -enum data_structure_type infer_data_structure(const void *compressed_data, - size_t compressed_size) { - if (!verify(compressed_data, compressed_size)) { - std::cout << log::log_err << "cannot verify the data!\n"; - exit(-1); - } - Metadata meta; - uint32_t meta_size = - *(uint32_t *)compressed_data + meta.metadata_size_offset(); - meta.Deserialize((SERIALIZED_TYPE *)compressed_data, meta_size); - return meta.dstype; -} - -template -std::vector infer_coords(const void *compressed_data, - size_t compressed_size) { - if (!verify(compressed_data, compressed_size)) { - std::cout << log::log_err << "cannot verify the data!\n"; - exit(-1); - } - Metadata meta; - uint32_t meta_size = - *(uint32_t *)compressed_data + meta.metadata_size_offset(); - meta.Deserialize((SERIALIZED_TYPE *)compressed_data, meta_size); - std::vector shape(meta.total_dims); - for (DIM d = 0; d < meta.total_dims; d++) { - shape[d] = (SIZE)(*(meta.shape + d)); - } - std::vector coords(meta.total_dims); - for (DIM d = 0; d < meta.total_dims; d++) { - coords[d] = (T *)std::malloc(shape[d] * sizeof(T)); - std::memcpy(coords[d], meta.coords[d], shape[d] * sizeof(T)); - } - return coords; -} - -std::string infer_nonuniform_coords_file(const void *compressed_data, - size_t compressed_size) { - if (!verify(compressed_data, compressed_size)) { - std::cout << log::log_err << "cannot verify the data!\n"; - exit(-1); - } - Metadata meta; - uint32_t meta_size = - *(uint32_t *)compressed_data + meta.metadata_size_offset(); - meta.Deserialize((SERIALIZED_TYPE *)compressed_data, meta_size); - return std::string(meta.nonuniform_coords_file); -} - -template -void compress(std::vector shape, T tol, T s, enum error_bound_type mode, - const void *original_data, void *&compressed_data, - size_t &compressed_size, Config config) { - Handle handle(shape, config); - mgard_cuda::Array in_array(shape); - in_array.loadData((const T *)original_data); - Array<1, unsigned char> compressed_array = - compress(handle, in_array, mode, tol, s); - compressed_size = compressed_array.getShape()[0]; - if (isGPUPointer(original_data)) { - cudaMallocHelper(handle, (void **)&compressed_data, compressed_size); - cudaMemcpyAsyncHelper(handle, compressed_data, compressed_array.get_dv(), - compressed_size, AUTO, 0); - handle.sync(0); - } else { - compressed_data = (unsigned char *)malloc(compressed_size); - memcpy(compressed_data, compressed_array.getDataHost(), compressed_size); - } -} - -template -void compress(std::vector shape, T tol, T s, enum error_bound_type mode, - const void *original_data, void *&compressed_data, - size_t &compressed_size, Config config, std::vector coords) { - Handle handle(shape, coords, config); - mgard_cuda::Array in_array(shape); - in_array.loadData((const T *)original_data); - Array<1, unsigned char> compressed_array = - compress(handle, in_array, mode, tol, s); - compressed_size = compressed_array.getShape()[0]; - if (isGPUPointer(original_data)) { - cudaMallocHelper(handle, (void **)&compressed_data, compressed_size); - cudaMemcpyAsyncHelper(handle, compressed_data, compressed_array.get_dv(), - compressed_size, AUTO, 0); - handle.sync(0); - } else { - compressed_data = (unsigned char *)malloc(compressed_size); - memcpy(compressed_data, compressed_array.getDataHost(), compressed_size); - } -} - -template -void decompress(std::vector shape, const void *compressed_data, - size_t compressed_size, void *&decompressed_data, - std::vector coords, Config config) { - size_t original_size = 1; - for (int i = 0; i < D; i++) { - original_size *= shape[i]; - } - Handle handle(shape, coords, config); - std::vector compressed_shape(1); - compressed_shape[0] = compressed_size; - Array<1, unsigned char> compressed_array(compressed_shape); - compressed_array.loadData((const unsigned char *)compressed_data); - Array out_array = decompress(handle, compressed_array); - - if (isGPUPointer(compressed_data)) { - cudaMallocHelper(handle, (void **)&decompressed_data, - original_size * sizeof(T)); - cudaMemcpyAsyncHelper(handle, decompressed_data, out_array.get_dv(), - original_size * sizeof(T), AUTO, 0); - handle.sync(0); - } else { - decompressed_data = (T *)malloc(original_size * sizeof(T)); - memcpy(decompressed_data, out_array.getDataHost(), - original_size * sizeof(T)); - } -} - -template -void decompress(std::vector shape, const void *compressed_data, - size_t compressed_size, void *&decompressed_data, - Config config) { - size_t original_size = 1; - for (int i = 0; i < D; i++) - original_size *= shape[i]; - Handle handle(shape, config); - std::vector compressed_shape(1); - compressed_shape[0] = compressed_size; - Array<1, unsigned char> compressed_array(compressed_shape); - compressed_array.loadData((const unsigned char *)compressed_data); - Array out_array = decompress(handle, compressed_array); - if (isGPUPointer(compressed_data)) { - cudaMallocHelper(handle, (void **)&decompressed_data, - original_size * sizeof(T)); - cudaMemcpyAsyncHelper(handle, decompressed_data, out_array.get_dv(), - original_size * sizeof(T), AUTO, 0); - handle.sync(0); - } else { - decompressed_data = (T *)malloc(original_size * sizeof(T)); - memcpy(decompressed_data, out_array.getDataHost(), - original_size * sizeof(T)); - } -} - -void compress(DIM D, data_type dtype, std::vector shape, double tol, - double s, enum error_bound_type mode, const void *original_data, - void *&compressed_data, size_t &compressed_size, Config config) { - if (dtype == data_type::Float) { - if (D == 1) { - compress<1, float>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config); - } else if (D == 2) { - compress<2, float>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config); - } else if (D == 3) { - compress<3, float>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config); - } else if (D == 4) { - compress<4, float>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config); - } else if (D == 5) { - compress<5, float>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config); - } else { - std::cout << log::log_err - << "do not support higher than five dimentions!\n"; - exit(-1); - } - } else if (dtype == data_type::Double) { - if (D == 1) { - compress<1, double>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config); - } else if (D == 2) { - compress<2, double>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config); - } else if (D == 3) { - compress<3, double>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config); - } else if (D == 4) { - compress<4, double>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config); - } else if (D == 5) { - compress<5, double>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config); - } else { - std::cout << log::log_err - << "do not support higher than five dimentions!\n"; - exit(-1); - } - } else { - std::cout << log::log_err - << "do not support types other than double and float!\n"; - exit(-1); - } -} - -void compress(DIM D, data_type dtype, std::vector shape, double tol, - double s, enum error_bound_type mode, const void *original_data, - void *&compressed_data, size_t &compressed_size) { - compress(D, dtype, shape, tol, s, mode, original_data, compressed_data, - compressed_size, Config()); -} - -void compress(DIM D, data_type dtype, std::vector shape, double tol, - double s, enum error_bound_type mode, const void *original_data, - void *&compressed_data, size_t &compressed_size, - std::vector coords, Config config) { - - if (dtype == data_type::Float) { - std::vector float_coords; - for (auto &coord : coords) - float_coords.push_back((float *)coord); - if (D == 1) { - compress<1, float>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config, float_coords); - } else if (D == 2) { - compress<2, float>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config, float_coords); - } else if (D == 3) { - compress<3, float>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config, float_coords); - } else if (D == 4) { - compress<4, float>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config, float_coords); - } else if (D == 5) { - compress<5, float>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config, float_coords); - } else { - std::cout << log::log_err - << "do not support higher than five dimentions!\n"; - exit(-1); - } - } else if (dtype == data_type::Double) { - std::vector double_coords; - for (auto &coord : coords) - double_coords.push_back((double *)coord); - if (D == 1) { - compress<1, double>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config, double_coords); - } else if (D == 2) { - compress<2, double>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config, double_coords); - } else if (D == 3) { - compress<3, double>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config, double_coords); - } else if (D == 4) { - compress<4, double>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config, double_coords); - } else if (D == 5) { - compress<5, double>(shape, tol, s, mode, original_data, compressed_data, - compressed_size, config, double_coords); - } else { - std::cout << log::log_err - << "do not support higher than five dimentions!\n"; - exit(-1); - } - } else { - std::cout << log::log_err - << "do not support types other than double and float!\n"; - exit(-1); - } -} - -void compress(DIM D, data_type dtype, std::vector shape, double tol, - double s, enum error_bound_type mode, const void *original_data, - void *&compressed_data, size_t &compressed_size, - std::vector coords) { - compress(D, dtype, shape, tol, s, mode, original_data, compressed_data, - compressed_size, coords, Config()); -} - -void decompress(const void *compressed_data, size_t compressed_size, - void *&decompressed_data, Config config) { - - std::vector shape = infer_shape(compressed_data, compressed_size); - data_type dtype = infer_data_type(compressed_data, compressed_size); - data_structure_type dstype = - infer_data_structure(compressed_data, compressed_size); - - if (dtype == data_type::Float) { - if (dstype == data_structure_type::Cartesian_Grid_Uniform) { - if (shape.size() == 1) { - decompress<1, float>(shape, compressed_data, compressed_size, - decompressed_data, config); - } else if (shape.size() == 2) { - decompress<2, float>(shape, compressed_data, compressed_size, - decompressed_data, config); - } else if (shape.size() == 3) { - decompress<3, float>(shape, compressed_data, compressed_size, - decompressed_data, config); - } else if (shape.size() == 4) { - decompress<4, float>(shape, compressed_data, compressed_size, - decompressed_data, config); - } else if (shape.size() == 5) { - decompress<5, float>(shape, compressed_data, compressed_size, - decompressed_data, config); - } else { - std::cout << log::log_err - << "do not support higher than five dimentions!\n"; - exit(-1); - } - } else if (dstype == data_structure_type::Cartesian_Grid_Non_Uniform) { - - std::vector coords = - infer_coords(compressed_data, compressed_size); - - if (shape.size() == 1) { - decompress<1, float>(shape, compressed_data, compressed_size, - decompressed_data, coords, config); - } else if (shape.size() == 2) { - decompress<2, float>(shape, compressed_data, compressed_size, - decompressed_data, coords, config); - } else if (shape.size() == 3) { - decompress<3, float>(shape, compressed_data, compressed_size, - decompressed_data, coords, config); - } else if (shape.size() == 4) { - decompress<4, float>(shape, compressed_data, compressed_size, - decompressed_data, coords, config); - } else if (shape.size() == 5) { - decompress<5, float>(shape, compressed_data, compressed_size, - decompressed_data, coords, config); - } else { - std::cout << log::log_err - << "do not support higher than five dimentions!\n"; - exit(-1); - } - } - } else if (dtype == data_type::Double) { - if (dstype == data_structure_type::Cartesian_Grid_Uniform) { - if (shape.size() == 1) { - decompress<1, double>(shape, compressed_data, compressed_size, - decompressed_data, config); - } else if (shape.size() == 2) { - decompress<2, double>(shape, compressed_data, compressed_size, - decompressed_data, config); - } else if (shape.size() == 3) { - decompress<3, double>(shape, compressed_data, compressed_size, - decompressed_data, config); - } else if (shape.size() == 4) { - decompress<4, double>(shape, compressed_data, compressed_size, - decompressed_data, config); - } else if (shape.size() == 5) { - decompress<5, double>(shape, compressed_data, compressed_size, - decompressed_data, config); - } else { - std::cout << log::log_err - << "do not support higher than five dimentions!\n"; - exit(-1); - } - } else { - std::cout << log::log_err - << "do not support types other than double and float!\n"; - exit(-1); - } - } else if (dstype == data_structure_type::Cartesian_Grid_Non_Uniform) { - - std::vector coords = - infer_coords(compressed_data, compressed_size); - - if (shape.size() == 1) { - decompress<1, double>(shape, compressed_data, compressed_size, - decompressed_data, coords, config); - } else if (shape.size() == 2) { - decompress<2, double>(shape, compressed_data, compressed_size, - decompressed_data, coords, config); - } else if (shape.size() == 3) { - decompress<3, double>(shape, compressed_data, compressed_size, - decompressed_data, coords, config); - } else if (shape.size() == 4) { - decompress<4, double>(shape, compressed_data, compressed_size, - decompressed_data, coords, config); - } else if (shape.size() == 5) { - decompress<5, double>(shape, compressed_data, compressed_size, - decompressed_data, coords, config); - } else { - std::cout << log::log_err - << "do not support higher than five dimentions!\n"; - exit(-1); - } - } -} - -void decompress(const void *compressed_data, size_t compressed_size, - void *&decompressed_data) { - decompress(compressed_data, compressed_size, decompressed_data, Config()); -} - -} // namespace mgard_cuda diff --git a/src/cuda/Array.cu b/src/cuda/Array.cu deleted file mode 100644 index aaf632425b..0000000000 --- a/src/cuda/Array.cu +++ /dev/null @@ -1,316 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. -<<<<<<< HEAD - * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: December 1, 2021 -======= - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 ->>>>>>> Add legacy CUDA code - */ - -#include - -#include "cuda/Array.h" -#include "cuda/CommonInternal.h" - -namespace mgard_cuda { - -template Array::Array() { - this->host_allocated = false; - this->device_allocated = false; -} - -template Array::Array(std::vector shape) { - this->host_allocated = false; - this->device_allocated = false; - std::reverse(shape.begin(), shape.end()); - this->shape = shape; - int ret = check_shape(shape); - if (ret == -1) { - std::cerr << log::log_err << "Number of dimensions mismatch (" << D - << " != " << shape.size() - << "). mgard_cuda::Array not " - "initialized!\n"; - return; - } - this->D_padded = D; - if (D < 3) { - this->D_padded = 3; - } - if (D % 2 == 0) { - this->D_padded = D + 1; - } - // padding dimensions - for (int d = this->shape.size(); d < D_padded; d++) { - this->shape.push_back(1); - } - this->linearized_depth = 1; - for (int i = 2; i < D_padded; i++) { - this->linearized_depth *= this->shape[i]; - } - Handle<1, float> handle; - size_t dv_pitch; - cudaMalloc3DHelper(handle, (void **)&(this->dv), &dv_pitch, - this->shape[0] * sizeof(T), this->shape[1], - this->linearized_depth); - this->ldvs_h.push_back((SIZE)dv_pitch / sizeof(T)); - for (int i = 1; i < D_padded; i++) { - this->ldvs_h.push_back(this->shape[i]); - } - - cudaMallocHelper(handle, (void **)&(this->ldvs_d), - this->ldvs_h.size() * sizeof(int)); - cudaMemcpyAsyncHelper(handle, this->ldvs_d, this->ldvs_h.data(), - this->ldvs_h.size() * sizeof(SIZE), AUTO, 0); - - this->device_allocated = true; -} - -template Array::Array(const Array &array) { - this->host_allocated = false; - this->device_allocated = false; - this->shape = array.shape; - this->D_padded = D; - if (D < 3) { - this->D_padded = 3; - } - if (D % 2 == 0) { - this->D_padded = D + 1; - } - // padding dimensions - for (int d = this->shape.size(); d < this->D_padded; d++) { - this->shape.push_back(1); - } - - this->linearized_depth = 1; - for (int i = 2; i < this->D_padded; i++) { - this->linearized_depth *= this->shape[i]; - } - Handle<1, float> handle; - size_t dv_pitch; - cudaMalloc3DHelper(handle, (void **)&dv, &dv_pitch, - this->shape[0] * sizeof(T), this->shape[1], - linearized_depth); - ldvs_h.push_back((SIZE)dv_pitch / sizeof(T)); - for (int i = 1; i < this->D_padded; i++) { - this->ldvs_h.push_back(this->shape[i]); - } - - cudaMallocHelper(handle, (void **)&(this->ldvs_d), - this->ldvs_h.size() * sizeof(SIZE)); - cudaMemcpyAsyncHelper(handle, this->ldvs_d, this->ldvs_h.data(), - this->ldvs_h.size() * sizeof(SIZE), AUTO, 0); - cudaMemcpy3DAsyncHelper( - handle, this->dv, this->ldvs_h[0] * sizeof(T), this->shape[0] * sizeof(T), - this->shape[1], array.dv, this->ldvs_h[0] * sizeof(T), - this->shape[0] * sizeof(T), this->shape[1], this->shape[0] * sizeof(T), - this->shape[1], this->linearized_depth, AUTO, 0); - this->device_allocated = true; -} - -template Array::Array(Array &array) { - this->host_allocated = false; - this->device_allocated = false; - this->shape = array.shape; - this->D_padded = D; - if (D < 3) { - this->D_padded = 3; - } - if (D % 2 == 0) { - this->D_padded = D + 1; - } - // padding dimensions - for (int d = this->shape.size(); d < this->D_padded; d++) { - this->shape.push_back(1); - } - - this->linearized_depth = 1; - for (int i = 2; i < this->D_padded; i++) { - this->linearized_depth *= this->shape[i]; - } - Handle<1, float> handle; - size_t dv_pitch; - cudaMalloc3DHelper(handle, (void **)&dv, &dv_pitch, - this->shape[0] * sizeof(T), this->shape[1], - linearized_depth); - ldvs_h.push_back((SIZE)dv_pitch / sizeof(T)); - for (int i = 1; i < this->D_padded; i++) { - this->ldvs_h.push_back(this->shape[i]); - } - - cudaMallocHelper(handle, (void **)&(this->ldvs_d), - this->ldvs_h.size() * sizeof(SIZE)); - cudaMemcpyAsyncHelper(handle, this->ldvs_d, this->ldvs_h.data(), - this->ldvs_h.size() * sizeof(SIZE), AUTO, 0); - cudaMemcpy3DAsyncHelper( - handle, this->dv, this->ldvs_h[0] * sizeof(T), this->shape[0] * sizeof(T), - this->shape[1], array.dv, this->ldvs_h[0] * sizeof(T), - this->shape[0] * sizeof(T), this->shape[1], this->shape[0] * sizeof(T), - this->shape[1], this->linearized_depth, AUTO, 0); - this->device_allocated = true; -} - -template -Array &Array::operator=(const Array &array) { - this->host_allocated = false; - this->device_allocated = false; - this->shape = array.shape; - this->D_padded = D; - if (D < 3) { - this->D_padded = 3; - } - if (D % 2 == 0) { - this->D_padded = D + 1; - } - // padding dimensions - for (int d = this->shape.size(); d < this->D_padded; d++) { - this->shape.push_back(1); - } - - this->linearized_depth = 1; - for (int i = 2; i < this->D_padded; i++) { - this->linearized_depth *= this->shape[i]; - } - Handle<1, float> handle; - size_t dv_pitch; - cudaMalloc3DHelper(handle, (void **)&dv, &dv_pitch, - this->shape[0] * sizeof(T), this->shape[1], - linearized_depth); - ldvs_h.push_back((SIZE)dv_pitch / sizeof(T)); - for (int i = 1; i < this->D_padded; i++) { - this->ldvs_h.push_back(this->shape[i]); - } - - cudaMallocHelper(handle, (void **)&(this->ldvs_d), - this->ldvs_h.size() * sizeof(SIZE)); - cudaMemcpyAsyncHelper(handle, this->ldvs_d, this->ldvs_h.data(), - this->ldvs_h.size() * sizeof(SIZE), AUTO, 0); - cudaMemcpy3DAsyncHelper( - handle, this->dv, this->ldvs_h[0] * sizeof(T), this->shape[0] * sizeof(T), - this->shape[1], array.dv, this->ldvs_h[0] * sizeof(T), - this->shape[0] * sizeof(T), this->shape[1], this->shape[0] * sizeof(T), - this->shape[1], this->linearized_depth, AUTO, 0); - this->device_allocated = true; - return *this; -} - -template Array::Array(Array &&array) { - this->host_allocated = false; - this->device_allocated = false; - this->shape = array.shape; - this->D_padded = D; - if (D < 3) { - this->D_padded = 3; - } - if (D % 2 == 0) { - this->D_padded = D + 1; - } - // padding dimensions - for (int d = this->shape.size(); d < this->D_padded; d++) { - this->shape.push_back(1); - } - - this->linearized_depth = 1; - for (int i = 2; i < this->D_padded; i++) { - this->linearized_depth *= this->shape[i]; - } - this->ldvs_h = array.ldvs_h; - this->ldvs_d = array.ldvs_d; - this->dv = array.dv; - array.device_allocated = false; - this->device_allocated = true; -} - -template Array::~Array() { - if (device_allocated) { - cudaFreeHelper(ldvs_d); - cudaFreeHelper(dv); - } - if (host_allocated) { - cudaFreeHostHelper(hv); - } -} - -template -void Array::loadData(const T *data, SIZE ld) { - if (ld == 0) { - ld = shape[0]; - } - Handle<1, float> handle; - cudaMemcpy3DAsyncHelper(handle, dv, ldvs_h[0] * sizeof(T), - shape[0] * sizeof(T), shape[1], data, ld * sizeof(T), - shape[0] * sizeof(T), shape[1], shape[0] * sizeof(T), - shape[1], linearized_depth, AUTO, 0); - handle.sync(0); -} - -template T *Array::getDataHost() { - Handle<1, float> handle; - if (!host_allocated) { - cudaMallocHostHelper((void **)&hv, - sizeof(T) * shape[0] * shape[1] * linearized_depth); - host_allocated = true; - } - cudaMemcpy3DAsyncHelper( - handle, hv, shape[0] * sizeof(T), shape[0] * sizeof(T), shape[1], dv, - ldvs_h[0] * sizeof(T), shape[0] * sizeof(T), shape[1], - shape[0] * sizeof(T), shape[1], linearized_depth, AUTO, 0); - handle.sync(0); - - return hv; -} - -template T *Array::getDataDevice(SIZE &ld) { - ld = ldvs_h[0]; - return dv; -} - -template std::vector Array::getShape() { - return shape; -} - -template T *Array::get_dv() { return dv; } - -template std::vector Array::get_ldvs_h() { - return ldvs_h; -} - -template SIZE *Array::get_ldvs_d() { return ldvs_d; } - -template class Array<1, double>; -template class Array<1, float>; -template class Array<2, double>; -template class Array<2, float>; -template class Array<3, double>; -template class Array<3, float>; -template class Array<4, double>; -template class Array<4, float>; -template class Array<5, double>; -template class Array<5, float>; - -// template class Array<1, unsigned char>; - -template class Array<1, bool>; - -template class Array<1, uint8_t>; -template class Array<1, uint16_t>; -template class Array<1, uint32_t>; -template class Array<1, uint64_t>; - -template class Array<2, uint8_t>; -template class Array<2, uint16_t>; -template class Array<2, uint32_t>; -template class Array<2, uint64_t>; - -template class Array<1, unsigned long long>; - -// template class Array<1, QUANTIZED_INT>; -// template class Array<2, QUANTIZED_INT>; -// template class Array<3, QUANTIZED_INT>; -// template class Array<4, QUANTIZED_INT>; -// template class Array<5, QUANTIZED_INT>; - -} // namespace mgard_cuda diff --git a/src/cuda/CMakeLists.txt b/src/cuda/CMakeLists.txt deleted file mode 100644 index 968378e985..0000000000 --- a/src/cuda/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ -set(MGARD_CUDA_SRC - ${CMAKE_CURRENT_SOURCE_DIR}/API.cu - ${CMAKE_CURRENT_SOURCE_DIR}/Array.cu - ${CMAKE_CURRENT_SOURCE_DIR}/SubArray.cu - ${CMAKE_CURRENT_SOURCE_DIR}/Message.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/CompressionWorkflow.cu - ${CMAKE_CURRENT_SOURCE_DIR}/Handle.cu - ${CMAKE_CURRENT_SOURCE_DIR}/MemoryManagement.cu - ${CMAKE_CURRENT_SOURCE_DIR}/Metadata.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/CommonInternal.cu - ${CMAKE_CURRENT_SOURCE_DIR}/PrecomputeKernels.cu - ${CMAKE_CURRENT_SOURCE_DIR}/DataRefactoring.cu - ${CMAKE_CURRENT_SOURCE_DIR}/GridProcessingKernels/InterpCalcCoeff.cu - ${CMAKE_CURRENT_SOURCE_DIR}/GridProcessingKernels/InterpCalcCoeff_3D.cu - ${CMAKE_CURRENT_SOURCE_DIR}/GridProcessingKernels/Interpolation.cu - ${CMAKE_CURRENT_SOURCE_DIR}/GridProcessingKernels/CalcCoefficients.cu - ${CMAKE_CURRENT_SOURCE_DIR}/GridProcessingKernels/Reorder.cu - ${CMAKE_CURRENT_SOURCE_DIR}/GridProcessingKernels/InterpCoeffRestore.cu - ${CMAKE_CURRENT_SOURCE_DIR}/GridProcessingKernels/InterpCoeffRestore_3D.cu - ${CMAKE_CURRENT_SOURCE_DIR}/GridProcessingKernels/InterpolationRev.cu - ${CMAKE_CURRENT_SOURCE_DIR}/GridProcessingKernels/CoefficientRestore.cu - ${CMAKE_CURRENT_SOURCE_DIR}/GridProcessingKernels/ReorderRestore.cu - ${CMAKE_CURRENT_SOURCE_DIR}/LinearProcessingKernels/MassTrans1.cu - ${CMAKE_CURRENT_SOURCE_DIR}/LinearProcessingKernels/MassTrans2.cu - ${CMAKE_CURRENT_SOURCE_DIR}/LinearProcessingKernels/MassTrans3.cu - ${CMAKE_CURRENT_SOURCE_DIR}/LinearProcessingKernels/MassTrans1_3D.cu - ${CMAKE_CURRENT_SOURCE_DIR}/LinearProcessingKernels/MassTrans2_3D.cu - ${CMAKE_CURRENT_SOURCE_DIR}/LinearProcessingKernels/MassTrans3_3D.cu - ${CMAKE_CURRENT_SOURCE_DIR}/IterativeProcessingKernels/SolveTridiag1.cu - ${CMAKE_CURRENT_SOURCE_DIR}/IterativeProcessingKernels/SolveTridiag2.cu - ${CMAKE_CURRENT_SOURCE_DIR}/IterativeProcessingKernels/SolveTridiag3.cu - ${CMAKE_CURRENT_SOURCE_DIR}/IterativeProcessingKernels/SolveTridiag1_3D.cu - ${CMAKE_CURRENT_SOURCE_DIR}/IterativeProcessingKernels/SolveTridiag2_3D.cu - ${CMAKE_CURRENT_SOURCE_DIR}/IterativeProcessingKernels/SolveTridiag3_3D.cu - ${CMAKE_CURRENT_SOURCE_DIR}/LevelwiseProcessingKernels/CopyLevel.cu - ${CMAKE_CURRENT_SOURCE_DIR}/LevelwiseProcessingKernels/AddLevel.cu - ${CMAKE_CURRENT_SOURCE_DIR}/LevelwiseProcessingKernels/SubtractLevel.cu - ${CMAKE_CURRENT_SOURCE_DIR}/LinearQuantization/LevelwiseLinearQuantization.cu - ${CMAKE_CURRENT_SOURCE_DIR}/LinearQuantization/LevelwiseLinearDequantization.cu - ${CMAKE_CURRENT_SOURCE_DIR}/LosslessCompression.cu - ${CMAKE_CURRENT_SOURCE_DIR}/ErrorCalculator.cu - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/canonical.cu - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/constants.cc - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/cuda_mem.cu - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/format.cc - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/histogram.cu - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/huffman_codec.cu - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/huffman_workflow.cu - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/huffman.cu - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/par_huffman.cu - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/par_huffman_sortbyfreq.cu - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/par_merge.cu - ${CMAKE_CURRENT_SOURCE_DIR}/ParallelHuffman/types.cc - PARENT_SCOPE) \ No newline at end of file diff --git a/src/cuda/CommonInternal.cu b/src/cuda/CommonInternal.cu deleted file mode 100644 index e47b7b99a9..0000000000 --- a/src/cuda/CommonInternal.cu +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" -#include -#include - -namespace mgard_cuda { -bool is_2kplus1_cuda(double num) { - float frac_part, f_level, int_part; - - f_level = std::log2(num - 1); - frac_part = modff(f_level, &int_part); - - if (frac_part == 0) { - return 1; - } else { - return 0; - } -} - -// __device__ int get_idx(const int ld, const int i, const int j) { -// return ld * i + j; -// } - -// // ld2 = nrow -// // ld1 = pitch -// __device__ int get_idx(const int ld1, const int ld2, const int z, const int -// y, -// const int x) { -// return ld2 * ld1 * z + ld1 * y + x; -// } - -template int check_shape(std::vector shape) { - if (D != shape.size()) { - return -1; - } - for (DIM i = 0; i < shape.size(); i++) { - if (shape[i] < 3) - return -2; - } - return 0; -} - -template int check_shape<1>(std::vector shape); -template int check_shape<2>(std::vector shape); -template int check_shape<3>(std::vector shape); -template int check_shape<4>(std::vector shape); -template int check_shape<5>(std::vector shape); - -template T max_norm_cuda(const T *v, size_t size) { - double norm = 0; - - for (int i = 0; i < size; ++i) { - T ntest = std::abs(v[i]); - if (ntest > norm) - norm = ntest; - } - return norm; -} - -template double max_norm_cuda(const double *v, size_t size); -template float max_norm_cuda(const float *v, size_t size); - -template __device__ T _get_dist(T *coords, int i, int j) { - return coords[j] - coords[i]; -} - -template __device__ double _get_dist(double *coords, int i, int j); -template __device__ float _get_dist(float *coords, int i, int j); - -__host__ __device__ int get_lindex_cuda(const int n, const int no, - const int i) { - // no: original number of points - // n : number of points at next coarser level (L-1) with 2^k+1 nodes - int lindex; - // return floor((no-2)/(n-2)*i); - if (i != n - 1) { - lindex = floor(((double)no - 2.0) / ((double)n - 2.0) * i); - } else if (i == n - 1) { - lindex = no - 1; - } - - return lindex; -} - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/CompressionWorkflow.cu b/src/cuda/CompressionWorkflow.cu deleted file mode 100644 index 95f6e6607c..0000000000 --- a/src/cuda/CompressionWorkflow.cu +++ /dev/null @@ -1,836 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: September 27, 2021 - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "cuda/CommonInternal.h" - -#include "cuda/CompressionWorkflow.h" - -#include "cuda/MemoryManagement.h" - -#include "cuda/DataRefactoring.h" -#include "cuda/LinearQuantization.h" -#include "cuda/LosslessCompression.h" - -#define BLOCK_SIZE 64 - -using namespace std::chrono; - -namespace mgard_cuda { - -template -struct linf_norm : public thrust::binary_function { - __host__ __device__ T operator()(T x, T y) { return max(abs(x), abs(y)); } -}; - -template struct l2_norm : public thrust::unary_function { - __host__ __device__ T operator()(T x) { return x * x; } -}; - -template -Array<1, unsigned char> compress(Handle &handle, Array &in_array, - enum error_bound_type type, T tol, T s) { - - cudaSetDeviceHelper(handle.dev_id); - - for (DIM i = 0; i < D; i++) { - if (handle.shapes_h[0][i] != in_array.getShape()[i]) { - std::cout << log::log_err - << "The shape of input array does not match the shape " - "initilized in handle!\n"; - std::vector empty_shape; - empty_shape.push_back(1); - Array<1, unsigned char> empty(empty_shape); - return empty; - } - } - // handle.l_target = 3; - high_resolution_clock::time_point t1, t2, start, end; - duration time_span; - size_t free, total; - - // cudaMemGetInfo(&free, &total); - // printf("Mem: %f/%f\n", (double)(total-free)/1e9, (double)total/1e9); - - if (handle.timing) - start = high_resolution_clock::now(); - T norm = (T)1.0; - - SIZE total_elems = - handle.dofs[0][0] * handle.dofs[1][0] * handle.linearized_depth; - - if (type == error_bound_type::REL) { - // printf("Calculate norm\n"); - if (handle.timing) - t1 = high_resolution_clock::now(); - thrust::device_vector v_vec(handle.dofs[0][0] * handle.dofs[1][0] * - handle.linearized_depth); - cudaMemcpy3DAsyncHelper( - handle, thrust::raw_pointer_cast(v_vec.data()), - handle.dofs[0][0] * sizeof(T), handle.dofs[0][0] * sizeof(T), - handle.dofs[1][0], in_array.get_dv(), - in_array.get_ldvs_h()[0] * sizeof(T), handle.dofs[0][0] * sizeof(T), - handle.dofs[1][0], handle.dofs[0][0] * sizeof(T), handle.dofs[1][0], - handle.linearized_depth, AUTO, 0); - handle.sync(0); - if (s == std::numeric_limits::infinity()) { - norm = thrust::reduce(v_vec.begin(), v_vec.end(), (T)0, linf_norm()); - } else { - thrust::transform(v_vec.begin(), v_vec.end(), v_vec.begin(), - l2_norm()); - norm = thrust::reduce(v_vec.begin(), v_vec.end(), (T)0); - norm = std::sqrt(norm); - } - if (handle.timing) { - t2 = high_resolution_clock::now(); - time_span = duration_cast>(t2 - t1); - std::cout << log::log_time << "Calculating norm using NVIDIA::Thrust: " - << time_span.count() << " s\n"; - } - } - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - handle.allocate_workspace(); - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - - // Decomposition - if (handle.timing) - t1 = high_resolution_clock::now(); - decompose(handle, in_array.get_dv(), in_array.get_ldvs_h(), - in_array.get_ldvs_d(), handle.l_target, 0); - handle.sync_all(); - if (handle.timing) { - t2 = high_resolution_clock::now(); - time_span = duration_cast>(t2 - t1); - std::cout << log::log_time << "decomposition time: " << time_span.count() - << " s\n"; - } - - // PrintSubarray("decomposed", SubArray(in_array)); - - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - // printf("sync_all 2\n"); - // handle.sync_all(); - handle.free_workspace(); - - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - - /**** refactoring ****/ - // outsize = (handle.linearized_depth * handle.dofs[1][0] * handle.dofs[0][0]) - // * sizeof(T); unsigned char *buffer = (unsigned char *)malloc(outsize); - // cudaMemcpy3DAsyncHelper( - // handle, - // buffer, handle.dofs[0][0] * sizeof(T), handle.dofs[0][0] * sizeof(T), - // handle.dofs[1][0], dv, lddv1 * sizeof(T), handle.dofs[0][0] * - // sizeof(T), handle.dofs[1][0], handle.dofs[0][0] * sizeof(T), - // handle.dofs[1][0], handle.linearized_depth, D2H, 0); - - // Quantization - if (handle.timing) - t1 = high_resolution_clock::now(); - bool prep_huffman = handle.lossless == lossless_type::GPU_Huffman || - handle.lossless == lossless_type::GPU_Huffman_LZ4; - SIZE dict_size = handle.huff_dict_size, block_size = handle.huff_block_size; - LENGTH quantized_count = - handle.dofs[0][0] * handle.dofs[1][0] * handle.linearized_depth; - QUANTIZED_INT *dqv; - cudaMallocHelper( - handle, (void **)&dqv, - (handle.dofs[0][0] * handle.dofs[1][0] * handle.linearized_depth) * - sizeof(QUANTIZED_INT)); - - thrust::device_vector ldqvs(handle.D_padded); - ldqvs[0] = handle.dofs[0][0]; - for (int i = 1; i < handle.D_padded; i++) { - ldqvs[i] = handle.dofs[i][0]; - } - - LENGTH estimate_outlier_count = (double)handle.dofs[0][0] * - handle.dofs[1][0] * handle.linearized_depth * - 1; - // printf("estimate_outlier_count: %llu\n", estimate_outlier_count); - LENGTH *outlier_count_d; - LENGTH *outlier_idx_d; - QUANTIZED_INT *outliers; - cudaMallocHelper(handle, (void **)&outliers, - estimate_outlier_count * sizeof(QUANTIZED_INT)); - cudaMallocHelper(handle, (void **)&outlier_count_d, sizeof(LENGTH)); - cudaMallocHelper(handle, (void **)&outlier_idx_d, - estimate_outlier_count * sizeof(LENGTH)); - LENGTH zero = 0, outlier_count, *outlier_idx_h; - cudaMemcpyAsyncHelper(handle, outlier_count_d, &zero, sizeof(LENGTH), H2D, 0); - - Metadata m; - m.ptype = processor_type::GPU_CUDA; - m.ebtype = type; - if (type == error_bound_type::REL) { - m.norm = norm; - } - m.tol = tol; - if (s == std::numeric_limits::infinity()) { - m.ntype = norm_type::L_Inf; - } else { - m.ntype = norm_type::L_2; - m.s = s; - } - m.l_target = handle.l_target; - m.ltype = handle.lossless; - m.dict_size = dict_size; - - m.dtype = - std::is_same::value ? data_type::Double : data_type::Float; - m.etype = CheckEndianess(); - m.dstype = handle.dstype; - m.total_dims = D; - m.shape = new uint64_t[D]; - for (int d = 0; d < D; d++) { - m.shape[d] = (uint64_t)handle.dofs[D - 1 - d][0]; - } - if (m.dstype == data_structure_type::Cartesian_Grid_Non_Uniform) { - m.cltype = coordinate_location::Embedded; - for (int d = 0; d < D; d++) { - m.coords.push_back((Byte *)handle.coords_h[D - 1 - d]); - } - } - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - - levelwise_linear_quantize( - handle, handle.ranges_d, handle.l_target, handle.volumes, - handle.ldvolumes, m, in_array.get_dv(), in_array.get_ldvs_d(), dqv, - thrust::raw_pointer_cast(ldqvs.data()), prep_huffman, handle.shapes_d[0], - outlier_count_d, outlier_idx_d, outliers, 0); - - cudaMemcpyAsyncHelper(handle, &outlier_count, outlier_count_d, sizeof(LENGTH), - D2H, 0); - - // printf("outlier_count: %llu\n", outlier_count); - - // printf("dqv\n"); - // print_matrix_cuda(1, quantized_count, dqv, quantized_count); - - // printf("outlier_idx_d\n"); - // print_matrix_cuda(1, outlier_count, outlier_idx_d, quantized_count); - - // printf("outliers\n"); - // print_matrix_cuda(1, outlier_count, outliers, quantized_count); - - std::vector outlier_idx; - - if (handle.timing) { - t2 = high_resolution_clock::now(); - time_span = duration_cast>(t2 - t1); - std::cout << log::log_time << "Quantization time: " << time_span.count() - << " s\n"; - std::cout << log::log_info - << "Outlier ratio: " << (double)100 * outlier_count / total_elems - << "%\n"; - } - - // cudaFreeHelper(dv); - - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - if (handle.lossless == lossless_type::GPU_Huffman || - handle.lossless == lossless_type::GPU_Huffman_LZ4) { - // printf("gpu lossless\n"); - // Huffman compression - if (handle.timing) - t1 = high_resolution_clock::now(); - uint64_t *hufmeta; - uint64_t *hufdata; - size_t hufmeta_size; - size_t hufdata_size; - huffman_compress( - handle, dqv, quantized_count, outlier_idx, hufmeta, hufmeta_size, - hufdata, hufdata_size, block_size, dict_size, 0); - // printf("sync_all 3\n"); - handle.sync_all(); - cudaFreeHelper(dqv); - if (handle.timing) { - t2 = high_resolution_clock::now(); - time_span = duration_cast>(t2 - t1); - std::cout << log::log_time - << "GPU Huffman encoding time: " << time_span.count() << " s\n"; - std::cout << log::log_info << "Huffman block size: " << block_size - << "\n"; - std::cout << log::log_info << "Huffman dictionary size: " << dict_size - << "\n"; - std::cout << log::log_info - << "Huffman compress ratio: " << total_elems * sizeof(int) - << "/" << hufmeta_size + hufdata_size << " (" - << (double)total_elems * sizeof(int) / - (hufmeta_size + hufdata_size) - << ")\n"; - } - - // SubArray<1, Byte> lossless_compressed_subarray({(SIZE)(hufdata_size)}, - // (Byte*)hufdata); PrintSubarray("Huffman lossless_compressed_subarray", - // lossless_compressed_subarray); - - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - - // LZ4 compression - void *lz4_hufmeta; - size_t lz4_hufmeta_size; - void *lz4_hufdata; - size_t lz4_hufdata_size; - - if (handle.lossless == lossless_type::GPU_Huffman_LZ4) { - SIZE lz4_before_size = hufdata_size; - if (handle.timing) - t1 = high_resolution_clock::now(); - lz4_compress(handle, hufdata, hufdata_size / sizeof(uint64_t), - lz4_hufdata, lz4_hufdata_size, handle.lz4_block_size, 0); - // printf("sync_all 4\n"); - handle.sync_all(); - cudaFreeHelper(hufdata); - hufdata = (uint64_t *)lz4_hufdata; - hufdata_size = lz4_hufdata_size; - SIZE lz4_after_size = lz4_hufdata_size; - if (handle.timing) { - t2 = high_resolution_clock::now(); - time_span = duration_cast>(t2 - t1); - std::cout << log::log_info - << "LZ4 block size: " << handle.lz4_block_size << "\n"; - std::cout << log::log_time - << "NVComp::LZ4 compression time: " << time_span.count() - << " s\n"; - - std::cout << log::log_info << "LZ4 compress ratio: " - << (double)lz4_before_size / lz4_after_size << "\n"; - } - - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - } - - if (handle.timing) { - end = high_resolution_clock::now(); - time_span = duration_cast>(end - start); - std::cout << log::log_time - << "Overall compression time: " << time_span.count() << " s (" - << (double)(handle.dofs[0][0] * handle.dofs[1][0] * - handle.linearized_depth * sizeof(T)) / - time_span.count() / 1e9 - << " GB/s)\n"; - } - - // Output serilization - if (handle.timing) - t1 = high_resolution_clock::now(); - - SIZE metadata_size; - SERIALIZED_TYPE *serizalied_meta = m.Serialize(metadata_size); - delete[] m.shape; - SIZE outsize = 0; - outsize += metadata_size; - outsize += sizeof(LENGTH) + outlier_count * sizeof(LENGTH) + - outlier_count * sizeof(QUANTIZED_INT); - outsize += sizeof(size_t) + hufmeta_size; - outsize += sizeof(size_t) + hufdata_size; - - std::vector out_shape(1); - out_shape[0] = outsize; - gpuErrchk(cudaDeviceSynchronize()); - Array<1, unsigned char> compressed_array(out_shape); - SERIALIZED_TYPE *buffer = compressed_array.get_dv(); - void *buffer_p = (void *)buffer; - - cudaMemcpyAsyncHelper(handle, buffer_p, serizalied_meta, metadata_size, - AUTO, 0); - buffer_p = buffer_p + metadata_size; - cudaMemcpyAsyncHelper(handle, buffer_p, outlier_count_d, sizeof(LENGTH), - AUTO, 0); - buffer_p = buffer_p + sizeof(LENGTH); - cudaMemcpyAsyncHelper(handle, buffer_p, outlier_idx_d, - outlier_count * sizeof(LENGTH), AUTO, 0); - buffer_p = buffer_p + outlier_count * sizeof(LENGTH); - cudaMemcpyAsyncHelper(handle, buffer_p, outliers, - outlier_count * sizeof(QUANTIZED_INT), AUTO, 0); - buffer_p = buffer_p + outlier_count * sizeof(QUANTIZED_INT); - - // memcpy(buffer_p, &hufmeta_size, sizeof(size_t)); - cudaMemcpyAsyncHelper(handle, buffer_p, &hufmeta_size, sizeof(size_t), AUTO, - 0); - - buffer_p = buffer_p + sizeof(size_t); - cudaMemcpyAsyncHelper(handle, buffer_p, hufmeta, hufmeta_size, AUTO, 0); - buffer_p = buffer_p + hufmeta_size; - - cudaMemcpyAsyncHelper(handle, buffer_p, &hufdata_size, sizeof(size_t), AUTO, - 0); - buffer_p = buffer_p + sizeof(size_t); - - cudaMemcpyAsyncHelper(handle, buffer_p, hufdata, hufdata_size, AUTO, 0); - buffer_p = buffer_p + hufdata_size; - // printf("sync_all 5\n"); - handle.sync_all(); - if (handle.timing) { - t2 = high_resolution_clock::now(); - time_span = duration_cast>(t2 - t1); - std::cout << log::log_time - << "Compressed output seralization time: " << time_span.count() - << " s\n"; - } - - delete serizalied_meta; - cudaFreeHelper(outlier_count_d); - cudaFreeHelper(outlier_idx_d); - cudaFreeHelper(outliers); - cudaFreeHelper(hufmeta); - cudaFreeHelper(hufdata); - - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - return compressed_array; - } else { // cpu lossless - // printf("cpu lossless\n"); - if (handle.timing) - t1 = high_resolution_clock::now(); - unsigned char *cpu_lossless_data; // on GPU memory - size_t cpu_lossless_size; - cpu_lossless_compression(handle, dqv, quantized_count, cpu_lossless_data, - cpu_lossless_size); - cudaFreeHelper(dqv); - if (handle.timing) { - t2 = high_resolution_clock::now(); - time_span = duration_cast>(t2 - t1); - std::cout << log::log_time - << "CPU lossless compression time: " << time_span.count() - << " s\n"; - } - - if (handle.timing) { - end = high_resolution_clock::now(); - time_span = duration_cast>(end - start); - std::cout << log::log_time - << "Overall compression time: " << time_span.count() << " s (" - << (double)(handle.dofs[0][0] * handle.dofs[1][0] * - handle.linearized_depth * sizeof(T)) / - time_span.count() / 1e9 - << " GB/s)\n"; - } - - if (handle.timing) - t1 = high_resolution_clock::now(); - - SIZE metadata_size; - SERIALIZED_TYPE *serizalied_meta = m.Serialize(metadata_size); - delete[] m.shape; - - SIZE outsize = 0; - outsize += metadata_size; - outsize += sizeof(size_t) + cpu_lossless_size; - // printf("cpu_lossless_size: %llu\n", cpu_lossless_size); - std::vector out_shape(1); - out_shape[0] = outsize; - Array<1, unsigned char> compressed_array(out_shape); - - unsigned char *buffer = compressed_array.get_dv(); - // cudaMallocHostHelper((void**)&buffer, outsize); - // else cudaMallocHelper((void**)&buffer, outsize); - // unsigned char *buffer = (unsigned char *)malloc(outsize); - - void *buffer_p = (void *)buffer; - cudaMemcpyAsyncHelper(handle, buffer_p, serizalied_meta, metadata_size, - AUTO, 0); - buffer_p = buffer_p + metadata_size; - cudaMemcpyAsyncHelper(handle, buffer_p, &cpu_lossless_size, sizeof(size_t), - AUTO, 0); - buffer_p = buffer_p + sizeof(size_t); - cudaMemcpyAsyncHelper(handle, buffer_p, cpu_lossless_data, - cpu_lossless_size, AUTO, 0); - buffer_p = buffer_p + cpu_lossless_size; - - delete[] serizalied_meta; - cudaFreeHelper(cpu_lossless_data); - if (handle.timing) { - t2 = high_resolution_clock::now(); - time_span = duration_cast>(t2 - t1); - std::cout << log::log_time - << "Compressed data serialization time: " << time_span.count() - << " s\n"; - } - - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - //(double)(total-free)/1e9, (double)total/1e9); - return compressed_array; - } -} - -template -Array decompress(Handle &handle, - Array<1, unsigned char> &compressed_array) { - - cudaSetDeviceHelper(handle.dev_id); - high_resolution_clock::time_point t1, t2, start, end; - duration time_span; - - size_t free, total; - - QUANTIZED_INT *dqv; - LENGTH quantized_count = - handle.dofs[0][0] * handle.dofs[1][0] * handle.linearized_depth; - - LENGTH outlier_count; - LENGTH *outlier_idx_d; - QUANTIZED_INT *outliers; - - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - - void *data_p = compressed_array.get_dv(); //(void *)data; - - Metadata m; - SIZE metadata_size; - cudaMemcpyAsyncHelper(handle, &metadata_size, - data_p + m.metadata_size_offset(), sizeof(uint32_t), - AUTO, 0); - SERIALIZED_TYPE *serizalied_meta = - (SERIALIZED_TYPE *)std::malloc(metadata_size); - cudaMemcpyAsyncHelper(handle, serizalied_meta, data_p, metadata_size, AUTO, - 0); - data_p = data_p + metadata_size; - m.Deserialize(serizalied_meta, metadata_size); - - if (m.etype != CheckEndianess()) { - std::cout - << log::log_err - << "This data was compressed on a machine with different endianess!\n"; - exit(-1); - } - - if (strcmp(m.magic_word, MAGIC_WORD) != 0) { - std::cout << log::log_err - << "This data was not compressed with MGARD or corrupted!\n"; - exit(-1); - } - - if (m.ptype != processor_type::GPU_CUDA) { - std::cout << log::log_err - << "This data was not compressed with GPU, please use CPU to " - "decompress!\n"; - exit(-1); - } - - // printf("m.cpu_lossless: %d\n", m.cpu_lossless); - if (m.ltype == lossless_type::GPU_Huffman || - m.ltype == lossless_type::GPU_Huffman_LZ4) { - // printf("gpu lossless\n"); - if (handle.timing) - t1 = high_resolution_clock::now(); - void *lz4_hufmeta; - size_t lz4_hufmeta_size; - void *lz4_hufdata; - size_t lz4_hufdata_size; - - uint8_t *hufmeta; - uint64_t *hufdata; - size_t hufmeta_size; - size_t hufdata_size; - size_t outsize; - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - - cudaMemcpyAsyncHelper(handle, &outlier_count, data_p, sizeof(LENGTH), AUTO, - 0); - data_p = data_p + sizeof(LENGTH); - handle.sync(0); - cudaMallocHelper(handle, (void **)&outlier_idx_d, - outlier_count * sizeof(LENGTH)); - cudaMemcpyAsyncHelper(handle, outlier_idx_d, data_p, - outlier_count * sizeof(LENGTH), AUTO, 0); - // outlier_idx_d = (LENGTH *) data_p; - data_p = data_p + outlier_count * sizeof(LENGTH); - cudaMallocHelper(handle, (void **)&outliers, - outlier_count * sizeof(QUANTIZED_INT)); - cudaMemcpyAsyncHelper(handle, outliers, data_p, - outlier_count * sizeof(QUANTIZED_INT), AUTO, 0); - // outliers = (QUANTIZED_INT *) data_p; - data_p = data_p + outlier_count * sizeof(QUANTIZED_INT); - cudaMemcpyAsyncHelper(handle, &hufmeta_size, data_p, sizeof(size_t), AUTO, - 0); - data_p = data_p + sizeof(size_t); - handle.sync(0); - - cudaMallocHelper(handle, (void **)&hufmeta, hufmeta_size); - cudaMemcpyAsyncHelper(handle, hufmeta, data_p, hufmeta_size, AUTO, 0); - // hufmeta = (uint8_t *)data_p; - data_p = data_p + hufmeta_size; - cudaMemcpyAsyncHelper(handle, &hufdata_size, data_p, sizeof(size_t), AUTO, - 0); - data_p = data_p + sizeof(size_t); - handle.sync(0); - cudaMallocHelper(handle, (void **)&hufdata, hufdata_size); - cudaMemcpyAsyncHelper(handle, hufdata, data_p, hufdata_size, H2D, 0); - // hufdata = (uint64_t *)data_p; - data_p = data_p + hufdata_size; - handle.sync(0); - - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - - if (handle.timing) { - t2 = high_resolution_clock::now(); - time_span = duration_cast>(t2 - t1); - std::cout << log::log_time - << "Compressed data deserialization time: " << time_span.count() - << " s\n"; - } - - if (handle.timing) - start = high_resolution_clock::now(); - - if (m.ltype == lossless_type::GPU_Huffman_LZ4) { - if (handle.timing) - t1 = high_resolution_clock::now(); - uint64_t *lz4_decompressed_hufdata; - size_t lz4_decompressed_hufdata_size; - lz4_decompress(handle, (void *)hufdata, hufdata_size, - lz4_decompressed_hufdata, lz4_decompressed_hufdata_size, - 0); - // printf("sync_all 6\n"); - handle.sync_all(); - cudaFreeHelper(hufdata); - hufdata = lz4_decompressed_hufdata; - hufdata_size = lz4_decompressed_hufdata_size; - t2 = high_resolution_clock::now(); - time_span = duration_cast>(t2 - t1); - if (handle.timing) { - std::cout << log::log_time - << "NVComp::LZ4 decompression time: " << time_span.count() - << " s\n"; - } - } - - if (handle.timing) - t1 = high_resolution_clock::now(); - huffman_decompress(handle, (uint64_t *)hufmeta, - hufmeta_size, hufdata, - hufdata_size, dqv, outsize, 0); - handle.sync_all(); - cudaFreeHelper(hufmeta); - cudaFreeHelper(hufdata); - if (handle.timing) { - t2 = high_resolution_clock::now(); - time_span = duration_cast>(t2 - t1); - std::cout << log::log_time - << "GPU Huffman decoding time: " << time_span.count() << " s\n"; - } - - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - - } else { // cpu lossless - // printf("cpu lossless\n"); - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - //(double)(total-free)/1e9, (double)total/1e9); - if (handle.timing) - start = high_resolution_clock::now(); - if (handle.timing) - t1 = high_resolution_clock::now(); - unsigned char *cpu_lossless_data; // on GPU memory - size_t cpu_lossless_size; - cudaMemcpyAsyncHelper(handle, &cpu_lossless_size, data_p, sizeof(size_t), - AUTO, 0); - data_p = data_p + sizeof(size_t); - handle.sync(0); - // cudaMallocHelper(handle, (void **)&cpu_lossless_data, - // cpu_lossless_size * sizeof(unsigned char)); - // cudaMemcpyAsyncHelper(handle, cpu_lossless_data, data_p, - // cpu_lossless_size * sizeof(unsigned char), AUTO, 0); - cpu_lossless_data = (unsigned char *)data_p; - cpu_lossless_decompression(handle, cpu_lossless_data, cpu_lossless_size, - dqv, quantized_count); - // cudaFreeHelper(cpu_lossless_data); - if (handle.timing) { - t2 = high_resolution_clock::now(); - time_span = duration_cast>(t2 - t1); - std::cout << log::log_time - << "CPU lossless decompression time: " << time_span.count() - << " s\n"; - } - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - //(double)(total-free)/1e9, (double)total/1e9); - } - - if (handle.timing) - t1 = high_resolution_clock::now(); - thrust::device_vector ldqvs(handle.D_padded); - ldqvs[0] = handle.dofs[0][0]; - for (int i = 1; i < handle.D_padded; i++) { - ldqvs[i] = handle.dofs[i][0]; - } - - std::vector decompressed_shape(D); - for (int i = 0; i < D; i++) - decompressed_shape[i] = handle.shapes_h[0][i]; - std::reverse(decompressed_shape.begin(), decompressed_shape.end()); - Array decompressed_data(decompressed_shape); - - // printf("sync_all 7.5\n"); - handle.sync_all(); - - // printf("dqv\n"); - // print_matrix_cuda(1, quantized_count, dqv, quantized_count); - - bool prep_huffman = m.ltype == lossless_type::GPU_Huffman || - m.ltype == lossless_type::GPU_Huffman_LZ4; - levelwise_linear_dequantize( - handle, handle.ranges_d, handle.l_target, handle.volumes, - handle.ldvolumes, m, dqv, thrust::raw_pointer_cast(ldqvs.data()), - decompressed_data.get_dv(), decompressed_data.get_ldvs_d(), prep_huffman, - outlier_count, outlier_idx_d, outliers, 0); - handle.sync_all(); - cudaFreeHelper(dqv); - if (prep_huffman) { - cudaFreeHelper(outlier_idx_d); - cudaFreeHelper(outliers); - } - if (handle.timing) { - t2 = high_resolution_clock::now(); - time_span = duration_cast>(t2 - t1); - std::cout << log::log_time << "Dequantization time: " << time_span.count() - << " s\n"; - } - - // cudaMemGetInfo(&free, &total); - // printf("Mem: %f/%f\n", (double)(total-free)/1e9, (double)total/1e9); - - // printf("dv:\n"); - // print_matrix_cuda(1, quantized_count, dv, quantized_count); - - /**** refactoring ****/ - - // cudaMemcpy3DAsyncHelper( handle, - // dv, lddv1 * sizeof(T), handle.dofs[0][0] * sizeof(T), handle.dofs[1][0], - // data, handle.dofs[0][0] * sizeof(T), handle.dofs[0][0] * sizeof(T), - // handle.dofs[1][0], handle.dofs[0][0] * sizeof(T), handle.dofs[1][0], - // handle.linearized_depth, H2D, 0); - - handle.allocate_workspace(); - - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - - if (handle.timing) - t1 = high_resolution_clock::now(); - recompose(handle, decompressed_data.get_dv(), - decompressed_data.get_ldvs_h(), - decompressed_data.get_ldvs_d(), m.l_target, 0); - - { - // int block_size = BLOCK_SIZE; - // for (int i = 0; i < handle.dofs[0][0]; i += block_size) { - // for (int j = 0; j < handle.dofs[1][0]; j += block_size) { - // for (int k = 0; k < handle.dofs[2][0]; k += block_size) { - // size_t b0 = std::min(block_size, handle.dofs[0][0] - i); - // size_t b1 = std::min(block_size, handle.dofs[1][0] - j); - // size_t b2 = std::min(block_size, handle.dofs[2][0] - k); - // std::vector block_shape = {b2, b1, b0}; - // // mgard_cuda::Array<3, T> block_array(block_shape); - // mgard_cuda::Handle<3, T> block_handle(block_shape); - // std::vector idx = {(int)i, (int)j, (int)k}; - // // printf("recompose: %llu, %llu, %llu\n", i, j, k); - // // printf("block_array: %llu, %llu, %llu ld %d %d %d\n", b0, b1, - // b2, block_array.get_ldvs_h()[0], - // // block_array.get_ldvs_h()[1], - // block_array.get_ldvs_h()[2]); - - // // - // block_array.loadData(in_array.get_dv()+get_idx(in_array.get_ldvs_h(), - // idx), in_array.get_ldvs_h()[0]); block_handle.allocate_workspace(); - // recompose<3, T>(block_handle, - // decompressed_data.get_dv()+get_idx(decompressed_data.get_ldvs_h(), - // idx), decompressed_data.get_ldvs_h(), - // block_handle.l_target); - // block_handle.free_workspace(); - // } - // } - // } - } - - // printf("sync_all 9\n"); - handle.sync_all(); - if (handle.timing) { - t2 = high_resolution_clock::now(); - time_span = duration_cast>(t2 - t1); - std::cout << log::log_time << "Recomposition time: " << time_span.count() - << " s\n"; - } - - handle.free_workspace(); - - // printf("sync_all 10\n"); - handle.sync_all(); - if (handle.timing) { - end = high_resolution_clock::now(); - time_span = duration_cast>(end - start); - std::cout << log::log_time - << "Overall decompression time: " << time_span.count() << " s (" - << (double)(handle.dofs[0][0] * handle.dofs[1][0] * - handle.linearized_depth * sizeof(T)) / - time_span.count() / 1e9 - << " GB/s)\n"; - } - - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - - // T *v; - // cudaMallocHostHelper((void **)&v, handle.dofs[0][0] * handle.dofs[1][0] * - // handle.linearized_depth * sizeof(T)); - // // = (T *)malloc(handle.dofs[0][0] * handle.dofs[1][0] * - // handle.linearized_depth * sizeof(T)); - - // cudaMemcpy3DAsyncHelper( - // handle, v, handle.dofs[0][0] * sizeof(T), handle.dofs[0][0] * - // sizeof(T), handle.dofs[1][0], dv, lddv1 * sizeof(T), handle.dofs[0][0] - // * sizeof(T), handle.dofs[1][0], handle.dofs[0][0] * sizeof(T), - // handle.dofs[1][0], handle.linearized_depth, D2H, 0); - - // cudaFreeHelper(dv); - - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - return decompressed_data; -} - -#define KERNELS(D, T) \ - template Array<1, unsigned char> compress( \ - Handle & handle, Array & in_array, \ - enum error_bound_type type, T tol, T s); \ - template Array decompress( \ - Handle & handle, Array<1, unsigned char> & compressed_array); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) -#undef KERNELS - -} // namespace mgard_cuda diff --git a/src/cuda/DataRefactoring.cu b/src/cuda/DataRefactoring.cu deleted file mode 100644 index eb675dcb8f..0000000000 --- a/src/cuda/DataRefactoring.cu +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/DataRefactoring.hpp" - -#include - -#include -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void calc_coeff_pointers( \ - Handle & handle, DIM curr_dims[3], DIM l, SubArray doutput, \ - SubArray & dcoarse, SubArray & dcoeff_f, \ - SubArray & dcoeff_c, SubArray & dcoeff_r, \ - SubArray & dcoeff_cf, SubArray & dcoeff_rf, \ - SubArray & dcoeff_rc, SubArray & dcoeff_rcf); \ - template void decompose(Handle & handle, T * dv, \ - std::vector ldvs_h, SIZE * ldvs_d, \ - SIZE l_target, int queue_idx); \ - template void recompose(Handle & handle, T * dv, \ - std::vector ldvs_h, SIZE * ldvs_d, \ - SIZE l_target, int queue_idx); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) -#undef KERNELS - -#define KERNELS(D, T) \ - template void calc_coefficients_3d( \ - Handle & handle, SubArray dinput, SubArray & doutput, \ - SIZE l, int queue_idx); \ - template void coefficients_restore_3d( \ - Handle & handle, SubArray dinput, SubArray & doutput, \ - SIZE l, int queue_idx); \ - template void calc_correction_3d( \ - Handle & handle, SubArray dcoeff, \ - SubArray & dcorrection, SIZE l, int queue_idx); -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -#undef KERNELS - -#define KERNELS(D, T) \ - template void calc_coefficients_nd( \ - Handle & handle, SubArray dinput1, SubArray dinput2, \ - SubArray & doutput, SIZE l, int queue_idx); \ - template void coefficients_restore_nd( \ - Handle & handle, SubArray dinput1, SubArray dinput2, \ - SubArray & doutput, SIZE l, int queue_idx); \ - template void calc_correction_nd( \ - Handle & handle, SubArray dcoeff, \ - SubArray & dcorrection, SIZE l, int queue_idx); - -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) -#undef KERNELS - -} // namespace mgard_cuda diff --git a/src/cuda/ErrorCalculator.cu b/src/cuda/ErrorCalculator.cu deleted file mode 100644 index 02bc48b34d..0000000000 --- a/src/cuda/ErrorCalculator.cu +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include -#include - -namespace mgard_cuda { - -template T L_inf_norm(size_t n, T *data) { - T L_inf = 0; - for (int i = 0; i < n; ++i) { - T temp = fabs(data[i]); - if (temp > L_inf) - L_inf = temp; - } - return L_inf; -} - -template T L_2_norm(size_t n, T *data) { - T L_2 = 0; - for (int i = 0; i < n; ++i) { - T temp = fabs(data[i]); - L_2 += temp * temp; - } - return std::sqrt(L_2); -} - -template -T L_inf_error(size_t n, T *original_data, T *decompressed_data, - enum error_bound_type mode) { - T error_L_inf_norm = 0; - for (int i = 0; i < n; ++i) { - T temp = fabs(original_data[i] - decompressed_data[i]); - if (temp > error_L_inf_norm) - error_L_inf_norm = temp; - } - if (mode == error_bound_type::ABS) { - return error_L_inf_norm; - } else if (mode == error_bound_type::REL) { - return error_L_inf_norm / L_inf_norm(n, original_data); - } else { - return 0; - } -} - -template -T L_2_error(size_t n, T *original_data, T *decompressed_data, - enum error_bound_type mode) { - T error_L_2_norm = 0; - for (int i = 0; i < n; ++i) { - T temp = fabs(original_data[i] - decompressed_data[i]); - error_L_2_norm += temp * temp; - } - if (mode == error_bound_type::ABS) { - return std::sqrt(error_L_2_norm); - } else if (mode == error_bound_type::REL) { - return std::sqrt(error_L_2_norm) / L_2_norm(n, original_data); - } else { - return 0; - } -} - -template T MSE(size_t n, T *original_data, T *decompressed_data) { - T mse = 0; - for (int i = 0; i < n; ++i) { - T temp = fabs(original_data[i] - decompressed_data[i]); - mse += temp * temp; - } - return mse / n; -} - -template T PSNR(size_t n, T *original_data, T *decompressed_data) { - T mse = MSE(n, original_data, decompressed_data); - T max = 0, min = std::numeric_limits::max(); - for (int i = 0; i < n; ++i) { - if (max < original_data[i]) - max = original_data[i]; - if (min > original_data[i]) - min = original_data[i]; - } - T range = max - min; - return 20 * std::log10(range / std::sqrt(mse)); -} - -// double max = 0, min = std::numeric_limits::max(), range = 0; -// double error_sum = 0, mse = 0, psnr = 0; -// for (int i = 0; i < num_double; ++i) { -// if (max < in_buff[i]) max = in_buff[i]; -// if (min > in_buff[i]) min = in_buff[i]; -// double err = fabs(in_buff[i] - mgard_out_buff[i]); -// error_sum += err * err; -// } -// range = max - min; -// mse = error_sum / num_double; -// psnr = 20*log::log10(range)-10*log::log10(mse); - -template float L_inf_norm(size_t n, float *data); -template double L_inf_norm(size_t n, double *data); -template float L_2_norm(size_t n, float *data); -template double L_2_norm(size_t n, double *data); - -template float L_inf_error(size_t n, float *original_data, - float *decompressed_data, - enum error_bound_type mode); -template double L_inf_error(size_t n, double *original_data, - double *decompressed_data, - enum error_bound_type mode); -template float L_2_error(size_t n, float *original_data, - float *decompressed_data, - enum error_bound_type mode); -template double L_2_error(size_t n, double *original_data, - double *decompressed_data, - enum error_bound_type mode); -template float MSE(size_t n, float *original_data, - float *decompressed_data); -template double MSE(size_t n, double *original_data, - double *decompressed_data); -template float PSNR(size_t n, float *original_data, - float *decompressed_data); -template double PSNR(size_t n, double *original_data, - double *decompressed_data); -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/GridProcessingKernels/CalcCoefficients.cu b/src/cuda/GridProcessingKernels/CalcCoefficients.cu deleted file mode 100644 index 30ce12fde0..0000000000 --- a/src/cuda/GridProcessingKernels/CalcCoefficients.cu +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ -#include "cuda/CommonInternal.h" - -#include "cuda/GridProcessingKernel.h" -#include "cuda/GridProcessingKernel.hpp" - -namespace mgard_cuda { - -#define KERNELS(D_GLOBAL, D_LOCAL, T, INTERPOLATION, CALC_COEFF, TYPE) \ - template void \ - gpk_reo( \ - Handle & handle, SIZE * shape_h, SIZE * shape_d, \ - SIZE * shape_c_d, SIZE * ldvs, SIZE * ldws, DIM unprocessed_n, \ - DIM * unprocessed_dims, DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, \ - T * dratio_r, T * dratio_c, T * dratio_f, T * dv, LENGTH lddv1, \ - LENGTH lddv2, T * dw, LENGTH lddw1, LENGTH lddw2, T * dwf, \ - LENGTH lddwf1, LENGTH lddwf2, T * dwc, LENGTH lddwc1, LENGTH lddwc2, \ - T * dwr, LENGTH lddwr1, LENGTH lddwr2, T * dwcf, LENGTH lddwcf1, \ - LENGTH lddwcf2, T * dwrf, LENGTH lddwrf1, LENGTH lddwrf2, T * dwrc, \ - LENGTH lddwrc1, LENGTH lddwrc2, T * dwrcf, LENGTH lddwrcf1, \ - LENGTH lddwrcf2, int queue_idx, int config); - -KERNELS(1, 1, double, false, true, 1) -KERNELS(1, 1, float, false, true, 1) -KERNELS(2, 2, double, false, true, 1) -KERNELS(2, 2, float, false, true, 1) -KERNELS(3, 3, double, false, true, 1) -KERNELS(3, 3, float, false, true, 1) - -KERNELS(4, 2, double, false, true, 2) -KERNELS(4, 2, float, false, true, 2) -KERNELS(5, 2, double, false, true, 2) -KERNELS(5, 2, float, false, true, 2) - -KERNELS(4, 3, double, false, true, 2) -KERNELS(4, 3, float, false, true, 2) -KERNELS(5, 3, double, false, true, 2) -KERNELS(5, 3, float, false, true, 2) -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/GridProcessingKernels/CoefficientRestore.cu b/src/cuda/GridProcessingKernels/CoefficientRestore.cu deleted file mode 100644 index 63b40b30bb..0000000000 --- a/src/cuda/GridProcessingKernels/CoefficientRestore.cu +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ -#include "cuda/CommonInternal.h" - -#include "cuda/GridProcessingKernel.h" -#include "cuda/GridProcessingKernel.hpp" - -namespace mgard_cuda { - -#define KERNELS(D_GLOBAL, D_LOCAL, T, INTERPOLATION, COEFF_RESTORE, TYPE) \ - template void \ - gpk_rev( \ - Handle & handle, SIZE * shape_h, SIZE * shape_d, \ - SIZE * shape_c_d, SIZE * ldvs, SIZE * ldws, DIM unprocessed_n, \ - DIM * unprocessed_dims, DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, \ - T * dratio_r, T * dratio_c, T * dratio_f, T * dv, LENGTH lddv1, \ - LENGTH lddv2, T * dw, LENGTH lddw1, LENGTH lddw2, T * dwf, \ - LENGTH lddwf1, LENGTH lddwf2, T * dwc, LENGTH lddwc1, LENGTH lddwc2, \ - T * dwr, LENGTH lddwr1, LENGTH lddwr2, T * dwcf, LENGTH lddwcf1, \ - LENGTH lddwcf2, T * dwrf, LENGTH lddwrf1, LENGTH lddwrf2, T * dwrc, \ - LENGTH lddwrc1, LENGTH lddwrc2, T * dwrcf, LENGTH lddwrcf1, \ - LENGTH lddwrcf2, SIZE svr, SIZE svc, SIZE svf, SIZE nvr, SIZE nvc, \ - SIZE nvf, int queue_idx, int config); - -KERNELS(1, 1, double, false, true, 1) -KERNELS(1, 1, float, false, true, 1) -KERNELS(2, 2, double, false, true, 1) -KERNELS(2, 2, float, false, true, 1) -KERNELS(3, 3, double, false, true, 1) -KERNELS(3, 3, float, false, true, 1) - -KERNELS(4, 2, double, false, true, 2) -KERNELS(4, 2, float, false, true, 2) -KERNELS(5, 2, double, false, true, 2) -KERNELS(5, 2, float, false, true, 2) - -KERNELS(4, 3, double, false, true, 2) -KERNELS(4, 3, float, false, true, 2) -KERNELS(5, 3, double, false, true, 2) -KERNELS(5, 3, float, false, true, 2) - -// #undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/GridProcessingKernels/InterpCalcCoeff.cu b/src/cuda/GridProcessingKernels/InterpCalcCoeff.cu deleted file mode 100644 index 7e95278222..0000000000 --- a/src/cuda/GridProcessingKernels/InterpCalcCoeff.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ -#include "cuda/CommonInternal.h" - -#include "cuda/GridProcessingKernel.h" -#include "cuda/GridProcessingKernel.hpp" - -namespace mgard_cuda { - -#define KERNELS(D_GLOBAL, D_LOCAL, T, INTERPOLATION, CALC_COEFF, TYPE) \ - template void \ - gpk_reo( \ - Handle & handle, SIZE * shape_h, SIZE * shape_d, \ - SIZE * shape_c_d, SIZE * ldvs, SIZE * ldws, DIM unprocessed_n, \ - DIM * unprocessed_dims, DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, \ - T * dratio_r, T * dratio_c, T * dratio_f, T * dv, LENGTH lddv1, \ - LENGTH lddv2, T * dw, LENGTH lddw1, LENGTH lddw2, T * dwf, \ - LENGTH lddwf1, LENGTH lddwf2, T * dwc, LENGTH lddwc1, LENGTH lddwc2, \ - T * dwr, LENGTH lddwr1, LENGTH lddwr2, T * dwcf, LENGTH lddwcf1, \ - LENGTH lddwcf2, T * dwrf, LENGTH lddwrf1, LENGTH lddwrf2, T * dwrc, \ - LENGTH lddwrc1, LENGTH lddwrc2, T * dwrcf, LENGTH lddwrcf1, \ - LENGTH lddwrcf2, int queue_idx, int config); - -KERNELS(1, 1, double, true, true, 1) -KERNELS(1, 1, float, true, true, 1) -KERNELS(2, 2, double, true, true, 1) -KERNELS(2, 2, float, true, true, 1) -KERNELS(3, 3, double, true, true, 1) -KERNELS(3, 3, float, true, true, 1) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/GridProcessingKernels/InterpCalcCoeff_3D.cu b/src/cuda/GridProcessingKernels/InterpCalcCoeff_3D.cu deleted file mode 100644 index e5e833ed6a..0000000000 --- a/src/cuda/GridProcessingKernels/InterpCalcCoeff_3D.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ -#include "cuda/CommonInternal.h" - -#include "cuda/GridProcessingKernel3D.h" -#include "cuda/GridProcessingKernel3D.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void gpk_reo_3d( \ - Handle & handle, SIZE nr, SIZE nc, SIZE nf, T * dratio_r, \ - T * dratio_c, T * dratio_f, T * dv, SIZE lddv1, SIZE lddv2, T * dw, \ - SIZE lddw1, SIZE lddw2, T * dwf, SIZE lddwf1, SIZE lddwf2, T * dwc, \ - SIZE lddwc1, SIZE lddwc2, T * dwr, SIZE lddwr1, SIZE lddwr2, T * dwcf, \ - SIZE lddwcf1, SIZE lddwcf2, T * dwrf, SIZE lddwrf1, SIZE lddwrf2, \ - T * dwrc, SIZE lddwrc1, SIZE lddwrc2, T * dwrcf, SIZE lddwrcf1, \ - SIZE lddwrcf2, int queue_idx, int config); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/GridProcessingKernels/InterpCoeffRestore.cu b/src/cuda/GridProcessingKernels/InterpCoeffRestore.cu deleted file mode 100644 index 9b34f5eacc..0000000000 --- a/src/cuda/GridProcessingKernels/InterpCoeffRestore.cu +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ -#include "cuda/CommonInternal.h" - -#include "cuda/GridProcessingKernel.h" -#include "cuda/GridProcessingKernel.hpp" - -namespace mgard_cuda { - -#define KERNELS(D_GLOBAL, D_LOCAL, T, INTERPOLATION, COEFF_RESTORE, TYPE) \ - template void \ - gpk_rev( \ - Handle & handle, SIZE * shape_h, SIZE * shape_d, \ - SIZE * shape_c_d, SIZE * ldvs, SIZE * ldws, DIM unprocessed_n, \ - DIM * unprocessed_dims, DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, \ - T * dratio_r, T * dratio_c, T * dratio_f, T * dv, LENGTH lddv1, \ - LENGTH lddv2, T * dw, LENGTH lddw1, LENGTH lddw2, T * dwf, \ - LENGTH lddwf1, LENGTH lddwf2, T * dwc, LENGTH lddwc1, LENGTH lddwc2, \ - T * dwr, LENGTH lddwr1, LENGTH lddwr2, T * dwcf, LENGTH lddwcf1, \ - LENGTH lddwcf2, T * dwrf, LENGTH lddwrf1, LENGTH lddwrf2, T * dwrc, \ - LENGTH lddwrc1, LENGTH lddwrc2, T * dwrcf, LENGTH lddwrcf1, \ - LENGTH lddwrcf2, SIZE svr, SIZE svc, SIZE svf, SIZE nvr, SIZE nvc, \ - SIZE nvf, int queue_idx, int config); - -KERNELS(1, 1, double, true, true, 1) -KERNELS(1, 1, float, true, true, 1) -KERNELS(2, 2, double, true, true, 1) -KERNELS(2, 2, float, true, true, 1) -KERNELS(3, 3, double, true, true, 1) -KERNELS(3, 3, float, true, true, 1) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/GridProcessingKernels/InterpCoeffRestore_3D.cu b/src/cuda/GridProcessingKernels/InterpCoeffRestore_3D.cu deleted file mode 100644 index d20af7395f..0000000000 --- a/src/cuda/GridProcessingKernels/InterpCoeffRestore_3D.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/GridProcessingKernel3D.h" -#include "cuda/GridProcessingKernel3D.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void gpk_rev_3d( \ - Handle & handle, SIZE nr, SIZE nc, SIZE nf, T * dratio_r, \ - T * dratio_c, T * dratio_f, T * dv, SIZE lddv1, SIZE lddv2, T * dw, \ - SIZE lddw1, SIZE lddw2, T * dwf, SIZE lddwf1, SIZE lddwf2, T * dwc, \ - SIZE lddwc1, SIZE lddwc2, T * dwr, SIZE lddwr1, SIZE lddwr2, T * dwcf, \ - SIZE lddwcf1, SIZE lddwcf2, T * dwrf, SIZE lddwrf1, SIZE lddwrf2, \ - T * dwrc, SIZE lddwrc1, SIZE lddwrc2, T * dwrcf, SIZE lddwrcf1, \ - SIZE lddwrcf2, SIZE svr, SIZE svc, SIZE svf, SIZE nvr, SIZE nvc, \ - SIZE nvf, int queue_idx, int config); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/GridProcessingKernels/Interpolation.cu b/src/cuda/GridProcessingKernels/Interpolation.cu deleted file mode 100644 index e33801a759..0000000000 --- a/src/cuda/GridProcessingKernels/Interpolation.cu +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/GridProcessingKernel.h" -#include "cuda/GridProcessingKernel.hpp" - -namespace mgard_cuda { - -#define KERNELS(D_GLOBAL, D_LOCAL, T, INTERPOLATION, CALC_COEFF, TYPE) \ - template void \ - gpk_reo( \ - Handle & handle, SIZE * shape_h, SIZE * shape_d, \ - SIZE * shape_c_d, SIZE * ldvs, SIZE * ldws, DIM unprocessed_n, \ - DIM * unprocessed_dims, DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, \ - T * dratio_r, T * dratio_c, T * dratio_f, T * dv, LENGTH lddv1, \ - LENGTH lddv2, T * dw, LENGTH lddw1, LENGTH lddw2, T * dwf, \ - LENGTH lddwf1, LENGTH lddwf2, T * dwc, LENGTH lddwc1, LENGTH lddwc2, \ - T * dwr, LENGTH lddwr1, LENGTH lddwr2, T * dwcf, LENGTH lddwcf1, \ - LENGTH lddwcf2, T * dwrf, LENGTH lddwrf1, LENGTH lddwrf2, T * dwrc, \ - LENGTH lddwrc1, LENGTH lddwrc2, T * dwrcf, LENGTH lddwrcf1, \ - LENGTH lddwrcf2, int queue_idx, int config); - -KERNELS(1, 1, double, true, false, 1) -KERNELS(1, 1, float, true, false, 1) -KERNELS(2, 2, double, true, false, 1) -KERNELS(2, 2, float, true, false, 1) -KERNELS(3, 3, double, true, false, 1) -KERNELS(3, 3, float, true, false, 1) - -KERNELS(4, 3, double, true, false, 1) -KERNELS(4, 3, float, true, false, 1) -KERNELS(5, 3, double, true, false, 1) -KERNELS(5, 3, float, true, false, 1) - -KERNELS(4, 2, double, true, false, 2) -KERNELS(4, 2, float, true, false, 2) -KERNELS(5, 2, double, true, false, 2) -KERNELS(5, 2, float, true, false, 2) - -KERNELS(4, 3, double, true, false, 2) -KERNELS(4, 3, float, true, false, 2) -KERNELS(5, 3, double, true, false, 2) -KERNELS(5, 3, float, true, false, 2) -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/GridProcessingKernels/InterpolationRev.cu b/src/cuda/GridProcessingKernels/InterpolationRev.cu deleted file mode 100644 index 5a2aabcaca..0000000000 --- a/src/cuda/GridProcessingKernels/InterpolationRev.cu +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/GridProcessingKernel.h" -#include "cuda/GridProcessingKernel.hpp" - -namespace mgard_cuda { - -#define KERNELS(D_GLOBAL, D_LOCAL, T, INTERPOLATION, COEFF_RESTORE, TYPE) \ - template void \ - gpk_rev( \ - Handle & handle, SIZE * shape_h, SIZE * shape_d, \ - SIZE * shape_c_d, SIZE * ldvs, SIZE * ldws, DIM unprocessed_n, \ - DIM * unprocessed_dims, DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, \ - T * dratio_r, T * dratio_c, T * dratio_f, T * dv, LENGTH lddv1, \ - LENGTH lddv2, T * dw, LENGTH lddw1, LENGTH lddw2, T * dwf, \ - LENGTH lddwf1, LENGTH lddwf2, T * dwc, LENGTH lddwc1, LENGTH lddwc2, \ - T * dwr, LENGTH lddwr1, LENGTH lddwr2, T * dwcf, LENGTH lddwcf1, \ - LENGTH lddwcf2, T * dwrf, LENGTH lddwrf1, LENGTH lddwrf2, T * dwrc, \ - LENGTH lddwrc1, LENGTH lddwrc2, T * dwrcf, LENGTH lddwrcf1, \ - LENGTH lddwrcf2, SIZE svr, SIZE svc, SIZE svf, SIZE nvr, SIZE nvc, \ - SIZE nvf, int queue_idx, int config); - -KERNELS(1, 1, double, true, false, 1) -KERNELS(1, 1, float, true, false, 1) -KERNELS(2, 2, double, true, false, 1) -KERNELS(2, 2, float, true, false, 1) -KERNELS(3, 3, double, true, false, 1) -KERNELS(3, 3, float, true, false, 1) - -KERNELS(4, 3, double, true, false, 1) -KERNELS(4, 3, float, true, false, 1) -KERNELS(5, 3, double, true, false, 1) -KERNELS(5, 3, float, true, false, 1) - -KERNELS(4, 2, double, true, false, 2) -KERNELS(4, 2, float, true, false, 2) -KERNELS(5, 2, double, true, false, 2) -KERNELS(5, 2, float, true, false, 2) - -KERNELS(4, 3, double, true, false, 2) -KERNELS(4, 3, float, true, false, 2) -KERNELS(5, 3, double, true, false, 2) -KERNELS(5, 3, float, true, false, 2) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/GridProcessingKernels/Reorder.cu b/src/cuda/GridProcessingKernels/Reorder.cu deleted file mode 100644 index 08113d3282..0000000000 --- a/src/cuda/GridProcessingKernels/Reorder.cu +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/GridProcessingKernel.h" -#include "cuda/GridProcessingKernel.hpp" - -namespace mgard_cuda { - -#define KERNELS(D_GLOBAL, D_LOCAL, T, INTERPOLATION, CALC_COEFF, TYPE) \ - template void \ - gpk_reo( \ - Handle & handle, SIZE * shape_h, SIZE * shape_d, \ - SIZE * shape_c_d, SIZE * ldvs, SIZE * ldws, DIM unprocessed_n, \ - DIM * unprocessed_dims, DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, \ - T * dratio_r, T * dratio_c, T * dratio_f, T * dv, LENGTH lddv1, \ - LENGTH lddv2, T * dw, LENGTH lddw1, LENGTH lddw2, T * dwf, \ - LENGTH lddwf1, LENGTH lddwf2, T * dwc, LENGTH lddwc1, LENGTH lddwc2, \ - T * dwr, LENGTH lddwr1, LENGTH lddwr2, T * dwcf, LENGTH lddwcf1, \ - LENGTH lddwcf2, T * dwrf, LENGTH lddwrf1, LENGTH lddwrf2, T * dwrc, \ - LENGTH lddwrc1, LENGTH lddwrc2, T * dwrcf, LENGTH lddwrcf1, \ - LENGTH lddwrcf2, int queue_idx, int config); - -KERNELS(1, 1, double, false, false, 1) -KERNELS(1, 1, float, false, false, 1) -KERNELS(2, 2, double, false, false, 1) -KERNELS(2, 2, float, false, false, 1) -KERNELS(3, 3, double, false, false, 1) -KERNELS(3, 3, float, false, false, 1) - -KERNELS(4, 3, double, false, false, 1) -KERNELS(4, 3, float, false, false, 1) -KERNELS(5, 3, double, false, false, 1) -KERNELS(5, 3, float, false, false, 1) - -KERNELS(4, 2, double, false, false, 2) -KERNELS(4, 2, float, false, false, 2) -KERNELS(5, 2, double, false, false, 2) -KERNELS(5, 2, float, false, false, 2) - -KERNELS(4, 3, double, false, false, 2) -KERNELS(4, 3, float, false, false, 2) -KERNELS(5, 3, double, false, false, 2) -KERNELS(5, 3, float, false, false, 2) - -// // for debug -// KERNELS(1, 1, QUANTIZED_INT, false, false, 1) -// KERNELS(2, 2, QUANTIZED_INT, false, false, 1) -// KERNELS(3, 3, QUANTIZED_INT, false, false, 1) - -// KERNELS(4, 3, QUANTIZED_INT, false, false, 1) -// KERNELS(5, 3, QUANTIZED_INT, false, false, 1) - -// KERNELS(4, 2, QUANTIZED_INT, false, false, 2) -// KERNELS(5, 2, QUANTIZED_INT, false, false, 2) - -// KERNELS(4, 3, QUANTIZED_INT, false, false, 2) -// KERNELS(5, 3, QUANTIZED_INT, false, false, 2) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/GridProcessingKernels/ReorderRestore.cu b/src/cuda/GridProcessingKernels/ReorderRestore.cu deleted file mode 100644 index e72988596a..0000000000 --- a/src/cuda/GridProcessingKernels/ReorderRestore.cu +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/GridProcessingKernel.h" -#include "cuda/GridProcessingKernel.hpp" - -namespace mgard_cuda { - -#define KERNELS(D_GLOBAL, D_LOCAL, T, INTERPOLATION, COEFF_RESTORE, TYPE) \ - template void \ - gpk_rev( \ - Handle & handle, SIZE * shape_h, SIZE * shape_d, \ - SIZE * shape_c_d, SIZE * ldvs, SIZE * ldws, DIM unprocessed_n, \ - DIM * unprocessed_dims, DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, \ - T * dratio_r, T * dratio_c, T * dratio_f, T * dv, LENGTH lddv1, \ - LENGTH lddv2, T * dw, LENGTH lddw1, LENGTH lddw2, T * dwf, \ - LENGTH lddwf1, LENGTH lddwf2, T * dwc, LENGTH lddwc1, LENGTH lddwc2, \ - T * dwr, LENGTH lddwr1, LENGTH lddwr2, T * dwcf, LENGTH lddwcf1, \ - LENGTH lddwcf2, T * dwrf, LENGTH lddwrf1, LENGTH lddwrf2, T * dwrc, \ - LENGTH lddwrc1, LENGTH lddwrc2, T * dwrcf, LENGTH lddwrcf1, \ - LENGTH lddwrcf2, SIZE svr, SIZE svc, SIZE svf, SIZE nvr, SIZE nvc, \ - SIZE nvf, int queue_idx, int config); - -KERNELS(1, 1, double, false, false, 1) -KERNELS(1, 1, float, false, false, 1) -KERNELS(2, 2, double, false, false, 1) -KERNELS(2, 2, float, false, false, 1) -KERNELS(3, 3, double, false, false, 1) -KERNELS(3, 3, float, false, false, 1) - -KERNELS(4, 3, double, false, false, 1) -KERNELS(4, 3, float, false, false, 1) -KERNELS(5, 3, double, false, false, 1) -KERNELS(5, 3, float, false, false, 1) - -KERNELS(4, 2, double, false, false, 2) -KERNELS(4, 2, float, false, false, 2) -KERNELS(5, 2, double, false, false, 2) -KERNELS(5, 2, float, false, false, 2) - -KERNELS(4, 3, double, false, false, 2) -KERNELS(4, 3, float, false, false, 2) -KERNELS(5, 3, double, false, false, 2) -KERNELS(5, 3, float, false, false, 2) - -// // for debug -// KERNELS(1, 1, QUANTIZED_INT, false, false, 1) -// KERNELS(2, 2, QUANTIZED_INT, false, false, 1) -// KERNELS(3, 3, QUANTIZED_INT, false, false, 1) - -// KERNELS(4, 3, QUANTIZED_INT, false, false, 1) -// KERNELS(5, 3, QUANTIZED_INT, false, false, 1) - -// KERNELS(4, 2, QUANTIZED_INT, false, false, 2) -// KERNELS(5, 2, QUANTIZED_INT, false, false, 2) - -// KERNELS(4, 3, QUANTIZED_INT, false, false, 2) -// KERNELS(5, 3, QUANTIZED_INT, false, false, 2) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/Handle.cu b/src/cuda/Handle.cu deleted file mode 100644 index 6046e874d4..0000000000 --- a/src/cuda/Handle.cu +++ /dev/null @@ -1,1146 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: September 27, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/MemoryManagement.h" -#include "cuda/PrecomputeKernels.h" - -#include "cuda/Handle.h" - -#include -#include -#include -#include - -namespace mgard_cuda { - -template -void Handle::coord_to_dist(SIZE dof, T *coord, T *dist) { - if (dof <= 1) - return; - // printf("coord_to_dist\n"); - T *h_coord = new T[dof]; - T *h_dist = new T[dof]; - for (int i = 0; i < dof; i++) - h_dist[i] = 0.0; - cudaMemcpyAsyncHelper(*this, h_coord, coord, dof * sizeof(T), AUTO, 0); - this->sync(0); - for (int i = 0; i < dof - 1; i++) { - h_dist[i] = h_coord[i + 1] - h_coord[i]; - } - if (dof != 2 && dof % 2 == 0) { - T last_dist = h_dist[dof - 2]; - h_dist[dof - 2] = last_dist / 2.0; - h_dist[dof - 1] = last_dist / 2.0; - } - cudaMemcpyAsyncHelper(*this, dist, h_dist, dof * sizeof(T), AUTO, 0); - this->sync(0); - delete[] h_coord; - delete[] h_dist; -} - -template -void Handle::dist_to_ratio(SIZE dof, T *dist, T *ratio) { - if (dof <= 1) - return; - // printf("dist_to_ratio %llu\n", dof); - T *h_dist = new T[dof]; - T *h_ratio = new T[dof]; - for (int i = 0; i < dof; i++) - h_ratio[i] = 0.0; - cudaMemcpyAsyncHelper(*this, h_dist, dist, dof * sizeof(T), AUTO, 0); - this->sync(0); - for (int i = 0; i < dof - 2; i++) { - h_ratio[i] = h_dist[i] / (h_dist[i + 1] + h_dist[i]); - // printf("dof: %llu ratio: %f\n", dof, h_ratio[i]); - } - if (dof % 2 == 0) { - h_ratio[dof - 2] = h_dist[dof - 2] / (h_dist[dof - 1] + h_dist[dof - 2]); - // printf("dof: %llu ratio: %f\n", dof, h_ratio[dof - 2]); - } - cudaMemcpyAsyncHelper(*this, ratio, h_ratio, dof * sizeof(T), AUTO, 0); - this->sync(0); - delete[] h_dist; - delete[] h_ratio; -} - -template -void Handle::reduce_dist(SIZE dof, T *dist, T *dist2) { - if (dof <= 1) - return; - // printf("reduce_dist\n"); - SIZE dof2 = dof / 2 + 1; - T *h_dist = new T[dof]; - T *h_dist2 = new T[dof2]; - for (int i = 0; i < dof2; i++) - h_dist2[i] = 0.0; - cudaMemcpyAsyncHelper(*this, h_dist, dist, dof * sizeof(T), AUTO, 0); - this->sync(0); - for (int i = 0; i < dof2 - 1; i++) { - h_dist2[i] = h_dist[i * 2] + h_dist[i * 2 + 1]; - } - if (dof2 != 2 && dof2 % 2 == 0) { - T last_dist = h_dist2[dof2 - 2]; - h_dist2[dof2 - 2] = last_dist / 2.0; - h_dist2[dof2 - 1] = last_dist / 2.0; - } - cudaMemcpyAsyncHelper(*this, dist2, h_dist2, dof2 * sizeof(T), AUTO, 0); - this->sync(0); - delete[] h_dist; - delete[] h_dist2; -} - -template -void Handle::calc_am_bm(SIZE dof, T *dist, T *am, T *bm) { - T *h_dist = new T[dof]; - T *h_am = new T[dof + 1]; - T *h_bm = new T[dof + 1]; - for (int i = 0; i < dof + 1; i++) { - h_am[i] = 0.0; - h_bm[i] = 0.0; - } - cudaMemcpyAsyncHelper(*this, h_dist, dist, dof * sizeof(T), AUTO, 0); - this->sync(0); - h_bm[0] = 2 * h_dist[0] / 6; - h_am[0] = 0.0; - - for (int i = 1; i < dof - 1; i++) { - T a_j = h_dist[i - 1] / 6; - T w = a_j / h_bm[i - 1]; - h_bm[i] = 2 * (h_dist[i - 1] + h_dist[i]) / 6 - w * a_j; - h_am[i] = a_j; - } - T a_j = h_dist[dof - 2] / 6; - T w = a_j / h_bm[dof - 2]; - h_bm[dof - 1] = 2 * h_dist[dof - 2] / 6 - w * a_j; - h_am[dof - 1] = a_j; -#ifdef MGARD_CUDA_FMA - for (int i = 0; i < dof + 1; i++) { - h_am[i] = -1 * h_am[i]; - h_bm[i] = 1 / h_bm[i]; - } -#endif - cudaMemcpyAsyncHelper(*this, am, h_am, dof * sizeof(T), AUTO, 0); - cudaMemcpyAsyncHelper(*this, bm + 1, h_bm, dof * sizeof(T), AUTO, - 0); // add offset - T one = 1; - cudaMemcpyAsyncHelper(*this, bm, &one, sizeof(T), AUTO, 0); // add offset - T zero = 0; - cudaMemcpyAsyncHelper(*this, am + dof, &zero, sizeof(T), AUTO, 0); - - this->sync(0); - delete[] h_dist; - delete[] h_am; - delete[] h_bm; -} - -template -void Handle::calc_volume(SIZE dof, T *dist, T *volume) { - T *h_dist = new T[dof]; - T *h_volume = new T[dof]; - for (int i = 0; i < dof; i++) { - h_volume[i] = 0.0; - } - cudaMemcpyAsyncHelper(*this, h_dist, dist, dof * sizeof(T), AUTO, 0); - this->sync(0); - if (dof == 2) { - h_volume[0] = h_dist[0] / 2; - h_volume[1] = h_dist[0] / 2; - } else { - int node_coeff_div = dof / 2 + 1; - h_volume[0] = h_dist[0] / 2; - for (int i = 1; i < dof - 1; i++) { - if (i % 2 == 0) { // node - h_volume[i / 2] = (h_dist[i - 1] + h_dist[i]) / 2; - } else { // coeff - h_volume[node_coeff_div + i / 2] = (h_dist[i - 1] + h_dist[i]) / 2; - } - } - if (dof % 2 != 0) { - h_volume[node_coeff_div - 1] = h_dist[dof - 2] / 2; - } else { - h_volume[node_coeff_div - 1] = h_dist[dof - 1] / 2; - } - } - - for (int i = 0; i < dof; i++) { - h_volume[i] = 1.0 / h_volume[i]; - } - cudaMemcpyAsyncHelper(*this, volume, h_volume, dof * sizeof(T), AUTO, 0); - this->sync(0); - delete[] h_dist; - delete[] h_volume; -} - -template -void Handle::init(std::vector shape, std::vector coords, - Config config) { - - this->shape = shape; - // determine dof - for (DIM i = 0; i < shape.size(); i++) { - std::vector curr_dofs; - int n = shape[i]; - // printf("shape[%d] = %d\n", i, shape[i]); - while (n > 2) { - curr_dofs.push_back(n); - n = n / 2 + 1; - } - if (shape[i] > 1) - curr_dofs.push_back(2); - dofs.push_back(curr_dofs); - // printf("dofs[%d].size() = %d\n", i, dofs[i].size()); - } - - // printf("isGPUPointer: %d\n", isGPUPointer(shape.data())); - - linearized_depth = 1; - for (int i = 2; i < shape.size(); i++) { - linearized_depth *= shape[i]; - } - - // workspace (assume 3d and above) - padded_linearized_depth = 1; - for (int i = 2; i < D; i++) { - padded_linearized_depth *= (shape[i] + 2); - } - - for (SIZE i = 1; i < shape.size(); i++) { - if (shape[i] == 1) { - for (SIZE l = 0; l < dofs[0].size(); l++) { - dofs[i].push_back(1); - } - } - } - - // for (int d = 0; d < std::max(3, (int)shape.size()); d++ ) { - // printf("shape[%d]: %d dofs[%d]: ", d, shape[d], d); - // for (int l = 0 ; l < dofs[d].size(); l++) { - // printf("%d ", dofs[d][l]); - // } - // printf("\n"); - // } - - // determine l target - SIZE nlevel = dofs[0].size(); - for (int i = 1; i < shape.size(); i++) { - nlevel = std::min(nlevel, (SIZE)dofs[i].size()); - } - l_target = nlevel - 1; - if (config.l_target != -1) { - l_target = std::min(nlevel - 1, config.l_target); - } - // l_target = nlevel; - // printf("nlevel - 1 %d, l_target: %d\n", nlevel - 1, config.l_target); - - // shapes - for (int l = 0; l < l_target + 1; l++) { - SIZE *curr_shape_h = new SIZE[D_padded]; - for (int d = 0; d < D_padded; d++) { - curr_shape_h[d] = dofs[d][l]; - } - shapes_h.push_back(curr_shape_h); - SIZE *curr_shape_d; - cudaMallocHelper(*this, (void **)&(curr_shape_d), D_padded * sizeof(SIZE)); - cudaMemcpyAsyncHelper(*this, curr_shape_d, curr_shape_h, - D_padded * sizeof(SIZE), mgard_cuda::H2D, 0); - shapes_d.push_back(curr_shape_d); - } - - // ranges - ranges_h = new SIZE[D * (l_target + 2)]; - for (int d = 0; d < D; d++) { - ranges_h[d * (l_target + 2)] = 0; - for (int l = 1; l < l_target + 2; l++) { - ranges_h[d * (l_target + 2) + l] = dofs[d][l_target + 1 - l]; - } - // printf("hshapes[%d]: ", d); - // for (int l = 0; l < handle.l_target+2; l++) { printf("%d ", hshapes[d * - // (handle.l_target+2)+l]); } printf("\n"); - } - cudaMallocHelper(*this, (void **)&ranges_d, - D * (l_target + 2) * sizeof(SIZE)); - cudaMemcpyAsyncHelper(*this, ranges_d, ranges_h, - D * (l_target + 2) * sizeof(SIZE), H2D, 0); - - processed_n = new DIM[D]; - processed_dims_h = new DIM *[D]; - processed_dims_d = new DIM *[D]; - - { - thrust::device_vector tmp(0); - for (int d = 0; d < D; d++) { - processed_n[d] = tmp.size(); - processed_dims_h[d] = new DIM[processed_n[d]]; - cudaMemcpyAsyncHelper(*this, processed_dims_h[d], - thrust::raw_pointer_cast(tmp.data()), - processed_n[d] * sizeof(DIM), mgard_cuda::D2H, 0); - cudaMallocHelper(*this, (void **)&processed_dims_d[d], - processed_n[d] * sizeof(DIM)); - cudaMemcpyAsyncHelper(*this, processed_dims_d[d], - thrust::raw_pointer_cast(tmp.data()), - processed_n[d] * sizeof(DIM), mgard_cuda::D2D, 0); - tmp.push_back(d); - } - } - { - thrust::device_vector tmp(0); - for (int i = 3; i < D; i++) { - tmp.push_back(i); - } - unprocessed_n = new DIM[tmp.size()]; - unprocessed_dims_h = new DIM *[tmp.size()]; - unprocessed_dims_d = new DIM *[tmp.size()]; - - //+1 is used for storing empty status - for (int d = 0; d < (int)D - 3 + 1; d++) { - unprocessed_n[d] = tmp.size(); - unprocessed_dims_h[d] = new DIM[unprocessed_n[d]]; - cudaMemcpyAsyncHelper(*this, unprocessed_dims_h[d], - thrust::raw_pointer_cast(tmp.data()), - unprocessed_n[d] * sizeof(DIM), mgard_cuda::D2H, 0); - cudaMallocHelper(*this, (void **)&unprocessed_dims_d[d], - unprocessed_n[d] * sizeof(DIM)); - cudaMemcpyAsyncHelper(*this, unprocessed_dims_d[d], - thrust::raw_pointer_cast(tmp.data()), - unprocessed_n[d] * sizeof(DIM), mgard_cuda::D2D, 0); - tmp.pop_back(); - } - } - - cudaMallocHelper(*this, (void **)&(quantizers), (l_target + 1) * sizeof(T)); - - // handle coords - this->coords_h = coords; - for (int i = 0; i < shape.size(); i++) { - T *curr_dcoords; - cudaMallocHelper(*this, (void **)&(curr_dcoords), shape[i] * sizeof(T)); - cudaMemcpyAsyncHelper(*this, curr_dcoords, this->coords_h[i], - shape[i] * sizeof(T), AUTO, 0); - this->coords_d.push_back(curr_dcoords); - } - - // calculate dist and ratio - for (int i = 0; i < shape.size(); i++) { - std::vector curr_ddist_l, curr_dratio_l; - // for level 0 - int last_dist = dofs[i][0] - 1; - T *curr_ddist0, *curr_dratio0; - cudaMallocHelper(*this, (void **)&curr_ddist0, dofs[i][0] * sizeof(T)); - cudaMallocHelper(*this, (void **)&curr_dratio0, dofs[i][0] * sizeof(T)); - curr_ddist_l.push_back(curr_ddist0); - curr_dratio_l.push_back(curr_dratio0); - coord_to_dist(dofs[i][0], this->coords_d[i], curr_ddist_l[0]); - dist_to_ratio(dofs[i][0], curr_ddist_l[0], curr_dratio_l[0]); - - // for l = 1 ... l_target - for (int l = 1; l < l_target + 1; l++) { - T *curr_ddist, *curr_dratio; - cudaMallocHelper(*this, (void **)&curr_ddist, dofs[i][l] * sizeof(T)); - cudaMallocHelper(*this, (void **)&curr_dratio, dofs[i][l] * sizeof(T)); - curr_ddist_l.push_back(curr_ddist); - curr_dratio_l.push_back(curr_dratio); - reduce_dist(dofs[i][l - 1], curr_ddist_l[l - 1], curr_ddist_l[l]); - dist_to_ratio(dofs[i][l], curr_ddist_l[l], curr_dratio_l[l]); - } - dist.push_back(curr_ddist_l); - ratio.push_back(curr_dratio_l); - } - - // for (int l = 0; l < l_target+1; l++) { - // printf("l: %d\n", l); - // for (int d = 0; d < D; d++) { - // printf("dist: "); - // print_matrix_cuda(1, dofs[d][l], dist[d][l], dofs[d][l]); - // printf("ratio: "); - // print_matrix_cuda(1, dofs[d][l], ratio[d][l], dofs[d][l]); - // } - // } - - // volume for quantization - SIZE volumes_width = 0; - for (int d = 0; d < D; d++) { - volumes_width = std::max(volumes_width, dofs[d][0]); - } - size_t volumes_pitch; - cudaMallocPitchHelper(*this, (void **)&volumes, &volumes_pitch, - volumes_width * sizeof(T), D * (l_target + 1)); - ldvolumes = (SIZE)volumes_pitch / sizeof(T); - for (int d = 0; d < D; d++) { - for (int l = 0; l < l_target + 1; l++) { - calc_volume(dofs[d][l], dist[d][l], - volumes + ldvolumes * (d * (l_target + 1) + (l_target - l))); - } - } - - // printf("volumes:\n"); - // print_matrix_cuda(D * (l_target+1), volumes_width, volumes, ldvolumes); - - for (DIM i = 0; i < D; i++) { - std::vector curr_am_l, curr_bm_l; - for (SIZE l = 0; l < l_target + 1; l++) { - T *curr_am, *curr_bm; - cudaMallocHelper(*this, (void **)&curr_am, (dofs[i][l] + 1) * sizeof(T)); - cudaMallocHelper(*this, (void **)&curr_bm, (dofs[i][l] + 1) * sizeof(T)); - cudaMemsetHelper((void **)&curr_am, (dofs[i][l] + 1) * sizeof(T), 0); - cudaMemsetHelper((void **)&curr_bm, (dofs[i][l] + 1) * sizeof(T), 0); - curr_am_l.push_back(curr_am); - curr_bm_l.push_back(curr_bm); - calc_am_bm(dofs[i][l], dist[i][l], curr_am_l[l], curr_bm_l[l]); - // printf("d: %d, l: %d\n", i, l); - // printf("am: "); - // print_matrix_cuda(1, dofs[i][l]+1, curr_am_l[l], dofs[i][l]+1); - // printf("bm: "); - // print_matrix_cuda(1, dofs[i][l]+1, curr_bm_l[l], dofs[i][l]+1); - } - am.push_back(curr_am_l); - bm.push_back(curr_bm_l); - } - - lossless = config.lossless; - huff_dict_size = config.huff_dict_size; - huff_block_size = config.huff_block_size; - lz4_block_size = config.lz4_block_size; - reduce_memory_footprint = config.reduce_memory_footprint; - profile_kernels = config.profile_kernels; - sync_and_check_all_kernels = config.sync_and_check_all_kernels; - timing = config.timing; - - initialized = true; -} - -template void Handle::destroy() { - - for (int i = 0; i < shapes_d.size(); i++) { - cudaFreeHelper(shapes_d[i]); - } - - delete[] ranges_h; - cudaFreeHelper(ranges_d); - - for (int d = 0; d < D; d++) { - delete[] processed_dims_h[d]; - cudaFreeHelper(processed_dims_d[d]); - } - delete[] processed_n; - delete[] processed_dims_h; - delete[] processed_dims_d; - - for (int d = 0; d < (int)D - 3; d++) { - // printf("d=%d D-3=%d\n",d, D-3); - delete[] unprocessed_dims_h[d]; - cudaFreeHelper(unprocessed_dims_d[d]); - } - delete[] unprocessed_n; - delete[] unprocessed_dims_h; - delete[] unprocessed_dims_d; - - cudaFreeHelper(quantizers); - - for (int i = 0; i < D_padded; i++) { - cudaFreeHelper(coords_d[i]); - } - - for (int i = 0; i < dist.size(); i++) { - for (int l = 0; l < dist[i].size(); l++) { - cudaFreeHelper(dist[i][l]); - cudaFreeHelper(ratio[i][l]); - } - } - - for (int i = 0; i < am.size(); i++) { - for (int l = 0; l < am[i].size(); l++) { - cudaFreeHelper(am[i][l]); - cudaFreeHelper(bm[i][l]); - } - } - - if (uniform_coords_created) { - for (int d = 0; d < D; d++) { - // delete [] this->coords_h[d]; - } - uniform_coords_created = false; - } -} - -template -void Handle::padding_dimensions(std::vector &shape, - std::vector &coords) { - D_padded = D; - if (D < 3) { - D_padded = 3; - } - if (D % 2 == 0) { - D_padded = D + 1; - } - // padding dimensions - for (int d = shape.size(); d < D_padded; d++) { - shape.push_back(1); - T *curr_coords = new T[shape[d]]; - for (int i = 0; i < shape[d]; i++) { - curr_coords[i] = (T)i; - } - coords.push_back(curr_coords); - } - // printf("D: %d, D_padded: %d\n", D, D_padded); -} - -template void Handle::create_queues() { - num_of_queues = 16; - cudaStream_t *ptr = new cudaStream_t[num_of_queues]; - for (int i = 0; i < num_of_queues; i++) { - gpuErrchk(cudaStreamCreate(ptr + i)); - } - queues = (void *)ptr; -} - -template void Handle::destroy_queues() { - cudaStream_t *ptr = (cudaStream_t *)queues; - for (int i = 0; i < num_of_queues; i++) { - gpuErrchk(cudaStreamDestroy(ptr[i])); - } -} - -template -std::vector Handle::create_uniform_coords(std::vector shape, - int mode) { - - std::vector coords(D); - for (int d = 0; d < D; d++) { - T *curr_coords = new T[shape[d]]; - for (int i = 0; i < shape[d]; i++) { - // 0...n-1 - if (mode == 0) { - // printf("create_uniform_coords %d\n", mode); - curr_coords[i] = (T)i; - } else if (mode == 1) { - // 0...1 - curr_coords[i] = (T)i / (shape[d] - 1); - } else { - std::cout << log::log_err << "wrong uniform coordinates mode!\n"; - exit(-1); - } - } - coords[d] = curr_coords; - } - uniform_coords_created = true; - return coords; -} - -template void Handle::init_auto_tuning_table() { - - cudaDeviceProp prop; - cudaGetDeviceProperties(&prop, dev_id); - - arch = 1; // default optimized for Volta - - if (prop.major == 7 && prop.minor == 0) { - arch = 1; - // printf("Optimized: Volta\n"); - } - - if (prop.major == 7 && (prop.minor == 2 || prop.minor == 5)) { - arch = 2; - // printf("Optimized: Turing\n"); - } - cudaDeviceSetSharedMemConfig(cudaSharedMemBankSizeEightByte); - cudaDeviceSetCacheConfig(cudaFuncCachePreferShared); - - if (sizeof(T) == sizeof(float)) { - precision = 0; - } else if (sizeof(T) == sizeof(double)) { - precision = 1; - } - - this->auto_tuning_cc = new int **[num_arch]; - this->auto_tuning_mr1 = new int **[num_arch]; - this->auto_tuning_mr2 = new int **[num_arch]; - this->auto_tuning_mr3 = new int **[num_arch]; - this->auto_tuning_ts1 = new int **[num_arch]; - this->auto_tuning_ts2 = new int **[num_arch]; - this->auto_tuning_ts3 = new int **[num_arch]; - for (int i = 0; i < num_arch; i++) { - this->auto_tuning_cc[i] = new int *[num_precision]; - this->auto_tuning_mr1[i] = new int *[num_precision]; - this->auto_tuning_mr2[i] = new int *[num_precision]; - this->auto_tuning_mr3[i] = new int *[num_precision]; - this->auto_tuning_ts1[i] = new int *[num_precision]; - this->auto_tuning_ts2[i] = new int *[num_precision]; - this->auto_tuning_ts3[i] = new int *[num_precision]; - for (int j = 0; j < num_precision; j++) { - this->auto_tuning_cc[i][j] = new int[num_range]; - this->auto_tuning_mr1[i][j] = new int[num_range]; - this->auto_tuning_mr2[i][j] = new int[num_range]; - this->auto_tuning_mr3[i][j] = new int[num_range]; - this->auto_tuning_ts1[i][j] = new int[num_range]; - this->auto_tuning_ts2[i][j] = new int[num_range]; - this->auto_tuning_ts3[i][j] = new int[num_range]; - } - } - - // Default - for (int i = 0; i < num_arch; i++) { - for (int j = 0; j < num_precision; j++) { - for (int k = 0; k < num_range; k++) { - this->auto_tuning_cc[i][j][k] = 0; - this->auto_tuning_mr1[i][j][k] = 0; - this->auto_tuning_mr2[i][j][k] = 0; - this->auto_tuning_mr3[i][j][k] = 0; - this->auto_tuning_ts1[i][j][k] = 0; - this->auto_tuning_ts2[i][j][k] = 0; - this->auto_tuning_ts3[i][j][k] = 0; - } - } - } - - // Volta-Single - this->auto_tuning_cc[1][0][0] = 1; - this->auto_tuning_cc[1][0][1] = 1; - this->auto_tuning_cc[1][0][2] = 1; - this->auto_tuning_cc[1][0][3] = 1; - this->auto_tuning_cc[1][0][4] = 1; - this->auto_tuning_cc[1][0][5] = 5; - this->auto_tuning_cc[1][0][6] = 5; - this->auto_tuning_cc[1][0][7] = 5; - this->auto_tuning_cc[1][0][8] = 5; - - this->auto_tuning_mr1[1][0][0] = 1; - this->auto_tuning_mr2[1][0][0] = 1; - this->auto_tuning_mr3[1][0][0] = 1; - this->auto_tuning_mr1[1][0][1] = 1; - this->auto_tuning_mr2[1][0][1] = 1; - this->auto_tuning_mr3[1][0][1] = 1; - this->auto_tuning_mr1[1][0][2] = 1; - this->auto_tuning_mr2[1][0][2] = 1; - this->auto_tuning_mr3[1][0][2] = 1; - this->auto_tuning_mr1[1][0][3] = 3; - this->auto_tuning_mr2[1][0][3] = 3; - this->auto_tuning_mr3[1][0][3] = 3; - this->auto_tuning_mr1[1][0][4] = 4; - this->auto_tuning_mr2[1][0][4] = 1; - this->auto_tuning_mr3[1][0][4] = 3; - this->auto_tuning_mr1[1][0][5] = 5; - this->auto_tuning_mr2[1][0][5] = 3; - this->auto_tuning_mr3[1][0][5] = 3; - this->auto_tuning_mr1[1][0][6] = 5; - this->auto_tuning_mr2[1][0][6] = 4; - this->auto_tuning_mr3[1][0][6] = 4; - this->auto_tuning_mr1[1][0][7] = 3; - this->auto_tuning_mr2[1][0][7] = 4; - this->auto_tuning_mr3[1][0][7] = 4; - this->auto_tuning_mr1[1][0][8] = 3; - this->auto_tuning_mr2[1][0][8] = 4; - this->auto_tuning_mr3[1][0][8] = 4; - - this->auto_tuning_ts1[1][0][0] = 1; - this->auto_tuning_ts2[1][0][0] = 1; - this->auto_tuning_ts3[1][0][0] = 1; - this->auto_tuning_ts1[1][0][1] = 1; - this->auto_tuning_ts2[1][0][1] = 1; - this->auto_tuning_ts3[1][0][1] = 1; - this->auto_tuning_ts1[1][0][2] = 2; - this->auto_tuning_ts2[1][0][2] = 2; - this->auto_tuning_ts3[1][0][2] = 2; - this->auto_tuning_ts1[1][0][3] = 3; - this->auto_tuning_ts2[1][0][3] = 2; - this->auto_tuning_ts3[1][0][3] = 2; - this->auto_tuning_ts1[1][0][4] = 3; - this->auto_tuning_ts2[1][0][4] = 2; - this->auto_tuning_ts3[1][0][4] = 2; - this->auto_tuning_ts1[1][0][5] = 3; - this->auto_tuning_ts2[1][0][5] = 2; - this->auto_tuning_ts3[1][0][5] = 2; - this->auto_tuning_ts1[1][0][6] = 5; - this->auto_tuning_ts2[1][0][6] = 3; - this->auto_tuning_ts3[1][0][6] = 2; - this->auto_tuning_ts1[1][0][7] = 5; - this->auto_tuning_ts2[1][0][7] = 6; - this->auto_tuning_ts3[1][0][7] = 5; - this->auto_tuning_ts1[1][0][8] = 5; - this->auto_tuning_ts2[1][0][8] = 6; - this->auto_tuning_ts3[1][0][8] = 5; - // Volta-Double - - this->auto_tuning_cc[1][1][0] = 1; - this->auto_tuning_cc[1][1][1] = 1; - this->auto_tuning_cc[1][1][2] = 1; - this->auto_tuning_cc[1][1][3] = 1; - this->auto_tuning_cc[1][1][4] = 4; - this->auto_tuning_cc[1][1][5] = 5; - this->auto_tuning_cc[1][1][6] = 6; - this->auto_tuning_cc[1][1][7] = 6; - this->auto_tuning_cc[1][1][8] = 5; - - this->auto_tuning_mr1[1][1][0] = 1; - this->auto_tuning_mr2[1][1][0] = 1; - this->auto_tuning_mr3[1][1][0] = 1; - this->auto_tuning_mr1[1][1][1] = 1; - this->auto_tuning_mr2[1][1][1] = 1; - this->auto_tuning_mr3[1][1][1] = 1; - this->auto_tuning_mr1[1][1][2] = 1; - this->auto_tuning_mr2[1][1][2] = 1; - this->auto_tuning_mr3[1][1][2] = 1; - this->auto_tuning_mr1[1][1][3] = 1; - this->auto_tuning_mr2[1][1][3] = 3; - this->auto_tuning_mr3[1][1][3] = 1; - this->auto_tuning_mr1[1][1][4] = 4; - this->auto_tuning_mr2[1][1][4] = 3; - this->auto_tuning_mr3[1][1][4] = 3; - this->auto_tuning_mr1[1][1][5] = 5; - this->auto_tuning_mr2[1][1][5] = 5; - this->auto_tuning_mr3[1][1][5] = 5; - this->auto_tuning_mr1[1][1][6] = 4; - this->auto_tuning_mr2[1][1][6] = 6; - this->auto_tuning_mr3[1][1][6] = 6; - this->auto_tuning_mr1[1][1][7] = 6; - this->auto_tuning_mr2[1][1][7] = 6; - this->auto_tuning_mr3[1][1][7] = 5; - this->auto_tuning_mr1[1][1][8] = 6; - this->auto_tuning_mr2[1][1][8] = 6; - this->auto_tuning_mr3[1][1][8] = 5; - - this->auto_tuning_ts1[1][1][0] = 1; - this->auto_tuning_ts2[1][1][0] = 1; - this->auto_tuning_ts3[1][1][0] = 1; - this->auto_tuning_ts1[1][1][1] = 1; - this->auto_tuning_ts2[1][1][1] = 1; - this->auto_tuning_ts3[1][1][1] = 1; - this->auto_tuning_ts1[1][1][2] = 2; - this->auto_tuning_ts2[1][1][2] = 2; - this->auto_tuning_ts3[1][1][2] = 2; - this->auto_tuning_ts1[1][1][3] = 3; - this->auto_tuning_ts2[1][1][3] = 2; - this->auto_tuning_ts3[1][1][3] = 2; - this->auto_tuning_ts1[1][1][4] = 3; - this->auto_tuning_ts2[1][1][4] = 2; - this->auto_tuning_ts3[1][1][4] = 2; - this->auto_tuning_ts1[1][1][5] = 4; - this->auto_tuning_ts2[1][1][5] = 2; - this->auto_tuning_ts3[1][1][5] = 2; - this->auto_tuning_ts1[1][1][6] = 5; - this->auto_tuning_ts2[1][1][6] = 5; - this->auto_tuning_ts3[1][1][6] = 2; - this->auto_tuning_ts1[1][1][7] = 5; - this->auto_tuning_ts2[1][1][7] = 6; - this->auto_tuning_ts3[1][1][7] = 6; - this->auto_tuning_ts1[1][1][8] = 5; - this->auto_tuning_ts2[1][1][8] = 6; - this->auto_tuning_ts3[1][1][8] = 6; - - // Turing-Single - this->auto_tuning_cc[2][0][0] = 1; - this->auto_tuning_cc[2][0][1] = 1; - this->auto_tuning_cc[2][0][2] = 1; - this->auto_tuning_cc[2][0][3] = 1; - this->auto_tuning_cc[2][0][4] = 3; - this->auto_tuning_cc[2][0][5] = 5; - this->auto_tuning_cc[2][0][6] = 5; - this->auto_tuning_cc[2][0][7] = 5; - this->auto_tuning_cc[2][0][8] = 4; - - this->auto_tuning_mr1[2][0][0] = 1; - this->auto_tuning_mr2[2][0][0] = 1; - this->auto_tuning_mr3[2][0][0] = 1; - this->auto_tuning_mr1[2][0][1] = 1; - this->auto_tuning_mr2[2][0][1] = 1; - this->auto_tuning_mr3[2][0][1] = 1; - this->auto_tuning_mr1[2][0][2] = 1; - this->auto_tuning_mr2[2][0][2] = 1; - this->auto_tuning_mr3[2][0][2] = 1; - this->auto_tuning_mr1[2][0][3] = 1; - this->auto_tuning_mr2[2][0][3] = 1; - this->auto_tuning_mr3[2][0][3] = 3; - this->auto_tuning_mr1[2][0][4] = 4; - this->auto_tuning_mr2[2][0][4] = 3; - this->auto_tuning_mr3[2][0][4] = 4; - this->auto_tuning_mr1[2][0][5] = 4; - this->auto_tuning_mr2[2][0][5] = 3; - this->auto_tuning_mr3[2][0][5] = 3; - this->auto_tuning_mr1[2][0][6] = 6; - this->auto_tuning_mr2[2][0][6] = 3; - this->auto_tuning_mr3[2][0][6] = 3; - this->auto_tuning_mr1[2][0][7] = 5; - this->auto_tuning_mr2[2][0][7] = 4; - this->auto_tuning_mr3[2][0][7] = 4; - this->auto_tuning_mr1[2][0][8] = 5; - this->auto_tuning_mr2[2][0][8] = 4; - this->auto_tuning_mr3[2][0][8] = 4; - - this->auto_tuning_ts1[2][0][0] = 1; - this->auto_tuning_ts2[2][0][0] = 1; - this->auto_tuning_ts3[2][0][0] = 1; - this->auto_tuning_ts1[2][0][1] = 1; - this->auto_tuning_ts2[2][0][1] = 1; - this->auto_tuning_ts3[2][0][1] = 1; - this->auto_tuning_ts1[2][0][2] = 2; - this->auto_tuning_ts2[2][0][2] = 2; - this->auto_tuning_ts3[2][0][2] = 2; - this->auto_tuning_ts1[2][0][3] = 3; - this->auto_tuning_ts2[2][0][3] = 2; - this->auto_tuning_ts3[2][0][3] = 2; - this->auto_tuning_ts1[2][0][4] = 3; - this->auto_tuning_ts2[2][0][4] = 2; - this->auto_tuning_ts3[2][0][4] = 2; - this->auto_tuning_ts1[2][0][5] = 3; - this->auto_tuning_ts2[2][0][5] = 2; - this->auto_tuning_ts3[2][0][5] = 2; - this->auto_tuning_ts1[2][0][6] = 5; - this->auto_tuning_ts2[2][0][6] = 5; - this->auto_tuning_ts3[2][0][6] = 2; - this->auto_tuning_ts1[2][0][7] = 5; - this->auto_tuning_ts2[2][0][7] = 6; - this->auto_tuning_ts3[2][0][7] = 6; - this->auto_tuning_ts1[2][0][8] = 5; - this->auto_tuning_ts2[2][0][8] = 6; - this->auto_tuning_ts3[2][0][8] = 6; - // Turing-Double - - this->auto_tuning_cc[2][1][0] = 0; - this->auto_tuning_cc[2][1][1] = 0; - this->auto_tuning_cc[2][1][2] = 2; - this->auto_tuning_cc[2][1][3] = 2; - this->auto_tuning_cc[2][1][4] = 3; - this->auto_tuning_cc[2][1][5] = 4; - this->auto_tuning_cc[2][1][6] = 4; - this->auto_tuning_cc[2][1][7] = 6; - this->auto_tuning_cc[2][1][8] = 3; - - this->auto_tuning_mr1[2][1][0] = 1; - this->auto_tuning_mr2[2][1][0] = 1; - this->auto_tuning_mr3[2][1][0] = 1; - this->auto_tuning_mr1[2][1][1] = 1; - this->auto_tuning_mr2[2][1][1] = 1; - this->auto_tuning_mr3[2][1][1] = 1; - this->auto_tuning_mr1[2][1][2] = 1; - this->auto_tuning_mr2[2][1][2] = 1; - this->auto_tuning_mr3[2][1][2] = 1; - this->auto_tuning_mr1[2][1][3] = 1; - this->auto_tuning_mr2[2][1][3] = 1; - this->auto_tuning_mr3[2][1][3] = 1; - this->auto_tuning_mr1[2][1][4] = 4; - this->auto_tuning_mr2[2][1][4] = 4; - this->auto_tuning_mr3[2][1][4] = 1; - this->auto_tuning_mr1[2][1][5] = 1; - this->auto_tuning_mr2[2][1][5] = 1; - this->auto_tuning_mr3[2][1][5] = 1; - this->auto_tuning_mr1[2][1][6] = 1; - this->auto_tuning_mr2[2][1][6] = 1; - this->auto_tuning_mr3[2][1][6] = 1; - this->auto_tuning_mr1[2][1][7] = 1; - this->auto_tuning_mr2[2][1][7] = 1; - this->auto_tuning_mr3[2][1][7] = 1; - this->auto_tuning_mr1[2][1][8] = 1; - this->auto_tuning_mr2[2][1][8] = 1; - this->auto_tuning_mr3[2][1][8] = 1; - - this->auto_tuning_ts1[2][1][0] = 1; - this->auto_tuning_ts2[2][1][0] = 1; - this->auto_tuning_ts3[2][1][0] = 1; - this->auto_tuning_ts1[2][1][1] = 1; - this->auto_tuning_ts2[2][1][1] = 1; - this->auto_tuning_ts3[2][1][1] = 1; - this->auto_tuning_ts1[2][1][2] = 2; - this->auto_tuning_ts2[2][1][2] = 2; - this->auto_tuning_ts3[2][1][2] = 2; - this->auto_tuning_ts1[2][1][3] = 3; - this->auto_tuning_ts2[2][1][3] = 2; - this->auto_tuning_ts3[2][1][3] = 2; - this->auto_tuning_ts1[2][1][4] = 2; - this->auto_tuning_ts2[2][1][4] = 2; - this->auto_tuning_ts3[2][1][4] = 2; - this->auto_tuning_ts1[2][1][5] = 2; - this->auto_tuning_ts2[2][1][5] = 2; - this->auto_tuning_ts3[2][1][5] = 2; - this->auto_tuning_ts1[2][1][6] = 3; - this->auto_tuning_ts2[2][1][6] = 5; - this->auto_tuning_ts3[2][1][6] = 3; - this->auto_tuning_ts1[2][1][7] = 3; - this->auto_tuning_ts2[2][1][7] = 6; - this->auto_tuning_ts3[2][1][7] = 6; - this->auto_tuning_ts1[2][1][8] = 3; - this->auto_tuning_ts2[2][1][8] = 6; - this->auto_tuning_ts3[2][1][8] = 6; - auto_tuning_table_created = true; -} - -template void Handle::destroy_auto_tuning_table() { - for (int i = 0; i < num_arch; i++) { - for (int j = 0; j < num_precision; j++) { - delete[] this->auto_tuning_cc[i][j]; - delete[] this->auto_tuning_mr1[i][j]; - delete[] this->auto_tuning_mr2[i][j]; - delete[] this->auto_tuning_mr3[i][j]; - delete[] this->auto_tuning_ts1[i][j]; - delete[] this->auto_tuning_ts2[i][j]; - delete[] this->auto_tuning_ts3[i][j]; - } - delete[] this->auto_tuning_cc[i]; - delete[] this->auto_tuning_mr1[i]; - delete[] this->auto_tuning_mr2[i]; - delete[] this->auto_tuning_mr3[i]; - delete[] this->auto_tuning_ts1[i]; - delete[] this->auto_tuning_ts2[i]; - delete[] this->auto_tuning_ts3[i]; - } - delete[] this->auto_tuning_cc; - delete[] this->auto_tuning_mr1; - delete[] this->auto_tuning_mr2; - delete[] this->auto_tuning_mr3; - delete[] this->auto_tuning_ts1; - delete[] this->auto_tuning_ts2; - delete[] this->auto_tuning_ts3; -} - -template void Handle::allocate_workspace() { - - // size_t free, total; - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - - // printf("allocate_workspace: %llu\n", (shapes_h[0][0] + 2) * sizeof(T) * - // (shapes_h[0][1] + 2) * padded_linearized_depth); - size_t dw_pitch; - mgard_cuda::cudaMalloc3DHelper(*this, (void **)&(dw), &dw_pitch, - (shapes_h[0][0] + 2) * sizeof(T), - shapes_h[0][1] + 2, padded_linearized_depth); - // printf("pitch %llu\n", dw_pitch); - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); - - // ldws.push_back(dw_pitch / sizeof(T)); - // for (int i = 1; i < D_padded; i++) { - // ldws.push_back(shapes_h[0][i] + 2); - // } - lddw1 = (SIZE)dw_pitch / sizeof(T); - lddw2 = shapes_h[0][1] + 2; - - // ldws_h = new int[D_padded]; - // ldws_h[0] = dw_pitch / sizeof(T); - // for (int i = 1; i < D_padded; i++) { - // ldws_h[i] = shapes_h[0][i] + 2; - // } - - ldws_h.push_back(dw_pitch / sizeof(T)); - for (int i = 1; i < D_padded; i++) { - ldws_h.push_back(shapes_h[0][i] + 2); - } - - mgard_cuda::cudaMallocHelper(*this, (void **)&ldws_d, - D_padded * sizeof(SIZE)); - mgard_cuda::cudaMemcpyAsyncHelper(*this, ldws_d, ldws_h.data(), - D_padded * sizeof(SIZE), mgard_cuda::H2D, - 0); - - if (D > 3) { - // printf("allocate_workspace: %llu\n", (shapes_h[0][0] + 2) * sizeof(T) * - // (shapes_h[0][1] + 2) * padded_linearized_depth); - size_t db_pitch; - mgard_cuda::cudaMalloc3DHelper(*this, (void **)&(db), &db_pitch, - (shapes_h[0][0] + 2) * sizeof(T), - shapes_h[0][1] + 2, padded_linearized_depth); - - // ldbs.push_back(db_pitch / sizeof(T)); - // for (int i = 1; i < D_padded; i++) { - // ldbs.push_back(shapes_h[0][i] + 2); - // } - lddb1 = (SIZE)db_pitch / sizeof(T); - lddb2 = shapes_h[0][1] + 2; - - // ldbs_h = new int[D_padded]; - // ldbs_h[0] = db_pitch / sizeof(T); - // for (int i = 1; i < D_padded; i++) { - // ldbs_h[i] = shapes_h[0][i] + 2; - // } - - ldbs_h.push_back(db_pitch / sizeof(T)); - for (int i = 1; i < D_padded; i++) { - ldbs_h.push_back(shapes_h[0][i] + 2); - } - - mgard_cuda::cudaMallocHelper(*this, (void **)&ldbs_d, - D_padded * sizeof(SIZE)); - mgard_cuda::cudaMemcpyAsyncHelper(*this, ldbs_d, ldbs_h.data(), - D_padded * sizeof(SIZE), mgard_cuda::H2D, - 0); - } - // cudaMemGetInfo(&free, &total); printf("Mem: %f/%f\n", - // (double)(total-free)/1e9, (double)total/1e9); -} - -template void Handle::free_workspace() { - mgard_cuda::cudaFreeHelper(dw); - mgard_cuda::cudaFreeHelper(ldws_d); - if (D > 3) { - mgard_cuda::cudaFreeHelper(db); - // delete [] ldbs_h; - mgard_cuda::cudaFreeHelper(ldbs_d); - } -} - -// This constructor is for internal use only -template Handle::Handle() { - cudaSetDeviceHelper(dev_id); - create_queues(); -} - -template Handle::Handle(std::vector shape) { - Config config; - dev_id = config.dev_id; - cudaSetDeviceHelper(dev_id); - std::reverse(shape.begin(), shape.end()); - int ret = check_shape(shape); - if (ret == -1) { - std::cerr << log::log_err - << "Number of dimensions mismatch. mgard_cuda::Hanlde not " - "initialized!\n"; - return; - } - if (ret == -2) { - std::cerr << log::log_err - << "Size of any dimensions cannot be smaller than 3. " - "mgard_cuda::Hanlde not " - "initialized!\n"; - } - dstype = data_structure_type::Cartesian_Grid_Uniform; - std::vector coords = - create_uniform_coords(shape, config.uniform_coord_mode); - padding_dimensions(shape, coords); - create_queues(); - init_auto_tuning_table(); - init(shape, coords, config); -} - -template -Handle::Handle(std::vector shape, std::vector coords) { - Config config; - dev_id = config.dev_id; - cudaSetDeviceHelper(dev_id); - std::reverse(shape.begin(), shape.end()); - std::reverse(coords.begin(), coords.end()); - int ret = check_shape(shape); - if (ret == -1) { - std::cerr << log::log_err - << "Number of dimensions mismatch. mgard_cuda::Hanlde not " - "initialized!\n"; - return; - } - if (ret == -2) { - std::cerr << log::log_err - << "Size of any dimensions cannot be smaller than 3. " - "mgard_cuda::Hanlde not " - "initialized!\n"; - } - - dstype = data_structure_type::Cartesian_Grid_Non_Uniform; - padding_dimensions(shape, coords); - create_queues(); - init_auto_tuning_table(); - init(shape, coords, config); -} - -template -Handle::Handle(std::vector shape, Config config) { - dev_id = config.dev_id; - - cudaSetDeviceHelper(dev_id); - - std::reverse(shape.begin(), shape.end()); - std::vector coords = - create_uniform_coords(shape, config.uniform_coord_mode); - int ret = check_shape(shape); - if (ret == -1) { - std::cerr << log::log_err - << "Number of dimensions mismatch. mgard_cuda::Hanlde not " - "initialized!\n"; - return; - } - if (ret == -2) { - std::cerr << log::log_err - << "Size of any dimensions cannot be smaller than 3. " - "mgard_cuda::Hanlde not " - "initialized!\n"; - } - - dstype = data_structure_type::Cartesian_Grid_Uniform; - padding_dimensions(shape, coords); - create_queues(); - init_auto_tuning_table(); - init(shape, coords, config); -} - -template -Handle::Handle(std::vector shape, std::vector coords, - Config config) { - dev_id = config.dev_id; - cudaSetDeviceHelper(dev_id); - std::reverse(shape.begin(), shape.end()); - std::reverse(coords.begin(), coords.end()); - int ret = check_shape(shape); - if (ret == -1) { - std::cerr << log::log_err - << "Number of dimensions mismatch. mgard_cuda::Hanlde not " - "initialized!\n"; - return; - } - if (ret == -2) { - std::cerr << log::log_err - << "Size of any dimensions cannot be smaller than 3. " - "mgard_cuda::Hanlde not " - "initialized!\n"; - } - - dstype = data_structure_type::Cartesian_Grid_Non_Uniform; - padding_dimensions(shape, coords); - create_queues(); - init_auto_tuning_table(); - init(shape, coords, config); -} - -template void *Handle::get(int i) { - cudaSetDeviceHelper(dev_id); - cudaStream_t *ptr = (cudaStream_t *)(this->queues); - return (void *)(ptr + i); -} - -template void Handle::sync(int i) { - cudaSetDeviceHelper(dev_id); - cudaStream_t *ptr = (cudaStream_t *)(this->queues); - gpuErrchk(cudaStreamSynchronize(ptr[i])); -} - -template void Handle::sync_all() { - cudaSetDeviceHelper(dev_id); - cudaStream_t *ptr = (cudaStream_t *)(this->queues); - for (int i = 0; i < this->num_of_queues; i++) { - gpuErrchk(cudaStreamSynchronize(ptr[i])); - } -} - -template Handle::~Handle() { - cudaSetDeviceHelper(dev_id); - if (initialized) { - destroy(); - } - if (auto_tuning_table_created) { - destroy_auto_tuning_table(); - } - destroy_queues(); -} - -template class Handle<1, double>; -template class Handle<1, float>; -template class Handle<2, double>; -template class Handle<2, float>; -template class Handle<3, double>; -template class Handle<3, float>; -template class Handle<4, double>; -template class Handle<4, float>; -template class Handle<5, double>; -template class Handle<5, float>; - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/IterativeProcessingKernels/SolveTridiag1.cu b/src/cuda/IterativeProcessingKernels/SolveTridiag1.cu deleted file mode 100644 index 5899251532..0000000000 --- a/src/cuda/IterativeProcessingKernels/SolveTridiag1.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/IterativeProcessingKernel.h" -#include "cuda/IterativeProcessingKernel.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void ipk_1( \ - Handle & handle, SIZE * shape_h, SIZE * shape_c_h, SIZE * shape_d, \ - SIZE * shape_c_d, SIZE * ldvs, SIZE * ldws, DIM processed_n, \ - DIM * processed_dims_h, DIM * processed_dims_d, DIM curr_dim_r, \ - DIM curr_dim_c, DIM curr_dim_f, T * am, T * bm, T * ddist_f, T * dv, \ - LENGTH lddv1, LENGTH lddv2, int queue_idx, int config); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/IterativeProcessingKernels/SolveTridiag1_3D.cu b/src/cuda/IterativeProcessingKernels/SolveTridiag1_3D.cu deleted file mode 100644 index 583fa2a257..0000000000 --- a/src/cuda/IterativeProcessingKernels/SolveTridiag1_3D.cu +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/IterativeProcessingKernel3D.h" -#include "cuda/IterativeProcessingKernel3D.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void ipk_1_3d( \ - Handle & handle, SIZE nr, SIZE nc, SIZE nf_c, T * am, T * bm, \ - T * ddist_f, T * dv, SIZE lddv1, SIZE lddv2, int queue_idx, int config); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/IterativeProcessingKernels/SolveTridiag2.cu b/src/cuda/IterativeProcessingKernels/SolveTridiag2.cu deleted file mode 100644 index c749c7dea1..0000000000 --- a/src/cuda/IterativeProcessingKernels/SolveTridiag2.cu +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ -#include "cuda/CommonInternal.h" - -#include "cuda/IterativeProcessingKernel.h" -#include "cuda/IterativeProcessingKernel.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void ipk_2( \ - Handle & handle, SIZE * shape_h, SIZE * shape_c_h, SIZE * shape_d, \ - SIZE * shape_c_d, SIZE * ldvs, SIZE * ldws, DIM processed_n, \ - DIM * processed_dims_h, DIM * processed_dims_d, DIM curr_dim_r, \ - DIM curr_dim_c, DIM curr_dim_f, T * am, T * bm, T * ddist_c, T * dv, \ - LENGTH lddv1, LENGTH lddv2, int queue_idx, int config); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/IterativeProcessingKernels/SolveTridiag2_3D.cu b/src/cuda/IterativeProcessingKernels/SolveTridiag2_3D.cu deleted file mode 100644 index 4fa46f632e..0000000000 --- a/src/cuda/IterativeProcessingKernels/SolveTridiag2_3D.cu +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ -#include "cuda/CommonInternal.h" - -#include "cuda/IterativeProcessingKernel3D.h" -#include "cuda/IterativeProcessingKernel3D.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void ipk_2_3d( \ - Handle & handle, SIZE nr, SIZE nc_c, SIZE nf_c, T * am, T * bm, \ - T * ddist_c, T * dv, SIZE lddv1, SIZE lddv2, int queue_idx, int config); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/IterativeProcessingKernels/SolveTridiag3.cu b/src/cuda/IterativeProcessingKernels/SolveTridiag3.cu deleted file mode 100644 index bc8db526c0..0000000000 --- a/src/cuda/IterativeProcessingKernels/SolveTridiag3.cu +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ -#include "cuda/CommonInternal.h" - -#include "cuda/IterativeProcessingKernel.h" -#include "cuda/IterativeProcessingKernel.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void ipk_3( \ - Handle & handle, SIZE * shape_h, SIZE * shape_c_h, SIZE * shape_d, \ - SIZE * shape_c_d, SIZE * ldvs, SIZE * ldws, DIM processed_n, \ - DIM * processed_dims_h, DIM * processed_dims_d, DIM curr_dim_r, \ - DIM curr_dim_c, DIM curr_dim_f, T * am, T * bm, T * ddist_r, T * dv, \ - LENGTH lddv1, LENGTH lddv2, int queue_idx, int config); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/IterativeProcessingKernels/SolveTridiag3_3D.cu b/src/cuda/IterativeProcessingKernels/SolveTridiag3_3D.cu deleted file mode 100644 index 3f95fffc97..0000000000 --- a/src/cuda/IterativeProcessingKernels/SolveTridiag3_3D.cu +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ -#include "cuda/CommonInternal.h" - -#include "cuda/IterativeProcessingKernel3D.h" -#include "cuda/IterativeProcessingKernel3D.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void ipk_3_3d( \ - Handle & handle, SIZE nr_c, SIZE nc_c, SIZE nf_c, T * am, T * bm, \ - T * ddist_r, T * dv, SIZE lddv1, SIZE lddv2, int queue_idx, int config); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/LevelwiseProcessingKernels/AddLevel.cu b/src/cuda/LevelwiseProcessingKernels/AddLevel.cu deleted file mode 100644 index 5163e3f1a6..0000000000 --- a/src/cuda/LevelwiseProcessingKernels/AddLevel.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/LevelwiseProcessingKernel.h" -#include "cuda/LevelwiseProcessingKernel.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void lwpk(Handle & handle, SIZE * shape_h, \ - SIZE * shape_d, T * dv, SIZE * ldvs, \ - T * dwork, SIZE * ldws, int queue_idx); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/LevelwiseProcessingKernels/CopyLevel.cu b/src/cuda/LevelwiseProcessingKernels/CopyLevel.cu deleted file mode 100644 index fe8807e629..0000000000 --- a/src/cuda/LevelwiseProcessingKernels/CopyLevel.cu +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/LevelwiseProcessingKernel.h" -#include "cuda/LevelwiseProcessingKernel.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void lwpk(Handle & handle, SIZE * shape_h, \ - SIZE * shape_d, T * dv, SIZE * ldvs, \ - T * dwork, SIZE * ldws, int queue_idx); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) - -#undef KERNELS -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/LevelwiseProcessingKernels/SubtractLevel.cu b/src/cuda/LevelwiseProcessingKernels/SubtractLevel.cu deleted file mode 100644 index b32c32e5d7..0000000000 --- a/src/cuda/LevelwiseProcessingKernels/SubtractLevel.cu +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/LevelwiseProcessingKernel.h" -#include "cuda/LevelwiseProcessingKernel.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void lwpk(Handle & handle, SIZE * shape_h, \ - SIZE * shape_d, T * dv, SIZE * ldvs, \ - T * dwork, SIZE * ldws, int queue_idx); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/LinearProcessingKernels/MassTrans1.cu b/src/cuda/LinearProcessingKernels/MassTrans1.cu deleted file mode 100644 index 05dcd3d941..0000000000 --- a/src/cuda/LinearProcessingKernels/MassTrans1.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/LinearProcessingKernel.h" -#include "cuda/LinearProcessingKernel.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void lpk_reo_1( \ - Handle & handle, SIZE * shape_h, SIZE * shape_c_h, SIZE * shape_d, \ - SIZE * shape_c_d, SIZE * ldvs, SIZE * ldws, DIM processed_n, \ - DIM * processed_dims_h, DIM * processed_dims_d, DIM curr_dim_r, \ - DIM curr_dim_c, DIM curr_dim_f, T * ddist_f, T * dratio_f, T * dv1, \ - LENGTH lddv11, LENGTH lddv12, T * dv2, LENGTH lddv21, LENGTH lddv22, \ - T * dw, LENGTH lddw1, LENGTH lddw2, int queue_idx, int config); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/LinearProcessingKernels/MassTrans1_3D.cu b/src/cuda/LinearProcessingKernels/MassTrans1_3D.cu deleted file mode 100644 index 1b80ae65e3..0000000000 --- a/src/cuda/LinearProcessingKernels/MassTrans1_3D.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/LinearProcessingKernel3D.h" -#include "cuda/LinearProcessingKernel3D.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void lpk_reo_1_3d( \ - Handle & handle, SIZE nr, SIZE nc, SIZE nf, SIZE nf_c, \ - SIZE zero_r, SIZE zero_c, SIZE zero_f, T * ddist_f, T * dratio_f, \ - T * dv1, SIZE lddv11, SIZE lddv12, T * dv2, SIZE lddv21, SIZE lddv22, \ - T * dw, SIZE lddw1, SIZE lddw2, int queue_idx, int config); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/LinearProcessingKernels/MassTrans2.cu b/src/cuda/LinearProcessingKernels/MassTrans2.cu deleted file mode 100644 index e22cd88f86..0000000000 --- a/src/cuda/LinearProcessingKernels/MassTrans2.cu +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ -#include "cuda/CommonInternal.h" - -#include "cuda/LinearProcessingKernel.h" -#include "cuda/LinearProcessingKernel.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void lpk_reo_2( \ - Handle & handle, SIZE * shape_h, SIZE * shape_c_h, SIZE * shape_d, \ - SIZE * shape_c_d, SIZE * ldvs, SIZE * ldws, DIM processed_n, \ - DIM * processed_dims_h, DIM * processed_dims_d, DIM curr_dim_r, \ - DIM curr_dim_c, DIM curr_dim_f, T * ddist_c, T * dratio_c, T * dv1, \ - LENGTH lddv11, LENGTH lddv12, T * dv2, LENGTH lddv21, LENGTH lddv22, \ - T * dw, LENGTH lddw1, LENGTH lddw2, int queue_idx, int config); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/LinearProcessingKernels/MassTrans2_3D.cu b/src/cuda/LinearProcessingKernels/MassTrans2_3D.cu deleted file mode 100644 index f735fcfa9e..0000000000 --- a/src/cuda/LinearProcessingKernels/MassTrans2_3D.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/LinearProcessingKernel3D.h" -#include "cuda/LinearProcessingKernel3D.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void lpk_reo_2_3d( \ - Handle & handle, SIZE nr, SIZE nc, SIZE nf_c, SIZE nc_c, \ - T * ddist_c, T * dratio_c, T * dv1, SIZE lddv11, SIZE lddv12, T * dv2, \ - SIZE lddv21, SIZE lddv22, T * dw, SIZE lddw1, SIZE lddw2, int queue_idx, \ - int config); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/LinearProcessingKernels/MassTrans3.cu b/src/cuda/LinearProcessingKernels/MassTrans3.cu deleted file mode 100644 index 2c8e7718fb..0000000000 --- a/src/cuda/LinearProcessingKernels/MassTrans3.cu +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/LinearProcessingKernel.h" -#include "cuda/LinearProcessingKernel.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void lpk_reo_3( \ - Handle & handle, SIZE * shape_h, SIZE * shape_c_h, SIZE * shape_d, \ - SIZE * shape_c_d, SIZE * ldvs, SIZE * ldws, DIM processed_n, \ - DIM * processed_dims_h, DIM * processed_dims_d, DIM curr_dim_r, \ - DIM curr_dim_c, DIM curr_dim_f, T * ddist_r, T * dratio_r, T * dv1, \ - LENGTH lddv11, LENGTH lddv12, T * dv2, LENGTH lddv21, LENGTH lddv22, \ - T * dw, LENGTH lddw1, LENGTH lddw2, int queue_idx, int config); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/LinearProcessingKernels/MassTrans3_3D.cu b/src/cuda/LinearProcessingKernels/MassTrans3_3D.cu deleted file mode 100644 index 16b86aab77..0000000000 --- a/src/cuda/LinearProcessingKernels/MassTrans3_3D.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/LinearProcessingKernel3D.h" -#include "cuda/LinearProcessingKernel3D.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void lpk_reo_3_3d( \ - Handle & handle, SIZE nr, SIZE nc_c, SIZE nf_c, SIZE nr_c, \ - T * ddist_r, T * dratio_r, T * dv1, SIZE lddv11, SIZE lddv12, T * dv2, \ - SIZE lddv21, SIZE lddv22, T * dw, SIZE lddw1, SIZE lddw2, int queue_idx, \ - int config); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/LinearQuantization/LevelwiseLinearDequantization.cu b/src/cuda/LinearQuantization/LevelwiseLinearDequantization.cu deleted file mode 100644 index 5557a6843b..0000000000 --- a/src/cuda/LinearQuantization/LevelwiseLinearDequantization.cu +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/LinearQuantization.h" -#include "cuda/LinearQuantization.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void levelwise_linear_dequantize( \ - Handle & handle, SIZE * shapes, SIZE l_target, T * volumes, \ - SIZE ldvolumes, Metadata & m, QUANTIZED_INT * dv, SIZE * ldvs, \ - T * dwork, SIZE * ldws, bool prep_huffman, LENGTH outlier_count, \ - LENGTH *outlier_idx, QUANTIZED_INT *outliers, int queue_idx); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/LinearQuantization/LevelwiseLinearQuantization.cu b/src/cuda/LinearQuantization/LevelwiseLinearQuantization.cu deleted file mode 100644 index dc06a5e285..0000000000 --- a/src/cuda/LinearQuantization/LevelwiseLinearQuantization.cu +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" - -#include "cuda/LinearQuantization.h" -#include "cuda/LinearQuantization.hpp" - -namespace mgard_cuda { - -#define KERNELS(D, T) \ - template void levelwise_linear_quantize( \ - Handle & handle, SIZE * shapes, SIZE l_target, T * volumes, \ - SIZE ldvolumes, Metadata & m, T * dv, SIZE * ldvs,\ - int *dwork, \ - SIZE *ldws,\ - bool prep_huffmam, \ - SIZE *shape, LENGTH *outlier_count, LENGTH *outlier_idx, \ - QUANTIZED_INT *outliers, int queue_idx); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/LosslessCompression.cu b/src/cuda/LosslessCompression.cu deleted file mode 100644 index feb61ab2d4..0000000000 --- a/src/cuda/LosslessCompression.cu +++ /dev/null @@ -1,679 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: September 27, 2021 - */ - -// #include "compressors.hpp" -#include "cuda/Common.h" -#include "cuda/CommonInternal.h" -#include "cuda/LosslessCompression.h" -#include "cuda/ParallelHuffman/huffman_workflow.cuh" -#include -#include - -namespace mgard { -void huffman_encoding(long int *quantized_data, const std::size_t n, - unsigned char **out_data_hit, size_t *out_data_hit_size, - unsigned char **out_data_miss, size_t *out_data_miss_size, - unsigned char **out_tree, size_t *out_tree_size); -void huffman_decoding(long int *quantized_data, - const std::size_t quantized_data_size, - unsigned char *out_data_hit, size_t out_data_hit_size, - unsigned char *out_data_miss, size_t out_data_miss_size, - unsigned char *out_tree, size_t out_tree_size); -} // namespace mgard - -namespace mgard_cuda { - -/*! CHECK - * Check that the condition holds. If it doesn't print a message and die. - */ -#define CHECK(cond, ...) \ - do { \ - if (!(cond)) { \ - fprintf(stderr, "%s:%d CHECK(%s) failed: ", __FILE__, __LINE__, #cond); \ - fprintf(stderr, "" __VA_ARGS__); \ - fprintf(stderr, "\n"); \ - exit(1); \ - } \ - } while (0) - -/*! CHECK_ZSTD - * Check the zstd error code and die if an error occurred after printing a - * message. - */ -/*! CHECK_ZSTD - * Check the zstd error code and die if an error occurred after printing a - * message. - */ -#define CHECK_ZSTD(fn, ...) \ - do { \ - size_t const err = (fn); \ - CHECK(!ZSTD_isError(err), "%s", ZSTD_getErrorName(err)); \ - } while (0) - -unsigned char *compress_memory_huffman(long int *const src, - const std::size_t srcLen, - std::size_t outsize) { - unsigned char *out_data_hit = 0; - size_t out_data_hit_size; - unsigned char *out_data_miss = 0; - size_t out_data_miss_size; - unsigned char *out_tree = 0; - size_t out_tree_size; - mgard::huffman_encoding(src, srcLen, &out_data_hit, &out_data_hit_size, - &out_data_miss, &out_data_miss_size, &out_tree, - &out_tree_size); - - const size_t total_size = - out_data_hit_size / 8 + 4 + out_data_miss_size + out_tree_size; - unsigned char *payload = (unsigned char *)malloc(total_size); - unsigned char *bufp = payload; - - if (out_tree_size) { - std::memcpy(bufp, out_tree, out_tree_size); - bufp += out_tree_size; - } - - std::memcpy(bufp, out_data_hit, out_data_hit_size / 8 + 4); - bufp += out_data_hit_size / 8 + 4; - - if (out_data_miss_size) { - std::memcpy(bufp, out_data_miss, out_data_miss_size); - bufp += out_data_miss_size; - } - - free(out_tree); - free(out_data_hit); - free(out_data_miss); - - // const MemoryBuffer out_data = - // compress_memory_zstd(payload, total_size); - - const size_t cBuffSize = ZSTD_compressBound(total_size); - unsigned char *const zstd_buffer = new unsigned char[cBuffSize]; - const std::size_t cSize = - ZSTD_compress(zstd_buffer, cBuffSize, payload, total_size, 1); - CHECK_ZSTD(cSize); - // return MemoryBuffer(buffer, cSize); - - free(payload); - payload = 0; - - const std::size_t bufferLen = 3 * sizeof(size_t) + cSize; - unsigned char *const buffer = new unsigned char[bufferLen]; - outsize = bufferLen; - - bufp = buffer; - *(size_t *)bufp = out_tree_size; - bufp += sizeof(size_t); - - *(size_t *)bufp = out_data_hit_size; - bufp += sizeof(size_t); - - *(size_t *)bufp = out_data_miss_size; - bufp += sizeof(size_t); - - { - unsigned char const *const p = zstd_buffer; - std::copy(p, p + cSize, bufp); - } - // return MemoryBuffer(buffer, bufferLen); - return buffer; -} - -void decompress_memory_huffman(unsigned char *const src, - const std::size_t srcLen, long int *const dst, - const std::size_t dstLen) { - unsigned char *out_data_hit = 0; - size_t out_data_hit_size; - unsigned char *out_data_miss = 0; - size_t out_data_miss_size; - unsigned char *out_tree = 0; - size_t out_tree_size; - - unsigned char *buf = src; - - out_tree_size = *(size_t *)buf; - buf += sizeof(size_t); - - out_data_hit_size = *(size_t *)buf; - buf += sizeof(size_t); - - out_data_miss_size = *(size_t *)buf; - buf += sizeof(size_t); - size_t total_huffman_size = - out_tree_size + out_data_hit_size / 8 + 4 + out_data_miss_size; - unsigned char *huffman_encoding_p = - (unsigned char *)malloc(total_huffman_size); - // decompress_memory_zstd(buf, srcLen - 3 * sizeof(size_t), - // huffman_encoding_p, - // total_huffman_size); - - size_t const dSize = ZSTD_decompress(huffman_encoding_p, total_huffman_size, - buf, srcLen - 3 * sizeof(size_t)); - CHECK_ZSTD(dSize); - - /* When zstd knows the content size, it will error if it doesn't match. */ - CHECK(dstLen == dSize, "Impossible because zstd will check this condition!"); - - out_tree = huffman_encoding_p; - out_data_hit = huffman_encoding_p + out_tree_size; - out_data_miss = - huffman_encoding_p + out_tree_size + out_data_hit_size / 8 + 4; - - mgard::huffman_decoding(dst, dstLen, out_data_hit, out_data_hit_size, - out_data_miss, out_data_miss_size, out_tree, - out_tree_size); - - free(huffman_encoding_p); -} - -template -void cascaded_compress(Handle &handle, C *input_data, size_t input_count, - void *&output_data, size_t &output_size, int n_rle, - int n_de, bool bitpack, int queue_idx) { - - // nvcomp::CascadedCompressor compressor(nvcomp::TypeOf(), n_rle, n_de, - // bitpack); - - nvcompBatchedCascadedOpts_t options = nvcompBatchedCascadedDefaultOpts; - options.type = nvcomp::TypeOf(); - options.num_RLEs = n_rle; - options.num_deltas = n_de; - options.use_bp = bitpack; - nvcomp::CascadedManager nvcomp_manager{ - options, *(cudaStream_t *)handle.get(queue_idx)}; - - // size_t *temp_bytes; - // cudaMallocHostHelper((void **)&temp_bytes, sizeof(size_t)); - // size_t *output_bytes; - // cudaMallocHostHelper((void **)&output_bytes, sizeof(size_t)); - - // compressor.configure(input_count * sizeof(C), temp_bytes, output_bytes); - auto comp_config = - nvcomp_manager.configure_compression(input_count * sizeof(C)); - - // void *temp_space; - // cudaMallocHelper(handle, &temp_space, *temp_bytes); - // cudaMallocHelper(handle, &output_data, *output_bytes); - - cudaMalloc(&output_data, comp_config.max_compressed_buffer_size); - - // compressor.compress_async(input_data, input_count * sizeof(C), temp_space, - // *temp_bytes, output_data, output_bytes, - // *(cudaStream_t *)handle.get(queue_idx)); - - uint8_t *output_data_uint8_t = (uint8_t *)output_data; - nvcomp_manager.compress((uint8_t *)input_data, output_data_uint8_t, - comp_config); - output_size = nvcomp_manager.get_compressed_output_size(output_data_uint8_t); - handle.sync(queue_idx); - - // cudaFreeHelper(temp_space); - // cudaFreeHostHelper(temp_bytes); - // cudaFreeHostHelper(output_bytes); -} - -template -void cascaded_decompress(Handle &handle, void *input_data, - size_t input_size, C *&output_data, int queue_idx) { - - // nvcomp::Decompressor decompressor(input_data, input_size, - // *(cudaStream_t - // *)handle.get(queue_idx)); - - // nvcomp::CascadedDecompressor decompressor; - auto decomp_nvcomp_manager = nvcomp::create_manager( - (uint8_t *)input_data, *(cudaStream_t *)handle.get(queue_idx)); - - // size_t *temp_bytes; - // cudaMallocHostHelper((void **)&temp_bytes, sizeof(size_t)); - // size_t *output_bytes; - // cudaMallocHostHelper((void **)&output_bytes, sizeof(size_t)); - - // decompressor.configure(input_data, input_size, temp_bytes, output_bytes, - // *(cudaStream_t *)handle.get(queue_idx)); - nvcomp::DecompressionConfig decomp_config = - decomp_nvcomp_manager->configure_decompression((uint8_t *)input_data); - - // void *temp_space; - // cudaMallocHelper(handle, (void **)&temp_space, *temp_bytes); - // cudaMallocHelper(handle, (void **)&output_data, *output_bytes); - cudaMalloc(&output_data, decomp_config.decomp_data_size); - - uint8_t *output_data_uint8_t = (uint8_t *)output_data; - decomp_nvcomp_manager->decompress(output_data_uint8_t, (uint8_t *)input_data, - decomp_config); - // output_size = decomp_config.decomp_data_size; - handle.sync(queue_idx); - // decompressor.decompress_async(input_data, input_size, temp_space, - // *temp_bytes, - // output_data, *output_bytes, - // *(cudaStream_t *)handle.get(queue_idx)); - // handle.sync(queue_idx); - // cudaFreeHelper(temp_space); - // cudaFreeHostHelper(temp_bytes); - // cudaFreeHostHelper(output_bytes); -} - -template -void lz4_compress(Handle &handle, C *input_data, size_t input_count, - void *&output_data, size_t &output_size, size_t chunk_size, - int queue_idx) { - nvcompType_t dtype = NVCOMP_TYPE_UCHAR; - // nvcomp::LZ4Compressor compressor(chunk_size, dtype); - nvcomp::LZ4Manager nvcomp_manager{chunk_size, dtype, - *(cudaStream_t *)handle.get(queue_idx)}; - - // size_t *temp_bytes; - // cudaMallocHostHelper((void **)&temp_bytes, sizeof(size_t)); - // size_t *output_bytes; - // cudaMallocHostHelper((void **)&output_bytes, sizeof(size_t)); - - // compressor.configure(input_count * sizeof(C), temp_bytes, output_bytes); - nvcomp::CompressionConfig comp_config = - nvcomp_manager.configure_compression(input_count * sizeof(C)); - - // void *temp_space; - // cudaMallocHelper(handle, &temp_space, *temp_bytes); - cudaMallocHelper(handle, &output_data, - comp_config.max_compressed_buffer_size); - - // compressor.compress_async(input_data, input_count * sizeof(C), temp_space, - // *temp_bytes, output_data, output_bytes, - // *(cudaStream_t *)handle.get(queue_idx)); - uint8_t *output_data_uint8_t = (uint8_t *)output_data; - nvcomp_manager.compress((uint8_t *)input_data, output_data_uint8_t, - comp_config); - output_size = nvcomp_manager.get_compressed_output_size(output_data_uint8_t); - - handle.sync(queue_idx); - // output_size = *output_bytes; - // cudaFreeHelper(temp_space); - // cudaFreeHostHelper(temp_bytes); - // cudaFreeHostHelper(output_bytes); -} - -template -void lz4_decompress(Handle &handle, void *input_data, size_t input_size, - C *&output_data, size_t &output_size, int queue_idx) { - - auto decomp_nvcomp_manager = nvcomp::create_manager( - (uint8_t *)input_data, *(cudaStream_t *)handle.get(queue_idx)); - - // size_t *temp_bytes; - // cudaMallocHostHelper((void **)&temp_bytes, sizeof(size_t)); - // size_t *output_bytes; - // cudaMallocHostHelper((void **)&output_bytes, sizeof(size_t)); - - // decompressor.configure(input_data, input_size, temp_bytes, output_bytes, - // *(cudaStream_t *)handle.get(queue_idx)); - nvcomp::DecompressionConfig decomp_config = - decomp_nvcomp_manager->configure_decompression((uint8_t *)input_data); - - // void *temp_space; - // cudaMallocHelper(handle, (void **)&temp_space, *temp_bytes); - // cudaMallocHelper(handle, (void **)&output_data, *output_bytes); - - cudaMalloc(&output_data, decomp_config.decomp_data_size); - - // decompressor.decompress_async(input_data, input_size, temp_space, - // *temp_bytes, - // output_data, *output_bytes, - // *(cudaStream_t *)handle.get(queue_idx)); - - uint8_t *output_data_uint8_t = (uint8_t *)output_data; - decomp_nvcomp_manager->decompress(output_data_uint8_t, (uint8_t *)input_data, - decomp_config); - output_size = decomp_config.decomp_data_size; - handle.sync(queue_idx); - - // cudaFreeHelper(temp_space); - // cudaFreeHostHelper(temp_bytes); - // cudaFreeHostHelper(output_bytes); -} - -#define KERNELS(D, T, C) \ - template void cascaded_compress( \ - Handle & handle, C * input_data, size_t input_count, \ - void *&output_data, size_t &output_size, int n_rle, int n_de, \ - bool bitpack, int queue_idx); \ - template void cascaded_decompress( \ - Handle & handle, void *input_data, size_t input_size, \ - C *&output_data, int queue_idx); \ - template void lz4_compress(Handle & handle, C * input_data, \ - size_t input_count, void *&output_data, \ - size_t &output_size, size_t chunk_size, \ - int queue_idx); \ - template void lz4_decompress( \ - Handle & handle, void *input_data, size_t input_size, \ - C *&output_data, size_t &output_count, int queue_idx); - -KERNELS(1, double, uint8_t) -KERNELS(1, float, uint8_t) -KERNELS(2, double, uint8_t) -KERNELS(2, float, uint8_t) -KERNELS(3, double, uint8_t) -KERNELS(3, float, uint8_t) -KERNELS(4, double, uint8_t) -KERNELS(4, float, uint8_t) -KERNELS(5, double, uint8_t) -KERNELS(5, float, uint8_t) -KERNELS(1, double, uint32_t) -KERNELS(1, float, uint32_t) -KERNELS(2, double, uint32_t) -KERNELS(2, float, uint32_t) -KERNELS(3, double, uint32_t) -KERNELS(3, float, uint32_t) -KERNELS(4, double, uint32_t) -KERNELS(4, float, uint32_t) -KERNELS(5, double, uint32_t) -KERNELS(5, float, uint32_t) -KERNELS(1, double, uint64_t) -KERNELS(1, float, uint64_t) -KERNELS(2, double, uint64_t) -KERNELS(2, float, uint64_t) -KERNELS(3, double, uint64_t) -KERNELS(3, float, uint64_t) -KERNELS(4, double, uint64_t) -KERNELS(4, float, uint64_t) -KERNELS(5, double, uint64_t) -KERNELS(5, float, uint64_t) -#undef KERNELS - -template -void SeparateOutlierAndPrimary(Handle &handle, S *dqv, size_t n, - size_t *outlier_idx, size_t outlier_count, - size_t primary_count, S *doutlier, Q *dprimary, - int queue_idx) { - - // printf("compress outlier_idx: "); for(int i = 0; i < outlier_count; i++) - // {printf("%llu ", outlier_idx[i]);} printf("\n"); - printf("compress outlier_count: %llu\n", outlier_count); - printf("compress primary_count: %llu\n", primary_count); - printf("start separating primary and outlier\n"); - - size_t p = 0; - size_t pp = 0; - size_t op = 0; - size_t size = outlier_idx[0] - 0; - // printf("copy primary\n"); - if (size > 0) { - mgard_cuda::cudaMemcpyAsyncHelper(handle, dprimary + pp, dqv + p, - size * sizeof(Q), mgard_cuda::D2D, - queue_idx); - } - pp += size; - p += size; - - for (int i = 0; i < outlier_count - 1; i++) { - size = 1; - // printf("copy outlier\n"); - mgard_cuda::cudaMemcpyAsyncHelper(handle, doutlier + op, dqv + p, - size * sizeof(S), mgard_cuda::D2D, - queue_idx); - op += size; - p += size; - size = outlier_idx[i + 1] - outlier_idx[i] - 1; - // printf("copy primary %d %d %d\n", p, size, outlier_idx[outlier_idx.size() - // - 1]); - if (size > 0) { - mgard_cuda::cudaMemcpyAsyncHelper(handle, dprimary + pp, dqv + p, - size * sizeof(Q), mgard_cuda::D2D, - queue_idx); - } - pp += size; - p += size; - } - size = 1; - // printf("copy outlier\n"); - mgard_cuda::cudaMemcpyAsyncHelper(handle, doutlier + op, dqv + p, - size * sizeof(S), mgard_cuda::D2D, - queue_idx); - op += size; - p += size; - size = n - outlier_idx[outlier_count - 1] - 1; - // printf("copy primary %d %d %d\n", p, size, outlier_idx[outlier_idx.size() - - // 1]); - if (size > 0) { - mgard_cuda::cudaMemcpyAsyncHelper(handle, dprimary + pp, dqv + p, - size * sizeof(Q), mgard_cuda::D2D, - queue_idx); - } - // printf("done copy primary\n"); - pp += size; - p += size; - - if (pp != primary_count || op != outlier_count) { - printf("Primary or outlier size mismatch!\n"); - } - printf("done separating primary and outlier\n"); -} - -template -void CombineOutlierAndPrimary(Handle &handle, S *dqv, size_t n, - size_t *outlier_idx, size_t outlier_count, - size_t primary_count, S *doutlier, Q *dprimary, - int queue_idx) { - size_t p = 0; - size_t pp = 0; - size_t op = 0; - size_t size = outlier_idx[0] - 0; - // printf("copy primary\n"); - if (size > 0) { - mgard_cuda::cudaMemcpyAsyncHelper(handle, dqv + p, dprimary + pp, - size * sizeof(Q), mgard_cuda::D2D, - queue_idx); - } - pp += size; - p += size; - - for (int i = 0; i < outlier_count - 1; i++) { - size = 1; - // printf("copy outlier\n"); - mgard_cuda::cudaMemcpyAsyncHelper(handle, dqv + p, doutlier + op, - size * sizeof(S), mgard_cuda::D2D, - queue_idx); - op += size; - p += size; - size = outlier_idx[i + 1] - outlier_idx[i] - 1; - // printf("copy primary %d %d %d\n", p, size, outlier_idx[outlier_idx.size() - // - 1]); - if (size > 0) { - mgard_cuda::cudaMemcpyAsyncHelper(handle, dqv + p, dprimary + pp, - size * sizeof(Q), mgard_cuda::D2D, - queue_idx); - } - pp += size; - p += size; - } - size = 1; - // printf("copy outlier\n"); - mgard_cuda::cudaMemcpyAsyncHelper(handle, dqv + p, doutlier + op, - size * sizeof(S), mgard_cuda::D2D, - queue_idx); - op += size; - p += size; - size = n - outlier_idx[outlier_count - 1] - 1; - // printf("copy primary %d %d %d\n", p, size, outlier_idx[outlier_idx.size() - - // 1]); - if (size > 0) { - mgard_cuda::cudaMemcpyAsyncHelper(handle, dqv + p, dprimary + pp, - size * sizeof(Q), mgard_cuda::D2D, - queue_idx); - } - // printf("done copy primary\n"); - pp += size; - p += size; -} - -#define KERNELS(D, T, S, Q) \ - template void SeparateOutlierAndPrimary( \ - Handle & handle, S * dqv, size_t n, size_t * outlier_idx,\ - size_t outlier_count, \ - size_t primary_count,\ - S * doutlier, \ - Q * dprimary, int queue_idx); \ - template void CombineOutlierAndPrimary( \ - Handle & handle, S * dqv, size_t n, size_t * outlier_idx,\ - size_t outlier_count, \ - size_t primary_count,\ - S * doutlier, \ - Q * dprimary, int queue_idx); - -KERNELS(1, double, int, uint32_t) -KERNELS(1, float, int, uint32_t) -KERNELS(2, double, int, uint32_t) -KERNELS(2, float, int, uint32_t) -KERNELS(3, double, int, uint32_t) -KERNELS(3, float, int, uint32_t) -KERNELS(4, double, int, uint32_t) -KERNELS(4, float, int, uint32_t) -KERNELS(5, double, int, uint32_t) -KERNELS(5, float, int, uint32_t) -#undef KERNELS - -template -void huffman_compress(Handle &handle, S *input_data, size_t input_count, - std::vector &outlier_idx, H *&out_meta, - size_t &out_meta_size, H *&out_data, - size_t &out_data_size, int chunk_size, int dict_size, - int queue_idx) { - - HuffmanEncode(handle, input_data, input_count, outlier_idx, - out_meta, out_meta_size, out_data, out_data_size, - chunk_size, dict_size); -} - -template -void huffman_decompress(Handle &handle, H *in_meta, size_t in_meta_size, - H *in_data, size_t in_data_size, S *&output_data, - size_t &output_count, int queue_idx) { - HuffmanDecode(handle, output_data, output_count, in_meta, - in_meta_size, in_data, in_data_size); -} - -#define KERNELS(D, T, S, Q, H) \ - template void huffman_compress( \ - Handle & handle, S * input_data, size_t input_count, \ - std::vector & outlier_idx, H * &out_meta, \ - size_t & out_meta_size, H * &out_data, size_t & out_data_size, \ - int chunk_size, int dict_size, int queue_idx); \ - template void huffman_decompress( \ - Handle & handle, H * in_meta, size_t in_meta_size, H * in_data, \ - size_t in_data_size, S * &output_data, size_t & output_count, \ - int queue_idx); - -KERNELS(1, double, int, uint32_t, uint32_t) -KERNELS(1, float, int, uint32_t, uint32_t) -KERNELS(2, double, int, uint32_t, uint32_t) -KERNELS(2, float, int, uint32_t, uint32_t) -KERNELS(3, double, int, uint32_t, uint32_t) -KERNELS(3, float, int, uint32_t, uint32_t) -KERNELS(4, double, int, uint32_t, uint32_t) -KERNELS(4, float, int, uint32_t, uint32_t) -KERNELS(5, double, int, uint32_t, uint32_t) -KERNELS(5, float, int, uint32_t, uint32_t) -KERNELS(1, double, int, uint32_t, uint64_t) -KERNELS(1, float, int, uint32_t, uint64_t) -KERNELS(2, double, int, uint32_t, uint64_t) -KERNELS(2, float, int, uint32_t, uint64_t) -KERNELS(3, double, int, uint32_t, uint64_t) -KERNELS(3, float, int, uint32_t, uint64_t) -KERNELS(4, double, int, uint32_t, uint64_t) -KERNELS(4, float, int, uint32_t, uint64_t) -KERNELS(5, double, int, uint32_t, uint64_t) -KERNELS(5, float, int, uint32_t, uint64_t) - -template -void cpu_lossless_compression(Handle &handle, S *input_data, - size_t input_count, H *&out_data, - size_t &out_data_size) { - - int *int_vector = new int[input_count]; - - cudaMemcpyAsyncHelper(handle, int_vector, input_data, input_count * sizeof(S), - AUTO, 0); - handle.sync(0); - - std::vector input_vector(input_count); - for (int i = 0; i < input_count; i++) - input_vector[i] = int_vector[i]; - - // printf("%u %u\n", sizeof(long int), sizeof(int)); - // printf("dqv\n"); - // print_matrix_cuda(1, input_count, input_data, input_count); - - // printf("input_vector: "); - // for (int i = 0; i < input_vector.size(); i++) printf("%d ", - // input_vector[i]); printf("\n"); Compress an array of data using `zstd`. - std::size_t zstd_outsize; - - unsigned char *buffer = compress_memory_huffman( - input_vector.data(), input_vector.size() * sizeof(long int), - zstd_outsize); - - out_data_size = zstd_outsize; - - cudaMallocHelper(handle, (void **)&out_data, out_data_size); - cudaMemcpyAsyncHelper(handle, out_data, buffer, out_data_size, AUTO, 0); - handle.sync(0); - delete[] int_vector; -} - -template -void cpu_lossless_decompression(Handle &handle, H *input_data, - size_t input_count, S *&out_data, - size_t output_count) { - - // printf("cpu decompression: %llu\n", input_count); - std::vector input_vector(input_count); - cudaMemcpyAsyncHelper(handle, input_vector.data(), input_data, input_count, - AUTO, 0); - handle.sync(0); - // printf("copy done\n"); - - long int *output_vector = new long int[output_count]; - int *int_vector = new int[output_count]; - - decompress_memory_huffman( - reinterpret_cast(input_vector.data()), - input_vector.size(), output_vector, - output_count * sizeof(*output_vector)); - - for (int i = 0; i < output_count; i++) - int_vector[i] = output_vector[i]; - cudaMallocHelper(handle, (void **)&out_data, output_count * sizeof(S)); - cudaMemcpyAsyncHelper(handle, out_data, int_vector, output_count * sizeof(S), - AUTO, 0); - handle.sync(0); - delete[] output_vector; - delete[] int_vector; - - // printf("dqv\n"); - // print_matrix_cuda(1, output_count, out_data, output_count); -} - -#define KERNELS(D, T, S, H) \ - template void cpu_lossless_compression( \ - Handle & handle, S * input_data, size_t input_count, \ - H * &out_data, size_t & out_data_size); \ - template void cpu_lossless_decompression( \ - Handle & handle, H * input_data, size_t input_count, \ - S * &out_data, size_t output_count); - -KERNELS(1, double, int, unsigned char) -KERNELS(1, float, int, unsigned char) -KERNELS(2, double, int, unsigned char) -KERNELS(2, float, int, unsigned char) -KERNELS(3, double, int, unsigned char) -KERNELS(3, float, int, unsigned char) -KERNELS(4, double, int, unsigned char) -KERNELS(4, float, int, unsigned char) -KERNELS(5, double, int, unsigned char) -KERNELS(5, float, int, unsigned char) - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/MemoryManagement.cu b/src/cuda/MemoryManagement.cu deleted file mode 100644 index 6aa4d2ac44..0000000000 --- a/src/cuda/MemoryManagement.cu +++ /dev/null @@ -1,746 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: September 27, 2021 - */ - -#include -#include -#include -#include -#include -#include // std::stringstream -#include // std::runtime_error -#include -#include // std::pair -#include - -#include "cuda/CommonInternal.h" - -#include "cuda/Handle.h" -#include "cuda/MemoryManagement.h" - -#define ANSI_RED "\x1b[31m" -#define ANSI_GREEN "\x1b[32m" -#define ANSI_RESET "\x1b[0m" - -namespace mgard_cuda { - -enum endiness_type CheckEndianess() { - int i = 1; - char *p = (char *)&i; - if (p[0] == 1) { - return endiness_type::Little_Endian; - } else { - return endiness_type::Big_Endian; - } -} - -template -void PrintSubarray(std::string name, SubArrayType subArray) { - Handle<1, float> tmp_handle; - - SIZE nrow = 1; - SIZE ncol = 1; - SIZE nfib = 1; - - nfib = subArray.shape[0]; - if (SubArrayType::NumDims >= 2) - ncol = subArray.shape[1]; - if (SubArrayType::NumDims >= 3) - nrow = subArray.shape[2]; - - using T = typename SubArrayType::DataType; - T *v = new T[nrow * ncol * nfib]; - cudaMemcpy3DAsyncHelper(tmp_handle, v, nfib * sizeof(T), nfib * sizeof(T), - ncol, subArray.data(), subArray.lddv1 * sizeof(T), - nfib * sizeof(T), subArray.lddv2, nfib * sizeof(T), - ncol, nrow, D2H, 0); - tmp_handle.sync(0); - - std::cout << "SubArray: " << name << "(" << nrow << " * " << ncol << " * " - << nfib << ") sizeof(T) = " << sizeof(T) << std::endl; - for (int i = 0; i < nrow; i++) { - printf("[i = %d]\n", i); - for (int j = 0; j < ncol; j++) { - for (int k = 0; k < nfib; k++) { - if (std::is_same::value) { - std::cout << std::setw(8) - << (unsigned int)v[nfib * ncol * i + nfib * j + k] << ", "; - } else { - std::cout << std::setw(8) << std::setprecision(6) << std::fixed - << v[nfib * ncol * i + nfib * j + k] << ", "; - } - } - std::cout << std::endl; - } - std::cout << std::endl; - } - std::cout << std::endl; - delete[] v; -} - -// print 2D CPU -template void print_matrix(SIZE nrow, SIZE ncol, T *v, SIZE ldv) { - // std::cout << std::setw(10); - // std::cout << std::setprecision(2) << std::fixed; - // for (int i = 0; i < nrow; i++) { - // for (int j = 0; j < ncol; j++) { - // if (isnan(v[ldv * i + j])) { - // std::cout << "nan\n"; - // return; - // } - // if (isinf(v[ldv * i + j])) { - // std::cout << "inf\n"; - // return; - // } - // if (abs(v[ldv * i + j]) > 10000) { - // std::cout << "LARGE" << " (" << i << ", " << j << ")\n"; - // return; - // } - // // if ((int)(v[ldv * i + j]*10)%10 != 0) { - // // std::cout << "NOT INTEGER" << " (" << i << ", " << j << ")\n"; - // // return; - // // } - // } - // } - - for (int i = 0; i < nrow; i++) { - for (int j = 0; j < ncol; j++) { - std::cout << std::setw(8) << std::setprecision(6) << std::fixed - << v[ldv * i + j] << ", "; - } - std::cout << std::endl; - } - std::cout << std::endl; -} - -// print 2D GPU -template -void print_matrix_cuda(SIZE nrow, SIZE ncol, T *dv, SIZE lddv) { - // std::cout << std::setw(10); - // std::cout << std::setprecision(2) << std::fixed; - Handle<2, float> *tmp_handle = new Handle<2, float>(); - int queue_idx = 0; - T *v = new T[nrow * ncol]; - cudaMemcpy2DAsyncHelper(*tmp_handle, v, ncol * sizeof(T), dv, - lddv * sizeof(T), ncol * sizeof(T), nrow, D2H, - queue_idx); - tmp_handle->sync(queue_idx); - print_matrix(nrow, ncol, v, ncol); - delete[] v; - delete tmp_handle; -} - -// print 3D GPU -template -void print_matrix_cuda(SIZE nrow, SIZE ncol, SIZE nfib, T *dv, SIZE lddv1, - SIZE lddv2, SIZE sizex) { - std::cout << std::setw(10); - std::cout << std::setprecision(2) << std::fixed; - Handle<3, float> *tmp_handle = new Handle<3, float>(); - int queue_idx = 0; - gpuErrchk(cudaDeviceSynchronize()); - T *v = new T[nrow * ncol * nfib]; - cudaMemcpy3DAsyncHelper(*tmp_handle, v, nfib * sizeof(T), nfib * sizeof(T), - ncol, dv, lddv1 * sizeof(T), sizex * sizeof(T), lddv2, - nfib * sizeof(T), ncol, nrow, D2H, queue_idx); - tmp_handle->sync(queue_idx); - gpuErrchk(cudaDeviceSynchronize()); - print_matrix(nrow, ncol, nfib, v, nfib, ncol); - delete[] v; - delete tmp_handle; - gpuErrchk(cudaDeviceSynchronize()); -} - -// print 3D CPU -template -void print_matrix(SIZE nrow, SIZE ncol, SIZE nfib, T *v, SIZE ldv1, SIZE ldv2) { - // std::cout << std::setw(10); - // std::cout << std::setprecision(2) << std::fixed; - for (int i = 0; i < nrow; i++) { - std::cout << "[ i = " << i << " ]\n"; - print_matrix(ncol, nfib, v + i * ldv1 * ldv2, ldv1); - // std::cout << std::endl; - } -} - -// compare 2D CPU -template -bool compare_matrix(SIZE nrow, SIZE ncol, T *v1, SIZE ldv1, T *v2, SIZE ldv2) { - // std::cout << std::setw(10); - // std::cout << std::setprecision(2) << std::fixed; - bool correct = true; - bool nan = false; - double E = 1e-6; - for (int i = 0; i < nrow; i++) { - for (int j = 0; j < ncol; j++) { - T a = v1[ldv1 * i + j]; - T b = v2[ldv2 * i + j]; - double diff = a - b; - diff = abs(diff); - if (diff > E) { - correct = false; - std::cout << "Diff at (" << i << ", " << j << ") "; - std::cout << a << " - " << b << " = " << diff << std::endl; - } - if (isnan(a) || isnan(b)) { - nan = true; - // std::cout << "NAN at (" << i << ", " << j << ") "; - // std::cout << a << " - " << b << " = " << abs(a-b) << std::endl; - } - } - } - if (correct) - printf("Compare: correct.\n"); - else - printf("Compare: wrong.\n"); - if (nan) - printf("Nan: include.\n"); - // else printf("Nan: not include.\n"); - return correct; -} - -// compare 2D GPU -template -bool compare_matrix_cuda(SIZE nrow, SIZE ncol, T *dv1, SIZE lddv1, T *dv2, - SIZE lddv2) { - Handle<2, float> *tmp_handle = new Handle<2, float>(); - int queue_idx = 0; - - T *v1 = new T[nrow * ncol]; - SIZE ldv1 = ncol; - cudaMemcpy2DAsyncHelper(*tmp_handle, v1, ldv1 * sizeof(T), dv1, - lddv1 * sizeof(T), ncol * sizeof(T), nrow, D2H, - queue_idx); - T *v2 = new T[nrow * ncol]; - SIZE ldv2 = ncol; - cudaMemcpy2DAsyncHelper(*tmp_handle, v2, ldv2 * sizeof(T), dv2, - lddv2 * sizeof(T), ncol * sizeof(T), nrow, D2H, - queue_idx); - tmp_handle->sync(queue_idx); - bool ret = compare_matrix(nrow, ncol, v1, ldv1, v2, ldv2); - delete[] v1; - delete[] v2; - delete tmp_handle; - return ret; -} - -// compare 3D CPU -template -bool compare_matrix(SIZE nrow, SIZE ncol, SIZE nfib, T *v1, SIZE ldv11, - SIZE ldv12, T *v2, SIZE ldv21, SIZE ldv22, - bool print_matrix) { - // std::cout << std::setw(10); - // std::cout << std::setprecision(2) << std::fixed; - bool correct = true; - bool nan = false; - double E = 1e-5; - for (int i = 0; i < nrow; i++) { - for (int j = 0; j < ncol; j++) { - for (int k = 0; k < nfib; k++) { - T a = v1[ldv11 * ldv12 * i + ldv11 * j + k]; - T b = v2[ldv21 * ldv22 * i + ldv21 * j + k]; - double diff = a - b; - diff = abs(diff); - if (diff > E) { - correct = false; - // std::cout << "Diff at (" << i << ", " << j << ", " << k <<") "; - // std::cout << a << " - " << b << " = " << abs(a-b) << std::endl; - if (print_matrix) - std::cout << ANSI_RED << std::setw(9) << std::setprecision(6) - << std::fixed << b << ", " << ANSI_RESET; - //<< b << "(" << a << ")"<< ", " << ANSI_RESET; - } else { - if (isnan(b)) { - if (print_matrix) - std::cout << ANSI_RED << std::setw(9) << std::setprecision(6) - << std::fixed << b << ", " << ANSI_RESET; - } else { - if (print_matrix) - std::cout << ANSI_GREEN << std::setw(9) << std::setprecision(6) - << std::fixed << b << ", " << ANSI_RESET; - } - } - - if (std::isnan(a) || std::isnan(b)) { - nan = true; - // std::cout << "NAN at (" << i << ", " << j << ") "; - // std::cout << a << " - " << b << " = " << abs(a-b) << std::endl; - } - } - if (print_matrix) - std::cout << std::endl; - } - if (print_matrix) - std::cout << std::endl; - } - if (correct && !nan) - printf(ANSI_GREEN "Compare: correct.\n" ANSI_RESET); - else - printf(ANSI_RED "Compare: wrong.\n" ANSI_RESET); - if (nan) - printf(ANSI_RED "Nan: include.\n" ANSI_RESET); - // else printf("Nan: not include.\n"); - return correct; -} - -// compare 3D GPU -template -bool compare_matrix_cuda(SIZE nrow, SIZE ncol, SIZE nfib, T *dv1, SIZE lddv11, - SIZE lddv12, SIZE sizex1, T *dv2, SIZE lddv21, - SIZE lddv22, SIZE sizex2, bool print_matrix) { - Handle<3, float> *tmp_handle = new Handle<3, float>(); - int queue_idx = 0; - - T *v1 = new T[nrow * ncol * nfib]; - SIZE ldv11 = nfib; - SIZE ldv12 = ncol; - cudaMemcpy3DAsyncHelper(*tmp_handle, v1, ldv11 * sizeof(T), nfib * sizeof(T), - ldv12, dv1, lddv11 * sizeof(T), sizex1 * sizeof(T), - lddv12, nfib * sizeof(T), ncol, nrow, D2H, queue_idx); - - T *v2 = new T[nrow * ncol * nfib]; - SIZE ldv21 = nfib; - SIZE ldv22 = ncol; - cudaMemcpy3DAsyncHelper(*tmp_handle, v2, ldv21 * sizeof(T), nfib * sizeof(T), - ldv22, dv2, lddv21 * sizeof(T), sizex2 * sizeof(T), - lddv22, nfib * sizeof(T), ncol, nrow, D2H, queue_idx); - tmp_handle->sync(queue_idx); - bool ret = compare_matrix(nrow, ncol, nfib, v1, ldv11, ldv12, v2, ldv21, - ldv22, print_matrix); - delete[] v1; - delete[] v2; - delete tmp_handle; - return ret; -} - -// print 3D CPU -template -void verify_matrix(SIZE nrow, SIZE ncol, SIZE nfib, T *v, SIZE ldv1, SIZE ldv2, - std::string file_prefix, bool store, bool verify) { - std::string filename = file_prefix + ".dat"; - if (store) { - std::ofstream myfile; - myfile.open(filename, std::ios::out | std::ios::binary); - if (!myfile) { - printf("Error: cannot write file\n"); - return; - } - myfile.write((char *)v, nrow * ncol * nfib * sizeof(T)); - myfile.close(); - if (!myfile.good()) { - printf("Error occurred at write time!\n"); - return; - } - } - if (verify) { - std::fstream fin; - fin.open(filename, std::ios::in | std::ios::binary); - if (!fin) { - printf("Error: cannot read file\n"); - return; - } - T *v2 = new T[nrow * ncol * nfib]; - fin.read((char *)v2, nrow * ncol * nfib * sizeof(T)); - fin.close(); - if (!fin.good()) { - printf("Error occurred at reading time!\n"); - return; - } - - bool mismatch = false; - for (int i = 0; i < nrow; i++) { - for (int j = 0; j < ncol; j++) { - for (int k = 0; k < nfib; k++) { - if (v[get_idx(ldv1, ldv2, i, j, k)] != - v2[get_idx(nfib, ncol, i, j, k)]) { - std::cout << filename << ": "; - printf("Mismatch[%d %d %d] %f - %f\n", i, j, k, - v[get_idx(ldv1, ldv2, i, j, k)], - v2[get_idx(nfib, ncol, i, j, k)]); - mismatch = true; - } - } - } - } - - delete v2; - if (mismatch) - exit(-1); - } -} - -// print 3D GPU -template -void verify_matrix_cuda(SIZE nrow, SIZE ncol, SIZE nfib, T *dv, SIZE lddv1, - SIZE lddv2, SIZE sizex, std::string file_prefix, - bool store, bool verify) { - // std::cout << std::setw(10); - // std::cout << std::setprecision(2) << std::fixed; - if (store || verify) { - Handle<3, float> *tmp_handle = new Handle<3, float>(); - int queue_idx = 0; - - T *v = new T[nrow * ncol * nfib]; - cudaMemcpy3DAsyncHelper(*tmp_handle, v, nfib * sizeof(T), nfib * sizeof(T), - ncol, dv, lddv1 * sizeof(T), sizex * sizeof(T), - lddv2, nfib * sizeof(T), ncol, nrow, D2H, - queue_idx); - tmp_handle->sync(queue_idx); - verify_matrix(nrow, ncol, nfib, v, nfib, ncol, file_prefix, store, verify); - delete[] v; - delete tmp_handle; - } -} - -// Allocate 1D -template -void cudaMallocHelper(Handle &handle, void **devPtr, size_t size) { - gpuErrchk(cudaMalloc(devPtr, size)); -} - -// Allocate 2D -template -void cudaMallocPitchHelper(Handle &handle, void **devPtr, size_t *pitch, - size_t width, size_t height) { - if (handle.reduce_memory_footprint) { - cudaMallocHelper(handle, devPtr, width * height); - *pitch = width; - } else { - gpuErrchk(cudaMallocPitch(devPtr, pitch, width, height)); - } -} - -// Allocate 3D -template -void cudaMalloc3DHelper(Handle &handle, void **devPtr, size_t *pitch, - size_t width, size_t height, size_t depth) { - - // if (handle.reduce_memory_footprint) { - cudaMallocHelper(handle, devPtr, width * height * depth); - *pitch = width; - // } else { - // cudaPitchedPtr devPitchedPtr; - // cudaExtent extent = make_cudaExtent(width, height, depth); - // gpuErrchk(cudaMalloc3D(&devPitchedPtr, extent)); - // *devPtr = devPitchedPtr.ptr; - // *pitch = devPitchedPtr.pitch; - // } -} - -// Allocate page-locked memory on host -void cudaMallocHostHelper(void **ptr, size_t size) { - gpuErrchk(cudaMallocHost(ptr, size)); -} - -enum cudaMemcpyKind inferTransferType(enum copy_type kind) { - switch (kind) { - case H2D: - return cudaMemcpyHostToDevice; - case D2H: - return cudaMemcpyDeviceToHost; - case D2D: - return cudaMemcpyDeviceToDevice; - case H2H: - return cudaMemcpyHostToHost; - case AUTO: - return cudaMemcpyDefault; - } -} - -// Copy 1D -template -void cudaMemcpyAsyncHelper(Handle &handle, void *dst, const void *src, - size_t count, enum copy_type kind, int queue_idx) { - - // printf("copu: %llu\n", count); - cudaStream_t stream = *(cudaStream_t *)handle.get(queue_idx); - enum cudaMemcpyKind cuda_copy_type = inferTransferType(kind); - // switch (kind) { - // case H2D: - // cuda_copy_type = cudaMemcpyHostToDevice; - // break; - // case D2H: - // cuda_copy_type = cudaMemcpyDeviceToHost; - // break; - // case D2D: - // cuda_copy_type = cudaMemcpyDeviceToDevice; - // break; - // } - gpuErrchk(cudaMemcpyAsync(dst, src, count, cuda_copy_type, stream)); -#ifdef MGARD_CUDA_DEBUG - gpuErrchk(cudaDeviceSynchronize()); -#endif -} - -// Copy 2D -template -void cudaMemcpy2DAsyncHelper(Handle &handle, void *dst, size_t dpitch, - void *src, size_t spitch, size_t width, - size_t height, enum copy_type kind, - int queue_idx) { - - cudaStream_t stream = *(cudaStream_t *)handle.get(queue_idx); - enum cudaMemcpyKind cuda_copy_type = inferTransferType(kind); - // switch (kind) { - // case H2D: - // cuda_copy_type = cudaMemcpyHostToDevice; - // break; - // case D2H: - // cuda_copy_type = cudaMemcpyDeviceToHost; - // break; - // case D2D: - // cuda_copy_type = cudaMemcpyDeviceToDevice; - // break; - // } - gpuErrchk(cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, - cuda_copy_type, stream)); -#ifdef MGARD_CUDA_DEBUG - gpuErrchk(cudaDeviceSynchronize()); -#endif -} - -void *cast_to_non_const(const void *const_ptr) { - const unsigned long long int const_address = - (unsigned long long int)const_ptr; - unsigned long long int address = const_address; - return (void *)address; -} - -// Copy 3D -template -void cudaMemcpy3DAsyncHelper(Handle &handle, void *dst, size_t dpitch, - size_t dwidth, size_t dheight, const void *src, - size_t spitch, size_t swidth, size_t sheight, - size_t width, size_t height, size_t depth, - enum copy_type kind, int queue_idx) { - - cudaStream_t stream = *(cudaStream_t *)handle.get(queue_idx); - - cudaExtent extent = make_cudaExtent(width, height, depth); - cudaMemcpy3DParms p = {0}; - p.dstPtr.ptr = dst; - p.dstPtr.pitch = dpitch; - p.dstPtr.xsize = dwidth; - p.dstPtr.ysize = dheight; - - p.srcPtr.ptr = cast_to_non_const(src); - p.srcPtr.pitch = spitch; - p.srcPtr.xsize = swidth; - p.srcPtr.ysize = sheight; - - p.extent = extent; - enum cudaMemcpyKind cuda_copy_type = inferTransferType(kind); - // switch (kind) { - // case H2D: - // cuda_copy_type = cudaMemcpyHostToDevice; - // break; - // case D2H: - // cuda_copy_type = cudaMemcpyDeviceToHost; - // break; - // case D2D: - // cuda_copy_type = cudaMemcpyDeviceToDevice; - // break; - // } - p.kind = cuda_copy_type; - - gpuErrchk(cudaMemcpy3DAsync(&p, stream)); -#ifdef MGARD_CUDA_DEBUG - gpuErrchk(cudaDeviceSynchronize()); -#endif -} - -void cudaFreeHelper(void *devPtr) { gpuErrchk(cudaFree(devPtr)); } - -void cudaFreeHostHelper(void *ptr) { gpuErrchk(cudaFreeHost(ptr)); } - -void cudaMemsetHelper(void *devPtr, int value, size_t count) { - gpuErrchk(cudaMemset(devPtr, value, count)); -} - -void cudaMemset2DHelper(void *devPtr, size_t pitch, int value, size_t width, - size_t height) { - gpuErrchk(cudaMemset2D(devPtr, pitch, value, width, height)); -} - -void cudaMemset3DHelper(void *devPtr, size_t pitch, size_t dwidth, - size_t dheight, int value, size_t width, size_t height, - size_t depth) { - cudaExtent extent = make_cudaExtent(width, height, depth); - cudaPitchedPtr devPitchedPtr; - devPitchedPtr.ptr = devPtr; - devPitchedPtr.pitch = pitch; - devPitchedPtr.xsize = dwidth; - devPitchedPtr.ysize = dheight; - gpuErrchk(cudaMemset3D(devPitchedPtr, value, extent)); -} - -void cudaSetDeviceHelper(int dev_id) { gpuErrchk(cudaSetDevice(dev_id)); } - -// Copy 1D Peer -template -void cudaMemcpyPeerAsyncHelper(Handle &handle, void *dst, int dst_dev, - const void *src, int src_dev, size_t count, - int queue_idx) { - - cudaStream_t stream = *(cudaStream_t *)handle.get(queue_idx); - gpuErrchk(cudaMemcpyPeerAsync(dst, dst_dev, src, src_dev, count, stream)); -#ifdef MGARD_CUDA_DEBUG - gpuErrchk(cudaDeviceSynchronize()); -#endif -} - -// Copy 3D peer -template -void cudaMemcpy3DPeerAsyncHelper(Handle &handle, void *dst, int dst_dev, - size_t dpitch, size_t dwidth, size_t dheight, - const void *src, int src_dev, size_t spitch, - size_t swidth, size_t sheight, size_t width, - size_t height, size_t depth, int queue_idx) { - - cudaStream_t stream = *(cudaStream_t *)handle.get(queue_idx); - - cudaExtent extent = make_cudaExtent(width, height, depth); - cudaMemcpy3DPeerParms p = {0}; - p.dstPtr.ptr = dst; - p.dstPtr.pitch = dpitch; - p.dstPtr.xsize = dwidth; - p.dstPtr.ysize = dheight; - - p.srcPtr.ptr = cast_to_non_const(src); - p.srcPtr.pitch = spitch; - p.srcPtr.xsize = swidth; - p.srcPtr.ysize = sheight; - - p.extent = extent; - - p.dstDevice = dst_dev; - p.srcDevice = src_dev; - - // printf("src_dev: %d - dst_dev: %d\n", dst_dev, src_dev); - - gpuErrchk(cudaMemcpy3DPeerAsync(&p, stream)); -#ifdef MGARD_CUDA_DEBUG - gpuErrchk(cudaDeviceSynchronize()); -#endif -} - -bool isGPUPointer(const void *ptr) { - cudaPointerAttributes attr; - cudaPointerGetAttributes(&attr, ptr); - return attr.type == cudaMemoryTypeDevice; -} - -#define KERNELS(T) \ - template bool compare_matrix(SIZE nrow, SIZE ncol, T * v1, SIZE ldv1, \ - T * v2, SIZE ldv2); \ - template bool compare_matrix_cuda(SIZE nrow, SIZE ncol, T * dv1, \ - SIZE lddv1, T * dv2, SIZE lddv2); \ - template bool compare_matrix(SIZE nrow, SIZE ncol, SIZE nfib, T * v1, \ - SIZE ldv11, SIZE ldv12, T * v2, SIZE ldv21, \ - SIZE ldv22, bool print_matrix); \ - template bool compare_matrix_cuda( \ - SIZE nrow, SIZE ncol, SIZE nfib, T * dv1, SIZE lddv11, SIZE lddv12, \ - SIZE sizex1, T * dv2, SIZE lddv21, SIZE lddv22, SIZE sizex2, \ - bool print_matrix); \ - template void verify_matrix( \ - SIZE nrow, SIZE ncol, SIZE nfib, T * v, SIZE ldv1, SIZE ldv2, \ - std::string file_prefix, bool save, bool verify); \ - template void verify_matrix_cuda( \ - SIZE nrow, SIZE ncol, SIZE nfib, T * dv, SIZE lddv1, SIZE lddv2, \ - SIZE sizex, std::string file_prefix, bool save, bool verify); - -KERNELS(double) -KERNELS(float) -KERNELS(int) -KERNELS(unsigned int) -KERNELS(size_t) -KERNELS(uint8_t) -#undef KERNELS - -#define KERNELS(T) \ - template void print_matrix(SIZE nrow, SIZE ncol, T * v, SIZE ldv); \ - template void print_matrix_cuda(SIZE nrow, SIZE ncol, T * dv, SIZE lddv); \ - template void print_matrix_cuda(SIZE nrow, SIZE ncol, SIZE nfib, T * dv, \ - SIZE lddv1, SIZE lddv2, SIZE sizex); \ - template void print_matrix(SIZE nrow, SIZE ncol, SIZE nfib, T * v, \ - SIZE ldv1, SIZE ldv2); - -KERNELS(double) -KERNELS(float) -KERNELS(int) -KERNELS(unsigned int) -KERNELS(LENGTH) -#undef KERNELS - -#define KERNELS(D, T) \ - template void cudaMallocHelper(Handle & handle, void **devPtr, \ - size_t size); \ - template void cudaMallocPitchHelper(Handle & handle, \ - void **devPtr, size_t *pitch, \ - size_t width, size_t height); \ - template void cudaMalloc3DHelper(Handle & handle, void **devPtr, \ - size_t *pitch, size_t width, \ - size_t height, size_t depth); \ - template void cudaMemcpyAsyncHelper( \ - Handle & handle, void *dst, const void *src, size_t count, \ - enum copy_type kind, int queue_idx); \ - template void cudaMemcpy2DAsyncHelper( \ - Handle & handle, void *dst, size_t dpitch, void *src, \ - size_t spitch, size_t width, size_t height, enum copy_type kind, \ - int queue_idx); \ - template void cudaMemcpy3DAsyncHelper( \ - Handle & handle, void *dst, size_t dpitch, size_t dwidth, \ - size_t dheight, const void *src, size_t spitch, size_t swidth, \ - size_t sheight, size_t width, size_t height, size_t depth, \ - enum copy_type kind, int queue_idx); \ - template void cudaMemcpyPeerAsyncHelper( \ - Handle & handle, void *dst, int dst_dev, const void *src, \ - int src_dev, size_t count, int queue_idx); \ - template void cudaMemcpy3DPeerAsyncHelper( \ - Handle & handle, void *dst, int dst_dev, size_t dpitch, \ - size_t dwidth, size_t dheight, const void *src, int src_dev, \ - size_t spitch, size_t swidth, size_t sheight, size_t width, \ - size_t height, size_t depth, int queue_idx); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) -#undef KERNELS - -#define KERNELS(D, T) \ - template void PrintSubarray>(std::string name, \ - SubArray subArray); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) -KERNELS(1, uint8_t) -KERNELS(1, uint16_t) -KERNELS(1, uint32_t) -KERNELS(1, uint64_t) -KERNELS(2, uint8_t) -KERNELS(2, uint16_t) -KERNELS(2, uint32_t) -KERNELS(2, uint64_t) -KERNELS(1, bool) - -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/Message.cpp b/src/cuda/Message.cpp deleted file mode 100644 index 050cf78b69..0000000000 --- a/src/cuda/Message.cpp +++ /dev/null @@ -1,39 +0,0 @@ -#include -#include -#include - -#include "cuda/Message.h" - -using std::string; - -namespace mgard_cuda { - -namespace log { - -const string log_null = " "; -const string log_err = "\e[31m[ERR]\e[0m "; -const string log_dbg = "\e[34m[dbg]\e[0m "; -const string log_info = "\e[32m[info]\e[0m "; -const string log_warn = "\e[31m[WARN]\e[0m "; -const string log_time = "\e[34m[time]\e[0m "; - -// https://stackoverflow.com/a/26080768/8740097 -template void build(std::ostream &o, T t) { o << t << std::endl; } - -template -void build(std::ostream &o, T t, - Args... args) // recursive variadic function -{ - build(o, t); - build(o, args...); -} - -template void print(string log_head, Args... args) { - std::ostringstream oss; - build(oss, args...); - std::cout << log_head << oss.str(); -} - -} // namespace log - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/Metadata.cpp b/src/cuda/Metadata.cpp deleted file mode 100644 index befc44bca6..0000000000 --- a/src/cuda/Metadata.cpp +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: September 27, 2021 - */ -#include "cuda/CommonInternal.h" - -#include "cuda/Metadata.h" - -#include "MGARDConfig.hpp" - -namespace mgard_cuda { - -SERIALIZED_TYPE *Metadata::Serialize(uint32_t &total_size) { - total_size = 0; - - // about MGARD software - total_size += sizeof(char) * strlen(magic_word); - total_size += sizeof(software_version); - total_size += sizeof(file_version); - total_size += sizeof(metadata_size); - total_size += sizeof(metadata_crc32); - total_size += sizeof(ptype); - - // about compression - total_size += sizeof(ebtype); - if (ebtype == error_bound_type::REL) { - total_size += sizeof(norm); // norm - } - total_size += sizeof(tol); // tol - total_size += sizeof(ntype); - if (ntype == norm_type::L_2) { - total_size += sizeof(s); // s - } - total_size += sizeof(l_target); // l_target; - total_size += sizeof(ltype); - if (ltype == lossless_type::GPU_Huffman || - ltype == lossless_type::GPU_Huffman_LZ4) { - total_size += sizeof(dict_size); // dict size - } - - // about data - total_size += sizeof(dtype); - total_size += sizeof(etype); - total_size += sizeof(dstype); - total_size += sizeof(total_dims); // total_dims; - total_size += sizeof(shape[0]) * total_dims; // shape; - if (dstype == data_structure_type::Cartesian_Grid_Non_Uniform) { - total_size += sizeof(cltype); - if (cltype == coordinate_location::Embedded) { - size_t coord_size = 0; - for (DIM d = 0; d < total_dims; d++) { - if (dtype == data_type::Float) { - coord_size += shape[d] * sizeof(float); - } else if (dtype == data_type::Double) { - coord_size += shape[d] * sizeof(double); - } - } - total_size += coord_size; - } else if (cltype == coordinate_location::External) { - total_size += sizeof(char) * strlen(nonuniform_coords_file); - } - } - - // initialize some fields - metadata_size = total_size; - - software_version[0] = MGARD_VERSION_MAJOR; - software_version[1] = MGARD_VERSION_MINOR; - software_version[2] = MGARD_VERSION_PATCH; - - file_version[0] = MGARD_FILE_VERSION_MAJOR; - file_version[1] = MGARD_FILE_VERSION_MINOR; - file_version[2] = MGARD_FILE_VERSION_PATCH; - - // to be replaced with actual CRC-32 checksum - metadata_crc32 = 0; - - // start serializing - SERIALIZED_TYPE *serialized_data = (SERIALIZED_TYPE *)std::malloc(total_size); - SERIALIZED_TYPE *p = serialized_data; - Serialize(&magic_word[0], p); - Serialize(software_version, p); - Serialize(file_version, p); - Serialize(metadata_size, p); - Serialize(metadata_crc32, p); - Serialize(ptype, p); - - Serialize(ebtype, p); - if (ebtype == error_bound_type::REL) { - Serialize(norm, p); - } - Serialize(tol, p); - Serialize(ntype, p); - if (ntype == norm_type::L_2) { - Serialize(s, p); - } - Serialize(l_target, p); - Serialize(ltype, p); - if (ltype == lossless_type::GPU_Huffman || - ltype == lossless_type::GPU_Huffman_LZ4) { - Serialize(dict_size, p); - } - - Serialize(dtype, p); - Serialize(etype, p); - Serialize(dstype, p); - Serialize(total_dims, p); - Serialize(shape, total_dims, p); - if (dstype == data_structure_type::Cartesian_Grid_Non_Uniform) { - Serialize(cltype, p); - if (cltype == coordinate_location::Embedded) { - Serialize(coords, shape, dtype, p); - } else if (cltype == coordinate_location::External) { - Serialize(nonuniform_coords_file, p); - } - } - self_initialized = false; - return serialized_data; -} - -void Metadata::Deserialize(SERIALIZED_TYPE *serialized_data, - uint32_t &total_size) { - SERIALIZED_TYPE *p = serialized_data; - - Deserialize(&magic_word[0], p); - Deserialize(software_version, p); - Deserialize(file_version, p); - Deserialize(metadata_size, p); - Deserialize(metadata_crc32, p); - Deserialize(ptype, p); - - Deserialize(ebtype, p); - if (ebtype == error_bound_type::REL) { - Deserialize(norm, p); - } - Deserialize(tol, p); - Deserialize(ntype, p); - if (ntype == norm_type::L_2) { - Deserialize(s, p); - } - Deserialize(l_target, p); - Deserialize(ltype, p); - if (ltype == lossless_type::GPU_Huffman || - ltype == lossless_type::GPU_Huffman_LZ4) { - Deserialize(dict_size, p); - } - - Deserialize(dtype, p); - Deserialize(etype, p); - Deserialize(dstype, p); - Deserialize(total_dims, p); - shape = new uint64_t[total_dims]; - Deserialize(shape, total_dims, p); - - if (dstype == data_structure_type::Cartesian_Grid_Non_Uniform) { - // printf("Deserialize Non_Uniform\n"); - Deserialize(cltype, p); - if (cltype == coordinate_location::Embedded) { - coords = std::vector(total_dims); - Deserialize(coords, shape, dtype, p); - } else if (cltype == coordinate_location::External) { - Deserialize(nonuniform_coords_file, p); - } - } - total_size = p - serialized_data; - self_initialized = true; -} - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/MgardCudaExec.cpp b/src/cuda/MgardCudaExec.cpp deleted file mode 100644 index a044e3864d..0000000000 --- a/src/cuda/MgardCudaExec.cpp +++ /dev/null @@ -1,469 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: September 27, 2021 - */ - -#include -#include -#include -#include -#include -#include - -#include "compress_cuda.hpp" - -using namespace std::chrono; - -void print_usage_message(std::string error) { - if (error.compare("") != 0) { - std::cout << mgard_cuda::log::log_err << error << std::endl; - } - printf("Options\n\ -\t -z: compress data\n\ -\t\t -i \n\ -\t\t -c \n\ -\t\t -t : data type (s: single; d:double)\n\ -\t\t -n : total number of dimensions\n\ -\t\t\t [dim1]: slowest dimention\n\ -\t\t\t [dim2]: 2nd slowest dimention\n\ -\t\t\t ...\n\ -\t\t\t [dimN]: fastest dimention\n\ -\t\t -u \n\ -\t\t -m : error bound mode (abs: abolute; rel: relative)\n\ -\t\t -e : error bound\n\ -\t\t -s : smoothness parameter\n\ -\t\t -l choose lossless compressor (0:ZSTD@CPU 1:Huffman@GPU 2:Huffman@GPU+LZ4@GPU)\n\ -\t\t -v enable verbose (show timing and statistics)\n\ -\n\ -\t -x: decompress data\n\ -\t\t -c \n\ -\t\t -d \n"); - exit(0); -} - -bool has_arg(int argc, char *argv[], std::string option) { - for (int i = 0; i < argc; i++) { - if (option.compare(std::string(argv[i])) == 0) { - return true; - } - } - return false; -} - -bool require_arg(int argc, char *argv[], std::string option) { - for (int i = 0; i < argc; i++) { - if (option.compare(std::string(argv[i])) == 0) { - return true; - } - } - print_usage_message("missing option: " + option + "."); - return false; -} - -std::string get_arg(int argc, char *argv[], std::string option) { - if (require_arg(argc, argv, option)) { - for (int i = 0; i < argc; i++) { - if (option.compare(std::string(argv[i])) == 0) { - return std::string(argv[i + 1]); - } - } - } - return std::string(""); -} - -int get_arg_int(int argc, char *argv[], std::string option) { - if (require_arg(argc, argv, option)) { - std::string arg; - int i; - for (i = 0; i < argc; i++) { - if (option.compare(std::string(argv[i])) == 0) { - arg = std::string(argv[i + 1]); - } - } - try { - int d = std::stoi(arg); - return d; - } catch (std::invalid_argument const &e) { - print_usage_message("illegal argument for option " + option + "."); - return 0; - } - } - return 0; -} - -std::vector get_arg_dims(int argc, char *argv[], - std::string option) { - std::vector shape; - if (require_arg(argc, argv, option)) { - std::string arg; - int arg_idx = 0, i; - for (i = 0; i < argc; i++) { - if (option.compare(std::string(argv[i])) == 0) { - arg = std::string(argv[i + 1]); - arg_idx = i + 1; - } - } - try { - int d = std::stoi(arg); - for (int i = 0; i < d; i++) { - shape.push_back(std::stoi(argv[arg_idx + 1 + i])); - } - return shape; - } catch (std::invalid_argument const &e) { - print_usage_message("illegal argument for option " + option + "."); - return shape; - } - } - return shape; -} - -double get_arg_double(int argc, char *argv[], std::string option) { - if (require_arg(argc, argv, option)) { - std::string arg; - int i; - for (i = 0; i < argc; i++) { - if (option.compare(std::string(argv[i])) == 0) { - arg = std::string(argv[i + 1]); - } - } - try { - double d = std::stod(arg); - return d; - } catch (std::invalid_argument const &e) { - print_usage_message("illegal argument for option " + option + "."); - } - } - return 0; -} - -template void min_max(size_t n, T *in_buff) { - T min = std::numeric_limits::infinity(); - T max = 0; - for (size_t i = 0; i < n; i++) { - if (min > in_buff[i]) { - min = in_buff[i]; - } - if (max < in_buff[i]) { - max = in_buff[i]; - } - } - printf("Min: %f, Max: %f\n", min, max); -} - -template size_t readfile(const char *input_file, T *&in_buff) { - std::cout << mgard_cuda::log::log_info << "Loading file: " << input_file - << "\n"; - - FILE *pFile; - pFile = fopen(input_file, "rb"); - if (pFile == NULL) { - std::cout << mgard_cuda::log::log_err << "file open error!\n"; - exit(1); - } - fseek(pFile, 0, SEEK_END); - size_t lSize = ftell(pFile); - rewind(pFile); - in_buff = (T *)malloc(lSize); - lSize = fread(in_buff, 1, lSize, pFile); - fclose(pFile); - // min_max(lSize/sizeof(T), in_buff); - return lSize; -} - -template -std::vector readcoords(const char *input_file, mgard_cuda::DIM D, - std::vector shape) { - std::cout << mgard_cuda::log::log_info - << "Loading coordinate file: " << input_file << "\n"; - FILE *pFile; - pFile = fopen(input_file, "rb"); - if (pFile == NULL) { - std::cout << mgard_cuda::log::log_err << "coordinate file open error!\n"; - exit(1); - } - fseek(pFile, 0, SEEK_END); - size_t lSize = ftell(pFile); - size_t expected_size = 0; - for (mgard_cuda::DIM d = 0; d < D; d++) { - expected_size += sizeof(T) * shape[d]; - } - if (lSize < expected_size) { - std::cout << mgard_cuda::log::log_err << "coordinate file read error!\n"; - exit(-1); - } - rewind(pFile); - std::vector coords(D); - for (mgard_cuda::DIM d = 0; d < D; d++) { - coords[d] = (T *)malloc(shape[d]); - lSize = fread(coords[d], sizeof(T), shape[d], pFile); - } - fclose(pFile); - return coords; -} - -template -void writefile(const char *output_file, size_t num_bytes, T *out_buff) { - FILE *file = fopen(output_file, "w"); - fwrite(out_buff, 1, num_bytes, file); - fclose(file); -} - -template -void print_statistics(double s, enum mgard_cuda::error_bound_type mode, - size_t n, T *original_data, T *decompressed_data) { - std::cout << std::scientific; - if (s == std::numeric_limits::infinity()) { - if (mode == mgard_cuda::error_bound_type::ABS) { - std::cout << mgard_cuda::log::log_info << "Absoluate L_inf error: " - << mgard_cuda::L_inf_error(n, original_data, decompressed_data, - mode) - << "\n"; - } else if (mode == mgard_cuda::error_bound_type::REL) { - std::cout << mgard_cuda::log::log_info << "Relative L_inf error: " - << mgard_cuda::L_inf_error(n, original_data, decompressed_data, - mode) - << "\n"; - } - } else { - if (mode == mgard_cuda::error_bound_type::ABS) { - std::cout << mgard_cuda::log::log_info << "Absoluate L_2 error: " - << mgard_cuda::L_2_error(n, original_data, decompressed_data, - mode) - << "\n"; - } else if (mode == mgard_cuda::error_bound_type::REL) { - std::cout << mgard_cuda::log::log_info << "Relative L_2 error: " - << mgard_cuda::L_2_error(n, original_data, decompressed_data, - mode) - << "\n"; - } - } - // std::cout << mgard_cuda::log::log_info << "L_2 error: " << - // mgard_cuda::L_2_error(n, original_data, decompressed_data) << "\n"; - std::cout << mgard_cuda::log::log_info - << "MSE: " << mgard_cuda::MSE(n, original_data, decompressed_data) - << "\n"; - std::cout << std::defaultfloat; - std::cout << mgard_cuda::log::log_info - << "PSNR: " << mgard_cuda::PSNR(n, original_data, decompressed_data) - << "\n"; -} - -template -int launch_compress(mgard_cuda::DIM D, enum mgard_cuda::data_type dtype, - const char *input_file, const char *output_file, - std::vector shape, bool non_uniform, - const char *coords_file, double tol, double s, - enum mgard_cuda::error_bound_type mode, int lossless, - bool verbose) { - - mgard_cuda::Config config; - config.timing = verbose; - - if (lossless == 0) { - config.lossless = mgard_cuda::lossless_type::CPU_Lossless; - } else if (lossless == 1) { - config.lossless = mgard_cuda::lossless_type::GPU_Huffman; - } else if (lossless == 2) { - config.lossless = mgard_cuda::lossless_type::GPU_Huffman_LZ4; - } - - size_t original_size = 1; - for (mgard_cuda::DIM i = 0; i < D; i++) - original_size *= shape[i]; - T *original_data; - size_t in_size = 0; - if (std::string(input_file).compare("random") == 0) { - in_size = original_size * sizeof(T); - original_data = new T[original_size]; - for (size_t i = 0; i < original_size; i++) - original_data[i] = rand() % 10 + 1; - } else { - in_size = readfile(input_file, original_data); - } - if (in_size != original_size * sizeof(T)) { - std::cout << mgard_cuda::log::log_err << "input file size mismatch!\n"; - } - - void *compressed_data = NULL; - size_t compressed_size = 0; - void *decompressed_data = NULL; - std::vector coords_byte; - if (!non_uniform) { - mgard_cuda::compress(D, dtype, shape, tol, s, mode, original_data, - compressed_data, compressed_size, config); - } else { - std::vector coords; - if (non_uniform) { - coords = readcoords(coords_file, D, shape); - } - for (auto &coord : coords) { - coords_byte.push_back((const mgard_cuda::Byte *)coord); - } - mgard_cuda::compress(D, dtype, shape, tol, s, mode, original_data, - compressed_data, compressed_size, coords_byte, config); - } - - writefile(output_file, compressed_size, compressed_data); - - printf("In size: %10ld Out size: %10ld Compression ratio: %f \n", - original_size * sizeof(T), compressed_size, - (double)original_size * sizeof(T) / compressed_size); - - if (verbose) { - config.timing = verbose; - - mgard_cuda::decompress(compressed_data, compressed_size, decompressed_data, - config); - - print_statistics(s, mode, original_size, original_data, - (T *)decompressed_data); - } - - delete[](T *) original_data; - return 0; -} - -int launch_decompress(const char *input_file, const char *output_file, - bool verbose) { - - mgard_cuda::Config config; - config.timing = verbose; - - mgard_cuda::SERIALIZED_TYPE *compressed_data; - size_t compressed_size = readfile(input_file, compressed_data); - std::vector shape = - mgard_cuda::infer_shape(compressed_data, compressed_size); - mgard_cuda::data_type dtype = - mgard_cuda::infer_data_type(compressed_data, compressed_size); - - size_t original_size = 1; - for (mgard_cuda::DIM i = 0; i < shape.size(); i++) { - original_size *= shape[i]; - } - - void *decompressed_data; - - mgard_cuda::decompress(compressed_data, compressed_size, decompressed_data, - config); - - int elem_size = 0; - if (dtype == mgard_cuda::data_type::Double) { - elem_size = 8; - } else if (dtype == mgard_cuda::data_type::Float) { - elem_size = 4; - } - writefile(output_file, original_size * elem_size, decompressed_data); - - delete[] compressed_data; - return 0; -} - -bool try_compression(int argc, char *argv[]) { - if (!has_arg(argc, argv, "-z")) - return false; - std::cout << mgard_cuda::log::log_info << "mode: compression\n"; - std::string input_file = get_arg(argc, argv, "-i"); - std::string output_file = get_arg(argc, argv, "-c"); - - std::cout << mgard_cuda::log::log_info << "original data: " << input_file - << "\n"; - std::cout << mgard_cuda::log::log_info << "compressed data: " << output_file - << "\n"; - - enum mgard_cuda::data_type dtype; - std::string dt = get_arg(argc, argv, "-t"); - if (dt.compare("s") == 0) { - dtype = mgard_cuda::data_type::Float; - std::cout << mgard_cuda::log::log_info << "data type: Single precision\n"; - } else if (dt.compare("d") == 0) { - dtype = mgard_cuda::data_type::Double; - std::cout << mgard_cuda::log::log_info << "data type: Double precision\n"; - } else - print_usage_message("wrong data type."); - - mgard_cuda::DIM D = get_arg_int(argc, argv, "-n"); - std::vector shape = get_arg_dims(argc, argv, "-n"); - std::string shape_string = "shape ("; - for (mgard_cuda::DIM d = 0; d < shape.size(); d++) - shape_string = shape_string + std::to_string(shape[d]) + " "; - shape_string = shape_string + ")"; - - bool non_uniform = false; - std::string non_uniform_coords_file; - if (has_arg(argc, argv, "-u")) { - non_uniform = true; - non_uniform_coords_file = get_arg(argc, argv, "-u"); - std::cout << mgard_cuda::log::log_info - << "non-uniform coordinate file: " << non_uniform_coords_file - << "\n"; - } - - enum mgard_cuda::error_bound_type mode; // REL or ABS - std::string em = get_arg(argc, argv, "-m"); - if (em.compare("rel") == 0) { - mode = mgard_cuda::error_bound_type::REL; - std::cout << mgard_cuda::log::log_info << "error bound mode: Relative\n"; - } else if (em.compare("abs") == 0) { - mode = mgard_cuda::error_bound_type::ABS; - std::cout << mgard_cuda::log::log_info << "error bound mode: Absolute\n"; - } else - print_usage_message("wrong error bound mode."); - - double tol = get_arg_double(argc, argv, "-e"); - double s = get_arg_double(argc, argv, "-s"); - - std::cout << std::scientific; - std::cout << mgard_cuda::log::log_info << "error bound: " << tol << "\n"; - std::cout << std::defaultfloat; - std::cout << mgard_cuda::log::log_info << "s: " << s << "\n"; - - int lossless_level = get_arg_int(argc, argv, "-l"); - if (lossless_level == 0) { - std::cout << mgard_cuda::log::log_info << "lossless: ZSTD@CPU\n"; - } else if (lossless_level == 1) { - std::cout << mgard_cuda::log::log_info << "lossless: Huffman@GPU\n"; - } else if (lossless_level == 2) { - std::cout << mgard_cuda::log::log_info - << "lossless: Huffman@GPU + LZ4@GPU\n"; - } - bool verbose = has_arg(argc, argv, "-v"); - if (verbose) - std::cout << mgard_cuda::log::log_info << "Verbose: enabled\n"; - if (dtype == mgard_cuda::data_type::Double) { - launch_compress(D, dtype, input_file.c_str(), output_file.c_str(), - shape, non_uniform, non_uniform_coords_file.c_str(), - tol, s, mode, lossless_level, verbose); - } else if (dtype == mgard_cuda::data_type::Float) { - launch_compress(D, dtype, input_file.c_str(), output_file.c_str(), - shape, non_uniform, non_uniform_coords_file.c_str(), - tol, s, mode, lossless_level, verbose); - } - return true; -} - -bool try_decompression(int argc, char *argv[]) { - if (!has_arg(argc, argv, "-x")) - return false; - std::cout << mgard_cuda::log::log_info << "mode: decompress\n"; - std::string input_file = get_arg(argc, argv, "-c"); - std::string output_file = get_arg(argc, argv, "-d"); - std::cout << mgard_cuda::log::log_info << "compressed data: " << input_file - << "\n"; - std::cout << mgard_cuda::log::log_info << "decompressed data: " << output_file - << "\n"; - bool verbose = has_arg(argc, argv, "-v"); - if (verbose) - std::cout << mgard_cuda::log::log_info << "verbose: enabled.\n"; - launch_decompress(input_file.c_str(), output_file.c_str(), verbose); - return true; -} - -int main(int argc, char *argv[]) { - - if (!try_compression(argc, argv) && !try_decompression(argc, argv)) { - print_usage_message(""); - } - return 0; -} diff --git a/src/cuda/ParallelHuffman/canonical.cu b/src/cuda/ParallelHuffman/canonical.cu deleted file mode 100644 index 884eb4d55e..0000000000 --- a/src/cuda/ParallelHuffman/canonical.cu +++ /dev/null @@ -1,103 +0,0 @@ -#include -#include -#include - -#include "cuda/ParallelHuffman/canonical.cuh" - -namespace cg = cooperative_groups; - -__device__ int max_bw = 0; - -// TODO change H Q order -template -__global__ void GPU::GetCanonicalCode(uint8_t *singleton, int DICT_SIZE) { - auto type_bw = sizeof(H) * 8; - auto codebooks = reinterpret_cast(singleton); - auto metadata = - reinterpret_cast(singleton + sizeof(H) * (3 * DICT_SIZE)); - auto keys = reinterpret_cast(singleton + sizeof(H) * (3 * DICT_SIZE) + - sizeof(int) * (4 * type_bw)); - H *i_cb = codebooks; - H *o_cb = codebooks + DICT_SIZE; - H *canonical = codebooks + DICT_SIZE * 2; - auto numl = metadata; - auto iter_by_ = metadata + type_bw; - auto first = metadata + type_bw * 2; - auto entry = metadata + type_bw * 3; - - cg::grid_group g = cg::this_grid(); - - int gid = blockDim.x * blockIdx.x + threadIdx.x; - // TODO - auto c = i_cb[gid]; - int bw = *((uint8_t *)&c + (sizeof(H) - 1)); - - if (c != ~((H)0x0)) { - atomicMax(&max_bw, bw); - atomicAdd(&numl[bw], 1); - } - g.sync(); - - if (gid == 0) { - // printf("\0"); - // atomicMax(&max_bw, max_bw + 0); - memcpy(entry + 1, numl, (type_bw - 1) * sizeof(int)); - // for (int i = 1; i < type_bw; i++) entry[i] = numl[i - 1]; - for (int i = 1; i < type_bw; i++) - entry[i] += entry[i - 1]; - } - g.sync(); - - if (gid < type_bw) - iter_by_[gid] = entry[gid]; - __syncthreads(); - // atomicMax(&max_bw, bw); - - if (gid == 0) { //////// first code - for (int l = max_bw - 1; l >= 1; l--) - first[l] = static_cast((first[l + 1] + numl[l + 1]) / 2.0 + 0.5); - first[0] = 0xff; // no off-by-one error - } - g.sync(); - - canonical[gid] = ~((H)0x0); - g.sync(); - o_cb[gid] = ~((H)0x0); - g.sync(); - - // Reverse Codebook Generation -- TODO isolate - if (gid == 0) { - // no atomicRead to handle read-after-write (true dependency) - for (int i = 0; i < DICT_SIZE; i++) { - auto _c = i_cb[i]; - uint8_t _bw = *((uint8_t *)&_c + (sizeof(H) - 1)); - - if (_c == ~((H)0x0)) - continue; - canonical[iter_by_[_bw]] = - static_cast(first[_bw] + iter_by_[_bw] - entry[_bw]); - keys[iter_by_[_bw]] = i; - - *((uint8_t *)&canonical[iter_by_[_bw]] + sizeof(H) - 1) = _bw; - iter_by_[_bw]++; - } - } - g.sync(); - - if (canonical[gid] == ~((H)0x0u)) - return; - o_cb[keys[gid]] = canonical[gid]; -} - -template __global__ void -GPU::GetCanonicalCode(uint8_t *singleton, int DICT_SIZE); -template __global__ void -GPU::GetCanonicalCode(uint8_t *singleton, int DICT_SIZE); -template __global__ void -GPU::GetCanonicalCode(uint8_t *singleton, int DICT_SIZE); -template __global__ void -GPU::GetCanonicalCode(uint8_t *singleton, int DICT_SIZE); -template __global__ void -GPU::GetCanonicalCode(uint8_t *singleton, int DICT_SIZE); -template __global__ void -GPU::GetCanonicalCode(uint8_t *singleton, int DICT_SIZE); diff --git a/src/cuda/ParallelHuffman/constants.cc b/src/cuda/ParallelHuffman/constants.cc deleted file mode 100644 index b3b8b5289a..0000000000 --- a/src/cuda/ParallelHuffman/constants.cc +++ /dev/null @@ -1,25 +0,0 @@ -#include - -#include "cuda/ParallelHuffman/constants.hh" - -const size_t DIM0 = 0; -const size_t DIM1 = 1; -const size_t DIM2 = 2; -const size_t DIM3 = 3; -const size_t nBLK0 = 4; -const size_t nBLK1 = 5; -const size_t nBLK2 = 6; -const size_t nBLK3 = 7; -const size_t nDIM = 8; -const size_t LEN = 12; -const size_t CAP = 13; -const size_t RADIUS = 14; - -const size_t EB = 0; -const size_t EBr = 1; -const size_t EBx2 = 2; -const size_t EBx2_r = 3; - -const int B_1d = 256; -const int B_2d = 16; -const int B_3d = 8; diff --git a/src/cuda/ParallelHuffman/cuda_mem.cu b/src/cuda/ParallelHuffman/cuda_mem.cu deleted file mode 100644 index bc20454b4f..0000000000 --- a/src/cuda/ParallelHuffman/cuda_mem.cu +++ /dev/null @@ -1,92 +0,0 @@ -#include -#include -#include -#include - -#include "cuda/ParallelHuffman/cuda_mem.cuh" - -template inline T *mem::CreateCUDASpace(size_t l, uint8_t i) { - T *d_var; - cudaMalloc(&d_var, l * sizeof(T)); - cudaMemset(d_var, i, l * sizeof(T)); - return d_var; -} - -// enum MemcpyDirection { h2d, d2h }; - -template -void mem::CopyBetweenSpaces(T *src, T *dst, size_t l, MemcpyDirection direct) { - assert(src != nullptr); - assert(dst != nullptr); - if (direct == h2d) { - cudaMemcpy(dst, src, sizeof(T) * l, cudaMemcpyHostToDevice); - } else if (direct == d2h) { - cudaMemcpy(dst, src, sizeof(T) * l, cudaMemcpyDeviceToHost); - } else { - // TODO log - exit(1); - } -} - -template -inline T *mem::CreateDeviceSpaceAndMemcpyFromHost(T *var, size_t l) { - T *d_var; - cudaMalloc(&d_var, l * sizeof(T)); - cudaMemcpy(d_var, var, l * sizeof(T), cudaMemcpyHostToDevice); - return d_var; -} -template -inline T *mem::CreateHostSpaceAndMemcpyFromDevice(T *d_var, size_t l) { - auto var = new T[l]; - cudaMemcpy(var, d_var, l * sizeof(T), cudaMemcpyDeviceToHost); - return var; -} - -template uint8_t *mem::CreateCUDASpace(size_t l, uint8_t i); -template uint16_t *mem::CreateCUDASpace(size_t l, uint8_t i); -template uint32_t *mem::CreateCUDASpace(size_t l, uint8_t i); -template uint64_t *mem::CreateCUDASpace(size_t l, uint8_t i); -template int8_t *mem::CreateCUDASpace(size_t l, uint8_t i); -template int16_t *mem::CreateCUDASpace(size_t l, uint8_t i); -template int32_t *mem::CreateCUDASpace(size_t l, uint8_t i); -template int64_t *mem::CreateCUDASpace(size_t l, uint8_t i); -template float *mem::CreateCUDASpace(size_t l, uint8_t i); -template double *mem::CreateCUDASpace(size_t l, uint8_t i); - -template int8_t *mem::CreateDeviceSpaceAndMemcpyFromHost(int8_t *var, size_t l); -template int16_t *mem::CreateDeviceSpaceAndMemcpyFromHost(int16_t *var, - size_t l); -template int32_t *mem::CreateDeviceSpaceAndMemcpyFromHost(int32_t *var, - size_t l); -template int64_t *mem::CreateDeviceSpaceAndMemcpyFromHost(int64_t *var, - size_t l); -template uint8_t *mem::CreateDeviceSpaceAndMemcpyFromHost(uint8_t *var, - size_t l); -template uint16_t *mem::CreateDeviceSpaceAndMemcpyFromHost(uint16_t *var, - size_t l); -template uint32_t *mem::CreateDeviceSpaceAndMemcpyFromHost(uint32_t *var, - size_t l); -template uint64_t *mem::CreateDeviceSpaceAndMemcpyFromHost(uint64_t *var, - size_t l); -template float *mem::CreateDeviceSpaceAndMemcpyFromHost(float *var, size_t l); -template double *mem::CreateDeviceSpaceAndMemcpyFromHost(double *var, size_t l); - -template int8_t *mem::CreateHostSpaceAndMemcpyFromDevice(int8_t *d_var, - size_t l); -template int16_t *mem::CreateHostSpaceAndMemcpyFromDevice(int16_t *d_var, - size_t l); -template int32_t *mem::CreateHostSpaceAndMemcpyFromDevice(int32_t *d_var, - size_t l); -template int64_t *mem::CreateHostSpaceAndMemcpyFromDevice(int64_t *d_var, - size_t l); -template uint8_t *mem::CreateHostSpaceAndMemcpyFromDevice(uint8_t *d_var, - size_t l); -template uint16_t *mem::CreateHostSpaceAndMemcpyFromDevice(uint16_t *d_var, - size_t l); -template uint32_t *mem::CreateHostSpaceAndMemcpyFromDevice(uint32_t *d_var, - size_t l); -template uint64_t *mem::CreateHostSpaceAndMemcpyFromDevice(uint64_t *d_var, - size_t l); -template float *mem::CreateHostSpaceAndMemcpyFromDevice(float *d_var, size_t l); -template double *mem::CreateHostSpaceAndMemcpyFromDevice(double *d_var, - size_t l); diff --git a/src/cuda/ParallelHuffman/format.cc b/src/cuda/ParallelHuffman/format.cc deleted file mode 100644 index 9f72256dce..0000000000 --- a/src/cuda/ParallelHuffman/format.cc +++ /dev/null @@ -1,27 +0,0 @@ -#include -#include -#include - -#include "cuda/ParallelHuffman/format.hh" - -using std::string; - -// https://stackoverflow.com/a/26080768/8740097 -template void huffman_gpu::log::build(std::ostream &o, T t) { - o << t << std::endl; -} - -template -void huffman_gpu::log::build(std::ostream &o, T t, - Args... args) // recursive variadic function -{ - huffman_gpu::log::build(o, t); - huffman_gpu::log::build(o, args...); -} - -template -void huffman_gpu::log::print(string log_head, Args... args) { - std::ostringstream oss; - huffman_gpu::log::build(oss, args...); - std::cout << log_head << oss.str(); -} diff --git a/src/cuda/ParallelHuffman/histogram.cu b/src/cuda/ParallelHuffman/histogram.cu deleted file mode 100644 index 643e6d60a7..0000000000 --- a/src/cuda/ParallelHuffman/histogram.cu +++ /dev/null @@ -1,75 +0,0 @@ -// includes CUDA Runtime -#include - -#include -#include - -#include "cuda/ParallelHuffman/histogram.cuh" - -using uint8__t = uint8_t; - -__global__ void naiveHistogram(int input_data[], int output[], int N, - int symbols_per_thread) { - unsigned int i = blockDim.x * blockIdx.x + threadIdx.x; - unsigned int j; - if (i * symbols_per_thread < N) { // if there is a symbol to count, - for (j = i * symbols_per_thread; j < (i + 1) * symbols_per_thread; j++) { - if (j < N) { - unsigned int item = input_data[j]; // Symbol to count - atomicAdd(&output[item], 1); // update bin count by 1 - } - } - } -} - -// const static unsigned int WARP_SIZE = 32; - -#define MIN(a, b) ((a) < (b)) ? (a) : (b) - -template -__global__ void p2013Histogram(T *input_data, Q *output, size_t N, int bins, - int R) { - extern __shared__ int Hs[/*(bins + 1) * R*/]; - - const unsigned int warpid = (int)(threadIdx.x / WARP_SIZE); - const unsigned int lane = threadIdx.x % WARP_SIZE; - const unsigned int warps_block = blockDim.x / WARP_SIZE; - - const unsigned int off_rep = (bins + 1) * (threadIdx.x % R); - - const unsigned int begin = - (N / warps_block) * warpid + WARP_SIZE * blockIdx.x + lane; - unsigned int end = (N / warps_block) * (warpid + 1); - const unsigned int step = WARP_SIZE * gridDim.x; - - // final warp handles data outside of the warps_block partitions - if (warpid >= warps_block - 1) - end = N; - - for (unsigned int pos = threadIdx.x; pos < (bins + 1) * R; pos += blockDim.x) - Hs[pos] = 0; - - __syncthreads(); - - for (unsigned int i = begin; i < end; i += step) { - int d = input_data[i]; - atomicAdd(&Hs[off_rep + d], 1); - } - - __syncthreads(); - - for (unsigned int pos = threadIdx.x; pos < bins; pos += blockDim.x) { - int sum = 0; - for (int base = 0; base < (bins + 1) * R; base += bins + 1) { - sum += Hs[base + pos]; - } - atomicAdd(output + pos, sum); - } -} - -template __global__ void p2013Histogram( - uint8__t *input_data, unsigned int *output, size_t N, int bins, int R); -template __global__ void p2013Histogram( - uint16_t *input_data, unsigned int *output, size_t N, int bins, int R); -template __global__ void p2013Histogram( - uint32_t *input_data, unsigned int *output, size_t N, int bins, int R); diff --git a/src/cuda/ParallelHuffman/huffman.cu b/src/cuda/ParallelHuffman/huffman.cu deleted file mode 100644 index 0bd0c75f3d..0000000000 --- a/src/cuda/ParallelHuffman/huffman.cu +++ /dev/null @@ -1,267 +0,0 @@ -#include -#include -#include - -#include "cuda/ParallelHuffman/huffman.cuh" - -using namespace std; - -__device__ HuffmanTree *global_gpuTree; - -template -__global__ void prototype::GPU_Histogram(T *input_data, Q *output, size_t N, - int symbols_per_thread) { - unsigned int i = blockDim.x * blockIdx.x + threadIdx.x; - unsigned int j; - if (i * symbols_per_thread < N) { // if there is a symbol to count - for (j = i * symbols_per_thread; j < (i + 1) * symbols_per_thread; j++) { - if (j < N) { - unsigned int item = input_data[j]; // Symbol to count - atomicAdd(&output[item], 1); // update bin count by 1 - } - } - } -} - -template -__global__ void prototype::EncodeFixedLen(T *data, Q *hcoded, size_t data_len, - Q *codebook) { - size_t gid = blockDim.x * blockIdx.x + threadIdx.x; - if (gid >= data_len) - return; - hcoded[gid] = codebook[data[gid]]; // try to exploit cache? - __syncthreads(); -} - -// auxiliary functions done -__host__ HuffmanTree *createHuffmanTreeCPU(int stateNum) { - auto ht = (HuffmanTree *)malloc(sizeof(HuffmanTree)); - memset(ht, 0, sizeof(HuffmanTree)); - ht->stateNum = stateNum; - ht->allNodes = 2 * stateNum; - - ht->pool = (struct node_t *)malloc(ht->allNodes * 2 * sizeof(struct node_t)); - ht->qqq = (node_list *)malloc(ht->allNodes * 2 * sizeof(node_list)); - ht->code = (uint64_t **)malloc(ht->stateNum * sizeof(uint64_t *)); - ht->cout = (uint8_t *)malloc(ht->stateNum * sizeof(uint8_t)); - - memset(ht->pool, 0, ht->allNodes * 2 * sizeof(struct node_t)); - memset(ht->qqq, 0, ht->allNodes * 2 * sizeof(node_list)); - memset(ht->code, 0, ht->stateNum * sizeof(uint64_t *)); - memset(ht->cout, 0, ht->stateNum * sizeof(uint8_t)); - ht->qq = ht->qqq - 1; - ht->n_nodes = 0; - ht->n_inode = 0; - ht->qend = 1; - - return ht; -} - -__device__ HuffmanTree *createHuffmanTreeGPU(int stateNum) { - auto ht = (HuffmanTree *)malloc(sizeof(HuffmanTree)); - memset(ht, 0, sizeof(HuffmanTree)); - ht->stateNum = stateNum; - ht->allNodes = 2 * stateNum; - - ht->pool = (struct node_t *)malloc(ht->allNodes * 2 * sizeof(struct node_t)); - ht->qqq = (node_list *)malloc(ht->allNodes * 2 * sizeof(node_list)); - ht->code = (uint64_t **)malloc(ht->stateNum * sizeof(uint64_t *)); - ht->cout = (uint8_t *)malloc(ht->stateNum * sizeof(uint8_t)); - - memset(ht->pool, 0, ht->allNodes * 2 * sizeof(struct node_t)); - memset(ht->qqq, 0, ht->allNodes * 2 * sizeof(node_list)); - memset(ht->code, 0, ht->stateNum * sizeof(uint64_t *)); - memset(ht->cout, 0, ht->stateNum * sizeof(uint8_t)); - ht->qq = ht->qqq - 1; - ht->n_nodes = 0; - ht->n_inode = 0; - ht->qend = 1; - - return ht; -} - -__host__ __device__ node_list new_node(HuffmanTree *huffmanTree, size_t freq, - uint32_t c, node_list a, node_list b) { - node_list n = huffmanTree->pool + huffmanTree->n_nodes++; - if (freq) { - n->c = c; - n->freq = freq; - n->t = 1; - } else { - n->left = a; - n->right = b; - n->freq = a->freq + b->freq; - n->t = 0; - // n->c = 0; - } - return n; -} - -/* priority queue */ -__host__ __device__ void qinsert(HuffmanTree *ht, node_list n) { - int j, i = ht->qend++; - while ((j = (i >> 1))) { // j=i/2 - if (ht->qq[j]->freq <= n->freq) - break; - ht->qq[i] = ht->qq[j], i = j; - } - ht->qq[i] = n; -} - -__host__ __device__ node_list qremove(HuffmanTree *ht) { - int i, l; - node_list n = ht->qq[i = 1]; - - if (ht->qend < 2) - return 0; - ht->qend--; - while ((l = (i << 1)) < ht->qend) { // l=(i*2) - if (l + 1 < ht->qend && ht->qq[l + 1]->freq < ht->qq[l]->freq) - l++; - ht->qq[i] = ht->qq[l], i = l; - } - ht->qq[i] = ht->qq[ht->qend]; - return n; -} - -/* walk the tree and put 0s and 1s */ -/** - * @out1 should be set to 0. - * @out2 should be 0 as well. - * @index: the index of the byte - * */ -__host__ __device__ void build_code(HuffmanTree *ht, node_list n, int len, - uint64_t out1, uint64_t out2) { - if (n->t) { - ht->code[n->c] = (uint64_t *)malloc(2 * sizeof(uint64_t)); - if (len <= 64) { - (ht->code[n->c])[0] = out1 << (64 - len); - (ht->code[n->c])[1] = out2; - } else { - (ht->code[n->c])[0] = out1; - (ht->code[n->c])[1] = out2 << (128 - len); - } - ht->cout[n->c] = (uint8_t)len; - return; - } - - int index = len >> 6; //=len/64 - if (index == 0) { - out1 = out1 << 1; - out1 = out1 | 0; - build_code(ht, n->left, len + 1, out1, 0); - out1 = out1 | 1; - build_code(ht, n->right, len + 1, out1, 0); - } else { - if (len % 64 != 0) - out2 = out2 << 1; - out2 = out2 | 0; - build_code(ht, n->left, len + 1, out1, out2); - out2 = out2 | 1; - build_code(ht, n->right, len + 1, out1, out2); - } -} - -//////////////////////////////////////////////////////////////////////////////// -// internal functions -//////////////////////////////////////////////////////////////////////////////// - -__device__ __forceinline__ node_list top(internal_stack_t *s) { - return s->_a[s->depth - 1]; -} - -template -__device__ __forceinline__ void push_v2(internal_stack_t *s, node_list n, - T path, T len) { - if (s->depth + 1 <= MAX_DEPTH) { - s->depth += 1; - - s->_a[s->depth - 1] = n; - s->saved_path[s->depth - 1] = path; - s->saved_length[s->depth - 1] = len; - } else - printf("Error: stack overflow\n"); -} - -__device__ __forceinline__ bool isEmpty(internal_stack_t *s) { - return (s->depth == 0); -} - -// TODO check with typing -template -__device__ __forceinline__ node_list pop_v2(internal_stack_t *s, - T *path_to_restore, - T *length_to_restore) { - node_list n; - - if (isEmpty(s)) { - printf("Error: stack underflow, exiting...\n"); - return nullptr; - // exit(0); - } else { - // TODO holding array -> __a - n = s->_a[s->depth - 1]; - s->_a[s->depth - 1] = nullptr; - - *length_to_restore = s->saved_length[s->depth - 1]; - *path_to_restore = s->saved_path[s->depth - 1]; - s->depth -= 1; - - return n; - } -} - -template -__device__ void InOrderTraverse_v2(HuffmanTree *ht, Q *codebook) { - node_list root = ht->qq[1]; - auto s = new internal_stack_t(); - - bool done = 0; - Q out1 = 0, len = 0; - - while (!done) { - if (root->left or root->right) { - push_v2(s, root, out1, len); - root = root->left; - out1 <<= 1u; - out1 |= 0u; - len += 1; - } else { - uint32_t bincode = root->c; - codebook[bincode] = out1 | ((len & (Q)0xffu) << (sizeof(Q) * 8 - 8)); - if (!isEmpty(s)) { - root = pop_v2(s, &out1, &len); - root = root->right; - out1 <<= 1u; - out1 |= 1u; - len += 1; - } else - done = true; - } - } /* end of while */ -} - -template -__global__ void InitHuffTreeAndGetCodebook( - int stateNum, unsigned int *freq, - H *codebook) { // length known as huffmanTree->allNodes - if (threadIdx.x != 0) - return; - global_gpuTree = createHuffmanTreeGPU(stateNum); - for (size_t i = 0; i < global_gpuTree->allNodes; i++) - if (freq[i]) - qinsert(global_gpuTree, new_node(global_gpuTree, freq[i], i, 0, 0)); - while (global_gpuTree->qend > 2) - qinsert(global_gpuTree, - new_node(global_gpuTree, 0, 0, qremove(global_gpuTree), - qremove(global_gpuTree))); - InOrderTraverse_v2(global_gpuTree, codebook); -} - -// TODO `unsigned int` seems trivial to pick up -template __global__ void -InitHuffTreeAndGetCodebook(int stateNum, unsigned int *freq, - uint32_t *codebook); -template __global__ void -InitHuffTreeAndGetCodebook(int stateNum, unsigned int *freq, - uint64_t *codebook); diff --git a/src/cuda/ParallelHuffman/huffman_codec.cu b/src/cuda/ParallelHuffman/huffman_codec.cu deleted file mode 100644 index e2f5dd4edc..0000000000 --- a/src/cuda/ParallelHuffman/huffman_codec.cu +++ /dev/null @@ -1,192 +0,0 @@ -#include -#include -#include -#include - -#include "cuda/ParallelHuffman/huffman_codec.cuh" - -using uint8__t = uint8_t; - -template -__global__ void EncodeFixedLen(Q *data, H *hcoded, size_t data_len, - H *codebook) { - size_t gid = blockDim.x * blockIdx.x + threadIdx.x; - if (gid >= data_len) - return; - hcoded[gid] = codebook[data[gid]]; // try to exploit cache? - __syncthreads(); -} - -template -__global__ void Deflate(Q *hcoded, // - size_t len, size_t *densely_meta, int PART_SIZE) { - size_t gid = blockIdx.x * blockDim.x + threadIdx.x; - if (gid >= (len - 1) / PART_SIZE + 1) - return; - uint8_t bitwidth; - size_t densely_coded_lsb_pos = sizeof(Q) * 8, total_bitwidth = 0; - size_t ending = - (gid + 1) * PART_SIZE <= len ? PART_SIZE : len - gid * PART_SIZE; - // if ((gid + 1) * PART_SIZE > len) printf("\n\ngid %lu\tending %lu\n\n", - // gid, ending); - Q msb_bw_word_lsb, _1, _2; - Q *current = hcoded + gid * PART_SIZE; - for (size_t i = 0; i < ending; i++) { - msb_bw_word_lsb = hcoded[gid * PART_SIZE + i]; - bitwidth = *((uint8_t *)&msb_bw_word_lsb + (sizeof(Q) - 1)); - - *((uint8_t *)&msb_bw_word_lsb + sizeof(Q) - 1) = 0x0; - if (densely_coded_lsb_pos == sizeof(Q) * 8) - *current = 0x0; // a new unit of data type - if (bitwidth <= densely_coded_lsb_pos) { - densely_coded_lsb_pos -= bitwidth; - *current |= msb_bw_word_lsb << densely_coded_lsb_pos; - if (densely_coded_lsb_pos == 0) { - densely_coded_lsb_pos = sizeof(Q) * 8; - ++current; - } - } else { - // example: we have 5-bit code 11111 but 3 bits left for (*current) - // we put first 3 bits of 11111 to the last 3 bits of (*current) - // and put last 2 bits from MSB of (*(++current)) - // the comment continues with the example - _1 = msb_bw_word_lsb >> (bitwidth - densely_coded_lsb_pos); - _2 = msb_bw_word_lsb << (sizeof(Q) * 8 - - (bitwidth - densely_coded_lsb_pos)); - *current |= _1; - *(++current) = 0x0; - *current |= _2; - densely_coded_lsb_pos = - sizeof(Q) * 8 - (bitwidth - densely_coded_lsb_pos); - } - total_bitwidth += bitwidth; - } - *(densely_meta + gid) = total_bitwidth; -} - -template -__device__ void InflateChunkwise(H *in_huff, T *out_quant, size_t total_bw, - uint8_t *singleton) { - uint8_t next_bit; - size_t idx_bit; - size_t idx_byte = 0; - size_t idx_bcoded = 0; - auto first = reinterpret_cast(singleton); - auto entry = first + sizeof(H) * 8; - auto keys = - reinterpret_cast(singleton + sizeof(H) * (2 * sizeof(H) * 8)); - H v = (in_huff[idx_byte] >> (sizeof(H) * 8 - 1)) & 0x1; // get the first bit - size_t l = 1; - size_t i = 0; - while (i < total_bw) { - while (v < first[l]) { // append next i_cb bit - ++i; - idx_byte = i / (sizeof(H) * 8); - idx_bit = i % (sizeof(H) * 8); - next_bit = ((in_huff[idx_byte] >> (sizeof(H) * 8 - 1 - idx_bit)) & 0x1); - v = (v << 1) | next_bit; - ++l; - } - out_quant[idx_bcoded++] = keys[entry[l] + v - first[l]]; - { - ++i; - idx_byte = i / (sizeof(H) * 8); - idx_bit = i % (sizeof(H) * 8); - next_bit = ((in_huff[idx_byte] >> (sizeof(H) * 8 - 1 - idx_bit)) & 0x1); - v = 0x0 | next_bit; - } - l = 1; - } -} - -template -__global__ void Decode(H *densely, // - size_t *dH_meta, // - Q *bcode, // - size_t len, // - int chunk_size, // - int n_chunk, uint8_t *singleton, size_t singleton_size) { - extern __shared__ uint8_t _s_singleton[]; - if (threadIdx.x == 0) - memcpy(_s_singleton, singleton, singleton_size); - __syncthreads(); - - auto dH_bit_meta = dH_meta; - auto dH_uInt_entry = dH_meta + n_chunk; - - size_t chunk_id = blockIdx.x * blockDim.x + threadIdx.x; - // if (chunk_id == 0) printf("n_chunk: %lu\n", n_chunk); - if (chunk_id >= n_chunk) - return; - - InflateChunkwise( // - densely + dH_uInt_entry[chunk_id], // - bcode + chunk_size * chunk_id, // - dH_bit_meta[chunk_id], // - _s_singleton); - __syncthreads(); -}; - -template __global__ void -EncodeFixedLen(uint8__t *, uint32_t *, size_t, uint32_t *); -template __global__ void -EncodeFixedLen(uint8__t *, uint64_t *, size_t, uint64_t *); -template __global__ void -EncodeFixedLen(uint16_t *, uint32_t *, size_t, uint32_t *); -template __global__ void -EncodeFixedLen(uint16_t *, uint64_t *, size_t, uint64_t *); -template __global__ void -EncodeFixedLen(uint32_t *, uint32_t *, size_t, uint32_t *); -template __global__ void -EncodeFixedLen(uint32_t *, uint64_t *, size_t, uint64_t *); - -template __global__ void Deflate(uint32_t *hcoded, size_t len, - size_t *densely_meta, int PART_SIZE); -template __global__ void Deflate(uint64_t *hcoded, size_t len, - size_t *densely_meta, int PART_SIZE); - -// H for Huffman, uint{32,64}_t -// T for quant code, uint{8,16,32}_t -template __device__ void InflateChunkwise(uint32_t *, - uint8__t *, - size_t, - uint8__t *); -template __device__ void InflateChunkwise(uint32_t *, - uint16_t *, - size_t, - uint8__t *); -template __device__ void InflateChunkwise(uint32_t *, - uint32_t *, - size_t, - uint8__t *); -template __device__ void InflateChunkwise(uint64_t *, - uint8__t *, - size_t, - uint8__t *); -template __device__ void InflateChunkwise(uint64_t *, - uint16_t *, - size_t, - uint8__t *); -template __device__ void InflateChunkwise(uint64_t *, - uint32_t *, - size_t, - uint8__t *); - -template __global__ void Decode(uint32_t *, size_t *, - uint8__t *, size_t, int, - int, uint8__t *, size_t); -template __global__ void Decode(uint64_t *, size_t *, - uint8__t *, size_t, int, - int, uint8__t *, size_t); -template __global__ void Decode(uint32_t *, size_t *, - uint16_t *, size_t, int, - int, uint8__t *, size_t); -template __global__ void Decode(uint64_t *, size_t *, - uint16_t *, size_t, int, - int, uint8__t *, size_t); -template __global__ void Decode(uint32_t *, size_t *, - uint32_t *, size_t, int, - int, uint8__t *, size_t); -template __global__ void Decode(uint64_t *, size_t *, - uint32_t *, size_t, int, - int, uint8__t *, size_t); diff --git a/src/cuda/ParallelHuffman/huffman_workflow.cu b/src/cuda/ParallelHuffman/huffman_workflow.cu deleted file mode 100644 index 24cd484141..0000000000 --- a/src/cuda/ParallelHuffman/huffman_workflow.cu +++ /dev/null @@ -1,587 +0,0 @@ -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -using namespace std::chrono; - -#include "cuda/Common.h" -#include "cuda/CommonInternal.h" - -#include "cuda/ParallelHuffman/canonical.cuh" -#include "cuda/ParallelHuffman/cuda_error_handling.cuh" -#include "cuda/ParallelHuffman/cuda_mem.cuh" -#include "cuda/ParallelHuffman/dbg_gpu_printing.cuh" -#include "cuda/ParallelHuffman/format.hh" -#include "cuda/ParallelHuffman/histogram.cuh" -#include "cuda/ParallelHuffman/huffman.cuh" -#include "cuda/ParallelHuffman/huffman_codec.cuh" -#include "cuda/ParallelHuffman/huffman_workflow.cuh" -#include "cuda/ParallelHuffman/par_huffman.cuh" -#include "cuda/ParallelHuffman/types.hh" - -int ht_state_num; -int ht_all_nodes; -using uint8__t = uint8_t; - -template -void wrapper::GetFrequency(Q *d_bcode, size_t len, unsigned int *d_freq, - int dict_size) { - // Parameters for thread and block count optimization - - // Initialize to device-specific values - int deviceId; - int maxbytes; - int maxbytesOptIn; - int numSMs; - - cudaGetDevice(&deviceId); - cudaDeviceGetAttribute(&maxbytes, cudaDevAttrMaxSharedMemoryPerBlock, - deviceId); - cudaDeviceGetAttribute(&numSMs, cudaDevAttrMultiProcessorCount, deviceId); - - // Account for opt-in extra shared memory on certain architectures - cudaDeviceGetAttribute(&maxbytesOptIn, - cudaDevAttrMaxSharedMemoryPerBlockOptin, deviceId); - maxbytes = std::max(maxbytes, maxbytesOptIn); - - // Optimize launch - int numBuckets = dict_size; - int numValues = len; - int itemsPerThread = 1; - int RPerBlock = (maxbytes / (int)sizeof(int)) / (numBuckets + 1); - int numBlocks = numSMs; - cudaFuncSetAttribute(p2013Histogram, - cudaFuncAttributeMaxDynamicSharedMemorySize, maxbytes); - // fits to size - int threadsPerBlock = - ((((numValues / (numBlocks * itemsPerThread)) + 1) / 64) + 1) * 64; - while (threadsPerBlock > 1024) { - if (RPerBlock <= 1) { - threadsPerBlock = 1024; - } else { - RPerBlock /= 2; - numBlocks *= 2; - threadsPerBlock = - ((((numValues / (numBlocks * itemsPerThread)) + 1) / 64) + 1) * 64; - } - } - - // mgard_cuda::print_matrix_cuda(1, 10, (int *)d_bcode, 10); - - // printf("maxbytes: %d, p2013Histogram: %d\n", maxbytes,(numBuckets + 1) * - // sizeof(int)); - - p2013Histogram // - <<>> // - (d_bcode, d_freq, numValues, numBuckets, RPerBlock); - cudaDeviceSynchronize(); - - // TODO make entropy optional - // { - // auto freq = mem::CreateHostSpaceAndMemcpyFromDevice(d_freq, - // dict_size); double entropy = 0.0; for (auto i = 0; i < dict_size; i++) - // if (freq[i]) { - // auto possibility = freq[i] / (1.0 * len); - // entropy -= possibility * log(possibility); - // cout << i << ": " << freq[i] << "\n"; - // } - // cout << log_info << "entropy:\t\t" << entropy << endl; - // delete[] freq; - // } - - // #ifdef DEBUG_PRINT - // print_histogram<<<1, 32>>>(d_freq, dict_size, dict_size / - // 2); cudaDeviceSynchronize(); - // #endif -} - -template -void PrintChunkHuffmanCoding(size_t *dH_bit_meta, // - size_t *dH_uInt_meta, size_t len, int chunk_size, - size_t total_bits, size_t total_uInts) { - cout << "\n" << log_dbg << "Huffman coding detail start ------" << endl; - printf("| %s\t%s\t%s\t%s\t%9s\n", "chunk", "bits", "bytes", "uInt", - "chunkCR"); - for (size_t i = 0; i < 8; i++) { - size_t n_byte = (dH_bit_meta[i] - 1) / 8 + 1; - auto chunk_CR = ((double)chunk_size * sizeof(float) / - (1.0 * (double)dH_uInt_meta[i] * sizeof(H))); - printf("| %lu\t%lu\t%lu\t%lu\t%9.6lf\n", i, dH_bit_meta[i], n_byte, - dH_uInt_meta[i], chunk_CR); - } - cout << "| ..." << endl - << "| Huff.total.bits:\t" << total_bits << endl - << "| Huff.total.bytes:\t" << total_uInts * sizeof(H) << endl - << "| Huff.CR (uInt):\t" - << (double)len * sizeof(float) / (total_uInts * 1.0 * sizeof(H)) << endl; - cout << log_dbg << "coding detail end ----------------" << endl; - cout << endl; -} - -template -void HuffmanEncode(mgard_cuda::Handle &handle, S *dqv, size_t n, - std::vector &outlier_idx, H *&dmeta, - size_t &dmeta_size, H *&ddata, size_t &ddata_size, - int chunk_size, int dict_size) { - - high_resolution_clock::time_point t1, t2, start, end; - duration time_span; - - // high_resolution_clock::time_point t1 = high_resolution_clock::now(); - - // size_t outlier_count = outlier_idx.size(); - // size_t primary_count = n - outlier_count; - - // // printf("compress outlier_idx: "); for(int i = 0; i < outlier_count; i++) - // {printf("%llu ", outlier_idx[i]);} printf("\n"); - - // printf("compress outlier_count: %llu\n", outlier_count); - // printf("compress primary_count: %llu\n", primary_count); - - // Q * dprimary; - // S * doutlier; - // mgard_cuda::cudaMallocHelper(handle, (void **)&dprimary, primary_count * - // sizeof(Q)); mgard_cuda::cudaMallocHelper(handle, (void **)&doutlier, - // outlier_count - // * sizeof(S)); - - // // for (int i = 0 ; i < outlier_count; i++) { printf("%d\n", - // outlier_idx[i]);} - - int queue_idx = 0; - - // size_t p = 0; - // size_t pp = 0; - // size_t op = 0; - // size_t size = outlier_idx[0] - 0; - // // printf("copy primary\n"); - // if (size > 0) { mgard_cuda::cudaMemcpyAsyncHelper(handle, dprimary + pp, - // dqv + p, size * sizeof(Q), mgard_cuda::D2D, - // (queue_idx++)%handle.num_of_queues); } pp += size; p += size; - - // for (int i = 0; i < outlier_idx.size() - 1; i++) { - // size = 1; - // // printf("copy outlier\n"); - // mgard_cuda::cudaMemcpyAsyncHelper(handle, doutlier + op, dqv + p, size * - // sizeof(S), mgard_cuda::D2D, (queue_idx++)%handle.num_of_queues); op += - // size; p += size; size = outlier_idx[i + 1] - outlier_idx[i] - 1; - // // printf("copy primary %d %d %d\n", p, size, - // outlier_idx[outlier_idx.size() - 1]); if (size > 0) { - // mgard_cuda::cudaMemcpyAsyncHelper(handle, dprimary + pp, dqv + p, size * - // sizeof(Q), mgard_cuda::D2D, (queue_idx++)%handle.num_of_queues); } pp += - // size; p += size; - // } - // size = 1; - // // printf("copy outlier\n"); - // mgard_cuda::cudaMemcpyAsyncHelper(handle, doutlier + op, dqv + p, size * - // sizeof(S), mgard_cuda::D2D, (queue_idx++)%handle.num_of_queues); op += - // size; p += size; size = n - outlier_idx[outlier_idx.size() - 1] - 1; - // // printf("copy primary %d %d %d\n", p, size, - // outlier_idx[outlier_idx.size() - 1]); if (size > 0) { - // mgard_cuda::cudaMemcpyAsyncHelper(handle, dprimary + pp, dqv + p, size * - // sizeof(Q), mgard_cuda::D2D, (queue_idx++)%handle.num_of_queues); } - // // printf("done copy primary\n"); - // pp += size; - // p += size; - - // if (pp != primary_count || op != outlier_count) { printf("Primary or - // outlier size mismatch!\n"); } handle.sync_all(); - - // gpuErrchk(cudaDeviceSynchronize()); - - Q *dprimary = (Q *)dqv; - size_t primary_count = n; - // printf("primary_count: %lld\n", primary_count); - - // high_resolution_clock::time_point t2 = high_resolution_clock::now(); - // duration time_span = duration_cast>(t2 - t1); - // printf("separate time: %.6f s\n", time_span.count()); - - t1 = high_resolution_clock::now(); - // start huffman - // histogram - ht_state_num = 2 * dict_size; - ht_all_nodes = 2 * ht_state_num; - auto freq = mem::CreateCUDASpace(ht_all_nodes); - wrapper::GetFrequency(dprimary, primary_count, freq, dict_size); - gpuErrchk(cudaDeviceSynchronize()); - - // Allocate cb memory - auto codebook = mem::CreateCUDASpace(dict_size, 0xff); - // canonical Huffman; follows H to decide first and entry type - auto type_bw = sizeof(H) * 8; - // first, entry, reversed codebook - // CHANGED first and entry to H type - size_t decodebook_size = sizeof(H) * (2 * type_bw) + sizeof(S) * dict_size; - uint8_t *decodebook = mem::CreateCUDASpace(decodebook_size); - - // Get codebooks - ParGetCodebook(dict_size, freq, codebook, decodebook); - cudaDeviceSynchronize(); - - // auto decode_meta = mem::CreateHostSpaceAndMemcpyFromDevice(d_decode_meta, - // decode_meta_size); gpuErrchk(cudaDeviceSynchronize()); - - // Non-deflated output - auto huff = mem::CreateCUDASpace(primary_count); - gpuErrchk(cudaDeviceSynchronize()); - // fix-length space - auto blockDim = tBLK_ENCODE; - auto gridDim = (primary_count - 1) / blockDim + 1; - EncodeFixedLen - <<>>(dprimary, huff, primary_count, codebook); - gpuErrchk(cudaDeviceSynchronize()); - - // deflate - auto nchunk = (primary_count - 1) / chunk_size + 1; // | - auto huff_bitwidths = mem::CreateCUDASpace(nchunk); - blockDim = tBLK_DEFLATE; - gridDim = (nchunk - 1) / blockDim + 1; - Deflate - <<>>(huff, primary_count, huff_bitwidths, chunk_size); - gpuErrchk(cudaDeviceSynchronize()); - - // dump TODO change to int - auto h_meta = new size_t[nchunk * 3](); - auto dH_uInt_meta = h_meta; - auto dH_bit_meta = h_meta + nchunk; - auto dH_uInt_entry = h_meta + nchunk * 2; - // copy back densely Huffman code (dHcode) - cudaMemcpy(dH_bit_meta, huff_bitwidths, nchunk * sizeof(size_t), - cudaMemcpyDeviceToHost); - gpuErrchk(cudaDeviceSynchronize()); - // transform in uInt - memcpy(dH_uInt_meta, dH_bit_meta, nchunk * sizeof(size_t)); - for_each(dH_uInt_meta, dH_uInt_meta + nchunk, - [&](size_t &i) { i = (i - 1) / (sizeof(H) * 8) + 1; }); - // make it entries - memcpy(dH_uInt_entry + 1, dH_uInt_meta, (nchunk - 1) * sizeof(size_t)); - for (auto i = 1; i < nchunk; i++) - dH_uInt_entry[i] += dH_uInt_entry[i - 1]; - - // sum bits from each chunk - auto total_bits = - std::accumulate(dH_bit_meta, dH_bit_meta + nchunk, (size_t)0); - auto total_uInts = - std::accumulate(dH_uInt_meta, dH_uInt_meta + nchunk, (size_t)0); - - gpuErrchk(cudaDeviceSynchronize()); - t2 = high_resolution_clock::now(); - time_span = duration_cast>(t2 - t1); - // printf("huffman encode time: %.6f s\n", time_span.count()); - - // out_meta: |outlier count|outlier idx|outlier data|primary count|dict - // size|chunk size|huffmeta size|huffmeta|decodebook size|decodebook| - // out_data: |huffman data| - - t1 = high_resolution_clock::now(); - dmeta_size = // sizeof(size_t) + outlier_count * sizeof(size_t) + - // outlier_count * sizeof(S) + //outlier - sizeof(size_t) + sizeof(int) + sizeof(int) + // primary - sizeof(size_t) + 2 * nchunk * sizeof(size_t) + sizeof(size_t) + - (sizeof(H) * (2 * type_bw) + sizeof(S) * dict_size) * sizeof(uint8_t); - - mgard_cuda::cudaMallocHelper(handle, (void **)&dmeta, dmeta_size); - ddata_size = total_uInts * sizeof(H); - mgard_cuda::cudaMallocHelper(handle, (void **)&ddata, ddata_size); - - void *dmeta_p = (void *)dmeta; - // //outlier - // mgard_cuda::cudaMemcpyAsyncHelper(handle, dmeta_p, &outlier_count, - // sizeof(size_t), mgard_cuda::H2D, (queue_idx++)%handle.num_of_queues); - // dmeta_p = dmeta_p + sizeof(size_t); - // mgard_cuda::cudaMemcpyAsyncHelper(handle, dmeta_p, outlier_idx.data(), - // outlier_count * sizeof(size_t), mgard_cuda::H2D, - // (queue_idx++)%handle.num_of_queues); dmeta_p = dmeta_p + outlier_count * - // sizeof(size_t); mgard_cuda::cudaMemcpyAsyncHelper(handle, dmeta_p, - // doutlier, outlier_count * sizeof(S), mgard_cuda::D2D, - // (queue_idx++)%handle.num_of_queues); dmeta_p = dmeta_p + outlier_count * - // sizeof(S); - - // primary - mgard_cuda::cudaMemcpyAsyncHelper(handle, dmeta_p, &primary_count, - sizeof(size_t), mgard_cuda::H2D, - (queue_idx++) % handle.num_of_queues); - dmeta_p = dmeta_p + sizeof(size_t); - mgard_cuda::cudaMemcpyAsyncHelper(handle, dmeta_p, &dict_size, sizeof(int), - mgard_cuda::H2D, - (queue_idx++) % handle.num_of_queues); - dmeta_p = dmeta_p + sizeof(int); - mgard_cuda::cudaMemcpyAsyncHelper(handle, dmeta_p, &chunk_size, sizeof(int), - mgard_cuda::H2D, - (queue_idx++) % handle.num_of_queues); - dmeta_p = dmeta_p + sizeof(int); - size_t huffmeta_size = 2 * nchunk * sizeof(size_t); - // printf("compress huffmeta_size: %llu\n", huffmeta_size); - mgard_cuda::cudaMemcpyAsyncHelper(handle, dmeta_p, &huffmeta_size, - sizeof(size_t), mgard_cuda::H2D, - (queue_idx++) % handle.num_of_queues); - dmeta_p = dmeta_p + sizeof(size_t); - mgard_cuda::cudaMemcpyAsyncHelper(handle, dmeta_p, h_meta + nchunk, - huffmeta_size, mgard_cuda::H2D, - (queue_idx++) % handle.num_of_queues); - dmeta_p = dmeta_p + huffmeta_size; - mgard_cuda::cudaMemcpyAsyncHelper(handle, dmeta_p, &decodebook_size, - sizeof(size_t), mgard_cuda::H2D, - (queue_idx++) % handle.num_of_queues); - dmeta_p = dmeta_p + sizeof(size_t); - // printf("compress decodebook_size: %llu\n", decodebook_size); - mgard_cuda::cudaMemcpyAsyncHelper(handle, dmeta_p, decodebook, - decodebook_size, mgard_cuda::H2D, - (queue_idx++) % handle.num_of_queues); - dmeta_p = dmeta_p + decodebook_size; - - gpuErrchk(cudaDeviceSynchronize()); - t2 = high_resolution_clock::now(); - time_span = duration_cast>(t2 - t1); - // printf("serilization time1: %.6f s\n", time_span.count()); - - t1 = high_resolution_clock::now(); - - for (auto i = 0; i < nchunk; i++) { - mgard_cuda::cudaMemcpyAsyncHelper( - handle, ddata + dH_uInt_entry[i], (void *)(huff + i * chunk_size), - dH_uInt_meta[i] * sizeof(H), mgard_cuda::D2D, - (queue_idx++) % handle.num_of_queues); - } - - gpuErrchk(cudaDeviceSynchronize()); - t2 = high_resolution_clock::now(); - time_span = duration_cast>(t2 - t1); - // printf("serilization time2: %.6f s\n", time_span.count()); - - //////// clean up - // cudaFreeHost(flags); - // cudaFree(doutlier); - // cudaFree(dprimary); - cudaFree(freq); - cudaFree(codebook); - cudaFree(decodebook); - cudaFree(huff); - cudaFree(huff_bitwidths); - delete[] h_meta; -} - -template -void HuffmanDecode(mgard_cuda::Handle &handle, S *&dqv, size_t &n, - H *dmeta, size_t dmeta_size, H *ddata, size_t ddata_size) { - - Q *dprimary; - S *doutlier; - size_t primary_count; - size_t outlier_count; - size_t *outlier_idx; - size_t huffmeta_size; - - int dict_size; - int chunk_size; - size_t *huffmeta; - uint8_t *decodebook; - size_t decodebook_size; - - void *dmeta_p = (void *)dmeta; - - // outlier - // mgard_cuda::cudaMemcpyAsyncHelper(handle, &outlier_counD, Tmeta_p, - // sizeof(size_t), mgard_cuda::D2H, 0); dmeta_p = dmeta_p + sizeof(size_t); - // // printf("decompress outlier_count: %llu\n", outlier_count); - // mgard_cuda::cudaMallocHelper(handle, (void**)&doutlier, - // outlier_count*sizeof(S)); outlier_idx = new size_t[outlier_count]; - // mgard_cuda::cudaMemcpyAsyncHelper(handle, outlier_idx, dmeta_p, - // outlier_count * sizeof(size_t), mgard_cuda::D2H, 0); - // // printf("decompress outlier_idx: "); for(int i = 0; i < outlier_count; - // i++) {printf("%llu ", outlier_idx[i]);} printf("\n"); dmeta_p = dmeta_p + - // outlier_count * sizeof(size_t); mgard_cuda::cudaMemcpyAsyncHelper(handle, - // doutlier, dmeta_p, outlier_count * sizeof(S), mgard_cuda::D2D, 0); - // // doutlier = (Q *)dmeta_p; - // dmeta_p = dmeta_p + outlier_count * sizeof(S); - - // primary - mgard_cuda::cudaMemcpyAsyncHelper(handle, &primary_count, dmeta_p, - sizeof(size_t), mgard_cuda::D2H, 0); - dmeta_p = dmeta_p + sizeof(size_t); - // printf("decompress primary_count: %llu\n", primary_count); - mgard_cuda::cudaMallocHelper(handle, (void **)&dprimary, - primary_count * sizeof(Q)); - - mgard_cuda::cudaMemcpyAsyncHelper(handle, &dict_size, dmeta_p, sizeof(int), - mgard_cuda::D2H, 0); - dmeta_p = dmeta_p + sizeof(int); - mgard_cuda::cudaMemcpyAsyncHelper(handle, &chunk_size, dmeta_p, sizeof(int), - mgard_cuda::D2H, 0); - dmeta_p = dmeta_p + sizeof(int); - mgard_cuda::cudaMemcpyAsyncHelper(handle, &huffmeta_size, dmeta_p, - sizeof(size_t), mgard_cuda::D2H, 0); - dmeta_p = dmeta_p + sizeof(size_t); - // printf("decompress huffmeta_size: %llu\n", huffmeta_size); - mgard_cuda::cudaMallocHelper(handle, (void **)&huffmeta, huffmeta_size); - mgard_cuda::cudaMemcpyAsyncHelper(handle, huffmeta, dmeta_p, huffmeta_size, - mgard_cuda::D2D, 0); - // // huffmeta = (size_t *)dmeta_p; - dmeta_p = dmeta_p + huffmeta_size; - mgard_cuda::cudaMemcpyAsyncHelper(handle, &decodebook_size, dmeta_p, - sizeof(size_t), mgard_cuda::D2H, 0); - dmeta_p = dmeta_p + sizeof(size_t); - // printf("decompress decodebook_size: %llu\n", decodebook_size); - mgard_cuda::cudaMallocHelper(handle, (void **)&decodebook, decodebook_size); - mgard_cuda::cudaMemcpyAsyncHelper(handle, decodebook, dmeta_p, - decodebook_size, mgard_cuda::D2D, 0); - // // decodebook = (uint8_t *)dmeta_p; - dmeta_p = dmeta_p + decodebook_size; - - // printf("start decoding\n"); - int nchunk = (primary_count - 1) / chunk_size + 1; - auto blockDim = tBLK_DEFLATE; // the same as deflating - auto gridDim = (nchunk - 1) / blockDim + 1; - - Decode<<>>( // - ddata, huffmeta, dprimary, primary_count, chunk_size, nchunk, - (uint8_t *)decodebook, (size_t)decodebook_size); - cudaDeviceSynchronize(); - - dqv = (S *)dprimary; - n = primary_count; - - // n = primary_count + outlier_count; - // // printf("start combine\n"); - // mgard_cuda::cudaMallocHelper(handle, (void **)&dqv, n * sizeof(S)); - - // size_t p = 0; - // size_t pp = 0; - // size_t op = 0; - // size_t size = outlier_idx[0] - 0; - // // printf("copy primary\n"); - // if (size > 0) { mgard_cuda::cudaMemcpyAsyncHelper(handle, dqv + p, dprimary - // + pp, size * sizeof(Q), mgard_cuda::D2D, 0); } pp += size; p += size; - - // for (int i = 0; i < outlier_count - 1; i++) { - // size = 1; - // // printf("copy outlier\n"); - // mgard_cuda::cudaMemcpyAsyncHelper(handle, dqv + p, doutlier + op, size * - // sizeof(S), mgard_cuda::D2D, 0); op += size; p += size; size = - // outlier_idx[i + 1] - outlier_idx[i] - 1; - // // printf("copy primary %d %d %d\n", p, size, - // outlier_idx[outlier_idx.size() - 1]); if (size > 0) { - // mgard_cuda::cudaMemcpyAsyncHelper(handle, dqv + p, dprimary + pp, size * - // sizeof(Q), mgard_cuda::D2D, 0); } pp += size; p += size; - // } - // size = 1; - // // printf("copy outlier\n"); - // mgard_cuda::cudaMemcpyAsyncHelper(handle, dqv + p, doutlier + op, size * - // sizeof(S), mgard_cuda::D2D, 0); op += size; p += size; size = n - - // outlier_idx[outlier_count - 1] - 1; - // // printf("copy primary %d %d %d\n", p, size, - // outlier_idx[outlier_idx.size() - 1]); if (size > 0) { - // mgard_cuda::cudaMemcpyAsyncHelper(handle, dqv + p, dprimary + pp, size * - // sizeof(Q), mgard_cuda::D2D, 0); } - // // printf("done copy primary\n"); - // pp += size; - // p += size; - - // Q*hqv = new Q[output_count]; - // mgard_cuda::cudaMemcpyAsyncHelper(handle, hqv, dqv, output_count * - // sizeof(Q), mgard_cuda::D2H, 0); Q*hqv2 = new Q[output_count]; - // mgard_cuda::cudaMemcpyAsyncHelper(handle, hqv2, dqv2, output_count * - // sizeof(Q), mgard_cuda::D2H, 0); for (int i = 0; i < output_count; i++) { - // if(hqv[i] != hqv2[i]) { - // cout << "diff at " << i << " " << hqv[i] << " - " << hqv2[i] << - // endl; - // } - // } -} - -template void wrapper::GetFrequency(uint8__t *, size_t, - unsigned int *, int); -template void wrapper::GetFrequency(uint16_t *, size_t, - unsigned int *, int); -template void wrapper::GetFrequency(uint32_t *, size_t, - unsigned int *, int); - -template void PrintChunkHuffmanCoding(size_t *, size_t *, size_t, int, - size_t, size_t); -template void PrintChunkHuffmanCoding(size_t *, size_t *, size_t, int, - size_t, size_t); - -// template tuple3ul HuffmanEncode(Handle -// &, string&, uint8__t*, size_t, void * &, size_t &, int, int); template -// tuple3ul HuffmanEncode(Handle &, -// string&, uint16_t*, size_t, void * &, size_t &, int, int); template tuple3ul -// HuffmanEncode(Handle &, string&, -// uint32_t*, size_t, void * &, size_t &, int, int); template tuple3ul -// HuffmanEncode(Handle &, string&, -// uint8__t*, size_t, void * &, size_t &, int, int); template tuple3ul -// HuffmanEncode(Handle &, string&, -// uint16_t*, size_t, void * &, size_t &, int, int); - -// template uint8__t* HuffmanDecode(std::string&, -// void * d_in, size_t, int, int, int); template uint16_t* -// HuffmanDecode(std::string&, void * d_in, size_t, -// int, int, int); template uint32_t* HuffmanDecode(std::string&, void * d_in, size_t, int, int, int); template uint8__t* -// HuffmanDecode(std::string&, void * d_in, size_t, -// int, int, int); template uint16_t* HuffmanDecode(std::string&, void * d_in, size_t, int, int, int); template uint32_t* -// HuffmanDecode(std::string&, void * d_in, size_t, -// int, int, int); - -// template void HuffmanEncode(Handle &handle, -// int* dqv, size_t n, bool * dflags, uint32_t * &dmeta, size_t &dmeta_size, -// uint32_t * &ddata, size_t &ddata_size, int chunk_size, int dict_size); -// template void HuffmanEncode(Handle &handle, -// int* dqv, size_t n, bool * dflags, uint32_t * &dmeta, size_t &dmeta_size, -// uint32_t * &ddata, size_t &ddata_size, int chunk_size, int dict_size); - -// template void HuffmanDecode(Handle &handle, -// int* &dqv, size_t &n, uint32_t * dmeta, size_t -// dmeta_size, uint32_t * ddata, size_t ddata_size); -// template void HuffmanDecode(Handle &handle, -// int* &dqv, size_t &n, uint32_t * dmeta, size_t -// dmeta_size, uint32_t * ddata, size_t ddata_size); - -#define KERNELS(D, T, S, Q, H) \ - template void HuffmanEncode( \ - mgard_cuda::Handle & handle, S * dqv, size_t n, \ - std::vector & outlier_idx, H * &dmeta, size_t & dmeta_size, \ - H * &ddata, size_t & ddata_size, int chunk_size, int dict_size); \ - template void HuffmanDecode( \ - mgard_cuda::Handle & handle, S * &dqv, size_t & n, H * dmeta, \ - size_t dmeta_size, H * ddata, size_t ddata_size); - -KERNELS(1, double, int, uint32_t, uint32_t) -KERNELS(1, float, int, uint32_t, uint32_t) -KERNELS(2, double, int, uint32_t, uint32_t) -KERNELS(2, float, int, uint32_t, uint32_t) -KERNELS(3, double, int, uint32_t, uint32_t) -KERNELS(3, float, int, uint32_t, uint32_t) -KERNELS(4, double, int, uint32_t, uint32_t) -KERNELS(4, float, int, uint32_t, uint32_t) -KERNELS(5, double, int, uint32_t, uint32_t) -KERNELS(5, float, int, uint32_t, uint32_t) -KERNELS(1, double, int, uint32_t, uint64_t) -KERNELS(1, float, int, uint32_t, uint64_t) -KERNELS(2, double, int, uint32_t, uint64_t) -KERNELS(2, float, int, uint32_t, uint64_t) -KERNELS(3, double, int, uint32_t, uint64_t) -KERNELS(3, float, int, uint32_t, uint64_t) -KERNELS(4, double, int, uint32_t, uint64_t) -KERNELS(4, float, int, uint32_t, uint64_t) -KERNELS(5, double, int, uint32_t, uint64_t) -KERNELS(5, float, int, uint32_t, uint64_t) - -// clang-format off diff --git a/src/cuda/ParallelHuffman/par_huffman.cu b/src/cuda/ParallelHuffman/par_huffman.cu deleted file mode 100644 index 96f6cbf7d6..0000000000 --- a/src/cuda/ParallelHuffman/par_huffman.cu +++ /dev/null @@ -1,690 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "cuda/ParallelHuffman/cuda_error_handling.cuh" -#include "cuda/ParallelHuffman/cuda_mem.cuh" -#include "cuda/ParallelHuffman/dbg_gpu_printing.cuh" -#include "cuda/ParallelHuffman/format.hh" -#include "cuda/ParallelHuffman/par_huffman.cuh" -#include "cuda/ParallelHuffman/par_merge.cuh" - -__device__ int iNodesFront = 0; -__device__ int iNodesRear = 0; -__device__ int lNodesCur = 0; - -__device__ int iNodesSize = 0; -__device__ int curLeavesNum; - -__device__ int minFreq; - -__device__ int tempLength; - -__device__ int mergeFront; -__device__ int mergeRear; - -__device__ int lNodesIndex; - -// GenerateCW Locals -__device__ int CCL; -__device__ int CDPI; -__device__ int newCDPI; - -// Profiling -__device__ long long int s[10]; -__device__ long long int st[10]; - -// Mathematically correct mod -#define MOD(a, b) ((((a) % (b)) + (b)) % (b)) - -// Parallel huffman code generation -// clang-format off -template -__global__ void parHuff::GPU_GenerateCL( - F* histogram, F* CL, int size, - /* Global Arrays */ - F* lNodesFreq, int* lNodesLeader, - F* iNodesFreq, int* iNodesLeader, - F* tempFreq, int* tempIsLeaf, int* tempIndex, - F* copyFreq, int* copyIsLeaf, int* copyIndex, - uint32_t* diagonal_path_intersections, int mblocks, int mthreads) -{ - // clang-format on - - extern __shared__ int32_t shmem[]; - // Shared variables - int32_t &x_top = shmem[0]; - int32_t &y_top = shmem[1]; - int32_t &x_bottom = shmem[2]; - int32_t &y_bottom = shmem[3]; - int32_t &found = shmem[4]; - int32_t *oneorzero = &shmem[5]; - - unsigned int thread = (blockIdx.x * blockDim.x) + threadIdx.x; - const unsigned int i = thread; // Adaptation for easier porting - auto current_grid = this_grid(); - - /* Initialization */ - if (thread < size) { - lNodesLeader[i] = -1; - CL[i] = 0; - } - - if (thread == 0) { - iNodesFront = 0; - iNodesRear = 0; - lNodesCur = 0; - - iNodesSize = 0; - } - current_grid.sync(); - - /* While there is not exactly one internal node */ - while (lNodesCur < size || iNodesSize > 1) { - /* Combine two most frequent nodes on same level */ - if (thread == 0) { - F midFreq[4]; - int midIsLeaf[4]; - for (int i = 0; i < 4; ++i) - midFreq[i] = UINT_MAX; - - if (lNodesCur < size) { - midFreq[0] = lNodesFreq[lNodesCur]; - midIsLeaf[0] = 1; - } - if (lNodesCur < size - 1) { - midFreq[1] = lNodesFreq[lNodesCur + 1]; - midIsLeaf[1] = 1; - } - if (iNodesSize >= 1) { - midFreq[2] = iNodesFreq[iNodesFront]; - midIsLeaf[2] = 0; - } - if (iNodesSize >= 2) { - midFreq[3] = iNodesFreq[MOD(iNodesFront + 1, size)]; - midIsLeaf[3] = 0; - } - - /* Select the minimum of minimums - 4elt sorting network */ - /* TODO There's likely a good 1-warp faster way to do this */ - { - F tempFreq; - int tempIsLeaf; - if (midFreq[1] > midFreq[3]) { - tempFreq = midFreq[1]; - midFreq[1] = midFreq[3]; - midFreq[3] = tempFreq; - tempIsLeaf = midIsLeaf[1]; - midIsLeaf[1] = midIsLeaf[3]; - midIsLeaf[3] = tempIsLeaf; - } - if (midFreq[0] > midFreq[2]) { - tempFreq = midFreq[0]; - midFreq[0] = midFreq[2]; - midFreq[2] = tempFreq; - tempIsLeaf = midIsLeaf[0]; - midIsLeaf[0] = midIsLeaf[2]; - midIsLeaf[2] = tempIsLeaf; - } - if (midFreq[0] > midFreq[1]) { - tempFreq = midFreq[0]; - midFreq[0] = midFreq[1]; - midFreq[1] = tempFreq; - tempIsLeaf = midIsLeaf[0]; - midIsLeaf[0] = midIsLeaf[1]; - midIsLeaf[1] = tempIsLeaf; - } - if (midFreq[2] > midFreq[3]) { - tempFreq = midFreq[2]; - midFreq[2] = midFreq[3]; - midFreq[3] = tempFreq; - tempIsLeaf = midIsLeaf[2]; - midIsLeaf[2] = midIsLeaf[3]; - midIsLeaf[3] = tempIsLeaf; - } - if (midFreq[1] > midFreq[2]) { - tempFreq = midFreq[1]; - midFreq[1] = midFreq[2]; - midFreq[2] = tempFreq; - tempIsLeaf = midIsLeaf[1]; - midIsLeaf[1] = midIsLeaf[2]; - midIsLeaf[2] = tempIsLeaf; - } - } - - minFreq = midFreq[0]; - if (midFreq[1] < UINT_MAX) { - minFreq += midFreq[1]; - } - iNodesFreq[iNodesRear] = minFreq; - iNodesLeader[iNodesRear] = -1; - - /* If is leaf */ - if (midIsLeaf[0]) { - lNodesLeader[lNodesCur] = iNodesRear; - ++CL[lNodesCur], ++lNodesCur; - } else { - iNodesLeader[iNodesFront] = iNodesRear; - iNodesFront = MOD(iNodesFront + 1, size); - } - if (midIsLeaf[1]) { - lNodesLeader[lNodesCur] = iNodesRear; - ++CL[lNodesCur], ++lNodesCur; - } else { - iNodesLeader[iNodesFront] = iNodesRear; - iNodesFront = MOD(iNodesFront + 1, size); /* ? */ - } - - // iNodesRear = MOD(iNodesRear + 1, size); - - iNodesSize = MOD(iNodesRear - iNodesFront, size); - } - - // int curLeavesNum; - /* Select elements to copy -- parallelized */ - curLeavesNum = 0; - current_grid.sync(); - if (i >= lNodesCur && i < size) { - // Parallel component - int threadCurLeavesNum; - if (lNodesFreq[i] <= minFreq) { - threadCurLeavesNum = i - lNodesCur + 1; - // Atomic max -- Largest valid index - atomicMax(&curLeavesNum, threadCurLeavesNum); - } - - if (i - lNodesCur < curLeavesNum) { - copyFreq[i - lNodesCur] = lNodesFreq[i]; - copyIndex[i - lNodesCur] = i; - copyIsLeaf[i - lNodesCur] = 1; - } - } - - current_grid.sync(); - - /* Updates Iterators */ - if (thread == 0) { - mergeRear = iNodesRear; - mergeFront = iNodesFront; - - if ((curLeavesNum + iNodesSize) % 2 == 0) { - iNodesFront = iNodesRear; - } - /* Odd number of nodes to merge - leave out one*/ - else if ((iNodesSize != 0) // - and (curLeavesNum == 0 // - or (histogram[lNodesCur + curLeavesNum] <= - iNodesFreq[MOD(iNodesRear - 1, size)])) // - ) { - mergeRear = MOD(mergeRear - 1, size); - iNodesFront = MOD(iNodesRear - 1, size); - } else { - iNodesFront = iNodesRear; - --curLeavesNum; - } - - lNodesCur = lNodesCur + curLeavesNum; - iNodesRear = MOD(iNodesRear + 1, size); - } - current_grid.sync(); - - /* Parallelized Merging Phase */ - - /*if (thread == 0) { - merge(copyFreq, copyIndex, copyIsLeaf, 0, curLeavesNum, - iNodesFreq, mergeFront, mergeRear, size, - tempFreq, tempIndex, tempIsLeaf, tempLength); - }*/ - - parMerge(copyFreq, copyIndex, copyIsLeaf, 0, curLeavesNum, // - iNodesFreq, mergeFront, mergeRear, size, // - tempFreq, tempIndex, tempIsLeaf, tempLength, // - diagonal_path_intersections, mblocks, mthreads, // - x_top, y_top, x_bottom, y_bottom, found, oneorzero); - current_grid.sync(); - - /* Melding phase -- New */ - if (thread < tempLength / 2) { - int ind = MOD(iNodesRear + i, size); - iNodesFreq[ind] = tempFreq[(2 * i)] + tempFreq[(2 * i) + 1]; - iNodesLeader[ind] = -1; - - if (tempIsLeaf[(2 * i)]) { - lNodesLeader[tempIndex[(2 * i)]] = ind; - ++CL[tempIndex[(2 * i)]]; - } else { - iNodesLeader[tempIndex[(2 * i)]] = ind; - } - if (tempIsLeaf[(2 * i) + 1]) { - lNodesLeader[tempIndex[(2 * i) + 1]] = ind; - ++CL[tempIndex[(2 * i) + 1]]; - } else { - iNodesLeader[tempIndex[(2 * i) + 1]] = ind; - } - } - current_grid.sync(); - - if (thread == 0) { - iNodesRear = MOD(iNodesRear + (tempLength / 2), size); - } - current_grid.sync(); - - /* Update leaders */ - if (thread < size) { - if (lNodesLeader[i] != -1) { - if (iNodesLeader[lNodesLeader[i]] != -1) { - lNodesLeader[i] = iNodesLeader[lNodesLeader[i]]; - ++CL[i]; - } - } - } - current_grid.sync(); - - if (thread == 0) { - iNodesSize = MOD(iNodesRear - iNodesFront, size); - } - current_grid.sync(); - } -} - -// Parallelized with atomic writes, but could replace with Jiannan's similar -// code -template -__global__ void parHuff::GPU_GenerateCW(F *CL, H *CW, H *first, H *entry, - int size) { - unsigned int thread = (blockIdx.x * blockDim.x) + threadIdx.x; - const unsigned int i = thread; // Porting convenience - auto current_grid = this_grid(); - auto type_bw = sizeof(H) * 8; - - /* Reverse in place - Probably a more CUDA-appropriate way */ - if (thread < size / 2) { - F temp = CL[i]; - CL[i] = CL[size - i - 1]; - CL[size - i - 1] = temp; - } - current_grid.sync(); - - if (thread == 0) { - CCL = CL[0]; - CDPI = 0; - newCDPI = size - 1; - entry[CCL] = 0; - - // Edge case -- only one input symbol - CW[CDPI] = 0; - first[CCL] = CW[CDPI] ^ (((H)1 << (H)CL[CDPI]) - 1); - entry[CCL + 1] = 1; - } - current_grid.sync(); - - // Initialize first and entry arrays - if (thread < CCL) { - // Initialization of first to Max ensures that unused code - // lengths are skipped over in decoding. - first[i] = std::numeric_limits::max(); - entry[i] = 0; - } - // Initialize first element of entry - current_grid.sync(); - - while (CDPI < size - 1) { - // CDPI update - if (i < size - 1 && CL[i + 1] > CCL) { - atomicMin(&newCDPI, i); - } - current_grid.sync(); - - // Last element to update - const int updateEnd = (newCDPI >= size - 1) ? type_bw : CL[newCDPI + 1]; - // Fill base - const int curEntryVal = entry[CCL]; - // Number of elements of length CCL - const int numCCL = (newCDPI - CDPI + 1); - - // Get first codeword - if (i == 0) { - if (CDPI == 0) { - CW[newCDPI] = 0; - } else { - CW[newCDPI] = CW[CDPI]; // Pre-stored - } - } - current_grid.sync(); - - if (i < size) { - // Parallel canonical codeword generation - if (i >= CDPI && i < newCDPI) { - CW[i] = CW[newCDPI] + (newCDPI - i); - } - } - - // Update entry and first arrays in O(1) time - if (thread > CCL && thread < updateEnd) { - entry[i] = curEntryVal + numCCL; - } - // Add number of entries to next CCL - if (thread == 0) { - if (updateEnd < type_bw) { - entry[updateEnd] = curEntryVal + numCCL; - } - } - current_grid.sync(); - - // Update first array in O(1) time - if (thread == CCL) { - // Flip least significant CL[CDPI] bits - first[CCL] = CW[CDPI] ^ (((H)1 << (H)CL[CDPI]) - 1); - } - if (thread > CCL && thread < updateEnd) { - first[i] = std::numeric_limits::max(); - } - current_grid.sync(); - - if (thread == 0) { - if (newCDPI < size - 1) { - int CLDiff = CL[newCDPI + 1] - CL[newCDPI]; - // Add and shift -- Next canonical code - CW[newCDPI + 1] = ((CW[CDPI] + 1) << CLDiff); - CCL = CL[newCDPI + 1]; - - ++newCDPI; - } - - // Update CDPI to newCDPI after codeword length increase - CDPI = newCDPI; - newCDPI = size - 1; - } - current_grid.sync(); - } - - if (thread < size) { - CW[i] = (CW[i] | (((H)CL[i] & (H)0xffu) << ((sizeof(H) * 8) - 8))) ^ - (((H)1 << (H)CL[i]) - 1); - } - current_grid.sync(); - - /* Reverse partial codebook */ - if (thread < size / 2) { - H temp = CW[i]; - CW[i] = CW[size - i - 1]; - CW[size - i - 1] = temp; - } -} - -// Helper implementations -template -__global__ void GPU_FillArraySequence(T *array, unsigned int size) { - unsigned int thread = (blockIdx.x * blockDim.x) + threadIdx.x; - if (thread < size) { - array[thread] = thread; - } -} - -// Precondition -- Result is preset to be equal to size -template -__global__ void GPU_GetFirstNonzeroIndex(T *array, unsigned int size, - unsigned int *result) { - unsigned int thread = (blockIdx.x * blockDim.x) + threadIdx.x; - if (array[thread] != 0) { - atomicMin(result, thread); - } -} - -__global__ void GPU_GetMaxCWLength(unsigned int *CL, unsigned int size, - unsigned int *result) { - (void)size; - unsigned int thread = (blockIdx.x * blockDim.x) + threadIdx.x; - if (thread == 0) { - *result = CL[0]; - } -} - -// Reorders given a set of indices. Programmer must ensure that all index[i] -// are unique or else race conditions may occur -template -__global__ void GPU_ReorderByIndex(T *array, Q *index, unsigned int size) { - unsigned int thread = (blockIdx.x * blockDim.x) + threadIdx.x; - T temp; - Q newIndex; - if (thread < size) { - temp = array[thread]; - newIndex = index[thread]; - array[newIndex] = temp; - } -} - -// Reverses a given array. -template -__global__ void GPU_ReverseArray(T *array, unsigned int size) { - unsigned int thread = (blockIdx.x * blockDim.x) + threadIdx.x; - if (thread < size / 2) { - T temp = array[thread]; - array[thread] = array[size - thread - 1]; - array[size - thread - 1] = temp; - } -} - -// Parallel codebook generation wrapper -template -void ParGetCodebook(int dict_size, unsigned int *_d_freq, H *_d_codebook, - uint8_t *_d_decode_meta) { - // Metadata - auto type_bw = sizeof(H) * 8; - auto _d_first = reinterpret_cast(_d_decode_meta); - auto _d_entry = reinterpret_cast(_d_decode_meta + (sizeof(H) * type_bw)); - auto _d_qcode = - reinterpret_cast(_d_decode_meta + (sizeof(H) * 2 * type_bw)); - - // Sort Qcodes by frequency - int nblocks = (dict_size / 1024) + 1; - GPU_FillArraySequence - <<>>(_d_qcode, (unsigned int)dict_size); - cudaDeviceSynchronize(); - - SortByFreq(_d_freq, _d_qcode, dict_size); - cudaDeviceSynchronize(); - - unsigned int *d_first_nonzero_index; - unsigned int first_nonzero_index = dict_size; - cudaMalloc(&d_first_nonzero_index, sizeof(unsigned int)); - cudaMemcpy(d_first_nonzero_index, &first_nonzero_index, sizeof(unsigned int), - cudaMemcpyHostToDevice); - GPU_GetFirstNonzeroIndex - <<>>(_d_freq, dict_size, d_first_nonzero_index); - cudaDeviceSynchronize(); - cudaMemcpy(&first_nonzero_index, d_first_nonzero_index, sizeof(unsigned int), - cudaMemcpyDeviceToHost); - cudaFree(d_first_nonzero_index); - - int nz_dict_size = dict_size - first_nonzero_index; - unsigned int *_nz_d_freq = _d_freq + first_nonzero_index; - H *_nz_d_codebook = _d_codebook + first_nonzero_index; - int nz_nblocks = (nz_dict_size / 1024) + 1; - - // Memory Allocation -- Perhaps put in another wrapper - // clang-format off - unsigned int *CL = nullptr; - /*unsigned int* lNodesFreq*/ int *lNodesLeader = nullptr; - unsigned int *iNodesFreq = nullptr; int *iNodesLeader = nullptr; - unsigned int *tempFreq = nullptr; int *tempIsLeaf = nullptr; int *tempIndex = nullptr; - unsigned int *copyFreq = nullptr; int *copyIsLeaf = nullptr; int *copyIndex = nullptr; - cudaMalloc(&CL, nz_dict_size * sizeof(unsigned int) ); - cudaMalloc(&lNodesLeader, nz_dict_size * sizeof(int) ); - cudaMalloc(&iNodesFreq, nz_dict_size * sizeof(unsigned int) ); - cudaMalloc(&iNodesLeader, nz_dict_size * sizeof(int) ); - cudaMalloc(&tempFreq, nz_dict_size * sizeof(unsigned int) ); - cudaMalloc(&tempIsLeaf, nz_dict_size * sizeof(int) ); - cudaMalloc(&tempIndex, nz_dict_size * sizeof(int) ); - cudaMalloc(©Freq, nz_dict_size * sizeof(unsigned int) ); - cudaMalloc(©IsLeaf, nz_dict_size * sizeof(int) ); - cudaMalloc(©Index, nz_dict_size * sizeof(int) ); - cudaMemset(CL, 0, nz_dict_size * sizeof(int) ); - // clang-format on - - // Grid configuration for CL -- based on Cooperative Groups - int cg_mblocks; - int cg_blocks_sm; - int device_id; - int mthreads = 32; // 1 warp - cudaDeviceProp deviceProp; - cudaGetDevice(&device_id); - cudaGetDeviceProperties(&deviceProp, device_id); - cudaOccupancyMaxActiveBlocksPerMultiprocessor( - &cg_blocks_sm, parHuff::GPU_GenerateCL, mthreads, - 5 * sizeof(int32_t) + 32 * sizeof(int32_t)); - cg_mblocks = deviceProp.multiProcessorCount * cg_blocks_sm; - - int ELTS_PER_SEQ_MERGE = 16; - int mblocks = std::min(cg_mblocks, (nz_dict_size / ELTS_PER_SEQ_MERGE) + 1); - - // Exit if not enough exposed parallelism -- TODO modify kernels so this is - // unneeded - int tthreads = mthreads * mblocks; - if (tthreads < nz_dict_size) { - cout << log_err << "Insufficient on-device parallelism to construct a " - << nz_dict_size << " non-zero item codebook" << endl; - cout << log_err << "Provided parallelism: " << mblocks << " blocks, " - << mthreads << " threads, " << tthreads << " total" << endl - << endl; - exit(1); - } - - uint32_t *diagonal_path_intersections; - cudaMalloc(&diagonal_path_intersections, - (2 * (mblocks + 1)) * sizeof(uint32_t)); - - // Codebook already init'ed - cudaDeviceSynchronize(); - - // Call first kernel - // Collect arguments - void *CL_Args[] = { - (void *)&_nz_d_freq, (void *)&CL, - (void *)&nz_dict_size, (void *)&_nz_d_freq, - (void *)&lNodesLeader, (void *)&iNodesFreq, - (void *)&iNodesLeader, (void *)&tempFreq, - (void *)&tempIsLeaf, (void *)&tempIndex, - (void *)©Freq, (void *)©IsLeaf, - (void *)©Index, (void *)&diagonal_path_intersections, - (void *)&mblocks, (void *)&mthreads}; - // Cooperative Launch - cudaLaunchCooperativeKernel((void *)parHuff::GPU_GenerateCL, - mblocks, mthreads, CL_Args, - 5 * sizeof(int32_t) + 32 * sizeof(int32_t)); - cudaDeviceSynchronize(); - - // Exits if the highest codeword length is greater than what - // the adaptive representation can handle - // TODO do proper cleanup - - unsigned int *d_max_CL; - unsigned int max_CL; - cudaMalloc(&d_max_CL, sizeof(unsigned int)); - GPU_GetMaxCWLength<<<1, 1>>>(CL, nz_dict_size, d_max_CL); - cudaDeviceSynchronize(); - cudaMemcpy(&max_CL, d_max_CL, sizeof(unsigned int), cudaMemcpyDeviceToHost); - cudaFree(d_max_CL); - - int max_CW_bits = (sizeof(H) * 8) - 8; - if (max_CL > max_CW_bits) { - cout << log_err << "Cannot store all Huffman codewords in " - << max_CW_bits + 8 << "-bit representation" << endl; - cout << log_err << "Huffman codeword representation requires at least " - << max_CL + 8 << " bits (longest codeword: " << max_CL << " bits)" - << endl; - exit(1); - } - - // Configure CW for 1024 threads/block - int cg_cw_mblocks = (cg_mblocks * mthreads) / 1024; - int cw_mblocks = std::min(cg_cw_mblocks, nz_nblocks); - - // Exit if not enough exposed parallelism -- TODO modify kernels so this is - // unneeded - int cw_tthreads = cw_mblocks * 1024; - if (cw_tthreads < nz_dict_size) { - cout << log_err << "Insufficient on-device parallelism to construct a " - << nz_dict_size << " non-zero item codebook" << endl; - cout << log_err << "Provided parallelism: " << cw_mblocks << " blocks, " - << 1024 << " threads, " << cw_tthreads << " total" << endl - << endl; - exit(1); - } - - void *CW_Args[] = {(void *)&CL, // - (void *)&_nz_d_codebook, // - (void *)&_d_first, // - (void *)&_d_entry, // - (void *)&nz_dict_size}; - - // Call second kernel - cudaLaunchCooperativeKernel( - (void *)parHuff::GPU_GenerateCW, // - cw_mblocks, // - 1024, // - CW_Args); - cudaDeviceSynchronize(); - -#ifdef D_DEBUG_PRINT - print_codebook<<<1, 32>>>(_d_codebook, dict_size); // PASS - cudaDeviceSynchronize(); -#endif - - // Reverse _d_qcode and _d_codebook - GPU_ReverseArray<<>>(_d_codebook, (unsigned int)dict_size); - GPU_ReverseArray<<>>(_d_qcode, (unsigned int)dict_size); - cudaDeviceSynchronize(); - - GPU_ReorderByIndex - <<>>(_d_codebook, _d_qcode, (unsigned int)dict_size); - cudaDeviceSynchronize(); - - // Cleanup - cudaFree(CL); - cudaFree(lNodesLeader); - cudaFree(iNodesFreq); - cudaFree(iNodesLeader); - cudaFree(tempFreq); - cudaFree(tempIsLeaf); - cudaFree(tempIndex); - cudaFree(copyFreq); - cudaFree(copyIsLeaf); - cudaFree(copyIndex); - cudaFree(diagonal_path_intersections); - cudaDeviceSynchronize(); - -#ifdef D_DEBUG_PRINT - print_codebook<<<1, 32>>>(_d_codebook, dict_size); // PASS - cudaDeviceSynchronize(); -#endif -} - -// Specialize wrapper -template void ParGetCodebook(int dict_size, - unsigned int *freq, - uint32_t *codebook, - uint8_t *meta); -template void ParGetCodebook(int dict_size, - unsigned int *freq, - uint64_t *codebook, - uint8_t *meta); -template void ParGetCodebook(int dict_size, - unsigned int *freq, - uint32_t *codebook, - uint8_t *meta); -template void ParGetCodebook(int dict_size, - unsigned int *freq, - uint64_t *codebook, - uint8_t *meta); -template void ParGetCodebook(int dict_size, - unsigned int *freq, - uint32_t *codebook, - uint8_t *meta); -template void ParGetCodebook(int dict_size, - unsigned int *freq, - uint64_t *codebook, - uint8_t *meta); diff --git a/src/cuda/ParallelHuffman/par_huffman_sortbyfreq.cu b/src/cuda/ParallelHuffman/par_huffman_sortbyfreq.cu deleted file mode 100644 index c2920f9ea7..0000000000 --- a/src/cuda/ParallelHuffman/par_huffman_sortbyfreq.cu +++ /dev/null @@ -1,16 +0,0 @@ -#include -#include -#include - -template void SortByFreq(K *freq, V *qcode, int size) { - using namespace thrust; - sort_by_key(device_ptr(freq), // - device_ptr(freq + size), // - device_ptr(qcode)); -} - -template void SortByFreq(unsigned int *, uint8_t *, int); -template void SortByFreq(unsigned int *, uint16_t *, - int); -template void SortByFreq(unsigned int *, uint32_t *, - int); diff --git a/src/cuda/ParallelHuffman/par_merge.cu b/src/cuda/ParallelHuffman/par_merge.cu deleted file mode 100644 index 6ef2a57cf4..0000000000 --- a/src/cuda/ParallelHuffman/par_merge.cu +++ /dev/null @@ -1,279 +0,0 @@ -#include -#include -#include -#include -#include - -#include "cuda/ParallelHuffman/par_merge.cuh" -#include - -using namespace cooperative_groups; - -#define MAX(X, Y) (((X) > (Y)) ? (X) : (Y)) -#define MIN(X, Y) (((X) < (Y)) ? (X) : (Y)) -// Mathematically correct modulo -#define MOD(a, b) ((((a) % (b)) + (b)) % (b)) - -/* MERGETYPE - * Performs merges of two sorted pseudorandom arrays of length - * Times the runs and reports on the average time Checks the output of - * each merge for correctness - */ -#define PADDING 1024 - -// clang-format off -template -__device__ void parMerge( - F* copyFreq, int* copyIndex, int* copyIsLeaf, int cStart, int cEnd, - F* iNodesFreq, int iStart, int iEnd, int iNodesCap, - F* tempFreq, int* tempIndex, int* tempIsLeaf, int& tempLength, - uint32_t* diagonal_path_intersections, int blocks, int threads, - /* Shared Memory */ - int32_t& x_top, int32_t& y_top, int32_t& x_bottom, int32_t& y_bottom, - int32_t& found, int32_t* oneorzero) -{ - // clang-format on - auto current_grid = this_grid(); - current_grid.sync(); - tempLength = (cEnd - cStart) + MOD(iEnd - iStart, iNodesCap); - - if (tempLength == 0) - return; - - // Perform the global diagonal intersection serach to divide work among SMs - cudaWorkloadDiagonals(copyFreq, copyIndex, copyIsLeaf, cStart, cEnd, // - iNodesFreq, iStart, iEnd, iNodesCap, // - diagonal_path_intersections, // - x_top, y_top, x_bottom, y_bottom, found, oneorzero); - current_grid.sync(); - - // Merge between global diagonals independently on each block - cudaMergeSinglePath(copyFreq, copyIndex, copyIsLeaf, cStart, cEnd, // - iNodesFreq, iStart, iEnd, iNodesCap, // - diagonal_path_intersections, // - tempFreq, tempIndex, tempIsLeaf, tempLength); - current_grid.sync(); -} - -/* CUDAWORKLOADDIAGONALS - * Performs a 32-wide binary search on one glboal diagonal per block to find the - * intersection with the path. This divides the workload into independent merges - * for the next step - */ -// clang-format off -template -__device__ void cudaWorkloadDiagonals( - F* copyFreq, int* copyIndex, int* copyIsLeaf, - int cStart, int cEnd, - F* iNodesFreq, - int iStart, int iEnd, int iNodesCap, - uint32_t* diagonal_path_intersections, - /* Shared Memory */ - int32_t& x_top, int32_t& y_top, int32_t& x_bottom, int32_t& y_bottom, - int32_t& found, int32_t* oneorzero) -{ - // clang-format on - uint32_t A_length = cEnd - cStart; - uint32_t B_length = MOD(iEnd - iStart, iNodesCap); - // Calculate combined index around the MergePath "matrix" - int32_t combinedIndex = - ((uint64_t)blockIdx.x * ((uint64_t)A_length + (uint64_t)B_length)) / - (uint64_t)gridDim.x; - /* - __shared__ int32_t x_top, y_top, x_bottom, y_bottom, found; - __shared__ int32_t oneorzero[32]; - */ - int threadOffset = threadIdx.x - 16; - - if (threadIdx.x < 32) { - // Figure out the coordinates of our diagonal - if (A_length >= B_length) { - x_top = MIN(combinedIndex, A_length); - y_top = combinedIndex > A_length ? combinedIndex - (A_length) : 0; - x_bottom = y_top; - y_bottom = x_top; - } else { - y_bottom = MIN(combinedIndex, B_length); - x_bottom = combinedIndex > B_length ? combinedIndex - (B_length) : 0; - y_top = x_bottom; - x_top = y_bottom; - } - } - - // if (threadIdx.x == 0) { - // printf("Diagonal block %d: (%d, %d) to (%d, %d)\n", blockIdx.x, x_top, - // y_top, x_bottom, y_bottom); - //} - - found = 0; - - // Search the diagonal - while (!found) { - // Update our coordinates within the 32-wide section of the diagonal - int32_t current_x = x_top - ((x_top - x_bottom) >> 1) - threadOffset; - int32_t current_y = y_top + ((y_bottom - y_top) >> 1) + threadOffset; - int32_t getfrom_x = current_x + cStart - 1; - // Below statement is a more efficienD, Tivmodless version of the following - // int32_t getfrom_y = MOD(iStart + current_y, iNodesCap); - int32_t getfrom_y = iStart + current_y; - - if (threadIdx.x < 32) { - if (getfrom_y >= iNodesCap) - getfrom_y -= iNodesCap; - - // Are we a '1' or '0' with respect to A[x] <= B[x] - if (current_x > (int32_t)A_length or current_y < 0) { - oneorzero[threadIdx.x] = 0; - } else if (current_y >= (int32_t)B_length || current_x < 1) { - oneorzero[threadIdx.x] = 1; - } else { - oneorzero[threadIdx.x] = - (copyFreq[getfrom_x] <= iNodesFreq[getfrom_y]) ? 1 : 0; - } - } - - __syncthreads(); - - // If we find the meeting of the '1's and '0's, we found the - // intersection of the path and diagonal - if (threadIdx.x > 0 and // - threadIdx.x < 32 and // - (oneorzero[threadIdx.x] != oneorzero[threadIdx.x - 1]) // - ) { - found = 1; - - diagonal_path_intersections[blockIdx.x] = current_x; - diagonal_path_intersections[blockIdx.x + gridDim.x + 1] = current_y; - } - - __syncthreads(); - - // Adjust the search window on the diagonal - if (threadIdx.x == 16) { - if (oneorzero[31] != 0) { - x_bottom = current_x; - y_bottom = current_y; - } else { - x_top = current_x; - y_top = current_y; - } - } - __syncthreads(); - } - - // Set the boundary diagonals (through 0,0 and A_length,B_length) - if (threadIdx.x == 0 && blockIdx.x == 0) { - diagonal_path_intersections[0] = 0; - diagonal_path_intersections[gridDim.x + 1] = 0; - diagonal_path_intersections[gridDim.x] = A_length; - diagonal_path_intersections[gridDim.x + gridDim.x + 1] = B_length; - } -} - -// Serial merge -// clang-format off -template -__device__ void merge( - F* copyFreq, int* copyIndex, int* copyIsLeaf, int cStart, int cEnd, - F* iNodesFreq, int iStart, int iEnd, int iNodesCap, - F* tempFreq, int* tempIndex, int* tempIsLeaf, int& tempLength) -{ - // clang-format on - int len = 0; - int iterCopy = cStart, iterINodes = iStart; - - while (iterCopy < cEnd && MOD(iEnd - iterINodes, iNodesCap) > 0) { - if (copyFreq[iterCopy] <= iNodesFreq[iterINodes]) { - tempFreq[len] = copyFreq[iterCopy]; - tempIndex[len] = copyIndex[iterCopy]; - tempIsLeaf[len] = copyIsLeaf[iterCopy]; - ++iterCopy; - } else { - tempFreq[len] = iNodesFreq[iterINodes]; - tempIndex[len] = iterINodes; - tempIsLeaf[len] = 0; - iterINodes = MOD(iterINodes + 1, iNodesCap); - } - ++len; - } - - while (iterCopy < cEnd) { - tempFreq[len] = copyFreq[iterCopy]; - tempIndex[len] = copyIndex[iterCopy]; - tempIsLeaf[len] = copyIsLeaf[iterCopy]; - ++iterCopy; - ++len; - } - while (MOD(iEnd - iterINodes, iNodesCap) > 0) { - tempFreq[len] = iNodesFreq[iterINodes]; - tempIndex[len] = iterINodes; - tempIsLeaf[len] = 0; - iterINodes = MOD(iterINodes + 1, iNodesCap); - ++len; - } - - tempLength = len; -} - -/* CUDAMERGESINGLEPATH - * Performs merge windows within a thread block from that block's global - * diagonal intersection to the next - */ -#define K 512 -#define PAD_SIZE 0 - -// clang-format off -template -__device__ void cudaMergeSinglePath( - F* copyFreq, int* copyIndex, int* copyIsLeaf, - int cStart, int cEnd, - F* iNodesFreq, - int iStart, int iEnd, int iNodesCap, - uint32_t* diagonal_path_intersections, - F* tempFreq, int* tempIndex, int* tempIsLeaf, - int tempLength) -{ - // clang-format on - // Temporary Code -- Serial Merge Per Block - if (threadIdx.x == 0) { - // Boundaries - int x_block_top = diagonal_path_intersections[blockIdx.x]; - int y_block_top = diagonal_path_intersections[blockIdx.x + gridDim.x + 1]; - int x_block_stop = diagonal_path_intersections[blockIdx.x + 1]; - int y_block_stop = diagonal_path_intersections[blockIdx.x + gridDim.x + 2]; - - // Actual indexes - int x_start = x_block_top + cStart; - int x_end = x_block_stop + cStart; - int y_start = MOD(iStart + y_block_top, iNodesCap); - int y_end = MOD(iStart + y_block_stop, iNodesCap); - - int offset = x_block_top + y_block_top; - - int dummy; // Unused result - // TODO optimize serial merging of each partition - merge(copyFreq, copyIndex, copyIsLeaf, x_start, x_end, // - iNodesFreq, y_start, y_end, iNodesCap, // - tempFreq + offset, tempIndex + offset, tempIsLeaf + offset, dummy); - if (0) { - printf("block: %d x: %d %d, y: %d %d, contrib: %d\n", blockIdx.x, - x_block_top, x_block_stop, y_block_top, y_block_stop, dummy); - } - } -} - -// `unsigned int` instantiations -template __device__ void parMerge( - unsigned int *copyFreq, int *copyIndex, int *copyIsLeaf, int cStart, - int cEnd, unsigned int *iNodesFreq, int iStart, int iEnd, int iNodesCap, - unsigned int *tempFreq, int *tempIndex, int *tempIsLeaf, int &tempLength, - uint32_t *diagonal_path_intersections, int blocks, int threads, - /* Shared Memory */ - int32_t &x_top, int32_t &y_top, int32_t &x_bottom, int32_t &y_bottom, - int32_t &found, int32_t *oneorzero); - -template __device__ void -merge(unsigned int *copyFreq, int *copyIndex, int *copyIsLeaf, - int cStart, int cEnd, unsigned int *iNodesFreq, int iStart, - int iEnd, int iNodesCap, unsigned int *tempFreq, - int *tempIndex, int *tempIsLeaf, int &tempLength); diff --git a/src/cuda/ParallelHuffman/types.cc b/src/cuda/ParallelHuffman/types.cc deleted file mode 100644 index 0b18c939bd..0000000000 --- a/src/cuda/ParallelHuffman/types.cc +++ /dev/null @@ -1,146 +0,0 @@ -#include -#include // for FP32 bit representation -#include // size_t -#include -#include -#include -#include -#include -#include - -#include "cuda/ParallelHuffman/constants.hh" -#include "cuda/ParallelHuffman/format.hh" -#include "cuda/ParallelHuffman/io.hh" -#include "cuda/ParallelHuffman/timer.hh" -#include "cuda/ParallelHuffman/types.hh" - -using namespace std; - -template double GetDatumValueRange(string fname, size_t l) { - auto d = io::ReadBinaryFile(fname, l); - T max_ = *std::max_element(d, d + l); - T min_ = *std::min_element(d, d + l); - delete[] d; - return max_ - min_; -} - -template double GetDatumValueRange(string fname, size_t l); -template double GetDatumValueRange(string fname, size_t l); - -size_t *InitializeDims(size_t cap, size_t n_dims, size_t dim0, size_t dim1, - size_t dim2, size_t dim3) { - auto dims_L16 = new size_t[16](); - - size_t dims[] = {dim0, dim1, dim2, dim3}; - std::copy(dims, dims + 4, dims_L16); - dims_L16[nDIM] = n_dims; - - int BLK = 0; - if (dims_L16[nDIM] == 1) - BLK = B_1d; - else if (dims_L16[nDIM] == 2) - BLK = B_2d; - else if (dims_L16[nDIM] == 3) - BLK = B_3d; - - dims_L16[nBLK0] = (dims_L16[DIM0] - 1) / (size_t)BLK + 1; - dims_L16[nBLK1] = (dims_L16[DIM1] - 1) / (size_t)BLK + 1; - dims_L16[nBLK2] = (dims_L16[DIM2] - 1) / (size_t)BLK + 1; - dims_L16[nBLK3] = (dims_L16[DIM3] - 1) / (size_t)BLK + 1; - dims_L16[LEN] = - dims_L16[DIM0] * dims_L16[DIM1] * dims_L16[DIM2] * dims_L16[DIM3]; - dims_L16[CAP] = cap; - dims_L16[RADIUS] = cap / 2; - - return dims_L16; -} - -// for example, binning needs to set new dimensions -void SetDims(size_t *dims_L16, size_t new_dims[4]) { - std::copy(new_dims, new_dims + 4, dims_L16); - int BLK = 0; - if (dims_L16[nDIM] == 1) - BLK = B_1d; - else if (dims_L16[nDIM] == 2) - BLK = B_2d; - else if (dims_L16[nDIM] == 3) - BLK = B_3d; - dims_L16[nBLK0] = (dims_L16[DIM0] - 1) / (size_t)BLK + 1; - dims_L16[nBLK1] = (dims_L16[DIM1] - 1) / (size_t)BLK + 1; - dims_L16[nBLK2] = (dims_L16[DIM2] - 1) / (size_t)BLK + 1; - dims_L16[nBLK3] = (dims_L16[DIM3] - 1) / (size_t)BLK + 1; - dims_L16[LEN] = - dims_L16[DIM0] * dims_L16[DIM1] * dims_L16[DIM2] * dims_L16[DIM3]; -} - -// typedef struct ErrorBoundConfigurator { -// int capacity, radius; -// double base, exp_base2, exp_base10; -// double eb_base2, eb_base10, eb_final; -// std::string mode; -ErrorBoundConfigurator::ErrorBoundConfigurator(int _capacity, double _precision, - double _exponent, int _base) { - capacity = _capacity; - radius = capacity / 2; - mode = std::string("ABS"); - - if (_precision != 1 and _base == 2) { - cerr << "tmp.ly we only support 1 x pow(2, \?\?)" << endl; - } - eb_final = _precision * pow(_base, _exponent); - base = _base; - exp_base10 = _base == 10 ? _exponent : log10(eb_final); - exp_base2 = _base == 2 ? _exponent : log2(eb_final); - - cout << log_info << "quant.cap:\t" << _capacity << "\tinput eb:\t" << eb_final - << "\n"; - // if (_base == 10) { - // cout << log_info << "input eb:\t" << _precision; - // cout << " x 10^(" << _exponent << ") = " << eb_final << endl; - // } - // else if (_base == 2) { - // cout << "eb.set.to:\t" - // << "2^(" << _exponent << ") = " << eb_final << endl; - // } -} - -void ErrorBoundConfigurator::ChangeToRelativeMode(double value_range) { - if (value_range == 0) { - cerr << log_err << "INVALID VALUE RANGE!" << endl; - exit(1); - } - cout << log_info << "eb change:\t(input eb) x " << value_range << " (rng) = "; - this->eb_final *= value_range; - cout << eb_final; - cout << " \e[2m(relative-to-range)\e[0m" << endl; - mode = std::string("VRREL"); -} - -void ErrorBoundConfigurator::ChangeToTightBase2() { - base = 2; - cout << log_info << "switch.to.tight.base2.mode, eb changed from " << eb_final - << " = 2^(" << exp_base2 << ") to "; - cout << "the exp base2 before changing:\t" << exp_base2 << endl; - exp_base2 = floor(exp_base2); - cout << "the exp base2 after changing:\t" << exp_base2 << endl; - eb_final = pow(2, exp_base2); - cout << eb_final << " = 2^(" << exp_base2 << ")" << endl; -} - -void ErrorBoundConfigurator::debug() const { - cout << log_dbg; - printf("exponent = %.3f (base10) (or) %.3f (base2)\n", exp_base10, exp_base2); -} - -//} config_t; - -typedef struct ErrorBoundConfigurator config_t; - -double *InitializeErrorBoundFamily(config_t *eb_config) { - auto ebs_L4 = new double[4](); - ebs_L4[0] = eb_config->eb_final; // eb - ebs_L4[1] = 1 / eb_config->eb_final; // 1/eb - ebs_L4[2] = 2 * eb_config->eb_final; // 2* eb - ebs_L4[3] = 1 / (2 * eb_config->eb_final); // 1/(2*eb) - return ebs_L4; -} diff --git a/src/cuda/PrecomputeKernels.cu b/src/cuda/PrecomputeKernels.cu deleted file mode 100644 index 4f82904717..0000000000 --- a/src/cuda/PrecomputeKernels.cu +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: April 2, 2021 - */ - -#include "cuda/CommonInternal.h" -#include "cuda/PrecomputeKernels.h" -#include -#include - -namespace mgard_cuda { - -template -__global__ void _calc_cpt_dist(int n, T *dcoord, T *ddist) { - - // extern __shared__ __align__(sizeof(T)) unsigned char smem[]; - // T * sm = reinterpret_cast(smem); - T *sm = SharedMemory(); - // extern __shared__ double sm[]; //size = blockDim.x + 1 - - int x0 = blockIdx.x * blockDim.x + threadIdx.x; - int x0_sm = threadIdx.x; - T dist; - for (int x = x0; x < n; x += blockDim.x * gridDim.x) { - // Load coordinates - sm[x0_sm] = dcoord[x]; - // printf("sm[%d] block %d thread %d load[%d] %f\n", x0_sm, blockIdx.x, - // threadIdx.x, x, dcoord[x * stride]); - if (x0_sm == 0) { - // sm[blockDim.x] = dcoord[(x + blockDim.x) * stride]; - int left = n - blockIdx.x * blockDim.x; - if (left >= blockDim.x + 1) { - sm[blockDim.x] = dcoord[blockDim.x + x]; - } - // sm[min(blockDim.x, left - 1)] = - // dcoord[min((x + blockDim.x) * stride, n - 1)]; - // printf("sm[%d] extra block %d thread %d load[%d] %f\n", min(blockDim.x, - // left-1), blockIdx.x, threadIdx.x, min((x + blockDim.x) * stride, n-1), - // dcoord[min((x + blockDim.x) * stride, n-1)]); - } - __syncthreads(); - - // Compute distance - - if (x < n - 1) { - ddist[x] = _get_dist(sm, x0_sm, x0_sm + 1); - } - __syncthreads(); - // ddist[x] = dist; - // __syncthreads(); - } -} - -template -void calc_cpt_dist(Handle &handle, int n, T *dcoord, T *ddist, - int queue_idx) { - - int total_thread_x = std::max(n, 1); - int total_thread_y = 1; - int tbx = std::min(16, total_thread_x); - int tby = 1; - int gridx = ceil((double)total_thread_x / tbx); - int gridy = ceil((double)total_thread_y / tby); - dim3 threadsPerBlock(tbx, tby); - dim3 blockPerGrid(gridx, gridy); - size_t sm_size = (tbx + 1) * sizeof(T); - - // printf("sm %d (%d %d) (%d %d)\n", sm_size, tbx, tby, gridx, gridy); - _calc_cpt_dist<<>>(n, dcoord, ddist); - gpuErrchk(cudaGetLastError()); -#ifdef MGARD_CUDA_DEBUG - gpuErrchk(cudaDeviceSynchronize()); -#endif -} - -template -__global__ void _reduce_two_dist(int n, T *ddist, T *ddist_reduced) { - int x_gl = blockIdx.x * blockDim.x + threadIdx.x; - int x_sm = threadIdx.x; - T *sm = SharedMemory(); - if (x_gl < n) { - sm[x_sm] = ddist[x_gl]; - // printf("thread %d load %f\n", x_gl, ddist[x_gl]); - __syncthreads(); - if (x_gl % 2 == 0) { - ddist_reduced[x_gl / 2] = sm[x_sm] + sm[x_sm + 1]; - // printf("thread %d compute %f + %f -> [%d]%f\n", x_gl, sm[x_sm], - // sm[x_sm+1], x_gl / 2, ddist_reduced[x_gl / 2]); - } - // __syncthreads(); - // if (x_gl % 2 == 0) { - // dratio[x_gl / 2] = ddist_reduced[x_gl / 2] / (ddist_reduced[x_gl / 2] + - // ddist_reduced[x_gl / 2 + 1]); - // } - } -} - -template -void reduce_two_dist(Handle &handle, int n, T *ddist, T *ddist_reduced, - int queue_idx) { - - int total_thread_x = std::max(n, 1); - int total_thread_y = 1; - int tbx = std::min(16, total_thread_x); - int tby = 1; - int gridx = ceil((double)total_thread_x / tbx); - int gridy = ceil((double)total_thread_y / tby); - dim3 threadsPerBlock(tbx, tby); - dim3 blockPerGrid(gridx, gridy); - size_t sm_size = tbx * sizeof(T); - // printf("reduce_two_dist: n: %d\n", n); - // printf("sm %d (%d %d) (%d %d)\n", sm_size, tbx, tby, gridx, gridy); - _reduce_two_dist<<>>(n, ddist, - ddist_reduced); - gpuErrchk(cudaGetLastError()); -#ifdef MGARD_CUDA_DEBUG - gpuErrchk(cudaDeviceSynchronize()); -#endif -} - -template -__global__ void _dist_to_ratio(int n, T *ddist, T *dratio) { - - T *sm = SharedMemory(); - - int x = blockIdx.x * blockDim.x + threadIdx.x; - int x_sm = threadIdx.x; - if (x < n) { - // Load dists - sm[x_sm] = ddist[x]; - if (x_sm == 0) { - int left = n - blockIdx.x * blockDim.x; - if (left >= blockDim.x + 1) { - sm[blockDim.x] = ddist[blockDim.x + x]; - } - // if (blockIdx.x == 1) { - // for (int i = 0; i < blockDim.x + 1; i++) { - // printf("%f ", sm[i]); - // } - // printf("\n"); - // } - } - } - __syncthreads(); - // Compute distance - if (x < n - 1) { - dratio[x] = sm[x_sm] / (sm[x_sm] + sm[x_sm + 1]); - // if (blockIdx.x == 1) { - // printf("x(%d) %f %f %f\n", x, dratio[x], sm[x_sm],sm[x_sm+1]);} - } -} - -template -void dist_to_ratio(Handle &handle, int n, T *ddist, T *dratio, - int queue_idx) { - - int total_thread_x = std::max(n, 1); - int total_thread_y = 1; - int tbx = std::min(16, total_thread_x); - int tby = 1; - int gridx = ceil((double)total_thread_x / tbx); - int gridy = ceil((double)total_thread_y / tby); - dim3 threadsPerBlock(tbx, tby); - dim3 blockPerGrid(gridx, gridy); - size_t sm_size = (tbx + 1) * sizeof(T); - // printf("reduce_two_dist: n: %d\n", n); - // printf("sm %d (%d %d) (%d %d)\n", sm_size, tbx, tby, gridx, gridy); - _dist_to_ratio<<>>(n, ddist, dratio); - gpuErrchk(cudaGetLastError()); -#ifdef MGARD_CUDA_DEBUG - gpuErrchk(cudaDeviceSynchronize()); -#endif -} - -template -__global__ void _dist_to_volume(int n, T *ddist, T *dvolume) { - T *sm = SharedMemory(); - int x = blockIdx.x * blockDim.x + threadIdx.x; - int x_sm = threadIdx.x; - if (x < n - 1) { - // Load dist - sm[x_sm + 1] = ddist[x]; - } - if (x == n - 1) { - sm[x_sm + 1] = 0; - } - int left = n - blockIdx.x * blockDim.x; - if (x_sm == 0) { // load extra - sm[0] = x - 1 < 0 ? 0 : ddist[x - 1]; - } - - // if (threadIdx.x == 0) { - // for (int i = 0; i < blockDim.x+2; i++) { - // printf("dist[%d] = %f\n", i, sm[i]); - // } - // } - int node_coeff_div = n / 2 + 1; - if (n == 2) { - dvolume[x] = (sm[x_sm] + sm[x_sm + 1]) / 2; - } else { - if (n % 2 != 0) { - if (x % 2 == 0) { // node - dvolume[x / 2] = (sm[x_sm] + sm[x_sm + 1]) / 2; - } else { // coeff - dvolume[node_coeff_div + x / 2] = (sm[x_sm] + sm[x_sm + 1]) / 2; - } - } else { - if (x != n - 1) { - if (x % 2 == 0) { // node - dvolume[x / 2] = (sm[x_sm] + sm[x_sm + 1]) / 2; - // printf("%f <- %f %f\n", dvolume[x/2], sm[x_sm], sm[x_sm+1]); - } else { // coeff - dvolume[node_coeff_div + x / 2] = (sm[x_sm] + sm[x_sm + 1]) / 2; - // printf("%f <- %f %f\n", dvolume[node_coeff_div + x/2], sm[x_sm], - // sm[x_sm+1]); - } - } else { - dvolume[x / 2 + 1] = (sm[x_sm] + sm[x_sm + 1]) / 2; - // printf("%f <- %f %f\n", dvolume[x/2+1], sm[x_sm], sm[x_sm+1]); - } - } - } -} - -template -void dist_to_volume(Handle &handle, int n, T *ddist, T *dvolume, - int queue_idx) { - - int total_thread_x = std::max(n, 1); - int total_thread_y = 1; - int tbx = std::min(16, total_thread_x); - int tby = 1; - int gridx = ceil((double)total_thread_x / tbx); - int gridy = ceil((double)total_thread_y / tby); - dim3 threadsPerBlock(tbx, tby); - dim3 blockPerGrid(gridx, gridy); - size_t sm_size = (tbx + 2) * sizeof(T); - // printf("reduce_two_dist: n: %d\n", n); - // printf("sm %d (%d %d) (%d %d)\n", sm_size, tbx, tby, gridx, gridy); - _dist_to_volume<<>>(n, ddist, - dvolume); - gpuErrchk(cudaGetLastError()); -#ifdef MGARD_CUDA_DEBUG - gpuErrchk(cudaDeviceSynchronize()); -#endif -} - -template -__global__ void _calc_am_bm(int n, T *ddist, T *am, T *bm) { - int c = threadIdx.x; - int c_sm = threadIdx.x; - // extern __shared__ __align__(sizeof(T)) unsigned char smem[]; - // T * sm = reinterpret_cast(smem); - T *sm = SharedMemory(); - // extern __shared__ double sm[]; - T *ddist_sm = sm; - T *am_sm = sm + blockDim.x; - T *bm_sm = am_sm + blockDim.x; - - T prev_am = 1.0; - T prev_dist = 0.0; - int rest = n; - - while (rest > blockDim.x) { - /* Load ddsist */ - ddist_sm[c_sm] = ddist[c]; - __syncthreads(); - /* Calculation on one thread*/ - if (c_sm == 0) { - bm_sm[0] = prev_dist / prev_am; - am_sm[0] = 2.0 * (ddist_sm[0] + prev_dist) - bm_sm[0] * prev_dist; - for (int i = 1; i < blockDim.x; i++) { - bm_sm[i] = ddist_sm[i - 1] / am_sm[i - 1]; - am_sm[i] = - 2.0 * (ddist_sm[i - 1] + ddist_sm[i]) - bm_sm[i] * ddist_sm[i - 1]; - } - prev_am = am_sm[blockDim.x - 1]; - prev_dist = ddist_sm[blockDim.x - 1]; - } - __syncthreads(); -#ifdef MGARD_CUDA_FMA - am[c] = 1 / am_sm[c_sm]; - bm[c] = bm_sm[c_sm] * -1; -#else - am[c] = am_sm[c_sm]; - bm[c] = bm_sm[c_sm]; -#endif - __syncthreads(); - c += blockDim.x; - rest -= blockDim.x; - __syncthreads(); - } // end of while - - if (c_sm < rest - 1) { - ddist_sm[c_sm] = ddist[c]; - } - - __syncthreads(); - if (c_sm == 0) { - if (rest == 1) { - bm_sm[rest - 1] = prev_dist / prev_am; - am_sm[rest - 1] = 2.0 * prev_dist - bm_sm[rest - 1] * prev_dist; - // printf("bm = %f\n", bm_sm[rest-1]); - // printf("am = %f\n", am_sm[rest-1]); - } else { - bm_sm[0] = prev_dist / prev_am; - am_sm[0] = 2.0 * (ddist_sm[0] + prev_dist) - bm_sm[0] * prev_dist; - for (int i = 1; i < rest - 1; i++) { - bm_sm[i] = ddist_sm[i - 1] / am_sm[i - 1]; - am_sm[i] = - 2.0 * (ddist_sm[i - 1] + ddist_sm[i]) - bm_sm[i] * ddist_sm[i - 1]; - } - bm_sm[rest - 1] = ddist_sm[rest - 2] / am_sm[rest - 2]; - am_sm[rest - 1] = - 2.0 * ddist_sm[rest - 2] - bm_sm[rest - 1] * ddist_sm[rest - 2]; - } - } - __syncthreads(); - if (c_sm < rest) { -#ifdef MGARD_CUDA_FMA - am[c] = 1 / am_sm[c_sm]; - bm[c] = bm_sm[c_sm] * -1; -#else - am[c] = am_sm[c_sm]; - bm[c] = bm_sm[c_sm]; -#endif - } -} - -template -void calc_am_bm(Handle &handle, int n, T *ddist, T *am, T *bm, - int queue_idx) { - - // int total_thread_y = 1; - int total_thread_x = 16; - int tby = 1; - int tbx = std::min(16, total_thread_x); - size_t sm_size = 16 * 3 * sizeof(T); - int gridy = 1; - int gridx = 1; - dim3 threadsPerBlock(tbx, tby); - dim3 blockPerGrid(gridx, gridy); - _calc_am_bm<<>>(n, ddist, am, bm); - gpuErrchk(cudaGetLastError()); -#ifdef MGARD_CUDA_DEBUG - gpuErrchk(cudaDeviceSynchronize()); -#endif -} - -#define KERNELS(D, T) \ - template void calc_cpt_dist(Handle &handle, int n, T *dcoord, \ - T *ddist, int queue_idx); \ - template void reduce_two_dist(Handle & handle, int n, T *ddist, \ - T *ddist_reduced, int queue_idx); \ - template void dist_to_ratio(Handle & handle, int n, T *ddist, \ - T *dratio, int queue_idx); \ - template void dist_to_volume(Handle & handle, int n, T *ddist, \ - T *dvolume, int queue_idx); \ - template void calc_am_bm(Handle & handle, int n, T *ddist, \ - T *am, T *bm, int queue_idx); - -KERNELS(1, double) -KERNELS(1, float) -KERNELS(2, double) -KERNELS(2, float) -KERNELS(3, double) -KERNELS(3, float) -KERNELS(4, double) -KERNELS(4, float) -KERNELS(5, double) -KERNELS(5, float) -#undef KERNELS - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/cuda/SubArray.cu b/src/cuda/SubArray.cu deleted file mode 100644 index d0ff98fee8..0000000000 --- a/src/cuda/SubArray.cu +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright 2021, Oak Ridge National Laboratory. - * MGARD-GPU: MultiGrid Adaptive Reduction of Data Accelerated by GPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: Jul 10, 2021 - */ - -#include - -#include "cuda/CommonInternal.h" - -#include "cuda/SubArray.h" - -namespace mgard_cuda { - -template SubArray::SubArray() { - lddv1 = 1; - lddv2 = 1; -} - -template SubArray::SubArray(Array &array) { - this->shape = array.getShape(); - this->dv = array.get_dv(); - this->ldvs_h = array.get_ldvs_h(); - this->ldvs_d = array.get_ldvs_d(); - lddv1 = ldvs_h[0]; - lddv2 = ldvs_h[1]; -} - -template -SubArray::SubArray(std::vector shape, T *dv, - std::vector ldvs_h, SIZE *ldvs_d) { - this->shape = shape; - this->dv = dv; - this->ldvs_h = ldvs_h; - this->ldvs_d = ldvs_d; - lddv1 = ldvs_h[0]; - lddv2 = ldvs_h[1]; -} - -template -SubArray::SubArray(std::vector shape, T *dv) { - this->shape = shape; - this->dv = dv; - this->lddv1 = shape[0]; - if (D > 1) { - this->lddv2 = shape[1]; - } else { - this->lddv2 = 1; - } -} - -template -SubArray::SubArray(SubArray &subArray) { - this->shape = subArray.shape; - this->dv = subArray.dv; - this->ldvs_h = subArray.ldvs_h; - this->ldvs_d = subArray.ldvs_d; - - this->lddv1 = subArray.lddv1; - this->lddv2 = subArray.lddv2; - - this->projected_dim0 = subArray.projected_dim0; - this->projected_dim1 = subArray.projected_dim1; - this->projected_dim2 = subArray.projected_dim2; -} - -template -SubArray::SubArray(const SubArray &subArray) { - this->shape = subArray.shape; - this->dv = subArray.dv; - this->ldvs_h = subArray.ldvs_h; - this->ldvs_d = subArray.ldvs_d; - - this->lddv1 = subArray.lddv1; - this->lddv2 = subArray.lddv2; - - this->projected_dim0 = subArray.projected_dim0; - this->projected_dim1 = subArray.projected_dim1; - this->projected_dim2 = subArray.projected_dim2; -} - -template -SubArray &SubArray::operator=(const SubArray &subArray) { - this->shape = subArray.shape; - this->dv = subArray.dv; - this->ldvs_h = subArray.ldvs_h; - this->ldvs_d = subArray.ldvs_d; - - this->lddv1 = subArray.lddv1; - this->lddv2 = subArray.lddv2; - - this->projected_dim0 = subArray.projected_dim0; - this->projected_dim1 = subArray.projected_dim1; - this->projected_dim2 = subArray.projected_dim2; - return *this; -} - -template -void SubArray::offset(std::vector idx) { - dv += get_idx(ldvs_h, idx); -} - -template -void SubArray::resize(std::vector shape) { - this->shape = shape; -} - -template -void SubArray::offset(SIZE dim, SIZE offset_value) { - std::vector idx(D, 0); - idx[dim] = offset_value; - dv += get_idx(ldvs_h, idx); -} - -template -void SubArray::resize(SIZE dim, SIZE new_size) { - shape[dim] = new_size; -} - -template -void SubArray::project(DIM dim0, DIM dim1, DIM dim2) { - projected_dim0 = dim0; - projected_dim1 = dim1; - projected_dim2 = dim2; - lddv1 = 1, lddv2 = 1; - for (DIM d = projected_dim0; d < projected_dim1; d++) { - lddv1 *= ldvs_h[d]; - } - for (DIM d = projected_dim1; d < projected_dim2; d++) { - lddv2 *= ldvs_h[d]; - } -} - -// template -// MGARDm_EXEC -// T* SubArray::operator()(IDX z, IDX y, IDX x) { -// return dv + lddv2 * lddv1 * z + lddv1 * y + x; -// } - -// template __forceinline__ __device__ LENGTH get_idx(SIZE *lds, SIZE -// *idx) { -// LENGTH curr_stride = 1; -// LENGTH ret_idx = 0; -// for (DIM i = 0; i < D; i++) { -// ret_idx += idx[i] * curr_stride; -// curr_stride *= lds[i]; -// } -// return ret_idx; -// } - -template SubArray::~SubArray() { - // nothing needs to be released -} - -template class SubArray<1, double>; -template class SubArray<1, float>; -template class SubArray<2, double>; -template class SubArray<2, float>; -template class SubArray<3, double>; -template class SubArray<3, float>; -template class SubArray<4, double>; -template class SubArray<4, float>; -template class SubArray<5, double>; -template class SubArray<5, float>; - -template class SubArray<1, bool>; - -template class SubArray<1, uint8_t>; -template class SubArray<1, uint16_t>; -template class SubArray<1, uint32_t>; -template class SubArray<1, uint64_t>; - -template class SubArray<2, uint8_t>; -template class SubArray<2, uint16_t>; -template class SubArray<2, uint32_t>; -template class SubArray<2, uint64_t>; - -template class SubArray<1, unsigned long long>; - -// template class SubArray<1, QUANTIZED_INT>; -// template class SubArray<2, QUANTIZED_INT>; -// template class SubArray<3, QUANTIZED_INT>; -// template class SubArray<4, QUANTIZED_INT>; -// template class SubArray<5, QUANTIZED_INT>; - -} // namespace mgard_cuda \ No newline at end of file diff --git a/src/mgard-x/CompressionHighLevel/DynamicAPI.cpp b/src/mgard-x/CompressionHighLevel/DynamicAPI.cpp index b98376537d..2f578d4b3e 100644 --- a/src/mgard-x/CompressionHighLevel/DynamicAPI.cpp +++ b/src/mgard-x/CompressionHighLevel/DynamicAPI.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include @@ -45,8 +45,7 @@ enum device_type auto_detect_device() { } #endif if (dev_type == device_type::NONE) { - log::err("MGARD-X was not built with any backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with any backend."); } return dev_type; } @@ -521,39 +520,34 @@ void BeginAutoTuning(enum device_type dev_type) { #if MGARD_ENABLE_SERIAL BeginAutoTuning(); #else - log::err("MGARD-X was not built with SERIAL backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with SERIAL backend."); #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP BeginAutoTuning(); #else - log::err("MGARD-X was not built with OPENMP backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with OPENMP backend."); #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA BeginAutoTuning(); #else - log::err("MGARD-X was not built with CUDA backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with CUDA backend."); #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP BeginAutoTuning(); #else - log::err("MGARD-X was not built with HIP backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with HIP backend."); #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL BeginAutoTuning(); #else - log::err("MGARD-X was not built with SYCL backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with SYCL backend."); #endif } else { - log::err("Unsupported backend."); + throw std::runtime_error("Unsupported backend."); } } @@ -567,39 +561,34 @@ void EndAutoTuning(enum device_type dev_type) { #if MGARD_ENABLE_SERIAL EndAutoTuning(); #else - log::err("MGARD-X was not built with SERIAL backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with SERIAL backend."); #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP EndAutoTuning(); #else - log::err("MGARD-X was not built with OPENMP backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with OPENMP backend."); #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA EndAutoTuning(); #else - log::err("MGARD-X was not built with CUDA backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with CUDA backend."); #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP EndAutoTuning(); #else - log::err("MGARD-X was not built with HIP backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with HIP backend."); #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL EndAutoTuning(); #else - log::err("MGARD-X was not built with SYCL backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with SYCL backend."); #endif } else { - log::err("Unsupported backend."); + throw std::runtime_error("Unsupported backend."); } } @@ -614,39 +603,34 @@ void pin_memory(void *ptr, SIZE num_bytes, Config config) { #if MGARD_ENABLE_SERIAL pin_memory(ptr, num_bytes); #else - log::err("MGARD-X was not built with SERIAL backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with SERIAL backend."); #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP pin_memory(ptr, num_bytes); #else - log::err("MGARD-X was not built with OPENMP backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with OPENMP backend."); #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA pin_memory(ptr, num_bytes); #else - log::err("MGARD-X was not built with CUDA backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with CUDA backend."); #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP pin_memory(ptr, num_bytes); #else - log::err("MGARD-X was not built with HIP backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with HIP backend."); #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL pin_memory(ptr, num_bytes); #else - log::err("MGARD-X was not built with SYCL backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with SYCL backend."); #endif } else { - log::err("Unsupported backend."); + throw std::runtime_error("Unsupported backend."); } } @@ -661,39 +645,34 @@ bool check_memory_pinned(void *ptr, Config config) { #if MGARD_ENABLE_SERIAL return check_memory_pinned(ptr); #else - log::err("MGARD-X was not built with SERIAL backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with SERIAL backend."); #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP return check_memory_pinned(ptr); #else - log::err("MGARD-X was not built with OPENMP backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with OPENMP backend."); #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA return check_memory_pinned(ptr); #else - log::err("MGARD-X was not built with CUDA backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with CUDA backend."); #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP return check_memory_pinned(ptr); #else - log::err("MGARD-X was not built with HIP backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with HIP backend."); #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL return check_memory_pinned(ptr); #else - log::err("MGARD-X was not built with SYCL backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with SYCL backend."); #endif } else { - log::err("Unsupported backend."); + throw std::runtime_error("Unsupported backend."); } } @@ -708,39 +687,34 @@ void unpin_memory(void *ptr, Config config) { #if MGARD_ENABLE_SERIAL unpin_memory(ptr); #else - log::err("MGARD-X was not built with SERIAL backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with SERIAL backend."); #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP unpin_memory(ptr); #else - log::err("MGARD-X was not built with OPENMP backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with OPENMP backend."); #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA unpin_memory(ptr); #else - log::err("MGARD-X was not built with CUDA backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with CUDA backend."); #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP unpin_memory(ptr); #else - log::err("MGARD-X was not built with HIP backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with HIP backend."); #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL unpin_memory(ptr); #else - log::err("MGARD-X was not built with SYCL backend."); - exit(-1); + throw std::runtime_error("MGARD-X was not built with SYCL backend."); #endif } else { - log::err("Unsupported backend."); + throw std::runtime_error("Unsupported backend."); } } diff --git a/src/mgard-x/Config/Config.cpp b/src/mgard-x/Config/Config.cpp index 12f35c9b9c..37901afe6c 100644 --- a/src/mgard-x/Config/Config.cpp +++ b/src/mgard-x/Config/Config.cpp @@ -1,14 +1,32 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ +#include #include #include "mgard-x/Config/Config.h" +namespace { +// CUDA 12 defaults to lazy module loading, which pulls each kernel's code from +// the (large) fatbin on its first launch. For backends with many/large kernels +// (e.g. the rANS lossless path) those scattered first-launch loads add hundreds +// of ms per process. Eager loading front-loads the module once at context +// creation, making launches instant (wall time unchanged or lower). The CUDA +// runtime reads this env var while registering fatbins in static initializers, +// so we must set it from an early-priority constructor (before those run) -- a +// setenv in main() is too late. Only set when the user hasn't chosen, so it +// stays overridable. +__attribute__((constructor(101))) void mgard_x_set_cuda_module_loading() { + if (std::getenv("CUDA_MODULE_LOADING") == nullptr) { + setenv("CUDA_MODULE_LOADING", "EAGER", 0); + } +} +} // namespace + namespace mgard_x { Config::Config() { @@ -19,13 +37,13 @@ Config::Config() { decomposition = decomposition_type::MultiDim; estimate_outlier_ratio = 1.0; huff_dict_size = 8192; - huff_block_size = 1024 * 20; + huff_block_size = 1024; + block_delta_block_size = 256; + block_delta_mode = block_delta_mode_type::Delta; lz4_block_size = 1 << 15; zstd_compress_level = 3; normalize_coordinates = true; lossless = lossless_type::Huffman; - reorder = 0; - prefetch = false; log_level = log::ERR; max_larget_level = std::numeric_limits::max(); // no limit auto_pin_host_buffers = true; @@ -38,8 +56,15 @@ Config::Config() { adjust_shape = false; compress_with_dryrun = false; num_local_refactoring_level = 1; + num_global_refactoring_level = 0; + enable_roi = false; + projection_mode = compression_projection_mode_type::Auto; auto_cache_release = false; cpu_mode = cpu_parallelization_mode::INTER_BLOCK; + mdr_qoi_mode = false; + mdr_qoi_num_variables = 3; + fuse_decompose_quantize = true; + fuse_dequantize_recompose = true; } void Config::apply() { log::level = log_level; } diff --git a/src/mgard-x/DataRefactoring/InCacheBlock/Decompose.cpp.in b/src/mgard-x/DataRefactoring/InCacheBlock/Decompose.cpp.in index 6bdad7feb7..8711b02dad 100644 --- a/src/mgard-x/DataRefactoring/InCacheBlock/Decompose.cpp.in +++ b/src/mgard-x/DataRefactoring/InCacheBlock/Decompose.cpp.in @@ -14,12 +14,21 @@ namespace data_refactoring { namespace in_cache_block { template void decompose<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( - SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> v, - SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> coarse, - SubArray<1, @DATA_TYPE@, @DEVICE_TYPE@> coeff, int queue_idx); + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> v, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> coarse, + SubArray<1, @DATA_TYPE@, @DEVICE_TYPE@> coeff, + bool orthogonal_projection, int queue_idx); + +template void decompose_quantize<@NUM_DIM@, @DATA_TYPE@, QUANTIZED_INT, @DEVICE_TYPE@>( + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> v, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> coarse, + SubArray<1, QUANTIZED_INT, @DEVICE_TYPE@> quantized_coeff, + @DATA_TYPE@ quantizer, + SubArray<1, @DATA_TYPE@, @DEVICE_TYPE@> block_quantizers, + bool use_block_quantizers, bool prep_huffman, + SIZE dict_size, bool orthogonal_projection, int queue_idx); } // namespace in_cache_block } // namespace data_refactoring } // namespace mgard_x// clang-format on - diff --git a/src/mgard-x/DataRefactoring/InCacheBlock/Recompose.cpp.in b/src/mgard-x/DataRefactoring/InCacheBlock/Recompose.cpp.in index 1eb31a26eb..d6e781a75c 100644 --- a/src/mgard-x/DataRefactoring/InCacheBlock/Recompose.cpp.in +++ b/src/mgard-x/DataRefactoring/InCacheBlock/Recompose.cpp.in @@ -15,11 +15,21 @@ namespace in_cache_block { template void recompose<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> v, - SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> coarse, - SubArray<1, @DATA_TYPE@, @DEVICE_TYPE@> coeff, int queue_idx); + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> coarse, + SubArray<1, @DATA_TYPE@, @DEVICE_TYPE@> coeff, + bool orthogonal_projection, int queue_idx); + +template void recompose_dequantize<@NUM_DIM@, @DATA_TYPE@, QUANTIZED_INT, @DEVICE_TYPE@>( + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> v, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> coarse, + SubArray<1, QUANTIZED_INT, @DEVICE_TYPE@> quantized_coeff, + @DATA_TYPE@ quantizer, + SubArray<1, @DATA_TYPE@, @DEVICE_TYPE@> block_quantizers, + bool use_block_quantizers, bool prep_huffman, + SIZE dict_size, bool orthogonal_projection, int queue_idx); } // namespace in_cache_block } // namespace data_refactoring -} // namespace mgard_x// clang-format on \ No newline at end of file +} // namespace mgard_x// clang-format on diff --git a/src/mgard-x/DataRefactoring/MultiDimension/Decompose.cpp.in b/src/mgard-x/DataRefactoring/MultiDimension/Decompose.cpp.in index 8d73c0b886..e360e5d9c9 100644 --- a/src/mgard-x/DataRefactoring/MultiDimension/Decompose.cpp.in +++ b/src/mgard-x/DataRefactoring/MultiDimension/Decompose.cpp.in @@ -18,7 +18,7 @@ template void decompose<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &v, SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> w, SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> b, - int start_level, int stop_level, int queue_idx); + int start_level, int stop_level, bool orthogonal_projection, int queue_idx); } // namespace multi_dimension diff --git a/src/mgard-x/DataRefactoring/MultiDimension/Recompose.cpp.in b/src/mgard-x/DataRefactoring/MultiDimension/Recompose.cpp.in index daff11f410..66d974d52a 100644 --- a/src/mgard-x/DataRefactoring/MultiDimension/Recompose.cpp.in +++ b/src/mgard-x/DataRefactoring/MultiDimension/Recompose.cpp.in @@ -18,7 +18,7 @@ template void recompose<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &v, SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> w, SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> b, - int start_level, int stop_level, int queue_idx); + int start_level, int stop_level, bool orthogonal_projection, int queue_idx); } // namespace multi_dimension diff --git a/src/mgard-x/Executables/MgardSerialExec.cpp b/src/mgard-x/Executables/MgardSerialExec.cpp index 16db219b69..884900efc0 100644 --- a/src/mgard-x/Executables/MgardSerialExec.cpp +++ b/src/mgard-x/Executables/MgardSerialExec.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include @@ -13,7 +13,6 @@ #include #include "compress.hpp" -// #include "compress_cuda.hpp" #include "mgard-x/Utilities/ErrorCalculator.h" using namespace std::chrono; diff --git a/src/mgard-x/Executables/mdr-x-qoi.cpp b/src/mgard-x/Executables/mdr-x-qoi.cpp new file mode 100644 index 0000000000..22b17492c0 --- /dev/null +++ b/src/mgard-x/Executables/mdr-x-qoi.cpp @@ -0,0 +1,831 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "compress_x.hpp" +#include "mdr_x.hpp" +#include "mgard-x/RuntimeX/Utilities/Log.h" +#include "mgard-x/Utilities/ErrorCalculator.h" + +#include "ArgumentParser.h" +using namespace std::chrono; + +void print_usage_message(std::string error) { + if (error.compare("") != 0) { + std::cout << mgard_x::log::log_err << error << std::endl; + } + printf("Options\n\ +\t -z / --refactor: refactor data\n\ +\t\t -i / --input \n\ +\t\t -o / --output \n\ +\t\t -dt / --data-type : data type (s: single; d:double)\n\ +\t\t -dim / --dimension : total number of dimensions\n\ +\t\t\t [dim1]: slowest dimention\n\ +\t\t\t [dim2]: 2nd slowest dimention\n\ +\t\t\t ...\n\ +\t\t\t [dimN]: fastest dimention\n\ +\t\t -d / --device : device type\n\ +\t\t (optional) -v / --verbose <0|1|2|3> 0: error; 1: error+info; 2: error+timing; 3: all\n\ +\t\t (optional) -m / --max-memory \n\ +\t\t (optional) -dd / --domain-decomposition \n\ +\t\t\t (optional) -dd-size / --domain-decomposition-size (for block domain decomposition only) \n\ +\n\ +\t -x / --reconstruct: reconstruct data\n\ +\t\t -i / --input \n\ +\t\t -o / --output \n\ +\t\t (optional) -g / --orginal (optinal)\n\ +\t\t -e / --error-bound : error bound\n\ +\t\t -me / --multi-error-bounds ..: multiple error bounds\n\ +\t\t -s / --smoothness : smoothness parameter\n\ +\t\t -d : device type\n\ +\t\t (optional) -v / --verbose <0|1|2|3> 0: error; 1: error+info; 2: error+timing; 3: all\n"); + exit(0); +} + +template void min_max(size_t n, T *in_buff) { + T min = std::numeric_limits::infinity(); + T max = 0; + for (size_t i = 0; i < n; i++) { + if (min > in_buff[i]) { + min = in_buff[i]; + } + if (max < in_buff[i]) { + max = in_buff[i]; + } + } + printf("Min: %f, Max: %f\n", min, max); +} + +template size_t readfile(std::string input_file, T *&in_buff) { + // std::cout << mgard_x::log::log_info << "Loading file: " << input_file << + // "\n"; + + FILE *pFile; + pFile = fopen(input_file.c_str(), "rb"); + if (pFile == NULL) { + std::cout << mgard_x::log::log_err << "file open error!\n"; + exit(1); + } + fseek(pFile, 0, SEEK_END); + size_t lSize = ftell(pFile); + rewind(pFile); + in_buff = (T *)malloc(lSize); + lSize = fread(in_buff, 1, lSize, pFile); + fclose(pFile); + // min_max(lSize/sizeof(T), in_buff); + return lSize; +} + +template +size_t readfile_header_metadata(std::string input_file, + std::vector &in_buff) { + // std::cout << mgard_x::log::log_info << "Loading file: " << input_file << + // "\n"; + + FILE *pFile; + pFile = fopen(input_file.c_str(), "rb"); + if (pFile == NULL) { + std::cout << mgard_x::log::log_err << "file open error!\n"; + exit(1); + } + fseek(pFile, 0, SEEK_END); + size_t lSize = ftell(pFile); + rewind(pFile); + in_buff.resize(lSize / sizeof(T)); + lSize = fread(in_buff.data(), 1, lSize, pFile); + fclose(pFile); + return lSize; +} + +template +void writefile(std::string output_file, T *out_buff, size_t num_bytes) { + FILE *file = fopen(output_file.c_str(), "w"); + fwrite(out_buff, 1, num_bytes, file); + fclose(file); +} + +template +void print_statistics(double s, enum mgard_x::error_bound_type mode, + std::vector shape, T *original_data, + T *decompressed_data, T tol, bool normalize_coordinates) { + mgard_x::SIZE n = 1; + for (mgard_x::DIM d = 0; d < shape.size(); d++) + n *= shape[d]; + T actual_error = 0.0; + std::cout << std::scientific; + if (s == std::numeric_limits::infinity()) { + actual_error = + mgard_x::L_inf_error(n, original_data, decompressed_data, mode); + if (mode == mgard_x::error_bound_type::ABS) { + std::cout << mgard_x::log::log_info + << "Absoluate L_inf error: " << actual_error << " (" + << (actual_error < tol ? "\e[32mSatisified\e[0m" + : "\e[31mNot Satisified\e[0m") + << ")" + << "\n"; + } else if (mode == mgard_x::error_bound_type::REL) { + std::cout << mgard_x::log::log_info + << "Relative L_inf error: " << actual_error << " (" + << (actual_error < tol ? "\e[32mSatisified\e[0m" + : "\e[31mNot Satisified\e[0m") + << ")" + << "\n"; + } + } else { + actual_error = mgard_x::L_2_error(shape, original_data, decompressed_data, + mode, normalize_coordinates); + if (mode == mgard_x::error_bound_type::ABS) { + std::cout << mgard_x::log::log_info + << "Absoluate L_2 error: " << actual_error << " (" + << (actual_error < tol ? "\e[32mSatisified\e[0m" + : "\e[31mNot Satisified\e[0m") + << ")" + << "\n"; + } else if (mode == mgard_x::error_bound_type::REL) { + std::cout << mgard_x::log::log_info + << "Relative L_2 error: " << actual_error << " (" + << (actual_error < tol ? "\e[32mSatisified\e[0m" + : "\e[31mNot Satisified\e[0m") + << ")" + << "\n"; + } + } + + std::cout << mgard_x::log::log_info + << "MSE: " << mgard_x::MSE(n, original_data, decompressed_data) + << "\n"; + std::cout << std::defaultfloat; + std::cout << mgard_x::log::log_info + << "PSNR: " << mgard_x::PSNR(n, original_data, decompressed_data) + << "\n"; + + // if (actual_error > tol) + // exit(-1); +} + +void create_dir(std::string name) { + struct stat st = {0}; + if (stat(name.c_str(), &st) == -1) { + mkdir(name.c_str(), 0700); + } +} + +void write_mdr(mgard_x::MDR::RefactoredMetadata &refactored_metadata, + mgard_x::MDR::RefactoredData &refactored_data, + std::string output) { + size_t size_written = 0; + create_dir(output); + std::vector serialized_metadata = + refactored_metadata.Serialize(); + writefile(output + "/header", refactored_metadata.header.data(), + refactored_metadata.header.size()); + writefile(output + "/metadata", serialized_metadata.data(), + serialized_metadata.size()); + for (int subdomain_id = 0; subdomain_id < refactored_metadata.metadata.size(); + subdomain_id++) { + for (int level_idx = 0; + level_idx < + refactored_metadata.metadata[subdomain_id].level_sizes.size(); + level_idx++) { + for (int bitplane_idx = 0; + bitplane_idx < refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx] + .size(); + bitplane_idx++) { + if (refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx] > 0) { + std::string filename = "component_" + std::to_string(subdomain_id) + + "_" + std::to_string(level_idx) + "_" + + std::to_string(bitplane_idx); + writefile(output + "/" + filename, + refactored_data.data[subdomain_id][level_idx][bitplane_idx], + refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx]); + size_written += refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx]; + } + } + } + } + std::cout << mgard_x::log::log_info << size_written << " bytes written\n"; +} + +size_t read_mdr_metadata(mgard_x::MDR::RefactoredMetadata &refactored_metadata, + mgard_x::MDR::RefactoredData &refactored_data, + std::string input) { + + size_t metadata_size = 0; + metadata_size += + readfile_header_metadata(input + "/header", refactored_metadata.header); + std::vector serialized_metadata; + metadata_size += + readfile_header_metadata(input + "/metadata", serialized_metadata); + refactored_metadata.Deserialize(serialized_metadata); + refactored_metadata.InitializeForReconstruction(); + refactored_data.InitializeForReconstruction(refactored_metadata); + return metadata_size; +} + +// size_t read_mdr(mgard_x::MDR::RefactoredMetadata &refactored_metadata, +// mgard_x::MDR::RefactoredData &refactored_data, std::string +// input, bool initialize_signs, mgard_x::Config config) { + +// size_t size_read = 0; +// int num_subdomains = refactored_metadata.metadata.size(); +// for (int subdomain_id = 0; subdomain_id < num_subdomains; subdomain_id++) { +// mgard_x::MDR::MDRMetadata metadata = +// refactored_metadata.metadata[subdomain_id]; +// int num_levels = metadata.level_sizes.size(); +// for (int level_idx = 0; level_idx < num_levels; level_idx++) { +// int num_bitplanes = metadata.level_sizes[level_idx].size(); +// int loaded_bitplanes = metadata.loaded_level_num_bitplanes[level_idx]; +// int reqested_bitplanes = +// metadata.requested_level_num_bitplanes[level_idx]; +// for (int bitplane_idx = loaded_bitplanes; +// bitplane_idx < reqested_bitplanes; bitplane_idx++) { +// std::string filename = "component_" + std::to_string(subdomain_id) + +// "_" + std::to_string(level_idx) + "_" + +// std::to_string(bitplane_idx); +// mgard_x::SIZE level_size = readfile( +// input + "/" + filename, +// refactored_data.data[subdomain_id][level_idx][bitplane_idx]); +// mgard_x::pin_memory( +// refactored_data.data[subdomain_id][level_idx][bitplane_idx], +// level_size, config); +// if (level_size != refactored_metadata.metadata[subdomain_id] +// .level_sizes[level_idx][bitplane_idx]) { +// std::cout << "mdr component size mismatch."; +// exit(-1); +// } +// size_read += level_size; +// } +// if (initialize_signs) { +// // level sign +// refactored_data.level_signs[subdomain_id][level_idx] = +// (bool *)malloc(sizeof(bool) * +// metadata.level_num_elems[level_idx]); +// memset(refactored_data.level_signs[subdomain_id][level_idx], 0, +// sizeof(bool) * metadata.level_num_elems[level_idx]); +// mgard_x::pin_memory( +// refactored_data.level_signs[subdomain_id][level_idx], +// sizeof(bool) * metadata.level_num_elems[level_idx], config); +// } +// } +// } +// return size_read; +// } + +size_t read_mdr(mgard_x::MDR::RefactoredMetadata &refactored_metadata, + mgard_x::MDR::RefactoredData &refactored_data, + std::string input, bool initialize_signs, + mgard_x::Config config) { + + size_t size_read = 0; + int num_subdomains = refactored_metadata.metadata.size(); + for (int subdomain_id = 0; subdomain_id < num_subdomains; subdomain_id++) { + mgard_x::MDR::MDRMetadata metadata = + refactored_metadata.metadata[subdomain_id]; + int num_levels = metadata.level_sizes.size(); + for (int level_idx = 0; level_idx < num_levels; level_idx++) { + int num_bitplanes = metadata.level_sizes[level_idx].size(); + for (int bitplane_idx = 0; bitplane_idx < num_bitplanes; bitplane_idx++) { + if (refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx] > 0) { + std::string filename = "component_" + std::to_string(subdomain_id) + + "_" + std::to_string(level_idx) + "_" + + std::to_string(bitplane_idx); + mgard_x::SIZE level_size = readfile( + input + "/" + filename, + refactored_data.data[subdomain_id][level_idx][bitplane_idx]); + mgard_x::pin_memory( + refactored_data.data[subdomain_id][level_idx][bitplane_idx], + level_size, config); + if (level_size != refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx]) { + std::cout << "mdr component size mismatch."; + exit(-1); + } + size_read += level_size; + } + } + if (initialize_signs) { + // level sign + refactored_data.level_signs[subdomain_id][level_idx] = + (bool *)malloc(sizeof(bool) * metadata.level_num_elems[level_idx]); + memset(refactored_data.level_signs[subdomain_id][level_idx], 0, + sizeof(bool) * metadata.level_num_elems[level_idx]); + mgard_x::pin_memory( + refactored_data.level_signs[subdomain_id][level_idx], + sizeof(bool) * metadata.level_num_elems[level_idx], config); + } + } + } + return size_read; +} + +int verbose_to_log_level(int verbose) { + if (verbose == 0) { + return mgard_x::log::ERR; + } else if (verbose == 1) { + return mgard_x::log::ERR | mgard_x::log::INFO; + } else if (verbose == 2) { + return mgard_x::log::ERR | mgard_x::log::TIME; + } else if (verbose == 3) { + return mgard_x::log::ERR | mgard_x::log::INFO | mgard_x::log::TIME; + } +} + +template +int launch_refactor(mgard_x::DIM D, enum mgard_x::data_type dtype, + std::string input_file, std::string output_file, + std::vector shape, + std::string domain_decomposition, mgard_x::SIZE block_size, + enum mgard_x::device_type dev_type, int verbose, + mgard_x::SIZE max_memory_footprint) { + + mgard_x::Config config; + config.normalize_coordinates = false; + config.log_level = verbose_to_log_level(verbose); + config.decomposition = mgard_x::decomposition_type::MultiDim; + if (domain_decomposition == "max-dim") { + config.domain_decomposition = mgard_x::domain_decomposition_type::MaxDim; + } else if (domain_decomposition == "block") { + config.domain_decomposition = mgard_x::domain_decomposition_type::Block; + config.block_size = block_size; + } else if (domain_decomposition == "variable") { + config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; + } + + config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; + config.domain_decomposition_dim = 0; + config.domain_decomposition_sizes = {shape[0] / 3, shape[0] / 3, + shape[0] / 3}; + + config.dev_type = dev_type; + config.max_memory_footprint = max_memory_footprint; + if (dtype == mgard_x::data_type::Float) { + config.total_num_bitplanes = 32; + } else if (dtype == mgard_x::data_type::Double) { + config.total_num_bitplanes = 64; + } + + size_t original_size = 1; + for (mgard_x::DIM i = 0; i < D; i++) + original_size *= shape[i]; + T *original_data = (T *)malloc(original_size * sizeof(T)); + size_t in_size = 0; + if (std::string(input_file).compare("random") == 0) { + in_size = original_size * sizeof(T); + srand(7117); + T c = 0; + for (size_t i = 0; i < original_size; i++) { + original_data[i] = rand() % 10 + 1; + } + } else { + T *file_data; + in_size = readfile(input_file, file_data); + + size_t loaded_size = 0; + while (loaded_size < original_size) { + std::memcpy(original_data + loaded_size, file_data, + std::min(in_size / sizeof(T), original_size - loaded_size) * + sizeof(T)); + loaded_size += std::min(in_size / sizeof(T), original_size - loaded_size); + } + in_size = loaded_size * sizeof(T); + } + if (in_size != original_size * sizeof(T)) { + std::cout << mgard_x::log::log_warn << "input file size mismatch " + << in_size << " vs. " << original_size * sizeof(T) << "!\n"; + } + + std::cout << mgard_x::log::log_info << "Max output data size: " + << mgard_x::MDR::MDRMaxOutputDataSize(D, dtype, shape, config) + << " bytes\n"; + + mgard_x::MDR::RefactoredMetadata refactored_metadata; + mgard_x::MDR::RefactoredData refactored_data; + mgard_x::pin_memory(original_data, original_size * sizeof(T), config); + + mgard_x::MDR::MDRefactor(D, dtype, shape, original_data, refactored_metadata, + refactored_data, config, false); + + write_mdr(refactored_metadata, refactored_data, output_file); + + mgard_x::unpin_memory(original_data, config); + delete[](T *) original_data; + + return 0; +} + +template +T compute_max_abs_error(const T *vec_ori, const T *vec_rec, size_t n) { + T error = fabs(vec_ori[0] - vec_rec[0]); + T max = error; + for (int i = 1; i < n; i++) { + error = fabs(vec_ori[i] - vec_rec[i]); + if (max < error) + max = error; + } + return max; +} + +template T compute_value_range(const T *vec, size_t n) { + T min = vec[0]; + T max = vec[0]; + for (int i = 0; i < n; i++) { + if (vec[i] < min) + min = vec[i]; + if (vec[i] > max) + max = vec[i]; + } + return max - min; +} + +template T compute_max_abs_value(const T *data, size_t n) { + T max_val = 0; + for (int i = 0; i < n; i++) { + T val = fabs(data[i]); + if (val > max_val) + max_val = val; + } + return max_val; +} + +template +void compute_VTOT(const T *Vx, const T *Vy, const T *Vz, size_t n, T *V_TOT_) { + for (int i = 0; i < n; i++) { + double V_TOT_2 = Vx[i] * Vx[i] + Vy[i] * Vy[i] + Vz[i] * Vz[i]; + double V_TOT = sqrt(V_TOT_2); + V_TOT_[i] = V_TOT; + } +} + +int launch_reconstruct(std::string input_file, std::string output_file, + std::string original_file, enum mgard_x::data_type dtype, + std::vector shape, + std::vector tols, double s, + enum mgard_x::error_bound_type mode, + bool adaptive_resolution, + enum mgard_x::device_type dev_type, int verbose, + int decrease_method = 0) { + + double bitrate = 0; + mgard_x::Config config; + config.normalize_coordinates = false; + config.log_level = verbose_to_log_level(verbose); + config.dev_type = dev_type; + config.mdr_adaptive_resolution = adaptive_resolution; + + config.mdr_qoi_mode = true; + config.mdr_qoi_num_variables = shape.size(); + config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; + config.domain_decomposition_dim = 0; + config.domain_decomposition_sizes = {shape[0] / 3, shape[0] / 3, + shape[0] / 3}; + + mgard_x::Byte *original_data; + size_t in_size = 0; + size_t original_size = 1; + for (mgard_x::DIM i = 0; i < shape.size(); i++) + original_size *= shape[i]; + if (original_file.compare("none") != 0 && !config.mdr_adaptive_resolution) { + if (original_file.compare("random") == 0) { + if (dtype == mgard_x::data_type::Float) { + in_size = original_size * sizeof(float); + original_data = (mgard_x::Byte *)new float[original_size]; + srand(7117); + for (size_t i = 0; i < original_size; i++) { + ((float *)original_data)[i] = rand() % 10 + 1; + } + } else if (dtype == mgard_x::data_type::Double) { + in_size = original_size * sizeof(double); + original_data = (mgard_x::Byte *)new double[original_size]; + srand(7117); + for (size_t i = 0; i < original_size; i++) { + ((double *)original_data)[i] = rand() % 10 + 1; + } + } + } else { + mgard_x::Byte *file_data; + in_size = readfile(original_file, file_data); + + if (dtype == mgard_x::data_type::Float) { + original_size *= sizeof(float); + } else if (dtype == mgard_x::data_type::Double) { + original_size *= sizeof(double); + } + + original_data = (mgard_x::Byte *)malloc(original_size); + + size_t loaded_size = 0; + while (loaded_size < original_size) { + + std::memcpy(original_data + loaded_size, file_data, + std::min(in_size, original_size - loaded_size)); + loaded_size += std::min(in_size, original_size - loaded_size); + } + in_size = loaded_size; + } + } + mgard_x::Byte *V_TOT_ori; + std::vector ebs; + size_t num_elements; + double tau = 0; + V_TOT_ori = (mgard_x::Byte *)malloc(in_size / config.mdr_qoi_num_variables); + mgard_x::Byte *org_Vx_ptr = original_data + original_size / 3 * 0; + mgard_x::Byte *org_Vy_ptr = original_data + original_size / 3 * 1; + mgard_x::Byte *org_Vz_ptr = original_data + original_size / 3 * 2; + if (dtype == mgard_x::data_type::Float) { + num_elements = (in_size / config.mdr_qoi_num_variables) / sizeof(float); + compute_VTOT((float *)org_Vx_ptr, (float *)org_Vy_ptr, + (float *)org_Vz_ptr, num_elements, (float *)V_TOT_ori); + tau = compute_value_range((float *)V_TOT_ori, num_elements) * tols[0]; + if (decrease_method < 4) { + ebs.push_back(compute_value_range((float *)org_Vx_ptr, num_elements) * + tols[0]); + ebs.push_back(compute_value_range((float *)org_Vy_ptr, num_elements) * + tols[0]); + ebs.push_back(compute_value_range((float *)org_Vz_ptr, num_elements) * + tols[0]); + } else { + std::vector var_value_range; + var_value_range.push_back( + compute_value_range((float *)org_Vx_ptr, num_elements) * tols[0]); + var_value_range.push_back( + compute_value_range((float *)org_Vy_ptr, num_elements) * tols[0]); + var_value_range.push_back( + compute_value_range((float *)org_Vz_ptr, num_elements) * tols[0]); + for (int i = 0; i < config.mdr_qoi_num_variables; i++) { + ebs.push_back( + *std::min_element(var_value_range.begin(), var_value_range.end())); + } + } + } else if (dtype == mgard_x::data_type::Double) { + num_elements = (in_size / config.mdr_qoi_num_variables) / sizeof(double); + compute_VTOT((double *)org_Vx_ptr, (double *)org_Vy_ptr, + (double *)org_Vz_ptr, num_elements, + (double *)V_TOT_ori); + tau = compute_value_range((double *)V_TOT_ori, num_elements) * tols[0]; + if (decrease_method < 2) { + ebs.push_back(compute_value_range((double *)org_Vx_ptr, num_elements) * + tols[0]); + ebs.push_back(compute_value_range((double *)org_Vy_ptr, num_elements) * + tols[0]); + ebs.push_back(compute_value_range((double *)org_Vz_ptr, num_elements) * + tols[0]); + } else { + std::vector var_value_range; + var_value_range.push_back( + compute_value_range((double *)org_Vx_ptr, num_elements) * tols[0]); + var_value_range.push_back( + compute_value_range((double *)org_Vy_ptr, num_elements) * tols[0]); + var_value_range.push_back( + compute_value_range((double *)org_Vz_ptr, num_elements) * tols[0]); + for (int i = 0; i < config.mdr_qoi_num_variables; i++) { + ebs.push_back( + *std::min_element(var_value_range.begin(), var_value_range.end())); + } + } + } + + mgard_x::MDR::RefactoredMetadata refactored_metadata; + mgard_x::MDR::RefactoredData refactored_data; + mgard_x::MDR::ReconstructedData reconstructed_data; + size_t metadata_size = + read_mdr_metadata(refactored_metadata, refactored_data, input_file); + refactored_metadata.total_size += metadata_size; + + refactored_metadata.relative_eb = tols[0]; + refactored_metadata.decrease_method = decrease_method; + for (int i = 0; i < config.mdr_qoi_num_variables; i++) { + refactored_metadata.metadata[i].num_elements = num_elements; + if (decrease_method == 0) { + refactored_metadata.metadata[i].requested_tol = ebs[i]; + } else if (decrease_method == 1) { + refactored_metadata.metadata[i].requested_size = 1; + refactored_metadata.metadata[i].segmented = true; + } else if (decrease_method >= 2) { + refactored_metadata.metadata[i].requested_tol = ebs[i]; + refactored_metadata.metadata[i].corresponding_error_return = true; + } + refactored_metadata.metadata[i].tau = tau; + refactored_metadata.metadata[i].requested_s = s; + } + // std::cout << "refactored_metadata.total_size = " << + // refactored_metadata.total_size << std::endl; + mgard_x::MDR::MDRequest(refactored_metadata, config); + // refactored_metadata.total_size += + // refactored_metadata.metadata[0].retrieved_size + // + + // refactored_metadata.metadata[1].retrieved_size + // + + // refactored_metadata.metadata[2].retrieved_size; + // for (auto &metadata : refactored_metadata.metadata) { + // metadata.PrintStatus(); + // } + size_t size_read = + read_mdr(refactored_metadata, refactored_data, input_file, true, config); + // refactored_metadata.total_size += size_read; + + mgard_x::MDR::MDReconstruct(refactored_metadata, refactored_data, + reconstructed_data, config, false); + + // we can check reconstructed_data.qoi_in_progress here + + // std::cout << mgard_x::log::log_info << "Additional " << size_read + // << " bytes read for reconstruction\n"; + + std::vector rec_var_ptrs; + if (original_file.compare("none") != 0 && !config.mdr_adaptive_resolution) { + for (int i = 0; i < config.mdr_qoi_num_variables; i++) { + std::vector var_shape = shape; + var_shape[0] /= config.mdr_qoi_num_variables; + mgard_x::Byte *org_var_ptr = original_data + original_size / 3 * i; + mgard_x::Byte *rec_var_ptr = + reconstructed_data.data[0] + original_size / 3 * i; + rec_var_ptrs.push_back(rec_var_ptr); + // if (dtype == mgard_x::data_type::Float) { + // print_statistics(s, mode, var_shape, (float *)org_var_ptr, + // (float *)rec_var_ptr, + // refactored_metadata.metadata[i].requested_tol, + // config.normalize_coordinates); + // } else if (dtype == mgard_x::data_type::Double) { + // print_statistics(s, mode, var_shape, (double *)org_var_ptr, + // (double *)rec_var_ptr, + // refactored_metadata.metadata[i].requested_tol, + // config.normalize_coordinates); + // } + } + } + mgard_x::Byte *V_TOT_rec; + V_TOT_rec = (mgard_x::Byte *)malloc(in_size / config.mdr_qoi_num_variables); + if (dtype == mgard_x::data_type::Float) { + compute_VTOT((float *)rec_var_ptrs[0], (float *)rec_var_ptrs[1], + (float *)rec_var_ptrs[2], num_elements, + (float *)V_TOT_rec); + } else if (dtype == mgard_x::data_type::Double) { + compute_VTOT((double *)rec_var_ptrs[0], (double *)rec_var_ptrs[1], + (double *)rec_var_ptrs[2], num_elements, + (double *)V_TOT_rec); + } + std::vector var_shape = shape; + var_shape[0] /= config.mdr_qoi_num_variables; + for (auto &metadata : refactored_metadata.metadata) { + refactored_metadata.total_size += metadata.GetLoadedBitPlaneSizes(); + } + if (dtype == mgard_x::data_type::Float) { + // print_statistics(s, mode, var_shape, (float *) V_TOT_ori, + // (float *) V_TOT_rec, tau, + // config.normalize_coordinates); + bitrate = 32 / ((double)in_size / refactored_metadata.total_size); + } else if (dtype == mgard_x::data_type::Double) { + // print_statistics(s, mode, var_shape, (double *) V_TOT_ori, + // (double *) V_TOT_rec, tau, + // config.normalize_coordinates); + bitrate = 64 / ((double)in_size / refactored_metadata.total_size); + } + // std::cout << "refactored_metadata.total_size = " << + // refactored_metadata.total_size << std::endl; std::cout << "in_size = " << + // in_size << std::endl; + std::cout << "Bitrate = " << bitrate << std::endl; + // std::cout << "Original Vx[35345] = " << ((float*) org_Vx_ptr)[35345] << ", + // Reconstructed Vx[35345] = " << ((float*) rec_var_ptrs[0])[35345] << + // std::endl; + std::cout << "Requested_Tau = " << tau << std::endl; + std::cout << "Est_max_error = " << refactored_metadata.max_est_error + << std::endl; + std::cout << "Real_max_error = " + << compute_max_abs_error((float *)V_TOT_ori, (float *)V_TOT_rec, + num_elements) + << std::endl; + return 0; +} + +bool try_refactoring(int argc, char *argv[]) { + if (!has_arg(argc, argv, "-z", "--refactor")) + return false; + mgard_x::log::info("Mode: refactor", true); + + std::string input_file = + get_arg(argc, argv, "Original data", "-i", "--input"); + std::string output_file = + get_arg(argc, argv, "Refactored data", "-o", "--output"); + enum mgard_x::data_type dtype = get_data_type(argc, argv); + std::vector shape = + get_args(argc, argv, "Dimensions", "-dim", "--dimension"); + // std::string lossless_level = get_arg(argc, argv, "Lossless", + // "-l", "--lossless"); + enum mgard_x::device_type dev_type = get_device_type(argc, argv); + int verbose = 0; + if (has_arg(argc, argv, "-v", "--verbose")) { + verbose = get_arg(argc, argv, "Verbose", "-v", "--verbose"); + } + mgard_x::SIZE max_memory_footprint = + std::numeric_limits::max(); + if (has_arg(argc, argv, "-m", "--max-memory")) { + max_memory_footprint = (mgard_x::SIZE)get_arg( + argc, argv, "Max memory", "-m", "--max-memory"); + } + std::string domain_decomposition = "max-dim"; + mgard_x::SIZE block_size = 0; + if (has_arg(argc, argv, "-dd", "--domain-decomposition")) { + domain_decomposition = get_arg( + argc, argv, "Domain decomposition", "-dd", "--domain-decomposition"); + if (domain_decomposition == "block") { + block_size = get_arg(argc, argv, "Block size", "-dd-size", + "--domain-decomposition-size"); + } + } + if (dtype == mgard_x::data_type::Double) { + launch_refactor(shape.size(), dtype, input_file.c_str(), + output_file.c_str(), shape, domain_decomposition, + block_size, dev_type, verbose, + max_memory_footprint); + } else if (dtype == mgard_x::data_type::Float) { + launch_refactor(shape.size(), dtype, input_file.c_str(), + output_file.c_str(), shape, domain_decomposition, + block_size, dev_type, verbose, max_memory_footprint); + } + return true; +} + +bool try_reconstruction(int argc, char *argv[]) { + if (!has_arg(argc, argv, "-x", "--reconstruct")) + return false; + mgard_x::log::info("mode: reconstruct", true); + std::string input_file = + get_arg(argc, argv, "Refactored data", "-i", "--input"); + std::string output_file = + get_arg(argc, argv, "Reconstructed data", "-o", "--output"); + // default is none (means original data not provided) + std::string original_file = "none"; + enum mgard_x::data_type dtype; + std::vector shape; + if (has_arg(argc, argv, "-g", "--orignal")) { + original_file = + get_arg(argc, argv, "Original data", "-g", "--orignal"); + dtype = get_data_type(argc, argv); + shape = get_args(argc, argv, "Dimensions", "-dim", + "--dimension"); + } + // only abs mode is supported now + enum mgard_x::error_bound_type mode = + mgard_x::error_bound_type::ABS; // REL or ABS + + std::vector tols; + if (has_arg(argc, argv, "-e", "--error-bound")) { + tols.push_back( + get_arg(argc, argv, "Error bound", "-e", "--error-bound")); + } else if (has_arg(argc, argv, "-me", "--multi-error-bounds")) { + tols = get_args(argc, argv, "Multi error bounds", "-me", + "--multi-error-bounds"); + } else { + throw std::runtime_error( + "Missing option -e/--error-bound or -me/--multi-error-bounds"); + } + double s = get_arg(argc, argv, "Smoothness", "-s", "--smoothness"); + enum mgard_x::device_type dev_type = get_device_type(argc, argv); + int verbose = 0; + if (has_arg(argc, argv, "-v", "--verbose")) { + verbose = get_arg(argc, argv, "Verbose", "-v", "--verbose"); + } + bool adaptive_resolution = false; + if (has_arg(argc, argv, "-ar", "--adaptive-resolution")) { + adaptive_resolution = get_arg(argc, argv, "Adaptive resolution", "-ar", + "--adaptive-resolution"); + } + if (verbose) + std::cout << mgard_x::log::log_info << "verbose: enabled.\n"; + int decrease_method; + if (has_arg(argc, argv, "-dm", "--decrease-method")) { + decrease_method = + get_arg(argc, argv, "Decrease method", "-dm", "--decrease-method"); + } + launch_reconstruct(input_file, output_file, original_file, dtype, shape, tols, + s, mode, adaptive_resolution, dev_type, verbose, + decrease_method); + return true; +} + +int main(int argc, char *argv[]) { + + if (!try_refactoring(argc, argv) && !try_reconstruction(argc, argv)) { + print_usage_message(""); + } + return 0; +} \ No newline at end of file diff --git a/src/mgard-x/Executables/mdr-x.cpp b/src/mgard-x/Executables/mdr-x.cpp index a6c10d3310..e6c84382c6 100644 --- a/src/mgard-x/Executables/mdr-x.cpp +++ b/src/mgard-x/Executables/mdr-x.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include @@ -172,7 +172,7 @@ void print_statistics(double s, enum mgard_x::error_bound_type mode, << "\n"; if (actual_error > tol) - exit(-1); + throw std::runtime_error("Error tolerance exceeded"); } void create_dir(std::string name) { @@ -204,19 +204,24 @@ void write_mdr(mgard_x::MDR::RefactoredMetadata &refactored_metadata, .level_sizes[level_idx] .size(); bitplane_idx++) { - std::string filename = "component_" + std::to_string(subdomain_id) + - "_" + std::to_string(level_idx) + "_" + - std::to_string(bitplane_idx); - writefile(output + "/" + filename, - refactored_data.data[subdomain_id][level_idx][bitplane_idx], - refactored_metadata.metadata[subdomain_id] - .level_sizes[level_idx][bitplane_idx]); - size_written += refactored_metadata.metadata[subdomain_id] - .level_sizes[level_idx][bitplane_idx]; + if (refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx] > 0) { + std::string filename = "component_" + std::to_string(subdomain_id) + + "_" + std::to_string(level_idx) + "_" + + std::to_string(bitplane_idx); + + writefile(output + "/" + filename, + refactored_data.data[subdomain_id][level_idx][bitplane_idx], + refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx]); + size_written += refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx]; + } } } } - std::cout << mgard_x::log::log_info << size_written << " bytes written\n"; + mgard_x::log::info(std::to_string(size_written) + " bytes written"); + mgard_x::log::csv("size.csv", size_written); } void read_mdr_metadata(mgard_x::MDR::RefactoredMetadata &refactored_metadata, @@ -231,10 +236,12 @@ void read_mdr_metadata(mgard_x::MDR::RefactoredMetadata &refactored_metadata, refactored_data.InitializeForReconstruction(refactored_metadata); } -void read_mdr(mgard_x::MDR::RefactoredMetadata &refactored_metadata, - mgard_x::MDR::RefactoredData &refactored_data, std::string input, - bool initialize_signs) { +size_t read_mdr(mgard_x::MDR::RefactoredMetadata &refactored_metadata, + mgard_x::MDR::RefactoredData &refactored_data, + std::string input, bool initialize_signs, + mgard_x::Config config) { + size_t size_read = 0; int num_subdomains = refactored_metadata.metadata.size(); for (int subdomain_id = 0; subdomain_id < num_subdomains; subdomain_id++) { mgard_x::MDR::MDRMetadata metadata = @@ -247,16 +254,23 @@ void read_mdr(mgard_x::MDR::RefactoredMetadata &refactored_metadata, metadata.requested_level_num_bitplanes[level_idx]; for (int bitplane_idx = loaded_bitplanes; bitplane_idx < reqested_bitplanes; bitplane_idx++) { - std::string filename = "component_" + std::to_string(subdomain_id) + - "_" + std::to_string(level_idx) + "_" + - std::to_string(bitplane_idx); - mgard_x::SIZE level_size = readfile( - input + "/" + filename, - refactored_data.data[subdomain_id][level_idx][bitplane_idx]); - if (level_size != refactored_metadata.metadata[subdomain_id] - .level_sizes[level_idx][bitplane_idx]) { - std::cout << "mdr component size mismatch."; - exit(-1); + if (refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx] > 0) { + std::string filename = "component_" + std::to_string(subdomain_id) + + "_" + std::to_string(level_idx) + "_" + + std::to_string(bitplane_idx); + mgard_x::SIZE level_size = readfile( + input + "/" + filename, + refactored_data.data[subdomain_id][level_idx][bitplane_idx]); + mgard_x::pin_memory( + refactored_data.data[subdomain_id][level_idx][bitplane_idx], + level_size, config); + if (level_size != refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx]) { + std::cout << "mdr component size mismatch."; + exit(-1); + } + size_read += level_size; } } if (initialize_signs) { @@ -265,9 +279,13 @@ void read_mdr(mgard_x::MDR::RefactoredMetadata &refactored_metadata, (bool *)malloc(sizeof(bool) * metadata.level_num_elems[level_idx]); memset(refactored_data.level_signs[subdomain_id][level_idx], 0, sizeof(bool) * metadata.level_num_elems[level_idx]); + mgard_x::pin_memory( + refactored_data.level_signs[subdomain_id][level_idx], + sizeof(bool) * metadata.level_num_elems[level_idx], config); } } } + return size_read; } int verbose_to_log_level(int verbose) { @@ -303,6 +321,13 @@ int launch_refactor(mgard_x::DIM D, enum mgard_x::data_type dtype, config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; } + // config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; + // config.domain_decomposition_dim = 0; + // config.domain_decomposition_sizes = {512, 512, 512, 512, 512, 512, 512, + // 512}; config.domain_decomposition_sizes = {98, 98, 98, 98, 98, 98, 98, 98}; + // config.domain_decomposition_sizes = std::vector(8, 256); + // config.domain_decomposition_sizes = std::vector(8, 100); + config.dev_type = dev_type; config.max_memory_footprint = max_memory_footprint; if (dtype == mgard_x::data_type::Float) { @@ -341,9 +366,10 @@ int launch_refactor(mgard_x::DIM D, enum mgard_x::data_type dtype, << in_size << " vs. " << original_size * sizeof(T) << "!\n"; } - std::cout << mgard_x::log::log_info << "Max output data size: " - << mgard_x::MDR::MDRMaxOutputDataSize(D, dtype, shape, config) - << " bytes\n"; + mgard_x::log::info("Max output data size: " + + std::to_string(mgard_x::MDR::MDRMaxOutputDataSize( + D, dtype, shape, config)) + + " bytes"); mgard_x::MDR::RefactoredMetadata refactored_metadata; mgard_x::MDR::RefactoredData refactored_data; @@ -360,6 +386,7 @@ int launch_refactor(mgard_x::DIM D, enum mgard_x::data_type dtype, return 0; } +template int launch_reconstruct(std::string input_file, std::string output_file, std::string original_file, enum mgard_x::data_type dtype, std::vector shape, @@ -374,31 +401,44 @@ int launch_reconstruct(std::string input_file, std::string output_file, config.dev_type = dev_type; config.mdr_adaptive_resolution = adaptive_resolution; - mgard_x::Byte *original_data; - size_t in_size = 0; - if (original_file.compare("none") != 0 && !config.mdr_adaptive_resolution) { - if (original_file.compare("random") == 0) { - size_t original_size = 1; - for (mgard_x::DIM i = 0; i < shape.size(); i++) - original_size *= shape[i]; - if (dtype == mgard_x::data_type::Float) { - in_size = original_size * sizeof(float); - original_data = (mgard_x::Byte *)new float[original_size]; - srand(7117); - for (size_t i = 0; i < original_size; i++) { - ((float *)original_data)[i] = rand() % 10 + 1; - } - } else if (dtype == mgard_x::data_type::Double) { - in_size = original_size * sizeof(double); - original_data = (mgard_x::Byte *)new double[original_size]; - srand(7117); - for (size_t i = 0; i < original_size; i++) { - ((double *)original_data)[i] = rand() % 10 + 1; - } - } + // config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; + // config.domain_decomposition_dim = 0; + // config.domain_decomposition_sizes = {512, 512, 512, 512, 512, 512, 512, + // 512}; config.domain_decomposition_sizes = {98, 98, 98, 98, 98, 98, 98, 98}; + // config.domain_decomposition_sizes = std::vector(8, 256); + // config.domain_decomposition_sizes = std::vector(8, 100); + size_t original_size = 1; + T *original_data = nullptr; + if (original_file.compare("none") != 0) { + for (mgard_x::DIM i = 0; i < shape.size(); i++) + original_size *= shape[i]; + original_data = (T *)malloc(original_size * sizeof(T)); + size_t in_size = 0; + if (std::string(original_file).compare("random") == 0) { + in_size = original_size * sizeof(T); + srand(7117); + T c = 0; + for (size_t i = 0; i < original_size; i++) { + original_data[i] = rand() % 10 + 1; + } } else { - in_size = readfile(original_file, original_data); + T *file_data; + in_size = readfile(original_file, file_data); + + size_t loaded_size = 0; + while (loaded_size < original_size) { + std::memcpy(original_data + loaded_size, file_data, + std::min(in_size / sizeof(T), original_size - loaded_size) * + sizeof(T)); + loaded_size += + std::min(in_size / sizeof(T), original_size - loaded_size); + } + in_size = loaded_size * sizeof(T); + } + if (in_size != original_size * sizeof(T)) { + std::cout << mgard_x::log::log_warn << "input file size mismatch " + << in_size << " vs. " << original_size * sizeof(T) << "!\n"; } } @@ -416,14 +456,21 @@ int launch_reconstruct(std::string input_file, std::string output_file, for (auto &metadata : refactored_metadata.metadata) { metadata.PrintStatus(); } - read_mdr(refactored_metadata, refactored_data, input_file, - first_reconstruction); + size_t size_read = read_mdr(refactored_metadata, refactored_data, + input_file, first_reconstruction, config); mgard_x::MDR::MDReconstruct(refactored_metadata, refactored_data, reconstructed_data, config, false); first_reconstruction = false; + mgard_x::log::info( + "Additional " + std::to_string(size_read) + " bytes (" + + std::to_string((float)100 * size_read / (original_size * sizeof(T))) + + "%) read for reconstruction"); + + // mgard_x::log::csv("size.csv", size_read); + if (original_file.compare("none") != 0 && !config.mdr_adaptive_resolution) { if (dtype == mgard_x::data_type::Float) { print_statistics(s, mode, shape, (float *)original_data, @@ -534,8 +581,16 @@ bool try_reconstruction(int argc, char *argv[]) { } if (verbose) std::cout << mgard_x::log::log_info << "verbose: enabled.\n"; - launch_reconstruct(input_file, output_file, original_file, dtype, shape, tols, - s, mode, adaptive_resolution, dev_type, verbose); + if (dtype == mgard_x::data_type::Double) { + launch_reconstruct(input_file, output_file, original_file, dtype, + shape, tols, s, mode, adaptive_resolution, + dev_type, verbose); + } else if (dtype == mgard_x::data_type::Float) { + launch_reconstruct(input_file, output_file, original_file, dtype, + shape, tols, s, mode, adaptive_resolution, + dev_type, verbose); + } + return true; } diff --git a/src/mgard-x/Executables/mgard-x-autotuner.cpp b/src/mgard-x/Executables/mgard-x-autotuner.cpp index b198fa26ee..7d7d98f118 100644 --- a/src/mgard-x/Executables/mgard-x-autotuner.cpp +++ b/src/mgard-x/Executables/mgard-x-autotuner.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include @@ -207,8 +207,7 @@ mgard_x::device_type get_arg_dev_type(int argc, char *argv[]) { dev_type = mgard_x::device_type::SYCL; std::cout << mgard_x::log::log_info << "device type: SYCL\n"; } else { - std::cout << "wrong device type.\n"; - exit(-1); + throw std::runtime_error("wrong device type."); } return dev_type; } diff --git a/src/mgard-x/Executables/mgard-x.cpp b/src/mgard-x/Executables/mgard-x.cpp index 009488cf46..891ce94dff 100644 --- a/src/mgard-x/Executables/mgard-x.cpp +++ b/src/mgard-x/Executables/mgard-x.cpp @@ -1,13 +1,16 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ +#include #include #include #include +#include +#include #include #include #include @@ -38,15 +41,31 @@ void print_usage_message(std::string error) { \t\t\t [int]: fastest dimention\n\ \t\t -em / --error-bound-mode : error bound mode (abs: abolute; rel: relative)\n\ \t\t -e / --error-bound : error bound\n\ +\t\t -r / --roi-tolerance-map : path to ROI tolerance map file\n\ +\t\t -roi / --enable-roi: enable ROI mode (use per-block tolerances from -r)\n\ \t\t -s / --smoothness : smoothness parameter\n\ -\t\t -l / --lossless : lossless compression\n\ +\t\t -l / --lossless : lossless compression\n\ \t\t -d / --device : device type\n\ +\t\t (optional) -hh / --hybrid: use hybrid (block-local + global) hierarchy\n\ +\t\t (optional) -ll / --local-levels : number of local refactoring levels (default: 1)\n\ +\t\t (optional) -gl / --global-levels : number of global refactoring levels (default: 0)\n\ +\t\t (optional) -pm / --projection-mode :\n\ +\t\t\t transform basis, plain or hybrid (default: auto -- hierarchical\n\ +\t\t\t under an L-inf bound, orthogonal otherwise; hierarchical is L-inf only)\n\ +\t\t (optional) -nkf / --no-kernel-fusion: run the hybrid local stage as\n\ +\t\t\t separate decompose and quantize passes instead of fused kernels\n\ +\t\t\t (same reconstruction either way, but slower -- use it to time the\n\ +\t\t\t two stages apart). Fusion is on by default.\n\ \t\t (optional) -v / --verbose <0|1|2|3> 0: error; 1: error+info; 2: error+timing; 3: all\n\ +\t\t (optional) -w / --warm-up: run a throwaway compress+decompress pass on a small\n\ +\t\t\t array first to pay HIP's one-time per-kernel load cost before timing\n\ \n\ \t -x / --decompress: decompress mode\n\ \t\t -i / --input \n\ \t\t -o / --output \n\ \t\t -d / --device : device type\n\ +\t\t (optional) -nkf / --no-kernel-fusion: as above, for the dequantize+\n\ +\t\t\t recompose stage\n\ \t\t (optional) -v / --verbose <0|1|2|3> 0: error; 1: error+info; 2: error+timing; 3: all\n"); exit(0); } @@ -91,6 +110,377 @@ void writefile(const char *output_file, size_t num_bytes, T *out_buff) { fclose(file); } +// ============================ +// ROI Block-wise Verification +// ============================ + +std::vector +LinearToCoord(mgard_x::SIZE linear_idx, + const std::vector &dims) { + std::vector coord(dims.size()); + for (int d = dims.size() - 1; d >= 0; --d) { + coord[d] = linear_idx % dims[d]; + linear_idx /= dims[d]; + } + return coord; +} + +mgard_x::SIZE CoordToLinear(const std::vector &coord, + const std::vector &dims) { + mgard_x::SIZE linear = 0; + mgard_x::SIZE stride = 1; + for (int d = dims.size() - 1; d >= 0; --d) { + linear += coord[d] * stride; + stride *= dims[d]; + } + return linear; +} + +struct BlockViolation { + std::vector block_coord; // Block coordinate + double tolerance; // Expected tolerance + double actual_error; // Actual L_inf error in this block + double violation_ratio; // actual_error / tolerance +}; + +// Block-wise ROI verification +template +std::vector +verify_roi_blocks(const std::vector &shape, T *original_data, + T *decompressed_data, const std::vector &tol_map, + enum mgard_x::error_bound_type mode, T global_norm) { + + const mgard_x::SIZE BLOCK_SIZE = 8; + size_t D = shape.size(); + + // Compute number of blocks in each dimension + std::vector num_blocks(D); + mgard_x::SIZE total_blocks = 1; + for (size_t d = 0; d < D; d++) { + num_blocks[d] = (shape[d] + BLOCK_SIZE - 1) / BLOCK_SIZE; + total_blocks *= num_blocks[d]; + } + + std::vector violations; + + // Iterate over all blocks + for (mgard_x::SIZE block_idx = 0; block_idx < total_blocks; block_idx++) { + // Get block coordinate + std::vector block_coord = + LinearToCoord(block_idx, num_blocks); + + // Get tolerance for this block + double block_tol = tol_map[block_idx]; + + // Compute block boundaries in data space + std::vector block_start(D), block_end(D); + for (size_t d = 0; d < D; d++) { + block_start[d] = block_coord[d] * BLOCK_SIZE; + block_end[d] = std::min(block_start[d] + BLOCK_SIZE, shape[d]); + } + + // Compute L_inf error within this block + T block_max_error = 0; + + // Iterate over all elements in this block using nested approach + std::vector elem_coord(D); + std::function iterate_block = [&](size_t dim) { + if (dim == D) { + // Compute linear index in data + mgard_x::SIZE data_idx = CoordToLinear(elem_coord, shape); + + // Compute error for this element + T error = + std::abs(original_data[data_idx] - decompressed_data[data_idx]); + if (mode == mgard_x::error_bound_type::REL) { + error = error / global_norm; + } + block_max_error = std::max(block_max_error, error); + return; + } + + for (mgard_x::SIZE i = block_start[dim]; i < block_end[dim]; i++) { + elem_coord[dim] = i; + iterate_block(dim + 1); + } + }; + + iterate_block(0); + + // Check if this block violates its tolerance + if (block_max_error > block_tol) { + BlockViolation v; + v.block_coord = block_coord; + v.tolerance = block_tol; + v.actual_error = block_max_error; + v.violation_ratio = block_max_error / block_tol; + violations.push_back(v); + } + } + + return violations; +} + +// Print ROI block-wise statistics (NEW) +template +void print_statistics_roi(double s, enum mgard_x::error_bound_type mode, + std::vector shape, T *original_data, + T *decompressed_data, + const std::vector &tol_map, + bool normalize_coordinates) { + const mgard_x::SIZE BLOCK_SIZE = 8; + size_t D = shape.size(); + + // Compute number of blocks + std::vector num_blocks(D); + mgard_x::SIZE total_blocks = 1; + for (size_t d = 0; d < D; d++) { + num_blocks[d] = (shape[d] + BLOCK_SIZE - 1) / BLOCK_SIZE; + total_blocks *= num_blocks[d]; + } + + // Verify tol_map size + if (tol_map.size() != static_cast(total_blocks)) { + std::cout << mgard_x::log::log_err + << "ROI tolerance map size mismatch: expected " << total_blocks + << ", got " << tol_map.size() << "\n"; + return; + } + + std::cout << mgard_x::log::log_info + << "=== ROI Block-wise Verification ===\n"; + std::cout << mgard_x::log::log_info << "Block size: " << BLOCK_SIZE; + for (size_t d = 1; d < D; d++) + std::cout << "x" << BLOCK_SIZE; + std::cout << "\n"; + + std::cout << mgard_x::log::log_info << "Number of blocks: "; + for (size_t d = 0; d < D; d++) { + std::cout << num_blocks[d]; + if (d < D - 1) + std::cout << " x "; + } + std::cout << " = " << total_blocks << " total\n"; + + // Compute global norm for relative error mode + T global_norm = 1; + if (mode == mgard_x::error_bound_type::REL) { + mgard_x::SIZE n = 1; + for (size_t d = 0; d < D; d++) + n *= shape[d]; + global_norm = 0; + for (mgard_x::SIZE i = 0; i < n; i++) { + global_norm = std::max(global_norm, std::abs(original_data[i])); + } + std::cout << mgard_x::log::log_info + << "Global L_inf norm: " << std::scientific << global_norm + << std::defaultfloat << "\n"; + } + + // Perform block-wise verification + std::vector violations = verify_roi_blocks( + shape, original_data, decompressed_data, tol_map, mode, global_norm); + + mgard_x::SIZE num_violations = violations.size(); + mgard_x::SIZE num_satisfied = total_blocks - num_violations; + double satisfaction_rate = 100.0 * num_satisfied / total_blocks; + + std::cout << mgard_x::log::log_info << "Blocks satisfied: " << num_satisfied + << "/" << total_blocks << " (" << std::fixed << std::setprecision(2) + << satisfaction_rate << "%)\n"; + std::cout << mgard_x::log::log_info << "Blocks violated: " << num_violations + << "/" << total_blocks << " (" << std::fixed << std::setprecision(2) + << (100.0 - satisfaction_rate) << "%)\n"; + std::cout << std::defaultfloat; + + if (num_violations == 0) { + std::cout << mgard_x::log::log_info + << "\e[32mAll blocks satisfied their tolerances!\e[0m\n"; + } else { + std::cout << mgard_x::log::log_info << "\e[31mViolated blocks:\e[0m\n"; + + // Sort violations by violation ratio (worst first) + std::sort(violations.begin(), violations.end(), + [](const BlockViolation &a, const BlockViolation &b) { + return a.violation_ratio > b.violation_ratio; + }); + + // Print header + std::cout << mgard_x::log::log_info << std::setw(20) << "Block Coord" + << std::setw(15) << "Tolerance" << std::setw(15) << "Actual Error" + << std::setw(10) << "Ratio" + << "\n"; + std::cout << mgard_x::log::log_info << std::string(60, '-') << "\n"; + + // Print all violations + std::cout << std::scientific << std::setprecision(4); + for (const auto &v : violations) { + // Format block coordinate + std::ostringstream coord_str; + coord_str << "("; + for (size_t d = 0; d < v.block_coord.size(); d++) { + coord_str << v.block_coord[d]; + if (d < v.block_coord.size() - 1) + coord_str << ","; + } + coord_str << ")"; + + std::cout << mgard_x::log::log_info << std::setw(20) << coord_str.str() + << std::setw(15) << v.tolerance << std::setw(15) + << v.actual_error << std::setw(10) << std::fixed + << std::setprecision(2) << v.violation_ratio << "x\n"; + std::cout << std::scientific << std::setprecision(4); + } + std::cout << std::defaultfloat; + + // Print worst violation summary + const auto &worst = violations[0]; + std::ostringstream worst_coord; + worst_coord << "("; + for (size_t d = 0; d < worst.block_coord.size(); d++) { + worst_coord << worst.block_coord[d]; + if (d < worst.block_coord.size() - 1) + worst_coord << ","; + } + worst_coord << ")"; + std::cout << mgard_x::log::log_info << "Worst violation: block " + << worst_coord.str() << " with " << std::scientific + << worst.actual_error << " > " << worst.tolerance << " (" + << std::fixed << std::setprecision(2) << worst.violation_ratio + << "x)\n"; + std::cout << std::defaultfloat; + } + + // Also print global statistics + std::cout << mgard_x::log::log_info << "=== Global Statistics ===\n"; + mgard_x::SIZE n = 1; + for (size_t d = 0; d < shape.size(); d++) + n *= shape[d]; + + std::cout << mgard_x::log::log_info + << "MSE: " << mgard_x::MSE(n, original_data, decompressed_data) + << "\n"; + std::cout << mgard_x::log::log_info + << "PSNR: " << mgard_x::PSNR(n, original_data, decompressed_data) + << "\n"; +} + +// ROI vs background error breakdown for standalone decompression +template +void print_decompress_roi_statistics(std::vector shape, + T *original_data, T *decompressed_data, + const std::vector &tol_map, + enum mgard_x::error_bound_type mode) { + + const mgard_x::SIZE BLOCK_SIZE = 8; + size_t D = shape.size(); + + std::vector num_blocks(D); + mgard_x::SIZE total_blocks = 1; + for (size_t d = 0; d < D; d++) { + num_blocks[d] = (shape[d] + BLOCK_SIZE - 1) / BLOCK_SIZE; + total_blocks *= num_blocks[d]; + } + + // Compute global norm for REL mode + T global_norm = 1; + if (mode == mgard_x::error_bound_type::REL) { + mgard_x::SIZE n = 1; + for (size_t d = 0; d < D; d++) + n *= shape[d]; + for (mgard_x::SIZE i = 0; i < n; i++) + global_norm = std::max(global_norm, std::abs(original_data[i])); + std::cout << mgard_x::log::log_info + << "Global L_inf norm: " << std::scientific << global_norm + << std::defaultfloat << "\n"; + } + + // Identify ROI tolerance (minimum) vs background (maximum) + double min_tol = *std::min_element(tol_map.begin(), tol_map.end()); + double max_tol = *std::max_element(tol_map.begin(), tol_map.end()); + double split = (min_tol + max_tol) * 0.5; + + struct GroupStats { + mgard_x::SIZE total = 0, satisfied = 0; + double sum_error = 0, max_error = 0, max_ratio = 0; + }; + GroupStats roi_stats, bg_stats; + + for (mgard_x::SIZE block_idx = 0; block_idx < total_blocks; block_idx++) { + std::vector block_coord = + LinearToCoord(block_idx, num_blocks); + double block_tol = tol_map[block_idx]; + bool is_roi = (block_tol <= split); + + std::vector block_start(D), block_end(D); + for (size_t d = 0; d < D; d++) { + block_start[d] = block_coord[d] * BLOCK_SIZE; + block_end[d] = std::min(block_start[d] + BLOCK_SIZE, shape[d]); + } + + T block_max_error = 0; + std::vector elem_coord(D); + std::function iterate = [&](size_t dim) { + if (dim == D) { + mgard_x::SIZE idx = CoordToLinear(elem_coord, shape); + T err = std::abs(original_data[idx] - decompressed_data[idx]); + if (mode == mgard_x::error_bound_type::REL) + err /= global_norm; + block_max_error = std::max(block_max_error, err); + return; + } + for (mgard_x::SIZE i = block_start[dim]; i < block_end[dim]; i++) { + elem_coord[dim] = i; + iterate(dim + 1); + } + }; + iterate(0); + + GroupStats &g = is_roi ? roi_stats : bg_stats; + g.total++; + g.sum_error += block_max_error; + g.max_error = std::max(g.max_error, (double)block_max_error); + if (block_max_error <= block_tol) { + g.satisfied++; + } else { + g.max_ratio = std::max(g.max_ratio, (double)block_max_error / block_tol); + } + } + + auto print_group = [&](const char *label, const GroupStats &g, double tol) { + if (g.total == 0) + return; + double avg_err = g.sum_error / g.total; + double sat_pct = 100.0 * g.satisfied / g.total; + std::cout << mgard_x::log::log_info << "--- " << label + << " (tolerance=" << std::scientific << tol + << ", blocks=" << g.total << ") ---\n" + << std::defaultfloat; + std::cout << mgard_x::log::log_info << " Satisfied: " << g.satisfied << "/" + << g.total << " (" << std::fixed << std::setprecision(2) + << sat_pct << "%)\n"; + std::cout << mgard_x::log::log_info + << " Avg block L_inf error: " << std::scientific << avg_err + << "\n"; + std::cout << mgard_x::log::log_info + << " Max block L_inf error: " << std::scientific << g.max_error + << "\n"; + if (g.satisfied < g.total) + std::cout << mgard_x::log::log_info + << " Worst violation ratio: " << std::fixed + << std::setprecision(2) << g.max_ratio << "x\n"; + std::cout << std::defaultfloat; + }; + + std::cout << mgard_x::log::log_info + << "=== Decompression ROI Error Verification ===\n"; + print_group("ROI blocks", roi_stats, min_tol); + print_group("Background blocks", bg_stats, max_tol); + std::cout << mgard_x::log::log_info << "Total blocks: " << total_blocks + << " (ROI=" << roi_stats.total << ", BG=" << bg_stats.total + << ")\n"; +} + template void print_statistics(double s, enum mgard_x::error_bound_type mode, std::vector shape, T *original_data, @@ -145,9 +535,6 @@ void print_statistics(double s, enum mgard_x::error_bound_type mode, std::cout << mgard_x::log::log_info << "PSNR: " << mgard_x::PSNR(n, original_data, decompressed_data) << "\n"; - - // if (actual_error > tol) - // exit(-1); } int verbose_to_log_level(int verbose) { @@ -168,17 +555,41 @@ int verbose_to_log_level(int verbose) { template int launch_compress(mgard_x::DIM D, enum mgard_x::data_type dtype, const char *input_file, const char *output_file, - std::vector shape, double tol, double s, + std::vector shape, double tol, + std::vector tol_map, bool enable_roi, double s, enum mgard_x::error_bound_type mode, std::string lossless, std::string domain_decomposition, mgard_x::SIZE block_size, enum mgard_x::device_type dev_type, int verbose, - mgard_x::SIZE max_memory_footprint) { - + mgard_x::SIZE max_memory_footprint, int num_local_levels, + int num_global_levels, bool use_hybrid, bool warm_up, + bool kernel_fusion, + mgard_x::compression_projection_mode_type projection_mode) { mgard_x::Config config; config.log_level = verbose_to_log_level(verbose); - config.decomposition = mgard_x::decomposition_type::MultiDim; - // config.decomposition = mgard_x::decomposition_type::Hybrid; - // config.num_local_refactoring_level = 1; + config.fuse_decompose_quantize = kernel_fusion; + config.fuse_dequantize_recompose = kernel_fusion; + // Hybrid (block-local + global) hierarchy decomposition is opt-in via + // -hh/--hybrid; the default remains the standard multi-dim decomposition. + if (use_hybrid) { + config.decomposition = mgard_x::decomposition_type::Hybrid; + } else { + config.decomposition = mgard_x::decomposition_type::MultiDim; + } + config.num_local_refactoring_level = num_local_levels; + config.num_global_refactoring_level = num_global_levels; + config.projection_mode = projection_mode; + + // Switch for ROI + config.enable_roi = enable_roi; + if (enable_roi) { + config.roi_tolerance_map = tol_map; + } + + if (!enable_roi && tol <= 0) { + std::cout << mgard_x::log::log_err + << "Error tolerance (-e) is required when not using ROI mode\n"; + exit(-1); + } // config.max_larget_level = 1; @@ -215,13 +626,16 @@ int launch_compress(mgard_x::DIM D, enum mgard_x::data_type dtype, // config.domain_decomposition_sizes = {180, 368, 463, 529, 605, 692, 43}; // config.domain_decomposition_sizes = std::vector(192, 15); - config.estimate_outlier_ratio = 0.3; + config.estimate_outlier_ratio = 1.0; config.dev_type = dev_type; - config.reorder = 0; config.auto_pin_host_buffers = true; config.max_memory_footprint = max_memory_footprint; + // config.huff_dict_size = 32768; + // config.huff_dict_size = 16384; config.huff_dict_size = 8192; + // config.huff_dict_size = 4096; + // config.huff_dict_size = 2048; config.adjust_shape = false; config.auto_cache_release = false; @@ -229,8 +643,23 @@ int launch_compress(mgard_x::DIM D, enum mgard_x::data_type dtype, config.lossless = mgard_x::lossless_type::Huffman; } else if (lossless == "huffman-lz4") { config.lossless = mgard_x::lossless_type::Huffman_LZ4; + } else if (lossless == "lz4") { + config.lossless = mgard_x::lossless_type::LZ4; } else if (lossless == "huffman-zstd") { config.lossless = mgard_x::lossless_type::Huffman_Zstd; + } else if (lossless == "blockdelta" || lossless == "blockdelta-delta") { + config.lossless = mgard_x::lossless_type::BlockDelta; + config.block_delta_mode = mgard_x::block_delta_mode_type::Delta; + } else if (lossless == "blockdelta-fixed") { + config.lossless = mgard_x::lossless_type::BlockDelta; + config.block_delta_mode = mgard_x::block_delta_mode_type::Fixed; + } else if (lossless == "blockdelta-outlier") { + config.lossless = mgard_x::lossless_type::BlockDelta; + config.block_delta_mode = mgard_x::block_delta_mode_type::Outlier; + } else if (lossless == "zerorle-rans") { + config.lossless = mgard_x::lossless_type::ZeroRLE_Rans; + } else if (lossless == "symbol-rans") { + config.lossless = mgard_x::lossless_type::SymbolRans; } size_t original_size = 1; @@ -264,6 +693,42 @@ int launch_compress(mgard_x::DIM D, enum mgard_x::data_type dtype, << in_size << " vs. " << original_size * sizeof(T) << "!\n"; } + // HIP pays a one-time cold-start cost (~10-50ms) the first time each + // distinct kernel template is launched in a process (lazy code-object + // loading), which otherwise gets fully attributed to whichever pipeline + // stage happens to launch that kernel first. Stages launched many times + // per compress() call (decomposition) amortize it away; single-shot + // stages (quantization, most Huffman kernels) pay it in full. Run a + // throwaway pass on a small array of the same dtype/config first so the + // real, timed run below only measures steady-state performance. + if (warm_up && !enable_roi) { + std::vector warmup_shape(D); + for (mgard_x::DIM i = 0; i < D; i++) { + warmup_shape[i] = std::min(shape[i], (mgard_x::SIZE)33); + } + size_t warmup_size = 1; + for (mgard_x::DIM i = 0; i < D; i++) + warmup_size *= warmup_shape[i]; + T *warmup_data = (T *)malloc(warmup_size * sizeof(T)); + for (size_t i = 0; i < warmup_size; i++) + warmup_data[i] = (T)(i % 10 + 1); + size_t warmup_compressed_size = warmup_size * sizeof(T) * 2; + void *warmup_compressed_data = (void *)malloc(warmup_compressed_size); + void *warmup_decompressed_data = malloc(warmup_size * sizeof(T)); + + mgard_x::Config warmup_config = config; + warmup_config.log_level = mgard_x::log::ERR; + mgard_x::compress(D, dtype, warmup_shape, tol, s, mode, warmup_data, + warmup_compressed_data, warmup_compressed_size, + warmup_config, false); + mgard_x::decompress(warmup_compressed_data, warmup_compressed_size, + warmup_decompressed_data, warmup_config, false); + + free(warmup_data); + free(warmup_compressed_data); + free(warmup_decompressed_data); + } + size_t compressed_size = original_size * sizeof(T) * 2; void *compressed_data = (void *)malloc(compressed_size); mgard_x::pin_memory(original_data, original_size * sizeof(T), config); @@ -272,8 +737,7 @@ int launch_compress(mgard_x::DIM D, enum mgard_x::data_type dtype, ret = mgard_x::compress(D, dtype, shape, tol, s, mode, original_data, compressed_data, compressed_size, config, true); if (ret != mgard_x::compress_status_type::Success) { - std::cout << mgard_x::log::log_err << "Compression failed\n"; - exit(-1); + throw std::runtime_error("Compression failed"); } writefile(output_file, compressed_size, compressed_data); std::cout << mgard_x::log::log_info << "Compression ratio: " @@ -284,8 +748,14 @@ int launch_compress(mgard_x::DIM D, enum mgard_x::data_type dtype, mgard_x::decompress(compressed_data, compressed_size, decompressed_data, config, true); - print_statistics(s, mode, shape, original_data, (T *)decompressed_data, - tol, config.normalize_coordinates); + if (config.enable_roi) { + print_statistics_roi(s, mode, shape, original_data, + (T *)decompressed_data, tol_map, + config.normalize_coordinates); + } else { + print_statistics(s, mode, shape, original_data, (T *)decompressed_data, + tol, config.normalize_coordinates); + } mgard_x::unpin_memory(decompressed_data, config); free(decompressed_data); @@ -297,13 +767,51 @@ int launch_compress(mgard_x::DIM D, enum mgard_x::data_type dtype, return 0; } -int launch_decompress(const char *input_file, const char *output_file, - enum mgard_x::device_type dev_type, int verbose) { +// Decompression parameters the user may override on the command line. The +// hybrid (BlockMGARD) parameters now travel in the file header, so every field +// here is unset by default and the metadata drives decompression; a field is +// only applied when the corresponding flag was actually passed. Overriding is +// kept for debugging a file whose header disagrees with the data. +struct DecompressOverrides { + bool has_local_levels = false; + int num_local_levels = 0; + bool has_global_levels = false; + int num_global_levels = 0; + bool has_roi = false; + bool enable_roi = false; + // Also used, independently of any override, as the reference map for the + // optional -orig block-error report. + std::vector tol_map; +}; + +int launch_decompress( + const char *input_file, const char *output_file, + enum mgard_x::device_type dev_type, int verbose, bool kernel_fusion, + const DecompressOverrides &overrides, const char *original_file = nullptr, + enum mgard_x::error_bound_type ebtype = mgard_x::error_bound_type::ABS) { mgard_x::Config config; config.log_level = verbose_to_log_level(verbose); + config.fuse_decompose_quantize = kernel_fusion; + config.fuse_dequantize_recompose = kernel_fusion; config.dev_type = dev_type; config.auto_pin_host_buffers = true; config.auto_cache_release = true; + // Leave the hybrid fields at their defaults unless explicitly overridden: + // decompress() restores them from the header, and overwriting them here with + // guesses is exactly the bug this replaces. + if (overrides.has_local_levels) { + config.num_local_refactoring_level = overrides.num_local_levels; + } + if (overrides.has_global_levels) { + config.num_global_refactoring_level = overrides.num_global_levels; + } + if (overrides.has_roi) { + config.enable_roi = overrides.enable_roi; + if (overrides.enable_roi) { + config.roi_tolerance_map = overrides.tol_map; + } + } + const std::vector &tol_map = overrides.tol_map; mgard_x::SERIALIZED_TYPE *compressed_data; size_t compressed_size = readfile(input_file, compressed_data); @@ -328,6 +836,29 @@ int launch_decompress(const char *input_file, const char *output_file, writefile(output_file, original_size * elem_size, decompressed_data); + // Block-wise error verification. Purely diagnostic and independent of how + // the file was decompressed, so it runs whenever the user supplied both the + // original data and a reference tolerance map (-orig and -r). + if (original_file != nullptr && !tol_map.empty()) { + void *orig_raw; + size_t orig_bytes = readfile(original_file, orig_raw); + if (orig_bytes == original_size * elem_size) { + if (dtype == mgard_x::data_type::Float) { + print_decompress_roi_statistics(shape, (float *)orig_raw, + (float *)decompressed_data, + tol_map, ebtype); + } else if (dtype == mgard_x::data_type::Double) { + print_decompress_roi_statistics(shape, (double *)orig_raw, + (double *)decompressed_data, + tol_map, ebtype); + } + free(orig_raw); + } else { + std::cout << mgard_x::log::log_warn + << "Original file size mismatch, skipping verification\n"; + } + } + delete[] compressed_data; return 0; } @@ -345,8 +876,45 @@ bool try_compression(int argc, char *argv[]) { get_args(argc, argv, "Dimensions", "-dim", "--dimension"); enum mgard_x::error_bound_type mode = get_error_bound_mode(argc, argv); // REL or ABS - double tol = - get_arg(argc, argv, "Error bound", "-e", "--error-bound"); + double tol = -1.0; + if (has_arg(argc, argv, "-e", "--error-bound")) { + tol = get_arg(argc, argv, "Error bound", "-e", "--error-bound"); + } + bool enable_roi = has_arg(argc, argv, "-roi", "--enable-roi"); + std::vector tol_map; + if (has_arg(argc, argv, "-r", "--roi-tolerance-map")) { + std::string roi_file = get_arg(argc, argv, "ROI tolerance map", + "-r", "--roi-tolerance-map"); + + double *roi_map_buffer; + size_t roi_map_bytes = readfile(roi_file.c_str(), roi_map_buffer); + size_t roi_map_size = roi_map_bytes / sizeof(double); + tol_map.resize(roi_map_size); + for (size_t i = 0; i < roi_map_size; i++) { + tol_map[i] = static_cast(roi_map_buffer[i]); + } + free(roi_map_buffer); + + size_t expected_roi_map_size = 1; + for (mgard_x::DIM i = 0; i < shape.size(); i++) { + expected_roi_map_size *= (shape[i] + 8 - 1) / 8; + } + if (tol_map.size() != expected_roi_map_size) { + std::cout << mgard_x::log::log_warn << "ROI map size mismatch: expected " + << expected_roi_map_size << ", got " << tol_map.size() << "\n"; + } + } + + if (enable_roi && tol_map.empty()) { + std::cout << mgard_x::log::log_err + << "--enable-roi requires -r/--roi-tolerance-map\n"; + exit(-1); + } + if (!enable_roi && tol <= 0) { + std::cout << mgard_x::log::log_err + << "-e/--tolerance is required when not using ROI mode\n"; + exit(-1); + } double s = get_arg(argc, argv, "Smoothness", "-s", "--smoothness"); std::string lossless = get_arg(argc, argv, "Lossless", "-l", "--lossless"); @@ -355,12 +923,50 @@ bool try_compression(int argc, char *argv[]) { if (has_arg(argc, argv, "-v", "--verbose")) { verbose = get_arg(argc, argv, "Verbose", "-v", "--verbose"); } + bool warm_up = has_arg(argc, argv, "-w", "--warm-up"); mgard_x::SIZE max_memory_footprint = std::numeric_limits::max(); if (has_arg(argc, argv, "-m", "--max-memory")) { max_memory_footprint = (mgard_x::SIZE)get_arg( argc, argv, "Max memory", "-m", "--max-memory"); } + + int num_local_levels = 1; // default value + if (has_arg(argc, argv, "-ll", "--local-levels")) { + num_local_levels = + get_arg(argc, argv, "Local levels", "-ll", "--local-levels"); + } + + bool use_hybrid = has_arg(argc, argv, "-hh", "--hybrid"); + + mgard_x::compression_projection_mode_type projection_mode = + mgard_x::compression_projection_mode_type::Auto; + if (has_arg(argc, argv, "-pm", "--projection-mode")) { + std::string value = get_arg(argc, argv, "Projection mode", + "-pm", "--projection-mode"); + if (value == "auto") { + projection_mode = mgard_x::compression_projection_mode_type::Auto; + } else if (value == "orthogonal") { + projection_mode = mgard_x::compression_projection_mode_type::Orthogonal; + } else if (value == "hierarchical") { + projection_mode = mgard_x::compression_projection_mode_type::Hierarchical; + } else { + std::cout << mgard_x::log::log_err + << "--projection-mode must be auto, orthogonal, or " + "hierarchical\n"; + exit(-1); + } + } + + // Fusion is on by default; the flag selects the separate-pass path. + bool kernel_fusion = !has_arg(argc, argv, "-nkf", "--no-kernel-fusion"); + + int num_global_levels = 0; // default value + if (has_arg(argc, argv, "-gl", "--global-levels")) { + num_global_levels = + get_arg(argc, argv, "Global levels", "-gl", "--global-levels"); + } + std::string domain_decomposition = "max-dim"; mgard_x::SIZE block_size = 0; if (has_arg(argc, argv, "-dd", "--domain-decomposition")) { @@ -373,15 +979,17 @@ bool try_compression(int argc, char *argv[]) { } if (dtype == mgard_x::data_type::Double) { - launch_compress(shape.size(), dtype, input_file.c_str(), - output_file.c_str(), shape, tol, s, mode, lossless, - domain_decomposition, block_size, dev_type, verbose, - max_memory_footprint); + launch_compress( + shape.size(), dtype, input_file.c_str(), output_file.c_str(), shape, + tol, tol_map, enable_roi, s, mode, lossless, domain_decomposition, + block_size, dev_type, verbose, max_memory_footprint, num_local_levels, + num_global_levels, use_hybrid, warm_up, kernel_fusion, projection_mode); } else if (dtype == mgard_x::data_type::Float) { - launch_compress(shape.size(), dtype, input_file.c_str(), - output_file.c_str(), shape, tol, s, mode, lossless, - domain_decomposition, block_size, dev_type, verbose, - max_memory_footprint); + launch_compress( + shape.size(), dtype, input_file.c_str(), output_file.c_str(), shape, + tol, tol_map, enable_roi, s, mode, lossless, domain_decomposition, + block_size, dev_type, verbose, max_memory_footprint, num_local_levels, + num_global_levels, use_hybrid, warm_up, kernel_fusion, projection_mode); } mgard_x::release_cache(mgard_x::Config()); return true; @@ -400,15 +1008,58 @@ bool try_decompression(int argc, char *argv[]) { if (has_arg(argc, argv, "-v", "--verbose")) { verbose = get_arg(argc, argv, "Verbose", "-v", "--verbose"); } - launch_decompress(input_file.c_str(), output_file.c_str(), dev_type, verbose); + // All of these are optional overrides: the hybrid parameters are restored + // from the file header, so a plain "mgard-x -x -i f.mgard -o f.raw" now + // decompresses a BlockMGARD file correctly with no extra flags. + DecompressOverrides overrides; + if (has_arg(argc, argv, "-roi", "--enable-roi")) { + overrides.has_roi = true; + overrides.enable_roi = true; + } + if (has_arg(argc, argv, "-r", "--roi-tolerance-map")) { + std::string roi_file = get_arg(argc, argv, "ROI tolerance map", + "-r", "--roi-tolerance-map"); + double *roi_map_buffer; + size_t roi_map_bytes = readfile(roi_file.c_str(), roi_map_buffer); + size_t roi_map_size = roi_map_bytes / sizeof(double); + overrides.tol_map.resize(roi_map_size); + for (size_t i = 0; i < roi_map_size; i++) { + overrides.tol_map[i] = static_cast(roi_map_buffer[i]); + } + free(roi_map_buffer); + } + if (has_arg(argc, argv, "-ll", "--local-levels")) { + overrides.has_local_levels = true; + overrides.num_local_levels = + get_arg(argc, argv, "Local levels", "-ll", "--local-levels"); + } + if (has_arg(argc, argv, "-gl", "--global-levels")) { + overrides.has_global_levels = true; + overrides.num_global_levels = + get_arg(argc, argv, "Global levels", "-gl", "--global-levels"); + } + // Optional: original data file for error verification + std::string original_file; + if (has_arg(argc, argv, "-orig", "--original-data")) { + original_file = get_arg(argc, argv, "Original data", "-orig", + "--original-data"); + } + enum mgard_x::error_bound_type ebtype = mgard_x::error_bound_type::REL; + if (has_arg(argc, argv, "-em", "--error-bound-mode")) { + ebtype = get_error_bound_mode(argc, argv); + } + bool kernel_fusion = !has_arg(argc, argv, "-nkf", "--no-kernel-fusion"); + launch_decompress(input_file.c_str(), output_file.c_str(), dev_type, verbose, + kernel_fusion, overrides, + original_file.empty() ? nullptr : original_file.c_str(), + ebtype); mgard_x::release_cache(mgard_x::Config()); return true; } int main(int argc, char *argv[]) { - if (!try_compression(argc, argv) && !try_decompression(argc, argv)) { print_usage_message(""); } return 0; -} \ No newline at end of file +} diff --git a/src/mgard-x/Executables/pmdr-x-qoi-io.cpp b/src/mgard-x/Executables/pmdr-x-qoi-io.cpp new file mode 100644 index 0000000000..60816c24af --- /dev/null +++ b/src/mgard-x/Executables/pmdr-x-qoi-io.cpp @@ -0,0 +1,1286 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "compress_x.hpp" +#include "mdr_x.hpp" +#include "mgard-x/RuntimeX/Utilities/Log.h" +#include "mgard-x/Utilities/ErrorCalculator.h" + +#include "ArgumentParser.h" + +#include +using namespace std::chrono; + +void print_usage_message(std::string error) { + if (error.compare("") != 0) { + std::cout << mgard_x::log::log_err << error << std::endl; + } + printf("Options\n\ + \t -z / --refactor: refactor data\n\ + \t\t -i / --input \n\ + \t\t -o / --output \n\ + \t\t -dt / --data-type : data type (s: single; d:double)\n\ + \t\t -dim / --dimension : total number of dimensions\n\ + \t\t\t [dim1]: slowest dimention\n\ + \t\t\t [dim2]: 2nd slowest dimention\n\ + \t\t\t ...\n\ + \t\t\t [dimN]: fastest dimention\n\ + \t\t -d / --device : device type\n\ + \t\t (optional) -v / --verbose <0|1|2|3> 0: error; 1: error+info; 2: error+timing; 3: all\n\ + \t\t (optional) -m / --max-memory \n\ + \t\t (optional) -dd / --domain-decomposition \n\ + \t\t\t (optional) -dd-size / --domain-decomposition-size (for block domain decomposition only) \n\ + \n\ + \t -x / --reconstruct: reconstruct data\n\ + \t\t -i / --input \n\ + \t\t -o / --output \n\ + \t\t (optional) -g / --orginal (optinal)\n\ + \t\t -e / --error-bound : error bound\n\ + \t\t -me / --multi-error-bounds ..: multiple error bounds\n\ + \t\t -s / --smoothness : smoothness parameter\n\ + \t\t -d : device type\n\ + \t\t (optional) -v / --verbose <0|1|2|3> 0: error; 1: error+info; 2: error+timing; 3: all\n"); + exit(0); +} + +template void min_max(size_t n, T *in_buff) { + T min = std::numeric_limits::infinity(); + T max = 0; + for (size_t i = 0; i < n; i++) { + if (min > in_buff[i]) { + min = in_buff[i]; + } + if (max < in_buff[i]) { + max = in_buff[i]; + } + } + printf("Min: %f, Max: %f\n", min, max); +} + +template size_t readfile(std::string input_file, T *&in_buff) { + // std::cout << mgard_x::log::log_info << "Loading file: " << input_file << + // "\n"; + + FILE *pFile; + pFile = fopen(input_file.c_str(), "rb"); + if (pFile == NULL) { + std::cout << mgard_x::log::log_err << "file open error!\n"; + exit(1); + } + fseek(pFile, 0, SEEK_END); + size_t lSize = ftell(pFile); + rewind(pFile); + in_buff = (T *)malloc(lSize); + lSize = fread(in_buff, 1, lSize, pFile); + fclose(pFile); + // min_max(lSize/sizeof(T), in_buff); + return lSize; +} + +template +size_t readfile_header_metadata(std::string input_file, + std::vector &in_buff) { + // std::cout << mgard_x::log::log_info << "Loading file: " << input_file << + // "\n"; + + FILE *pFile; + pFile = fopen(input_file.c_str(), "rb"); + if (pFile == NULL) { + std::cout << mgard_x::log::log_err << "file open error!\n"; + exit(1); + } + fseek(pFile, 0, SEEK_END); + size_t lSize = ftell(pFile); + rewind(pFile); + in_buff.resize(lSize / sizeof(T)); + lSize = fread(in_buff.data(), 1, lSize, pFile); + fclose(pFile); + return lSize; +} + +template +void writefile(std::string output_file, T *out_buff, size_t num_bytes) { + FILE *file = fopen(output_file.c_str(), "w"); + fwrite(out_buff, 1, num_bytes, file); + fclose(file); +} + +template +void print_statistics(double s, enum mgard_x::error_bound_type mode, + std::vector shape, T *original_data, + T *decompressed_data, T tol, bool normalize_coordinates) { + mgard_x::SIZE n = 1; + for (mgard_x::DIM d = 0; d < shape.size(); d++) + n *= shape[d]; + T actual_error = 0.0; + std::cout << std::scientific; + if (s == std::numeric_limits::infinity()) { + actual_error = + mgard_x::L_inf_error(n, original_data, decompressed_data, mode); + if (mode == mgard_x::error_bound_type::ABS) { + std::cout << mgard_x::log::log_info + << "Absoluate L_inf error: " << actual_error << " (" + << (actual_error < tol ? "\e[32mSatisified\e[0m" + : "\e[31mNot Satisified\e[0m") + << ")" + << "\n"; + } else if (mode == mgard_x::error_bound_type::REL) { + std::cout << mgard_x::log::log_info + << "Relative L_inf error: " << actual_error << " (" + << (actual_error < tol ? "\e[32mSatisified\e[0m" + : "\e[31mNot Satisified\e[0m") + << ")" + << "\n"; + } + } else { + actual_error = mgard_x::L_2_error(shape, original_data, decompressed_data, + mode, normalize_coordinates); + if (mode == mgard_x::error_bound_type::ABS) { + std::cout << mgard_x::log::log_info + << "Absoluate L_2 error: " << actual_error << " (" + << (actual_error < tol ? "\e[32mSatisified\e[0m" + : "\e[31mNot Satisified\e[0m") + << ")" + << "\n"; + } else if (mode == mgard_x::error_bound_type::REL) { + std::cout << mgard_x::log::log_info + << "Relative L_2 error: " << actual_error << " (" + << (actual_error < tol ? "\e[32mSatisified\e[0m" + : "\e[31mNot Satisified\e[0m") + << ")" + << "\n"; + } + } + + std::cout << mgard_x::log::log_info + << "MSE: " << mgard_x::MSE(n, original_data, decompressed_data) + << "\n"; + std::cout << std::defaultfloat; + std::cout << mgard_x::log::log_info + << "PSNR: " << mgard_x::PSNR(n, original_data, decompressed_data) + << "\n"; + + // if (actual_error > tol) + // exit(-1); +} + +void create_dir(std::string name) { + struct stat st = {0}; + if (stat(name.c_str(), &st) == -1) { + mkdir(name.c_str(), 0700); + } +} + +void write_mdr(mgard_x::MDR::RefactoredMetadata &refactored_metadata, + mgard_x::MDR::RefactoredData &refactored_data, + std::string output) { + size_t size_written = 0; + create_dir(output); + std::vector serialized_metadata = + refactored_metadata.Serialize(); + writefile(output + "/header", refactored_metadata.header.data(), + refactored_metadata.header.size()); + writefile(output + "/metadata", serialized_metadata.data(), + serialized_metadata.size()); + for (int subdomain_id = 0; subdomain_id < refactored_metadata.metadata.size(); + subdomain_id++) { + for (int level_idx = 0; + level_idx < + refactored_metadata.metadata[subdomain_id].level_sizes.size(); + level_idx++) { + for (int bitplane_idx = 0; + bitplane_idx < refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx] + .size(); + bitplane_idx++) { + if (refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx] > 0) { + std::string filename = "component_" + std::to_string(subdomain_id) + + "_" + std::to_string(level_idx) + "_" + + std::to_string(bitplane_idx); + writefile(output + "/" + filename, + refactored_data.data[subdomain_id][level_idx][bitplane_idx], + refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx]); + size_written += refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx]; + } + } + } + } + std::cout << mgard_x::log::log_info << size_written << " bytes written\n"; +} + +size_t read_mdr_metadata(mgard_x::MDR::RefactoredMetadata &refactored_metadata, + mgard_x::MDR::RefactoredData &refactored_data, + std::string input) { + + size_t metadata_size = 0; + metadata_size += + readfile_header_metadata(input + "/header", refactored_metadata.header); + std::vector serialized_metadata; + metadata_size += + readfile_header_metadata(input + "/metadata", serialized_metadata); + refactored_metadata.Deserialize(serialized_metadata); + refactored_metadata.InitializeForReconstruction(); + refactored_data.InitializeForReconstruction(refactored_metadata); + return metadata_size; +} + +size_t read_mdr(mgard_x::MDR::RefactoredMetadata &refactored_metadata, + mgard_x::MDR::RefactoredData &refactored_data, + std::string input, bool initialize_signs, + mgard_x::Config config) { + + size_t size_read = 0; + int num_subdomains = refactored_metadata.metadata.size(); + for (int subdomain_id = 0; subdomain_id < num_subdomains; subdomain_id++) { + mgard_x::MDR::MDRMetadata metadata = + refactored_metadata.metadata[subdomain_id]; + int num_levels = metadata.level_sizes.size(); + for (int level_idx = 0; level_idx < num_levels; level_idx++) { + int num_bitplanes = metadata.level_sizes[level_idx].size(); + int loaded_bitplanes = metadata.loaded_level_num_bitplanes[level_idx]; + int reqested_bitplanes = + metadata.requested_level_num_bitplanes[level_idx]; + for (int bitplane_idx = loaded_bitplanes; + bitplane_idx < reqested_bitplanes; bitplane_idx++) { + if (refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx] > 0) { + std::string filename = "component_" + std::to_string(subdomain_id) + + "_" + std::to_string(level_idx) + "_" + + std::to_string(bitplane_idx); + mgard_x::SIZE level_size = readfile( + input + "/" + filename, + refactored_data.data[subdomain_id][level_idx][bitplane_idx]); + mgard_x::pin_memory( + refactored_data.data[subdomain_id][level_idx][bitplane_idx], + level_size, config); + if (level_size != refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx]) { + std::cout << "mdr component size mismatch."; + exit(-1); + } + size_read += level_size; + } + } + if (initialize_signs) { + // level sign + refactored_data.level_signs[subdomain_id][level_idx] = + (bool *)malloc(sizeof(bool) * metadata.level_num_elems[level_idx]); + memset(refactored_data.level_signs[subdomain_id][level_idx], 0, + sizeof(bool) * metadata.level_num_elems[level_idx]); + mgard_x::pin_memory( + refactored_data.level_signs[subdomain_id][level_idx], + sizeof(bool) * metadata.level_num_elems[level_idx], config); + } + } + } + return size_read; +} + +// size_t read_mdr(mgard_x::MDR::RefactoredMetadata &refactored_metadata, +// mgard_x::MDR::RefactoredData &refactored_data, std::string +// input, bool initialize_signs, mgard_x::Config config) { + +// size_t size_read = 0; +// int num_subdomains = refactored_metadata.metadata.size(); +// for (int subdomain_id = 0; subdomain_id < num_subdomains; subdomain_id++) +// { +// mgard_x::MDR::MDRMetadata metadata = +// refactored_metadata.metadata[subdomain_id]; +// int num_levels = metadata.level_sizes.size(); +// for (int level_idx = 0; level_idx < num_levels; level_idx++) { +// int num_bitplanes = metadata.level_sizes[level_idx].size(); +// for (int bitplane_idx = 0; +// bitplane_idx < num_bitplanes; bitplane_idx++) { +// std::string filename = "component_" + std::to_string(subdomain_id) + +// "_" + std::to_string(level_idx) + "_" + +// std::to_string(bitplane_idx); +// mgard_x::SIZE level_size = readfile( +// input + "/" + filename, +// refactored_data.data[subdomain_id][level_idx][bitplane_idx]); +// mgard_x::pin_memory( +// refactored_data.data[subdomain_id][level_idx][bitplane_idx], +// level_size, config); +// if (level_size != refactored_metadata.metadata[subdomain_id] +// .level_sizes[level_idx][bitplane_idx]) { +// std::cout << "mdr component size mismatch."; +// exit(-1); +// } +// size_read += level_size; +// } +// if (initialize_signs) { +// // level sign +// refactored_data.level_signs[subdomain_id][level_idx] = +// (bool *)malloc(sizeof(bool) * +// metadata.level_num_elems[level_idx]); +// memset(refactored_data.level_signs[subdomain_id][level_idx], 0, +// sizeof(bool) * metadata.level_num_elems[level_idx]); +// mgard_x::pin_memory( +// refactored_data.level_signs[subdomain_id][level_idx], +// sizeof(bool) * metadata.level_num_elems[level_idx], config); +// } +// } +// } +// return size_read; +// } + +void posterior_bp_request(mgard_x::MDR::RefactoredMetadata &refactored_metadata, + double tol, int rank) { + if (tol == 1e-1) { + switch (rank) { + case 0: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 28, 28, 24, 24, 20, 16, 12, 12, 8}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 28, 28, 24, 24, 20, 16, 12, 12, 8}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 28, 28, 24, 24, 20, 16, 12, 12, 8}; + break; + case 1: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 28, 28, 24, 20, 20, 16, 12, 12, 8}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 28, 28, 24, 20, 20, 16, 12, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 28, 28, 24, 20, 20, 16, 12, 12, 8}; + break; + case 2: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 28, 28, 24, 20, 20, 16, 12, 12, 8, 8}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 28, 28, 24, 20, 20, 16, 12, 12, 8, 8}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 28, 28, 24, 20, 20, 16, 12, 12, 8}; + break; + case 3: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 28, 28, 24, 24, 20, 16, 12, 12, 8}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 28, 28, 24, 20, 20, 16, 12, 12, 8}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 28, 28, 24, 20, 20, 16, 12, 12, 8}; + break; + case 4: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 28, 28, 24, 24, 20, 16, 12, 12, 8}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 28, 28, 24, 24, 20, 16, 12, 12, 8}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 28, 24, 20, 20, 16, 12, 12}; + break; + case 5: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 28, 28, 24, 24, 20, 16, 12, 12, 8, 8}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 28, 28, 24, 20, 20, 16, 12, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 28, 28, 24, 24, 20, 16, 12, 12, 8}; + break; + case 6: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 28, 28, 24, 24, 20, 16, 12, 12, 8}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 28, 28, 24, 24, 20, 16, 12, 12, 8}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 28, 28, 24, 20, 20, 16, 12, 12}; + break; + case 7: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 28, 28, 24, 20, 16, 16, 12, 12}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 28, 28, 24, 20, 16, 16, 12, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 28, 28, 24, 24, 20, 16, 12, 8, 8}; + break; + default: + break; + } + } else if (tol == 1e-2) { + switch (rank) { + case 0: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 28, 24, 24, 20, 16, 12, 12}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 16, 12, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 28, 24, 24, 20, 16, 12, 12}; + break; + case 1: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 28, 24, 20, 20, 16, 12, 12}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 24, 20, 20, 16, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 28, 24, 20, 20, 16, 12, 12}; + break; + case 2: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 28, 28, 24, 20, 16, 16, 12, 12}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 28, 28, 24, 20, 16, 16, 12, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 28, 28, 24, 20, 16, 16, 12, 12}; + break; + case 3: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 28, 28, 24, 20, 20, 16, 12, 12}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 28, 28, 24, 20, 20, 16, 12, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 28, 28, 24, 20, 20, 16, 12, 12}; + break; + case 4: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 28, 28, 24, 20, 16, 16, 12, 12}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 28, 24, 20, 20, 16, 12, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 28, 24, 20, 20, 16, 12, 12}; + break; + case 5: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 28, 28, 24, 20, 16, 16, 12, 12}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 28, 28, 24, 20, 16, 16, 12, 12}; + break; + case 6: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 28, 28, 24, 20, 20, 16, 12, 12}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 28, 28, 24, 20, 20, 16, 12, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 28, 28, 24, 20, 20, 16, 12, 12}; + break; + case 7: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 20, 16, 12}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 20, 16, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 28, 28, 24, 24, 20, 16, 16, 12, 8}; + break; + default: + break; + } + } else if (tol == 1e-3) { + switch (rank) { + case 0: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 28, 24, 24, 20, 16, 16, 12}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 20, 16, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 20, 16, 12}; + break; + case 1: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 28, 24, 24, 20, 16, 16, 12}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 20, 16, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; + break; + case 2: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 16, 12, 12}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 28, 24, 24, 20, 16, 16, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 28, 24, 24, 20, 16, 16, 12}; + break; + case 3: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; + break; + case 4: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 28, 24, 24, 20, 16, 16, 12}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 20, 16, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 20, 16, 12}; + break; + case 5: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; + break; + case 6: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; + break; + case 7: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 28, 24, 24, 20, 16, 16, 12}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 28, 24, 24, 20, 16, 16, 12}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 24, 20, 20, 16, 12}; + break; + default: + break; + } + } else if (tol == 1e-4) { + switch (rank) { + case 0: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 24, 20, 16, 16}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + break; + case 1: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 24, 24, 20, 16}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + break; + case 2: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 20, 16}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + break; + case 3: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + break; + case 4: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + break; + case 5: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + break; + case 6: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; + break; + case 7: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 20, 16}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 28, 24, 20, 20, 16}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 32, 28, 24, 20, 20, 16, 12}; + break; + default: + break; + } + } else if (tol == 1e-5) { + switch (rank) { + case 0: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 32, 28, 24, 20, 20}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 32, 28, 28, 20, 20}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 32, 28, 24, 20, 20}; + break; + case 1: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 32, 32, 28, 24, 20}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; + break; + case 2: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 32, 28, 24, 20, 20}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 32, 28, 24, 20, 16}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; + break; + case 3: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; + break; + case 4: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; + break; + case 5: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 32, 32, 28, 24, 20}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; + break; + case 6: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; + break; + case 7: + refactored_metadata.metadata[0].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; + refactored_metadata.metadata[1].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; + refactored_metadata.metadata[2].requested_level_num_bitplanes = { + 32, 32, 32, 32, 32, 28, 28, 24, 20, 16}; + break; + default: + break; + } + } +} + +int verbose_to_log_level(int verbose) { + if (verbose == 0) { + return mgard_x::log::ERR; + } else if (verbose == 1) { + return mgard_x::log::ERR | mgard_x::log::INFO; + } else if (verbose == 2) { + return mgard_x::log::ERR | mgard_x::log::TIME; + } else if (verbose == 3) { + return mgard_x::log::ERR | mgard_x::log::INFO | mgard_x::log::TIME; + } +} + +template +int launch_refactor(mgard_x::DIM D, enum mgard_x::data_type dtype, + std::string input_file, std::string output_file, + std::vector shape, + std::string domain_decomposition, mgard_x::SIZE block_size, + enum mgard_x::device_type dev_type, int verbose, + mgard_x::SIZE max_memory_footprint, int rank) { + + mgard_x::Config config; + config.normalize_coordinates = false; + config.log_level = verbose_to_log_level(verbose); + config.decomposition = mgard_x::decomposition_type::MultiDim; + if (domain_decomposition == "max-dim") { + config.domain_decomposition = mgard_x::domain_decomposition_type::MaxDim; + } else if (domain_decomposition == "block") { + config.domain_decomposition = mgard_x::domain_decomposition_type::Block; + config.block_size = block_size; + } else if (domain_decomposition == "variable") { + config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; + } + + config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; + config.domain_decomposition_dim = 0; + config.domain_decomposition_sizes = {shape[0] / 3, shape[0] / 3, + shape[0] / 3}; + + config.dev_type = dev_type; + config.max_memory_footprint = max_memory_footprint; + if (dtype == mgard_x::data_type::Float) { + config.total_num_bitplanes = 32; + } else if (dtype == mgard_x::data_type::Double) { + config.total_num_bitplanes = 64; + } + + size_t original_size = 1; + for (mgard_x::DIM i = 0; i < D; i++) + original_size *= shape[i]; + T *original_data = (T *)malloc(original_size * sizeof(T)); + size_t in_size = 0; + if (std::string(input_file).compare("random") == 0) { + in_size = original_size * sizeof(T); + srand(7117); + T c = 0; + for (size_t i = 0; i < original_size; i++) { + original_data[i] = rand() % 10 + 1; + } + } else { + T *file_data; + in_size = readfile(input_file, file_data); + + size_t loaded_size = 0; + while (loaded_size < original_size) { + std::memcpy(original_data + loaded_size, file_data, + std::min(in_size / sizeof(T), original_size - loaded_size) * + sizeof(T)); + loaded_size += std::min(in_size / sizeof(T), original_size - loaded_size); + } + in_size = loaded_size * sizeof(T); + } + if (in_size != original_size * sizeof(T)) { + std::cout << mgard_x::log::log_warn << "input file size mismatch " + << in_size << " vs. " << original_size * sizeof(T) << "!\n"; + } + + std::cout << mgard_x::log::log_info << "Max output data size: " + << mgard_x::MDR::MDRMaxOutputDataSize(D, dtype, shape, config) + << " bytes\n"; + + mgard_x::MDR::RefactoredMetadata refactored_metadata; + mgard_x::MDR::RefactoredData refactored_data; + mgard_x::pin_memory(original_data, original_size * sizeof(T), config); + double local_elapsed_time = 0, max_time = 0; + local_elapsed_time = -MPI_Wtime(); + mgard_x::MDR::MDRefactor(D, dtype, shape, original_data, refactored_metadata, + refactored_data, config, false); + local_elapsed_time += MPI_Wtime(); + MPI_Reduce(&local_elapsed_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, + MPI_COMM_WORLD); + if (!rank) + std::cout << "max_elapsed_time = " << max_time << std::endl; + write_mdr(refactored_metadata, refactored_data, output_file); + + mgard_x::unpin_memory(original_data, config); + delete[](T *) original_data; + + return 0; +} + +template +T compute_max_abs_error(const T *vec_ori, const T *vec_rec, size_t n) { + T error = fabs(vec_ori[0] - vec_rec[0]); + T max = error; + for (int i = 1; i < n; i++) { + error = fabs(vec_ori[i] - vec_rec[i]); + if (max < error) + max = error; + } + return max; +} + +template T compute_value_range(const T *vec, size_t n) { + T min = vec[0]; + T max = vec[0]; + for (int i = 0; i < n; i++) { + if (vec[i] < min) + min = vec[i]; + if (vec[i] > max) + max = vec[i]; + } + return max - min; +} + +template +void compute_VTOT(const T *Vx, const T *Vy, const T *Vz, size_t n, T *V_TOT_) { + for (int i = 0; i < n; i++) { + double V_TOT_2 = Vx[i] * Vx[i] + Vy[i] * Vy[i] + Vz[i] * Vz[i]; + double V_TOT = sqrt(V_TOT_2); + V_TOT_[i] = V_TOT; + } +} + +int launch_reconstruct(std::string input_file, std::string output_file, + std::string original_file, enum mgard_x::data_type dtype, + std::vector shape, + std::vector tols, double s, + enum mgard_x::error_bound_type mode, + bool adaptive_resolution, + enum mgard_x::device_type dev_type, int verbose, + int rank, int decrease_method = 0) { + + double bitrate = 0; + mgard_x::Config config; + config.normalize_coordinates = false; + config.log_level = verbose_to_log_level(verbose); + config.dev_type = dev_type; + config.mdr_adaptive_resolution = adaptive_resolution; + + config.mdr_qoi_mode = true; + config.mdr_qoi_num_variables = shape.size(); + config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; + config.domain_decomposition_dim = 0; + config.domain_decomposition_sizes = {shape[0] / 3, shape[0] / 3, + shape[0] / 3}; + + mgard_x::Byte *original_data; + size_t in_size = 0; + size_t original_size = 1; + for (mgard_x::DIM i = 0; i < shape.size(); i++) + original_size *= shape[i]; + if (original_file.compare("none") != 0 && !config.mdr_adaptive_resolution) { + if (original_file.compare("random") == 0) { + if (dtype == mgard_x::data_type::Float) { + in_size = original_size * sizeof(float); + original_data = (mgard_x::Byte *)new float[original_size]; + srand(7117); + for (size_t i = 0; i < original_size; i++) { + ((float *)original_data)[i] = rand() % 10 + 1; + } + } else if (dtype == mgard_x::data_type::Double) { + in_size = original_size * sizeof(double); + original_data = (mgard_x::Byte *)new double[original_size]; + srand(7117); + for (size_t i = 0; i < original_size; i++) { + ((double *)original_data)[i] = rand() % 10 + 1; + } + } + } else { + mgard_x::Byte *file_data; + in_size = readfile(original_file, file_data); + + if (dtype == mgard_x::data_type::Float) { + original_size *= sizeof(float); + } else if (dtype == mgard_x::data_type::Double) { + original_size *= sizeof(double); + } + + original_data = (mgard_x::Byte *)malloc(original_size); + + size_t loaded_size = 0; + while (loaded_size < original_size) { + + std::memcpy(original_data + loaded_size, file_data, + std::min(in_size, original_size - loaded_size)); + loaded_size += std::min(in_size, original_size - loaded_size); + } + in_size = loaded_size; + } + } + mgard_x::Byte *V_TOT_ori; + std::vector ebs; + size_t num_elements; + double tau = 0; + V_TOT_ori = (mgard_x::Byte *)malloc(in_size / config.mdr_qoi_num_variables); + mgard_x::Byte *org_Vx_ptr = original_data + original_size / 3 * 0; + mgard_x::Byte *org_Vy_ptr = original_data + original_size / 3 * 1; + mgard_x::Byte *org_Vz_ptr = original_data + original_size / 3 * 2; + if (dtype == mgard_x::data_type::Float) { + num_elements = (in_size / config.mdr_qoi_num_variables) / sizeof(float); + compute_VTOT((float *)org_Vx_ptr, (float *)org_Vy_ptr, + (float *)org_Vz_ptr, num_elements, (float *)V_TOT_ori); + // tau = compute_value_range((float *) V_TOT_ori, num_elements) * tols[0]; + if (decrease_method < 4) { + ebs.push_back(compute_value_range((float *)org_Vx_ptr, num_elements) * + tols[0]); + ebs.push_back(compute_value_range((float *)org_Vy_ptr, num_elements) * + tols[0]); + ebs.push_back(compute_value_range((float *)org_Vz_ptr, num_elements) * + tols[0]); + } else { + std::vector var_value_range; + var_value_range.push_back( + compute_value_range((float *)org_Vx_ptr, num_elements) * tols[0]); + var_value_range.push_back( + compute_value_range((float *)org_Vy_ptr, num_elements) * tols[0]); + var_value_range.push_back( + compute_value_range((float *)org_Vz_ptr, num_elements) * tols[0]); + for (int i = 0; i < config.mdr_qoi_num_variables; i++) { + ebs.push_back( + *std::min_element(var_value_range.begin(), var_value_range.end())); + } + } + float local_max = -std::numeric_limits::max(); + float local_min = std::numeric_limits::max(); + float global_max = 0, global_min = 0; + float *V_TOT = (float *)V_TOT_ori; + for (int i = 0; i < num_elements; i++) { + if (V_TOT[i] > local_max) + local_max = V_TOT[i]; + if (V_TOT[i] < local_min) + local_min = V_TOT[i]; + } + // std::cout << "local_min = " << local_min << ", local_max = " << + // local_max << std::endl; + MPI_Allreduce(&local_min, &global_min, 1, MPI_FLOAT, MPI_MIN, + MPI_COMM_WORLD); + MPI_Allreduce(&local_max, &global_max, 1, MPI_FLOAT, MPI_MAX, + MPI_COMM_WORLD); + tau = (double)(global_max - global_min) * tols[0]; + } else if (dtype == mgard_x::data_type::Double) { + num_elements = (in_size / config.mdr_qoi_num_variables) / sizeof(double); + compute_VTOT((double *)org_Vx_ptr, (double *)org_Vy_ptr, + (double *)org_Vz_ptr, num_elements, + (double *)V_TOT_ori); + // tau = compute_value_range((double *) V_TOT_ori, num_elements) * tols[0]; + if (decrease_method < 4) { + ebs.push_back(compute_value_range((double *)org_Vx_ptr, num_elements) * + tols[0]); + ebs.push_back(compute_value_range((double *)org_Vy_ptr, num_elements) * + tols[0]); + ebs.push_back(compute_value_range((double *)org_Vz_ptr, num_elements) * + tols[0]); + } else { + std::vector var_value_range; + var_value_range.push_back( + compute_value_range((double *)org_Vx_ptr, num_elements) * tols[0]); + var_value_range.push_back( + compute_value_range((double *)org_Vy_ptr, num_elements) * tols[0]); + var_value_range.push_back( + compute_value_range((double *)org_Vz_ptr, num_elements) * tols[0]); + for (int i = 0; i < config.mdr_qoi_num_variables; i++) { + ebs.push_back( + *std::min_element(var_value_range.begin(), var_value_range.end())); + } + } + double local_min = -std::numeric_limits::max(); + double local_max = std::numeric_limits::max(); + double global_max = 0, global_min = 0; + double *V_TOT = (double *)V_TOT_ori; + for (int i = 0; i < num_elements; i++) { + if (V_TOT[i] > local_max) + local_max = V_TOT[i]; + if (V_TOT[i] < local_min) + local_min = V_TOT[i]; + } + MPI_Allreduce(&local_min, &global_min, 1, MPI_DOUBLE, MPI_MIN, + MPI_COMM_WORLD); + MPI_Allreduce(&local_max, &global_max, 1, MPI_DOUBLE, MPI_MAX, + MPI_COMM_WORLD); + tau = (double)(global_max - global_min) * tols[0]; + } + + mgard_x::MDR::RefactoredMetadata refactored_metadata; + mgard_x::MDR::RefactoredData refactored_data; + mgard_x::MDR::ReconstructedData reconstructed_data; + size_t metadata_size = + read_mdr_metadata(refactored_metadata, refactored_data, input_file); + refactored_metadata.total_size += metadata_size; + + refactored_metadata.relative_eb = tols[0]; + refactored_metadata.decrease_method = decrease_method; + refactored_metadata.MPI_enabled = true; + refactored_metadata.input_path = input_file; + for (int i = 0; i < config.mdr_qoi_num_variables; i++) { + refactored_metadata.metadata[i].num_elements = num_elements; + if (decrease_method == 0) { + refactored_metadata.metadata[i].requested_tol = ebs[i]; + } else if (decrease_method == 1) { + refactored_metadata.metadata[i].requested_size = 1; + refactored_metadata.metadata[i].segmented = true; + } else if (decrease_method >= 2) { + refactored_metadata.metadata[i].requested_tol = ebs[i]; + refactored_metadata.metadata[i].corresponding_error_return = true; + } + refactored_metadata.metadata[i].tau = tau; + refactored_metadata.metadata[i].requested_s = s; + } + mgard_x::MDR::MDRequest(refactored_metadata, config); + + posterior_bp_request(refactored_metadata, tols[0], rank); + double local_IO_time = 0, global_IO_time = 0; + MPI_Barrier(MPI_COMM_WORLD); + local_IO_time = -MPI_Wtime(); + size_t size_read = + read_mdr(refactored_metadata, refactored_data, input_file, true, config); + MPI_Barrier(MPI_COMM_WORLD); + local_IO_time += MPI_Wtime(); + MPI_Reduce(&local_IO_time, &global_IO_time, 1, MPI_DOUBLE, MPI_MAX, 0, + MPI_COMM_WORLD); + if (!rank) + std::cout << "IO_time = " << global_IO_time << std::endl; + + // refactored_metadata.total_size += size_read; + double local_elapsed_time = 0, max_time = 0; + local_elapsed_time = -MPI_Wtime(); + mgard_x::MDR::MDReconstruct(refactored_metadata, refactored_data, + reconstructed_data, config, false); + local_elapsed_time += MPI_Wtime(); + MPI_Reduce(&local_elapsed_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, + MPI_COMM_WORLD); + if (!rank) + std::cout << "max_elapsed_time = " << max_time << std::endl; + double local_kernel_time = 0, global_kernel_time = 0; + local_kernel_time = refactored_metadata.kernel_time; + MPI_Reduce(&local_kernel_time, &global_kernel_time, 1, MPI_DOUBLE, MPI_MAX, 0, + MPI_COMM_WORLD); + if (!rank) + std::cout << "max_kernel_time = " << global_kernel_time << std::endl; + for (int i = 0; i < 8; ++i) { + MPI_Barrier(MPI_COMM_WORLD); + if (rank == i) { + std::cout << "From Rank " << i << ": " + << "kernel_time = " << local_kernel_time << std::endl; + } + } + + // we can check reconstructed_data.qoi_in_progress here + + // std::cout << mgard_x::log::log_info << "Additional " << size_read + // << " bytes read for reconstruction\n"; + + std::vector rec_var_ptrs; + if (original_file.compare("none") != 0 && !config.mdr_adaptive_resolution) { + for (int i = 0; i < config.mdr_qoi_num_variables; i++) { + std::vector var_shape = shape; + var_shape[0] /= config.mdr_qoi_num_variables; + mgard_x::Byte *org_var_ptr = original_data + original_size / 3 * i; + mgard_x::Byte *rec_var_ptr = + reconstructed_data.data[0] + original_size / 3 * i; + rec_var_ptrs.push_back(rec_var_ptr); + } + } + mgard_x::Byte *V_TOT_rec; + V_TOT_rec = (mgard_x::Byte *)malloc(in_size / config.mdr_qoi_num_variables); + if (dtype == mgard_x::data_type::Float) { + compute_VTOT((float *)rec_var_ptrs[0], (float *)rec_var_ptrs[1], + (float *)rec_var_ptrs[2], num_elements, + (float *)V_TOT_rec); + } else if (dtype == mgard_x::data_type::Double) { + compute_VTOT((double *)rec_var_ptrs[0], (double *)rec_var_ptrs[1], + (double *)rec_var_ptrs[2], num_elements, + (double *)V_TOT_rec); + } + for (auto &metadata : refactored_metadata.metadata) { + refactored_metadata.total_size += metadata.GetLoadedBitPlaneSizes(); + } + + // Bitrate + unsigned long long int local_total_size = refactored_metadata.total_size; + unsigned long long int global_total_size = 0; + MPI_Reduce(&local_total_size, &global_total_size, 1, MPI_UNSIGNED_LONG_LONG, + MPI_SUM, 0, MPI_COMM_WORLD); + unsigned long long int local_in_size = in_size; + unsigned long long int global_in_size = 0; + MPI_Reduce(&local_in_size, &global_in_size, 1, MPI_UNSIGNED_LONG_LONG, + MPI_SUM, 0, MPI_COMM_WORLD); + if (dtype == mgard_x::data_type::Float) { + bitrate = 32 / ((double)global_in_size / global_total_size); + } else if (dtype == mgard_x::data_type::Double) { + bitrate = 64 / ((double)global_in_size / global_total_size); + } + if (!rank) + std::cout << "Bitrate = " << bitrate << std::endl; // MPI_REDUCE SUM + + // std::cout << "Original Vx[35345] = " << ((float*) org_Vx_ptr)[35345] << ", + // Reconstructed Vx[35345] = " << ((float*) rec_var_ptrs[0])[35345] << + // std::endl; + if (!rank) + std::cout << "Requested_Tau = " << tau << std::endl; + + // Max_est_error + double local_max_est_error = refactored_metadata.max_est_error; + double global_max_est_error = 0; + MPI_Reduce(&local_max_est_error, &global_max_est_error, 1, MPI_DOUBLE, + MPI_MAX, 0, MPI_COMM_WORLD); + if (!rank) + std::cout << "Est_max_error = " << global_max_est_error << std::endl; + + // Max_real_error + double local_real_max_error = (double)compute_max_abs_error( + (float *)V_TOT_ori, (float *)V_TOT_rec, num_elements); + double global_real_max_error = 0; + MPI_Reduce(&local_real_max_error, &global_real_max_error, 1, MPI_DOUBLE, + MPI_MAX, 0, MPI_COMM_WORLD); + if (!rank) + std::cout << "Real_max_error = " << global_real_max_error << std::endl; + // for (int i = 0; i < 8; ++i) { + // MPI_Barrier(MPI_COMM_WORLD); + // if (rank == i) { + // std::cout << "From Rank " << i << ": " << std::endl; + // int count = 0; + // for (auto &metadata : refactored_metadata.metadata){ + // if (count == 0) std::cout << "Vx:" << std::endl; + // else if (count == 1) std::cout << "Vy:" << std::endl; + // else if (count == 2) std::cout << "Vz:" << std::endl; + // for (size_t i = 0; i < metadata.loaded_level_num_bitplanes.size(); + // ++i) { + // std::cout << "Level " << i << ": bitplane = " + // << + // static_cast(metadata.loaded_level_num_bitplanes[i]) + // << std::endl; + // } + // count ++; + // } + // } + // } + + return 0; +} + +bool try_refactoring(int argc, char *argv[]) { + if (!has_arg(argc, argv, "-z", "--refactor")) + return false; + mgard_x::log::info("Mode: refactor", true); + + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + std::ostringstream oss; + oss << rank; + + std::string input_file = + get_arg(argc, argv, "Original data", "-i", "--input"); + input_file += oss.str() + ".dat"; + // std::cout << input_file << std::endl; + std::string output_file = + get_arg(argc, argv, "Refactored data", "-o", "--output"); + output_file += oss.str(); + // std::cout << output_file << std::endl; + enum mgard_x::data_type dtype = get_data_type(argc, argv); + std::vector shape = + get_args(argc, argv, "Dimensions", "-dim", "--dimension"); + // std::string lossless_level = get_arg(argc, argv, "Lossless", + // "-l", "--lossless"); + enum mgard_x::device_type dev_type = get_device_type(argc, argv); + int verbose = 0; + if (has_arg(argc, argv, "-v", "--verbose")) { + verbose = get_arg(argc, argv, "Verbose", "-v", "--verbose"); + } + mgard_x::SIZE max_memory_footprint = + std::numeric_limits::max(); + if (has_arg(argc, argv, "-m", "--max-memory")) { + max_memory_footprint = (mgard_x::SIZE)get_arg( + argc, argv, "Max memory", "-m", "--max-memory"); + } + std::string domain_decomposition = "max-dim"; + mgard_x::SIZE block_size = 0; + if (has_arg(argc, argv, "-dd", "--domain-decomposition")) { + domain_decomposition = get_arg( + argc, argv, "Domain decomposition", "-dd", "--domain-decomposition"); + if (domain_decomposition == "block") { + block_size = get_arg(argc, argv, "Block size", "-dd-size", + "--domain-decomposition-size"); + } + } + if (dtype == mgard_x::data_type::Double) { + launch_refactor(shape.size(), dtype, input_file.c_str(), + output_file.c_str(), shape, domain_decomposition, + block_size, dev_type, verbose, max_memory_footprint, + rank); + } else if (dtype == mgard_x::data_type::Float) { + launch_refactor(shape.size(), dtype, input_file.c_str(), + output_file.c_str(), shape, domain_decomposition, + block_size, dev_type, verbose, max_memory_footprint, + rank); + } + return true; +} + +bool try_reconstruction(int argc, char *argv[]) { + if (!has_arg(argc, argv, "-x", "--reconstruct")) + return false; + + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + std::ostringstream oss; + // oss << "JHTDB_" << rank; + oss << rank; + + mgard_x::log::info("mode: reconstruct", true); + std::string input_file = + get_arg(argc, argv, "Refactored data", "-i", "--input"); + // if (!input_file.empty() && input_file.back() == '/') input_file += + // oss.str(); else input_file += "/" + oss.str(); + input_file += oss.str(); + // std::cout << input_file << std::endl; + std::string output_file = + get_arg(argc, argv, "Reconstructed data", "-o", "--output"); + // default is none (means original data not provided) + std::string original_file = "none"; + enum mgard_x::data_type dtype; + std::vector shape; + if (has_arg(argc, argv, "-g", "--orignal")) { + original_file = + get_arg(argc, argv, "Original data", "-g", "--orignal"); + // if (!original_file.empty() && original_file.back() == '/') original_file + // += oss.str() + ".dat"; else original_file += "/" + oss.str() + ".dat"; + original_file += oss.str() + ".dat"; + // std::cout << original_file << std::endl; + dtype = get_data_type(argc, argv); + shape = get_args(argc, argv, "Dimensions", "-dim", + "--dimension"); + } + // only abs mode is supported now + enum mgard_x::error_bound_type mode = + mgard_x::error_bound_type::ABS; // REL or ABS + + std::vector tols; + if (has_arg(argc, argv, "-e", "--error-bound")) { + tols.push_back( + get_arg(argc, argv, "Error bound", "-e", "--error-bound")); + } else if (has_arg(argc, argv, "-me", "--multi-error-bounds")) { + tols = get_args(argc, argv, "Multi error bounds", "-me", + "--multi-error-bounds"); + } else { + throw std::runtime_error( + "Missing option -e/--error-bound or -me/--multi-error-bounds"); + } + double s = get_arg(argc, argv, "Smoothness", "-s", "--smoothness"); + enum mgard_x::device_type dev_type = get_device_type(argc, argv); + int verbose = 0; + if (has_arg(argc, argv, "-v", "--verbose")) { + verbose = get_arg(argc, argv, "Verbose", "-v", "--verbose"); + } + bool adaptive_resolution = false; + if (has_arg(argc, argv, "-ar", "--adaptive-resolution")) { + adaptive_resolution = get_arg(argc, argv, "Adaptive resolution", "-ar", + "--adaptive-resolution"); + } + if (verbose) + std::cout << mgard_x::log::log_info << "verbose: enabled.\n"; + int decrease_method; + if (has_arg(argc, argv, "-dm", "--decrease-method")) { + decrease_method = + get_arg(argc, argv, "Decrease method", "-dm", "--decrease-method"); + } + launch_reconstruct(input_file, output_file, original_file, dtype, shape, tols, + s, mode, adaptive_resolution, dev_type, verbose, rank, + decrease_method); + return true; +} + +int main(int argc, char *argv[]) { + + MPI_Init(&argc, &argv); + + if (!try_refactoring(argc, argv) && !try_reconstruction(argc, argv)) { + print_usage_message(""); + } + + MPI_Finalize(); + return 0; +} \ No newline at end of file diff --git a/src/mgard-x/Executables/pmdr-x-qoi.cpp b/src/mgard-x/Executables/pmdr-x-qoi.cpp new file mode 100644 index 0000000000..f731a11064 --- /dev/null +++ b/src/mgard-x/Executables/pmdr-x-qoi.cpp @@ -0,0 +1,1237 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "compress_x.hpp" +#include "mdr_x.hpp" +#include "mgard-x/RuntimeX/Utilities/Log.h" +#include "mgard-x/Utilities/ErrorCalculator.h" + +#include "ArgumentParser.h" + +#include +using namespace std::chrono; + +void print_usage_message(std::string error) { + if (error.compare("") != 0) { + std::cout << mgard_x::log::log_err << error << std::endl; + } + printf("Options\n\ + \t -z / --refactor: refactor data\n\ + \t\t -i / --input \n\ + \t\t -o / --output \n\ + \t\t -dt / --data-type : data type (s: single; d:double)\n\ + \t\t -dim / --dimension : total number of dimensions\n\ + \t\t\t [dim1]: slowest dimention\n\ + \t\t\t [dim2]: 2nd slowest dimention\n\ + \t\t\t ...\n\ + \t\t\t [dimN]: fastest dimention\n\ + \t\t -d / --device : device type\n\ + \t\t (optional) -v / --verbose <0|1|2|3> 0: error; 1: error+info; 2: error+timing; 3: all\n\ + \t\t (optional) -m / --max-memory \n\ + \t\t (optional) -dd / --domain-decomposition \n\ + \t\t\t (optional) -dd-size / --domain-decomposition-size (for block domain decomposition only) \n\ + \n\ + \t -x / --reconstruct: reconstruct data\n\ + \t\t -i / --input \n\ + \t\t -o / --output \n\ + \t\t (optional) -g / --orginal (optinal)\n\ + \t\t -e / --error-bound : error bound\n\ + \t\t -me / --multi-error-bounds ..: multiple error bounds\n\ + \t\t -s / --smoothness : smoothness parameter\n\ + \t\t -d : device type\n\ + \t\t (optional) -v / --verbose <0|1|2|3> 0: error; 1: error+info; 2: error+timing; 3: all\n"); + exit(0); +} + +template void min_max(size_t n, T *in_buff) { + T min = std::numeric_limits::infinity(); + T max = 0; + for (size_t i = 0; i < n; i++) { + if (min > in_buff[i]) { + min = in_buff[i]; + } + if (max < in_buff[i]) { + max = in_buff[i]; + } + } + printf("Min: %f, Max: %f\n", min, max); +} + +template size_t readfile(std::string input_file, T *&in_buff) { + // std::cout << mgard_x::log::log_info << "Loading file: " << input_file << + // "\n"; + + FILE *pFile; + pFile = fopen(input_file.c_str(), "rb"); + if (pFile == NULL) { + std::cout << mgard_x::log::log_err << "file open error!\n"; + exit(1); + } + fseek(pFile, 0, SEEK_END); + size_t lSize = ftell(pFile); + rewind(pFile); + in_buff = (T *)malloc(lSize); + lSize = fread(in_buff, 1, lSize, pFile); + fclose(pFile); + // min_max(lSize/sizeof(T), in_buff); + return lSize; +} + +template +size_t readfile_header_metadata(std::string input_file, + std::vector &in_buff) { + // std::cout << mgard_x::log::log_info << "Loading file: " << input_file << + // "\n"; + + FILE *pFile; + pFile = fopen(input_file.c_str(), "rb"); + if (pFile == NULL) { + std::cout << mgard_x::log::log_err << "file open error!\n"; + exit(1); + } + fseek(pFile, 0, SEEK_END); + size_t lSize = ftell(pFile); + rewind(pFile); + in_buff.resize(lSize / sizeof(T)); + lSize = fread(in_buff.data(), 1, lSize, pFile); + fclose(pFile); + return lSize; +} + +template +void writefile(std::string output_file, T *out_buff, size_t num_bytes) { + FILE *file = fopen(output_file.c_str(), "w"); + fwrite(out_buff, 1, num_bytes, file); + fclose(file); +} + +template +void print_statistics(double s, enum mgard_x::error_bound_type mode, + std::vector shape, T *original_data, + T *decompressed_data, T tol, bool normalize_coordinates) { + mgard_x::SIZE n = 1; + for (mgard_x::DIM d = 0; d < shape.size(); d++) + n *= shape[d]; + T actual_error = 0.0; + std::cout << std::scientific; + if (s == std::numeric_limits::infinity()) { + actual_error = + mgard_x::L_inf_error(n, original_data, decompressed_data, mode); + if (mode == mgard_x::error_bound_type::ABS) { + std::cout << mgard_x::log::log_info + << "Absoluate L_inf error: " << actual_error << " (" + << (actual_error < tol ? "\e[32mSatisified\e[0m" + : "\e[31mNot Satisified\e[0m") + << ")" + << "\n"; + } else if (mode == mgard_x::error_bound_type::REL) { + std::cout << mgard_x::log::log_info + << "Relative L_inf error: " << actual_error << " (" + << (actual_error < tol ? "\e[32mSatisified\e[0m" + : "\e[31mNot Satisified\e[0m") + << ")" + << "\n"; + } + } else { + actual_error = mgard_x::L_2_error(shape, original_data, decompressed_data, + mode, normalize_coordinates); + if (mode == mgard_x::error_bound_type::ABS) { + std::cout << mgard_x::log::log_info + << "Absoluate L_2 error: " << actual_error << " (" + << (actual_error < tol ? "\e[32mSatisified\e[0m" + : "\e[31mNot Satisified\e[0m") + << ")" + << "\n"; + } else if (mode == mgard_x::error_bound_type::REL) { + std::cout << mgard_x::log::log_info + << "Relative L_2 error: " << actual_error << " (" + << (actual_error < tol ? "\e[32mSatisified\e[0m" + : "\e[31mNot Satisified\e[0m") + << ")" + << "\n"; + } + } + + std::cout << mgard_x::log::log_info + << "MSE: " << mgard_x::MSE(n, original_data, decompressed_data) + << "\n"; + std::cout << std::defaultfloat; + std::cout << mgard_x::log::log_info + << "PSNR: " << mgard_x::PSNR(n, original_data, decompressed_data) + << "\n"; + + // if (actual_error > tol) + // exit(-1); +} + +void create_dir(std::string name) { + struct stat st = {0}; + if (stat(name.c_str(), &st) == -1) { + mkdir(name.c_str(), 0700); + } +} + +void write_mdr(mgard_x::MDR::RefactoredMetadata &refactored_metadata, + mgard_x::MDR::RefactoredData &refactored_data, + std::string output) { + size_t size_written = 0; + create_dir(output); + std::vector serialized_metadata = + refactored_metadata.Serialize(); + writefile(output + "/header", refactored_metadata.header.data(), + refactored_metadata.header.size()); + writefile(output + "/metadata", serialized_metadata.data(), + serialized_metadata.size()); + for (int subdomain_id = 0; subdomain_id < refactored_metadata.metadata.size(); + subdomain_id++) { + for (int level_idx = 0; + level_idx < + refactored_metadata.metadata[subdomain_id].level_sizes.size(); + level_idx++) { + for (int bitplane_idx = 0; + bitplane_idx < refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx] + .size(); + bitplane_idx++) { + if (refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx] > 0) { + std::string filename = "component_" + std::to_string(subdomain_id) + + "_" + std::to_string(level_idx) + "_" + + std::to_string(bitplane_idx); + writefile(output + "/" + filename, + refactored_data.data[subdomain_id][level_idx][bitplane_idx], + refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx]); + size_written += refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx]; + } + } + } + } + std::cout << mgard_x::log::log_info << size_written << " bytes written\n"; +} + +size_t read_mdr_metadata(mgard_x::MDR::RefactoredMetadata &refactored_metadata, + mgard_x::MDR::RefactoredData &refactored_data, + std::string input) { + + size_t metadata_size = 0; + metadata_size += + readfile_header_metadata(input + "/header", refactored_metadata.header); + std::vector serialized_metadata; + metadata_size += + readfile_header_metadata(input + "/metadata", serialized_metadata); + refactored_metadata.Deserialize(serialized_metadata); + refactored_metadata.InitializeForReconstruction(); + refactored_data.InitializeForReconstruction(refactored_metadata); + return metadata_size; +} + +// size_t read_mdr(mgard_x::MDR::RefactoredMetadata &refactored_metadata, +// mgard_x::MDR::RefactoredData &refactored_data, std::string +// input, bool initialize_signs, mgard_x::Config config) { + +// size_t size_read = 0; +// int num_subdomains = refactored_metadata.metadata.size(); +// for (int subdomain_id = 0; subdomain_id < num_subdomains; subdomain_id++) +// { +// mgard_x::MDR::MDRMetadata metadata = +// refactored_metadata.metadata[subdomain_id]; +// int num_levels = metadata.level_sizes.size(); +// for (int level_idx = 0; level_idx < num_levels; level_idx++) { +// int num_bitplanes = metadata.level_sizes[level_idx].size(); +// int loaded_bitplanes = metadata.loaded_level_num_bitplanes[level_idx]; +// int reqested_bitplanes = +// metadata.requested_level_num_bitplanes[level_idx]; +// for (int bitplane_idx = loaded_bitplanes; +// bitplane_idx < reqested_bitplanes; bitplane_idx++) { +// std::string filename = "component_" + std::to_string(subdomain_id) + +// "_" + std::to_string(level_idx) + "_" + +// std::to_string(bitplane_idx); +// mgard_x::SIZE level_size = readfile( +// input + "/" + filename, +// refactored_data.data[subdomain_id][level_idx][bitplane_idx]); +// mgard_x::pin_memory( +// refactored_data.data[subdomain_id][level_idx][bitplane_idx], +// level_size, config); +// if (level_size != refactored_metadata.metadata[subdomain_id] +// .level_sizes[level_idx][bitplane_idx]) { +// std::cout << "mdr component size mismatch."; +// exit(-1); +// } +// size_read += level_size; +// } +// if (initialize_signs) { +// // level sign +// refactored_data.level_signs[subdomain_id][level_idx] = +// (bool *)malloc(sizeof(bool) * +// metadata.level_num_elems[level_idx]); +// memset(refactored_data.level_signs[subdomain_id][level_idx], 0, +// sizeof(bool) * metadata.level_num_elems[level_idx]); +// mgard_x::pin_memory( +// refactored_data.level_signs[subdomain_id][level_idx], +// sizeof(bool) * metadata.level_num_elems[level_idx], config); +// } +// } +// } +// return size_read; +// } + +size_t read_mdr(mgard_x::MDR::RefactoredMetadata &refactored_metadata, + mgard_x::MDR::RefactoredData &refactored_data, + std::string input, bool initialize_signs, + mgard_x::Config config) { + + size_t size_read = 0; + int num_subdomains = refactored_metadata.metadata.size(); + for (int subdomain_id = 0; subdomain_id < num_subdomains; subdomain_id++) { + mgard_x::MDR::MDRMetadata metadata = + refactored_metadata.metadata[subdomain_id]; + int num_levels = metadata.level_sizes.size(); + for (int level_idx = 0; level_idx < num_levels; level_idx++) { + int num_bitplanes = metadata.level_sizes[level_idx].size(); + for (int bitplane_idx = 0; bitplane_idx < num_bitplanes; bitplane_idx++) { + + if (refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx] > 0) { + std::string filename = "component_" + std::to_string(subdomain_id) + + "_" + std::to_string(level_idx) + "_" + + std::to_string(bitplane_idx); + mgard_x::SIZE level_size = readfile( + input + "/" + filename, + refactored_data.data[subdomain_id][level_idx][bitplane_idx]); + mgard_x::pin_memory( + refactored_data.data[subdomain_id][level_idx][bitplane_idx], + level_size, config); + if (level_size != refactored_metadata.metadata[subdomain_id] + .level_sizes[level_idx][bitplane_idx]) { + std::cout << "mdr component size mismatch."; + exit(-1); + } + size_read += level_size; + } + } + if (initialize_signs) { + // level sign + refactored_data.level_signs[subdomain_id][level_idx] = + (bool *)malloc(sizeof(bool) * metadata.level_num_elems[level_idx]); + memset(refactored_data.level_signs[subdomain_id][level_idx], 0, + sizeof(bool) * metadata.level_num_elems[level_idx]); + mgard_x::pin_memory( + refactored_data.level_signs[subdomain_id][level_idx], + sizeof(bool) * metadata.level_num_elems[level_idx], config); + } + } + } + return size_read; +} + +// void posterior_bp_request(mgard_x::MDR::RefactoredMetadata +// &refactored_metadata, double tol, int rank){ if (tol == 1e-1) { +// switch (rank) { case 0: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 28, 28, +// 24, 24, 20, 16, 12, 12, 8}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 28, 28, 24, 24, 20, 16, 12, 12, 8}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 28, 28, 24, 24, 20, 16, 12, 12, 8}; break; +// case 1: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 28, 28, +// 24, 20, 20, 16, 12, 12, 8}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 28, 28, 24, 20, 20, 16, 12, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 28, 28, 24, 20, 20, 16, 12, 12, 8}; break; +// case 2: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {28, 28, 24, +// 20, 20, 16, 12, 12, 8, 8}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {28, 28, 24, 20, 20, 16, 12, 12, 8, 8}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 28, 28, 24, 20, 20, 16, 12, 12, 8}; break; +// case 3: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 28, 28, +// 24, 24, 20, 16, 12, 12, 8}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 28, 28, 24, 20, 20, 16, 12, 12, 8}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 28, 28, 24, 20, 20, 16, 12, 12, 8}; break; +// case 4: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 28, 28, +// 24, 24, 20, 16, 12, 12, 8}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 28, 28, 24, 24, 20, 16, 12, 12, 8}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 28, 24, 20, 20, 16, 12, 12}; break; +// case 5: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {28, 28, 24, +// 24, 20, 16, 12, 12, 8, 8}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 28, 28, 24, 20, 20, 16, 12, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 28, 28, 24, 24, 20, 16, 12, 12, 8}; break; +// case 6: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 28, 28, +// 24, 24, 20, 16, 12, 12, 8}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 28, 28, 24, 24, 20, 16, 12, 12, 8}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 28, 28, 24, 20, 20, 16, 12, 12}; break; +// case 7: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 28, +// 28, 24, 20, 16, 16, 12, 12}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 28, 28, 24, 20, 16, 16, 12, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 28, 28, 24, 24, 20, 16, 12, 8, 8}; break; +// default: break; +// } +// } else if (tol == 1e-2) { +// switch (rank) { +// case 0: +// refactored_metadata.metadata[0].requested_level_num_bitplanes +// = {32, 32, 32, 28, 24, 24, 20, 16, 12, 12}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 28, 28, 24, 20, 16, 12, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 28, 24, 24, 20, 16, 12, 12}; break; +// case 1: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 28, 24, 20, 20, 16, 12, 12}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 32, 28, 24, 20, 20, 16, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 28, 24, 20, 20, 16, 12, 12}; break; +// case 2: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 28, +// 28, 24, 20, 16, 16, 12, 12}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 28, 28, 24, 20, 16, 16, 12, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 28, 28, 24, 20, 16, 16, 12, 12}; break; +// case 3: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 28, +// 28, 24, 20, 20, 16, 12, 12}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 28, 28, 24, 20, 20, 16, 12, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 28, 28, 24, 20, 20, 16, 12, 12}; break; +// case 4: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 28, +// 28, 24, 20, 16, 16, 12, 12}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 28, 24, 20, 20, 16, 12, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 28, 24, 20, 20, 16, 12, 12}; break; +// case 5: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 28, +// 28, 24, 20, 16, 16, 12, 12}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 28, 28, 24, 20, 16, 16, 12, 12}; break; +// case 6: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 28, +// 28, 24, 20, 20, 16, 12, 12}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 28, 28, 24, 20, 20, 16, 12, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 28, 28, 24, 20, 20, 16, 12, 12}; break; +// case 7: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 28, 28, 24, 20, 20, 16, 12}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 28, 28, 24, 20, 20, 16, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 28, 28, 24, 24, 20, 16, 16, 12, 8}; break; +// default: break; +// } +// } else if (tol == 1e-3) { +// switch (rank) { +// case 0: +// refactored_metadata.metadata[0].requested_level_num_bitplanes +// = {32, 32, 32, 28, 24, 24, 20, 16, 16, 12}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 28, 28, 24, 20, 20, 16, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 28, 28, 24, 20, 20, 16, 12}; break; +// case 1: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 28, 24, 24, 20, 16, 16, 12}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 28, 28, 24, 20, 20, 16, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; break; +// case 2: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 28, 28, 24, 20, 16, 12, 12}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 28, 24, 24, 20, 16, 16, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 28, 24, 24, 20, 16, 16, 12}; break; +// case 3: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 28, 28, 24, 20, 16, 16, 12}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; break; +// case 4: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 28, 24, 24, 20, 16, 16, 12}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 28, 28, 24, 20, 20, 16, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 28, 28, 24, 20, 20, 16, 12}; break; +// case 5: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 28, 28, 24, 20, 16, 16, 12}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; break; +// case 6: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 28, 28, 24, 20, 16, 16, 12}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 28, 28, 24, 20, 16, 16, 12}; break; +// case 7: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 28, 24, 24, 20, 16, 16, 12}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 28, 24, 24, 20, 16, 16, 12}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 32, 28, 24, 20, 20, 16, 12}; break; +// default: break; +// } +// } else if (tol == 1e-4) { +// switch (rank) { +// case 0: +// refactored_metadata.metadata[0].requested_level_num_bitplanes +// = {32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 28, 24, 20, 16, 16}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; break; +// case 1: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 32, 28, 28, 24, 20, 16, 16}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 28, 24, 24, 20, 16}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; break; +// case 2: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 32, 28, 28, 24, 20, 16, 16}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 32, 28, 28, 24, 20, 20, 16}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; break; +// case 3: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 32, 28, 28, 24, 20, 16, 16}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; break; +// case 4: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 32, 28, 28, 24, 20, 16, 16}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; break; +// case 5: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 32, 28, 28, 24, 20, 16, 16}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; break; +// case 6: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 32, 28, 28, 24, 20, 16, 16}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 32, 28, 28, 24, 20, 16, 16}; break; +// case 7: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 32, 28, 28, 24, 20, 20, 16}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 32, 28, 28, 24, 20, 20, 16}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 32, 28, 24, 20, 20, 16, 12}; break; +// default: break; +// } +// } else if (tol == 1e-5) { +// switch (rank) { +// case 0: +// refactored_metadata.metadata[0].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 32, 28, 24, 20, 20}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 32, 28, 28, 20, 20}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 32, 28, 24, 20, 20}; break; +// case 1: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 32, 32, 28, 28, 24, 20, 20}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 32, 32, 28, 24, 20}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; break; +// case 2: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 32, 32, 32, 28, 24, 20, 20}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 32, 28, 24, 20, 16}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; break; +// case 3: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 32, 32, 28, 28, 24, 20, 20}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; break; +// case 4: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 32, 32, 28, 28, 24, 20, 20}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; break; +// case 5: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 32, 32, 28, 28, 24, 20, 20}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 32, 32, 28, 24, 20}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; break; +// case 6: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 32, 32, 28, 28, 24, 20, 20}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; break; +// case 7: +// refactored_metadata.metadata[0].requested_level_num_bitplanes = {32, 32, 32, +// 32, 32, 28, 28, 24, 20, 20}; +// refactored_metadata.metadata[1].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 28, 28, 24, 20, 20}; +// refactored_metadata.metadata[2].requested_level_num_bitplanes +// = {32, 32, 32, 32, 32, 28, 28, 24, 20, 16}; break; +// default: break; +// } +// }} + +int verbose_to_log_level(int verbose) { + if (verbose == 0) { + return mgard_x::log::ERR; + } else if (verbose == 1) { + return mgard_x::log::ERR | mgard_x::log::INFO; + } else if (verbose == 2) { + return mgard_x::log::ERR | mgard_x::log::TIME; + } else if (verbose == 3) { + return mgard_x::log::ERR | mgard_x::log::INFO | mgard_x::log::TIME; + } +} + +template +int launch_refactor(mgard_x::DIM D, enum mgard_x::data_type dtype, + std::string input_file, std::string output_file, + std::vector shape, + std::string domain_decomposition, mgard_x::SIZE block_size, + enum mgard_x::device_type dev_type, int verbose, + mgard_x::SIZE max_memory_footprint, int rank) { + + mgard_x::Config config; + config.normalize_coordinates = false; + config.log_level = verbose_to_log_level(verbose); + config.decomposition = mgard_x::decomposition_type::MultiDim; + if (domain_decomposition == "max-dim") { + config.domain_decomposition = mgard_x::domain_decomposition_type::MaxDim; + } else if (domain_decomposition == "block") { + config.domain_decomposition = mgard_x::domain_decomposition_type::Block; + config.block_size = block_size; + } else if (domain_decomposition == "variable") { + config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; + } + + config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; + config.domain_decomposition_dim = 0; + config.domain_decomposition_sizes = {shape[0] / 3, shape[0] / 3, + shape[0] / 3}; + + config.dev_type = dev_type; + config.max_memory_footprint = max_memory_footprint; + if (dtype == mgard_x::data_type::Float) { + config.total_num_bitplanes = 32; + } else if (dtype == mgard_x::data_type::Double) { + config.total_num_bitplanes = 64; + } + + size_t original_size = 1; + for (mgard_x::DIM i = 0; i < D; i++) + original_size *= shape[i]; + T *original_data = (T *)malloc(original_size * sizeof(T)); + size_t in_size = 0; + if (std::string(input_file).compare("random") == 0) { + in_size = original_size * sizeof(T); + srand(7117); + T c = 0; + for (size_t i = 0; i < original_size; i++) { + original_data[i] = rand() % 10 + 1; + } + } else { + T *file_data; + in_size = readfile(input_file, file_data); + + size_t loaded_size = 0; + while (loaded_size < original_size) { + std::memcpy(original_data + loaded_size, file_data, + std::min(in_size / sizeof(T), original_size - loaded_size) * + sizeof(T)); + loaded_size += std::min(in_size / sizeof(T), original_size - loaded_size); + } + in_size = loaded_size * sizeof(T); + } + if (in_size != original_size * sizeof(T)) { + std::cout << mgard_x::log::log_warn << "input file size mismatch " + << in_size << " vs. " << original_size * sizeof(T) << "!\n"; + } + + std::cout << mgard_x::log::log_info << "Max output data size: " + << mgard_x::MDR::MDRMaxOutputDataSize(D, dtype, shape, config) + << " bytes\n"; + + mgard_x::MDR::RefactoredMetadata refactored_metadata; + mgard_x::MDR::RefactoredData refactored_data; + mgard_x::pin_memory(original_data, original_size * sizeof(T), config); + double local_elapsed_time = 0, max_time = 0; + local_elapsed_time = -MPI_Wtime(); + mgard_x::MDR::MDRefactor(D, dtype, shape, original_data, refactored_metadata, + refactored_data, config, false); + local_elapsed_time += MPI_Wtime(); + MPI_Reduce(&local_elapsed_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, + MPI_COMM_WORLD); + if (!rank) + std::cout << "max_elapsed_time = " << max_time << std::endl; + write_mdr(refactored_metadata, refactored_data, output_file); + + mgard_x::unpin_memory(original_data, config); + delete[](T *) original_data; + + return 0; +} + +template +T compute_max_abs_error(const T *vec_ori, const T *vec_rec, size_t n) { + T error = fabs(vec_ori[0] - vec_rec[0]); + T max = error; + for (int i = 1; i < n; i++) { + error = fabs(vec_ori[i] - vec_rec[i]); + if (max < error) + max = error; + } + return max; +} + +template T compute_value_range(const T *vec, size_t n) { + T min = vec[0]; + T max = vec[0]; + for (int i = 0; i < n; i++) { + if (vec[i] < min) + min = vec[i]; + if (vec[i] > max) + max = vec[i]; + } + return max - min; +} + +template +void compute_VTOT(const T *Vx, const T *Vy, const T *Vz, size_t n, T *V_TOT_) { + for (int i = 0; i < n; i++) { + double V_TOT_2 = Vx[i] * Vx[i] + Vy[i] * Vy[i] + Vz[i] * Vz[i]; + double V_TOT = sqrt(V_TOT_2); + V_TOT_[i] = V_TOT; + } +} + +int launch_reconstruct(std::string input_file, std::string output_file, + std::string original_file, enum mgard_x::data_type dtype, + std::vector shape, + std::vector tols, double s, + enum mgard_x::error_bound_type mode, + bool adaptive_resolution, + enum mgard_x::device_type dev_type, int verbose, + int rank, int decrease_method = 0) { + + double bitrate = 0; + mgard_x::Config config; + config.normalize_coordinates = false; + config.log_level = verbose_to_log_level(verbose); + config.dev_type = dev_type; + config.mdr_adaptive_resolution = adaptive_resolution; + + config.mdr_qoi_mode = true; + config.mdr_qoi_num_variables = shape.size(); + config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; + config.domain_decomposition_dim = 0; + config.domain_decomposition_sizes = {shape[0] / 3, shape[0] / 3, + shape[0] / 3}; + + mgard_x::Byte *original_data; + size_t in_size = 0; + size_t original_size = 1; + for (mgard_x::DIM i = 0; i < shape.size(); i++) + original_size *= shape[i]; + if (original_file.compare("none") != 0 && !config.mdr_adaptive_resolution) { + if (original_file.compare("random") == 0) { + if (dtype == mgard_x::data_type::Float) { + in_size = original_size * sizeof(float); + original_data = (mgard_x::Byte *)new float[original_size]; + srand(7117); + for (size_t i = 0; i < original_size; i++) { + ((float *)original_data)[i] = rand() % 10 + 1; + } + } else if (dtype == mgard_x::data_type::Double) { + in_size = original_size * sizeof(double); + original_data = (mgard_x::Byte *)new double[original_size]; + srand(7117); + for (size_t i = 0; i < original_size; i++) { + ((double *)original_data)[i] = rand() % 10 + 1; + } + } + } else { + mgard_x::Byte *file_data; + in_size = readfile(original_file, file_data); + + if (dtype == mgard_x::data_type::Float) { + original_size *= sizeof(float); + } else if (dtype == mgard_x::data_type::Double) { + original_size *= sizeof(double); + } + + original_data = (mgard_x::Byte *)malloc(original_size); + + size_t loaded_size = 0; + while (loaded_size < original_size) { + + std::memcpy(original_data + loaded_size, file_data, + std::min(in_size, original_size - loaded_size)); + loaded_size += std::min(in_size, original_size - loaded_size); + } + in_size = loaded_size; + } + } + mgard_x::Byte *V_TOT_ori; + std::vector ebs; + size_t num_elements; + double tau = 0; + V_TOT_ori = (mgard_x::Byte *)malloc(in_size / config.mdr_qoi_num_variables); + mgard_x::Byte *org_Vx_ptr = original_data + original_size / 3 * 0; + mgard_x::Byte *org_Vy_ptr = original_data + original_size / 3 * 1; + mgard_x::Byte *org_Vz_ptr = original_data + original_size / 3 * 2; + if (dtype == mgard_x::data_type::Float) { + num_elements = (in_size / config.mdr_qoi_num_variables) / sizeof(float); + compute_VTOT((float *)org_Vx_ptr, (float *)org_Vy_ptr, + (float *)org_Vz_ptr, num_elements, (float *)V_TOT_ori); + // tau = compute_value_range((float *) V_TOT_ori, num_elements) * tols[0]; + if (decrease_method < 4) { + ebs.push_back(compute_value_range((float *)org_Vx_ptr, num_elements) * + tols[0]); + ebs.push_back(compute_value_range((float *)org_Vy_ptr, num_elements) * + tols[0]); + ebs.push_back(compute_value_range((float *)org_Vz_ptr, num_elements) * + tols[0]); + } else { + std::vector var_value_range; + var_value_range.push_back( + compute_value_range((float *)org_Vx_ptr, num_elements) * tols[0]); + var_value_range.push_back( + compute_value_range((float *)org_Vy_ptr, num_elements) * tols[0]); + var_value_range.push_back( + compute_value_range((float *)org_Vz_ptr, num_elements) * tols[0]); + for (int i = 0; i < config.mdr_qoi_num_variables; i++) { + ebs.push_back( + *std::min_element(var_value_range.begin(), var_value_range.end())); + } + } + float local_max = -std::numeric_limits::max(); + float local_min = std::numeric_limits::max(); + float global_max = 0, global_min = 0; + float *V_TOT = (float *)V_TOT_ori; + for (int i = 0; i < num_elements; i++) { + if (V_TOT[i] > local_max) + local_max = V_TOT[i]; + if (V_TOT[i] < local_min) + local_min = V_TOT[i]; + } + // std::cout << "local_min = " << local_min << ", local_max = " << + // local_max << std::endl; + MPI_Allreduce(&local_min, &global_min, 1, MPI_FLOAT, MPI_MIN, + MPI_COMM_WORLD); + MPI_Allreduce(&local_max, &global_max, 1, MPI_FLOAT, MPI_MAX, + MPI_COMM_WORLD); + tau = (double)(global_max - global_min) * tols[0]; + } else if (dtype == mgard_x::data_type::Double) { + num_elements = (in_size / config.mdr_qoi_num_variables) / sizeof(double); + compute_VTOT((double *)org_Vx_ptr, (double *)org_Vy_ptr, + (double *)org_Vz_ptr, num_elements, + (double *)V_TOT_ori); + // tau = compute_value_range((double *) V_TOT_ori, num_elements) * tols[0]; + if (decrease_method < 4) { + ebs.push_back(compute_value_range((double *)org_Vx_ptr, num_elements) * + tols[0]); + ebs.push_back(compute_value_range((double *)org_Vy_ptr, num_elements) * + tols[0]); + ebs.push_back(compute_value_range((double *)org_Vz_ptr, num_elements) * + tols[0]); + } else { + std::vector var_value_range; + var_value_range.push_back( + compute_value_range((double *)org_Vx_ptr, num_elements) * tols[0]); + var_value_range.push_back( + compute_value_range((double *)org_Vy_ptr, num_elements) * tols[0]); + var_value_range.push_back( + compute_value_range((double *)org_Vz_ptr, num_elements) * tols[0]); + for (int i = 0; i < config.mdr_qoi_num_variables; i++) { + ebs.push_back( + *std::min_element(var_value_range.begin(), var_value_range.end())); + } + } + double local_min = -std::numeric_limits::max(); + double local_max = std::numeric_limits::max(); + double global_max = 0, global_min = 0; + double *V_TOT = (double *)V_TOT_ori; + for (int i = 0; i < num_elements; i++) { + if (V_TOT[i] > local_max) + local_max = V_TOT[i]; + if (V_TOT[i] < local_min) + local_min = V_TOT[i]; + } + MPI_Allreduce(&local_min, &global_min, 1, MPI_DOUBLE, MPI_MIN, + MPI_COMM_WORLD); + MPI_Allreduce(&local_max, &global_max, 1, MPI_DOUBLE, MPI_MAX, + MPI_COMM_WORLD); + tau = (double)(global_max - global_min) * tols[0]; + } + + mgard_x::MDR::RefactoredMetadata refactored_metadata; + mgard_x::MDR::RefactoredData refactored_data; + mgard_x::MDR::ReconstructedData reconstructed_data; + size_t metadata_size = + read_mdr_metadata(refactored_metadata, refactored_data, input_file); + refactored_metadata.total_size += metadata_size; + + refactored_metadata.relative_eb = tols[0]; + refactored_metadata.decrease_method = decrease_method; + refactored_metadata.MPI_enabled = true; + refactored_metadata.input_path = input_file; + for (int i = 0; i < config.mdr_qoi_num_variables; i++) { + refactored_metadata.metadata[i].num_elements = num_elements; + if (decrease_method == 0) { + refactored_metadata.metadata[i].requested_tol = ebs[i]; + } else if (decrease_method == 1) { + refactored_metadata.metadata[i].requested_size = 1; + refactored_metadata.metadata[i].segmented = true; + } else if (decrease_method >= 2) { + refactored_metadata.metadata[i].requested_tol = ebs[i]; + refactored_metadata.metadata[i].corresponding_error_return = true; + } + refactored_metadata.metadata[i].tau = tau; + refactored_metadata.metadata[i].requested_s = s; + } + mgard_x::MDR::MDRequest(refactored_metadata, config); + + // posterior_bp_request(refactored_metadata, tols[0], rank); + // double local_IO_time = 0, global_IO_time = 0; + // MPI_Barrier(MPI_COMM_WORLD); + // local_IO_time = -MPI_Wtime(); + size_t size_read = + read_mdr(refactored_metadata, refactored_data, input_file, true, config); + // MPI_Barrier(MPI_COMM_WORLD); + // local_IO_time += MPI_Wtime(); + // MPI_Reduce(&local_IO_time, &global_IO_time, 1, MPI_DOUBLE, MPI_MAX, 0, + // MPI_COMM_WORLD); if(!rank) std::cout << "IO_time = " << global_IO_time << + // std::endl; + + // refactored_metadata.total_size += size_read; + double local_elapsed_time = 0, max_time = 0; + local_elapsed_time = -MPI_Wtime(); + mgard_x::MDR::MDReconstruct(refactored_metadata, refactored_data, + reconstructed_data, config, false); + local_elapsed_time += MPI_Wtime(); + MPI_Reduce(&local_elapsed_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, + MPI_COMM_WORLD); + if (!rank) + std::cout << "max_elapsed_time = " << max_time << std::endl; + double local_kernel_time = 0, global_kernel_time = 0; + local_kernel_time = refactored_metadata.kernel_time; + MPI_Reduce(&local_kernel_time, &global_kernel_time, 1, MPI_DOUBLE, MPI_MAX, 0, + MPI_COMM_WORLD); + if (!rank) + std::cout << "max_kernel_time = " << global_kernel_time << std::endl; + for (int i = 0; i < 8; ++i) { + MPI_Barrier(MPI_COMM_WORLD); + if (rank == i) { + std::cout << "From Rank " << i << ": " + << "kernel_time = " << local_kernel_time << std::endl; + } + } + + // we can check reconstructed_data.qoi_in_progress here + + // std::cout << mgard_x::log::log_info << "Additional " << size_read + // << " bytes read for reconstruction\n"; + + std::vector rec_var_ptrs; + if (original_file.compare("none") != 0 && !config.mdr_adaptive_resolution) { + for (int i = 0; i < config.mdr_qoi_num_variables; i++) { + std::vector var_shape = shape; + var_shape[0] /= config.mdr_qoi_num_variables; + mgard_x::Byte *org_var_ptr = original_data + original_size / 3 * i; + mgard_x::Byte *rec_var_ptr = + reconstructed_data.data[0] + original_size / 3 * i; + rec_var_ptrs.push_back(rec_var_ptr); + } + } + mgard_x::Byte *V_TOT_rec; + V_TOT_rec = (mgard_x::Byte *)malloc(in_size / config.mdr_qoi_num_variables); + if (dtype == mgard_x::data_type::Float) { + compute_VTOT((float *)rec_var_ptrs[0], (float *)rec_var_ptrs[1], + (float *)rec_var_ptrs[2], num_elements, + (float *)V_TOT_rec); + } else if (dtype == mgard_x::data_type::Double) { + compute_VTOT((double *)rec_var_ptrs[0], (double *)rec_var_ptrs[1], + (double *)rec_var_ptrs[2], num_elements, + (double *)V_TOT_rec); + } + for (auto &metadata : refactored_metadata.metadata) { + refactored_metadata.total_size += metadata.GetLoadedBitPlaneSizes(); + } + + // Bitrate + unsigned long long int local_total_size = refactored_metadata.total_size; + unsigned long long int global_total_size = 0; + MPI_Reduce(&local_total_size, &global_total_size, 1, MPI_UNSIGNED_LONG_LONG, + MPI_SUM, 0, MPI_COMM_WORLD); + unsigned long long int local_in_size = in_size; + unsigned long long int global_in_size = 0; + MPI_Reduce(&local_in_size, &global_in_size, 1, MPI_UNSIGNED_LONG_LONG, + MPI_SUM, 0, MPI_COMM_WORLD); + if (dtype == mgard_x::data_type::Float) { + bitrate = 32 / ((double)global_in_size / global_total_size); + } else if (dtype == mgard_x::data_type::Double) { + bitrate = 64 / ((double)global_in_size / global_total_size); + } + if (!rank) + std::cout << "Bitrate = " << bitrate << std::endl; // MPI_REDUCE SUM + + // std::cout << "Original Vx[35345] = " << ((float*) org_Vx_ptr)[35345] << ", + // Reconstructed Vx[35345] = " << ((float*) rec_var_ptrs[0])[35345] << + // std::endl; + if (!rank) + std::cout << "Requested_Tau = " << tau << std::endl; + + // Max_est_error + double local_max_est_error = refactored_metadata.max_est_error; + double global_max_est_error = 0; + MPI_Reduce(&local_max_est_error, &global_max_est_error, 1, MPI_DOUBLE, + MPI_MAX, 0, MPI_COMM_WORLD); + if (!rank) + std::cout << "Est_max_error = " << global_max_est_error << std::endl; + + // Max_real_error + double local_real_max_error = (double)compute_max_abs_error( + (float *)V_TOT_ori, (float *)V_TOT_rec, num_elements); + double global_real_max_error = 0; + MPI_Reduce(&local_real_max_error, &global_real_max_error, 1, MPI_DOUBLE, + MPI_MAX, 0, MPI_COMM_WORLD); + if (!rank) + std::cout << "Real_max_error = " << global_real_max_error << std::endl; + // for (int i = 0; i < 8; ++i) { + // MPI_Barrier(MPI_COMM_WORLD); + // if (rank == i) { + // std::cout << "From Rank " << i << ": " << std::endl; + // int count = 0; + // for (auto &metadata : refactored_metadata.metadata){ + // if (count == 0) std::cout << "Vx:" << std::endl; + // else if (count == 1) std::cout << "Vy:" << std::endl; + // else if (count == 2) std::cout << "Vz:" << std::endl; + // for (size_t i = 0; i < metadata.loaded_level_num_bitplanes.size(); + // ++i) { + // std::cout << "Level " << i << ": bitplane = " + // << + // static_cast(metadata.loaded_level_num_bitplanes[i]) + // << std::endl; + // } + // count ++; + // } + // } + // } + + return 0; +} + +bool try_refactoring(int argc, char *argv[]) { + if (!has_arg(argc, argv, "-z", "--refactor")) + return false; + mgard_x::log::info("Mode: refactor", true); + + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + std::ostringstream oss; + oss << rank; + + std::string input_file = + get_arg(argc, argv, "Original data", "-i", "--input"); + input_file += oss.str() + ".dat"; + // std::cout << input_file << std::endl; + std::string output_file = + get_arg(argc, argv, "Refactored data", "-o", "--output"); + output_file += oss.str(); + // std::cout << output_file << std::endl; + enum mgard_x::data_type dtype = get_data_type(argc, argv); + std::vector shape = + get_args(argc, argv, "Dimensions", "-dim", "--dimension"); + // std::string lossless_level = get_arg(argc, argv, "Lossless", + // "-l", "--lossless"); + enum mgard_x::device_type dev_type = get_device_type(argc, argv); + int verbose = 0; + if (has_arg(argc, argv, "-v", "--verbose")) { + verbose = get_arg(argc, argv, "Verbose", "-v", "--verbose"); + } + mgard_x::SIZE max_memory_footprint = + std::numeric_limits::max(); + if (has_arg(argc, argv, "-m", "--max-memory")) { + max_memory_footprint = (mgard_x::SIZE)get_arg( + argc, argv, "Max memory", "-m", "--max-memory"); + } + std::string domain_decomposition = "max-dim"; + mgard_x::SIZE block_size = 0; + if (has_arg(argc, argv, "-dd", "--domain-decomposition")) { + domain_decomposition = get_arg( + argc, argv, "Domain decomposition", "-dd", "--domain-decomposition"); + if (domain_decomposition == "block") { + block_size = get_arg(argc, argv, "Block size", "-dd-size", + "--domain-decomposition-size"); + } + } + if (dtype == mgard_x::data_type::Double) { + launch_refactor(shape.size(), dtype, input_file.c_str(), + output_file.c_str(), shape, domain_decomposition, + block_size, dev_type, verbose, max_memory_footprint, + rank); + } else if (dtype == mgard_x::data_type::Float) { + launch_refactor(shape.size(), dtype, input_file.c_str(), + output_file.c_str(), shape, domain_decomposition, + block_size, dev_type, verbose, max_memory_footprint, + rank); + } + return true; +} + +bool try_reconstruction(int argc, char *argv[]) { + if (!has_arg(argc, argv, "-x", "--reconstruct")) + return false; + + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + std::ostringstream oss; + // oss << "JHTDB_" << rank; + oss << rank; + + mgard_x::log::info("mode: reconstruct", true); + std::string input_file = + get_arg(argc, argv, "Refactored data", "-i", "--input"); + // if (!input_file.empty() && input_file.back() == '/') input_file += + // oss.str(); else input_file += "/" + oss.str(); + input_file += oss.str(); + // std::cout << input_file << std::endl; + std::string output_file = + get_arg(argc, argv, "Reconstructed data", "-o", "--output"); + // default is none (means original data not provided) + std::string original_file = "none"; + enum mgard_x::data_type dtype; + std::vector shape; + if (has_arg(argc, argv, "-g", "--orignal")) { + original_file = + get_arg(argc, argv, "Original data", "-g", "--orignal"); + // if (!original_file.empty() && original_file.back() == '/') original_file + // += oss.str() + ".dat"; else original_file += "/" + oss.str() + ".dat"; + original_file += oss.str() + ".dat"; + // std::cout << original_file << std::endl; + dtype = get_data_type(argc, argv); + shape = get_args(argc, argv, "Dimensions", "-dim", + "--dimension"); + } + // only abs mode is supported now + enum mgard_x::error_bound_type mode = + mgard_x::error_bound_type::ABS; // REL or ABS + + std::vector tols; + if (has_arg(argc, argv, "-e", "--error-bound")) { + tols.push_back( + get_arg(argc, argv, "Error bound", "-e", "--error-bound")); + } else if (has_arg(argc, argv, "-me", "--multi-error-bounds")) { + tols = get_args(argc, argv, "Multi error bounds", "-me", + "--multi-error-bounds"); + } else { + throw std::runtime_error( + "Missing option -e/--error-bound or -me/--multi-error-bounds"); + } + double s = get_arg(argc, argv, "Smoothness", "-s", "--smoothness"); + enum mgard_x::device_type dev_type = get_device_type(argc, argv); + int verbose = 0; + if (has_arg(argc, argv, "-v", "--verbose")) { + verbose = get_arg(argc, argv, "Verbose", "-v", "--verbose"); + } + bool adaptive_resolution = false; + if (has_arg(argc, argv, "-ar", "--adaptive-resolution")) { + adaptive_resolution = get_arg(argc, argv, "Adaptive resolution", "-ar", + "--adaptive-resolution"); + } + if (verbose) + std::cout << mgard_x::log::log_info << "verbose: enabled.\n"; + int decrease_method; + if (has_arg(argc, argv, "-dm", "--decrease-method")) { + decrease_method = + get_arg(argc, argv, "Decrease method", "-dm", "--decrease-method"); + } + launch_reconstruct(input_file, output_file, original_file, dtype, shape, tols, + s, mode, adaptive_resolution, dev_type, verbose, rank, + decrease_method); + return true; +} + +int main(int argc, char *argv[]) { + + MPI_Init(&argc, &argv); + + if (!try_refactoring(argc, argv) && !try_reconstruction(argc, argv)) { + print_usage_message(""); + } + + MPI_Finalize(); + return 0; +} \ No newline at end of file diff --git a/src/mgard-x/MDRHighLevel/CMakeLists.txt b/src/mgard-x/MDRHighLevel/CMakeLists.txt index a85ef2a42c..aa11352658 100644 --- a/src/mgard-x/MDRHighLevel/CMakeLists.txt +++ b/src/mgard-x/MDRHighLevel/CMakeLists.txt @@ -1,4 +1,7 @@ -MgardXGenerateSourceAllDevices("MDRHighLevel") +MgardXGenerateSourceAllCombinations("MDRHighLevel") +# MgardXGenerateSourceAllCombinations("MDRefactor") +# MgardXGenerateSourceAllCombinations("MDRecontruct") +# MgardXGenerateSourceAllCombinations("MDRequest") set(MGARD_X_SERIAL_SRC ${MGARD_X_SERIAL_SRC} PARENT_SCOPE) set(MGARD_X_OPENMP_SRC ${MGARD_X_OPENMP_SRC} PARENT_SCOPE) set(MGARD_X_CUDA_SRC ${MGARD_X_CUDA_SRC} PARENT_SCOPE) diff --git a/src/mgard-x/MDRHighLevel/DynamicAPI.cpp b/src/mgard-x/MDRHighLevel/DynamicAPI.cpp index 121cc104a4..6b041ccdc0 100644 --- a/src/mgard-x/MDRHighLevel/DynamicAPI.cpp +++ b/src/mgard-x/MDRHighLevel/DynamicAPI.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include @@ -16,10 +16,288 @@ #include "mgard-x/RuntimeX/DataTypes.h" #include "mgard-x/Utilities/Types.h" +#include "mgard-x/Metadata/Metadata.hpp" + #include "mgard-x/MDRHighLevel/MDRHighLevel.h" namespace mgard_x { namespace MDR { + +template +void MDRefactor(DIM D, data_type dtype, std::vector shape, + const void *original_data, + RefactoredMetadata &refactored_metadata, + RefactoredData &refactored_data, Config config, + bool output_pre_allocated) { + if (dtype == data_type::Float) { + if (D == 1) { + MDRefactor<1, float, DeviceType>(shape, original_data, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else if (D == 2) { + MDRefactor<2, float, DeviceType>(shape, original_data, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else if (D == 3) { + MDRefactor<3, float, DeviceType>(shape, original_data, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else if (D == 4) { + MDRefactor<4, float, DeviceType>(shape, original_data, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else if (D == 5) { + MDRefactor<5, float, DeviceType>(shape, original_data, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else { + throw std::runtime_error("do not support higher than five dimentions"); + } + } else if (dtype == data_type::Double) { + if (D == 1) { + MDRefactor<1, double, DeviceType>(shape, original_data, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else if (D == 2) { + MDRefactor<2, double, DeviceType>(shape, original_data, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else if (D == 3) { + MDRefactor<3, double, DeviceType>(shape, original_data, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else if (D == 4) { + MDRefactor<4, double, DeviceType>(shape, original_data, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else if (D == 5) { + MDRefactor<5, double, DeviceType>(shape, original_data, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else { + throw std::runtime_error("do not support higher than five dimentions"); + } + } else { + throw std::runtime_error( + "do not support types other than double and float!"); + } +} + +template +void MDRefactor(DIM D, data_type dtype, std::vector shape, + const void *original_data, std::vector coords, + RefactoredMetadata &refactored_metadata, + RefactoredData &refactored_data, Config config, + bool output_pre_allocated) { + if (dtype == data_type::Float) { + std::vector float_coords; + for (auto &coord : coords) + float_coords.push_back((float *)coord); + if (D == 1) { + MDRefactor<1, float, DeviceType>(shape, original_data, float_coords, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else if (D == 2) { + MDRefactor<2, float, DeviceType>(shape, original_data, float_coords, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else if (D == 3) { + MDRefactor<3, float, DeviceType>(shape, original_data, float_coords, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else if (D == 4) { + MDRefactor<4, float, DeviceType>(shape, original_data, float_coords, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else if (D == 5) { + MDRefactor<5, float, DeviceType>(shape, original_data, float_coords, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else { + throw std::runtime_error("do not support higher than five dimentions"); + } + } else if (dtype == data_type::Double) { + std::vector double_coords; + for (auto &coord : coords) + double_coords.push_back((double *)coord); + if (D == 1) { + MDRefactor<1, double, DeviceType>(shape, original_data, double_coords, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else if (D == 2) { + MDRefactor<2, double, DeviceType>(shape, original_data, double_coords, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else if (D == 3) { + MDRefactor<3, double, DeviceType>(shape, original_data, double_coords, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else if (D == 4) { + MDRefactor<4, double, DeviceType>(shape, original_data, double_coords, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else if (D == 5) { + MDRefactor<5, double, DeviceType>(shape, original_data, double_coords, + refactored_metadata, refactored_data, + config, output_pre_allocated); + } else { + throw std::runtime_error("do not support higher than five dimentions"); + } + } else { + throw std::runtime_error( + "do not support types other than double and float!"); + } +} + +template +void MDRequest(RefactoredMetadata &refactored_metadata, Config config) { + Metadata meta; + meta.Deserialize((SERIALIZED_TYPE *)refactored_metadata.header.data()); + + std::vector shape = std::vector(meta.total_dims); + for (DIM d = 0; d < shape.size(); d++) + shape[d] = (SIZE)meta.shape[d]; + data_type dtype = meta.dtype; + + if (dtype == data_type::Float) { + if (shape.size() == 1) { + MDRequest<1, float, DeviceType>(shape, refactored_metadata, config); + } else if (shape.size() == 2) { + MDRequest<2, float, DeviceType>(shape, refactored_metadata, config); + } else if (shape.size() == 3) { + MDRequest<3, float, DeviceType>(shape, refactored_metadata, config); + } else if (shape.size() == 4) { + MDRequest<4, float, DeviceType>(shape, refactored_metadata, config); + } else if (shape.size() == 5) { + MDRequest<5, float, DeviceType>(shape, refactored_metadata, config); + } else { + throw std::runtime_error("do not support higher than five dimentions"); + } + } else if (dtype == data_type::Double) { + if (shape.size() == 1) { + MDRequest<1, double, DeviceType>(shape, refactored_metadata, config); + } else if (shape.size() == 2) { + MDRequest<2, double, DeviceType>(shape, refactored_metadata, config); + } else if (shape.size() == 3) { + MDRequest<3, double, DeviceType>(shape, refactored_metadata, config); + } else if (shape.size() == 4) { + MDRequest<4, double, DeviceType>(shape, refactored_metadata, config); + } else if (shape.size() == 5) { + MDRequest<5, double, DeviceType>(shape, refactored_metadata, config); + } else { + throw std::runtime_error("do not support higher than five dimentions"); + } + } else { + throw std::runtime_error( + "do not support types other than double and float!"); + } +} + +template +SIZE MDRMaxOutputDataSize(DIM D, data_type dtype, std::vector shape, + Config config) { + if (dtype == data_type::Float) { + if (shape.size() == 1) { + return MDRMaxOutputDataSize<1, float, DeviceType>(shape, config); + } else if (shape.size() == 2) { + return MDRMaxOutputDataSize<2, float, DeviceType>(shape, config); + } else if (shape.size() == 3) { + return MDRMaxOutputDataSize<3, float, DeviceType>(shape, config); + } else if (shape.size() == 4) { + return MDRMaxOutputDataSize<4, float, DeviceType>(shape, config); + } else if (shape.size() == 5) { + return MDRMaxOutputDataSize<5, float, DeviceType>(shape, config); + } else { + throw std::runtime_error("do not support higher than five dimentions"); + } + } else if (dtype == data_type::Double) { + if (shape.size() == 1) { + return MDRMaxOutputDataSize<1, double, DeviceType>(shape, config); + } else if (shape.size() == 2) { + return MDRMaxOutputDataSize<2, double, DeviceType>(shape, config); + } else if (shape.size() == 3) { + return MDRMaxOutputDataSize<3, double, DeviceType>(shape, config); + } else if (shape.size() == 4) { + return MDRMaxOutputDataSize<4, double, DeviceType>(shape, config); + } else if (shape.size() == 5) { + return MDRMaxOutputDataSize<5, double, DeviceType>(shape, config); + } else { + throw std::runtime_error("do not support higher than five dimentions"); + } + } else { + throw std::runtime_error( + "do not support types other than double and float!"); + } +} + +template +void MDReconstruct(RefactoredMetadata &refactored_metadata, + RefactoredData &refactored_data, + ReconstructedData &reconstructed_data, Config config, + bool output_pre_allocated) { + + Metadata meta; + meta.Deserialize((SERIALIZED_TYPE *)refactored_metadata.header.data()); + + std::vector shape = std::vector(meta.total_dims); + for (DIM d = 0; d < shape.size(); d++) + shape[d] = (SIZE)meta.shape[d]; + data_type dtype = meta.dtype; + + if (dtype == data_type::Float) { + if (shape.size() == 1) { + MDReconstruct<1, float, DeviceType>(shape, refactored_metadata, + refactored_data, reconstructed_data, + config, output_pre_allocated); + } else if (shape.size() == 2) { + MDReconstruct<2, float, DeviceType>(shape, refactored_metadata, + refactored_data, reconstructed_data, + config, output_pre_allocated); + } else if (shape.size() == 3) { + MDReconstruct<3, float, DeviceType>(shape, refactored_metadata, + refactored_data, reconstructed_data, + config, output_pre_allocated); + } else if (shape.size() == 4) { + MDReconstruct<4, float, DeviceType>(shape, refactored_metadata, + refactored_data, reconstructed_data, + config, output_pre_allocated); + } else if (shape.size() == 5) { + MDReconstruct<5, float, DeviceType>(shape, refactored_metadata, + refactored_data, reconstructed_data, + config, output_pre_allocated); + } else { + throw std::runtime_error("do not support higher than five dimentions"); + } + } else if (dtype == data_type::Double) { + if (shape.size() == 1) { + MDReconstruct<1, double, DeviceType>(shape, refactored_metadata, + refactored_data, reconstructed_data, + config, output_pre_allocated); + } else if (shape.size() == 2) { + MDReconstruct<2, double, DeviceType>(shape, refactored_metadata, + refactored_data, reconstructed_data, + config, output_pre_allocated); + } else if (shape.size() == 3) { + MDReconstruct<3, double, DeviceType>(shape, refactored_metadata, + refactored_data, reconstructed_data, + config, output_pre_allocated); + } else if (shape.size() == 4) { + MDReconstruct<4, double, DeviceType>(shape, refactored_metadata, + refactored_data, reconstructed_data, + config, output_pre_allocated); + } else if (shape.size() == 5) { + MDReconstruct<5, double, DeviceType>(shape, refactored_metadata, + refactored_data, reconstructed_data, + config, output_pre_allocated); + } else { + throw std::runtime_error("do not support higher than five dimentions"); + } + } else { + throw std::runtime_error( + "do not support types other than double and float!"); + } +} + enum device_type auto_detect_device() { enum device_type dev_type = device_type::NONE; #if MGARD_ENABLE_SERIAL @@ -44,8 +322,7 @@ enum device_type auto_detect_device() { } #endif if (dev_type == device_type::NONE) { - log::err("MDR-X was not built with any backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with any backend."); } return dev_type; } @@ -66,44 +343,38 @@ void MDRefactor(DIM D, data_type dtype, std::vector shape, MDRefactor(D, dtype, shape, original_data, refactored_metadata, refactored_data, config, output_pre_allocated); #else - log::err("MDR-X was not built with SERIAL backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with SERIAL backend."); #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP MDRefactor(D, dtype, shape, original_data, refactored_metadata, refactored_data, config, output_pre_allocated); #else - log::err("MDR-X was not built with OPENMP backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with OPENMP backend."); #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA MDRefactor(D, dtype, shape, original_data, refactored_metadata, refactored_data, config, output_pre_allocated); #else - log::err("MDR-X was not built with CUDA backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with CUDA backend."); #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP MDRefactor(D, dtype, shape, original_data, refactored_metadata, refactored_data, config, output_pre_allocated); #else - log::err("MDR-X was not built with HIP backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with HIP backend."); #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL MDRefactor(D, dtype, shape, original_data, refactored_metadata, refactored_data, config, output_pre_allocated); #else - log::err("MDR-X was not built with SYCL backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with SYCL backend."); #endif } else { - log::err("Unsupported backend."); - exit(-1); + throw std::runtime_error("Unsupported backend."); } } @@ -124,8 +395,7 @@ void MDRefactor(DIM D, data_type dtype, std::vector shape, refactored_metadata, refactored_data, config, output_pre_allocated); #else - log::err("MDR-X was not built with SERIAL backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with SERIAL backend."); #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP @@ -133,8 +403,7 @@ void MDRefactor(DIM D, data_type dtype, std::vector shape, refactored_metadata, refactored_data, config, output_pre_allocated); #else - log::err("MDR-X was not built with OPENMP backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with OPENMP backend."); #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA @@ -142,16 +411,14 @@ void MDRefactor(DIM D, data_type dtype, std::vector shape, refactored_metadata, refactored_data, config, output_pre_allocated); #else - log::err("MDR-X was not built with CUDA backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with CUDA backend."); #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP MDRefactor(D, dtype, shape, original_data, coords, refactored_metadata, refactored_data, config, output_pre_allocated); #else - log::err("MDR-X was not built with HIP backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with HIP backend."); #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL @@ -159,12 +426,10 @@ void MDRefactor(DIM D, data_type dtype, std::vector shape, refactored_metadata, refactored_data, config, output_pre_allocated); #else - log::err("MDR-X was not built with SYCL backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with SYCL backend."); #endif } else { - log::err("Unsupported backend."); - exit(-1); + throw std::runtime_error("Unsupported backend."); } } @@ -177,42 +442,36 @@ void MDRequest(RefactoredMetadata &refactored_metadata, Config config) { if (dev_type == device_type::SERIAL) { #if MGARD_ENABLE_SERIAL - MDRequest(refactored_metadata); + MDRequest(refactored_metadata, config); #else - log::err("MDR-X was not built with SERIAL backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with SERIAL backend."); #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP - MDRequest(refactored_metadata); + MDRequest(refactored_metadata, config); #else - log::err("MDR-X was not built with OPENMP backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with OPENMP backend."); #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA - MDRequest(refactored_metadata); + MDRequest(refactored_metadata, config); #else - log::err("MDR-X was not built with CUDA backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with CUDA backend."); #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP - MDRequest(refactored_metadata); + MDRequest(refactored_metadata, config); #else - log::err("MDR-X was not built with HIP backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with HIP backend."); #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL - MDRequest(refactored_metadata); + MDRequest(refactored_metadata, config); #else - log::err("MDR-X was not built with SYCL backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with SYCL backend."); #endif } else { - log::err("Unsupported backend."); - exit(-1); + throw std::runtime_error("Unsupported backend."); } } @@ -228,40 +487,34 @@ SIZE MDRMaxOutputDataSize(DIM D, data_type dtype, std::vector shape, #if MGARD_ENABLE_SERIAL return MDRMaxOutputDataSize(D, dtype, shape, config); #else - log::err("MDR-X was not built with SERIAL backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with SERIAL backend."); #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP return MDRMaxOutputDataSize(D, dtype, shape, config); #else - log::err("MDR-X was not built with OPENMP backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with OPENMP backend."); #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA return MDRMaxOutputDataSize(D, dtype, shape, config); #else - log::err("MDR-X was not built with CUDA backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with CUDA backend."); #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP return MDRMaxOutputDataSize(D, dtype, shape, config); #else - log::err("MDR-X was not built with HIP backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with HIP backend."); #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL return MDRMaxOutputDataSize(D, dtype, shape, config); #else - log::err("MDR-X was not built with SYCL backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with SYCL backend."); #endif } else { - log::err("Unsupported backend."); - exit(-1); + throw std::runtime_error("Unsupported backend."); } } @@ -280,47 +533,54 @@ void MDReconstruct(RefactoredMetadata &refactored_metadata, MDReconstruct(refactored_metadata, refactored_data, reconstructed_data, config, output_pre_allocated); #else - log::err("MDR-X was not built with SERIAL backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with SERIAL backend."); #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP MDReconstruct(refactored_metadata, refactored_data, reconstructed_data, config, output_pre_allocated); #else - log::err("MDR-X was not built with OPENMP backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with OPENMP backend."); #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA MDReconstruct(refactored_metadata, refactored_data, reconstructed_data, config, output_pre_allocated); #else - log::err("MDR-X was not built with CUDA backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with CUDA backend."); #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP MDReconstruct(refactored_metadata, refactored_data, reconstructed_data, config, output_pre_allocated); #else - log::err("MDR-X was not built with HIP backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with HIP backend."); #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL MDReconstruct(refactored_metadata, refactored_data, reconstructed_data, config, output_pre_allocated); #else - log::err("MDR-X was not built with SYCL backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with SYCL backend."); #endif } else { - log::err("Unsupported backend."); - exit(-1); + throw std::runtime_error("Unsupported backend."); } } +template void release_cache() { + release_cache<1, T, DeviceType>(); + release_cache<2, T, DeviceType>(); + release_cache<3, T, DeviceType>(); + release_cache<4, T, DeviceType>(); + release_cache<5, T, DeviceType>(); +} + +template void release_cache() { + release_cache(); + release_cache(); +} + void release_cache(Config config) { enum device_type dev_type = config.dev_type; @@ -332,40 +592,34 @@ void release_cache(Config config) { #if MGARD_ENABLE_SERIAL release_cache(); #else - log::err("MDR-X was not built with SERIAL backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with SERIAL backend."); #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP release_cache(); #else - log::err("MDR-X was not built with OPENMP backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with OPENMP backend."); #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA release_cache(); #else - log::err("MDR-X was not built with CUDA backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with CUDA backend."); #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP release_cache(); #else - log::err("MDR-X was not built with HIP backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with HIP backend."); #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL elease_cache(); #else - log::err("MDR-X was not built with SYCL backend."); - exit(-1); + throw std::runtime_error("MDR-X was not built with SYCL backend."); #endif } else { - log::err("Unsupported backend."); - exit(-1); + throw std::runtime_error("Unsupported backend."); } } diff --git a/src/mgard-x/MDRHighLevel/MDRHighLevel.cpp.in b/src/mgard-x/MDRHighLevel/MDRHighLevel.cpp.in index b2cf7fa571..2ab97d5061 100644 --- a/src/mgard-x/MDRHighLevel/MDRHighLevel.cpp.in +++ b/src/mgard-x/MDRHighLevel/MDRHighLevel.cpp.in @@ -10,26 +10,31 @@ namespace mgard_x { namespace MDR { -template void MDRefactor<@DEVICE_TYPE@>(DIM D, data_type dtype, std::vector shape, const void *original_data, - RefactoredMetadata &refactored_metadata, - RefactoredData &refactored_data, - Config config, bool output_pre_allocated); +template void MDRefactor<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>(std::vector shape, const void *original_data, + RefactoredMetadata &refactored_metadata, + RefactoredData &refactored_data, Config config, + bool output_pre_allocated); -template void MDRefactor<@DEVICE_TYPE@>(DIM D, data_type dtype, std::vector shape, const void *original_data, - std::vector coords, RefactoredMetadata &refactored_metadata, - RefactoredData &refactored_data, - Config config, bool output_pre_allocated); +template void MDRefactor<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>(std::vector shape, const void *original_data, + std::vector<@DATA_TYPE@ *> coords, + RefactoredMetadata &refactored_metadata, + RefactoredData &refactored_data, Config config, + bool output_pre_allocated); -template void MDRequest<@DEVICE_TYPE@>(RefactoredMetadata &refactored_metadata); +template void MDRequest<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>(std::vector shape, RefactoredMetadata &refactored_metadata, + Config config); -template SIZE MDRMaxOutputDataSize<@DEVICE_TYPE@>(DIM D, data_type dtype, std::vector shape, Config config); +template SIZE MDRMaxOutputDataSize<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>(std::vector shape, Config config); + +template void MDReconstruct<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>(std::vector shape, + RefactoredMetadata &refactored_metadata, + RefactoredData &refactored_data, + ReconstructedData &reconstructed_data, Config config, + bool output_pre_allocated); + +template void release_cache<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>(); -template void MDReconstruct<@DEVICE_TYPE@>(RefactoredMetadata &refactored_metadata, - RefactoredData &refactored_data, - ReconstructedData &reconstructed_data, Config config, - bool output_pre_allocated); -template void release_cache<@DEVICE_TYPE@>(); } } // namespace mgard_x // clang-format on \ No newline at end of file diff --git a/src/mgard-x/MDRHighLevel/MDRecontruct.cpp.in b/src/mgard-x/MDRHighLevel/MDRecontruct.cpp.in new file mode 100644 index 0000000000..f82d39fa3d --- /dev/null +++ b/src/mgard-x/MDRHighLevel/MDRecontruct.cpp.in @@ -0,0 +1,21 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x/MDRHighLevel/MDRHighLevel.hpp" +// clang-format off +namespace mgard_x { +namespace MDR { + +template void MDReconstruct<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>(std::vector shape, + RefactoredMetadata &refactored_metadata, + RefactoredData &refactored_data, + ReconstructedData &reconstructed_data, Config config, + bool output_pre_allocated); + +} +} // namespace mgard_x +// clang-format on \ No newline at end of file diff --git a/src/mgard-x/MDRHighLevel/MDRefactor.cpp.in b/src/mgard-x/MDRHighLevel/MDRefactor.cpp.in new file mode 100644 index 0000000000..ea513cfe33 --- /dev/null +++ b/src/mgard-x/MDRHighLevel/MDRefactor.cpp.in @@ -0,0 +1,26 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x/MDRHighLevel/MDRHighLevel.hpp" +// clang-format off +namespace mgard_x { +namespace MDR { + +template void MDRefactor<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>(std::vector shape, const void *original_data, + RefactoredMetadata &refactored_metadata, + RefactoredData &refactored_data, Config config, + bool output_pre_allocated); + +template void MDRefactor<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>(std::vector shape, const void *original_data, + std::vector<@DATA_TYPE@ *> coords, + RefactoredMetadata &refactored_metadata, + RefactoredData &refactored_data, Config config, + bool output_pre_allocated); + +} +} // namespace mgard_x +// clang-format on \ No newline at end of file diff --git a/src/mgard-x/MDRHighLevel/MDRequest.cpp.in b/src/mgard-x/MDRHighLevel/MDRequest.cpp.in new file mode 100644 index 0000000000..1feab639a7 --- /dev/null +++ b/src/mgard-x/MDRHighLevel/MDRequest.cpp.in @@ -0,0 +1,25 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x/MDRHighLevel/MDRHighLevel.hpp" +// clang-format off +namespace mgard_x { +namespace MDR { + + +template void MDRequest<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>(std::vector shape, RefactoredMetadata &refactored_metadata, + Config config); + +template SIZE MDRMaxOutputDataSize<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>(std::vector shape, Config config); + + +template void release_cache<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>(); + + +} +} // namespace mgard_x +// clang-format on \ No newline at end of file diff --git a/src/mgard-x/Metadata/Metadata.cpp b/src/mgard-x/Metadata/Metadata.cpp index 4ffaae3de0..05f0afcfbf 100644 --- a/src/mgard-x/Metadata/Metadata.cpp +++ b/src/mgard-x/Metadata/Metadata.cpp @@ -6,7 +6,8 @@ */ #include "mgard-x/Metadata/Metadata.hpp" -#include "format.hpp" +#include "MGARDConfig.hpp" +#include "mgard-x/RuntimeX/Utilities/Exceptions.h" #include "proto/mgard.pb.h" #include @@ -24,6 +25,41 @@ template bool big_endian() { return not *reinterpret_cast(&n); } +// The width of the quantized-coefficient type is a build-time choice +// (QUANTIZED_INT in DataTypes.h) that the reader must agree with, so it is +// recorded in the header rather than assumed. Writing it means the width can +// be changed later without turning existing files into garbage: a mismatch is +// then a clean error instead of a misparse. +mgard::pb::Quantization::Type QuantizationTypeForWidth(std::size_t width) { + switch (width) { + case 1: + return mgard::pb::Quantization::INT8_T; + case 2: + return mgard::pb::Quantization::INT16_T; + case 4: + return mgard::pb::Quantization::INT32_T; + case 8: + return mgard::pb::Quantization::INT64_T; + default: + throw mgard_x::InvalidDataException("unsupported quantized integer width."); + } +} + +std::size_t WidthForQuantizationType(mgard::pb::Quantization::Type type) { + switch (type) { + case mgard::pb::Quantization::INT8_T: + return 1; + case mgard::pb::Quantization::INT16_T: + return 2; + case mgard::pb::Quantization::INT32_T: + return 4; + case mgard::pb::Quantization::INT64_T: + return 8; + default: + throw mgard_x::InvalidDataException("unrecognized quantization type."); + } +} + uint32_t ComputeCRC32(const std::vector &data, std::size_t start = 0) { // `crc32_z` takes a `z_size_t`. @@ -111,8 +147,7 @@ void DeserializeSignature(SerializedIter &iter) { char c_; Deserialize(c_, iter); if (c_ != c) { - std::cout << mgard_x::log::log_err << "signature mismatch.\n"; - exit(-1); + throw mgard_x::InvalidDataException("signature mismatch."); } } } @@ -122,17 +157,115 @@ void SerializeBytes(const std::vector &data, vec.insert(vec.end(), data.begin(), data.end()); } +// Write `tolerances` into `roi` using whichever of the two encodings is +// smaller. ROI maps are piecewise constant in practice (a few distinct +// tolerances over long runs of blocks), where run-length is dramatically +// smaller; a fully irregular map is bigger under run-length, so both are +// built and the smaller one wins. Both decode to the same vector. +void SerializeROITolerances(const std::vector &tolerances, + mgard::pb::RegionOfInterest &roi) { + mgard::pb::PackedTolerances packed; + google::protobuf::RepeatedField &packed_values = + *packed.mutable_tolerances(); + packed_values.Reserve(tolerances.size()); + for (double t : tolerances) { + packed_values.Add(t); + } + + mgard::pb::RunLengthTolerances run_length; + google::protobuf::RepeatedField &rl_values = + *run_length.mutable_values(); + google::protobuf::RepeatedField &rl_lengths = + *run_length.mutable_run_lengths(); + for (std::size_t i = 0; i < tolerances.size();) { + std::size_t j = i; + // Bit-exact comparison: the decoded tolerances must reproduce the + // compressor's quantization steps exactly, so runs may only merge values + // that are identical, not merely close. + while (j < tolerances.size() && tolerances[j] == tolerances[i]) { + j++; + } + rl_values.Add(tolerances[i]); + rl_lengths.Add(static_cast(j - i)); + i = j; + } + + if (run_length.ByteSizeLong() < packed.ByteSizeLong()) { + *roi.mutable_run_length() = run_length; + } else { + *roi.mutable_packed() = packed; + } +} + +// Inverse of SerializeROITolerances. +std::vector +DeserializeROITolerances(const mgard::pb::RegionOfInterest &roi) { + std::vector tolerances; + if (roi.has_packed()) { + const google::protobuf::RepeatedField &values = + roi.packed().tolerances(); + tolerances.assign(values.begin(), values.end()); + } else if (roi.has_run_length()) { + const mgard::pb::RunLengthTolerances &run_length = roi.run_length(); + if (run_length.values_size() != run_length.run_lengths_size()) { + throw mgard_x::InvalidDataException( + "ROI tolerance map has mismatched run values and run lengths."); + } + std::size_t total = 0; + for (int i = 0; i < run_length.run_lengths_size(); i++) { + total += run_length.run_lengths(i); + } + tolerances.reserve(total); + for (int i = 0; i < run_length.values_size(); i++) { + tolerances.insert(tolerances.end(), run_length.run_lengths(i), + run_length.values(i)); + } + } else { + throw mgard_x::InvalidDataException( + "ROI tolerance map is present but carries no tolerance encoding."); + } + return tolerances; +} + } // anonymous namespace namespace mgard_x { void MetadataBase::InitializeConfig(Config &config) { config.domain_decomposition = ddtype; + config.domain_decomposition_dim = domain_decomposed_dim; config.decomposition = decomposition; config.lossless = ltype; - config.huff_dict_size = huff_dict_size; - config.huff_block_size = huff_block_size; - config.reorder = reorder; + // Only overwrite a backend's parameters when the metadata actually carries + // them (non-zero). A file compressed with a non-Huffman backend leaves the + // Huffman fields at 0; copying those zeros would make + // HuffmanWorkspace::resize divide by a zero block size. Leaving config + // defaults intact avoids that (the unused backend's workspace is harmlessly + // sized with defaults). + if (huff_dict_size != 0) { + config.huff_dict_size = huff_dict_size; + } + if (huff_block_size != 0) { + config.huff_block_size = huff_block_size; + } + if (block_delta_block_size != 0) { + config.block_delta_block_size = block_delta_block_size; + } + // The hybrid parameters are only meaningful for a hybrid file, and + // Deserialize refuses to produce a hybrid file without them, so a Hybrid + // decomposition here always carries a full set. For non-hybrid files leave + // the caller's Config alone -- those fields are unused and overwriting them + // with zeros would break a subsequent hybrid compression through the same + // Config. + if (decomposition == decomposition_type::Hybrid) { + config.num_local_refactoring_level = (int)hybrid_num_local_levels; + config.num_global_refactoring_level = (int)hybrid_num_global_levels; + config.projection_mode = hybrid_projection_mode; + config.enable_roi = hybrid_enable_roi; + if (hybrid_enable_roi) { + config.roi_tolerance_map = hybrid_roi_tolerance_map; + } + } } void MetadataBase::PrintSummary() { @@ -175,8 +308,33 @@ void MetadataBase::PrintSummary() { std::cout << "MultiDim\n"; } else if (decomposition == decomposition_type::SingleDim) { std::cout << "SingleDim\n"; + } else if (decomposition == decomposition_type::Hybrid) { + std::cout << "Hybrid\n"; + std::cout << "Local refactoring levels: " << hybrid_num_local_levels + << "\n"; + std::cout << "Global refactoring levels: " << hybrid_num_global_levels + << "\n"; + std::cout << "Local block size: " << hybrid_local_block_size << "\n"; + std::cout << "Projection: " + << (hybrid_projection_mode == + compression_projection_mode_type::Hierarchical + ? "hierarchical" + : "orthogonal") + << "\n"; + std::cout << "ROI: "; + if (hybrid_enable_roi) { + std::cout << "enabled (" << hybrid_roi_tolerance_map.size() + << " level-0 blocks"; + if (!hybrid_roi_block_dimensions.empty()) { + std::cout << ", grid"; + for (uint64_t n : hybrid_roi_block_dimensions) + std::cout << " " << n; + } + std::cout << ")\n"; + } else { + std::cout << "disabled\n"; + } } - std::cout << "Reorder: " << reorder << "\n"; std::cout << "Domain Decomposition: "; if (domain_decomposed) { if (ddtype == domain_decomposition_type::MaxDim) { @@ -216,12 +374,17 @@ void MetadataBase::PrintSummary() { std::cout << "Huffman_LZ4\n"; std::cout << "Huffman dictionary size: " << huff_dict_size << "\n"; std::cout << "Huffman block size: " << huff_block_size << "\n"; + } else if (ltype == mgard_x::lossless_type::LZ4) { + std::cout << "LZ4\n"; } else if (ltype == mgard_x::lossless_type::Huffman_Zstd) { std::cout << "Huffman_Zstd\n"; std::cout << "Huffman dictionary size: " << huff_dict_size << "\n"; std::cout << "Huffman block size: " << huff_block_size << "\n"; } else if (ltype == mgard_x::lossless_type::CPU_Lossless) { std::cout << "CPU_Lossless\n"; + } else if (ltype == mgard_x::lossless_type::BlockDelta) { + std::cout << "BlockDelta\n"; + std::cout << "BlockDelta block size: " << block_delta_block_size << "\n"; } std::cout << "Backend: "; @@ -367,7 +530,40 @@ std::vector MetadataBase::Serialize() { } else if (decomposition == decomposition_type::Hybrid) { function_decomposition.set_hierarchy( mgard::pb::FunctionDecomposition::HYBRID_HIERARCHY); + // The hybrid hierarchy cannot be reconstructed from the hierarchy enum + // alone: the local/global level counts fix the layout of the decomposed + // buffer and the ROI map fixes the per-block quantization step. Without + // them the decompressor would have to be handed the same parameters out + // of band, and silently produce garbage when it was not. + mgard::pb::HybridHierarchy &hybrid = + *function_decomposition.mutable_hybrid_hierarchy(); + hybrid.set_num_local_levels(hybrid_num_local_levels); + hybrid.set_num_global_levels(hybrid_num_global_levels); + hybrid.set_local_block_size(hybrid_local_block_size); + if (hybrid_projection_mode == + compression_projection_mode_type::Hierarchical) { + hybrid.set_projection(mgard::pb::HybridHierarchy::HIERARCHICAL); + } else if (hybrid_projection_mode == + compression_projection_mode_type::Orthogonal) { + hybrid.set_projection(mgard::pb::HybridHierarchy::ORTHOGONAL); + } else { + throw InvalidDataException( + "cannot serialize an invalid hybrid projection mode."); + } + if (hybrid_enable_roi) { + mgard::pb::RegionOfInterest &roi = *hybrid.mutable_region_of_interest(); + google::protobuf::RepeatedField + &block_dimensions = *roi.mutable_block_dimensions(); + block_dimensions.Reserve(hybrid_roi_block_dimensions.size()); + for (uint64_t n : hybrid_roi_block_dimensions) { + block_dimensions.Add(n); + } + SerializeROITolerances(hybrid_roi_tolerance_map, roi); + } } + // Not populated by MGARD-X: the reader rebuilds the hierarchy (and hence + // the level count) from the shape, and under domain decomposition there is + // no single value to record. Written for format compatibility only. function_decomposition.set_l_target(l_target); } @@ -376,9 +572,10 @@ std::vector MetadataBase::Serialize() { if (otype == operation_type::Compression) { quantization.set_method(mgard::pb::Quantization::COEFFICIENTWISE_LINEAR); quantization.set_bin_widths(mgard::pb::Quantization::PER_COEFFICIENT); - quantization.set_type(mgard::pb::Quantization::INT64_T); - quantization.set_big_endian(big_endian()); - if (big_endian()) { + quantization.set_type( + ::QuantizationTypeForWidth(sizeof(mgard_x::QUANTIZED_INT))); + quantization.set_big_endian(big_endian()); + if (big_endian()) { etype = endiness_type::Big_Endian; } else { etype = endiness_type::Little_Endian; @@ -405,11 +602,7 @@ std::vector MetadataBase::Serialize() { { // Encoding mgard::pb::Encoding &encoding = *header.mutable_encoding(); - if (reorder == 0) { - encoding.set_preprocessor(mgard::pb::Encoding::NOOP_PREPROCESSOR); - } else { - encoding.set_preprocessor(mgard::pb::Encoding::SHUFFLE); - } + encoding.set_preprocessor(mgard::pb::Encoding::NOOP_PREPROCESSOR); if (ltype == mgard_x::lossless_type::Huffman) { encoding.set_compressor(mgard::pb::Encoding::X_HUFFMAN); encoding.set_huffman_dictionary_size(huff_dict_size); @@ -418,12 +611,22 @@ std::vector MetadataBase::Serialize() { encoding.set_compressor(mgard::pb::Encoding::X_HUFFMAN_LZ4); encoding.set_huffman_dictionary_size(huff_dict_size); encoding.set_huffman_block_size(huff_block_size); + } else if (ltype == mgard_x::lossless_type::LZ4) { + encoding.set_compressor(mgard::pb::Encoding::X_LZ4); } else if (ltype == mgard_x::lossless_type::Huffman_Zstd) { encoding.set_compressor(mgard::pb::Encoding::X_HUFFMAN_ZSTD); encoding.set_huffman_dictionary_size(huff_dict_size); encoding.set_huffman_block_size(huff_block_size); } else if (ltype == mgard_x::lossless_type::CPU_Lossless) { encoding.set_compressor(mgard::pb::Encoding::CPU_HUFFMAN_ZSTD); + } else if (ltype == mgard_x::lossless_type::BlockDelta) { + encoding.set_compressor(mgard::pb::Encoding::X_BLOCK_DELTA); + encoding.set_block_delta_block_size(block_delta_block_size); + } else if (ltype == mgard_x::lossless_type::SymbolRans) { + encoding.set_compressor(mgard::pb::Encoding::X_SYMBOL_RANS); + encoding.set_huffman_dictionary_size(huff_dict_size); + } else if (ltype == mgard_x::lossless_type::ZeroRLE_Rans) { + encoding.set_compressor(mgard::pb::Encoding::X_ZERORLE_RANS); } } @@ -486,8 +689,7 @@ void MetadataBase::Deserialize( uint64_t offset = std::distance(serialized_data.begin(), iter); if (header_crc32 != ComputeCRC32(serialized_data, offset)) { - std::cout << log::log_err << "header CRC32 mismatch.\n"; - exit(-1); + throw InvalidDataException("header CRC32 mismatch."); } mgard::pb::Header header = DeserializeProtoBuf(serialized_data, offset); @@ -499,8 +701,7 @@ void MetadataBase::Deserialize( software_version[1] = mgard_version_number.minor_(); software_version[2] = mgard_version_number.patch_(); if (software_version[0] > MGARD_VERSION_MAJOR) { - std::cout << log::log_err << "MGARD version mismatch.\n"; - exit(-1); + throw InvalidDataException("MGARD version mismatch."); } const mgard::pb::VersionNumber format_version_number = @@ -509,8 +710,7 @@ void MetadataBase::Deserialize( file_version[1] = format_version_number.minor_(); file_version[2] = format_version_number.patch_(); if (file_version[0] > MGARD_FILE_VERSION_MAJOR) { - std::cout << log::log_err << "MGARD file format version mismatch.\n"; - exit(-1); + throw InvalidDataException("MGARD file format version mismatch."); } } @@ -522,9 +722,7 @@ void MetadataBase::Deserialize( const google::protobuf::RepeatedField shape_ = cartesian_grid_topology.shape(); if (total_dims != shape_.size()) { - std::cout << log::log_err - << "grid shape does not match given dimension.\n"; - exit(-1); + throw InvalidDataException("grid shape does not match given dimension."); } shape = std::vector(total_dims); std::copy(shape_.begin(), shape_.end(), shape.begin()); @@ -542,10 +740,8 @@ void MetadataBase::Deserialize( for (DIM d = 0; d < total_dims; d++) totel_len += shape[d]; if (totel_len != coordinates.size()) { - std::cout << log::log_err - << "mismatch between number of node coordinates and grid " - "shape.\n"; - exit(-1); + throw InvalidDataException( + "mismatch between number of node coordinates and grid shape."); } using It = google::protobuf::RepeatedField::const_iterator; It p = coordinates.begin(); @@ -628,11 +824,76 @@ void MetadataBase::Deserialize( } else if (function_decomposition.hierarchy() == mgard::pb::FunctionDecomposition::HYBRID_HIERARCHY) { decomposition = decomposition_type::Hybrid; + if (!function_decomposition.has_hybrid_hierarchy()) { + // Written by a build from before the hybrid parameters were added to + // the header. Its level counts and ROI map only ever existed in the + // caller's Config, so we cannot reconstruct it here. Failing loudly is + // the point: silently falling back to the Config defaults is what + // produced wrong output without any error. + throw InvalidDataException( + "this file uses the hybrid hierarchy but predates the hybrid " + "parameters being recorded in the header; it can only be " + "decompressed by supplying the original local/global level counts " + "and ROI tolerance map through Config."); + } + const mgard::pb::HybridHierarchy &hybrid = + function_decomposition.hybrid_hierarchy(); + hybrid_num_local_levels = hybrid.num_local_levels(); + hybrid_num_global_levels = hybrid.num_global_levels(); + hybrid_local_block_size = hybrid.local_block_size(); + switch (hybrid.projection()) { + case mgard::pb::HybridHierarchy::HIERARCHICAL: + hybrid_projection_mode = compression_projection_mode_type::Hierarchical; + break; + case mgard::pb::HybridHierarchy::PROJECTION_UNSPECIFIED: + case mgard::pb::HybridHierarchy::ORTHOGONAL: + // A missing field denotes a legacy BlockMGARD stream, all of which + // used the orthogonal basis. + hybrid_projection_mode = compression_projection_mode_type::Orthogonal; + break; + default: + throw InvalidDataException( + "hybrid hierarchy records an unknown projection mode."); + } + if (hybrid_num_local_levels == 0 && hybrid_num_global_levels == 0) { + throw InvalidDataException( + "hybrid hierarchy records zero local and zero global levels."); + } + if (hybrid_local_block_size != MGARDX_HYBRID_LOCAL_BLOCK_SIZE) { + throw InvalidDataException( + "hybrid hierarchy was written with local block size " + + std::to_string(hybrid_local_block_size) + + ", but this build of MGARD-X only implements block size " + + std::to_string(MGARDX_HYBRID_LOCAL_BLOCK_SIZE) + "."); + } + hybrid_enable_roi = hybrid.has_region_of_interest(); + if (hybrid_enable_roi) { + const mgard::pb::RegionOfInterest &roi = hybrid.region_of_interest(); + const google::protobuf::RepeatedField + &block_dimensions = roi.block_dimensions(); + hybrid_roi_block_dimensions.assign(block_dimensions.begin(), + block_dimensions.end()); + hybrid_roi_tolerance_map = ::DeserializeROITolerances(roi); + if (hybrid_roi_block_dimensions.size() != total_dims) { + throw InvalidDataException( + "ROI block grid has a different dimension than the data."); + } + std::size_t expected = 1; + for (uint64_t n : hybrid_roi_block_dimensions) { + expected *= n; + } + if (hybrid_roi_tolerance_map.size() != expected) { + throw InvalidDataException( + "ROI tolerance map holds " + + std::to_string(hybrid_roi_tolerance_map.size()) + + " entries but its block grid describes " + + std::to_string(expected) + "."); + } + } } else { - std::cout << log::log_err - << "this decomposition hierarchy mismatch the hierarchy used " - "in MGARD-X.\n"; - exit(-1); + throw InvalidDataException( + "this decomposition hierarchy mismatch the hierarchy used " + "in MGARD-X."); } l_target = function_decomposition.l_target(); } @@ -642,9 +903,22 @@ void MetadataBase::Deserialize( if (quantization.method() != mgard::pb::Quantization::NOOP_QUANTIZATION) { assert(quantization.bin_widths() == mgard::pb::Quantization::PER_COEFFICIENT); - assert(quantization.type() == mgard::pb::Quantization::INT64_T); - assert(quantization.big_endian() == big_endian()); - if (big_endian()) { + // Checked rather than asserted: this decides how the payload is read, so + // a mismatch must fail in release builds too. It fires when a file + // written by a build with a different QUANTIZED_INT width is read. + const std::size_t file_width = + ::WidthForQuantizationType(quantization.type()); + if (file_width != sizeof(mgard_x::QUANTIZED_INT)) { + throw InvalidDataException( + "this file quantizes to " + std::to_string(file_width * 8) + + "-bit integers, but this build of MGARD-X uses " + + std::to_string(sizeof(mgard_x::QUANTIZED_INT) * 8) + "-bit."); + } + if (quantization.big_endian() != big_endian()) { + throw InvalidDataException( + "this file was written with the opposite endianness."); + } + if (big_endian()) { etype = endiness_type::Big_Endian; } else { etype = endiness_type::Little_Endian; @@ -672,10 +946,8 @@ void MetadataBase::Deserialize( quantization.method() == mgard::pb::Quantization::NOOP_QUANTIZATION && bitplane_encoding.method() == mgard::pb::BitplaneEncoding::NOOP_BITPLANE_ENCODING) { - std::cout << log::log_err - << "cannot determine whether this is compressed or " - "refactored data.\n"; - exit(-1); + throw InvalidDataException("cannot determine whether this is compressed " + "or refactored data."); } } @@ -691,11 +963,6 @@ void MetadataBase::Deserialize( { // Encoding const mgard::pb::Encoding encoding = header.encoding(); - if (encoding.preprocessor() == mgard::pb::Encoding::SHUFFLE) { - reorder = 1; - } else { - reorder = 0; - } if (encoding.compressor() == mgard::pb::Encoding::X_HUFFMAN) { ltype = mgard_x::lossless_type::Huffman; huff_dict_size = encoding.huffman_dictionary_size(); @@ -704,15 +971,24 @@ void MetadataBase::Deserialize( ltype = mgard_x::lossless_type::Huffman_LZ4; huff_dict_size = encoding.huffman_dictionary_size(); huff_block_size = encoding.huffman_block_size(); + } else if (encoding.compressor() == mgard::pb::Encoding::X_LZ4) { + ltype = mgard_x::lossless_type::LZ4; } else if (encoding.compressor() == mgard::pb::Encoding::X_HUFFMAN_ZSTD) { ltype = mgard_x::lossless_type::Huffman_Zstd; huff_dict_size = encoding.huffman_dictionary_size(); huff_block_size = encoding.huffman_block_size(); } else if (encoding.compressor() == mgard::pb::Encoding::CPU_HUFFMAN_ZSTD) { ltype = mgard_x::lossless_type::CPU_Lossless; + } else if (encoding.compressor() == mgard::pb::Encoding::X_BLOCK_DELTA) { + ltype = mgard_x::lossless_type::BlockDelta; + block_delta_block_size = encoding.block_delta_block_size(); + } else if (encoding.compressor() == mgard::pb::Encoding::X_SYMBOL_RANS) { + ltype = mgard_x::lossless_type::SymbolRans; + huff_dict_size = encoding.huffman_dictionary_size(); + } else if (encoding.compressor() == mgard::pb::Encoding::X_ZERORLE_RANS) { + ltype = mgard_x::lossless_type::ZeroRLE_Rans; } else { - std::cout << log::log_err << "unknown lossless compressor type.\n"; - exit(-1); + throw InvalidDataException("unknown lossless compressor type."); } } @@ -729,9 +1005,7 @@ void MetadataBase::Deserialize( } else if (device.backend() == mgard::pb::Device::X_SYCL) { ptype = processor_type::X_SYCL; } else if (device.backend() == mgard::pb::Device::CPU) { - std::cout << log::log_err - << "this data was not compressed with MGARD-X.\n"; - exit(-1); + throw InvalidDataException("this data was not compressed with MGARD-X."); } } } diff --git a/src/mgard-x/RuntimeX/AutoTuners/AutoTunerCuda.cu b/src/mgard-x/RuntimeX/AutoTuners/AutoTunerCuda.cu index 0a600469f5..b593370178 100644 --- a/src/mgard-x/RuntimeX/AutoTuners/AutoTunerCuda.cu +++ b/src/mgard-x/RuntimeX/AutoTuners/AutoTunerCuda.cu @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "mgard-x/RuntimeX/RuntimeX.h" namespace mgard_x { diff --git a/src/mgard-x/RuntimeX/AutoTuners/AutoTunerHip.cpp b/src/mgard-x/RuntimeX/AutoTuners/AutoTunerHip.cpp index 8535e5fe7e..dbb37c9d6e 100644 --- a/src/mgard-x/RuntimeX/AutoTuners/AutoTunerHip.cpp +++ b/src/mgard-x/RuntimeX/AutoTuners/AutoTunerHip.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "mgard-x/RuntimeX/RuntimeX.h" namespace mgard_x { diff --git a/src/mgard-x/RuntimeX/AutoTuners/AutoTunerOpenmp.cpp b/src/mgard-x/RuntimeX/AutoTuners/AutoTunerOpenmp.cpp index 45c487aa8f..2988714e1b 100644 --- a/src/mgard-x/RuntimeX/AutoTuners/AutoTunerOpenmp.cpp +++ b/src/mgard-x/RuntimeX/AutoTuners/AutoTunerOpenmp.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "mgard-x/RuntimeX/RuntimeX.h" namespace mgard_x { diff --git a/src/mgard-x/RuntimeX/AutoTuners/AutoTunerSerial.cpp b/src/mgard-x/RuntimeX/AutoTuners/AutoTunerSerial.cpp index 2926954736..fbca23163a 100644 --- a/src/mgard-x/RuntimeX/AutoTuners/AutoTunerSerial.cpp +++ b/src/mgard-x/RuntimeX/AutoTuners/AutoTunerSerial.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "mgard-x/RuntimeX/RuntimeX.h" namespace mgard_x { diff --git a/src/mgard-x/RuntimeX/AutoTuners/AutoTunerSycl.cpp b/src/mgard-x/RuntimeX/AutoTuners/AutoTunerSycl.cpp index 31630a9aeb..e26ab8be49 100644 --- a/src/mgard-x/RuntimeX/AutoTuners/AutoTunerSycl.cpp +++ b/src/mgard-x/RuntimeX/AutoTuners/AutoTunerSycl.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "mgard-x/RuntimeX/RuntimeX.h" namespace mgard_x { diff --git a/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterCuda.cu b/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterCuda.cu index 2639bc5b29..99d2a9a71e 100644 --- a/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterCuda.cu +++ b/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterCuda.cu @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "mgard-x/RuntimeX/RuntimeX.h" @@ -14,7 +14,6 @@ DeviceQueues DeviceRuntime::queues; DeviceSpecification DeviceRuntime::DeviceSpecs; bool DeviceRuntime::SyncAllKernelsAndCheckErrors = false; -bool MemoryManager::ReduceMemoryFootprint = false; bool DeviceRuntime::TimingAllKernels = false; bool DeviceRuntime::PrintKernelConfig = false; diff --git a/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterHip.cpp b/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterHip.cpp index 79b0c4ab31..5875188370 100644 --- a/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterHip.cpp +++ b/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterHip.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "mgard-x/RuntimeX/RuntimeX.h" @@ -14,7 +14,6 @@ DeviceQueues DeviceRuntime::queues; DeviceSpecification DeviceRuntime::DeviceSpecs; bool DeviceRuntime::SyncAllKernelsAndCheckErrors = false; -bool MemoryManager::ReduceMemoryFootprint = false; bool DeviceRuntime::TimingAllKernels = false; bool DeviceRuntime::PrintKernelConfig = false; diff --git a/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.cpp b/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.cpp index 56f35dccbc..4f0cd953ca 100644 --- a/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.cpp +++ b/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "mgard-x/RuntimeX/RuntimeX.h" diff --git a/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.cpp b/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.cpp index 3afd774f9a..163ebc417d 100644 --- a/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.cpp +++ b/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "mgard-x/RuntimeX/RuntimeX.h" @@ -14,7 +14,6 @@ DeviceQueues DeviceRuntime::queues; DeviceSpecification DeviceRuntime::DeviceSpecs; bool DeviceRuntime::SyncAllKernelsAndCheckErrors = false; -bool MemoryManager::ReduceMemoryFootprint = false; bool DeviceRuntime::TimingAllKernels = false; bool DeviceRuntime::PrintKernelConfig = false; diff --git a/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSerial.cpp b/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSerial.cpp index a7f37dd2dc..2c93983f50 100644 --- a/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSerial.cpp +++ b/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSerial.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "mgard-x/RuntimeX/RuntimeX.h" @@ -14,7 +14,6 @@ DeviceQueues DeviceRuntime::queues; DeviceSpecification DeviceRuntime::DeviceSpecs; bool DeviceRuntime::SyncAllKernelsAndCheckErrors = false; -bool MemoryManager::ReduceMemoryFootprint = false; bool DeviceRuntime::TimingAllKernels = false; bool DeviceRuntime::PrintKernelConfig = false; diff --git a/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSycl.cpp b/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSycl.cpp index 3ebde3098a..86f61df8d1 100644 --- a/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSycl.cpp +++ b/src/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSycl.cpp @@ -1,8 +1,8 @@ /* - * Copyright 2022, Oak Ridge National Laboratory. + * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs - * Author: Jieyang Chen (chenj3@ornl.gov) - * Date: March 17, 2022 + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 */ #include "mgard-x/RuntimeX/RuntimeX.h" @@ -15,7 +15,6 @@ DeviceSpecification DeviceRuntime::DeviceSpecs; // SyncAllKernelsAndCheckErrors needs to be always ON for SYCL bool DeviceRuntime::SyncAllKernelsAndCheckErrors = true; -bool MemoryManager::ReduceMemoryFootprint = false; bool DeviceRuntime::TimingAllKernels = false; bool DeviceRuntime::PrintKernelConfig = false; diff --git a/src/mgard.proto b/src/mgard.proto index f2f25efdbe..0a20ab8069 100644 --- a/src/mgard.proto +++ b/src/mgard.proto @@ -78,6 +78,63 @@ message DomainDecomposition { uint64 decomposition_size = 3; } +// Per-block error tolerances for the region-of-interest (ROI) mode of the +// hybrid hierarchy. The tolerances are the level-0 block tolerances the +// compressor was given; the decompressor re-derives the deeper levels from +// them exactly as the compressor did, so only level 0 is stored. +// +// Block order is the local quantizer's: row-major over +// ceil(padded_shape[d] / local_block_size) for d = 0..dimension-1, with +// padded_shape[d] the shape rounded up to a multiple of local_block_size. +message RegionOfInterest { + // Number of blocks along each dimension at level 0. Stored so the reader can + // reject a tolerance map that does not match the grid it is applied to. + repeated uint64 block_dimensions = 1; + + // ROI maps are piecewise constant in practice (a handful of distinct + // tolerances over large runs of blocks), so the run-length form is usually + // far smaller. The writer emits whichever encoding serializes smaller; both + // decode to the same block_count-length tolerance vector. + oneof encoding { + PackedTolerances packed = 2; + RunLengthTolerances run_length = 3; + } +} + +message PackedTolerances { repeated double tolerances = 1; } + +message RunLengthTolerances { + repeated double values = 1; + repeated uint64 run_lengths = 2; +} + +// Parameters of the hybrid (block-local + global) hierarchy, a.k.a. BlockMGARD. +// Required to reconstruct: the number of local/global levels determines the +// layout of the decomposed buffer and which (de)quantizers run, and the ROI +// tolerance map determines the per-block quantization step. +message HybridHierarchy { + enum Projection { + // Streams written before this field existed used the orthogonal basis. + PROJECTION_UNSPECIFIED = 0; + ORTHOGONAL = 1; + HIERARCHICAL = 2; + } + // Number of block-local refactoring levels (Config:: + // num_local_refactoring_level, "L"). + uint64 num_local_levels = 1; + // Number of global refactoring levels (Config:: + // num_global_refactoring_level, "M"). At least one of L and M is non-zero. + uint64 num_global_levels = 2; + // Edge length of the block-local transform block. Currently a compile-time + // constant (8), recorded so a future change stays readable. + uint64 local_block_size = 3; + // Absent when ROI mode was disabled. + RegionOfInterest region_of_interest = 4; + // Resolved transform basis. This must be persisted because two L-infinity + // streams may be produced with different user-selected projection modes. + Projection projection = 5; +} + message FunctionDecomposition { enum Transform { MULTILEVEL_COEFFICIENTS = 0; } enum Hierarchy { @@ -91,6 +148,8 @@ message FunctionDecomposition { Hierarchy hierarchy = 2; // Currently only relevant when `hierarchy == GHOST_NODES`. uint64 L_target = 3; + // Required when `hierarchy == HYBRID_HIERARCHY`, absent otherwise. + HybridHierarchy hybrid_hierarchy = 4; } message Quantization { @@ -140,8 +199,13 @@ message Encoding { CPU_HUFFMAN_ZLIB = 1; CPU_HUFFMAN_ZSTD = 2; X_HUFFMAN = 3; - X_HUFFMAN_LZ4 = 4; + X_HUFFMAN_LZ4 = 4; // Huffman + the portable (nvcomp-free) LZ4 backend X_HUFFMAN_ZSTD = 5; + X_BLOCK_DELTA = 6; + reserved 7; // was X_HUFFMAN_LZ4_PORTABLE (folded into X_HUFFMAN_LZ4) + X_LZ4 = 8; // portable LZ4 directly on the quantized stream + X_SYMBOL_RANS = 9; // symbol-alphabet rANS (outlier separation + rANS) + X_ZERORLE_RANS = 10; // zero-RLE then byte-alphabet rANS } Preprocessor preprocessor = 1; @@ -150,6 +214,8 @@ message Encoding { // X_HUFFMAN_LZ4` or `compressor == X_HUFFMAN_ZSTD` uint64 huffman_dictionary_size = 3; uint64 huffman_block_size = 4; + // Only relevant when `compressor == X_BLOCK_DELTA` + uint64 block_delta_block_size = 5; } message Device { diff --git a/src/mgard/CMakeLists.txt b/src/mgard/CMakeLists.txt new file mode 100644 index 0000000000..447db80dd2 --- /dev/null +++ b/src/mgard/CMakeLists.txt @@ -0,0 +1,6 @@ +target_sources(mgard-library PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/compress.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/compress_internal.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/compressors.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/format.cpp" +) diff --git a/src/cli/CMakeLists.txt b/src/mgard/cli/CMakeLists.txt similarity index 100% rename from src/cli/CMakeLists.txt rename to src/mgard/cli/CMakeLists.txt diff --git a/src/cli/arguments.cpp b/src/mgard/cli/arguments.cpp similarity index 97% rename from src/cli/arguments.cpp rename to src/mgard/cli/arguments.cpp index 5f99640a6b..383958ab3c 100644 --- a/src/cli/arguments.cpp +++ b/src/mgard/cli/arguments.cpp @@ -1,4 +1,4 @@ -#include "cli/arguments.hpp" +#include "mgard/cli/arguments.hpp" #include diff --git a/src/cli/cli_internal.cpp b/src/mgard/cli/cli_internal.cpp similarity index 95% rename from src/cli/cli_internal.cpp rename to src/mgard/cli/cli_internal.cpp index 4b36ab158d..63bed009bf 100644 --- a/src/cli/cli_internal.cpp +++ b/src/mgard/cli/cli_internal.cpp @@ -1,9 +1,9 @@ -#include "cli/cli_internal.hpp" +#include "mgard/cli/cli_internal.hpp" #include #include "compress.hpp" -#include "utilities.hpp" +#include "mgard/utilities.hpp" namespace cli { diff --git a/src/cli/cmdline.cpp b/src/mgard/cli/cmdline.cpp similarity index 98% rename from src/cli/cmdline.cpp rename to src/mgard/cli/cmdline.cpp index d7b0680b3c..63d22d7dc1 100644 --- a/src/cli/cmdline.cpp +++ b/src/mgard/cli/cmdline.cpp @@ -1,4 +1,4 @@ -#include "cli/cmdline.hpp" +#include "mgard/cli/cmdline.hpp" #include @@ -6,7 +6,7 @@ #include -#include "cli/output.hpp" +#include "mgard/cli/output.hpp" namespace cli { diff --git a/src/cli/executable.cpp b/src/mgard/cli/executable.cpp similarity index 96% rename from src/cli/executable.cpp rename to src/mgard/cli/executable.cpp index 7991419335..2c6d77d5bd 100644 --- a/src/cli/executable.cpp +++ b/src/mgard/cli/executable.cpp @@ -6,9 +6,9 @@ #include "MGARDConfig.hpp" -#include "cli/arguments.hpp" -#include "cli/cli_internal.hpp" -#include "cli/cmdline.hpp" +#include "mgard/cli/arguments.hpp" +#include "mgard/cli/cli_internal.hpp" +#include "mgard/cli/cmdline.hpp" int main(const int argc, char const *const *const argv) { cli::SubCmdLine subcompress(MGARD_VERSION_STR, diff --git a/src/cli/output.cpp b/src/mgard/cli/output.cpp similarity index 99% rename from src/cli/output.cpp rename to src/mgard/cli/output.cpp index 63ed0b9aeb..5be4e685ea 100644 --- a/src/cli/output.cpp +++ b/src/mgard/cli/output.cpp @@ -1,4 +1,4 @@ -#include "cli/output.hpp" +#include "mgard/cli/output.hpp" #include diff --git a/src/compress.cpp b/src/mgard/compress.cpp similarity index 90% rename from src/compress.cpp rename to src/mgard/compress.cpp index d711b5d25e..43ef2d184a 100644 --- a/src/compress.cpp +++ b/src/mgard/compress.cpp @@ -2,8 +2,8 @@ #include -#include "compress_internal.hpp" -#include "format.hpp" +#include "mgard/compress_internal.hpp" +#include "mgard/format.hpp" namespace mgard { diff --git a/src/compress_internal.cpp b/src/mgard/compress_internal.cpp similarity index 92% rename from src/compress_internal.cpp rename to src/mgard/compress_internal.cpp index 3b614a53d6..8bcbfa074b 100644 --- a/src/compress_internal.cpp +++ b/src/mgard/compress_internal.cpp @@ -1,4 +1,4 @@ -#include "compress_internal.hpp" +#include "mgard/compress_internal.hpp" namespace mgard { diff --git a/src/compressors.cpp b/src/mgard/compressors.cpp similarity index 99% rename from src/compressors.cpp rename to src/mgard/compressors.cpp index 915912f7f9..c1443cc3ed 100644 --- a/src/compressors.cpp +++ b/src/mgard/compressors.cpp @@ -1,4 +1,4 @@ -#include "compressors.hpp" +#include "mgard/compressors.hpp" #include #include @@ -13,7 +13,7 @@ #include -#include "format.hpp" +#include "mgard/format.hpp" #ifdef MGARD_TIMING #include diff --git a/src/format.cpp b/src/mgard/format.cpp similarity index 99% rename from src/format.cpp rename to src/mgard/format.cpp index ea16c508a0..e5164adfee 100644 --- a/src/format.cpp +++ b/src/mgard/format.cpp @@ -1,4 +1,4 @@ -#include "format.hpp" +#include "mgard/format.hpp" #include #include @@ -16,7 +16,7 @@ #ifdef __NVCC__ // `utilities.hpp` wasn't included in the header. -#include "utilities.hpp" +#include "mgard/utilities.hpp" #endif namespace mgard { diff --git a/src/interpolation.tpp b/src/mgard/interpolation.tpp similarity index 100% rename from src/interpolation.tpp rename to src/mgard/interpolation.tpp diff --git a/src/unstructured/CMakeLists.txt b/src/mgard/unstructured/CMakeLists.txt similarity index 100% rename from src/unstructured/CMakeLists.txt rename to src/mgard/unstructured/CMakeLists.txt diff --git a/src/unstructured/EnumeratedMeshRange.cpp b/src/mgard/unstructured/EnumeratedMeshRange.cpp similarity index 74% rename from src/unstructured/EnumeratedMeshRange.cpp rename to src/mgard/unstructured/EnumeratedMeshRange.cpp index 54fd74bd9a..7ee4dbbd14 100644 --- a/src/unstructured/EnumeratedMeshRange.cpp +++ b/src/mgard/unstructured/EnumeratedMeshRange.cpp @@ -1,4 +1,4 @@ -#include "unstructured/EnumeratedMeshRange.hpp" +#include "mgard/unstructured/EnumeratedMeshRange.hpp" namespace mgard { diff --git a/src/unstructured/IndicatorInput.cpp b/src/mgard/unstructured/IndicatorInput.cpp similarity index 98% rename from src/unstructured/IndicatorInput.cpp rename to src/mgard/unstructured/IndicatorInput.cpp index e12c6383c2..bb2e22d987 100644 --- a/src/unstructured/IndicatorInput.cpp +++ b/src/mgard/unstructured/IndicatorInput.cpp @@ -1,4 +1,4 @@ -#include "unstructured/IndicatorInput.hpp" +#include "mgard/unstructured/IndicatorInput.hpp" namespace mgard { diff --git a/src/unstructured/LinearOperator.cpp b/src/mgard/unstructured/LinearOperator.cpp similarity index 92% rename from src/unstructured/LinearOperator.cpp rename to src/mgard/unstructured/LinearOperator.cpp index 05386dac8e..fab8b0cd29 100644 --- a/src/unstructured/LinearOperator.cpp +++ b/src/mgard/unstructured/LinearOperator.cpp @@ -1,4 +1,4 @@ -#include "unstructured/LinearOperator.hpp" +#include "mgard/unstructured/LinearOperator.hpp" namespace mgard { diff --git a/src/unstructured/MassMatrix.cpp b/src/mgard/unstructured/MassMatrix.cpp similarity index 97% rename from src/unstructured/MassMatrix.cpp rename to src/mgard/unstructured/MassMatrix.cpp index 7b3fe70812..77208fcaa6 100644 --- a/src/unstructured/MassMatrix.cpp +++ b/src/mgard/unstructured/MassMatrix.cpp @@ -1,9 +1,9 @@ -#include "unstructured/MassMatrix.hpp" +#include "mgard/unstructured/MassMatrix.hpp" #include #include -#include "utilities.hpp" +#include "mgard/utilities.hpp" namespace mgard { diff --git a/src/unstructured/MeshHierarchy.cpp b/src/mgard/unstructured/MeshHierarchy.cpp similarity index 99% rename from src/unstructured/MeshHierarchy.cpp rename to src/mgard/unstructured/MeshHierarchy.cpp index 0b1b426d45..d9b2da4e40 100644 --- a/src/unstructured/MeshHierarchy.cpp +++ b/src/mgard/unstructured/MeshHierarchy.cpp @@ -1,4 +1,4 @@ -#include "unstructured/MeshHierarchy.hpp" +#include "mgard/unstructured/MeshHierarchy.hpp" #include @@ -6,8 +6,8 @@ #include #include -#include "unstructured/MassMatrix.hpp" -#include "unstructured/pcg.hpp" +#include "mgard/unstructured/MassMatrix.hpp" +#include "mgard/unstructured/pcg.hpp" namespace mgard { diff --git a/src/unstructured/MeshLevel.cpp b/src/mgard/unstructured/MeshLevel.cpp similarity index 99% rename from src/unstructured/MeshLevel.cpp rename to src/mgard/unstructured/MeshLevel.cpp index 7781cdceb3..5fc79a190b 100644 --- a/src/unstructured/MeshLevel.cpp +++ b/src/mgard/unstructured/MeshLevel.cpp @@ -1,4 +1,4 @@ -#include "unstructured/MeshLevel.hpp" +#include "mgard/unstructured/MeshLevel.hpp" #include @@ -8,7 +8,7 @@ #include "moab/Interface.hpp" -#include "unstructured/measure.hpp" +#include "mgard/unstructured/measure.hpp" typedef double (*EntityMeasureFunction)(double const *const); diff --git a/src/unstructured/MeshRefiner.cpp b/src/mgard/unstructured/MeshRefiner.cpp similarity index 75% rename from src/unstructured/MeshRefiner.cpp rename to src/mgard/unstructured/MeshRefiner.cpp index 652d5e0a2a..844024187f 100644 --- a/src/unstructured/MeshRefiner.cpp +++ b/src/mgard/unstructured/MeshRefiner.cpp @@ -1,4 +1,4 @@ -#include "unstructured/MeshRefiner.hpp" +#include "mgard/unstructured/MeshRefiner.hpp" namespace mgard { diff --git a/src/unstructured/UniformMeshHierarchy.cpp b/src/mgard/unstructured/UniformMeshHierarchy.cpp similarity index 98% rename from src/unstructured/UniformMeshHierarchy.cpp rename to src/mgard/unstructured/UniformMeshHierarchy.cpp index d12853fc19..17ba30fb09 100644 --- a/src/unstructured/UniformMeshHierarchy.cpp +++ b/src/mgard/unstructured/UniformMeshHierarchy.cpp @@ -1,4 +1,4 @@ -#include "unstructured/UniformMeshHierarchy.hpp" +#include "mgard/unstructured/UniformMeshHierarchy.hpp" #include #include @@ -9,9 +9,9 @@ #include #include -#include "utilities.hpp" +#include "mgard/utilities.hpp" -#include "unstructured/UniformMeshRefiner.hpp" +#include "mgard/unstructured/UniformMeshRefiner.hpp" namespace mgard { diff --git a/src/unstructured/UniformMeshRefiner.cpp b/src/mgard/unstructured/UniformMeshRefiner.cpp similarity index 99% rename from src/unstructured/UniformMeshRefiner.cpp rename to src/mgard/unstructured/UniformMeshRefiner.cpp index d4fac91870..c2ae887e97 100644 --- a/src/unstructured/UniformMeshRefiner.cpp +++ b/src/mgard/unstructured/UniformMeshRefiner.cpp @@ -1,4 +1,4 @@ -#include "unstructured/UniformMeshRefiner.hpp" +#include "mgard/unstructured/UniformMeshRefiner.hpp" #include #include @@ -9,9 +9,9 @@ #include #include -#include "blas.hpp" +#include "mgard/blas.hpp" -#include "utilities.hpp" +#include "mgard/utilities.hpp" //! Find the node of a triangle not included in an edge. //! diff --git a/src/unstructured/UniformRestriction.cpp b/src/mgard/unstructured/UniformRestriction.cpp similarity index 90% rename from src/unstructured/UniformRestriction.cpp rename to src/mgard/unstructured/UniformRestriction.cpp index 73cce6fccc..820a215a32 100644 --- a/src/unstructured/UniformRestriction.cpp +++ b/src/mgard/unstructured/UniformRestriction.cpp @@ -1,6 +1,6 @@ -#include "unstructured/UniformRestriction.hpp" +#include "mgard/unstructured/UniformRestriction.hpp" -#include "unstructured/UniformEdgeFamilies.hpp" +#include "mgard/unstructured/UniformEdgeFamilies.hpp" namespace mgard { diff --git a/src/unstructured/estimators.cpp b/src/mgard/unstructured/estimators.cpp similarity index 96% rename from src/unstructured/estimators.cpp rename to src/mgard/unstructured/estimators.cpp index 2ef7eb4b78..0d1863fc13 100644 --- a/src/unstructured/estimators.cpp +++ b/src/mgard/unstructured/estimators.cpp @@ -1,13 +1,13 @@ -#include "unstructured/estimators.hpp" +#include "mgard/unstructured/estimators.hpp" #include #include #include -#include "blas.hpp" +#include "mgard/blas.hpp" -#include "unstructured/MassMatrix.hpp" +#include "mgard/unstructured/MassMatrix.hpp" namespace mgard { diff --git a/src/unstructured/indicators.cpp b/src/mgard/unstructured/indicators.cpp similarity index 91% rename from src/unstructured/indicators.cpp rename to src/mgard/unstructured/indicators.cpp index f2e86014ae..7db5664dc2 100644 --- a/src/unstructured/indicators.cpp +++ b/src/mgard/unstructured/indicators.cpp @@ -1,4 +1,4 @@ -#include "unstructured/indicators.hpp" +#include "mgard/unstructured/indicators.hpp" #include diff --git a/src/unstructured/measure.cpp b/src/mgard/unstructured/measure.cpp similarity index 97% rename from src/unstructured/measure.cpp rename to src/mgard/unstructured/measure.cpp index af3fb2d77a..917fb56e57 100644 --- a/src/unstructured/measure.cpp +++ b/src/mgard/unstructured/measure.cpp @@ -1,6 +1,6 @@ -#include "unstructured/measure.hpp" +#include "mgard/unstructured/measure.hpp" -#include "blas.hpp" +#include "mgard/blas.hpp" #include #include diff --git a/src/unstructured/norms.cpp b/src/mgard/unstructured/norms.cpp similarity index 95% rename from src/unstructured/norms.cpp rename to src/mgard/unstructured/norms.cpp index 797ff958e5..a1d0cb9af6 100644 --- a/src/unstructured/norms.cpp +++ b/src/mgard/unstructured/norms.cpp @@ -1,4 +1,4 @@ -#include "unstructured/norms.hpp" +#include "mgard/unstructured/norms.hpp" #include #include @@ -8,13 +8,13 @@ #include #include -#include "blas.hpp" +#include "mgard/blas.hpp" -#include "utilities.hpp" +#include "mgard/utilities.hpp" -#include "unstructured/MassMatrix.hpp" -#include "unstructured/UniformRestriction.hpp" -#include "unstructured/pcg.hpp" +#include "mgard/unstructured/MassMatrix.hpp" +#include "mgard/unstructured/UniformRestriction.hpp" +#include "mgard/unstructured/pcg.hpp" namespace mgard { diff --git a/src/unstructured/pcg.cpp b/src/mgard/unstructured/pcg.cpp similarity index 98% rename from src/unstructured/pcg.cpp rename to src/mgard/unstructured/pcg.cpp index 6de7a5b09d..59c9f6d5ae 100644 --- a/src/unstructured/pcg.cpp +++ b/src/mgard/unstructured/pcg.cpp @@ -1,4 +1,4 @@ -#include "unstructured/pcg.hpp" +#include "mgard/unstructured/pcg.hpp" #include @@ -6,7 +6,7 @@ #include #include -#include "blas.hpp" +#include "mgard/blas.hpp" //! Calculate the residual 'from scratch.' //! diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1e67174fac..e61ceef207 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,63 +1,10 @@ -set( - MGARD_TESTS_CPP - "src/testing_utilities.cpp" - "src/test_utilities.cpp" - "src/test_blas.cpp" - "src/test_shuffle.cpp" - "src/test_TensorMeshHierarchy.cpp" - "src/test_TensorMeshHierarchyIteration.cpp" - "src/test_TensorLinearOperator.cpp" - "src/test_TensorMassMatrix.cpp" - "src/test_TensorProlongation.cpp" - "src/test_TensorRestriction.cpp" - "src/test_LinearQuantizer.cpp" - "src/test_TensorMultilevelCoefficientQuantizer.cpp" - "src/test_TensorNorms.cpp" - "src/test_TensorQuantityOfInterest.cpp" - "src/test_compress.cpp" - "src/test_decompose.cpp" - "src/test_format.cpp" - "src/test_quantize.cpp" - "src/test_compressors.cpp" - "src/test_CompressedDataset.cpp" -) - -if(MGARD_ENABLE_UNSTRUCTURED AND MOAB_FOUND) - list( - APPEND - MGARD_TESTS_CPP - "src/unstructured/test_estimators.cpp" - "src/unstructured/test_indicators.cpp" - "src/unstructured/test_norms.cpp" - "src/unstructured/test_data.cpp" - "src/unstructured/test_pcg.cpp" - "src/unstructured/test_measure.cpp" - "src/unstructured/test_MeshLevel.cpp" - "src/unstructured/test_MeshHierarchy.cpp" - "src/unstructured/test_MeshRefiner.cpp" - "src/unstructured/test_LinearOperator.cpp" - "src/unstructured/test_MassMatrix.cpp" - "src/unstructured/test_UniformMeshHierarchy.cpp" - "src/unstructured/test_UniformMeshRefiner.cpp" - "src/unstructured/test_UniformEdgeFamilies.cpp" - "src/unstructured/test_UniformRestriction.cpp" - "src/unstructured/test_EnumeratedMeshRange.cpp" - "src/unstructured/test_SituatedCoefficientRange.cpp" - "src/unstructured/test_IndicatorInput.cpp" - "src/unstructured/test_MultilevelCoefficientQuantizer.cpp" - ) -endif() - find_package(Catch2 3.0.0) if(Catch2_FOUND) - configure_file("include/testing_paths.hpp.in" "include/testing_paths.hpp") - add_executable(mgard-tests ${MGARD_TESTS_CPP}) - target_include_directories(mgard-tests PRIVATE "include" "${CMAKE_CURRENT_BINARY_DIR}/include") - target_link_libraries(mgard-tests PRIVATE mgard-library Catch2::Catch2WithMain) - set_target_properties(mgard-tests PROPERTIES OUTPUT_NAME tests) -else() - message(STATUS "Will not build tests (Catch2 not found).") + include(Catch) endif() +add_subdirectory(mgard) +add_subdirectory(mgard-x) + set(Catch2_FOUND "${Catch2_FOUND}" PARENT_SCOPE) diff --git a/tests/mgard-x/CMakeLists.txt b/tests/mgard-x/CMakeLists.txt new file mode 100644 index 0000000000..7cd53ea12d --- /dev/null +++ b/tests/mgard-x/CMakeLists.txt @@ -0,0 +1,153 @@ +set(_MGARD_X_TEST_DIR "${CMAKE_CURRENT_LIST_DIR}") + +if(NOT Catch2_FOUND) + message(STATUS "Will not build mgard-x tests (Catch2 not found).") + return() +endif() + +if(NOT DEVICE_TYPE_LIST) + message(STATUS "Will not build mgard-x tests (no device backend enabled).") + return() +endif() + +# Generate one test source file from a .cpp.in template for the given device type, +# set its compile language, and append it to MGARD_X_TEST_SOURCES. +macro(mgard_x_add_test SRC_NAME DEVICE_TYPE) + set(_src "${_MGARD_X_TEST_DIR}/src/${SRC_NAME}.cpp.in") + set(_gen "${CMAKE_CURRENT_BINARY_DIR}/src/${SRC_NAME}_${DEVICE_TYPE}.cpp") + configure_file(${_src} ${_gen}) + if(${DEVICE_TYPE} STREQUAL "CUDA") + set_source_files_properties(${_gen} PROPERTIES LANGUAGE CUDA) + elseif(${DEVICE_TYPE} STREQUAL "HIP") + set_source_files_properties(${_gen} PROPERTIES LANGUAGE HIP) + else() + set_source_files_properties(${_gen} PROPERTIES LANGUAGE CXX) + endif() + list(APPEND MGARD_X_TEST_SOURCES ${_gen}) +endmacro() + +# BPEncoderRegisterBlock's progressive_decode() unconditionally macro-unrolls +# 64 kernel variants per (T_data, T_bitplane, NegaBinary, ControlL2) +# configuration it is instantiated with. Templating across all of them in one +# translation unit (as test_Compressor etc. do for their own axes) overwhelms +# the CUDA/HIP device compiler, so each configuration is generated into its +# own small file instead -- the same reason the library itself generates +# e.g. Compressor_1D_double_CUDA.cpp per type/dim combination rather than +# templating across them. +set(BP_ENCODER_LIST BPEncoderLocalityBlock BPEncoderRegisterBlock) +set(BP_DATA_TYPE_LIST float double) +set(BP_BITPLANE_TYPE_LIST uint32_t uint64_t) +set(BP_BOOL_LIST true false) + +macro(mgard_x_add_bitplane_encoder_tests DEVICE_TYPE) + foreach(ENCODER_NAME IN LISTS BP_ENCODER_LIST) + # BPEncoderLocalityBlock has no batch-alignment requirement on n (its + # encode/decode loops guard every element access with `< n`), so it is + # deliberately tested with a size that is not a multiple of either + # bitplane word's batch size (32 or 64). BPEncoderRegisterBlock requires n + # to be a multiple of its batch size; 4096 is a multiple of both. + if(ENCODER_NAME STREQUAL "BPEncoderLocalityBlock") + set(BP_TEST_N 4001) + else() + set(BP_TEST_N 4096) + endif() + foreach(T_DATA IN LISTS BP_DATA_TYPE_LIST) + foreach(T_BITPLANE IN LISTS BP_BITPLANE_TYPE_LIST) + foreach(NEGA_BINARY IN LISTS BP_BOOL_LIST) + foreach(COLLECT_ERROR IN LISTS BP_BOOL_LIST) + set(_src "${_MGARD_X_TEST_DIR}/src/test_BitplaneEncoderConfig.cpp.in") + set(_tag "${ENCODER_NAME}_${T_DATA}_${T_BITPLANE}_nega${NEGA_BINARY}_l2${COLLECT_ERROR}") + set(_gen "${CMAKE_CURRENT_BINARY_DIR}/src/test_BitplaneEncoder_${_tag}_${DEVICE_TYPE}.cpp") + configure_file(${_src} ${_gen}) + if(${DEVICE_TYPE} STREQUAL "CUDA") + set_source_files_properties(${_gen} PROPERTIES LANGUAGE CUDA) + elseif(${DEVICE_TYPE} STREQUAL "HIP") + set_source_files_properties(${_gen} PROPERTIES LANGUAGE HIP) + else() + set_source_files_properties(${_gen} PROPERTIES LANGUAGE CXX) + endif() + list(APPEND MGARD_X_TEST_SOURCES ${_gen}) + endforeach() + endforeach() + endforeach() + endforeach() + endforeach() + mgard_x_add_test(test_BitplaneEncoderRegisterBlockThrows ${DEVICE_TYPE}) +endmacro() + +# Same reasoning as mgard_x_add_bitplane_encoder_tests: ComposedRefactor/ +# Reconstructor pull in BPEncoderRegisterBlock's 64-way macro-unrolled decode +# kernels too, so each (D, T_data, Basis, ControlL2, NegaBinary) configuration +# gets its own file. Hierarchical + ControlL2 is skipped: L2 error tracking +# needs an orthogonal basis to be meaningful. +set(CR_D_LIST 1 2 3) +set(CR_T_LIST float double) +set(CR_BASIS_LIST Hierarchical Orthogonal) +set(CR_BOOL_LIST true false) + +macro(mgard_x_add_composed_refactor_tests DEVICE_TYPE) + foreach(CR_D IN LISTS CR_D_LIST) + if(CR_D STREQUAL "1") + set(CR_SHAPE "{4097}") + elseif(CR_D STREQUAL "2") + set(CR_SHAPE "{129, 129}") + else() + set(CR_SHAPE "{33, 33, 33}") + endif() + foreach(CR_T IN LISTS CR_T_LIST) + foreach(CR_BASIS IN LISTS CR_BASIS_LIST) + foreach(CR_CONTROL_L2 IN LISTS CR_BOOL_LIST) + if(NOT (CR_BASIS STREQUAL "Hierarchical" AND CR_CONTROL_L2 STREQUAL "true")) + foreach(CR_NEGA_BINARY IN LISTS CR_BOOL_LIST) + set(_src "${_MGARD_X_TEST_DIR}/src/test_ComposedRefactorReconstructor.cpp.in") + set(_tag "${CR_D}D_${CR_T}_${CR_BASIS}_l2${CR_CONTROL_L2}_nega${CR_NEGA_BINARY}") + set(_gen "${CMAKE_CURRENT_BINARY_DIR}/src/test_ComposedRefactorReconstructor_${_tag}_${DEVICE_TYPE}.cpp") + configure_file(${_src} ${_gen}) + if(${DEVICE_TYPE} STREQUAL "CUDA") + set_source_files_properties(${_gen} PROPERTIES LANGUAGE CUDA) + elseif(${DEVICE_TYPE} STREQUAL "HIP") + set_source_files_properties(${_gen} PROPERTIES LANGUAGE HIP) + else() + set_source_files_properties(${_gen} PROPERTIES LANGUAGE CXX) + endif() + list(APPEND MGARD_X_TEST_SOURCES ${_gen}) + endforeach() + endif() + endforeach() + endforeach() + endforeach() + endforeach() +endmacro() + +foreach(DEVICE_TYPE IN LISTS DEVICE_TYPE_LIST) + set(MGARD_X_TEST_SOURCES "") + + mgard_x_add_test(test_Huffman ${DEVICE_TYPE}) + mgard_x_add_test(test_BlockDelta ${DEVICE_TYPE}) + mgard_x_add_test(test_DataRefactoring ${DEVICE_TYPE}) + mgard_x_add_test(test_HybridHierarchy ${DEVICE_TYPE}) + mgard_x_add_test(test_HybridMetadata ${DEVICE_TYPE}) + mgard_x_add_test(test_KernelFusionSwitch ${DEVICE_TYPE}) + mgard_x_add_test(test_Compressor ${DEVICE_TYPE}) + mgard_x_add_test(test_Zstd ${DEVICE_TYPE}) + mgard_x_add_test(test_RLE ${DEVICE_TYPE}) + mgard_x_add_test(test_ZeroRLE ${DEVICE_TYPE}) + mgard_x_add_bitplane_encoder_tests(${DEVICE_TYPE}) + mgard_x_add_test(test_DirectInterleaver ${DEVICE_TYPE}) + mgard_x_add_test(test_MGARDDecomposer ${DEVICE_TYPE}) + mgard_x_add_test(test_HybridLevelCompressor ${DEVICE_TYPE}) + mgard_x_add_composed_refactor_tests(${DEVICE_TYPE}) + mgard_x_add_test(test_Rans ${DEVICE_TYPE}) + mgard_x_add_test(test_ComposedHuffmanZstd ${DEVICE_TYPE}) + + # Portable (nvcomp-free) LZ4 backend: available on every device type. + mgard_x_add_test(test_LZ4 ${DEVICE_TYPE}) + mgard_x_add_test(test_ComposedHuffmanLZ4 ${DEVICE_TYPE}) + + set(TEST_TARGET "mgard-x-tests-${DEVICE_TYPE}") + add_executable(${TEST_TARGET} ${MGARD_X_TEST_SOURCES}) + target_link_libraries(${TEST_TARGET} PRIVATE mgard-library Catch2::Catch2WithMain) + + string(TOLOWER "${DEVICE_TYPE}" DEVICE_TYPE_LOWER) + catch_discover_tests(${TEST_TARGET} TEST_PREFIX "mgard-x/${DEVICE_TYPE_LOWER}/") +endforeach() diff --git a/tests/mgard-x/src/test_BitplaneEncoderConfig.cpp.in b/tests/mgard-x/src/test_BitplaneEncoderConfig.cpp.in new file mode 100644 index 0000000000..307542705d --- /dev/null +++ b/tests/mgard-x/src/test_BitplaneEncoderConfig.cpp.in @@ -0,0 +1,178 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +// One (encoder, T_data, T_bitplane, NegaBinary, ControlL2) configuration per +// generated file -- see tests/mgard-x/CMakeLists.txt. BPEncoderRegisterBlock's +// progressive_decode() unconditionally macro-unrolls 64 kernel variants per +// compile-time configuration it is instantiated with; cramming more than a +// couple of configurations into one translation unit exhausts the CUDA/HIP +// device compiler, so each configuration gets its own small file instead of +// one file looping over all of them (the same reason the library itself +// generates e.g. Compressor_1D_double_CUDA.cpp per type/dim combination +// rather than templating across them in one TU). + +#include "catch2/catch_test_macros.hpp" + +#include +#include +#include +#include +#include + +#include "mgard-x/Hierarchy/Hierarchy.hpp" +#include "mgard-x/MDR-X/BitplaneEncoder/BitplaneEncoder.hpp" +#include "mgard-x/RuntimeX/RuntimeX.h" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; +using T_data = @T_DATA@; +using T_bitplane = @T_BITPLANE@; +constexpr bool NegaBinary = @NEGA_BINARY@; +constexpr bool ControlL2 = @COLLECT_ERROR@; +constexpr SIZE BP_TEST_N = @BP_TEST_N@; + +// Bitplane encoders work on a flat SubArray<1, ...> and never use the +// Hierarchy's dimensionality beyond passing it through, so every case below +// uses a fixed placeholder D. +constexpr DIM BP_TEST_D = 1; + +// Deterministic data with a pinned exact max magnitude (so the exponent the +// encoders derive via frexp, and hence the expected quantization step, is +// known exactly) plus an exact zero to exercise the sign/zero edge case. +// +// `static`: every generated configuration file defines a function with this +// exact name and signature, and they all link into the same test executable, +// so this needs internal linkage to avoid an ODR violation at link time. +static std::vector make_bitplane_test_data(SIZE n) { + std::vector data(n); + std::srand(42); + for (SIZE i = 0; i < n; i++) { + double frac = (double)std::rand() / (double)RAND_MAX; // [0, 1) + double sign = (std::rand() % 2 == 0) ? 1.0 : -1.0; + data[i] = static_cast(sign * frac * 14.0); // stay clear of +/-15 + } + data[0] = (T_data)0; // exact zero + data[n / 2] = (T_data)15.0; // pins abs-max to an exactly representable value + return data; +} + +// Encodes `original` at full precision (MAX_BITPLANES bitplanes), then +// decodes it back twice: once from all the bitplanes (round trip should be +// limited only by floating point rounding, not by quantization) and once +// from half of them (to confirm progressive_decode actually is progressive, +// i.e. it produces a coarser but still correctly bounded result rather than +// silently reusing the full decode). +// +// `static`: see make_bitplane_test_data above -- same name/signature in every +// generated configuration file, linked into the same test executable. +static void run_bitplane_encoder_roundtrip(SIZE n) { + using T_error = double; + using Encoder = MDR::@ENCODER_NAME@; + constexpr int MAX_BITPLANES = Encoder::MAX_BITPLANES; + + DeviceRuntime::Initialize(); + + std::vector original = make_bitplane_test_data(n); + + // Adapt() sizes the encoder's internal error-reduction workspace from + // hierarchy.level_num_elems(hierarchy.l_target()), which for a real + // multilevel Hierarchy is only the finest level's own coefficient count -- + // roughly half of the hierarchy's total shape, not `n`. Since this test + // drives the encoder directly on a flat n-element buffer (bypassing actual + // multilevel decomposition), the sizing hierarchy has to be built oversized + // so that its "finest level" count still covers n; otherwise the ControlL2 + // workspace is undersized and encode() overruns it. + Hierarchy hierarchy({n * 4}, Config()); + Encoder encoder(hierarchy); + + Array<1, T_data, TestDevice> v_array({n}); + v_array.load(original.data()); + + Array<1, T_data, TestDevice> abs_max_array({1}); + Array<1, Byte, TestDevice> abs_max_workspace; + DeviceCollective::AbsMax(n, SubArray<1, T_data, TestDevice>(), + SubArray<1, T_data, TestDevice>(), + abs_max_workspace, false, 0); + DeviceCollective::AbsMax( + n, SubArray(v_array), SubArray(abs_max_array), abs_max_workspace, true, + 0); + // hostCopy's device-to-host copy runs on TestDevice's queue/stream (async + // on a real GPU backend), so the pointer it returns must not be dereferenced + // until after the matching SyncQueue -- reading it first (as an earlier + // version of this test did) silently returned stale/zero host memory on + // CUDA/HIP while happening to work on SERIAL, where everything is + // synchronous anyway. + T_data *abs_max_host = abs_max_array.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + T_data abs_max = abs_max_host[0]; + // Sanity check on the pinned test data itself, not the encoder: if this + // ever fails, make_bitplane_test_data (or AbsMax) is the place to look, not + // the round trip below. + REQUIRE((double)abs_max == 15.0); + + Array<2, T_bitplane, TestDevice> encoded_bitplanes_array( + {(SIZE)MAX_BITPLANES, Encoder::bitplane_length(n)}); + Array<1, T_error, TestDevice> level_errors_array({(SIZE)MAX_BITPLANES + 1}); + + encoder.encode(n, MAX_BITPLANES, SubArray(abs_max_array), SubArray(v_array), + SubArray(encoded_bitplanes_array), + SubArray(level_errors_array), 0); + DeviceRuntime::SyncQueue(0); + + auto decode_and_check = [&](int num_bitplanes, double tolerance) { + Array<1, bool, TestDevice> signs_array({n}); + Array<1, T_data, TestDevice> decoded_array({n}); + decoded_array.memset(0, 0); + encoder.progressive_decode(n, 0, num_bitplanes, SubArray(abs_max_array), + SubArray(encoded_bitplanes_array), + SubArray(signs_array), 0, + SubArray(decoded_array), 0); + DeviceRuntime::SyncQueue(0); + T_data *result = decoded_array.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + double max_err = 0; + for (SIZE i = 0; i < n; i++) + max_err = std::max( + max_err, std::fabs((double)original[i] - (double)result[i])); + CAPTURE(num_bitplanes, max_err, tolerance); + REQUIRE(max_err <= tolerance); + }; + + // Full precision: the only error left is float/double round trip through + // the fixed-point representation, not bitplane truncation. + const double full_tolerance = + (double)abs_max * (std::is_same::value ? 1e-12 : 1e-5); + decode_and_check(MAX_BITPLANES, full_tolerance); + + // Half the bitplanes: the quantization step is ~abs_max * 2^-(k - + // headroom); the headroom covers the +2 exponent shift the NegaBinary path + // applies before truncating. + const int half_bitplanes = MAX_BITPLANES / 2; + const double half_tolerance = + (double)abs_max * std::pow(2.0, -(half_bitplanes - 3)); + decode_and_check(half_bitplanes, half_tolerance); + + if constexpr (ControlL2) { + T_error *errors = level_errors_array.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + for (int i = 0; i <= MAX_BITPLANES; i++) { + CAPTURE(i, errors[i]); + REQUIRE(errors[i] >= 0.0); + REQUIRE(std::isfinite(errors[i])); + } + } +} + +TEST_CASE("MDR-X @ENCODER_NAME@ bitplane round trip " + "[@T_DATA@/@T_BITPLANE@/nega=@NEGA_BINARY@/L2=@COLLECT_ERROR@]", + "[mgard-x][mdr-x][bitplane-encoder][@DEVICE_TYPE@]") { + run_bitplane_encoder_roundtrip(BP_TEST_N); +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_BitplaneEncoderRegisterBlockThrows.cpp.in b/tests/mgard-x/src/test_BitplaneEncoderRegisterBlockThrows.cpp.in new file mode 100644 index 0000000000..228cc4ee00 --- /dev/null +++ b/tests/mgard-x/src/test_BitplaneEncoderRegisterBlockThrows.cpp.in @@ -0,0 +1,56 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +#include "catch2/catch_test_macros.hpp" + +#include +#include +#include + +#include "mgard-x/Hierarchy/Hierarchy.hpp" +#include "mgard-x/MDR-X/BitplaneEncoder/BitplaneEncoder.hpp" +#include "mgard-x/RuntimeX/RuntimeX.h" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; +constexpr DIM BP_TEST_D = 1; + +// Unlike BPEncoderLocalityBlock, RegisterBlock requires n to be a multiple of +// its batch size (bits of T_bitplane); this pins that guard rather than +// letting a misaligned caller silently corrupt memory. +TEST_CASE("MDR-X BPEncoderRegisterBlock rejects sizes not a batch multiple", + "[mgard-x][mdr-x][bitplane-encoder][@DEVICE_TYPE@]") { + using T_data = float; + using T_bitplane = uint32_t; + using T_error = double; + using Encoder = + MDR::BPEncoderRegisterBlock; + DeviceRuntime::Initialize(); + const SIZE n = 100; // not a multiple of 32 + Hierarchy hierarchy({n}, Config()); + Encoder encoder(hierarchy); + + std::vector original(n, 1.0f); + Array<1, T_data, TestDevice> v_array({n}); + v_array.load(original.data()); + Array<1, T_data, TestDevice> abs_max_array({1}); + abs_max_array.memset(0, 0); + Array<2, T_bitplane, TestDevice> encoded_bitplanes_array( + {(SIZE)Encoder::MAX_BITPLANES, Encoder::bitplane_length(n)}); + Array<1, T_error, TestDevice> level_errors_array( + {(SIZE)Encoder::MAX_BITPLANES + 1}); + + REQUIRE_THROWS_AS( + encoder.encode(n, Encoder::MAX_BITPLANES, SubArray(abs_max_array), + SubArray(v_array), SubArray(encoded_bitplanes_array), + SubArray(level_errors_array), 0), + std::runtime_error); +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_BlockDelta.cpp.in b/tests/mgard-x/src/test_BlockDelta.cpp.in new file mode 100644 index 0000000000..7040ca8b39 --- /dev/null +++ b/tests/mgard-x/src/test_BlockDelta.cpp.in @@ -0,0 +1,89 @@ +/* + * Copyright 2025, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +#include "catch2/catch_test_macros.hpp" + +#include +#include +#include + +#include "mgard-x/RuntimeX/RuntimeX.h" +#include "mgard-x/Lossless/BlockDelta/BlockDelta.hpp" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; + +namespace { +template +void roundtrip(const std::vector &original, int block_size, + block_delta_mode_type mode) { + const SIZE n = (SIZE)original.size(); + Array<1, T, TestDevice> input_data({n}); + input_data.load(original.data()); + + BlockDeltaLossless compressor; + compressor.Resize(n, block_size, mode, 0); + DeviceRuntime::SyncQueue(0); + + Array<1, Byte, TestDevice> compressed_data; + compressor.Compress(input_data, compressed_data, 0); // computation + compressor.Serialize(compressed_data, 0); // memory movement + + // Separate object: no shared internal state across compress/decompress. + BlockDeltaLossless decompressor; + decompressor.Resize(n, block_size, mode, 0); + DeviceRuntime::SyncQueue(0); + + decompressor.Deserialize(compressed_data, 0); + Array<1, T, TestDevice> decompressed_data({n}); + decompressor.Decompress(compressed_data, decompressed_data, 0); + + T *result = decompressed_data.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + for (SIZE i = 0; i < n; i++) { + REQUIRE(result[i] == original[i]); + } +} +} // namespace + +// Round-trip each mode on a signed quantized-integer stream with zero regions, +// small values, and sparse wide outliers. +TEST_CASE("BlockDelta round-trip (Fixed/Delta/Outlier)", + "[mgard-x][@DEVICE_TYPE@]") { + using T = QUANTIZED_INT; + const SIZE n = 10000; + DeviceRuntime::Initialize(); + + std::vector original(n); + std::srand(42); + for (SIZE i = 0; i < n; i++) { + int r = std::rand() % 100; + if (r < 20) + original[i] = 0; + else if (r < 95) + original[i] = static_cast(std::rand() % 128) - 64; + else + original[i] = static_cast(std::rand()) - static_cast(std::rand()); + } + + roundtrip(original, 256, block_delta_mode_type::Fixed); + roundtrip(original, 256, block_delta_mode_type::Delta); + roundtrip(original, 256, block_delta_mode_type::Outlier); +} + +// Edge case: all-zero input (every block has bit-width 0, packed size 0). +TEST_CASE("BlockDelta all-zero input", "[mgard-x][@DEVICE_TYPE@]") { + using T = QUANTIZED_INT; + const SIZE n = 3000; + DeviceRuntime::Initialize(); + std::vector original(n, 0); + roundtrip(original, 256, block_delta_mode_type::Fixed); + roundtrip(original, 256, block_delta_mode_type::Delta); + roundtrip(original, 256, block_delta_mode_type::Outlier); +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_ComposedHuffmanLZ4.cpp.in b/tests/mgard-x/src/test_ComposedHuffmanLZ4.cpp.in new file mode 100644 index 0000000000..f30d7543aa --- /dev/null +++ b/tests/mgard-x/src/test_ComposedHuffmanLZ4.cpp.in @@ -0,0 +1,64 @@ +/* + * Copyright 2025, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +#include "catch2/catch_test_macros.hpp" + +#include +#include +#include + +#include "mgard-x/RuntimeX/RuntimeX.h" +#include "mgard-x/Lossless/Lossless.hpp" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; + +// ComposedLosslessCompressor with the Huffman + portable-LZ4 pipeline (nvcomp +// free, runs on every backend): Huffman codes the integer symbols, then the +// portable LZ4 backend compresses the serialized Huffman stream. +TEST_CASE("Composed Lossless Huffman+LZ4", "[mgard-x][@DEVICE_TYPE@]") { + using T = QUANTIZED_INT; // int64_t + using H = HUFFMAN_CODE; // uint64_t + + const SIZE n = 100000; + + DeviceRuntime::Initialize(); + + Config config; + config.lossless = lossless_type::Huffman_LZ4; + + // Skewed distribution (values 0-63) so Huffman achieves real compression. + std::vector original(n); + std::srand(42); + for (SIZE i = 0; i < n; i++) { + original[i] = static_cast(std::rand() % 64); + } + + // Keep the host copy: Huffman compression shifts the device array in place. + Array<1, T, TestDevice> original_data({n}); + original_data.load(original.data()); + + // Use Adapt (not the parameterized constructor) so the Huffman workspace is + // fully set up. Separate compressor and decompressor objects. + ComposedLosslessCompressor compressor; + compressor.Adapt(n, config, 0); + Array<1, Byte, TestDevice> compressed_data; + compressor.Compress(original_data, compressed_data, 0); + + ComposedLosslessCompressor decompressor; + decompressor.Adapt(n, config, 0); + Array<1, T, TestDevice> decompressed_data({n}); + decompressor.Decompress(compressed_data, decompressed_data, 0); + + T *result = decompressed_data.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + for (SIZE i = 0; i < n; i++) { + REQUIRE(result[i] == original[i]); + } +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_ComposedHuffmanZstd.cpp.in b/tests/mgard-x/src/test_ComposedHuffmanZstd.cpp.in new file mode 100644 index 0000000000..4fcaeefd11 --- /dev/null +++ b/tests/mgard-x/src/test_ComposedHuffmanZstd.cpp.in @@ -0,0 +1,63 @@ +/* + * Copyright 2025, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +#include "catch2/catch_test_macros.hpp" + +#include +#include +#include + +#include "mgard-x/RuntimeX/RuntimeX.h" +#include "mgard-x/Lossless/Lossless.hpp" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; + +// ComposedLosslessCompressor with the Huffman + Zstd pipeline: Huffman codes +// the integer symbols, then Zstd compresses the serialized Huffman stream. +TEST_CASE("Composed Lossless Huffman+Zstd", "[mgard-x][@DEVICE_TYPE@]") { + using T = QUANTIZED_INT; // int64_t + using H = HUFFMAN_CODE; // uint64_t + + const SIZE n = 100000; + + DeviceRuntime::Initialize(); + + Config config; + config.lossless = lossless_type::Huffman_Zstd; + + // Skewed distribution (values 0-63) so Huffman achieves real compression. + std::vector original(n); + std::srand(42); + for (SIZE i = 0; i < n; i++) { + original[i] = static_cast(std::rand() % 64); + } + + // Keep the host copy: Huffman compression shifts the device array in place. + Array<1, T, TestDevice> original_data({n}); + original_data.load(original.data()); + + // Use Adapt (not the parameterized constructor) so the Huffman workspace is + // fully set up. Separate compressor and decompressor objects. + ComposedLosslessCompressor compressor; + compressor.Adapt(n, config, 0); + Array<1, Byte, TestDevice> compressed_data; + compressor.Compress(original_data, compressed_data, 0); + + ComposedLosslessCompressor decompressor; + decompressor.Adapt(n, config, 0); + Array<1, T, TestDevice> decompressed_data({n}); + decompressor.Decompress(compressed_data, decompressed_data, 0); + + T *result = decompressed_data.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + for (SIZE i = 0; i < n; i++) { + REQUIRE(result[i] == original[i]); + } +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_ComposedRefactorReconstructor.cpp.in b/tests/mgard-x/src/test_ComposedRefactorReconstructor.cpp.in new file mode 100644 index 0000000000..595fb5ac9f --- /dev/null +++ b/tests/mgard-x/src/test_ComposedRefactorReconstructor.cpp.in @@ -0,0 +1,189 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +// One (D, T_data, Basis, ControlL2, NegaBinary) configuration per generated +// file -- see tests/mgard-x/CMakeLists.txt. ComposedRefactor/Reconstructor +// pull in the full decompose/interleave/encode/compress pipeline (including +// BPEncoderRegisterBlock's 64-way macro-unrolled decode kernels), so +// combining more than one configuration in a translation unit risks the same +// CUDA/HIP compile-time blowup test_BitplaneEncoderConfig.cpp.in works +// around. +// +// Semantics pinned here (from ComposedReconstructor::GenerateRequest): +// - s == inf drives the max-error (L-infinity) estimator, which reads +// level_error_bounds only -- it works for either Basis and regardless of +// ControlL2. +// - s != inf (e.g. s == 0, L2) drives an s-norm/squared-error estimator +// that reads level_squared_errors directly. Those are only populated +// with real values when the encoder actually collected them, i.e. when +// ControlL2 is on. So an L2 request is only meaningful with ControlL2 on, +// and separately, ControlL2 with a Hierarchical basis isn't a supported +// combination (L2 tracking needs an orthogonal basis) -- excluded at the +// CMake level, not tested here. +// - Refactor and Reconstructor are instantiated with identical template +// arguments; the bitplanes one writes are meaningless to the other +// otherwise. + +#include "catch2/catch_test_macros.hpp" + +#include +#include +#include + +#include "mgard-x/Hierarchy/Hierarchy.hpp" +#include "mgard-x/MDR-X/Reconstructor/ComposedReconstructor.hpp" +#include "mgard-x/MDR-X/Refactor/ComposedRefactor.hpp" +#include "mgard-x/RuntimeX/RuntimeX.h" +#include "mgard-x/Utilities/ErrorCalculator.h" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; +using T_data = @CR_T@; +using Basis = MDR::@CR_BASIS@; +constexpr bool ControlL2 = @CR_CONTROL_L2@; +constexpr bool NegaBinary = @CR_NEGA_BINARY@; +constexpr DIM CR_D = @CR_D@; + +// Separable product of sines, amplitude in [-1, 1] -- same generator as +// test_HybridHierarchy/test_DataRefactoring -- so a single set of absolute +// tolerances means roughly the same thing across every shape this is called +// with. +static std::vector fill_smooth_data(std::vector shape) { + const double pi = std::acos(-1.0); + SIZE n = 1; + for (auto s : shape) + n *= s; + std::vector data(n); + for (SIZE idx = 0; idx < n; idx++) { + SIZE r = idx; + double v = 1.0; + for (int d = static_cast(CR_D) - 1; d >= 0; d--) { + SIZE coord = r % shape[d]; + r /= shape[d]; + double x = static_cast(coord) / static_cast(shape[d]); + v *= std::sin(2.0 * pi * x + 0.5 * d); + } + data[idx] = static_cast(v); + } + return data; +} + +// Refactors once, then reconstructs progressively against a decreasing +// sequence of absolute error bounds on the *same* mdr_metadata/mdr_data -- +// exactly the progressive-request pattern examples/mgard-x/MDR-X/*/ +// refactor.* drives, just with an assertion after every step instead of a +// print. Covers "single bound" (check just the first entry), "progressive to +// several bounds" (check every entry against the running state) and "near +// lossless" (the tightest, last entry) all in one pass, since they are the +// same operation repeated -- MDR's whole point is that asking again for a +// tighter bound refines rather than redoes the previous reconstruction. +static void +run_composed_pipeline_test(std::vector shape, double s, + const std::vector &tolerances) { + std::vector original = fill_smooth_data(shape); + SIZE n = original.size(); + + Config config; + Hierarchy hierarchy(shape, config); + + using Refactor = MDR::ComposedRefactor; + using Reconstructor = + MDR::ComposedReconstructor; + + Array input_array(shape); + input_array.load(original.data()); + + MDR::MDRData mdr_data; + MDR::MDRMetadata mdr_metadata; + { + Refactor refactor(hierarchy, config); + refactor.Refactor(input_array, mdr_metadata, mdr_data, 0); + refactor.Compress(mdr_metadata, mdr_data, 0); + refactor.StoreMetadata(mdr_metadata, mdr_data, 0); + } + + Reconstructor reconstructor(hierarchy, config); + mdr_metadata.InitializeForReconstruction(); + + Array reconstructed(shape); + reconstructed.memset(0, 0); + + for (double tol : tolerances) { + mdr_metadata.requested_tol = tol; + mdr_metadata.requested_s = s; + reconstructor.GenerateRequest(mdr_metadata); + mdr_metadata.DoneLoadingBitplans(); + // LoadMetadata/Decompress must run before ProgressiveReconstruct: it + // reads decoded bitplanes out of the reconstructor's own + // encoded_bitplanes buffer, which only Decompress fills in (from + // mdr_data.compressed_bitplanes) and only after LoadMetadata has set the + // per-level bitplane counts and abs-max the decoder needs. Skipping + // these -- as examples/mgard-x/MDR-X/*/refactor.* currently does -- + // leaves that buffer at whatever it was (zero, on the first request), + // so progressive_decode silently reconstructs nothing regardless of + // what GenerateRequest asked for. + reconstructor.LoadMetadata(mdr_metadata, mdr_data, 0); + reconstructor.Decompress(mdr_metadata, mdr_data, 0); + reconstructor.ProgressiveReconstruct(mdr_metadata, mdr_data, false, + reconstructed, 0); + // Advances prev_used_level_num_bitplanes so the next iteration's + // LoadMetadata computes the incremental (not full-again) bitplane count. + mdr_metadata.DoneReconstruct(); + + T_data *result = reconstructed.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + double achieved; + if (s == std::numeric_limits::infinity()) { + achieved = L_inf_error(n, original.data(), result, error_bound_type::ABS); + } else { + achieved = + L_2_error(shape, original.data(), result, error_bound_type::ABS, + /*normalize_coordinates=*/false); + } + CAPTURE(tol, achieved, s); + REQUIRE(achieved <= tol); + } +} + +// Tight enough to be "near lossless" without asking for less than this exact +// transform's own floating point round trip floor delivers at full bitplane +// precision (test_DataRefactoring/test_MGARDDecomposer pin that floor at the +// same order of magnitude for this data range: ~1e-3 relative for float, +// ~1e-10 for double). +static std::vector progressive_tolerances() { + const double floor = std::is_same::value ? 1e-9 : 1e-3; + return {0.5, 0.1, 0.01, floor}; +} + +TEST_CASE("MDR-X ComposedRefactor/Reconstructor L-infinity round trip " + "[@CR_D@D/@CR_T@/@CR_BASIS@/l2=@CR_CONTROL_L2@/nega=@CR_NEGA_BINARY@]", + "[mgard-x][mdr-x][composed-refactor][@DEVICE_TYPE@]") { + DeviceRuntime::Initialize(); + run_composed_pipeline_test(@CR_SHAPE@, + std::numeric_limits::infinity(), + progressive_tolerances()); +} + +TEST_CASE("MDR-X ComposedRefactor/Reconstructor L2 round trip " + "[@CR_D@D/@CR_T@/@CR_BASIS@/l2=@CR_CONTROL_L2@/nega=@CR_NEGA_BINARY@]", + "[mgard-x][mdr-x][composed-refactor][@DEVICE_TYPE@]") { + DeviceRuntime::Initialize(); + // L2 (s = 0) requests are only meaningful with ControlL2 on -- without it + // level_squared_errors is never populated by the encoder. Skip cleanly + // rather than exercise an unsupported configuration. + if constexpr (ControlL2) { + run_composed_pipeline_test(@CR_SHAPE@, 0.0, progressive_tolerances()); + } else { + SUCCEED("L2 (s=0) request is only meaningful with ControlL2 on; skipped " + "for this configuration."); + } +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_Compressor.cpp.in b/tests/mgard-x/src/test_Compressor.cpp.in new file mode 100644 index 0000000000..63930faa2f --- /dev/null +++ b/tests/mgard-x/src/test_Compressor.cpp.in @@ -0,0 +1,321 @@ +/* + * Copyright 2025, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +#include "catch2/catch_approx.hpp" +#include "catch2/catch_template_test_macros.hpp" +#include "catch2/catch_test_macros.hpp" +#include "catch2/generators/catch_generators.hpp" + +#include +#include +#include + +#include "mgard-x/RuntimeX/RuntimeX.h" +#include "mgard-x/CompressionLowLevel/Compressor.hpp" +#include "mgard-x/Utilities/ErrorCalculator.h" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; + +// Smooth, multi-dimensional test data: a separable product of sines over the +// normalized grid coordinates. Smoothness across the actual grid (not just the +// linearized index) is what lets MGARD reach the requested error bound. +template +void fill_smooth_data(std::vector &data, std::vector shape) { + const double pi = std::acos(-1.0); + SIZE n = data.size(); + for (SIZE idx = 0; idx < n; idx++) { + SIZE r = idx; + double v = 1.0; + // Row-major: the last dimension is contiguous. + for (int d = static_cast(D) - 1; d >= 0; d--) { + SIZE coord = r % shape[d]; + r /= shape[d]; + double x = static_cast(coord) / static_cast(shape[d]); + v *= std::sin(2.0 * pi * x + 0.5 * d); + } + data[idx] = static_cast(v); + } +} + +// Compress -> Decompress round trip for the low-level Compressor. Verifies, the +// same way the mgard-x executable's print_statistics does: +// 1. The true error stays under the requested error bound (tol). +// 2. The norm returned by Compress matches the norm of the input data +// (L_inf when s == inf, L_2 when s == 0). The norm is only computed by +// Compress in REL mode, so it is only checked there. +template +void run_compressor_test( + std::vector shape, enum error_bound_type ebtype, T s, T tol, + compression_projection_mode_type projection_mode = + compression_projection_mode_type::Auto) { + SIZE n = 1; + for (auto sz : shape) + n *= sz; + + std::vector original(n); + fill_smooth_data(original, shape); + + Config config; + config.projection_mode = projection_mode; + + // Separate compressor and decompressor objects so they share no internal + // state — the round trip only communicates through the serialized bytes + // (and the norm, as it would be carried in metadata in real usage). + // Use Adapt (not the parameterized constructor) to initialize: only the + // Adapt path allocates the Huffman signature_verify buffer that Deserialize + // reads during decompression. This matches the high-level pipeline. + Hierarchy hierarchy(shape, config); + Compressor compressor; + compressor.Adapt(hierarchy, config, 0); + Compressor decompressor; + decompressor.Adapt(hierarchy, config, 0); + DeviceRuntime::SyncQueue(0); + + // Compress modifies the device array in place (in-place decomposition), so + // keep the host copy in `original` for the error comparison below. + Array original_array(shape); + original_array.load(original.data()); + + T norm = 0; + Array<1, Byte, DevType> compressed_array; + compressor.Compress(original_array, ebtype, tol, s, norm, compressed_array, 0); + DeviceRuntime::SyncQueue(0); + + Array decompressed_array; + decompressor.Decompress(compressed_array, ebtype, tol, s, norm, + decompressed_array, 0); + T *result = decompressed_array.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + // (1) True error under the requested bound. Mirrors print_statistics: L_inf + // metric when s == inf, L_2 metric otherwise; the metric already returns the + // relative error in REL mode, so comparing against tol works for both modes. + double actual_error; + if (s == std::numeric_limits::infinity()) { + actual_error = L_inf_error(n, original.data(), result, ebtype); + } else { + actual_error = L_2_error(shape, original.data(), result, ebtype, + config.normalize_coordinates); + } + REQUIRE(actual_error < static_cast(tol)); + + // (2) Returned norm matches the input data norm (REL mode only). + if (ebtype == error_bound_type::REL) { + double expected_norm; + if (s == std::numeric_limits::infinity()) { + expected_norm = L_inf_norm(n, original.data()); + } else { + expected_norm = + L_2_norm(shape, original.data(), config.normalize_coordinates); + } + REQUIRE(static_cast(norm) == + Catch::Approx(expected_norm).epsilon(1e-4)); + } +} + +// Exercises the cartesian product of error_bound_type {REL, ABS}, s {inf, 0}, +// and tol {1e-2, 1e-3, 1e-4} via GENERATE, for each data type. +#define MGARD_X_COMPRESSOR_TEST_BODY(D, SHAPE) \ + DeviceRuntime::Initialize(); \ + enum error_bound_type ebtype = \ + GENERATE(error_bound_type::REL, error_bound_type::ABS); \ + TestType s = GENERATE(std::numeric_limits::infinity(), \ + static_cast(0)); \ + TestType tol = GENERATE(static_cast(1e-2), \ + static_cast(1e-3), \ + static_cast(1e-4)); \ + CAPTURE(static_cast(ebtype), s, tol); \ + run_compressor_test(SHAPE, ebtype, s, tol); + +TEMPLATE_TEST_CASE("Low-Level Compressor 1D", "[mgard-x][@DEVICE_TYPE@]", + float, double) { + MGARD_X_COMPRESSOR_TEST_BODY(1, (std::vector{4097})) +} + +TEMPLATE_TEST_CASE("Low-Level Compressor 2D", "[mgard-x][@DEVICE_TYPE@]", + float, double) { + MGARD_X_COMPRESSOR_TEST_BODY(2, (std::vector{129, 129})) +} + +TEMPLATE_TEST_CASE("Low-Level Compressor 3D", "[mgard-x][@DEVICE_TYPE@]", + float, double) { + MGARD_X_COMPRESSOR_TEST_BODY(3, (std::vector{33, 33, 33})) +} + +// --------------------------------------------------------------------------- +// Projection mode vs. error norm +// +// Config::projection_mode is shared by this plain Compressor and +// HybridHierarchyCompressor (see resolve_projection_mode in +// Utilities/ProjectionMode.h): Auto (the default) resolves to the +// hierarchical basis under an L-infinity bound and orthogonal otherwise; +// Orthogonal/Hierarchical force a specific basis, and Hierarchical still only +// supports L-infinity error control. These cases pin that contract through a +// real Compress()/Decompress() round trip, for every combination that is +// supposed to work. +// --------------------------------------------------------------------------- + +TEMPLATE_TEST_CASE("Low-Level Compressor projection mode vs error norm", + "[mgard-x][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + const TestType inf = std::numeric_limits::infinity(); + const TestType l2_s = static_cast(0); + const TestType tol = static_cast(1e-2); + + SECTION("3D") { + SECTION("orthogonal, L_inf") { + run_compressor_test<3, TestType, TestDevice>( + {33, 33, 33}, error_bound_type::REL, inf, tol, + compression_projection_mode_type::Orthogonal); + } + SECTION("orthogonal, L_2") { + run_compressor_test<3, TestType, TestDevice>( + {33, 33, 33}, error_bound_type::REL, l2_s, tol, + compression_projection_mode_type::Orthogonal); + } + SECTION("hierarchical, L_inf") { + run_compressor_test<3, TestType, TestDevice>( + {33, 33, 33}, error_bound_type::REL, inf, tol, + compression_projection_mode_type::Hierarchical); + } + SECTION("auto, L_inf") { + run_compressor_test<3, TestType, TestDevice>( + {33, 33, 33}, error_bound_type::REL, inf, tol, + compression_projection_mode_type::Auto); + } + SECTION("auto, L_2") { + run_compressor_test<3, TestType, TestDevice>( + {33, 33, 33}, error_bound_type::REL, l2_s, tol, + compression_projection_mode_type::Auto); + } + } + SECTION("2D") { + SECTION("orthogonal, L_inf") { + run_compressor_test<2, TestType, TestDevice>( + {129, 129}, error_bound_type::REL, inf, tol, + compression_projection_mode_type::Orthogonal); + } + SECTION("orthogonal, L_2") { + run_compressor_test<2, TestType, TestDevice>( + {129, 129}, error_bound_type::REL, l2_s, tol, + compression_projection_mode_type::Orthogonal); + } + SECTION("hierarchical, L_inf") { + run_compressor_test<2, TestType, TestDevice>( + {129, 129}, error_bound_type::REL, inf, tol, + compression_projection_mode_type::Hierarchical); + } + SECTION("auto, L_inf") { + run_compressor_test<2, TestType, TestDevice>( + {129, 129}, error_bound_type::REL, inf, tol, + compression_projection_mode_type::Auto); + } + SECTION("auto, L_2") { + run_compressor_test<2, TestType, TestDevice>( + {129, 129}, error_bound_type::REL, l2_s, tol, + compression_projection_mode_type::Auto); + } + } + SECTION("1D") { + SECTION("orthogonal, L_inf") { + run_compressor_test<1, TestType, TestDevice>( + {4097}, error_bound_type::REL, inf, tol, + compression_projection_mode_type::Orthogonal); + } + SECTION("orthogonal, L_2") { + run_compressor_test<1, TestType, TestDevice>( + {4097}, error_bound_type::REL, l2_s, tol, + compression_projection_mode_type::Orthogonal); + } + SECTION("hierarchical, L_inf") { + run_compressor_test<1, TestType, TestDevice>( + {4097}, error_bound_type::REL, inf, tol, + compression_projection_mode_type::Hierarchical); + } + SECTION("auto, L_inf") { + run_compressor_test<1, TestType, TestDevice>( + {4097}, error_bound_type::REL, inf, tol, + compression_projection_mode_type::Auto); + } + SECTION("auto, L_2") { + run_compressor_test<1, TestType, TestDevice>( + {4097}, error_bound_type::REL, l2_s, tol, + compression_projection_mode_type::Auto); + } + } +} + +// The hierarchical basis rejects any error control other than L_inf, at the +// level callers actually hit -- Compress() itself. +TEMPLATE_TEST_CASE( + "Low-Level Compressor rejects hierarchical projection under L_2", + "[mgard-x][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + constexpr DIM D = 3; + std::vector shape{33, 33, 33}; + SIZE n = shape[0] * shape[1] * shape[2]; + + std::vector original(n); + fill_smooth_data(original, shape); + + Config config; + config.projection_mode = compression_projection_mode_type::Hierarchical; + Hierarchy hierarchy(shape, config); + Compressor compressor; + compressor.Adapt(hierarchy, config, 0); + DeviceRuntime::SyncQueue(0); + + Array original_array(shape); + original_array.load(original.data()); + + // s = 0 selects the L_2 norm, which the hierarchical basis cannot control. + TestType norm = 0; + Array<1, Byte, TestDevice> compressed_array; + REQUIRE_THROWS_AS( + compressor.Compress(original_array, error_bound_type::REL, + static_cast(1e-2), + static_cast(0), norm, compressed_array, 0), + ProcessingException); +} + +// The hierarchical basis is only implemented for 1D/2D/3D (the +// multi-dimensional decompose/recompose kernels only honor the flag there); +// requesting it explicitly above 3-D must throw rather than silently falling +// back the way Auto does. +TEMPLATE_TEST_CASE( + "Low-Level Compressor rejects hierarchical projection above 3-D", + "[mgard-x][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + constexpr DIM D = 4; + std::vector shape{9, 9, 9, 9}; + SIZE n = shape[0] * shape[1] * shape[2] * shape[3]; + + std::vector original(n); + fill_smooth_data(original, shape); + + Config config; + config.projection_mode = compression_projection_mode_type::Hierarchical; + Hierarchy hierarchy(shape, config); + Compressor compressor; + compressor.Adapt(hierarchy, config, 0); + DeviceRuntime::SyncQueue(0); + + Array original_array(shape); + original_array.load(original.data()); + + const TestType inf = std::numeric_limits::infinity(); + TestType norm = 0; + Array<1, Byte, TestDevice> compressed_array; + REQUIRE_THROWS_AS( + compressor.Compress(original_array, error_bound_type::REL, + static_cast(1e-2), inf, norm, + compressed_array, 0), + ProcessingException); +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_DataRefactoring.cpp.in b/tests/mgard-x/src/test_DataRefactoring.cpp.in new file mode 100644 index 0000000000..76dfae37ae --- /dev/null +++ b/tests/mgard-x/src/test_DataRefactoring.cpp.in @@ -0,0 +1,537 @@ +/* + * Copyright 2025, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +#include "catch2/catch_template_test_macros.hpp" +#include "catch2/catch_test_macros.hpp" +#include "catch2/generators/catch_generators.hpp" + +#include +#include +#include + +#include "mgard-x/RuntimeX/RuntimeX.h" +#include "mgard-x/DataRefactoring/DataRefactor.hpp" +#include "mgard-x/DataRefactoring/InCacheBlock/DataRefactoring.h" +#include "mgard-x/Utilities/ErrorCalculator.h" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; + +// Tolerance reflects floating-point accumulation over O(log N) decomposition +// levels: tight for double (~1e-10), generous for float (~1e-3). +template +void run_refactoring_test(std::vector shape, bool orthogonal_projection) { + SIZE n = 1; + for (auto s : shape) + n *= s; + + const double pi = std::acos(-1.0); + std::vector original(n); + for (SIZE i = 0; i < n; i++) + original[i] = static_cast(std::sin(2.0 * pi * i / n)); + + Config config; + Hierarchy hierarchy(shape, config); + + Array data(shape); + data.load(original.data()); + + data_refactoring::DataRefactor refactor(hierarchy, config); + SubArray data_subarray(data); + + refactor.Decompose(data_subarray, orthogonal_projection, 0); + refactor.Recompose(data_subarray, orthogonal_projection, 0); + + T *result = data.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + const double tolerance = + std::is_same::value ? 1e-10 : 1e-3; + double error = + L_inf_error(n, original.data(), result, error_bound_type::REL); + REQUIRE(error < tolerance); +} + +TEMPLATE_TEST_CASE("Orthogonal Decomposition 1D", "[mgard-x][@DEVICE_TYPE@]", + float, double) { + DeviceRuntime::Initialize(); + SECTION("small") { run_refactoring_test<1, TestType, TestDevice>({33}, true); } + SECTION("medium") { run_refactoring_test<1, TestType, TestDevice>({129}, true); } + SECTION("large") { run_refactoring_test<1, TestType, TestDevice>({4097}, true); } +} + +TEMPLATE_TEST_CASE("Orthogonal Decomposition 2D", "[mgard-x][@DEVICE_TYPE@]", + float, double) { + DeviceRuntime::Initialize(); + SECTION("small") { run_refactoring_test<2, TestType, TestDevice>({33, 33}, true); } + SECTION("medium") { run_refactoring_test<2, TestType, TestDevice>({129, 129}, true); } + SECTION("large") { run_refactoring_test<2, TestType, TestDevice>({513, 513}, true); } +} + +TEMPLATE_TEST_CASE("Orthogonal Decomposition 3D", "[mgard-x][@DEVICE_TYPE@]", + float, double) { + DeviceRuntime::Initialize(); + SECTION("small") { run_refactoring_test<3, TestType, TestDevice>({17, 17, 17}, true); } + SECTION("medium") { run_refactoring_test<3, TestType, TestDevice>({33, 33, 33}, true); } + SECTION("large") { run_refactoring_test<3, TestType, TestDevice>({65, 65, 65}, true); } +} + +TEMPLATE_TEST_CASE("Hierarchical Decomposition 1D", "[mgard-x][@DEVICE_TYPE@]", + float, double) { + DeviceRuntime::Initialize(); + SECTION("small") { run_refactoring_test<1, TestType, TestDevice>({33}, false); } + SECTION("medium") { run_refactoring_test<1, TestType, TestDevice>({129}, false); } + SECTION("large") { run_refactoring_test<1, TestType, TestDevice>({4097}, false); } +} + +TEMPLATE_TEST_CASE("Hierarchical Decomposition 2D", "[mgard-x][@DEVICE_TYPE@]", + float, double) { + DeviceRuntime::Initialize(); + SECTION("small") { run_refactoring_test<2, TestType, TestDevice>({33, 33}, false); } + SECTION("medium") { run_refactoring_test<2, TestType, TestDevice>({129, 129}, false); } + SECTION("large") { run_refactoring_test<2, TestType, TestDevice>({513, 513}, false); } +} + +TEMPLATE_TEST_CASE("Hierarchical Decomposition 3D", "[mgard-x][@DEVICE_TYPE@]", + float, double) { + DeviceRuntime::Initialize(); + SECTION("small") { run_refactoring_test<3, TestType, TestDevice>({17, 17, 17}, false); } + SECTION("medium") { run_refactoring_test<3, TestType, TestDevice>({33, 33, 33}, false); } + SECTION("large") { run_refactoring_test<3, TestType, TestDevice>({65, 65, 65}, false); } +} + + + +// --------------------------------------------------------------------------- +// In-cache block (8x8x8) refactoring +// +// The in-cache path decomposes each 8x8x8 block independently in shared +// memory, emitting a 5x5x5 coarse brick plus 8^3 - 5^3 = 387 coefficients per +// block. These cases drive the exported kernels directly rather than through +// BlockLocalHierarchyDataRefactor (covered by test_HybridHierarchy), so a +// layout or boundary regression shows up on a single level. +// --------------------------------------------------------------------------- + +namespace in_cache_block = data_refactoring::in_cache_block; + +constexpr SIZE in_cache_block_size = 8; +constexpr SIZE in_cache_coarse_size = 5; +constexpr SIZE in_cache_coeff_per_block = 387; + +// Number of 8x8x8 blocks along each dimension (partial blocks included). +static std::vector in_cache_grid(const std::vector &shape) { + std::vector grid(shape.size()); + for (DIM d = 0; d < shape.size(); d++) + grid[d] = (shape[d] - 1) / in_cache_block_size + 1; + return grid; +} + +static std::vector in_cache_coarse_shape(const std::vector &shape) { + std::vector coarse_shape = in_cache_grid(shape); + for (DIM d = 0; d < coarse_shape.size(); d++) + coarse_shape[d] *= in_cache_coarse_size; + return coarse_shape; +} + +static SIZE in_cache_num_coeff(const std::vector &shape) { + std::vector grid = in_cache_grid(shape); + SIZE num_blocks = 1; + for (auto g : grid) + num_blocks *= g; + return num_blocks * in_cache_coeff_per_block; +} + +static SIZE num_elems(const std::vector &shape) { + SIZE n = 1; + for (auto s : shape) + n *= s; + return n; +} + +// Separable product of sines: smooth along every dimension, so the block-local +// interpolation has structure to resolve in all three directions (the 1D +// linear-index sine used above degenerates to noise across y and z). +template +static void fill_smooth_3d(std::vector &data, + const std::vector &shape) { + const double pi = std::acos(-1.0); + for (SIZE z = 0; z < shape[0]; z++) { + for (SIZE y = 0; y < shape[1]; y++) { + for (SIZE x = 0; x < shape[2]; x++) { + double v = std::sin(2.0 * pi * z / shape[0]) * + std::sin(2.0 * pi * y / shape[1] + 0.5) * + std::sin(2.0 * pi * x / shape[2] + 1.0); + data[(z * shape[1] + y) * shape[2] + x] = static_cast(v); + } + } + } +} + +// Host mirror of the in-kernel quantizer. Must stay bit-identical to +// DecomposeQuantize8x8x8Functor::Operation9 (and QuantizeLocalLevelFunctor +// with volume == 1): scale, round half away from zero by truncating +// 0.5 + |t * q|, then fold in the Huffman dictionary shift. +template +static Q host_quantize(T t, T quantizer, bool prep_huffman, SIZE dict_size) { + Q quantized = static_cast(std::copysign((T)0.5 + std::fabs(t * quantizer), t)); + if (prep_huffman) + quantized += dict_size / 2; + return quantized; +} + +template +static T host_dequantize(Q quantized, T dequantizer, bool prep_huffman, + SIZE dict_size) { + if (prep_huffman) + quantized -= dict_size / 2; + return dequantizer * (T)quantized; +} + +// Copy an Array back to the host into a vector we own: hostCopy() hands back a +// buffer tied to the Array, and several of these tests hold two of them alive +// at once. +template +static std::vector device_to_host(Array &array, SIZE n) { + T *host = array.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + return std::vector(host, host + n); +} + +// decompose -> recompose through the unfused in-cache kernels is a numerical +// identity: no quantization is involved, only the forward and inverse block +// transforms. The unfused kernels read and write v without bounds checks, so +// every extent must be a multiple of 8. +template +void run_in_cache_block_test(std::vector shape) { + constexpr DIM D = 3; + SIZE n = num_elems(shape); + + std::vector original(n); + fill_smooth_3d(original, shape); + + Array v(shape); + v.load(original.data()); + Array coarse(in_cache_coarse_shape(shape)); + Array<1, T, DevType> coeff({in_cache_num_coeff(shape)}); + + in_cache_block::decompose(SubArray(v), SubArray(coarse), + SubArray(coeff), true, 0); + DeviceRuntime::SyncQueue(0); + + // Zero v so the reconstruction can only come from coarse + coeff. + v.memset(0, 0); + in_cache_block::recompose(SubArray(v), SubArray(coarse), + SubArray(coeff), true, 0); + T *result = v.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + const double tolerance = std::is_same::value ? 1e-10 : 1e-3; + double error = L_inf_error(n, original.data(), result, error_bound_type::REL); + CAPTURE(error); + REQUIRE(error < tolerance); +} + +// A constant field is reproduced exactly by the multilinear interpolant, so +// every coefficient must vanish and the coarse brick must carry the constant +// through unchanged (a zero correction on top of a constant coarse grid). +template +void run_in_cache_block_constant_test(std::vector shape) { + constexpr DIM D = 3; + const T value = static_cast(3.25); + std::vector coarse_shape = in_cache_coarse_shape(shape); + SIZE num_coeff = in_cache_num_coeff(shape); + SIZE num_coarse = num_elems(coarse_shape); + + std::vector original(num_elems(shape), value); + + Array v(shape); + v.load(original.data()); + Array coarse(coarse_shape); + Array<1, T, DevType> coeff({num_coeff}); + + in_cache_block::decompose(SubArray(v), SubArray(coarse), + SubArray(coeff), true, 0); + DeviceRuntime::SyncQueue(0); + + std::vector coeff_h = device_to_host(coeff, num_coeff); + std::vector coarse_h = device_to_host(coarse, num_coarse); + + double max_coeff = 0.0, max_coarse_deviation = 0.0; + for (SIZE i = 0; i < num_coeff; i++) + max_coeff = std::max(max_coeff, std::abs((double)coeff_h[i])); + for (SIZE i = 0; i < num_coarse; i++) + max_coarse_deviation = + std::max(max_coarse_deviation, std::abs((double)coarse_h[i] - value)); + + const double tolerance = std::is_same::value ? 1e-12 : 1e-5; + CAPTURE(max_coeff, max_coarse_deviation); + REQUIRE(max_coeff < tolerance); + REQUIRE(max_coarse_deviation < tolerance); +} + +// The fused decompose+quantize / dequantize+recompose kernels run the same +// transform as the unfused pair and only differ in how coefficients cross the +// kernel boundary. So they must agree exactly with running the unfused kernels +// and quantizing on the host -- not merely to within a tolerance. +template +void run_in_cache_block_fused_test(std::vector shape, bool prep_huffman) { + constexpr DIM D = 3; + using Q = QUANTIZED_INT; + const SIZE dict_size = 8192; + const T quantizer = static_cast(1024); // reciprocal of the bin width + const T dequantizer = (T)1 / quantizer; + + SIZE n = num_elems(shape); + std::vector coarse_shape = in_cache_coarse_shape(shape); + SIZE num_coarse = num_elems(coarse_shape); + SIZE num_coeff = in_cache_num_coeff(shape); + + std::vector original(n); + fill_smooth_3d(original, shape); + + // Unfused reference decomposition. + Array v_ref(shape); + v_ref.load(original.data()); + Array coarse_ref(coarse_shape); + Array<1, T, DevType> coeff_ref({num_coeff}); + in_cache_block::decompose( + SubArray(v_ref), SubArray(coarse_ref), SubArray(coeff_ref), true, 0); + DeviceRuntime::SyncQueue(0); + std::vector coarse_ref_h = device_to_host(coarse_ref, num_coarse); + std::vector coeff_ref_h = device_to_host(coeff_ref, num_coeff); + + // Fused decomposition + quantization. + Array v(shape); + v.load(original.data()); + Array coarse_fused(coarse_shape); + Array<1, Q, DevType> quantized({num_coeff}); + in_cache_block::decompose_quantize( + SubArray(v), SubArray(coarse_fused), SubArray(quantized), quantizer, + SubArray<1, T, DevType>(), false, prep_huffman, dict_size, true, 0); + DeviceRuntime::SyncQueue(0); + std::vector coarse_fused_h = device_to_host(coarse_fused, num_coarse); + std::vector quantized_h = device_to_host(quantized, num_coeff); + + // The coarse brick is never quantized, so it must come out bit-identical. + SIZE coarse_mismatches = 0; + for (SIZE i = 0; i < num_coarse; i++) + coarse_mismatches += (coarse_fused_h[i] != coarse_ref_h[i]); + REQUIRE(coarse_mismatches == 0); + + SIZE coeff_mismatches = 0; + for (SIZE i = 0; i < num_coeff; i++) { + Q expected = host_quantize(coeff_ref_h[i], quantizer, prep_huffman, + dict_size); + coeff_mismatches += (quantized_h[i] != expected); + } + REQUIRE(coeff_mismatches == 0); + + // Inverse: unfused recompose fed host-dequantized coefficients must match + // the fused dequantize+recompose bit for bit. + std::vector dequantized_h(num_coeff); + for (SIZE i = 0; i < num_coeff; i++) + dequantized_h[i] = host_dequantize(quantized_h[i], dequantizer, + prep_huffman, dict_size); + Array<1, T, DevType> coeff_deq({num_coeff}); + coeff_deq.load(dequantized_h.data()); + + Array v_unfused(shape); + v_unfused.memset(0, 0); + in_cache_block::recompose( + SubArray(v_unfused), SubArray(coarse_ref), SubArray(coeff_deq), true, 0); + DeviceRuntime::SyncQueue(0); + std::vector unfused_h = device_to_host(v_unfused, n); + + Array v_fused(shape); + v_fused.memset(0, 0); + in_cache_block::recompose_dequantize( + SubArray(v_fused), SubArray(coarse_fused), SubArray(quantized), + dequantizer, SubArray<1, T, DevType>(), false, prep_huffman, dict_size, + true, 0); + DeviceRuntime::SyncQueue(0); + std::vector fused_h = device_to_host(v_fused, n); + + SIZE data_mismatches = 0; + for (SIZE i = 0; i < n; i++) + data_mismatches += (fused_h[i] != unfused_h[i]); + REQUIRE(data_mismatches == 0); + + // Bound the round trip against the quantization step itself. Recompose is a + // linear operator, so perturbing every coefficient by at most half a bin + // (0.5 / quantizer) perturbs the reconstruction by at most that half-bin + // times the operator's gain. Measured gain here is ~1.55 (worst case across + // these shapes and both precisions: 7.56e-4 against a 4.88e-4 half-bin), so + // 4x leaves headroom for a different data set without letting a real + // amplification regression through. + // + // For scale: the shipped error budget in LocalQuantization::CalcQuantizers + // spends a factor of 2^(l+1) * (1 + 3^D) = 112 on one local level in 3D, so + // a step of 1/1024 is what the compressor would pick for tol = 0.055. That + // is the worst case the scheme is designed to survive; this assertion is the + // much tighter statement that the actual gain is O(1). + const double half_bin = 0.5 / (double)quantizer; + double error = + L_inf_error(n, original.data(), fused_h.data(), error_bound_type::ABS); + CAPTURE(error, half_bin, error / half_bin); + REQUIRE(error < 4.0 * half_bin); +} + +// The fused kernels bounds-check their global reads and writes so they can +// consume and produce unpadded arrays directly, standing in for the explicit +// zero-padding the unfused path needs. Running them on an unpadded shape must +// reproduce the unfused result on the same data padded up to a multiple of 8. +template +void run_in_cache_block_padding_test(std::vector shape) { + constexpr DIM D = 3; + using Q = QUANTIZED_INT; + const SIZE dict_size = 8192; + const T quantizer = static_cast(1024); + const T dequantizer = (T)1 / quantizer; + + std::vector padded_shape(D); + for (DIM d = 0; d < D; d++) + padded_shape[d] = in_cache_grid(shape)[d] * in_cache_block_size; + + SIZE n = num_elems(shape); + SIZE n_padded = num_elems(padded_shape); + std::vector coarse_shape = in_cache_coarse_shape(shape); + SIZE num_coarse = num_elems(coarse_shape); + SIZE num_coeff = in_cache_num_coeff(shape); + + std::vector original(n); + fill_smooth_3d(original, shape); + + // The same data zero-padded to the next multiple of 8 in every dimension. + std::vector padded(n_padded, (T)0); + for (SIZE z = 0; z < shape[0]; z++) + for (SIZE y = 0; y < shape[1]; y++) + for (SIZE x = 0; x < shape[2]; x++) + padded[(z * padded_shape[1] + y) * padded_shape[2] + x] = + original[(z * shape[1] + y) * shape[2] + x]; + + // Unfused reference on the padded input. + Array v_padded(padded_shape); + v_padded.load(padded.data()); + Array coarse_ref(coarse_shape); + Array<1, T, DevType> coeff_ref({num_coeff}); + in_cache_block::decompose( + SubArray(v_padded), SubArray(coarse_ref), SubArray(coeff_ref), true, 0); + DeviceRuntime::SyncQueue(0); + std::vector coarse_ref_h = device_to_host(coarse_ref, num_coarse); + std::vector coeff_ref_h = device_to_host(coeff_ref, num_coeff); + + // Fused on the unpadded input. + Array v(shape); + v.load(original.data()); + Array coarse_fused(coarse_shape); + Array<1, Q, DevType> quantized({num_coeff}); + in_cache_block::decompose_quantize( + SubArray(v), SubArray(coarse_fused), SubArray(quantized), quantizer, + SubArray<1, T, DevType>(), false, false, dict_size, true, 0); + DeviceRuntime::SyncQueue(0); + std::vector coarse_fused_h = device_to_host(coarse_fused, num_coarse); + std::vector quantized_h = device_to_host(quantized, num_coeff); + + SIZE coarse_mismatches = 0; + for (SIZE i = 0; i < num_coarse; i++) + coarse_mismatches += (coarse_fused_h[i] != coarse_ref_h[i]); + REQUIRE(coarse_mismatches == 0); + + SIZE coeff_mismatches = 0; + for (SIZE i = 0; i < num_coeff; i++) { + Q expected = + host_quantize(coeff_ref_h[i], quantizer, false, dict_size); + coeff_mismatches += (quantized_h[i] != expected); + } + REQUIRE(coeff_mismatches == 0); + + // Inverse: the fused kernel writes only the in-range positions, so it must + // match the padded unfused reconstruction cropped back to the true extent. + std::vector dequantized_h(num_coeff); + for (SIZE i = 0; i < num_coeff; i++) + dequantized_h[i] = + host_dequantize(quantized_h[i], dequantizer, false, dict_size); + Array<1, T, DevType> coeff_deq({num_coeff}); + coeff_deq.load(dequantized_h.data()); + + Array v_padded_out(padded_shape); + v_padded_out.memset(0, 0); + in_cache_block::recompose( + SubArray(v_padded_out), SubArray(coarse_ref), SubArray(coeff_deq), true, + 0); + DeviceRuntime::SyncQueue(0); + std::vector padded_out_h = device_to_host(v_padded_out, n_padded); + + // Seed with a sentinel: the fused store is bounds-checked against the + // unpadded extent, so it must still write every in-range element (the + // threads it drops are the ones with no position in the array at all). + const T sentinel = static_cast(-12345); + std::vector seeded(n, sentinel); + Array v_fused(shape); + v_fused.load(seeded.data()); + in_cache_block::recompose_dequantize( + SubArray(v_fused), SubArray(coarse_fused), SubArray(quantized), + dequantizer, SubArray<1, T, DevType>(), false, false, dict_size, true, + 0); + DeviceRuntime::SyncQueue(0); + std::vector fused_h = device_to_host(v_fused, n); + + SIZE unwritten = 0; + for (SIZE i = 0; i < n; i++) + unwritten += (fused_h[i] == sentinel); + REQUIRE(unwritten == 0); + + SIZE data_mismatches = 0; + for (SIZE z = 0; z < shape[0]; z++) + for (SIZE y = 0; y < shape[1]; y++) + for (SIZE x = 0; x < shape[2]; x++) + data_mismatches += + (fused_h[(z * shape[1] + y) * shape[2] + x] != + padded_out_h[(z * padded_shape[1] + y) * padded_shape[2] + x]); + REQUIRE(data_mismatches == 0); +} + +TEMPLATE_TEST_CASE("In-Cache Block Decomposition 3D", + "[mgard-x][in-cache][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + SECTION("single block") { run_in_cache_block_test({8, 8, 8}); } + SECTION("cubic") { run_in_cache_block_test({64, 64, 64}); } + SECTION("non-cubic") { run_in_cache_block_test({40, 64, 24}); } +} + +TEMPLATE_TEST_CASE("In-Cache Block Decomposition of a constant field", + "[mgard-x][in-cache][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + SECTION("single block") { run_in_cache_block_constant_test({8, 8, 8}); } + SECTION("multi block") { run_in_cache_block_constant_test({32, 24, 16}); } +} + +TEMPLATE_TEST_CASE("In-Cache Block fused quantization matches the unfused path", + "[mgard-x][in-cache][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + bool prep_huffman = GENERATE(false, true); + CAPTURE(prep_huffman); + SECTION("single block") { + run_in_cache_block_fused_test({8, 8, 8}, prep_huffman); + } + SECTION("cubic") { + run_in_cache_block_fused_test({64, 64, 64}, prep_huffman); + } + SECTION("non-cubic") { + run_in_cache_block_fused_test({40, 64, 24}, prep_huffman); + } +} + +TEMPLATE_TEST_CASE("In-Cache Block fused path handles unpadded shapes", + "[mgard-x][in-cache][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + SECTION("partial block") { run_in_cache_block_padding_test({5, 5, 5}); } + SECTION("ragged, all dims") { run_in_cache_block_padding_test({60, 60, 60}); } + SECTION("ragged, one dim") { run_in_cache_block_padding_test({64, 64, 39}); } +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_DirectInterleaver.cpp.in b/tests/mgard-x/src/test_DirectInterleaver.cpp.in new file mode 100644 index 0000000000..c9a42220a8 --- /dev/null +++ b/tests/mgard-x/src/test_DirectInterleaver.cpp.in @@ -0,0 +1,143 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +#include "catch2/catch_template_test_macros.hpp" +#include "catch2/catch_test_macros.hpp" + +#include +#include +#include + +#include "mgard-x/Hierarchy/Hierarchy.hpp" +#include "mgard-x/MDR-X/Interleaver/DirectInterleaver.hpp" +#include "mgard-x/RuntimeX/RuntimeX.h" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; + +// Smooth, multi-dimensional test data (same generator as test_Compressor / +// test_HybridHierarchy): a separable product of sines over the normalized +// grid coordinates. The actual values don't matter here -- DirectInterleaver +// only moves elements around by index, it never touches their content -- but +// a smooth field makes it easy to spot a misplacement by eye if this ever +// needs debugging. +template +void fill_smooth_data(std::vector &data, std::vector shape) { + const double pi = std::acos(-1.0); + SIZE n = data.size(); + for (SIZE idx = 0; idx < n; idx++) { + SIZE r = idx; + double v = 1.0; + for (int d = static_cast(D) - 1; d >= 0; d--) { + SIZE coord = r % shape[d]; + r /= shape[d]; + double x = static_cast(coord) / static_cast(shape[d]); + v *= std::sin(2.0 * pi * x + 0.5 * d); + } + data[idx] = static_cast(v); + } +} + +// DirectInterleaver::interleave scatters a D-dimensional decomposed array +// into l_target+1 flat per-level buffers by index (which MG level each grid +// point belongs to), and reposition is the exact inverse gather. Since this +// is a pure permutation -- no quantization, no arithmetic on the values -- +// interleave followed by reposition into a fresh, zeroed array of the same +// shape must reproduce the original data bit-for-bit, for every element. +template +void run_direct_interleaver_roundtrip(std::vector shape) { + SIZE n = 1; + for (auto s : shape) + n *= s; + + std::vector original(n); + fill_smooth_data(original, shape); + + Config config; + Hierarchy hierarchy(shape, config); + MDR::DirectInterleaver interleaver(hierarchy); + DeviceRuntime::SyncQueue(0); + + Array data(shape); + data.load(original.data()); + + SIZE l_target = hierarchy.l_target(); + std::vector> level_arrays(l_target + 1); + std::vector> level_subarrays(l_target + 1); + for (SIZE l = 0; l < l_target + 1; l++) { + level_arrays[l].resize({hierarchy.level_num_elems(l)}, 0); + level_subarrays[l] = SubArray<1, T, DevType>(level_arrays[l]); + } + + interleaver.interleave(SubArray(data), level_subarrays, l_target, 0); + DeviceRuntime::SyncQueue(0); + + // Every level buffer must have been fully claimed: reposition reads every + // slot of every level buffer, so if interleave left any element + // unwritten (a gap in the level-to-index mapping), the round trip below + // would read stale/uninitialized memory there instead of failing loudly. + // Summing the per-level counts against the total catches that directly. + SIZE total_level_elems = 0; + for (SIZE l = 0; l < l_target + 1; l++) + total_level_elems += hierarchy.level_num_elems(l); + REQUIRE(total_level_elems == n); + + Array reconstructed(shape); + reconstructed.memset(0, 0); + interleaver.reposition(level_subarrays, SubArray(reconstructed), l_target, 0); + DeviceRuntime::SyncQueue(0); + + T *result = reconstructed.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + SIZE mismatches = 0; + for (SIZE i = 0; i < n; i++) + mismatches += (result[i] != original[i]); + CAPTURE(mismatches, n); + REQUIRE(mismatches == 0); +} + +TEMPLATE_TEST_CASE("MDR-X DirectInterleaver round trip 1D", + "[mgard-x][mdr-x][direct-interleaver][@DEVICE_TYPE@]", + float, double) { + DeviceRuntime::Initialize(); + SECTION("power-of-two-ish") { + run_direct_interleaver_roundtrip<1, TestType, TestDevice>({1024}); + } + SECTION("irregular size") { + run_direct_interleaver_roundtrip<1, TestType, TestDevice>({1003}); + } + SECTION("smaller than one thread block") { + run_direct_interleaver_roundtrip<1, TestType, TestDevice>({64}); + } +} + +TEMPLATE_TEST_CASE("MDR-X DirectInterleaver round trip 2D", + "[mgard-x][mdr-x][direct-interleaver][@DEVICE_TYPE@]", + float, double) { + DeviceRuntime::Initialize(); + SECTION("square") { + run_direct_interleaver_roundtrip<2, TestType, TestDevice>({64, 64}); + } + SECTION("non-square, irregular") { + run_direct_interleaver_roundtrip<2, TestType, TestDevice>({60, 47}); + } +} + +TEMPLATE_TEST_CASE("MDR-X DirectInterleaver round trip 3D", + "[mgard-x][mdr-x][direct-interleaver][@DEVICE_TYPE@]", + float, double) { + DeviceRuntime::Initialize(); + SECTION("cubic") { + run_direct_interleaver_roundtrip<3, TestType, TestDevice>({32, 32, 32}); + } + SECTION("non-cubic, irregular") { + run_direct_interleaver_roundtrip<3, TestType, TestDevice>({33, 31, 17}); + } +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_Huffman.cpp.in b/tests/mgard-x/src/test_Huffman.cpp.in new file mode 100644 index 0000000000..b5676ec493 --- /dev/null +++ b/tests/mgard-x/src/test_Huffman.cpp.in @@ -0,0 +1,131 @@ +/* + * Copyright 2025, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +#include "catch2/catch_test_macros.hpp" + +#include +#include +#include + +#include "mgard-x/RuntimeX/RuntimeX.h" +#include "mgard-x/Lossless/ParallelHuffman/Huffman.hpp" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; + +// Typical configuration matching DefaultLevelCompressor: +// Q = uint8_t (primary byte type) +// S = uint8_t (same — no signed outlier separation on primary path) +// H = uint64_t (Huffman codeword type) +// dict_size = 256, chunk_size = 1024 +TEST_CASE("Huffman CompressPrimary", "[mgard-x][@DEVICE_TYPE@]") { + using Q = uint8_t; + using S = uint8_t; + using H = uint64_t; + + const int dict_size = 256; + const int chunk_size = 1024; + const SIZE n = 10000; + + DeviceRuntime::Initialize(); + + // Skewed distribution (values 0–63) so Huffman achieves meaningful compression + std::vector original(n); + std::srand(42); + for (SIZE i = 0; i < n; i++) { + original[i] = static_cast(std::rand() % 64); + } + + // Compressor — use Resize so that signature_verify is allocated. + Huffman compressor; + compressor.Resize(n, dict_size, chunk_size, 0.0, 0); + DeviceRuntime::SyncQueue(0); + + // Primary path has no outliers — must be set to 0 before Serialize. + compressor.outlier_count = 0; + + Array<1, Q, TestDevice> primary_data({n}); + primary_data.load(original.data()); + + Array<1, Byte, TestDevice> compressed_data; + bool success = compressor.CompressPrimary(primary_data, compressed_data, 0.0, 0); + REQUIRE(success == true); + + compressor.Serialize(compressed_data, 0); + + // Decompressor — entirely separate object with no shared internal state. + Huffman decompressor; + decompressor.Resize(n, dict_size, chunk_size, 0.0, 0); + DeviceRuntime::SyncQueue(0); + + decompressor.Deserialize(compressed_data, 0); + + Array<1, Q, TestDevice> decompressed_data; + decompressor.DecompressPrimary(compressed_data, decompressed_data, 0); + + Q *result = decompressed_data.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + for (SIZE i = 0; i < n; i++) { + REQUIRE(result[i] == original[i]); + } +} + +// Tests the full pipeline: dictionary shift + outlier separation + Huffman coding. +// Compress modifies the device input in-place, so original must be saved on host. +// Decompress needs a pre-allocated output array because it wraps it via an +// external pointer before calling DecompressPrimary internally. +TEST_CASE("Huffman Compress", "[mgard-x][@DEVICE_TYPE@]") { + using Q = uint8_t; + using S = uint8_t; + using H = uint64_t; + + const int dict_size = 256; + const int chunk_size = 1024; + const SIZE n = 10000; + + DeviceRuntime::Initialize(); + + std::vector original(n); + std::srand(42); + for (SIZE i = 0; i < n; i++) { + original[i] = static_cast(std::rand() % 64); + } + + // Load to device before compressing; keep host copy for comparison since + // Compress shifts values in the device array in-place. + Array<1, S, TestDevice> input_data({n}); + input_data.load(original.data()); + + Huffman compressor; + compressor.Resize(n, dict_size, chunk_size, 0.0, 0); + DeviceRuntime::SyncQueue(0); + + Array<1, Byte, TestDevice> compressed_data; + bool success = compressor.Compress(input_data, compressed_data, 0.0, 0); + REQUIRE(success == true); + + compressor.Serialize(compressed_data, 0); + + Huffman decompressor; + decompressor.Resize(n, dict_size, chunk_size, 0.0, 0); + DeviceRuntime::SyncQueue(0); + + decompressor.Deserialize(compressed_data, 0); + + Array<1, S, TestDevice> decompressed_data({n}); + decompressor.Decompress(compressed_data, decompressed_data, 0); + + S *result = decompressed_data.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + for (SIZE i = 0; i < n; i++) { + REQUIRE(result[i] == original[i]); + } +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_HybridHierarchy.cpp.in b/tests/mgard-x/src/test_HybridHierarchy.cpp.in new file mode 100644 index 0000000000..b3ba6d439c --- /dev/null +++ b/tests/mgard-x/src/test_HybridHierarchy.cpp.in @@ -0,0 +1,1598 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +#include "catch2/catch_template_test_macros.hpp" +#include "catch2/catch_test_macros.hpp" +#include "catch2/generators/catch_generators.hpp" + +#include +#include +#include +#include +#include +#include + +#include "mgard-x/RuntimeX/RuntimeX.h" +// CompressionHighLevel.h declares the public API in terms of Config but does +// not include it. +#include "mgard-x/Config/Config.h" +#include "mgard-x/CompressionHighLevel/CompressionHighLevel.h" +#include "mgard-x/CompressionLowLevel/HybridHierarchyCompressor.hpp" +#include "mgard-x/DataRefactoring/BlockLocalHierarchyDataRefactor.hpp" +#include "mgard-x/DataRefactoring/HybridHierarchyDataRefactor.hpp" +#include "mgard-x/Utilities/ErrorCalculator.h" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; + +// Smooth, multi-dimensional test data: a separable product of sines over the +// normalized grid coordinates (same generator as test_Compressor). +template +void fill_smooth_data(std::vector &data, std::vector shape) { + const double pi = std::acos(-1.0); + SIZE n = data.size(); + for (SIZE idx = 0; idx < n; idx++) { + SIZE r = idx; + double v = 1.0; + for (int d = static_cast(D) - 1; d >= 0; d--) { + SIZE coord = r % shape[d]; + r /= shape[d]; + double x = static_cast(coord) / static_cast(shape[d]); + v *= std::sin(2.0 * pi * x + 0.5 * d); + } + data[idx] = static_cast(v); + } +} + +// Decompose -> Recompose with the block-local (in-cache) refactor must be a +// numerical identity: no quantization is involved, only the forward and +// inverse block transforms. Shapes must be multiples of 8 per level (the +// hybrid compressor pads to this internally). Templated on D so the 1D (8), +// 2D (8x8) and 3D (8x8x8) block kernels are all covered. +template +void run_block_local_roundtrip(std::vector shape, int num_local_levels, + bool orthogonal_projection = true) { + SIZE n = 1; + for (auto s : shape) + n *= s; + + std::vector original(n); + fill_smooth_data(original, shape); + + Config config; + config.num_local_refactoring_level = num_local_levels; + Hierarchy hierarchy(shape, config); + + data_refactoring::BlockLocalHierarchyDataRefactor refactor; + refactor.Adapt(hierarchy, config, 0); + DeviceRuntime::SyncQueue(0); + + Array data(shape); + data.load(original.data()); + Array<1, T, DevType> decomposed({(SIZE)refactor.DecomposedDataSize()}); + + refactor.Decompose(SubArray(data), SubArray(decomposed), 0, + orthogonal_projection); + DeviceRuntime::SyncQueue(0); + + // Recompose into a fresh output array so reconstruction can only come from + // the decomposed representation. + Array reconstructed(shape); + reconstructed.memset(0, 0); + refactor.Recompose(SubArray(reconstructed), SubArray(decomposed), 0, + orthogonal_projection); + + T *result = reconstructed.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + const double tolerance = std::is_same::value ? 1e-10 : 1e-3; + double error = + L_inf_error(n, original.data(), result, error_bound_type::REL); + REQUIRE(error < tolerance); +} + +// Decompose -> Recompose with the hybrid refactor (block-local levels chained +// into the standard global decomposition) must also be a numerical identity. +template +void run_hybrid_refactor_roundtrip(std::vector shape, + int num_local_levels, + int num_global_levels, + bool orthogonal_projection = true) { + SIZE n = 1; + for (auto s : shape) + n *= s; + + std::vector original(n); + fill_smooth_data(original, shape); + + Config config; + config.num_local_refactoring_level = num_local_levels; + config.num_global_refactoring_level = num_global_levels; + Hierarchy hierarchy(shape, config); + + data_refactoring::HybridHierarchyDataRefactor refactor; + refactor.Adapt(hierarchy, config, 0); + DeviceRuntime::SyncQueue(0); + + Array data(shape); + data.load(original.data()); + Array<1, T, DevType> decomposed({(SIZE)refactor.DecomposedDataSize()}); + + refactor.Decompose(SubArray(data), SubArray(decomposed), 0, + orthogonal_projection); + DeviceRuntime::SyncQueue(0); + + Array reconstructed(shape); + reconstructed.memset(0, 0); + refactor.Recompose(SubArray(reconstructed), SubArray(decomposed), 0, + orthogonal_projection); + + T *result = reconstructed.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + const double tolerance = std::is_same::value ? 1e-10 : 1e-3; + double error = + L_inf_error(n, original.data(), result, error_bound_type::REL); + REQUIRE(error < tolerance); +} + +// Full hybrid compress -> decompress round trip through serialized bytes. +template +void run_hybrid_compressor_test( + std::vector shape, int num_local_levels, int num_global_levels, + enum error_bound_type ebtype, T tol, bool fused = true, + T s = std::numeric_limits::infinity(), + compression_projection_mode_type projection_mode = + compression_projection_mode_type::Orthogonal) { + SIZE n = 1; + for (auto sz : shape) + n *= sz; + + std::vector original(n); + fill_smooth_data(original, shape); + + Config config; + config.decomposition = decomposition_type::Hybrid; + config.num_local_refactoring_level = num_local_levels; + config.num_global_refactoring_level = num_global_levels; + config.projection_mode = projection_mode; + // The fused and unfused local kernels are separate implementations of the + // same transform, so both have to be driven. + config.fuse_decompose_quantize = fused; + config.fuse_dequantize_recompose = fused; + + // Separate compressor and decompressor objects so the round trip only + // communicates through the serialized bytes (and the norm, as it would be + // carried in metadata in real usage). + Hierarchy hierarchy(shape, config); + HybridHierarchyCompressor compressor; + compressor.Adapt(hierarchy, config, 0); + HybridHierarchyCompressor decompressor; + decompressor.Adapt(hierarchy, config, 0); + DeviceRuntime::SyncQueue(0); + + Array original_array(shape); + original_array.load(original.data()); + + T norm = 0; + Array<1, Byte, DevType> compressed_array; + compressor.Compress(original_array, ebtype, tol, s, norm, compressed_array, + 0); + DeviceRuntime::SyncQueue(0); + + Array decompressed_array; + decompressor.Decompress(compressed_array, ebtype, tol, s, norm, + decompressed_array, 0); + T *result = decompressed_array.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + // Match the metric to the requested norm, as print_statistics and + // test_Compressor do: L_inf when s == inf, L_2 otherwise. Both metrics + // already return a relative error in REL mode, so tol compares directly. + double actual_error; + if (s == std::numeric_limits::infinity()) { + actual_error = L_inf_error(n, original.data(), result, ebtype); + } else { + actual_error = L_2_error(shape, original.data(), result, ebtype, + config.normalize_coordinates); + } + CAPTURE(actual_error, s); + REQUIRE(actual_error < static_cast(tol)); +} + +TEST_CASE("compression projection mode defaults to Auto", + "[mgard-x][hybrid]") { + const float inf = std::numeric_limits::infinity(); + + REQUIRE(Config().projection_mode == compression_projection_mode_type::Auto); + // Auto resolves from s alone: Hierarchical under an L-infinity bound, + // Orthogonal otherwise. + REQUIRE(resolve_projection_mode(compression_projection_mode_type::Auto, + inf) == + compression_projection_mode_type::Hierarchical); + REQUIRE(resolve_projection_mode(compression_projection_mode_type::Auto, + 0.0f) == + compression_projection_mode_type::Orthogonal); + REQUIRE(resolve_projection_mode( + compression_projection_mode_type::Orthogonal, inf) == + compression_projection_mode_type::Orthogonal); + REQUIRE_THROWS_AS(resolve_projection_mode( + compression_projection_mode_type::Hierarchical, 0.0f), + ProcessingException); + REQUIRE_THROWS_AS(resolve_projection_mode( + static_cast(255), + inf), + ProcessingException); +} + +TEMPLATE_TEST_CASE("Block-Local (In-Cache 8x8x8) Decomposition 3D", + "[mgard-x][hybrid][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + bool orthogonal_projection = GENERATE(true, false); + CAPTURE(orthogonal_projection); + SECTION("one level, cubic") { + run_block_local_roundtrip<3, TestType, TestDevice>( + {64, 64, 64}, 1, orthogonal_projection); + } + SECTION("one level, non-cubic") { + run_block_local_roundtrip<3, TestType, TestDevice>( + {40, 64, 56}, 1, orthogonal_projection); + } + SECTION("one level, padded (non-multiple-of-8)") { + run_block_local_roundtrip<3, TestType, TestDevice>( + {60, 60, 60}, 1, orthogonal_projection); + } + SECTION("one level, padded non-cubic") { + run_block_local_roundtrip<3, TestType, TestDevice>( + {65, 48, 39}, 1, orthogonal_projection); + } + SECTION("two levels") { + run_block_local_roundtrip<3, TestType, TestDevice>( + {64, 64, 64}, 2, orthogonal_projection); + } + SECTION("two levels, padded") { + run_block_local_roundtrip<3, TestType, TestDevice>( + {60, 60, 60}, 2, orthogonal_projection); + } +} + +TEMPLATE_TEST_CASE("Hybrid Hierarchy Decomposition 3D", + "[mgard-x][hybrid][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + bool orthogonal_projection = GENERATE(true, false); + CAPTURE(orthogonal_projection); + SECTION("pure local (L=1, M=0)") { + run_hybrid_refactor_roundtrip<3, TestType, TestDevice>( + {64, 64, 64}, 1, 0, orthogonal_projection); + } + SECTION("pure local (L=2, M=0)") { + run_hybrid_refactor_roundtrip<3, TestType, TestDevice>( + {64, 64, 64}, 2, 0, orthogonal_projection); + } + SECTION("pure global (L=0, M=2)") { + run_hybrid_refactor_roundtrip<3, TestType, TestDevice>( + {64, 64, 64}, 0, 2, orthogonal_projection); + } + SECTION("hybrid (L=1, M=2)") { + run_hybrid_refactor_roundtrip<3, TestType, TestDevice>( + {64, 64, 64}, 1, 2, orthogonal_projection); + } + SECTION("hybrid, padded (L=1, M=2)") { + run_hybrid_refactor_roundtrip<3, TestType, TestDevice>( + {60, 60, 60}, 1, 2, orthogonal_projection); + } +} + +TEMPLATE_TEST_CASE("Hybrid Hierarchy Compressor 3D", + "[mgard-x][hybrid][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + enum error_bound_type ebtype = + GENERATE(error_bound_type::REL, error_bound_type::ABS); + TestType tol = GENERATE(static_cast(1e-2), + static_cast(1e-3)); + bool fused = GENERATE(true, false); + const TestType s = std::numeric_limits::infinity(); + compression_projection_mode_type projection_mode = + GENERATE(compression_projection_mode_type::Orthogonal, + compression_projection_mode_type::Hierarchical); + CAPTURE(static_cast(ebtype), tol, fused, + static_cast(projection_mode)); + SECTION("pure local (L=1, M=0)") { + run_hybrid_compressor_test<3, TestType, TestDevice>({64, 64, 64}, 1, 0, + ebtype, tol, fused, s, + projection_mode); + } + SECTION("hybrid (L=1, M=2)") { + run_hybrid_compressor_test<3, TestType, TestDevice>({64, 64, 64}, 1, 2, + ebtype, tol, fused, s, + projection_mode); + } + SECTION("padded (L=1, M=0)") { + run_hybrid_compressor_test<3, TestType, TestDevice>({60, 60, 60}, 1, 0, + ebtype, tol, fused, s, + projection_mode); + } +} + +// The block transform is an MGARD level transform, not just some invertible +// shuffle: its interpolation is exact for multilinear data. So for +// f = prod_d (1 + a_d * coord_d) every coefficient must be exactly zero +// (each is a value minus the average of its coarse neighbours, which for a +// multilinear f reproduces it), and with a zero coefficient field the +// correction vanishes too, leaving every coarse output equal to f at that +// coarse node. +// +// A round trip cannot see this: an index or sign slip made symmetrically in +// decompose and recompose still round trips. This pins the forward transform +// on its own. +template +void run_block_local_multilinear_test(std::vector shape) { + SIZE n = 1; + for (auto sz : shape) + n *= sz; + for (auto sz : shape) + REQUIRE(sz % 8 == 0); // no padding: f is undefined outside the extent + + // f(coords) = prod_d (1 + a_d * coord_d), multilinear by construction. + const double a[3] = {0.25, -0.5, 0.125}; + std::vector original(n); + auto value_at = [&](const std::vector &coord) { + double v = 1.0; + for (DIM d = 0; d < D; d++) + v *= 1.0 + a[d] * (double)coord[d]; + return (T)v; + }; + for (SIZE idx = 0; idx < n; idx++) { + SIZE r = idx; + std::vector coord(D); + for (int d = (int)D - 1; d >= 0; d--) { + coord[d] = r % shape[d]; + r /= shape[d]; + } + original[idx] = value_at(coord); + } + + Config config; + config.num_local_refactoring_level = 1; + Hierarchy hierarchy(shape, config); + + data_refactoring::BlockLocalHierarchyDataRefactor refactor; + refactor.Adapt(hierarchy, config, 0); + DeviceRuntime::SyncQueue(0); + + Array data(shape); + data.load(original.data()); + Array<1, T, DevType> decomposed({(SIZE)refactor.DecomposedDataSize()}); + refactor.Decompose(SubArray(data), SubArray(decomposed), 0); + DeviceRuntime::SyncQueue(0); + + T *out = decomposed.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + // Scale the tolerance by the data range: these are absolute comparisons. + double magnitude = 0; + for (SIZE i = 0; i < n; i++) + magnitude = std::max(magnitude, (double)std::abs(original[i])); + const double tolerance = + magnitude * (std::is_same::value ? 1e-12 : 1e-5); + + // Every coefficient is zero. + SIZE num_coarse = 1; + for (DIM d = 0; d < D; d++) + num_coarse *= refactor.coarse_shapes[0][d]; + double worst_coeff = 0; + for (SIZE i = num_coarse; i < (SIZE)refactor.DecomposedDataSize(); i++) + worst_coeff = std::max(worst_coeff, (double)std::abs(out[i])); + CAPTURE(worst_coeff, tolerance); + REQUIRE(worst_coeff < tolerance); + + // Every coarse value is f at that coarse node: coarse index (b, c) along a + // dimension sits at fine node b * 8 + {0, 2, 4, 6, 7}[c]. + const SIZE coarse_node[5] = {0, 2, 4, 6, 7}; + double worst_coarse = 0; + for (SIZE i = 0; i < num_coarse; i++) { + SIZE r = i; + std::vector coord(D); + for (int d = (int)D - 1; d >= 0; d--) { + SIZE c = r % refactor.coarse_shapes[0][d]; + r /= refactor.coarse_shapes[0][d]; + coord[d] = (c / 5) * 8 + coarse_node[c % 5]; + } + worst_coarse = + std::max(worst_coarse, (double)std::abs(out[i] - value_at(coord))); + } + CAPTURE(worst_coarse); + REQUIRE(worst_coarse < tolerance); +} + +TEMPLATE_TEST_CASE("Block-local transform is exact for multilinear data", + "[mgard-x][hybrid][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + SECTION("1D") { + run_block_local_multilinear_test<1, TestType, TestDevice>({1024}); + } + SECTION("2D") { + run_block_local_multilinear_test<2, TestType, TestDevice>({64, 48}); + } + SECTION("3D") { + run_block_local_multilinear_test<3, TestType, TestDevice>({32, 24, 40}); + } +} + +// --------------------------------------------------------------------------- +// 1D and 2D block-local transforms +// +// The block geometry is per-dimension identical to the 3D block -- 8 fine +// nodes coarsen to the 5 nodes {0, 2, 4, 6, 7}, with the trailing cell split +// around a phantom node -- so the same cases apply: exact shapes, shapes that +// need padding up to a multiple of 8, and more than one level. +// +// The 1D kernel additionally packs several 8-element tiles into one thread +// block, so its shapes deliberately mix a whole number of thread blocks with +// one that leaves the last block partly idle. +// --------------------------------------------------------------------------- + +TEMPLATE_TEST_CASE("Block-Local (In-Cache 8x8) Decomposition 2D", + "[mgard-x][hybrid][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + SECTION("one level, square") { + run_block_local_roundtrip<2, TestType, TestDevice>({64, 64}, 1); + } + SECTION("one level, non-square") { + run_block_local_roundtrip<2, TestType, TestDevice>({40, 56}, 1); + } + SECTION("one level, padded (non-multiple-of-8)") { + run_block_local_roundtrip<2, TestType, TestDevice>({60, 60}, 1); + } + SECTION("one level, padded non-square") { + run_block_local_roundtrip<2, TestType, TestDevice>({65, 39}, 1); + } + SECTION("two levels") { + run_block_local_roundtrip<2, TestType, TestDevice>({64, 64}, 2); + } + SECTION("two levels, padded") { + run_block_local_roundtrip<2, TestType, TestDevice>({60, 60}, 2); + } +} + +TEMPLATE_TEST_CASE("Block-Local (In-Cache 8) Decomposition 1D", + "[mgard-x][hybrid][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + SECTION("one level, whole thread blocks") { + run_block_local_roundtrip<1, TestType, TestDevice>({1024}, 1); + } + SECTION("one level, partial last thread block") { + run_block_local_roundtrip<1, TestType, TestDevice>({1000}, 1); + } + SECTION("one level, padded (non-multiple-of-8)") { + run_block_local_roundtrip<1, TestType, TestDevice>({1003}, 1); + } + SECTION("one level, smaller than one thread block") { + run_block_local_roundtrip<1, TestType, TestDevice>({64}, 1); + } + SECTION("two levels") { + run_block_local_roundtrip<1, TestType, TestDevice>({1024}, 2); + } + SECTION("two levels, padded") { + run_block_local_roundtrip<1, TestType, TestDevice>({1003}, 2); + } +} + +TEMPLATE_TEST_CASE("Hybrid Hierarchy Decomposition 2D", + "[mgard-x][hybrid][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + SECTION("pure local (L=1, M=0)") { + run_hybrid_refactor_roundtrip<2, TestType, TestDevice>({64, 64}, 1, 0); + } + SECTION("pure local (L=2, M=0)") { + run_hybrid_refactor_roundtrip<2, TestType, TestDevice>({64, 64}, 2, 0); + } + SECTION("hybrid (L=1, M=2)") { + run_hybrid_refactor_roundtrip<2, TestType, TestDevice>({64, 64}, 1, 2); + } + SECTION("hybrid, padded (L=1, M=2)") { + run_hybrid_refactor_roundtrip<2, TestType, TestDevice>({60, 60}, 1, 2); + } +} + +TEMPLATE_TEST_CASE("Hybrid Hierarchy Decomposition 1D", + "[mgard-x][hybrid][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + SECTION("pure local (L=1, M=0)") { + run_hybrid_refactor_roundtrip<1, TestType, TestDevice>({1024}, 1, 0); + } + SECTION("pure local (L=2, M=0)") { + run_hybrid_refactor_roundtrip<1, TestType, TestDevice>({1024}, 2, 0); + } + SECTION("hybrid (L=1, M=2)") { + run_hybrid_refactor_roundtrip<1, TestType, TestDevice>({1024}, 1, 2); + } + SECTION("hybrid, padded (L=1, M=2)") { + run_hybrid_refactor_roundtrip<1, TestType, TestDevice>({1000}, 1, 2); + } +} + +TEMPLATE_TEST_CASE("Hybrid Hierarchy Compressor 2D", + "[mgard-x][hybrid][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + enum error_bound_type ebtype = + GENERATE(error_bound_type::REL, error_bound_type::ABS); + TestType tol = GENERATE(static_cast(1e-2), + static_cast(1e-3)); + bool fused = GENERATE(true, false); + const TestType s = std::numeric_limits::infinity(); + compression_projection_mode_type projection_mode = + GENERATE(compression_projection_mode_type::Orthogonal, + compression_projection_mode_type::Hierarchical); + CAPTURE(static_cast(ebtype), tol, fused, + static_cast(projection_mode)); + SECTION("pure local (L=1, M=0)") { + run_hybrid_compressor_test<2, TestType, TestDevice>({256, 256}, 1, 0, + ebtype, tol, fused, s, + projection_mode); + } + SECTION("hybrid (L=1, M=2)") { + run_hybrid_compressor_test<2, TestType, TestDevice>({256, 256}, 1, 2, + ebtype, tol, fused, s, + projection_mode); + } + SECTION("padded (L=1, M=0)") { + run_hybrid_compressor_test<2, TestType, TestDevice>({250, 250}, 1, 0, + ebtype, tol, fused, s, + projection_mode); + } +} + +TEMPLATE_TEST_CASE("Hybrid Hierarchy Compressor 1D", + "[mgard-x][hybrid][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + enum error_bound_type ebtype = + GENERATE(error_bound_type::REL, error_bound_type::ABS); + TestType tol = GENERATE(static_cast(1e-2), + static_cast(1e-3)); + bool fused = GENERATE(true, false); + const TestType s = std::numeric_limits::infinity(); + compression_projection_mode_type projection_mode = + GENERATE(compression_projection_mode_type::Orthogonal, + compression_projection_mode_type::Hierarchical); + CAPTURE(static_cast(ebtype), tol, fused, + static_cast(projection_mode)); + SECTION("pure local (L=1, M=0)") { + run_hybrid_compressor_test<1, TestType, TestDevice>({65536}, 1, 0, ebtype, + tol, fused, s, + projection_mode); + } + SECTION("hybrid (L=1, M=2)") { + run_hybrid_compressor_test<1, TestType, TestDevice>({65536}, 1, 2, ebtype, + tol, fused, s, + projection_mode); + } + SECTION("padded (L=1, M=0)") { + run_hybrid_compressor_test<1, TestType, TestDevice>({65000}, 1, 0, ebtype, + tol, fused, s, + projection_mode); + } +} + +// --------------------------------------------------------------------------- +// Projection mode vs. error norm +// +// resolve_projection_mode() (tested directly above, in "compression +// projection mode defaults to Auto") says the hierarchical (BlockMGARD) basis +// only supports L-infinity error control, while the orthogonal basis supports +// both L_inf and L_2. These cases pin that contract +// at the level callers actually hit: a real Compress()/Decompress() round +// trip for every combination that is supposed to work, and Compress() itself +// throwing for the one that is not. +// +// An s-norm (L_2) bound has its own, separate requirement, enforced by +// HybridHierarchyQuantizer::CheckSNormConfiguration: exactly one block-local +// level and no global stage (L=1, M=0), since only then is every block +// transformed independently and the error budget divides evenly across them. +// L=1, M=0 is used throughout so that varies only what this suite is actually +// after -- projection mode and norm -- without also tripping that unrelated +// constraint. +// --------------------------------------------------------------------------- + +TEMPLATE_TEST_CASE("Hybrid Hierarchy Compressor projection mode vs error norm", + "[mgard-x][hybrid][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + const TestType inf = std::numeric_limits::infinity(); + const TestType l2_s = static_cast(0); + const TestType tol = static_cast(1e-2); + + SECTION("3D") { + SECTION("orthogonal, L_inf") { + run_hybrid_compressor_test<3, TestType, TestDevice>( + {64, 64, 64}, 1, 0, error_bound_type::REL, tol, true, inf, + compression_projection_mode_type::Orthogonal); + } + SECTION("orthogonal, L_2") { + run_hybrid_compressor_test<3, TestType, TestDevice>( + {64, 64, 64}, 1, 0, error_bound_type::REL, tol, true, l2_s, + compression_projection_mode_type::Orthogonal); + } + SECTION("hierarchical, L_inf") { + run_hybrid_compressor_test<3, TestType, TestDevice>( + {64, 64, 64}, 1, 0, error_bound_type::REL, tol, true, inf, + compression_projection_mode_type::Hierarchical); + } + SECTION("auto, L_inf") { + run_hybrid_compressor_test<3, TestType, TestDevice>( + {64, 64, 64}, 1, 0, error_bound_type::REL, tol, true, inf, + compression_projection_mode_type::Auto); + } + SECTION("auto, L_2") { + run_hybrid_compressor_test<3, TestType, TestDevice>( + {64, 64, 64}, 1, 0, error_bound_type::REL, tol, true, l2_s, + compression_projection_mode_type::Auto); + } + } + SECTION("2D") { + SECTION("orthogonal, L_inf") { + run_hybrid_compressor_test<2, TestType, TestDevice>( + {256, 256}, 1, 0, error_bound_type::REL, tol, true, inf, + compression_projection_mode_type::Orthogonal); + } + SECTION("orthogonal, L_2") { + run_hybrid_compressor_test<2, TestType, TestDevice>( + {256, 256}, 1, 0, error_bound_type::REL, tol, true, l2_s, + compression_projection_mode_type::Orthogonal); + } + SECTION("hierarchical, L_inf") { + run_hybrid_compressor_test<2, TestType, TestDevice>( + {256, 256}, 1, 0, error_bound_type::REL, tol, true, inf, + compression_projection_mode_type::Hierarchical); + } + SECTION("auto, L_inf") { + run_hybrid_compressor_test<2, TestType, TestDevice>( + {256, 256}, 1, 0, error_bound_type::REL, tol, true, inf, + compression_projection_mode_type::Auto); + } + SECTION("auto, L_2") { + run_hybrid_compressor_test<2, TestType, TestDevice>( + {256, 256}, 1, 0, error_bound_type::REL, tol, true, l2_s, + compression_projection_mode_type::Auto); + } + } + SECTION("1D") { + SECTION("orthogonal, L_inf") { + run_hybrid_compressor_test<1, TestType, TestDevice>( + {65536}, 1, 0, error_bound_type::REL, tol, true, inf, + compression_projection_mode_type::Orthogonal); + } + SECTION("orthogonal, L_2") { + run_hybrid_compressor_test<1, TestType, TestDevice>( + {65536}, 1, 0, error_bound_type::REL, tol, true, l2_s, + compression_projection_mode_type::Orthogonal); + } + SECTION("hierarchical, L_inf") { + run_hybrid_compressor_test<1, TestType, TestDevice>( + {65536}, 1, 0, error_bound_type::REL, tol, true, inf, + compression_projection_mode_type::Hierarchical); + } + SECTION("auto, L_inf") { + run_hybrid_compressor_test<1, TestType, TestDevice>( + {65536}, 1, 0, error_bound_type::REL, tol, true, inf, + compression_projection_mode_type::Auto); + } + SECTION("auto, L_2") { + run_hybrid_compressor_test<1, TestType, TestDevice>( + {65536}, 1, 0, error_bound_type::REL, tol, true, l2_s, + compression_projection_mode_type::Auto); + } + } +} + +TEMPLATE_TEST_CASE( + "Hybrid Hierarchy Compressor rejects hierarchical projection under L_2", + "[mgard-x][hybrid][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + constexpr DIM D = 3; + std::vector shape{64, 64, 64}; + SIZE n = shape[0] * shape[1] * shape[2]; + + std::vector original(n); + fill_smooth_data(original, shape); + + // L=1, M=0: the one decomposition an s-norm bound is otherwise allowed to + // use (see CheckSNormConfiguration), so the only thing left to reject is + // the projection mode itself. + Config config; + config.decomposition = decomposition_type::Hybrid; + config.num_local_refactoring_level = 1; + config.num_global_refactoring_level = 0; + config.projection_mode = compression_projection_mode_type::Hierarchical; + Hierarchy hierarchy(shape, config); + HybridHierarchyCompressor compressor; + compressor.Adapt(hierarchy, config, 0); + DeviceRuntime::SyncQueue(0); + + Array original_array(shape); + original_array.load(original.data()); + + // s = 0 selects the L_2 norm, which the hierarchical basis cannot control; + // Compress() must surface that rather than silently mishandling it. + TestType norm = 0; + Array<1, Byte, TestDevice> compressed_array; + REQUIRE_THROWS_AS( + compressor.Compress(original_array, error_bound_type::REL, + static_cast(1e-2), + static_cast(0), norm, compressed_array, 0), + ProcessingException); +} + +// --------------------------------------------------------------------------- +// Hybrid refactor internals +// +// The cases above drive Decompose/Recompose end to end. These pin the seams +// the rest of the hybrid stack leans on but that no test reached: the +// factored-out global entry points, reconstruction from a refactor that never +// ran Decompose, the shape contract between the local and global stages, and +// the L == M == 0 guard. +// --------------------------------------------------------------------------- + +// Copy a decomposed buffer back to the host so two variants can be compared. +template +static std::vector decomposed_to_host(Array<1, T, DevType> &array, SIZE n) { + T *host = array.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + return std::vector(host, host + n); +} + +template +static void adapt_refactor( + data_refactoring::HybridHierarchyDataRefactor<3, T, DevType> &refactor, + Hierarchy<3, T, DevType> &hierarchy, Config config) { + refactor.Adapt(hierarchy, config, 0); + DeviceRuntime::SyncQueue(0); +} + +// Decompose() with both stages enabled must be exactly the local stage +// followed by DecomposeGlobal(), and Recompose() exactly RecomposeGlobal() +// followed by the local stage. The fused quantization path drives the stages +// through those two entry points while the unfused path calls Decompose / +// Recompose, so if the two ever drift the fused and unfused compressors +// silently disagree -- a divergence that today only surfaces end to end. +template +void run_hybrid_stage_split_test(std::vector shape, int num_local_levels, + int num_global_levels) { + constexpr DIM D = 3; + SIZE n = 1; + for (auto sz : shape) + n *= sz; + + std::vector original(n); + fill_smooth_data(original, shape); + + Config config; + config.num_local_refactoring_level = num_local_levels; + config.num_global_refactoring_level = num_global_levels; + Hierarchy hierarchy(shape, config); + + data_refactoring::HybridHierarchyDataRefactor monolithic; + data_refactoring::HybridHierarchyDataRefactor staged; + adapt_refactor(monolithic, hierarchy, config); + adapt_refactor(staged, hierarchy, config); + + SIZE decomposed_size = (SIZE)monolithic.DecomposedDataSize(); + + Array data_a(shape); + data_a.load(original.data()); + Array<1, T, DevType> decomposed_a({decomposed_size}); + monolithic.Decompose(SubArray(data_a), SubArray(decomposed_a), 0); + DeviceRuntime::SyncQueue(0); + + Array data_b(shape); + data_b.load(original.data()); + Array<1, T, DevType> decomposed_b({decomposed_size}); + staged.local_refactor.Decompose(SubArray(data_b), SubArray(decomposed_b), 0); + staged.DecomposeGlobal(SubArray(decomposed_b), 0); + DeviceRuntime::SyncQueue(0); + + std::vector host_a = decomposed_to_host(decomposed_a, decomposed_size); + std::vector host_b = decomposed_to_host(decomposed_b, decomposed_size); + + SIZE decompose_mismatches = 0; + for (SIZE i = 0; i < decomposed_size; i++) + decompose_mismatches += (host_a[i] != host_b[i]); + REQUIRE(decompose_mismatches == 0); + + // Same split on the way back. Both stages run in place over the decomposed + // buffer, so each variant needs its own copy of it. + Array<1, T, DevType> input_c({decomposed_size}); + input_c.load(host_a.data()); + Array out_c(shape); + out_c.memset(0, 0); + monolithic.Recompose(SubArray(out_c), SubArray(input_c), 0); + DeviceRuntime::SyncQueue(0); + + Array<1, T, DevType> input_d({decomposed_size}); + input_d.load(host_a.data()); + Array out_d(shape); + out_d.memset(0, 0); + staged.RecomposeGlobal(SubArray(input_d), 0); + staged.local_refactor.Recompose(SubArray(out_d), SubArray(input_d), 0); + DeviceRuntime::SyncQueue(0); + + T *host_c = out_c.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + std::vector reconstructed_c(host_c, host_c + n); + T *host_d = out_d.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + SIZE recompose_mismatches = 0; + for (SIZE i = 0; i < n; i++) + recompose_mismatches += (reconstructed_c[i] != host_d[i]); + REQUIRE(recompose_mismatches == 0); + + // And the split path is still a faithful round trip, not merely consistent. + const double tolerance = std::is_same::value ? 1e-10 : 1e-3; + double error = L_inf_error(n, original.data(), host_d, error_bound_type::REL); + CAPTURE(error); + REQUIRE(error < tolerance); +} + +// Decompression in the wild runs on a refactor that never saw Decompose: the +// object is Adapted from file metadata and handed the decomposed bytes. The +// block-local stage restores its coarsest staging buffer and coefficient +// offset from that input precisely so this works, so reconstruct through a +// second, independently Adapted refactor rather than reusing the first. +template +void run_hybrid_standalone_recompose_test(std::vector shape, + int num_local_levels, + int num_global_levels) { + constexpr DIM D = 3; + SIZE n = 1; + for (auto sz : shape) + n *= sz; + + std::vector original(n); + fill_smooth_data(original, shape); + + Config config; + config.num_local_refactoring_level = num_local_levels; + config.num_global_refactoring_level = num_global_levels; + Hierarchy hierarchy(shape, config); + + data_refactoring::HybridHierarchyDataRefactor compressor_side; + adapt_refactor(compressor_side, hierarchy, config); + + SIZE decomposed_size = (SIZE)compressor_side.DecomposedDataSize(); + Array data(shape); + data.load(original.data()); + Array<1, T, DevType> decomposed({decomposed_size}); + compressor_side.Decompose(SubArray(data), SubArray(decomposed), 0); + DeviceRuntime::SyncQueue(0); + + // Hand the decomposed buffer to a refactor with no decomposition history. + Hierarchy decompress_hierarchy(shape, config); + data_refactoring::HybridHierarchyDataRefactor + decompressor_side; + adapt_refactor(decompressor_side, decompress_hierarchy, config); + + REQUIRE((SIZE)decompressor_side.DecomposedDataSize() == decomposed_size); + + Array reconstructed(shape); + reconstructed.memset(0, 0); + decompressor_side.Recompose(SubArray(reconstructed), SubArray(decomposed), 0); + + T *result = reconstructed.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + const double tolerance = std::is_same::value ? 1e-10 : 1e-3; + double error = L_inf_error(n, original.data(), result, error_bound_type::REL); + CAPTURE(error); + REQUIRE(error < tolerance); +} + +// The global stage is built on the output shape of the local stage. Adapt() +// derives that shape independently of the block-local refactor's own +// bookkeeping, so a change to either 8 -> 5 coarsening rule must move both. +template +void run_hybrid_shape_contract_test(std::vector shape, + int num_local_levels, + int num_global_levels) { + constexpr DIM D = 3; + Config config; + config.num_local_refactoring_level = num_local_levels; + config.num_global_refactoring_level = num_global_levels; + Hierarchy hierarchy(shape, config); + + data_refactoring::HybridHierarchyDataRefactor refactor; + adapt_refactor(refactor, hierarchy, config); + + std::vector local_coarsest = + refactor.local_refactor.coarse_shapes[num_local_levels - 1]; + std::vector global_finest = + refactor.global_hierarchy.level_shape( + refactor.global_hierarchy.l_target()); + + REQUIRE(global_finest.size() == local_coarsest.size()); + for (DIM d = 0; d < D; d++) { + CAPTURE(d, local_coarsest[d], global_finest[d]); + REQUIRE(global_finest[d] == local_coarsest[d]); + } + + // Each local level coarsens every extent by ceil(n / 8) * 5. + std::vector expected = shape; + for (int l = 0; l < num_local_levels; l++) + for (DIM d = 0; d < D; d++) + expected[d] = ((expected[d] - 1) / 8 + 1) * 5; + for (DIM d = 0; d < D; d++) { + CAPTURE(d, expected[d], local_coarsest[d]); + REQUIRE(local_coarsest[d] == expected[d]); + } +} + +TEMPLATE_TEST_CASE("Hybrid stages compose into Decompose/Recompose", + "[mgard-x][hybrid][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + SECTION("L=1, M=1") { + run_hybrid_stage_split_test({64, 64, 64}, 1, 1); + } + SECTION("L=1, M=2") { + run_hybrid_stage_split_test({64, 64, 64}, 1, 2); + } + SECTION("L=2, M=2") { + run_hybrid_stage_split_test({64, 64, 64}, 2, 2); + } + SECTION("non-cubic (L=1, M=2)") { + run_hybrid_stage_split_test({40, 64, 56}, 1, 2); + } + SECTION("padded (L=1, M=1)") { + run_hybrid_stage_split_test({60, 60, 60}, 1, 1); + } +} + +TEMPLATE_TEST_CASE("Hybrid Recompose works without a prior Decompose", + "[mgard-x][hybrid][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + SECTION("pure local (L=1, M=0)") { + run_hybrid_standalone_recompose_test({64, 64, 64}, 1, + 0); + } + SECTION("pure local (L=2, M=0)") { + run_hybrid_standalone_recompose_test({64, 64, 64}, 2, + 0); + } + SECTION("pure global (L=0, M=2)") { + run_hybrid_standalone_recompose_test({64, 64, 64}, 0, + 2); + } + SECTION("hybrid (L=1, M=2)") { + run_hybrid_standalone_recompose_test({64, 64, 64}, 1, + 2); + } + SECTION("hybrid, padded (L=1, M=2)") { + run_hybrid_standalone_recompose_test({60, 60, 60}, 1, + 2); + } +} + +TEMPLATE_TEST_CASE("Hybrid global stage is built on the local coarse shape", + "[mgard-x][hybrid][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + SECTION("cubic (L=1)") { + run_hybrid_shape_contract_test({64, 64, 64}, 1, 2); + } + SECTION("cubic (L=2)") { + run_hybrid_shape_contract_test({64, 64, 64}, 2, 2); + } + SECTION("non-cubic (L=1)") { + run_hybrid_shape_contract_test({40, 64, 56}, 1, 2); + } + SECTION("padded, ragged (L=1)") { + run_hybrid_shape_contract_test({65, 48, 39}, 1, 1); + } +} + +TEST_CASE("Hybrid refactor rejects having no levels at all", + "[mgard-x][hybrid][@DEVICE_TYPE@]") { + DeviceRuntime::Initialize(); + using T = float; + constexpr DIM D = 3; + std::vector shape{64, 64, 64}; + + Config config; + config.num_local_refactoring_level = 1; + Hierarchy hierarchy(shape, config); + + Config empty_config; + empty_config.num_local_refactoring_level = 0; + empty_config.num_global_refactoring_level = 0; + + data_refactoring::HybridHierarchyDataRefactor refactor; + REQUIRE_THROWS_AS(refactor.Adapt(hierarchy, empty_config, 0), + ProcessingException); +} + + +// --------------------------------------------------------------------------- +// End-to-end pipeline (public compress/decompress API) +// +// The compressor case above drives HybridHierarchyCompressor directly: no +// metadata header, no serialization, and the same Config handed to both +// sides. test_HybridMetadata covers the header in isolation, with no +// compression. Nothing joined the two, so the contract that actually matters +// to a caller -- write a hybrid file, hand the bytes back with no Config at +// all, get the data within tolerance -- went unchecked, as did the public API +// itself. +// --------------------------------------------------------------------------- + +// A free() that matches the allocation the API performs when +// output_pre_allocated is false. +struct MallocedBuffer { + void *ptr = nullptr; + size_t size = 0; + ~MallocedBuffer() { std::free(ptr); } + MallocedBuffer() = default; + MallocedBuffer(const MallocedBuffer &) = delete; + MallocedBuffer &operator=(const MallocedBuffer &) = delete; +}; + +template static data_type data_type_of() { + return std::is_same::value ? data_type::Double : data_type::Float; +} + +static Config hybrid_config(int num_local_levels, int num_global_levels) { + Config config; + config.decomposition = decomposition_type::Hybrid; + config.num_local_refactoring_level = num_local_levels; + config.num_global_refactoring_level = num_global_levels; + return config; +} + +// Compress through the public API and hand the bytes back to decompress with a +// *default* Config. The hybrid level counts have to survive the round trip +// through the metadata header (MetadataBase::InitializeConfig restores them), +// because a real decompressor has only the file. +template +void run_pipeline_test(std::vector shape, int num_local_levels, + int num_global_levels, enum error_bound_type ebtype, + double tol) { + // Start from a clean CompressorCache so this case measures one shape in + // isolation rather than inheriting buffers sized by whatever ran before. + // The shape-switch case below deliberately does not do this. + release_cache(); + SIZE n = 1; + for (auto sz : shape) + n *= sz; + + std::vector original(n); + fill_smooth_data(original, shape); + + const double s = std::numeric_limits::infinity(); + + MallocedBuffer compressed; + REQUIRE(compress(D, data_type_of(), shape, tol, s, ebtype, + original.data(), compressed.ptr, compressed.size, + hybrid_config(num_local_levels, num_global_levels), + false) == compress_status_type::Success); + REQUIRE(compressed.ptr != nullptr); + REQUIRE(compressed.size > 0); + // Deliberately no assertion that the stream beats the raw size: nothing in + // the API promises a ratio at an arbitrary tolerance, and at tol=1e-3 a + // 60^3 float block genuinely does not compress. Recorded for context only. + CAPTURE(compressed.size, n * sizeof(T)); + + // Decompress knowing nothing but the bytes. + MallocedBuffer decompressed; + data_type recovered_dtype; + std::vector recovered_shape; + REQUIRE(decompress(compressed.ptr, compressed.size, + decompressed.ptr, recovered_dtype, + recovered_shape, false) == + compress_status_type::Success); + + REQUIRE(recovered_dtype == data_type_of()); + REQUIRE(recovered_shape == shape); + + const T *result = (const T *)decompressed.ptr; + double error = L_inf_error(n, original.data(), result, ebtype); + CAPTURE(error, tol, compressed.size, n * sizeof(T)); + REQUIRE(error < tol); +} + +// Passing a Config to decompress must not change the result: the hybrid +// parameters come from the file either way. If the decompressor ever started +// honoring caller-supplied level counts over the file's, a hybrid stream would +// decode against the wrong buffer layout -- and the default-Config path above +// would be the one that broke. +template +void run_pipeline_config_independence_test(std::vector shape, + int num_local_levels, + int num_global_levels) { + constexpr DIM D = 3; + SIZE n = 1; + for (auto sz : shape) + n *= sz; + + std::vector original(n); + fill_smooth_data(original, shape); + + // Start from a clean CompressorCache so this case measures one shape in + // isolation rather than inheriting buffers sized by whatever ran before. + // The shape-switch case below deliberately does not do this. + release_cache(); + const double s = std::numeric_limits::infinity(); + const double tol = 1e-3; + + MallocedBuffer compressed; + REQUIRE(compress(D, data_type_of(), shape, tol, s, + error_bound_type::REL, original.data(), + compressed.ptr, compressed.size, + hybrid_config(num_local_levels, num_global_levels), + false) == compress_status_type::Success); + + MallocedBuffer from_default; + REQUIRE(decompress(compressed.ptr, compressed.size, + from_default.ptr, false) == + compress_status_type::Success); + + // A Config that disagrees with the file about the level split. + MallocedBuffer from_wrong_config; + REQUIRE(decompress(compressed.ptr, compressed.size, + from_wrong_config.ptr, hybrid_config(2, 1), + false) == compress_status_type::Success); + + const T *a = (const T *)from_default.ptr; + const T *b = (const T *)from_wrong_config.ptr; + SIZE mismatches = 0; + for (SIZE i = 0; i < n; i++) + mismatches += (a[i] != b[i]); + REQUIRE(mismatches == 0); +} + +// The hybrid switch must actually reach the pipeline. Compressing the same +// data with and without it has to produce different streams while both stay +// inside the requested bound -- otherwise the flag could be silently dropped +// somewhere between Config and the compressor and every tolerance check above +// would still pass. +template +void run_pipeline_hybrid_is_distinct_test(std::vector shape, + int num_local_levels, + int num_global_levels) { + constexpr DIM D = 3; + SIZE n = 1; + for (auto sz : shape) + n *= sz; + + std::vector original(n); + fill_smooth_data(original, shape); + + const double s = std::numeric_limits::infinity(); + const double tol = 1e-3; + + // Start from a clean CompressorCache so this case measures one shape in + // isolation rather than inheriting buffers sized by whatever ran before. + // The shape-switch case below deliberately does not do this. + release_cache(); + MallocedBuffer hybrid_stream; + REQUIRE(compress(D, data_type_of(), shape, tol, s, + error_bound_type::REL, original.data(), + hybrid_stream.ptr, hybrid_stream.size, + hybrid_config(num_local_levels, num_global_levels), + false) == compress_status_type::Success); + + Config plain; // default decomposition, no hybrid levels + MallocedBuffer plain_stream; + REQUIRE(compress(D, data_type_of(), shape, tol, s, + error_bound_type::REL, original.data(), + plain_stream.ptr, plain_stream.size, plain, + false) == compress_status_type::Success); + + bool streams_differ = hybrid_stream.size != plain_stream.size || + std::memcmp(hybrid_stream.ptr, plain_stream.ptr, + hybrid_stream.size) != 0; + CAPTURE(hybrid_stream.size, plain_stream.size); + REQUIRE(streams_differ); + + // Both must still honor the bound, so "different" cannot mean "broken". + for (MallocedBuffer *stream : {&hybrid_stream, &plain_stream}) { + MallocedBuffer out; + REQUIRE(decompress(stream->ptr, stream->size, out.ptr, false) == + compress_status_type::Success); + double error = L_inf_error(n, original.data(), (const T *)out.ptr, + error_bound_type::REL); + CAPTURE(error); + REQUIRE(error < tol); + } +} + +// Compressing a second, differently-shaped dataset in the same process must +// not disturb the first result. The public API keeps a CompressorCache whose +// device_subdomain_buffer is resized per call, so a shape change exercises +// Array::resize's in-place path -- and a buffer left describing the previous +// shape corrupts the output silently: Success is returned, the ratio looks +// ordinary, and only the data is wrong. +template +void run_pipeline_shape_switch_test(std::vector> shapes, + int num_local_levels, + int num_global_levels) { + constexpr DIM D = 3; + const double s = std::numeric_limits::infinity(); + const double tol = 1e-3; + + for (const auto &shape : shapes) { + SIZE n = 1; + for (auto sz : shape) + n *= sz; + std::vector original(n); + fill_smooth_data(original, shape); + + MallocedBuffer compressed; + REQUIRE(compress( + D, data_type_of(), shape, tol, s, error_bound_type::REL, + original.data(), compressed.ptr, compressed.size, + hybrid_config(num_local_levels, num_global_levels), false) == + compress_status_type::Success); + + MallocedBuffer decompressed; + REQUIRE(decompress(compressed.ptr, compressed.size, + decompressed.ptr, false) == + compress_status_type::Success); + + double error = L_inf_error(n, original.data(), (const T *)decompressed.ptr, + error_bound_type::REL); + CAPTURE(shape[0], shape[1], shape[2], error, tol); + REQUIRE(error < tol); + } +} + +TEMPLATE_TEST_CASE("Hybrid pipeline survives a shape change in one process", + "[mgard-x][hybrid][pipeline][@DEVICE_TYPE@]", float, + double) { + DeviceRuntime::Initialize(); + // Shrinking the fastest-varying extent is the case that breaks: the cached + // subdomain buffer keeps the wider leading dimension of the first shape. + SECTION("shrink fastest dim") { + run_pipeline_shape_switch_test( + {{64, 64, 64}, {64, 64, 56}}, 1, 0); + } + SECTION("shrink middle dim") { + run_pipeline_shape_switch_test( + {{64, 64, 64}, {64, 56, 64}}, 1, 0); + } + SECTION("shrink slowest dim only") { + run_pipeline_shape_switch_test( + {{64, 64, 64}, {56, 64, 64}}, 1, 0); + } + SECTION("grow then shrink") { + run_pipeline_shape_switch_test( + {{40, 40, 40}, {64, 64, 64}, {60, 60, 60}}, 1, 0); + } +} + +TEMPLATE_TEST_CASE("Hybrid end-to-end pipeline round trip", + "[mgard-x][hybrid][pipeline][@DEVICE_TYPE@]", float, + double) { + DeviceRuntime::Initialize(); + enum error_bound_type ebtype = + GENERATE(error_bound_type::REL, error_bound_type::ABS); + double tol = GENERATE(1e-2, 1e-3); + CAPTURE(static_cast(ebtype), tol); + SECTION("pure local (L=1, M=0)") { + run_pipeline_test<3, TestType, TestDevice>({64, 64, 64}, 1, 0, ebtype, tol); + } + SECTION("hybrid (L=1, M=2)") { + run_pipeline_test<3, TestType, TestDevice>({64, 64, 64}, 1, 2, ebtype, tol); + } + SECTION("two local levels (L=2, M=1)") { + run_pipeline_test<3, TestType, TestDevice>({64, 64, 64}, 2, 1, ebtype, tol); + } + SECTION("padded (L=1, M=0)") { + run_pipeline_test<3, TestType, TestDevice>({60, 60, 60}, 1, 0, ebtype, tol); + } + SECTION("non-cubic (L=1, M=2)") { + run_pipeline_test<3, TestType, TestDevice>({40, 64, 56}, 1, 2, ebtype, tol); + } + SECTION("2D, pure local (L=1, M=0)") { + run_pipeline_test<2, TestType, TestDevice>({256, 256}, 1, 0, ebtype, tol); + } + SECTION("2D, hybrid, padded (L=1, M=2)") { + run_pipeline_test<2, TestType, TestDevice>({250, 250}, 1, 2, ebtype, tol); + } + SECTION("1D, pure local (L=1, M=0)") { + run_pipeline_test<1, TestType, TestDevice>({65536}, 1, 0, ebtype, tol); + } + SECTION("1D, hybrid, padded (L=1, M=2)") { + run_pipeline_test<1, TestType, TestDevice>({65000}, 1, 2, ebtype, tol); + } +} + +// ROI mode, driven so that compress and decompress communicate only through +// the serialized bytes. +// +// The hybrid compressor is cached across calls, so a compress immediately +// followed by a decompress in the same process can read the coarsest layer +// straight out of the still-live decomposed buffer. That is exactly how a +// missing coarsest-layer quantization stayed invisible: the in-process check +// passed while a separate decompressor got zeros. Releasing the cache between +// the two calls is what makes this test see what a fresh process would. +// +// The assertion is the actual ROI contract: every level-0 block must meet its +// own tolerance. That also pins the per-block quantizer indexing +// (idx / hybrid_local_coeff_per_block(D)) to the right blocks in space -- a +// spatial mix-up would let a loose block bleed into a tight one. +template +void run_roi_pipeline_test(std::vector shape, int num_local_levels, + int num_global_levels, bool fused) { + SIZE n = 1; + for (auto sz : shape) + n *= sz; + + std::vector original(n); + fill_smooth_data(original, shape); + + // One tolerance per level-0 block; tight over the first half of the slowest + // dimension, loose elsewhere, so the two regions are easy to tell apart. + const double tight = 1e-5, loose = 1e-2; + std::vector block_dims(D); + SIZE num_blocks = 1; + for (DIM d = 0; d < D; d++) { + block_dims[d] = (shape[d] + 7) / 8; + num_blocks *= block_dims[d]; + } + std::vector tol_map(num_blocks); + for (SIZE b = 0; b < num_blocks; b++) { + SIZE slowest = b; + for (DIM d = 1; d < D; d++) + slowest /= block_dims[d]; + tol_map[b] = (slowest < block_dims[0] / 2) ? tight : loose; + } + + Config config; + config.decomposition = decomposition_type::Hybrid; + config.num_local_refactoring_level = num_local_levels; + config.num_global_refactoring_level = num_global_levels; + config.enable_roi = true; + config.roi_tolerance_map = tol_map; + config.fuse_decompose_quantize = fused; + config.fuse_dequantize_recompose = fused; + + const T s = std::numeric_limits::infinity(); + + // Two compressor objects, both alive at the same time. That is the + // isolation this test needs: their buffers are distinct live allocations, + // so the decompressor cannot read a value the compressor left behind. + // + // Releasing a cached compressor and building a new one is NOT enough. The + // allocator can hand the replacement the very same buffers, still holding + // the previous run's coarsest values, so a stream missing its coarsest + // layer still reconstructs correctly. That is how this bug hid: it showed + // up only in a separate decompressor process, and an in-process check + // caught it or not depending on which allocation it happened to get -- + // it caught it for double and missed it for float. + Hierarchy hierarchy(shape, config); + HybridHierarchyCompressor compressor; + compressor.Adapt(hierarchy, config, 0); + HybridHierarchyCompressor decompressor; + decompressor.Adapt(hierarchy, config, 0); + DeviceRuntime::SyncQueue(0); + + Array original_array(shape); + original_array.load(original.data()); + + T norm = 0; + Array<1, Byte, DevType> compressed_array; + compressor.Compress(original_array, error_bound_type::ABS, (T)loose, s, norm, + compressed_array, 0); + DeviceRuntime::SyncQueue(0); + + Array decompressed_array; + decompressor.Decompress(compressed_array, error_bound_type::ABS, (T)loose, s, + norm, decompressed_array, 0); + const T *result = decompressed_array.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + // Every block within its own tolerance, and the tight region genuinely + // tighter than the loose one (which a zeroed coarsest layer would fail). + std::vector block_err(num_blocks, 0.0); + for (SIZE idx = 0; idx < n; idx++) { + SIZE r = idx, block = 0; + for (int d = (int)D - 1; d >= 0; d--) { + SIZE coord = r % shape[d]; + r /= shape[d]; + SIZE stride = 1; + for (DIM dd = (DIM)d + 1; dd < D; dd++) + stride *= block_dims[dd]; + block += (coord / 8) * stride; + } + double e = std::abs((double)original[idx] - (double)result[idx]); + block_err[block] = std::max(block_err[block], e); + } + + double worst_tight = 0, worst_loose = 0, worst_ratio = 0; + for (SIZE b = 0; b < num_blocks; b++) { + worst_ratio = std::max(worst_ratio, block_err[b] / tol_map[b]); + if (tol_map[b] == tight) + worst_tight = std::max(worst_tight, block_err[b]); + else + worst_loose = std::max(worst_loose, block_err[b]); + } + CAPTURE(worst_tight, worst_loose, worst_ratio, tight, loose); + REQUIRE(worst_ratio <= 1.0); + REQUIRE(worst_tight <= tight); +} + +// s != inf (L2 family) through the hybrid compressor, non-ROI only. +// +// The block-local quantizer weights each layer by 2^(-s*l) and by the layer's +// node volume, both folded into the per-layer scalar. The assertion is the +// contract itself: the achieved L_2 error must stay under the requested +// tolerance, checked with the same metric test_Compressor uses for s != inf. +// Both level splits matter -- M == 0 leaves every layer to the block-local +// quantizer, while M > 0 hands the coarsest region to the global stage. +TEMPLATE_TEST_CASE("Hybrid Hierarchy Compressor with an L2 error bound", + "[mgard-x][hybrid][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + enum error_bound_type ebtype = + GENERATE(error_bound_type::REL, error_bound_type::ABS); + TestType tol = GENERATE(static_cast(1e-2), + static_cast(1e-3)); + bool fused = GENERATE(true, false); + TestType s = static_cast(0); + CAPTURE(static_cast(ebtype), tol, fused, s); + // One block-local level, no global stage: the only configuration where the + // blocks are independent, so the budget divides evenly in quadrature and the + // s-norm bound holds. Anything else is refused / falls back (below). + SECTION("3D") { + run_hybrid_compressor_test<3, TestType, TestDevice>({64, 64, 64}, 1, 0, + ebtype, tol, fused, s); + } + SECTION("2D") { + run_hybrid_compressor_test<2, TestType, TestDevice>({256, 256}, 1, 0, + ebtype, tol, fused, s); + } + SECTION("1D") { + run_hybrid_compressor_test<1, TestType, TestDevice>({65536}, 1, 0, ebtype, + tol, fused, s); + } + SECTION("padded shape") { + run_hybrid_compressor_test<3, TestType, TestDevice>({60, 60, 60}, 1, 0, + ebtype, tol, fused, s); + } + SECTION("non-cubic") { + run_hybrid_compressor_test<3, TestType, TestDevice>({40, 64, 56}, 1, 0, + ebtype, tol, fused, s); + } +} + +// The low-level quantizer cannot re-configure itself -- L and M fixed the +// buffer layout at Adapt() -- so it must refuse an s-norm bound it cannot +// honour rather than silently returning an unbounded result. +TEMPLATE_TEST_CASE("Hybrid refuses an s-norm bound it cannot bound", + "[mgard-x][hybrid][@DEVICE_TYPE@]", float, double) { + DeviceRuntime::Initialize(); + const TestType s = static_cast(0); + auto attempt = [&](int L, int M) { + std::vector shape{64, 64, 64}; + Config config; + config.decomposition = decomposition_type::Hybrid; + config.num_local_refactoring_level = L; + config.num_global_refactoring_level = M; + Hierarchy<3, TestType, TestDevice> hierarchy(shape, config); + HybridHierarchyCompressor<3, TestType, TestDevice> compressor; + compressor.Adapt(hierarchy, config, 0); + DeviceRuntime::SyncQueue(0); + std::vector original(64 * 64 * 64); + fill_smooth_data<3, TestType>(original, shape); + Array<3, TestType, TestDevice> data(shape); + data.load(original.data()); + TestType norm = 0; + Array<1, Byte, TestDevice> compressed; + compressor.Compress(data, error_bound_type::ABS, (TestType)1e-2, s, norm, + compressed, 0); + }; + SECTION("two local levels") { REQUIRE_THROWS(attempt(2, 0)); } + SECTION("a global stage") { REQUIRE_THROWS(attempt(1, 2)); } + SECTION("both") { REQUIRE_THROWS(attempt(2, 1)); } + // The supported configuration still works. + SECTION("one local level, no global stage") { + REQUIRE_NOTHROW(attempt(1, 0)); + } +} + +// The public compress path does not refuse: it falls back to the one +// configuration that can be bounded, warns, and records the adjusted level +// counts in the header so the decompressor agrees. +TEMPLATE_TEST_CASE("Hybrid falls back to one local level for an s-norm bound", + "[mgard-x][hybrid][pipeline][@DEVICE_TYPE@]", float, + double) { + DeviceRuntime::Initialize(); + release_cache(); + std::vector shape{64, 64, 64}; + SIZE n = 64 * 64 * 64; + std::vector original(n); + fill_smooth_data<3, TestType>(original, shape); + const double s = 0.0; + const double tol = 1e-2; + + // Ask for a configuration that cannot be bounded; it must still succeed. + MallocedBuffer compressed; + REQUIRE(compress(3, data_type_of(), shape, tol, s, + error_bound_type::ABS, original.data(), + compressed.ptr, compressed.size, + hybrid_config(2, 2), + false) == compress_status_type::Success); + + MallocedBuffer decompressed; + data_type recovered_dtype; + std::vector recovered_shape; + REQUIRE(decompress(compressed.ptr, compressed.size, + decompressed.ptr, recovered_dtype, + recovered_shape, false) == + compress_status_type::Success); + REQUIRE(recovered_shape == shape); + + // And the bound the fallback exists to provide must actually hold. + const TestType *result = (const TestType *)decompressed.ptr; + Config probe; + double error = L_2_error(shape, original.data(), result, + error_bound_type::ABS, probe.normalize_coordinates); + CAPTURE(error, tol); + REQUIRE(error < tol); +} + +TEMPLATE_TEST_CASE("Hybrid ROI round trip through the bytes alone", + "[mgard-x][hybrid][pipeline][@DEVICE_TYPE@]", float, + double) { + DeviceRuntime::Initialize(); + bool fused = GENERATE(true, false); + CAPTURE(fused); + // M == 0 is the case with no global stage to cover the coarsest layer. + SECTION("1D, no global stage") { + run_roi_pipeline_test<1, TestType, TestDevice>({4096}, 1, 0, fused); + } + SECTION("2D, no global stage") { + run_roi_pipeline_test<2, TestType, TestDevice>({128, 96}, 1, 0, fused); + } + SECTION("3D, no global stage") { + run_roi_pipeline_test<3, TestType, TestDevice>({32, 32, 32}, 1, 0, fused); + } + SECTION("2D, no global stage, two local levels") { + run_roi_pipeline_test<2, TestType, TestDevice>({128, 96}, 2, 0, fused); + } + // M > 0 already worked; keep it so the fix cannot regress it. + SECTION("2D, with a global stage") { + run_roi_pipeline_test<2, TestType, TestDevice>({128, 96}, 1, 2, fused); + } + SECTION("3D, with a global stage") { + run_roi_pipeline_test<3, TestType, TestDevice>({32, 32, 32}, 1, 2, fused); + } +} + +TEMPLATE_TEST_CASE("Hybrid decompression reads its levels from the file", + "[mgard-x][hybrid][pipeline][@DEVICE_TYPE@]", float, + double) { + DeviceRuntime::Initialize(); + SECTION("pure local (L=1, M=0)") { + run_pipeline_config_independence_test({64, 64, 64}, 1, + 0); + } + SECTION("hybrid (L=1, M=2)") { + run_pipeline_config_independence_test({64, 64, 64}, 1, + 2); + } +} + +TEMPLATE_TEST_CASE("Hybrid pipeline differs from the standard pipeline", + "[mgard-x][hybrid][pipeline][@DEVICE_TYPE@]", float, + double) { + DeviceRuntime::Initialize(); + SECTION("pure local (L=1, M=0)") { + run_pipeline_hybrid_is_distinct_test({64, 64, 64}, 1, + 0); + } + SECTION("hybrid (L=1, M=2)") { + run_pipeline_hybrid_is_distinct_test({64, 64, 64}, 1, + 2); + } +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_HybridLevelCompressor.cpp.in b/tests/mgard-x/src/test_HybridLevelCompressor.cpp.in new file mode 100644 index 0000000000..1ebed6a05a --- /dev/null +++ b/tests/mgard-x/src/test_HybridLevelCompressor.cpp.in @@ -0,0 +1,155 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +// HybridLevelCompressor (include/mgard-x/MDR-X/LosslessCompressor/ +// HybridLevelCompressor.hpp) is the lossless back end ComposedRefactor uses +// on the bitplanes BPEncoder{LocalityBlock,RegisterBlock} produce (see +// test_BitplaneEncoder*): every num_merged_bitplanes (4) consecutive rows of +// the encoded_bitplanes 2D buffer are byte-reinterpreted as one flat stream +// and, if large enough, compressed with RLE first, then Huffman, falling +// back to a raw copy if neither reaches cr_threshold or the stream is too +// small to bother. decompress_level must invert whichever of the three it +// picked. It has no dimensionality of its own (it operates on a flat +// 2D bitplane buffer, not the D-dimensional decomposed data), so the only +// type axis here is T_bitplane. + +#include "catch2/catch_template_test_macros.hpp" +#include "catch2/catch_test_macros.hpp" + +#include +#include +#include + +#include "mgard-x/RuntimeX/RuntimeX.h" +// HybridLevelCompressor::Adapt takes a Config, but nothing it includes +// brings in the Config type itself. +#include "mgard-x/Config/Config.h" +#include "mgard-x/MDR-X/LosslessCompressor/HybridLevelCompressor.hpp" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; + +enum class BitplaneContent { Compressible, Incompressible }; + +// Compressible: only a handful of distinct values, repeated in long runs -- +// RLE (tried first) should win outright. Incompressible: every word +// independent noise, so RLE and Huffman both fail to clear cr_threshold and +// compress_level must fall back to a raw copy even though the stream is +// large enough to have tried. +template +std::vector make_bitplane_data(SIZE num_bitplanes, + SIZE stream_length, + BitplaneContent content) { + std::vector data(num_bitplanes * stream_length); + std::srand(42); + for (SIZE i = 0; i < data.size(); i++) { + if (content == BitplaneContent::Compressible) { + data[i] = static_cast((i / 4096) % 3); + } else { + uint64_t hi = (uint64_t)std::rand(); + uint64_t lo = (uint64_t)std::rand(); + data[i] = static_cast((hi << 32) ^ lo); + } + } + return data; +} + +// compress_level then decompress_level must reproduce the original bitplane +// words exactly -- RLE, Huffman and the raw-copy fallback are all lossless, +// so unlike the encoders this feeds (which quantize) there is no tolerance +// here, only bit-for-bit equality. +template +void run_hybrid_level_compressor_roundtrip(SIZE num_bitplanes, + SIZE stream_length, + BitplaneContent content) { + using Compressor = MDR::HybridLevelCompressor; + constexpr SIZE merge = Compressor::num_merged_bitplanes; + REQUIRE(num_bitplanes % merge == 0); + + Config config; + Compressor compressor; + compressor.Adapt(stream_length, num_bitplanes, config, 0); + DeviceRuntime::SyncQueue(0); + + std::vector original = + make_bitplane_data(num_bitplanes, stream_length, content); + + Array<2, T_bitplane, DevType> encoded_bitplanes_array( + {num_bitplanes, stream_length}); + encoded_bitplanes_array.load(original.data()); + SubArray<2, T_bitplane, DevType> encoded_bitplanes(encoded_bitplanes_array); + + std::vector> compressed(num_bitplanes); + compressor.compress_level(encoded_bitplanes, compressed, 0, 0); + DeviceRuntime::SyncQueue(0); + + const SIZE merged_bytes = stream_length * sizeof(T_bitplane) * merge; + const SIZE size_threshold = compressor.size_threshold; + for (SIZE bp = 0; bp < num_bitplanes; bp++) { + if (bp % merge == 0) { + SIZE compressed_size = compressed[bp].shape(0); + CAPTURE(bp, compressed_size, merged_bytes, size_threshold); + REQUIRE(compressed_size > 0); + if (content == BitplaneContent::Compressible && + merged_bytes > size_threshold) { + // Large and genuinely compressible: RLE or Huffman must have been + // used, not the raw-copy fallback. + REQUIRE(compressed_size < merged_bytes); + } + } else { + // Every non-leading row in a merged group carries no payload of its + // own; its bytes were folded into the leading row's stream. + REQUIRE(compressed[bp].shape(0) == 0); + } + } + + // Overwrite the source buffer so decompress_level can only be reading from + // the compressed streams, not leftover device memory. + encoded_bitplanes_array.memset(0xff, 0); + DeviceRuntime::SyncQueue(0); + + compressor.decompress_level(compressed, encoded_bitplanes, 0, num_bitplanes, + 0, 0); + DeviceRuntime::SyncQueue(0); + + T_bitplane *result = encoded_bitplanes_array.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + SIZE mismatches = 0; + for (SIZE i = 0; i < original.size(); i++) + mismatches += (result[i] != original[i]); + CAPTURE(mismatches); + REQUIRE(mismatches == 0); +} + +// num_bitplanes must be a multiple of num_merged_bitplanes (4); stream_length +// is chosen so merged_bytes = stream_length * sizeof(T_bitplane) * 4 lands +// clearly above (100000) or below (100) HybridLevelCompressor's +// size_threshold (1e6 bytes) for every T_bitplane this is instantiated with. +TEMPLATE_TEST_CASE("MDR-X HybridLevelCompressor round trip", + "[mgard-x][mdr-x][hybrid-level-compressor][@DEVICE_TYPE@]", + uint32_t, uint64_t) { + DeviceRuntime::Initialize(); + SECTION("large, compressible -> RLE/Huffman actually used") { + run_hybrid_level_compressor_roundtrip( + 8, 100000, BitplaneContent::Compressible); + } + SECTION("large, incompressible -> falls back to raw copy") { + run_hybrid_level_compressor_roundtrip( + 8, 100000, BitplaneContent::Incompressible); + } + SECTION("below size threshold -> raw copy without attempting compression") { + run_hybrid_level_compressor_roundtrip( + 8, 100, BitplaneContent::Compressible); + } + SECTION("single merged group (num_bitplanes == num_merged_bitplanes)") { + run_hybrid_level_compressor_roundtrip( + 4, 100000, BitplaneContent::Compressible); + } +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_HybridMetadata.cpp.in b/tests/mgard-x/src/test_HybridMetadata.cpp.in new file mode 100644 index 0000000000..ecb6f49244 --- /dev/null +++ b/tests/mgard-x/src/test_HybridMetadata.cpp.in @@ -0,0 +1,223 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + */ + +#include "catch2/catch_test_macros.hpp" + +#include +#include + +#include "mgard-x/Config/Config.h" +#include "mgard-x/Metadata/Metadata.hpp" +#include "mgard-x/RuntimeX/RuntimeX.h" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; + +namespace { + +// Fill the non-hybrid fields with something valid so Serialize/Deserialize +// have a well-formed header to work with. +Metadata base_metadata(std::vector shape) { + Metadata m; + m.FillForCompression( + error_bound_type::ABS, (float)1e-3, std::numeric_limits::infinity(), + (float)1.0, decomposition_type::Hybrid, + lossless_type::Huffman, /*huff_dict_size=*/8192, + /*huff_block_size=*/1024, shape, /*domain_decomposed=*/false, + domain_decomposition_type::MaxDim, /*domain_decomposed_dim=*/0, + /*domain_decomposed_size=*/0); + return m; +} + +std::vector block_grid(const std::vector &shape) { + std::vector dims(shape.size()); + for (size_t d = 0; d < shape.size(); d++) { + dims[d] = (shape[d] + MGARDX_HYBRID_LOCAL_BLOCK_SIZE - 1) / + MGARDX_HYBRID_LOCAL_BLOCK_SIZE; + } + return dims; +} + +} // namespace + +TEST_CASE("hybrid level counts survive a metadata round trip", + "[HybridMetadata]") { + std::vector shape = {64, 64, 64}; + Metadata out = base_metadata(shape); + out.hybrid_num_local_levels = 2; + out.hybrid_num_global_levels = 1; + out.hybrid_local_block_size = MGARDX_HYBRID_LOCAL_BLOCK_SIZE; + out.hybrid_projection_mode = compression_projection_mode_type::Hierarchical; + out.hybrid_enable_roi = false; + + std::vector bytes = out.MetadataBase::Serialize(); + + Metadata in; + in.MetadataBase::Deserialize(bytes); + + REQUIRE(in.decomposition == decomposition_type::Hybrid); + REQUIRE(in.hybrid_num_local_levels == 2); + REQUIRE(in.hybrid_num_global_levels == 1); + REQUIRE(in.hybrid_local_block_size == MGARDX_HYBRID_LOCAL_BLOCK_SIZE); + REQUIRE(in.hybrid_projection_mode == + compression_projection_mode_type::Hierarchical); + REQUIRE(in.hybrid_enable_roi == false); + + // The whole point of recording these: a Config that never saw the original + // compression parameters comes back configured correctly. + Config config; + config.num_local_refactoring_level = 99; + config.num_global_refactoring_level = 99; + in.InitializeConfig(config); + REQUIRE(config.num_local_refactoring_level == 2); + REQUIRE(config.num_global_refactoring_level == 1); + REQUIRE(config.projection_mode == + compression_projection_mode_type::Hierarchical); + REQUIRE(config.enable_roi == false); +} + +TEST_CASE("an invalid hybrid projection mode is not serialized", + "[HybridMetadata]") { + Metadata out = base_metadata({64, 64, 64}); + out.hybrid_num_local_levels = 1; + out.hybrid_num_global_levels = 0; + out.hybrid_local_block_size = MGARDX_HYBRID_LOCAL_BLOCK_SIZE; + out.hybrid_projection_mode = + static_cast(255); + + REQUIRE_THROWS_AS(out.MetadataBase::Serialize(), InvalidDataException); +} + +// Auto is a compression-side policy resolved to a concrete basis before +// metadata is ever filled (see resolve_projection_mode in +// CompressionHighLevel.hpp); Serialize() must refuse it too, in case that +// resolution step is ever skipped. +TEST_CASE("Auto is not a valid projection mode to serialize", + "[HybridMetadata]") { + Metadata out = base_metadata({64, 64, 64}); + out.hybrid_num_local_levels = 1; + out.hybrid_num_global_levels = 0; + out.hybrid_local_block_size = MGARDX_HYBRID_LOCAL_BLOCK_SIZE; + out.hybrid_projection_mode = compression_projection_mode_type::Auto; + + REQUIRE_THROWS_AS(out.MetadataBase::Serialize(), InvalidDataException); +} + +TEST_CASE("ROI tolerance map survives a metadata round trip", + "[HybridMetadata]") { + std::vector shape = {64, 64, 64}; + std::vector dims = block_grid(shape); + size_t num_blocks = 1; + for (uint64_t n : dims) + num_blocks *= n; + + // Piecewise constant, so the run-length encoding is the one that gets used. + std::vector tolerances(num_blocks); + for (size_t i = 0; i < num_blocks; i++) { + tolerances[i] = (i < num_blocks / 2) ? 1e-3 : 1e-5; + } + + Metadata out = base_metadata(shape); + out.hybrid_num_local_levels = 1; + out.hybrid_num_global_levels = 0; + out.hybrid_local_block_size = MGARDX_HYBRID_LOCAL_BLOCK_SIZE; + out.hybrid_enable_roi = true; + out.hybrid_roi_block_dimensions = dims; + out.hybrid_roi_tolerance_map = tolerances; + + std::vector bytes = out.MetadataBase::Serialize(); + + // Two runs must encode far smaller than one double per block. + REQUIRE(bytes.size() < num_blocks * sizeof(double)); + + Metadata in; + in.MetadataBase::Deserialize(bytes); + + REQUIRE(in.hybrid_enable_roi == true); + REQUIRE(in.hybrid_roi_block_dimensions == dims); + REQUIRE(in.hybrid_roi_tolerance_map.size() == num_blocks); + // Tolerances must come back bit-exact: they set the quantization step, so a + // rounded value would change the reconstruction. + for (size_t i = 0; i < num_blocks; i++) { + REQUIRE(in.hybrid_roi_tolerance_map[i] == tolerances[i]); + } + + Config config; + in.InitializeConfig(config); + REQUIRE(config.enable_roi == true); + REQUIRE(config.roi_tolerance_map == tolerances); +} + +TEST_CASE("an irregular ROI map still round trips exactly", + "[HybridMetadata]") { + // Every block distinct: run-length is the larger encoding here, so this + // exercises the packed branch of the writer. + std::vector shape = {16, 16, 16}; + std::vector dims = block_grid(shape); + size_t num_blocks = 1; + for (uint64_t n : dims) + num_blocks *= n; + + std::vector tolerances(num_blocks); + for (size_t i = 0; i < num_blocks; i++) { + tolerances[i] = 1e-3 * std::pow(0.9, (double)i); + } + + Metadata out = base_metadata(shape); + out.hybrid_num_local_levels = 1; + out.hybrid_num_global_levels = 0; + out.hybrid_local_block_size = MGARDX_HYBRID_LOCAL_BLOCK_SIZE; + out.hybrid_enable_roi = true; + out.hybrid_roi_block_dimensions = dims; + out.hybrid_roi_tolerance_map = tolerances; + + std::vector bytes = out.MetadataBase::Serialize(); + Metadata in; + in.MetadataBase::Deserialize(bytes); + + REQUIRE(in.hybrid_roi_tolerance_map.size() == num_blocks); + for (size_t i = 0; i < num_blocks; i++) { + REQUIRE(in.hybrid_roi_tolerance_map[i] == tolerances[i]); + } +} + +TEST_CASE("a non-hybrid file leaves the caller's hybrid Config alone", + "[HybridMetadata]") { + std::vector shape = {64, 64, 64}; + Metadata out = base_metadata(shape); + out.decomposition = decomposition_type::MultiDim; + + std::vector bytes = out.MetadataBase::Serialize(); + Metadata in; + in.MetadataBase::Deserialize(bytes); + REQUIRE(in.decomposition == decomposition_type::MultiDim); + + Config config; + config.num_local_refactoring_level = 3; + config.num_global_refactoring_level = 2; + in.InitializeConfig(config); + // Untouched: overwriting these with zeros would break a later hybrid + // compression driven by the same Config. + REQUIRE(config.num_local_refactoring_level == 3); + REQUIRE(config.num_global_refactoring_level == 2); +} + +TEST_CASE("the quantized integer width is recorded and checked", + "[HybridMetadata]") { + std::vector shape = {64, 64, 64}; + Metadata out = base_metadata(shape); + out.hybrid_num_local_levels = 1; + out.hybrid_num_global_levels = 0; + out.hybrid_local_block_size = MGARDX_HYBRID_LOCAL_BLOCK_SIZE; + + std::vector bytes = out.MetadataBase::Serialize(); + Metadata in; + // Round trips cleanly against a build with the same QUANTIZED_INT width; + // Deserialize throws when the widths disagree. + REQUIRE_NOTHROW(in.MetadataBase::Deserialize(bytes)); +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_KernelFusionSwitch.cpp.in b/tests/mgard-x/src/test_KernelFusionSwitch.cpp.in new file mode 100644 index 0000000000..2fbb14af88 --- /dev/null +++ b/tests/mgard-x/src/test_KernelFusionSwitch.cpp.in @@ -0,0 +1,209 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + */ + +#include "catch2/catch_template_test_macros.hpp" +#include "catch2/catch_test_macros.hpp" + +#include +#include +#include +#include + +#include "mgard-x/CompressionLowLevel/HybridHierarchyCompressor.hpp" +#include "mgard-x/RuntimeX/RuntimeX.h" +#include "mgard-x/Utilities/ErrorCalculator.h" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; + +namespace { + +template +void fill_smooth_data(std::vector &data, std::vector shape) { + const double pi = std::acos(-1.0); + SIZE n = data.size(); + for (SIZE idx = 0; idx < n; idx++) { + SIZE r = idx; + double v = 1.0; + for (int d = static_cast(D) - 1; d >= 0; d--) { + SIZE coord = r % shape[d]; + r /= shape[d]; + double x = static_cast(coord) / static_cast(shape[d]); + v *= std::sin(2.0 * pi * x + 0.5 * d); + } + data[idx] = static_cast(v); + } +} + +template +Config hybrid_config(int num_local_levels, int num_global_levels) { + Config config; + config.decomposition = decomposition_type::Hybrid; + config.num_local_refactoring_level = num_local_levels; + config.num_global_refactoring_level = num_global_levels; + return config; +} + +// Compress with `fuse_compress`, decompress with `fuse_decompress`, and return +// both the compressed bytes and the reconstruction. +template +void round_trip(std::vector shape, int num_local_levels, + int num_global_levels, bool fuse_compress, bool fuse_decompress, + enum error_bound_type ebtype, T tol, + const std::vector &original, std::vector &bytes_out, + std::vector &result_out) { + constexpr DIM D = 3; + const T s = std::numeric_limits::infinity(); + + Config compress_config = + hybrid_config(num_local_levels, num_global_levels); + compress_config.fuse_decompose_quantize = fuse_compress; + Config decompress_config = + hybrid_config(num_local_levels, num_global_levels); + decompress_config.fuse_dequantize_recompose = fuse_decompress; + + Hierarchy compress_hierarchy(shape, compress_config); + HybridHierarchyCompressor compressor; + compressor.Adapt(compress_hierarchy, compress_config, 0); + + Hierarchy decompress_hierarchy(shape, decompress_config); + HybridHierarchyCompressor decompressor; + decompressor.Adapt(decompress_hierarchy, decompress_config, 0); + DeviceRuntime::SyncQueue(0); + + Array original_array(shape); + original_array.load(original.data()); + + T norm = 0; + Array<1, Byte, DevType> compressed_array; + compressor.Compress(original_array, ebtype, tol, s, norm, compressed_array, + 0); + DeviceRuntime::SyncQueue(0); + + Byte *bytes = compressed_array.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + bytes_out.assign(bytes, bytes + compressed_array.shape(0)); + + Array decompressed_array; + decompressor.Decompress(compressed_array, ebtype, tol, s, norm, + decompressed_array, 0); + T *result = decompressed_array.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + SIZE n = 1; + for (auto sz : shape) + n *= sz; + result_out.assign(result, result + n); +} + +} // namespace + +// The switch is only meaningful if it does not change what comes out. What +// "does not change" can mean here is bounded by the backends themselves: +// neither produces a reproducible byte stream. On the GPU backends the Huffman +// outlier separator hands out slots with block-aggregated atomics, so outlier +// order follows thread-block scheduling. On the CPU backends the result +// depends on memory layout -- compressing the same data twice in one process, +// or running the same binary with a different-sized environment block, shifts +// stack alignment, changes how the vectorized loops round, and moves a few +// coefficients across a quantization boundary. Measured, back when the switch +// was an environment variable: a 4 KB environment variable changes the +// compressed size of a 64^3 float input by 424 bytes. +// +// So equality of bytes, of size, or of the reconstruction is not a property +// this code has, with or without fusion. What must hold -- and what the switch +// depends on -- is that either path meets the requested error bound, and that +// a file compressed one way decompresses the other way and still meets it. +TEMPLATE_TEST_CASE("Kernel fusion switch does not change the compressed data", + "[mgard-x][hybrid][fusion][@DEVICE_TYPE@]", float, double) { + using T = TestType; + std::vector shape = {64, 64, 64}; + const enum error_bound_type ebtype = error_bound_type::ABS; + const T tol = (T)1e-3; + + SIZE n = 1; + for (auto sz : shape) + n *= sz; + std::vector original(n); + fill_smooth_data<3, T>(original, shape); + + auto check = [&](int L, int M) { + std::vector fused_bytes, separate_bytes; + std::vector fused_result, separate_result; + round_trip(shape, L, M, /*fuse_compress=*/true, + /*fuse_decompress=*/true, ebtype, tol, original, + fused_bytes, fused_result); + round_trip(shape, L, M, /*fuse_compress=*/false, + /*fuse_decompress=*/false, ebtype, tol, original, + separate_bytes, separate_result); + + double fused_error = + L_inf_error(n, original.data(), fused_result.data(), ebtype); + double separate_error = + L_inf_error(n, original.data(), separate_result.data(), ebtype); + CAPTURE(L, M, fused_bytes.size(), separate_bytes.size(), fused_error, + separate_error); + + // Both paths honour the tolerance the caller asked for. + REQUIRE(fused_error <= static_cast(tol)); + REQUIRE(separate_error <= static_cast(tol)); + + // Compression ratio must not change meaningfully. Loose enough to absorb + // the last-bit rounding differences described above (measured at 0.08%), + // tight enough to catch a path that quantizes wrongly. + double size_ratio = (double)fused_bytes.size() / separate_bytes.size(); + REQUIRE(size_ratio > 0.99); + REQUIRE(size_ratio < 1.01); + }; + + SECTION("L = 1, M = 0") { check(1, 0); } + SECTION("L = 2, M = 0") { check(2, 0); } + SECTION("L = 1, M = 2") { check(1, 2); } +} + +// Compressing fused and decompressing separately (or the reverse) has to work, +// since the header records nothing about which was used and the two sides can +// be configured independently. +TEMPLATE_TEST_CASE("Fused and separate paths interoperate across a round trip", + "[mgard-x][hybrid][fusion][@DEVICE_TYPE@]", float, double) { + using T = TestType; + std::vector shape = {64, 64, 64}; + const enum error_bound_type ebtype = error_bound_type::ABS; + const T tol = (T)1e-3; + + SIZE n = 1; + for (auto sz : shape) + n *= sz; + std::vector original(n); + fill_smooth_data<3, T>(original, shape); + + std::vector bytes; + std::vector both_fused, compress_fused, decompress_fused, neither; + round_trip(shape, 2, 0, true, true, ebtype, tol, original, + bytes, both_fused); + round_trip(shape, 2, 0, true, false, ebtype, tol, original, + bytes, compress_fused); + round_trip(shape, 2, 0, false, true, ebtype, tol, original, + bytes, decompress_fused); + round_trip(shape, 2, 0, false, false, ebtype, tol, original, + bytes, neither); + + // All four combinations must honour the tolerance. This is what makes the + // switch safe to flip on either side independently: nothing in the file + // records which path wrote it, so a file compressed fused may well be + // decompressed separately, and vice versa. + double both = L_inf_error(n, original.data(), both_fused.data(), ebtype); + double cf = L_inf_error(n, original.data(), compress_fused.data(), ebtype); + double df = L_inf_error(n, original.data(), decompress_fused.data(), ebtype); + double none = L_inf_error(n, original.data(), neither.data(), ebtype); + CAPTURE(both, cf, df, none); + REQUIRE(both <= static_cast(tol)); + REQUIRE(cf <= static_cast(tol)); + REQUIRE(df <= static_cast(tol)); + REQUIRE(none <= static_cast(tol)); +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_LZ4.cpp.in b/tests/mgard-x/src/test_LZ4.cpp.in new file mode 100644 index 0000000000..089b38d9dd --- /dev/null +++ b/tests/mgard-x/src/test_LZ4.cpp.in @@ -0,0 +1,96 @@ +/* + * Copyright 2025, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +#include "catch2/catch_test_macros.hpp" + +#include +#include +#include + +#include "mgard-x/RuntimeX/RuntimeX.h" +#include "mgard-x/Lossless/LZ4/LZ4.hpp" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; + +// LZ4 is the nvcomp-free, backend-portable byte-stream codec operating +// in place on an Array<1, Byte>: Compress replaces the array with the +// compressed stream (header + per-chunk sizes + packed LZ4 blocks), Decompress +// restores the original bytes (size recovered from the stream). +TEST_CASE("LZ4 round-trip", "[mgard-x][@DEVICE_TYPE@]") { + const SIZE n = 100000; + const SIZE chunk_size = 1 << 15; + + DeviceRuntime::Initialize(); + + // Low-entropy data (values 0-7) so LZ4 achieves meaningful compression. + std::vector original(n); + std::srand(42); + for (SIZE i = 0; i < n; i++) { + original[i] = static_cast(std::rand() % 8); + } + + Array<1, Byte, TestDevice> data({n}); + data.load(original.data()); + + // Separate codec objects; each needs its own scratch buffer via Resize. + LZ4 compressor; + compressor.Resize(n, chunk_size, 0); + compressor.Compress(data, 0); + DeviceRuntime::SyncQueue(0); + + // The compressed stream should be strictly smaller than the input. + REQUIRE(data.shape(0) < n); + + LZ4 decompressor; + decompressor.Resize(n, chunk_size, 0); + decompressor.Decompress(data, 0); + DeviceRuntime::SyncQueue(0); + + REQUIRE(data.shape(0) == n); + Byte *result = data.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + for (SIZE i = 0; i < n; i++) { + REQUIRE(result[i] == original[i]); + } +} + +// Incompressible (high-entropy) data must still round-trip exactly, exercising +// the literal-only / extension-length paths of the encoder and decoder. +TEST_CASE("LZ4 incompressible round-trip", "[mgard-x][@DEVICE_TYPE@]") { + const SIZE n = 70000; // not a multiple of chunk_size -> exercises last chunk + const SIZE chunk_size = 1 << 14; + + DeviceRuntime::Initialize(); + + std::vector original(n); + std::srand(7); + for (SIZE i = 0; i < n; i++) { + original[i] = static_cast(std::rand() % 256); + } + + Array<1, Byte, TestDevice> data({n}); + data.load(original.data()); + + LZ4 compressor; + compressor.Resize(n, chunk_size, 0); + compressor.Compress(data, 0); + + LZ4 decompressor; + decompressor.Resize(n, chunk_size, 0); + decompressor.Decompress(data, 0); + DeviceRuntime::SyncQueue(0); + + REQUIRE(data.shape(0) == n); + Byte *result = data.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + for (SIZE i = 0; i < n; i++) { + REQUIRE(result[i] == original[i]); + } +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_MGARDDecomposer.cpp.in b/tests/mgard-x/src/test_MGARDDecomposer.cpp.in new file mode 100644 index 0000000000..91dd9961dd --- /dev/null +++ b/tests/mgard-x/src/test_MGARDDecomposer.cpp.in @@ -0,0 +1,176 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +// MGARDDecomposer (include/mgard-x/MDR-X/Decomposer/MGARD.hpp) is the +// DecomposerInterface adapter ComposedRefactor/ComposedReconstructor actually +// drive: it owns a data_refactoring::DataRefactor and maps the Basis template +// parameter (Orthogonal/Hierarchical) onto that refactor's +// `orthogonal_projection` bool, working on a plain Array& +// rather than a SubArray. test_DataRefactoring.cpp.in already pins the +// underlying transform's numerics; this pins the adapter itself -- the +// Basis-to-bool mapping, Adapt(), and the Array-based decompose/recompose +// signatures MGARDDecomposer actually exposes. + +#include "catch2/catch_template_test_macros.hpp" +#include "catch2/catch_test_macros.hpp" + +#include +#include + +#include "mgard-x/RuntimeX/RuntimeX.h" +#include "mgard-x/MDR-X/Decomposer/MGARD.hpp" +#include "mgard-x/Utilities/ErrorCalculator.h" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; + +// decompose() then recompose() over the same in-place Array must reproduce +// the original data. MGARD's decomposition is a floating point transform +// (not a pure permutation, unlike DirectInterleaver), so the round trip is +// bounded by a tolerance rather than required to be bit-exact -- the same +// tolerance test_DataRefactoring.cpp.in uses for the identical underlying +// transform: tight for double (~1e-10), generous for float (~1e-3), +// reflecting float/double accumulation over O(log N) levels. +template +void run_mgard_decomposer_roundtrip(std::vector shape) { + SIZE n = 1; + for (auto s : shape) + n *= s; + + const double pi = std::acos(-1.0); + std::vector original(n); + for (SIZE i = 0; i < n; i++) + original[i] = static_cast(std::sin(2.0 * pi * i / n)); + + Config config; + Hierarchy hierarchy(shape, config); + MDR::MGARDDecomposer decomposer(hierarchy, config); + + Array data(shape); + data.load(original.data()); + + decomposer.decompose(data, hierarchy.l_target(), 0, 0); + decomposer.recompose(data, 0, hierarchy.l_target(), 0); + + T *result = data.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + + const double tolerance = std::is_same::value ? 1e-10 : 1e-3; + double error = L_inf_error(n, original.data(), result, error_bound_type::REL); + CAPTURE(error); + REQUIRE(error < tolerance); +} + +TEMPLATE_TEST_CASE("MDR-X MGARDDecomposer Orthogonal basis 1D", + "[mgard-x][mdr-x][mgard-decomposer][@DEVICE_TYPE@]", float, + double) { + DeviceRuntime::Initialize(); + SECTION("small") { + run_mgard_decomposer_roundtrip<1, TestType, MDR::Orthogonal, TestDevice>( + {33}); + } + SECTION("medium") { + run_mgard_decomposer_roundtrip<1, TestType, MDR::Orthogonal, TestDevice>( + {129}); + } + SECTION("large") { + run_mgard_decomposer_roundtrip<1, TestType, MDR::Orthogonal, TestDevice>( + {4097}); + } +} + +TEMPLATE_TEST_CASE("MDR-X MGARDDecomposer Orthogonal basis 2D", + "[mgard-x][mdr-x][mgard-decomposer][@DEVICE_TYPE@]", float, + double) { + DeviceRuntime::Initialize(); + SECTION("small") { + run_mgard_decomposer_roundtrip<2, TestType, MDR::Orthogonal, TestDevice>( + {33, 33}); + } + SECTION("medium") { + run_mgard_decomposer_roundtrip<2, TestType, MDR::Orthogonal, TestDevice>( + {129, 129}); + } + SECTION("large") { + run_mgard_decomposer_roundtrip<2, TestType, MDR::Orthogonal, TestDevice>( + {513, 513}); + } +} + +TEMPLATE_TEST_CASE("MDR-X MGARDDecomposer Orthogonal basis 3D", + "[mgard-x][mdr-x][mgard-decomposer][@DEVICE_TYPE@]", float, + double) { + DeviceRuntime::Initialize(); + SECTION("small") { + run_mgard_decomposer_roundtrip<3, TestType, MDR::Orthogonal, TestDevice>( + {17, 17, 17}); + } + SECTION("medium") { + run_mgard_decomposer_roundtrip<3, TestType, MDR::Orthogonal, TestDevice>( + {33, 33, 33}); + } + SECTION("large") { + run_mgard_decomposer_roundtrip<3, TestType, MDR::Orthogonal, TestDevice>( + {65, 65, 65}); + } +} + +TEMPLATE_TEST_CASE("MDR-X MGARDDecomposer Hierarchical basis 1D", + "[mgard-x][mdr-x][mgard-decomposer][@DEVICE_TYPE@]", float, + double) { + DeviceRuntime::Initialize(); + SECTION("small") { + run_mgard_decomposer_roundtrip<1, TestType, MDR::Hierarchical, TestDevice>( + {33}); + } + SECTION("medium") { + run_mgard_decomposer_roundtrip<1, TestType, MDR::Hierarchical, TestDevice>( + {129}); + } + SECTION("large") { + run_mgard_decomposer_roundtrip<1, TestType, MDR::Hierarchical, TestDevice>( + {4097}); + } +} + +TEMPLATE_TEST_CASE("MDR-X MGARDDecomposer Hierarchical basis 2D", + "[mgard-x][mdr-x][mgard-decomposer][@DEVICE_TYPE@]", float, + double) { + DeviceRuntime::Initialize(); + SECTION("small") { + run_mgard_decomposer_roundtrip<2, TestType, MDR::Hierarchical, TestDevice>( + {33, 33}); + } + SECTION("medium") { + run_mgard_decomposer_roundtrip<2, TestType, MDR::Hierarchical, TestDevice>( + {129, 129}); + } + SECTION("large") { + run_mgard_decomposer_roundtrip<2, TestType, MDR::Hierarchical, TestDevice>( + {513, 513}); + } +} + +TEMPLATE_TEST_CASE("MDR-X MGARDDecomposer Hierarchical basis 3D", + "[mgard-x][mdr-x][mgard-decomposer][@DEVICE_TYPE@]", float, + double) { + DeviceRuntime::Initialize(); + SECTION("small") { + run_mgard_decomposer_roundtrip<3, TestType, MDR::Hierarchical, TestDevice>( + {17, 17, 17}); + } + SECTION("medium") { + run_mgard_decomposer_roundtrip<3, TestType, MDR::Hierarchical, TestDevice>( + {33, 33, 33}); + } + SECTION("large") { + run_mgard_decomposer_roundtrip<3, TestType, MDR::Hierarchical, TestDevice>( + {65, 65, 65}); + } +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_RLE.cpp.in b/tests/mgard-x/src/test_RLE.cpp.in new file mode 100644 index 0000000000..dc5b720723 --- /dev/null +++ b/tests/mgard-x/src/test_RLE.cpp.in @@ -0,0 +1,71 @@ +/* + * Copyright 2025, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +#include "catch2/catch_test_macros.hpp" + +#include +#include +#include + +#include "mgard-x/RuntimeX/RuntimeX.h" +#include "mgard-x/Lossless/ParallelRLE/RunLengthEncoding.hpp" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; + +// Run-length encoding round trip. Compress encodes (count, symbol) runs into a +// serialized byte stream; the decompressor Deserializes the metadata (which +// zero-copies into the compressed stream) and Decompresses back to the symbols. +TEST_CASE("RLE", "[mgard-x][@DEVICE_TYPE@]") { + using T_symbol = uint32_t; + using C_run = uint32_t; + using C_global = uint32_t; + + const SIZE n = 100000; + + DeviceRuntime::Initialize(); + + // Generate run-friendly data: repeated blocks of a small symbol set. + std::vector original(n); + std::srand(42); + SIZE i = 0; + while (i < n) { + SIZE run = std::rand() % 20 + 1; + T_symbol val = static_cast(std::rand() % 16); + for (SIZE j = 0; j < run && i < n; j++, i++) { + original[i] = val; + } + } + + Array<1, T_symbol, TestDevice> original_data({n}); + original_data.load(original.data()); + + // Separate compressor and decompressor objects. + parallel_rle::RunLengthEncoding + compressor; + compressor.Resize(n, 0); + Array<1, Byte, TestDevice> compressed_data; + // target_cr == 0 disables the compression-ratio early-out, so it always runs. + bool success = compressor.Compress(original_data, compressed_data, 0.0, 0); + REQUIRE(success == true); + + parallel_rle::RunLengthEncoding + decompressor; + decompressor.Resize(n, 0); + decompressor.Deserialize(compressed_data, 0); + Array<1, T_symbol, TestDevice> decompressed_data; + decompressor.Decompress(compressed_data, decompressed_data, 0); + + REQUIRE(decompressed_data.shape(0) == n); + T_symbol *result = decompressed_data.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + for (SIZE k = 0; k < n; k++) { + REQUIRE(result[k] == original[k]); + } +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_Rans.cpp.in b/tests/mgard-x/src/test_Rans.cpp.in new file mode 100644 index 0000000000..a5d1611d5d --- /dev/null +++ b/tests/mgard-x/src/test_Rans.cpp.in @@ -0,0 +1,141 @@ +/* + * Copyright 2025, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +#include "catch2/catch_test_macros.hpp" + +#include +#include +#include + +#include "mgard-x/RuntimeX/RuntimeX.h" +#include "mgard-x/Lossless/rANS/Rans.hpp" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; + +namespace { + +// Compress a byte stream with rANS then decompress through a fresh object +// (metadata travels inside the serialized stream) and return the bytes plus the +// compressed size, so tests can assert both round-trip correctness and that the +// codec actually shrinks compressible input. +struct RansResult { + std::vector bytes; + SIZE compressed_size; + SIZE original_size; +}; + +RansResult RoundTrip(const std::vector &original) { + const SIZE n = static_cast(original.size()); + + Array<1, Byte, TestDevice> input_data({n}); + if (n > 0) { + input_data.load(original.data()); + } + + rans::Rans compressor; + compressor.Resize(n == 0 ? 1 : n, 256, 0, 0, /*interleaved=*/true); + Array<1, Byte, TestDevice> compressed_data; + bool success = compressor.Compress(input_data, compressed_data, 0); + REQUIRE(success == true); + SIZE compressed_size = compressed_data.shape(0); + + rans::Rans decompressor; + decompressor.Resize(n == 0 ? 1 : n, 256, 0, 0, /*interleaved=*/true); + decompressor.Deserialize(compressed_data, 0); + Array<1, Byte, TestDevice> decompressed_data; + decompressor.Decompress(compressed_data, decompressed_data, 0); + + REQUIRE(decompressed_data.shape(0) == n); + RansResult res; + res.compressed_size = compressed_size; + res.original_size = n; + if (n > 0) { + Byte *result = decompressed_data.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + res.bytes.assign(result, result + n); + } + return res; +} + +void RequireRoundTrip(const std::vector &original) { + RansResult res = RoundTrip(original); + REQUIRE(res.bytes.size() == original.size()); + for (SIZE k = 0; k < original.size(); k++) { + REQUIRE(res.bytes[k] == original[k]); + } +} + +} // namespace + +TEST_CASE("Rans", "[mgard-x][@DEVICE_TYPE@]") { + DeviceRuntime::Initialize(); + + SECTION("uniform random bytes (round trip, ~incompressible)") { + const SIZE n = 200000; + std::vector original(n); + std::srand(1); + for (SIZE i = 0; i < n; i++) { + original[i] = static_cast(std::rand() & 0xFF); + } + RequireRoundTrip(original); + } + + SECTION("highly skewed (breaks the Huffman 1-bit floor)") { + // ~99% zeros: a per-symbol prefix code would spend >= 1 bit/symbol here; + // rANS should compress far past that. + const SIZE n = 200000; + std::vector original(n, 0); + std::srand(2); + for (SIZE i = 0; i < n; i++) { + if (std::rand() % 100 == 0) { + original[i] = static_cast(std::rand() % 255 + 1); + } + } + RansResult res = RoundTrip(original); + REQUIRE(res.bytes == original); + // Far better than the 32x/8 = ... ; for a byte alphabet Huffman floors at + // 8x (1 bit/byte). rANS must beat that comfortably on 99%-zero data. + REQUIRE(res.compressed_size * 8 < res.original_size); + } + + SECTION("single symbol (near-unbounded ratio)") { + const SIZE n = 100000; + std::vector original(n, 7); + RansResult res = RoundTrip(original); + REQUIRE(res.bytes == original); + REQUIRE(res.compressed_size * 50 < res.original_size); + } + + SECTION("skewed multi-symbol distribution") { + const SIZE n = 150000; + std::vector original(n); + std::srand(3); + for (SIZE i = 0; i < n; i++) { + int r = std::rand() % 100; + original[i] = static_cast(r < 70 ? 0 : (r < 90 ? 1 : (r % 7) + 2)); + } + RequireRoundTrip(original); + } + + SECTION("size not a multiple of segment size") { + const SIZE n = 2048 * 3 + 17; // forces a short final segment + std::vector original(n); + std::srand(4); + for (SIZE i = 0; i < n; i++) { + original[i] = static_cast(std::rand() % 4); + } + RequireRoundTrip(original); + } + + SECTION("small input (single partial segment)") { + std::vector original = {3, 3, 3, 1, 2, 3, 0, 0, 0, 9, 9, 1}; + RequireRoundTrip(original); + } +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_ZeroRLE.cpp.in b/tests/mgard-x/src/test_ZeroRLE.cpp.in new file mode 100644 index 0000000000..1b649aad92 --- /dev/null +++ b/tests/mgard-x/src/test_ZeroRLE.cpp.in @@ -0,0 +1,116 @@ +/* + * Copyright 2025, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +#include "catch2/catch_test_macros.hpp" + +#include +#include +#include + +#include "mgard-x/RuntimeX/RuntimeX.h" +#include "mgard-x/Lossless/ParallelRLE/ZeroRunLengthEncoding.hpp" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; + +namespace { + +using T_symbol = uint32_t; +using C_run = uint32_t; +using C_global = uint32_t; + +// Compress with zero-RLE then decompress through a fresh object (mirroring how +// the codec is used: metadata travels inside the serialized stream) and return +// the reconstructed values for comparison against the input. +std::vector RoundTrip(const std::vector &original) { + const SIZE n = static_cast(original.size()); + + Array<1, T_symbol, TestDevice> original_data({n}); + original_data.load(original.data()); + + parallel_rle::ZeroRunLengthEncoding + compressor; + compressor.Resize(n, 0); + Array<1, Byte, TestDevice> compressed_data; + // target_cr == 0 disables the compression-ratio early-out, so it always runs. + bool success = compressor.Compress(original_data, compressed_data, 0.0, 0); + REQUIRE(success == true); + + parallel_rle::ZeroRunLengthEncoding + decompressor; + decompressor.Resize(n, 0); + decompressor.Deserialize(compressed_data, 0); + Array<1, T_symbol, TestDevice> decompressed_data; + decompressor.Decompress(compressed_data, decompressed_data, 0); + + REQUIRE(decompressed_data.shape(0) == n); + T_symbol *result = decompressed_data.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + return std::vector(result, result + n); +} + +void RequireRoundTrip(const std::vector &original) { + std::vector result = RoundTrip(original); + REQUIRE(result.size() == original.size()); + for (SIZE k = 0; k < original.size(); k++) { + REQUIRE(result[k] == original[k]); + } +} + +} // namespace + +// Zero-RLE (RLE0) round trip across the sparsity edge cases that distinguish it +// from full RLE: leading/interior/trailing zero runs, all zeros, and no zeros. +TEST_CASE("ZeroRLE", "[mgard-x][@DEVICE_TYPE@]") { + DeviceRuntime::Initialize(); + + SECTION("sparse with leading and trailing zeros") { + // Long leading zero run, a few sparse nonzeros, long trailing zero run. + std::vector original(50000, 0); + original[12345] = 7; + original[12346] = 9; // adjacent nonzeros (gap == 0) + original[20000] = 3; + original[33333] = 42; + // [33334, 49999] stay zero -> trailing zeros are implied, not stored. + RequireRoundTrip(original); + } + + SECTION("first and last element nonzero") { + std::vector original(1000, 0); + original[0] = 11; // gap before first symbol == 0 + original[999] = 22; // no trailing zeros + original[500] = 33; + RequireRoundTrip(original); + } + + SECTION("all zeros") { + std::vector original(4096, 0); + RequireRoundTrip(original); + } + + SECTION("no zeros (dense)") { + std::vector original(4096); + for (SIZE i = 0; i < original.size(); i++) { + original[i] = static_cast(i % 255 + 1); // never zero + } + RequireRoundTrip(original); + } + + SECTION("random sparse (~10% nonzero)") { + const SIZE n = 100000; + std::vector original(n, 0); + std::srand(42); + for (SIZE i = 0; i < n; i++) { + if (std::rand() % 10 == 0) { + original[i] = static_cast(std::rand() % 255 + 1); + } + } + RequireRoundTrip(original); + } +} + +} // namespace mgard_x diff --git a/tests/mgard-x/src/test_Zstd.cpp.in b/tests/mgard-x/src/test_Zstd.cpp.in new file mode 100644 index 0000000000..105a2d6b3e --- /dev/null +++ b/tests/mgard-x/src/test_Zstd.cpp.in @@ -0,0 +1,55 @@ +/* + * Copyright 2025, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + */ + +#include "catch2/catch_test_macros.hpp" + +#include +#include +#include + +#include "mgard-x/RuntimeX/RuntimeX.h" +#include "mgard-x/Lossless/Zstd.hpp" + +namespace mgard_x { + +using TestDevice = @DEVICE_TYPE@; + +// Zstd is a host-side byte-stream codec that operates in place on an +// Array<1, Byte>: Compress turns the array into the compressed stream (with a +// size_t length prefix), Decompress turns it back into the original bytes. +TEST_CASE("Zstd", "[mgard-x][@DEVICE_TYPE@]") { + const SIZE n = 100000; + const int compression_level = 3; + + DeviceRuntime::Initialize(); + + // Low-entropy data (values 0-7) so Zstd achieves meaningful compression. + std::vector original(n); + std::srand(42); + for (SIZE i = 0; i < n; i++) { + original[i] = static_cast(std::rand() % 8); + } + + Array<1, Byte, TestDevice> data({n}); + data.load(original.data()); + + // Separate codec objects so they share no internal state. + Zstd compressor(n, compression_level); + compressor.Compress(data, 0); + + Zstd decompressor(n, compression_level); + decompressor.Decompress(data, 0); + DeviceRuntime::SyncQueue(0); + + REQUIRE(data.shape(0) == n); + Byte *result = data.hostCopy(false, 0); + DeviceRuntime::SyncQueue(0); + for (SIZE i = 0; i < n; i++) { + REQUIRE(result[i] == original[i]); + } +} + +} // namespace mgard_x diff --git a/tests/mgard/CMakeLists.txt b/tests/mgard/CMakeLists.txt new file mode 100644 index 0000000000..bb78482e16 --- /dev/null +++ b/tests/mgard/CMakeLists.txt @@ -0,0 +1,70 @@ +set(_MGARD_TEST_DIR "${CMAKE_CURRENT_LIST_DIR}") + +set( + MGARD_TESTS_CPP + "${_MGARD_TEST_DIR}/src/testing_utilities.cpp" + "${_MGARD_TEST_DIR}/src/test_utilities.cpp" + "${_MGARD_TEST_DIR}/src/test_blas.cpp" + "${_MGARD_TEST_DIR}/src/test_shuffle.cpp" + "${_MGARD_TEST_DIR}/src/test_TensorMeshHierarchy.cpp" + "${_MGARD_TEST_DIR}/src/test_TensorMeshHierarchyIteration.cpp" + "${_MGARD_TEST_DIR}/src/test_TensorLinearOperator.cpp" + "${_MGARD_TEST_DIR}/src/test_TensorMassMatrix.cpp" + "${_MGARD_TEST_DIR}/src/test_TensorProlongation.cpp" + "${_MGARD_TEST_DIR}/src/test_TensorRestriction.cpp" + "${_MGARD_TEST_DIR}/src/test_LinearQuantizer.cpp" + "${_MGARD_TEST_DIR}/src/test_TensorMultilevelCoefficientQuantizer.cpp" + "${_MGARD_TEST_DIR}/src/test_TensorNorms.cpp" + "${_MGARD_TEST_DIR}/src/test_TensorQuantityOfInterest.cpp" + "${_MGARD_TEST_DIR}/src/test_compress.cpp" + "${_MGARD_TEST_DIR}/src/test_decompose.cpp" + "${_MGARD_TEST_DIR}/src/test_format.cpp" + "${_MGARD_TEST_DIR}/src/test_quantize.cpp" + # "${_MGARD_TEST_DIR}/src/test_compressors.cpp" + "${_MGARD_TEST_DIR}/src/test_CompressedDataset.cpp" +) + +if(MGARD_ENABLE_UNSTRUCTURED AND MOAB_FOUND) + list( + APPEND + MGARD_TESTS_CPP + "${_MGARD_TEST_DIR}/src/unstructured/test_estimators.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_indicators.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_norms.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_data.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_pcg.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_measure.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_MeshLevel.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_MeshHierarchy.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_MeshRefiner.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_LinearOperator.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_MassMatrix.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_UniformMeshHierarchy.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_UniformMeshRefiner.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_UniformEdgeFamilies.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_UniformRestriction.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_EnumeratedMeshRange.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_SituatedCoefficientRange.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_IndicatorInput.cpp" + "${_MGARD_TEST_DIR}/src/unstructured/test_MultilevelCoefficientQuantizer.cpp" + ) +endif() + +if(NOT Catch2_FOUND) + message(STATUS "Will not build mgard tests (Catch2 not found).") + return() +endif() + +configure_file( + "${_MGARD_TEST_DIR}/include/testing_paths.hpp.in" + "${CMAKE_CURRENT_BINARY_DIR}/include/testing_paths.hpp" +) +add_executable(mgard-tests ${MGARD_TESTS_CPP}) +target_include_directories( + mgard-tests PRIVATE + "${_MGARD_TEST_DIR}/include" + "${CMAKE_CURRENT_BINARY_DIR}/include" +) +target_link_libraries(mgard-tests PRIVATE mgard-library Catch2::Catch2WithMain) +set_target_properties(mgard-tests PROPERTIES OUTPUT_NAME tests) +catch_discover_tests(mgard-tests TEST_PREFIX "mgard/") diff --git a/tests/include/testing_paths.hpp.in b/tests/mgard/include/testing_paths.hpp.in similarity index 100% rename from tests/include/testing_paths.hpp.in rename to tests/mgard/include/testing_paths.hpp.in diff --git a/tests/include/testing_random.hpp b/tests/mgard/include/testing_random.hpp similarity index 98% rename from tests/include/testing_random.hpp rename to tests/mgard/include/testing_random.hpp index 2af8d69605..c927f37484 100644 --- a/tests/include/testing_random.hpp +++ b/tests/mgard/include/testing_random.hpp @@ -4,7 +4,7 @@ #include #include -#include "TensorMeshHierarchy.hpp" +#include "mgard/TensorMeshHierarchy.hpp" //! Random polynomial with the exponent of each variable in each term being //! either zero or one. diff --git a/tests/include/testing_random.tpp b/tests/mgard/include/testing_random.tpp similarity index 98% rename from tests/include/testing_random.tpp rename to tests/mgard/include/testing_random.tpp index 051048cbe3..44a514aa4a 100644 --- a/tests/include/testing_random.tpp +++ b/tests/mgard/include/testing_random.tpp @@ -5,9 +5,9 @@ #include #include -#include "blas.hpp" +#include "mgard/blas.hpp" -#include "TensorMeshHierarchyIteration.hpp" +#include "mgard/TensorMeshHierarchyIteration.hpp" #include "testing_utilities.hpp" template diff --git a/tests/include/testing_utilities.hpp b/tests/mgard/include/testing_utilities.hpp similarity index 97% rename from tests/include/testing_utilities.hpp rename to tests/mgard/include/testing_utilities.hpp index 318d521d6d..9e0a8952a6 100644 --- a/tests/include/testing_utilities.hpp +++ b/tests/mgard/include/testing_utilities.hpp @@ -10,7 +10,7 @@ static const double APPROX_MARGIN_DEFAULT = 0; -#include "TensorMeshHierarchy.hpp" +#include "mgard/TensorMeshHierarchy.hpp" // TODO: Change these? See . diff --git a/tests/include/testing_utilities.tpp b/tests/mgard/include/testing_utilities.tpp similarity index 100% rename from tests/include/testing_utilities.tpp rename to tests/mgard/include/testing_utilities.tpp diff --git a/tests/include/unstructured/testing_utilities.hpp b/tests/mgard/include/unstructured/testing_utilities.hpp similarity index 100% rename from tests/include/unstructured/testing_utilities.hpp rename to tests/mgard/include/unstructured/testing_utilities.hpp diff --git a/tests/meshes/circle.msh b/tests/mgard/meshes/circle.msh similarity index 100% rename from tests/meshes/circle.msh rename to tests/mgard/meshes/circle.msh diff --git a/tests/meshes/hexahedron.msh b/tests/mgard/meshes/hexahedron.msh similarity index 100% rename from tests/meshes/hexahedron.msh rename to tests/mgard/meshes/hexahedron.msh diff --git a/tests/meshes/lopsided.msh b/tests/mgard/meshes/lopsided.msh similarity index 100% rename from tests/meshes/lopsided.msh rename to tests/mgard/meshes/lopsided.msh diff --git a/tests/meshes/pyramid.msh b/tests/mgard/meshes/pyramid.msh similarity index 100% rename from tests/meshes/pyramid.msh rename to tests/mgard/meshes/pyramid.msh diff --git a/tests/meshes/seated.msh b/tests/mgard/meshes/seated.msh similarity index 100% rename from tests/meshes/seated.msh rename to tests/mgard/meshes/seated.msh diff --git a/tests/meshes/slope.msh b/tests/mgard/meshes/slope.msh similarity index 100% rename from tests/meshes/slope.msh rename to tests/mgard/meshes/slope.msh diff --git a/tests/meshes/tetrahedron.msh b/tests/mgard/meshes/tetrahedron.msh similarity index 100% rename from tests/meshes/tetrahedron.msh rename to tests/mgard/meshes/tetrahedron.msh diff --git a/tests/meshes/triangle.msh b/tests/mgard/meshes/triangle.msh similarity index 100% rename from tests/meshes/triangle.msh rename to tests/mgard/meshes/triangle.msh diff --git a/tests/outputs/circle_L=2_coefficients.txt b/tests/mgard/outputs/circle_L=2_coefficients.txt similarity index 100% rename from tests/outputs/circle_L=2_coefficients.txt rename to tests/mgard/outputs/circle_L=2_coefficients.txt diff --git a/tests/src/test_CompressedDataset.cpp b/tests/mgard/src/test_CompressedDataset.cpp similarity index 94% rename from tests/src/test_CompressedDataset.cpp rename to tests/mgard/src/test_CompressedDataset.cpp index c92dffc362..d74ef0bc6e 100644 --- a/tests/src/test_CompressedDataset.cpp +++ b/tests/mgard/src/test_CompressedDataset.cpp @@ -4,12 +4,12 @@ #include #include -#include "CompressedDataset.hpp" #include "compress.hpp" +#include "mgard/CompressedDataset.hpp" #include "testing_random.hpp" -TEST_CASE("data buffers and sizes", "[CompressedDataset]") { +TEST_CASE("data buffers and sizes", "[mgard][CompressedDataset]") { { const mgard::TensorMeshHierarchy<2, float> hierarchy({6, 3}); const float s = 0.25; @@ -76,7 +76,8 @@ void test_serialization(const mgard::TensorMeshHierarchy &hierarchy, } // namespace -TEST_CASE("compressed dataset (de)serialization", "[CompressedDataset]") { +TEST_CASE("compressed dataset (de)serialization", + "[mgard][CompressedDataset]") { std::default_random_engine gen(534393); { const mgard::TensorMeshHierarchy<1, float> hierarchy({381}); diff --git a/tests/src/test_LinearQuantizer.cpp b/tests/mgard/src/test_LinearQuantizer.cpp similarity index 94% rename from tests/src/test_LinearQuantizer.cpp rename to tests/mgard/src/test_LinearQuantizer.cpp index 0cfe96ac69..414869f48f 100644 --- a/tests/src/test_LinearQuantizer.cpp +++ b/tests/mgard/src/test_LinearQuantizer.cpp @@ -12,7 +12,7 @@ #include "testing_utilities.hpp" -#include "LinearQuantizer.hpp" +#include "mgard/LinearQuantizer.hpp" // Not immediately seeing a way to handle these templates using Catch2 macros. @@ -67,14 +67,14 @@ static void test_dequantization_inversion(const Real quantum, REQUIRE(tracker); } -TEST_CASE("quantization error", "[LinearQuantizer]") { +TEST_CASE("quantization error", "[mgard][LinearQuantizer]") { test_quantization_error(0.01); test_quantization_error(2.4); test_quantization_error(0.5); test_quantization_error(0.89327); } -TEST_CASE("quantization exceptions", "[LinearQuantizer]") { +TEST_CASE("quantization exceptions", "[mgard][LinearQuantizer]") { SECTION("quantum must be positive") { REQUIRE_THROWS(mgard::LinearQuantizer(-12.2)); REQUIRE_THROWS(mgard::LinearQuantizer(0)); @@ -89,7 +89,7 @@ TEST_CASE("quantization exceptions", "[LinearQuantizer]") { } } -TEST_CASE("quantization of a range", "[LinearQuantizer]") { +TEST_CASE("quantization of a range", "[mgard][LinearQuantizer]") { // This originally tested a quantizer applied to a range 'in its entirety' // (rather than element-by-element). SECTION("basic quantization iteration") { diff --git a/tests/src/test_TensorLinearOperator.cpp b/tests/mgard/src/test_TensorLinearOperator.cpp similarity index 97% rename from tests/src/test_TensorLinearOperator.cpp rename to tests/mgard/src/test_TensorLinearOperator.cpp index 8d667df160..c4017c3ace 100644 --- a/tests/src/test_TensorLinearOperator.cpp +++ b/tests/mgard/src/test_TensorLinearOperator.cpp @@ -8,9 +8,9 @@ #include "testing_utilities.hpp" -#include "TensorLinearOperator.hpp" -#include "TensorMeshHierarchy.hpp" -#include "shuffle.hpp" +#include "mgard/TensorLinearOperator.hpp" +#include "mgard/TensorMeshHierarchy.hpp" +#include "mgard/shuffle.hpp" namespace { @@ -103,7 +103,7 @@ class ThreeByThreeMatrix : public mgard::ConstituentLinearOperator<2, float> { } // namespace -TEST_CASE("simple constituent operators", "[TensorLinearOperator]") { +TEST_CASE("simple constituent operators", "[mgard][TensorLinearOperator]") { SECTION("diagonal constituent operators") { const mgard::TensorMeshHierarchy<2, float> hierarchy({3, 3}); const std::size_t ndof = 3 * 3; @@ -178,7 +178,7 @@ TEST_CASE("simple constituent operators", "[TensorLinearOperator]") { } TEST_CASE("tensor products of simple constituent operators", - "[TensorLinearOperator]") { + "[mgard][TensorLinearOperator]") { { const mgard::TensorMeshHierarchy<3, float> hierarchy({2, 2, 2}); const std::size_t ndof = 2 * 2 * 2; @@ -235,7 +235,7 @@ TEST_CASE("tensor products of simple constituent operators", } TEST_CASE("tensor product linear operators on 'flat' meshes", - "[TensorLinearOperator]") { + "[mgard][TensorLinearOperator]") { const std::vector A_diagonal = {2, 2, 3, 5}; const std::vector B_diagonal = {-2, -1, 0, 1, 2}; const std::size_t L = 2; diff --git a/tests/src/test_TensorMassMatrix.cpp b/tests/mgard/src/test_TensorMassMatrix.cpp similarity index 97% rename from tests/src/test_TensorMassMatrix.cpp rename to tests/mgard/src/test_TensorMassMatrix.cpp index 525bddb1c0..2cfab84855 100644 --- a/tests/src/test_TensorMassMatrix.cpp +++ b/tests/mgard/src/test_TensorMassMatrix.cpp @@ -12,13 +12,13 @@ #include "testing_random.hpp" #include "testing_utilities.hpp" -#include "TensorMassMatrix.hpp" -#include "TensorMeshHierarchy.hpp" -#include "blas.hpp" -#include "shuffle.hpp" -#include "utilities.hpp" +#include "mgard/TensorMassMatrix.hpp" +#include "mgard/TensorMeshHierarchy.hpp" +#include "mgard/blas.hpp" +#include "mgard/shuffle.hpp" +#include "mgard/utilities.hpp" -TEST_CASE("constituent mass matrices", "[TensorMassMatrix]") { +TEST_CASE("constituent mass matrices", "[mgard][TensorMassMatrix]") { SECTION("1D and default spacing") { const mgard::TensorMeshHierarchy<1, float> hierarchy({9}); const std::size_t ndof = 9; @@ -206,7 +206,7 @@ TEST_CASE("constituent mass matrices", "[TensorMassMatrix]") { } } -TEST_CASE("tensor product mass matrices", "[TensorMassMatrix]") { +TEST_CASE("tensor product mass matrices", "[mgard][TensorMassMatrix]") { const mgard::TensorMeshHierarchy<2, double> hierarchy( {3, 3}, {{{0, 0.5, 1}, {1, 1.25, 2}}}); const std::size_t ndof = 3 * 3; @@ -301,7 +301,7 @@ void exhaustive_constituent_inverse_test( } // namespace -TEST_CASE("constituent mass matrix inverses", "[TensorMassMatrix]") { +TEST_CASE("constituent mass matrix inverses", "[mgard][TensorMassMatrix]") { SECTION("1D and default spacing") { const mgard::TensorMeshHierarchy<1, float> hierarchy({9}); const std::size_t ndof = 9; @@ -425,7 +425,7 @@ void test_mass_matrix_inversion( } // namespace -TEST_CASE("tensor product mass matrix inverses", "[TensorMassMatrix]") { +TEST_CASE("tensor product mass matrix inverses", "[mgard][TensorMassMatrix]") { std::default_random_engine generator(741495); std::array u_; { @@ -451,7 +451,8 @@ TEST_CASE("tensor product mass matrix inverses", "[TensorMassMatrix]") { } } -TEST_CASE("mass matrices and inverses on 'flat' meshes", "[TensorMassMatrix]") { +TEST_CASE("mass matrices and inverses on 'flat' meshes", + "[mgard][TensorMassMatrix]") { const std::size_t ndof = 36; const std::size_t l = 2; std::vector u_(ndof); diff --git a/tests/src/test_TensorMeshHierarchy.cpp b/tests/mgard/src/test_TensorMeshHierarchy.cpp similarity index 97% rename from tests/src/test_TensorMeshHierarchy.cpp rename to tests/mgard/src/test_TensorMeshHierarchy.cpp index 121a752915..3cfa3e8dcf 100644 --- a/tests/src/test_TensorMeshHierarchy.cpp +++ b/tests/mgard/src/test_TensorMeshHierarchy.cpp @@ -11,12 +11,12 @@ #include "testing_utilities.hpp" -#include "TensorMeshHierarchy.hpp" -#include "TensorMeshHierarchyIteration.hpp" -#include "shuffle.hpp" -#include "utilities.hpp" +#include "mgard/TensorMeshHierarchy.hpp" +#include "mgard/TensorMeshHierarchyIteration.hpp" +#include "mgard/shuffle.hpp" +#include "mgard/utilities.hpp" -TEST_CASE("hierarchy mesh shapes", "[TensorMeshHierarchy]") { +TEST_CASE("hierarchy mesh shapes", "[mgard][TensorMeshHierarchy]") { { const std::array shape = {5}; const mgard::TensorMeshHierarchy<1, float> hierarchy(shape); @@ -50,7 +50,7 @@ TEST_CASE("hierarchy mesh shapes", "[TensorMeshHierarchy]") { REQUIRE_THROWS(mgard::TensorMeshHierarchy<2, float>({17, 0})); } -TEST_CASE("TensorMeshHierarchy construction", "[TensorMeshHierarchy]") { +TEST_CASE("TensorMeshHierarchy construction", "[mgard][TensorMeshHierarchy]") { { const mgard::TensorMeshHierarchy<1, float> hierarchy({17}); REQUIRE(hierarchy.uniform); @@ -266,7 +266,7 @@ void test_entry_indexing_exhaustive(const std::array shape) { } // namespace -TEST_CASE("TensorMeshHierarchy indexing", "[TensorMeshHierarchy]") { +TEST_CASE("TensorMeshHierarchy indexing", "[mgard][TensorMeshHierarchy]") { SECTION("accessing elements") { { const std::vector> multiindices = { @@ -379,7 +379,7 @@ void test_index_iteration( } // namespace -TEST_CASE("index iteration", "[TensorMeshHierarchy]") { +TEST_CASE("index iteration", "[mgard][TensorMeshHierarchy]") { { const std::array shape = {9, 5}; const std::vector, 2>> expected = { @@ -407,7 +407,7 @@ TEST_CASE("index iteration", "[TensorMeshHierarchy]") { } } -TEST_CASE("node iteration", "[TensorMeshHierarchy]") { +TEST_CASE("node iteration", "[mgard][TensorMeshHierarchy]") { // The largest of the mesh sizes used below. const std::size_t N = 11 * 14; float *const buffer = new float[N]; @@ -523,7 +523,7 @@ TEST_CASE("node iteration", "[TensorMeshHierarchy]") { delete[] buffer; } -TEST_CASE("dates of birth", "[TensorMeshHierarchy]") { +TEST_CASE("dates of birth", "[mgard][TensorMeshHierarchy]") { { const mgard::TensorMeshHierarchy<1, float> hierarchy({9}); std::vector encountered; @@ -593,7 +593,7 @@ void check_decomposition_hierarchy(const mgard::pb::Header &header) { } // namespace -TEST_CASE("header field population", "[TensorMeshHierarchy]") { +TEST_CASE("header field population", "[mgard][TensorMeshHierarchy]") { { mgard::pb::Header header; const std::array shape{123}; diff --git a/tests/src/test_TensorMeshHierarchyIteration.cpp b/tests/mgard/src/test_TensorMeshHierarchyIteration.cpp similarity index 97% rename from tests/src/test_TensorMeshHierarchyIteration.cpp rename to tests/mgard/src/test_TensorMeshHierarchyIteration.cpp index 662e7fdb25..9de6f14a86 100644 --- a/tests/src/test_TensorMeshHierarchyIteration.cpp +++ b/tests/mgard/src/test_TensorMeshHierarchyIteration.cpp @@ -6,12 +6,12 @@ #include "testing_utilities.hpp" -#include "TensorMeshHierarchy.hpp" -#include "TensorMeshHierarchyIteration.hpp" -#include "utilities.hpp" +#include "mgard/TensorMeshHierarchy.hpp" +#include "mgard/TensorMeshHierarchyIteration.hpp" +#include "mgard/utilities.hpp" TEST_CASE("TensorIndexRange size and iteration", - "[TensorMeshHierarchyIteration]") { + "[mgard][TensorMeshHierarchyIteration]") { // Dyadic. { const mgard::TensorMeshHierarchy<1, float> hierarchy({17}); @@ -93,7 +93,7 @@ void increment_and_test_neighbors( } // namespace TEST_CASE("TensorNode predecessors and successors", - "[TensorMeshHierarchyIteration]") { + "[mgard][TensorMeshHierarchyIteration]") { const mgard::TensorMeshHierarchy<2, float> hierarchy({3, 3}); SECTION("'normal' nodes") { // Finest level. @@ -216,7 +216,7 @@ void test_shuffled_dereferencing( } // namespace TEST_CASE("ShuffledTensorNodeRange dereferencing", - "[TensorMeshHierarchyIteration]") { + "[mgard][TensorMeshHierarchyIteration]") { SECTION("1D") { const std::vector> expected = {{0}, {5}, {2}, {1}, {3}, {4}}; diff --git a/tests/src/test_TensorMultilevelCoefficientQuantizer.cpp b/tests/mgard/src/test_TensorMultilevelCoefficientQuantizer.cpp similarity index 95% rename from tests/src/test_TensorMultilevelCoefficientQuantizer.cpp rename to tests/mgard/src/test_TensorMultilevelCoefficientQuantizer.cpp index 103e8ff0ef..0505726459 100644 --- a/tests/src/test_TensorMultilevelCoefficientQuantizer.cpp +++ b/tests/mgard/src/test_TensorMultilevelCoefficientQuantizer.cpp @@ -6,12 +6,12 @@ #include "proto/mgard.pb.h" -#include "blas.hpp" +#include "mgard/blas.hpp" -#include "TensorMeshHierarchy.hpp" -#include "TensorMultilevelCoefficientQuantizer.hpp" -#include "TensorNorms.hpp" -#include "decompose.hpp" +#include "mgard/TensorMeshHierarchy.hpp" +#include "mgard/TensorMultilevelCoefficientQuantizer.hpp" +#include "mgard/TensorNorms.hpp" +#include "mgard/decompose.hpp" #include "testing_random.hpp" #include "testing_utilities.hpp" @@ -57,7 +57,7 @@ void test_mc_quantization_iteration(const std::array shape, } // namespace TEST_CASE("tensor multilevel coefficient (de)quantization iteration", - "[TensorMultilevelCoefficientQuantizer]") { + "[mgard][TensorMultilevelCoefficientQuantizer]") { test_mc_quantization_iteration<1, float, int>( {65}, std::numeric_limits::infinity(), 0.1); test_mc_quantization_iteration<2, double, long int>( @@ -170,7 +170,7 @@ void test_mc_quantization_error(const std::array shape, } // namespace TEST_CASE("tensor multilevel coefficient (de)quantization inversion", - "[TensorMultilevelCoefficientQuantizer]") { + "[mgard][TensorMultilevelCoefficientQuantizer]") { std::default_random_engine generator; const std::vector smoothness_parameters = { std::numeric_limits::infinity(), -0.75, 0, 1.5}; @@ -189,7 +189,7 @@ TEST_CASE("tensor multilevel coefficient (de)quantization inversion", } TEST_CASE("tensor multilevel coefficient (de)quantization error", - "[TensorMultilevelCoefficientQuantizer]") { + "[mgard][TensorMultilevelCoefficientQuantizer]") { std::default_random_engine generator; const std::vector smoothness_parameters = { std::numeric_limits::infinity(), -3, 0, 0.75}; diff --git a/tests/src/test_TensorNorms.cpp b/tests/mgard/src/test_TensorNorms.cpp similarity index 95% rename from tests/src/test_TensorNorms.cpp rename to tests/mgard/src/test_TensorNorms.cpp index 393d334508..f9fd306287 100644 --- a/tests/src/test_TensorNorms.cpp +++ b/tests/mgard/src/test_TensorNorms.cpp @@ -11,11 +11,11 @@ #include #include -#include "blas.hpp" +#include "mgard/blas.hpp" -#include "TensorMeshHierarchy.hpp" -#include "TensorMeshHierarchyIteration.hpp" -#include "TensorNorms.hpp" +#include "mgard/TensorMeshHierarchy.hpp" +#include "mgard/TensorMeshHierarchyIteration.hpp" +#include "mgard/TensorNorms.hpp" #include "testing_random.hpp" #include "testing_utilities.hpp" @@ -76,7 +76,7 @@ void test_tensor_basic_norm_properties( } // namespace -TEST_CASE("tensor basic norm properties", "[norms]") { +TEST_CASE("tensor basic norm properties", "[mgard][norms]") { std::default_random_engine generator; { std::uniform_real_distribution node_spacing_distribution(1, 3); @@ -114,7 +114,8 @@ float f(const std::array xyz) { } // namespace -TEST_CASE("comparison with Python implementation: tensor norms", "[norms]") { +TEST_CASE("comparison with Python implementation: tensor norms", + "[mgard][norms]") { const mgard::TensorMeshHierarchy<3, float> hierarchy({9, 9, 9}); const std::size_t ndof = hierarchy.ndof(); std::vector u_(ndof); @@ -181,7 +182,7 @@ void test_component_norms_sum( } // namespace -TEST_CASE("orthogonal component norms", "[norms]") { +TEST_CASE("orthogonal component norms", "[mgard][norms]") { std::default_random_engine gen; // Node spacing distribution. std::uniform_real_distribution dis(0.01, 0.02); diff --git a/tests/src/test_TensorProlongation.cpp b/tests/mgard/src/test_TensorProlongation.cpp similarity index 95% rename from tests/src/test_TensorProlongation.cpp rename to tests/mgard/src/test_TensorProlongation.cpp index a1225ed4d8..2bcd016241 100644 --- a/tests/src/test_TensorProlongation.cpp +++ b/tests/mgard/src/test_TensorProlongation.cpp @@ -7,13 +7,13 @@ #include "testing_random.hpp" #include "testing_utilities.hpp" -#include "TensorMeshHierarchy.hpp" -#include "TensorMeshHierarchyIteration.hpp" -#include "TensorProlongation.hpp" -#include "shuffle.hpp" -#include "utilities.hpp" +#include "mgard/TensorMeshHierarchy.hpp" +#include "mgard/TensorMeshHierarchyIteration.hpp" +#include "mgard/TensorProlongation.hpp" +#include "mgard/shuffle.hpp" +#include "mgard/utilities.hpp" -TEST_CASE("constituent prolongations", "[TensorProlongation]") { +TEST_CASE("constituent prolongations", "[mgard][TensorProlongation]") { SECTION("1D and default spacing") { const mgard::TensorMeshHierarchy<1, float> hierarchy({9}); const std::size_t ndof = 9; @@ -196,7 +196,7 @@ void test_tensor_product_prolongations(std::default_random_engine &generator, } // namespace -TEST_CASE("tensor product prolongations", "[TensorProlongation]") { +TEST_CASE("tensor product prolongations", "[mgard][TensorProlongation]") { std::default_random_engine generator(176067); SECTION("dyadic") { @@ -216,7 +216,7 @@ TEST_CASE("tensor product prolongations", "[TensorProlongation]") { } } -TEST_CASE("prolongations on 'flat' meshes", "[TensorProlongation]") { +TEST_CASE("prolongations on 'flat' meshes", "[mgard][TensorProlongation]") { const std::size_t ndof = 12; const std::size_t l = 3; std::vector u_(ndof); diff --git a/tests/src/test_TensorQuantityOfInterest.cpp b/tests/mgard/src/test_TensorQuantityOfInterest.cpp similarity index 94% rename from tests/src/test_TensorQuantityOfInterest.cpp rename to tests/mgard/src/test_TensorQuantityOfInterest.cpp index ef3062290f..9010f1de30 100644 --- a/tests/src/test_TensorQuantityOfInterest.cpp +++ b/tests/mgard/src/test_TensorQuantityOfInterest.cpp @@ -8,10 +8,10 @@ #include #include -#include "TensorMeshHierarchy.hpp" -#include "TensorNorms.hpp" -#include "TensorQuantityOfInterest.hpp" -#include "blas.hpp" +#include "mgard/TensorMeshHierarchy.hpp" +#include "mgard/TensorNorms.hpp" +#include "mgard/TensorQuantityOfInterest.hpp" +#include "mgard/blas.hpp" #include "testing_random.hpp" @@ -106,7 +106,7 @@ void test_qoi_norm_equality(std::default_random_engine &generator, } // namespace -TEST_CASE("Riesz representative norm equality", "[qoi]") { +TEST_CASE("Riesz representative norm equality", "[mgard][qoi]") { std::default_random_engine gen; // Node spacing distribution. std::uniform_real_distribution dis(0.05, 0.075); @@ -144,7 +144,7 @@ void test_average_norms(const std::array shape) { } // namespace -TEST_CASE("average quantity of interest", "[qoi]") { +TEST_CASE("average quantity of interest", "[mgard][qoi]") { // Could be any function contained in the coarsest function space here. test_average_norms<1, float>({25}); test_average_norms<2, double>({12, 14}); diff --git a/tests/src/test_TensorRestriction.cpp b/tests/mgard/src/test_TensorRestriction.cpp similarity index 95% rename from tests/src/test_TensorRestriction.cpp rename to tests/mgard/src/test_TensorRestriction.cpp index 436366786e..cd17c9d7b6 100644 --- a/tests/src/test_TensorRestriction.cpp +++ b/tests/mgard/src/test_TensorRestriction.cpp @@ -7,15 +7,15 @@ #include "testing_random.hpp" #include "testing_utilities.hpp" -#include "TensorMassMatrix.hpp" -#include "TensorMeshHierarchy.hpp" -#include "TensorMeshHierarchyIteration.hpp" -#include "TensorProlongation.hpp" -#include "TensorRestriction.hpp" -#include "shuffle.hpp" -#include "utilities.hpp" +#include "mgard/TensorMassMatrix.hpp" +#include "mgard/TensorMeshHierarchy.hpp" +#include "mgard/TensorMeshHierarchyIteration.hpp" +#include "mgard/TensorProlongation.hpp" +#include "mgard/TensorRestriction.hpp" +#include "mgard/shuffle.hpp" +#include "mgard/utilities.hpp" -TEST_CASE("constituent restrictions", "[TensorRestriction]") { +TEST_CASE("constituent restrictions", "[mgard][TensorRestriction]") { SECTION("1D and default spacing") { const mgard::TensorMeshHierarchy<1, float> hierarchy({9}); const std::size_t ndof = 9; @@ -197,7 +197,7 @@ void test_tensor_projection_identity(std::default_random_engine &generator, } // namespace -TEST_CASE("tensor product restrictions", "[TensorRestriction]") { +TEST_CASE("tensor product restrictions", "[mgard][TensorRestriction]") { { const mgard::TensorMeshHierarchy<2, double> hierarchy( {3, 3}, {{{0, 0.5, 1}, {-1, -0.5, 1}}}); @@ -236,7 +236,7 @@ TEST_CASE("tensor product restrictions", "[TensorRestriction]") { } } -TEST_CASE("restrictions on 'flat' meshes", "[TensorRestriction]") { +TEST_CASE("restrictions on 'flat' meshes", "[mgard][TensorRestriction]") { const std::size_t ndof = 90; const std::size_t l = 2; std::vector u_(ndof); diff --git a/tests/src/test_blas.cpp b/tests/mgard/src/test_blas.cpp similarity index 66% rename from tests/src/test_blas.cpp rename to tests/mgard/src/test_blas.cpp index 760c476c70..7f603f91e2 100644 --- a/tests/src/test_blas.cpp +++ b/tests/mgard/src/test_blas.cpp @@ -1,5 +1,5 @@ #include "catch2/catch_test_macros.hpp" -#include "blas.hpp" +#include "mgard/blas.hpp" // Empty. diff --git a/tests/src/test_compress.cpp b/tests/mgard/src/test_compress.cpp similarity index 95% rename from tests/src/test_compress.cpp rename to tests/mgard/src/test_compress.cpp index ebb41eecfd..1ec47358ad 100644 --- a/tests/src/test_compress.cpp +++ b/tests/mgard/src/test_compress.cpp @@ -13,12 +13,12 @@ #include "testing_random.hpp" #include "testing_utilities.hpp" -#include "TensorMeshHierarchy.hpp" -#include "TensorMeshHierarchyIteration.hpp" -#include "TensorNorms.hpp" -#include "blas.hpp" #include "compress.hpp" -#include "shuffle.hpp" +#include "mgard/TensorMeshHierarchy.hpp" +#include "mgard/TensorMeshHierarchyIteration.hpp" +#include "mgard/TensorNorms.hpp" +#include "mgard/blas.hpp" +#include "mgard/shuffle.hpp" namespace { @@ -72,8 +72,8 @@ void test_compression_decompression( } // namespace -TEMPLATE_TEST_CASE("compression followed by decompression", "[compress]", float, - double) { +TEMPLATE_TEST_CASE("compression followed by decompression", "[mgard][compress]", + float, double) { std::default_random_engine gen(343873); const std::vector smoothness_parameters = { -1.5, -0.5, 0.0, 0.5, 1.5, std::numeric_limits::infinity()}; @@ -117,7 +117,7 @@ void test_compression_error_bound( } // namespace -TEST_CASE("1D quadratic data", "[compress]") { +TEST_CASE("1D quadratic data", "[mgard][compress]") { { const mgard::TensorMeshHierarchy<1, float> hierarchy({64}); const std::size_t ndof = hierarchy.ndof(); @@ -144,7 +144,7 @@ TEST_CASE("1D quadratic data", "[compress]") { } } -TEST_CASE("3D constant data", "[compress]") { +TEST_CASE("3D constant data", "[mgard][compress]") { const mgard::TensorMeshHierarchy<3, float> hierarchy({16, 16, 16}); const std::size_t ndof = hierarchy.ndof(); float *const v = new float[ndof]; @@ -155,7 +155,7 @@ TEST_CASE("3D constant data", "[compress]") { delete[] v; } -TEST_CASE("1D cosine data", "[compress]") { +TEST_CASE("1D cosine data", "[mgard][compress]") { const mgard::TensorMeshHierarchy<1, double> hierarchy({4096}); const std::size_t ndof = hierarchy.ndof(); double *const v = new double[ndof]; @@ -169,7 +169,7 @@ TEST_CASE("1D cosine data", "[compress]") { delete[] v; } -TEST_CASE("2D cosine data", "[compress]") { +TEST_CASE("2D cosine data", "[mgard][compress]") { const mgard::TensorMeshHierarchy<2, float> hierarchy({256, 16}); const std::size_t ndof = hierarchy.ndof(); float *const v = new float[ndof]; @@ -211,7 +211,7 @@ void test_compression_on_flat_mesh( } // namespace -TEST_CASE("compressing on 'flat' meshes", "[compress]") { +TEST_CASE("compressing on 'flat' meshes", "[mgard][compress]") { std::default_random_engine gen(799875); std::uniform_real_distribution dis(0.01, 0.011); const mgard::TensorMeshHierarchy<2, float> hierarchy = @@ -293,7 +293,7 @@ void test_decompression_on_flat_mesh( } // namespace -TEST_CASE("decompressing on 'flat' meshes", "[compress]") { +TEST_CASE("decompressing on 'flat' meshes", "[mgard][compress]") { std::default_random_engine gen(780037); std::uniform_real_distribution dis(2, 3); const mgard::TensorMeshHierarchy<3, double> hierarchy = @@ -396,8 +396,8 @@ void test_self_describing_decompression( } // namespace -TEMPLATE_TEST_CASE("decompressing self-describing buffer", "[compress]", float, - double) { +TEMPLATE_TEST_CASE("decompressing self-describing buffer", "[mgard][compress]", + float, double) { std::default_random_engine gen(32094); const std::vector smoothness_parameters = { -1.5, -0.5, 0.0, 0.5, 1.5, std::numeric_limits::infinity()}; diff --git a/tests/src/test_compressors.cpp b/tests/mgard/src/test_compressors.cpp similarity index 94% rename from tests/src/test_compressors.cpp rename to tests/mgard/src/test_compressors.cpp index 8ab071fb6f..20636b8246 100644 --- a/tests/src/test_compressors.cpp +++ b/tests/mgard/src/test_compressors.cpp @@ -6,8 +6,8 @@ #include #include -#include "compressors.hpp" -#include "format.hpp" +#include "mgard/compressors.hpp" +#include "mgard/format.hpp" namespace { @@ -30,7 +30,7 @@ void test_huffman_identity(std::default_random_engine &gen, } // namespace -TEST_CASE("Huffman compression", "[compressors] [!mayfail]") { +TEST_CASE("Huffman compression", "[mgard][compressors] [!mayfail]") { std::default_random_engine gen(257100); const std::size_t n = 5000; SECTION("signed characters") { test_huffman_identity(gen, n); } @@ -60,7 +60,7 @@ void test_zstd_identity(std::uniform_int_distribution &dis, } // namespace #ifdef MGARD_ZSTD -TEST_CASE("zstd compression", "[compressors]") { +TEST_CASE("zstd compression", "[mgard][compressors]") { std::uniform_int_distribution dis; std::default_random_engine gen(158648); const std::vector ns{10, 10, 1000, 10000}; @@ -90,7 +90,7 @@ void test_zlib_identity(std::uniform_int_distribution &dis, } // namespace -TEST_CASE("zlib compression", "[compressors]") { +TEST_CASE("zlib compression", "[mgard][compressors]") { std::uniform_int_distribution dis; std::default_random_engine gen(252315); const std::vector ns{10, 10, 1000, 10000}; @@ -99,7 +99,7 @@ TEST_CASE("zlib compression", "[compressors]") { } } -TEST_CASE("compression with header configuration", "[compressors]") { +TEST_CASE("compression with header configuration", "[mgard][compressors]") { mgard::pb::Header header; // TODO: Once Huffman trees can be built for types other than `long int`, use // something other than `std::int64_t` here. @@ -137,7 +137,7 @@ TEST_CASE("compression with header configuration", "[compressors]") { delete[] quantized; } -TEST_CASE("decompression with header configuration", "[compressors]") { +TEST_CASE("decompression with header configuration", "[mgard][compressors]") { mgard::pb::Header header; // TODO: Once Huffman trees can be built for types other than `long int`, use // something other than `std::int64_t` here. @@ -216,7 +216,7 @@ TEST_CASE("decompression with header configuration", "[compressors]") { delete[] quantized; } -TEST_CASE("compression and decompression with header", "[compressors]") { +TEST_CASE("compression and decompression with header", "[mgard][compressors]") { mgard::pb::Header header; // TODO: Once Huffman trees can be built for types other than `long int`, use // something other than `std::int64_t` here. diff --git a/tests/src/test_decompose.cpp b/tests/mgard/src/test_decompose.cpp similarity index 97% rename from tests/src/test_decompose.cpp rename to tests/mgard/src/test_decompose.cpp index e56d822c3d..8be7e7155e 100644 --- a/tests/src/test_decompose.cpp +++ b/tests/mgard/src/test_decompose.cpp @@ -13,12 +13,12 @@ #include "testing_random.hpp" #include "testing_utilities.hpp" -#include "TensorMeshHierarchy.hpp" -#include "TensorProlongation.hpp" -#include "blas.hpp" -#include "decompose.hpp" -#include "format.hpp" -#include "shuffle.hpp" +#include "mgard/TensorMeshHierarchy.hpp" +#include "mgard/TensorProlongation.hpp" +#include "mgard/blas.hpp" +#include "mgard/decompose.hpp" +#include "mgard/format.hpp" +#include "mgard/shuffle.hpp" namespace { @@ -273,7 +273,7 @@ void test_recomposition_with_zero_coefficients( } // namespace -TEST_CASE("decomposition", "[decompose]") { +TEST_CASE("decomposition", "[mgard][decompose]") { SECTION("1D, dyadic, uniform") { const std::vector u_ = {10, 3, -8, -6, 3, 0, -5, 0, 0, -2, -8, -5, -10, -7, 8, -2, 3, -1, 0, 9, -4, -6, @@ -456,7 +456,7 @@ TEST_CASE("decomposition", "[decompose]") { REQUIRE(tracker); } - SECTION("linear dependence on input", "[decompose]") { + SECTION("linear dependence on input", "[mgard][decompose]") { std::default_random_engine generator(469957); std::uniform_real_distribution node_spacing_distribution(0.2, 1.1); std::uniform_real_distribution nodal_coefficient_distribution(0.5, @@ -474,7 +474,7 @@ TEST_CASE("decomposition", "[decompose]") { } // Piecewise linear on a coarser grid. - SECTION("coefficients of linear functions", "[decompose]") { + SECTION("coefficients of linear functions", "[mgard][decompose]") { std::default_random_engine generator(841397); std::uniform_real_distribution node_spacing_distribution(0.1, 0.3); std::uniform_real_distribution nodal_coefficient_distribution(-2, @@ -491,7 +491,7 @@ TEST_CASE("decomposition", "[decompose]") { {14, 10, 17}); } - SECTION("on 'flat' meshes", "[decompose]") { + SECTION("on 'flat' meshes", "[mgard][decompose]") { std::default_random_engine gen(731641); // Node spacing distribution. std::uniform_real_distribution dis(1, 1.1); @@ -546,7 +546,7 @@ TEST_CASE("decomposition", "[decompose]") { } } -TEST_CASE("recomposition", "[decompose]") { +TEST_CASE("recomposition", "[mgard][decompose]") { SECTION("1D, dyadic, uniform") { const std::vector u_ = {-4, 2, -4, 2, 7, -10, -4, -9, 9, 6, -1, 7, 9, 2, -9, -8, -1}; @@ -744,7 +744,7 @@ TEST_CASE("recomposition", "[decompose]") { test_dyadic_uniform_recomposition<4, float>(u_, expecteds); } - SECTION("linear dependence on input", "[decompose]") { + SECTION("linear dependence on input", "[mgard][decompose]") { std::default_random_engine generator(860343); std::uniform_real_distribution node_spacing_distribution(0.1, 0.3); std::uniform_real_distribution multilevel_coefficient_distribution( @@ -761,7 +761,7 @@ TEST_CASE("recomposition", "[decompose]") { {10, 15, 10}); } - SECTION("zero coefficients", "[decompose]") { + SECTION("zero coefficients", "[mgard][decompose]") { std::default_random_engine generator(848733); std::uniform_real_distribution node_spacing_distribution(1, 1.1); std::uniform_real_distribution multilevel_coefficient_distribution( @@ -778,7 +778,7 @@ TEST_CASE("recomposition", "[decompose]") { multilevel_coefficient_distribution, {17, 15, 9}); } - SECTION("on 'flat' meshes", "[decompose]") { + SECTION("on 'flat' meshes", "[mgard][decompose]") { std::default_random_engine gen(679382); // Node spacing distribution. std::uniform_real_distribution dis(0.25, 0.35); @@ -837,7 +837,7 @@ TEST_CASE("recomposition", "[decompose]") { } } -TEST_CASE("configuration by header", "[decompose]") { +TEST_CASE("configuration by header", "[mgard][decompose]") { mgard::pb::Header header; header.mutable_function_decomposition()->set_transform( mgard::pb::FunctionDecomposition::MULTILEVEL_COEFFICIENTS); diff --git a/tests/src/test_format.cpp b/tests/mgard/src/test_format.cpp similarity index 91% rename from tests/src/test_format.cpp rename to tests/mgard/src/test_format.cpp index 64943b2421..f77712623f 100644 --- a/tests/src/test_format.cpp +++ b/tests/mgard/src/test_format.cpp @@ -13,14 +13,14 @@ #include "testing_utilities.hpp" #include "MGARDConfig.hpp" -#include "format.hpp" +#include "mgard/format.hpp" namespace { using SizeBytes = std::array; using CRC32Bytes = std::array; } // namespace -TEST_CASE("header size and CRC32 deserialization", "[format]") { +TEST_CASE("header size and CRC32 deserialization", "[mgard][format]") { { const SizeBytes bytes{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf7}; REQUIRE(mgard::deserialize_header_size(bytes) == 247ULL); @@ -35,7 +35,7 @@ TEST_CASE("header size and CRC32 deserialization", "[format]") { } } -TEST_CASE("header size and CRC32 serialization", "[format]") { +TEST_CASE("header size and CRC32 serialization", "[mgard][format]") { { const SizeBytes expected{0x02, 0x03, 0x05, 0x07, 0x0b, 0x0d, 0x11, 0x13}; REQUIRE(mgard::serialize_header_size(144965140814303507ULL) == expected); @@ -50,7 +50,7 @@ TEST_CASE("header size and CRC32 serialization", "[format]") { } } -TEST_CASE("deserialization inverts serialization", "[format]") { +TEST_CASE("deserialization inverts serialization", "[mgard][format]") { const std::size_t ntrials = 500; std::default_random_engine gen(963912); { @@ -78,7 +78,7 @@ TEST_CASE("deserialization inverts serialization", "[format]") { } } -TEST_CASE("checking alignment", "[format]") { +TEST_CASE("checking alignment", "[mgard][format]") { double x; double const *const p = &x; REQUIRE_NOTHROW(mgard::check_alignment(p)); @@ -100,7 +100,7 @@ void check_version_number(const mgard::pb::VersionNumber &version_number, } // namespace -TEST_CASE("setting version numbers", "[format]") { +TEST_CASE("setting version numbers", "[mgard][format]") { mgard::pb::Header header; mgard::populate_version_numbers(header); check_version_number(header.mgard_version(), MGARD_VERSION_MAJOR, @@ -109,7 +109,7 @@ TEST_CASE("setting version numbers", "[format]") { MGARD_FILE_VERSION_MINOR, MGARD_FILE_VERSION_PATCH); } -TEST_CASE("advancing buffer windows", "[format]") { +TEST_CASE("advancing buffer windows", "[mgard][format]") { const std::size_t N = 10; unsigned char const *const p = new unsigned char[N]; mgard::BufferWindow window(p, N); @@ -122,7 +122,7 @@ TEST_CASE("advancing buffer windows", "[format]") { delete[] p; } -TEST_CASE("magic number", "[format]") { +TEST_CASE("magic number", "[mgard][format]") { unsigned char buffer[5]; for (std::size_t i = 0; i < 5; ++i) { buffer[i] = i; @@ -139,7 +139,7 @@ TEST_CASE("magic number", "[format]") { REQUIRE(window.current == buffer + 5); } -TEST_CASE("reading header size and CRC32", "[format]") { +TEST_CASE("reading header size and CRC32", "[mgard][format]") { const std::uint_least64_t header_size = 20; const std::uint_least32_t header_crc32 = 0x670b; const std::size_t n = mgard::HEADER_SIZE_SIZE + mgard::HEADER_CRC32_SIZE; @@ -160,7 +160,7 @@ TEST_CASE("reading header size and CRC32", "[format]") { REQUIRE(mgard::read_header_crc32(window) == header_crc32); } -TEST_CASE("checking header CRC32", "[format]") { +TEST_CASE("checking header CRC32", "[mgard][format]") { const std::uint_least64_t header_size = 40; unsigned char buffer[header_size]; for (std::uint_least64_t i = 0; i < header_size; ++i) { @@ -175,12 +175,12 @@ TEST_CASE("checking header CRC32", "[format]") { REQUIRE_NOTHROW(check_header_crc32(window, header_size, header_crc32)); } -TEST_CASE("dataset types", "[format]") { +TEST_CASE("dataset types", "[mgard][format]") { REQUIRE(mgard::type_to_dataset_type() == mgard::pb::Dataset::FLOAT); REQUIRE(mgard::type_to_dataset_type() == mgard::pb::Dataset::DOUBLE); } -TEST_CASE("quantization type sizes", "[format]") { +TEST_CASE("quantization type sizes", "[mgard][format]") { mgard::pb::Header header; mgard::pb::Quantization &quantization = *header.mutable_quantization(); const std::size_t ndof = 1; @@ -218,7 +218,7 @@ TEST_CASE("quantization type sizes", "[format]") { } } -TEST_CASE("reading topology and geometry", "[format]") { +TEST_CASE("reading topology and geometry", "[mgard][format]") { mgard::pb::Domain domain; const std::size_t dimension = 3; const std::vector shape{5, 5, 6}; @@ -269,7 +269,7 @@ TEST_CASE("reading topology and geometry", "[format]") { // TODO: Test storage of coefficients in separate file. } -TEST_CASE("reading dataset type", "[format]") { +TEST_CASE("reading dataset type", "[mgard][format]") { mgard::pb::Header header; mgard::pb::Dataset &d = *header.mutable_dataset(); d.set_dimension(1); @@ -283,7 +283,7 @@ TEST_CASE("reading dataset type", "[format]") { } } -TEST_CASE("reading error control parameters", "[format]") { +TEST_CASE("reading error control parameters", "[mgard][format]") { mgard::pb::Header header; mgard::pb::ErrorControl &e = *header.mutable_error_control(); { @@ -314,7 +314,7 @@ TEST_CASE("reading error control parameters", "[format]") { } } -TEST_CASE("checking decomposition parameters", "[format]") { +TEST_CASE("checking decomposition parameters", "[mgard][format]") { mgard::pb::Header header; mgard::pb::FunctionDecomposition &d = *header.mutable_function_decomposition(); @@ -335,7 +335,7 @@ TEST_CASE("checking decomposition parameters", "[format]") { } } -TEST_CASE("reading quantization parameters", "[format]") { +TEST_CASE("reading quantization parameters", "[mgard][format]") { mgard::pb::Header header; mgard::pb::Quantization &q = *header.mutable_quantization(); q.set_method(mgard::pb::Quantization::COEFFICIENTWISE_LINEAR); @@ -354,7 +354,7 @@ TEST_CASE("reading quantization parameters", "[format]") { } } -TEST_CASE("reading encoding compressor", "[format]") { +TEST_CASE("reading encoding compressor", "[mgard][format]") { mgard::pb::Header header; mgard::pb::Encoding &e = *header.mutable_encoding(); e.set_preprocessor(mgard::pb::Encoding::SHUFFLE); @@ -382,7 +382,7 @@ template void test_big_endian() { } // namespace -TEST_CASE("endianness", "[format]") { +TEST_CASE("endianness", "[mgard][format]") { test_big_endian(); test_big_endian(); test_big_endian(); @@ -404,7 +404,7 @@ void test_serialization_deserialization(const mgard::pb::Header &header) { } // namespace -TEST_CASE("metadata (de)serialization", "[format]") { +TEST_CASE("metadata (de)serialization", "[mgard][format]") { mgard::pb::Header header; mgard::populate_defaults(header); { test_serialization_deserialization(header); } diff --git a/tests/src/test_quantize.cpp b/tests/mgard/src/test_quantize.cpp similarity index 92% rename from tests/src/test_quantize.cpp rename to tests/mgard/src/test_quantize.cpp index a5fbaee321..0455c30abf 100644 --- a/tests/src/test_quantize.cpp +++ b/tests/mgard/src/test_quantize.cpp @@ -6,12 +6,12 @@ #include "proto/mgard.pb.h" -#include "TensorMeshHierarchy.hpp" -#include "TensorMultilevelCoefficientQuantizer.hpp" -#include "format.hpp" -#include "quantize.hpp" +#include "mgard/TensorMeshHierarchy.hpp" +#include "mgard/TensorMultilevelCoefficientQuantizer.hpp" +#include "mgard/format.hpp" +#include "mgard/quantize.hpp" -TEST_CASE("quantization", "[quantize]") { +TEST_CASE("quantization", "[mgard][quantize]") { const mgard::TensorMeshHierarchy<2, float> hierarchy({9, 10}); const std::size_t ndof = hierarchy.ndof(); const float s = 0; @@ -42,7 +42,7 @@ TEST_CASE("quantization", "[quantize]") { delete[] coefficients; } -TEST_CASE("dequantization", "[quantize]") { +TEST_CASE("dequantization", "[mgard][quantize]") { const mgard::TensorMeshHierarchy<1, double> hierarchy({148}); const std::size_t ndof = hierarchy.ndof(); const double s = 0.5; @@ -79,7 +79,7 @@ TEST_CASE("dequantization", "[quantize]") { delete[] quantized; } -TEST_CASE("alignment and endianness", "[quantize]") { +TEST_CASE("alignment and endianness", "[mgard][quantize]") { const mgard::TensorMeshHierarchy<3, float> hierarchy({5, 12, 13}); const std::size_t ndof = hierarchy.ndof(); const float s = 0; diff --git a/tests/src/test_shuffle.cpp b/tests/mgard/src/test_shuffle.cpp similarity index 94% rename from tests/src/test_shuffle.cpp rename to tests/mgard/src/test_shuffle.cpp index d2e2fba23b..8881abc257 100644 --- a/tests/src/test_shuffle.cpp +++ b/tests/mgard/src/test_shuffle.cpp @@ -3,8 +3,8 @@ #include #include -#include "TensorMeshHierarchy.hpp" -#include "shuffle.hpp" +#include "mgard/TensorMeshHierarchy.hpp" +#include "mgard/shuffle.hpp" #include "testing_utilities.hpp" @@ -30,7 +30,7 @@ void test_shuffle(const std::array shape, } // namespace -TEST_CASE("shuffle behavior", "[shuffle]") { +TEST_CASE("shuffle behavior", "[mgard][shuffle]") { SECTION("1D") { const std::vector expected = {0, 8, 4, 2, 6, 1, 3, 5, 7}; test_shuffle<1, float>({9}, expected); @@ -71,7 +71,7 @@ void test_unshuffle(const std::array shape, } // namespace -TEST_CASE("unshuffle behavior", "[shuffle]") { +TEST_CASE("unshuffle behavior", "[mgard][shuffle]") { SECTION("1D") { const std::vector expected = {0, 3, 5, 2, 6, 4, 7, 1}; test_unshuffle<1, double>({8}, expected); @@ -126,7 +126,7 @@ void test_inversion(const std::array shape) { } // namespace -TEST_CASE("shuffle inversion", "[shuffle]") { +TEST_CASE("shuffle inversion", "[mgard][shuffle]") { SECTION("1D") { test_inversion<1, float>({27}); // test_inversion<1, double>({33}); diff --git a/tests/src/test_utilities.cpp b/tests/mgard/src/test_utilities.cpp similarity index 92% rename from tests/src/test_utilities.cpp rename to tests/mgard/src/test_utilities.cpp index 3d665e331e..08c4826757 100644 --- a/tests/src/test_utilities.cpp +++ b/tests/mgard/src/test_utilities.cpp @@ -9,9 +9,9 @@ #include "testing_utilities.hpp" -#include "utilities.hpp" +#include "mgard/utilities.hpp" -TEST_CASE("PseudoArray iteration", "[utilities]") { +TEST_CASE("PseudoArray iteration", "[mgard][utilities]") { int values[10] = {0, 1, 4, 9, 16, 25, 36, 0, -2, 1}; SECTION("comparison with manual iteration") { @@ -41,7 +41,7 @@ TEST_CASE("PseudoArray iteration", "[utilities]") { } } -TEST_CASE("Enumeration iteration", "[utilities]") { +TEST_CASE("Enumeration iteration", "[mgard][utilities]") { const std::vector xs = {-1.375, 0, 732.5, -0.875}; std::vector indices; std::vector values; @@ -64,7 +64,7 @@ TEST_CASE("Enumeration iteration", "[utilities]") { } } -TEST_CASE("ZippedRange iteration", "[utilities]") { +TEST_CASE("ZippedRange iteration", "[mgard][utilities]") { using T = std::vector; using U = std::array; const T xs = {-3.28, 17.37, 0, 0.2388, -99.1}; @@ -80,7 +80,7 @@ TEST_CASE("ZippedRange iteration", "[utilities]") { REQUIRE(tracker); } -TEST_CASE("RangeSlice iteration", "[utilities]") { +TEST_CASE("RangeSlice iteration", "[mgard][utilities]") { const std::array xs = {2, 3, 5, 7, 11, 13, 17, 19}; using It = std::array::const_iterator; const mgard::RangeSlice slice{xs.begin() + 2, xs.end() - 2}; @@ -89,7 +89,7 @@ TEST_CASE("RangeSlice iteration", "[utilities]") { REQUIRE(middle == expected_middle); } -TEST_CASE("CartesianProduct iterator", "[utilities]") { +TEST_CASE("CartesianProduct iterator", "[mgard][utilities]") { { const std::vector a = {1, 3, 5}; const std::vector b = {2, 4, 6}; @@ -111,7 +111,8 @@ TEST_CASE("CartesianProduct iterator", "[utilities]") { } } -TEST_CASE("CartesianProduct predecessors and successors", "[utilities]") { +TEST_CASE("CartesianProduct predecessors and successors", + "[mgard][utilities]") { const std::vector a = {'a', 'b', 'c'}; const std::vector b = {'d', 'e'}; const mgard::CartesianProduct, 2> product({a, b}); diff --git a/tests/src/testing_utilities.cpp b/tests/mgard/src/testing_utilities.cpp similarity index 100% rename from tests/src/testing_utilities.cpp rename to tests/mgard/src/testing_utilities.cpp diff --git a/tests/src/unstructured/test_EnumeratedMeshRange.cpp b/tests/mgard/src/unstructured/test_EnumeratedMeshRange.cpp similarity index 82% rename from tests/src/unstructured/test_EnumeratedMeshRange.cpp rename to tests/mgard/src/unstructured/test_EnumeratedMeshRange.cpp index 0a5ca8dc7e..4d3194261a 100644 --- a/tests/src/unstructured/test_EnumeratedMeshRange.cpp +++ b/tests/mgard/src/unstructured/test_EnumeratedMeshRange.cpp @@ -7,11 +7,11 @@ #include "testing_utilities.hpp" -#include "unstructured/EnumeratedMeshRange.hpp" -#include "unstructured/MeshLevel.hpp" -#include "unstructured/UniformMeshHierarchy.hpp" +#include "mgard/unstructured/EnumeratedMeshRange.hpp" +#include "mgard/unstructured/MeshLevel.hpp" +#include "mgard/unstructured/UniformMeshHierarchy.hpp" -TEST_CASE("EnumeratedMeshRange iteration", "[EnumeratedMeshRange]") { +TEST_CASE("EnumeratedMeshRange iteration", "[mgard][EnumeratedMeshRange]") { const auto [filename, expected_ndofs] = GENERATE(table>( {{"seated.msh", {5, 12, 35}}, {"tetrahedron.msh", {4, 10, 35}}})); diff --git a/tests/src/unstructured/test_IndicatorInput.cpp b/tests/mgard/src/unstructured/test_IndicatorInput.cpp similarity index 86% rename from tests/src/unstructured/test_IndicatorInput.cpp rename to tests/mgard/src/unstructured/test_IndicatorInput.cpp index 44cafc84ad..1c87905672 100644 --- a/tests/src/unstructured/test_IndicatorInput.cpp +++ b/tests/mgard/src/unstructured/test_IndicatorInput.cpp @@ -11,10 +11,10 @@ #include "testing_utilities.hpp" -#include "unstructured/IndicatorInput.hpp" -#include "unstructured/MeshLevel.hpp" -#include "unstructured/UniformMeshHierarchy.hpp" -#include "unstructured/data.hpp" +#include "mgard/unstructured/IndicatorInput.hpp" +#include "mgard/unstructured/MeshLevel.hpp" +#include "mgard/unstructured/UniformMeshHierarchy.hpp" +#include "mgard/unstructured/data.hpp" // Function defined on the mesh. static double f(const mgard::MeshLevel &mesh, const moab::EntityHandle node) { @@ -24,7 +24,7 @@ static double f(const mgard::MeshLevel &mesh, const moab::EntityHandle node) { return (2 * xyz[0] + 1) * (0.5 * xyz[1] - 1) / (std::abs(xyz[2]) + 1); } -TEST_CASE("IndicatorInput iteration", "[IndicatorInput]") { +TEST_CASE("IndicatorInput iteration", "[mgard][IndicatorInput]") { const std::string filename = GENERATE("triangle.msh", "tetrahedron.msh"); moab::ErrorCode ecode; moab::Core mbcore; diff --git a/tests/src/unstructured/test_LinearOperator.cpp b/tests/mgard/src/unstructured/test_LinearOperator.cpp similarity index 88% rename from tests/src/unstructured/test_LinearOperator.cpp rename to tests/mgard/src/unstructured/test_LinearOperator.cpp index be5ce993ee..057c2009a5 100644 --- a/tests/src/unstructured/test_LinearOperator.cpp +++ b/tests/mgard/src/unstructured/test_LinearOperator.cpp @@ -1,6 +1,6 @@ #include "catch2/catch_test_macros.hpp" -#include "unstructured/LinearOperator.hpp" +#include "mgard/unstructured/LinearOperator.hpp" class DoNothingOperator : public mgard::LinearOperator { public: @@ -11,7 +11,7 @@ class DoNothingOperator : public mgard::LinearOperator { double *const) const override {} }; -TEST_CASE("basic member functions", "[LinearOperator]") { +TEST_CASE("basic member functions", "[mgard][LinearOperator]") { { DoNothingOperator A(5, 4); REQUIRE(!A.is_square()); diff --git a/tests/src/unstructured/test_MassMatrix.cpp b/tests/mgard/src/unstructured/test_MassMatrix.cpp similarity index 96% rename from tests/src/unstructured/test_MassMatrix.cpp rename to tests/mgard/src/unstructured/test_MassMatrix.cpp index c13ad113f8..9cc2360064 100644 --- a/tests/src/unstructured/test_MassMatrix.cpp +++ b/tests/mgard/src/unstructured/test_MassMatrix.cpp @@ -14,14 +14,14 @@ #include #include -#include "blas.hpp" +#include "mgard/blas.hpp" -#include "unstructured/MassMatrix.hpp" -#include "unstructured/pcg.hpp" +#include "mgard/unstructured/MassMatrix.hpp" +#include "mgard/unstructured/pcg.hpp" #include "testing_utilities.hpp" -TEST_CASE("mass matrix and mass matrix preconditioner", "[MassMatrix]") { +TEST_CASE("mass matrix and mass matrix preconditioner", "[mgard][MassMatrix]") { SECTION("triangles") { moab::ErrorCode ecode; const std::size_t num_nodes = 5; @@ -157,7 +157,7 @@ TEST_CASE("mass matrix and mass matrix preconditioner", "[MassMatrix]") { } } -TEST_CASE("contiguous subset mass matrix", "[MassMatrix]") { +TEST_CASE("contiguous subset mass matrix", "[mgard][MassMatrix]") { const auto [filename, i, n] = GENERATE(table( {{"circle.msh", 10, 30}, diff --git a/tests/src/unstructured/test_MeshHierarchy.cpp b/tests/mgard/src/unstructured/test_MeshHierarchy.cpp similarity index 52% rename from tests/src/unstructured/test_MeshHierarchy.cpp rename to tests/mgard/src/unstructured/test_MeshHierarchy.cpp index e560bfd3f4..c494115f2f 100644 --- a/tests/src/unstructured/test_MeshHierarchy.cpp +++ b/tests/mgard/src/unstructured/test_MeshHierarchy.cpp @@ -1,5 +1,5 @@ #include "catch2/catch_test_macros.hpp" -#include "unstructured/MeshHierarchy.hpp" +#include "mgard/unstructured/MeshHierarchy.hpp" // Empty. diff --git a/tests/src/unstructured/test_MeshLevel.cpp b/tests/mgard/src/unstructured/test_MeshLevel.cpp similarity index 96% rename from tests/src/unstructured/test_MeshLevel.cpp rename to tests/mgard/src/unstructured/test_MeshLevel.cpp index 0f2c0bf233..4ba858232b 100644 --- a/tests/src/unstructured/test_MeshLevel.cpp +++ b/tests/mgard/src/unstructured/test_MeshLevel.cpp @@ -7,8 +7,8 @@ #include "moab/Core.hpp" -#include "unstructured/MeshLevel.hpp" -#include "unstructured/pcg.hpp" +#include "mgard/unstructured/MeshLevel.hpp" +#include "mgard/unstructured/pcg.hpp" #include "testing_utilities.hpp" @@ -44,7 +44,7 @@ static moab::ErrorCode check_edges(mgard::MeshLevel &mesh, return moab::MB_SUCCESS; } -TEST_CASE("MeshLevel construction", "[MeshLevel]") { +TEST_CASE("MeshLevel construction", "[mgard][MeshLevel]") { const std::size_t num_nodes = 6; const std::size_t num_edges = 9; const std::size_t num_tris = 4; @@ -108,7 +108,7 @@ TEST_CASE("MeshLevel construction", "[MeshLevel]") { } } -TEST_CASE("edge generation", "[MeshLevel]") { +TEST_CASE("edge generation", "[mgard][MeshLevel]") { moab::ErrorCode ecode; moab::Core mbcore; diff --git a/tests/src/unstructured/test_MeshRefiner.cpp b/tests/mgard/src/unstructured/test_MeshRefiner.cpp similarity index 53% rename from tests/src/unstructured/test_MeshRefiner.cpp rename to tests/mgard/src/unstructured/test_MeshRefiner.cpp index 03d23b496c..4b87f83406 100644 --- a/tests/src/unstructured/test_MeshRefiner.cpp +++ b/tests/mgard/src/unstructured/test_MeshRefiner.cpp @@ -1,5 +1,5 @@ #include "catch2/catch_test_macros.hpp" -#include "unstructured/MeshRefiner.hpp" +#include "mgard/unstructured/MeshRefiner.hpp" // Empty. diff --git a/tests/src/unstructured/test_MultilevelCoefficientQuantizer.cpp b/tests/mgard/src/unstructured/test_MultilevelCoefficientQuantizer.cpp similarity index 93% rename from tests/src/unstructured/test_MultilevelCoefficientQuantizer.cpp rename to tests/mgard/src/unstructured/test_MultilevelCoefficientQuantizer.cpp index 45d6fbda5b..a0c2f638f9 100644 --- a/tests/src/unstructured/test_MultilevelCoefficientQuantizer.cpp +++ b/tests/mgard/src/unstructured/test_MultilevelCoefficientQuantizer.cpp @@ -13,14 +13,14 @@ #include "testing_utilities.hpp" -#include "unstructured/MeshLevel.hpp" -#include "unstructured/MultilevelCoefficientQuantizer.hpp" -#include "unstructured/UniformMeshHierarchy.hpp" -#include "unstructured/data.hpp" -#include "unstructured/norms.hpp" +#include "mgard/unstructured/MeshLevel.hpp" +#include "mgard/unstructured/MultilevelCoefficientQuantizer.hpp" +#include "mgard/unstructured/UniformMeshHierarchy.hpp" +#include "mgard/unstructured/data.hpp" +#include "mgard/unstructured/norms.hpp" TEST_CASE("multilevel coefficient (de)quantization iteration", - "[MultilevelCoefficientQuantizer]") { + "[mgard][MultilevelCoefficientQuantizer]") { const std::string filename = GENERATE("triangle.msh", "tetrahedron.msh"); moab::Core mbcore; const moab::ErrorCode ecode = mbcore.load_file(mesh_path(filename).c_str()); @@ -60,7 +60,7 @@ TEST_CASE("multilevel coefficient (de)quantization iteration", } TEST_CASE("quantization respects error bound", - "[MultilevelCoefficientQuantizer]") { + "[mgard][MultilevelCoefficientQuantizer]") { const std::string filename = GENERATE("lopsided.msh", "hexahedron.msh"); moab::Core mbcore; const moab::ErrorCode ecode = mbcore.load_file(mesh_path(filename).c_str()); @@ -109,7 +109,7 @@ TEST_CASE("quantization respects error bound", } TEST_CASE("multilevel coefficient (de)quantization inversion", - "[MultilevelCoefficientQuantizer]") { + "[mgard][MultilevelCoefficientQuantizer]") { const std::string filename = GENERATE("triangle.msh", "tetrahedron.msh"); moab::Core mbcore; const moab::ErrorCode ecode = mbcore.load_file(mesh_path(filename).c_str()); diff --git a/tests/src/unstructured/test_SituatedCoefficientRange.cpp b/tests/mgard/src/unstructured/test_SituatedCoefficientRange.cpp similarity index 84% rename from tests/src/unstructured/test_SituatedCoefficientRange.cpp rename to tests/mgard/src/unstructured/test_SituatedCoefficientRange.cpp index 4f2320e2dc..4164ba457a 100644 --- a/tests/src/unstructured/test_SituatedCoefficientRange.cpp +++ b/tests/mgard/src/unstructured/test_SituatedCoefficientRange.cpp @@ -11,10 +11,10 @@ #include "testing_utilities.hpp" -#include "unstructured/MeshLevel.hpp" -#include "unstructured/SituatedCoefficientRange.hpp" -#include "unstructured/UniformMeshHierarchy.hpp" -#include "unstructured/data.hpp" +#include "mgard/unstructured/MeshLevel.hpp" +#include "mgard/unstructured/SituatedCoefficientRange.hpp" +#include "mgard/unstructured/UniformMeshHierarchy.hpp" +#include "mgard/unstructured/data.hpp" // Function defined on the mesh. static double f(const mgard::MeshLevel &mesh, const moab::EntityHandle node) { @@ -25,7 +25,8 @@ static double f(const mgard::MeshLevel &mesh, const moab::EntityHandle node) { 2 * std::cos(xyz[1] - 4 * xyz[2])); } -TEST_CASE("SituatedCoefficientRange iteration", "[SituatedCoefficientRange]") { +TEST_CASE("SituatedCoefficientRange iteration", + "[mgard][SituatedCoefficientRange]") { const std::string filename = GENERATE("pyramid.msh", "hexahedron.msh"); moab::ErrorCode ecode; moab::Core mbcore; diff --git a/tests/mgard/src/unstructured/test_UniformEdgeFamilies.cpp b/tests/mgard/src/unstructured/test_UniformEdgeFamilies.cpp new file mode 100644 index 0000000000..e289eb5bdb --- /dev/null +++ b/tests/mgard/src/unstructured/test_UniformEdgeFamilies.cpp @@ -0,0 +1,5 @@ +#include "catch2/catch_test_macros.hpp" + +#include "mgard/unstructured/UniformEdgeFamilies.hpp" + +// Empty. diff --git a/tests/src/unstructured/test_UniformMeshHierarchy.cpp b/tests/mgard/src/unstructured/test_UniformMeshHierarchy.cpp similarity index 95% rename from tests/src/unstructured/test_UniformMeshHierarchy.cpp rename to tests/mgard/src/unstructured/test_UniformMeshHierarchy.cpp index 0594a75492..b2bed2637e 100644 --- a/tests/src/unstructured/test_UniformMeshHierarchy.cpp +++ b/tests/mgard/src/unstructured/test_UniformMeshHierarchy.cpp @@ -12,20 +12,20 @@ #include #include -#include "blas.hpp" +#include "mgard/blas.hpp" #include "moab/Core.hpp" #include "testing_utilities.hpp" -#include "unstructured/MeshLevel.hpp" -#include "unstructured/UniformMeshHierarchy.hpp" -#include "unstructured/data.hpp" +#include "mgard/unstructured/MeshLevel.hpp" +#include "mgard/unstructured/UniformMeshHierarchy.hpp" +#include "mgard/unstructured/data.hpp" // These tests call `decompose` and `recompose` in the old style and read the // transformed coefficients from the same identifier. -TEST_CASE("basic properties", "[UniformMeshHierarchy]") { +TEST_CASE("basic properties", "[mgard][UniformMeshHierarchy]") { const std::string filename = GENERATE("pyramid.msh", "hexahedron.msh"); moab::ErrorCode ecode; moab::Core mbcore; @@ -103,7 +103,7 @@ TEST_CASE("basic properties", "[UniformMeshHierarchy]") { } TEST_CASE("comparison with Python implementation: refinement and decomposition", - "[UniformMeshHierarchy]") { + "[mgard][UniformMeshHierarchy]") { moab::ErrorCode ecode; moab::Core mbcore; ecode = mbcore.load_file(mesh_path("circle.msh").c_str()); @@ -184,7 +184,7 @@ static double f(const mgard::MeshLevel &mesh, const moab::EntityHandle node) { return 4.27 * square(xyz[0]) - 9.28 * square(xyz[1]) + 0.288 * square(xyz[2]); } -TEST_CASE("iteration over nodes and values", "[UniformMeshHierarchy]") { +TEST_CASE("iteration over nodes and values", "[mgard][UniformMeshHierarchy]") { const std::string filename = GENERATE("slope.msh", "hexahedron.msh"); moab::ErrorCode ecode; moab::Core mbcore; diff --git a/tests/src/unstructured/test_UniformMeshRefiner.cpp b/tests/mgard/src/unstructured/test_UniformMeshRefiner.cpp similarity index 94% rename from tests/src/unstructured/test_UniformMeshRefiner.cpp rename to tests/mgard/src/unstructured/test_UniformMeshRefiner.cpp index 34d9427902..41e34ef128 100644 --- a/tests/src/unstructured/test_UniformMeshRefiner.cpp +++ b/tests/mgard/src/unstructured/test_UniformMeshRefiner.cpp @@ -5,9 +5,9 @@ #include "moab/Core.hpp" -#include "utilities.hpp" +#include "mgard/utilities.hpp" -#include "unstructured/UniformMeshRefiner.hpp" +#include "mgard/unstructured/UniformMeshRefiner.hpp" #include "testing_utilities.hpp" @@ -40,7 +40,7 @@ check_elements(mgard::MeshLevel &MESH, return moab::MB_SUCCESS; } -TEST_CASE("refining multiple triangles", "[UniformMeshRefiner]") { +TEST_CASE("refining multiple triangles", "[mgard][UniformMeshRefiner]") { moab::ErrorCode ecode; moab::Core mbcore; ecode = mbcore.load_file(mesh_path("seated.msh").c_str()); @@ -73,7 +73,7 @@ TEST_CASE("refining multiple triangles", "[UniformMeshRefiner]") { require_moab_success(ecode); } -TEST_CASE("refining multiple tetrahedra", "[UniformMeshRefiner]") { +TEST_CASE("refining multiple tetrahedra", "[mgard][UniformMeshRefiner]") { moab::ErrorCode ecode; moab::Core mbcore; ecode = mbcore.load_file(mesh_path("hexahedron.msh").c_str()); @@ -113,7 +113,7 @@ TEST_CASE("refining multiple tetrahedra", "[UniformMeshRefiner]") { require_moab_success(ecode); } -TEST_CASE("refining triangle multiply", "[UniformMeshRefiner]") { +TEST_CASE("refining triangle multiply", "[mgard][UniformMeshRefiner]") { moab::ErrorCode ecode; moab::Core mbcore; ecode = mbcore.load_file(mesh_path("triangle.msh").c_str()); @@ -159,7 +159,7 @@ TEST_CASE("refining triangle multiply", "[UniformMeshRefiner]") { require_moab_success(ecode); } -TEST_CASE("refining tetrahedron multiply", "[UniformMeshRefiner]") { +TEST_CASE("refining tetrahedron multiply", "[mgard][UniformMeshRefiner]") { moab::ErrorCode ecode; moab::Core mbcore; ecode = mbcore.load_file(mesh_path("tetrahedron.msh").c_str()); diff --git a/tests/src/unstructured/test_UniformRestriction.cpp b/tests/mgard/src/unstructured/test_UniformRestriction.cpp similarity index 87% rename from tests/src/unstructured/test_UniformRestriction.cpp rename to tests/mgard/src/unstructured/test_UniformRestriction.cpp index 22669eb38f..c3628d3c28 100644 --- a/tests/src/unstructured/test_UniformRestriction.cpp +++ b/tests/mgard/src/unstructured/test_UniformRestriction.cpp @@ -2,13 +2,13 @@ #include "moab/Core.hpp" -#include "unstructured/MeshLevel.hpp" -#include "unstructured/UniformMeshHierarchy.hpp" -#include "unstructured/UniformRestriction.hpp" +#include "mgard/unstructured/MeshLevel.hpp" +#include "mgard/unstructured/UniformMeshHierarchy.hpp" +#include "mgard/unstructured/UniformRestriction.hpp" #include "testing_utilities.hpp" -TEST_CASE("uniform functional restriction", "[UniformRestriction]") { +TEST_CASE("uniform functional restriction", "[mgard][UniformRestriction]") { SECTION("triangles") { moab::Core mbcore; moab::ErrorCode ecode; diff --git a/tests/src/unstructured/test_data.cpp b/tests/mgard/src/unstructured/test_data.cpp similarity index 57% rename from tests/src/unstructured/test_data.cpp rename to tests/mgard/src/unstructured/test_data.cpp index ae4ae09d2b..a1a9ff95ae 100644 --- a/tests/src/unstructured/test_data.cpp +++ b/tests/mgard/src/unstructured/test_data.cpp @@ -1,5 +1,5 @@ #include "catch2/catch_test_macros.hpp" -#include "unstructured/data.hpp" +#include "mgard/unstructured/data.hpp" // Empty. diff --git a/tests/src/unstructured/test_estimators.cpp b/tests/mgard/src/unstructured/test_estimators.cpp similarity index 90% rename from tests/src/unstructured/test_estimators.cpp rename to tests/mgard/src/unstructured/test_estimators.cpp index b35779355b..b66b50d3cc 100644 --- a/tests/src/unstructured/test_estimators.cpp +++ b/tests/mgard/src/unstructured/test_estimators.cpp @@ -13,19 +13,20 @@ #include "moab/Core.hpp" -#include "blas.hpp" +#include "mgard/blas.hpp" -#include "unstructured/MeshLevel.hpp" -#include "unstructured/UniformMeshHierarchy.hpp" -#include "unstructured/data.hpp" -#include "unstructured/estimators.hpp" -#include "unstructured/norms.hpp" +#include "mgard/unstructured/MeshLevel.hpp" +#include "mgard/unstructured/UniformMeshHierarchy.hpp" +#include "mgard/unstructured/data.hpp" +#include "mgard/unstructured/estimators.hpp" +#include "mgard/unstructured/norms.hpp" #include "testing_utilities.hpp" static const double inf = std::numeric_limits::infinity(); -TEST_CASE("comparison with Python implementation: estimators", "[estimators]") { +TEST_CASE("comparison with Python implementation: estimators", + "[mgard][estimators]") { moab::ErrorCode ecode; moab::Core mbcore; ecode = mbcore.load_file(mesh_path("slope.msh").c_str()); @@ -72,7 +73,7 @@ TEST_CASE("comparison with Python implementation: estimators", "[estimators]") { Catch::Approx(914.1806446523887)); } -TEST_CASE("estimators should track norms", "[estimators]") { +TEST_CASE("estimators should track norms", "[mgard][estimators]") { const std::string filename = GENERATE("pyramid.msh", "hexahedron.msh"); moab::ErrorCode ecode; moab::Core mbcore; diff --git a/tests/src/unstructured/test_indicators.cpp b/tests/mgard/src/unstructured/test_indicators.cpp similarity index 89% rename from tests/src/unstructured/test_indicators.cpp rename to tests/mgard/src/unstructured/test_indicators.cpp index ff21625716..e1cdd1ba9c 100644 --- a/tests/src/unstructured/test_indicators.cpp +++ b/tests/mgard/src/unstructured/test_indicators.cpp @@ -13,14 +13,14 @@ #include "moab/Core.hpp" -#include "blas.hpp" +#include "mgard/blas.hpp" -#include "unstructured/IndicatorInput.hpp" -#include "unstructured/MeshLevel.hpp" -#include "unstructured/UniformMeshHierarchy.hpp" -#include "unstructured/data.hpp" -#include "unstructured/estimators.hpp" -#include "unstructured/indicators.hpp" +#include "mgard/unstructured/IndicatorInput.hpp" +#include "mgard/unstructured/MeshLevel.hpp" +#include "mgard/unstructured/UniformMeshHierarchy.hpp" +#include "mgard/unstructured/data.hpp" +#include "mgard/unstructured/estimators.hpp" +#include "mgard/unstructured/indicators.hpp" #include "testing_utilities.hpp" @@ -38,7 +38,8 @@ unscaled_indicator(const mgard::MultilevelCoefficients u_mc, return std::sqrt(unscaled_square_indicator); } -TEST_CASE("comparison with Python implementation: indicators", "[indicators]") { +TEST_CASE("comparison with Python implementation: indicators", + "[mgard][indicators]") { moab::ErrorCode ecode; moab::Core mbcore; ecode = mbcore.load_file(mesh_path("seated.msh").c_str()); @@ -76,7 +77,7 @@ TEST_CASE("comparison with Python implementation: indicators", "[indicators]") { Catch::Approx(112.02614999556158)); } -TEST_CASE("indicators should track estimators", "[indicators]") { +TEST_CASE("indicators should track estimators", "[mgard][indicators]") { const std::string filename = GENERATE("lopsided.msh", "hexahedron.msh"); moab::ErrorCode ecode; moab::Core mbcore; diff --git a/tests/src/unstructured/test_measure.cpp b/tests/mgard/src/unstructured/test_measure.cpp similarity index 93% rename from tests/src/unstructured/test_measure.cpp rename to tests/mgard/src/unstructured/test_measure.cpp index 657a57182d..10febd28f8 100644 --- a/tests/src/unstructured/test_measure.cpp +++ b/tests/mgard/src/unstructured/test_measure.cpp @@ -3,11 +3,11 @@ #include -#include "blas.hpp" +#include "mgard/blas.hpp" -#include "unstructured/measure.hpp" +#include "mgard/unstructured/measure.hpp" -TEST_CASE("`orient_2d`", "[measure]") { +TEST_CASE("`orient_2d`", "[mgard][measure]") { // Just basic tests. Relying mostly on `test_tri_measure`. const std::size_t N = 2; const double a[N] = {5, 3}; @@ -18,7 +18,7 @@ TEST_CASE("`orient_2d`", "[measure]") { REQUIRE(mgard::orient_2d(a, a, c) == 0); } -TEST_CASE("`orient_3d`", "[measure]") { +TEST_CASE("`orient_3d`", "[mgard][measure]") { // Just basic tests. Relying mostly on `test_tet_measure`. const std::size_t N = 3; const double a[N] = {0, 23, 1}; @@ -30,7 +30,7 @@ TEST_CASE("`orient_3d`", "[measure]") { REQUIRE(mgard::orient_3d(a, b, d, c) == -determinant); } -TEST_CASE("edge measure", "[measure]") { +TEST_CASE("edge measure", "[mgard][measure]") { const std::size_t N = 6; const double a[N] = {0, 0, 0, 1, -2, 3}; const double base_length = mgard::edge_measure(a); @@ -67,7 +67,7 @@ TEST_CASE("edge measure", "[measure]") { } } -TEST_CASE("triangle measure", "[measure]") { +TEST_CASE("triangle measure", "[mgard][measure]") { const std::size_t N = 9; const double a[N] = {3, 1, 1, 0, 2, 0, 0, 4, -2}; const double base_area = mgard::tri_measure(a); @@ -107,7 +107,7 @@ TEST_CASE("triangle measure", "[measure]") { } } -TEST_CASE("tetrahedron measure", "[measure]") { +TEST_CASE("tetrahedron measure", "[mgard][measure]") { const std::size_t N = 12; const double a[N] = {0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, -4}; const double base_volume = mgard::tet_measure(a); diff --git a/tests/src/unstructured/test_norms.cpp b/tests/mgard/src/unstructured/test_norms.cpp similarity index 93% rename from tests/src/unstructured/test_norms.cpp rename to tests/mgard/src/unstructured/test_norms.cpp index 799daf36b9..59c950df9a 100644 --- a/tests/src/unstructured/test_norms.cpp +++ b/tests/mgard/src/unstructured/test_norms.cpp @@ -12,19 +12,19 @@ #include "moab/Core.hpp" -#include "blas.hpp" +#include "mgard/blas.hpp" -#include "unstructured/MeshLevel.hpp" -#include "unstructured/UniformMeshHierarchy.hpp" -#include "unstructured/data.hpp" -#include "unstructured/norms.hpp" +#include "mgard/unstructured/MeshLevel.hpp" +#include "mgard/unstructured/UniformMeshHierarchy.hpp" +#include "mgard/unstructured/data.hpp" +#include "mgard/unstructured/norms.hpp" #include "testing_random.hpp" #include "testing_utilities.hpp" static const double inf = std::numeric_limits::infinity(); -TEST_CASE("unstructured basic norm properties", "[norms]") { +TEST_CASE("unstructured basic norm properties", "[mgard][norms]") { const std::string filename = GENERATE("pyramid.msh", "tetrahedron.msh"); moab::ErrorCode ecode; moab::Core mbcore; @@ -95,7 +95,7 @@ TEST_CASE("unstructured basic norm properties", "[norms]") { } TEST_CASE("comparison with Python implementation: unstructured norms", - "[norms]") { + "[mgard][norms]") { moab::ErrorCode ecode; moab::Core mbcore; ecode = mbcore.load_file(mesh_path("circle.msh").c_str()); diff --git a/tests/src/unstructured/test_pcg.cpp b/tests/mgard/src/unstructured/test_pcg.cpp similarity index 96% rename from tests/src/unstructured/test_pcg.cpp rename to tests/mgard/src/unstructured/test_pcg.cpp index 8b6fdb708f..236077c4df 100644 --- a/tests/src/unstructured/test_pcg.cpp +++ b/tests/mgard/src/unstructured/test_pcg.cpp @@ -4,10 +4,10 @@ #include #include -#include "blas.hpp" +#include "mgard/blas.hpp" -#include "unstructured/LinearOperator.hpp" -#include "unstructured/pcg.hpp" +#include "mgard/unstructured/LinearOperator.hpp" +#include "mgard/unstructured/pcg.hpp" #include "testing_utilities.hpp" @@ -80,7 +80,7 @@ static void diagonal_scaling(double const *const x, double *const y) { y[3] = x[3] / (1); } -TEST_CASE("preconditioned conjugate gradient algorithm", "[pcg]") { +TEST_CASE("preconditioned conjugate gradient algorithm", "[mgard][pcg]") { SECTION("diagonal system") { const std::size_t Ns[4] = {1, 11, 111, 1111}; for (std::size_t N : Ns) { diff --git a/tests/src/unstructured/testing_utilities.cpp b/tests/mgard/src/unstructured/testing_utilities.cpp similarity index 100% rename from tests/src/unstructured/testing_utilities.cpp rename to tests/mgard/src/unstructured/testing_utilities.cpp diff --git a/tests/src/unstructured/test_UniformEdgeFamilies.cpp b/tests/src/unstructured/test_UniformEdgeFamilies.cpp deleted file mode 100644 index e1ef60fe9e..0000000000 --- a/tests/src/unstructured/test_UniformEdgeFamilies.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "catch2/catch_test_macros.hpp" - -#include "unstructured/UniformEdgeFamilies.hpp" - -// Empty.