From 76c89d1fb62b409a61b5246c9bc641caedf29e98 Mon Sep 17 00:00:00 2001 From: Gary Liu Date: Thu, 21 Nov 2019 15:55:24 -0500 Subject: [PATCH 1/5] updated travis. --- scripts/travis/run-gs.sh | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/scripts/travis/run-gs.sh b/scripts/travis/run-gs.sh index ee769776eb..4b3a9c77ae 100755 --- a/scripts/travis/run-gs.sh +++ b/scripts/travis/run-gs.sh @@ -15,4 +15,23 @@ cd ${SOURCE_DIR} ./build/bin/cosine -exit $EXITCODE +if [ $? -eq 0 ] +then + echo "Successfully compress/decompress consine" + exit 0 +else + echo "Error in compressing/decompressing consine" >&2 + exit 1 +fi + +./build/bin/constant3d + +if [ $? -eq 0 ] +then + echo "Successfully compress/decompress constant3d" + exit 0 +else + echo "Error in compressing/decompressing constant3d" >&2 + exit 1 +fi + From 1daf46dd71f2a8a275976e26bef660213babc60a Mon Sep 17 00:00:00 2001 From: Gary Liu Date: Thu, 21 Nov 2019 17:07:24 -0500 Subject: [PATCH 2/5] updated travis. --- scripts/travis/run-gs.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/travis/run-gs.sh b/scripts/travis/run-gs.sh index 4b3a9c77ae..4d975282b5 100755 --- a/scripts/travis/run-gs.sh +++ b/scripts/travis/run-gs.sh @@ -18,7 +18,6 @@ cd ${SOURCE_DIR} if [ $? -eq 0 ] then echo "Successfully compress/decompress consine" - exit 0 else echo "Error in compressing/decompressing consine" >&2 exit 1 @@ -29,9 +28,9 @@ fi if [ $? -eq 0 ] then echo "Successfully compress/decompress constant3d" - exit 0 else echo "Error in compressing/decompressing constant3d" >&2 exit 1 fi +exit 0 From 7c32998e7ea514adacd23248867e5794af82ce04 Mon Sep 17 00:00:00 2001 From: Gary Liu Date: Wed, 26 Aug 2020 22:46:22 -0400 Subject: [PATCH 3/5] initial support for zstd using the fast mode (level = 1). --- CMakeLists.txt | 4 ++ cmake/FindZSTD.cmake | 62 ++++++++++++++++++++++++++++++ include/mgard_compress.hpp | 10 ++++- src/mgard_compress.cpp | 77 +++++++++++++++++++++++++++++++++++--- 4 files changed, 147 insertions(+), 6 deletions(-) create mode 100644 cmake/FindZSTD.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index c5fa34735f..3cff0ac7fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -147,6 +147,7 @@ endif() # Dependencies find_package(ZLIB REQUIRED) find_package(ADIOS2) +find_package(ZSTD) # Set library add_library(mgard @@ -164,6 +165,9 @@ target_include_directories(mgard $ ) target_link_libraries(mgard PRIVATE ZLIB::ZLIB ${CMAKE_DL_LIBS}) +if(ZSTD_FOUND) + target_link_libraries(mgard PRIVATE ZSTD::ZSTD ${CMAKE_DL_LIBS}) +endif() # Make sure we require C++11. Use meta-compile features if available, diff --git a/cmake/FindZSTD.cmake b/cmake/FindZSTD.cmake new file mode 100644 index 0000000000..16b9e95d08 --- /dev/null +++ b/cmake/FindZSTD.cmake @@ -0,0 +1,62 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# +# +# FindZSTD +# ----------- +# +# Try to find the ZSTD library +# +# This module defines the following variables: +# +# ZSTD_FOUND - System has ZSTD +# ZSTD_INCLUDE_DIRS - The ZSTD include directory +# ZSTD_LIBRARIES - Link these to use ZSTD +# +# and the following imported targets: +# ZSTD::ZSTD - The ZSTD compression library target +# +# You can also set the following variable to help guide the search: +# ZSTD_ROOT - The install prefix for ZSTD containing the +# include and lib folders +# Note: this can be set as a CMake variable or an +# environment variable. If specified as a CMake +# variable, it will override any setting specified +# as an environment variable. + +if(NOT ZSTD_FOUND) + if((NOT ZSTD_ROOT) AND (NOT (ENV{ZSTD_ROOT} STREQUAL ""))) + set(ZSTD_ROOT "$ENV{ZSTD_ROOT}") + endif() + if(ZSTD_ROOT) + set(ZSTD_INCLUDE_OPTS HINTS ${ZSTD_ROOT}/include NO_DEFAULT_PATHS) + set(ZSTD_LIBRARY_OPTS + HINTS ${ZSTD_ROOT}/lib ${ZSTD_ROOT}/lib64 + NO_DEFAULT_PATHS + ) + endif() + + find_path(ZSTD_INCLUDE_DIR zstd.h ${ZSTD_INCLUDE_OPTS}) + find_library(ZSTD_LIBRARY NAMES zstd ${ZSTD_LIBRARY_OPTS}) + + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args(ZSTD + FOUND_VAR ZSTD_FOUND + REQUIRED_VARS ZSTD_LIBRARY ZSTD_INCLUDE_DIR + ) + if(ZSTD_FOUND) + set(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR}) + set(ZSTD_LIBRARIES ${ZSTD_LIBRARY}) + add_definitions(-DMGARD_ZSTD) + include_directories(${ZSTD_INCLUDE_DIR}) + if(ZSTD_FOUND AND NOT TARGET ZSTD::ZSTD) + add_library(ZSTD::ZSTD UNKNOWN IMPORTED) + set_target_properties(ZSTD::ZSTD PROPERTIES + IMPORTED_LOCATION "${ZSTD_LIBRARY}" + INTERFACE_LINK_LIBRARIES "${ZSTD_LIBRARIES}" + INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_INCLUDE_DIR}" + ) + endif() + endif() +endif() diff --git a/include/mgard_compress.hpp b/include/mgard_compress.hpp index cfcc50f596..9665a14b14 100644 --- a/include/mgard_compress.hpp +++ b/include/mgard_compress.hpp @@ -18,7 +18,11 @@ void huffman_decoding(int *const in_data, const std::size_t in_data_size, unsigned char *out_data_hit, size_t out_data_hit_size, unsigned char *out_data_miss, size_t out_data_miss_size, unsigned char *out_tree, size_t out_tree_size); - +#ifdef MGARD_ZSTD +//! Compress an array of data using `zstd`. +void compress_memory_zstd(void *const in_data, const std::size_t in_data_size, + std::vector &out_data); +#endif //! Compress an array of data using `zlib`. //! //!\param in_data Pointer to data to be compressed. @@ -39,4 +43,8 @@ void decompress_memory_z(void *const src, const int srcLen, int *const dst, void decompress_memory_z_huffman(void *const src, const int srcLen, unsigned char *const dst, const int dstLen); +#ifdef MGARD_ZSTD +void decompress_memory_zstd_huffman(void *const src, const int srcLen, + unsigned char *const dst, const int dstLen); +#endif } // namespace mgard diff --git a/src/mgard_compress.cpp b/src/mgard_compress.cpp index 2b8c322272..7d905ea7a1 100644 --- a/src/mgard_compress.cpp +++ b/src/mgard_compress.cpp @@ -11,6 +11,10 @@ #include +#ifdef MGARD_ZSTD +#include +#endif + namespace mgard { const int nql = 32768 * 4; @@ -179,10 +183,13 @@ void decompress_memory_huffman(unsigned char *data, int data_len, out_tree_size + out_data_hit_size / 8 + 4 + out_data_miss_size; unsigned char *huffman_encoding_p = (unsigned char *)malloc(total_huffman_size); - +#ifndef MGARD_ZSTD mgard::decompress_memory_z_huffman(buf, data_len - 3 * sizeof(size_t), huffman_encoding_p, total_huffman_size); - +#else + mgard::decompress_memory_zstd_huffman(buf, data_len - 3 * sizeof(size_t), + huffman_encoding_p, total_huffman_size); +#endif out_tree = huffman_encoding_p; out_data_hit = huffman_encoding_p + out_tree_size; out_data_miss = @@ -294,9 +301,11 @@ unsigned char *compress_memory_huffman(std::vector &qv, free(out_tree); free(out_data_hit); free(out_data_miss); - +#ifndef MGARD_ZSTD mgard::compress_memory_z(payload, total_size, out_data); - +#else + mgard::compress_memory_zstd(payload, total_size, out_data); +#endif outsize = out_data.size() + 3 * sizeof(size_t); unsigned char *buffer = (unsigned char *)malloc(outsize); @@ -387,7 +396,8 @@ void huffman_encoding(int *const quantized_data, const std::size_t n, } } - std::cout << "num hit: " << n - num_miss << " num_miss: " << num_miss << "\n"; + // std::cout << "num hit: " << n - num_miss << " num_miss: " << num_miss << + // "\n"; // Note: hit size is in bits, while miss size is in bytes. *out_data_hit_size = start_bit; *out_data_miss_size = num_miss * sizeof(int); @@ -416,6 +426,51 @@ void huffman_encoding(int *const quantized_data, const std::size_t n, ft = 0; } +#ifdef MGARD_ZSTD +/*! CHECK + * Check that the condition holds. If it doesn't print a message and die. + */ +#define CHECK(cond, ...) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "%s:%d CHECK(%s) failed: ", __FILE__, __LINE__, #cond); \ + fprintf(stderr, "" __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + exit(1); \ + } \ + } while (0) + +/*! CHECK_ZSTD + * Check the zstd error code and die if an error occurred after printing a + * message. + */ +/*! CHECK_ZSTD + * Check the zstd error code and die if an error occurred after printing a + * message. + */ +#define CHECK_ZSTD(fn, ...) \ + do { \ + size_t const err = (fn); \ + CHECK(!ZSTD_isError(err), "%s", ZSTD_getErrorName(err)); \ + } while (0) + +void compress_memory_zstd(void *const in_data, const std::size_t in_data_size, + std::vector &out_data) { + size_t const cBuffSize = ZSTD_compressBound(in_data_size); + uint8_t *cBuff = (uint8_t *)malloc(cBuffSize); + + assert(cBuff); + + size_t const cSize = + ZSTD_compress(cBuff, cBuffSize, in_data, in_data_size, 1); + CHECK_ZSTD(cSize); + + std::copy(cBuff, cBuff + cSize, back_inserter(out_data)); + + free(cBuff); +} +#endif + void compress_memory_z(void *const in_data, const std::size_t in_data_size, std::vector &out_data) { std::vector buffer; @@ -484,6 +539,18 @@ void decompress_memory_z(void *const src, const int srcLen, int *const dst, assert(res == Z_OK); } +#ifdef MGARD_ZSTD +void decompress_memory_zstd_huffman(void *const src, const int srcLen, + unsigned char *const dst, + const int dstLen) { + size_t const dSize = ZSTD_decompress(dst, dstLen, src, srcLen); + CHECK_ZSTD(dSize); + + /* When zstd knows the content size, it will error if it doesn't match. */ + CHECK(dSize == dstLen, "Impossible because zstd will check this condition!"); +} +#endif + void decompress_memory_z_huffman(void *const src, const int srcLen, unsigned char *const dst, const int dstLen) { z_stream strm = {0}; From 86aa62566f47657f980ad5bb1d553120cb6dab3c Mon Sep 17 00:00:00 2001 From: Qing Liu Date: Tue, 1 Sep 2020 18:50:57 -0400 Subject: [PATCH 4/5] fixed a linking issue on Rhea for template. --- include/mgard_mesh.hpp | 6 ------ src/mgard_mesh.cpp | 6 ++++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/mgard_mesh.hpp b/include/mgard_mesh.hpp index e52f4577da..bb663102d2 100644 --- a/include/mgard_mesh.hpp +++ b/include/mgard_mesh.hpp @@ -31,12 +31,6 @@ template struct Dimensions2kPlus1 { int nlevel; }; -template struct Dimensions2kPlus1<1>; - -template struct Dimensions2kPlus1<2>; - -template struct Dimensions2kPlus1<3>; - // As of this writing, these are only needed in the implementations of the // `Dimensions2kPlus1` constructor and `is_2kplus1`. int nlevel_from_size(const int n); diff --git a/src/mgard_mesh.cpp b/src/mgard_mesh.cpp index b6f200a2de..53708d409b 100644 --- a/src/mgard_mesh.cpp +++ b/src/mgard_mesh.cpp @@ -40,4 +40,10 @@ int get_lindex(const int n, const int no, const int i) { : no - 1); } +template struct Dimensions2kPlus1<1>; + +template struct Dimensions2kPlus1<2>; + +template struct Dimensions2kPlus1<3>; + } // namespace mgard From 4364e2ea81bc9937e18c9936c7fed6d0fff9f58d Mon Sep 17 00:00:00 2001 From: Gary Liu Date: Thu, 3 Sep 2020 17:01:24 -0400 Subject: [PATCH 5/5] added some timing code (incomplete). --- CMakeLists.txt | 4 ++++ src/mgard.tpp | 24 ++++++++++++++++++++---- src/mgard_compress.cpp | 29 ++++++++++++++++++++++++++++- tests/test_1d/test_1d.cpp | 15 ++++++++++++--- 4 files changed, 64 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3cff0ac7fc..1449cc1dc9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,6 +169,10 @@ if(ZSTD_FOUND) target_link_libraries(mgard PRIVATE ZSTD::ZSTD ${CMAKE_DL_LIBS}) endif() +option(DEFINE_MGARD_TIMING "Enable/disable MGARD timing" OFF) +if(DEFINE_MGARD_TIMING) + add_definitions(-DMGARD_TIMING) +endif() # Make sure we require C++11. Use meta-compile features if available, # otherwise use specific language features diff --git a/src/mgard.tpp b/src/mgard.tpp index 87a5270588..002e155cc6 100644 --- a/src/mgard.tpp +++ b/src/mgard.tpp @@ -21,6 +21,10 @@ #include #include +#ifdef MGARD_TIMING +#include +#endif + #include "mgard_compress.hpp" #include "mgard_mesh.hpp" #include "mgard_nuni.h" @@ -498,7 +502,9 @@ unsigned char *refactor_qz_1D(int ncol, const Real *u, int &outsize, Real tol) { int nlevel; set_number_of_levels(1, ncol, nlevel); tol /= nlevel + 1; - +#ifdef MGARD_TIMING + auto start = std::chrono::high_resolution_clock::now(); +#endif const int l_target = nlevel - 1; mgard::refactor_1D(ncol, l_target, v.data(), work, row_vec); @@ -509,7 +515,11 @@ unsigned char *refactor_qz_1D(int ncol, const Real *u, int &outsize, Real tol) { std::vector qv(ncol + size_ratio); mgard::quantize_2D_interleave(1, ncol, v.data(), qv, norm, tol); - +#ifdef MGARD_TIMING + auto stop = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(stop - start); + std::cout << "Refactor Time = " << (double)duration.count()/1000000 << "\n"; +#endif std::vector out_data; return mgard::compress_memory_huffman(qv, out_data, outsize); @@ -522,7 +532,9 @@ unsigned char *refactor_qz_1D(int ncol, const Real *u, int &outsize, Real tol) { tol /= dims.nlevel + 1; const int l_target = dims.nlevel - 1; - +#ifdef MGARD_TIMING + auto start = std::chrono::high_resolution_clock::now(); +#endif mgard_2d::mgard_gen::prep_1D(dims.rnded[0], dims.input[0], l_target, v.data(), work, coords_x, row_vec); @@ -536,7 +548,11 @@ unsigned char *refactor_qz_1D(int ncol, const Real *u, int &outsize, Real tol) { std::vector qv(ncol + size_ratio); mgard::quantize_2D_interleave(1, ncol, v.data(), qv, norm, tol); - +#ifdef MGARD_TIMING + auto stop = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(stop - start); + std::cout << "Refactor Time = " << duration.count() << "\n"; +#endif std::vector out_data; return mgard::compress_memory_huffman(qv, out_data, outsize); diff --git a/src/mgard_compress.cpp b/src/mgard_compress.cpp index 7d905ea7a1..84581a855a 100644 --- a/src/mgard_compress.cpp +++ b/src/mgard_compress.cpp @@ -9,6 +9,10 @@ #include #include +#ifdef MGARD_TIMING +#include +#endif + #include #ifdef MGARD_ZSTD @@ -280,10 +284,17 @@ unsigned char *compress_memory_huffman(std::vector &qv, size_t out_data_miss_size; unsigned char *out_tree = 0; size_t out_tree_size; +#ifdef MGARD_TIMING + auto huff_time1 = std::chrono::high_resolution_clock::now(); +#endif mgard::huffman_encoding(qv.data(), qv.size(), &out_data_hit, &out_data_hit_size, &out_data_miss, &out_data_miss_size, &out_tree, &out_tree_size); - +#ifdef MGARD_TIMING + auto huff_time2 = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(huff_time2 - huff_time1); + std::cout << "Huffman tree time = " << (double)duration.count()/1000000 << "\n"; +#endif size_t total_size = out_data_hit_size / 8 + 4 + out_data_miss_size + out_tree_size; unsigned char *payload = (unsigned char *)malloc(total_size); @@ -302,9 +313,25 @@ unsigned char *compress_memory_huffman(std::vector &qv, free(out_data_hit); free(out_data_miss); #ifndef MGARD_ZSTD +#ifdef MGARD_TIMING + auto z_time1 = std::chrono::high_resolution_clock::now(); +#endif mgard::compress_memory_z(payload, total_size, out_data); +#ifdef MGARD_TIMING + auto z_time2 = std::chrono::high_resolution_clock::now(); + auto z_duration = std::chrono::duration_cast(z_time2 - z_time1); + std::cout << "ZLIB compression time = " << (double)z_duration.count()/1000000 << "\n"; +#endif #else +#ifdef MGARD_TIMING + auto zstd_time1 = std::chrono::high_resolution_clock::now(); +#endif mgard::compress_memory_zstd(payload, total_size, out_data); +#ifdef MGARD_TIMING + auto zstd_time2 = std::chrono::high_resolution_clock::now(); + auto zstd_duration = std::chrono::duration_cast(zstd_time2 - zstd_time1); + std::cout << "ZSTD compression time = " << (double)zstd_duration.count()/1000000 << "\n"; +#endif #endif outsize = out_data.size() + 3 * sizeof(size_t); unsigned char *buffer = (unsigned char *)malloc(outsize); diff --git a/tests/test_1d/test_1d.cpp b/tests/test_1d/test_1d.cpp index 0a0098c0f2..f74ee025ec 100644 --- a/tests/test_1d/test_1d.cpp +++ b/tests/test_1d/test_1d.cpp @@ -3,6 +3,9 @@ #include #include #include +#ifdef MGARD_TIMING +#include +#endif using namespace std; @@ -29,12 +32,18 @@ int main(int argc, char **argv) { std::vector data(num_elements); datafile.read(reinterpret_cast(&data[0]), num_elements * sizeof(double)); - +#ifdef MGARD_TIMING + auto start = chrono::high_resolution_clock::now(); +#endif compressed_data = mgard_compress(0, data.data(), out_size, 1, num_elements, 1, tol); +#ifdef MGARD_TIMING + auto stop = chrono::high_resolution_clock::now(); + auto duration = chrono::duration_cast(stop - start); cout << "Original size = " << num_elements * 8 << " out_size = " << out_size - << " CR = " << num_elements * 8.0 / out_size << endl; - + << " CR = " << num_elements * 8.0 / out_size + << " Time = " << (double) duration.count() / 1000000<< endl; +#endif double quantizer; double *decompressed_data = mgard_decompress(0, quantizer, compressed_data, out_size, 1, num_elements, 1);