Rewrite Huffman Coding Implementation

CMakeLists.txt

@@ -11,11 +11,11 @@ endif()
 list(INSERT CMAKE_MODULE_PATH 0 "${CMAKE_CURRENT_LIST_DIR}/cmake")
 
 set(MGARD_VERSION_MAJOR "1")
-set(MGARD_VERSION_MINOR "2")
+set(MGARD_VERSION_MINOR "3")
 set(MGARD_VERSION_PATCH "0")
 
 set(MGARD_FILE_VERSION_MAJOR "1")
-set(MGARD_FILE_VERSION_MINOR "0")
+set(MGARD_FILE_VERSION_MINOR "1")
 set(MGARD_FILE_VERSION_PATCH "0")
 
 project(
@@ -201,9 +201,15 @@ set(
   MGARD_LIBRARY_CPP
         src/compress.cpp
         src/compress_internal.cpp
-  src/compressors.cpp
+  src/utilities.cpp
+  src/huffman.cpp
+  src/lossless_zlib.cpp
+  src/lossless_dispatcher.cpp
   src/format.cpp
 )
+if(zstd_FOUND)
+	list(APPEND MGARD_LIBRARY_CPP src/lossless_zstd.cpp)
+endif()
 
 set(MAXIMUM_DIMENSION 4 CACHE STRING "Maximum supported dimension for self-describing decompression.")
 

include/compress.tpp

@@ -20,16 +20,14 @@
 #include "MGARDConfig.hpp"
 #include "TensorMultilevelCoefficientQuantizer.hpp"
 #include "TensorNorms.hpp"
-#include "compressors.hpp"
 #include "decompose.hpp"
 #include "format.hpp"
+#include "lossless.hpp"
 #include "quantize.hpp"
 #include "shuffle.hpp"
 
 namespace mgard {
 
-using DEFAULT_INT_T = std::int64_t;
-
 template <std::size_t N, typename Real>
 CompressedDataset<N, Real>
 compress(const TensorMeshHierarchy<N, Real> &hierarchy, Real *const v,

include/compress_internal.tpp

@@ -1,8 +1,8 @@
 #include <cstdlib>
 
 #include "compress.hpp"
-#include "compressors.hpp"
 #include "decompose.hpp"
+#include "lossless.hpp"
 #include "quantize.hpp"
 #include "shuffle.hpp"
 

include/format.hpp

@@ -66,6 +66,14 @@ serialize_header_crc32(std::uint_least64_t crc32);
 //!\param p Pointer whose alignment will be checked.
 template <typename T> void check_alignment(void const *const p);
 
+//! Check that a quantization buffer has the right alignment and a valid size.
+//!
+//!\param header Self-describing dataset header.
+//!\param p Quantization buffer.
+//!\param n Size in bytes of quantization buffer.
+void check_quantization_buffer(const pb::Header &header, void const *const p,
+                               const std::size_t n);
+
 //! Determine whether an integral type is big endian.
 template <typename Int> bool big_endian();
 
@@ -74,6 +82,11 @@ template <typename Int> bool big_endian();
 //!\return `Dataset::Type` corresponding to `Real`.
 template <typename Real> pb::Dataset::Type type_to_dataset_type();
 
+//! Return the `Quantization::Type` value corrresponding to an integral type.
+//!
+//!\return `Quantization::Type` corresponding to `Int`.
+template <typename Int> pb::Quantization::Type type_to_quantization_type();
+
 //! Allocate a quantization buffer of the proper alignment and size.
 //!
 //!\param header Self-describing dataset header.
@@ -165,16 +178,19 @@ pb::Header read_metadata(BufferWindow &window);
 //!\param header Header of the self-describing buffer.
 void write_metadata(std::ostream &ostream, const pb::Header &header);
 
-//! Parse the header of a self-describing buffer.
+template <typename T>
+//! Parse a message from a buffer window.
 //!
 //! The buffer pointer will be advanced past the header.
 //!
-//!\param window Window into the self-describing buffer. The current position
-//! should be the start of the header.
-//!\param header_size Size in bytes of the header.
-//!\return Header of the self-describing buffer.
-pb::Header read_header(BufferWindow &window,
-                       const std::uint_least64_t header_size);
+//! This function was originally written to parse the header from a
+//! self-describing buffer.
+//
+//!\param window Buffer window containing the serialized message. The current
+//! position should be the start of the message.
+//!\param nmessage Size in bytes of the message.
+//!\return Parsed message.
+T read_message(BufferWindow &window, const std::uint_least64_t nmessage);
 
 //! Check that a dataset was compressed with a compatible version of MGARD.
 //!

include/format.tpp

@@ -61,4 +61,26 @@ template <typename Int> bool big_endian() {
   return not*reinterpret_cast<unsigned char const *>(&n);
 }
 
+template <typename T>
+T read_message(BufferWindow &window, const std::uint_least64_t nmessage) {
+  // The `CodedInputStream` constructor takes an `int`.
+  if (nmessage > std::numeric_limits<int>::max()) {
+    throw std::runtime_error("message is too large (size would overflow)");
+  }
+  // Check that the read will stay in the buffer.
+  unsigned char const *const next = window.next(nmessage);
+  T message;
+  google::protobuf::io::CodedInputStream stream(
+      static_cast<google::protobuf::uint8 const *>(window.current), nmessage);
+  if (not message.ParseFromCodedStream(&stream)) {
+    throw std::runtime_error(
+        "message parsing encountered read or format error");
+  }
+  if (not stream.ConsumedEntireMessage()) {
+    throw std::runtime_error("part of message left unparsed");
+  }
+  window.current = next;
+  return message;
+}
+
 } // namespace mgard