From 2509c56c3b4d8ed93eb25af6b5afcbb11b6ff9c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20H=C3=B6gqvist?= Date: Mon, 16 Sep 2024 08:47:22 +0200 Subject: [PATCH 1/3] LZ4 dependency fix Serialization split into header and source file to avoid external dependency to LZ4 library. Also fixed linkage to LZ4 by changing variables used in CMakeLists.txt. --- cmake/flann.pc.in | 2 +- src/cpp/CMakeLists.txt | 16 +- src/cpp/flann/util/serialization.cpp | 276 +++++++ src/cpp/flann/util/serialization.h | 1019 ++++++++------------------ 4 files changed, 614 insertions(+), 699 deletions(-) create mode 100644 src/cpp/flann/util/serialization.cpp diff --git a/cmake/flann.pc.in b/cmake/flann.pc.in index 07b8d649..82eee5e6 100644 --- a/cmake/flann.pc.in +++ b/cmake/flann.pc.in @@ -8,6 +8,6 @@ Name: @PROJECT_NAME@ Description: @PKG_DESC@ Version: @FLANN_VERSION@ Requires: @PKG_EXTERNAL_DEPS@ -Libs: -L${libdir} @LZ4_STATIC_LDFLAGS@ -lflann -lflann_cpp +Libs: -L${libdir} -lflann -lflann_cpp Cflags: -I${includedir} diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt index 7603abc8..726d8dab 100644 --- a/src/cpp/CMakeLists.txt +++ b/src/cpp/CMakeLists.txt @@ -5,11 +5,13 @@ add_definitions(-D_FLANN_VERSION=${FLANN_VERSION}) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/flann/config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/flann/config.h) file(GLOB_RECURSE C_SOURCES flann.cpp) -file(GLOB_RECURSE CPP_SOURCES flann_cpp.cpp) +file(GLOB_RECURSE CPP_SOURCES flann_cpp.cpp util/serialization.cpp) file(GLOB_RECURSE CU_SOURCES *.cu) add_library(flann_cpp_s STATIC ${CPP_SOURCES}) -target_link_libraries(flann_cpp_s PUBLIC ${LZ4_LINK_LIBRARIES}) +target_link_directories(flann_cpp_s PRIVATE ${LZ4_LIBRARY_DIRS}) +target_link_libraries(flann_cpp_s PRIVATE ${LZ4_LIBRARIES}) +set_target_properties(flann_cpp_s PROPERTIES OUTPUT_NAME flann_cpp) if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG) set_target_properties(flann_cpp_s PROPERTIES COMPILE_FLAGS -fPIC) endif() @@ -20,7 +22,8 @@ else() endif() add_library(flann_cpp SHARED ${CPP_SOURCES}) -target_link_libraries(flann_cpp ${LZ4_LINK_LIBRARIES}) +target_link_directories(flann_cpp PRIVATE ${LZ4_LIBRARY_DIRS}) +target_link_libraries(flann_cpp PRIVATE ${LZ4_LIBRARIES}) # export lz4 headers, so that MSVC to creates flann_cpp.lib set_target_properties(flann_cpp PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS YES) @@ -81,7 +84,9 @@ endif() if (BUILD_C_BINDINGS) add_library(flann_s STATIC ${C_SOURCES}) - target_link_libraries(flann_s ${LZ4_LINK_LIBRARIES}) + target_link_directories(flann_s PRIVATE ${LZ4_LIBRARY_DIRS}) + target_link_libraries(flann_s ${LZ4_LIBRARIES}) + set_target_properties(flann_s PROPERTIES OUTPUT_NAME flann) if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG) set_target_properties(flann_s PROPERTIES COMPILE_FLAGS -fPIC) endif() @@ -93,7 +98,8 @@ if (BUILD_C_BINDINGS) endif() add_library(flann SHARED ${C_SOURCES}) - target_link_libraries(flann ${LZ4_LINK_LIBRARIES}) + target_link_directories(flann PRIVATE ${LZ4_LIBRARY_DIRS}) + target_link_libraries(flann ${LZ4_LIBRARIES}) list(APPEND flann_install_targets flann) if(MINGW AND OPENMP_FOUND) diff --git a/src/cpp/flann/util/serialization.cpp b/src/cpp/flann/util/serialization.cpp new file mode 100644 index 00000000..c0c817f0 --- /dev/null +++ b/src/cpp/flann/util/serialization.cpp @@ -0,0 +1,276 @@ +#include "serialization.h" + +#include +#include + +namespace { +LZ4_streamHC_t lz4Stream_body; +LZ4_streamHC_t *lz4Stream; + +LZ4_streamDecode_t lz4StreamDecode_body; +LZ4_streamDecode_t *lz4StreamDecode; + +} // namespace + +namespace flann { +namespace serialization { +void SaveArchive::initBlock() { + // Alloc the space for both buffer blocks (each compressed block + // references the previous) + buffer_ = buffer_blocks_ = (char *)malloc(BLOCK_BYTES * 2); + compressed_buffer_ = + (char *)malloc(LZ4_COMPRESSBOUND(BLOCK_BYTES) + sizeof(size_t)); + if (buffer_ == NULL || compressed_buffer_ == NULL) { + throw FLANNException("Error allocating compression buffer"); + } + + // Init the LZ4 stream + lz4Stream = &lz4Stream_body; + LZ4_resetStreamHC(lz4Stream, 9); + first_block_ = true; + + offset_ = 0; +} + +void SaveArchive::flushBlock() { + size_t compSz = 0; + // Handle header + if (first_block_) { + // Copy & set the header + IndexHeaderStruct *head = (IndexHeaderStruct *)buffer_; + size_t headSz = sizeof(IndexHeaderStruct); + + assert(head->compression == 0); + head->compression = 1; // Bool now, enum later + + // Do the compression for the block + compSz = LZ4_compress_HC_continue( + lz4Stream, buffer_ + headSz, compressed_buffer_ + headSz, + offset_ - headSz, LZ4_COMPRESSBOUND(BLOCK_BYTES)); + + if (compSz <= 0) { + throw FLANNException("Error compressing (first block)"); + } + + // Handle header + head->first_block_size = compSz; + memcpy(compressed_buffer_, buffer_, headSz); + + compSz += headSz; + first_block_ = false; + } else { + size_t headSz = sizeof(compSz); + + // Do the compression for the block + compSz = LZ4_compress_HC_continue(lz4Stream, buffer_, + compressed_buffer_ + headSz, offset_, + LZ4_COMPRESSBOUND(BLOCK_BYTES)); + + if (compSz <= 0) { + throw FLANNException("Error compressing"); + } + + // Save the size of the compressed block as the header + memcpy(compressed_buffer_, &compSz, headSz); + compSz += headSz; + } + + // Write the compressed buffer + fwrite(compressed_buffer_, compSz, 1, stream_); + + // Switch the buffer to the *other* block + if (buffer_ == buffer_blocks_) + buffer_ = &buffer_blocks_[BLOCK_BYTES]; + else + buffer_ = buffer_blocks_; + offset_ = 0; +} + +void SaveArchive::endBlock() { + // Cleanup memory + free(buffer_blocks_); + buffer_blocks_ = NULL; + buffer_ = NULL; + free(compressed_buffer_); + compressed_buffer_ = NULL; + + // Write a '0' size for next block + size_t z = 0; + fwrite(&z, sizeof(z), 1, stream_); +} + +void LoadArchive::decompressAndLoadV10(FILE *stream) { + buffer_ = NULL; + + // Find file size + size_t pos = ftell(stream); + fseek(stream, 0, SEEK_END); + size_t fileSize = ftell(stream) - pos; + fseek(stream, pos, SEEK_SET); + size_t headSz = sizeof(IndexHeaderStruct); + + // Read the (compressed) file to a buffer + char *compBuffer = (char *)malloc(fileSize); + if (compBuffer == NULL) { + throw FLANNException("Error allocating file buffer space"); + } + if (fread(compBuffer, fileSize, 1, stream) != 1) { + free(compBuffer); + throw FLANNException( + "Invalid index file, cannot read from disk (compressed)"); + } + + // Extract header + IndexHeaderStruct *head = (IndexHeaderStruct *)(compBuffer); + + // Backward compatability + size_t compressedSz = fileSize - headSz; + size_t uncompressedSz = head->first_block_size - headSz; + + // Check for compression type + if (head->compression != 1) { + free(compBuffer); + throw FLANNException("Compression type not supported"); + } + + // Allocate a decompressed buffer + ptr_ = buffer_ = (char *)malloc(uncompressedSz + headSz); + if (buffer_ == NULL) { + free(compBuffer); + throw FLANNException("Error (re)allocating decompression buffer"); + } + + // Extract body + size_t usedSz = LZ4_decompress_safe(compBuffer + headSz, buffer_ + headSz, + compressedSz, uncompressedSz); + + // Check if the decompression was the expected size. + if (usedSz != uncompressedSz) { + free(compBuffer); + throw FLANNException("Unexpected decompression size"); + } + + // Copy header data + memcpy(buffer_, compBuffer, headSz); + free(compBuffer); + + // Put the file pointer at the end of the data we've read + if (compressedSz + headSz + pos != fileSize) + fseek(stream, compressedSz + headSz + pos, SEEK_SET); + block_sz_ = uncompressedSz + headSz; +} + +void LoadArchive::initBlock(FILE *stream) { + size_t pos = ftell(stream); + buffer_ = NULL; + buffer_blocks_ = NULL; + compressed_buffer_ = NULL; + size_t headSz = sizeof(IndexHeaderStruct); + + // Read the file header to a buffer + IndexHeaderStruct *head = (IndexHeaderStruct *)malloc(headSz); + if (head == NULL) { + throw FLANNException("Error allocating header buffer space"); + } + if (fread(head, headSz, 1, stream) != 1) { + free(head); + throw FLANNException("Invalid index file, cannot read from disk (header)"); + } + + // Backward compatability + if (head->signature[13] == '1' && head->signature[15] == '0') { + free(head); + fseek(stream, pos, SEEK_SET); + return decompressAndLoadV10(stream); + } + + // Alloc the space for both buffer blocks (each block + // references the previous) + buffer_ = buffer_blocks_ = (char *)malloc(BLOCK_BYTES * 2); + compressed_buffer_ = (char *)malloc(LZ4_COMPRESSBOUND(BLOCK_BYTES)); + if (buffer_ == NULL || compressed_buffer_ == NULL) { + free(head); + throw FLANNException("Error allocating compression buffer"); + } + + // Init the LZ4 stream + lz4StreamDecode = &lz4StreamDecode_body; + LZ4_setStreamDecode(lz4StreamDecode, NULL, 0); + + // Read first block + memcpy(buffer_, head, headSz); + loadBlock(buffer_ + headSz, head->first_block_size, stream); + block_sz_ += headSz; + ptr_ = buffer_; + free(head); +} + +void loadBlock(char *buffer_, size_t compSz, FILE *stream) { + if (compSz >= LZ4_COMPRESSBOUND(BLOCK_BYTES)) { + throw FLANNException("Requested block size too large"); + } + + // Read the block into the compressed buffer + if (fread(compressed_buffer_, compSz, 1, stream) != 1) { + throw FLANNException("Invalid index file, cannot read from disk (block)"); + } + + // Decompress into the regular buffer + const int decBytes = LZ4_decompress_safe_continue( + lz4StreamDecode, compressed_buffer_, buffer_, compSz, BLOCK_BYTES); + if (decBytes <= 0) { + throw FLANNException("Invalid index file, cannot decompress block"); + } + block_sz_ = decBytes; +} + +void LoadArchive::preparePtr(size_t size) { + // Return if the new size is less than (or eq) the size of a block + if (ptr_ + size <= buffer_ + block_sz_) + return; + + // Switch the buffer to the *other* block + if (buffer_ == buffer_blocks_) + buffer_ = &buffer_blocks_[BLOCK_BYTES]; + else + buffer_ = buffer_blocks_; + + // Find the size of the next block + size_t cmpSz = 0; + size_t readCnt = fread(&cmpSz, sizeof(cmpSz), 1, stream_); + if (cmpSz <= 0 || readCnt != 1) { + throw FLANNException("Requested to read next block past end of file"); + } + + // Load block & init ptr + loadBlock(buffer_, cmpSz, stream_); + ptr_ = buffer_; +} + +void LoadArchive::endBlock() { + // If not v1.0 format hack... + if (buffer_blocks_ != NULL) { + // Read the last '0' in the file + size_t zero = 1; + if (fread(&zero, sizeof(zero), 1, stream_) != 1) { + throw FLANNException("Invalid index file, cannot read from disk (end)"); + } + if (zero != 0) { + throw FLANNException("Invalid index file, last block not zero length"); + } + } + + // Free resources + if (buffer_blocks_ != NULL) { + free(buffer_blocks_); + buffer_blocks_ = NULL; + } + if (compressed_buffer_ != NULL) { + free(compressed_buffer_); + compressed_buffer_ = NULL; + } + ptr_ = NULL; +} + +} // namespace serialization +} // namespace flann \ No newline at end of file diff --git a/src/cpp/flann/util/serialization.h b/src/cpp/flann/util/serialization.h index ac4484b6..2dd41a3a 100644 --- a/src/cpp/flann/util/serialization.h +++ b/src/cpp/flann/util/serialization.h @@ -1,97 +1,75 @@ #ifndef SERIALIZATION_H_ #define SERIALIZATION_H_ -#include -#include #include #include +#include #include -#include -#include - - -namespace flann -{ - struct IndexHeaderStruct { - char signature[24]; - char version[16]; - flann_datatype_t data_type; - flann_algorithm_t index_type; - size_t rows; - size_t cols; - size_t compression; - size_t first_block_size; - }; - -namespace serialization -{ - -struct access -{ - template - static inline void serialize(Archive& ar, T& type) - { - type.serialize(ar); - } -}; +#include +namespace flann { +struct IndexHeaderStruct { + char signature[24]; + char version[16]; + flann_datatype_t data_type; + flann_algorithm_t index_type; + size_t rows; + size_t cols; + size_t compression; + size_t first_block_size; +}; -template -inline void serialize(Archive& ar, T& type) -{ - access::serialize(ar,type); -} +namespace serialization { -template -struct Serializer -{ - template - static inline void load(InputArchive& ar, T& val) - { - serialization::serialize(ar,val); - } - template - static inline void save(OutputArchive& ar, const T& val) - { - serialization::serialize(ar,const_cast(val)); - } +struct access { + template + static inline void serialize(Archive &ar, T &type) { + type.serialize(ar); + } }; -#define BASIC_TYPE_SERIALIZER(type)\ -template<> \ -struct Serializer \ -{\ - template\ - static inline void load(InputArchive& ar, type& val)\ - {\ - ar.load(val);\ - }\ - template\ - static inline void save(OutputArchive& ar, const type& val)\ - {\ - ar.save(val);\ - }\ +template +inline void serialize(Archive &ar, T &type) { + access::serialize(ar, type); } -#define ENUM_SERIALIZER(type)\ -template<>\ -struct Serializer\ -{\ - template\ - static inline void load(InputArchive& ar, type& val)\ - {\ - int int_val;\ - ar & int_val;\ - val = (type) int_val;\ - }\ - template\ - static inline void save(OutputArchive& ar, const type& val)\ - {\ - int int_val = (int)val;\ - ar & int_val;\ - }\ -} +template struct Serializer { + template + static inline void load(InputArchive &ar, T &val) { + serialization::serialize(ar, val); + } + template + static inline void save(OutputArchive &ar, const T &val) { + serialization::serialize(ar, const_cast(val)); + } +}; +#define BASIC_TYPE_SERIALIZER(type) \ + template <> struct Serializer { \ + template \ + static inline void load(InputArchive &ar, type &val) { \ + ar.load(val); \ + } \ + template \ + static inline void save(OutputArchive &ar, const type &val) { \ + ar.save(val); \ + } \ + } + +#define ENUM_SERIALIZER(type) \ + template <> struct Serializer { \ + template \ + static inline void load(InputArchive &ar, type &val) { \ + int int_val; \ + ar & int_val; \ + val = (type)int_val; \ + } \ + template \ + static inline void save(OutputArchive &ar, const type &val) { \ + int int_val = (int)val; \ + ar & int_val; \ + } \ + } // declare serializers for simple types BASIC_TYPE_SERIALIZER(char); @@ -114,250 +92,182 @@ BASIC_TYPE_SERIALIZER(unsigned __int64); #endif #endif - // serializer for std::vector -template -struct Serializer > -{ - template - static inline void load(InputArchive& ar, std::vector& val) - { - size_t size; - ar & size; - val.resize(size); - for (size_t i=0;i - static inline void save(OutputArchive& ar, const std::vector& val) - { - ar & val.size(); - for (size_t i=0;i struct Serializer> { + template + static inline void load(InputArchive &ar, std::vector &val) { + size_t size; + ar & size; + val.resize(size); + for (size_t i = 0; i < size; ++i) { + ar &val[i]; + } + } + + template + static inline void save(OutputArchive &ar, const std::vector &val) { + ar & val.size(); + for (size_t i = 0; i < val.size(); ++i) { + ar &val[i]; + } + } }; // serializer for std::vector -template -struct Serializer > -{ - template - static inline void load(InputArchive& ar, std::map& map_val) - { - size_t size; - ar & size; - for (size_t i = 0; i < size; ++i) - { - K key; - ar & key; - V value; - ar & value; - map_val[key] = value; - } - } - - template - static inline void save(OutputArchive& ar, const std::map& map_val) - { - ar & map_val.size(); - for (typename std::map::const_iterator i=map_val.begin(); i!=map_val.end(); ++i) { - ar & i->first; - ar & i->second; - } - } +template struct Serializer> { + template + static inline void load(InputArchive &ar, std::map &map_val) { + size_t size; + ar & size; + for (size_t i = 0; i < size; ++i) { + K key; + ar & key; + V value; + ar & value; + map_val[key] = value; + } + } + + template + static inline void save(OutputArchive &ar, const std::map &map_val) { + ar & map_val.size(); + for (typename std::map::const_iterator i = map_val.begin(); + i != map_val.end(); ++i) { + ar & i->first; + ar & i->second; + } + } }; -template -struct Serializer -{ - template - static inline void load(InputArchive& ar, T*& val) - { - ar.load(val); - } +template struct Serializer { + template + static inline void load(InputArchive &ar, T *&val) { + ar.load(val); + } - template - static inline void save(OutputArchive& ar, T* const& val) - { - ar.save(val); - } + template + static inline void save(OutputArchive &ar, T *const &val) { + ar.save(val); + } }; -template -struct Serializer -{ - template - static inline void load(InputArchive& ar, T (&val)[N]) - { - ar.load(val); - } +template struct Serializer { + template + static inline void load(InputArchive &ar, T (&val)[N]) { + ar.load(val); + } - template - static inline void save(OutputArchive& ar, T const (&val)[N]) - { - ar.save(val); - } + template + static inline void save(OutputArchive &ar, T const (&val)[N]) { + ar.save(val); + } }; +struct binary_object { + void const *ptr_; + size_t size_; + binary_object(void *const ptr, size_t size) : ptr_(ptr), size_(size) {} + binary_object(const binary_object &rhs) : ptr_(rhs.ptr_), size_(rhs.size_) {} - -struct binary_object -{ - void const * ptr_; - size_t size_; - - binary_object( void * const ptr, size_t size) : - ptr_(ptr), - size_(size) - {} - binary_object(const binary_object & rhs) : - ptr_(rhs.ptr_), - size_(rhs.size_) - {} - - binary_object & operator=(const binary_object & rhs) { - ptr_ = rhs.ptr_; - size_ = rhs.size_; - return *this; - } + binary_object &operator=(const binary_object &rhs) { + ptr_ = rhs.ptr_; + size_ = rhs.size_; + return *this; + } }; -inline const binary_object make_binary_object(/* const */ void * t, size_t size){ - return binary_object(t, size); +inline const binary_object make_binary_object(/* const */ void *t, + size_t size) { + return binary_object(t, size); } -template<> -struct Serializer -{ - template - static inline void load(InputArchive& ar, const binary_object& b) - { - ar.load_binary(const_cast(b.ptr_), b.size_); - } +template <> struct Serializer { + template + static inline void load(InputArchive &ar, const binary_object &b) { + ar.load_binary(const_cast(b.ptr_), b.size_); + } - template - static inline void save(OutputArchive& ar, const binary_object& b) - { - ar.save_binary(b.ptr_, b.size_); - } + template + static inline void save(OutputArchive &ar, const binary_object &b) { + ar.save_binary(b.ptr_, b.size_); + } }; -template<> -struct Serializer -{ - template - static inline void load(InputArchive& ar, binary_object& b) - { - ar.load_binary(const_cast(b.ptr_), b.size_); - } +template <> struct Serializer { + template + static inline void load(InputArchive &ar, binary_object &b) { + ar.load_binary(const_cast(b.ptr_), b.size_); + } - template - static inline void save(OutputArchive& ar, const binary_object& b) - { - ar.save_binary(b.ptr_, b.size_); - } + template + static inline void save(OutputArchive &ar, const binary_object &b) { + ar.save_binary(b.ptr_, b.size_); + } }; - - -template -struct bool_ { - static const bool value = C_; - typedef bool value_type; +template struct bool_ { + static const bool value = C_; + typedef bool value_type; }; - -class ArchiveBase -{ +class ArchiveBase { public: - void* getObject() { return object_; } + void *getObject() { return object_; } - void setObject(void* object) { object_ = object; } + void setObject(void *object) { object_ = object; } private: - void* object_; + void *object_; }; - -template -class InputArchive : public ArchiveBase -{ +template class InputArchive : public ArchiveBase { protected: - InputArchive() {}; + InputArchive(){}; + public: - typedef bool_ is_loading; - typedef bool_ is_saving; - - template - Archive& operator& (T& val) - { - Serializer::load(*static_cast(this),val); - return *static_cast(this); - } -}; + typedef bool_ is_loading; + typedef bool_ is_saving; + template Archive &operator&(T &val) { + Serializer::load(*static_cast(this), val); + return *static_cast(this); + } +}; -template -class OutputArchive : public ArchiveBase -{ +template class OutputArchive : public ArchiveBase { protected: - OutputArchive() {}; + OutputArchive(){}; + public: - typedef bool_ is_loading; - typedef bool_ is_saving; - - template - Archive& operator& (const T& val) - { - Serializer::save(*static_cast(this),val); - return *static_cast(this); - } -}; + typedef bool_ is_loading; + typedef bool_ is_saving; + template Archive &operator&(const T &val) { + Serializer::save(*static_cast(this), val); + return *static_cast(this); + } +}; +class SizeArchive : public OutputArchive { + size_t size_; -class SizeArchive : public OutputArchive -{ - size_t size_; public: + SizeArchive() : size_(0) {} - SizeArchive() : size_(0) - { - } - - template - void save(const T& val) - { - size_ += sizeof(val); - } + template void save(const T &val) { size_ += sizeof(val); } - template - void save_binary(T* ptr, size_t size) - { - size_ += size; - } + template void save_binary(T *ptr, size_t size) { size_ += size; } + void reset() { size_ = 0; } - void reset() - { - size_ = 0; - } - - size_t size() - { - return size_; - } + size_t size() { return size_; } }; - // -//class PrintArchive : public OutputArchive +// class PrintArchive : public OutputArchive //{ -//public: +// public: // template // void save(const T& val) // { @@ -370,442 +280,165 @@ class SizeArchive : public OutputArchive // std::cout << "" << std::endl; // } //}; - + #define BLOCK_BYTES (1024 * 64) -class SaveArchive : public OutputArchive -{ - /** - * Based on blockStreaming_doubleBuffer code at: - * https://github.com/Cyan4973/lz4/blob/master/examples/blockStreaming_doubleBuffer.c - */ - - FILE* stream_; - bool own_stream_; - char *buffer_; - size_t offset_; - - int first_block_; - char *buffer_blocks_; - char *compressed_buffer_; - LZ4_streamHC_t lz4Stream_body; - LZ4_streamHC_t* lz4Stream; - - void initBlock() - { - // Alloc the space for both buffer blocks (each compressed block - // references the previous) - buffer_ = buffer_blocks_ = (char *)malloc(BLOCK_BYTES*2); - compressed_buffer_ = (char *)malloc(LZ4_COMPRESSBOUND(BLOCK_BYTES) + sizeof(size_t)); - if (buffer_ == NULL || compressed_buffer_ == NULL) { - throw FLANNException("Error allocating compression buffer"); - } - - // Init the LZ4 stream - lz4Stream = &lz4Stream_body; - LZ4_resetStreamHC(lz4Stream, 9); - first_block_ = true; - - offset_ = 0; - } - - void flushBlock() - { - size_t compSz = 0; - // Handle header - if (first_block_) { - // Copy & set the header - IndexHeaderStruct *head = (IndexHeaderStruct *)buffer_; - size_t headSz = sizeof(IndexHeaderStruct); - - assert(head->compression == 0); - head->compression = 1; // Bool now, enum later - - // Do the compression for the block - compSz = LZ4_compress_HC_continue( - lz4Stream, buffer_+headSz, compressed_buffer_+headSz, offset_-headSz, - LZ4_COMPRESSBOUND(BLOCK_BYTES)); - - if(compSz <= 0) { - throw FLANNException("Error compressing (first block)"); - } - - // Handle header - head->first_block_size = compSz; - memcpy(compressed_buffer_, buffer_, headSz); - - compSz += headSz; - first_block_ = false; - } else { - size_t headSz = sizeof(compSz); - - // Do the compression for the block - compSz = LZ4_compress_HC_continue( - lz4Stream, buffer_, compressed_buffer_+headSz, offset_, - LZ4_COMPRESSBOUND(BLOCK_BYTES)); - - if(compSz <= 0) { - throw FLANNException("Error compressing"); - } - - // Save the size of the compressed block as the header - memcpy(compressed_buffer_, &compSz, headSz); - compSz += headSz; - } - - // Write the compressed buffer - fwrite(compressed_buffer_, compSz, 1, stream_); - - // Switch the buffer to the *other* block - if (buffer_ == buffer_blocks_) - buffer_ = &buffer_blocks_[BLOCK_BYTES]; - else - buffer_ = buffer_blocks_; - offset_ = 0; - } - - void endBlock() - { - // Cleanup memory - free(buffer_blocks_); - buffer_blocks_ = NULL; - buffer_ = NULL; - free(compressed_buffer_); - compressed_buffer_ = NULL; - - // Write a '0' size for next block - size_t z = 0; - fwrite(&z, sizeof(z), 1, stream_); - } +class SaveArchive : public OutputArchive { + /** + * Based on blockStreaming_doubleBuffer code at: + * https://github.com/Cyan4973/lz4/blob/master/examples/blockStreaming_doubleBuffer.c + */ -public: - SaveArchive(const char* filename) - { - stream_ = fopen(filename, "wb"); - own_stream_ = true; - initBlock(); - } + FILE *stream_; + bool own_stream_; + char *buffer_; + size_t offset_; - SaveArchive(FILE* stream) : stream_(stream), own_stream_(false) - { - initBlock(); - } + int first_block_; + char *buffer_blocks_; + char *compressed_buffer_; - ~SaveArchive() - { - flushBlock(); - endBlock(); - if (buffer_) { - free(buffer_); - buffer_ = NULL; - } - if (own_stream_) { - fclose(stream_); - } - } + void initBlock(); + void flushBlock(); - template - void save(const T& val) - { - assert(sizeof(val) < BLOCK_BYTES); - if (offset_+sizeof(val) > BLOCK_BYTES) - flushBlock(); - memcpy(buffer_+offset_, &val, sizeof(val)); - offset_ += sizeof(val); - } - - template - void save(T* const& val) - { - // don't save pointers - //fwrite(&val, sizeof(val), 1, handle_); - } - - template - void save_binary(T* ptr, size_t size) - { - while (size > BLOCK_BYTES) { - // Flush existing block - flushBlock(); - - // Save large chunk - memcpy(buffer_, ptr, BLOCK_BYTES); - offset_ += BLOCK_BYTES; - ptr = ((char *)ptr) + BLOCK_BYTES; - size -= BLOCK_BYTES; - } - - // Save existing block if new data will make it too big - if (offset_+size > BLOCK_BYTES) - flushBlock(); - - // Copy out requested data - memcpy(buffer_+offset_, ptr, size); - offset_ += size; - } + void endBlock(); +public: + SaveArchive(const char *filename) { + stream_ = fopen(filename, "wb"); + own_stream_ = true; + initBlock(); + } + + SaveArchive(FILE *stream) : stream_(stream), own_stream_(false) { + initBlock(); + } + + ~SaveArchive() { + flushBlock(); + endBlock(); + if (buffer_) { + free(buffer_); + buffer_ = NULL; + } + if (own_stream_) { + fclose(stream_); + } + } + + template void save(const T &val) { + assert(sizeof(val) < BLOCK_BYTES); + if (offset_ + sizeof(val) > BLOCK_BYTES) + flushBlock(); + memcpy(buffer_ + offset_, &val, sizeof(val)); + offset_ += sizeof(val); + } + + template void save(T *const &val) { + // don't save pointers + // fwrite(&val, sizeof(val), 1, handle_); + } + + template void save_binary(T *ptr, size_t size) { + while (size > BLOCK_BYTES) { + // Flush existing block + flushBlock(); + + // Save large chunk + memcpy(buffer_, ptr, BLOCK_BYTES); + offset_ += BLOCK_BYTES; + ptr = ((char *)ptr) + BLOCK_BYTES; + size -= BLOCK_BYTES; + } + + // Save existing block if new data will make it too big + if (offset_ + size > BLOCK_BYTES) + flushBlock(); + + // Copy out requested data + memcpy(buffer_ + offset_, ptr, size); + offset_ += size; + } }; +class LoadArchive : public InputArchive { + /** + * Based on blockStreaming_doubleBuffer code at: + * https://github.com/Cyan4973/lz4/blob/master/examples/blockStreaming_doubleBuffer.c + */ -class LoadArchive : public InputArchive -{ - /** - * Based on blockStreaming_doubleBuffer code at: - * https://github.com/Cyan4973/lz4/blob/master/examples/blockStreaming_doubleBuffer.c - */ - - FILE* stream_; - bool own_stream_; - char *buffer_; - char *ptr_; - - char *buffer_blocks_; - char *compressed_buffer_; - LZ4_streamDecode_t lz4StreamDecode_body; - LZ4_streamDecode_t* lz4StreamDecode; - size_t block_sz_; - - void decompressAndLoadV10(FILE* stream) - { - buffer_ = NULL; - - // Find file size - size_t pos = ftell(stream); - fseek(stream, 0, SEEK_END); - size_t fileSize = ftell(stream)-pos; - fseek(stream, pos, SEEK_SET); - size_t headSz = sizeof(IndexHeaderStruct); - - // Read the (compressed) file to a buffer - char *compBuffer = (char *)malloc(fileSize); - if (compBuffer == NULL) { - throw FLANNException("Error allocating file buffer space"); - } - if (fread(compBuffer, fileSize, 1, stream) != 1) { - free(compBuffer); - throw FLANNException("Invalid index file, cannot read from disk (compressed)"); - } - - // Extract header - IndexHeaderStruct *head = (IndexHeaderStruct *)(compBuffer); - - // Backward compatability - size_t compressedSz = fileSize-headSz; - size_t uncompressedSz = head->first_block_size-headSz; - - // Check for compression type - if (head->compression != 1) { - free(compBuffer); - throw FLANNException("Compression type not supported"); - } - - // Allocate a decompressed buffer - ptr_ = buffer_ = (char *)malloc(uncompressedSz+headSz); - if (buffer_ == NULL) { - free(compBuffer); - throw FLANNException("Error (re)allocating decompression buffer"); - } - - // Extract body - size_t usedSz = LZ4_decompress_safe(compBuffer+headSz, - buffer_+headSz, - compressedSz, - uncompressedSz); - - // Check if the decompression was the expected size. - if (usedSz != uncompressedSz) { - free(compBuffer); - throw FLANNException("Unexpected decompression size"); - } - - // Copy header data - memcpy(buffer_, compBuffer, headSz); - free(compBuffer); - - // Put the file pointer at the end of the data we've read - if (compressedSz+headSz+pos != fileSize) - fseek(stream, compressedSz+headSz+pos, SEEK_SET); - block_sz_ = uncompressedSz+headSz; - } - - void initBlock(FILE *stream) - { - size_t pos = ftell(stream); - buffer_ = NULL; - buffer_blocks_ = NULL; - compressed_buffer_ = NULL; - size_t headSz = sizeof(IndexHeaderStruct); - - // Read the file header to a buffer - IndexHeaderStruct *head = (IndexHeaderStruct *)malloc(headSz); - if (head == NULL) { - throw FLANNException("Error allocating header buffer space"); - } - if (fread(head, headSz, 1, stream) != 1) { - free(head); - throw FLANNException("Invalid index file, cannot read from disk (header)"); - } - - // Backward compatability - if (head->signature[13] == '1' && head->signature[15] == '0') { - free(head); - fseek(stream, pos, SEEK_SET); - return decompressAndLoadV10(stream); - } - - // Alloc the space for both buffer blocks (each block - // references the previous) - buffer_ = buffer_blocks_ = (char *)malloc(BLOCK_BYTES*2); - compressed_buffer_ = (char *)malloc(LZ4_COMPRESSBOUND(BLOCK_BYTES)); - if (buffer_ == NULL || compressed_buffer_ == NULL) { - free(head); - throw FLANNException("Error allocating compression buffer"); - } - - // Init the LZ4 stream - lz4StreamDecode = &lz4StreamDecode_body; - LZ4_setStreamDecode(lz4StreamDecode, NULL, 0); - - // Read first block - memcpy(buffer_, head, headSz); - loadBlock(buffer_+headSz, head->first_block_size, stream); - block_sz_ += headSz; - ptr_ = buffer_; - free(head); - } + FILE *stream_; + bool own_stream_; + char *buffer_; + char *ptr_; - void loadBlock(char* buffer_, size_t compSz, FILE* stream) - { - if(compSz >= LZ4_COMPRESSBOUND(BLOCK_BYTES)) { - throw FLANNException("Requested block size too large"); - } - - // Read the block into the compressed buffer - if (fread(compressed_buffer_, compSz, 1, stream) != 1) { - throw FLANNException("Invalid index file, cannot read from disk (block)"); - } - - // Decompress into the regular buffer - const int decBytes = LZ4_decompress_safe_continue( - lz4StreamDecode, compressed_buffer_, buffer_, compSz, BLOCK_BYTES); - if(decBytes <= 0) { - throw FLANNException("Invalid index file, cannot decompress block"); - } - block_sz_ = decBytes; - } - - void preparePtr(size_t size) - { - // Return if the new size is less than (or eq) the size of a block - if (ptr_+size <= buffer_+block_sz_) - return; - - // Switch the buffer to the *other* block - if (buffer_ == buffer_blocks_) - buffer_ = &buffer_blocks_[BLOCK_BYTES]; - else - buffer_ = buffer_blocks_; - - // Find the size of the next block - size_t cmpSz = 0; - size_t readCnt = fread(&cmpSz, sizeof(cmpSz), 1, stream_); - if(cmpSz <= 0 || readCnt != 1) { - throw FLANNException("Requested to read next block past end of file"); - } - - // Load block & init ptr - loadBlock(buffer_, cmpSz, stream_); - ptr_ = buffer_; - } - - void endBlock() - { - // If not v1.0 format hack... - if (buffer_blocks_ != NULL) { - // Read the last '0' in the file - size_t zero = 1; - if (fread(&zero, sizeof(zero), 1, stream_) != 1) { - throw FLANNException("Invalid index file, cannot read from disk (end)"); - } - if (zero != 0) { - throw FLANNException("Invalid index file, last block not zero length"); - } - } - - // Free resources - if (buffer_blocks_ != NULL) { - free(buffer_blocks_); - buffer_blocks_ = NULL; - } - if (compressed_buffer_ != NULL) { - free(compressed_buffer_); - compressed_buffer_ = NULL; - } - ptr_ = NULL; - } - -public: - LoadArchive(const char* filename) - { - // Open the file - stream_ = fopen(filename, "rb"); - own_stream_ = true; + char *buffer_blocks_; + char *compressed_buffer_; + size_t block_sz_; - initBlock(stream_); - } + void decompressAndLoadV10(FILE *stream); - LoadArchive(FILE* stream) - { - stream_ = stream; - own_stream_ = false; + void initBlock(FILE *stream); - initBlock(stream); - } + void loadBlock(char *buffer_, size_t compSz, FILE *stream); - ~LoadArchive() - { - endBlock(); - if (own_stream_) { - fclose(stream_); - } - } + void preparePtr(size_t size); - template - void load(T& val) - { - preparePtr(sizeof(val)); - memcpy(&val, ptr_, sizeof(val)); - ptr_ += sizeof(val); - } + void endBlock(); - template - void load(T*& val) - { - // don't load pointers - //fread(&val, sizeof(val), 1, handle_); - } - - template - void load_binary(T* ptr, size_t size) - { - while (size > BLOCK_BYTES) { - // Load next block - preparePtr(BLOCK_BYTES); - - // Load large chunk - memcpy(ptr, ptr_, BLOCK_BYTES); - ptr_ += BLOCK_BYTES; - ptr = ((char *)ptr) + BLOCK_BYTES; - size -= BLOCK_BYTES; - } - - // Load next block if needed - preparePtr(size); - - // Load the data - memcpy(ptr, ptr_, size); - ptr_ += size; - } +public: + LoadArchive(const char *filename) { + // Open the file + stream_ = fopen(filename, "rb"); + own_stream_ = true; + + initBlock(stream_); + } + + LoadArchive(FILE *stream) { + stream_ = stream; + own_stream_ = false; + + initBlock(stream); + } + + ~LoadArchive() { + endBlock(); + if (own_stream_) { + fclose(stream_); + } + } + + template void load(T &val) { + preparePtr(sizeof(val)); + memcpy(&val, ptr_, sizeof(val)); + ptr_ += sizeof(val); + } + + template void load(T *&val) { + // don't load pointers + // fread(&val, sizeof(val), 1, handle_); + } + + template void load_binary(T *ptr, size_t size) { + while (size > BLOCK_BYTES) { + // Load next block + preparePtr(BLOCK_BYTES); + + // Load large chunk + memcpy(ptr, ptr_, BLOCK_BYTES); + ptr_ += BLOCK_BYTES; + ptr = ((char *)ptr) + BLOCK_BYTES; + size -= BLOCK_BYTES; + } + + // Load next block if needed + preparePtr(size); + + // Load the data + memcpy(ptr, ptr_, size); + ptr_ += size; + } }; } // namespace serialization From 25d1112f7e840aaf272095324a65f1cb13d0bfcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20H=C3=B6gqvist?= Date: Mon, 16 Sep 2024 09:08:37 +0200 Subject: [PATCH 2/3] HDF5 only if building tests HDF5 is only required if building tests. Also fixed so that both C and CPP static libraries are installed. --- CMakeLists.txt | 12 +++++++----- src/cpp/CMakeLists.txt | 5 ++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f79d173a..572deee7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,11 +76,13 @@ if (NOT PYTHON_EXECUTABLE) endif() endif() -find_hdf5() -if (NOT HDF5_FOUND) - message(WARNING "hdf5 library not found, some tests will not be run") -else() - include_directories(${HDF5_INCLUDE_DIR}) +if(BUILD_TESTS) + find_hdf5() + if (NOT HDF5_FOUND) + message(WARNING "hdf5 library not found, some tests will not be run") + else() + include_directories(${HDF5_INCLUDE_DIR}) + endif() endif() if (USE_MPI OR HDF5_IS_PARALLEL) diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt index 726d8dab..51475752 100644 --- a/src/cpp/CMakeLists.txt +++ b/src/cpp/CMakeLists.txt @@ -4,7 +4,7 @@ add_definitions(-D_FLANN_VERSION=${FLANN_VERSION}) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/flann/config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/flann/config.h) -file(GLOB_RECURSE C_SOURCES flann.cpp) +file(GLOB_RECURSE C_SOURCES flann.cpp util/serialization.cpp) file(GLOB_RECURSE CPP_SOURCES flann_cpp.cpp util/serialization.cpp) file(GLOB_RECURSE CU_SOURCES *.cu) @@ -24,10 +24,9 @@ endif() add_library(flann_cpp SHARED ${CPP_SOURCES}) target_link_directories(flann_cpp PRIVATE ${LZ4_LIBRARY_DIRS}) target_link_libraries(flann_cpp PRIVATE ${LZ4_LIBRARIES}) -# export lz4 headers, so that MSVC to creates flann_cpp.lib set_target_properties(flann_cpp PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS YES) -set(flann_install_targets flann_cpp) +list(APPEND flann_install_targets flann_cpp) if (BUILD_CUDA_LIB) SET(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};-DFLANN_USE_CUDA;-Xcudafe \"--diag_suppress=partial_override\" ;-gencode=arch=compute_52,code=\"sm_52,compute_52\";-gencode=arch=compute_61,code=\"sm_61,compute_61\"") From e2a7b41bd3f2d987d63c394055508a3cd96d6582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20H=C3=B6gqvist?= Date: Mon, 16 Sep 2024 10:28:02 +0200 Subject: [PATCH 3/3] Removed renaming static libs The static libs now have the _s suffix as before. --- src/cpp/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt index 51475752..42c61105 100644 --- a/src/cpp/CMakeLists.txt +++ b/src/cpp/CMakeLists.txt @@ -11,7 +11,6 @@ file(GLOB_RECURSE CU_SOURCES *.cu) add_library(flann_cpp_s STATIC ${CPP_SOURCES}) target_link_directories(flann_cpp_s PRIVATE ${LZ4_LIBRARY_DIRS}) target_link_libraries(flann_cpp_s PRIVATE ${LZ4_LIBRARIES}) -set_target_properties(flann_cpp_s PROPERTIES OUTPUT_NAME flann_cpp) if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG) set_target_properties(flann_cpp_s PROPERTIES COMPILE_FLAGS -fPIC) endif() @@ -85,7 +84,6 @@ if (BUILD_C_BINDINGS) add_library(flann_s STATIC ${C_SOURCES}) target_link_directories(flann_s PRIVATE ${LZ4_LIBRARY_DIRS}) target_link_libraries(flann_s ${LZ4_LIBRARIES}) - set_target_properties(flann_s PROPERTIES OUTPUT_NAME flann) if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG) set_target_properties(flann_s PROPERTIES COMPILE_FLAGS -fPIC) endif()