From e339b030fe60fb4c8959f3bf08c87bf8d406176e Mon Sep 17 00:00:00 2001 From: Dimitri Vlachos Date: Mon, 27 Jan 2025 11:09:22 +0000 Subject: [PATCH 01/16] Create h5 directory in include and move relevant files --- include/dx2/{ => h5}/h5read_processed.h | 0 tests/test_read_h5_array.cxx | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename include/dx2/{ => h5}/h5read_processed.h (100%) diff --git a/include/dx2/h5read_processed.h b/include/dx2/h5/h5read_processed.h similarity index 100% rename from include/dx2/h5read_processed.h rename to include/dx2/h5/h5read_processed.h diff --git a/tests/test_read_h5_array.cxx b/tests/test_read_h5_array.cxx index 3b96cc3..995f60e 100644 --- a/tests/test_read_h5_array.cxx +++ b/tests/test_read_h5_array.cxx @@ -1,4 +1,4 @@ -#include +#include #include #include #include From 6493c85fe9dba19b9975ca1e0140c95d332cd4bc Mon Sep 17 00:00:00 2001 From: Dimitri Vlachos Date: Mon, 27 Jan 2025 11:13:07 +0000 Subject: [PATCH 02/16] Create basic h5 filewriter --- include/dx2/h5/h5write.h | 85 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 include/dx2/h5/h5write.h diff --git a/include/dx2/h5/h5write.h b/include/dx2/h5/h5write.h new file mode 100644 index 0000000..72b451d --- /dev/null +++ b/include/dx2/h5/h5write.h @@ -0,0 +1,85 @@ +#ifndef H5WRITE_H +#define H5WRITE_H + +#include +#include +#include +#include +#include +#include + +// Function to create a group if it does not exist +hid_t create_or_open_group(hid_t parent, const std::string &group_name) { + hid_t group = H5Gopen(parent, group_name.c_str(), H5P_DEFAULT); + if (group < 0) { + group = H5Gcreate(parent, group_name.c_str(), H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT); + if (group < 0) { + std::cerr << "Error: Unable to create group " << group_name << std::endl; + std::exit(1); + } + } + return group; +} + +// Function to write a 3D vector dataset to an HDF5 file +void write_xyzobs_to_h5_file( + const std::string &filename, const std::string &group_path, + const std::string &dataset_name, + const std::vector> &xyz_data) { + hid_t file = H5Fopen(filename.c_str(), H5F_ACC_RDWR, H5P_DEFAULT); + if (file < 0) { + file = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + if (file < 0) { + std::cerr << "Error: Unable to create or open file " << filename + << std::endl; + std::exit(1); + } + } + + // Create or open the hierarchy: /dials/processing/group_0 + hid_t dials_group = create_or_open_group(file, "dials"); + hid_t processing_group = create_or_open_group(dials_group, group_path); + hid_t group = create_or_open_group(processing_group, "group_0"); + + // Create dataspace for 3D vector data + hsize_t dims[2] = {xyz_data.size(), 3}; // Number of elements x 3D coordinates + hid_t dataspace = H5Screate_simple(2, dims, NULL); + + // Create dataset + hid_t dataset = H5Dcreate(group, dataset_name.c_str(), H5T_NATIVE_DOUBLE, + dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if (dataset < 0) { + std::cerr << "Error: Unable to create dataset " << dataset_name + << std::endl; + std::exit(1); + } + + // Flatten 3D vector data for HDF5 storage + std::vector flat_data; + for (const auto &xyz : xyz_data) { + flat_data.insert(flat_data.end(), xyz.begin(), xyz.end()); + } + + // Write data to dataset + herr_t status = H5Dwrite(dataset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, + H5P_DEFAULT, flat_data.data()); + if (status < 0) { + std::cerr << "Error: Unable to write data to dataset " << dataset_name + << std::endl; + std::exit(1); + } + + // Cleanup + H5Dclose(dataset); + H5Sclose(dataspace); + H5Gclose(group); + H5Gclose(processing_group); + H5Gclose(dials_group); + H5Fclose(file); + + std::cout << "Successfully wrote 3D vector dataset " << dataset_name << " to " + << filename << std::endl; +} + +#endif // H5WRITE_H From da24d268df0e35eecfecde2eac4ae3056ad04c80 Mon Sep 17 00:00:00 2001 From: Dimitri Vlachos Date: Mon, 27 Jan 2025 11:13:32 +0000 Subject: [PATCH 03/16] Add unit test for h5 filewriter --- tests/CMakeLists.txt | 4 ++++ tests/test_write_h5_array.cxx | 43 +++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 tests/test_write_h5_array.cxx diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2de1601..1216b17 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,7 +5,11 @@ target_link_libraries(test_make_models GTest::gtest_main dx2 Eigen3::Eigen nlohm add_executable(test_read_h5_array test_read_h5_array.cxx) target_link_libraries(test_read_h5_array GTest::gtest_main dx2 hdf5::hdf5) +add_executable(test_write_h5_array test_write_h5_array.cxx) +target_link_libraries(test_write_h5_array GTest::gtest_main dx2 hdf5::hdf5) + include(GoogleTest) gtest_discover_tests(test_make_models PROPERTIES LABELS dx2tests) gtest_discover_tests(test_read_h5_array WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}/tests" PROPERTIES LABELS dx2tests) +gtest_discover_tests(test_write_h5_array WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}/tests" PROPERTIES LABELS dx2tests) diff --git a/tests/test_write_h5_array.cxx b/tests/test_write_h5_array.cxx new file mode 100644 index 0000000..8166381 --- /dev/null +++ b/tests/test_write_h5_array.cxx @@ -0,0 +1,43 @@ +#include +#include +#include +#include +#include + +// Test writing a 3D vector dataset to an HDF5 file and verifying the written +// content +TEST(ExampleTests, WriteArrayTest) { + // Define the test file path + std::filesystem::path cwd = std::filesystem::current_path(); + std::string test_file_path = cwd.generic_string(); + test_file_path.append("/data/test_write.h5"); + + // Prepare test data (3D vectors) + std::vector> xyzobs_test_data = { + {100.1, 200.2, 300.3}, {400.4, 500.5, 600.6}, {700.7, 800.8, 900.9}}; + + // Call the function to write the test data to the HDF5 file + std::string dataset_name = "xyzobs.px.value"; + std::string group_path = "processing"; + + write_xyzobs_to_h5_file(test_file_path, group_path, dataset_name, + xyzobs_test_data); + + // Read the written data back from the HDF5 file + std::string full_dataset_path = "/dials/processing/group_0/xyzobs.px.value"; + std::vector read_xyzobs = + read_array_from_h5_file(test_file_path, full_dataset_path); + + // Validate the size of the read data + ASSERT_EQ(read_xyzobs.size(), xyzobs_test_data.size() * 3); + + // Validate the content of the read data + for (std::size_t i = 0; i < xyzobs_test_data.size(); ++i) { + EXPECT_DOUBLE_EQ(read_xyzobs[i * 3 + 0], xyzobs_test_data[i][0]); + EXPECT_DOUBLE_EQ(read_xyzobs[i * 3 + 1], xyzobs_test_data[i][1]); + EXPECT_DOUBLE_EQ(read_xyzobs[i * 3 + 2], xyzobs_test_data[i][2]); + } + + // Clean up test file after successful run (comment out to keep the test file) + std::filesystem::remove(test_file_path); +} From d8fb7defc04d374e8ca9ca674d3418c1de0c5d12 Mon Sep 17 00:00:00 2001 From: Dimitri Vlachos Date: Tue, 28 Jan 2025 16:16:24 +0000 Subject: [PATCH 04/16] Create recursive group creation and traversal for generalisablility --- include/dx2/h5/h5write.h | 110 +++++++++++++++------------------- tests/test_write_h5_array.cxx | 89 +++++++++++++++------------ 2 files changed, 98 insertions(+), 101 deletions(-) diff --git a/include/dx2/h5/h5write.h b/include/dx2/h5/h5write.h index 72b451d..3cf0ff2 100644 --- a/include/dx2/h5/h5write.h +++ b/include/dx2/h5/h5write.h @@ -8,78 +8,64 @@ #include #include -// Function to create a group if it does not exist -hid_t create_or_open_group(hid_t parent, const std::string &group_name) { - hid_t group = H5Gopen(parent, group_name.c_str(), H5P_DEFAULT); - if (group < 0) { - group = H5Gcreate(parent, group_name.c_str(), H5P_DEFAULT, H5P_DEFAULT, - H5P_DEFAULT); - if (group < 0) { - std::cerr << "Error: Unable to create group " << group_name << std::endl; - std::exit(1); - } +/** + * @brief Recursively traverses or creates groups in an HDF5 file based + * on the given path. + * + * This function takes a parent group identifier and a path string, and + * recursively traverses or creates the groups specified in the path. If + * a group in the path does not exist, it is created. + * + * @param parent The identifier of the parent group in the HDF5 file. + * @param path The path of groups to traverse or create, specified as a + * string with '/' as the delimiter. + * @return The identifier of the final group in the path. + * @throws std::runtime_error If a group cannot be created or opened. + */ +hid_t traverse_or_create_groups(hid_t parent, const std::string &path) { + // Strip leading '/' characters, if any, to prevent empty group names + size_t start_pos = path.find_first_not_of('/'); + if (start_pos == std::string::npos) { + return parent; // Return parent if the path is entirely '/' } - return group; -} + std::string cleaned_path = path.substr(start_pos); -// Function to write a 3D vector dataset to an HDF5 file -void write_xyzobs_to_h5_file( - const std::string &filename, const std::string &group_path, - const std::string &dataset_name, - const std::vector> &xyz_data) { - hid_t file = H5Fopen(filename.c_str(), H5F_ACC_RDWR, H5P_DEFAULT); - if (file < 0) { - file = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - if (file < 0) { - std::cerr << "Error: Unable to create or open file " << filename - << std::endl; - std::exit(1); - } + /* + * This is the base case for recursion. When the path is empty, we + * have reached the final group in the path and we return the parent + * group. + */ + if (cleaned_path.empty()) { + return parent; } - // Create or open the hierarchy: /dials/processing/group_0 - hid_t dials_group = create_or_open_group(file, "dials"); - hid_t processing_group = create_or_open_group(dials_group, group_path); - hid_t group = create_or_open_group(processing_group, "group_0"); - - // Create dataspace for 3D vector data - hsize_t dims[2] = {xyz_data.size(), 3}; // Number of elements x 3D coordinates - hid_t dataspace = H5Screate_simple(2, dims, NULL); + // Split the path into the current group name and the remaining path + size_t pos = cleaned_path.find('/'); + std::string group_name = + (pos == std::string::npos) ? cleaned_path : cleaned_path.substr(0, pos); + std::string remaining_path = + (pos == std::string::npos) ? "" : cleaned_path.substr(pos + 1); - // Create dataset - hid_t dataset = H5Dcreate(group, dataset_name.c_str(), H5T_NATIVE_DOUBLE, - dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (dataset < 0) { - std::cerr << "Error: Unable to create dataset " << dataset_name - << std::endl; - std::exit(1); + // Attempt to open the group. If it does not exist, create it. + hid_t next_group = H5Gopen(parent, group_name.c_str(), H5P_DEFAULT); + if (next_group < 0) { + next_group = H5Gcreate(parent, group_name.c_str(), H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT); + if (next_group < 0) { + std::runtime_error("Error: Unable to create or open group: " + + group_name); + } } - // Flatten 3D vector data for HDF5 storage - std::vector flat_data; - for (const auto &xyz : xyz_data) { - flat_data.insert(flat_data.end(), xyz.begin(), xyz.end()); - } + // Recurse to the next group in the hierarchy + hid_t final_group = traverse_or_create_groups(next_group, remaining_path); - // Write data to dataset - herr_t status = H5Dwrite(dataset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, - H5P_DEFAULT, flat_data.data()); - if (status < 0) { - std::cerr << "Error: Unable to write data to dataset " << dataset_name - << std::endl; - std::exit(1); + // Close the current group to avoid resource leaks, except for the final group + if (next_group != final_group) { + H5Gclose(next_group); } - // Cleanup - H5Dclose(dataset); - H5Sclose(dataspace); - H5Gclose(group); - H5Gclose(processing_group); - H5Gclose(dials_group); - H5Fclose(file); - - std::cout << "Successfully wrote 3D vector dataset " << dataset_name << " to " - << filename << std::endl; + return final_group; } #endif // H5WRITE_H diff --git a/tests/test_write_h5_array.cxx b/tests/test_write_h5_array.cxx index 8166381..62b8c3a 100644 --- a/tests/test_write_h5_array.cxx +++ b/tests/test_write_h5_array.cxx @@ -1,43 +1,54 @@ -#include #include #include #include -#include - -// Test writing a 3D vector dataset to an HDF5 file and verifying the written -// content -TEST(ExampleTests, WriteArrayTest) { - // Define the test file path - std::filesystem::path cwd = std::filesystem::current_path(); - std::string test_file_path = cwd.generic_string(); - test_file_path.append("/data/test_write.h5"); - - // Prepare test data (3D vectors) - std::vector> xyzobs_test_data = { - {100.1, 200.2, 300.3}, {400.4, 500.5, 600.6}, {700.7, 800.8, 900.9}}; - - // Call the function to write the test data to the HDF5 file - std::string dataset_name = "xyzobs.px.value"; - std::string group_path = "processing"; - - write_xyzobs_to_h5_file(test_file_path, group_path, dataset_name, - xyzobs_test_data); - - // Read the written data back from the HDF5 file - std::string full_dataset_path = "/dials/processing/group_0/xyzobs.px.value"; - std::vector read_xyzobs = - read_array_from_h5_file(test_file_path, full_dataset_path); - - // Validate the size of the read data - ASSERT_EQ(read_xyzobs.size(), xyzobs_test_data.size() * 3); - - // Validate the content of the read data - for (std::size_t i = 0; i < xyzobs_test_data.size(); ++i) { - EXPECT_DOUBLE_EQ(read_xyzobs[i * 3 + 0], xyzobs_test_data[i][0]); - EXPECT_DOUBLE_EQ(read_xyzobs[i * 3 + 1], xyzobs_test_data[i][1]); - EXPECT_DOUBLE_EQ(read_xyzobs[i * 3 + 2], xyzobs_test_data[i][2]); - } - - // Clean up test file after successful run (comment out to keep the test file) - std::filesystem::remove(test_file_path); +#include +#include + +// Test the traverse_or_create_groups function +TEST(HDF5Tests, TraverseOrCreateGroupsTest) { + // Define the test file path + std::filesystem::path cwd = std::filesystem::current_path(); + std::string test_file_path = cwd.generic_string(); + test_file_path.append("/data/test_traverse_or_create_groups.h5"); + + // Create or open an HDF5 file + hid_t file = H5Fcreate(test_file_path.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + ASSERT_GE(file, 0) << "Failed to create HDF5 file."; + + try { + // Define a hierarchical path + std::string group_path = "/dials/processing/group_0"; + + // Call the function to traverse or create groups + hid_t final_group = traverse_or_create_groups(file, group_path); + ASSERT_GE(final_group, 0) << "Failed to create or open the final group."; + + // Verify that each group in the hierarchy exists + hid_t dials_group = H5Gopen(file, "dials", H5P_DEFAULT); + ASSERT_GE(dials_group, 0) << "Failed to open the 'dials' group."; + + hid_t processing_group = H5Gopen(dials_group, "processing", H5P_DEFAULT); + ASSERT_GE(processing_group, 0) << "Failed to open the 'processing' group."; + + hid_t group_0 = H5Gopen(processing_group, "group_0", H5P_DEFAULT); + ASSERT_GE(group_0, 0) << "Failed to open the 'group_0' group."; + + // Close all opened groups + H5Gclose(group_0); + H5Gclose(processing_group); + H5Gclose(dials_group); + H5Gclose(final_group); + } catch (const std::runtime_error &e) { + FAIL() << "Runtime error occurred: " << e.what(); + } + + // Close the file + H5Fclose(file); + + // Validate that the HDF5 file was created + ASSERT_TRUE(std::filesystem::exists(test_file_path)) + << "HDF5 file was not created."; + + // Clean up test file after successful run (comment out to keep the test file) + std::filesystem::remove(test_file_path); } From 0a4dcc35e7e5801e272066b00c063e594de10272 Mon Sep 17 00:00:00 2001 From: Dimitri Vlachos Date: Tue, 28 Jan 2025 17:55:45 +0000 Subject: [PATCH 05/16] Add functions to deduce shape and flatten nested containers for HDF5 writing --- include/dx2/h5/h5write.h | 85 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/include/dx2/h5/h5write.h b/include/dx2/h5/h5write.h index 3cf0ff2..2bc1c05 100644 --- a/include/dx2/h5/h5write.h +++ b/include/dx2/h5/h5write.h @@ -68,4 +68,89 @@ hid_t traverse_or_create_groups(hid_t parent, const std::string &path) { return final_group; } +/** + * @brief Deduce the shape of a nested container. + * + * This helper function recursively determines the shape of nested + * containers. This is used to determine the shape of the dataset to be + * created in an HDF5 file. + * + * @tparam Container The type of the container. + * @param container The container whose shape is to be determined. + * @return A vector of dimensions representing the shape of the + * container. + */ +template +std::vector deduce_shape(const Container &container) { + if (container.empty()) { + return {0}; + } + if constexpr (std::is_arithmetic_v) { + // Base case: container holds arithmetic types (e.g., double, int) + return {container.size()}; + } else { + // Recursive case: container holds other containers + + // Check that all inner containers have the same size + size_t inner_size = container.begin()->size(); + for (const auto &sub_container : container) { + if (sub_container.size() != inner_size) { + throw std::runtime_error("Cannot deduce shape: inner containers have " + "different sizes."); + } + } + + auto sub_shape = deduce_shape(*container.begin()); + sub_shape.insert(sub_shape.begin(), container.size()); + return sub_shape; + } +} + +/** + * @brief Flatten nested containers into a 1D vector. + * + * This helper function recursively flattens nested containers into a 1D + * vector for writing to HDF5. If the input container is already 1D, it + * simply returns it. + * + * @tparam Container The type of the container. + * @param container The container to flatten. + * @return A flat vector containing all elements of the input container + * in a linear order. + */ +template auto flatten(const Container &container) { + // Determine the type of the elements in the container + using ValueType = typename Container::value_type; + + // Base case: If the container holds arithmetic types (e.g., int, double), + // it is already 1D, so we return a copy of the container as a std::vector. + if constexpr (std::is_arithmetic_v) { + return std::vector(container.begin(), container.end()); + } else { + // Recursive case: The container holds nested containers, so we need to + // flatten them. + + // Determine the type of elements in the flattened result. + // This is deduced by recursively calling flatten on the first + // sub-container. + using InnerType = + typename decltype(flatten(*container.begin()))::value_type; + + // Create a vector to store the flattened data + std::vector flat_data; + + // Iterate over the outer container + for (const auto &sub_container : container) { + // Recursively flatten each sub-container + auto sub_flat = flatten(sub_container); + + // Append the flattened sub-container to the result + flat_data.insert(flat_data.end(), sub_flat.begin(), sub_flat.end()); + } + + // Return the fully flattened data + return flat_data; + } +} + #endif // H5WRITE_H From 4a3118a851185890aeac343ced393a93dad454de Mon Sep 17 00:00:00 2001 From: Dimitri Vlachos Date: Tue, 28 Jan 2025 17:56:34 +0000 Subject: [PATCH 06/16] Add function to write arbitrarily dimensional data to HDF5 files --- include/dx2/h5/h5write.h | 87 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/include/dx2/h5/h5write.h b/include/dx2/h5/h5write.h index 2bc1c05..d41850c 100644 --- a/include/dx2/h5/h5write.h +++ b/include/dx2/h5/h5write.h @@ -153,4 +153,91 @@ template auto flatten(const Container &container) { } } +/** + * @brief Writes multidimensional data to an HDF5 file. + * + * This function writes a dataset to an HDF5 file. The dataset's shape + * is determined dynamically based on the input container. + * + * @tparam Container The type of the container holding the data. + * @param filename The path to the HDF5 file. + * @param dataset_path The full path to the dataset, including group + * hierarchies. + * @param data The data to write to the dataset. + * @throws std::runtime_error If the dataset cannot be created or data + * cannot be written. + */ +template +void write_data_to_h5_file(const std::string &filename, + const std::string &dataset_path, + const Container &data) { + // Open or create the HDF5 file + hid_t file = H5Fopen(filename.c_str(), H5F_ACC_RDWR, H5P_DEFAULT); + if (file < 0) { + file = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + if (file < 0) { + throw std::runtime_error("Error: Unable to create or open file: " + + filename); + } + } + + try { + // Separate the dataset path into group path and dataset name + size_t last_slash_pos = dataset_path.find_last_of('/'); + if (last_slash_pos == std::string::npos) { + throw std::runtime_error("Error: Invalid dataset path, no '/' found: " + + dataset_path); + } + + std::string group_path = dataset_path.substr(0, last_slash_pos); + std::string dataset_name = dataset_path.substr(last_slash_pos + 1); + + // Traverse or create the groups leading to the dataset + hid_t group = traverse_or_create_groups(file, group_path); + + // Deduce the shape of the data + std::vector shape = deduce_shape(data); + + // Flatten the data into a 1D vector + auto flat_data = flatten(data); + + // Create dataspace for the dataset + hid_t dataspace = H5Screate_simple(shape.size(), shape.data(), NULL); + if (dataspace < 0) { + throw std::runtime_error( + "Error: Unable to create dataspace for dataset: " + dataset_name); + } + + // Create the dataset + hid_t dataset = H5Dcreate(group, dataset_name.c_str(), H5T_NATIVE_DOUBLE, + dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if (dataset < 0) { + H5Sclose(dataspace); + throw std::runtime_error("Error: Unable to create dataset: " + + dataset_name); + } + + // Write the data to the dataset + herr_t status = H5Dwrite(dataset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, + H5P_DEFAULT, flat_data.data()); + if (status < 0) { + H5Dclose(dataset); + H5Sclose(dataspace); + throw std::runtime_error("Error: Unable to write data to dataset: " + + dataset_name); + } + + // Cleanup resources + H5Dclose(dataset); + H5Sclose(dataspace); + H5Gclose(group); + } catch (...) { + H5Fclose(file); + throw; // Re-throw the exception to propagate it upwards + } + + // Close the file + H5Fclose(file); +} + #endif // H5WRITE_H From 97361a9088476d27c653f469423aa508764ddfe4 Mon Sep 17 00:00:00 2001 From: Dimitri Vlachos Date: Tue, 28 Jan 2025 17:57:01 +0000 Subject: [PATCH 07/16] Add comprehensive unit tests for HDF5 data writing functions --- tests/test_write_h5_array.cxx | 272 +++++++++++++++++++++++++++++----- 1 file changed, 237 insertions(+), 35 deletions(-) diff --git a/tests/test_write_h5_array.cxx b/tests/test_write_h5_array.cxx index 62b8c3a..7e323ea 100644 --- a/tests/test_write_h5_array.cxx +++ b/tests/test_write_h5_array.cxx @@ -1,54 +1,256 @@ +#include +#include #include #include #include #include #include +#include +#pragma region traverse_or_create_groups tests // Test the traverse_or_create_groups function TEST(HDF5Tests, TraverseOrCreateGroupsTest) { - // Define the test file path - std::filesystem::path cwd = std::filesystem::current_path(); - std::string test_file_path = cwd.generic_string(); - test_file_path.append("/data/test_traverse_or_create_groups.h5"); + // Define the test file path + std::filesystem::path cwd = std::filesystem::current_path(); + std::string test_file_path = cwd.generic_string(); + test_file_path.append("/data/test_traverse_or_create_groups.h5"); - // Create or open an HDF5 file - hid_t file = H5Fcreate(test_file_path.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - ASSERT_GE(file, 0) << "Failed to create HDF5 file."; + // Create or open an HDF5 file + hid_t file = H5Fcreate(test_file_path.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, + H5P_DEFAULT); + ASSERT_GE(file, 0) << "Failed to create HDF5 file."; - try { - // Define a hierarchical path - std::string group_path = "/dials/processing/group_0"; + try { + // Define a hierarchical path + std::string group_path = "/dials/processing/group_0"; - // Call the function to traverse or create groups - hid_t final_group = traverse_or_create_groups(file, group_path); - ASSERT_GE(final_group, 0) << "Failed to create or open the final group."; + // Call the function to traverse or create groups + hid_t final_group = traverse_or_create_groups(file, group_path); + ASSERT_GE(final_group, 0) << "Failed to create or open the final group."; - // Verify that each group in the hierarchy exists - hid_t dials_group = H5Gopen(file, "dials", H5P_DEFAULT); - ASSERT_GE(dials_group, 0) << "Failed to open the 'dials' group."; + // Verify that each group in the hierarchy exists + hid_t dials_group = H5Gopen(file, "dials", H5P_DEFAULT); + ASSERT_GE(dials_group, 0) << "Failed to open the 'dials' group."; - hid_t processing_group = H5Gopen(dials_group, "processing", H5P_DEFAULT); - ASSERT_GE(processing_group, 0) << "Failed to open the 'processing' group."; + hid_t processing_group = H5Gopen(dials_group, "processing", H5P_DEFAULT); + ASSERT_GE(processing_group, 0) << "Failed to open the 'processing' group."; - hid_t group_0 = H5Gopen(processing_group, "group_0", H5P_DEFAULT); - ASSERT_GE(group_0, 0) << "Failed to open the 'group_0' group."; + hid_t group_0 = H5Gopen(processing_group, "group_0", H5P_DEFAULT); + ASSERT_GE(group_0, 0) << "Failed to open the 'group_0' group."; - // Close all opened groups - H5Gclose(group_0); - H5Gclose(processing_group); - H5Gclose(dials_group); - H5Gclose(final_group); - } catch (const std::runtime_error &e) { - FAIL() << "Runtime error occurred: " << e.what(); - } + // Close all opened groups + H5Gclose(group_0); + H5Gclose(processing_group); + H5Gclose(dials_group); + H5Gclose(final_group); + } catch (const std::runtime_error &e) { + FAIL() << "Runtime error occurred: " << e.what(); + } - // Close the file - H5Fclose(file); + // Close the file + H5Fclose(file); - // Validate that the HDF5 file was created - ASSERT_TRUE(std::filesystem::exists(test_file_path)) - << "HDF5 file was not created."; + // Validate that the HDF5 file was created + ASSERT_TRUE(std::filesystem::exists(test_file_path)) + << "HDF5 file was not created."; - // Clean up test file after successful run (comment out to keep the test file) - std::filesystem::remove(test_file_path); + // Clean up test file after successful run (comment out to keep the test file) + std::filesystem::remove(test_file_path); } +#pragma endregion traverse_or_create_groups tests + +#pragma region deduce_shape tests +// Test deducing shape for 1D container (std::vector) +TEST(DeduceShapeTests, OneDimensionalVector) { + std::vector data = {1.0, 2.0, 3.0, 4.0}; + std::vector expected_shape = {4}; // Single dimension of size 4 + auto shape = deduce_shape(data); + EXPECT_EQ(shape, expected_shape); +} + +// Test deducing shape for 2D container (std::vector) +TEST(DeduceShapeTests, TwoDimensionalVector) { + std::vector> data = {{1.0, 2.0, 3.0}, {4.0, 5.0, 6.0}}; + std::vector expected_shape = {2, 3}; // 2 rows, 3 columns + auto shape = deduce_shape(data); + EXPECT_EQ(shape, expected_shape); +} + +// Test deducing shape for 2D container (std::vector) +TEST(DeduceShapeTests, TwoDimensionalVectorArray) { + std::vector> data = { + {1.0, 2.0, 3.0}, {4.0, 5.0, 6.0}, {7.0, 8.0, 9.0}}; + std::vector expected_shape = {3, 3}; // 3 rows, 3 columns + auto shape = deduce_shape(data); + EXPECT_EQ(shape, expected_shape); +} + +// Test deducing shape for 3D container (std::vector>) +TEST(DeduceShapeTests, ThreeDimensionalVector) { + std::vector>> data = {{{1, 2}, {3, 4}}, + {{5, 6}, {7, 8}}}; + std::vector expected_shape = {2, 2, 2}; // 2x2x2 structure + auto shape = deduce_shape(data); + EXPECT_EQ(shape, expected_shape); +} + +// Test deducing shape for empty container +TEST(DeduceShapeTests, EmptyContainer) { + std::vector> data = {}; + std::vector expected_shape = {0}; // Outer container size is 0 + auto shape = deduce_shape(data); + EXPECT_EQ(shape, expected_shape); +} + +// Test deducing shape for mixed-sized inner containers (should throw or handle +// gracefully) +TEST(DeduceShapeTests, MixedSizeInnerContainers) { + std::vector> data = { + {1.0, 2.0}, {3.0, 4.0, 5.0} // Different size + }; + + // deduce_shape assumes uniformity. If it encounters mixed sizes, it might + // throw or return the first subcontainer's size. + try { + auto shape = deduce_shape(data); + FAIL() << "Expected deduce_shape to throw due to inconsistent inner " + "container sizes."; + } catch (const std::exception &e) { + SUCCEED(); // deduce_shape threw an exception as expected + } catch (...) { + FAIL() << "Expected deduce_shape to throw a standard exception."; + } +} +#pragma endregion deduce_shape tests + +#pragma region flatten tests +// Test flattening a 1D vector +TEST(FlattenTests, OneDimensionalVector) { + std::vector data = {1.0, 2.0, 3.0, 4.0}; + auto flat_data = flatten(data); // Should return the same as the input + + std::vector expected_flat_data = {1.0, 2.0, 3.0, 4.0}; + EXPECT_EQ(flat_data, expected_flat_data); +} + +// Test flattening a 2D vector +TEST(FlattenTests, TwoDimensionalVector) { + std::vector> data = {{1.0, 2.0, 3.0}, {4.0, 5.0, 6.0}}; + auto flat_data = flatten(data); + + std::vector expected_flat_data = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; + EXPECT_EQ(flat_data, expected_flat_data); +} + +// Test flattening a 2D vector of arrays +TEST(FlattenTests, TwoDimensionalVectorArray) { + std::vector> data = { + {1.0, 2.0, 3.0}, {4.0, 5.0, 6.0}, {7.0, 8.0, 9.0}}; + auto flat_data = flatten(data); + + std::vector expected_flat_data = {1.0, 2.0, 3.0, 4.0, 5.0, + 6.0, 7.0, 8.0, 9.0}; + EXPECT_EQ(flat_data, expected_flat_data); +} + +// Test flattening a 3D vector +TEST(FlattenTests, ThreeDimensionalVector) { + std::vector>> data = {{{1, 2}, {3, 4}}, + {{5, 6}, {7, 8}}}; + auto flat_data = flatten(data); + + std::vector expected_flat_data = {1, 2, 3, 4, 5, 6, 7, 8}; + EXPECT_EQ(flat_data, expected_flat_data); +} + +// Test flattening an empty container +TEST(FlattenTests, EmptyContainer) { + std::vector> data = {}; + auto flat_data = flatten(data); + + std::vector expected_flat_data = {}; + EXPECT_EQ(flat_data, expected_flat_data); +} + +// Test flattening a container with mixed sizes (shouldn't be an issue here) +TEST(FlattenTests, MixedSizeInnerContainers) { + std::vector> data = {{1.0, 2.0}, {3.0, 4.0, 5.0}}; + + // Flatten should concatenate all elements into a single 1D vector + auto flat_data = flatten(data); + std::vector expected_flat_data = {1.0, 2.0, 3.0, 4.0, 5.0}; + EXPECT_EQ(flat_data, expected_flat_data); +} +#pragma endregion flatten tests + +#pragma region write_h5 tests +// Test writing a 1D vector to an HDF5 file +TEST(WriteDataTests, WriteOneDimensionalVector) { + std::string filename = "test_1d_vector.h5"; + std::string dataset_path = "/group_1/dataset_1d"; + + std::vector data = {1.0, 2.0, 3.0, 4.0}; + write_data_to_h5_file(filename, dataset_path, data); + + auto read_data = read_array_from_h5_file(filename, dataset_path); + + EXPECT_EQ(data, read_data); + + // Clean up test file + std::filesystem::remove(filename); +} + +// Test writing a 2D vector to an HDF5 file +TEST(WriteDataTests, WriteTwoDimensionalVector) { + std::string filename = "test_2d_vector.h5"; + std::string dataset_path = "/group_2/dataset_2d"; + + std::vector> data = {{1.0, 2.0, 3.0}, {4.0, 5.0, 6.0}}; + write_data_to_h5_file(filename, dataset_path, data); + + auto read_data = read_array_from_h5_file(filename, dataset_path); + + // Flatten the original data for comparison + std::vector expected_data = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; + EXPECT_EQ(expected_data, read_data); + + // Clean up test file + std::filesystem::remove(filename); +} + +// Test writing a 2D vector of arrays to an HDF5 file +TEST(WriteDataTests, WriteTwoDimensionalVectorArray) { + std::string filename = "test_2d_vector_array.h5"; + std::string dataset_path = "/group_3/dataset_2d_array"; + + std::vector> data = { + {1.0, 2.0, 3.0}, {4.0, 5.0, 6.0}, {7.0, 8.0, 9.0}}; + write_data_to_h5_file(filename, dataset_path, data); + + auto read_data = read_array_from_h5_file(filename, dataset_path); + + // Flatten the original data for comparison + std::vector expected_data = {1.0, 2.0, 3.0, 4.0, 5.0, + 6.0, 7.0, 8.0, 9.0}; + EXPECT_EQ(expected_data, read_data); + + // Clean up test file + std::filesystem::remove(filename); +} + +// Test writing an empty dataset to an HDF5 file +TEST(WriteDataTests, WriteEmptyDataset) { + std::string filename = "test_empty_dataset.h5"; + std::string dataset_path = "/group_empty/dataset_empty"; + + std::vector data = {}; // Empty dataset + write_data_to_h5_file(filename, dataset_path, data); + + auto read_data = read_array_from_h5_file(filename, dataset_path); + + EXPECT_TRUE(read_data.empty()); + + // Clean up test file + std::filesystem::remove(filename); +} +#pragma endregion write_h5 tests \ No newline at end of file From 963878c685796165f9bde2e169d498c92713c99a Mon Sep 17 00:00:00 2001 From: Dimitri Vlachos Date: Tue, 28 Jan 2025 18:02:49 +0000 Subject: [PATCH 08/16] Enhance HDF5 dataset reading function with improved error handling --- include/dx2/h5/h5read_processed.h | 98 +++++++++++++++++++++++-------- 1 file changed, 74 insertions(+), 24 deletions(-) diff --git a/include/dx2/h5/h5read_processed.h b/include/dx2/h5/h5read_processed.h index 1b636ca..628fa7e 100644 --- a/include/dx2/h5/h5read_processed.h +++ b/include/dx2/h5/h5read_processed.h @@ -1,3 +1,6 @@ +#ifndef DX2_H5_H5READ_PROCESSED_HPP +#define DX2_H5_H5READ_PROCESSED_HPP + #include #include #include @@ -7,34 +10,81 @@ #include #include +/** + * @brief Reads a dataset from an HDF5 file into an std::vector. + * + * @tparam T The type of data to read (e.g., int, double). + * @param filename The path to the HDF5 file. + * @param dataset_name The name of the dataset to read. + * @return A std::vector containing the data from the dataset. + * @throws std::runtime_error If the file, dataset, or datatype cannot be opened + * or read. + */ template -std::vector read_array_from_h5_file(std::string filename, - std::string array_name) { +std::vector read_array_from_h5_file(const std::string &filename, + const std::string &dataset_name) { + // Start measuring time auto start_time = std::chrono::high_resolution_clock::now(); + + // Open the HDF5 file hid_t file = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); if (file < 0) { - std::cout << "Error: Unable to open " << filename.c_str() - << " as a hdf5 reflection table" << std::endl; - std::exit(1); + throw std::runtime_error("Error: Unable to open file: " + filename); } - hid_t dataset = H5Dopen(file, array_name.c_str(), H5P_DEFAULT); - hid_t datatype = H5Dget_type(dataset); - size_t datatype_size = H5Tget_size(datatype); - assert((datatype_size == sizeof(T))); - hid_t dataspace = H5Dget_space(dataset); - size_t num_elements = H5Sget_simple_extent_npoints(dataspace); - hid_t space = H5Dget_space(dataset); - std::vector data_out(num_elements); - - H5Dread(dataset, datatype, H5S_ALL, space, H5P_DEFAULT, &data_out[0]); - float total_time = std::chrono::duration_cast>( - std::chrono::high_resolution_clock::now() - start_time) - .count(); - std::cout << "READ TIME for " << array_name << " : " << total_time << "s" - << std::endl; - - H5Dclose(dataset); - H5Fclose(file); - return data_out; + try { + // Open the dataset + hid_t dataset = H5Dopen(file, dataset_name.c_str(), H5P_DEFAULT); + if (dataset < 0) { + throw std::runtime_error("Error: Unable to open dataset: " + + dataset_name); + } + + try { + // Get the datatype and check size + hid_t datatype = H5Dget_type(dataset); + size_t datatype_size = H5Tget_size(datatype); + if (datatype_size != sizeof(T)) { + throw std::runtime_error( + "Error: Dataset type size does not match expected type size."); + } + + // Get the dataspace and the number of elements + hid_t dataspace = H5Dget_space(dataset); + size_t num_elements = H5Sget_simple_extent_npoints(dataspace); + + // Allocate a vector to hold the data + std::vector data_out(num_elements); + + // Read the data into the vector + herr_t status = H5Dread(dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, + data_out.data()); + if (status < 0) { + throw std::runtime_error("Error: Unable to read dataset: " + + dataset_name); + } + + // Close the dataset and return the data + H5Dclose(dataset); + H5Fclose(file); + + // Log timing + auto end_time = std::chrono::high_resolution_clock::now(); + double elapsed_time = + std::chrono::duration(end_time - start_time).count(); + std::cout << "READ TIME for " << dataset_name << " : " << elapsed_time + << "s" << std::endl; + + return data_out; + + } catch (...) { + H5Dclose(dataset); // Ensure dataset is closed in case of failure + throw; + } + } catch (...) { + H5Fclose(file); // Ensure file is closed in case of failure + throw; + } } + +#endif // DX2_H5_H5READ_PROCESSED_HPP \ No newline at end of file From 5e1a186121c720c8ea093d9d0f097821fae8ea43 Mon Sep 17 00:00:00 2001 From: Dimitri Vlachos Date: Wed, 29 Jan 2025 12:47:14 +0000 Subject: [PATCH 09/16] Update CMake to allow inclusion --- CMakeLists.txt | 76 +++++++++++++++++++++++----------------------- dx2/CMakeLists.txt | 3 +- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bcf5570..dd87a15 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,44 +68,44 @@ add_subdirectory(tests) # ####################################################################### # Installation -install( - TARGETS dx2 - EXPORT DX2Targets - LIBRARY DESTINATION lib +# install( +# TARGETS dx2 +# EXPORT DX2Targets +# LIBRARY DESTINATION lib - # FILE_SET HEADERS DESTINATION include/dx2 -) -install( - FILES - include/dx2/dx2.h - DESTINATION include/dx2 -) +# # FILE_SET HEADERS DESTINATION include/dx2 +# ) +# install( +# FILES +# include/dx2/dx2.h +# DESTINATION include/dx2 +# ) -install( - EXPORT DX2Targets - FILE DX2Targets.cmake - DESTINATION lib/cmake/DX2 -) +# install( +# EXPORT DX2Targets +# FILE DX2Targets.cmake +# DESTINATION lib/cmake/DX2 +# ) -include(CMakePackageConfigHelpers) -configure_package_config_file( - ${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in - "${CMAKE_CURRENT_BINARY_DIR}/DX2Config.cmake" - INSTALL_DESTINATION "lib/cmake/DX2" - NO_SET_AND_CHECK_MACRO - NO_CHECK_REQUIRED_COMPONENTS_MACRO -) -write_basic_package_version_file( - "${CMAKE_CURRENT_BINARY_DIR}/DX2ConfigVersion.cmake" - VERSION "${DX2_VERSION_MAJOR}.${DX2_VERSION_MINOR}" - COMPATIBILITY AnyNewerVersion -) -install( - FILES - ${CMAKE_CURRENT_BINARY_DIR}/DX2Config.cmake - ${CMAKE_CURRENT_BINARY_DIR}/DX2ConfigVersion.cmake - DESTINATION lib/cmake/DX2 -) -export(EXPORT DX2Targets - FILE "${CMAKE_CURRENT_BINARY_DIR}/DX2Targets.cmake" -) +# include(CMakePackageConfigHelpers) +# configure_package_config_file( +# ${CMAKE_CURRENT_SOURCE_DIR}/Config.cmake.in +# "${CMAKE_CURRENT_BINARY_DIR}/DX2Config.cmake" +# INSTALL_DESTINATION "lib/cmake/DX2" +# NO_SET_AND_CHECK_MACRO +# NO_CHECK_REQUIRED_COMPONENTS_MACRO +# ) +# write_basic_package_version_file( +# "${CMAKE_CURRENT_BINARY_DIR}/DX2ConfigVersion.cmake" +# VERSION "${DX2_VERSION_MAJOR}.${DX2_VERSION_MINOR}" +# COMPATIBILITY AnyNewerVersion +# ) +# install( +# FILES +# ${CMAKE_CURRENT_BINARY_DIR}/DX2Config.cmake +# ${CMAKE_CURRENT_BINARY_DIR}/DX2ConfigVersion.cmake +# DESTINATION lib/cmake/DX2 +# ) +# export(EXPORT DX2Targets +# FILE "${CMAKE_CURRENT_BINARY_DIR}/DX2Targets.cmake" +# ) diff --git a/dx2/CMakeLists.txt b/dx2/CMakeLists.txt index b5c3d73..b2a8921 100644 --- a/dx2/CMakeLists.txt +++ b/dx2/CMakeLists.txt @@ -4,7 +4,7 @@ add_library( ) target_link_libraries( dx2 - PRIVATE + PUBLIC Eigen3::Eigen nlohmann_json::nlohmann_json hdf5::hdf5 @@ -16,4 +16,3 @@ target_include_directories( $ $ ) -target_link_libraries(dx2 PUBLIC $) From 3ee666826d6e9f2b59302233a4ccaf6e7f0eaf83 Mon Sep 17 00:00:00 2001 From: Dimitri Vlachos Date: Wed, 29 Jan 2025 13:09:02 +0000 Subject: [PATCH 10/16] Update h5write to handle pre-existing datasets and add test cases --- include/dx2/h5/h5write.h | 47 ++++++++++++------ tests/test_write_h5_array.cxx | 93 +++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 14 deletions(-) diff --git a/include/dx2/h5/h5write.h b/include/dx2/h5/h5write.h index d41850c..2b1615e 100644 --- a/include/dx2/h5/h5write.h +++ b/include/dx2/h5/h5write.h @@ -201,20 +201,41 @@ void write_data_to_h5_file(const std::string &filename, // Flatten the data into a 1D vector auto flat_data = flatten(data); - // Create dataspace for the dataset - hid_t dataspace = H5Screate_simple(shape.size(), shape.data(), NULL); - if (dataspace < 0) { - throw std::runtime_error( - "Error: Unable to create dataspace for dataset: " + dataset_name); - } - - // Create the dataset - hid_t dataset = H5Dcreate(group, dataset_name.c_str(), H5T_NATIVE_DOUBLE, - dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + // Check if dataset exists + hid_t dataset = H5Dopen(group, dataset_name.c_str(), H5P_DEFAULT); if (dataset < 0) { + // Dataset does not exist, create it + hid_t dataspace = H5Screate_simple(shape.size(), shape.data(), NULL); + if (dataspace < 0) { + throw std::runtime_error( + "Error: Unable to create dataspace for dataset: " + dataset_name); + } + + dataset = H5Dcreate(group, dataset_name.c_str(), H5T_NATIVE_DOUBLE, + dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if (dataset < 0) { + H5Sclose(dataspace); + throw std::runtime_error("Error: Unable to create dataset: " + + dataset_name); + } + H5Sclose(dataspace); - throw std::runtime_error("Error: Unable to create dataset: " + - dataset_name); + } else { + // Dataset exists, check if the shape matches + hid_t existing_space = H5Dget_space(dataset); + int ndims = H5Sget_simple_extent_ndims(existing_space); + std::vector existing_dims(ndims); + H5Sget_simple_extent_dims(existing_space, existing_dims.data(), NULL); + H5Sclose(existing_space); + + if (existing_dims != shape) { + H5Dclose(dataset); + throw std::runtime_error( + "Error: Dataset shape mismatch. Cannot overwrite dataset: " + + dataset_name); + } + + // Dataset exists and has the correct shape, proceed to overwrite } // Write the data to the dataset @@ -222,14 +243,12 @@ void write_data_to_h5_file(const std::string &filename, H5P_DEFAULT, flat_data.data()); if (status < 0) { H5Dclose(dataset); - H5Sclose(dataspace); throw std::runtime_error("Error: Unable to write data to dataset: " + dataset_name); } // Cleanup resources H5Dclose(dataset); - H5Sclose(dataspace); H5Gclose(group); } catch (...) { H5Fclose(file); diff --git a/tests/test_write_h5_array.cxx b/tests/test_write_h5_array.cxx index 7e323ea..f85ecd1 100644 --- a/tests/test_write_h5_array.cxx +++ b/tests/test_write_h5_array.cxx @@ -253,4 +253,97 @@ TEST(WriteDataTests, WriteEmptyDataset) { // Clean up test file std::filesystem::remove(filename); } + +// Test writing to a file that already exists +TEST(WriteDataTests, WriteToExistingFile) { + std::string filename = "test_existing_file.h5"; + std::string dataset_path = "/existing_group/existing_dataset"; + + // Create the file first + hid_t file = + H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + ASSERT_GE(file, 0) << "Failed to create HDF5 file."; + H5Fclose(file); + + // Now attempt to write data into it + std::vector data = {1.0, 2.0, 3.0, 4.0}; + write_data_to_h5_file(filename, dataset_path, data); + + // Read back the data to verify it was written correctly + auto read_data = read_array_from_h5_file(filename, dataset_path); + EXPECT_EQ(data, read_data); + + // Clean up test file + std::filesystem::remove(filename); +} + +// Test writing to a group that already exists +TEST(WriteDataTests, WriteToExistingGroup) { + std::string filename = "test_existing_group.h5"; + std::string dataset_path = "/group_1/dataset_1"; + + // Create file and group manually + hid_t file = + H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + ASSERT_GE(file, 0) << "Failed to create HDF5 file."; + + hid_t group = + H5Gcreate(file, "/group_1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + ASSERT_GE(group, 0) << "Failed to create group."; + + H5Gclose(group); + H5Fclose(file); + + // Now attempt to write data into the existing group + std::vector data = {10.0, 20.0, 30.0, 40.0}; + write_data_to_h5_file(filename, dataset_path, data); + + // Read back the data to verify it was written correctly + auto read_data = read_array_from_h5_file(filename, dataset_path); + EXPECT_EQ(data, read_data); + + // Clean up test file + std::filesystem::remove(filename); +} + +// Test writing to a dataset that already exists +TEST(WriteDataTests, WriteToExistingDataset) { + std::string filename = "test_existing_dataset.h5"; + std::string dataset_path = "/group_2/dataset_existing"; + + // Create file and dataset manually + hid_t file = + H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + ASSERT_GE(file, 0) << "Failed to create HDF5 file."; + + hid_t group = + H5Gcreate(file, "/group_2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + ASSERT_GE(group, 0) << "Failed to create group."; + + hsize_t dims[1] = {4}; // 1D dataset with 4 elements + hid_t dataspace = H5Screate_simple(1, dims, NULL); + hid_t dataset = H5Dcreate(group, "dataset_existing", H5T_NATIVE_DOUBLE, + dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + ASSERT_GE(dataset, 0) << "Failed to create dataset."; + + std::vector initial_data = {1.1, 2.2, 3.3, 4.4}; + H5Dwrite(dataset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, + initial_data.data()); + + H5Dclose(dataset); + H5Sclose(dataspace); + H5Gclose(group); + H5Fclose(file); + + // Now attempt to overwrite the dataset + std::vector new_data = {5.5, 6.6, 7.7, 8.8}; + write_data_to_h5_file(filename, dataset_path, new_data); + + // Read back the data to verify it was overwritten correctly + auto read_data = read_array_from_h5_file(filename, dataset_path); + EXPECT_EQ(new_data, read_data); + + // Clean up test file + std::filesystem::remove(filename); +} #pragma endregion write_h5 tests \ No newline at end of file From 930bcc000eac31713a4962d6125ffdbe1df06117 Mon Sep 17 00:00:00 2001 From: James Beilsten-Edmands <30625594+jbeilstenedmands@users.noreply.github.com> Date: Fri, 31 Jan 2025 15:15:05 +0000 Subject: [PATCH 11/16] Suppress HDF5 stdout error logging to stdout --- include/dx2/h5/h5write.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/dx2/h5/h5write.h b/include/dx2/h5/h5write.h index 2b1615e..a254eed 100644 --- a/include/dx2/h5/h5write.h +++ b/include/dx2/h5/h5write.h @@ -47,6 +47,8 @@ hid_t traverse_or_create_groups(hid_t parent, const std::string &path) { (pos == std::string::npos) ? "" : cleaned_path.substr(pos + 1); // Attempt to open the group. If it does not exist, create it. + H5Eset_auto2(H5E_DEFAULT, NULL, NULL); // Suppress errors to stdout when + // trying to open a file/group that may not exist. hid_t next_group = H5Gopen(parent, group_name.c_str(), H5P_DEFAULT); if (next_group < 0) { next_group = H5Gcreate(parent, group_name.c_str(), H5P_DEFAULT, H5P_DEFAULT, @@ -172,6 +174,8 @@ void write_data_to_h5_file(const std::string &filename, const std::string &dataset_path, const Container &data) { // Open or create the HDF5 file + H5Eset_auto2(H5E_DEFAULT, NULL, NULL); // Suppress errors to stdout when + // trying to open a file/group that may not exist. hid_t file = H5Fopen(filename.c_str(), H5F_ACC_RDWR, H5P_DEFAULT); if (file < 0) { file = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); From 2bd2c0274fc4c79208d026b2060d54571a6b48e9 Mon Sep 17 00:00:00 2001 From: Dimitri Vlachos Date: Tue, 4 Feb 2025 18:03:45 +0000 Subject: [PATCH 12/16] Refactor HDF5 write tests to use test fixtures for better structure and cleanup --- tests/test_write_h5_array.cxx | 279 ++++++++++------------------------ 1 file changed, 81 insertions(+), 198 deletions(-) diff --git a/tests/test_write_h5_array.cxx b/tests/test_write_h5_array.cxx index f85ecd1..b2e84eb 100644 --- a/tests/test_write_h5_array.cxx +++ b/tests/test_write_h5_array.cxx @@ -7,28 +7,37 @@ #include #include +// Test fixture for HDF5-related tests +class HDF5Test : public ::testing::Test { +protected: + std::filesystem::path test_file_path; + + void SetUp() override { + test_file_path = std::filesystem::current_path() / "test_hdf5_file.h5"; + } + + void TearDown() override { + if (std::filesystem::exists(test_file_path)) { + std::filesystem::remove(test_file_path); + } + } +}; + +// --------------- traverse_or_create_groups TESTS --------------- #pragma region traverse_or_create_groups tests -// Test the traverse_or_create_groups function -TEST(HDF5Tests, TraverseOrCreateGroupsTest) { - // Define the test file path - std::filesystem::path cwd = std::filesystem::current_path(); - std::string test_file_path = cwd.generic_string(); - test_file_path.append("/data/test_traverse_or_create_groups.h5"); - - // Create or open an HDF5 file + +TEST_F(HDF5Test, TraverseOrCreateGroups) { + // Create an HDF5 file hid_t file = H5Fcreate(test_file_path.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); ASSERT_GE(file, 0) << "Failed to create HDF5 file."; try { - // Define a hierarchical path std::string group_path = "/dials/processing/group_0"; - - // Call the function to traverse or create groups hid_t final_group = traverse_or_create_groups(file, group_path); ASSERT_GE(final_group, 0) << "Failed to create or open the final group."; - // Verify that each group in the hierarchy exists + // Verify group hierarchy exists hid_t dials_group = H5Gopen(file, "dials", H5P_DEFAULT); ASSERT_GE(dials_group, 0) << "Failed to open the 'dials' group."; @@ -38,7 +47,7 @@ TEST(HDF5Tests, TraverseOrCreateGroupsTest) { hid_t group_0 = H5Gopen(processing_group, "group_0", H5P_DEFAULT); ASSERT_GE(group_0, 0) << "Failed to open the 'group_0' group."; - // Close all opened groups + // Close all groups H5Gclose(group_0); H5Gclose(processing_group); H5Gclose(dials_group); @@ -47,280 +56,157 @@ TEST(HDF5Tests, TraverseOrCreateGroupsTest) { FAIL() << "Runtime error occurred: " << e.what(); } - // Close the file + // Close and validate file existence H5Fclose(file); - - // Validate that the HDF5 file was created ASSERT_TRUE(std::filesystem::exists(test_file_path)) << "HDF5 file was not created."; - - // Clean up test file after successful run (comment out to keep the test file) - std::filesystem::remove(test_file_path); } -#pragma endregion traverse_or_create_groups tests +#pragma endregion traverse_or_create_groups tests +// --------------- deduce_shape TESTS --------------- #pragma region deduce_shape tests -// Test deducing shape for 1D container (std::vector) + TEST(DeduceShapeTests, OneDimensionalVector) { std::vector data = {1.0, 2.0, 3.0, 4.0}; - std::vector expected_shape = {4}; // Single dimension of size 4 - auto shape = deduce_shape(data); - EXPECT_EQ(shape, expected_shape); + EXPECT_EQ(deduce_shape(data), (std::vector{4})); } -// Test deducing shape for 2D container (std::vector) TEST(DeduceShapeTests, TwoDimensionalVector) { std::vector> data = {{1.0, 2.0, 3.0}, {4.0, 5.0, 6.0}}; - std::vector expected_shape = {2, 3}; // 2 rows, 3 columns - auto shape = deduce_shape(data); - EXPECT_EQ(shape, expected_shape); + EXPECT_EQ(deduce_shape(data), (std::vector{2, 3})); } -// Test deducing shape for 2D container (std::vector) TEST(DeduceShapeTests, TwoDimensionalVectorArray) { std::vector> data = { {1.0, 2.0, 3.0}, {4.0, 5.0, 6.0}, {7.0, 8.0, 9.0}}; - std::vector expected_shape = {3, 3}; // 3 rows, 3 columns - auto shape = deduce_shape(data); - EXPECT_EQ(shape, expected_shape); + EXPECT_EQ(deduce_shape(data), (std::vector{3, 3})); } -// Test deducing shape for 3D container (std::vector>) TEST(DeduceShapeTests, ThreeDimensionalVector) { std::vector>> data = {{{1, 2}, {3, 4}}, {{5, 6}, {7, 8}}}; - std::vector expected_shape = {2, 2, 2}; // 2x2x2 structure - auto shape = deduce_shape(data); - EXPECT_EQ(shape, expected_shape); + EXPECT_EQ(deduce_shape(data), (std::vector{2, 2, 2})); } -// Test deducing shape for empty container TEST(DeduceShapeTests, EmptyContainer) { std::vector> data = {}; - std::vector expected_shape = {0}; // Outer container size is 0 - auto shape = deduce_shape(data); - EXPECT_EQ(shape, expected_shape); + EXPECT_EQ(deduce_shape(data), (std::vector{0})); } -// Test deducing shape for mixed-sized inner containers (should throw or handle -// gracefully) TEST(DeduceShapeTests, MixedSizeInnerContainers) { - std::vector> data = { - {1.0, 2.0}, {3.0, 4.0, 5.0} // Different size - }; - - // deduce_shape assumes uniformity. If it encounters mixed sizes, it might - // throw or return the first subcontainer's size. - try { - auto shape = deduce_shape(data); - FAIL() << "Expected deduce_shape to throw due to inconsistent inner " - "container sizes."; - } catch (const std::exception &e) { - SUCCEED(); // deduce_shape threw an exception as expected - } catch (...) { - FAIL() << "Expected deduce_shape to throw a standard exception."; - } + std::vector> data = {{1.0, 2.0}, {3.0, 4.0, 5.0}}; + EXPECT_THROW(deduce_shape(data), std::exception); } -#pragma endregion deduce_shape tests +#pragma endregion deduce_shape tests +// --------------- flatten TESTS --------------- #pragma region flatten tests -// Test flattening a 1D vector + TEST(FlattenTests, OneDimensionalVector) { std::vector data = {1.0, 2.0, 3.0, 4.0}; - auto flat_data = flatten(data); // Should return the same as the input - - std::vector expected_flat_data = {1.0, 2.0, 3.0, 4.0}; - EXPECT_EQ(flat_data, expected_flat_data); + EXPECT_EQ(flatten(data), data); } -// Test flattening a 2D vector TEST(FlattenTests, TwoDimensionalVector) { std::vector> data = {{1.0, 2.0, 3.0}, {4.0, 5.0, 6.0}}; - auto flat_data = flatten(data); - - std::vector expected_flat_data = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; - EXPECT_EQ(flat_data, expected_flat_data); + EXPECT_EQ(flatten(data), (std::vector{1.0, 2.0, 3.0, 4.0, 5.0, 6.0})); } -// Test flattening a 2D vector of arrays TEST(FlattenTests, TwoDimensionalVectorArray) { std::vector> data = { {1.0, 2.0, 3.0}, {4.0, 5.0, 6.0}, {7.0, 8.0, 9.0}}; - auto flat_data = flatten(data); - - std::vector expected_flat_data = {1.0, 2.0, 3.0, 4.0, 5.0, - 6.0, 7.0, 8.0, 9.0}; - EXPECT_EQ(flat_data, expected_flat_data); + EXPECT_EQ(flatten(data), + (std::vector{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0})); } -// Test flattening a 3D vector TEST(FlattenTests, ThreeDimensionalVector) { std::vector>> data = {{{1, 2}, {3, 4}}, {{5, 6}, {7, 8}}}; - auto flat_data = flatten(data); - - std::vector expected_flat_data = {1, 2, 3, 4, 5, 6, 7, 8}; - EXPECT_EQ(flat_data, expected_flat_data); + EXPECT_EQ(flatten(data), (std::vector{1, 2, 3, 4, 5, 6, 7, 8})); } -// Test flattening an empty container TEST(FlattenTests, EmptyContainer) { std::vector> data = {}; - auto flat_data = flatten(data); - - std::vector expected_flat_data = {}; - EXPECT_EQ(flat_data, expected_flat_data); + EXPECT_EQ(flatten(data), (std::vector{})); } -// Test flattening a container with mixed sizes (shouldn't be an issue here) -TEST(FlattenTests, MixedSizeInnerContainers) { - std::vector> data = {{1.0, 2.0}, {3.0, 4.0, 5.0}}; - - // Flatten should concatenate all elements into a single 1D vector - auto flat_data = flatten(data); - std::vector expected_flat_data = {1.0, 2.0, 3.0, 4.0, 5.0}; - EXPECT_EQ(flat_data, expected_flat_data); -} #pragma endregion flatten tests - +// --------------- write_h5 TESTS --------------- #pragma region write_h5 tests -// Test writing a 1D vector to an HDF5 file -TEST(WriteDataTests, WriteOneDimensionalVector) { - std::string filename = "test_1d_vector.h5"; - std::string dataset_path = "/group_1/dataset_1d"; +TEST_F(HDF5Test, WriteOneDimensionalVector) { + std::string dataset_path = "/group_1/dataset_1d"; std::vector data = {1.0, 2.0, 3.0, 4.0}; - write_data_to_h5_file(filename, dataset_path, data); - - auto read_data = read_array_from_h5_file(filename, dataset_path); - EXPECT_EQ(data, read_data); - - // Clean up test file - std::filesystem::remove(filename); + write_data_to_h5_file(test_file_path, dataset_path, data); + EXPECT_EQ(read_array_from_h5_file(test_file_path, dataset_path), + data); } -// Test writing a 2D vector to an HDF5 file -TEST(WriteDataTests, WriteTwoDimensionalVector) { - std::string filename = "test_2d_vector.h5"; +TEST_F(HDF5Test, WriteTwoDimensionalVector) { std::string dataset_path = "/group_2/dataset_2d"; - std::vector> data = {{1.0, 2.0, 3.0}, {4.0, 5.0, 6.0}}; - write_data_to_h5_file(filename, dataset_path, data); - auto read_data = read_array_from_h5_file(filename, dataset_path); - - // Flatten the original data for comparison - std::vector expected_data = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; - EXPECT_EQ(expected_data, read_data); - - // Clean up test file - std::filesystem::remove(filename); -} - -// Test writing a 2D vector of arrays to an HDF5 file -TEST(WriteDataTests, WriteTwoDimensionalVectorArray) { - std::string filename = "test_2d_vector_array.h5"; - std::string dataset_path = "/group_3/dataset_2d_array"; - - std::vector> data = { - {1.0, 2.0, 3.0}, {4.0, 5.0, 6.0}, {7.0, 8.0, 9.0}}; - write_data_to_h5_file(filename, dataset_path, data); - - auto read_data = read_array_from_h5_file(filename, dataset_path); - - // Flatten the original data for comparison - std::vector expected_data = {1.0, 2.0, 3.0, 4.0, 5.0, - 6.0, 7.0, 8.0, 9.0}; - EXPECT_EQ(expected_data, read_data); - - // Clean up test file - std::filesystem::remove(filename); + write_data_to_h5_file(test_file_path, dataset_path, data); + EXPECT_EQ(read_array_from_h5_file(test_file_path, dataset_path), + flatten(data)); } -// Test writing an empty dataset to an HDF5 file -TEST(WriteDataTests, WriteEmptyDataset) { - std::string filename = "test_empty_dataset.h5"; +TEST_F(HDF5Test, WriteEmptyDataset) { std::string dataset_path = "/group_empty/dataset_empty"; + std::vector data = {}; - std::vector data = {}; // Empty dataset - write_data_to_h5_file(filename, dataset_path, data); - - auto read_data = read_array_from_h5_file(filename, dataset_path); - - EXPECT_TRUE(read_data.empty()); - - // Clean up test file - std::filesystem::remove(filename); + write_data_to_h5_file(test_file_path, dataset_path, data); + EXPECT_TRUE( + read_array_from_h5_file(test_file_path, dataset_path).empty()); } // Test writing to a file that already exists -TEST(WriteDataTests, WriteToExistingFile) { - std::string filename = "test_existing_file.h5"; - std::string dataset_path = "/existing_group/existing_dataset"; - - // Create the file first - hid_t file = - H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); +TEST_F(HDF5Test, WriteToExistingFile) { + hid_t file = H5Fcreate(test_file_path.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, + H5P_DEFAULT); ASSERT_GE(file, 0) << "Failed to create HDF5 file."; H5Fclose(file); - // Now attempt to write data into it + std::string dataset_path = "/existing_group/existing_dataset"; std::vector data = {1.0, 2.0, 3.0, 4.0}; - write_data_to_h5_file(filename, dataset_path, data); - // Read back the data to verify it was written correctly - auto read_data = read_array_from_h5_file(filename, dataset_path); - EXPECT_EQ(data, read_data); - - // Clean up test file - std::filesystem::remove(filename); + write_data_to_h5_file(test_file_path, dataset_path, data); + EXPECT_EQ(read_array_from_h5_file(test_file_path, dataset_path), + data); } // Test writing to a group that already exists -TEST(WriteDataTests, WriteToExistingGroup) { - std::string filename = "test_existing_group.h5"; - std::string dataset_path = "/group_1/dataset_1"; - - // Create file and group manually - hid_t file = - H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); +TEST_F(HDF5Test, WriteToExistingGroup) { + hid_t file = H5Fcreate(test_file_path.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, + H5P_DEFAULT); ASSERT_GE(file, 0) << "Failed to create HDF5 file."; hid_t group = H5Gcreate(file, "/group_1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); ASSERT_GE(group, 0) << "Failed to create group."; - H5Gclose(group); H5Fclose(file); - // Now attempt to write data into the existing group + std::string dataset_path = "/group_1/dataset_1"; std::vector data = {10.0, 20.0, 30.0, 40.0}; - write_data_to_h5_file(filename, dataset_path, data); - - // Read back the data to verify it was written correctly - auto read_data = read_array_from_h5_file(filename, dataset_path); - EXPECT_EQ(data, read_data); - // Clean up test file - std::filesystem::remove(filename); + write_data_to_h5_file(test_file_path, dataset_path, data); + EXPECT_EQ(read_array_from_h5_file(test_file_path, dataset_path), + data); } // Test writing to a dataset that already exists -TEST(WriteDataTests, WriteToExistingDataset) { - std::string filename = "test_existing_dataset.h5"; - std::string dataset_path = "/group_2/dataset_existing"; - - // Create file and dataset manually - hid_t file = - H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); +TEST_F(HDF5Test, WriteToExistingDataset) { + hid_t file = H5Fcreate(test_file_path.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, + H5P_DEFAULT); ASSERT_GE(file, 0) << "Failed to create HDF5 file."; hid_t group = H5Gcreate(file, "/group_2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); ASSERT_GE(group, 0) << "Failed to create group."; - hsize_t dims[1] = {4}; // 1D dataset with 4 elements + hsize_t dims[1] = {4}; hid_t dataspace = H5Screate_simple(1, dims, NULL); hid_t dataset = H5Dcreate(group, "dataset_existing", H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); @@ -337,13 +223,10 @@ TEST(WriteDataTests, WriteToExistingDataset) { // Now attempt to overwrite the dataset std::vector new_data = {5.5, 6.6, 7.7, 8.8}; - write_data_to_h5_file(filename, dataset_path, new_data); - - // Read back the data to verify it was overwritten correctly - auto read_data = read_array_from_h5_file(filename, dataset_path); - EXPECT_EQ(new_data, read_data); - - // Clean up test file - std::filesystem::remove(filename); + write_data_to_h5_file(test_file_path, "/group_2/dataset_existing", new_data); + EXPECT_EQ(read_array_from_h5_file(test_file_path, + "/group_2/dataset_existing"), + new_data); } -#pragma endregion write_h5 tests \ No newline at end of file + +#pragma endregion write_h5 tests From a32d6d1cda63a873e4ff6d2bab30417165a00468 Mon Sep 17 00:00:00 2001 From: Dimitri Vlachos Date: Wed, 5 Feb 2025 11:23:22 +0000 Subject: [PATCH 13/16] Add empty dataset for testing purposes --- tests/data/cut_strong.refl | Bin 15937 -> 16209 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/data/cut_strong.refl b/tests/data/cut_strong.refl index 4a751dacdaa6c4c2976853678a2c3847ec3a28f6..b5156651fd5bec78a428784fdffd87d2acce8967 100644 GIT binary patch delta 166 zcmX?DbFprM22-H@MlCBgsb`*x)0I~Tro-q5OiBzOz|1t+fmvhnYF6>dd)YWN*c3p5 zj4X^G3J7>pa|=o;<5Ln#5{pwy9PJ>Y3@H;gX0Yh0w(D<>WPf4C%E-XRz&LrGqV&XV e0+S1Mt Date: Tue, 4 Feb 2025 18:14:02 +0000 Subject: [PATCH 14/16] Refactor and expand HDF5 read tests --- tests/test_read_h5_array.cxx | 117 +++++++++++++++++++++++++++++++---- 1 file changed, 104 insertions(+), 13 deletions(-) diff --git a/tests/test_read_h5_array.cxx b/tests/test_read_h5_array.cxx index 995f60e..acef842 100644 --- a/tests/test_read_h5_array.cxx +++ b/tests/test_read_h5_array.cxx @@ -1,29 +1,120 @@ #include #include #include -#include +#include +#include +#include -// A test that we can read data arrays from a h5 processing file. +// Test fixture for HDF5 read tests +class HDF5ReadTest : public ::testing::Test { +protected: + std::filesystem::path test_file_path; -TEST(ExampleTests, ReadArrayTest) { + void SetUp() override { + // Set the test file path (assumes the tests directory as the working + // directory) + test_file_path = std::filesystem::current_path() / "data/cut_strong.refl"; + // Create the empty dataset if it does not exist + create_empty_dataset(test_file_path, "/dials/processing/empty_dataset"); + } + + void create_empty_dataset(const std::string &filename, + const std::string &dataset_path) { + // Open the HDF5 file + hid_t file = H5Fopen(filename.c_str(), H5F_ACC_RDWR, H5P_DEFAULT); + if (file < 0) { + throw std::runtime_error("Error: Unable to open file: " + filename); + } + + // Check if the dataset exists + hid_t dataset = H5Dopen(file, dataset_path.c_str(), H5P_DEFAULT); + if (dataset >= 0) { + // Dataset already exists, close and return + H5Dclose(dataset); + H5Fclose(file); + return; + } + + // Create the empty dataset + hsize_t dims[1] = {0}; // Zero elements + hid_t dataspace = H5Screate_simple(1, dims, NULL); + dataset = H5Dcreate(file, dataset_path.c_str(), H5T_NATIVE_DOUBLE, + dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if (dataset < 0) { + H5Sclose(dataspace); + H5Fclose(file); + throw std::runtime_error("Error: Unable to create empty dataset: " + + dataset_path); + } + + // Close handles + H5Dclose(dataset); + H5Sclose(dataspace); + H5Fclose(file); + } +}; + +// --------------- read_array_from_h5_file TESTS --------------- +#pragma region read_array_from_h5_file tests + +TEST_F(HDF5ReadTest, ReadDoubleArrayFromH5) { std::string array_name = "/dials/processing/group_0/xyzobs.px.value"; - std::string flags = "/dials/processing/group_0/flags"; - // The test has been given the tests directory as the working directory - // so we can get the path to the test file. - std::filesystem::path cwd = std::filesystem::current_path(); - std::string fpath = cwd.generic_string(); - fpath.append("/data/cut_strong.refl"); + // Read array from the test HDF5 file std::vector xyzobs_px = - read_array_from_h5_file(fpath, array_name); - // check a random value + read_array_from_h5_file(test_file_path, array_name); + + // Check a specific value double expected_value = 528.86470588235295; EXPECT_EQ(xyzobs_px[10], expected_value); +} + +TEST_F(HDF5ReadTest, ReadSizeTArrayFromH5) { + std::string flags_name = "/dials/processing/group_0/flags"; + // Read array from the test HDF5 file std::vector flags_array = - read_array_from_h5_file(fpath, flags); - // check a random value + read_array_from_h5_file(test_file_path, flags_name); + + // Check a specific value std::size_t expected_flag_value = 32; EXPECT_EQ(flags_array[5], expected_flag_value); } + +// Test reading from a non-existent file +TEST_F(HDF5ReadTest, ReadFromNonExistentFile) { + std::string invalid_file = "invalid_file.h5"; + std::string dataset_name = "/some/dataset"; + + EXPECT_THROW(read_array_from_h5_file(invalid_file, dataset_name), + std::runtime_error); +} + +// Test reading a non-existent dataset +TEST_F(HDF5ReadTest, ReadNonExistentDataset) { + std::string invalid_dataset = "/this/does/not/exist"; + + EXPECT_THROW(read_array_from_h5_file(test_file_path, invalid_dataset), + std::runtime_error); +} + +// Test reading an empty dataset +TEST_F(HDF5ReadTest, ReadEmptyDataset) { + std::string empty_dataset = "/dials/processing/empty_dataset"; + + std::vector result = + read_array_from_h5_file(test_file_path, empty_dataset); + EXPECT_TRUE(result.empty()) << "Expected an empty vector for empty dataset."; +} + +// Test data type mismatch +TEST_F(HDF5ReadTest, ReadWithIncorrectType) { + std::string dataset = "/dials/processing/group_0/xyzobs.px.value"; + + // Try to read a float dataset as int (should fail) + EXPECT_THROW(read_array_from_h5_file(test_file_path, dataset), + std::runtime_error); +} + +#pragma endregion read_array_from_h5_file tests From 7c675d7a7a12ffaab3bd14f64f6f09ca5910e793 Mon Sep 17 00:00:00 2001 From: Dimitri Vlachos Date: Wed, 5 Feb 2025 12:02:02 +0000 Subject: [PATCH 15/16] Update header file extensions from .h to .hpp --- include/dx2/h5/{h5read_processed.h => h5read_processed.hpp} | 0 include/dx2/h5/{h5write.h => h5write.hpp} | 0 tests/test_read_h5_array.cxx | 2 +- tests/test_write_h5_array.cxx | 4 ++-- 4 files changed, 3 insertions(+), 3 deletions(-) rename include/dx2/h5/{h5read_processed.h => h5read_processed.hpp} (100%) rename include/dx2/h5/{h5write.h => h5write.hpp} (100%) diff --git a/include/dx2/h5/h5read_processed.h b/include/dx2/h5/h5read_processed.hpp similarity index 100% rename from include/dx2/h5/h5read_processed.h rename to include/dx2/h5/h5read_processed.hpp diff --git a/include/dx2/h5/h5write.h b/include/dx2/h5/h5write.hpp similarity index 100% rename from include/dx2/h5/h5write.h rename to include/dx2/h5/h5write.hpp diff --git a/tests/test_read_h5_array.cxx b/tests/test_read_h5_array.cxx index acef842..cedecb3 100644 --- a/tests/test_read_h5_array.cxx +++ b/tests/test_read_h5_array.cxx @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/tests/test_write_h5_array.cxx b/tests/test_write_h5_array.cxx index b2e84eb..010aeb2 100644 --- a/tests/test_write_h5_array.cxx +++ b/tests/test_write_h5_array.cxx @@ -1,6 +1,6 @@ #include -#include -#include +#include +#include #include #include #include From b666dc516ce760f000cb2540e1977c567919b21d Mon Sep 17 00:00:00 2001 From: Dimitri Vlachos Date: Fri, 14 Feb 2025 17:13:46 +0000 Subject: [PATCH 16/16] Replace include guards with #pragma once in HDF5 read and write headers --- include/dx2/h5/h5read_processed.hpp | 5 +---- include/dx2/h5/h5write.hpp | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/include/dx2/h5/h5read_processed.hpp b/include/dx2/h5/h5read_processed.hpp index 628fa7e..2c21da5 100644 --- a/include/dx2/h5/h5read_processed.hpp +++ b/include/dx2/h5/h5read_processed.hpp @@ -1,5 +1,4 @@ -#ifndef DX2_H5_H5READ_PROCESSED_HPP -#define DX2_H5_H5READ_PROCESSED_HPP +#pragma once #include #include @@ -86,5 +85,3 @@ std::vector read_array_from_h5_file(const std::string &filename, throw; } } - -#endif // DX2_H5_H5READ_PROCESSED_HPP \ No newline at end of file diff --git a/include/dx2/h5/h5write.hpp b/include/dx2/h5/h5write.hpp index a254eed..98f5e85 100644 --- a/include/dx2/h5/h5write.hpp +++ b/include/dx2/h5/h5write.hpp @@ -1,5 +1,4 @@ -#ifndef H5WRITE_H -#define H5WRITE_H +#pragma once #include #include @@ -262,5 +261,3 @@ void write_data_to_h5_file(const std::string &filename, // Close the file H5Fclose(file); } - -#endif // H5WRITE_H