diff --git a/.gitignore b/.gitignore index 286e8dda5..8da0b83df 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ tests/unittests/Manifest.toml # Data Files *.root *.dat +*.sio # Spack build folders spack* diff --git a/include/podio/Reader.h b/include/podio/Reader.h new file mode 100644 index 000000000..e644a8238 --- /dev/null +++ b/include/podio/Reader.h @@ -0,0 +1,110 @@ +#ifndef PODIO_READER_H +#define PODIO_READER_H + +#include "podio/Frame.h" +#include "podio/podioVersion.h" + +namespace podio { + +class Reader { +public: + struct ReaderConcept { + virtual ~ReaderConcept() = default; + + virtual podio::Frame readNextFrame(const std::string& name) = 0; + virtual podio::Frame readFrame(const std::string& name, size_t index) = 0; + virtual size_t getEntries(const std::string& name) = 0; + virtual podio::version::Version currentFileVersion() const = 0; + virtual std::vector getAvailableCategories() const = 0; + virtual const std::string_view getDatamodelDefinition(const std::string& name) const = 0; + virtual std::vector getAvailableDatamodels() const = 0; + }; + + template + struct ReaderModel final : public ReaderConcept { + ReaderModel(std::unique_ptr reader) : m_reader(std::move(reader)) { + } + ReaderModel(const ReaderModel&) = delete; + ReaderModel& operator=(const ReaderModel&) = delete; + + podio::Frame readNextFrame(const std::string& name) override { + auto maybeFrame = m_reader->readNextEntry(name); + if (maybeFrame) { + return maybeFrame; + } + throw std::runtime_error("Failed reading category " + name + " (reading beyond bounds?)"); + } + + podio::Frame readFrame(const std::string& name, size_t index) override { + auto maybeFrame = m_reader->readEntry(name, index); + if (maybeFrame) { + return maybeFrame; + } + throw std::runtime_error("Failed reading category " + name + " at frame " + std::to_string(index) + + " (reading beyond bounds?)"); + } + size_t getEntries(const std::string& name) override { + return m_reader->getEntries(name); + } + podio::version::Version currentFileVersion() const override { + return m_reader->currentFileVersion(); + } + + std::vector getAvailableCategories() const override { + return m_reader->getAvailableCategories(); + } + + const std::string_view getDatamodelDefinition(const std::string& name) const override { + return m_reader->getDatamodelDefinition(name); + } + + std::vector getAvailableDatamodels() const override { + return m_reader->getAvailableDatamodels(); + } + + std::unique_ptr m_reader; + }; + + std::unique_ptr m_self{nullptr}; + + template + Reader(std::unique_ptr); + + podio::Frame readNextFrame(const std::string& name) { + return m_self->readNextFrame(name); + } + podio::Frame readNextEvent() { + return readNextFrame(podio::Category::Event); + } + podio::Frame readFrame(const std::string& name, size_t index) { + return m_self->readFrame(name, index); + } + podio::Frame readEvent(size_t index) { + return readFrame(podio::Category::Event, index); + } + size_t getEntries(const std::string& name) { + return m_self->getEntries(name); + } + size_t getEvents() { + return getEntries(podio::Category::Event); + } + podio::version::Version currentFileVersion() const { + return m_self->currentFileVersion(); + } + std::vector getAvailableCategories() const { + return m_self->getAvailableCategories(); + } + const std::string_view getDatamodelDefinition(const std::string& name) const { + return m_self->getDatamodelDefinition(name); + } + std::vector getAvailableDatamodels() const { + return m_self->getAvailableDatamodels(); + } +}; + +Reader makeReader(const std::string& filename); +Reader makeReader(const std::vector& filename); + +} // namespace podio + +#endif // PODIO_READER_H diff --git a/include/podio/Writer.h b/include/podio/Writer.h new file mode 100644 index 000000000..74c02025d --- /dev/null +++ b/include/podio/Writer.h @@ -0,0 +1,86 @@ +#ifndef PODIO_WRITER_H +#define PODIO_WRITER_H + +#include "podio/Frame.h" +#include "podio/podioVersion.h" + +namespace podio { + +class Writer { +public: + struct WriterConcept { + virtual ~WriterConcept() = default; + + virtual void writeFrame(const podio::Frame& frame, const std::string& category) = 0; + virtual void writeFrame(const podio::Frame& frame, const std::string& category, + const std::vector& collections) = 0; + virtual void writeEvent(const podio::Frame& frame) = 0; + virtual void writeEvent(const podio::Frame& frame, const std::vector& collections) = 0; + virtual void finish() = 0; + }; + + template + struct WriterModel final : public WriterConcept { + WriterModel(std::unique_ptr writer) : m_writer(std::move(writer)) { + } + WriterModel(const WriterModel&) = delete; + WriterModel& operator=(const WriterModel&) = delete; + WriterModel(WriterModel&&) = default; + WriterModel& operator=(WriterModel&&) = default; + + ~WriterModel() = default; + + void writeFrame(const podio::Frame& frame, const std::string& category) override { + return m_writer->writeFrame(frame, category); + } + void writeFrame(const podio::Frame& frame, const std::string& category, + const std::vector& collections) override { + return m_writer->writeFrame(frame, category, collections); + } + void writeEvent(const podio::Frame& frame) override { + return writeFrame(frame, podio::Category::Event); + } + void writeEvent(const podio::Frame& frame, const std::vector& collections) override { + return writeFrame(frame, podio::Category::Event, collections); + } + void finish() override { + return m_writer->finish(); + } + std::unique_ptr m_writer{nullptr}; + }; + + std::unique_ptr m_self{nullptr}; + + template + Writer(std::unique_ptr reader) : m_self(std::make_unique>(std::move(reader))) { + } + + Writer(const Writer&) = delete; + Writer& operator=(const Writer&) = delete; + Writer(Writer&&) = default; + Writer& operator=(Writer&&) = default; + + ~Writer() = default; + + void writeFrame(const podio::Frame& frame, const std::string& category) { + return m_self->writeFrame(frame, category); + } + void writeFrame(const podio::Frame& frame, const std::string& category, const std::vector& collections) { + return m_self->writeFrame(frame, category, collections); + } + void writeEvent(const podio::Frame& frame) { + return writeFrame(frame, podio::Category::Event); + } + void writeEvent(const podio::Frame& frame, const std::vector& collections) { + return writeFrame(frame, podio::Category::Event, collections); + } + void finish() { + return m_self->finish(); + } +}; + +Writer makeWriter(const std::string& filename, const std::string& type = "default"); + +} // namespace podio + +#endif // PODIO_WRITER_H diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1397f2e35..f129467d8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -135,8 +135,25 @@ if(ENABLE_SIO) LIST(APPEND INSTALL_LIBRARIES podioSioIO podioSioIODict) endif() +# --- IO +set(io_sources + Writer.cc + Reader.cc + ) + +set(io_headers + ${PROJECT_SOURCE_DIR}/include/podio/Writer.h + ${PROJECT_SOURCE_DIR}/include/podio/Reader.h + ) + +PODIO_ADD_LIB_AND_DICT(podioIO "${io_headers}" "${io_sources}" io_selection.xml) +target_link_libraries(podioIO PUBLIC podio::podio podio::podioRootIO) +if(ENABLE_SIO) + target_link_libraries(podioIO PUBLIC podio::podioSioIO) +endif() + # --- Install everything -install(TARGETS podio podioDict podioRootIO podioRootIODict ${INSTALL_LIBRARIES} +install(TARGETS podio podioDict podioRootIO podioRootIODict podioIO ${INSTALL_LIBRARIES} EXPORT podioTargets DESTINATION "${CMAKE_INSTALL_LIBDIR}") @@ -153,6 +170,8 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libpodioDict_rdict.pcm ${CMAKE_CURRENT_BINARY_DIR}/podioRootIODictDict.rootmap ${CMAKE_CURRENT_BINARY_DIR}/libpodioRootIODict_rdict.pcm + ${CMAKE_CURRENT_BINARY_DIR}/podioIODictDict.rootmap + ${CMAKE_CURRENT_BINARY_DIR}/libpodioIODict_rdict.pcm DESTINATION "${CMAKE_INSTALL_LIBDIR}") if (ENABLE_SIO) diff --git a/src/Reader.cc b/src/Reader.cc new file mode 100644 index 000000000..2686419b8 --- /dev/null +++ b/src/Reader.cc @@ -0,0 +1,75 @@ +#include "podio/Reader.h" + +#include "podio/ROOTReader.h" +#if PODIO_ENABLE_RNTUPLE + #include "podio/RNTupleReader.h" +#endif +#if PODIO_ENABLE_SIO + #include "podio/SIOReader.h" +#endif + +#include "TFile.h" +#include "TKey.h" +#include + +namespace podio { + +template +Reader::Reader(std::unique_ptr reader) : m_self(std::make_unique>(std::move(reader))) { +} + +Reader makeReader(const std::string& filename) { + return makeReader(std::vector{filename}); +} + +Reader makeReader(const std::vector& filenames) { + + auto suffix = filenames[0].substr(filenames[0].find_last_of(".") + 1); + for (size_t i = 1; i < filenames.size(); ++i) { + if (filenames[i].substr(filenames[i].find_last_of(".") + 1) != suffix) { + throw std::runtime_error("All files must have the same extension"); + } + } + + if (suffix == "root") { + TFile* file = TFile::Open(filenames[0].c_str()); + bool hasRNTuple = false; + + for (auto key : *file->GetListOfKeys()) { + auto tkey = dynamic_cast(key); + + if (tkey && std::string(tkey->GetClassName()) == "ROOT::Experimental::RNTuple") { + hasRNTuple = true; + break; + } + } + if (hasRNTuple) { +#if PODIO_ENABLE_RNTUPLE + auto actualReader = std::make_unique(); + actualReader->openFiles(filenames); + Reader reader{std::move(actualReader)}; + return reader; +#else + throw std::runtime_error("ROOT RNTuple reader not available. Please recompile with ROOT RNTuple support."); +#endif + } else { + auto actualReader = std::make_unique(); + actualReader->openFiles(filenames); + Reader reader{std::move(actualReader)}; + return reader; + } + } else if (suffix == "sio") { +#if PODIO_ENABLE_SIO + auto actualReader = std::make_unique(); + actualReader->openFile(filenames[0]); + Reader reader{std::move(actualReader)}; + return reader; +#else + throw std::runtime_error("SIO reader not available. Please recompile with SIO support."); +#endif + } + + throw std::runtime_error("Unknown file extension: " + suffix); +} + +} // namespace podio diff --git a/src/Writer.cc b/src/Writer.cc new file mode 100644 index 000000000..13f979ce8 --- /dev/null +++ b/src/Writer.cc @@ -0,0 +1,44 @@ +#include "podio/Writer.h" + +#include "podio/ROOTWriter.h" +#if PODIO_ENABLE_RNTUPLE + #include "podio/RNTupleWriter.h" +#endif +#if PODIO_ENABLE_SIO + #include "podio/SIOWriter.h" +#endif + +#include + +namespace podio { + +Writer makeWriter(const std::string& filename, const std::string& type) { + + auto endsWith = [](const std::string& str, const std::string& suffix) { + return str.size() >= suffix.size() && 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix); + }; + + auto lower = [](std::string str) { + std::transform(str.begin(), str.end(), str.begin(), [](unsigned char c) { return std::tolower(c); }); + return str; + }; + + if ((type == "default" && endsWith(filename, ".root")) || lower(type) == "root") { + return Writer{std::make_unique(filename)}; + } else if (lower(type) == "rntuple") { +#if PODIO_ENABLE_RNTUPLE + return Writer{std::make_unique(filename)}; +#else + throw std::runtime_error("ROOT RNTuple writer not available. Please recompile with ROOT RNTuple support."); +#endif + } else if (endsWith(filename, ".sio")) { +#if PODIO_ENABLE_SIO + return Writer{std::make_unique(filename)}; +#else + throw std::runtime_error("SIO writer not available. Please recompile with SIO support."); +#endif + } + throw std::runtime_error("Unknown file type for file " + filename + " with type " + type); +} + +} // namespace podio diff --git a/src/io_selection.xml b/src/io_selection.xml new file mode 100644 index 000000000..ffeeb5eef --- /dev/null +++ b/src/io_selection.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/tests/CTestCustom.cmake b/tests/CTestCustom.cmake index 396c32b78..3c660068a 100644 --- a/tests/CTestCustom.cmake +++ b/tests/CTestCustom.cmake @@ -22,6 +22,9 @@ if ((NOT "@FORCE_RUN_ALL_TESTS@" STREQUAL "ON") AND (NOT "@USE_SANITIZER@" STREQ write_frame_root read_frame_root + write_interface_root + read_interface_root + write_python_frame_sio read_python_frame_sio @@ -71,13 +74,12 @@ if ((NOT "@FORCE_RUN_ALL_TESTS@" STREQUAL "ON") AND (NOT "@USE_SANITIZER@" STREQ set(CTEST_CUSTOM_TESTS_IGNORE ${CTEST_CUSTOM_TESTS_IGNORE} - write_sio read_sio read_and_write_sio write_timed_sio read_timed_sio - write_frame_sio read_frame_sio + read_interface_sio read_frame_legacy_sio read_and_write_frame_sio ) diff --git a/tests/read_interface.h b/tests/read_interface.h new file mode 100644 index 000000000..278e03742 --- /dev/null +++ b/tests/read_interface.h @@ -0,0 +1,87 @@ +#ifndef PODIO_TESTS_READ_INTERFACE_H // NOLINT(llvm-header-guard): folder structure not suitable +#define PODIO_TESTS_READ_INTERFACE_H // NOLINT(llvm-header-guard): folder structure not suitable + +#include "read_frame.h" + +#include "podio/Reader.h" + +int read_frames(podio::Reader& reader) { + + if (reader.getEntries(podio::Category::Event) != 10) { + std::cerr << "Could not read back the number of events correctly. " + << "(expected:" << 10 << ", actual: " << reader.getEntries(podio::Category::Event) << ")" << std::endl; + return 1; + } + + if (reader.getEntries(podio::Category::Event) != reader.getEntries("other_events")) { + std::cerr << "Could not read back the number of events correctly. " + << "(expected:" << 10 << ", actual: " << reader.getEntries("other_events") << ")" << std::endl; + return 1; + } + + // Read the frames in a different order than when writing them here to make + // sure that the writing/reading order does not impose any usage requirements + for (size_t i = 0; i < reader.getEntries(podio::Category::Event); ++i) { + auto frame = reader.readNextFrame(podio::Category::Event); + if (frame.get("emptySubsetColl") == nullptr) { + std::cerr << "Could not retrieve an empty subset collection" << std::endl; + return 1; + } + if (frame.get("emptyCollection") == nullptr) { + std::cerr << "Could not retrieve an empty collection" << std::endl; + return 1; + } + + processEvent(frame, i, reader.currentFileVersion()); + + auto otherFrame = reader.readNextFrame("other_events"); + processEvent(otherFrame, i + 100, reader.currentFileVersion()); + // The other_events category also holds external collections + processExtensions(otherFrame, i + 100, reader.currentFileVersion()); + // As well as a test for the vector members subset category + checkVecMemSubsetColl(otherFrame); + } + + // if (reader.readNextFrame(podio::Category::Event)) { + // std::cerr << "Trying to read more frame data than is present should return a nullptr" << std::endl; + // return 1; + // } + + std::cout << "========================================================\n" << std::endl; + // if (reader.readNextFrame("not_present")) { + // std::cerr << "Trying to read non-existant frame data should return a nullptr" << std::endl; + // return 1; + // } + + // Reading specific (jumping to) entry + { + auto frame = reader.readFrame(podio::Category::Event, 4); + processEvent(frame, 4, reader.currentFileVersion()); + // Reading the next entry after jump, continues from after the jump + auto nextFrame = reader.readNextFrame(podio::Category::Event); + processEvent(nextFrame, 5, reader.currentFileVersion()); + + auto otherFrame = reader.readFrame("other_events", 4); + processEvent(otherFrame, 4 + 100, reader.currentFileVersion()); + if (reader.currentFileVersion() > podio::version::Version{0, 16, 2}) { + processExtensions(otherFrame, 4 + 100, reader.currentFileVersion()); + } + + // Jumping back also works + auto previousFrame = reader.readFrame("other_events", 2); + processEvent(previousFrame, 2 + 100, reader.currentFileVersion()); + if (reader.currentFileVersion() > podio::version::Version{0, 16, 2}) { + processExtensions(previousFrame, 2 + 100, reader.currentFileVersion()); + } + + // Trying to read a Frame that is not present returns a nullptr + // if (reader.readFrame(podio::Category::Event, 10)) { + // std::cerr << "Trying to read a specific entry that does not exist should return a nullptr" << std::endl; + // return 1; + // } + } + + return 0; +} + +#endif // PODIO_TESTS_READ_INTERFACE_H diff --git a/tests/root_io/CMakeLists.txt b/tests/root_io/CMakeLists.txt index 1bc906755..6a4069a48 100644 --- a/tests/root_io/CMakeLists.txt +++ b/tests/root_io/CMakeLists.txt @@ -6,6 +6,8 @@ set(root_dependent_tests read_python_frame_root.cpp read_frame_root_multiple.cpp read_and_write_frame_root.cpp + write_interface_root.cpp + read_interface_root.cpp ) if(ENABLE_RNTUPLE) set(root_dependent_tests @@ -13,13 +15,17 @@ if(ENABLE_RNTUPLE) write_rntuple.cpp read_rntuple.cpp read_python_frame_rntuple.cpp + write_interface_rntuple.cpp + read_interface_rntuple.cpp ) endif() -set(root_libs TestDataModelDict ExtensionDataModelDict podio::podioRootIO) +set(root_libs TestDataModelDict ExtensionDataModelDict podio::podioRootIO podio::podioIO) foreach( sourcefile ${root_dependent_tests} ) CREATE_PODIO_TEST(${sourcefile} "${root_libs}") endforeach() +set_property(TEST read_interface_root PROPERTY DEPENDS write_interface_root) + set_tests_properties( read_frame_root read_frame_root_multiple @@ -31,6 +37,7 @@ set_tests_properties( if(ENABLE_RNTUPLE) set_property(TEST read_rntuple PROPERTY DEPENDS write_rntuple) + set_property(TEST read_interface_rntuple PROPERTY DEPENDS write_interface_rntuple) endif() add_executable(read_frame_legacy_root read_frame_legacy_root.cpp) diff --git a/tests/root_io/read_interface_rntuple.cpp b/tests/root_io/read_interface_rntuple.cpp new file mode 100644 index 000000000..e2ee2a421 --- /dev/null +++ b/tests/root_io/read_interface_rntuple.cpp @@ -0,0 +1,10 @@ +#include "read_interface.h" + +int main(int, char**) { + auto reader = podio::makeReader("example_from_rntuple_interface.root"); + if (read_frames(reader)) { + return 1; + } + + return 0; +} diff --git a/tests/root_io/read_interface_root.cpp b/tests/root_io/read_interface_root.cpp new file mode 100644 index 000000000..8d33ec1c3 --- /dev/null +++ b/tests/root_io/read_interface_root.cpp @@ -0,0 +1,11 @@ +#include "read_interface.h" + +int main(int, char**) { + + auto reader = podio::makeReader("example_frame_interface.root"); + if (read_frames(reader)) { + return 1; + } + + return 0; +} diff --git a/tests/root_io/write_interface_rntuple.cpp b/tests/root_io/write_interface_rntuple.cpp new file mode 100644 index 000000000..c13658c61 --- /dev/null +++ b/tests/root_io/write_interface_rntuple.cpp @@ -0,0 +1,8 @@ +#include "write_interface.h" + +int main(int, char**) { + auto writer = podio::makeWriter("example_from_rntuple_interface.root", "rntuple"); + write_frames(writer); + + return 0; +} diff --git a/tests/root_io/write_interface_root.cpp b/tests/root_io/write_interface_root.cpp new file mode 100644 index 000000000..84775e6de --- /dev/null +++ b/tests/root_io/write_interface_root.cpp @@ -0,0 +1,9 @@ +#include "write_interface.h" + +int main(int, char**) { + + auto writer = podio::makeWriter("example_frame_interface.root"); + write_frames(writer); + + return 0; +} diff --git a/tests/sio_io/CMakeLists.txt b/tests/sio_io/CMakeLists.txt index b3987ba5a..7102c36c7 100644 --- a/tests/sio_io/CMakeLists.txt +++ b/tests/sio_io/CMakeLists.txt @@ -3,8 +3,10 @@ set(sio_dependent_tests write_frame_sio.cpp read_and_write_frame_sio.cpp read_python_frame_sio.cpp + write_interface_sio.cpp + read_interface_sio.cpp ) -set(sio_libs podio::podioSioIO) +set(sio_libs podio::podioSioIO podio::podioIO) foreach( sourcefile ${sio_dependent_tests} ) CREATE_PODIO_TEST(${sourcefile} "${sio_libs}") endforeach() @@ -18,6 +20,8 @@ set_tests_properties( write_frame_sio ) +set_property(TEST read_interface_sio PROPERTY DEPENDS write_interface_sio) + #--- Write via python and the SIO backend and see if we can read it back in in #--- c++ add_test(NAME write_python_frame_sio COMMAND python3 ${PROJECT_SOURCE_DIR}/tests/write_frame.py example_frame_with_py.sio sio_io.Writer) diff --git a/tests/sio_io/read_interface_sio.cpp b/tests/sio_io/read_interface_sio.cpp new file mode 100644 index 000000000..e411f2a96 --- /dev/null +++ b/tests/sio_io/read_interface_sio.cpp @@ -0,0 +1,11 @@ +#include "read_interface.h" + +int main(int, char**) { + + auto readerSIO = podio::makeReader("example_frame_sio_interface.sio"); + if (read_frames(readerSIO)) { + return 1; + } + + return 0; +} diff --git a/tests/sio_io/write_interface_sio.cpp b/tests/sio_io/write_interface_sio.cpp new file mode 100644 index 000000000..9def5db18 --- /dev/null +++ b/tests/sio_io/write_interface_sio.cpp @@ -0,0 +1,9 @@ +#include "write_interface.h" + +int main(int, char**) { + + auto writerSIO = podio::makeWriter("example_frame_sio_interface.sio", "sio"); + write_frames(writerSIO); + + return 0; +} diff --git a/tests/write_interface.h b/tests/write_interface.h new file mode 100644 index 000000000..1acaa2383 --- /dev/null +++ b/tests/write_interface.h @@ -0,0 +1,21 @@ +#ifndef PODIO_TESTS_WRITE_INTERFACE_H // NOLINT(llvm-header-guard): folder structure not suitable +#define PODIO_TESTS_WRITE_INTERFACE_H // NOLINT(llvm-header-guard): folder structure not suitable + +#include "write_frame.h" + +#include "podio/Writer.h" + +void write_frames(podio::Writer& frameWriter) { + + for (int i = 0; i < 10; ++i) { + auto frame = makeFrame(i); + frameWriter.writeFrame(frame, podio::Category::Event, collsToWrite); + } + + for (int i = 100; i < 110; ++i) { + auto frame = makeFrame(i); + frameWriter.writeFrame(frame, "other_events"); + } +} + +#endif // PODIO_TESTS_WRITE_INTERFACE_H