From c37df2318dd2a9fde6f256a9a861f4b281938a45 Mon Sep 17 00:00:00 2001 From: Punya Biswal Date: Tue, 10 Sep 2024 10:27:48 -0400 Subject: [PATCH] Implement in-memory metric exporter (#3043) Fixes #1405 --- CHANGELOG.md | 3 + exporters/memory/BUILD | 78 +++++++++++++++- exporters/memory/CMakeLists.txt | 38 ++++++++ .../exporters/memory/in_memory_metric_data.h | 72 ++++++++++++++ .../in_memory_metric_exporter_factory.h | 44 +++++++++ exporters/memory/src/in_memory_metric_data.cc | 54 +++++++++++ .../src/in_memory_metric_exporter_factory.cc | 93 +++++++++++++++++++ .../memory/test/in_memory_metric_data_test.cc | 55 +++++++++++ .../test/in_memory_metric_exporter_test.cc | 60 ++++++++++++ 9 files changed, 496 insertions(+), 1 deletion(-) create mode 100644 exporters/memory/include/opentelemetry/exporters/memory/in_memory_metric_data.h create mode 100644 exporters/memory/include/opentelemetry/exporters/memory/in_memory_metric_exporter_factory.h create mode 100644 exporters/memory/src/in_memory_metric_data.cc create mode 100644 exporters/memory/src/in_memory_metric_exporter_factory.cc create mode 100644 exporters/memory/test/in_memory_metric_data_test.cc create mode 100644 exporters/memory/test/in_memory_metric_exporter_test.cc diff --git a/CHANGELOG.md b/CHANGELOG.md index bce84c84df..66d60f3513 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,9 @@ Increment the: * [REMOVAL] Remove build option `WITH_DEPRECATED_SDK_FACTORY` [#2717](https://github.com/open-telemetry/opentelemetry-cpp/pull/2717) +* [EXPORTER] Add in-memory metric exporter + [#3043](https://github.com/open-telemetry/opentelemetry-cpp/pull/3043) + Breaking changes: * [REMOVAL] Remove build option `WITH_DEPRECATED_SDK_FACTORY` diff --git a/exporters/memory/BUILD b/exporters/memory/BUILD index a65066480c..033c85d369 100644 --- a/exporters/memory/BUILD +++ b/exporters/memory/BUILD @@ -4,14 +4,90 @@ package(default_visibility = ["//visibility:public"]) cc_library( - name = "in_memory_span_data", + name = "in_memory_metric_data", + srcs = [ + "src/in_memory_metric_data.cc", + ], + hdrs = [ + "include/opentelemetry/exporters/memory/in_memory_metric_data.h", + ], + strip_include_prefix = "include", + tags = [ + "memory", + "test", + ], + deps = [ + ":in_memory_data", + "//sdk/src/metrics", + ], +) + +cc_test( + name = "in_memory_metric_data_test", + srcs = ["test/in_memory_metric_data_test.cc"], + tags = [ + "memory", + "test", + ], + deps = [ + ":in_memory_metric_data", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "in_memory_metric_exporter_factory", + srcs = [ + "src/in_memory_metric_exporter_factory.cc", + ], + hdrs = [ + "include/opentelemetry/exporters/memory/in_memory_metric_exporter_factory.h", + ], + strip_include_prefix = "include", + tags = [ + "memory", + "test", + ], + deps = [ + ":in_memory_metric_data", + "//sdk/src/metrics", + ], +) + +cc_test( + name = "in_memory_metric_exporter_test", + srcs = ["test/in_memory_metric_exporter_test.cc"], + tags = [ + "memory", + "test", + ], + deps = [ + ":in_memory_metric_exporter_factory", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "in_memory_data", hdrs = [ "include/opentelemetry/exporters/memory/in_memory_data.h", + ], + strip_include_prefix = "include", + tags = ["memory"], + deps = [ + "//sdk:headers", + ], +) + +cc_library( + name = "in_memory_span_data", + hdrs = [ "include/opentelemetry/exporters/memory/in_memory_span_data.h", ], strip_include_prefix = "include", tags = ["memory"], deps = [ + ":in_memory_data", "//api", "//sdk/src/resource", "//sdk/src/trace", diff --git a/exporters/memory/CMakeLists.txt b/exporters/memory/CMakeLists.txt index 3dff044018..0738759a96 100644 --- a/exporters/memory/CMakeLists.txt +++ b/exporters/memory/CMakeLists.txt @@ -16,9 +16,26 @@ set_target_version(opentelemetry_exporter_in_memory) target_link_libraries(opentelemetry_exporter_in_memory PUBLIC opentelemetry_trace) +add_library( + opentelemetry_exporter_in_memory_metric + src/in_memory_metric_exporter_factory.cc src/in_memory_metric_data.cc) + +target_include_directories( + opentelemetry_exporter_in_memory_metric + PUBLIC "$" + "$") + +set_target_properties(opentelemetry_exporter_in_memory_metric + PROPERTIES EXPORT_NAME in_memory_metric_exporter) +set_target_version(opentelemetry_exporter_in_memory_metric) + +target_link_libraries(opentelemetry_exporter_in_memory_metric + PUBLIC opentelemetry_metrics) + if(OPENTELEMETRY_INSTALL) install( TARGETS opentelemetry_exporter_in_memory + opentelemetry_exporter_in_memory_metric EXPORT "${PROJECT_NAME}-target" RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} @@ -35,16 +52,29 @@ if(BUILD_TESTING) add_executable(in_memory_span_data_test test/in_memory_span_data_test.cc) add_executable(in_memory_span_exporter_test test/in_memory_span_exporter_test.cc) + add_executable(in_memory_metric_data_test test/in_memory_metric_data_test.cc) + add_executable(in_memory_metric_exporter_test + test/in_memory_metric_exporter_test.cc) target_link_libraries( in_memory_span_data_test ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} opentelemetry_exporter_in_memory opentelemetry_resources) + target_link_libraries( + in_memory_metric_data_test ${GTEST_BOTH_LIBRARIES} + ${CMAKE_THREAD_LIBS_INIT} opentelemetry_exporter_in_memory_metric + opentelemetry_resources) + target_link_libraries( in_memory_span_exporter_test ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} opentelemetry_exporter_in_memory opentelemetry_resources) + target_link_libraries( + in_memory_metric_exporter_test ${GTEST_BOTH_LIBRARIES} + ${CMAKE_THREAD_LIBS_INIT} opentelemetry_exporter_in_memory_metric + opentelemetry_resources) + gtest_add_tests( TARGET in_memory_span_data_test TEST_PREFIX exporter. @@ -53,4 +83,12 @@ if(BUILD_TESTING) TARGET in_memory_span_exporter_test TEST_PREFIX exporter. TEST_LIST in_memory_span_exporter_test) + gtest_add_tests( + TARGET in_memory_metric_data_test + TEST_PREFIX exporter. + TEST_LIST in_memory_metric_data_test) + gtest_add_tests( + TARGET in_memory_metric_exporter_test + TEST_PREFIX exporter. + TEST_LIST in_memory_metric_exporter_test) endif() diff --git a/exporters/memory/include/opentelemetry/exporters/memory/in_memory_metric_data.h b/exporters/memory/include/opentelemetry/exporters/memory/in_memory_metric_data.h new file mode 100644 index 0000000000..8d24c586b2 --- /dev/null +++ b/exporters/memory/include/opentelemetry/exporters/memory/in_memory_metric_data.h @@ -0,0 +1,72 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include + +#include "opentelemetry/exporters/memory/in_memory_data.h" +#include "opentelemetry/sdk/metrics/data/metric_data.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace sdk +{ +namespace metrics +{ +struct ResourceMetrics; +} +} // namespace sdk +namespace exporter +{ +namespace memory +{ + +/// The abstract base class for types used to store in-memory data backing an +/// InMemoryMetricExporter. +class InMemoryMetricData +{ +public: + InMemoryMetricData() = default; + virtual ~InMemoryMetricData() = default; + + InMemoryMetricData(const InMemoryMetricData &) = delete; + InMemoryMetricData(InMemoryMetricData &&) = delete; + InMemoryMetricData &operator=(const InMemoryMetricData &) = delete; + InMemoryMetricData &operator=(InMemoryMetricData &&) = delete; + + virtual void Add(std::unique_ptr resource_metrics) = 0; +}; + +/// An implementation of InMemoryMetricData that stores full-fidelity data points in a circular +/// buffer. This allows tests to inspect every aspect of exported data, in exchange for a somewhat +/// cumbersome API. +class CircularBufferInMemoryMetricData final : public InMemoryMetricData, + public InMemoryData +{ +public: + explicit CircularBufferInMemoryMetricData(size_t buffer_size); + void Add(std::unique_ptr resource_metrics) override; +}; + +/// An implementation of InMemoryMetricData that stores only the most recent data point in each time +/// series, and allows convenient lookups of time series. This makes simple tests easier to write. +class SimpleAggregateInMemoryMetricData final : public InMemoryMetricData +{ +public: + using AttributeToPoint = std::map; + + void Add(std::unique_ptr resource_metrics) override; + const AttributeToPoint &Get(const std::string &scope, const std::string &metric); + void Clear(); + +private: + std::map, AttributeToPoint> data_; +}; + +} // namespace memory +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/memory/include/opentelemetry/exporters/memory/in_memory_metric_exporter_factory.h b/exporters/memory/include/opentelemetry/exporters/memory/in_memory_metric_exporter_factory.h new file mode 100644 index 0000000000..f6934df727 --- /dev/null +++ b/exporters/memory/include/opentelemetry/exporters/memory/in_memory_metric_exporter_factory.h @@ -0,0 +1,44 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include "opentelemetry/sdk/metrics/instruments.h" +#include "opentelemetry/version.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace sdk +{ +namespace metrics +{ +class PushMetricExporter; +} // namespace metrics +} // namespace sdk +namespace exporter +{ +namespace memory +{ +class InMemoryMetricData; + +/// A factory for InMemoryMetricExporter +class InMemoryMetricExporterFactory +{ +public: + /// Create a InMemoryMetricExporter with a default buffer size and aggregation + /// temporality selector. + /// @param [out] data the InMemoryMetricData the exporter will write to, + /// for the caller to inspect + /// @param [in] buffer_size number of entries to save in the circular buffer + /// @param [in] temporality output temporality as a function of instrument kind + static std::unique_ptr Create( + const std::shared_ptr &data, + const sdk::metrics::AggregationTemporalitySelector &temporality); + + static std::unique_ptr Create( + const std::shared_ptr &data); +}; +} // namespace memory +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/memory/src/in_memory_metric_data.cc b/exporters/memory/src/in_memory_metric_data.cc new file mode 100644 index 0000000000..2a77e0b5a3 --- /dev/null +++ b/exporters/memory/src/in_memory_metric_data.cc @@ -0,0 +1,54 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/exporters/memory/in_memory_metric_data.h" +#include "opentelemetry/sdk/instrumentationscope/instrumentation_scope.h" +#include "opentelemetry/sdk/metrics/export/metric_producer.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace memory +{ +using sdk::metrics::ResourceMetrics; + +CircularBufferInMemoryMetricData::CircularBufferInMemoryMetricData(size_t buffer_size) + : InMemoryData(buffer_size) +{} + +void CircularBufferInMemoryMetricData::Add(std::unique_ptr resource_metrics) +{ + InMemoryData::Add(std::move(resource_metrics)); +} + +void SimpleAggregateInMemoryMetricData::Add(std::unique_ptr resource_metrics) +{ + for (const auto &sm : resource_metrics->scope_metric_data_) + { + const auto &scope = sm.scope_->GetName(); + for (const auto &m : sm.metric_data_) + { + const auto &metric = m.instrument_descriptor.name_; + for (const auto &pda : m.point_data_attr_) + { + data_[{scope, metric}].insert({pda.attributes, pda.point_data}); + } + } + } +} + +const SimpleAggregateInMemoryMetricData::AttributeToPoint &SimpleAggregateInMemoryMetricData::Get( + const std::string &scope, + const std::string &metric) +{ + return data_[{scope, metric}]; +} + +void SimpleAggregateInMemoryMetricData::Clear() +{ + data_.clear(); +} + +} // namespace memory +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/memory/src/in_memory_metric_exporter_factory.cc b/exporters/memory/src/in_memory_metric_exporter_factory.cc new file mode 100644 index 0000000000..f2577c4e9b --- /dev/null +++ b/exporters/memory/src/in_memory_metric_exporter_factory.cc @@ -0,0 +1,93 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/exporters/memory/in_memory_metric_exporter_factory.h" +#include "opentelemetry/exporters/memory/in_memory_metric_data.h" +#include "opentelemetry/sdk/common/global_log_handler.h" +#include "opentelemetry/sdk/metrics/export/metric_producer.h" +#include "opentelemetry/sdk/metrics/push_metric_exporter.h" + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace memory +{ + +using opentelemetry::sdk::metrics::PushMetricExporter; +using sdk::common::ExportResult; +using sdk::metrics::AggregationTemporality; +using sdk::metrics::AggregationTemporalitySelector; +using sdk::metrics::InstrumentType; +using sdk::metrics::ResourceMetrics; + +namespace +{ + +/// A Push Metric Exporter which accumulates metrics data in memory and allows it to be inspected. +/// It is not thread-safe. +class InMemoryMetricExporter final : public sdk::metrics::PushMetricExporter +{ +public: + /// @param buffer_size a required value that sets the size of the CircularBuffer + /// @param temporality Output temporality as a function of instrument kind. + InMemoryMetricExporter(const std::shared_ptr &data, + const sdk::metrics::AggregationTemporalitySelector &temporality) + : data_(data), temporality_(temporality) + {} + + ~InMemoryMetricExporter() override = default; + + InMemoryMetricExporter(const InMemoryMetricExporter &) = delete; + InMemoryMetricExporter(const InMemoryMetricExporter &&) = delete; + void operator=(const InMemoryMetricExporter &) = delete; + void operator=(const InMemoryMetricExporter &&) = delete; + + ExportResult Export(const ResourceMetrics &data) noexcept override + { + if (is_shutdown_) + { + OTEL_INTERNAL_LOG_ERROR("[In Memory Metric Exporter] Exporting failed, exporter is shutdown"); + return ExportResult::kFailure; + } + data_->Add(std::make_unique(data)); + return ExportResult::kSuccess; + } + + AggregationTemporality GetAggregationTemporality( + InstrumentType instrument_type) const noexcept override + { + return temporality_(instrument_type); + } + + bool ForceFlush(std::chrono::microseconds /* timeout */) noexcept override { return true; } + + bool Shutdown(std::chrono::microseconds /* timeout */) noexcept override + { + is_shutdown_ = true; + return true; + } + +private: + std::shared_ptr data_; + std::atomic is_shutdown_{false}; + sdk::metrics::AggregationTemporalitySelector temporality_; +}; + +} // namespace + +std::unique_ptr InMemoryMetricExporterFactory::Create( + const std::shared_ptr &data) +{ + return Create(data, [](auto) { return AggregationTemporality::kCumulative; }); +} + +std::unique_ptr InMemoryMetricExporterFactory::Create( + const std::shared_ptr &data, + const AggregationTemporalitySelector &temporality) +{ + return std::make_unique(data, temporality); +} + +} // namespace memory +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/memory/test/in_memory_metric_data_test.cc b/exporters/memory/test/in_memory_metric_data_test.cc new file mode 100644 index 0000000000..ffaba2cfb9 --- /dev/null +++ b/exporters/memory/test/in_memory_metric_data_test.cc @@ -0,0 +1,55 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/exporters/memory/in_memory_metric_data.h" +#include "opentelemetry/sdk/instrumentationscope/instrumentation_scope.h" +#include "opentelemetry/sdk/metrics/export/metric_producer.h" +#include "opentelemetry/sdk/resource/resource.h" + +#include + +#include + +using opentelemetry::exporter::memory::CircularBufferInMemoryMetricData; +using opentelemetry::exporter::memory::SimpleAggregateInMemoryMetricData; +using opentelemetry::sdk::metrics::MetricData; +using opentelemetry::sdk::metrics::PointDataAttributes; +using opentelemetry::sdk::metrics::ResourceMetrics; +using opentelemetry::sdk::metrics::ScopeMetrics; +using opentelemetry::sdk::metrics::SumPointData; +using opentelemetry::sdk::resource::Resource; + +TEST(InMemoryMetricDataTest, CircularBuffer) +{ + CircularBufferInMemoryMetricData buf(10); + Resource resource = Resource::GetEmpty(); + buf.Add(std::make_unique( + &resource, std::vector{{nullptr, std::vector{}}})); + EXPECT_EQ((*buf.Get().begin())->resource_, &resource); +} + +TEST(InMemoryMetricDataTest, SimpleAggregate) +{ + SimpleAggregateInMemoryMetricData agg; + + Resource resource = Resource::GetEmpty(); + + auto scope = opentelemetry::sdk::instrumentationscope::InstrumentationScope::Create( + "my-scope", "1.0.0", "http://example.com"); + + SumPointData spd; + spd.value_ = 42.0; + PointDataAttributes pda{{{"hello", "world"}}, spd}; + + MetricData md; + md.instrument_descriptor.name_ = "my-metric"; + md.point_data_attr_.push_back(pda); + + agg.Add(std::make_unique( + &resource, std::vector{{scope.get(), std::vector{md}}})); + auto it = agg.Get("my-scope", "my-metric").begin(); + + auto saved_point = opentelemetry::nostd::get(it->second); + + EXPECT_EQ(opentelemetry::nostd::get(saved_point.value_), 42.0); +} diff --git a/exporters/memory/test/in_memory_metric_exporter_test.cc b/exporters/memory/test/in_memory_metric_exporter_test.cc new file mode 100644 index 0000000000..adaa322f71 --- /dev/null +++ b/exporters/memory/test/in_memory_metric_exporter_test.cc @@ -0,0 +1,60 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "opentelemetry/exporters/memory/in_memory_metric_data.h" +#include "opentelemetry/exporters/memory/in_memory_metric_exporter_factory.h" +#include "opentelemetry/sdk/metrics/export/metric_producer.h" +#include "opentelemetry/sdk/metrics/instruments.h" +#include "opentelemetry/sdk/metrics/push_metric_exporter.h" +#include "opentelemetry/sdk/resource/resource.h" + +#include + +using opentelemetry::exporter::memory::CircularBufferInMemoryMetricData; +using opentelemetry::exporter::memory::InMemoryMetricExporterFactory; +using opentelemetry::sdk::common::ExportResult; +using opentelemetry::sdk::metrics::AggregationTemporality; +using opentelemetry::sdk::metrics::InstrumentType; +using opentelemetry::sdk::metrics::PushMetricExporter; +using opentelemetry::sdk::metrics::ResourceMetrics; +using opentelemetry::sdk::metrics::ScopeMetrics; +using opentelemetry::sdk::resource::Resource; + +class InMemoryMetricExporterTest : public ::testing::Test +{ +protected: + InMemoryMetricExporterTest() { exporter_ = InMemoryMetricExporterFactory::Create(data_); } + + std::unique_ptr exporter_; + std::shared_ptr data_ = + std::make_shared(10); + + Resource resource_ = Resource::GetEmpty(); + ResourceMetrics resource_metrics_{&resource_, std::vector{}}; +}; + +TEST_F(InMemoryMetricExporterTest, Export) +{ + EXPECT_EQ(exporter_->Export(resource_metrics_), ExportResult::kSuccess); + + auto data = data_->Get(); + EXPECT_EQ(data.size(), 1); + EXPECT_EQ((*data.begin())->resource_, &resource_); +} + +TEST_F(InMemoryMetricExporterTest, ForceFlush) +{ + EXPECT_TRUE(exporter_->ForceFlush()); +} + +TEST_F(InMemoryMetricExporterTest, Shutdown) +{ + EXPECT_TRUE(exporter_->Shutdown()); + EXPECT_EQ(exporter_->Export(resource_metrics_), ExportResult::kFailure); +} + +TEST_F(InMemoryMetricExporterTest, TemporalitySelector) +{ + EXPECT_EQ(exporter_->GetAggregationTemporality(InstrumentType::kCounter), + AggregationTemporality::kCumulative); +}