From 015453f4ac3e11bfebe6aa0cb29ddfa57627c491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Karczewski?= Date: Tue, 12 Nov 2024 14:12:12 +0100 Subject: [PATCH] Remove protobuf depenency Original motivation for this change was to remove unnecessary binary dependency on libatomic. As it appeared source of this dependency was protobuf. As this quite complex piece of software was used only to generate simple json I swiched it to boost::json. I decided to use boost::json because boost already was dependency for eBPF discovery. --- CMakeLists.txt | 4 +- conanfile.txt | 1 - ebpfdiscoverysrv/CMakeLists.txt | 4 + ebpfdiscoverysrv/test/dependencies.sh | 11 ++ libebpfdiscovery/CMakeLists.txt | 3 +- libebpfdiscovery/headers/ebpfdiscovery/Json.h | 73 +++++++ libebpfdiscovery/src/Discovery.cpp | 17 +- libebpfdiscovery/test/JsonTest.cpp | 186 ++++++++++++++++++ libebpfdiscoveryproto/CMakeLists.txt | 53 ----- .../ebpfdiscoveryproto/service.proto | 33 ---- .../headers/ebpfdiscoveryproto/Translator.h | 30 --- libebpfdiscoveryproto/src/Translator.cpp | 63 ------ libebpfdiscoveryproto/test/TranslatorTest.cpp | 92 --------- libservice/headers/service/Aggregator.h | 6 +- libservice/headers/service/Service.h | 40 +++- libservice/src/Aggregator.cpp | 37 ++-- libservice/test/AggregatorTest.cpp | 14 +- 17 files changed, 341 insertions(+), 326 deletions(-) create mode 100644 ebpfdiscoverysrv/test/dependencies.sh create mode 100644 libebpfdiscovery/headers/ebpfdiscovery/Json.h create mode 100644 libebpfdiscovery/test/JsonTest.cpp delete mode 100644 libebpfdiscoveryproto/CMakeLists.txt delete mode 100644 libebpfdiscoveryproto/ebpfdiscoveryproto/service.proto delete mode 100644 libebpfdiscoveryproto/headers/ebpfdiscoveryproto/Translator.h delete mode 100644 libebpfdiscoveryproto/src/Translator.cpp delete mode 100644 libebpfdiscoveryproto/test/TranslatorTest.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 22902c38..596bd433 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,7 +34,8 @@ if (CMAKE_BUILD_TYPE STREQUAL "Release") endif () if ("${PROJECT_VERSION}" STREQUAL "") - message(FATAL_ERROR "Project version has not been provided\n Please, provide project version using -DPROJECT_VERSION") + set(PROJECT_VERSION 0.0.0) + message(WARNING "Project version has not been provided so it is set to default value: ${PROJECT_VERSION}\n Please, provide project version using -DPROJECT_VERSION") endif () string(REGEX MATCH "([0-9]+)\.([0-9]+)\.([0-9]+)" _ "${PROJECT_VERSION}") @@ -126,7 +127,6 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/third_party/libbpf-boo add_subdirectory(third_party) add_subdirectory(libebpfdiscovery) -add_subdirectory(libebpfdiscoveryproto) add_subdirectory(libebpfdiscoveryshared) add_subdirectory(libebpfdiscoveryskel) add_subdirectory(libhttpparser) diff --git a/conanfile.txt b/conanfile.txt index 693eceb5..44db3821 100644 --- a/conanfile.txt +++ b/conanfile.txt @@ -1,5 +1,4 @@ [requires] -protobuf/3.21.9 ms-gsl/4.0.0 fmt/10.1.1 spdlog/1.12.0 diff --git a/ebpfdiscoverysrv/CMakeLists.txt b/ebpfdiscoverysrv/CMakeLists.txt index c8a0a270..303aacaf 100644 --- a/ebpfdiscoverysrv/CMakeLists.txt +++ b/ebpfdiscoverysrv/CMakeLists.txt @@ -20,3 +20,7 @@ add_executable(${TARGET} ${SOURCES}) target_link_libraries(${TARGET} PRIVATE ebpfdiscovery) target_link_options(${TARGET} PRIVATE "-static-libgcc" "-static-libstdc++") target_compile_definitions(${TARGET} PUBLIC PROJECT_VERSION="${PROJECT_VERSION}") + +if (BUILD_TESTS) + add_test(NAME binary_dependencies COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/test/dependencies.sh $) +endif () diff --git a/ebpfdiscoverysrv/test/dependencies.sh b/ebpfdiscoverysrv/test/dependencies.sh new file mode 100644 index 00000000..6e8a206e --- /dev/null +++ b/ebpfdiscoverysrv/test/dependencies.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +binary_path="${1}" +approved_binary_dependencies=("libelf|libz|libm|libdl|librt|libpthread|libc|ld-linux") + +depenencies=$(objdump -p "${binary_path}" 2>/dev/null | grep NEEDED | grep -Ev "${approved_binary_dependencies}") + +if [ ! -z "${depenencies}" ]; then + echo "Additional dependencies: ${depenencies}" + exit 1 +fi diff --git a/libebpfdiscovery/CMakeLists.txt b/libebpfdiscovery/CMakeLists.txt index 79b6db7d..f6b1f1a2 100644 --- a/libebpfdiscovery/CMakeLists.txt +++ b/libebpfdiscovery/CMakeLists.txt @@ -29,7 +29,6 @@ target_include_directories(${TARGET} PRIVATE src PUBLIC headers) target_link_libraries(${TARGET} PUBLIC libpf-tools - ebpfdiscoveryproto ebpfdiscoveryshared ebpfdiscoveryskel httpparser @@ -41,7 +40,7 @@ target_link_libraries(${TARGET} ) if (BUILD_TESTS) - list(APPEND TEST_SOURCES test/LRUCacheTest.cpp) + list(APPEND TEST_SOURCES test/LRUCacheTest.cpp test/JsonTest.cpp) set(TEST_TARGET test${TARGET}) add_executable(${TEST_TARGET} ${TEST_SOURCES}) diff --git a/libebpfdiscovery/headers/ebpfdiscovery/Json.h b/libebpfdiscovery/headers/ebpfdiscovery/Json.h new file mode 100644 index 00000000..97996ffb --- /dev/null +++ b/libebpfdiscovery/headers/ebpfdiscovery/Json.h @@ -0,0 +1,73 @@ +/* Copyright 2023 Dynatrace LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +namespace boost::json { +template +void tag_invoke(json::value_from_tag, json::value& v, std::reference_wrapper const& wrapped) { + v = json::value_from(wrapped.get()); +} +} // namespace boost::json + +namespace boost::json::ext { + +void print(std::ostream& os, boost::json::value const& jv) { + switch (jv.kind()) { + case json::kind::object: { + auto const& obj = jv.get_object(); + if (!obj.empty()) { + os << "{"; + for (auto it = obj.begin(); it != obj.end(); ++it) { + auto val = it->value(); + if (val.is_null() || (val.is_string() && (val.get_string().size() == 0))) { + continue; + } + if (it != obj.begin()) { + os << ","; + } + os << json::serialize(it->key()) << ":"; + print(os, val); + } + os << "}"; + } + break; + } + + case json::kind::array: { + auto const& arr = jv.get_array(); + if (!arr.empty()) { + os << "["; + auto it = arr.begin(); + for (auto it = arr.begin(); it != arr.end(); ++it) { + if (it != arr.begin()) { + os << ","; + } + print(os, *it); + } + } + os << "]"; + break; + } + default: + os << jv; + } +} +} // namespace boost::json::ext diff --git a/libebpfdiscovery/src/Discovery.cpp b/libebpfdiscovery/src/Discovery.cpp index 0d99e76b..8c52d695 100644 --- a/libebpfdiscovery/src/Discovery.cpp +++ b/libebpfdiscovery/src/Discovery.cpp @@ -17,7 +17,7 @@ #include "ebpfdiscovery/Discovery.h" #include "ebpfdiscovery/Session.h" -#include "ebpfdiscoveryproto/Translator.h" +#include "ebpfdiscovery/Json.h" #include "logging/Logger.h" #include "service/IpAddress.h" @@ -32,6 +32,7 @@ #include #include #include +#include namespace ebpfdiscovery { @@ -62,17 +63,9 @@ void Discovery::outputServicesToStdout() { return; } - ServicesList servicesProto{}; - bool isListEmpty = false; - { - std::lock_guard lock(serviceAggregator.getServicesMutex()); - std::tie(servicesProto, isListEmpty) = proto::internalToProto(services, serviceAggregator.getEnableNetworkCounters()); - } - if (!isListEmpty) { - const auto servicesJson{proto::protoToJson(servicesProto)}; - std::cout << servicesJson << std::endl; - } - + boost::json::object outJson{{"services", boost::json::value_from(services)}}; + boost::json::ext::print(std::cout, outJson); + serviceAggregator.clear(); } diff --git a/libebpfdiscovery/test/JsonTest.cpp b/libebpfdiscovery/test/JsonTest.cpp new file mode 100644 index 00000000..49e844ad --- /dev/null +++ b/libebpfdiscovery/test/JsonTest.cpp @@ -0,0 +1,186 @@ +/* + * Copyright 2023 Dynatrace LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ebpfdiscovery/Json.h" +#include + +#include "service/Service.h" +#include +#include +#include +#include + +#include + +class JsonTest : public testing::Test {}; + +struct testClass { + std::string str = "foo"; + std::string empty = ""; +}; + +BOOST_DESCRIBE_STRUCT(testClass, (), (str, empty)) + +bool is_parsable_back(std::string_view json_string) { + boost::system::error_code ec; + boost::json::parse(json_string, ec); + if (ec) { + return false; + } + return true; +} + +TEST_F(JsonTest, removeEmptyKeys) { + + std::vector vtc(4, {"bar", ""}); + boost::json::object json{{"key", boost::json::value_from(vtc)}}; + + std::stringstream result; + boost::json::ext::print(result, json); + + const std::string expected{"{\"key\":[{\"str\":\"bar\"},{\"str\":\"bar\"},{\"str\":\"bar\"},{\"str\":\"bar\"}]}"}; + + EXPECT_TRUE(is_parsable_back(result.str())); + EXPECT_EQ(result.str(), expected); +} + +TEST_F(JsonTest, servicesToJson) { + + std::vector> internalServices; + service::Service service1{.pid = 1, .endpoint = "/endpoint/1", .internalClientsNumber = 1, .externalClientsNumber = 2}; + service::Service service2{.pid = 2, .endpoint = "/endpoint/1", .internalClientsNumber = 1, .externalClientsNumber = 2}; + service::Service service3{.pid = 3, .endpoint = "/endpoint/2", .internalClientsNumber = 1, .externalClientsNumber = 2}; + + service::Service service4{ + .pid = 4, + .endpoint = "google.com/endpoint/3", + .domain = "google.com", + .scheme = "http", + .internalClientsNumber = 1, + .externalClientsNumber = 2}; + service::Service service5{ + .pid = 5, + .endpoint = "dynatrace.com/endpoint/4", + .domain = "dynatrace.com", + .scheme = "https", + .internalClientsNumber = 1, + .externalClientsNumber = 2}; + + internalServices.push_back(service1); + internalServices.push_back(service2); + internalServices.push_back(service3); + internalServices.push_back(service4); + internalServices.push_back(service5); + + boost::json::object outJson{{"service", boost::json::value_from(internalServices)}}; + + std::stringstream result; + boost::json::ext::print(result, outJson); + + // clang-format off + const std::string expected{"{\"service\":[" + "{\"pid\":1,\"endpoint\":\"/endpoint/1\",\"internalClientsNumber\":1,\"externalClientsNumber\":2}," + "{\"pid\":2,\"endpoint\":\"/endpoint/1\",\"internalClientsNumber\":1,\"externalClientsNumber\":2}," + "{\"pid\":3,\"endpoint\":\"/endpoint/2\",\"internalClientsNumber\":1,\"externalClientsNumber\":2}," + "{\"pid\":4,\"endpoint\":\"google.com/endpoint/3\",\"domain\":\"google.com\",\"scheme\":\"http\",\"internalClientsNumber\":1,\"externalClientsNumber\":2}," + "{\"pid\":5,\"endpoint\":\"dynatrace.com/endpoint/4\",\"domain\":\"dynatrace.com\",\"scheme\":\"https\",\"internalClientsNumber\":1,\"externalClientsNumber\":2}]}"}; + // clang-format on + EXPECT_TRUE(is_parsable_back(result.str())); + EXPECT_EQ(result.str(), expected); +} + +TEST_F(JsonTest, servicesToJsonNetworkCounters) { + std::unordered_map> detectedExternalIPv416Networks = { + {0xAC8F, std::chrono::steady_clock::now()}, {0xACC7, std::chrono::steady_clock::now()}}; + std::unordered_map> detectedExternalIPv424Networks = { + {0xAC8F04, std::chrono::steady_clock::now()}, + {0xAC8F06, std::chrono::steady_clock::now()}, + {0xACC72D, std::chrono::steady_clock::now()}}; + std::unordered_map< + std::array, + std::chrono::time_point, + service::ArrayHasher> + externalIPv6ClientsNets = { + {{0x20, 0x01, 0x48, 0x60, 0x48, 0x60}, std::chrono::steady_clock::now()}, + {{0x12, 0x34, 0x23, 0x45, 0x34, 0x56}, std::chrono::steady_clock::now()}}; + + std::vector> internalServices; + service::Service service1{ + .pid = 1, + .endpoint = "/endpoint/1", + .internalClientsNumber = 1, + .externalClientsNumber = 3, + .externalIPv4_16ClientNets = detectedExternalIPv416Networks, + .externalIPv4_24ClientNets = detectedExternalIPv424Networks}; + service::Service service2{ + .pid = 2, + .endpoint = "/endpoint/1", + .internalClientsNumber = 1, + .externalClientsNumber = 3, + .externalIPv4_16ClientNets = detectedExternalIPv416Networks, + .externalIPv4_24ClientNets = detectedExternalIPv424Networks}; + service::Service service3{ + .pid = 3, + .endpoint = "/endpoint/2", + .internalClientsNumber = 1, + .externalClientsNumber = 3, + .externalIPv4_16ClientNets = detectedExternalIPv416Networks, + .externalIPv4_24ClientNets = detectedExternalIPv424Networks}; + + service::Service service4{ + .pid = 4, + .endpoint = "google.com/endpoint/3", + .domain = "google.com", + .scheme = "http", + .internalClientsNumber = 1, + .externalClientsNumber = 2, + .externalIPv6ClientsNets = externalIPv6ClientsNets}; + service::Service service5{ + .pid = 5, + .endpoint = "dynatrace.com/endpoint/4", + .domain = "dynatrace.com", + .scheme = "https", + .internalClientsNumber = 1, + .externalClientsNumber = 2, + .externalIPv6ClientsNets = externalIPv6ClientsNets}; + + internalServices.emplace_back(service1); + internalServices.emplace_back(service2); + internalServices.emplace_back(service3); + internalServices.emplace_back(service4); + internalServices.emplace_back(service5); + + boost::json::object outJson{{"service", boost::json::value_from(internalServices)}}; + + std::stringstream result; + boost::json::ext::print(result, outJson); + const std::string expected{"{\"service\":[{\"pid\":1,\"endpoint\":\"/endpoint/" + "1\",\"internalClientsNumber\":1,\"externalClientsNumber\":3,\"externalIPv4_16ClientNets\":2,\"externalIPv4_" + "24ClientNets\":3},{\"pid\":2,\"endpoint\":\"/endpoint/" + "1\",\"internalClientsNumber\":1,\"externalClientsNumber\":3,\"externalIPv4_16ClientNets\":2,\"externalIPv4_" + "24ClientNets\":3},{\"pid\":3,\"endpoint\":\"/endpoint/" + "2\",\"internalClientsNumber\":1,\"externalClientsNumber\":3,\"externalIPv4_16ClientNets\":2,\"externalIPv4_" + "24ClientNets\":3},{\"pid\":4,\"endpoint\":\"google.com/" + "endpoint/" + "3\",\"domain\":\"google.com\",\"scheme\":\"http\",\"internalClientsNumber\":1,\"externalClientsNumber\":2," + "\"externalIPv6ClientsNets\":2},{\"pid\":5,\"endpoint\":\"dynatrace.com/" + "endpoint/" + "4\",\"domain\":\"dynatrace.com\",\"scheme\":\"https\",\"internalClientsNumber\":1," + "\"externalClientsNumber\":2,\"externalIPv6ClientsNets\":2}]}"}; + + EXPECT_TRUE(is_parsable_back(result.str())); + EXPECT_EQ(result.str(), expected); +} diff --git a/libebpfdiscoveryproto/CMakeLists.txt b/libebpfdiscoveryproto/CMakeLists.txt deleted file mode 100644 index d5db1976..00000000 --- a/libebpfdiscoveryproto/CMakeLists.txt +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2023 Dynatrace LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set(TARGET ebpfdiscoveryproto) - -list(APPEND SOURCES src/Translator.cpp) - -set(PROTO_FILES_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ebpfdiscoveryproto) - -file(GLOB PROTO_FILES "${PROTO_FILES_DIRECTORY}/*.proto") - -set(GENERATED_PROTO_FILES "") -foreach(PROTO_FILE ${PROTO_FILES}) - get_filename_component(PROTO_FILE_NAME ${PROTO_FILE} NAME_WE) - - set(PROTO_C_FILE "${CMAKE_CURRENT_BINARY_DIR}/ebpfdiscoveryproto/${PROTO_FILE_NAME}.pb.cc") - set(PROTO_H_FILE "${CMAKE_CURRENT_BINARY_DIR}/ebpfdiscoveryproto/${PROTO_FILE_NAME}.pb.h") - - add_custom_command( - OUTPUT ${PROTO_C_FILE} ${PROTO_H_FILE} - COMMAND ${CONAN_BIN_DIRS_PROTOBUF}/protoc - ARGS --cpp_out=${CMAKE_CURRENT_BINARY_DIR} --proto_path=${CMAKE_CURRENT_SOURCE_DIR} ${PROTO_FILE} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS ${PROTO_FILE} - ) - list(APPEND GENERATED_FILES ${PROTO_C_FILE}) -endforeach() - -add_library(${TARGET} STATIC ${SOURCES} ${GENERATED_FILES}) -target_link_libraries(${TARGET} CONAN_PKG::protobuf) -target_link_libraries(${TARGET} service) -target_include_directories(${TARGET} PUBLIC ${CMAKE_CURRENT_BINARY_DIR} headers) - -if (BUILD_TESTS) - list(APPEND TEST_SOURCES test/TranslatorTest.cpp) - set(TEST_TARGET test${TARGET}) - - add_executable(${TEST_TARGET} ${TEST_SOURCES}) - target_link_libraries(${TEST_TARGET} CONAN_PKG::gtest ${TARGET}) - target_include_directories(${TEST_TARGET} PRIVATE src) - gtest_discover_tests(${TEST_TARGET}) -endif () \ No newline at end of file diff --git a/libebpfdiscoveryproto/ebpfdiscoveryproto/service.proto b/libebpfdiscoveryproto/ebpfdiscoveryproto/service.proto deleted file mode 100644 index 238da0a0..00000000 --- a/libebpfdiscoveryproto/ebpfdiscoveryproto/service.proto +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 Dynatrace LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -syntax = "proto3"; - -message ServicesList { - repeated Service service = 1; -} - -message Service { - uint32 pid = 1; - string endpoint = 2; - string domain = 3; - string scheme = 4; - uint32 internalClientsNumber = 5; - uint32 externalClientsNumber = 6; - uint32 externalIPv4_16ClientNets = 7; - uint32 externalIPv4_24ClientNets = 8; - uint32 externalIPv6ClientsNets = 9; -} \ No newline at end of file diff --git a/libebpfdiscoveryproto/headers/ebpfdiscoveryproto/Translator.h b/libebpfdiscoveryproto/headers/ebpfdiscoveryproto/Translator.h deleted file mode 100644 index 8e82efc9..00000000 --- a/libebpfdiscoveryproto/headers/ebpfdiscoveryproto/Translator.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2023 Dynatrace LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include "ebpfdiscoveryproto/service.pb.h" -#include "service/Service.h" - -#include - -namespace proto { - -std::pair internalToProto(const std::vector>& services, const bool enableNetworkCounters); - -std::string protoToJson(const ServicesList& protoServices); - -} // namespace proto diff --git a/libebpfdiscoveryproto/src/Translator.cpp b/libebpfdiscoveryproto/src/Translator.cpp deleted file mode 100644 index 7a711ef1..00000000 --- a/libebpfdiscoveryproto/src/Translator.cpp +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2023 Dynatrace LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ebpfdiscoveryproto/Translator.h" - -#include - -namespace proto { - -std::pair internalToProto(const std::vector>& services, const bool enableNetworkCounters) { - ServicesList protoServicesList; - bool isListEmpty = true; - for (const auto& serviceRef : services) { - const auto& service{serviceRef.get()}; - - if (service.internalClientsNumber == 0 && service.externalClientsNumber == 0) { - continue; - } - - const auto protoService{protoServicesList.add_service()}; - - protoService->set_pid(service.pid); - protoService->set_endpoint(service.endpoint); - protoService->set_domain(service.domain); - protoService->set_scheme(service.scheme); - protoService->set_internalclientsnumber(service.internalClientsNumber); - protoService->set_externalclientsnumber(service.externalClientsNumber); - if (enableNetworkCounters) { - protoService->set_externalipv4_16clientnets(service.detectedExternalIPv4_16Networks.size()); - protoService->set_externalipv4_24clientnets(service.detectedExternalIPv4_24Networks.size()); - protoService->set_externalipv6clientsnets(service.detectedExternalIPv6Networks.size()); - } - - isListEmpty = false; - } - return {protoServicesList, isListEmpty}; -} - -std::string protoToJson(const ServicesList& protoServices) { - std::string json; - google::protobuf::util::JsonPrintOptions options; - options.add_whitespace = false; - options.always_print_primitive_fields = false; - options.preserve_proto_field_names = true; - - MessageToJsonString(protoServices, &json, options); - return json; -} - -} // namespace proto diff --git a/libebpfdiscoveryproto/test/TranslatorTest.cpp b/libebpfdiscoveryproto/test/TranslatorTest.cpp deleted file mode 100644 index a9ae6677..00000000 --- a/libebpfdiscoveryproto/test/TranslatorTest.cpp +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2023 Dynatrace LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ebpfdiscoveryproto/Translator.h" - -#include - -using namespace proto; - -class ProtobufTranslatorTest : public testing::Test {}; - -TEST_F(ProtobufTranslatorTest, successfulTranslationToJson) { - - std::vector> internalServices; - service::Service service1{.pid = 1, .endpoint = "/endpoint/1", .internalClientsNumber = 1, .externalClientsNumber = 2}; - service::Service service2{.pid = 2, .endpoint = "/endpoint/1", .internalClientsNumber = 1, .externalClientsNumber = 2}; - service::Service service3{.pid = 3, .endpoint = "/endpoint/2", .internalClientsNumber = 1, .externalClientsNumber = 2}; - - service::Service service4{.pid = 4, .endpoint = "google.com/endpoint/3", .domain = "google.com", .scheme = "http", .internalClientsNumber = 1, .externalClientsNumber = 2}; - service::Service service5{.pid = 5, .endpoint = "dynatrace.com/endpoint/4", .domain = "dynatrace.com", .scheme = "https", .internalClientsNumber = 1, .externalClientsNumber = 2}; - - internalServices.emplace_back(service1); - internalServices.emplace_back(service2); - internalServices.emplace_back(service3); - internalServices.emplace_back(service4); - internalServices.emplace_back(service5); - - const auto proto{internalToProto(internalServices, false)}; - ASSERT_FALSE(proto.second); - const auto result{protoToJson(proto.first)}; - const std::string expected{"{\"service\":[{\"pid\":1,\"endpoint\":\"/endpoint/" - "1\",\"internalClientsNumber\":1,\"externalClientsNumber\":2},{\"pid\":2,\"endpoint\":\"/endpoint/" - "1\",\"internalClientsNumber\":1,\"externalClientsNumber\":2},{\"pid\":3,\"endpoint\":\"/endpoint/" - "2\",\"internalClientsNumber\":1,\"externalClientsNumber\":2},{\"pid\":4,\"endpoint\":\"google.com/" - "endpoint/3\",\"domain\":\"google.com\",\"scheme\":\"http\",\"internalClientsNumber\":1,\"externalClientsNumber\":2},{\"pid\":5,\"endpoint\":\"dynatrace.com/" - "endpoint/4\",\"domain\":\"dynatrace.com\",\"scheme\":\"https\",\"internalClientsNumber\":1,\"externalClientsNumber\":2}]}"}; - EXPECT_EQ(result, expected); -} - -TEST_F(ProtobufTranslatorTest, successfulTranslationToJsonNetworkCounters) { - std::unordered_map> detectedExternalIPv416Networks = { - {0xAC8F, std::chrono::steady_clock::now()}, - {0xACC7, std::chrono::steady_clock::now()} - }; - std::unordered_map> detectedExternalIPv424Networks = { - {0xAC8F04, std::chrono::steady_clock::now()}, - {0xAC8F06, std::chrono::steady_clock::now()}, - {0xACC72D, std::chrono::steady_clock::now()} - }; - std::unordered_map, std::chrono::time_point, service::ArrayHasher> detectedExternalIPv6Networks = { - {{0x20, 0x01, 0x48, 0x60, 0x48, 0x60}, std::chrono::steady_clock::now()}, - {{0x12, 0x34, 0x23, 0x45, 0x34, 0x56}, std::chrono::steady_clock::now()} - }; - - std::vector> internalServices; - service::Service service1{.pid = 1, .endpoint = "/endpoint/1", .internalClientsNumber = 1, .externalClientsNumber = 3, .detectedExternalIPv4_16Networks = detectedExternalIPv416Networks, .detectedExternalIPv4_24Networks = detectedExternalIPv424Networks}; - service::Service service2{.pid = 2, .endpoint = "/endpoint/1", .internalClientsNumber = 1, .externalClientsNumber = 3, .detectedExternalIPv4_16Networks = detectedExternalIPv416Networks, .detectedExternalIPv4_24Networks = detectedExternalIPv424Networks}; - service::Service service3{.pid = 3, .endpoint = "/endpoint/2", .internalClientsNumber = 1, .externalClientsNumber = 3, .detectedExternalIPv4_16Networks = detectedExternalIPv416Networks, .detectedExternalIPv4_24Networks = detectedExternalIPv424Networks}; - - service::Service service4{.pid = 4, .endpoint = "google.com/endpoint/3", .domain = "google.com", .scheme = "http", .internalClientsNumber = 1, .externalClientsNumber = 2, .detectedExternalIPv6Networks = detectedExternalIPv6Networks}; - service::Service service5{.pid = 5, .endpoint = "dynatrace.com/endpoint/4", .domain = "dynatrace.com", .scheme = "https", .internalClientsNumber = 1, .externalClientsNumber = 2, .detectedExternalIPv6Networks = detectedExternalIPv6Networks}; - - internalServices.emplace_back(service1); - internalServices.emplace_back(service2); - internalServices.emplace_back(service3); - internalServices.emplace_back(service4); - internalServices.emplace_back(service5); - - const auto proto{internalToProto(internalServices, true)}; - ASSERT_FALSE(proto.second); - const auto result{protoToJson(proto.first)}; - const std::string expected{"{\"service\":[{\"pid\":1,\"endpoint\":\"/endpoint/" - "1\",\"internalClientsNumber\":1,\"externalClientsNumber\":3,\"externalIPv4_16ClientNets\":2,\"externalIPv4_24ClientNets\":3},{\"pid\":2,\"endpoint\":\"/endpoint/" - "1\",\"internalClientsNumber\":1,\"externalClientsNumber\":3,\"externalIPv4_16ClientNets\":2,\"externalIPv4_24ClientNets\":3},{\"pid\":3,\"endpoint\":\"/endpoint/" - "2\",\"internalClientsNumber\":1,\"externalClientsNumber\":3,\"externalIPv4_16ClientNets\":2,\"externalIPv4_24ClientNets\":3},{\"pid\":4,\"endpoint\":\"google.com/" - "endpoint/3\",\"domain\":\"google.com\",\"scheme\":\"http\",\"internalClientsNumber\":1,\"externalClientsNumber\":2,\"externalIPv6ClientsNets\":2},{\"pid\":5,\"endpoint\":\"dynatrace.com/" - "endpoint/4\",\"domain\":\"dynatrace.com\",\"scheme\":\"https\",\"internalClientsNumber\":1,\"externalClientsNumber\":2,\"externalIPv6ClientsNets\":2}]}"}; - EXPECT_EQ(result, expected); -} diff --git a/libservice/headers/service/Aggregator.h b/libservice/headers/service/Aggregator.h index c212b1d4..ed46c921 100644 --- a/libservice/headers/service/Aggregator.h +++ b/libservice/headers/service/Aggregator.h @@ -24,6 +24,7 @@ #include #include #include +#include template <> struct std::hash> { @@ -47,6 +48,7 @@ class Aggregator { private: using ServiceKey = std::pair; using ServiceStorage = std::unordered_map; + using ServicesList = std::vector>; public: Aggregator(const service::IpAddressChecker& ipChecker, bool _enableNetworkCounters); @@ -55,8 +57,6 @@ class Aggregator { void newRequest(const httpparser::HttpRequest& request, const DiscoverySessionMeta& meta); std::vector> collectServices(); void networkCountersCleaning(); - std::mutex& getServicesMutex(); - bool getEnableNetworkCounters() const; protected: virtual std::chrono::time_point getCurrentTime() const; @@ -64,7 +64,7 @@ class Aggregator { private: const IpAddressChecker& ipChecker; - std::mutex servicesMutex{}; + mutable std::mutex servicesMutex{}; ServiceStorage services; }; diff --git a/libservice/headers/service/Service.h b/libservice/headers/service/Service.h index 625ebdea..038cfeb9 100644 --- a/libservice/headers/service/Service.h +++ b/libservice/headers/service/Service.h @@ -22,6 +22,8 @@ #include #include #include +#include +#include namespace service { @@ -39,6 +41,10 @@ struct ArrayHasher { static constexpr uint8_t ipv6NetworkPrefixBytesLen = 6; struct Service { + + using IPv6NetworksMap_t = std::unordered_map, std::chrono::time_point, ArrayHasher>; + using IPv4NetworksMap_t = std::unordered_map>; + uint32_t pid; std::string endpoint; std::string domain; @@ -46,16 +52,38 @@ struct Service { uint32_t internalClientsNumber{0u}; uint32_t externalClientsNumber{0u}; - std::unordered_map> detectedExternalIPv4_16Networks; - std::unordered_map> detectedExternalIPv4_24Networks; - std::unordered_map, std::chrono::time_point, ArrayHasher> detectedExternalIPv6Networks; + IPv4NetworksMap_t externalIPv4_16ClientNets; + IPv4NetworksMap_t externalIPv4_24ClientNets; + IPv6NetworksMap_t externalIPv6ClientsNets; bool operator==(const Service& other) const { return pid == other.pid && endpoint == other.endpoint && domain == other.domain && scheme == other.scheme && internalClientsNumber == other.internalClientsNumber && externalClientsNumber == other.externalClientsNumber && - detectedExternalIPv4_16Networks == other.detectedExternalIPv4_16Networks && - detectedExternalIPv4_24Networks == other.detectedExternalIPv4_24Networks && detectedExternalIPv6Networks == other.detectedExternalIPv6Networks; + externalIPv4_16ClientNets == other.externalIPv4_16ClientNets && + externalIPv4_24ClientNets == other.externalIPv4_24ClientNets && externalIPv6ClientsNets == other.externalIPv6ClientsNets; } }; -} // namespace service \ No newline at end of file +BOOST_DESCRIBE_STRUCT(Service, (), (pid, endpoint, domain, scheme, internalClientsNumber, externalClientsNumber, externalIPv4_16ClientNets, externalIPv4_24ClientNets, externalIPv6ClientsNets)) + +} // namespace service + +namespace boost::json { + + inline void tag_invoke(value_from_tag, value& v, service::Service::IPv6NetworksMap_t map) { + if(map.empty()){ + v = value{}; + } else { + v = json::value_from(map.size()); + } + } + + + inline void tag_invoke(json::value_from_tag, json::value& v, service::Service::IPv4NetworksMap_t map) { + if(map.empty()){ + v = value{}; + } else { + v = json::value_from(map.size()); + } + } +} diff --git a/libservice/src/Aggregator.cpp b/libservice/src/Aggregator.cpp index 1dfd7dbb..c37bb9c5 100644 --- a/libservice/src/Aggregator.cpp +++ b/libservice/src/Aggregator.cpp @@ -91,13 +91,13 @@ static void incrementServiceClientsNumber( if (isIpv6) { std::array networkIPv6{}; std::memcpy(networkIPv6.data(), std::get(clientAddrBinary).s6_addr, service::ipv6NetworkPrefixBytesLen); - service.detectedExternalIPv6Networks[networkIPv6] = currentTime; + service.externalIPv6ClientsNets[networkIPv6] = currentTime; } else { uint32_t network24 = (std::get(clientAddrBinary).s_addr & 0xFFFFFF); - service.detectedExternalIPv4_24Networks[network24] = currentTime; + service.externalIPv4_24ClientNets[network24] = currentTime; uint32_t network16 = (std::get(clientAddrBinary).s_addr & 0xFFFF); - service.detectedExternalIPv4_16Networks[network16] = currentTime; + service.externalIPv4_16ClientNets[network16] = currentTime; } } catch (const std::bad_variant_access& e) { LOG_TRACE("Bad variant access during network counters processing: {} (client address: {})", e.what(), clientAddr); @@ -137,9 +137,9 @@ void Aggregator::clear() { std::lock_guard lock(servicesMutex); if (enableNetworkCounters) { for (auto it = services.begin(); it != services.end();) { - if (it->second.detectedExternalIPv4_16Networks.empty() && - it->second.detectedExternalIPv4_24Networks.empty() && - it->second.detectedExternalIPv6Networks.empty()) { + if (it->second.externalIPv4_16ClientNets.empty() && + it->second.externalIPv4_24ClientNets.empty() && + it->second.externalIPv6ClientsNets.empty()) { it = services.erase(it); } else { it->second.externalClientsNumber = 0; @@ -170,6 +170,7 @@ void Aggregator::newRequest(const httpparser::HttpRequest& request, const Discov std::vector> Aggregator::collectServices() { std::lock_guard lock(servicesMutex); std::vector> servicesVec; + servicesVec.reserve(services.size()); for (auto& pair : services) { @@ -183,38 +184,30 @@ void Aggregator::networkCountersCleaning() { std::lock_guard lock(servicesMutex); for (auto& service : services) { auto currentTime = getCurrentTime(); - for (auto detectedExternalIPv416NetworksIt = service.second.detectedExternalIPv4_16Networks.begin(); detectedExternalIPv416NetworksIt != service.second.detectedExternalIPv4_16Networks.end();) { + for (auto detectedExternalIPv416NetworksIt = service.second.externalIPv4_16ClientNets.begin(); detectedExternalIPv416NetworksIt != service.second.externalIPv4_16ClientNets.end();) { if (currentTime - detectedExternalIPv416NetworksIt->second >= retentionTime) { - detectedExternalIPv416NetworksIt = service.second.detectedExternalIPv4_16Networks.erase(detectedExternalIPv416NetworksIt); + detectedExternalIPv416NetworksIt = service.second.externalIPv4_16ClientNets.erase(detectedExternalIPv416NetworksIt); } else { ++detectedExternalIPv416NetworksIt; } } - for (auto detectedExternalIPv424NetworksIt = service.second.detectedExternalIPv4_24Networks.begin(); detectedExternalIPv424NetworksIt != service.second.detectedExternalIPv4_24Networks.end();) { + for (auto detectedExternalIPv424NetworksIt = service.second.externalIPv4_24ClientNets.begin(); detectedExternalIPv424NetworksIt != service.second.externalIPv4_24ClientNets.end();) { if (currentTime - detectedExternalIPv424NetworksIt->second >= retentionTime) { - detectedExternalIPv424NetworksIt = service.second.detectedExternalIPv4_24Networks.erase(detectedExternalIPv424NetworksIt); + detectedExternalIPv424NetworksIt = service.second.externalIPv4_24ClientNets.erase(detectedExternalIPv424NetworksIt); } else { ++detectedExternalIPv424NetworksIt; } } - for (auto detectedExternalIPv6NetworksIt = service.second.detectedExternalIPv6Networks.begin(); detectedExternalIPv6NetworksIt != service.second.detectedExternalIPv6Networks.end();) { - if (currentTime - detectedExternalIPv6NetworksIt->second >= retentionTime) { - detectedExternalIPv6NetworksIt = service.second.detectedExternalIPv6Networks.erase(detectedExternalIPv6NetworksIt); + for (auto externalIPv6ClientsNetsIt = service.second.externalIPv6ClientsNets.begin(); externalIPv6ClientsNetsIt != service.second.externalIPv6ClientsNets.end();) { + if (currentTime - externalIPv6ClientsNetsIt->second >= retentionTime) { + externalIPv6ClientsNetsIt = service.second.externalIPv6ClientsNets.erase(externalIPv6ClientsNetsIt); } else { - ++detectedExternalIPv6NetworksIt; + ++externalIPv6ClientsNetsIt; } } } } -std::mutex& Aggregator::getServicesMutex() { - return servicesMutex; -} - -bool Aggregator::getEnableNetworkCounters() const { - return enableNetworkCounters; -} - std::chrono::time_point Aggregator::getCurrentTime() const { return std::chrono::steady_clock::now(); } diff --git a/libservice/test/AggregatorTest.cpp b/libservice/test/AggregatorTest.cpp index ffd22d87..06be0c81 100644 --- a/libservice/test/AggregatorTest.cpp +++ b/libservice/test/AggregatorTest.cpp @@ -26,7 +26,7 @@ using namespace service; namespace service { void PrintTo(const Service& service, std::ostream* os) { *os << "(" << service.pid << ", " << service.endpoint << ", " << service.internalClientsNumber << ", " << service.externalClientsNumber - << ", " << service.detectedExternalIPv4_16Networks.size() << ", " << service.detectedExternalIPv4_24Networks.size() << ", " << service.detectedExternalIPv6Networks.size() + << ", " << service.externalIPv4_16ClientNets.size() << ", " << service.externalIPv4_24ClientNets.size() << ", " << service.externalIPv6ClientsNets.size() << ")"; } } // namespace service @@ -241,7 +241,7 @@ TEST_F(ServiceAggregatorTest, aggregateNetworkCounters) { {0x068FAC, std::chrono::time_point{}}, {0x2DC7AC, std::chrono::time_point{}} }; - const std::unordered_map, std::chrono::time_point, service::ArrayHasher> detectedExternalIPv6Networks = { + const std::unordered_map, std::chrono::time_point, service::ArrayHasher> externalIPv6ClientsNets = { {{0x20, 0x01, 0x48, 0x60, 0x48, 0x60}, std::chrono::time_point{}}, {{0x12, 0x34, 0x23, 0x45, 0x34, 0x56}, std::chrono::time_point{}} }; @@ -250,8 +250,8 @@ TEST_F(ServiceAggregatorTest, aggregateNetworkCounters) { auto services{aggregator.collectServices()}; EXPECT_EQ(services.size(), 2); - Service expectedService{.pid = 100, .endpoint{"host/url"}, .domain = "host", .scheme = "http", .internalClientsNumber = 0, .externalClientsNumber = 5, .detectedExternalIPv4_16Networks = detectedExternalIPv416Networks, .detectedExternalIPv4_24Networks = detectedExternalIPv424Networks, .detectedExternalIPv6Networks = detectedExternalIPv6Networks}; - Service expectedService2{.pid = 200, .endpoint{"host/url"}, .domain = "host", .scheme = "https", .internalClientsNumber = 0, .externalClientsNumber = 5, .detectedExternalIPv4_16Networks = detectedExternalIPv416Networks, .detectedExternalIPv4_24Networks = detectedExternalIPv424Networks, .detectedExternalIPv6Networks = detectedExternalIPv6Networks}; + Service expectedService{.pid = 100, .endpoint{"host/url"}, .domain = "host", .scheme = "http", .internalClientsNumber = 0, .externalClientsNumber = 5, .externalIPv4_16ClientNets = detectedExternalIPv416Networks, .externalIPv4_24ClientNets = detectedExternalIPv424Networks, .externalIPv6ClientsNets = externalIPv6ClientsNets}; + Service expectedService2{.pid = 200, .endpoint{"host/url"}, .domain = "host", .scheme = "https", .internalClientsNumber = 0, .externalClientsNumber = 5, .externalIPv4_16ClientNets = detectedExternalIPv416Networks, .externalIPv4_24ClientNets = detectedExternalIPv424Networks, .externalIPv6ClientsNets = externalIPv6ClientsNets}; std::vector servicesCopy; std::transform(services.begin(), services.end(), std::back_inserter(servicesCopy), [](const auto& ref) { return ref.get(); }); @@ -270,9 +270,9 @@ TEST_F(ServiceAggregatorTest, aggregateNetworkCounters) { for (const auto& service : collectedServices) { EXPECT_EQ(service.get().externalClientsNumber, 0); - EXPECT_EQ(service.get().detectedExternalIPv4_16Networks.size(), 2); - EXPECT_EQ(service.get().detectedExternalIPv4_24Networks.size(), 3); - EXPECT_EQ(service.get().detectedExternalIPv6Networks.size(), 2); + EXPECT_EQ(service.get().externalIPv4_16ClientNets.size(), 2); + EXPECT_EQ(service.get().externalIPv4_24ClientNets.size(), 3); + EXPECT_EQ(service.get().externalIPv6ClientsNets.size(), 2); } }