feat: add ebpf plugins

.devcontainer/devcontainer.json

@@ -6,6 +6,11 @@
     }
   },
   "initializeCommand": ".devcontainer/gen_env.sh",
+  "privileged": true,
+  "mounts": [
+    { "source": "/sys", "target": "/sys", "type": "bind" },
+    { "source": "/", "target": "/logtail_host", "type": "bind" }
+  ],
   "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ],
   "onCreateCommand": "sudo chown -R $(id -un):$(id -gn) /root",
   "customizations": {

.github/workflows/build-core-ut.yaml

@@ -83,10 +83,10 @@ jobs:
         run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage .
 
       - name: Unit Test
-        run: docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd) && make unittest_core"
+        run: docker run --privileged -v /sys:/sys -v /:/logtail_host -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd) && make unittest_core"
 
       - name: Unit Test Coverage
-        run: docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --gcov-ignore-errors=no_working_dir_found --root . --json coverage.json --json-summary-pretty --json-summary summary.json -e \".*\.pb\.cc\" -e \".*\.pb\.h\" -e \".*unittest.*\" -e \".*sdk.*\" -e \".*logger.*\" -e \".*config_server.*\" -e \".*go_pipeline.*\" -e \".*application.*\" -e \".*runner.*\""
+        run: docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --gcov-ignore-parse-errors --merge-mode-functions=separate --gcov-ignore-errors=no_working_dir_found --root . --json coverage.json --json-summary-pretty --json-summary summary.json -e \".*\.pb\.cc\" -e \".*\.pb\.h\" -e \".*unittest.*\" -e \".*sdk.*\" -e \".*logger.*\" -e \".*config_server.*\" -e \".*go_pipeline.*\" -e \".*application.*\" -e \".*runner.*\" -e \".*_thirdparty.*\""
 
       - name: Setup Python3.10
         uses: actions/setup-python@v5

.github/workflows/static-check.yaml

@@ -88,7 +88,7 @@ jobs:
         with:
           clang-format-version: '18'
           check-path: 'core'
-          exclude-regex: 'common/xxhash|labels/Relabel\.cpp|ProcessorParseContainerLogNative\.cpp|FlusherSLS\.cpp'
+          exclude-regex: 'common/xxhash|labels/Relabel\.cpp|ProcessorParseContainerLogNative\.cpp|FlusherSLS\.cpp|_thirdparty|core/common/LRUCache\.h|core/common/queue/.*|core/common/magic_enum\.hpp'
           include-regex: '.*\.(cpp|h)$'
 
       - name: Go Plugin Lint

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "core/_thirdparty/coolbpf"]
+	path = core/_thirdparty/coolbpf
+	url = https://gitee.com/anolis/coolbpf.git

.golangci.yml

@@ -31,6 +31,7 @@ run:
   # default value is empty list, but default dirs are skipped independently
   # from this option's value (see skip-dirs-use-default).
   skip-dirs:
+    - (^|/)core($|/)
     - (^|/)vendor($|/)
     - (^|/)external($|/)
     # todo delete or polish packages

core/CMakeLists.txt

@@ -92,7 +92,6 @@ endif ()
 # To be compatible with low version Linux.
 if (ENABLE_COMPATIBLE_MODE)
     message(STATUS "Enable compatible mode.")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c90")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--wrap=memcpy")
     add_definitions(-DENABLE_COMPATIBLE_MODE)
 endif ()
@@ -127,7 +126,7 @@ set(SUB_DIRECTORIES_LIST
         protobuf/sls protobuf/models
         file_server file_server/event file_server/event_handler file_server/event_listener file_server/reader file_server/polling
         prometheus prometheus/labels prometheus/schedulers prometheus/async prometheus/component
-        ebpf ebpf/observer ebpf/security ebpf/handler
+        ebpf ebpf/util ebpf/util/sampler ebpf/protocol/http ebpf/protocol ebpf/plugin/file_security ebpf/plugin/network_observer ebpf/plugin/process_security ebpf/plugin/network_security ebpf/plugin ebpf/observer ebpf/security
         parser
         )
 if (LINUX)
@@ -145,6 +144,7 @@ endif()
 
 # Module includes & set files.
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+include_directories("${DEPS_INCLUDE_ROOT}/coolbpf")
 
 foreach (DIR_NAME ${SUB_DIRECTORIES_LIST})
     include_directories(${CMAKE_CURRENT_SOURCE_DIR}/${DIR_NAME})
@@ -221,6 +221,9 @@ endif ()
 # Generate independent libraries.
 add_subdirectory(go_pipeline)
 add_subdirectory(common)
+# Build eBPF dependencies
+set(EBPF_DIRVER_TARGET "EbpfDriver")
+add_subdirectory(ebpf/driver)
 
 # Link libraries.
 if(BUILD_LOGTAIL OR BUILD_LOGTAIL_SHARED_LIBRARY)
@@ -230,6 +233,7 @@ if(BUILD_LOGTAIL OR BUILD_LOGTAIL_SHARED_LIBRARY)
     all_link(${LOGTAIL_TARGET})
     common_link(${LOGTAIL_TARGET})
     target_link_libraries(${LOGTAIL_TARGET} provider)
+    add_dependencies(${LOGTAIL_TARGET} install_coolbpf)
 endif()
 
 # Logtail UT.

core/app_config/AppConfig.cpp

@@ -166,6 +166,9 @@ DEFINE_FLAG_STRING(metrics_report_method,
 DEFINE_FLAG_STRING(operator_service, "loong collector operator service", "");
 DEFINE_FLAG_INT32(operator_service_port, "loong collector operator service port", 8888);
 DEFINE_FLAG_INT32(k8s_meta_service_port, "loong collector operator service port", 9000);
+
+DEFINE_FLAG_STRING(singleton_service, "loong collector singleton service", "loongcollector-singleton");
+DEFINE_FLAG_INT32(singleton_port, "loong collector singleton service port", 8899);
 DEFINE_FLAG_STRING(_pod_name_, "agent pod name", "");
 
 DEFINE_FLAG_STRING(app_info_file, "", "app_info.json");

core/collection_pipeline/serializer/SLSSerializer.cpp

@@ -18,6 +18,7 @@
 
 #include "json/json.h"
 
+#include "Logger.h"
 #include "collection_pipeline/serializer/JsonSerializer.h"
 #include "common/Flags.h"
 #include "common/compression/CompressType.h"
@@ -119,6 +120,19 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri
         case PipelineEvent::Type::METRIC: {
             for (size_t i = 0; i < group.mEvents.size(); ++i) {
                 const auto& e = group.mEvents[i].Cast<MetricEvent>();
+                if (SHOULD_LOG_DEBUG(sLogger)) {
+                    for (auto tag = e.TagsBegin(); tag != e.TagsEnd(); tag++) {
+                        LOG_DEBUG(
+                            sLogger,
+                            ("event tags for metricname", e.GetName().data())(tag->first.data(), tag->second.data()));
+                    }
+                    for (auto tag = group.mTags.mInner.begin(); tag != group.mTags.mInner.end(); tag++) {
+                        LOG_DEBUG(
+                            sLogger,
+                            ("group tags for metricname", e.GetName().data())(tag->first.data(), tag->second.data()));
+                    }
+                }
+
                 if (e.GetTimestamp() < 1e9) {
                     LOG_WARNING(sLogger,
                                 ("metric event timestamp is less than 1e9", "discard event")(
@@ -127,6 +141,8 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri
                 }
                 if (e.Is<UntypedSingleValue>()) {
                     metricEventContentCache[i].first = to_string(e.GetValue<UntypedSingleValue>()->mValue);
+                    // should not happen
+                    LOG_ERROR(sLogger, ("config", mFlusher->GetContext().GetConfigName())("metricname", e.GetName()));
                 } else {
                     // untyped multi value is not supported
                     LOG_WARNING(sLogger,
@@ -150,6 +166,18 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri
         case PipelineEvent::Type::SPAN:
             for (size_t i = 0; i < group.mEvents.size(); ++i) {
                 const auto& e = group.mEvents[i].Cast<SpanEvent>();
+                if (SHOULD_LOG_DEBUG(sLogger)) {
+                    for (auto tag = e.TagsBegin(); tag != e.TagsEnd(); tag++) {
+                        LOG_DEBUG(sLogger,
+                                  ("event tags for spanname", std::string(e.GetName()))(std::string(tag->first),
+                                                                                        std::string(tag->second)));
+                    }
+                    for (auto tag = group.mTags.mInner.begin(); tag != group.mTags.mInner.end(); tag++) {
+                        LOG_DEBUG(sLogger,
+                                  ("group tags for spanname", std::string(e.GetName()))(std::string(tag->first),
+                                                                                        std::string(tag->second)));
+                    }
+                }
                 size_t contentSZ = 0;
                 contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_TRACE_ID.size(), e.GetTraceId().size());
                 contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_SPAN_ID.size(), e.GetSpanId().size());

core/common/CapabilityUtil.cpp

@@ -0,0 +1,94 @@
+// Copyright 2023 iLogtail Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "CapabilityUtil.h"
+
+#include <array>
+#include <stdexcept>
+#include <string>
+
+namespace logtail {
+
+static const std::array<std::string, 41> CAPABILITY_STRINGS = {{"CAP_CHOWN",
+                                                                "DAC_OVERRIDE",
+                                                                "CAP_DAC_READ_SEARCH",
+                                                                "CAP_FOWNER",
+                                                                "CAP_FSETID",
+                                                                "CAP_KILL",
+                                                                "CAP_SETGID",
+                                                                "CAP_SETUID",
+                                                                "CAP_SETPCAP",
+                                                                "CAP_LINUX_IMMUTABLE",
+                                                                "CAP_NET_BIND_SERVICE",
+                                                                "CAP_NET_BROADCAST",
+                                                                "CAP_NET_ADMIN",
+                                                                "CAP_NET_RAW",
+                                                                "CAP_IPC_LOCK",
+                                                                "CAP_IPC_OWNER",
+                                                                "CAP_SYS_MODULE",
+                                                                "CAP_SYS_RAWIO",
+                                                                "CAP_SYS_CHROOT",
+                                                                "CAP_SYS_PTRACE",
+                                                                "CAP_SYS_PACCT",
+                                                                "CAP_SYS_ADMIN",
+                                                                "CAP_SYS_BOOT",
+                                                                "CAP_SYS_NICE",
+                                                                "CAP_SYS_RESOURCE",
+                                                                "CAP_SYS_TIME",
+                                                                "CAP_SYS_TTY_CONFIG",
+                                                                "CAP_MKNOD",
+                                                                "CAP_LEASE",
+                                                                "CAP_AUDIT_WRITE",
+                                                                "CAP_AUDIT_CONTROL",
+                                                                "CAP_SETFCAP",
+                                                                "CAP_MAC_OVERRIDE",
+                                                                "CAP_MAC_ADMIN",
+                                                                "CAP_SYSLOG",
+                                                                "CAP_WAKE_ALARM",
+                                                                "CAP_BLOCK_SUSPEND",
+                                                                "CAP_AUDIT_READ",
+                                                                "CAP_PERFMON",
+                                                                "CAP_BPF",
+                                                                "CAP_CHECKPOINT_RESTORE"}};
+
+static constexpr int32_t CAP_LAST_CAP = 40;
+
+const std::string& GetCapability(int32_t capInt) {
+    if (capInt < 0 || capInt > CAP_LAST_CAP) {
+        throw std::invalid_argument("invalid capability value " + std::to_string(capInt));
+    }
+    return CAPABILITY_STRINGS[capInt];
+}
+
+std::string GetCapabilities(uint64_t capInt) {
+    if (capInt == 0) {
+        return "";
+    }
+
+    std::string result;
+    result.reserve(CAP_LAST_CAP * 16);
+
+    for (uint64_t i = 0; i <= CAP_LAST_CAP; ++i) {
+        if ((1ULL << i) & capInt) {
+            if (!result.empty()) {
+                result.append(1, ' ');
+            }
+            result.append(CAPABILITY_STRINGS[i]);
+        }
+    }
+
+    return result;
+}
+
+} // namespace logtail

core/common/CapabilityUtil.h

@@ -0,0 +1,25 @@
+// Copyright 2023 iLogtail Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <string>
+
+namespace logtail {
+
+std::string GetCapabilities(uint64_t capInt);
+
+const std::string& GetCapability(int32_t capInt);
+
+} // namespace logtail