Merge opensource main

本次代码评审主要涉及添加新的依赖、引入新的变量、修改锁的使用、增加和修改函数功能、调整配置处理逻辑、更新测试用例以适应代码结构和逻辑的变化，以及一系列的代码优化和重构，特别是增加了对配置管理和指标统计的处理，加强了模块间的解耦和代码的可维护性。
Link: https://code.alibaba-inc.com/sls/ilogtail/codereview/18705573
* add setpipelineforitems for process queues (#1769)

* add metric for runner (#1766)

* fix deadlock in full drain mode on config update (#1776)

* remove InputStream (#1777)

* fix: do not use gtid if gtid is disabled (#1781)

Fixed an issue where the input_canal plugin used GTID as the sync method even when gtid_enabled was set to false. This led to syncing from the beginning due to receiving an empty GTID set. Syncing from the beginning is usually undesirable as it consumes significant CPU resources on the server side and deviates from the default plugin behavior. This commit ensures that the plugin respects the gtid_enabled switch and the gtid_mode query result.

* fix: gometric v2 not started and shared extensions (#1782)

* fix metrics v2 type assertion and shared extension initialiazation

* restore go mod

* test: calculate incremental unittest coverage (#1783)

* fix: given more time to fix ut failed of flusher_prometheus (#1784)

* Flusher  supports new metricstore endpoint, which optimizes time series data storage and query (#1731)

* rename self monitor metrics (#1779)

* c++ part

* go part

* fix go bug

* rename cpp metrics

* rename go metrics

* change parameters name

* delete unused metrics

* change name

* delete node id

* change name

* fix build

* change name

* 1. support honor_timestamps, 2. support honor_labels, apply global mTags to MetricEvent in relabel phase 3. optimization relabel, manage MetricEvent tags directly (#1742)

* update: remove enable-compression config temporarily (#1785)

* update: check pipeline queue before scrape, try again after 1 second if is not valid to push (#1757)

* Add a default content key for the service_kafka plugin (#1754)

* add K8s meta self metrics (#1765)

* support k8s meta to collect more resources

* feat: reload pipeline config independently (#1713)

* feat: reload Go pipeline config independently

* fix unittest

* fix unittest

* fix

* fix

* fix

* fix

* fix

* unittest

* fix

* fix

* fix

* fix

* fix

* self telemetry

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* test: calculate incremental unittest coverage

* fix

* fix

* fix

* fix

* fix conflict

* fix log

* feat: prometheus self monitor metrics (#1796)

* feat:iLogtail renamed to LoongCollector and directory layout changes (#1791)

```plaintext
/
└── /opt/loongcollector/
                       ├── loongcollector
                       ├── libPluginAdapter.so
                       ├── libPluginBase.so
                       ├── ca-bundle.crt
                       ├── plugins
                       │      └── custom plugins
                       ├── dump (used exclusively by the service_http_server input plugin)
                       ├── thirdparty
                       │      ├── jvm
                       │      └── telegraf
                       ├── conf/
                       │      ├── scripts
                       │      ├── apsara_log_conf.json
                       │      ├── plugin_logger.xml
                       │      ├── user_defined_id
                       │      ├── authorization.json
                       │      ├── pipelineconfig/
                       │      │                 ├── local/
                       │      │                 │         └── collect_stdout.json
                       │      │                 └── remote/
                       │      │                           └── collect_file.json
                       │      └── instanceconfig/
                       │                        ├── local/
                       │                        │         ├── ebpf.json
                       │                        │         └── loongcollector_config.json
                       │                        └── remote/
                       │                                  ├── region.json
                       │                                  └── resource.json
                       ├── data/
                       │       ├── file_check_point
                       │       ├── exactly_once_checkpoint/
                       │       ├── go_plugin_checkpoint/
                       │       ├── docker_path_config.json
                       │       ├── send_buffer_file_xxxxxxxxxxxx
                       │       └── backtrace.dat
                       ├── log/
                       │       ├── loongcollector.log
                       │       ├── loongcollector.log.1
                       │       ├── go_plugin.log
                       │       ├── go_plugin.log.1
                       │       ├── logger_initialization.log
                       │       └── snapshot/
                       └── run/
                               ├── loongcollector.pid
                               ├── inotify_watcher_dirs
                               └── app_info.json
```

* Support concurrency isolation between different sender queues (#1786)

* Merge remote-tracking branch 'upstream/main' into HEAD

* Internal code adaptation to ConcurrencyLimiter changes

.devcontainer/Dockerfile

@@ -29,15 +29,15 @@ RUN wget http://mirrors.ustc.edu.cn/gnu/libc/glibc-2.18.tar.gz && \
     rm -fr glibc-2.18*
 
 # install python3.8
-RUN cd /opt && curl -O https://cdn.npmmirror.com/binaries/python/3.8.12/Python-3.8.12.tgz && \
+RUN cd /opt && curl -O https://mirrors.aliyun.com/python-release/source/Python-3.8.12.tgz && \
     tar -zxvf Python-3.8.12.tgz && cd Python-3.8.12 && \
     mkdir /usr/local/python3 && \
     ./configure --prefix=/usr/local/python3 && \
     make clean && make && make install && \
     cp /usr/local/python3/bin/python3.8 /usr/bin/python3
 # install gcovr
 RUN python3 -m pip install --upgrade pip
-RUN cp /usr/local/python3/bin/pip3 /usr/bin/pip3 && pip3 install gcovr==7.0
+RUN cp /usr/local/python3/bin/pip3 /usr/bin/pip3 && pip3 config set global.index-url https://mirrors.aliyun.com/pypi/simple/ && pip3 install gcovr==7.0
 RUN cp /usr/local/python3/bin/gcovr /usr/bin/gcovr
 
 # Create the user

.github/workflows/build-core-ut.yaml

@@ -82,15 +82,15 @@ jobs:
         run: make unittest_core
 
       - name: Unit Test Coverage
-        run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage . && docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --root . --lcov coverage.lcov --txt coverage.txt -e \".*sdk.*\" -e \".*observer.*\" -e \".*protobuf.*\" -e \".*unittest.*\" -e \".*config_server.*\" -e \".*fuse.*\" -e \".*go_pipeline.*\""
+        run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage . && docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --root . --json coverage.json --json-summary-pretty --json-summary summary.json -e \".*sdk.*\" -e \".*observer.*\" -e \".*logger.*\" -e \".*unittest.*\" -e \".*config_server.*\" -e \".*go_pipeline.*\" -e \".*application.*\" -e \".*protobuf.*\" -e \".*runner.*\""
 
       - name: Setup Python3.10
         uses: actions/setup-python@v5
         with:
           python-version: "3.10"
 
       - name: Report code coverage
-        run: python3 tools/coverage-diff/main.py core/coverage.txt
+        run: python3 tools/coverage-diff/main.py --path core/coverage.json --summary core/summary.json
 
   result:
     runs-on: arc-runner-set-ilogtail

.github/workflows/static-check.yaml

@@ -72,7 +72,7 @@ jobs:
 
       - name: Copy Lib
         if: matrix.runner == 'ubuntu'
-        run: sudo cp ./pkg/logtail/libPluginAdapter.so /usr/lib/
+        run: sudo cp ./pkg/logtail/libGoPluginAdapter.so /usr/lib/
 
       - name: Check License Header
         if: matrix.runner == 'ubuntu'

CHANGELOG.md

@@ -36,3 +36,4 @@ your changes, such as:
 - [public] [both] [fix] do not read env config from exited containers
 
 ## [Unreleased]
+- [inner] [both] [updated] Support SLS Metricstore output

Makefile

@@ -13,10 +13,10 @@
 # limitations under the License.
 
 .DEFAULT_GOAL := all
-VERSION ?= 2.0.0
+VERSION ?= 0.0.1
 DOCKER_PUSH ?= false
-DOCKER_REPOSITORY ?= aliyun/ilogtail
-BUILD_REPOSITORY ?= aliyun/ilogtail_build
+DOCKER_REPOSITORY ?= aliyun/loongcollector
+BUILD_REPOSITORY ?= aliyun/loongcollector_build
 GENERATED_HOME ?= generated_files
 PLUGINS_CONFIG_FILE ?= plugins.yml,external_plugins.yml
 GO_MOD_FILE ?= go.mod
@@ -68,9 +68,9 @@ GO_BUILD_FLAGS = -v
 LICENSE_COVERAGE_FILE=license_coverage.txt
 OUT_DIR = output
 DIST_DIR = dist
-PACKAGE_DIR = ilogtail-$(VERSION)
+PACKAGE_DIR = loongcollector-$(VERSION)
 EXTERNAL_DIR = external
-DIST_FILE = $(DIST_DIR)/ilogtail-$(VERSION).linux-$(ARCH).tar.gz
+DIST_FILE = $(DIST_DIR)/loongcollector-$(VERSION).linux-$(ARCH).tar.gz
 
 .PHONY: tools
 tools:
@@ -137,8 +137,8 @@ upgrade_adapter_lib:
 .PHONY: plugin_main
 plugin_main: clean
 	./scripts/plugin_build.sh mod default $(OUT_DIR) $(VERSION) $(PLUGINS_CONFIG_FILE) $(GO_MOD_FILE)
-	cp pkg/logtail/libPluginAdapter.so $(OUT_DIR)/libPluginAdapter.so
-	cp pkg/logtail/PluginAdapter.dll $(OUT_DIR)/PluginAdapter.dll
+	cp pkg/logtail/libGoPluginAdapter.so $(OUT_DIR)/libGoPluginAdapter.so
+	cp pkg/logtail/GoPluginAdapter.dll $(OUT_DIR)/GoPluginAdapter.dll
 
 .PHONY: plugin_local
 plugin_local:
@@ -192,8 +192,8 @@ unittest_e2e_engine: clean gocdocker
 
 .PHONY: unittest_plugin
 unittest_plugin: clean import_plugins
-	cp pkg/logtail/libPluginAdapter.so ./plugin_main
-	cp pkg/logtail/PluginAdapter.dll ./plugin_main
+	cp pkg/logtail/libGoPluginAdapter.so ./plugin_main
+	cp pkg/logtail/GoPluginAdapter.dll ./plugin_main
 	mv ./plugins/input/prometheus/input_prometheus.go ./plugins/input/prometheus/input_prometheus.go.bak
 	go test $$(go list ./...|grep -Ev "telegraf|external|envconfig|(input\/prometheus)|(input\/syslog)"| grep -Ev "plugin_main|pluginmanager") -coverprofile .testCoverage.txt
 	mv ./plugins/input/prometheus/input_prometheus.go.bak ./plugins/input/prometheus/input_prometheus.go
@@ -205,9 +205,9 @@ unittest_core:
 
 .PHONY: unittest_pluginmanager
 unittest_pluginmanager: clean import_plugins
-	cp pkg/logtail/libPluginAdapter.so ./plugin_main
-	cp pkg/logtail/PluginAdapter.dll ./plugin_main
-	cp pkg/logtail/libPluginAdapter.so ./pluginmanager
+	cp pkg/logtail/libGoPluginAdapter.so ./plugin_main
+	cp pkg/logtail/GoPluginAdapter.dll ./plugin_main
+	cp pkg/logtail/libGoPluginAdapter.so ./pluginmanager
 	mv ./plugins/input/prometheus/input_prometheus.go ./plugins/input/prometheus/input_prometheus.go.bak
 	go test $$(go list ./...|grep -Ev "telegraf|external|envconfig"| grep -E "plugin_main|pluginmanager") -coverprofile .coretestCoverage.txt
 	mv ./plugins/input/prometheus/input_prometheus.go.bak ./plugins/input/prometheus/input_prometheus.go
@@ -223,7 +223,7 @@ dist: all
 	./scripts/dist.sh "$(OUT_DIR)" "$(DIST_DIR)" "$(PACKAGE_DIR)"
 
 $(DIST_FILE):
-	@echo 'ilogtail-$(VERSION) dist does not exist! Please download or run `make dist` first!'
+	@echo 'loongcollector-$(VERSION) dist does not exist! Please download or run `make dist` first!'
 	@false
 
 .PHONY: docker

core/CMakeLists.txt

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 cmake_minimum_required(VERSION 3.22)
-project(logtail)
+project(loongcollector)
 
 include(CMakeDependentOption)
 
@@ -113,7 +113,7 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/flusher/links.cmake)
 
 # Subdirectories (modules). except for common, input, processor, flusher, observer, helper, spl, and provider.
 set(SUB_DIRECTORIES_LIST
-        application app_config checkpoint container_manager logger go_pipeline monitor profile_sender models
+        application app_config checkpoint container_manager logger go_pipeline monitor monitor/metric_constants profile_sender models
         config config/watcher
         pipeline pipeline/batch pipeline/limiter pipeline/plugin pipeline/plugin/creator pipeline/plugin/instance pipeline/plugin/interface pipeline/queue pipeline/route pipeline/serializer
         runner runner/sink/http
@@ -125,7 +125,7 @@ set(SUB_DIRECTORIES_LIST
         )
 if (LINUX)
     if (ENABLE_ENTERPRISE)
-        set(SUB_DIRECTORIES_LIST ${SUB_DIRECTORIES_LIST} shennong shennong/sdk streamlog aggregator)
+        set(SUB_DIRECTORIES_LIST ${SUB_DIRECTORIES_LIST} shennong shennong/sdk)
     endif()
 elseif(MSVC)
     if (ENABLE_ENTERPRISE)