Merge branch 'main' into changelog-monitoring-fixes

.buildkite/hooks/pre-command

@@ -76,7 +76,7 @@ GH_TOKEN="${VAULT_GITHUB_TOKEN}"
 export GH_TOKEN
 
 echo "--- Setting up the :golang: environment..."
-GO_VERSION=$(cat .go-version)
+GO_VERSION=$(grep '^go' go.mod | cut -d' ' -f2)
 WORKSPACE=$(git rev-parse --show-toplevel)
 export GO_VERSION WORKSPACE
 get_os_details

.ci/scripts/bench.sh

@@ -170,7 +170,7 @@ if [ "${CI}" == "true" ] ; then
   export BUILDKITE_TOKEN="${BUILDKITE_TOKEN_SECRET}"
 
   echo "--- Setting up the :golang: environment..."
-  GO_VERSION=$(cat .go-version)
+  GO_VERSION=$(grep '^go' go.mod | cut -d' ' -f2)
   OS_NAME=linux
   OS_ARCH=amd64
   retry 5 curl -sL -o "${BASE_PROJECT}"/gvm "https://github.com/andrewkroh/gvm/releases/download/v0.5.2/gvm-${OS_NAME}-${OS_ARCH}"

.github/workflows/ci.yml

@@ -6,6 +6,7 @@ on:
       - main
       - 7.1*
       - 8.*
+      - 9.*
     paths-ignore: # When updating the list of expressions also update ci-docs.yml
       - '**.md'
       - '**.asciidoc'
@@ -36,9 +37,6 @@ jobs:
             go.sum
             tools/go.sum
       - run: make check-full
-      - name: Ensure .go-version is up to date
-        run: make update-go-version
-        if: ${{ github.ref == 'main' }}
 
   test:
     strategy:
@@ -63,21 +61,14 @@ jobs:
         with:
           go-version-file: systemtest/go.mod
           cache: true
+          cache-dependency-path: |
+            go.sum
+            systemtest/go.sum
       - run: docker compose up -d
       - env:
           GOTESTFLAGS: "-v -json"
-          OUTPUT_JSON_FILE: "build/TEST-go-system_tests.out.json"
-          OUTPUT_JUNIT_FILE: "build/TEST-go-system_tests.xml"
           GH_TOKEN: ${{ github.token }}
-        run: |
-          go run -modfile=tools/go.mod gotest.tools/gotestsum \
-          --no-color -f standard-quiet --jsonfile "$OUTPUT_JSON_FILE" --junitfile "$OUTPUT_JUNIT_FILE" \
-          --raw-command -- make system-test
-      - uses: actions/upload-artifact@v4  # upload test results
-        if: ${{ always() }} # run this step even if previous step failed
-        with:
-          name: test-results
-          path: 'build/*.xml'
+        run: make system-test
 
   test-package:
     runs-on: ubuntu-latest
@@ -87,6 +78,9 @@ jobs:
         with:
           go-version-file: go.mod
           cache: false
+          cache-dependency-path: |
+            go.sum
+            tools/go.sum
       - run: make package-snapshot
         env:
           GH_TOKEN: ${{ github.token }}
@@ -103,6 +97,9 @@ jobs:
         with:
           go-version-file: go.mod
           cache: false
+          cache-dependency-path: |
+            go.sum
+            tools/go.sum
       - name: Log in to the Elastic Container registry
         uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
@@ -123,7 +120,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Get changed files
         id: changed-files
-        uses: tj-actions/changed-files@d6e91a2266cdb9d62096cebf1e8546899c6aa18f # v45.0.6
+        uses: tj-actions/changed-files@dcc7a0cba800f454d79fff4b993e8c3555bcc0a8 # v45.0.7
         with:
           files: .go-version
 

.github/workflows/microbenchmark.yml

@@ -7,8 +7,8 @@ on:
   push:
     branches:
       - main
-      - "8.[0-9]+"
-      - "8.x"
+      - "8.*"
+      - "9.*"
     paths-ignore:
       - '**.md'
       - '**.asciidoc'

.github/workflows/setup-cluster-env/action.yml

@@ -9,6 +9,7 @@ runs:
     - uses: actions/setup-go@v5
       with:
         go-version-file: go.mod
+        cache: true
         cache-dependency-path: |
           go.sum
           tools/go.sum

.gitignore

@@ -47,6 +47,7 @@ testing/smoke/**/tf.log
 testing/smoke/**/apm-server-*.ndjson
 testing/smoke/**/provisioner_key*
 testing/smoke/**/main.tf
+testing/smoke/**/secret_token_value.json
 !testing/smoke/main.tf
 !testing/smoke/supported-os/main.tf
 !testing/smoke/managed/main.tf

.go-version

@@ -1 +1 @@
-1.23.5
+1.23.6

.mergify.yml

@@ -176,7 +176,9 @@ pull_request_rules:
           * `backport-8./d` is the label to automatically backport to the `8./d` branch. `/d` is the digit.
           * `backport-9./d` is the label to automatically backport to the `9./d` branch. `/d` is the digit.
           * `backport-8.x` is the label to automatically backport to the `8.x` branch.
-
+          * `backport-active-all` is the label that automatically backports to all active branches.
+          * `backport-active-8` is the label that automatically backports to all active minor branches for the 8 major.
+          * `backport-active-9` is the label that automatically backports to all active minor branches for the 9 major.
   - name: remove backport-skip label
     conditions:
       - label~=^backport-\d
@@ -434,3 +436,60 @@ pull_request_rules:
         labels:
           - "backport"
         title: "[{{ destination_branch }}] {{ title }} (backport #{{ number }})"
+
+  - name: backport patches to all active minor branches for the 8 major.
+    conditions:
+      - merged
+      - base=main
+      - label=backport-active-8
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        # NOTE: this list needs to be changed when a new minor branch is created
+        #       or an existing minor branch reached EOL.
+        branches:
+          - "8.x"
+          - "8.18"
+          - "8.17"
+          - "8.16"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: backport patches to all active minor branches for the 9 major.
+    conditions:
+      - merged
+      - base=main
+      - label=backport-active-9
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        # NOTE: this list needs to be changed when a new minor branch is created
+        #       or an existing minor branch reached EOL.
+        branches:
+          - "9.0"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: backport patches to all active branches
+    conditions:
+      - merged
+      - base=main
+      - label=backport-active-all
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        # NOTE: this list needs to be changed when a new minor branch is created
+        #       or an existing release branch reached EOL.
+        branches:
+          - "9.0"
+          - "8.18"
+          - "8.17"
+          - "8.16"
+          - "8.x"
+          - "7.17"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"

Makefile

@@ -148,7 +148,7 @@ get-version:
 get-version-only:
 	@echo $(APM_SERVER_ONLY_VERSION)
 
-# update-go-version updates .go-version, documentation, and build files
+# update-go-version updates documentation, and build files
 # to use the most recent patch version for the major.minor Go version
 # defined in go.mod.
 .PHONY: update-go-version

NOTICE.txt

@@ -455,11 +455,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/beats/v7
-Version: v7.0.0-alpha2.0.20250206132501-d9f5498ae84e
+Version: v7.0.0-alpha2.0.20250207124234-95b5dce16c07
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20250206132501-d9f5498ae84e/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20250207124234-95b5dce16c07/LICENSE.txt:
 
 Source code in this repository is variously licensed under the Apache License
 Version 2.0, an Apache compatible license, or the Elastic License. Outside of

cmd/intake-receiver/go.mod

@@ -1,6 +1,6 @@
 module github.com/elastic/apm-server/cmd/intake-receiver
 
-go 1.23.0
+go 1.23.6
 
 require (
 	github.com/stretchr/testify v1.6.1

docker-compose.yml

@@ -11,7 +11,7 @@ x-logging: &default-logging
 services:
   elasticsearch:
     # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-    image: docker.elastic.co/elasticsearch/elasticsearch:9.0.0-80928867-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.1.0-4b928ba7-SNAPSHOT
     ports:
       - 9200:9200
     healthcheck:
@@ -43,7 +43,7 @@ services:
 
   kibana:
     # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-    image: docker.elastic.co/kibana/kibana:9.0.0-80928867-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:9.1.0-4b928ba7-SNAPSHOT
     ports:
       - 5601:5601
     healthcheck:
@@ -63,7 +63,7 @@ services:
 
   metricbeat:
     # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-    image: docker.elastic.co/beats/metricbeat:9.0.0-80928867-SNAPSHOT
+    image: docker.elastic.co/beats/metricbeat:9.1.0-4b928ba7-SNAPSHOT
     environment:
       ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
       ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}"

functionaltests/go.mod

@@ -1,6 +1,6 @@
 module github.com/elastic/apm-server/functionaltests
 
-go 1.23.2
+go 1.23.6
 
 require (
 	github.com/elastic/apm-perf v0.0.0-20241230130730-2ad47482b731

go.mk

@@ -5,7 +5,7 @@ GITCOMMITTIMESTAMPUNIX ?= $(shell git log -1 --pretty=%ct)
 GITREFFILE ?= $(GITDIR)/$(shell git rev-parse --symbolic-full-name HEAD)
 GITROOT ?= $(shell git rev-parse --show-toplevel)
 
-GOLANG_VERSION=$(shell cat $(GITROOT)/.go-version)
+GOLANG_VERSION=$(shell grep '^go' go.mod | cut -d' ' -f2)
 GOARCH:=$(shell go env GOARCH)
 
 APM_SERVER_ONLY_VERSION=$(shell grep "const Version" $(GITROOT)/internal/version/version.go | cut -d'=' -f2 | tr -d '" ')

go.mod

@@ -1,6 +1,6 @@
 module github.com/elastic/apm-server
 
-go 1.23.0
+go 1.23.6
 
 require (
 	github.com/KimMachineGun/automemlimit v0.7.0
@@ -9,7 +9,7 @@ require (
 	github.com/dustin/go-humanize v1.0.1
 	github.com/elastic/apm-aggregation v1.2.0
 	github.com/elastic/apm-data v1.16.0
-	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250206132501-d9f5498ae84e
+	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250207124234-95b5dce16c07
 	github.com/elastic/elastic-agent-client/v7 v7.17.0
 	github.com/elastic/elastic-agent-libs v0.18.5
 	github.com/elastic/elastic-agent-system-metrics v0.11.7

go.sum

@@ -119,8 +119,8 @@ github.com/elastic/apm-aggregation v1.2.0 h1:UndqG3ccBTjyxTqHujBVjcbVLb9qG1clxRc
 github.com/elastic/apm-aggregation v1.2.0/go.mod h1:YllYwPYVV27pbuPfjRtQAKo6eSSrh13PZr38RKYd810=
 github.com/elastic/apm-data v1.16.0 h1:LkJFoNkadIyqXNo3EMm98J38j9HWDMWUe6F74GaXtJ4=
 github.com/elastic/apm-data v1.16.0/go.mod h1:ST2P1yshhN2U3IbBYyn+Ni3VOFTifavNzMfh7E9zLHY=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250206132501-d9f5498ae84e h1:aYTL11UD2hI1zmTIgfKJNjhKX4bhdoLAms7hdv7/YF0=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250206132501-d9f5498ae84e/go.mod h1:A3OQ9JLS2gNVO2MayjAqFDhqwCj3NN1GnVjramGhj+s=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250207124234-95b5dce16c07 h1:WXHQJBc3r96QgEWhc9SjU4ixQSnHwoS8lGn7Ftm/NGI=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20250207124234-95b5dce16c07/go.mod h1:A3OQ9JLS2gNVO2MayjAqFDhqwCj3NN1GnVjramGhj+s=
 github.com/elastic/elastic-agent-autodiscover v0.9.0 h1:+iWIKh0u3e8I+CJa3FfWe9h0JojNasPgYIA47gpuuns=
 github.com/elastic/elastic-agent-autodiscover v0.9.0/go.mod h1:5iUxLHhVdaGSWYTveSwfJEY4RqPXTG13LPiFoxcpFd4=
 github.com/elastic/elastic-agent-client/v7 v7.17.0 h1:TPLrEHF4kJ3RkmQzZPffrniY4WeW4vriHZbOAzM1hFo=

internal/beatcmd/cmd.go

@@ -74,7 +74,9 @@ func NewRootCommand(beatParams BeatParams) *cobra.Command {
 	// Register subcommands.
 	rootCommand.AddCommand(runCommand)
 	rootCommand.AddCommand(exportCommand)
-	rootCommand.AddCommand(keystoreCommand)
+	if keystoreCommand != nil {
+		rootCommand.AddCommand(keystoreCommand)
+	}
 	rootCommand.AddCommand(versionCommand)
 	rootCommand.AddCommand(genTestCmd(beatParams))
 

internal/beatcmd/config.go

@@ -23,7 +23,6 @@ import (
 
 	"github.com/elastic/beats/v7/libbeat/cfgfile"
 	"github.com/elastic/beats/v7/libbeat/cloudid"
-	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/pprof"
 	"github.com/elastic/elastic-agent-libs/config"
 	libkeystore "github.com/elastic/elastic-agent-libs/keystore"
@@ -94,10 +93,14 @@ func LoadConfig(opts ...LoadConfigOption) (*Config, *config.C, libkeystore.Keyst
 		configOpts = append(configOpts, ucfg.ResolveNOOP)
 	} else {
 		configOpts = append(configOpts,
-			ucfg.Resolve(libkeystore.ResolverWrap(keystore)),
 			ucfg.ResolveEnv,
 			ucfg.VarExp,
 		)
+		if keystore != nil {
+			configOpts = append(configOpts,
+				ucfg.Resolve(libkeystore.ResolverWrap(keystore)),
+			)
+		}
 	}
 	config.OverwriteConfigOpts(configOpts)
 
@@ -138,13 +141,6 @@ func WithMergeConfig(cfg ...*config.C) LoadConfigOption {
 	}
 }
 
-// loadKeystore returns the appropriate keystore based on the configuration.
-func loadKeystore(cfg *config.C) (libkeystore.Keystore, error) {
-	keystoreCfg, _ := cfg.Child("keystore", -1)
-	defaultPathConfig := paths.Resolve(paths.Data, "apm-server.keystore")
-	return libkeystore.Factory(keystoreCfg, defaultPathConfig, common.IsStrictPerms())
-}
-
 func initPaths(cfg *config.C) error {
 	// To Fix the chicken-egg problem with the Keystore and the loading of the configuration
 	// files we are doing a partial unpack of the configuration file and only take into consideration