Add tlscommon and httpcommon diagnostics hooks

NOTICE.txt

@@ -460,11 +460,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.9.11
+Version: v0.9.12
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].11/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].12/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

changelog/fragments/1718034684-Add-tlscommon-and-httpcommon-hooks.yaml

@@ -0,0 +1,36 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Add tlscommon and httpcommon diagnostic information
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: |
+  Add tlscommon and httpcommon hooks for fleet-server to add API and
+  output TLS diagnostics files when diagnostics are collected as well as
+  an httpcommon diagnostics trace for fleet-server's connection to
+  Elasticsearch.
+
+# Affected component; a word indicating the component this changeset affects.
+component:
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/fleet-server/pull/3587
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/Pallinder/go-randomdata v1.2.0
 	github.com/dgraph-io/ristretto v0.1.1
 	github.com/elastic/elastic-agent-client/v7 v7.11.0
-	github.com/elastic/elastic-agent-libs v0.9.11
+	github.com/elastic/elastic-agent-libs v0.9.12
 	github.com/elastic/elastic-agent-system-metrics v0.10.2
 	github.com/elastic/go-elasticsearch/v8 v8.14.0
 	github.com/elastic/go-ucfg v0.8.8

go.sum

@@ -36,8 +36,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/elastic/elastic-agent-client/v7 v7.11.0 h1:YpkFQyE3qPnVai2a2NiKTMpBXXmPcHRV86AtW7LdpA8=
 github.com/elastic/elastic-agent-client/v7 v7.11.0/go.mod h1:/AeiwX9zxG99eUNrLhpApTpwmE71Qwuh4ozObn7a0ss=
-github.com/elastic/elastic-agent-libs v0.9.11 h1:J4aduNJhVeb699FxJIW/dD4BPREILqXgpWD41sCw8Uc=
-github.com/elastic/elastic-agent-libs v0.9.11/go.mod h1:TLFd0T/e1SHmxnx9pbdm/pqOV9y+VMvHikDyPN4Owkw=
+github.com/elastic/elastic-agent-libs v0.9.12 h1:k3U+F4RA0kUYoP1o9YfXF9aGTmNHzvppmqQfxrA9qs4=
+github.com/elastic/elastic-agent-libs v0.9.12/go.mod h1:TLFd0T/e1SHmxnx9pbdm/pqOV9y+VMvHikDyPN4Owkw=
 github.com/elastic/elastic-agent-system-metrics v0.10.2 h1:AVW+YqgezR0mNOZ80NxPLH3tiYMenNGZ8SC/bIUf4Uc=
 github.com/elastic/elastic-agent-system-metrics v0.10.2/go.mod h1:0jJ2ARnzTTOEMmcRX9UNqSwbwguEluE/mK2HaM3GViI=
 github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA=

internal/pkg/config/output.go

@@ -5,6 +5,8 @@
 package config
 
 import (
+	"bytes"
+	"context"
 	"fmt"
 	"net"
 	"net/http"
@@ -16,8 +18,10 @@ import (
 	"time"
 
 	urlutil "github.com/elastic/elastic-agent-libs/kibana"
+	"github.com/elastic/elastic-agent-libs/transport/httpcommon"
 	"github.com/elastic/elastic-agent-libs/transport/tlscommon"
 	"github.com/elastic/go-elasticsearch/v8"
+	"github.com/rs/zerolog"
 )
 
 // The timeout would be driven by the server for long poll.
@@ -228,3 +232,52 @@ func makeURL(defaultScheme string, defaultPath string, rawURL string, defaultPor
 	addr.Host = host + ":" + port
 	return addr.String(), nil
 }
+
+func (c *Elasticsearch) DiagRequests(ctx context.Context) []byte {
+	pURL, err := httpcommon.NewProxyURIFromString(c.ProxyURL)
+	if err != nil {
+		zerolog.Ctx(ctx).Warn().Err(err).Msg("Unable to transform proxy_url to url.URL")
+	}
+	settings := httpcommon.HTTPTransportSettings{
+		TLS:     c.TLS,
+		Timeout: c.Timeout,
+		Proxy: httpcommon.HTTPClientProxySettings{
+			Disable: c.ProxyDisable,
+			URL:     pURL,
+			Headers: httpcommon.ProxyHeaders(c.ProxyHeaders),
+		},
+	}
+	headers := http.Header{}
+	for k, v := range c.Headers {
+		headers.Set(k, v)
+	}
+
+	reqs := make([]*http.Request, 0, len(c.Hosts))
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var res bytes.Buffer
+	for _, host := range c.Hosts {
+		u, err := url.Parse(host)
+		if err != nil {
+			zerolog.Ctx(ctx).Warn().Err(err).Str("host", host).Msg("Unable to transform host to url.URL")
+			res.WriteString(fmt.Sprintf("Unable to transform host %q to url.URL: %v\n", host, err))
+			continue
+		}
+		if u.Scheme == "" {
+			u.Scheme = c.Protocol
+		}
+		ctx, cancel := context.WithTimeout(ctx, time.Second*30) // TODO(michel-laterman): Configurable timeout; should this be part of config, or part of a diagnostics action?
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+		cancel()
+		if err != nil {
+			zerolog.Ctx(ctx).Warn().Err(err).Str("host", host).Msg("Unable to create request to host")
+			res.WriteString(fmt.Sprintf("Unable to create request to host %q: %v\n", host, err))
+			continue
+		}
+		req.Header = headers.Clone()
+		reqs = append(reqs, req)
+	}
+	res.Write(settings.DiagRequests(reqs)())
+	return res.Bytes()
+}

internal/pkg/config/output_test.go

@@ -8,8 +8,10 @@
 package config
 
 import (
+	"context"
 	"crypto/tls"
 	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"testing"
@@ -382,3 +384,19 @@ func setTestEnv(t *testing.T, env map[string]string) {
 		t.Setenv(k, v)
 	}
 }
+
+func Test_Elasticsearch_DiagRequests(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	es := &Elasticsearch{}
+	es.InitDefaults()
+	es.Hosts = []string{srv.URL}
+
+	p := es.DiagRequests(ctx)
+	require.NotEmpty(t, p)
+	require.Contains(t, string(p), "request 0 successful.")
+}

internal/pkg/server/agent.go

@@ -115,6 +115,42 @@ func (a *Agent) Run(ctx context.Context) error {
 		}
 		return p
 	})
+	a.agent.RegisterDiagnosticHook("fleet-server api tls diag", "fleet-server's API TLS config", "fleet-server-api-tls.txt", "text/plain", func() []byte {
+		if a.srv == nil {
+			log.Warn().Msg("Diagnostics hook failure fleet-server is nil.")
+			return []byte(`Diagnostics hook failure fleet-server is nil`)
+		}
+		cfg := a.srv.GetConfig()
+		if cfg == nil || len(cfg.Inputs) == 0 {
+			log.Warn().Msg("Diagnostics hook failure config is nil.")
+			return []byte(`Diagnostics hook failure config is nil`)
+		}
+		return cfg.Inputs[0].Server.TLS.DiagCerts()()
+	})
+	a.agent.RegisterDiagnosticHook("fleet-server output tls diag", "fleet-server's output TLS config", "fleet-server-output-tls.txt", "text/plain", func() []byte {
+		if a.srv == nil {
+			log.Warn().Msg("Diagnostics hook failure fleet-server is nil.")
+			return []byte(`Diagnostics hook failure fleet-server is nil`)
+		}
+		cfg := a.srv.GetConfig()
+		if cfg == nil {
+			log.Warn().Msg("Diagnostics hook failure config is nil.")
+			return []byte(`Diagnostics hook failure config is nil`)
+		}
+		return cfg.Output.Elasticsearch.TLS.DiagCerts()()
+	})
+	a.agent.RegisterOptionalDiagnosticHook("CONN", "fleet-server output request diag", "fleet-server output request trace diagnostics", "fleet-server-output-request.txt", "text/plain", func() []byte {
+		if a.srv == nil {
+			log.Warn().Msg("Diagnostics hook failure fleet-server is nil.")
+			return []byte(`Diagnostics hook failure fleet-server is nil`)
+		}
+		cfg := a.srv.GetConfig()
+		if cfg == nil {
+			log.Warn().Msg("Diagnostics hook failure config is nil.")
+			return []byte(`Diagnostics hook failure config is nil`)
+		}
+		return cfg.Output.Elasticsearch.DiagRequests(ctx)
+	})
 
 	subCtx, subCanceller := context.WithCancel(ctx)
 	defer subCanceller()

model/openapi.yml

@@ -557,6 +557,7 @@ components:
             type: string
             enum:
               - CPU
+              - CONN
     actionPolicyReassign:
       description: The POLICY_REASSIGN action data.
       type: object

testing/go.mod

@@ -31,7 +31,7 @@ require (
 	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/elastic/elastic-agent-libs v0.9.11 // indirect
+	github.com/elastic/elastic-agent-libs v0.9.12 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect

testing/go.sum

@@ -37,8 +37,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/elastic/elastic-agent-client/v7 v7.11.0 h1:YpkFQyE3qPnVai2a2NiKTMpBXXmPcHRV86AtW7LdpA8=
 github.com/elastic/elastic-agent-client/v7 v7.11.0/go.mod h1:/AeiwX9zxG99eUNrLhpApTpwmE71Qwuh4ozObn7a0ss=
-github.com/elastic/elastic-agent-libs v0.9.11 h1:J4aduNJhVeb699FxJIW/dD4BPREILqXgpWD41sCw8Uc=
-github.com/elastic/elastic-agent-libs v0.9.11/go.mod h1:TLFd0T/e1SHmxnx9pbdm/pqOV9y+VMvHikDyPN4Owkw=
+github.com/elastic/elastic-agent-libs v0.9.12 h1:k3U+F4RA0kUYoP1o9YfXF9aGTmNHzvppmqQfxrA9qs4=
+github.com/elastic/elastic-agent-libs v0.9.12/go.mod h1:TLFd0T/e1SHmxnx9pbdm/pqOV9y+VMvHikDyPN4Owkw=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=