1853 - Zarf Metrics & Tests (#1915)

## Description
Expose Prometheus metrics through /metrics HTTP endpoint.
Add example of how to scrape metrics.

This PR also tests:

- [Named Ports
PR](https://github.com/cmwylie19/zarf/blob/d5a9cf2a9a16d56a0fd6dca72ac6e695f5ae8835/examples/manifests/httpd-service.yaml#L13)
- [Wait-For PR
(jsonPath)](https://github.com/cmwylie19/zarf/blob/d5a9cf2a9a16d56a0fd6dca72ac6e695f5ae8835/examples/manifests/zarf.yaml#L26)
## Related Issue

Fixes #1849
<!-- or -->
Relates to #1693 #1873 
## Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [x] Other (security config, docs update, etc)

## Checklist before merging

- [x] Test, docs, adr added or updated as needed
- [x] [Contributor Guide
Steps](https://github.com/defenseunicorns/zarf/blob/main/CONTRIBUTING.md#developer-workflow)
followed

---------

Signed-off-by: Case Wylie <[email protected]>
Co-authored-by: Wayne Starr <[email protected]>

.github/workflows/release.yml

@@ -91,6 +91,9 @@ jobs:
           name: build-artifacts
           path: build/
 
+      - name: Setup golang
+        uses: ./.github/actions/golang
+
       - name: Make Zarf executable
         run: |
           chmod +x build/zarf

.github/workflows/test-bigbang.yml

@@ -22,7 +22,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  validate:
+  build:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -40,9 +40,6 @@ jobs:
           init-package: 'false'
           build-examples: 'false'
 
-      - name: Setup K3d
-        uses: ./.github/actions/k3d
-
       - name: Login to Iron Bank
         uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0
         if: ${{ env.IRONBANK_USERNAME != '' }}
@@ -54,8 +51,56 @@ jobs:
           password: ${{ secrets.IRONBANK_PASSWORD }}
 
       - name: Build a registry1.dso.mil Zarf 'init' package
+        if: ${{ env.IRONBANK_USERNAME != '' }}
+        env:
+          IRONBANK_USERNAME: ${{ secrets.IRONBANK_USERNAME }}
         run: make ib-init-package
 
+      # Upload the contents of the build directory for later stages to use
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: build-artifacts
+          path: build/
+          retention-days: 1
+
+  validate:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: build-artifacts
+          path: build/
+
+      - name: Setup golang
+        uses: ./.github/actions/golang
+
+      - name: Make Zarf executable
+        run: |
+          chmod +x build/zarf
+
+      # Before we run the tests we need to aggressively cleanup files to reduce disk pressure
+      - name: Cleanup files
+        uses: ./.github/actions/cleanup-files
+
+      - name: Setup K3d
+        uses: ./.github/actions/k3d
+
+      - name: Login to Iron Bank
+        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0
+        if: ${{ env.IRONBANK_USERNAME != '' }}
+        env:
+          IRONBANK_USERNAME: ${{ secrets.IRONBANK_USERNAME }}
+        with:
+          registry: registry1.dso.mil
+          username: ${{ secrets.IRONBANK_USERNAME }}
+          password: ${{ secrets.IRONBANK_PASSWORD }}
+
       - name: Run tests
         if: ${{ env.IRONBANK_USERNAME != '' }}
         env:

.github/workflows/test-upgrade.yml

@@ -60,6 +60,9 @@ jobs:
           name: build-artifacts
           path: build/
 
+      - name: Setup golang
+        uses: ./.github/actions/golang
+
       - name: Make Zarf executable
         run: |
           chmod +x build/zarf

examples/kiwix/manifests/deployment.yaml

@@ -37,7 +37,7 @@ spec:
               name: data
       containers:
         - name: kiwix-serve
-          image: "ghcr.io/kiwix/kiwix-serve:3.5.0"
+          image: "ghcr.io/kiwix/kiwix-serve:3.5.0-2"
           command:
             [
               "sh",

examples/kiwix/manifests/service.yaml

@@ -13,4 +13,4 @@ spec:
     - name: http
       port: 8080
       protocol: TCP
-      targetPort: 80
+      targetPort: http

examples/kiwix/zarf.yaml

@@ -17,7 +17,7 @@ components:
           - manifests/deployment.yaml
           - manifests/service.yaml
     images:
-      - ghcr.io/kiwix/kiwix-serve:3.5.0
+      - ghcr.io/kiwix/kiwix-serve:3.5.0-2
       - alpine:3.18
     # Add new data into the cluster, these will keep trying up until their timeout
     dataInjections:

examples/manifests/zarf.yaml

@@ -22,7 +22,7 @@ components:
                 kind: deployment
                 name: httpd-deployment
                 namespace: httpd
-                condition: available
+                condition: '{.status.readyReplicas}=2'
     # image discovery is supported in all manifests and charts using:
     # zarf prepare find-images
     images:

src/extensions/bigbang/test/package/zarf.yaml

@@ -24,3 +24,8 @@ components:
         valuesFiles:
           - disable-all-bb###ZARF_PKG_TMPL_BB_MAJOR###.yaml
           - enable-twistlock.yaml
+    actions:
+      onDeploy:
+        onFailure:
+          - cmd: ./zarf tools kubectl describe nodes
+          - cmd: ./zarf tools kubectl describe pods -A

src/internal/agent/http/server.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/defenseunicorns/zarf/src/internal/agent/hooks"
 	"github.com/defenseunicorns/zarf/src/pkg/message"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 // NewAdmissionServer creates an http.Server for the mutating webhook admission handler.
@@ -26,6 +27,7 @@ func NewAdmissionServer(port string) *http.Server {
 	mux.Handle("/healthz", healthz())
 	mux.Handle("/mutate/pod", ah.Serve(podsMutation))
 	mux.Handle("/mutate/flux-gitrepository", ah.Serve(gitRepositoryMutation))
+	mux.Handle("/metrics", promhttp.Handler())
 
 	return &http.Server{
 		Addr:    fmt.Sprintf(":%s", port),
@@ -40,6 +42,7 @@ func NewProxyServer(port string) *http.Server {
 	mux := http.NewServeMux()
 	mux.Handle("/healthz", healthz())
 	mux.Handle("/", ProxyHandler())
+	mux.Handle("/metrics", promhttp.Handler())
 
 	return &http.Server{
 		Addr:    fmt.Sprintf(":%s", port),

src/test/e2e/21_connect_test.go

@@ -5,6 +5,8 @@
 package test
 
 import (
+	"crypto/tls"
+	"io/ioutil"
 	"net/http"
 	"strings"
 	"testing"
@@ -58,3 +60,41 @@ func TestConnect(t *testing.T) {
 	stdOut, stdErr, err = e2e.Zarf("package", "remove", "init", "--components=logging", "--confirm")
 	require.NoError(t, err, stdOut, stdErr)
 }
+
+func TestMetrics(t *testing.T) {
+	t.Log("E2E: Emits metrics")
+	e2e.SetupWithCluster(t)
+
+	tunnel, err := cluster.NewTunnel("zarf", "svc", "agent-hook", 8888, 8443)
+
+	require.NoError(t, err)
+	err = tunnel.Connect("", false)
+	require.NoError(t, err)
+	defer tunnel.Close()
+
+	// Skip certificate verification
+	// this is an https endpoint being accessed through port-forwarding
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+
+	client := &http.Client{Transport: tr}
+	httpsEndpoint := strings.ReplaceAll(tunnel.HTTPEndpoint(), "http", "https")
+	resp, err := client.Get(httpsEndpoint + "/metrics")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+
+	// Read the response body
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	desiredString := "go_gc_duration_seconds_count"
+	require.Equal(t, true, strings.Contains(string(body), desiredString))
+	require.NoError(t, err, resp)
+	require.Equal(t, 200, resp.StatusCode)
+
+}

src/test/e2e/23_data_injection_test.go

@@ -7,10 +7,12 @@ package test
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"path/filepath"
 	"testing"
 	"time"
 
+	"github.com/defenseunicorns/zarf/src/internal/cluster"
 	"github.com/defenseunicorns/zarf/src/pkg/utils/exec"
 	"github.com/stretchr/testify/require"
 )
@@ -35,6 +37,21 @@ func TestDataInjection(t *testing.T) {
 	require.Contains(t, stdOut, "devops.stackexchange.com_en_all_2023-05.zim")
 	require.Contains(t, stdOut, ".zarf-injection-")
 
+	// Test Named Port
+	// Set remotePort to 0 so it checks for the targetPort on the pod
+	tunnel, err := cluster.NewTunnel("kiwix", "svc", "kiwix", 8080, 0)
+
+	require.NoError(t, err)
+	// need target equal svc that we are trying to connect to call checkForZarfConnectLabel
+	err = tunnel.Connect("kiwix", false)
+	require.NoError(t, err)
+	defer tunnel.Close()
+
+	// Ensure connection
+	resp, err := http.Get(tunnel.HTTPEndpoint())
+	require.NoError(t, err, resp)
+	require.Equal(t, 200, resp.StatusCode)
+
 	// Remove the data injection example
 	stdOut, stdErr, err = e2e.Zarf("package", "remove", path, "--confirm")
 	require.NoError(t, err, stdOut, stdErr)