add retry logic to flaky http test

service/telemetry/metrics_test.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"net/http"
 	"testing"
+	"time"
 
 	io_prometheus_client "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/expfmt"
@@ -201,14 +202,34 @@ func createTestMetrics(t *testing.T, mp metric.MeterProvider) {
 }
 
 func getMetricsFromPrometheus(t *testing.T, endpoint string) map[string]*io_prometheus_client.MetricFamily {
-	req, err := http.NewRequest(http.MethodGet, endpoint, nil)
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+	var (
+		req  *http.Request
+		err  error
+		resp *http.Response
+	)
+	req, err = http.NewRequest(http.MethodGet, endpoint, nil)
 	require.NoError(t, err)
 
-	rr, err := http.DefaultClient.Do(req)
-	require.NoError(t, err)
+	maxRetries := 5
+	for i := 0; i < maxRetries; i++ {
+		resp, err = client.Do(req)
+		if err == nil && resp.StatusCode == http.StatusOK {
+			break
+		}
+
+		if i < maxRetries-1 {
+			time.Sleep(2 * time.Second) // Wait before retrying
+		}
+	}
+	require.NoError(t, err, "failed to get metrics from Prometheus after %d attempts", maxRetries)
+	require.Equal(t, http.StatusOK, resp.StatusCode, "unexpected status code after %d attempts", maxRetries)
+	defer resp.Body.Close()
 
 	var parser expfmt.TextParser
-	parsed, err := parser.TextToMetricFamilies(rr.Body)
+	parsed, err := parser.TextToMetricFamilies(resp.Body)
 	require.NoError(t, err)
 
 	return parsed