Merge pull request #37 from ipfs-shipyard/fix-context-timout

Update vole, add authenticated metrics, change backend endpoint to `/check`, and cleanup frontend

README.md

@@ -16,7 +16,7 @@ A tool for checking the accessibility of your data by IPFS peers
 
 ### Deploy
 
-There are web assets in `web` that interacts with the Go HTTP server that can be deployed however you deploy web assets.
+There are web assets in `web` that interact with the Go HTTP server that can be deployed however you deploy web assets.
 Maybe just deploy it on IPFS and reference it with DNSLink.
 
 For anything other than local testing you're going to want to have a proxy to give you HTTPS support on the Go server.
@@ -56,6 +56,23 @@ npx -y serve -l 3000 web
 # Then open http://localhost:3000?backendURL=http://localhost:3333
 ```
 
+## Metrics
+
+The ipfs-check server is instrumented and exposes two Prometheus metrics endpoints:
+
+- `/metrics/libp2p` exposes [go-libp2p metrics](https://blog.libp2p.io/2023-08-15-metrics-in-go-libp2p/).
+- `/metrics/http` exposes http metrics for the check endpoint.
+
+### Securing the metrics endpoints
+
+To add HTTP basic auth to the two metrics endpoints, you can use the `--metrics-auth-username` and `--metrics-auth-password` flags:
+
+```
+./ipfs-check --metrics-auth-username=user --metrics-auth-password=pass
+```
+
+Alternatively, you can use the `IPFS_CHECK_METRICS_AUTH_USER` and `IPFS_CHECK_METRICS_AUTH_PASS` env vars.
+
 ## License
 
 [SPDX-License-Identifier: Apache-2.0 OR MIT](LICENSE.md)

daemon.go

@@ -123,6 +123,9 @@ func (d *daemon) runCheck(query url.Values) (*output, error) {
 		return nil, err
 	}
 
+	// User has only passed a PeerID without any maddrs
+	onlyPeerID := len(ai.Addrs) == 0
+
 	c, err := cid.Decode(cidStr)
 	if err != nil {
 		return nil, err
@@ -138,9 +141,10 @@ func (d *daemon) runCheck(query url.Values) (*output, error) {
 	addrMap, peerAddrDHTErr := peerAddrsInDHT(ctx, d.dht, d.dhtMessenger, ai.ID)
 	out.PeerFoundInDHT = addrMap
 
-	// If peerID given, but no addresses check the DHT
-	if len(ai.Addrs) == 0 {
+	// If peerID given,but no addresses check the DHT
+	if onlyPeerID {
 		if peerAddrDHTErr != nil {
+			// PeerID is not resolvable via the DHT
 			connectionFailed = true
 			out.ConnectionError = peerAddrDHTErr.Error()
 		}
@@ -166,7 +170,7 @@ func (d *daemon) runCheck(query url.Values) (*output, error) {
 		connErr := testHost.Connect(dialCtx, *ai)
 		dialCancel()
 		if connErr != nil {
-			out.ConnectionError = connErr.Error()
+			out.ConnectionError = fmt.Sprintf("error dialing to peer: %s", connErr.Error())
 			connectionFailed = true
 		}
 	}
@@ -175,17 +179,17 @@ func (d *daemon) runCheck(query url.Values) (*output, error) {
 		out.DataAvailableOverBitswap.Error = "could not connect to peer"
 	} else {
 		// If so is the data available over Bitswap?
-		out.DataAvailableOverBitswap = checkBitswapCID(ctx, c, ma)
+		out.DataAvailableOverBitswap = checkBitswapCID(ctx, testHost, c, ma)
 	}
 
 	return out, nil
 }
 
-func checkBitswapCID(ctx context.Context, c cid.Cid, ma multiaddr.Multiaddr) BitswapCheckOutput {
+func checkBitswapCID(ctx context.Context, host host.Host, c cid.Cid, ma multiaddr.Multiaddr) BitswapCheckOutput {
 	out := BitswapCheckOutput{}
 	start := time.Now()
 
-	bsOut, err := vole.CheckBitswapCID(ctx, c, ma, false)
+	bsOut, err := vole.CheckBitswapCID(ctx, host, c, ma, false)
 	if err != nil {
 		out.Error = err.Error()
 	} else {

go.mod

@@ -4,7 +4,7 @@ go 1.21
 
 require (
 	github.com/gavv/httpexpect/v2 v2.16.0
-	github.com/ipfs-shipyard/vole v0.0.0-20240704031213-bd92d8918fb2
+	github.com/ipfs-shipyard/vole v0.0.0-20240801195547-d7b80a461193
 	github.com/ipfs/boxo v0.21.0
 	github.com/ipfs/go-block-format v0.2.0
 	github.com/ipfs/go-cid v0.4.1
@@ -17,6 +17,7 @@ require (
 	github.com/libp2p/go-msgio v0.3.0
 	github.com/multiformats/go-multiaddr v0.13.0
 	github.com/multiformats/go-multihash v0.2.3
+	github.com/prometheus/client_golang v1.19.1
 	github.com/stretchr/testify v1.9.0
 	github.com/urfave/cli/v2 v2.27.3
 )
@@ -132,7 +133,6 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polydawn/refmt v0.89.0 // indirect
-	github.com/prometheus/client_golang v1.19.1 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect

go.sum

@@ -204,8 +204,8 @@ github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFck
 github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk=
 github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/ipfs-shipyard/vole v0.0.0-20240704031213-bd92d8918fb2 h1:ZoJvCQJpdrim33d21Rl0HCPA9oVT2ncpN/lZ9olz68o=
-github.com/ipfs-shipyard/vole v0.0.0-20240704031213-bd92d8918fb2/go.mod h1:ibnGHr4b6P1OYWIR2HLKT1ONUbmd1ZGe9gzRWb/Kcto=
+github.com/ipfs-shipyard/vole v0.0.0-20240801195547-d7b80a461193 h1:5HPfcUkFXM5KvIKzViKNs00NfLP22IW/KBuV7uFoQl0=
+github.com/ipfs-shipyard/vole v0.0.0-20240801195547-d7b80a461193/go.mod h1:ibnGHr4b6P1OYWIR2HLKT1ONUbmd1ZGe9gzRWb/Kcto=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
 github.com/ipfs/boxo v0.21.0 h1:XpGXb+TQQ0IUdYaeAxGzWjSs6ow/Lce148A/2IbRDVE=

integration_test.go

@@ -70,7 +70,7 @@ func TestBasicIntegration(t *testing.T) {
 					libp2p.EnableHolePunching())
 			},
 		}
-		_ = startServer(ctx, d, ":1234")
+		_ = startServer(ctx, d, ":1234", "", "")
 	}()
 
 	h, err := libp2p.New()

main.go

@@ -2,12 +2,16 @@ package main
 
 import (
 	"context"
+	"crypto/subtle"
 	"encoding/json"
 	"log"
 	"net"
 	"net/http"
 	"os"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
 	"github.com/urfave/cli/v2"
 )
 
@@ -28,14 +32,26 @@ func main() {
 			EnvVars: []string{"IPFS_CHECK_ACCELERATED_DHT"},
 			Usage:   "run the accelerated DHT client",
 		},
+		&cli.StringFlag{
+			Name:    "metrics-auth-username",
+			Value:   "",
+			EnvVars: []string{"IPFS_CHECK_METRICS_AUTH_USER"},
+			Usage:   "http basic auth user for the metrics endpoints",
+		},
+		&cli.StringFlag{
+			Name:    "metrics-auth-password",
+			Value:   "",
+			EnvVars: []string{"IPFS_CHECK_METRICS_AUTH_USER"},
+			Usage:   "http basic auth password for the metrics endpoints",
+		},
 	}
 	app.Action = func(cctx *cli.Context) error {
 		ctx := cctx.Context
 		d, err := newDaemon(ctx, cctx.Bool("accelerated-dht"))
 		if err != nil {
 			return err
 		}
-		return startServer(ctx, d, cctx.String("address"))
+		return startServer(ctx, d, cctx.String("address"), cctx.String("metrics-auth-username"), cctx.String("metrics-auth-password"))
 	}
 
 	err := app.Run(os.Args)
@@ -44,23 +60,22 @@ func main() {
 	}
 }
 
-func startServer(ctx context.Context, d *daemon, tcpListener string) error {
+func startServer(ctx context.Context, d *daemon, tcpListener, metricsUsername, metricPassword string) error {
 	l, err := net.Listen("tcp", tcpListener)
 	if err != nil {
 		return err
 	}
 
+	log.Printf("listening on %v\n", l.Addr())
+	log.Printf("Libp2p host peer id %s\n", d.h.ID())
+	log.Printf("Libp2p host listening on %v\n", d.h.Addrs())
 	log.Printf("listening on %v\n", l.Addr())
 
 	d.mustStart()
 
 	log.Printf("ready to start serving")
 
-	// 1. Is the peer findable in the DHT?
-	// 2. Does the multiaddr work? If not, what's the error?
-	// 3. Is the CID in the DHT?
-	// 4. Does the peer respond that it has the given data over Bitswap?
-	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+	checkHandler := func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Add("Access-Control-Allow-Origin", "*")
 		data, err := d.runCheck(r.URL.Query())
 		if err == nil {
@@ -70,8 +85,58 @@ func startServer(ctx context.Context, d *daemon, tcpListener string) error {
 			w.WriteHeader(http.StatusInternalServerError)
 			_, _ = w.Write([]byte(err.Error()))
 		}
+	}
+
+	// Create a custom registry
+	reg := prometheus.NewRegistry()
+
+	requestsTotal := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "http_requests_total",
+			Help: "Total number of slow requests",
+		},
+		[]string{"code"},
+	)
+
+	requestDuration := prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "http_request_duration_seconds",
+			Help:    "Duration of slow requests",
+			Buckets: prometheus.DefBuckets,
+		},
+		[]string{"code"},
+	)
+
+	requestsInFlight := prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "slow_requests_in_flight",
+		Help: "Number of slow requests currently being served",
 	})
 
+	// Register metrics with our custom registry
+	reg.MustRegister(requestsTotal)
+	reg.MustRegister(requestDuration)
+	reg.MustRegister(requestsInFlight)
+	// Instrument the slowHandler
+	instrumentedHandler := promhttp.InstrumentHandlerCounter(
+		requestsTotal,
+		promhttp.InstrumentHandlerDuration(
+			requestDuration,
+			promhttp.InstrumentHandlerInFlight(
+				requestsInFlight,
+				http.HandlerFunc(checkHandler),
+			),
+		),
+	)
+
+	// 1. Is the peer findable in the DHT?
+	// 2. Does the multiaddr work? If not, what's the error?
+	// 3. Is the CID in the DHT?
+	// 4. Does the peer respond that it has the given data over Bitswap?
+	http.Handle("/check", instrumentedHandler)
+
+	http.Handle("/metrics/libp2p", BasicAuth(promhttp.Handler(), metricsUsername, metricPassword))
+	http.Handle("/metrics/http", BasicAuth(promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), metricsUsername, metricPassword))
+
 	done := make(chan error, 1)
 	go func() {
 		defer close(done)
@@ -86,3 +151,22 @@ func startServer(ctx context.Context, d *daemon, tcpListener string) error {
 		return <-done
 	}
 }
+
+func BasicAuth(handler http.Handler, username, password string) http.Handler {
+	if username == "" || password == "" {
+		log.Println("Warning: no http basic auth for the metrics endpoint.")
+		return handler
+	}
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		user, pass, ok := r.BasicAuth()
+
+		if !ok || subtle.ConstantTimeCompare([]byte(user), []byte(username)) != 1 || subtle.ConstantTimeCompare([]byte(pass), []byte(password)) != 1 {
+			w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+			return
+		}
+
+		handler.ServeHTTP(w, r)
+	})
+}

test/tools.go

@@ -43,7 +43,7 @@ func Query(
 
 	e := httpexpect.Default(t, url)
 
-	return e.POST("/").
+	return e.POST("/check").
 		WithQuery("cid", cid).
 		WithQuery("multiaddr", multiaddr).
 		Expect().

web/index.html

@@ -111,16 +111,22 @@ <h2 class="f4">What does it mean if I get an error?</h2>
             const backendURL = getBackendUrl(formData)
 
             showInQuery(formData) // add `cid` and `multiaddr` to local url query to make it shareable
-            toggleSubmitButton()            
-            const res = await fetch(backendURL, { method: 'POST' })
             toggleSubmitButton()
-            if (res.ok) {
-                const respObj = await res.json()
-                const output = formatOutput(formData, respObj)
-                showOutput(output)
-            } else {
-                const resText = await res.text()
-                showOutput(`⚠️ backend returned an error: ${res.status} ${resText}`)
+            try {
+              const res = await fetch(backendURL, { method: 'POST' })
+
+              if (res.ok) {
+                  const respObj = await res.json()
+                  const output = formatOutput(formData, respObj)
+                  showOutput(output)
+              } else {
+                  const resText = await res.text()
+                  showOutput(`⚠️ backend returned an error: ${res.status} ${resText}`)
+              }
+            } catch (e) {
+              showOutput(`⚠️ backend error: ${e}`)
+            } finally {
+              toggleSubmitButton()
             }
         })
     })
@@ -163,31 +169,33 @@ <h2 class="f4">What does it mean if I get an error?</h2>
     function formatOutput (formData, respObj) {
         const ma = formData.get('multiaddr')
         const peerIDStartIndex = ma.lastIndexOf("/p2p/")
-        const peerID = ma.substr(peerIDStartIndex + 5, ma.length)
-        const addrPart = ma.substr(0, peerIDStartIndex)
+        const peerID = ma.slice(peerIDStartIndex + 5);
+        const addrPart = ma.slice(0, peerIDStartIndex);
         let outText = ""
 
         if (respObj.ConnectionError !== "") {
             outText += "❌ Could not connect to multiaddr: " + respObj.ConnectionError + "\n"
         } else {
-            outText += "✔ Successfully connected to multiaddr\n"
+            outText += "✅ Successfully connected to multiaddr\n"
         }
 
         if (ma.indexOf("/p2p/") === 0 && ma.lastIndexOf("/") === 4) {
+            // only peer id passed with /p2p/PeerID
             if (Object.keys(respObj.PeerFoundInDHT).length === 0) {
                 outText += "❌ Could not find any multiaddrs in the dht\n"
             } else {
-                outText += "✔ Found multiaddrs advertised in the DHT:\n"
+                outText += "✅ Found multiaddrs advertised in the DHT:\n"
                 for (const key in respObj.PeerFoundInDHT) {
                     outText += "\t" + key + "\n"
                 }
             }
         } else {
+            // a proper maddr with an IP was passed
             let foundAddr = false
             for (const key in respObj.PeerFoundInDHT) {
                 if (key === addrPart) {
                     foundAddr = true
-                    outText += "✔ Found multiaddr with " + respObj.PeerFoundInDHT[key] + " dht peers\n"
+                    outText += "✅ Found multiaddr with " + respObj.PeerFoundInDHT[key] + " dht peers\n"
                     break
                 }
             }
@@ -200,7 +208,7 @@ <h2 class="f4">What does it mean if I get an error?</h2>
         }
 
         if (respObj.CidInDHT === true) {
-            outText += "✔ Found multihash advertised in the dht\n"
+            outText += "✅ Found multihash advertised in the dht\n"
         } else {
             outText += "❌ Could not find the multihash in the dht\n"
         }
@@ -210,7 +218,7 @@ <h2 class="f4">What does it mean if I get an error?</h2>
         } else if (respObj.DataAvailableOverBitswap.Responded !== true) {
             outText += "❌ The peer did not quickly respond if it had the CID\n"
         } else if (respObj.DataAvailableOverBitswap.Found === true) {
-            outText += "✔ The peer responded that it has the CID\n"
+            outText += "✅ The peer responded that it has the CID\n"
         } else {
             outText += "❌ The peer responded that it does not have the CID\n"
         }