feat: Get code in that enables running a devnet on GCP

cmd/trustedpods/flags.go

@@ -70,7 +70,7 @@ var _ = func() error {
 	deploymentFlags.StringVar(&deploymentFormat, "deployment-format", "", fmt.Sprintf("Deployment format. One of %v (leave empty to auto-detect)", pb.FormatNames))
 	deploymentFlags.StringVar(&providerPeer, "provider", "", "provider peer id")
 	deploymentFlags.StringVar(&providerEthAddress, "provider-eth", "", "provider public address")
-	deploymentFlags.Int64Var(&expirationOffset, "token-expiration", 10, "authentication token expires after token-expiration seconds (expired after 10 seconds by default)")
+	deploymentFlags.Int64Var(&expirationOffset, "token-expiration", 60, "authentication token expires after token-expiration seconds (expires after 1 minute by default) (note: might need higher values with providers with out-of-sync clocks)")
 	deploymentFlags.StringVar(&ipfsApi, "ipfs", "/ip4/127.0.0.1/tcp/5001", "multiaddr where the ipfs/kubo api can be accessed")
 	deploymentFlags.BoolVar(&authorize, "authorize", false, "Create a key pair for the application and authorize the returned addresses to control the payment channel")
 	deploymentFlags.BoolVar(&verify, "verify", false, "verify the pod images (requires certificate-identity & certificate-oidc-issuer flags)")

deploy/Tiltfile

@@ -101,10 +101,10 @@ def apocryph_resource(
     if registry_contract == "":
         registry_contract = "$(jq .returns.registry.value %s -r)" % deploy_script_json
 
-    ethereum_port = (
-        "$(kubectl get svc -n %s eth-rpc -o jsonpath --template '{$.spec.ports[0].nodePort}')"
-        % ethereum_namespace
-    )
+    # ethereum_port = (
+    #     "$(kubectl get svc -n %s eth-rpc -o jsonpath --template '{$.spec.ports[0].nodePort}')"
+    #     % ethereum_namespace
+    # )
 
     if type(manifest_file) == "string":
         manifest = read_yaml(manifest_file)
@@ -137,7 +137,8 @@ def apocryph_resource(
         "--ipfs",
         "/ip4/127.0.0.1/tcp/5001",  # TODO: use docker's routing! e.g. '/dns4/%s/tcp/5001' % docker_ipfs
         "--ethereum-rpc",
-        "http://%s:%s" % (cluster_ip, ethereum_port),
+        "http://127.0.0.1:8549",
+        # "http://%s:%s" % (cluster_ip, ethereum_port),
     ] + flags
     apply_flags = [
         "--registry-contract",
@@ -534,6 +535,19 @@ def deploy_apocryph_local(
     cosign_key_path="deploy/keys/",
     resource_deps=["ipfs", "anvil"],
 ):
+    if len(resource_deps) == 0:  # HACK
+        local_resource(
+            "ingress-nginx-portforward",
+            serve_cmd="kubectl port-forward -n keda svc/ingress-nginx-controller 8004:80",
+        )
+        local_resource(
+            "anvil-portforward",
+            serve_cmd="kubectl port-forward -n eth svc/eth-rpc 8549:8545",
+        )
+    else:
+        k8s_resource(workload="ingress-nginx", port_forwards=["8004:80"])
+        k8s_resource(workload="anvil", port_forwards=["8549:8545"])
+
     docker_ipfs_resource(
         "ipfs-local",
         "docker.io/ipfs/kubo:v0.23.0",
@@ -547,8 +561,8 @@ def deploy_apocryph_local(
         labels=["apocryph", "flaky"],
         dir="./contracts/",
         # [ -f ./broadcast/Deploy.s.sol/31337/run-latest.json ] ||
-        cmd="forge script script/Deploy.s.sol --rpc-url http://%s:$(kubectl get svc -n eth eth-rpc -o jsonpath --template '{$.spec.ports[0].nodePort}') --private-key %s --broadcast"
-        % (cluster_ip, deployer_key),
+        cmd="forge script script/Deploy.s.sol --rpc-url http://127.0.0.1:8549 --private-key %s --broadcast"
+        % (deployer_key,),
         resource_deps=resource_deps,
         deps=["./contracts/src", "./contracts/script", "./contracts/lib"],
     )

deploy/charts/eth/anvil-proxy.yml

@@ -6,7 +6,7 @@ metadata:
   labels:
     app: eth
 spec:
-  type: NodePort
+  type: ClusterIP
   selector:
     app: anvil
   ports:

deploy/charts/ipfs/templates/ipfs.yml

@@ -44,7 +44,9 @@ data:
     #!/bin/sh
     set -ex
     ipfs config --json Experimental.Libp2pStreamMounting true
+    {{ if .Values.swarm.announceIp }}
     ipfs config --json Addresses.Announce '["/ip4/{{ .Values.swarm.announceIp }}/tcp/{{ .Values.swarm.nodePort }}", "/ip4/{{ .Values.swarm.announceIp }}/udp/{{ .Values.swarm.nodePort }}/quic", "/ip4/{{ .Values.swarm.announceIp }}/udp/{{ .Values.swarm.nodePort }}/quic-v1", "/ip4/{{ .Values.swarm.announceIp }}/udp/{{ .Values.swarm.nodePort }}/quic-v1/webtransport"]'
+    {{ end }}
 
 ---
 apiVersion: apps/v1

deploy/constellation/helmfile-gcp.yaml

@@ -0,0 +1,107 @@
+# NOTE: Code below duplicates deploy_apocryph_stack in ../Tiltfile
+repositories:
+  - name: kedacore
+    url: https://kedacore.github.io/charts
+  - name: ingress-nginx
+    url: https://kubernetes.github.io/ingress-nginx
+  - name: prometheus-community
+    url: https://prometheus-community.github.io/helm-charts
+  - name: grafana
+    url: https://grafana.github.io/helm-charts
+  - name: sigstore
+    url: https://sigstore.github.io/helm-charts
+
+releases:
+  - name: namespaces # Helmfile exclusive, see https://github.com/helm/helm/issues/9813
+    namespace: default
+    chart: ./namespaces
+
+  - name: policy-controller
+    namespace: policy-controller
+    chart: sigstore/policy-controller
+    needs:
+      - default/namespaces
+
+  - name: keda
+    namespace: keda
+    chart: kedacore/keda
+    needs:
+      - default/namespaces
+
+  - name: ingress-nginx
+    namespace: keda
+    chart: ingress-nginx/ingress-nginx
+    needs:
+      - default/namespaces
+
+  - name: keda-ingress
+    chart: ../charts/keda
+    namespace: keda
+    needs:
+      - keda-http-addon
+      - ingress-nginx # NOTE: nginx admission controllers typically fails to wait long enough for nginx to start at this step
+      - default/namespaces
+
+  - name: keda-http-addon
+    namespace: keda
+    chart: kedacore/keda-add-ons-http
+    set:
+      - name: interceptor.replicas.min
+        value: 1
+      - name: scaler.replicas
+        value: 1
+      - name: interceptor.waitTimeout
+        value: 40s
+    needs:
+      - keda
+      - default/namespaces
+
+  - name: prometheus
+    chart: prometheus-community/prometheus
+    namespace: prometheus
+    set:
+      - name: alertmanager.enabled
+        value: false
+      - name: prometheus-node-exporter.enabled
+        value: false
+      - name: server.persistentVolume.size # Helmfile exclusive, reduce prometheus PV
+        value: 1Gi
+    needs:
+      - default/namespaces
+
+  - name: loki
+    chart: grafana/loki-stack
+    namespace: loki
+    values:
+      - ../charts/loki/values.yml
+    needs:
+      - default/namespaces
+
+  - name: ipfs
+    chart: ../charts/ipfs
+    namespace: ipfs
+    set:
+      - name: swarm.announceIp # Helmfile exclusive, don't change announce IP
+        value: false
+      - name: p2phelper.image # Helmfile exclusive, use ghcr images
+        value: ghcr.io/comrade-coop/apocryph/p2p-helper:master
+    needs:
+      - default/namespaces
+
+  - name: eth
+    chart: ../charts/eth
+    namespace: eth
+    needs:
+      - default/namespaces
+
+  - name: trustedpods
+    chart: ../charts/trustedpods
+    namespace: trustedpods
+    set:
+      - name: policy.enable # Helmfile exclusive, disable policy
+        value: false
+      - name: image # Helmfile exclusive, use ghcr images
+        value: ghcr.io/comrade-coop/apocryph/server:master
+    needs:
+      - default/namespaces
+# NOTE: Code above duplicates deploy_apocryph_stack in ../Tiltfile

deploy/constellation/helmfile.yaml

@@ -120,7 +120,7 @@ releases:
       - name: policy.enable # Helmfile exclusive, disable policy
         value: false
       - name: image # Helmfile exclusive, use ghcr images
-        value: ttl.sh/47edb30b-apocryph-server:1h
+        value: ghcr.io/comrade-coop/apocryph/server:master
     needs:
       - default/namespaces
 # NOTE: Code above duplicates deploy_apocryph_stack in ../Tiltfile

pkg/proto/protoconnect/interceptors.go

@@ -127,7 +127,7 @@ func (a authInterceptor) authenticate(header http.Header) (common.Address, error
 	if err != nil {
 		return common.Address{}, connect.NewError(connect.CodeDataLoss, fmt.Errorf("Failed Unmarshalling token"))
 	}
-	if time.Now().After(token.ExpirationTime) {
+	if time.Now().UTC().After(token.ExpirationTime) {
 		return common.Address{}, connect.NewError(connect.CodeDeadlineExceeded, fmt.Errorf("Token Expired"))
 	}
 
@@ -218,7 +218,7 @@ func (a *AuthInterceptorClient) getOrCreateToken(operation string) (serializedTo
 		tokenData := Token{
 			PodId:          a.podId,
 			Operation:      operation,
-			ExpirationTime: time.Now().Add(a.expirationOffset),
+			ExpirationTime: time.Now().UTC().Add(a.expirationOffset),
 			Publisher:      a.publisher,
 		}
 		tokenDataBytes, err := json.Marshal(tokenData)

test/e2e/nginx/Tiltfile

@@ -6,7 +6,7 @@ load("../../../deploy/Tiltfile", "apocryph_resource", "cluster_ip")
 apocryph_resource(
     "nginx-example",
     "../common/manifests/manifest-nginx.yaml",
-    upload_images=False,
+    upload_images=True,
     apply_flags=["--mint-funds"],
     labels=["apocryph-pod", "flaky"],
 )
@@ -32,12 +32,6 @@ provider_balance_cmd = 'cast call "%s" "balanceOf(address)" "%s" -r %s | cast to
 )
 curl_cmd = 'curl --connect-timeout 40 -H "Host: %s" localhost:8004 --fail-with-body' % manifest_host
 
-local_resource(
-    "ingress-nginx-portforward",
-    serve_cmd="kubectl port-forward -n keda svc/ingress-nginx-controller 8004:80",
-)
-# k8s_resource(workload="ingress-nginx", port_forwards=["8004:80"])
-
 local_resource(
     "nginx-example-measure-and-query",
     [