From 300e0d338199aae3756d5b066e4282bd55b4a751 Mon Sep 17 00:00:00 2001 From: Bojidar Marinov Date: Thu, 24 Oct 2024 16:46:24 +0300 Subject: [PATCH] feat: Get code in that enables running a devnet on GCP --- cmd/trustedpods/flags.go | 2 +- deploy/Tiltfile | 28 +++++-- deploy/charts/eth/anvil-proxy.yml | 2 +- deploy/charts/ipfs/templates/ipfs.yml | 2 + deploy/constellation/helmfile-gcp.yaml | 107 +++++++++++++++++++++++++ deploy/constellation/helmfile.yaml | 2 +- pkg/proto/protoconnect/interceptors.go | 4 +- test/e2e/nginx/Tiltfile | 8 +- 8 files changed, 136 insertions(+), 19 deletions(-) create mode 100644 deploy/constellation/helmfile-gcp.yaml diff --git a/cmd/trustedpods/flags.go b/cmd/trustedpods/flags.go index 01e5ae5..3e1586d 100644 --- a/cmd/trustedpods/flags.go +++ b/cmd/trustedpods/flags.go @@ -70,7 +70,7 @@ var _ = func() error { deploymentFlags.StringVar(&deploymentFormat, "deployment-format", "", fmt.Sprintf("Deployment format. One of %v (leave empty to auto-detect)", pb.FormatNames)) deploymentFlags.StringVar(&providerPeer, "provider", "", "provider peer id") deploymentFlags.StringVar(&providerEthAddress, "provider-eth", "", "provider public address") - deploymentFlags.Int64Var(&expirationOffset, "token-expiration", 10, "authentication token expires after token-expiration seconds (expired after 10 seconds by default)") + deploymentFlags.Int64Var(&expirationOffset, "token-expiration", 60, "authentication token expires after token-expiration seconds (expires after 1 minute by default) (note: might need higher values with providers with out-of-sync clocks)") deploymentFlags.StringVar(&ipfsApi, "ipfs", "/ip4/127.0.0.1/tcp/5001", "multiaddr where the ipfs/kubo api can be accessed") deploymentFlags.BoolVar(&authorize, "authorize", false, "Create a key pair for the application and authorize the returned addresses to control the payment channel") deploymentFlags.BoolVar(&verify, "verify", false, "verify the pod images (requires certificate-identity & certificate-oidc-issuer flags)") diff --git a/deploy/Tiltfile b/deploy/Tiltfile index ad9d1fb..9865347 100644 --- a/deploy/Tiltfile +++ b/deploy/Tiltfile @@ -101,10 +101,10 @@ def apocryph_resource( if registry_contract == "": registry_contract = "$(jq .returns.registry.value %s -r)" % deploy_script_json - ethereum_port = ( - "$(kubectl get svc -n %s eth-rpc -o jsonpath --template '{$.spec.ports[0].nodePort}')" - % ethereum_namespace - ) + # ethereum_port = ( + # "$(kubectl get svc -n %s eth-rpc -o jsonpath --template '{$.spec.ports[0].nodePort}')" + # % ethereum_namespace + # ) if type(manifest_file) == "string": manifest = read_yaml(manifest_file) @@ -137,7 +137,8 @@ def apocryph_resource( "--ipfs", "/ip4/127.0.0.1/tcp/5001", # TODO: use docker's routing! e.g. '/dns4/%s/tcp/5001' % docker_ipfs "--ethereum-rpc", - "http://%s:%s" % (cluster_ip, ethereum_port), + "http://127.0.0.1:8549", + # "http://%s:%s" % (cluster_ip, ethereum_port), ] + flags apply_flags = [ "--registry-contract", @@ -534,6 +535,19 @@ def deploy_apocryph_local( cosign_key_path="deploy/keys/", resource_deps=["ipfs", "anvil"], ): + if len(resource_deps) == 0: # HACK + local_resource( + "ingress-nginx-portforward", + serve_cmd="kubectl port-forward -n keda svc/ingress-nginx-controller 8004:80", + ) + local_resource( + "anvil-portforward", + serve_cmd="kubectl port-forward -n eth svc/eth-rpc 8549:8545", + ) + else: + k8s_resource(workload="ingress-nginx", port_forwards=["8004:80"]) + k8s_resource(workload="anvil", port_forwards=["8549:8545"]) + docker_ipfs_resource( "ipfs-local", "docker.io/ipfs/kubo:v0.23.0", @@ -547,8 +561,8 @@ def deploy_apocryph_local( labels=["apocryph", "flaky"], dir="./contracts/", # [ -f ./broadcast/Deploy.s.sol/31337/run-latest.json ] || - cmd="forge script script/Deploy.s.sol --rpc-url http://%s:$(kubectl get svc -n eth eth-rpc -o jsonpath --template '{$.spec.ports[0].nodePort}') --private-key %s --broadcast" - % (cluster_ip, deployer_key), + cmd="forge script script/Deploy.s.sol --rpc-url http://127.0.0.1:8549 --private-key %s --broadcast" + % (deployer_key,), resource_deps=resource_deps, deps=["./contracts/src", "./contracts/script", "./contracts/lib"], ) diff --git a/deploy/charts/eth/anvil-proxy.yml b/deploy/charts/eth/anvil-proxy.yml index 4095ade..4e9dd20 100644 --- a/deploy/charts/eth/anvil-proxy.yml +++ b/deploy/charts/eth/anvil-proxy.yml @@ -6,7 +6,7 @@ metadata: labels: app: eth spec: - type: NodePort + type: ClusterIP selector: app: anvil ports: diff --git a/deploy/charts/ipfs/templates/ipfs.yml b/deploy/charts/ipfs/templates/ipfs.yml index fa462cb..15fe9d5 100644 --- a/deploy/charts/ipfs/templates/ipfs.yml +++ b/deploy/charts/ipfs/templates/ipfs.yml @@ -44,7 +44,9 @@ data: #!/bin/sh set -ex ipfs config --json Experimental.Libp2pStreamMounting true + {{ if .Values.swarm.announceIp }} ipfs config --json Addresses.Announce '["/ip4/{{ .Values.swarm.announceIp }}/tcp/{{ .Values.swarm.nodePort }}", "/ip4/{{ .Values.swarm.announceIp }}/udp/{{ .Values.swarm.nodePort }}/quic", "/ip4/{{ .Values.swarm.announceIp }}/udp/{{ .Values.swarm.nodePort }}/quic-v1", "/ip4/{{ .Values.swarm.announceIp }}/udp/{{ .Values.swarm.nodePort }}/quic-v1/webtransport"]' + {{ end }} --- apiVersion: apps/v1 diff --git a/deploy/constellation/helmfile-gcp.yaml b/deploy/constellation/helmfile-gcp.yaml new file mode 100644 index 0000000..b4939df --- /dev/null +++ b/deploy/constellation/helmfile-gcp.yaml @@ -0,0 +1,107 @@ +# NOTE: Code below duplicates deploy_apocryph_stack in ../Tiltfile +repositories: + - name: kedacore + url: https://kedacore.github.io/charts + - name: ingress-nginx + url: https://kubernetes.github.io/ingress-nginx + - name: prometheus-community + url: https://prometheus-community.github.io/helm-charts + - name: grafana + url: https://grafana.github.io/helm-charts + - name: sigstore + url: https://sigstore.github.io/helm-charts + +releases: + - name: namespaces # Helmfile exclusive, see https://github.com/helm/helm/issues/9813 + namespace: default + chart: ./namespaces + + - name: policy-controller + namespace: policy-controller + chart: sigstore/policy-controller + needs: + - default/namespaces + + - name: keda + namespace: keda + chart: kedacore/keda + needs: + - default/namespaces + + - name: ingress-nginx + namespace: keda + chart: ingress-nginx/ingress-nginx + needs: + - default/namespaces + + - name: keda-ingress + chart: ../charts/keda + namespace: keda + needs: + - keda-http-addon + - ingress-nginx # NOTE: nginx admission controllers typically fails to wait long enough for nginx to start at this step + - default/namespaces + + - name: keda-http-addon + namespace: keda + chart: kedacore/keda-add-ons-http + set: + - name: interceptor.replicas.min + value: 1 + - name: scaler.replicas + value: 1 + - name: interceptor.waitTimeout + value: 40s + needs: + - keda + - default/namespaces + + - name: prometheus + chart: prometheus-community/prometheus + namespace: prometheus + set: + - name: alertmanager.enabled + value: false + - name: prometheus-node-exporter.enabled + value: false + - name: server.persistentVolume.size # Helmfile exclusive, reduce prometheus PV + value: 1Gi + needs: + - default/namespaces + + - name: loki + chart: grafana/loki-stack + namespace: loki + values: + - ../charts/loki/values.yml + needs: + - default/namespaces + + - name: ipfs + chart: ../charts/ipfs + namespace: ipfs + set: + - name: swarm.announceIp # Helmfile exclusive, don't change announce IP + value: false + - name: p2phelper.image # Helmfile exclusive, use ghcr images + value: ghcr.io/comrade-coop/apocryph/p2p-helper:master + needs: + - default/namespaces + + - name: eth + chart: ../charts/eth + namespace: eth + needs: + - default/namespaces + + - name: trustedpods + chart: ../charts/trustedpods + namespace: trustedpods + set: + - name: policy.enable # Helmfile exclusive, disable policy + value: false + - name: image # Helmfile exclusive, use ghcr images + value: ghcr.io/comrade-coop/apocryph/server:master + needs: + - default/namespaces +# NOTE: Code above duplicates deploy_apocryph_stack in ../Tiltfile diff --git a/deploy/constellation/helmfile.yaml b/deploy/constellation/helmfile.yaml index a033097..952f8e6 100644 --- a/deploy/constellation/helmfile.yaml +++ b/deploy/constellation/helmfile.yaml @@ -120,7 +120,7 @@ releases: - name: policy.enable # Helmfile exclusive, disable policy value: false - name: image # Helmfile exclusive, use ghcr images - value: ttl.sh/47edb30b-apocryph-server:1h + value: ghcr.io/comrade-coop/apocryph/server:master needs: - default/namespaces # NOTE: Code above duplicates deploy_apocryph_stack in ../Tiltfile diff --git a/pkg/proto/protoconnect/interceptors.go b/pkg/proto/protoconnect/interceptors.go index dc7cbf6..d2e677d 100644 --- a/pkg/proto/protoconnect/interceptors.go +++ b/pkg/proto/protoconnect/interceptors.go @@ -127,7 +127,7 @@ func (a authInterceptor) authenticate(header http.Header) (common.Address, error if err != nil { return common.Address{}, connect.NewError(connect.CodeDataLoss, fmt.Errorf("Failed Unmarshalling token")) } - if time.Now().After(token.ExpirationTime) { + if time.Now().UTC().After(token.ExpirationTime) { return common.Address{}, connect.NewError(connect.CodeDeadlineExceeded, fmt.Errorf("Token Expired")) } @@ -218,7 +218,7 @@ func (a *AuthInterceptorClient) getOrCreateToken(operation string) (serializedTo tokenData := Token{ PodId: a.podId, Operation: operation, - ExpirationTime: time.Now().Add(a.expirationOffset), + ExpirationTime: time.Now().UTC().Add(a.expirationOffset), Publisher: a.publisher, } tokenDataBytes, err := json.Marshal(tokenData) diff --git a/test/e2e/nginx/Tiltfile b/test/e2e/nginx/Tiltfile index 757ae1f..0f122ab 100644 --- a/test/e2e/nginx/Tiltfile +++ b/test/e2e/nginx/Tiltfile @@ -6,7 +6,7 @@ load("../../../deploy/Tiltfile", "apocryph_resource", "cluster_ip") apocryph_resource( "nginx-example", "../common/manifests/manifest-nginx.yaml", - upload_images=False, + upload_images=True, apply_flags=["--mint-funds"], labels=["apocryph-pod", "flaky"], ) @@ -32,12 +32,6 @@ provider_balance_cmd = 'cast call "%s" "balanceOf(address)" "%s" -r %s | cast to ) curl_cmd = 'curl --connect-timeout 40 -H "Host: %s" localhost:8004 --fail-with-body' % manifest_host -local_resource( - "ingress-nginx-portforward", - serve_cmd="kubectl port-forward -n keda svc/ingress-nginx-controller 8004:80", -) -# k8s_resource(workload="ingress-nginx", port_forwards=["8004:80"]) - local_resource( "nginx-example-measure-and-query", [