Merge remote-tracking branch 'origin/sync-layer-stable' into ra/zk-as…

…-gateway-token
matter-labs · Oct 31, 2024 · 7eaf96f · 7eaf96f
2 parents abd6993 + c291a2b
commit 7eaf96f
Show file tree

Hide file tree

Showing 1,103 changed files with 44,145 additions and 39,335 deletions.
diff --git a/.githooks/pre-push b/.githooks/pre-push
@@ -6,14 +6,29 @@
 RED='\033[0;31m'
 NC='\033[0m' # No Color
 
+# Common prompts
+INSTALL_PROPT="Please install ZK Stack CLI using zkstackup from https://github.com/matter-labs/zksync-era/tree/main/zkstack_cli/zkstackup"
+FORMAT_PROMPT="Please format the code via 'zkstack dev fmt', cannot push unformatted code"
+
 # Check that prettier formatting rules are not violated.
-if which zk_supervisor >/dev/null; then
-  if ! zk_supervisor fmt --check; then
+if which zkstack >/dev/null; then
+  if ! zkstack dev fmt --check; then
       echo -e "${RED}Push error!${NC}"
-      echo "Please format the code via 'zks fmt', cannot push unformatted code"
+      echo -e "${FORMAT_PROMPT}"
       exit 1
   fi
 else
-  echo "Please install zk_toolbox using zkup from https://github.com/matter-labs/zksync-era/tree/main/zk_toolbox/zkup, and then run ./bin/zkt from the zksync-era repository."
-  exit 1
+  if which zk_supervisor >/dev/null; then
+    echo -e "${RED}WARNING: zkup, zk_inception/zki, and zk_supervisor/zks are DEPRECATED.${NC}"
+    echo -e "${RED}${INSTALL_PROPT}${NC}"
+
+    if ! zk_supervisor fmt --check; then
+      echo -e "${RED}Push error!${NC}"
+      echo -e "${FORMAT_PROMPT}"
+      exit 1
+    fi
+  else
+    echo -e "${INSTALL_PROPT}"
+    exit 1
+  fi
 fi
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -17,4 +17,4 @@
 - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs).
 - [ ] Tests for the changes have been added / updated.
 - [ ] Documentation comments have been added / updated.
-- [ ] Code has been formatted via `zk_supervisor fmt` and `zk_supervisor lint`.
+- [ ] Code has been formatted via `zkstack dev fmt` and `zkstack dev lint`.
diff --git a/.github/release-please/config.json b/.github/release-please/config.json
@@ -20,9 +20,9 @@
       "release-type": "simple",
       "component": "prover"
     },
-    "zk_toolbox": {
+    "zkstack_cli": {
       "release-type": "simple",
-      "component": "zk_toolbox",
+      "component": "zkstack_cli",
       "plugins": [
         "cargo-workspace"
       ]

diff --git a/.github/release-please/manifest.json b/.github/release-please/manifest.json
@@ -1,5 +1,5 @@
 {
-  "core": "24.27.0",
+  "core": "25.0.0",
   "prover": "16.5.0",
-  "zk_toolbox": "0.1.2"
+  "zkstack_cli": "0.1.2"
 }
diff --git a/.github/scripts/gateway.sh b/.github/scripts/gateway.sh
diff --git a/.github/workflows/build-contract-verifier-template.yml b/.github/workflows/build-contract-verifier-template.yml
@@ -101,7 +101,6 @@ jobs:
       - name: start-services
         run: |
           echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
-          mkdir -p ./volumes/postgres
           run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
           ci_run pre_download_compilers.sh
@@ -113,15 +112,19 @@ jobs:
           ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen
           ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts
           ci_run git config --global --add safe.directory /usr/src/zksync/contracts
-          ci_run ./bin/zkt || true
           ci_run ./bin/zk || true
           ci_run run_retried curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key
 
+      - name: install zkstack
+        run: |
+          ci_run ./zkstack_cli/zkstackup/install -g --path ./zkstack_cli/zkstackup/zkstackup || true
+          ci_run zkstackup -g --local
+
       - name: build contracts
         if: env.BUILD_CONTRACTS == 'true'
         run: |
           ci_run cp etc/tokens/{test,localhost}.json
-          ci_run zk_supervisor contracts
+          ci_run zkstack dev contracts
 
       - name: Login to Docker registries
         if: ${{ inputs.action == 'push' }}

diff --git a/.github/workflows/build-core-template.yml b/.github/workflows/build-core-template.yml
@@ -71,11 +71,15 @@ jobs:
             if [ $(jq length <<<"$tags") -eq 0 ]; then
               echo "No tag found on all pages."
               echo "BUILD_CONTRACTS=true" >> "$GITHUB_ENV"
+              # TODO Remove it when we migrate to foundry inside contracts repository
+              mkdir -p contracts/l1-contracts/artifacts/
               exit 0
             fi
             filtered_tag=$(jq -r --arg commit_sha "$commit_sha" 'map(select(.commit.sha == $commit_sha)) | .[].name' <<<"$tags")
             if [[ ! -z "$filtered_tag" ]]; then
               echo "BUILD_CONTRACTS=false" >> "$GITHUB_ENV"
+              # TODO Remove it when we migrate to foundry inside contracts repository
+              mkdir -p contracts/l1-contracts/out
               break
             fi
             ((page++))
@@ -110,7 +114,6 @@ jobs:
       - name: start-services
         run: |
           echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
-          mkdir -p ./volumes/postgres
           run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
           ci_run pre_download_compilers.sh
@@ -123,14 +126,19 @@ jobs:
           ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts
           ci_run git config --global --add safe.directory /usr/src/zksync/contracts
           ci_run ./bin/zk || true
-          ci_run ./bin/zkt || true
           ci_run run_retried curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key
 
+      - name: Install zkstack
+        if: env.BUILD_CONTRACTS == 'true'
+        run: |
+          ci_run ./zkstack_cli/zkstackup/install -g --path ./zkstack_cli/zkstackup/zkstackup || true
+          ci_run zkstackup -g --local
+
       - name: build contracts
         if: env.BUILD_CONTRACTS == 'true'
         run: |
           ci_run cp etc/tokens/{test,localhost}.json
-          ci_run zk_supervisor contracts
+          ci_run zkstack dev contracts --system-contracts --l1-contracts --l2-contracts 
 
       - name: Login to Docker registries
         if: ${{ inputs.action == 'push' }}

diff --git a/.github/workflows/build-local-node-docker.yml b/.github/workflows/build-local-node-docker.yml
@@ -50,9 +50,13 @@ jobs:
 
       - name: start-services
         run: |
-          mkdir -p ./volumes/postgres
           run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
+        
+      - name: Install zkstack
+        run: |
+          ci_run ./zkstack_cli/zkstackup/install -g --path ./zkstack_cli/zkstackup/zkstackup || true
+          ci_run zkstackup -g
 
       - name: init
         run: |
@@ -61,9 +65,11 @@ jobs:
           ci_run git config --global --add safe.directory /usr/src/zksync/contracts
 
           ci_run zk
-          ci_run zkt
           ci_run cp etc/tokens/{test,localhost}.json
-          ci_run zk_supervisor contracts 
+
+      - name: build contracts
+        run: |
+          ci_run zkstack dev contracts 
 
       - name: update-image
         run: |

diff --git a/.github/workflows/build-prover-fri-gpu-gar-and-circuit-prover-gpu-gar.yml b/.github/workflows/build-prover-fri-gpu-gar-and-circuit-prover-gpu-gar.yml
@@ -28,7 +28,6 @@ jobs:
       - name: Download Setup data
         run: |
           gsutil -m rsync -r gs://matterlabs-setup-data-us/${{ inputs.setup_keys_id }} docker/prover-gpu-fri-gar
-          cp -v docker/prover-gpu-fri-gar/*.bin docker/circuit-prover-gpu-gar/
 
       - name: Login to us-central1 GAR
         run: |
@@ -70,6 +69,10 @@ jobs:
             --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
+      - name: Move Setup data from prover-gpu-fri-gar to circuit-prover-gpu-gar
+        run: |
+          mv -v docker/prover-gpu-fri-gar/*.bin docker/circuit-prover-gpu-gar/
+
       - name: Build and push circuit-prover-gpu-gar
         uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
         with:

diff --git a/.github/workflows/build-prover-template.yml b/.github/workflows/build-prover-template.yml
@@ -45,7 +45,7 @@ jobs:
       RUNNER_COMPOSE_FILE: "docker-compose-runner-nightly.yml"
       ERA_BELLMAN_CUDA_RELEASE: ${{ inputs.ERA_BELLMAN_CUDA_RELEASE }}
       CUDA_ARCH: ${{ inputs.CUDA_ARCH }}
-    runs-on: [ matterlabs-ci-runner-high-performance ]
+    runs-on: [matterlabs-ci-runner-high-performance]
     strategy:
       matrix:
         component:
@@ -56,6 +56,7 @@ jobs:
           - prover-fri-gateway
           - prover-job-monitor
           - proof-fri-gpu-compressor
+          - prover-autoscaler
     outputs:
       protocol_version: ${{ steps.protocolversion.outputs.protocol_version }}
     steps:
@@ -74,7 +75,6 @@ jobs:
       - name: start-services
         run: |
           echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
-          mkdir -p ./volumes/postgres
           run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
           ci_run sccache --start-server
@@ -91,7 +91,6 @@ jobs:
         run: |
           ci_run run_retried curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^24.key
 
-
       - name: login to Docker registries
         if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
         run: |

diff --git a/.github/workflows/build-witness-generator-template.yml b/.github/workflows/build-witness-generator-template.yml
@@ -75,7 +75,6 @@ jobs:
       - name: start-services
         run: |
           echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
-          mkdir -p ./volumes/postgres
           run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
           ci_run sccache --start-server

diff --git a/.github/workflows/ci-common-reusable.yml b/.github/workflows/ci-common-reusable.yml
@@ -27,15 +27,15 @@ jobs:
       - name: Start services
         run: |
           run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
-          mkdir -p ./volumes/postgres
           docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres
-
-      - name: Init
+      
+      - name: Install zkstack
         run: |
-          ci_run zkt
+          ci_run ./zkstack_cli/zkstackup/install -g --path ./zkstack_cli/zkstackup/zkstackup
+          ci_run zkstackup -g --local
 
       # This does both linting and "building". We're using `zk lint prover` as it's common practice within our repo
       # `zk lint prover` = cargo clippy, which does cargo check behind the scenes, which is a lightweight version of cargo build
       - name: Lints
-        run: ci_run zk_supervisor lint -t rs --check
+        run: ci_run zkstack dev lint -t rs --check
 
diff --git a/.github/workflows/ci-core-lint-reusable.yml b/.github/workflows/ci-core-lint-reusable.yml
@@ -26,24 +26,31 @@ jobs:
       - name: Start services
         run: |
           ci_localnet_up
+      
+      - name: Install zkstack
+        run: |
+          ci_run ./zkstack_cli/zkstackup/install -g --path ./zkstack_cli/zkstackup/zkstackup
+          ci_run zkstackup -g --local
 
       - name: Build
         run: |
-          ci_run ./bin/zkt
           ci_run yarn install
           ci_run git config --global --add safe.directory /usr/src/zksync
-          ci_run zk_supervisor db setup --prover-url=${{ env.prover_url }} --core-url=${{ env.core_url }}
 
+      - name: Database setup
+        run: |
+          ci_run zkstack dev db setup --prover-url=${{ env.prover_url }} --core-url=${{ env.core_url }}
 
       - name: Lints
         run: |
-          ci_run zk_supervisor fmt --check
-          ci_run zk_supervisor lint -t md --check
-          ci_run zk_supervisor lint -t sol --check
-          ci_run zk_supervisor lint -t js --check
-          ci_run zk_supervisor lint -t ts --check
-          ci_run zk_supervisor lint -t rs --check
+          ci_run zkstack dev fmt --check
+          ci_run zkstack dev lint -t md --check
+          ci_run zkstack dev lint -t sol --check
+          ci_run zkstack dev lint -t js --check
+          ci_run zkstack dev lint -t ts --check
+          ci_run zkstack dev lint -t rs --check
+          ci_run zkstack dev lint -t autocompletion --check
 
       - name: Check Database
         run: |
-          ci_run zk_supervisor database check-sqlx-data --prover-url=${{ env.prover_url }} --core-url=${{ env.core_url }}
+          ci_run zkstack dev database check-sqlx-data --prover-url=${{ env.prover_url }} --core-url=${{ env.core_url }}