feat(raiko): generalized build pipeline for ZkVMs guests (#133)

* init + some taiko_util -> util

* pipeline

* example

* r0 bins & tests done

* exmple-sp1 done

* gen proof with unittests

* test harness for sp1 doen + split the builder and driver

* make example a workspace

* change dest param

* harness upgrade

* image id compute

* cleanup

* pipeline api change

* r0 switch builder-driver

* r0 switch builder-driver

* sp1 prover -> driver rename

* sp1 builder

* cargo check passed

* name changes

* commented out sp1 & ris0 flags in irrelavent setup

* fixes + makefile

* update

* clean up

* update CI r0

* update sp1

* add builder to workspace

* sp1 CI permission denied

* add example test in guest + clippy

* add test to CI

* fmt & fix CI bug

* test with --release + toolchain

* fix

* fix docker

* add CC support for c-kzg

* trait Pipeline impl in specific builder

* fix ci

* download & verify riscv-gcc-prebuilt

* Update pipeline/src/lib.rs

* update script

* Updated readme + fix local building

* cargo: -> cargo::

---------

Co-authored-by: Brechtpd <[email protected]>

.dockerignore

@@ -10,6 +10,7 @@
 !/primitives
 !/provers
 !/host
+!/pipeline
 !/rust-toolchain
 !/taiko.toml
 !/tests

.github/workflows/ci.yml

@@ -20,49 +20,51 @@ jobs:
     name: Build and test risc0
     runs-on: ubuntu-latest
     timeout-minutes: 60
-    env:
-      CC: gcc
-      CC_riscv32im_risc0_zkvm_elf: /tmp/riscv/bin/riscv32-unknown-elf-gcc
     steps:
     - uses: actions/checkout@v4
       with:
         submodules: recursive
     - run: scripts/toolchain-before.sh risc0
-    - run: scripts/toolchain.sh risc0
     - uses: actions-rs/toolchain@v1
       with:
         profile: minimal
     - run: scripts/toolchain-after.sh risc0
     - uses: risc0/risc0/.github/actions/[email protected]
     - name: Install cargo-binstall
       uses: cargo-bins/[email protected]
-    - run: cargo binstall -y --force cargo-risczero
-    - run: cargo risczero install
-    - run: cargo build -p risc0-prover --features enable
-    - run: cargo test -p risc0-prover --features enable
+    - name: Install cargo-risczero
+      run: |
+        cargo binstall -y --force cargo-risczero
+        cargo risczero install
+    - name: Build risc0 guest target
+      run: cargo run --bin risc0-builder
+    - name: Build risc0 prover
+      run: cargo build --release --features risc0
+    - name: Test risc0 prover
+      run: cargo test -p risc0-driver --features enable
 
   build-test-sp1:
     name: Build and test sp1
     runs-on: ubuntu-latest
     timeout-minutes: 60
-    env:
-      CC: gcc
-      CC_riscv32im_succinct_zkvm_elf: /tmp/riscv/bin/riscv32-unknown-elf-gcc
     steps:
       - uses: actions/checkout@v4
       - run: scripts/toolchain-before.sh sp1
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-      - run: scripts/toolchain.sh sp1
       - run: scripts/toolchain-after.sh sp1
-      - name: Setup and build
+      - name: Install sp1
         run: |
           curl -L https://sp1.succinct.xyz | bash
+          echo "/home/runner/.config/.sp1/bin" >> $GITHUB_PATH
           /home/runner/.config/.sp1/bin/sp1up
-          cargo build -p sp1-prover --features enable
-          cargo test -p sp1-prover --features enable
-        shell: bash
+      - name: Build sp1 guest target
+        run: cargo run --bin sp1-builder
+      - name: Build sp1 prover
+        run: cargo build --release --features sp1
+      - name: Test sp1 prover
+        run: cargo test -p sp1-driver --features enable --release
 
   build-test-sgx:
     name: Build and test sgx
@@ -78,7 +80,7 @@ jobs:
       - name: Setup and build
         run: |
           cargo build --features sgx
-          cargo test -p sgx-prover --features enable
+          cargo test -p sgx-prover --features enable --release
         shell: bash
 
   build-test-sgx-with-docker: