feat(raiko): install script + makefile CI integration (#159)

* test make in ci * update makefile * add make install * update makefile * back to stable * typo * fixes * caching * fix sp1 hopefully * fixes * fix SP1 * fix sgx * fix sp1 * fix SP1 for sure now ya * now 4 sure * enable testing * feedback fix * feedback * add run script
taikoxyz · May 6, 2024 · a6c1095 · a6c1095
1 parent 73a9da8
commit a6c1095
Show file tree

Hide file tree

Showing 10 changed files with 165 additions and 264 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -24,64 +24,61 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: recursive
-    - run: scripts/toolchain-before.sh risc0
     - uses: actions-rs/toolchain@v1
       with:
-        profile: minimal
-    - run: scripts/toolchain-after.sh risc0
-    - uses: risc0/risc0/.github/actions/[email protected]
+          toolchain: stable
+          profile: minimal
     - name: Install cargo-binstall
       uses: cargo-bins/[email protected]
-    - name: Install cargo-risczero
-      run: |
-        cargo binstall -y --force cargo-risczero
-        cargo risczero install
-    - name: Build risc0 guest target
-      run: cargo run --bin risc0-builder
+    - uses: risc0/risc0/.github/actions/[email protected]
+    - name: Install risc0
+      run: TARGET="risc0" CI=1 make install
     - name: Build risc0 prover
-      run: cargo build --release --features risc0
+      run: TARGET="risc0" make build
     - name: Test risc0 prover
-      run: cargo test -p risc0-driver --features enable
+      run: TARGET="risc0" make test
 
   build-test-sp1:
     name: Build and test sp1
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
-      - uses: actions/checkout@v4
-      - run: scripts/toolchain-before.sh sp1
-      - uses: actions-rs/toolchain@v1
-        with:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+    - uses: actions-rs/toolchain@v1
+      with:
+          toolchain: nightly-2024-02-06
           profile: minimal
-      - run: scripts/toolchain-after.sh sp1
-      - name: Install sp1
-        run: |
-          curl -L https://sp1.succinct.xyz | bash
-          echo "/home/runner/.config/.sp1/bin" >> $GITHUB_PATH
-          /home/runner/.config/.sp1/bin/sp1up
-      - name: Build sp1 guest target
-        run: cargo run --bin sp1-builder
-      - name: Build sp1 prover
-        run: cargo build --release --features sp1
-      - name: Test sp1 prover
-        run: cargo test -p sp1-driver --features enable --release
+    - name: Install cargo-binstall
+      uses: cargo-bins/[email protected]
+    - name: Install sp1
+      run: TARGET="sp1" CI=1 make install
+    - name: Build sp1 prover
+      run: TARGET="sp1" make build
+    - name: Test sp1 prover
+      run: TARGET="risc0" make test
 
   build-test-sgx:
     name: Build and test sgx
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
-      - run: scripts/toolchain-before.sh sgx
+        with:
+          submodules: recursive
       - uses: actions-rs/toolchain@v1
         with:
-          profile: minimal
-      - run: scripts/toolchain-after.sh sgx
-      - name: Setup and build
-        run: |
-          cargo build --features sgx
-          cargo test -p sgx-prover --features enable --release
-        shell: bash
+            toolchain: stable
+            profile: minimal
+      - name: Install cargo-binstall
+        uses: cargo-bins/[email protected]
+      - name: Install sgx
+        run: TARGET="sgx" CI=1 make install
+      - name: Build sgx prover
+        run: TARGET="sgx" make build
+      - name: Test sgx prover
+        run: TARGET="sgx" make test
 
   build-test-sgx-with-docker:
     name: Build and test sgx with Docker
@@ -127,4 +124,4 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - uses: risc0/risc0/.github/actions/[email protected]
-    - run: cargo fmt --all --check
+    - run: make fmt
diff --git a/README.md b/README.md
@@ -4,22 +4,18 @@
 
 ### Building
 
-- Install the `cargo risczero` tool and the `risc0` toolchain:
+- To download all dependencies for all provers you can run
 
 ```console
-$ cargo install cargo-risczero
-$ cargo risczero install --version v2024-02-08.1
+$ make install
 ```
-- Install the `cargo prove` tool and the `succinct` toolchain:
+
+You can also download all required dependencies for each prover separately, for example for SP1:
 
 ```console
-$ curl -L https://sp1.succinct.xyz | bash
-$ sp1up
-$ cargo prove --version
+$ TARGET="sp1" make install
 ```
 
-- For SGX, install gramine: https://github.com/gramineproject/gramine. If you're running ubuntu 22.04 (or a compatible distro) you can just download and install this deb file: https://packages.gramineproject.io/pool/main/g/gramine/gramine_1.6.2_amd64.deb
-
 - Clone the repository and build with `cargo`:
 
 ```console
@@ -51,22 +47,16 @@ You can also automatically sync with the tip of the chain and prove all new bloc
 
 ## Provers
 
-Provers can be enabled using features. To compile with all of them (using standard options):
-
-```
-cargo run --release --features "risc0 sp1"
-```
-
 ### risc zero
 
 Build using
 ```
-make risc0
+TARGET="risc0" make build
 ```
 
-#### Testing
+#### Running
 ```
-RISC0_DEV_MODE=1 cargo run --release --features risc0
+TARGET="risc0" make run
 ```
 
 #### Bonsai
@@ -79,7 +69,7 @@ prove_block.sh taiko_a7 risc0-bonsai 10
 
 #### CPU
 ```
-cargo run --release --features risc0
+TARGET="risc0" make run
 ```
 
 #### GPU
@@ -96,20 +86,18 @@ CUDA needs to be installed when using `cuda`: https://docs.nvidia.com/cuda/cuda-
 
 ### SP1:
 ```
-make sp1
-cargo run --release --features sp1
+TARGET="sp1" make build
+TARGET="sp1" make run
 ```
 
 ### SGX:
 ```
-make sgx
-cargo run --release --features sgx
+TARGET="sgx" make build
+TARGET="sgx" make run
 ```
 
-Make sure to first do a cargo build because cargo run does not build the sgx binary!
-
 If your CPU doesn't support SGX, you can still run the SGX code through gramine like it would on an SGX machine:
 
 ```
-SGX_DIRECT=1 cargo run --release --features sgx
+SGX_DIRECT=1 TARGET="sgx" make run
 ```
diff --git a/makefile b/makefile
@@ -1,14 +1,19 @@
 
-risc0:
-	@./toolchain.sh risc0
-	@cargo run --bin risc0-builder
-	cargo build --release --features risc0
-
-sp1:
-	@./toolchain.sh sp1
-	@cargo run --bin sp1-builder
-	cargo build --release --features sp1
-
-sgx:
-	@./toolchain.sh sgx
-	cargo build --release --features sgx
+install:
+	./scripts/install.sh $(TARGET)
+
+build:
+	./scripts/run.sh $(TARGET)
+
+run:
+	RUN=1 ./scripts/run.sh $(TARGET)
+
+test:
+	TEST=1 ./scripts/run.sh $(TARGET)
+
+fmt:
+	@cargo fmt --all --check
+
+clippy:
+	@cargo +nightly-2024-02-06 check --features "sgx,sp1,risc0"
+	@cargo +nightly-2024-02-06 clippy --workspace --features "sgx,sp1,risc0" --all-targets -- -Dwarnings
diff --git a/rust-toolchain b/rust-toolchain
@@ -1 +1 @@
-stable
+stable
diff --git a/scripts/install.sh b/scripts/install.sh
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+# toolchain necessary to compile c-kzg in SP1/risc0
+if [ -z "$1" ] || [ "$1" == "sp1" ] || [ "$1" == "risc0" ]; then
+	# Check if the RISC-V GCC prebuilt binary archive already exists
+	if [ -f /tmp/riscv32-unknown-elf.gcc-13.2.0.tar.gz ]; then
+		echo "riscv-gcc-prebuilt existed, please check the file manually"
+	else
+		# Download the file using wget
+		wget -O /tmp/riscv32-unknown-elf.gcc-13.2.0.tar.gz https://github.com/stnolting/riscv-gcc-prebuilt/releases/download/rv32i-131023/riscv32-unknown-elf.gcc-13.2.0.tar.gz
+		# Check if wget succeeded
+		if [ $? -ne 0 ]; then
+			echo "Failed to download riscv-gcc-prebuilt"
+			exit 1
+		fi
+		# Create the directory if it doesn't exist
+		if [ ! -d /opt/riscv ]; then
+			mkdir /opt/riscv
+		fi
+		# Extract the downloaded archive
+		tar -xzf /tmp/riscv32-unknown-elf.gcc-13.2.0.tar.gz -C /opt/riscv/
+		# Check if tar succeeded
+		if [ $? -ne 0 ]; then
+			echo "Failed to extract riscv-gcc-prebuilt"
+			exit 1
+		fi
+	fi
+fi
+
+# SGX
+if [ -z "$1" ] || [ "$1" == "sgx" ]; then
+	# For SGX, install gramine: https://github.com/gramineproject/gramine.
+	wget -O /tmp/gramine.deb https://packages.gramineproject.io/pool/main/g/gramine/gramine_1.6.2_amd64.deb
+	sudo apt install /tmp/gramine.deb
+fi
+# RISC0
+if [ -z "$1" ] || [ "$1" == "risc0" ]; then
+	cargo install cargo-risczero
+	cargo risczero install --version v2024-02-08.1
+fi
+# SP1
+if [ -z "$1" ] || [ "$1" == "sp1" ]; then
+	curl -L https://sp1.succinct.xyz | bash
+
+	if [ -z "${CI}" ]; then
+		# Need to add sp1up to the path here
+		PROFILE=$HOME/.bashrc
+		echo ${PROFILE}
+		source ${PROFILE}
+		sp1up
+	else
+		echo "/home/runner/.config/.sp1/bin" >> $GITHUB_PATH
+		/home/runner/.config/.sp1/bin/sp1up
+	fi
+fi
diff --git a/scripts/run.sh b/scripts/run.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+TOOLCHAIN_RISC0=+stable
+TOOLCHAIN_SP1=+nightly-2024-02-06
+TOOLCHAIN_SGX=+stable
+
+if [ -z "${DEBUG}" ]; then
+	FLAGS=--release
+fi
+
+
+if [ -z "${RUN}" ]; then
+	COMMAND=build
+else
+	COMMAND=run
+fi
+
+# SGX
+if [ -z "$1" ] || [ "$1" == "sgx" ]; then
+	if [ -z "${TEST}" ]; then
+		cargo ${TOOLCHAIN_SGX} ${COMMAND} ${FLAGS} --features sgx
+	else
+		cargo ${TOOLCHAIN_SGX} test ${FLAGS} -p sgx-prover --features enable
+	fi
+fi
+# RISC0
+if [ -z "$1" ] || [ "$1" == "risc0" ]; then
+	if [ -z "${TEST}" ]; then
+		cargo ${TOOLCHAIN_RISC0} run --bin risc0-builder
+		cargo ${TOOLCHAIN_RISC0} ${COMMAND} ${FLAGS} --features risc0
+	else
+		RISC0_DEV_MODE=1 cargo ${TOOLCHAIN_RISC0} test ${FLAGS} -p risc0-driver --features enable
+	fi
+fi
+# SP1
+if [ -z "$1" ] || [ "$1" == "sp1" ]; then
+	if [ -z "${TEST}" ]; then
+		cargo ${TOOLCHAIN_SP1} run --bin sp1-builder
+		cargo ${TOOLCHAIN_SP1} ${COMMAND} ${FLAGS} --features sp1
+	else
+		cargo ${TOOLCHAIN_SP1} test ${FLAGS} -p sp1-driver --features enable
+	fi
+fi
diff --git a/scripts/toolchain-after.sh b/scripts/toolchain-after.sh