Fix the SGX Docker container (#102)

Provide Docker container for running SGX prover. This commit aligns the Raiko Docker image to ensure compatibility with the latest changes made on the unstable branch. The purpose of introducing a Docker container for the SGX prover was to minimize the effort required to set up the machine's infrastructure, such as configuring and starting Intel's PCCS service, bootstrapping the Raiko SGX prover, and populating configuration files. Up until now, we have relied on manually pre-configured machines running in Tencent Cloud. This commit enables running Raiko within a fully-fledged k8s environment (already tested by @johntaiko). Additionally, a CI test was added to build the Docker image from scratch, ensuring that no unintended code changes can break it. --------- Signed-off-by: smtmfft <[email protected]> Co-authored-by: smtmfft <[email protected]> Co-authored-by: john xu <[email protected]> Co-authored-by: Patryk Bęza <[email protected]>
taikoxyz · Apr 18, 2024 · d5f7389 · d5f7389
1 parent 4a37427
commit d5f7389
Show file tree

Hide file tree

Showing 22 changed files with 276 additions and 151 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -4,13 +4,12 @@
 # Allow files and directories
 !/Cargo.lock
 !/Cargo.toml
-!/config.toml
+!/config.json
 !/docker/entrypoint.sh
 !/lib
 !/primitives
 !/provers
 !/host
 !/rust-toolchain
-!/ra-tls
 !/taiko.toml
 !/tests
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -73,6 +73,18 @@ jobs:
           cargo test -p sgx-prover --features enable
         shell: bash
 
+  build-test-sgx-with-docker:
+    name: Build and test sgx with Docker
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup and build
+        run: |
+          cd docker
+          docker compose build --no-cache
+        shell: bash
+
   test-lib:
     name: Test raiko-lib
     runs-on: ubuntu-latest

diff --git a/Dockerfile b/Dockerfile
@@ -2,8 +2,6 @@ FROM rust:1.75.0 as builder
 
 ENV DEBIAN_FRONTEND=noninteractive
 ARG BUILD_FLAGS=""
-WORKDIR /opt/raiko
-COPY . .
 RUN apt-get update && \
     apt-get install -y \
     cmake \
@@ -12,16 +10,13 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/*
 
 # risc0 dependencies
-
 RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
 RUN cargo binstall -y --force cargo-risczero
 RUN cargo risczero install
 
-RUN cargo build --release ${BUILD_FLAGS}
-RUN ls -la /opt/raiko
-RUN ls -la /opt/raiko/target
-RUN ls -la /opt/raiko/target/release
-RUN ls -la /opt/raiko/target/release/host
+WORKDIR /opt/raiko
+COPY . .
+RUN cargo build --release ${BUILD_FLAGS} --features "sgx" --features "docker_build"
 
 
 FROM gramineproject/gramine:1.6-jammy as runtime
@@ -44,34 +39,24 @@ RUN curl -o setup.sh -sL https://deb.nodesource.com/setup_18.x && \
 
 RUN sed -i 's/#default quoting type = ecdsa_256/default quoting type = ecdsa_256/' /etc/aesmd.conf
 RUN sed -i 's/,"use_secure_cert": true/,"use_secure_cert": false/' /etc/sgx_default_qcnl.conf
-RUN sed -i 's/https:\/\/localhost:8081/https:\/\/pccs:8081/g' /etc/sgx_default_qcnl.conf
 
 RUN mkdir -p \
     ./bin \
     ./provers/sgx \
     /var/log/raiko
 
 COPY --from=builder /opt/raiko/docker/entrypoint.sh ./bin/
-COPY --from=builder /opt/raiko/provers/sgx/config/raiko-guest.manifest.template ./provers/sgx/
-COPY --from=builder /opt/raiko/host/config/config.toml /etc/raiko/
-RUN ls -la /opt/raiko
-RUN ls -la /opt/raiko/bin
-RUN ls -la /opt/raiko/provers
-RUN ls -la /opt/raiko/provers/sgx
-COPY --from=builder /opt/raiko/target/release/provers ./provers/sgx/
-# ubuntu@VM-0-6-ubuntu:~/zeth-john/provers/sgx$ cargo build --release
-# GROTH16_VERIFIER_ADDRESS="" cargo build --features "sgx" --release
-# ./provers/sp1/target
-# ./provers/risc0/guest/target
-# /target/release/host
-# "/opt/raiko/target/release/provers": not found
-
-# COPY --from=builder /opt/raiko/target/release/host ./bin/
-
-# ARG EDMM=0
-# ENV EDMM=${EDMM}
-# RUN cd ./provers/sgx && \
-#     gramine-manifest -Dlog_level=error -Darch_libdir=/lib/x86_64-linux-gnu/ raiko-guest.manifest.template raiko-guest.manifest
-
-# ENTRYPOINT [ "/opt/raiko/bin/entrypoint.sh" ]
-ENTRYPOINT [ "/bin/bash" ]
+COPY --from=builder /opt/raiko/provers/sgx/config/sgx-guest.docker.manifest.template ./provers/sgx/config/
+COPY --from=builder /opt/raiko/host/config/config.sgx.json /etc/raiko/
+COPY --from=builder /opt/raiko/target/release/sgx-guest ./bin/
+COPY --from=builder /opt/raiko/target/release/raiko-host ./bin/
+
+ARG EDMM=0
+ENV EDMM=${EDMM}
+RUN cd ./bin && \
+    gramine-sgx-gen-private-key -f && \
+    gramine-manifest -Dlog_level=error -Ddirect_mode=0 -Darch_libdir=/lib/x86_64-linux-gnu/ ../provers/sgx/config/sgx-guest.docker.manifest.template sgx-guest.manifest && \
+    gramine-sgx-sign --manifest sgx-guest.manifest --output sgx-guest.manifest.sgx && \
+    gramine-sgx-sigstruct-view "sgx-guest.sig"
+
+ENTRYPOINT [ "/opt/raiko/bin/entrypoint.sh" ]
diff --git a/Dockerfile.pccs b/Dockerfile.pccs
@@ -6,6 +6,9 @@ FROM ubuntu:23.04 AS builder
 # Define arguments used across multiple stages
 ARG DCAP_VERSION=DCAP_1.20
 ARG NODE_MAJOR=20
+ARG NODE_MINOR=11
+ARG NODE_PATCH=1
+ARG NODE_REVISION=1nodesource1
 
 # update and install packages, nodejs
 RUN DEBIAN_FRONTEND=noninteractive \
@@ -21,12 +24,12 @@ RUN DEBIAN_FRONTEND=noninteractive \
     && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /usr/share/keyrings/nodesource.gpg \
     && echo "deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list \
     && apt-get update -yq \
-    && apt-get install -yq --no-install-recommends nodejs \
+    && apt-get install -yq --no-install-recommends nodejs=${NODE_MAJOR}.${NODE_MINOR}.${NODE_PATCH}-${NODE_REVISION} \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
 # Clone the specific branch or tag
-RUN git clone --recurse-submodules https://github.com/intel/SGXDataCenterAttestationPrimitives.git # -b ${DCAP_VERSION} --depth 1
+RUN git clone --recurse-submodules https://github.com/intel/SGXDataCenterAttestationPrimitives.git -b ${DCAP_VERSION} --depth 1
 
 # Build libPCKCertSelection library
 WORKDIR /SGXDataCenterAttestationPrimitives/tools/PCKCertSelection/

diff --git a/README_Docker.md b/README_Docker.md
@@ -196,8 +196,8 @@ Bootstrapping involves generating a public-private key pair, which is used for s
    /home/ubuntu/.config/raiko
    ├── config
    │   ├── bootstrap.json
-   │   ├── raiko-guest.manifest.sgx
-   │   └── raiko-guest.sig
+   │   ├── sgx-guest.manifest.sgx
+   │   └── sgx-guest.sig
    └── secrets
       └── priv.key
    ```

diff --git a/README_SGX.md b/README_SGX.md
@@ -1,5 +1,7 @@
 # Raiko
 
+> :warning: **DISCLAIMER**: This `README.md` hasn't been updated for a while and may be outdated. For a more current guide, please refer to `README_Docker.md`.
+
 This project is Taiko-specific, SGX-enabled fork of [Zeth][zeth] called _Raiko_. It consists of 2 'modules': `raiko-guest` and `host`.
 
 - `host` is capable of fetching relevant block data and saving it to the `*.json.gz` file. `host` is _not_ being run inside SGX enclave.
@@ -46,8 +48,8 @@ To run `raiko-guest` in _one-shot_ mode with SGX using Gramine:
 1. Compile Gramine's configuration file:
    ```console
    ubuntu@ubuntu:~/zeth$ cd target/debug
-   ubuntu@ubuntu:~/zeth/target/debug$ cp ../../raiko-guest/config/raiko-guest.manifest.template .
-   ubuntu@ubuntu:~/zeth/target/debug$ gramine-manifest -Dlog_level=error -Darch_libdir=/lib/x86_64-linux-gnu/ raiko-guest.manifest.template raiko-guest.manifest
+   ubuntu@ubuntu:~/zeth/target/debug$ cp ../../raiko-guest/config/raiko-guest.local.manifest.template .
+   ubuntu@ubuntu:~/zeth/target/debug$ gramine-manifest -Dlog_level=error -Darch_libdir=/lib/x86_64-linux-gnu/ raiko-guest.local.manifest.template raiko-guest.manifest
    ```
 1. Sign Gramine's configuration file. [`MRENCLAVE`][mrenclave] – a.k.a. [_measurement_][measurement] – is also calculated at this stage (see last line of the below log):
 
@@ -225,7 +227,7 @@ Copy `sgx-guest` binary:
 cargo build
 cp target/debug/raiko-guest host/provers/sgx
 cd host/provers/sgx
-gramine-manifest -Dlog_level=error -Darch_libdir=/lib/x86_64-linux-gnu/ raiko-guest.manifest.template raiko-guest.manifest
+gramine-manifest -Dlog_level=error -Darch_libdir=/lib/x86_64-linux-gnu/ raiko-guest.local.manifest.template raiko-guest.manifest
 gramine-sgx-sign --manifest raiko-guest.manifest --output raiko-guest.manifest.sgx
 cd -
 ```

diff --git a/docker/config_sgx.sh b/docker/config_sgx.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+if [ -z $1 ] ; then
+    echo "Please provide the new sgx ID"
+    exit 1
+fi
+NEW_ID=$1
+
+
+FILTER_NAME="raiko"
+CONTAINER_ID=$(docker ps --filter "name=$FILTER_NAME" --format "{{.ID}}")
+echo "Ready to config container: $CONTAINER_ID"
+
+# pre-check
+echo "Old config"
+docker exec $CONTAINER_ID cat /etc/raiko/config.sgx.json
+echo
+docker exec $CONTAINER_ID sed -i "s/123456/$NEW_ID/" /etc/raiko/config.sgx.json
+# post-check update
+echo "New config"
+docker exec $CONTAINER_ID cat /etc/raiko/config.sgx.json
diff --git a/docker/docker-compose-pos.yml b/docker/docker-compose-pos.yml
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -1,12 +1,31 @@
 version: "3.9"
 services:
+  init:
+    build:
+      context: ..
+      dockerfile: Dockerfile
+    image: gcr.io/evmchain/raiko:latest
+    container_name: raiko-init
+    command: --init
+    devices:
+      - "/dev/sgx_enclave:/dev/sgx_enclave"
+      - "/dev/sgx_provision:/dev/sgx_provision"
+    volumes:
+      - /var/log/raiko:/var/log/raiko
+      - ${HOME}/.config/gramine:/root/.config/gramine
+      - ${HOME}/.config/raiko:/root/.config/raiko
+    # environment:
+      # you can use your own PCCS host
+      #- PCCS_HOST=host.docker.internal:8081
+    depends_on:
+      - pccs
   raiko:
     build:
       context: ..
       dockerfile: Dockerfile
     image: gcr.io/evmchain/raiko:latest
     container_name: raiko
-    command: --config-path=/etc/raiko/config.toml
+    command: --config-path=/etc/raiko/config.sgx.json
     devices:
       - "/dev/sgx_enclave:/dev/sgx_enclave"
       - "/dev/sgx_provision:/dev/sgx_provision"
@@ -16,10 +35,17 @@ services:
       - ${HOME}/.config/raiko:/root/.config/raiko
     ports:
       - "8080:8080"
+    environment:
+      # Set to 0 (which is the default) to run on real hardware; use 1 for testing
+      - SGX_DIRECT=${SGX_DIRECT}
+      - SGX_INSTANCE_ID=${SGX_INSTANCE_ID}
+      # you can use your own PCCS host
+      #- PCCS_HOST=host.docker.internal:8081
+    # use the host's network to connect to the PCCS
+    #extra_hosts:
+    #  - "host.docker.internal:host-gateway"
     depends_on:
       - pccs
-    environment:
-      - GITHUB_TOKEN=${GITHUB_TOKEN}
   pccs:
     build:
       context: ..
@@ -30,7 +56,3 @@ services:
       - ${HOME}/.config/sgx-pccs/default.json:/opt/intel/pccs/config/default.json
       - ${HOME}/.config/sgx-pccs/file.crt:/opt/intel/pccs/ssl_key/file.crt
       - ${HOME}/.config/sgx-pccs/private.pem:/opt/intel/pccs/ssl_key/private.pem
-    ports:
-      - "8082:8081"
-    environment:
-      - GITHUB_TOKEN=${GITHUB_TOKEN}
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
@@ -7,14 +7,14 @@ RAIKO_DOCKER_VOLUME_PATH="/root/.config/raiko"
 RAIKO_DOCKER_VOLUME_CONFIG_PATH="$RAIKO_DOCKER_VOLUME_PATH/config"
 RAIKO_DOCKER_VOLUME_SECRETS_PATH="$RAIKO_DOCKER_VOLUME_PATH/secrets"
 RAIKO_DOCKER_VOLUME_PRIV_KEY_PATH="$RAIKO_DOCKER_VOLUME_SECRETS_PATH/priv.key"
-RAIKO_GUEST_APP_DIR="/opt/raiko/provers/sgx"
-RAIKO_GUEST_APP_FILENAME="raiko-guest"
-RAIKO_INPUT_MANIFEST_FILENAME="$RAIKO_GUEST_APP_FILENAME.manifest"
+RAIKO_APP_DIR="/opt/raiko/bin"
+RAIKO_GUEST_APP_FILENAME="sgx-guest"
+RAIKO_INPUT_MANIFEST_FILENAME="$RAIKO_GUEST_APP_FILENAME.docker.manifest.template"
 RAIKO_OUTPUT_MANIFEST_FILENAME="$RAIKO_GUEST_APP_FILENAME.manifest.sgx"
 RAIKO_SIGNED_MANIFEST_FILENAME="$RAIKO_GUEST_APP_FILENAME.sig"
 
 function sign_gramine_manifest() {
-    cd "$RAIKO_GUEST_APP_DIR"
+    cd "$RAIKO_APP_DIR"
     gramine-sgx-sign --manifest "$RAIKO_INPUT_MANIFEST_FILENAME" --output "$RAIKO_OUTPUT_MANIFEST_FILENAME"
     mkdir -p "$RAIKO_DOCKER_VOLUME_CONFIG_PATH"
     cp "$RAIKO_OUTPUT_MANIFEST_FILENAME" "$RAIKO_SIGNED_MANIFEST_FILENAME" "$RAIKO_DOCKER_VOLUME_CONFIG_PATH"
@@ -23,26 +23,32 @@ function sign_gramine_manifest() {
 
 function bootstrap() {
     mkdir -p "$RAIKO_DOCKER_VOLUME_SECRETS_PATH"
-    cd "$RAIKO_GUEST_APP_DIR"
+    cd "$RAIKO_APP_DIR"
     gramine-sgx "$RAIKO_GUEST_APP_FILENAME" bootstrap
     cd -
 }
 
+if [[ -z "${PCCS_HOST}" ]]; then
+    MY_PCCS_HOST=pccs:8081
+else
+    MY_PCCS_HOST=${PCCS_HOST}
+fi
+
+sed -i "s/https:\/\/localhost:8081/https:\/\/${MY_PCCS_HOST}/g" /etc/sgx_default_qcnl.conf
+sed -i "s/123456/${SGX_INSTANCE_ID}/" /etc/raiko/config.sgx.json
 /restart_aesm.sh
 
+echo $#
 if [[ $# -eq 1 && $1 == "--init" ]]; then
-    if [[ ! -f "$GRAMINE_PRIV_KEY" ]]; then
-        gramine-sgx-gen-private-key
-    fi
-    sign_gramine_manifest
+    echo "start bootstrap"
     bootstrap
 else
+    echo "start proving"
     if [[ ! -f "$RAIKO_DOCKER_VOLUME_PRIV_KEY_PATH" ]]; then
         echo "Application was not bootstrapped. "\
              "$RAIKO_DOCKER_VOLUME_PRIV_KEY_PATH is missing. Bootstrap it first." >&2
         exit 1
     fi
 
-    sign_gramine_manifest
-    /opt/raiko/bin/host "$@"
+    /opt/raiko/bin/raiko-host "$@"
 fi
diff --git a/docker/pccs-defaut-example.json b/docker/pccs-defaut-example.json
@@ -0,0 +1,54 @@
+{
+  "HTTPS_PORT": 8081,
+  "hosts": "0.0.0.0",
+  "uri": "https://api.trustedservices.intel.com/sgx/certification/v4/",
+  "ApiKey": "YOUR_API_KEY",
+  "proxy": "",
+  "RefreshSchedule": "0 0 1 * * *",
+  "UserTokenHash": "YOUR_USER_TOKEN_HASH",
+  "AdminTokenHash": "YOUR_ADMIN_TOKEN_HASH",
+  "CachingFillMode": "LAZY",
+  "OPENSSL_FIPS_MODE": false,
+  "LogLevel": "info",
+  "DB_CONFIG": "sqlite",
+  "sqlite": {
+    "database": "database",
+    "username": "username",
+    "password": "password",
+    "options": {
+      "host": "localhost",
+      "dialect": "sqlite",
+      "pool": {
+        "max": 5,
+        "min": 0,
+        "acquire": 30000,
+        "idle": 10000
+      },
+      "define": {
+        "freezeTableName": true
+      },
+      "logging": false,
+      "storage": "pckcache.db"
+    }
+  },
+  "mysql": {
+    "database": "pckcache",
+    "username": "root",
+    "password": "mypass",
+    "options": {
+      "host": "localhost",
+      "port": "3306",
+      "dialect": "mysql",
+      "pool": {
+        "max": 5,
+        "min": 0,
+        "acquire": 30000,
+        "idle": 10000
+      },
+      "define": {
+        "freezeTableName": true
+      },
+      "logging": false
+    }
+  }
+}
diff --git a/host/config/config.sgx.json b/host/config/config.sgx.json
@@ -0,0 +1,6 @@
+{
+    "network": "taiko_a7",
+    "sgx": {
+        "instance_id": 123456
+    }
+}