setup OIDC (#1325)

maxday · Jul 5, 2024 · 87e2bf6 · 87e2bf6
1 parent 38619f3
commit 87e2bf6
Show file tree

Hide file tree

Showing 5 changed files with 143 additions and 110 deletions.
diff --git a/.github/workflows/aws-deploy.yml b/.github/workflows/aws-deploy.yml
@@ -1,12 +1,12 @@
 name: deploy to aws
 
+permissions:
+  id-token: write
+  contents: read
+
 on:
   workflow_dispatch:
     inputs:
-      mfaCode:
-        type: string
-        description: MFA Code
-        required: true
       region:
         description: 'region on which you want to deploy'
         type: choice
@@ -36,15 +36,6 @@ jobs:
             ~/.cargo/git/db/
             target/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-      - name: AWS Authentication
-        env:
-          AWS_MFA_ARN: ${{ secrets.RESTRICTED_AWS_MFA_ARN }}
-          AWS_REGION: ${{ inputs.region }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.RESTRICTED_AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.RESTRICTED_AWS_SECRET_ACCESS_KEY }}
-        run: |
-          creds_json=$(aws sts get-session-token --serial-number ${AWS_MFA_ARN} --token-code ${{ inputs.mfaCode }})
-          echo $creds_json > /tmp/creds_json
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@stable
       - name: Install Zig toolchain
@@ -66,23 +57,21 @@ jobs:
           cd ./result-builder && yarn install
       - name: install Serverless Framework
         run: sudo yarn global add serverless@^3.27.0 --prefix /usr/local
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          role-session-name: GitHub_Action_LambdaPerf_Session
+          aws-region: ${{ inputs.region }}
       - name: deploy
         env:
           AWS_REGION: ${{ inputs.region }}
-          AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
           GH_AUTH_TOKEN: ${{ secrets.GH_AUTH_TOKEN }}
           LAMBDA_PERF_ENV: ${{ inputs.lambdaPerfEnv }}
-        run: |
-          export AWS_ACCESS_KEY_ID=$(cat /tmp/creds_json | jq .Credentials.AccessKeyId |tr -d '"')
-          export AWS_SECRET_ACCESS_KEY=$(cat /tmp/creds_json | jq .Credentials.SecretAccessKey| tr -d '"')
-          export AWS_SESSION_TOKEN=$(cat /tmp/creds_json | jq .Credentials.SessionToken|tr -d '"')
-          sls deploy --force
+        run: sls deploy --force
       - name: Remove the cron job scheduler in staging (eu-west-1)
         if: "${{ inputs.region == 'eu-west-1' }}"
         run: |
-          export AWS_ACCESS_KEY_ID=$(cat /tmp/creds_json | jq .Credentials.AccessKeyId |tr -d '"')
-          export AWS_SECRET_ACCESS_KEY=$(cat /tmp/creds_json | jq .Credentials.SecretAccessKey| tr -d '"')
-          export AWS_SESSION_TOKEN=$(cat /tmp/creds_json | jq .Credentials.SessionToken|tr -d '"')
           RULE_NAME=$(aws events list-rule-names-by-target --target-arn arn:aws:lambda:${{ inputs.region }}:${{ secrets.AWS_ACCOUNT_ID }}:function:lambda-perf-dev-functionTriggerDeployerRs --region ${{ inputs.region }} | jq -r '.RuleNames[0]')
           echo "Disabling rule ${RULE_NAME}"
           aws events disable-rule --name $RULE_NAME --region ${{ inputs.region }}

diff --git a/.github/workflows/aws-container-upload.yml → ...workflows/aws-publish-artifact-to-ecr.yml b/.github/workflows/aws-container-upload.yml → ...workflows/aws-publish-artifact-to-ecr.yml
@@ -1,12 +1,12 @@
-name: build and upload container to ECR
+name: AWS publish artifacts to ECR
+
+permissions:
+  id-token: write
+  contents: read
 
 on:
   workflow_dispatch:
     inputs:
-      mfaCode:
-        type: string
-        description: MFA Code
-        required: true
       region:
         description: 'region on which you want to deploy'
         type: choice
@@ -27,6 +27,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       runner: ${{ steps.select.outputs.runner }}
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
       - name: Check branch
         id: select
@@ -37,11 +38,27 @@ jobs:
             echo "runner=self-hosted" >> $GITHUB_OUTPUT
           fi
           cat $GITHUB_OUTPUT
+      - uses: actions/checkout@v4
+      - name: Build runtime matrix
+        id: set-matrix
+        run: |
+          jq -c '[.runtimes[] | select(.architectures | index("arm64")) | select(.image != null and .image.baseImage != null) | .path]' manifest.json
+          echo "matrix=$(jq -c '[.runtimes[] | select(.architectures | index("${{ inputs.architecture }}")) | select(.image != null and .image.baseImage != null) | .path]' manifest.json)" >> $GITHUB_OUTPUT
+
   build-upload:
     needs: setup
     runs-on: ${{ needs.setup.outputs.runner }}
+    strategy:
+      matrix: 
+        runtime_id: ${{fromJson(needs.setup.outputs.matrix)}}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          role-session-name: GitHub_Action_LambdaPerf_Session
+          aws-region: ${{ inputs.region }}
       - name: Delete huge unnecessary folders
         run: |
           rm -rf /opt/hostedtoolcache
@@ -52,19 +69,26 @@ jobs:
         env:
           ARCHITECTURE: ${{ inputs.architecture }}
           AWS_REGION: ${{ inputs.region }}
-          ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.RESTRICTED_AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.RESTRICTED_AWS_SECRET_ACCESS_KEY }}
-          AWS_MFA_ARN: ${{ secrets.RESTRICTED_AWS_MFA_ARN }}
+          RUNTIME_ID: ${{ matrix.runtime_id }}
+          AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
         run: |
-          creds_json=$(aws sts get-session-token --serial-number ${AWS_MFA_ARN} --token-code ${{ inputs.mfaCode }})
-          export AWS_ACCESS_KEY_ID=$(echo "$creds_json" | jq .Credentials.AccessKeyId |tr -d '"')
-          export AWS_SECRET_ACCESS_KEY=$(echo "$creds_json" | jq .Credentials.SecretAccessKey| tr -d '"')
-          export AWS_SESSION_TOKEN=$(echo "$creds_json" | jq .Credentials.SessionToken|tr -d '"')
-
           cd container-uploader
           yarn install
           node app.mjs
 
+  clean-untagged-image:
+    needs: build-upload
+    runs-on: ubuntu-latest
+    steps:
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          role-session-name: GitHub_Action_LambdaPerf_Session
+          aws-region: ${{ inputs.region }}
+      - name: clean untagged images
+        env:
+          AWS_REGION: ${{ inputs.region }}
+        run: |
           UNTAGGED_IMAGES=$( aws ecr list-images --region ${AWS_REGION} --repository-name lambda-perf --filter "tagStatus=UNTAGGED" --query 'imageIds[*]' --output json )
           aws ecr batch-delete-image --region ${AWS_REGION} --repository-name lambda-perf --image-ids "$UNTAGGED_IMAGES"
diff --git a/.github/workflows/aws-s3-uploader.yml → .../workflows/aws-publish-artifact-to-s3.yml b/.github/workflows/aws-s3-uploader.yml → .../workflows/aws-publish-artifact-to-s3.yml
@@ -1,12 +1,12 @@
-name: build and upload artifacts in S3
+name: AWS publish artifacts to S3
+
+permissions:
+  id-token: write
+  contents: read
 
 on:
   workflow_dispatch:
     inputs:
-      mfaCode:
-        type: string
-        description: MFA Code
-        required: true
       region:
         description: 'region on which you want to deploy'
         type: choice
@@ -27,6 +27,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       runner: ${{ steps.select.outputs.runner }}
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
       - name: Check branch
         id: select
@@ -37,11 +38,27 @@ jobs:
             echo "runner=self-hosted" >> $GITHUB_OUTPUT
           fi
           cat $GITHUB_OUTPUT
+      - uses: actions/checkout@v4
+      - name: Build runtime matrix
+        id: set-matrix
+        run: |
+          jq -c '[.runtimes[] | select(.architectures | index("${{ inputs.architecture }}")) | .path]' manifest.json
+          echo "matrix=$(jq -c '[.runtimes[] | select(.architectures | index("${{ inputs.architecture }}")) | .path]' manifest.json)" >> $GITHUB_OUTPUT
+
   build-upload:
     needs: setup
     runs-on: ${{ needs.setup.outputs.runner }}
+    strategy:
+      matrix: 
+        runtime_id: ${{fromJson(needs.setup.outputs.matrix)}}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+      - name: configure aws credentials
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          role-session-name: GitHub_Action_LambdaPerf_Session
+          aws-region: ${{ inputs.region }}
       - name: Delete huge unnecessary folders
         run: |
           rm -rf /opt/hostedtoolcache
@@ -52,15 +69,8 @@ jobs:
         env:
           ARCHITECTURE: ${{ inputs.architecture }}
           AWS_REGION: ${{ inputs.region }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.RESTRICTED_AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.RESTRICTED_AWS_SECRET_ACCESS_KEY }}
-          AWS_MFA_ARN: ${{ secrets.RESTRICTED_AWS_MFA_ARN }}
+          RUNTIME_ID: ${{ matrix.runtime_id }}
         run: |
-          creds_json=$(aws sts get-session-token --serial-number ${AWS_MFA_ARN} --token-code ${{ inputs.mfaCode }})
-          export AWS_ACCESS_KEY_ID=$(echo "$creds_json" | jq .Credentials.AccessKeyId |tr -d '"')
-          export AWS_SECRET_ACCESS_KEY=$(echo "$creds_json" | jq .Credentials.SecretAccessKey| tr -d '"')
-          export AWS_SESSION_TOKEN=$(echo "$creds_json" | jq .Credentials.SessionToken|tr -d '"')
-
           cd s3-uploader
           yarn install
-          node app.mjs
+          node app.mjs
diff --git a/container-uploader/app.mjs b/container-uploader/app.mjs
@@ -95,36 +95,39 @@ const getFromS3 = async (
   }
 };
 
-const run = async () => {
-  const REGION = process.env.AWS_REGION;
-  const ACCOUNT_ID = process.env.ACCOUNT_ID;
-  const ARCHITECTURE = process.env.ARCHITECTURE;
+const run = async (accountId, runtime, architecture, region) => {
   const SLEEP_DELAY_IN_MILLISEC = 5000;
-
   const s3Client = new S3Client();
-  for (const runtime of manifest.runtimes) {
-    console.log(runtime);
-    for (const architecture of runtime.architectures) {
-      console.log(architecture);
-      if (architecture === ARCHITECTURE) {
-        console.log("building image");
-        if (runtime.hasOwnProperty("image")) {
-          const hasSpecificImageBuild =
-            runtime.hasOwnProperty("hasSpecificImageBuild") &&
-            runtime.hasSpecificImageBuild === true;
-          await buildDockerImage(
-            ACCOUNT_ID,
-            s3Client,
-            REGION,
-            runtime,
-            architecture,
-            hasSpecificImageBuild,
-            SLEEP_DELAY_IN_MILLISEC
-          );
-        }
-      }
-    }
+  console.log("building image");
+  if (runtime.hasOwnProperty("image")) {
+    const hasSpecificImageBuild =
+      runtime.hasOwnProperty("hasSpecificImageBuild") &&
+      runtime.hasSpecificImageBuild === true;
+    await buildDockerImage(
+      accountId,
+      s3Client,
+      region,
+      runtime,
+      architecture,
+      hasSpecificImageBuild,
+      SLEEP_DELAY_IN_MILLISEC
+    );
   }
 };
 
-await run();
+const runtimeFromRuntimeId = (manifest, runtimeId) => {
+  const runtime = manifest.find(r => {
+    return r.path === runtimeId;
+  });
+  if(!runtime) {
+    throw "cound not find the runtime"
+  }
+  console.log(runtime);
+  return runtime;
+}
+
+console.log('region = ', process.env.AWS_REGION);
+console.log('architecure = ', process.env.ARCHITECTURE);
+console.log('runtimeId = ', process.env.RUNTIME_ID);
+
+await run(process.env.AWS_ACCOUNT_ID, runtimeFromRuntimeId(manifest.runtimes, process.env.RUNTIME_ID), process.env.ARCHITECTURE, process.env.AWS_REGION);
diff --git a/s3-uploader/app.mjs b/s3-uploader/app.mjs
@@ -33,6 +33,7 @@ const build = async (path, architecture, hasSpecificImageBuild, nbRetry) => {
   console.log(
     `start building the artifact for ${path} arch = ${architecture}, hasSpecificImageBuild = ${hasSpecificImageBuild} and retry = ${nbRetry}`
   );
+
   if (nbRetry > 5) {
     throw new Error("Too many retries");
   }
@@ -51,41 +52,47 @@ const build = async (path, architecture, hasSpecificImageBuild, nbRetry) => {
     }
   } catch (e) {
     console.error(e);
-    await build(path, architecture, nbRetry + 1);
+    await build(path, architecture, hasSpecificImageBuild, nbRetry + 1);
   }
 };
 
-const upload = async () => {
-  const REGION = process.env.AWS_REGION;
-  const ARCHITECTURE = process.env.ARCHITECTURE;
+const upload = async (region, architecture, runtime) => {
   const SLEEP_DELAY_IN_MILLISEC = 5000;
+  const path = runtime.path;
+  const hasSpecificImageBuild =
+    runtime.hasOwnProperty("hasSpecificImageBuild") &&
+    runtime.hasSpecificImageBuild === true;
+  await build(path, architecture, hasSpecificImageBuild, 0);
 
-  for (const runtime of manifest.runtimes) {
-    for (const architecture of runtime.architectures) {
-      if (architecture === ARCHITECTURE) {
-        const path = runtime.path;
-        const hasSpecificImageBuild =
-          runtime.hasOwnProperty("hasSpecificImageBuild") &&
-          runtime.hasSpecificImageBuild === true;
-        await build(path, architecture, hasSpecificImageBuild, 0);
-
-        let codeFilename = `code_${architecture}.zip`;
-        await sendToS3(REGION, path, codeFilename, SLEEP_DELAY_IN_MILLISEC, 0);
-        // we need to upload a different artifact if the runtime
-        // has specific image build instructions
-        codeFilename = `code_${architecture}_image.zip`;
-        if (hasSpecificImageBuild) {
-          await sendToS3(
-            REGION,
-            path,
-            codeFilename,
-            SLEEP_DELAY_IN_MILLISEC,
-            0
-          );
-        }
-      }
-    }
+  let codeFilename = `code_${architecture}.zip`;
+  await sendToS3(region, path, codeFilename, SLEEP_DELAY_IN_MILLISEC, 0);
+  // we need to upload a different artifact if the runtime
+  // has specific image build instructions
+  codeFilename = `code_${architecture}_image.zip`;
+  if (hasSpecificImageBuild) {
+    await sendToS3(
+      region,
+      path,
+      codeFilename,
+      SLEEP_DELAY_IN_MILLISEC,
+      0
+    );
   }
 };
 
-await upload();
+const runtimeFromRuntimeId = (manifest, runtimeId) => {
+  const runtime = manifest.find(r => {
+    return r.path === runtimeId;
+  });
+  if(!runtime) {
+    throw "cound not find the runtime"
+  }
+  console.log(runtime);
+  return runtime;
+}
+
+console.log('region = ', process.env.AWS_REGION);
+console.log('architecure = ', process.env.ARCHITECTURE);
+console.log('runtimeId = ', process.env.RUNTIME_ID);
+
+await upload(process.env.AWS_REGION, process.env.ARCHITECTURE, runtimeFromRuntimeId(manifest.runtimes, process.env.RUNTIME_ID));