Update external prs app to handle ossd PR validation (#1182)

* Update external-prs app to namespace

* Update gh workflows to use new namespacing

.github/workflows/external-prs-handle-comment.yml

@@ -30,7 +30,7 @@ jobs:
       - name: Parse the comment to see if it's a deploy comment
         id: parse_comment
         run: |
-          cd ops/external-prs && pnpm tools parse-comment ${{ github.repository }} ${{ github.event.comment.id }} $GITHUB_OUTPUT 
+          cd ops/external-prs && pnpm tools oso parse-comment --repo ${{ github.repository }} ${{ github.event.comment.id }} $GITHUB_OUTPUT 
 
       # - name: Deploy if the deployment is eligible
       #   run: |

.github/workflows/refresh-test-credentials.yml

@@ -8,6 +8,8 @@
 # This workflow will run every 30 mins to hopefully ensure that credentials
 # don't expire even if this script errors.
 name: refresh-test-credentials 
+env:
+  PR_TOOLS_REPO: ${{ github.repository }}
 
 # For now this only runs on a schedule once a day. Once we have made some of the
 # plugin workflows more incremental we will run this on _every_ commit to main
@@ -54,22 +56,23 @@ jobs:
         run: |
           cd ops/external-prs &&
           bash scripts/rotate-service-account.sh [email protected] dummy.json && 
-          pnpm tools refresh-gcp-credentials --secret=false ${{ github.repository }} testing dummy.json GOOGLE_TEST_DUMMY_CREDENTIALS_JSON
+          pnpm tools oso refresh-gcp-credentials --secret=false testing dummy.json GOOGLE_TEST_DUMMY_CREDENTIALS_JSON
+        
       
       # These credentials are intended to be secret
       - name: Refresh credentials for the bigquery-admin user on the external-prs-app environment
         shell: bash
         run: |
           cd ops/external-prs &&
           bash scripts/rotate-service-account.sh [email protected] bigquery-admin.json && 
-          pnpm tools refresh-gcp-credentials ${{ github.repository }} external-prs-app bigquery-admin.json GOOGLE_BQ_ADMIN_CREDENTIALS_JSON
+          pnpm tools oso refresh-gcp-credentials external-prs-app bigquery-admin.json GOOGLE_BQ_ADMIN_CREDENTIALS_JSON
       
       - name: Refresh credentials for the bigquery-admin user on the deploy environment
         shell: bash
         run: |
           cd ops/external-prs &&
           bash scripts/rotate-service-account.sh [email protected] bigquery-admin.json && 
-          pnpm tools refresh-gcp-credentials ${{ github.repository }} deploy bigquery-admin.json GOOGLE_BQ_ADMIN_CREDENTIALS_JSON
+          pnpm tools refresh-gcp-credentials oso deploy bigquery-admin.json GOOGLE_BQ_ADMIN_CREDENTIALS_JSON
 
   rebuild-docker-public-vars:
     name: rebuild-docker-public-vars

.github/workflows/test-deploy-clean.yml

@@ -39,5 +39,5 @@ jobs:
 
       - name: Clean up datasets that are older than 1 day
         run: |
-          cd ops/external-ops && pnpm tools test-deploy clean --project-id ${{ vars.GOOGLE_PROJECT_ID }} opensource-observer/oso 86400
+          cd ops/external-ops && pnpm tools oso test-deploy clean --project-id ${{ vars.GOOGLE_PROJECT_ID }} --repo opensource-observer/oso 86400
           

.github/workflows/test-deploy-owners.yml

@@ -11,6 +11,7 @@ env:
   # should not be set to a legitimate value for testing. This will use up API
   # quota otherwise
   DUNE_API_KEY: "none"
+  PR_TOOLS_REPO: ${{ github.repository }}
 
 # Trigger the workflow when:
 on:
@@ -40,7 +41,7 @@ jobs:
       - name: Initialize check
         run: |
           cd ops/external-prs &&
-          pnpm tools initialize-check ${{ github.repository }} ${{ github.event.pull_request.head.sha }} ${{ github.event.pull_request.user.login }}
+          pnpm tools oso initialize-check ${{ github.event.pull_request.head.sha }} ${{ github.event.pull_request.user.login }}
 
       - name: Author association debug
         run: |

.github/workflows/test-deploy/action.yml

@@ -43,7 +43,7 @@ runs:
     # - name: Evaluate if this is eligible for deployment
     #   id: is_eligible
     #   run: |
-    #     cd ops/external-prs && pnpm tools test-deploy is-eligible $GITHUB_OUTPUT
+    #     cd ops/external-prs && pnpm tools oso test-deploy is-eligible $GITHUB_OUTPUT
 
     - name: checkout the PR
       uses: actions/checkout@v3
@@ -72,6 +72,8 @@ runs:
       shell: bash
       run: |
         mkdir -p $HOME/.dbt &&
-        cd ops/external-prs && pnpm tools test-deploy --project-id ${{ inputs.google_project_id }} setup ${{ github.repository }} ${{ inputs.pr }} ${{ inputs.sha }} $HOME/.dbt/profiles.yml ${{ inputs.gcp_service_account_path }} pr-clone
+        cd ops/external-prs && pnpm tools oso test-deploy --project-id ${{ inputs.google_project_id }} setup ${{ inputs.pr }} ${{ inputs.sha }} $HOME/.dbt/profiles.yml ${{ inputs.gcp_service_account_path }} pr-clone
+      env:
+        PR_TOOLS_REPO: ${{ github.repository }}
 
 

ops/external-prs/README.md

@@ -3,3 +3,47 @@
 A Github App that allows us to accept external PRs by enabling PR checks by an
 out of band process. We need this because some of the checks require _some_ form
 of authentication.
+
+## Usage
+
+This is generally only supposed to be used on Github Actions. However, if you're
+developing this locally it's useful to know the available commands.
+
+### General commands
+
+```bash
+# Initialize a check with a given check name
+# If the user has write access then it automatically is listed as "queued"
+# If the user does not have write access then it automatically fails and waits for approval
+pnpm external-prs initialize-check {sha} {login} {check-name}
+```
+
+### OSO Specific
+
+```bash
+# Handle oso comments
+pnpm external-prs oso parse-comment {comment} {output}
+
+# Refresh gcp credentials for the test deployment infrastructure
+pnpm external-prs oso refresh-gcp-credentials {environment} {creds-path} {name}
+
+# Test deployment sub commands
+pnpm external-prs oso test-deploy --help
+
+# Test deployment setup
+pnpm external-prs oso test-deploy setup {pr} {sha} {profile-path} {service-account-path} {checkout-path}
+
+# Test deployment teardown
+pnpm external-prs oso test-deploy teardown {pr}
+
+# Test deployment clean
+pnpm external-prs oso test-deploy clean {ttl-seconds}
+```
+
+### OSS-Directory Specific
+
+```bash
+# Handle PR validations
+pnpm external-prs ossd validate-prs {pr} {sha} {main-path} {pr-path}
+
+```

ops/external-prs/package.json

@@ -5,6 +5,9 @@
   "author": "Kariba Labs",
   "license": "Apache-2.0",
   "private": true,
+  "bin": {
+    "external-prs": "./dist/src/cli.js"
+  },
   "main": "./dist/src/index.js",
   "types": "./dist/src/index.d.ts",
   "type": "module",
@@ -31,12 +34,17 @@
   },
   "dependencies": {
     "@google-cloud/bigquery": "^7.5.1",
+    "@types/columnify": "^1.5.4",
     "@types/libsodium-wrappers": "^0.7.13",
     "@types/yargs": "^17.0.32",
     "chalk": "^5.3.0",
+    "columnify": "^1.6.0",
     "dayjs": "^1.11.10",
+    "duckdb": "^0.10.1",
     "libsodium-wrappers": "^0.7.13",
     "octokit": "^3.1.0",
+    "oss-directory": "^0.0.10",
+    "tmp-promise": "^3.0.3",
     "ts-dedent": "^2.2.0",
     "winston": "^3.11.0",
     "yaml": "^2.3.1",

ops/external-prs/src/base.ts

@@ -0,0 +1,9 @@
+import { Repo } from "./github.js";
+import { App } from "octokit";
+
+export interface BaseArgs {
+  githubAppPrivateKey: string;
+  githubAppId: string;
+  repo: Repo;
+  app: App;
+}