[CORE-265] Replace toolsalpha with qa env (#178)

* Replace toolsalpha with tools env

The toolsalpha env was deleted

* Re-enable integration tests for Google resources

* Update actions/upload-artifact from deprecated v3 to v4

* Use tools client-sa in tests

* use new pubsub queue

* Update render config secrets

* Fix pubsub topic id

* Fix TestConfiguration comments

Env is now QA

* Make render secrets re-run

The actions were using the old cached secrets.
Update the step so that it re-runs

* Fix typos

Fix render-config variable and
pubsub project id for integration tests

* Uncomment two more tests

.github/workflows/test.yml

@@ -17,8 +17,7 @@ env:
   JANITOR_CLIENT_SA_FILE: src/test/resources/rendered/client-sa-account.json
   JANITOR_CLIENT_SA_TOOLS_FILE: src/test/resources/rendered/tools-client-sa-account.json
   JANITOR_CLOUD_ACCESS_SA_FILE: src/test/resources/rendered/cloud-access-sa-account.json
-  LOCAL_PROPERTIES_DIR: config
-
+  LOCAL_PROPERTIES_DIRECTORY: config
 jobs:
   unit-test:
     runs-on: ubuntu-latest
@@ -68,16 +67,16 @@ jobs:
           JANITOR_CLOUD_ACCESS_SA=$(echo $JANITOR_CLOUD_ACCESS_SA_B64 | base64 --decode)
           echo ::add-mask::$JANITOR_CLOUD_ACCESS_SA
           echo $JANITOR_CLOUD_ACCESS_SA > $JANITOR_CLOUD_ACCESS_SA_FILE
-
+          
           AZURE_PUBLISHER_CLIENT_ID=${{ secrets.AZURE_PUBLISHER_CLIENT_ID }}
           echo ::add-mask::$AZURE_PUBLISHER_CLIENT_ID
           AZURE_PUBLISHER_CLIENT_SECRET=${{ secrets.AZURE_PUBLISHER_CLIENT_SECRET }}
           echo ::add-mask::$AZURE_PUBLISHER_CLIENT_SECRET
           AZURE_PUBLISHER_TENANT_ID=${{ secrets.AZURE_PUBLISHER_TENANT_ID }}
           echo ::add-mask::$AZURE_PUBLISHER_TENANT_ID
-
-          mkdir -p "${LOCAL_PROPERTIES_DIR}"
-          cat << EOF > ${LOCAL_PROPERTIES_DIR}/local-properties.yml
+          
+          mkdir -p "${LOCAL_PROPERTIES_DIRECTORY}"
+          cat << EOF > ${LOCAL_PROPERTIES_DIRECTORY}/local-properties.yml
           janitor:
             azure:
               managed-app-client-id: ${AZURE_PUBLISHER_CLIENT_ID}

local-dev/render-config.sh

@@ -7,13 +7,13 @@
 # TODO: migrate vault secrets to GSM as needed
 VAULT_TOKEN=${1:-$(cat $HOME/.vault-token)}
 DSDE_TOOLBOX_DOCKER_IMAGE=broadinstitute/dsde-toolbox:dev
-VAULT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/integration/toolsalpha/crl_janitor/app-sa
-VAULT_CLIENT_SERVICE_ACCOUNT_PATH=secret/dsde/terra/kernel/integration/toolsalpha/crl_janitor/client-sa
 VAULT_CLOUD_ACCESS_SERVICE_ACCOUNT_PATH=secret/dsde/terra/janitor-test/default/cloud-access-sa
 VAULT_AZURE_MANAGED_APP_PUBLISHER_PATH=secret/dsde/terra/azure/common/managed-app-publisher
 
 # GSM secrets
 GSM_CLIENT_SERVICE_ACCOUNT_SECRET=crljanitor-client-sa
+GSM_QA_APP_SERVICE_ACCOUNT_SECRET=crljanitor-qa-sa
+GSM_QA_CLIENT_SERVICE_ACCOUNT_SECRET=crljanitor-client-qa-sa
 GSM_CLIENT_SERVICE_ACCOUNT_PROJECT=broad-dsde-qa
 
 # Rendered paths
@@ -25,13 +25,6 @@ AZURE_MANAGED_APP_PUBLISHER_OUTPUT_FILE_PATH="$(dirname $0)"/../src/test/resourc
 LOCAL_PROPERTIES_DIR="$(dirname $0)"/../config
 
 # Pull secrets from vault
-docker run --rm -e VAULT_TOKEN=$VAULT_TOKEN ${DSDE_TOOLBOX_DOCKER_IMAGE} \
-            vault read -format json ${VAULT_SERVICE_ACCOUNT_PATH} \
-            | jq -r .data.key | base64 -d > ${SERVICE_ACCOUNT_OUTPUT_FILE_PATH}
-docker run --rm --cap-add IPC_LOCK \
-            -e VAULT_TOKEN=$VAULT_TOKEN ${DSDE_TOOLBOX_DOCKER_IMAGE} \
-            vault read -format json ${VAULT_CLIENT_SERVICE_ACCOUNT_PATH} \
-            | jq -r .data.key | base64 -d > ${CLIENT_SERVICE_ACCOUNT_OUTPUT_FILE_PATH}
 docker run --rm --cap-add IPC_LOCK \
             -e VAULT_TOKEN=$VAULT_TOKEN ${DSDE_TOOLBOX_DOCKER_IMAGE} \
             vault read -format json ${VAULT_CLOUD_ACCESS_SERVICE_ACCOUNT_PATH} \
@@ -45,6 +38,12 @@ docker run --rm --cap-add IPC_LOCK \
 gcloud secrets versions access latest --project $GSM_CLIENT_SERVICE_ACCOUNT_PROJECT --secret $GSM_CLIENT_SERVICE_ACCOUNT_SECRET \
   | jq -r '.key' | base64 -d > "$TOOLS_CLIENT_SERVICE_ACCOUNT_OUTPUT_FILE_PATH"
 
+gcloud secrets versions access latest --project $GSM_CLIENT_SERVICE_ACCOUNT_PROJECT --secret $GSM_QA_APP_SERVICE_ACCOUNT_SECRET \
+  | jq -r '.key' | base64 -d > "$SERVICE_ACCOUNT_OUTPUT_FILE_PATH"
+
+gcloud secrets versions access latest --project $GSM_CLIENT_SERVICE_ACCOUNT_PROJECT --secret $GSM_QA_CLIENT_SERVICE_ACCOUNT_SECRET \
+  | jq -r '.key' | base64 -d > "$CLIENT_SERVICE_ACCOUNT_OUTPUT_FILE_PATH"
+
 # Write the Azure configuration into the local-properties.yml file
 mkdir -p "${LOCAL_PROPERTIES_DIR}"
 AZURE_MANAGED_APP_CLIENT_ID=$(jq -r '."client-id"' ${AZURE_MANAGED_APP_PUBLISHER_OUTPUT_FILE_PATH})

src/test/java/bio/terra/janitor/integration/TrackResourceIntegrationTest.java

@@ -206,7 +206,6 @@ public void tearDownPubsub() throws Exception {
     publisher.shutdown();
   }
 
-  @Disabled
   @Test
   public void subscribeAndCleanupResource_googleBucket() throws Exception {
     // Creates bucket and verify.
@@ -230,7 +229,6 @@ public void subscribeAndCleanupResource_googleBucket() throws Exception {
   }
 
   /** Try to let Janitor cleanup a Bucket that is already deleted in cloud. */
-  @Disabled
   @Test
   public void subscribeAndCleanupResource_alreadyDeletedBucket() throws Exception {
     // Creates bucket and verify.
@@ -248,7 +246,6 @@ public void subscribeAndCleanupResource_alreadyDeletedBucket() throws Exception
     publishAndVerify(resource, ResourceState.DONE);
   }
 
-  @Disabled
   @Test
   public void subscribeAndCleanupResource_googleBlob() throws Exception {
     // Creates Blob and verify.
@@ -273,7 +270,6 @@ public void subscribeAndCleanupResource_googleBlob() throws Exception {
   }
 
   /** Try to let Janitor cleanup a Blob that is already deleted in cloud. */
-  @Disabled
   @Test
   public void subscribeAndCleanupResource_alreadyDeletedBlob() throws Exception {
     // Creates Blob and verify.
@@ -298,7 +294,6 @@ public void subscribeAndCleanupResource_alreadyDeletedBlob() throws Exception {
     storageCow.delete(bucketName);
   }
 
-  @Disabled
   @Test
   public void subscribeAndCleanupResource_googleDataset() throws Exception {
     // Creates dataset and table.
@@ -371,7 +366,6 @@ public void subscribeAndCleanupResource_googleDataset() throws Exception {
     publishAndVerify(tableUid, ResourceState.DONE);
   }
 
-  @Disabled
   @Test
   public void subscribeAndCleanupResource_googleBigQueryTable() throws Exception {
     // Creates dataset and table.
@@ -431,7 +425,6 @@ public void subscribeAndCleanupResource_googleBigQueryTable() throws Exception {
     bigQueryCow.datasets().delete(projectId, datasetName).execute();
   }
 
-  @Disabled
   @Test
   public void subscribeAndCleanupResource_googleNotebookInstance() throws Exception {
     InstanceName instanceName =
@@ -458,7 +451,6 @@ public void subscribeAndCleanupResource_googleNotebookInstance() throws Exceptio
     assertEquals(404, e.getStatusCode());
   }
 
-  @Disabled
   @Test
   public void subscribeAndCleanupResource_alreadyDeletedGoogleNotebookInstance() throws Exception {
     InstanceName instanceName =
@@ -496,7 +488,6 @@ public void subscribeAndCleanupResource_alreadyDeletedGoogleNotebookInstance() t
     assertEquals(404, e.getStatusCode());
   }
 
-  @Disabled
   @Test
   public void subscribeAndCleanupResource_googleProject() throws Exception {
     String projectId = randomProjectId();
@@ -513,7 +504,6 @@ public void subscribeAndCleanupResource_googleProject() throws Exception {
     assertEquals("DELETE_REQUESTED", project.getState());
   }
 
-  @Disabled
   @Test
   public void subscribeAndCleanupResource_alreadyDeletedGoogleProject() throws Exception {
     String projectId = randomProjectId();
@@ -529,7 +519,6 @@ public void subscribeAndCleanupResource_alreadyDeletedGoogleProject() throws Exc
     assertEquals("DELETE_REQUESTED", project.getState());
   }
 
-  @Disabled
   @Test
   public void subscribeAndCleanupResource_neverCreatedGoogleProject_withMetadataOk()
       throws Exception {
@@ -550,7 +539,6 @@ public void subscribeAndCleanupResource_neverCreatedGoogleProject_withMetadataOk
     publishAndVerify(request, ResourceState.DONE);
   }
 
-  @Disabled
   @Test
   public void subscribeAndCleanupResource_neverCreatedGoogleProject_withoutMetadataError()
       throws Exception {
@@ -727,7 +715,6 @@ public void subscribeAndCleanupResource_azureVirtualMachine() throws Exception {
   }
 
   /** Clean up a fake WSM workspace. */
-  @Disabled
   @Test
   public void subscribeAndCleanupResource_terraWorkspace() throws Exception {
     // Cleaning up workspaces relies on domain-wide delegation to impersonate test users. The tools
@@ -749,7 +736,6 @@ public void subscribeAndCleanupResource_terraWorkspace() throws Exception {
   }
 
   /** Try to clean up an already deleted workspace, should succeed. */
-  @Disabled
   @Test
   public void subscribeAndCleanupResource_alreadyDeletedTerraWorkspace() throws Exception {
     UUID fakeWorkspaceId = UUID.randomUUID();

src/test/java/bio/terra/janitor/integration/common/configuration/TestConfiguration.java

@@ -19,19 +19,19 @@ public class TestConfiguration {
   /** How long to keep the resource before the 'prod' Janitor do the cleanup. */
   public static Duration RESOURCE_TIME_TO_LIVE_PROD = Duration.ofMinutes(30);
 
-  /** pubsub project id to publish track resource to Janitor test env(toolsalpha) */
+  /** pubsub project id to publish track resource to Janitor test env(qa) */
   private String resourceProjectId;
 
   /** pubsub project id to publish track resource to Janitor prod env(tools) */
   private String prodTrackResourceProjectId;
 
-  /** pubsub topic id to publish track resource to Janitor test env(toolsalpha) */
+  /** pubsub topic id to publish track resource to Janitor test env(qa) */
   private String trackResourceTopicId;
 
   /** pubsub topic id to publish track resource to Janitor prod env(tools) */
   private String prodTrackResourceTopicId;
 
-  /** Credential file path to be able to publish message to Janitor test env (toolsalpha). */
+  /** Credential file path to be able to publish message to Janitor test env (qa). */
   private String janitorClientServiceAccountPath;
 
   /** Credential file path to be able to publish message to Janitor prod env (tools). */

src/test/resources/application-integration.yml

@@ -6,8 +6,8 @@ janitor:
   pubsub:
     track-resource:
       enabled: true
-      project-id: terra-kernel-k8s
-      subscription: crljanitor-toolsalpha-pubsub-sub
+      project-id: broad-dsde-qa
+      subscription: crljanitor-qa-pubsub-sub
   test:
     janitor-client-service-account-path: rendered/client-sa-account.json
     parent-resource-id: folders/1074206284898
@@ -16,7 +16,7 @@ janitor:
     prod-track-resource-topic-id: crljanitor-tools-pubsub-topic
     resource-credential-file-path: rendered/cloud-access-sa-account.json
     resource-project-id: terra-janitor-test
-    track-resource-topic-id: crljanitor-toolsalpha-pubsub-topic
+    track-resource-topic-id: crljanitor-qa-pubsub-topic
     # Reusing static MRG from https://github.com/broadinstitute/terra-github-workflows/blob/main/.github/workflows/attach-billing-project-to-landing-zone.yaml
     azure-tenant-id: fad90753-2022-4456-9b0a-c7e5b934e408
     azure-subscription-id: f557c728-871d-408c-a28b-eb6b2141a087