DataBiosphere · wnojopra · Aug 19, 2024 · Aug 19, 2024 · Sep 13, 2024 · Oct 22, 2024
diff --git a/service/local-dev/run_server.sh b/service/local-dev/run_server.sh
@@ -54,7 +54,7 @@ fi
 
 if [[ ${useVerilyUnderlays} ]]; then
   echo "Using Verily underlays."
-  export TANAGRA_UNDERLAY_FILES=cmssynpuf_verily,aouSR2019q4r4_verily,sd20230831_verily,pilotsynthea2022q3_verily
+  export TANAGRA_UNDERLAY_FILES=cmssynpuf_verily,aouSR2019q4r4_verily,sd20230831_verily,pilotsynthea2022q3_verily,amp_pd,bhs
   export TANAGRA_EXPORT_SHARED_GCP_PROJECT_ID=verily-tanagra-test
   export TANAGRA_EXPORT_SHARED_BQ_DATASET_IDS=service_export_us,service_export_uscentral1
   export TANAGRA_EXPORT_SHARED_GCS_BUCKET_NAMES=verily-tanagra-test-export-bucket,verily-tanagra-test-export-bucket-uscentral1

diff --git a/underlay/src/main/java/bio/terra/tanagra/utils/GoogleBigQuery.java b/underlay/src/main/java/bio/terra/tanagra/utils/GoogleBigQuery.java
@@ -271,6 +271,9 @@ public JobStatistics.QueryStatistics dryRunQuery(
       @Nullable Integer pageSize,
       @Nullable TableId destinationTable,
       @Nullable Clustering clustering) {
+    LOGGER.info("HEY WILLY");
+    LOGGER.info(sql);
+    LOGGER.info("BYE WILLY");
     Pair<QueryJobConfiguration, List<BigQuery.QueryResultsOption>> queryJobConfig =
         buildQueryJobConfig(
             sql, true, queryParams, pageToken, pageSize, destinationTable, clustering, null);

diff --git a/underlay/src/main/resources/config/criteria/amp_pd/criteriaselector/age/age.json b/underlay/src/main/resources/config/criteria/amp_pd/criteriaselector/age/age.json
@@ -0,0 +1,3 @@
+{
+  "attribute": "age"
+}
diff --git a/underlay/src/main/resources/config/criteria/amp_pd/criteriaselector/age/selector.json b/underlay/src/main/resources/config/criteria/amp_pd/criteriaselector/age/selector.json
@@ -0,0 +1,15 @@
+{
+  "name": "amp-pd-age",
+  "displayName": "Age",
+  "isEnabledForCohorts": true,
+  "isEnabledForDataFeatureSets": true,
+  "display": {
+    "category": "Demographics",
+    "tags": null
+  },
+  "filterBuilder": "core.PrimaryEntityFilterBuilder",
+  "plugin": "attribute",
+  "pluginConfig": null,
+  "pluginConfigFile": "age.json",
+  "modifiers": null
+}
diff --git a/underlay/src/main/resources/config/criteria/amp_pd/criteriaselector/ethnicity/ethnicity.json b/underlay/src/main/resources/config/criteria/amp_pd/criteriaselector/ethnicity/ethnicity.json
@@ -0,0 +1,3 @@
+{
+  "attribute": "ethnicity"
+}
diff --git a/underlay/src/main/resources/config/criteria/amp_pd/criteriaselector/ethnicity/selector.json b/underlay/src/main/resources/config/criteria/amp_pd/criteriaselector/ethnicity/selector.json
@@ -0,0 +1,15 @@
+{
+  "name": "amp-pd-ethnicity",
+  "displayName": "Ethnicity",
+  "isEnabledForCohorts": true,
+  "isEnabledForDataFeatureSets": true,
+  "display": {
+    "category": "Demographics",
+    "tags": null
+  },
+  "filterBuilder": "core.PrimaryEntityFilterBuilder",
+  "plugin": "attribute",
+  "pluginConfig": null,
+  "pluginConfigFile": "ethnicity.json",
+  "modifiers": null
+}
diff --git a/underlay/src/main/resources/config/criteria/amp_pd/criteriaselector/race/race.json b/underlay/src/main/resources/config/criteria/amp_pd/criteriaselector/race/race.json
@@ -0,0 +1,3 @@
+{
+  "attribute": "race"
+}
diff --git a/underlay/src/main/resources/config/criteria/amp_pd/criteriaselector/race/selector.json b/underlay/src/main/resources/config/criteria/amp_pd/criteriaselector/race/selector.json
@@ -0,0 +1,15 @@
+{
+  "name": "amp-pd-race",
+  "displayName": "Race",
+  "isEnabledForCohorts": true,
+  "isEnabledForDataFeatureSets": true,
+  "display": {
+    "category": "Demographics",
+    "tags": null
+  },
+  "filterBuilder": "core.PrimaryEntityFilterBuilder",
+  "plugin": "attribute",
+  "pluginConfig": null,
+  "pluginConfigFile": "race.json",
+  "modifiers": null
+}
diff --git a/underlay/src/main/resources/config/criteria/amp_pd/criteriaselector/sex/selector.json b/underlay/src/main/resources/config/criteria/amp_pd/criteriaselector/sex/selector.json
@@ -0,0 +1,15 @@
+{
+  "name": "amp-pd-sex",
+  "displayName": "Sex",
+  "isEnabledForCohorts": true,
+  "isEnabledForDataFeatureSets": true,
+  "display": {
+    "category": "Demographics",
+    "tags": null
+  },
+  "filterBuilder": "core.PrimaryEntityFilterBuilder",
+  "plugin": "attribute",
+  "pluginConfig": null,
+  "pluginConfigFile": "sex.json",
+  "modifiers": null
+}
diff --git a/underlay/src/main/resources/config/criteria/amp_pd/criteriaselector/sex/sex.json b/underlay/src/main/resources/config/criteria/amp_pd/criteriaselector/sex/sex.json
@@ -0,0 +1,3 @@
+{
+  "attribute": "sex"
+}
diff --git a/underlay/src/main/resources/config/datamapping/amp_pd/entity/dummyEntity/all.sql b/underlay/src/main/resources/config/datamapping/amp_pd/entity/dummyEntity/all.sql
@@ -0,0 +1,5 @@
+SELECT
+    participant_id
+
+FROM `${omopDataset}.Demographics` d
+
diff --git a/underlay/src/main/resources/config/datamapping/amp_pd/entity/dummyEntity/entity.json b/underlay/src/main/resources/config/datamapping/amp_pd/entity/dummyEntity/entity.json
@@ -0,0 +1,9 @@
+{
+  "name": "dummyEntity",
+  "allInstancesSqlFile": "all.sql",
+  "attributes": [
+    { "name": "id", "dataType": "STRING", "valueFieldName": "participant_id" }
+  ],
+  "idAttribute": "id",
+  "sourceQueryTableName": "${omopDataset}.Demographics"
+}
diff --git a/underlay/src/main/resources/config/datamapping/amp_pd/entity/person/all.sql b/underlay/src/main/resources/config/datamapping/amp_pd/entity/person/all.sql
@@ -0,0 +1,9 @@
+SELECT
+    participant_id,
+    age_at_baseline,
+    sex,
+    race,
+    ethnicity
+
+FROM `${omopDataset}.Demographics` d
+
diff --git a/underlay/src/main/resources/config/datamapping/amp_pd/entity/person/entity.json b/underlay/src/main/resources/config/datamapping/amp_pd/entity/person/entity.json
@@ -0,0 +1,14 @@
+{
+  "name": "person",
+  "allInstancesSqlFile": "all.sql",
+  "attributes": [
+    { "name": "id", "dataType": "STRING", "valueFieldName": "participant_id" },
+    { "name": "age", "dataType": "INT64", "valueFieldName": "age_at_baseline"},
+    { "name": "sex", "dataType": "STRING" },
+    { "name": "race", "dataType": "STRING" },
+    { "name": "ethnicity", "dataType": "STRING" }
+  ],
+  "idAttribute": "id",
+  "optimizeGroupByAttributes": [ "sex", "race", "age" ],
+  "sourceQueryTableName": "${omopDataset}.Demographics"
+}
diff --git a/...rlay/src/main/resources/config/datamapping/amp_pd/entitygroup/dummyGroup/entityGroup.json b/...rlay/src/main/resources/config/datamapping/amp_pd/entitygroup/dummyGroup/entityGroup.json
@@ -0,0 +1,6 @@
+{
+    "name": "dummyGroup",
+    "groupEntity": "person",
+    "itemsEntity": "dummyEntity",
+    "foreignKeyAttributeItemsEntity": "id"
+}
diff --git a/underlay/src/main/resources/config/datamapping/bhs/entity/dummyEntity/all.sql b/underlay/src/main/resources/config/datamapping/bhs/entity/dummyEntity/all.sql
@@ -0,0 +1,4 @@
+SELECT
+    SUBJID as participant_id
+
+FROM `${omopDataset}.DM` d
diff --git a/underlay/src/main/resources/config/datamapping/bhs/entity/dummyEntity/entity.json b/underlay/src/main/resources/config/datamapping/bhs/entity/dummyEntity/entity.json
@@ -0,0 +1,9 @@
+{
+  "name": "dummyEntity",
+  "allInstancesSqlFile": "all.sql",
+  "attributes": [
+    { "name": "id", "dataType": "STRING", "valueFieldName": "participant_id" }
+  ],
+  "idAttribute": "id",
+  "sourceQueryTableName": "${dataset}.DM"
+}
diff --git a/underlay/src/main/resources/config/datamapping/bhs/entity/person/all.sql b/underlay/src/main/resources/config/datamapping/bhs/entity/person/all.sql
@@ -0,0 +1,7 @@
+SELECT
+    SUBJID as participant_id,
+    CAST(age_at_enrollment as INT64) as age,
+    SEX as sex,
+    RACE as race
+
+FROM `${omopDataset}.DM` d
diff --git a/underlay/src/main/resources/config/datamapping/bhs/entity/person/entity.json b/underlay/src/main/resources/config/datamapping/bhs/entity/person/entity.json
@@ -0,0 +1,12 @@
+{
+  "name": "person",
+  "allInstancesSqlFile": "all.sql",
+  "attributes": [
+    { "name": "id", "dataType": "STRING", "valueFieldName": "participant_id" },
+    { "name": "age", "dataType": "INT64" },
+    { "name": "sex", "dataType": "STRING" },
+    { "name": "race", "dataType": "STRING" }
+  ],
+  "idAttribute": "id",
+  "sourceQueryTableName": "${dataset}.DM"
+}
diff --git a/underlay/src/main/resources/config/datamapping/bhs/entitygroup/dummyGroup/entityGroup.json b/underlay/src/main/resources/config/datamapping/bhs/entitygroup/dummyGroup/entityGroup.json
@@ -0,0 +1,6 @@
+{
+    "name": "dummyGroup",
+    "groupEntity": "person",
+    "itemsEntity": "dummyEntity",
+    "foreignKeyAttributeItemsEntity": "id"
+}
diff --git a/underlay/src/main/resources/config/indexer/amp_pd.json b/underlay/src/main/resources/config/indexer/amp_pd.json
@@ -0,0 +1,27 @@
+{
+  "underlay": "amp_pd",
+  "bigQuery": {
+    "sourceData": {
+      "projectId": "amp-pd-research",
+      "datasetId": "2023_v4release_1027",
+      "sqlSubstitutions": {
+        "omopDataset": "amp-pd-research.2023_v4release_1027",
+        "staticTablesDataset": ""
+      }
+    },
+    "indexData": {
+      "projectId": "aaa-willyn-test",
+      "datasetId": "amp_pd_index_081624"
+    },
+    "queryProjectId": "aaa-willyn-test",
+    "dataLocation": "us"
+  },
+  "dataflow": {
+    "serviceAccountEmail": "[email protected]",
+    "dataflowLocation": "us-central1",
+    "gcsTempDirectory": "gs://dataflow-staging-us-central1-694046000181/temp/",
+    "workerMachineType": "n1-standard-4",
+    "usePublicIps": false,
+    "vpcSubnetworkName": null
+  }
+}
diff --git a/underlay/src/main/resources/config/indexer/bhs.json b/underlay/src/main/resources/config/indexer/bhs.json
@@ -0,0 +1,27 @@
+{
+  "underlay": "bhs",
+  "bigQuery": {
+    "sourceData": {
+      "projectId": "wb-steady-blueberry-8554",
+      "datasetId": "BF_20240610_Screener",
+      "sqlSubstitutions": {
+        "omopDataset": "wb-steady-blueberry-8554.BF_20240610_Screener",
+        "staticTablesDataset": ""
+      }
+    },
+    "indexData": {
+      "projectId": "aaa-willyn-test",
+      "datasetId": "bhs_index_081624"
+    },
+    "queryProjectId": "aaa-willyn-test",
+    "dataLocation": "us"
+  },
+  "dataflow": {
+    "serviceAccountEmail": "[email protected]",
+    "dataflowLocation": "us-central1",
+    "gcsTempDirectory": "gs://dataflow-staging-us-central1-694046000181/temp/",
+    "workerMachineType": "n1-standard-4",
+    "usePublicIps": false,
+    "vpcSubnetworkName": null
+  }
+}
diff --git a/underlay/src/main/resources/config/service/amp_pd.json b/underlay/src/main/resources/config/service/amp_pd.json
@@ -0,0 +1,21 @@
+{
+  "underlay": "amp_pd",
+  "bigQuery": {
+    "sourceData": {
+      "projectId": "amp-pd-research",
+      "datasetId": "2023_v4release_1027",
+      "sqlSubstitutions": {
+        "dataset": "amp-pd-research.2023_v4release_1027",
+        "staticTablesDataset": ""
+      }
+    },
+    "indexData": {
+      "projectId": "aaa-willyn-test",
+      "datasetId": "amp_pd_index_081624"
+    },
+    "queryProjectId": "aaa-willyn-test",
+    "dataLocation": "us"
+  },
+  "uiConfigFile": "ui.json"
+}
+
diff --git a/underlay/src/main/resources/config/service/bhs.json b/underlay/src/main/resources/config/service/bhs.json
@@ -0,0 +1,20 @@
+{
+  "underlay": "bhs",
+  "bigQuery": {
+    "sourceData": {
+      "projectId": "wb-steady-blueberry-8554",
+      "datasetId": "BF_20240610_Screener",
+      "sqlSubstitutions": {
+        "dataset": "wb-steady-blueberry-8554.BF_20240610_Screener",
+        "staticTablesDataset": ""
+      }
+    },
+    "indexData": {
+      "projectId": "aaa-willyn-test",
+      "datasetId": "bhs_index_081624"
+    },
+    "queryProjectId": "aaa-willyn-test",
+    "dataLocation": "us"
+  },
+  "uiConfigFile": "ui.json"
+}
diff --git a/underlay/src/main/resources/config/ui/omop/viz/sex/sex.json b/underlay/src/main/resources/config/ui/omop/viz/sex/sex.json
@@ -0,0 +1,13 @@
+{
+  "sources": [
+    {
+      "criteriaSelector": "demographics",
+      "joins": [],
+      "attributes": [
+        {
+          "attribute": "sex"
+        }
+      ]
+    }
+  ]
+}
diff --git a/underlay/src/main/resources/config/ui/omop/viz/sex/viz.json b/underlay/src/main/resources/config/ui/omop/viz/sex/viz.json
@@ -0,0 +1,6 @@
+{
+  "name": "sex",
+  "title": "Sex identity",
+  "dataConfigFile": "sex.json",
+  "plugin": "core/bar"
+}
diff --git a/underlay/src/main/resources/config/ui/omop/viz/sexAndAge/sexAndAge.json b/underlay/src/main/resources/config/ui/omop/viz/sexAndAge/sexAndAge.json
@@ -0,0 +1,20 @@
+{
+  "sources": [
+    {
+      "criteriaSelector": "demographics",
+      "joins": [],
+      "attributes": [
+        {
+          "attribute": "sex"
+        },
+        {
+          "attribute": "age",
+          "numericBucketing": {
+            "thresholds": [18, 45, 65],
+            "includeGreater": true
+          }
+        }
+      ]
+    }
+  ]
+}
diff --git a/underlay/src/main/resources/config/ui/omop/viz/sexAndAge/viz.json b/underlay/src/main/resources/config/ui/omop/viz/sexAndAge/viz.json
@@ -0,0 +1,6 @@
+{
+  "name": "sexAndAge",
+  "title": "Sex and age",
+  "dataConfigFile": "sexAndAge.json",
+  "plugin": "core/bar"
+}