[DC-1022] Fixes failing integration test.

Adds creating concept_ancestor_extension table in main_test.py

Signed-off-by: Krishna Kalluri <[email protected]>

data_steward/common.py

@@ -1,10 +1,11 @@
 # Python imports
 import os
 
+import jinja2
+
 # Project imports
 from constants.bq_utils import VALIDATION_DATASET_REGEX
 from constants.validation.participants.identity_match import REPORT_DIRECTORY_REGEX
-import jinja2
 
 # AOU required PII tables
 PII_WILDCARD = 'pii*'
@@ -112,6 +113,9 @@
     CONCEPT, CONCEPT_ANCESTOR, CONCEPT_CLASS, CONCEPT_RELATIONSHIP,
     CONCEPT_SYNONYM, DOMAIN, DRUG_STRENGTH, RELATIONSHIP, VOCABULARY
 ]
+
+CONCEPT_ANCESTOR_EXTENSION = 'concept_ancestor_extension'
+
 # Achilles
 ACHILLES_ANALYSIS = 'achilles_analysis'
 ACHILLES_RESULTS = 'achilles_results'

data_steward/tools/generate_concept_ancestor_extension.py

@@ -16,12 +16,11 @@
 import argparse
 import logging
 
+from common import CONCEPT_ANCESTOR_EXTENSION
 from utils import bq
 
 LOGGER = logging.getLogger(__name__)
 
-CONCEPT_ANCESTOR_EXTENSION = 'concept_ancestor_extension'
-
 CONCEPT_ANCESTOR_EXT_QUERY = '''
 DECLARE
   num_of_new_records INT64;
@@ -43,14 +42,14 @@
       c.vocabulary_id = 'LOINC'
       AND domain_id = 'Measurement' ) AS loinc_ids
   JOIN
-    `ehr_ops.concept_relationship` AS cr
+    `{project}.{dataset}.concept_relationship` AS cr
   ON
     loinc_ids.ancestor_concept_id = cr.concept_id_1
     AND relationship_id IN ('Subsumes',
       'Component of')
     AND cr.concept_id_1 <> cr.concept_id_2
   JOIN
-    `ehr_ops.concept` AS c2
+    `{project}.{dataset}.concept` AS c2
   ON
     cr.concept_id_2 = c2.concept_id
     AND c2.domain_id = 'Measurement' );
@@ -113,7 +112,7 @@
 END LOOP
   ;
 CREATE OR REPLACE TABLE
-  ehr_ops.concept_ancestor_extension AS
+  `{project}.{dataset}.{ancestor_extension}` AS
 SELECT
   ancestor_concept_id,
   descendant_concept_id,

data_steward/validation/metrics/required_labs_sql.py

@@ -27,8 +27,8 @@
 SELECT
   *
 FROM
-  `{project_id}.{vocab_dataset_id}.concept_ancestor_extension`
-)
+  `{project_id}.{ehr_ops_dataset_id}.concept_ancestor_extension`
+),
 
 get_direct_parents_loinc_group AS
 (

tests/integration_tests/data_steward/validation/main_test.py

@@ -2,6 +2,7 @@
 Unit test components of data_steward.validation.main
 """
 from __future__ import print_function
+
 import json
 import os
 import unittest
@@ -10,14 +11,15 @@
 import mock
 from bs4 import BeautifulSoup as bs
 
-import bq_utils
 import app_identity
+import bq_utils
 import common
-from constants import bq_utils as bq_consts
-from constants.validation import main as main_consts
 import gcs_utils
 import resources
+from constants import bq_utils as bq_consts
+from constants.validation import main as main_consts
 from tests import test_util
+from tools.generate_concept_ancestor_extension import generate_concept_ancestor_extension
 from validation import main
 from validation.metrics import required_labs
 
@@ -46,12 +48,19 @@ def setUp(self):
         self._empty_bucket()
         test_util.delete_all_tables(self.bigquery_dataset_id)
         self._create_drug_class_table(self.bigquery_dataset_id)
+        self._create_concept_ancestor_extension(self.project_id,
+                                                self.bigquery_dataset_id)
 
     def _empty_bucket(self):
         bucket_items = gcs_utils.list_bucket(self.hpo_bucket)
         for bucket_item in bucket_items:
             gcs_utils.delete_object(self.hpo_bucket, bucket_item['name'])
 
+    @staticmethod
+    def _create_concept_ancestor_extension(project_id, dataset_id):
+        if not bq_utils.table_exists(common.CONCEPT_ANCESTOR_EXTENSION):
+            generate_concept_ancestor_extension(project_id, dataset_id)
+
     @staticmethod
     def _create_drug_class_table(bigquery_dataset_id):
 
@@ -86,7 +95,7 @@ def _create_drug_class_table(bigquery_dataset_id):
         if not bq_utils.table_exists(common.CONCEPT_ANCESTOR):
             bq_utils.create_standard_table(common.CONCEPT_ANCESTOR,
                                            common.CONCEPT_ANCESTOR)
-            q = """INSERT INTO {dataset}.concept_ancestor
+            q = """INSERT INTO {dataset}.concept_ancestor 
             SELECT * FROM {vocab}.concept_ancestor""".format(
                 dataset=bigquery_dataset_id, vocab=common.VOCABULARY_DATASET)
             bq_utils.query(q)