Merge pull request #1664 from bcgov/bugfix/planning-review-decisions-etl

properly retrieve properties for PR insert
bcgov · May 8, 2024 · 7374e44 · 7374e44
2 parents 8195b25 + e1c6b43
commit 7374e44
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 6 deletions.
diff --git a/bin/migrate-oats-data/planning_review/decisions/init_planning_review_decisions.py b/bin/migrate-oats-data/planning_review/decisions/init_planning_review_decisions.py
@@ -53,10 +53,10 @@ def process_planning_review_decisions(conn=None, batch_size=BATCH_UPLOAD_SIZE):
             while True:
                 query_condition = ""
                 if last_decision_date:
-                    query_condition = f"WHERE pr.file_number::bigint >= {last_file_number} AND opd.decision_date >= {last_decision_date} AND opd.planning_decision_id > {last_planning_decision_id} ORDER BY pr.file_number::bigint, opd.decision_date, opd.planning_decision_id;"
-                else: 
+                    query_condition = f"WHERE pr.file_number::bigint >= {last_file_number} AND opd.decision_date >= '{last_decision_date}' AND opd.planning_decision_id > {last_planning_decision_id} ORDER BY pr.file_number::bigint, opd.decision_date, opd.planning_decision_id;"
+                else:
                     query_condition = f"WHERE pr.file_number::bigint >= {last_file_number} AND opd.planning_decision_id > {last_planning_decision_id} ORDER BY pr.file_number::bigint, opd.decision_date, opd.planning_decision_id;"
-                
+
                 cursor.execute(
                     f"""
                         {query} 
@@ -76,8 +76,16 @@ def process_planning_review_decisions(conn=None, batch_size=BATCH_UPLOAD_SIZE):
                     )
                     last_record = dict(inserted_data[-1])
                     last_planning_decision_id = last_record["planning_decision_id"]
-                    last_file_number = last_record["file_number"]
-                    last_decision_date = last_record["decision_date"]
+                    last_inserted_row = next(
+                        (
+                            row
+                            for row in rows
+                            if row["uuid"] == last_record["review_uuid"]
+                        ),
+                        None,
+                    ) 
+                    last_file_number = last_inserted_row["file_number"]
+                    last_decision_date = last_inserted_row["decision_date"]
 
                     logger.debug(
                         f"Retrieved/updated items count: {len(inserted_data)}; total successfully inserted planning referral so far {successful_inserts_count}; last updated planning_decision_id: {last_planning_decision_id}"
@@ -150,6 +158,7 @@ def _prepare_oats_planning_review_data(row_data_list):
                 "resolution_number": row["resolution_number"],
                 "resolution_year": _map_resolution_year(row),
                 "created_at": to_alcs_format(get_now_with_offset(insert_index)),
+                "file_number": row["file_number"],
             }
         )
         insert_index += 1

diff --git a/bin/migrate-oats-data/planning_review/sql/decisions/planning_review_decisions_insert.sql b/bin/migrate-oats-data/planning_review/sql/decisions/planning_review_decisions_insert.sql
@@ -6,7 +6,7 @@ SELECT
     pr."uuid",
     opd.resolution_number,
     opd.planning_decision_id,
-    pr.file_number
+    pr.file_number::bigint
 FROM
     oats.oats_planning_decisions opd
     JOIN alcs.planning_review pr ON opd.planning_review_id::TEXT = pr.file_number