From 468b93f3bcb162e5b84ae798023929254df5fe96 Mon Sep 17 00:00:00 2001 From: Bryan Marchena Date: Fri, 2 Feb 2024 17:09:58 -0500 Subject: [PATCH 1/3] Add index on geoid column for ACS db tables --- migrations/acs.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/migrations/acs.sql b/migrations/acs.sql index 38d7243..efba9f3 100644 --- a/migrations/acs.sql +++ b/migrations/acs.sql @@ -28,4 +28,6 @@ SELECT m as margin_of_error, p as percent, z as percent_margin_of_error -INTO acs.:"TABLE_NAME" FROM tmp; \ No newline at end of file +INTO acs.:"TABLE_NAME" FROM tmp; + +CREATE INDEX acs_:TABLE_NAME||_geoid_idx ON acs.:"TABLE_NAME" (geoid); \ No newline at end of file From 5837e9883b246006341125ad22c9b19d819ef6e2 Mon Sep 17 00:00:00 2001 From: Tyler Matteo Date: Mon, 5 Feb 2024 09:46:43 -0500 Subject: [PATCH 2/3] Add INDEX_NAME var to etl script and update acs.sql to use it to create geoid index --- migrations/acs.sql | 3 ++- migrations/etl.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/migrations/acs.sql b/migrations/acs.sql index efba9f3..8d6e3f1 100644 --- a/migrations/acs.sql +++ b/migrations/acs.sql @@ -1,5 +1,6 @@ CREATE SCHEMA IF NOT EXISTS acs; DROP TABLE IF EXISTS acs.:"TABLE_NAME"; +DROP INDEX IF EXISTS acs.:INDEX_NAME; CREATE TEMP TABLE tmp ( census_geoid text, @@ -30,4 +31,4 @@ SELECT z as percent_margin_of_error INTO acs.:"TABLE_NAME" FROM tmp; -CREATE INDEX acs_:TABLE_NAME||_geoid_idx ON acs.:"TABLE_NAME" (geoid); \ No newline at end of file +CREATE INDEX IF NOT EXISTS :INDEX_NAME ON acs.:"TABLE_NAME" (geoid); \ No newline at end of file diff --git a/migrations/etl.sh b/migrations/etl.sh index 85420ab..326fc87 100644 --- a/migrations/etl.sh +++ b/migrations/etl.sh @@ -38,7 +38,7 @@ if [[ $download -eq 1 ]]; then fi if [[ $load -eq 1 ]]; then - cat $filepath | psql $DATABASE_URL -v TABLE_NAME=$year -f migrations/$datasource.sql + cat $filepath | psql $DATABASE_URL -v TABLE_NAME=$year -v INDEX_NAME="${datasource}_${year}_geoid_idx" -f migrations/$datasource.sql fi if [[ $clean -eq 1 ]]; then From fac0a4881a3430b218a7442c755046a0b6cb6a41 Mon Sep 17 00:00:00 2001 From: Bryan Marchena Date: Tue, 6 Feb 2024 11:17:59 -0500 Subject: [PATCH 3/3] Update decennial.sql to use INDEX_NAME variable to create geoid index --- migrations/decennial.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/migrations/decennial.sql b/migrations/decennial.sql index d04da4f..a2d491d 100644 --- a/migrations/decennial.sql +++ b/migrations/decennial.sql @@ -1,5 +1,6 @@ CREATE SCHEMA IF NOT EXISTS decennial; DROP TABLE IF EXISTS decennial.:"TABLE_NAME"; +DROP INDEX IF EXISTS decennial.:INDEX_NAME; CREATE TEMP TABLE tmp ( year text, @@ -11,4 +12,6 @@ CREATE TEMP TABLE tmp ( \COPY tmp FROM PSTDIN WITH DELIMITER ',' CSV HEADER; DROP TABLE IF EXISTS decennial.:"TABLE_NAME"; -SELECT * INTO decennial.:"TABLE_NAME" FROM tmp; \ No newline at end of file +SELECT * INTO decennial.:"TABLE_NAME" FROM tmp; + +CREATE INDEX IF NOT EXISTS :INDEX_NAME ON decennial.:"TABLE_NAME" (geoid); \ No newline at end of file