From 84a48c54f27e999076a3ceff74c852a6a897add3 Mon Sep 17 00:00:00 2001 From: tangoyankee Date: Tue, 15 Aug 2023 14:02:38 -0400 Subject: [PATCH 01/26] point to versioned digital ocean space --- migrations/cli.sh | 2 +- migrations/etl.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/migrations/cli.sh b/migrations/cli.sh index 96afc69..d20b115 100755 --- a/migrations/cli.sh +++ b/migrations/cli.sh @@ -6,7 +6,7 @@ fi function support_geoids { local geography=${1:-2010_to_2020} - BASE_URL=https://nyc3.digitaloceanspaces.com/edm-publishing/db-factfinder/archive + BASE_URL=https://nyc3.digitaloceanspaces.com/edm-publishing/db-factfinder/main/archive fileurl=$BASE_URL/support_geoids/geography=$geography/support_geoids.csv curl $fileurl | psql $DATABASE_URL -f migrations/support_geoids.sql } diff --git a/migrations/etl.sh b/migrations/etl.sh index 0d6dcb4..e594547 100644 --- a/migrations/etl.sh +++ b/migrations/etl.sh @@ -20,7 +20,7 @@ case $i in esac done -BASE_URL=https://nyc3.digitaloceanspaces.com/edm-publishing/db-factfinder/archive +BASE_URL=https://nyc3.digitaloceanspaces.com/edm-publishing/db-factfinder/main fileurl=$BASE_URL/$datasource/year=$year/geography=$geography/$datasource.csv filepath=.migration/$datasource/year=$year/geography=$geography/$datasource.csv filedir=$(dirname $filepath) From f16018da86966d0f33966d22b853205fb6895790 Mon Sep 17 00:00:00 2001 From: tangoyankee Date: Wed, 16 Aug 2023 16:14:03 -0400 Subject: [PATCH 02/26] only specify major version of database --- compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index e5dc2e8..86149e8 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,6 +1,6 @@ services: database: - image: 'postgres:11.16-alpine3.16' + image: 'postgres:11' ports: - 5432:5432 environment: From 2519519ff6d330e87ac356c9a08f33ed950625df Mon Sep 17 00:00:00 2001 From: tangoyankee Date: Fri, 18 Aug 2023 14:14:56 -0400 Subject: [PATCH 03/26] expose port 5433 --- compose.yaml | 4 ++-- sample.env | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compose.yaml b/compose.yaml index 86149e8..214306b 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,7 +2,7 @@ services: database: image: 'postgres:11' ports: - - 5432:5432 + - 5433:5432 environment: POSTGRES_USER: factfinder-user POSTGRES_PASSWORD: factfinder-password @@ -20,4 +20,4 @@ services: ports: - 3001:3001 environment: - DATABASE_URL: postgresql://factfinder-user:factfinder-password@database:5432/factfinder-local + DATABASE_URL: postgresql://factfinder-user:factfinder-password@database:5433/factfinder-local diff --git a/sample.env b/sample.env index 0127efc..cbfbc04 100644 --- a/sample.env +++ b/sample.env @@ -1 +1 @@ -DATABASE_URL=postgresql://factfinder-user:factfinder-password@localhost:5432/factfinder-local +DATABASE_URL=postgresql://factfinder-user:factfinder-password@localhost:5433/factfinder-local From ba7b5424a790f12c5dab6e99c8b7f176618954bd Mon Sep 17 00:00:00 2001 From: tangoyankee Date: Mon, 21 Aug 2023 13:49:15 -0400 Subject: [PATCH 04/26] use 5432 within docker network --- compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index 214306b..32cdadf 100644 --- a/compose.yaml +++ b/compose.yaml @@ -20,4 +20,4 @@ services: ports: - 3001:3001 environment: - DATABASE_URL: postgresql://factfinder-user:factfinder-password@database:5433/factfinder-local + DATABASE_URL: postgresql://factfinder-user:factfinder-password@database:5432/factfinder-local From df77dfbec8ac1ddfec87ceafc2a0cf4d4d9d18bc Mon Sep 17 00:00:00 2001 From: tangoyankee Date: Mon, 21 Aug 2023 13:50:24 -0400 Subject: [PATCH 05/26] point to archive digital ocean space --- migrations/cli.sh | 2 +- migrations/etl.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/migrations/cli.sh b/migrations/cli.sh index d20b115..2ec03c6 100755 --- a/migrations/cli.sh +++ b/migrations/cli.sh @@ -6,7 +6,7 @@ fi function support_geoids { local geography=${1:-2010_to_2020} - BASE_URL=https://nyc3.digitaloceanspaces.com/edm-publishing/db-factfinder/main/archive + BASE_URL=https://nyc3.digitaloceanspaces.com/edm-publishing/db-factfinder/main fileurl=$BASE_URL/support_geoids/geography=$geography/support_geoids.csv curl $fileurl | psql $DATABASE_URL -f migrations/support_geoids.sql } diff --git a/migrations/etl.sh b/migrations/etl.sh index e594547..0d6dcb4 100644 --- a/migrations/etl.sh +++ b/migrations/etl.sh @@ -20,7 +20,7 @@ case $i in esac done -BASE_URL=https://nyc3.digitaloceanspaces.com/edm-publishing/db-factfinder/main +BASE_URL=https://nyc3.digitaloceanspaces.com/edm-publishing/db-factfinder/archive fileurl=$BASE_URL/$datasource/year=$year/geography=$geography/$datasource.csv filepath=.migration/$datasource/year=$year/geography=$geography/$datasource.csv filedir=$(dirname $filepath) From 154adf27093ccb3fbf9fd1ed67409dc0e3b25e84 Mon Sep 17 00:00:00 2001 From: tangoyankee Date: Tue, 22 Aug 2023 10:07:16 -0400 Subject: [PATCH 06/26] update postgres to 15 --- compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index 32cdadf..816d0c3 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,6 +1,6 @@ services: database: - image: 'postgres:11' + image: 'postgres:15' ports: - 5433:5432 environment: From d1ece57c1e73788703016df0d90f25e701155361 Mon Sep 17 00:00:00 2001 From: tangoyankee Date: Tue, 22 Aug 2023 16:25:03 -0400 Subject: [PATCH 07/26] build from spaces version folders --- bin/migrate | 20 ++++++++++++-------- migrations/cli.sh | 8 ++++---- migrations/etl.sh | 12 ++++++------ migrations/metadata.sh | 10 ++++++---- special-calculations/data/constants.js | 12 ++++++++++-- 5 files changed, 38 insertions(+), 24 deletions(-) diff --git a/bin/migrate b/bin/migrate index f8cb388..7e40df0 100755 --- a/bin/migrate +++ b/bin/migrate @@ -2,11 +2,15 @@ const exec = require('child_process').exec; const constants = require('../special-calculations/data/constants'); const { - GEOGRAPHY_YEAR, + GEOGRAPHY_VERSION, ACS_LATEST_TABLE_NAME, + ACS_LATEST_VERSION, ACS_EARLIEST_TABLE_NAME, + ACS_EARLIEST_VERSION, DECENNIAL_LATEST_TABLE_NAME, + DECENNIAL_LATEST_VERSION, DECENNIAL_EARLIEST_TABLE_NAME, + DECENNIAL_EARLIEST_VERSION, } = constants; const etl = (command) => { @@ -22,11 +26,11 @@ const etl = (command) => { }); }; -etl(`./migrations/cli.sh etl --datasource=acs --year=${ACS_LATEST_TABLE_NAME} --geography=${GEOGRAPHY_YEAR} --download --load`); -etl(`./migrations/cli.sh etl --datasource=acs --year=${ACS_EARLIEST_TABLE_NAME} --geography=${GEOGRAPHY_YEAR} --download --load`); -etl(`./migrations/cli.sh etl --datasource=decennial --year=${DECENNIAL_LATEST_TABLE_NAME} --geography=${GEOGRAPHY_YEAR} --download --load`); -etl(`./migrations/cli.sh etl --datasource=decennial --year=${DECENNIAL_EARLIEST_TABLE_NAME} --geography=${GEOGRAPHY_YEAR} --download --load`); -etl(`./migrations/cli.sh metadata --datasource=acs --year_curr=${ACS_LATEST_TABLE_NAME} --year_prev=${ACS_EARLIEST_TABLE_NAME}`); -etl(`./migrations/cli.sh metadata --datasource=decennial --year_curr=${DECENNIAL_LATEST_TABLE_NAME} --year_prev=${DECENNIAL_EARLIEST_TABLE_NAME}`); -etl(`./migrations/cli.sh support_geoids ${GEOGRAPHY_YEAR}`); +etl(`./migrations/cli.sh etl --datasource=acs --year=${ACS_LATEST_TABLE_NAME} --version=${ACS_LATEST_VERSION} --download --load`); +etl(`./migrations/cli.sh etl --datasource=acs --year=${ACS_EARLIEST_TABLE_NAME} --version=${ACS_EARLIEST_VERSION} --download --load`); +etl(`./migrations/cli.sh etl --datasource=decennial --year=${DECENNIAL_LATEST_TABLE_NAME} --version=${DECENNIAL_LATEST_VERSION} --download --load`); +etl(`./migrations/cli.sh etl --datasource=decennial --year=${DECENNIAL_EARLIEST_TABLE_NAME} --version=${DECENNIAL_EARLIEST_VERSION} --download --load`); +etl(`./migrations/cli.sh metadata --datasource=acs --year_curr=${ACS_LATEST_TABLE_NAME} --year_curr_vers=${ACS_LATEST_VERSION} --year_prev=${ACS_EARLIEST_TABLE_NAME} --year_prev_vers=${ACS_EARLIEST_VERSION}`); +etl(`./migrations/cli.sh metadata --datasource=decennial --year_curr=${DECENNIAL_LATEST_TABLE_NAME} --year_curr_vers=${DECENNIAL_LATEST_VERSION} --year_prev=${DECENNIAL_EARLIEST_TABLE_NAME} --year_prev_vers=${DECENNIAL_EARLIEST_VERSION}`); +etl(`./migrations/cli.sh support_geoids ${GEOGRAPHY_VERSION}`); etl('./migrations/cli.sh selection'); diff --git a/migrations/cli.sh b/migrations/cli.sh index 2ec03c6..c51a047 100755 --- a/migrations/cli.sh +++ b/migrations/cli.sh @@ -5,9 +5,9 @@ if [ -f .env ]; then fi function support_geoids { - local geography=${1:-2010_to_2020} + local version=${1:-latest} BASE_URL=https://nyc3.digitaloceanspaces.com/edm-publishing/db-factfinder/main - fileurl=$BASE_URL/support_geoids/geography=$geography/support_geoids.csv + fileurl=$BASE_URL/support_geoids/$version/support_geoids.csv curl $fileurl | psql $DATABASE_URL -f migrations/support_geoids.sql } @@ -34,8 +34,8 @@ case $1 in * ) echo echo "$1 command not found" - echo "e.g. ./migrations/cli.sh etl --datasource=acs --year=2019 --geography=2010_to_2020 --download --load" - echo "e.g. ./migrations/cli.sh etl -s={{ acs|decennial }} -y=2010 -g=2010_to_2020 --load --clean" + echo "e.g. ./migrations/cli.sh etl --datasource=acs --year=2019 --version=2023-03-27 --download --load" + echo "e.g. ./migrations/cli.sh etl -s={{ acs|decennial }} -y=2010 -v=2023-03-27 --load --clean" echo ;; esac \ No newline at end of file diff --git a/migrations/etl.sh b/migrations/etl.sh index 0d6dcb4..85420ab 100644 --- a/migrations/etl.sh +++ b/migrations/etl.sh @@ -5,24 +5,24 @@ do case $i in -s=*|--datasource=*) datasource="${i#*=}" ;; -y=*|--year=*) year="${i#*=}" ;; - -g=*|--geography=*) geography="${i#*=}";; + -v=*|--version=*) version="${i#*=}";; --download) download=1 ;; --load) load=1 ;; --clean) clean=1 ;; *) echo echo "invalid flag $i" - echo "e.g. --datasource=acs --year=2019 --geography=2010_to_2020 --download --load" - echo "e.g. -s=acs -y=2019 -g=2010_to_2020 --download --load" + echo "e.g. --datasource=acs --year=2019 --version=2023-03-27 --download --load" + echo "e.g. -s=acs -y=2019 --v=2023-03-27 --download --load" echo exit ;; esac done -BASE_URL=https://nyc3.digitaloceanspaces.com/edm-publishing/db-factfinder/archive -fileurl=$BASE_URL/$datasource/year=$year/geography=$geography/$datasource.csv -filepath=.migration/$datasource/year=$year/geography=$geography/$datasource.csv +BASE_URL=https://nyc3.digitaloceanspaces.com/edm-publishing/db-factfinder/main +fileurl=$BASE_URL/$datasource/$year/$version/$datasource.csv +filepath=.migration/$datasource/$year/$datasource.csv filedir=$(dirname $filepath) if [[ $download -eq 1 ]]; then mkdir -p .migration && ( diff --git a/migrations/metadata.sh b/migrations/metadata.sh index e6bbbad..66e522d 100644 --- a/migrations/metadata.sh +++ b/migrations/metadata.sh @@ -5,11 +5,13 @@ do case $i in -s=*|--datasource=*) datasource="${i#*=}" ;; --year_curr=*) year_curr="${i#*=}" ;; + --year_curr_vers=*) year_curr_vers="${i#*=}";; --year_prev=*) year_prev="${i#*=}" ;; + --year_prev_vers=*) year_prev_vers="${i#*=}";; *) echo echo "invalid flag $i" - echo "e.g. -s={{ acs|decennial }} -year_curr=2019 -year_prev=2010" + echo "e.g. -s={{ acs|decennial }} -year_curr=2019 -year_curr_vers=2023-03-27 -year_prev=2010 -year_prev_vers=2023-03-27" exit ;; esac @@ -17,9 +19,9 @@ done if [ ! -z "$datasource" ] && [ ! -z "$year_curr" ] && [ ! -z "$year_prev" ]; then echo "loading metadata for $datasource year_curr: $year_curr year_prev: $year_prev" - base_url="https://raw.githubusercontent.com/NYCPlanning/db-factfinder/main/factfinder/data" - url_curr="$base_url/$datasource/$year_curr/metadata.json" - url_prev="$base_url/$datasource/$year_prev/metadata.json" + base_url="https://nyc3.digitaloceanspaces.com/edm-publishing/db-factfinder/main" + url_curr="$base_url/$datasource/$year_curr/$year_curr_vers/metadata.json" + url_prev="$base_url/$datasource/$year_prev/$year_prev_vers/metadata.json" base_path="$( cd ../"$(dirname "$0")" ; pwd -P )" if [ $datasource == "acs" ]; then echo $base_path diff --git a/special-calculations/data/constants.js b/special-calculations/data/constants.js index 0cd32af..661e857 100644 --- a/special-calculations/data/constants.js +++ b/special-calculations/data/constants.js @@ -5,13 +5,17 @@ const DIFF_PERCENT_THRESHOLD = -0.05; const DESIGN_FACTOR = 1.5; const CUR_YEAR = 'Y2017-2021'; const PREV_YEAR = 'Y2006-2010'; // TODO: update with actual previous year -const GEOGRAPHY_YEAR = '2010_to_2020'; +const GEOGRAPHY_VERSION = 'latest'; const DECENNIAL_SCHEMA_NAME = 'decennial'; const DECENNIAL_LATEST_TABLE_NAME = '2020'; +const DECENNIAL_LATEST_VERSION = '2021'; const DECENNIAL_EARLIEST_TABLE_NAME = '2010'; +const DECENNIAL_EARLIEST_VERSION = '2021'; const ACS_SCHEMA_NAME = 'acs'; const ACS_LATEST_TABLE_NAME = '2021'; +const ACS_LATEST_VERSION = '2023-03-27'; const ACS_EARLIEST_TABLE_NAME = '2010'; +const ACS_EARLIEST_VERSION = '2023-03-27'; const ACS_METADATA_FULL_PATH = `"${ACS_SCHEMA_NAME}"."metadata"` const ACS_LATEST_TABLE_FULL_PATH = `"${ACS_SCHEMA_NAME}"."${ACS_LATEST_TABLE_NAME}"`; const ACS_EARLIEST_TABLE_FULL_PATH = `"${ACS_SCHEMA_NAME}"."${ACS_EARLIEST_TABLE_NAME}"`; @@ -26,15 +30,19 @@ module.exports = { DESIGN_FACTOR, CUR_YEAR, PREV_YEAR, - GEOGRAPHY_YEAR, + GEOGRAPHY_VERSION, DECENNIAL_LATEST_TABLE_NAME, + DECENNIAL_LATEST_VERSION, DECENNIAL_EARLIEST_TABLE_NAME, + DECENNIAL_EARLIEST_VERSION, DECENNIAL_LATEST_TABLE_FULL_PATH, DECENNIAL_EARLIEST_TABLE_FULL_PATH, DECENNIAL_METADATA_FULL_PATH, ACS_SCHEMA_NAME, ACS_LATEST_TABLE_NAME, + ACS_LATEST_VERSION, ACS_EARLIEST_TABLE_NAME, + ACS_EARLIEST_VERSION, ACS_METADATA_FULL_PATH, ACS_LATEST_TABLE_FULL_PATH, ACS_EARLIEST_TABLE_FULL_PATH, From 5a8d255bb1d6e8acc93f361154fd1f820d08920e Mon Sep 17 00:00:00 2001 From: tangoyankee Date: Tue, 22 Aug 2023 16:52:43 -0400 Subject: [PATCH 08/26] point to 2023-08-21 decennial data --- special-calculations/data/constants.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/special-calculations/data/constants.js b/special-calculations/data/constants.js index 661e857..3061bff 100644 --- a/special-calculations/data/constants.js +++ b/special-calculations/data/constants.js @@ -8,9 +8,9 @@ const PREV_YEAR = 'Y2006-2010'; // TODO: update with actual previous year const GEOGRAPHY_VERSION = 'latest'; const DECENNIAL_SCHEMA_NAME = 'decennial'; const DECENNIAL_LATEST_TABLE_NAME = '2020'; -const DECENNIAL_LATEST_VERSION = '2021'; +const DECENNIAL_LATEST_VERSION = '2023-08-21'; const DECENNIAL_EARLIEST_TABLE_NAME = '2010'; -const DECENNIAL_EARLIEST_VERSION = '2021'; +const DECENNIAL_EARLIEST_VERSION = '2023-08-21'; const ACS_SCHEMA_NAME = 'acs'; const ACS_LATEST_TABLE_NAME = '2021'; const ACS_LATEST_VERSION = '2023-03-27'; From 87b6d7bf110df0d13293f2b3e1cf75da93d38671 Mon Sep 17 00:00:00 2001 From: tangoyankee Date: Wed, 23 Aug 2023 15:52:12 -0400 Subject: [PATCH 09/26] update table variables that changed --- special-calculations/decennial.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/special-calculations/decennial.js b/special-calculations/decennial.js index af34dca..373a109 100644 --- a/special-calculations/decennial.js +++ b/special-calculations/decennial.js @@ -23,7 +23,7 @@ module.exports = [ }, // population density { - variable: 'popperacre', + variable: 'popacre', specialType: 'mean', options: { args: ['pop1', 'landacres'] }, }, @@ -31,7 +31,7 @@ module.exports = [ { variable: 'avghhsz', specialType: 'mean', - options: { args: ['popinhh', 'ochu_1'] }, + options: { args: ['popinhh_1', 'ochu_1'] }, }, { variable: 'avgfamsz', From 5d02464f346b5c5fb1994040fbab62c557159813 Mon Sep 17 00:00:00 2001 From: tangoyankee Date: Wed, 30 Aug 2023 09:49:54 -0400 Subject: [PATCH 10/26] Decennial data version with: - No space at end of male var - 2020 var metadata - ccd prefix in geoid --- special-calculations/data/constants.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/special-calculations/data/constants.js b/special-calculations/data/constants.js index 3061bff..c38f85c 100644 --- a/special-calculations/data/constants.js +++ b/special-calculations/data/constants.js @@ -8,9 +8,9 @@ const PREV_YEAR = 'Y2006-2010'; // TODO: update with actual previous year const GEOGRAPHY_VERSION = 'latest'; const DECENNIAL_SCHEMA_NAME = 'decennial'; const DECENNIAL_LATEST_TABLE_NAME = '2020'; -const DECENNIAL_LATEST_VERSION = '2023-08-21'; +const DECENNIAL_LATEST_VERSION = '2023-08-28'; const DECENNIAL_EARLIEST_TABLE_NAME = '2010'; -const DECENNIAL_EARLIEST_VERSION = '2023-08-21'; +const DECENNIAL_EARLIEST_VERSION = '2023-08-28'; const ACS_SCHEMA_NAME = 'acs'; const ACS_LATEST_TABLE_NAME = '2021'; const ACS_LATEST_VERSION = '2023-03-27'; From 6b964c80fd61a616bcd7b46210b544f8fa4d1956 Mon Sep 17 00:00:00 2001 From: dhochbaum-dcp Date: Wed, 30 Aug 2023 11:24:25 -0400 Subject: [PATCH 11/26] Updated geography version. --- special-calculations/data/constants.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/special-calculations/data/constants.js b/special-calculations/data/constants.js index c38f85c..5460fb6 100644 --- a/special-calculations/data/constants.js +++ b/special-calculations/data/constants.js @@ -5,7 +5,7 @@ const DIFF_PERCENT_THRESHOLD = -0.05; const DESIGN_FACTOR = 1.5; const CUR_YEAR = 'Y2017-2021'; const PREV_YEAR = 'Y2006-2010'; // TODO: update with actual previous year -const GEOGRAPHY_VERSION = 'latest'; +const GEOGRAPHY_VERSION = '2023-08-29'; const DECENNIAL_SCHEMA_NAME = 'decennial'; const DECENNIAL_LATEST_TABLE_NAME = '2020'; const DECENNIAL_LATEST_VERSION = '2023-08-28'; From c92047be0fe37807a8e3ac1d1986791043f3d637 Mon Sep 17 00:00:00 2001 From: tangoyankee Date: Wed, 30 Aug 2023 11:51:30 -0400 Subject: [PATCH 12/26] populate support_geoids table with ccds update selection table to support ccds --- README.md | 1 + migrations/selection.sql | 2 +- migrations/support_geoids.sql | 10 ++++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b888ca0..4591091 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,7 @@ See the table below for possible programmatic Geotype values, and a mapping of p | ntas | NTA2020 | ACS, Census | | tracts | CT2020 | ACS, Census | | cdtas | CDTA2020 | ACS | +| ccds | CCD2023 | Census | | | ??? | Census | | blocks | CB2020 | Census | | boroughs | Boro2020 | ACS, Census | diff --git a/migrations/selection.sql b/migrations/selection.sql index a350ec5..775bee9 100644 --- a/migrations/selection.sql +++ b/migrations/selection.sql @@ -2,7 +2,7 @@ This will be used to store custom geography selections */ CREATE TABLE IF NOT EXISTS selection ( - geotype text CHECK(geotype IN ('blocks','tracts','ntas','cdtas','boroughs','districts')), + geotype text CHECK(geotype IN ('blocks','tracts','ntas','cdtas','boroughs','districts', 'ccds')), geoids text[], hash text PRIMARY KEY ); \ No newline at end of file diff --git a/migrations/support_geoids.sql b/migrations/support_geoids.sql index e4c19a8..6a626e3 100644 --- a/migrations/support_geoids.sql +++ b/migrations/support_geoids.sql @@ -44,5 +44,15 @@ CREATE TABLE support_geoids AS ( FROM tmp WHERE geotype LIKE 'CDTA%' ORDER BY geogname ASC + ) UNION ALL + ( + SELECT + geoid, + geotype, + geogname as label, + 'City Council Districts (CCDs)' AS typelabel + FROM tmp + WHERE geotype LIKE 'CCD%' + ORDER BY geogname ASC ) ); \ No newline at end of file From 00697903c9ade45a9edd53e7a72d05c7ea36e174 Mon Sep 17 00:00:00 2001 From: dhochbaum-dcp Date: Thu, 31 Aug 2023 17:51:10 -0400 Subject: [PATCH 13/26] Add CCDs to summary level --- selection-helpers/summary-levels.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/selection-helpers/summary-levels.js b/selection-helpers/summary-levels.js index 9247ece..6fb01f6 100644 --- a/selection-helpers/summary-levels.js +++ b/selection-helpers/summary-levels.js @@ -24,6 +24,14 @@ const summaryLevels = { FROM pff_2020_census_tracts_21c `, + ccds: (webmercator = true) => ` + SELECT + ${webmercator ? 'the_geom_webmercator' : 'the_geom'}, + coundist as geolabel, + CONCAT('CCD', coundist) AS geoid + FROM dcp_city_council_districts + `, + cdtas: (webmercator = true) => ` SELECT ${webmercator ? 'the_geom_webmercator' : 'the_geom'}, From e4a5f3768625586037c1c0aad4012a0298c08aa5 Mon Sep 17 00:00:00 2001 From: tangoyankee Date: Thu, 31 Aug 2023 13:20:35 -0400 Subject: [PATCH 14/26] update ci to postgres:15 and node 18 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c7adf5..722c805 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres services: db: - image: postgis/postgis:11-3.0-alpine + image: postgres:15 env: POSTGRES_PASSWORD: postgres options: >- From 5fa00c9024a6174eb9c2af0b7d4e45f69fcea7a3 Mon Sep 17 00:00:00 2001 From: tangoyankee Date: Thu, 31 Aug 2023 16:19:08 -0400 Subject: [PATCH 15/26] update ci to run ubuntu 22 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 722c805..7da59d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ on: jobs: test: name: Testing ... - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres services: From 199884bf3d61aa1f53647adac488c21b76b06bf0 Mon Sep 17 00:00:00 2001 From: tangoyankee Date: Fri, 8 Sep 2023 17:10:03 -0400 Subject: [PATCH 16/26] add special variable configurations --- special-calculations/decennial.js | 93 ++++++++++++++++--------------- 1 file changed, 48 insertions(+), 45 deletions(-) diff --git a/special-calculations/decennial.js b/special-calculations/decennial.js index 373a109..bb626de 100644 --- a/special-calculations/decennial.js +++ b/special-calculations/decennial.js @@ -1,47 +1,50 @@ module.exports = [ - // housing-occupancy - { - variable: 'hmownrvcrt', - specialType: 'mean', - options: { args: ['vhufslo', 'oochu1'], formulaName: { sum: 'ratio' } }, - }, - { - variable: 'rntvcrt', - specialType: 'mean', - options: { args: ['vhufrnt', 'rochu_3'], formulaName: { sum: 'ratio' } }, - }, - // housing tenure - { - variable: 'avhhszooc', - specialType: 'mean', - options: { args: ['popoochu', 'oochu'] }, - }, - { - variable: 'avhhszroc', - specialType: 'mean', - options: { args: ['poprochu', 'rochu_1'] }, - }, - // population density - { - variable: 'popacre', - specialType: 'mean', - options: { args: ['pop1', 'landacres'] }, - }, - // relationship to head of household (householder) - { - variable: 'avghhsz', - specialType: 'mean', - options: { args: ['popinhh_1', 'ochu_1'] }, - }, - { - variable: 'avgfamsz', - specialType: 'mean', - options: { args: ['popinfam', 'fam1'] }, - }, - // sex and age - { - variable: 'mdage', - specialType: 'median', - options: {}, - }, + // Population per acre + { + variable: 'popacre', + specialType: 'mean', + options: { args: ['pop5', 'landacres'], formulaName: { sum: 'ratio' } }, + }, + // Median Age + { + variable: 'mdage', + specialType: 'median', + options: {}, + }, + // Age dependency ratio + { + variable: 'agdpdrt', + specialType: 'mean', + options: { args: ['agu1865p', 'ag18t64'], formulaName: { sum: 'ratio' } } + }, + // Old age dependency ratio + { + variable: 'odagdpdrt', + specialType: 'mean', + options: { args: ['odag65pl', 'odag18t64'], formulaName: { sum: 'ratio' } } + }, + // Child dependency ratio + { + variable: 'chlddpdrt', + specialType: 'mean', + options: { args: ['cdpdu18', 'cdpd18t64'], formulaName: { sum: 'ratio' } } + }, + // Home owner vacancy rate + { + variable: 'hmownvcrt', + specialType: 'mean', + options: { args: ['vacsale', 'hovacu'], formulaName: { sum: 'ratio' } }, + }, + // Rental vacancy rate + { + variable: 'rntvcrt', + specialType: 'mean', + options: { args: ['vacrnt', 'rntvacu'], formulaName: { sum: 'ratio' } }, + }, + // Average household size + { + variable: 'avghhsz', + specialType: 'mean', + options: { args: ['popinhh_1', 'ochu_1'] }, + } ]; From b07fe3f2716709e787375c308f3ee5dc044c2e69 Mon Sep 17 00:00:00 2001 From: tangoyankee Date: Mon, 11 Sep 2023 14:33:13 -0400 Subject: [PATCH 17/26] Revert "use 5432 within docker network" and "expose port 5433" This reverts commits b8a66f123d82861c9f3f1e38cba7a46f5bbcb9c9 and 479fc95a8ea25bfa917523a7c2e513b5d17182d7 --- compose.yaml | 2 +- sample.env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose.yaml b/compose.yaml index 816d0c3..4b00047 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,7 +2,7 @@ services: database: image: 'postgres:15' ports: - - 5433:5432 + - 5432:5432 environment: POSTGRES_USER: factfinder-user POSTGRES_PASSWORD: factfinder-password diff --git a/sample.env b/sample.env index cbfbc04..0127efc 100644 --- a/sample.env +++ b/sample.env @@ -1 +1 @@ -DATABASE_URL=postgresql://factfinder-user:factfinder-password@localhost:5433/factfinder-local +DATABASE_URL=postgresql://factfinder-user:factfinder-password@localhost:5432/factfinder-local From 704b8ad1b345e0efce334119f4b51cbbc2dd7843 Mon Sep 17 00:00:00 2001 From: Timothy Miller Date: Mon, 25 Sep 2023 13:47:37 -0400 Subject: [PATCH 18/26] Develop to staging bug fixes for epic 13898 (#249) * point to archive digital ocean space * Fix for AB#14700, invalid selection type will no cause server api * use node v18.17.1 for ci * point to versioned digital ocean space * only specify major version of database * expose port 5433 * use 5432 within docker network * point to archive digital ocean space * update postgres to 15 * build from spaces version folders * point to 2023-08-21 decennial data * update table variables that changed * Decennial data version with: - No space at end of male var - 2020 var metadata - ccd prefix in geoid * Updated geography version. * populate support_geoids table with ccds update selection table to support ccds * Add CCDs to summary level * update ci to postgres:15 and node 18 * update ci to run ubuntu 22 * add special variable configurations * Revert "use 5432 within docker network" and "expose port 5433" This reverts commits b8a66f123d82861c9f3f1e38cba7a46f5bbcb9c9 and 479fc95a8ea25bfa917523a7c2e513b5d17182d7 * remove CCDs from support geoids table migration * fix agdpdrt and popacre calculations * support decennial variable names for mdage bins * remove CCDs from support geoids table migration (reapplication of change) --------- Co-authored-by: dhochbaum-dcp --- migrations/support_geoids.sql | 12 +------- special-calculations/data/bins.js | 46 +++++++++++++++---------------- special-calculations/decennial.js | 4 +-- utils/interpolate.js | 2 +- 4 files changed, 27 insertions(+), 37 deletions(-) diff --git a/migrations/support_geoids.sql b/migrations/support_geoids.sql index 6a626e3..81af90a 100644 --- a/migrations/support_geoids.sql +++ b/migrations/support_geoids.sql @@ -44,15 +44,5 @@ CREATE TABLE support_geoids AS ( FROM tmp WHERE geotype LIKE 'CDTA%' ORDER BY geogname ASC - ) UNION ALL - ( - SELECT - geoid, - geotype, - geogname as label, - 'City Council Districts (CCDs)' AS typelabel - FROM tmp - WHERE geotype LIKE 'CCD%' - ORDER BY geogname ASC ) -); \ No newline at end of file +); diff --git a/special-calculations/data/bins.js b/special-calculations/data/bins.js index b896495..48d6016 100644 --- a/special-calculations/data/bins.js +++ b/special-calculations/data/bins.js @@ -244,28 +244,28 @@ module.exports = { ], }, mdage: [ - ['mdpop0t4', [ 0, 4.9999 ]], - ['mdpop5t9', [ 5, 9.9999 ]], - ['mdpop10t14', [ 10, 14.9999 ]], - ['mdpop15t17', [ 15, 17.9999 ]], - ['mdpop18t19', [ 18, 19.9999 ]], - ['mdpop20', [ 20, 20.9999 ]], - ['mdpop21', [ 21, 21.9999 ]], - ['mdpop22t24', [ 22, 24.9999 ]], - ['mdpop25t29', [ 25, 29.9999 ]], - ['mdpop30t34', [ 30, 34.9999 ]], - ['mdpop35t39', [ 35, 39.9999 ]], - ['mdpop40t44', [ 40, 44.9999 ]], - ['mdpop45t49', [ 45, 49.9999 ]], - ['mdpop50t54', [ 50, 54.9999 ]], - ['mdpop55t59', [ 55, 59.9999 ]], - ['mdpop60t61', [ 60, 61.9999 ]], - ['mdpop62t64', [ 62, 64.9999 ]], - ['mdpop65t66', [ 65, 66.9999 ]], - ['mdpop67t69', [ 67, 69.9999 ]], - ['mdpop70t74', [ 70, 74.9999 ]], - ['mdpop75t79', [ 75, 79.9999 ]], - ['mdpop80t84', [ 80, 84.9999 ]], - ['mdpop85pl', [ 85, 115 ]] + [/^(mdpop0t4|ageu5)$/, [ 0, 4.9999 ]], + [/^(mdpop|age)5t9$/, [ 5, 9.9999 ]], + [/^(mdpop|age)10t14$/, [ 10, 14.9999 ]], + [/^(mdpop|age)15t17$/, [ 15, 17.9999 ]], + [/^(mdpop|age)18t19$/, [ 18, 19.9999 ]], + [/^(mdpop|age)20$/, [ 20, 20.9999 ]], + [/^(mdpop|age)21$/, [ 21, 21.9999 ]], + [/^(mdpop|age)22t24$/, [ 22, 24.9999 ]], + [/^(mdpop|age)25t29$/, [ 25, 29.9999 ]], + [/^(mdpop|age)30t34$/, [ 30, 34.9999 ]], + [/^(mdpop|age)35t39$/, [ 35, 39.9999 ]], + [/^(mdpop|age)40t44$/, [ 40, 44.9999 ]], + [/^(mdpop|age)45t49$/, [ 45, 49.9999 ]], + [/^(mdpop|age)50t54$/, [ 50, 54.9999 ]], + [/^(mdpop|age)55t59$/, [ 55, 59.9999 ]], + [/^(mdpop|age)60t61$/, [ 60, 61.9999 ]], + [/^(mdpop|age)62t64$/, [ 62, 64.9999 ]], + [/^(mdpop|age)65t66$/, [ 65, 66.9999 ]], + [/^(mdpop|age)67t69$/, [ 67, 69.9999 ]], + [/^(mdpop|age)70t74$/, [ 70, 74.9999 ]], + [/^(mdpop|age)75t79$/, [ 75, 79.9999 ]], + [/^(mdpop|age)80t84$/, [ 80, 84.9999 ]], + [/^(mdpop|age)85pl$/, [ 85, 115 ]] ], }; diff --git a/special-calculations/decennial.js b/special-calculations/decennial.js index bb626de..d44cfdb 100644 --- a/special-calculations/decennial.js +++ b/special-calculations/decennial.js @@ -3,7 +3,7 @@ module.exports = [ { variable: 'popacre', specialType: 'mean', - options: { args: ['pop5', 'landacres'], formulaName: { sum: 'ratio' } }, + options: { args: ['pop5', 'landacres'] }, }, // Median Age { @@ -15,7 +15,7 @@ module.exports = [ { variable: 'agdpdrt', specialType: 'mean', - options: { args: ['agu1865p', 'ag18t64'], formulaName: { sum: 'ratio' } } + options: { args: ['agu1865pl', 'ag18t64'], formulaName: { sum: 'ratio' } } }, // Old age dependency ratio { diff --git a/utils/interpolate.js b/utils/interpolate.js index 1347755..c57c433 100644 --- a/utils/interpolate.js +++ b/utils/interpolate.js @@ -17,7 +17,7 @@ function interpolate(data, variable, year) { const scenario = bins.map((bin) => { const [key, range] = bin; const [min, max] = range; - const row = find(data, ['variable', key]); + const row = find(data, (row) => row.variable.match(key) !== null); if (!row) throw new Error(`${key} was not found in the ${year} dataset.`); From 3ed638fa1e2e23c176a4479e535ea2778659cd98 Mon Sep 17 00:00:00 2001 From: Timothy Miller Date: Mon, 2 Oct 2023 09:43:44 -0400 Subject: [PATCH 19/26] Develop -> Staging Bug/15025 pop acre (#251) * use data version with two decimal landacre precision * use regex match in median error calculation --- special-calculations/data/constants.js | 4 ++-- utils/calculate-median-error.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/special-calculations/data/constants.js b/special-calculations/data/constants.js index 5460fb6..035fe57 100644 --- a/special-calculations/data/constants.js +++ b/special-calculations/data/constants.js @@ -8,9 +8,9 @@ const PREV_YEAR = 'Y2006-2010'; // TODO: update with actual previous year const GEOGRAPHY_VERSION = '2023-08-29'; const DECENNIAL_SCHEMA_NAME = 'decennial'; const DECENNIAL_LATEST_TABLE_NAME = '2020'; -const DECENNIAL_LATEST_VERSION = '2023-08-28'; +const DECENNIAL_LATEST_VERSION = '2023-09-28'; const DECENNIAL_EARLIEST_TABLE_NAME = '2010'; -const DECENNIAL_EARLIEST_VERSION = '2023-08-28'; +const DECENNIAL_EARLIEST_VERSION = '2023-09-28'; const ACS_SCHEMA_NAME = 'acs'; const ACS_LATEST_TABLE_NAME = '2021'; const ACS_LATEST_VERSION = '2023-03-27'; diff --git a/utils/calculate-median-error.js b/utils/calculate-median-error.js index f79fab7..4250a2f 100644 --- a/utils/calculate-median-error.js +++ b/utils/calculate-median-error.js @@ -20,7 +20,7 @@ function calculateMedianError(data, variable, year, options) { const bins = getBins(variable, year); const scenario = bins.map((bin) => { const [key] = bin; - const { sum } = find(data, ['variable', key]); + const { sum } = find(data, (row) => row.variable.match(key) !== null); return { quantity: sum, From f09043c9375f325fc70503c6dad5c1eab79c88af Mon Sep 17 00:00:00 2001 From: Tyler Matteo Date: Thu, 21 Mar 2024 14:09:02 -0400 Subject: [PATCH 20/26] Merge data version update to staging (#287) * point to archive digital ocean space * Fix for AB#14700, invalid selection type will no cause server api * use node v18.17.1 for ci * point to versioned digital ocean space * only specify major version of database * expose port 5433 * use 5432 within docker network * point to archive digital ocean space * update postgres to 15 * build from spaces version folders * point to 2023-08-21 decennial data * update table variables that changed * Decennial data version with: - No space at end of male var - 2020 var metadata - ccd prefix in geoid * Updated geography version. * populate support_geoids table with ccds update selection table to support ccds * Add CCDs to summary level * update ci to postgres:15 and node 18 * update ci to run ubuntu 22 * add special variable configurations * Revert "use 5432 within docker network" and "expose port 5433" This reverts commits b8a66f123d82861c9f3f1e38cba7a46f5bbcb9c9 and 479fc95a8ea25bfa917523a7c2e513b5d17182d7 * remove CCDs from support geoids table migration * fix agdpdrt and popacre calculations * support decennial variable names for mdage bins * remove CCDs from support geoids table migration (reapplication of change) * use data version with two decimal landacre precision * use regex match in median error calculation * Add index on geoid column for ACS db tables (#261) * Add index on geoid column for ACS db tables * Add INDEX_NAME var to etl script and update acs.sql to use it to create geoid index * Update decennial.sql to use INDEX_NAME variable to create geoid index --------- Co-authored-by: Tyler Matteo * Issue/16067 PFF Updates (#272) * Population FactFinder 2018-2022 ACS & 2020 DDHC-A Updates * Issue 268 update constants * Issue 256 Update top and bottom thresholds * issue 274 update constants for data migration * Update to working version folders for acs and decennial csvs * Closes #1147 - updated decennial metadata script to reflect domain change from decennial to census - added semicolons in constants js * updated constants with new dates --------- Co-authored-by: pratishta * 279 updated constants * issue 284 updated constants to pull new data (#286) --------- Co-authored-by: tangoyankee Co-authored-by: dhochbaum-dcp Co-authored-by: Bryan Co-authored-by: Horatio Co-authored-by: pratishta Co-authored-by: horatio --- app.js | 6 +++--- special-calculations/data/constants.js | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app.js b/app.js index 8c2bce8..317296d 100644 --- a/app.js +++ b/app.js @@ -19,10 +19,10 @@ app.db = pgp(process.env.DATABASE_URL); // allows CORS app.use(cors({ - origin: ['https://staging--labs-factfinder.netlify.app','https://develop--labs-factfinder.netlify.app','https://popfactfinder.planning.nyc.gov', 'http://localhost:4200','https://factfinder-staging.planninglabs.nyc','https://factfinder-develop.planninglabs.nyc'], - methods: "GET,HEAD,PUT,PATCH,POST,DELETE", + origin: ['https://staging--labs-factfinder.netlify.app', 'https://develop--labs-factfinder.netlify.app', 'https://popfactfinder.planning.nyc.gov', 'http://localhost:4200', 'https://factfinder-staging.planninglabs.nyc', 'https://factfinder-develop.planninglabs.nyc'], + methods: 'GET,HEAD,PUT,PATCH,POST,DELETE', allowedHeaders: 'X-Requested-With,Content-Type,Authorization', -})) +})); // middleware app.use(logger('dev')); diff --git a/special-calculations/data/constants.js b/special-calculations/data/constants.js index 581796e..a019416 100644 --- a/special-calculations/data/constants.js +++ b/special-calculations/data/constants.js @@ -8,14 +8,14 @@ const PREV_YEAR = 'Y2006-2010'; // TODO: update with actual previous year const GEOGRAPHY_VERSION = '2023-08-29'; const DECENNIAL_SCHEMA_NAME = 'decennial'; const DECENNIAL_LATEST_TABLE_NAME = '2020'; -const DECENNIAL_LATEST_VERSION = '2024-03-15'; +const DECENNIAL_LATEST_VERSION = '2024-03-19'; const DECENNIAL_EARLIEST_TABLE_NAME = '2010'; -const DECENNIAL_EARLIEST_VERSION = '2024-03-15'; +const DECENNIAL_EARLIEST_VERSION = '2024-03-19'; const ACS_SCHEMA_NAME = 'acs'; const ACS_LATEST_TABLE_NAME = '2022'; -const ACS_LATEST_VERSION = '2024-03-15'; +const ACS_LATEST_VERSION = '2024-03-19'; const ACS_EARLIEST_TABLE_NAME = '2010'; -const ACS_EARLIEST_VERSION = '2024-03-15'; +const ACS_EARLIEST_VERSION = '2024-03-19'; const ACS_METADATA_FULL_PATH = `"${ACS_SCHEMA_NAME}"."metadata"`; const ACS_LATEST_TABLE_FULL_PATH = `"${ACS_SCHEMA_NAME}"."${ACS_LATEST_TABLE_NAME}"`; const ACS_EARLIEST_TABLE_FULL_PATH = `"${ACS_SCHEMA_NAME}"."${ACS_EARLIEST_TABLE_NAME}"`; From e86628fd3b39770bd3ae05e7ec591de5605830eb Mon Sep 17 00:00:00 2001 From: horatio Date: Mon, 1 Apr 2024 13:47:54 -0400 Subject: [PATCH 21/26] issue 290 updated constants --- special-calculations/data/constants.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/special-calculations/data/constants.js b/special-calculations/data/constants.js index a019416..4fbacea 100644 --- a/special-calculations/data/constants.js +++ b/special-calculations/data/constants.js @@ -8,14 +8,14 @@ const PREV_YEAR = 'Y2006-2010'; // TODO: update with actual previous year const GEOGRAPHY_VERSION = '2023-08-29'; const DECENNIAL_SCHEMA_NAME = 'decennial'; const DECENNIAL_LATEST_TABLE_NAME = '2020'; -const DECENNIAL_LATEST_VERSION = '2024-03-19'; +const DECENNIAL_LATEST_VERSION = '2024-04-01'; const DECENNIAL_EARLIEST_TABLE_NAME = '2010'; -const DECENNIAL_EARLIEST_VERSION = '2024-03-19'; +const DECENNIAL_EARLIEST_VERSION = '2024-04-01'; const ACS_SCHEMA_NAME = 'acs'; const ACS_LATEST_TABLE_NAME = '2022'; -const ACS_LATEST_VERSION = '2024-03-19'; +const ACS_LATEST_VERSION = '2024-04-01'; const ACS_EARLIEST_TABLE_NAME = '2010'; -const ACS_EARLIEST_VERSION = '2024-03-19'; +const ACS_EARLIEST_VERSION = '2024-04-01'; const ACS_METADATA_FULL_PATH = `"${ACS_SCHEMA_NAME}"."metadata"`; const ACS_LATEST_TABLE_FULL_PATH = `"${ACS_SCHEMA_NAME}"."${ACS_LATEST_TABLE_NAME}"`; const ACS_EARLIEST_TABLE_FULL_PATH = `"${ACS_SCHEMA_NAME}"."${ACS_EARLIEST_TABLE_NAME}"`; From ff45d47478af950fcf2e48d27bc78b805269a1c7 Mon Sep 17 00:00:00 2001 From: horatio Date: Tue, 9 Apr 2024 13:34:59 -0400 Subject: [PATCH 22/26] issue/282 clean up 2021 acs table --- migrations/acs.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/acs.sql b/migrations/acs.sql index 8d6e3f1..578540c 100644 --- a/migrations/acs.sql +++ b/migrations/acs.sql @@ -1,5 +1,5 @@ CREATE SCHEMA IF NOT EXISTS acs; -DROP TABLE IF EXISTS acs.:"TABLE_NAME"; +DROP TABLE IF EXISTS acs."2021", acs.:"TABLE_NAME"; DROP INDEX IF EXISTS acs.:INDEX_NAME; CREATE TEMP TABLE tmp ( From 6b0ff0d2ef570b92c3dc190c4249c4d198a94cad Mon Sep 17 00:00:00 2001 From: horatio Date: Wed, 17 Apr 2024 11:01:17 -0400 Subject: [PATCH 23/26] 296 updated decennial contants --- special-calculations/data/constants.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/special-calculations/data/constants.js b/special-calculations/data/constants.js index 4fbacea..86a338c 100644 --- a/special-calculations/data/constants.js +++ b/special-calculations/data/constants.js @@ -8,9 +8,9 @@ const PREV_YEAR = 'Y2006-2010'; // TODO: update with actual previous year const GEOGRAPHY_VERSION = '2023-08-29'; const DECENNIAL_SCHEMA_NAME = 'decennial'; const DECENNIAL_LATEST_TABLE_NAME = '2020'; -const DECENNIAL_LATEST_VERSION = '2024-04-01'; +const DECENNIAL_LATEST_VERSION = '2024-04-16'; const DECENNIAL_EARLIEST_TABLE_NAME = '2010'; -const DECENNIAL_EARLIEST_VERSION = '2024-04-01'; +const DECENNIAL_EARLIEST_VERSION = '2024-04-16'; const ACS_SCHEMA_NAME = 'acs'; const ACS_LATEST_TABLE_NAME = '2022'; const ACS_LATEST_VERSION = '2024-04-01'; From 0319022b04bb1af9412d0644c7a43b3ce2eb6a80 Mon Sep 17 00:00:00 2001 From: horatio Date: Fri, 19 Apr 2024 18:38:57 -0400 Subject: [PATCH 24/26] 299 updateds constants to fix ACS social control total population gray text bug --- special-calculations/data/constants.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/special-calculations/data/constants.js b/special-calculations/data/constants.js index 86a338c..38f50ae 100644 --- a/special-calculations/data/constants.js +++ b/special-calculations/data/constants.js @@ -13,9 +13,9 @@ const DECENNIAL_EARLIEST_TABLE_NAME = '2010'; const DECENNIAL_EARLIEST_VERSION = '2024-04-16'; const ACS_SCHEMA_NAME = 'acs'; const ACS_LATEST_TABLE_NAME = '2022'; -const ACS_LATEST_VERSION = '2024-04-01'; +const ACS_LATEST_VERSION = '2024-04-19'; const ACS_EARLIEST_TABLE_NAME = '2010'; -const ACS_EARLIEST_VERSION = '2024-04-01'; +const ACS_EARLIEST_VERSION = '2024-04-19'; const ACS_METADATA_FULL_PATH = `"${ACS_SCHEMA_NAME}"."metadata"`; const ACS_LATEST_TABLE_FULL_PATH = `"${ACS_SCHEMA_NAME}"."${ACS_LATEST_TABLE_NAME}"`; const ACS_EARLIEST_TABLE_FULL_PATH = `"${ACS_SCHEMA_NAME}"."${ACS_EARLIEST_TABLE_NAME}"`; From 525c2cd2ddd078e0a21846906107d75d52751dc4 Mon Sep 17 00:00:00 2001 From: horatio Date: Mon, 29 Apr 2024 11:10:36 -0400 Subject: [PATCH 25/26] 301 updated decennial constants --- special-calculations/data/constants.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/special-calculations/data/constants.js b/special-calculations/data/constants.js index 38f50ae..5eab6b4 100644 --- a/special-calculations/data/constants.js +++ b/special-calculations/data/constants.js @@ -8,9 +8,9 @@ const PREV_YEAR = 'Y2006-2010'; // TODO: update with actual previous year const GEOGRAPHY_VERSION = '2023-08-29'; const DECENNIAL_SCHEMA_NAME = 'decennial'; const DECENNIAL_LATEST_TABLE_NAME = '2020'; -const DECENNIAL_LATEST_VERSION = '2024-04-16'; +const DECENNIAL_LATEST_VERSION = '2024-04-29'; const DECENNIAL_EARLIEST_TABLE_NAME = '2010'; -const DECENNIAL_EARLIEST_VERSION = '2024-04-16'; +const DECENNIAL_EARLIEST_VERSION = '2024-04-29'; const ACS_SCHEMA_NAME = 'acs'; const ACS_LATEST_TABLE_NAME = '2022'; const ACS_LATEST_VERSION = '2024-04-19'; From 157cc5d267ae14572920afe811e9b8e74440b265 Mon Sep 17 00:00:00 2001 From: horatio Date: Fri, 10 May 2024 11:17:40 -0400 Subject: [PATCH 26/26] issue 303 get coded rows to set isReliable to false when top/bottom coded --- utils/data-processor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/data-processor.js b/utils/data-processor.js index ea4b6f4..da22b9f 100644 --- a/utils/data-processor.js +++ b/utils/data-processor.js @@ -174,7 +174,7 @@ class DataProcessor { */ recalculateIsReliable(row, wasCoded) { // top- or bottom-coded values are not reliable - if (!row.correlationCoefficient) return; + if (!row.correlationCoefficient && !wasCoded) return; row.isReliable = wasCoded ? false : executeFormula(this.data, row.variable, 'isReliable'); } }