diff --git a/.env_example b/.env_example index 00ede5275..0a4bc9962 100644 --- a/.env_example +++ b/.env_example @@ -32,12 +32,4 @@ princeApiPath=placeholder warmupSchedule="60 minutes" warmupConcurrency=5 iamPath=/ -iamPermissionsBoundary="bound" -runV2DataMigration=false - -# -- SET THESE TO JUNK IF NOT RUNNING MIGRATION -postgresDatabase=placeholder -postgresHost=placeholder -postgresUser=placeholder -postgresPassword=placeholder -# -- \ No newline at end of file +iamPermissionsBoundary="bound" \ No newline at end of file diff --git a/README.md b/README.md index ae1c7d302..e36968445 100644 --- a/README.md +++ b/README.md @@ -61,16 +61,6 @@ There are two mechanisms for seeding data. - This is useful for deploying data such as section base templates, and keeping it up to date with the code base. - Adding specific test seed data to environments may be useful for things like cypress tests. This can be accomplished with the test-tables directory, referencing the same seed-local tables if desired. -### V2 Data Migration - -The data migration for v2 -> v3 is controlled via ssm parameters, and can be kicked off in an env on deploy by setting /configuration/{env}/runV2DataMigration. If you have access to the v2 databases. -, it can also be run to write v2 data into the local database by setting the runV2DataMigration env flag to true, and invoking the function with: - -```aws lambda invoke /dev/null \ - --endpoint-url http://localhost:3003 \ - --function-name database-local-dataMigration -``` - ### Local Development Random Info Local dev is configured in typescript project in `./src`. The entrypoint is `./src/dev.ts`, it manages running the moving pieces locally: the API, the database, the filestore, and the frontend. diff --git a/services/database/handlers/dataMigration/dataMigration.js b/services/database/handlers/dataMigration/dataMigration.js deleted file mode 100644 index 7bc243d84..000000000 --- a/services/database/handlers/dataMigration/dataMigration.js +++ /dev/null @@ -1,18 +0,0 @@ -// eslint-disable-next-line no-unused-vars -async function myHandler(event, context, callback) { - // eslint-disable-next-line no-console - console.log("Beginning v2 Migration"); - - const buildRunner = require("./services/migrationRunner"); - const migrationRunner = buildRunner(); - - const { tables } = require("./tables/index"); - for (const table of tables) { - await migrationRunner.executeMigration(table); - } - - // eslint-disable-next-line no-console - console.log("V2 Migration Finished"); -} - -exports.handler = myHandler; diff --git a/services/database/handlers/dataMigration/services/migrationRunner.js b/services/database/handlers/dataMigration/services/migrationRunner.js deleted file mode 100644 index ec8f53d7a..000000000 --- a/services/database/handlers/dataMigration/services/migrationRunner.js +++ /dev/null @@ -1,101 +0,0 @@ -let dynamoClient; -let pgClient; -let dynamoPrefix; - -const runMigration = async (migrationInstructions) => { - const { name, query, transform, tableNameBuilder, keys } = - migrationInstructions; - // eslint-disable-next-line no-console - console.log(` - ${name}: Loading data from postgres (v2)`); - - // Extract - const result = await pgClient.query(query); - const rows = result.rows; - if (!rows || rows.length <= 0) return; - - // Transform - const transformed = transform ? transform(rows) : rows; - - // Load - if (transformed && transformed.length > 0) { - const tableName = tableNameBuilder(dynamoPrefix); - // eslint-disable-next-line no-console - console.log(` - ${tableName}: Saving ${rows.length} entries (v3)`); - await updateItems(tableName, transformed, keys); - } -}; - -const updateItems = async (tableName, items, keys) => { - try { - for (const item of items) { - let key = {}; - for (const k of keys) { - key[k] = item[k]; - delete item[k]; - } - - const params = { - TableName: tableName, - Key: key, - ...convertToDynamoExpression(item), - }; - await dynamoClient.update(params).promise(); - } - } catch (e) { - // eslint-disable-next-line no-console - console.log(` -- ERROR UPLOADING ${tableName}\n`, e); - } -}; - -const convertToDynamoExpression = (listOfVars) => { - let expressionAttributeNames = {}; - let expressionAttributeValues = {}; - let updateExpression = ""; - Object.keys(listOfVars).forEach((key, index) => { - expressionAttributeNames[`#${key}`] = key; - expressionAttributeValues[`:${key}`] = listOfVars[key]; - - updateExpression = - index === 0 - ? `set #${key}=:${key}` - : `${updateExpression}, #${key}=:${key}`; - }); - return { - UpdateExpression: updateExpression, - ExpressionAttributeNames: expressionAttributeNames, - ExpressionAttributeValues: expressionAttributeValues, - }; -}; - -const buildMigrationRunner = () => { - const aws = require("aws-sdk"); - const { Client } = require("pg"); - - const dynamoConfig = {}; - const endpoint = process.env.DYNAMODB_URL; - if (endpoint) { - dynamoConfig.endpoint = endpoint; - dynamoConfig.accessKeyId = "LOCALFAKEKEY"; // pragma: allowlist secret - dynamoConfig.secretAccessKey = "LOCALFAKESECRET"; // pragma: allowlist secret - dynamoPrefix = "local"; - } else { - dynamoConfig["region"] = "us-east-1"; - dynamoPrefix = process.env.dynamoPrefix; - } - dynamoClient = new aws.DynamoDB.DocumentClient(dynamoConfig); - - pgClient = new Client({ - host: process.env.postgresHost, - database: process.env.postgresDb, - port: 5432, - user: process.env.postgresUser, - password: process.env.postgresPassword, - }); - pgClient.connect(); - - return { - executeMigration: runMigration, - }; -}; - -module.exports = buildMigrationRunner; diff --git a/services/database/handlers/dataMigration/tables/acs.js b/services/database/handlers/dataMigration/tables/acs.js deleted file mode 100644 index 2946ba629..000000000 --- a/services/database/handlers/dataMigration/tables/acs.js +++ /dev/null @@ -1,12 +0,0 @@ -const query = `select state_id as "stateId", year, number_uninsured as "numberUninsured", number_uninsured_moe as "numberUninsuredMoe", - percent_uninsured as "percentUninsured", percent_uninsured_moe as "percentUninsuredMoe" - from carts_api_acs`; - -const migration = { - name: "ACS", - query, - tableNameBuilder: (stage) => `${stage}-acs`, - keys: ["stateId", "year"], -}; - -module.exports = migration; diff --git a/services/database/handlers/dataMigration/tables/enrollmentCounts.js b/services/database/handlers/dataMigration/tables/enrollmentCounts.js deleted file mode 100644 index ee33222b4..000000000 --- a/services/database/handlers/dataMigration/tables/enrollmentCounts.js +++ /dev/null @@ -1,29 +0,0 @@ -const query = `select distinct on (state_id, year_to_modify, type_of_enrollment, index_to_update) - filter_id as "filterId", state_id as "stateId", year_to_modify as "yearToModify", - type_of_enrollment as "typeOfEnrollment", index_to_update as "indexToUpdate", - enrollment_count as "enrollmentCount", id as "oldId" - from stg_enrollment_counts - order by state_id, year_to_modify, type_of_enrollment, index_to_update, id desc`; - -const typeMap = { - "Medicaid Expansion CHIP": "medicaid_exp_chip", - "Separate CHIP": "separate_chip", -}; - -const migration = { - name: "SEDS CHIP Enrollment Data", - query, - tableNameBuilder: (stage) => `${stage}-stg-enrollment-counts`, - transform: (rows) => { - const createdTime = new Date().toLocaleString(); - rows.map((row) => { - row.pk = `${row.stateId}-${row.yearToModify}`; - row.entryKey = `${typeMap[row.typeOfEnrollment]}-${row.indexToUpdate}`; - row.createdTime = createdTime; - }); - return rows; - }, - keys: ["pk", "entryKey"], -}; - -module.exports = migration; diff --git a/services/database/handlers/dataMigration/tables/fmap.js b/services/database/handlers/dataMigration/tables/fmap.js deleted file mode 100644 index 165886019..000000000 --- a/services/database/handlers/dataMigration/tables/fmap.js +++ /dev/null @@ -1,11 +0,0 @@ -const query = `select state_id as "stateId", fiscal_year as "fiscalYear", "enhanced_FMAP" as "enhancedFmap" - from carts_api_fmap`; - -const migration = { - name: "FMAP", - query, - tableNameBuilder: (stage) => `${stage}-fmap`, - keys: ["stateId", "fiscalYear"], -}; - -module.exports = migration; diff --git a/services/database/handlers/dataMigration/tables/index.js b/services/database/handlers/dataMigration/tables/index.js deleted file mode 100644 index eda645bc4..000000000 --- a/services/database/handlers/dataMigration/tables/index.js +++ /dev/null @@ -1,12 +0,0 @@ -const tables = [ - require("./acs"), - // require("./enrollmentCounts"), - require("./fmap"), - require("./section"), - require("./sectionBase"), - require("./state"), - require("./stateStatus"), - require("./uploads"), -]; - -module.exports = { tables }; diff --git a/services/database/handlers/dataMigration/tables/section.js b/services/database/handlers/dataMigration/tables/section.js deleted file mode 100644 index 76ff66a2e..000000000 --- a/services/database/handlers/dataMigration/tables/section.js +++ /dev/null @@ -1,18 +0,0 @@ -const query = `select contents->'section'->'state' as "stateId", contents->'section'->'year' as year, contents->'section'->'ordinal' as "sectionId", modified_on as "lastChanged", contents -from carts_api_section`; - -const migration = { - name: "Sections", - query, - transform: (rows) => { - rows.map((row) => { - row.pk = `${row.stateId}-${row.year}`; - row.lastChanged = row.lastChanged?.toString(); - }); - return rows; - }, - tableNameBuilder: (stage) => `${stage}-section`, - keys: ["pk", "sectionId"], -}; - -module.exports = migration; diff --git a/services/database/handlers/dataMigration/tables/sectionBase.js b/services/database/handlers/dataMigration/tables/sectionBase.js deleted file mode 100644 index bf55df664..000000000 --- a/services/database/handlers/dataMigration/tables/sectionBase.js +++ /dev/null @@ -1,11 +0,0 @@ -const query = `select contents->'section'->'year' as year, contents->'section'->'ordinal' as "sectionId", contents -from carts_api_sectionbase`; - -const migration = { - name: "Section Base", - query, - tableNameBuilder: (stage) => `${stage}-section-base`, - keys: ["sectionId", "year"], -}; - -module.exports = migration; diff --git a/services/database/handlers/dataMigration/tables/state.js b/services/database/handlers/dataMigration/tables/state.js deleted file mode 100644 index 92fc7134a..000000000 --- a/services/database/handlers/dataMigration/tables/state.js +++ /dev/null @@ -1,11 +0,0 @@ -const query = `select distinct on (code) code, name, program_type as "programType" - from carts_api_state`; - -const migration = { - name: "State", - query, - tableNameBuilder: (stage) => `${stage}-state`, - keys: ["code"], -}; - -module.exports = migration; diff --git a/services/database/handlers/dataMigration/tables/stateStatus.js b/services/database/handlers/dataMigration/tables/stateStatus.js deleted file mode 100644 index df06ac09c..000000000 --- a/services/database/handlers/dataMigration/tables/stateStatus.js +++ /dev/null @@ -1,19 +0,0 @@ -const query = `select state_id as "stateId", year, status, cas.program_type as "programType", cass.modified_by as "username", cass.modified_on as "lastChanged" -from carts_api_statestatus cass -join carts_api_state cas on cas.code = cass.state_id -where year <= 2021`; - -const migration = { - name: "State Status", - query, - transform: (rows) => { - rows.map((row) => { - row.lastChanged = row.lastChanged?.toString(); - }); - return rows; - }, - tableNameBuilder: (stage) => `${stage}-state-status`, - keys: ["stateId", "year"], -}; - -module.exports = migration; diff --git a/services/database/handlers/dataMigration/tables/uploads.js b/services/database/handlers/dataMigration/tables/uploads.js deleted file mode 100644 index 4b38378ac..000000000 --- a/services/database/handlers/dataMigration/tables/uploads.js +++ /dev/null @@ -1,21 +0,0 @@ -const query = `select filename, aws_filename as "awsFilename", question_id as "questionId", uploaded_date as "uploadedDate", - uploaded_username as "uploadedUsername", uploaded_state as "uploadedState" - from carts_api_uploadedfiles`; - -const migration = { - name: "Uploads", - query, - transform: (rows) => { - rows.map((row) => { - row.fileId = `${row.questionId.slice(0, 4)}-${row.questionId}_${ - row.awsFilename - }`; - row.uploadedDate = row.uploadedDate?.toString(); - }); - return rows; - }, - tableNameBuilder: (stage) => `${stage}-uploads`, - keys: ["uploadedState", "fileId"], -}; - -module.exports = migration; diff --git a/services/database/serverless.yml b/services/database/serverless.yml index f8ef41676..df5f4ba40 100644 --- a/services/database/serverless.yml +++ b/services/database/serverless.yml @@ -58,23 +58,10 @@ custom: sources: [./data/seed/seed-section-base.json] - table: ${self:custom.stageEnrollmentCountsTableName} sources: [./data/seed-local/seed-stg-enrollment-counts.json] - # Disabling migration handler to unattach from v2 security group for teardown. Remove this and dataMigration folder in future - # dataMigrationEnabled: ${env:runV2DataMigration, ssm:/configuration/${self:custom.stage}/runV2DataMigration, "false"} - # dataMigrationEnvMap: - # production: prod - # val: staging - # main: master - # default: master - # dataMigrationEnv: ${self:custom.dataMigrationEnvMap.${self:custom.stage}, self:custom.dataMigrationEnvMap.default} - dataMigrationEnabled: false scripts: hooks: deploy:finalize: | serverless invoke --stage ${self:custom.stage} --function seed - if [ ${self:custom.dataMigrationEnabled} = "true" ]; - then - serverless invoke --stage ${self:custom.stage} --function dataMigration - fi dynamodb: stages: - local @@ -105,21 +92,6 @@ provider: Resource: "*" functions: - # Disabling migration handler to unattach from v2 security group for teardown. Remove this and dataMigration folder in future - # dataMigration: - # handler: handlers/dataMigration/dataMigration.handler - # environment: - # postgresDb: ${env:postgresDatabase, ssm:/${self:custom.dataMigrationEnv}/postgres_db} - # postgresHost: ${env:postgresHost, ssm:/${self:custom.dataMigrationEnv}/postgres_host} - # postgresUser: ${env:postgresUser, ssm:/${self:custom.dataMigrationEnv}/postgres_user} - # postgresPassword: ${env:postgresPassword, ssm:/${self:custom.dataMigrationEnv}/custom/postgres_custom_password, ssm:/${self:custom.dataMigrationEnv}/postgres_password} - # dynamoPrefix: ${self:custom.stage} - # timeout: 120 - # vpc: - # securityGroupIds: - # - ${env:runV2DataMigration, ssm:/configuration/${self:custom.stage}/migrationSecurityGroup, ssm:/configuration/default/migrationSecurityGroup} - # subnetIds: - # - ${env:runV2DataMigration, ssm:/configuration/${self:custom.stage}/migrationSubnet, ssm:/configuration/default/migrationSubnet} seed: handler: handlers/seed/seed.handler environment: diff --git a/yarn.lock b/yarn.lock index 2fab8e9d4..7b9ade7ce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8850,19 +8850,10 @@ loader-runner@^4.2.0: resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-4.3.0.tgz#c1b4a163b99f614830353b16755e7149ac2314e1" integrity sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg== -loader-utils@^1.2.3: - version "1.4.2" - resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.4.2.tgz#29a957f3a63973883eb684f10ffd3d151fec01a3" - integrity sha512-I5d00Pd/jwMD2QCduo657+YM/6L3KZu++pmX9VFncxaxvHcru9jx1lBaFft+r4Mt2jK0Yhp41XlRAihzPxHNCg== - dependencies: - big.js "^5.2.2" - emojis-list "^3.0.0" - json5 "^1.0.1" - -loader-utils@^2.0.0: - version "2.0.2" - resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-2.0.2.tgz#d6e3b4fb81870721ae4e0868ab11dd638368c129" - integrity sha512-TM57VeHptv569d/GKh6TAYdzKblwDNiumOdkFnejjD0XwTH87K90w3O7AiJRqdQoXygvi1VQTJTLGhJl7WqA7A== +loader-utils@^1.2.3, loader-utils@^2.0.0, loader-utils@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-2.0.4.tgz#8b5cb38b5c34a9a018ee1fc0e6a066d1dfcc528c" + integrity sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw== dependencies: big.js "^5.2.2" emojis-list "^3.0.0"