From a077a29c3f39c453ed22bc45bdd6cb35da52ee91 Mon Sep 17 00:00:00 2001 From: Lu Cui Date: Fri, 31 Mar 2023 11:36:02 +0100 Subject: [PATCH 1/3] feat(DB): Introduce DB write concern env var for createStatements (LLC-2389) --- .env.example | 3 +++ .../repo/modelsRepo/createStatements/mongo.ts | 10 ++++++++-- .../repo/modelsRepo/utils/mongoModels/constants.ts | 8 ++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 687118814..f23dfed65 100644 --- a/.env.example +++ b/.env.example @@ -111,3 +111,6 @@ EXPRESS_PORT=8081 # If set this will override AWS access secret for FS_S3_SECRET_ACCESS_KEY and WINSTON_CLOUDWATCH_SECRET_ACCESS_KEY #GLOBAL_AWS_IAM_SECRET_ACCESS_KEY= + +# Write concern for statements insertMany +#STORE_STATEMENT_WRITE_CONCERN_DEFAULT= \ No newline at end of file diff --git a/src/apps/statements/repo/modelsRepo/createStatements/mongo.ts b/src/apps/statements/repo/modelsRepo/createStatements/mongo.ts index 8349adb0a..4d68a1e19 100644 --- a/src/apps/statements/repo/modelsRepo/createStatements/mongo.ts +++ b/src/apps/statements/repo/modelsRepo/createStatements/mongo.ts @@ -1,5 +1,9 @@ +/* eslint-disable no-console */ import { ObjectId } from 'mongodb'; -import { STATEMENTS_COLLECTION_NAME } from '../utils/mongoModels/constants'; +import { + STATEMENTS_COLLECTION_NAME, + STORE_STATEMENT_WRITE_CONCERN_DEFAULT, +} from '../utils/mongoModels/constants'; import FacadeConfig from '../utils/mongoModels/FacadeConfig'; import { encodeDotsInStatement } from '../utils/mongoModels/replaceDotsInStatement'; import Signature from './Signature'; @@ -22,7 +26,9 @@ export default (config: FacadeConfig): Signature => { }); const collection = (await config.db()).collection(STATEMENTS_COLLECTION_NAME); - await collection.insertMany(documents); + await collection.insertMany(documents, { + writeConcern: { w: STORE_STATEMENT_WRITE_CONCERN_DEFAULT }, + }); return opts.models; }; }; diff --git a/src/apps/statements/repo/modelsRepo/utils/mongoModels/constants.ts b/src/apps/statements/repo/modelsRepo/utils/mongoModels/constants.ts index f4fcdc738..9d8768fe2 100644 --- a/src/apps/statements/repo/modelsRepo/utils/mongoModels/constants.ts +++ b/src/apps/statements/repo/modelsRepo/utils/mongoModels/constants.ts @@ -1,3 +1,11 @@ +import getNumberOption from 'jscommons/dist/config/getNumberOption'; + export const STATEMENTS_COLLECTION_NAME = 'statements'; export const LRS_COLLECTION_NAME = 'lrs'; export const FULL_ACTIVITIES_COLLECTION_NAME = 'fullActivities'; + +const WRITE_CONCERN_DEFAULT = 3; +export const STORE_STATEMENT_WRITE_CONCERN_DEFAULT = + process.env.STORE_STATEMENT_WRITE_CONCERN_DEFAULT === 'majority' + ? process.env.STORE_STATEMENT_WRITE_CONCERN_DEFAULT + : getNumberOption(process.env.STORE_STATEMENT_WRITE_CONCERN_DEFAULT, WRITE_CONCERN_DEFAULT); From 3da315fbb11542686011bf3f2bfa3a31a63c1656 Mon Sep 17 00:00:00 2001 From: Lu Cui Date: Fri, 31 Mar 2023 12:07:46 +0100 Subject: [PATCH 2/3] fix(Tests): Use write concern of 1 for CI process due to no DB node replication (LLC-2389) --- .github/workflows/integration.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 20924991f..ea422bff9 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -42,6 +42,7 @@ jobs: env: WINSTON_CONSOLE_LEVEL: none EXPRESS_PORT: 1337 + STORE_STATEMENT_WRITE_CONCERN_DEFAULT: 1 services: mongodb: @@ -93,6 +94,7 @@ jobs: env: WINSTON_CONSOLE_LEVEL: none EXPRESS_PORT: 1337 + STORE_STATEMENT_WRITE_CONCERN_DEFAULT: 1 STORAGE_REPO: s3 SUB_FOLDER_ACTIVITIES: gha/$GITHUB_RUN_NUMBER_$GITHUB_RUN_ATTEMPT SUB_FOLDER_AGENTS: SUB_FOLDER_AGENTS=gha/$GITHUB_RUN_NUMBER_$GITHUB_RUN_ATTEMPT @@ -153,6 +155,7 @@ jobs: env: WINSTON_CONSOLE_LEVEL: none EXPRESS_PORT: 1337 + STORE_STATEMENT_WRITE_CONCERN_DEFAULT: 1 STORAGE_REPO: azure FS_AZURE_CONTAINER_SUBFOLDER: gha/$GITHUB_RUN_NUMBER_$GITHUB_RUN_ATTEMPT FS_AZURE_ACCOUNT: ${{ secrets.FS_AZURE_ACCOUNT }} @@ -209,6 +212,7 @@ jobs: env: WINSTON_CONSOLE_LEVEL: none EXPRESS_PORT: 1337 + STORE_STATEMENT_WRITE_CONCERN_DEFAULT: 1 STORAGE_REPO: google FS_GOOGLE_CLOUD_BUCKET: ${{ secrets.FS_GOOGLE_CLOUD_BUCKET }} FS_GOOGLE_CLOUD_PROJECT_ID: ${{ secrets.FS_GOOGLE_CLOUD_PROJECT_ID }} From 6484be43910ade39b6339aa0acb904c5534a83cd Mon Sep 17 00:00:00 2001 From: Lu Cui Date: Mon, 3 Apr 2023 15:40:36 +0100 Subject: [PATCH 3/3] refactor: Add timer log for testing only (LLC-2389) --- src/apps/statements/repo/modelsRepo/createStatements/mongo.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/apps/statements/repo/modelsRepo/createStatements/mongo.ts b/src/apps/statements/repo/modelsRepo/createStatements/mongo.ts index 4d68a1e19..f5b463816 100644 --- a/src/apps/statements/repo/modelsRepo/createStatements/mongo.ts +++ b/src/apps/statements/repo/modelsRepo/createStatements/mongo.ts @@ -26,9 +26,11 @@ export default (config: FacadeConfig): Signature => { }); const collection = (await config.db()).collection(STATEMENTS_COLLECTION_NAME); + console.time('STATEMENT INSERTION'); await collection.insertMany(documents, { writeConcern: { w: STORE_STATEMENT_WRITE_CONCERN_DEFAULT }, }); + console.timeEnd('STATEMENT INSERTION'); return opts.models; }; };