From a405449a9c60523a58b09ef2c3114386b1ed0183 Mon Sep 17 00:00:00 2001 From: Evans Dianga Date: Wed, 17 Jul 2024 13:11:38 +0300 Subject: [PATCH 1/5] feat(import-user-profiles): namespace `processedDocs` Refs Tangerine-Community/Tangerine#3696 --- .../import-user-profile/import-user-profile.component.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/src/app/user-profile/import-user-profile/import-user-profile.component.ts b/client/src/app/user-profile/import-user-profile/import-user-profile.component.ts index 89b524dda..e3ec22666 100644 --- a/client/src/app/user-profile/import-user-profile/import-user-profile.component.ts +++ b/client/src/app/user-profile/import-user-profile/import-user-profile.component.ts @@ -58,11 +58,12 @@ export class ImportUserProfileComponent implements AfterContentInit { this.shortCode = this.userShortCodeInput.nativeElement.value; let newUserProfile = await this.http.get(`${this.appConfig.serverUrl}api/${this.appConfig.groupId}/responsesByUserProfileShortCode/${this.shortCode}/?userProfile=true`).toPromise() if(!!newUserProfile){ + const username = this.userService.getCurrentUser() this.state = this.STATE_SYNCING await this.userService.saveUserAccount({ ...this.userAccount, userUUID: newUserProfile['_id'], initialProfileComplete: true }) this.totalDocs = (await this.http.get(`${this.appConfig.serverUrl}api/${this.appConfig.groupId}/responsesByUserProfileShortCode/${this.shortCode}/?totalRows=true`).toPromise())['totalDocs'] const docsToQuery = 1000; - let processedDocs = +localStorage.getItem('processedDocs') || 0; + let processedDocs = +localStorage.getItem(`${username}-processedDocs`) || 0; while (processedDocs < this.totalDocs) { this.docs = await this.http.get(`${this.appConfig.serverUrl}api/${this.appConfig.groupId}/responsesByUserProfileShortCode/${this.shortCode}/${docsToQuery}/${processedDocs}`).toPromise() for (let doc of this.docs) { @@ -71,7 +72,7 @@ export class ImportUserProfileComponent implements AfterContentInit { } processedDocs += this.docs.length; this.processedDocs = processedDocs - localStorage.setItem('processedDocs', String(processedDocs)) + localStorage.setItem(`${username}-processedDocs`, String(processedDocs)) } } else{ this.state = this.STATE_NOT_FOUND From eaf3ea83375e9ef217fccbe17dfca1d3219753ef Mon Sep 17 00:00:00 2001 From: esurface Date: Thu, 18 Jul 2024 14:33:24 -0400 Subject: [PATCH 2/5] Add client/dist to develop.sh --- develop.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/develop.sh b/develop.sh index a7af59e3c..154a5c64c 100755 --- a/develop.sh +++ b/develop.sh @@ -221,6 +221,7 @@ OPTIONS="--link $T_COUCHDB_CONTAINER_NAME:couchdb \ --volume $(pwd)/server/package.json:/tangerine/server/package.json:delegated \ --volume $(pwd)/server/src:/tangerine/server/src:delegated \ --volume $(pwd)/client/src:/tangerine/client/src:delegated \ + --volume $(pwd)/client/dist:/tangerine/client/dist:delegated \ --volume $(pwd)/server/reporting:/tangerine/server/reporting:delegated \ --volume $(pwd)/upgrades:/tangerine/upgrades:delegated \ --volume $(pwd)/scripts/generate-csv/bin.js:/tangerine/scripts/generate-csv/bin.js:delegated \ From 52eb6be4e877db74f8c49fb801c1612a5b3a4a41 Mon Sep 17 00:00:00 2001 From: esurface Date: Thu, 18 Jul 2024 14:33:44 -0400 Subject: [PATCH 3/5] Update debugging-reporting.md --- docs/developer/debugging-reporting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer/debugging-reporting.md b/docs/developer/debugging-reporting.md index a1f825ce1..5d8c97db8 100644 --- a/docs/developer/debugging-reporting.md +++ b/docs/developer/debugging-reporting.md @@ -11,7 +11,7 @@ Summary of steps: 1. Enter the container on command line with `docker exec -it tangerine bash`. 1. Clear reporting cache with command `reporting-cache-clear`. 1. Run a batch with debugger enabled by running command `node --inspect-brk=0.0.0.0:9228 $(which reporting-worker-batch)`. -1. Latch onto debugging session using Chrome inspect. You may need to click "configure" and add `localhost:9228` to "Target discovery settings". +1. Latch onto debugging session using [Chrome Inspect](chrome://inspect/#devices). You may need to click "configure" and add `localhost:9228` to "Target discovery settings". ## Instructions From c95317d1caea70df1bc75461be46b1557d412eb5 Mon Sep 17 00:00:00 2001 From: esurface Date: Thu, 18 Jul 2024 14:35:00 -0400 Subject: [PATCH 4/5] Use variable service to store -processedDocs in import-user-profile --- .../import-user-profile.component.ts | 12 +++++++----- server/src/group-views.js | 2 +- server/src/upgrade/v3.31.0.js | 13 +++++++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) create mode 100755 server/src/upgrade/v3.31.0.js diff --git a/client/src/app/user-profile/import-user-profile/import-user-profile.component.ts b/client/src/app/user-profile/import-user-profile/import-user-profile.component.ts index e3ec22666..3262fefbe 100644 --- a/client/src/app/user-profile/import-user-profile/import-user-profile.component.ts +++ b/client/src/app/user-profile/import-user-profile/import-user-profile.component.ts @@ -2,10 +2,10 @@ import { Component, ViewChild, ElementRef, AfterContentInit } from '@angular/cor import { HttpClient } from '@angular/common/http'; import { UserService } from '../../shared/_services/user.service'; import { Router } from '@angular/router'; -import PouchDB from 'pouchdb'; import { AppConfigService } from 'src/app/shared/_services/app-config.service'; import { AppConfig } from 'src/app/shared/_classes/app-config.class'; import { _TRANSLATE } from 'src/app/shared/translation-marker'; +import { VariableService } from 'src/app/shared/_services/variable.service'; @Component({ @@ -33,7 +33,8 @@ export class ImportUserProfileComponent implements AfterContentInit { private router: Router, private http: HttpClient, private userService: UserService, - private appConfigService: AppConfigService + private appConfigService: AppConfigService, + private variableService: VariableService ) { } ngAfterContentInit() { @@ -63,7 +64,8 @@ export class ImportUserProfileComponent implements AfterContentInit { await this.userService.saveUserAccount({ ...this.userAccount, userUUID: newUserProfile['_id'], initialProfileComplete: true }) this.totalDocs = (await this.http.get(`${this.appConfig.serverUrl}api/${this.appConfig.groupId}/responsesByUserProfileShortCode/${this.shortCode}/?totalRows=true`).toPromise())['totalDocs'] const docsToQuery = 1000; - let processedDocs = +localStorage.getItem(`${username}-processedDocs`) || 0; + let previousProcessedDocs = await this.variableService.get(`${username}-processedDocs`) + let processedDocs = parseInt(previousProcessedDocs) || 0; while (processedDocs < this.totalDocs) { this.docs = await this.http.get(`${this.appConfig.serverUrl}api/${this.appConfig.groupId}/responsesByUserProfileShortCode/${this.shortCode}/${docsToQuery}/${processedDocs}`).toPromise() for (let doc of this.docs) { @@ -72,9 +74,9 @@ export class ImportUserProfileComponent implements AfterContentInit { } processedDocs += this.docs.length; this.processedDocs = processedDocs - localStorage.setItem(`${username}-processedDocs`, String(processedDocs)) + await this.variableService.set(`${username}-processedDocs`, String(processedDocs)) } - } else{ + } else { this.state = this.STATE_NOT_FOUND } this.router.navigate([`/${this.appConfig.homeUrl}`] ); diff --git a/server/src/group-views.js b/server/src/group-views.js index db658c92b..533648624 100644 --- a/server/src/group-views.js +++ b/server/src/group-views.js @@ -78,7 +78,7 @@ module.exports.responsesByUserProfileShortCode = function(doc) { } module.exports.userProfileByUserProfileShortCode = function (doc) { - if (doc.collection === "TangyFormResponse"&&doc.form && doc.form.id === 'user-profile') { + if (doc.collection === "TangyFormResponse" && doc.form && doc.form.id === 'user-profile') { return emit(doc._id.substr(doc._id.length - 6, doc._id.length), true); } } diff --git a/server/src/upgrade/v3.31.0.js b/server/src/upgrade/v3.31.0.js new file mode 100755 index 000000000..0ec27efb8 --- /dev/null +++ b/server/src/upgrade/v3.31.0.js @@ -0,0 +1,13 @@ +#!/usr/bin/env node +const util = require("util"); +const exec = util.promisify(require('child_process').exec) + +async function go() { + console.log('Updating views with a new view used for the User Profile listing.') + try { + await exec(`/tangerine/server/src/scripts/push-all-groups-views.js `) + } catch (e) { + console.log(e) + } +} +go() From 15cbac9e6e042cd7ee71ec45a667e4f8cd81db1f Mon Sep 17 00:00:00 2001 From: esurface Date: Thu, 18 Jul 2024 15:49:29 -0400 Subject: [PATCH 5/5] Simplify user profile short code APIs and reduce views --- .../import-user-profile.component.ts | 6 ++-- server/src/express-app.js | 1 + server/src/group-views.js | 29 ++++--------------- ...up-responses-by-user-profile-short-code.js | 13 ++------- .../group-user-profile-by-short-code.js | 18 ++++++++++++ 5 files changed, 31 insertions(+), 36 deletions(-) create mode 100644 server/src/routes/group-user-profile-by-short-code.js diff --git a/client/src/app/user-profile/import-user-profile/import-user-profile.component.ts b/client/src/app/user-profile/import-user-profile/import-user-profile.component.ts index 3262fefbe..39a07a078 100644 --- a/client/src/app/user-profile/import-user-profile/import-user-profile.component.ts +++ b/client/src/app/user-profile/import-user-profile/import-user-profile.component.ts @@ -57,11 +57,11 @@ export class ImportUserProfileComponent implements AfterContentInit { try { this.appConfig = await this.appConfigService.getAppConfig() this.shortCode = this.userShortCodeInput.nativeElement.value; - let newUserProfile = await this.http.get(`${this.appConfig.serverUrl}api/${this.appConfig.groupId}/responsesByUserProfileShortCode/${this.shortCode}/?userProfile=true`).toPromise() - if(!!newUserProfile){ + let existingUserProfile = await this.http.get(`${this.appConfig.serverUrl}api/${this.appConfig.groupId}/userProfileByShortCode/${this.shortCode}`).toPromise() + if(!!existingUserProfile){ const username = this.userService.getCurrentUser() this.state = this.STATE_SYNCING - await this.userService.saveUserAccount({ ...this.userAccount, userUUID: newUserProfile['_id'], initialProfileComplete: true }) + await this.userService.saveUserAccount({ ...this.userAccount, userUUID: existingUserProfile['_id'], initialProfileComplete: true }) this.totalDocs = (await this.http.get(`${this.appConfig.serverUrl}api/${this.appConfig.groupId}/responsesByUserProfileShortCode/${this.shortCode}/?totalRows=true`).toPromise())['totalDocs'] const docsToQuery = 1000; let previousProcessedDocs = await this.variableService.get(`${username}-processedDocs`) diff --git a/server/src/express-app.js b/server/src/express-app.js index ad69f9cd5..56bd30a61 100644 --- a/server/src/express-app.js +++ b/server/src/express-app.js @@ -210,6 +210,7 @@ app.get('/app/:groupId/responsesByMonthAndFormId/:keys/:limit?/:skip?', isAuthen // Note that the lack of security middleware here is intentional. User IDs are UUIDs and thus sufficiently hard to guess. app.get('/api/:groupId/responsesByUserProfileId/:userProfileId/:limit?/:skip?', require('./routes/group-responses-by-user-profile-id.js')) app.get('/api/:groupId/responsesByUserProfileShortCode/:userProfileShortCode/:limit?/:skip?', require('./routes/group-responses-by-user-profile-short-code.js')) +app.get('/api/:groupId/userProfileByShortCode/:userProfileShortCode', require('./routes/group-user-profile-by-short-code.js')) app.get('/api/:groupId/:docId', isAuthenticatedOrHasUploadToken, require('./routes/group-doc-read.js')) app.put('/api/:groupId/:docId', isAuthenticated, require('./routes/group-doc-write.js')) app.post('/api/:groupId/:docId', isAuthenticated, require('./routes/group-doc-write.js')) diff --git a/server/src/group-views.js b/server/src/group-views.js index 533648624..8b979c3b9 100644 --- a/server/src/group-views.js +++ b/server/src/group-views.js @@ -59,10 +59,10 @@ module.exports.unpaid = function(doc) { } } -module.exports.responsesByUserProfileShortCode = function(doc) { - if (doc.collection === "TangyFormResponse") { +module.exports.responsesByUserProfileShortCode = { + map: function (doc) { if (doc.form && doc.form.id === 'user-profile') { - return emit(doc._id.substr(doc._id.length-6, doc._id.length), true) + return emit(doc._id.substr(doc._id.length-6, doc._id.length), 1) } var inputs = doc.items.reduce(function(acc, item) { return acc.concat(item.inputs)}, []) var userProfileInput = null @@ -72,9 +72,10 @@ module.exports.responsesByUserProfileShortCode = function(doc) { } }) if (userProfileInput) { - emit(userProfileInput.value.substr(userProfileInput.value.length-6, userProfileInput.value.length), true) + emit(userProfileInput.value.substr(userProfileInput.value.length-6, userProfileInput.value.length), 1) } - } + }, + reduce: '_count' } module.exports.userProfileByUserProfileShortCode = function (doc) { @@ -82,24 +83,6 @@ module.exports.userProfileByUserProfileShortCode = function (doc) { return emit(doc._id.substr(doc._id.length - 6, doc._id.length), true); } } -module.exports.totalDocsByUserProfileShortCode = { - map: function (doc) { - if (doc.form && doc.form.id === 'user-profile') { - return emit(doc._id.substr(doc._id.length-6, doc._id.length), 1) - } - var inputs = doc.items.reduce(function(acc, item) { return acc.concat(item.inputs)}, []) - var userProfileInput = null - inputs.forEach(function(input) { - if (input.name === 'userProfileId') { - userProfileInput = input - } - }) - if (userProfileInput) { - emit(userProfileInput.value.substr(userProfileInput.value.length-6, userProfileInput.value.length), 1) - } - }, - reduce: '_count' -} module.exports.groupIssues = function(doc) { if (doc.collection === "TangyFormResponse" && doc.type === "issue") { diff --git a/server/src/routes/group-responses-by-user-profile-short-code.js b/server/src/routes/group-responses-by-user-profile-short-code.js index 80e6041e6..19b82593b 100644 --- a/server/src/routes/group-responses-by-user-profile-short-code.js +++ b/server/src/routes/group-responses-by-user-profile-short-code.js @@ -1,12 +1,11 @@ const DB = require('../db.js') -const clog = require('tangy-log').clog const log = require('tangy-log').log module.exports = async (req, res) => { try { const groupDb = new DB(req.params.groupId) const userProfileShortCode = req.params.userProfileShortCode - let options = { key: userProfileShortCode, include_docs: true } + let options = { key: userProfileShortCode } if (req.params.limit) { options.limit = req.params.limit } @@ -14,16 +13,10 @@ module.exports = async (req, res) => { options.skip = req.params.skip } if (req.query.totalRows) { - const results = await groupDb.query('totalDocsByUserProfileShortCode', { key: userProfileShortCode, limit: 1,skip: 0, include_docs: false, reduce:true, group:true }); + const results = await groupDb.query('responsesByUserProfileShortCode', { key: userProfileShortCode, limit: 1,skip: 0, include_docs: false, reduce: true, group: true }); res.send({ totalDocs: results.rows[0].value }) - } else if (req.query.userProfile) { - await groupDb.query("userProfileByUserProfileShortCode", { limit: 0 }); - const result = await groupDb.query("userProfileByUserProfileShortCode", { key: userProfileShortCode, limit: 1, include_docs: true }); - const profile = result.rows[0] - const data = profile ? {_id: profile.id, key: profile.id, formId: profile.doc.form.id, collection: profile.doc.collection}: undefined - res.send(data) } else { - const results = await groupDb.query('responsesByUserProfileShortCode', options); + const results = await groupDb.query('responsesByUserProfileShortCode', { ...options, include_docs: true, reduce: false }); const docs = results.rows.map(row => row.doc) res.send(docs) } diff --git a/server/src/routes/group-user-profile-by-short-code.js b/server/src/routes/group-user-profile-by-short-code.js new file mode 100644 index 000000000..a5c08eb36 --- /dev/null +++ b/server/src/routes/group-user-profile-by-short-code.js @@ -0,0 +1,18 @@ +const DB = require('../db.js') +const log = require('tangy-log').log + +module.exports = async (req, res) => { + try { + const groupDb = new DB(req.params.groupId) + const userProfileShortCode = req.params.userProfileShortCode + + const result = await groupDb.query("userProfileByUserProfileShortCode", { key: userProfileShortCode, limit: 1, include_docs: true }); + const profile = result.rows[0] + const data = profile ? {_id: profile.id, key: profile.id, formId: profile.doc.form.id, collection: profile.doc.collection} : undefined + res.send(data) + + } catch (error) { + log.error(error); + res.status(500).send(error); + } +} \ No newline at end of file