diff --git a/config.defaults.sh b/config.defaults.sh index 92a5201207..ab8b302ae7 100755 --- a/config.defaults.sh +++ b/config.defaults.sh @@ -23,7 +23,7 @@ T_UPLOAD_TOKEN="password" # The database username and password. Please make this extrememly secure. T_COUCHDB_USER_ADMIN_NAME="admin" T_COUCHDB_USER_ADMIN_PASS="password" -# Mysql +# Mysql - Enter container name - usually "mysql" or the ip address or server name for an external service. T_MYSQL_CONTAINER_NAME="mysql" T_MYSQL_USER="admin" T_MYSQL_PASSWORD="password" @@ -59,9 +59,12 @@ T_PAID_ALLOWANCE="unlimited" # Reporting delay determines how quickly uploads will get processed and show up in reporting outputs such as CSV. Time is in milliseconds and default is 5 minutes. T_REPORTING_DELAY="300000" -# Limit the rebuild-mysql-db script to certain group dbs. Omit to rebuild all. +# Limit mysql processing to certain group dbs. T_REBUILD_MYSQL_DBS="" +# Enter "true" if using a mysql container instead of an external database service such as AWS RDS. This will launch a mysql container. +T_USE_MYSQL_CONTAINER="" + # When CSV is generated, this determines how many form responses are held in memory during a batch. The higher the number the more memory this process will take but the faster it will complete. T_CSV_BATCH_SIZE=50 diff --git a/develop.sh b/develop.sh index a86f04a025..9733a45e25 100755 --- a/develop.sh +++ b/develop.sh @@ -74,10 +74,14 @@ else fi if echo "$T_MODULES" | grep mysql; then - ./mysql-start.sh - echo "Waiting 60 seconds for myql to start..." - sleep 60 - ./mysql-setup.sh + ./mysql-create-dirs.sh +fi + +if echo "$T_USE_MYSQL_CONTAINER" | grep "true"; then + ./mysql-start-container.sh + echo "Waiting 60 seconds for mysql container to start..." + sleep 60 + ./mysql-setup.sh fi if echo "$T_MYSQL_PHPMYADMIN" | grep "TRUE"; then @@ -223,9 +227,16 @@ OPTIONS="--link $T_COUCHDB_CONTAINER_NAME:couchdb \ tangerine/tangerine:local " +if echo "$T_USE_MYSQL_CONTAINER" | grep "true"; then + echo "Linking mysql container ..." + OPTIONS=" + --link $T_MYSQL_CONTAINER_NAME:mysql \ + $OPTIONS + " +fi + if echo "$T_MODULES" | grep mysql; then OPTIONS=" - --link $T_MYSQL_CONTAINER_NAME:mysql \ --env \"T_MYSQL_CONTAINER_NAME=$T_MYSQL_CONTAINER_NAME\" \ --env \"T_MYSQL_USER=$T_MYSQL_USER\" \ --env \"T_MYSQL_PASSWORD=$T_MYSQL_PASSWORD\" \ diff --git a/mysql-create-dirs.sh b/mysql-create-dirs.sh new file mode 100755 index 0000000000..b2d132da3a --- /dev/null +++ b/mysql-create-dirs.sh @@ -0,0 +1,26 @@ + +# +# Load config. +# + +source ./config.defaults.sh +if [ -f "./config.sh" ]; then + source ./config.sh +else + echo "You have no config.sh. Copy config.defaults.sh to config.sh, change the passwords and try again." && exit 1; +fi + + +if [ ! -d data ]; then + mkdir data +fi +if [ ! -d data/mysql ]; then + mkdir data/mysql +fi +if [ ! -d data/mysql/databases ]; then + mkdir data/mysql/databases +fi +if [ ! -d data/mysql/state ]; then + mkdir data/mysql/state +fi + diff --git a/mysql-setup.sh b/mysql-setup.sh index 53d0655b3f..d4d2a763ff 100755 --- a/mysql-setup.sh +++ b/mysql-setup.sh @@ -17,4 +17,4 @@ else eval ${CMD} CMD="docker exec mysql mysql -u root -p'$T_MYSQL_PASSWORD' -e \"GRANT ALL PRIVILEGES ON \"*.*\" TO '$T_MYSQL_USER'@'%' WITH GRANT OPTION;\"" eval ${CMD} -fi \ No newline at end of file +fi diff --git a/mysql-start-container.sh b/mysql-start-container.sh new file mode 100755 index 0000000000..ad8101e716 --- /dev/null +++ b/mysql-start-container.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +source ./config.defaults.sh +if [ -f "./config.sh" ]; then + source ./config.sh +else + echo "You have no config.sh. Copy config.defaults.sh to config.sh, change the passwords and try again." && exit 1; +fi + + [ "$(docker ps | grep $T_MYSQL_CONTAINER_NAME)" ] && docker stop $T_MYSQL_CONTAINER_NAME + [ "$(docker ps -a | grep $T_MYSQL_CONTAINER_NAME)" ] && docker rm $T_MYSQL_CONTAINER_NAME + +docker run -v $(pwd)/server/src/modules/mysql/conf.d:/etc/mysql/conf.d --name "$T_MYSQL_CONTAINER_NAME" -v "$(pwd)/data/mysql/databases:/var/lib/mysql" -p 3306:3306 -e "MYSQL_ROOT_PASSWORD=$T_MYSQL_PASSWORD" -d mysql:latest + diff --git a/mysql-start.sh b/mysql-start.sh index 1cea7c4178..a9094fd105 100755 --- a/mysql-start.sh +++ b/mysql-start.sh @@ -25,8 +25,8 @@ if [ ! -d data/mysql/state ]; then fi -[ "$(docker ps | grep $T_MYSQL_CONTAINER_NAME)" ] && docker stop $T_MYSQL_CONTAINER_NAME -[ "$(docker ps -a | grep $T_MYSQL_CONTAINER_NAME)" ] && docker rm $T_MYSQL_CONTAINER_NAME +# [ "$(docker ps | grep $T_MYSQL_CONTAINER_NAME)" ] && docker stop $T_MYSQL_CONTAINER_NAME +# [ "$(docker ps -a | grep $T_MYSQL_CONTAINER_NAME)" ] && docker rm $T_MYSQL_CONTAINER_NAME #docker run -v $(pwd)/server/src/modules/mysql/conf.d:/etc/mysql/conf.d --name "$T_MYSQL_CONTAINER_NAME" -v "$(pwd)/data/mysql/databases:/var/lib/mysql" -p 3306:3306 -e "MYSQL_ROOT_PASSWORD=$T_MYSQL_PASSWORD" -d mysql:latest diff --git a/server/src/modules/mysql-js/index.js b/server/src/modules/mysql-js/index.js index 5fc24024ad..2857837b6c 100644 --- a/server/src/modules/mysql-js/index.js +++ b/server/src/modules/mysql-js/index.js @@ -247,6 +247,23 @@ module.exports = { const result = await saveToMysql(knex, uploadedDoc, tablenameSuffix, tableName, docType, primaryKey, createFunction) log.info('Processed: ' + JSON.stringify(result)) } + } else if (doc.type === 'issue') { + const uploadedDoc = await saveFlatResponse(doc, locationList, targetDb, sanitized); + // issue doc + tableName = 'issue' + tablenameSuffix + docType = 'issue' + primaryKey = 'ID' + createFunction = function (t) { + t.engine('InnoDB') + t.string(primaryKey, 36).notNullable().primary(); + t.string('caseId', 36) // .index('response_caseId_IDX'); + t.string('participantID', 36) //.index('case_instances_ParticipantID_IDX'); + t.string('caseEventId', 36) // .index('eventform_caseEventId_IDX'); + t.tinyint('complete'); + t.string('archived', 36); // + } + const result = await saveToMysql(knex, uploadedDoc, tablenameSuffix, tableName, docType, primaryKey, createFunction) + log.info('Processed: ' + JSON.stringify(result)) } else { const uploadedDoc = await saveFlatResponse(doc, locationList, targetDb, sanitized); tableName = null; @@ -748,6 +765,68 @@ async function convert_response(knex, doc, groupId, tableName) { return cleanData } +async function convert_issue(knex, doc, groupId, tableName) { + let data = doc.data + if (!data) { + data = {} + } + + const id= doc._id + const startDatetime = doc.startDatetime + const geoIp = doc.geoip + const caseId = doc.caseId + const eventId = doc.eventId + const eventFormId= doc.eventFormId + const participantId = doc.participantId + const caseEventId = doc.caseEventId + const formID = 'issue' + +// append geoIP to the data object + if (geoIp) { + for (const key in geoIp) { + if (geoIp.hasOwnProperty(key)) { + const value = geoIp[key] + data[`geoip_${key}`] = value + } + } + } + + if (!data['_id']) { + data['_id'] = id + } + if (!data['caseid']) { + data['caseId'] = caseId + } + if (!data['participantid']) { + data['participantid'] = participantId + } + if (!data['eventid']) { + data['eventid'] = eventId + } + if (!data['eventformid']) { + data['eventformid'] = eventFormId + } + if (!data['caseeventid']) { + data['caseeventid'] = caseEventId + } + if (!data['startdatetime']) { + data['startdatetime'] = startDatetime + } + // adding formID to the data object + if (!data['formID_sanitized']) { + data['formID_sanitized'] = formID + } + + doc.ID = doc._id + doc.dbRevision = doc._rev + + // # Delete the following keys; + const valuesToRemove = ['_id', '_rev','buildChannel','buildId','caseEventId','deviceId','eventFormId','eventId','groupId','participantId','startDatetime', 'startUnixtime'] + valuesToRemove.forEach(e => delete doc[e]); + const cleanData = populateDataFromDocument(doc, data); + return cleanData +} + async function saveToMysql(knex, doc, tablenameSuffix, tableName, docType, primaryKey, createFunction) { let data; let result = {id: doc._id, tableName, docType} @@ -759,7 +838,11 @@ async function saveToMysql(knex, doc, tablenameSuffix, tableName, docType, prima } // Docs of type response must be flattened first to get the table name. if (doc.type.toLowerCase() !== 'response') { - await createTable(knex, groupId, tableName, docType, createFunction, primaryKey) + try { + await createTable(knex, groupId, tableName, docType, createFunction, primaryKey) + } catch (e) { + log.error("Error creating table: " + e) + } } switch (doc.type.toLowerCase()) { case 'case': @@ -774,6 +857,9 @@ async function saveToMysql(knex, doc, tablenameSuffix, tableName, docType, prima case 'event-form': data = await convert_event_form(knex, doc, groupId, tableName) break; + case 'issue': + data = await convert_issue(knex, doc, groupId, tableName) + break; case 'response': data = await convert_response(knex, doc, groupId, tableName) // Check if table exists and create if needed: diff --git a/server/src/reporting/clear-reporting-cache.js b/server/src/reporting/clear-reporting-cache.js index 86e193ff58..071ef04868 100644 --- a/server/src/reporting/clear-reporting-cache.js +++ b/server/src/reporting/clear-reporting-cache.js @@ -33,6 +33,7 @@ async function clearReportingCache() { const newState = Object.assign({}, state, { databases: state.databases.map(({name, sequence}) => { return {name, sequence: 0}}) }) + console.log("newState: " + JSON.stringify(newState)) await writeFile(REPORTING_WORKER_STATE, JSON.stringify(newState), 'utf-8') await unlink(REPORTING_WORKER_PAUSE) console.log('Done!') diff --git a/server/src/reporting/reporting-worker.js b/server/src/reporting/reporting-worker.js index ad2aef441e..fd60e9c95c 100644 --- a/server/src/reporting/reporting-worker.js +++ b/server/src/reporting/reporting-worker.js @@ -154,34 +154,53 @@ async function batch() { const DB = PouchDB.defaults(workerState.pouchDbDefaults) const startTime = new Date().toISOString() let processed = 0 + let onlyProcessTheseGroups = [] + if (process.env.T_REBUILD_MYSQL_DBS && process.env.T_REBUILD_MYSQL_DBS !== '') { + onlyProcessTheseGroups = process.env.T_REBUILD_MYSQL_DBS + ? JSON.parse(process.env.T_REBUILD_MYSQL_DBS.replace(/\'/g, `"`)) + : [] + // log.info('onlyProcessTheseGroups from T_REBUILD_MYSQL_DBS: ' + onlyProcessTheseGroups) + } // Process batch. for (let database of workerState.databases) { - const db = new DB(database.name) - const changes = await db.changes({ since: database.sequence, limit: workerState.batchSizePerDatabase, include_docs: false }) - if (changes.results.length > 0) { - for (let change of changes.results) { - try { - await changeProcessor(change, db) - processed++ - } catch (error) { - let errorMessage = JSON.stringify(error) - let errorMessageText = error.message - - // Sometimes JSON.stringify wipes out the error. - console.log("typeof error message: " + typeof error.message + " errorMessage: " + errorMessage + " errorMessageText: " + errorMessageText) - if (typeof error.message === 'object') { - errorMessageText = JSON.stringify(error.message) - } - if (errorMessage === '{}') { - errorMessage = "Error : " + " message: " + errorMessageText - } else { - errorMessage = "Error : " + " message: " + errorMessageText + " errorMessage: " + errorMessage + let processGroup = false + if (onlyProcessTheseGroups.length === 0 || onlyProcessTheseGroups.includes(database.name)) { + processGroup = true + } else { + // log.debug("Excluding group: " + database.name + " from mysql processing.") + } + if (processGroup) { + const db = new DB(database.name) + const changes = await db.changes({ + since: database.sequence, + limit: workerState.batchSizePerDatabase, + include_docs: false + }) + if (changes.results.length > 0) { + for (let change of changes.results) { + try { + await changeProcessor(change, db) + processed++ + } catch (error) { + let errorMessage = JSON.stringify(error) + let errorMessageText = error.message + + // Sometimes JSON.stringify wipes out the error. + console.log("typeof error message: " + typeof error.message + " errorMessage: " + errorMessage + " errorMessageText: " + errorMessageText) + if (typeof error.message === 'object') { + errorMessageText = JSON.stringify(error.message) + } + if (errorMessage === '{}') { + errorMessage = "Error : " + " message: " + errorMessageText + } else { + errorMessage = "Error : " + " message: " + errorMessageText + " errorMessage: " + errorMessage + } + log.error(`Error on change sequence ${change.seq} with id ${change.id} - Error: ${errorMessage} ::::: `) } - log.error(`Error on change sequence ${change.seq} with id ${change.id} - Error: ${errorMessage} ::::: `) } + // Even if an error was thrown, continue on with the next sequences. + database.sequence = changes.results[changes.results.length - 1].seq } - // Even if an error was thrown, continue on with the next sequences. - database.sequence = changes.results[changes.results.length-1].seq } } // Persist state to disk. diff --git a/server/src/scripts/reporting-worker-pause.sh b/server/src/scripts/reporting-worker-pause.sh index 6f2162bfc8..2f0fa2cdd6 100755 --- a/server/src/scripts/reporting-worker-pause.sh +++ b/server/src/scripts/reporting-worker-pause.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + if [ "$2" = "--help" ]; then echo "Usage:" echo " reporting-worker-pause" diff --git a/start.sh b/start.sh index eec8f4bdc1..c981db8bce 100755 --- a/start.sh +++ b/start.sh @@ -72,10 +72,14 @@ else fi if echo "$T_MODULES" | grep mysql; then - ./mysql-start.sh - echo "Waiting 60 seconds for myql to start..." - sleep 60 - ./mysql-setup.sh + ./mysql-create-dirs.sh +fi + +if echo "$T_USE_MYSQL_CONTAINER" | grep "true"; then + ./mysql-start-container.sh + echo "Waiting 60 seconds for mysql container to start..." + sleep 60 + ./mysql-setup.sh fi if echo "$T_MYSQL_PHPMYADMIN" | grep "TRUE"; then @@ -227,6 +231,14 @@ else " fi +if echo "$T_USE_MYSQL_CONTAINER" | grep "true"; then + echo "Linking mysql container ..." + OPTIONS=" + --link $T_MYSQL_CONTAINER_NAME:mysql \ + $OPTIONS + " +fi + if echo "$T_MODULES" | grep mysql; then RUN_OPTIONS=" --link $T_MYSQL_CONTAINER_NAME:mysql \