Merge pull request #3680 from Tangerine-Community/import-archives-v2

feat(imports): Allow users to import exported data using new format

client/src/app/core/export-data/export-data/export-data.component.ts

@@ -275,11 +275,12 @@ export class ExportDataComponent implements OnInit {
           const stream = new window['Memorystream']
           let data = '';
           stream.on('data', function (chunk) {
-            data += chunk.toString();
+            data += chunk.toString() +',';// Add comma after each item - will be useful in creating a valid JSON object
           });
           await db.dump(stream)
           console.log('Successfully exported : ' + dbName);
           this.statusMessage += `<p>${_TRANSLATE('Successfully exported database ')} ${dbName}</p>`
+          data = `[${data.replace(/,([^,]*)$/, '$1')}]`//Make a valid JSON string - Wrap the items in [] and remove trailing comma
           const file = new Blob([data], {type: 'application/json'});
           this.downloadData(file, fileName, 'application/json');
           this.hideExportButton = false

server/package.json

@@ -29,6 +29,7 @@
     "update-group-archived-index": "./src/scripts/update-group-archived-index.js",
     "find-missing-records": "./src/scripts/find-missing-records.js",
     "import-archives": "./src/scripts/import-archives/bin.js",
+    "import-archives-v2": "./src/scripts/import-archives-v2/bin.js",
     "reset-all-devices": "./src/scripts/reset-all-devices/bin.js",
     "translations-update": "./src/scripts/translations-update.js",
     "release-dat": "./src/scripts/release-dat.sh",

server/src/scripts/import-archives-v2/bin.js

@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+if (!process.argv[2]) {
+  console.log('Place archives from clients into the ./data/archives folder on the host machine then run...')
+  console.log('       ./bin.js <groupName>')
+  process.exit()
+}
+
+const util = require('util')
+const readdir = util.promisify(require('fs').readdir)
+const readFile = util.promisify(require('fs').readFile)
+const pako = require('pako')
+const axios = require('axios')
+const url = `http://localhost/api/${process.argv[2]}/upload`
+const ARCHIVES_PATH = '/archives'
+
+
+async function go() {
+  const archivesList = await readdir(ARCHIVES_PATH)
+  for (const archivePath of archivesList) {
+    const archiveContents = await readFile(`${ARCHIVES_PATH}/${archivePath}`, 'utf-8')
+    const docsArray = ([...JSON.parse(archiveContents)]).find(item=>item.docs)?.docs
+    const userProfileDoc = docsArray.find(item=> item.form&& item.form.id=== 'user-profile')
+    console.log(userProfileDoc)
+    const docs = docsArray
+    .map(item => {
+      if (item.collection !== 'TangyFormResponse') return
+      if (item.form && item.form.id !== 'user-profile') {
+        item.items[0].inputs.push({
+          name: 'userProfileId',
+          value: userProfileDoc._id
+        })
+      }
+      return item 
+    })
+    .filter(doc => doc !== undefined)
+    for (const doc of docs) {
+      let body = pako.deflate(JSON.stringify({ doc }), {to: 'string'})
+      await axios({
+        method: 'post',
+        url,
+        data: `${body}`,
+        headers: {
+          'content-type': 'text/plain',
+          'Authorization': `${process.env.T_UPLOAD_TOKEN}`
+        }
+      })
+    }
+  }
+}
+
+try {
+  go()
+} catch(e) {
+  console.log(e)
+}

server/src/scripts/info.sh

@@ -14,7 +14,8 @@ echo "generate-cases           (Generate cases with group's case-export.json as
 echo "reset-all-devices        (Reset server tokens and device keys for all devices, requires reinstall and set up on all devices after)"
 echo "push-all-groups-views    (Push all database views into all groups)"
 echo "index-all-groups-views   (Index all database views into all groups)"
-echo "import-archives          (Import client archives from the ./data/archives folder)"
+echo "import-archives          (Import client archives from the ./data/archives folder using old format)"
+echo "import-archives-v2       (Import client archives from the ./data/archives folder using new format)"
 echo "release-apk              (Release a Group App as an APK)"
 echo "release-pwa              (Release a Group App as a PWA)"
 echo "release-dat              (Release a Group APP as a Dat Archive)"