From 98b35c8377987b041790b79470e7afc3cb7fa133 Mon Sep 17 00:00:00 2001
From: Evans Dianga <evans@itsevans.com>
Date: Thu, 21 Dec 2023 12:22:39 +0300
Subject: [PATCH] feat(imports): Allow users to import exported data using new
 format

- Added new import-archives-v2 command
- Exported docs now are valid JSON objects
- Users will need to update Tangerine client and re-export files

Refs Tangerine-Community/Tangerine#3145
---
 .../export-data/export-data.component.ts      |  3 +-
 server/package.json                           |  1 +
 server/src/scripts/import-archives-v2/bin.js  | 55 +++++++++++++++++++
 server/src/scripts/info.sh                    |  3 +-
 4 files changed, 60 insertions(+), 2 deletions(-)
 create mode 100755 server/src/scripts/import-archives-v2/bin.js

diff --git a/client/src/app/core/export-data/export-data/export-data.component.ts b/client/src/app/core/export-data/export-data/export-data.component.ts
index 870ad77e75..94157f4750 100644
--- a/client/src/app/core/export-data/export-data/export-data.component.ts
+++ b/client/src/app/core/export-data/export-data/export-data.component.ts
@@ -275,11 +275,12 @@ export class ExportDataComponent implements OnInit {
           const stream = new window['Memorystream']
           let data = '';
           stream.on('data', function (chunk) {
-            data += chunk.toString();
+            data += chunk.toString() +',';// Add comma after each item - will be useful in creating a valid JSON object
           });
           await db.dump(stream)
           console.log('Successfully exported : ' + dbName);
           this.statusMessage += `<p>${_TRANSLATE('Successfully exported database ')} ${dbName}</p>`
+          data = `[${data.replace(/,([^,]*)$/, '$1')}]`//Make a valid JSON string - Wrap the items in [] and remove trailing comma
           const file = new Blob([data], {type: 'application/json'});
           this.downloadData(file, fileName, 'application/json');
           this.hideExportButton = false
diff --git a/server/package.json b/server/package.json
index ca98b895bb..881db6d507 100644
--- a/server/package.json
+++ b/server/package.json
@@ -29,6 +29,7 @@
     "update-group-archived-index": "./src/scripts/update-group-archived-index.js",
     "find-missing-records": "./src/scripts/find-missing-records.js",
     "import-archives": "./src/scripts/import-archives/bin.js",
+    "import-archives-v2": "./src/scripts/import-archives-v2/bin.js",
     "reset-all-devices": "./src/scripts/reset-all-devices/bin.js",
     "translations-update": "./src/scripts/translations-update.js",
     "release-dat": "./src/scripts/release-dat.sh",
diff --git a/server/src/scripts/import-archives-v2/bin.js b/server/src/scripts/import-archives-v2/bin.js
new file mode 100755
index 0000000000..3f75e2ea8b
--- /dev/null
+++ b/server/src/scripts/import-archives-v2/bin.js
@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+if (!process.argv[2]) {
+  console.log('Place archives from clients into the ./data/archives folder on the host machine then run...')
+  console.log('       ./bin.js <groupName>')
+  process.exit()
+}
+
+const util = require('util')
+const readdir = util.promisify(require('fs').readdir)
+const readFile = util.promisify(require('fs').readFile)
+const pako = require('pako')
+const axios = require('axios')
+const url = `http://localhost/api/${process.argv[2]}/upload`
+const ARCHIVES_PATH = '/archives'
+
+
+async function go() {
+  const archivesList = await readdir(ARCHIVES_PATH)
+  for (const archivePath of archivesList) {
+    const archiveContents = await readFile(`${ARCHIVES_PATH}/${archivePath}`, 'utf-8')
+    const docsArray = ([...JSON.parse(archiveContents)]).find(item=>item.docs)?.docs
+    const userProfileDoc = docsArray.find(item=> item.form&& item.form.id=== 'user-profile')
+    console.log(userProfileDoc)
+    const docs = docsArray
+    .map(item => {
+      if (item.collection !== 'TangyFormResponse') return
+      if (item.form && item.form.id !== 'user-profile') {
+        item.items[0].inputs.push({
+          name: 'userProfileId',
+          value: userProfileDoc._id
+        })
+      }
+      return item 
+    })
+    .filter(doc => doc !== undefined)
+    for (const doc of docs) {
+      let body = pako.deflate(JSON.stringify({ doc }), {to: 'string'})
+      await axios({
+        method: 'post',
+        url,
+        data: `${body}`,
+        headers: {
+          'content-type': 'text/plain',
+          'Authorization': `${process.env.T_UPLOAD_TOKEN}`
+        }
+      })
+    }
+  }
+}
+
+try {
+  go()
+} catch(e) {
+  console.log(e)
+}
diff --git a/server/src/scripts/info.sh b/server/src/scripts/info.sh
index 6ebf79f85a..f36cb3b492 100755
--- a/server/src/scripts/info.sh
+++ b/server/src/scripts/info.sh
@@ -14,7 +14,8 @@ echo "generate-cases           (Generate cases with group's case-export.json as
 echo "reset-all-devices        (Reset server tokens and device keys for all devices, requires reinstall and set up on all devices after)"
 echo "push-all-groups-views    (Push all database views into all groups)"
 echo "index-all-groups-views   (Index all database views into all groups)"
-echo "import-archives          (Import client archives from the ./data/archives folder)"
+echo "import-archives          (Import client archives from the ./data/archives folder using old format)"
+echo "import-archives-v2       (Import client archives from the ./data/archives folder using new format)"
 echo "release-apk              (Release a Group App as an APK)"
 echo "release-pwa              (Release a Group App as a PWA)"
 echo "release-dat              (Release a Group APP as a Dat Archive)"