feat: add config to allow return if found key in S3

aura-nw · Feb 5, 2024 · 3bcfd0b · 3bcfd0b
1 parent 27316a3
commit 3bcfd0b
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 5 deletions.
diff --git a/ci/config.json.ci b/ci/config.json.ci
@@ -301,7 +301,8 @@
     "key": "uploadBlockRawLogToS3",
     "millisecondCrawl": 1000,
     "blocksPerCall": 100,
-    "overwriteS3IfFound": true
+    "overwriteS3IfFound": false,
+    "returnIfFound": true
   },
   "uploadTransactionRawLogToS3": {
     "key": "uploadTransactionRawLogToS3",

diff --git a/config.json b/config.json
@@ -304,7 +304,8 @@
     "key": "uploadBlockRawLogToS3",
     "millisecondCrawl": 1000,
     "blocksPerCall": 100,
-    "overwriteS3IfFound": true
+    "overwriteS3IfFound": false,
+    "returnIfFound": true
   },
   "uploadTransactionRawLogToS3": {
     "key": "uploadTransactionRawLogToS3",

diff --git a/src/common/utils/utils.ts b/src/common/utils/utils.ts
@@ -225,7 +225,8 @@ export default class Utils {
     data: Buffer,
     bucketName: string,
     s3Gateway: string,
-    overwrite = false
+    overwriteS3IfFound = false,
+    returnIfFound = false
   ) {
     const foundS3Object = await s3Client
       .headObject({
@@ -243,7 +244,13 @@ export default class Utils {
     if (foundS3Object) {
       const err = `This S3 key is found in S3: ${fileName}`;
       console.warn(err);
-      if (!overwrite) {
+      if (!overwriteS3IfFound) {
+        if (returnIfFound) {
+          return {
+            key: s3Gateway ? s3Gateway + fileName : fileName,
+            id,
+          };
+        }
         throw new Error(err);
       }
     }

diff --git a/src/services/crawl-block/upload_block_raw_log_to_s3.service.ts b/src/services/crawl-block/upload_block_raw_log_to_s3.service.ts
@@ -49,7 +49,8 @@ export default class UploadBlockRawLogToS3 extends BullableService {
             Buffer.from(JSON.stringify(block.data)),
             Config.BUCKET,
             Config.S3_GATEWAY,
-            config.uploadBlockRawLogToS3.overwriteS3IfFound
+            config.uploadBlockRawLogToS3.overwriteS3IfFound,
+            config.uploadBlockRawLogToS3.returnIfFound
           )
         )
       ).catch((err) => {