Merge remote-tracking branch 'origin/bugfix/S3UTILS-188-support-metri…

…cs-above-max-safe' into w/1.15/bugfix/S3UTILS-188-support-metrics-above-max-safe

.eslintrc

@@ -7,7 +7,7 @@
         "jest/globals": true
     },
     "parserOptions": {
-        "ecmaVersion": 9
+        "ecmaVersion": 2020
     },
     "rules": {
         "no-plusplus": 0,
@@ -23,7 +23,11 @@
         "no-lonely-if": 0,
         "max-classes-per-file": 0,
         "prefer-spread": 0,
-        "no-constructor-return": 0
+        "no-constructor-return": 0,
+        "new-cap": 0
+    },
+    "globals": {
+        "BigInt": "readonly"
     }
 }
 

.github/workflows/build.yml

@@ -39,6 +39,13 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ github.token }}
 
+      - name: Install Oras
+        run: |
+          curl -L https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz | \
+            tar -xz -C /usr/local/bin oras
+        env:
+          ORAS_VERSION: 1.2.2
+
       - name: Push dashboards into the development namespace
         run: |
           oras push ghcr.io/${{ github.repository }}/${{ env.PROJECT_NAME }}-dashboards:${{ inputs.tag }} \

.github/workflows/test.yml

@@ -15,7 +15,7 @@ env:
 
 jobs:
   prepare:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     permissions:
       # Need to explicitely add package write permissions for dependabot
       contents: read
@@ -45,7 +45,7 @@ jobs:
 
   tests:
     needs: prepare
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     services:
       mongodb:
         image: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}

CountItems/CountManager.js

@@ -61,8 +61,8 @@ class CountManager {
         if (!results) {
             return;
         }
-        this.store.versions += results.versions;
-        this.store.objects += results.objects;
+        this.store.versions += (results.versions || 0);
+        this.store.objects += (results.objects || 0);
         this.store.stalled += results.stalled;
         if (results.dataManaged
             && results.dataManaged.locations

CountItems/CountWorker.js

@@ -2,6 +2,7 @@ const assert = require('assert');
 const async = require('async');
 const { BucketInfo } = require('arsenal').models;
 const monitoring = require('../utils/monitoring');
+const { deserializeBigInts, serializeBigInts } = require('./utils/utils');
 
 class CountWorker {
     constructor(params) {
@@ -61,7 +62,7 @@ class CountWorker {
         }
         switch (data.type) {
         case 'count':
-            this.countItems(data.bucketInfo, (err, results) => {
+            this.countItems(deserializeBigInts(data.bucketInfo), (err, results) => {
                 if (err) {
                     return this._sendFn({
                         id: data.id,
@@ -76,7 +77,7 @@ class CountWorker {
                     owner: 'scality',
                     type: 'count',
                     status: 'passed',
-                    results,
+                    results: serializeBigInts(results),
                 });
             });
             break;

CountItems/CountWorkerObj.js

@@ -1,5 +1,6 @@
 const uuid = require('node-uuid');
 const { once } = require('arsenal').jsutil;
+const { deserializeBigInts, serializeBigInts } = require('./utils/utils');
 
 class CountWorkerObj {
     constructor(id, worker) {
@@ -118,12 +119,18 @@ class CountWorkerObj {
 
     count(bucketInfo, callback) {
         const id = uuid.v4();
-        this._addCallback(id, 'count', callback);
+        this._addCallback(id, 'count', (err, results) => {
+            if (err) {
+                return callback(err);
+            }
+            // Deserialize BigInts from the worker response
+            return callback(null, deserializeBigInts(results));
+        });
         this._worker.send({
             id,
             owner: 'scality',
             type: 'count',
-            bucketInfo,
+            bucketInfo: serializeBigInts(bucketInfo),
         });
     }
 

CountItems/utils/utils.js

@@ -9,79 +9,110 @@ function consolidateDataMetrics(target, source) {
     if (!resTarget.usedCapacity) {
         Object.assign(resTarget, {
             usedCapacity: {
-                current: 0,
-                nonCurrent: 0,
-                _currentCold: 0,
-                _nonCurrentCold: 0,
-                _currentRestored: 0,
-                _currentRestoring: 0,
-                _nonCurrentRestored: 0,
-                _nonCurrentRestoring: 0,
-                _inflightsPreScan: 0,
-                _incompleteMPUParts: 0,
+                current: 0n,
+                nonCurrent: 0n,
+                _currentCold: 0n,
+                _nonCurrentCold: 0n,
+                _currentRestored: 0n,
+                _currentRestoring: 0n,
+                _nonCurrentRestored: 0n,
+                _nonCurrentRestoring: 0n,
+                _inflightsPreScan: 0n,
+                _incompleteMPUParts: 0n,
             },
         });
     }
     if (!resTarget.objectCount) {
         Object.assign(resTarget, {
             objectCount: {
-                current: 0,
-                nonCurrent: 0,
-                _currentCold: 0,
-                _nonCurrentCold: 0,
-                _currentRestored: 0,
-                _currentRestoring: 0,
-                _nonCurrentRestored: 0,
-                _nonCurrentRestoring: 0,
-                _incompleteMPUUploads: 0,
-                deleteMarker: 0,
+                current: 0n,
+                nonCurrent: 0n,
+                _currentCold: 0n,
+                _nonCurrentCold: 0n,
+                _currentRestored: 0n,
+                _currentRestoring: 0n,
+                _nonCurrentRestored: 0n,
+                _nonCurrentRestoring: 0n,
+                _incompleteMPUUploads: 0n,
+                deleteMarker: 0n,
             },
         });
     }
     if (!source) {
         return resTarget;
     }
     const { usedCapacity, objectCount, accountOwnerID } = source;
-    resTarget.usedCapacity.current += usedCapacity && usedCapacity.current ? usedCapacity.current : 0;
-    resTarget.usedCapacity.nonCurrent += usedCapacity && usedCapacity.nonCurrent ? usedCapacity.nonCurrent : 0;
-    resTarget.usedCapacity._currentCold += usedCapacity && usedCapacity._currentCold ? usedCapacity._currentCold : 0;
-    resTarget.usedCapacity._nonCurrentCold += usedCapacity && usedCapacity._nonCurrentCold ? usedCapacity._nonCurrentCold : 0;
-    resTarget.usedCapacity._currentRestoring += usedCapacity && usedCapacity._currentRestoring ? usedCapacity._currentRestoring : 0;
-    resTarget.usedCapacity._currentRestored += usedCapacity && usedCapacity._currentRestored ? usedCapacity._currentRestored : 0;
-    resTarget.usedCapacity._nonCurrentRestoring += usedCapacity && usedCapacity._nonCurrentRestoring ? usedCapacity._nonCurrentRestoring : 0;
-    resTarget.usedCapacity._nonCurrentRestored += usedCapacity && usedCapacity._nonCurrentRestored ? usedCapacity._nonCurrentRestored : 0;
-    resTarget.usedCapacity._incompleteMPUParts += usedCapacity && usedCapacity._incompleteMPUParts ? usedCapacity._incompleteMPUParts : 0;
+    resTarget.usedCapacity.current += BigInt(usedCapacity?.current || 0n);
+    resTarget.usedCapacity.nonCurrent += BigInt(usedCapacity?.nonCurrent || 0n);
+    resTarget.usedCapacity._currentCold += BigInt(usedCapacity?._currentCold || 0n);
+    resTarget.usedCapacity._nonCurrentCold += BigInt(usedCapacity?._nonCurrentCold || 0n);
+    resTarget.usedCapacity._currentRestoring += BigInt(usedCapacity?._currentRestoring || 0n);
+    resTarget.usedCapacity._currentRestored += BigInt(usedCapacity?._currentRestored || 0n);
+    resTarget.usedCapacity._nonCurrentRestoring += BigInt(usedCapacity?._nonCurrentRestoring || 0n);
+    resTarget.usedCapacity._nonCurrentRestored += BigInt(usedCapacity?._nonCurrentRestored || 0n);
+    resTarget.usedCapacity._incompleteMPUParts += BigInt(usedCapacity?._incompleteMPUParts || 0n);
 
-    resTarget.objectCount.current += objectCount && objectCount.current ? objectCount.current : 0;
-    resTarget.objectCount.nonCurrent += objectCount && objectCount.nonCurrent ? objectCount.nonCurrent : 0;
-    resTarget.objectCount.deleteMarker += objectCount && objectCount.deleteMarker ? objectCount.deleteMarker : 0;
-    resTarget.objectCount._currentCold += objectCount && objectCount._currentCold ? objectCount._currentCold : 0;
-    resTarget.objectCount._nonCurrentCold += objectCount && objectCount._nonCurrentCold ? objectCount._nonCurrentCold : 0;
-    resTarget.objectCount._currentRestoring += objectCount && objectCount._currentRestoring ? objectCount._currentRestoring : 0;
-    resTarget.objectCount._currentRestored += objectCount && objectCount._currentRestored ? objectCount._currentRestored : 0;
-    resTarget.objectCount._nonCurrentRestoring += objectCount && objectCount._nonCurrentRestoring ? objectCount._nonCurrentRestoring : 0;
-    resTarget.objectCount._nonCurrentRestored += objectCount && objectCount._nonCurrentRestored ? objectCount._nonCurrentRestored : 0;
-    resTarget.objectCount._incompleteMPUUploads += objectCount && objectCount._incompleteMPUUploads ? objectCount._incompleteMPUUploads : 0;
+    resTarget.objectCount.current += BigInt(objectCount?.current || 0n);
+    resTarget.objectCount.nonCurrent += BigInt(objectCount?.nonCurrent || 0n);
+    resTarget.objectCount.deleteMarker += BigInt(objectCount?.deleteMarker || 0n);
+    resTarget.objectCount._currentCold += BigInt(objectCount?._currentCold || 0n);
+    resTarget.objectCount._nonCurrentCold += BigInt(objectCount?._nonCurrentCold || 0n);
+    resTarget.objectCount._currentRestoring += BigInt(objectCount?._currentRestoring || 0n);
+    resTarget.objectCount._currentRestored += BigInt(objectCount?._currentRestored || 0n);
+    resTarget.objectCount._nonCurrentRestoring += BigInt(objectCount?._nonCurrentRestoring || 0n);
+    resTarget.objectCount._nonCurrentRestored += BigInt(objectCount?._nonCurrentRestored || 0n);
+    resTarget.objectCount._incompleteMPUUploads += BigInt(objectCount?._incompleteMPUUploads || 0n);
 
-    resTarget.usedCapacity._inflightsPreScan += usedCapacity && usedCapacity._inflightsPreScan ? usedCapacity._inflightsPreScan : 0;
+    resTarget.usedCapacity._inflightsPreScan += BigInt(usedCapacity?._inflightsPreScan || 0n);
     if (accountOwnerID) {
         resTarget.accountOwnerID = accountOwnerID;
     }
 
     resTarget.usedCapacity.current += usedCapacity
-        ? usedCapacity._currentCold + usedCapacity._currentRestored + usedCapacity._currentRestoring
-        + usedCapacity._incompleteMPUParts : 0;
+        ? BigInt(usedCapacity._currentCold) + BigInt(usedCapacity._currentRestored) + BigInt(usedCapacity._currentRestoring)
+        + BigInt(usedCapacity._incompleteMPUParts) : 0n;
     resTarget.usedCapacity.nonCurrent += usedCapacity
-        ? usedCapacity._nonCurrentCold + usedCapacity._nonCurrentRestored + usedCapacity._nonCurrentRestoring : 0;
+        ? BigInt(usedCapacity._nonCurrentCold) + BigInt(usedCapacity._nonCurrentRestored) + BigInt(usedCapacity._nonCurrentRestoring) : 0n;
     resTarget.objectCount.current += objectCount
-        ? objectCount._currentCold + objectCount._currentRestored + objectCount._currentRestoring
-        + objectCount._incompleteMPUUploads : 0;
+        ? BigInt(objectCount._currentCold) + BigInt(objectCount._currentRestored) + BigInt(objectCount._currentRestoring)
+        + BigInt(objectCount._incompleteMPUUploads) : 0n;
     resTarget.objectCount.nonCurrent += objectCount
-        ? objectCount._nonCurrentCold + objectCount._nonCurrentRestored + objectCount._nonCurrentRestoring : 0;
+        ? BigInt(objectCount._nonCurrentCold) + BigInt(objectCount._nonCurrentRestored) + BigInt(objectCount._nonCurrentRestoring) : 0n;
 
     return resTarget;
 }
 
+function serializeBigInts(obj) {
+    if (typeof obj !== 'object' || obj === null) {
+        return typeof obj === 'bigint' ? { __bigint: obj.toString() } : obj;
+    }
+    const result = Array.isArray(obj) ? [] : {};
+    for (const key in obj) {
+        if (Object.prototype.hasOwnProperty.call(obj, key)) {
+            result[key] = serializeBigInts(obj[key]);
+        }
+    }
+    return result;
+}
+
+function deserializeBigInts(obj) {
+    if (!obj || typeof obj !== 'object') {
+        return obj;
+    }
+    if (obj.__bigint !== undefined) {
+        return BigInt(obj.__bigint);
+    }
+    const result = Array.isArray(obj) ? [] : {};
+    for (const key in obj) {
+        if (Object.prototype.hasOwnProperty.call(obj, key)) {
+            result[key] = deserializeBigInts(obj[key]);
+        }
+    }
+    return result;
+}
+
 module.exports = {
     consolidateDataMetrics,
+    serializeBigInts,
+    deserializeBigInts,
 };

DataReport/collectBucketMetricsAndUpdateBucketCapacityInfo.js

@@ -32,15 +32,19 @@ function isSOSCapacityInfoEnabled(bucketInfo) {
 
 function isValidBucketStorageMetrics(bucketMetric) {
     return bucketMetric
-    && bucketMetric.usedCapacity
-        && typeof bucketMetric.usedCapacity.current === 'number'
-        && typeof bucketMetric.usedCapacity.nonCurrent === 'number'
+        && bucketMetric.usedCapacity
+        // For backward compatibility reeasons, we accept bigint and numbers
+        // But only bigints will be stored in the database
+        && (typeof bucketMetric.usedCapacity.current === 'bigint' || typeof bucketMetric.usedCapacity.current === 'number')
+        && (typeof bucketMetric.usedCapacity.nonCurrent === 'bigint' || typeof bucketMetric.usedCapacity.nonCurrent === 'number')
         && bucketMetric.usedCapacity.current > -1
         && bucketMetric.usedCapacity.nonCurrent > -1;
 }
 
 function isValidCapacityValue(capacity) {
-    return (Number.isSafeInteger(capacity) && capacity >= 0);
+    // For backward compatibility reasons, we accept bigint and numbers
+    // But only bigints (Longs) will be stored in the database
+    return ((typeof capacity === 'bigint' || (typeof capacity === 'number' && Number.isInteger(capacity))) && capacity >= 0);
 }
 
 function collectBucketMetricsAndUpdateBucketCapacityInfo(mongoClient, log, callback) {
@@ -74,22 +78,28 @@ function collectBucketMetricsAndUpdateBucketCapacityInfo(mongoClient, log, callb
                         return nxt(null, doc);
                     }),
                     (storageMetricDoc, nxt) => {
-                        let bucketStorageUsed = -1;
+                        let bucketStorageUsed = -1n;
                         if (isValidBucketStorageMetrics(storageMetricDoc)) {
                             // Do not count the objects in cold for SOSAPI
-                            bucketStorageUsed = storageMetricDoc.usedCapacity.current
-                                + storageMetricDoc.usedCapacity.nonCurrent;
+                            // numbers are converted into bigints to ensure we support more
+                            // than 9PB of bytes stored and previous clusters where the
+                            // values were stored as numbers.
+                            bucketStorageUsed = BigInt(storageMetricDoc.usedCapacity.current)
+                                + BigInt(storageMetricDoc.usedCapacity.nonCurrent);
                         }
                         // read Capacity from bucket._capabilities
                         const { Capacity } = bucket.getCapabilities().VeeamSOSApi.CapacityInfo;
 
-                        let available = -1;
-                        let capacity = -1;
-                        if (isValidCapacityValue(Capacity)) { // is Capacity value is valid
+                        let available = -1n;
+                        let capacity = -1n;
+                        if (isValidCapacityValue(Capacity)) {
+                            // is Capacity value is valid
                             capacity = Capacity;
                             // if bucket storage used is valid and capacity is bigger than used
-                            if (bucketStorageUsed !== -1 && (capacity - bucketStorageUsed) >= 0) {
+                            if (bucketStorageUsed !== -1n && (capacity - bucketStorageUsed) >= 0n) {
                                 available = capacity - bucketStorageUsed;
+                            } else if (bucketStorageUsed !== -1n && (capacity - bucketStorageUsed) < 0n) {
+                                available = 0n;
                             }
                         }
                         return mongoClient.updateBucketCapacityInfo(bucketName, {

bucketVersionsStats.js

@@ -132,12 +132,12 @@ const s3 = new AWS.S3(Object.assign(options, s3Options));
 
 const stats = {
     current: {
-        count: 0,
-        size: 0,
+        count: 0n,
+        size: 0n,
     },
     noncurrent: {
-        count: 0,
-        size: 0,
+        count: 0n,
+        size: 0n,
     },
 };
 
@@ -147,8 +147,8 @@ let VersionIdMarker;
 function _logProgress(message) {
     const loggedStats = {
         total: {
-            count: stats.current.count + stats.noncurrent.count,
-            size: stats.current.size + stats.noncurrent.size,
+            count: BigInt(stats.current.count + stats.noncurrent.count),
+            size: BigInt(stats.current.size + stats.noncurrent.size),
         },
         ...stats,
     };
@@ -199,8 +199,8 @@ function listBucket(bucket, cb) {
                         }
                     }
                     const statObj = version.IsLatest ? stats.current : stats.noncurrent;
-                    statObj.count += 1;
-                    statObj.size += version.Size || 0;
+                    statObj.count += 1n;
+                    statObj.size += version.Size || 0n;
                     if (VERBOSE) {
                         log.info('version info', {
                             bucket: BUCKET,

package.json

@@ -28,7 +28,7 @@
   "dependencies": {
     "@senx/warp10": "^1.1.2",
     "JSONStream": "^1.3.5",
-    "arsenal": "git+https://github.com/scality/arsenal#8.1.142",
+    "arsenal": "git+https://github.com/scality/arsenal#8.1.146",
     "async": "^2.6.4",
     "aws-sdk": "^2.1005.0",
     "bucketclient": "git+https://github.com/scality/bucketclient#8.1.5",
@@ -56,8 +56,9 @@
     "eslint-config-scality": "github:scality/Guidelines#8.2.0",
     "eslint-plugin-import": "^2.20.1",
     "eslint-plugin-jest": "^23.6.0",
-    "jest": "^23.6.0",
+    "jest": "^29.7.0",
     "mongodb-memory-server": "^8.10.2",
+    "randomatic": "^3.1.1",
     "sinon": "^17.0.1"
   }
 }