Skip to content

Commit

Permalink
(╯°□°)╯︵ ┻━┻
Browse files Browse the repository at this point in the history
  • Loading branch information
@williamlardier
williamlardier committed Jan 23, 2025
1 parent 19b0bd9 commit be70dc3
Show file tree
Hide file tree
Showing 18 changed files with 4,087 additions and 5,451 deletions.
7 changes: 2 additions & 5 deletions .eslintrc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"jest/globals": true
},
"parserOptions": {
"ecmaVersion": 9
"ecmaVersion": 2020
},
"rules": {
"no-plusplus": 0,
Expand All @@ -23,10 +23,7 @@
"no-lonely-if": 0,
"max-classes-per-file": 0,
"prefer-spread": 0,
"no-constructor-return": 0,
"no-console": 0,
"object-curly-spacing": 0,
"no-extend-native": 0
"no-constructor-return": 0
},
"globals": {
"BigInt": "readonly"
Expand Down
18 changes: 5 additions & 13 deletions CountItems/CountManager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@ class CountManager {
this.maxConcurrent = params.maxConcurrent;
this.temporaryStore = {};
this.store = {
objects: BigInt(0),
versions: BigInt(0),
objects: 0n,
versions: 0n,
buckets: 0,
bucketList: [],
dataManaged: {
total: { curr: BigInt(0), prev: BigInt(0) },
total: { curr: 0n, prev: 0n },
byLocation: {},
},
stalled: 0,
Expand Down Expand Up @@ -61,22 +61,14 @@ class CountManager {
if (!results) {
return;
}
this.store.versions += results.versions ? results.versions : BigInt(0);
// eslint-disable-next-line no-console
console.log('results.objects', results.objects);
this.store.objects += results.objects ? results.objects : BigInt(0);
// eslint-disable-next-line no-console
console.log('this.store.objects', this.store.objects);
this.store.versions += results.versions ? results.versions : 0n;
this.store.objects += results.objects ? results.objects : 0n;
this.store.stalled += results.stalled;
if (results.dataManaged
&& results.dataManaged.locations
&& results.dataManaged.total) {
const { locations, total } = results.dataManaged;
// eslint-disable-next-line no-console
console.log('total.curr', total.curr);
this.store.dataManaged.total.curr += total.curr;
// eslint-disable-next-line no-console
console.log('total.prev', total.prev);
this.store.dataManaged.total.prev += total.prev;
Object.keys(locations).forEach(site => {
if (!this.store.dataManaged.byLocation[site]) {
Expand Down
4 changes: 0 additions & 4 deletions CountItems/CountWorker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,6 @@ const async = require('async');
const { BucketInfo } = require('arsenal').models;
const monitoring = require('../utils/monitoring');

/* eslint-disable no-extend-native */
BigInt.prototype.toJSON = () => this.toString();
/* eslint-disable no-extend-native */

class CountWorker {
constructor(params) {
this.log = params.log;
Expand Down
4 changes: 0 additions & 4 deletions CountItems/masterProcess.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,6 @@ const loggerConfig = {

let waitingForPromScraping = false;

/* eslint-disable no-extend-native */
BigInt.prototype.toJSON = () => this.toString();
/* eslint-disable no-extend-native */

werelogs.configure(loggerConfig);
const log = new werelogs.Logger('S3Utils::CountItems::Master');

Expand Down
88 changes: 44 additions & 44 deletions CountItems/utils/utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,75 +9,75 @@ function consolidateDataMetrics(target, source) {
if (!resTarget.usedCapacity) {
Object.assign(resTarget, {
usedCapacity: {
current: BigInt(0),
nonCurrent: BigInt(0),
_currentCold: BigInt(0),
_nonCurrentCold: BigInt(0),
_currentRestored: BigInt(0),
_currentRestoring: BigInt(0),
_nonCurrentRestored: BigInt(0),
_nonCurrentRestoring: BigInt(0),
_inflightsPreScan: BigInt(0),
_incompleteMPUParts: BigInt(0),
current: 0n,
nonCurrent: 0n,
_currentCold: 0n,
_nonCurrentCold: 0n,
_currentRestored: 0n,
_currentRestoring: 0n,
_nonCurrentRestored: 0n,
_nonCurrentRestoring: 0n,
_inflightsPreScan: 0n,
_incompleteMPUParts: 0n,
},
});
}
if (!resTarget.objectCount) {
Object.assign(resTarget, {
objectCount: {
current: BigInt(0),
nonCurrent: BigInt(0),
_currentCold: BigInt(0),
_nonCurrentCold: BigInt(0),
_currentRestored: BigInt(0),
_currentRestoring: BigInt(0),
_nonCurrentRestored: BigInt(0),
_nonCurrentRestoring: BigInt(0),
_incompleteMPUUploads: BigInt(0),
deleteMarker: BigInt(0),
current: 0n,
nonCurrent: 0n,
_currentCold: 0n,
_nonCurrentCold: 0n,
_currentRestored: 0n,
_currentRestoring: 0n,
_nonCurrentRestored: 0n,
_nonCurrentRestoring: 0n,
_incompleteMPUUploads: 0n,
deleteMarker: 0n,
},
});
}
if (!source) {
return resTarget;
}
const { usedCapacity, objectCount, accountOwnerID } = source;
resTarget.usedCapacity.current += usedCapacity && usedCapacity.current ? usedCapacity.current : BigInt(0);
resTarget.usedCapacity.nonCurrent += usedCapacity && usedCapacity.nonCurrent ? usedCapacity.nonCurrent : BigInt(0);
resTarget.usedCapacity._currentCold += usedCapacity && usedCapacity._currentCold ? usedCapacity._currentCold : BigInt(0);
resTarget.usedCapacity._nonCurrentCold += usedCapacity && usedCapacity._nonCurrentCold ? usedCapacity._nonCurrentCold : BigInt(0);
resTarget.usedCapacity._currentRestoring += usedCapacity && usedCapacity._currentRestoring ? usedCapacity._currentRestoring : BigInt(0);
resTarget.usedCapacity._currentRestored += usedCapacity && usedCapacity._currentRestored ? usedCapacity._currentRestored : BigInt(0);
resTarget.usedCapacity._nonCurrentRestoring += usedCapacity && usedCapacity._nonCurrentRestoring ? usedCapacity._nonCurrentRestoring : BigInt(0);
resTarget.usedCapacity._nonCurrentRestored += usedCapacity && usedCapacity._nonCurrentRestored ? usedCapacity._nonCurrentRestored : BigInt(0);
resTarget.usedCapacity._incompleteMPUParts += usedCapacity && usedCapacity._incompleteMPUParts ? usedCapacity._incompleteMPUParts : BigInt(0);
resTarget.usedCapacity.current += usedCapacity && usedCapacity.current ? usedCapacity.current : 0n;
resTarget.usedCapacity.nonCurrent += usedCapacity && usedCapacity.nonCurrent ? usedCapacity.nonCurrent : 0n;
resTarget.usedCapacity._currentCold += usedCapacity && usedCapacity._currentCold ? usedCapacity._currentCold : 0n;
resTarget.usedCapacity._nonCurrentCold += usedCapacity && usedCapacity._nonCurrentCold ? usedCapacity._nonCurrentCold : 0n;
resTarget.usedCapacity._currentRestoring += usedCapacity && usedCapacity._currentRestoring ? usedCapacity._currentRestoring : 0n;
resTarget.usedCapacity._currentRestored += usedCapacity && usedCapacity._currentRestored ? usedCapacity._currentRestored : 0n;
resTarget.usedCapacity._nonCurrentRestoring += usedCapacity && usedCapacity._nonCurrentRestoring ? usedCapacity._nonCurrentRestoring : 0n;
resTarget.usedCapacity._nonCurrentRestored += usedCapacity && usedCapacity._nonCurrentRestored ? usedCapacity._nonCurrentRestored : 0n;
resTarget.usedCapacity._incompleteMPUParts += usedCapacity && usedCapacity._incompleteMPUParts ? usedCapacity._incompleteMPUParts : 0n;

resTarget.objectCount.current += objectCount && objectCount.current ? objectCount.current : BigInt(0);
resTarget.objectCount.nonCurrent += objectCount && objectCount.nonCurrent ? objectCount.nonCurrent : BigInt(0);
resTarget.objectCount.deleteMarker += objectCount && objectCount.deleteMarker ? objectCount.deleteMarker : BigInt(0);
resTarget.objectCount._currentCold += objectCount && objectCount._currentCold ? objectCount._currentCold : BigInt(0);
resTarget.objectCount._nonCurrentCold += objectCount && objectCount._nonCurrentCold ? objectCount._nonCurrentCold : BigInt(0);
resTarget.objectCount._currentRestoring += objectCount && objectCount._currentRestoring ? objectCount._currentRestoring : BigInt(0);
resTarget.objectCount._currentRestored += objectCount && objectCount._currentRestored ? objectCount._currentRestored : BigInt(0);
resTarget.objectCount._nonCurrentRestoring += objectCount && objectCount._nonCurrentRestoring ? objectCount._nonCurrentRestoring : BigInt(0);
resTarget.objectCount._nonCurrentRestored += objectCount && objectCount._nonCurrentRestored ? objectCount._nonCurrentRestored : BigInt(0);
resTarget.objectCount._incompleteMPUUploads += objectCount && objectCount._incompleteMPUUploads ? objectCount._incompleteMPUUploads : BigInt(0);
resTarget.objectCount.current += objectCount && objectCount.current ? objectCount.current : 0n;
resTarget.objectCount.nonCurrent += objectCount && objectCount.nonCurrent ? objectCount.nonCurrent : 0n;
resTarget.objectCount.deleteMarker += objectCount && objectCount.deleteMarker ? objectCount.deleteMarker : 0n;
resTarget.objectCount._currentCold += objectCount && objectCount._currentCold ? objectCount._currentCold : 0n;
resTarget.objectCount._nonCurrentCold += objectCount && objectCount._nonCurrentCold ? objectCount._nonCurrentCold : 0n;
resTarget.objectCount._currentRestoring += objectCount && objectCount._currentRestoring ? objectCount._currentRestoring : 0n;
resTarget.objectCount._currentRestored += objectCount && objectCount._currentRestored ? objectCount._currentRestored : 0n;
resTarget.objectCount._nonCurrentRestoring += objectCount && objectCount._nonCurrentRestoring ? objectCount._nonCurrentRestoring : 0n;
resTarget.objectCount._nonCurrentRestored += objectCount && objectCount._nonCurrentRestored ? objectCount._nonCurrentRestored : 0n;
resTarget.objectCount._incompleteMPUUploads += objectCount && objectCount._incompleteMPUUploads ? objectCount._incompleteMPUUploads : 0n;

resTarget.usedCapacity._inflightsPreScan += usedCapacity && usedCapacity._inflightsPreScan ? usedCapacity._inflightsPreScan : BigInt(0);
resTarget.usedCapacity._inflightsPreScan += usedCapacity && usedCapacity._inflightsPreScan ? usedCapacity._inflightsPreScan : 0n;
if (accountOwnerID) {
resTarget.accountOwnerID = accountOwnerID;
}

resTarget.usedCapacity.current += usedCapacity
? usedCapacity._currentCold + usedCapacity._currentRestored + usedCapacity._currentRestoring
+ usedCapacity._incompleteMPUParts : BigInt(0);
+ usedCapacity._incompleteMPUParts : 0n;
resTarget.usedCapacity.nonCurrent += usedCapacity
? usedCapacity._nonCurrentCold + usedCapacity._nonCurrentRestored + usedCapacity._nonCurrentRestoring : BigInt(0);
? usedCapacity._nonCurrentCold + usedCapacity._nonCurrentRestored + usedCapacity._nonCurrentRestoring : 0n;
resTarget.objectCount.current += objectCount
? objectCount._currentCold + objectCount._currentRestored + objectCount._currentRestoring
+ objectCount._incompleteMPUUploads : BigInt(0);
+ objectCount._incompleteMPUUploads : 0n;
resTarget.objectCount.nonCurrent += objectCount
? objectCount._nonCurrentCold + objectCount._nonCurrentRestored + objectCount._nonCurrentRestoring : BigInt(0);
? objectCount._nonCurrentCold + objectCount._nonCurrentRestored + objectCount._nonCurrentRestoring : 0n;

return resTarget;
}
Expand Down
24 changes: 7 additions & 17 deletions DataReport/collectBucketMetricsAndUpdateBucketCapacityInfo.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,12 @@ function isValidBucketStorageMetrics(bucketMetric) {
&& bucketMetric.usedCapacity
&& typeof bucketMetric.usedCapacity.current === 'bigint'
&& typeof bucketMetric.usedCapacity.nonCurrent === 'bigint'
&& bucketMetric.usedCapacity.current > BigInt(-1)
&& bucketMetric.usedCapacity.nonCurrent > BigInt(-1);
&& bucketMetric.usedCapacity.current > -1n
&& bucketMetric.usedCapacity.nonCurrent > -1n;
}

function isValidCapacityValue(capacity) {
return (typeof capacity === 'bigint' && capacity >= BigInt(0));
return (typeof capacity === 'bigint' && capacity >= 0n);
}

function collectBucketMetricsAndUpdateBucketCapacityInfo(mongoClient, log, callback) {
Expand Down Expand Up @@ -74,33 +74,23 @@ function collectBucketMetricsAndUpdateBucketCapacityInfo(mongoClient, log, callb
return nxt(null, doc);
}),
(storageMetricDoc, nxt) => {
let bucketStorageUsed = BigInt(-1);
console.log('storageMetricDoc', storageMetricDoc);
let bucketStorageUsed = -1n;
if (isValidBucketStorageMetrics(storageMetricDoc)) {
console.log(' here in the isValidBucketStorageMetrics');
// Do not count the objects in cold for SOSAPI
bucketStorageUsed = storageMetricDoc.usedCapacity.current
+ storageMetricDoc.usedCapacity.nonCurrent;
console.log('bucketStorageUsed', { bucketStorageUsed });
}
// read Capacity from bucket._capabilities
const { Capacity } = bucket.getCapabilities().VeeamSOSApi.CapacityInfo;
console.log('Capacity', Capacity);

let available = BigInt(-1);
let capacity = BigInt(-1);
let available = -1n;
let capacity = -1n;
if (isValidCapacityValue(Capacity)) {
console.log('we are in the valid capacity value');
// is Capacity value is valid
capacity = Capacity;
// if bucket storage used is valid and capacity is bigger than used
if (bucketStorageUsed !== BigInt(-1) && (capacity - bucketStorageUsed) >= BigInt(0)) {
console.log('WE ARE IN THE CONDITION');
if (bucketStorageUsed !== -1n && (capacity - bucketStorageUsed) >= 0n) {
available = capacity - bucketStorageUsed;
console.log('available', available);
console.log('capacity', typeof capacity);
console.log('bucketStorageUsed', typeof bucketStorageUsed);
console.log('available', typeof available);
}
}
return mongoClient.updateBucketCapacityInfo(bucketName, {
Expand Down
12 changes: 6 additions & 6 deletions bucketVersionsStats.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,12 +132,12 @@ const s3 = new AWS.S3(Object.assign(options, s3Options));

const stats = {
current: {
count: BigInt(0),
size: BigInt(0),
count: 0n,
size: 0n,
},
noncurrent: {
count: BigInt(0),
size: BigInt(0),
count: 0n,
size: 0n,
},
};

Expand Down Expand Up @@ -199,8 +199,8 @@ function listBucket(bucket, cb) {
}
}
const statObj = version.IsLatest ? stats.current : stats.noncurrent;
statObj.count += BigInt(1);
statObj.size += version.Size || BigInt(0);
statObj.count += 1n;
statObj.size += version.Size || 0n;
if (VERBOSE) {
log.info('version info', {
bucket: BUCKET,
Expand Down
5 changes: 3 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
"dependencies": {
"@senx/warp10": "^1.1.2",
"JSONStream": "^1.3.5",
"arsenal": "git+https://github.com/scality/arsenal#3c8219beb3785a66d1a5d74b4dff14eda869348f",
"arsenal": "https://github.com/scality/arsenal#3c7581ef0734f6979b1525fc463253925ed6534f",
"async": "^2.6.4",
"aws-sdk": "^2.1005.0",
"bucketclient": "git+https://github.com/scality/bucketclient#8.1.5",
Expand Down Expand Up @@ -56,8 +56,9 @@
"eslint-config-scality": "github:scality/Guidelines#8.2.0",
"eslint-plugin-import": "^2.20.1",
"eslint-plugin-jest": "^23.6.0",
"jest": "^23.6.0",
"jest": "^29.7.0",
"mongodb-memory-server": "^8.10.2",
"randomatic": "^3.1.1",
"sinon": "^17.0.1"
}
}
4 changes: 2 additions & 2 deletions service-level-sidecar/report.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ const { getAccountIdForCanonicalId } = require('./vault');
class MetricReport {
constructor(factor = 1) {
this.factor = factor;
this.count = BigInt(0);
this.bytes = BigInt(0);
this.count = 0n;
this.bytes = 0n;
}

/**
Expand Down
Loading

0 comments on commit be70dc3

Please sign in to comment.