From 94e55de6956c2623d9ff56b9936bdc292db35b1c Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Sat, 26 Oct 2024 13:04:33 -0400 Subject: [PATCH 1/2] feat: incorporate DigCache --- package-lock.json | 99 +++++++++++++++++++++++-- package.json | 2 +- src/controllers/merkleTreeController.ts | 10 +-- src/tasks/sync_stores.ts | 4 +- src/utils/nonce.ts | 16 ++-- 5 files changed, 108 insertions(+), 23 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1f0d63b..e4cdbb0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "ISC", "dependencies": { "@dignetwork/datalayer-driver": "^0.1.28", - "@dignetwork/dig-sdk": "^0.0.1-alpha.169", + "@dignetwork/dig-sdk": "^0.0.1-alpha.173", "async-mutex": "^0.5.0", "busboy": "^1.6.0", "express": "^4.19.2", @@ -252,13 +252,14 @@ } }, "node_modules/@dignetwork/dig-sdk": { - "version": "0.0.1-alpha.169", - "resolved": "https://registry.npmjs.org/@dignetwork/dig-sdk/-/dig-sdk-0.0.1-alpha.169.tgz", - "integrity": "sha512-lDrsTbNuDz1KXRkAmOy6D07S4pUpoqZPBdeYf94pKIOuFyOd79HgI2n+afpIJTdx6PjZbyKWCKpIQXDNQ258Fw==", + "version": "0.0.1-alpha.173", + "resolved": "https://registry.npmjs.org/@dignetwork/dig-sdk/-/dig-sdk-0.0.1-alpha.173.tgz", + "integrity": "sha512-9zU6w+L3VOpSoR3tzHDgLeuep85sRfMPEQSuNy/gh89mfFjDwQaejCLCg8BQ1tE9eClwxRLZFux0DHiRYkXt4w==", "dependencies": { "@dignetwork/datalayer-driver": "^0.1.29", "@dignetwork/dig-sdk": "^0.0.1-alpha.158", "archiver": "^7.0.1", + "async-mutex": "^0.5.0", "axios": "^1.7.7", "bip39": "^3.1.0", "bottleneck": "^2.19.5", @@ -272,7 +273,7 @@ "figures": "^6.1.0", "fs-extra": "^11.2.0", "ignore": "^5.3.2", - "inquirer": "^10.1.8", + "inquirer": "^10.2.2", "lodash": "^4.17.21", "merkletreejs": "^0.4.0", "nanospinner": "^1.1.0", @@ -281,7 +282,8 @@ "node-stun": "^0.1.2", "progress-stream": "^2.0.0", "proper-lockfile": "^4.1.2", - "superagent": "^10.0.0", + "redis": "^4.7.0", + "superagent": "^10.1.0", "unzipper": "^0.12.3" } }, @@ -678,6 +680,59 @@ "node": ">=14" } }, + "node_modules/@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/client": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.0.tgz", + "integrity": "sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==", + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@redis/graph": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz", + "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/json": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.7.tgz", + "integrity": "sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/search": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.2.0.tgz", + "integrity": "sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/time-series": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.1.0.tgz", + "integrity": "sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, "node_modules/@scure/base": { "version": "1.1.9", "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.1.9.tgz", @@ -1704,6 +1759,14 @@ "node": ">=0.8" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -2967,6 +3030,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "engines": { + "node": ">= 4" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -4932,6 +5003,19 @@ "node": ">=8" } }, + "node_modules/redis": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.7.0.tgz", + "integrity": "sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==", + "dependencies": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.6.0", + "@redis/graph": "1.1.1", + "@redis/json": "1.0.7", + "@redis/search": "1.2.0", + "@redis/time-series": "1.1.0" + } + }, "node_modules/request-ip": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/request-ip/-/request-ip-3.3.0.tgz", @@ -6006,8 +6090,7 @@ "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "node_modules/yargs": { "version": "16.2.0", diff --git a/package.json b/package.json index a9ea979..a2ed8d4 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ ], "dependencies": { "@dignetwork/datalayer-driver": "^0.1.28", - "@dignetwork/dig-sdk": "^0.0.1-alpha.169", + "@dignetwork/dig-sdk": "^0.0.1-alpha.173", "async-mutex": "^0.5.0", "busboy": "^1.6.0", "express": "^4.19.2", diff --git a/src/controllers/merkleTreeController.ts b/src/controllers/merkleTreeController.ts index b585e92..3115771 100644 --- a/src/controllers/merkleTreeController.ts +++ b/src/controllers/merkleTreeController.ts @@ -11,12 +11,12 @@ import { getFilePathFromSha256, Environment, DigNetwork, + DigCache, } from "@dignetwork/dig-sdk"; import { promisify } from "util"; import { getStorageLocation } from "../utils/storage"; import tmp from "tmp"; import { PassThrough } from "stream"; -import NodeCache from "node-cache"; import { generateNonce, validateNonce } from "../utils/nonce"; import Busboy from "busboy"; import fsExtra from "fs-extra"; @@ -41,7 +41,7 @@ const sessionCache: { resetTtl: () => void; }; } = {}; -const ownerCache = new NodeCache({ stdTTL: ownerCacheTTL }); +const ownerCache = new DigCache({ stdTTL: ownerCacheTTL }); /** * Creates a session directory with custom TTL logic. Each session has a TTL that can be reset @@ -450,7 +450,7 @@ export const generateFileNonce = async ( // If file does not exist, generate a nonce for the file if (!fileExists) { const nonceKey = `${storeId}_${sessionId}_${filename}`; - const nonce = generateNonce(nonceKey); + const nonce = await generateNonce(nonceKey); res.setHeader("x-nonce", nonce); } @@ -492,7 +492,7 @@ export const uploadFile = async ( ); } - if (!validateNonce(`${storeId}_${sessionId}_${filename}`, nonce)) { + if (!(await validateNonce(`${storeId}_${sessionId}_${filename}`, nonce))) { throw new HttpError(401, "Invalid nonce."); } @@ -517,7 +517,7 @@ export const uploadFile = async ( // Check if the user has write permissions to the store const cacheKey = `${publicKey}_${storeId}`; - let isOwner = ownerCache.get(cacheKey); + let isOwner = await ownerCache.get(cacheKey); if (isOwner === undefined) { const dataStore = new DataStore(storeId, { disableInitialize: true }); diff --git a/src/tasks/sync_stores.ts b/src/tasks/sync_stores.ts index 1c2daca..cf79aec 100644 --- a/src/tasks/sync_stores.ts +++ b/src/tasks/sync_stores.ts @@ -169,7 +169,9 @@ const synchronizeStore = async ( console.log( `Store ${storeId} is not synced. Initiating synchronization from peers.` ); - await syncStoreFromNetwork(storeId); + if (fs.existsSync(path.join(STORE_PATH, storeId))) { + await syncStoreFromNetwork(storeId); + } } } }; diff --git a/src/utils/nonce.ts b/src/utils/nonce.ts index b6bfb57..d87b963 100644 --- a/src/utils/nonce.ts +++ b/src/utils/nonce.ts @@ -1,19 +1,19 @@ import crypto from "crypto"; -import NodeCache from "node-cache"; +import { DigCache } from '@dignetwork/dig-sdk' -// Create a new NodeCache instance with a 5-minute TTL for nonces -const nonceCache = new NodeCache({ stdTTL: 10 * 60 }); +// 5-minute TTL for nonces +const nonceCache = new DigCache({ stdTTL: 10 * 60 }); /** - * Function to generate and store nonce in NodeCache. + * Function to generate and store nonce in DigCache. * @param {string} userId - Unique identifier for the user (or session). * @returns {string} - The generated nonce. */ -export const generateNonce = (nonceKey: string): string => { +export const generateNonce = async (nonceKey: string): Promise => { const nonce = crypto.randomBytes(16).toString("hex"); // Store the nonce in the cache with userId as the key - nonceCache.set(nonceKey, nonce); + await nonceCache.set(nonceKey, nonce); return nonce; }; @@ -24,8 +24,8 @@ export const generateNonce = (nonceKey: string): string => { * @param {string} providedNonce - The nonce provided for validation. * @returns {boolean} - True if the nonce is valid, otherwise false. */ -export const validateNonce = (nonceKey: string, providedNonce: string): boolean => { - const cachedNonce = nonceCache.get(nonceKey); +export const validateNonce = async (nonceKey: string, providedNonce: string): Promise => { + const cachedNonce = await nonceCache.get(nonceKey); if (cachedNonce && cachedNonce === providedNonce) { nonceCache.del(nonceKey); // Delete nonce after successful validation From 66c06bfa74e7c16bffece094e0b823bf089cd019 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Sat, 26 Oct 2024 13:05:01 -0400 Subject: [PATCH 2/2] chore(release): 0.0.1-alpha.135 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85ee7e9..237c727 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [0.0.1-alpha.135](https://github.com/DIG-Network/dig-propagation-server/compare/v0.0.1-alpha.134...v0.0.1-alpha.135) (2024-10-26) + + +### Features + +* incorporate DigCache ([94e55de](https://github.com/DIG-Network/dig-propagation-server/commit/94e55de6956c2623d9ff56b9936bdc292db35b1c)) + ### [0.0.1-alpha.134](https://github.com/DIG-Network/dig-propagation-server/compare/v0.0.1-alpha.133...v0.0.1-alpha.134) (2024-10-10) diff --git a/package-lock.json b/package-lock.json index e4cdbb0..cd94d66 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "dig-propagation-server", - "version": "0.0.1-alpha.134", + "version": "0.0.1-alpha.135", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dig-propagation-server", - "version": "0.0.1-alpha.134", + "version": "0.0.1-alpha.135", "license": "ISC", "dependencies": { "@dignetwork/datalayer-driver": "^0.1.28", diff --git a/package.json b/package.json index a2ed8d4..ebbe8b3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dig-propagation-server", - "version": "0.0.1-alpha.134", + "version": "0.0.1-alpha.135", "description": "", "type": "commonjs", "main": "./dist/index.js",