chore: redis migration debug utils

bin/tmp/README.md

@@ -0,0 +1,23 @@
+# KV-migration utils
+
+Create KV DB:
+```bash
+docker compose up -d redis
+```
+
+Fill db with random keys using [fill-kv-db.js](fill-kv-db.js). Use empty prefix combined with other to mix DB conditions
+```bash
+node fill-kv-db.js
+```
+
+In parallel, run [rename-kv-db.js](rename-kv-db.js) to rename existing keys and [emulate-kv-load.js](emulate-kv-load.js) to emulate parallel write load
+
+```bash
+node rename-kv-db.js & node emulate-kv-load.js
+```
+
+Run [check-kv-db.js](check-kv-db.js) to verify, that all keys are renamed
+```bash
+node check-kv-db.js
+```
+

bin/tmp/check-kv-db.js

@@ -0,0 +1,47 @@
+const IORedis = require('ioredis')
+
+async function checkKVDB ({
+    connectionString,
+    prefix,
+    scanSize = 1_000,
+    logEvery = 1_000_000,
+}) {
+    const client = new IORedis(connectionString)
+
+    const size = await client.dbsize()
+    console.log(`Database size: ${size} keys`)
+
+    let cursor = '0'
+    const nonMatching = []
+    let scanned = 0
+    let lastLogged = 0
+
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const [updatedCursor, keys] = await client.scan(cursor, 'MATCH', '*', 'COUNT', scanSize)
+        cursor = updatedCursor
+        scanned += keys.length
+
+        if (scanned - lastLogged >= logEvery) {
+            console.log(`${scanned} keys scanned`)
+            lastLogged = scanned
+        }
+
+        nonMatching.push(...keys.filter(key => !key.startsWith(prefix)))
+
+        if (cursor === '0') {
+            console.log('All keys are successfully scanned!')
+            break
+        }
+    }
+
+    console.log(`Non-matching keys: ${JSON.stringify(nonMatching, null, 2)}`)
+}
+
+checkKVDB({
+    connectionString: 'redis://127.0.0.1:6379/32',
+    prefix: 'condo:',
+}).then(() => {
+    console.log('ALL DONE')
+    process.exit(0)
+})

bin/tmp/emulate-kv-load.js

@@ -0,0 +1,42 @@
+const { faker } = require('@faker-js/faker')
+const IORedis = require('ioredis')
+
+function randInt (min, max) {
+    return Math.floor(Math.random() * (max - min) + min)
+}
+
+async function sleep (ms) {
+    return new Promise((resolve) => {
+        setTimeout(() => resolve(ms), ms)
+    })
+}
+
+async function emulateKvLoad ({
+    connectionString,
+    prefixes = [''],
+    durationInMS = 90_000,
+    sleepIntervalInMS = 500,
+}) {
+    const client = new IORedis(connectionString)
+
+    const iterations = Math.floor(durationInMS / sleepIntervalInMS) + 1
+
+    for (let i = 0; i < iterations; i++) {
+        const rndIdx = randInt(0, prefixes.length)
+        const prefix = prefixes[rndIdx]
+        const key = prefix + faker.datatype.uuid()
+
+        await client.set(key, faker.datatype.uuid())
+        console.log(`SET ${key}`)
+
+        await sleep(sleepIntervalInMS)
+    }
+}
+
+emulateKvLoad({
+    connectionString: 'redis://127.0.0.1:6379/32',
+    prefixes: ['load:', 'condo:'],
+}).then(() => {
+    console.log('ALL DONE')
+    process.exit(0)
+})

bin/tmp/fill-kv-db.js

@@ -0,0 +1,48 @@
+const { faker } = require('@faker-js/faker')
+const IORedis = require('ioredis')
+
+function randInt (min, max) {
+    return Math.floor(Math.random() * (max - min) + min)
+}
+
+async function fillKVDB ({
+    connectionString,
+    amount,
+    prefixes = ['', '', 'condo:'], // 1/3 chance on "condo:"
+    batch = 10_000,
+    logEvery = 1_000_000,
+}) {
+    const client = new IORedis(connectionString)
+
+    const iterations = Math.ceil(amount / batch)
+
+    let lastLogged = 0
+    let added = 0
+
+    for (let i = 0; i < iterations; i++) {
+        const data = Object.fromEntries(
+            Array.from({ length: batch }, () => {
+                const prefix = prefixes[randInt(0, prefixes.length)]
+                return [prefix + faker.datatype.uuid(), faker.datatype.uuid()]
+            })
+        )
+
+        await client.mset(data)
+
+        added += batch
+
+        if (added - lastLogged >= logEvery) {
+            console.log(`${added} keys added`)
+            lastLogged = added
+        }
+    }
+}
+
+// NOTE: Use docker stat to monitor memory consumption
+fillKVDB({
+    connectionString: 'redis://127.0.0.1:6379/32',
+    amount: 40_000_000,
+}).then(() => {
+    console.log('ALL DONE')
+    process.exit(0)
+})

bin/tmp/rename-kv-db.js

@@ -0,0 +1,69 @@
+const IORedis = require('ioredis')
+
+async function renameKVDB ({
+    connectionString,
+    fromPrefix = '',
+    toPrefix = '',
+    logEvery = 100_000,
+    scanSize = 1_000,
+}) {
+    console.time('rename')
+    console.log(`Renaming keys with "${fromPrefix}" prefix to "${toPrefix}"`)
+    const client = new IORedis(connectionString)
+
+    const size = await client.dbsize()
+    console.log(`Database size: ${size} keys`)
+
+
+
+    let cursor = '0'
+    let renamed = 0
+    let lastLogged = 0
+
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const [updatedCursor, keys] = await client.scan(cursor, 'MATCH', '*', 'COUNT', scanSize)
+        cursor = updatedCursor
+
+        let tx = client.multi()
+
+        for (const key of keys) {
+            // NOTE: exclude non-matching keys if fromPrefix specified
+            if (fromPrefix.length && !key.startsWith(fromPrefix)) {
+                continue
+            }
+
+            // NOTE: skip already renamed keys
+            if (toPrefix.length && key.startsWith(toPrefix)) {
+                continue
+            }
+
+            tx = tx.renamenx(key, toPrefix + key.substring(fromPrefix.length))
+            renamed++
+        }
+
+        await tx.exec()
+
+        if (renamed - lastLogged >= logEvery) {
+            console.log(`${renamed} keys renamed`)
+            lastLogged = renamed
+        }
+
+        if (cursor === '0') {
+            console.log('All keys are successfully scanned!')
+            break
+        }
+    }
+
+    console.log(`${renamed} keys renamed`)
+
+    console.timeEnd('rename')
+}
+
+renameKVDB({
+    connectionString: 'redis://127.0.0.1:6379/32',
+    toPrefix: 'condo:',
+}).then(() => {
+    console.log('ALL DONE')
+    process.exit(0)
+})