fix(tokens-info): Refactor delete missing keys for keep internal keys

scripts/update-rtdb.ts

@@ -2,7 +2,7 @@
 import { diffString } from 'json-diff'
 import { loadUpdateRTDBConfig } from '../src/config'
 import { FirebaseClient } from '../src/clients/firebase-client'
-import { deleteMissingKeysUpdateRequest } from '../src/utils/utils'
+import { keepInternalKeys } from '../src/utils/utils'
 import { getCeloRTDBMetadata } from '../src'
 
 async function main() {
@@ -28,17 +28,20 @@ async function main() {
           await firebaseClient.writeToPath(rtdbLocation, data)
           console.log(`Wrote data to ${rtdbLocation}.`)
         } else {
-          let updateRequest = data
-          if (overrideType.deleteMissingKeys) {
-            const deleteMissingKeys = deleteMissingKeysUpdateRequest(
-              data,
-              rtdbData,
+          const updateRequest = keepInternalKeys(
+            data,
+            rtdbData,
+            overrideType.keptInternalKeys,
+          )
+          const updateDiff = diffString(rtdbData, updateRequest)
+          if (!updateDiff) {
+            console.log(
+              `Diff is empty for data at ${rtdbLocation}, skipping...`,
             )
-            updateRequest = { ...updateRequest, ...deleteMissingKeys }
+          } else {
+            await firebaseClient.writeToPath(rtdbLocation, updateRequest)
+            console.log(`Updated data at ${rtdbLocation}.`)
           }
-          // TODO: Avoid updating the node if we already know there aren't changes
-          await firebaseClient.updateToPath(rtdbLocation, updateRequest)
-          console.log(`Updated data at ${rtdbLocation}.`)
         }
       }
     }

src/clients/firebase-client.ts

@@ -1,5 +1,4 @@
 import * as admin from 'firebase-admin'
-import { mapNestedJsonIntoPlain } from '../utils/utils'
 import { UpdateRTDBConfig } from '../types'
 
 export class FirebaseClient {
@@ -21,11 +20,6 @@ export class FirebaseClient {
     return data.val()
   }
 
-  async updateToPath(path: string, data: any): Promise<void> {
-    const ref = await this.firebaseDb.ref(path)
-    await ref.update(mapNestedJsonIntoPlain(data))
-  }
-
   async writeToPath(path: string, data: any): Promise<void> {
     const ref = await this.firebaseDb.ref(path)
     await ref.set(data)

src/index.ts

@@ -27,7 +27,11 @@ export function getCeloRTDBMetadata(environment: Environment): RTDBMetadata[] {
       data: transformCeloTokensForRTDB(tokensInfo),
       schema: RTDBAddressToTokenInfoSchema,
       rtdbLocation: 'tokensInfo',
-      overrideType: OverrideType.DeleteMissingKeysAndUpdate,
+      overrideType: OverrideType.KeepInternalKeys([
+        'historicalUsdPrices',
+        'usdPrice',
+        'priceFetchedAt',
+      ]),
     },
   ]
 }

src/types.ts

@@ -1,14 +1,14 @@
 import Joi from 'joi'
 
 export class OverrideType {
-  static readonly OverrideAll = new OverrideType(true, true)
-  static readonly OnlyUpdates = new OverrideType(false, false)
-  static readonly DeleteMissingKeysAndUpdate = new OverrideType(false, true)
+  static readonly OverrideAll = new OverrideType(true, [])
+  static readonly KeepInternalKeys = (keepInternalKeys?: string[]) =>
+    new OverrideType(false, keepInternalKeys)
 
   // private to disallow creating other instances of this type
   private constructor(
     public readonly shouldOverride: boolean,
-    public readonly deleteMissingKeys: boolean,
+    public readonly keptInternalKeys: string[] = [],
   ) {}
 }
 

src/utils/utils.test.ts

@@ -1,128 +1,91 @@
-import { deleteMissingKeysUpdateRequest, mapNestedJsonIntoPlain } from './utils'
+import { keepInternalKeys } from './utils'
+
+describe('Keep Internal Keys', () => {
+  const valuesToUpdate = {
+    '0x1234': {
+      address: '0x1234',
+      decimal: 18,
+      imageUrl: 'https://example.com',
+    },
+    '0x1235': {
+      address: '0x1235',
+      decimal: 8,
+      imageUrl: 'https://example2.com',
+    },
+    '0x1236': {
+      address: '0x1236',
+      decimal: 18,
+      imageUrl: 'https://example3.com',
+    },
+  }
+
+  const currentValues = {
+    '0x1234': {
+      address: '0x1234old',
+      decimal: 18,
+      imageUrl: 'https://example.com',
+      usdPrice: '1',
+      historicalUsdPrices: {
+        lastDay: {
+          price: '1.2',
+        },
+      },
+    },
+    '0x1235': {
+      address: '0x1235old',
+      decimal: 8,
+      imageUrl: 'https://example2.com',
+      usdPrice: '2',
+      historicalUsdPrices: {
+        lastDay: {
+          price: '2.2',
+        },
+      },
+    },
+  }
 
-describe('Map Nested Json Into', () => {
-  it('should map correctly when the json contains one level', () => {
-    const inputJson = {
-      key1: 'stringValue',
-      key2: 3,
-    }
-    const expectedJson = inputJson
+  it("returns the expected object when there aren't keys to keep", () => {
+    const updateObject = keepInternalKeys(valuesToUpdate, currentValues, [])
 
-    const result = mapNestedJsonIntoPlain(inputJson)
-    expect(result).toMatchObject(expectedJson)
+    expect(updateObject).toMatchObject(valuesToUpdate)
   })
 
-  it('should map correctly when the json contains a nested level', () => {
-    const inputJson = {
-      key1: {
-        key3: 4,
-        key4: 'stringValue',
+  it('returns the expected object when there are internal keys to keep', () => {
+    const updateObject = keepInternalKeys(valuesToUpdate, currentValues, [
+      'usdPrice',
+      'historicalUsdPrices',
+    ])
+
+    const expectedObject = {
+      '0x1234': {
+        address: '0x1234',
+        decimal: 18,
+        imageUrl: 'https://example.com',
+        usdPrice: '1',
+        historicalUsdPrices: {
+          lastDay: {
+            price: '1.2',
+          },
+        },
       },
-      key2: 3,
-    }
-    const expectedJson = {
-      'key1/key3': 4,
-      'key1/key4': 'stringValue',
-      key2: 3,
-    }
-
-    const result = mapNestedJsonIntoPlain(inputJson)
-    expect(result).toMatchObject(expectedJson)
-  })
-
-  it('should map correctly when the json contains multiple nested level', () => {
-    const inputJson = {
-      key1: {
-        key3: 4,
-        key4: 5,
-        key5: {
-          key6: 'stringValue',
-          key7: {
-            key8: 9,
+      '0x1235': {
+        address: '0x1235',
+        decimal: 8,
+        imageUrl: 'https://example2.com',
+        usdPrice: '2',
+        historicalUsdPrices: {
+          lastDay: {
+            price: '2.2',
           },
         },
       },
-      key2: 3,
-    }
-    const expectedJson = {
-      'key1/key3': 4,
-      'key1/key4': 5,
-      'key1/key5/key6': 'stringValue',
-      'key1/key5/key7/key8': 9,
-      key2: 3,
-    }
-
-    const result = mapNestedJsonIntoPlain(inputJson)
-    expect(result).toMatchObject(expectedJson)
-  })
-
-  it('should map array to object with indexes as key', () => {
-    const inputJson = {
-      key1: ['this', 'is', 'a', 'test'],
-      key2: 3,
-    }
-
-    const expectedJson = {
-      'key1/0': 'this',
-      'key1/1': 'is',
-      'key1/2': 'a',
-      'key1/3': 'test',
-      key2: 3,
-    }
-
-    const result = mapNestedJsonIntoPlain(inputJson)
-    expect(result).toMatchObject(expectedJson)
-  })
-})
-
-describe('Delete Missing Keys UpdateRequest', () => {
-  it('return expected update request when there are missing keys', () => {
-    const expected = {
-      key1: 1,
-      key2: 'value',
-    }
-
-    const current = {
-      key1: 2,
-      key3: 'shouldBeDeleted',
-    }
-
-    const updateObject = deleteMissingKeysUpdateRequest(expected, current)
-
-    const expectedResult = {
-      key3: null,
-    }
-
-    expect(updateObject).toMatchObject(expectedResult)
-  })
-
-  it('return empty update request when there are not missing keys', () => {
-    const expected = {
-      key1: 1,
-      key2: 'value',
-      key3: 'extra',
-    }
-
-    const current = {
-      key1: 2,
-      key2: 'shouldNotBeDeleted',
-    }
-
-    const updateObject = deleteMissingKeysUpdateRequest(expected, current)
-
-    expect(updateObject).toMatchObject({})
-  })
-
-  it('creates all keys if the RTDB instance is clean', () => {
-    const expected = {
-      key1: 1,
-      key2: 'value',
+      '0x1236': {
+        address: '0x1236',
+        decimal: 18,
+        imageUrl: 'https://example3.com',
+      },
     }
 
-    const current = null
-
-    const updateObject = deleteMissingKeysUpdateRequest(expected, current)
-
-    expect(updateObject).toMatchObject({})
+    expect(expectedObject).toMatchObject(updateObject)
   })
 })

src/utils/utils.ts

@@ -1,41 +1,19 @@
-export function mapNestedJsonIntoPlain(json: any): any {
-  if (!isNonEmptyObject(json)) {
-    return json
-  }
+export function keepInternalKeys(
+  expected: any,
+  current: any,
+  keptInternalKeys: string[],
+) {
+  const result: any = {}
 
-  return Object.keys(json).reduce(
-    (result, current) => ({
-      ...result,
-      ...prependKey(current, mapNestedJsonIntoPlain(json[current])),
-    }),
-    {},
-  )
-}
+  for (const key of Object.keys(expected)) {
+    result[key] = expected[key]
 
-function prependKey(key: string, json: any) {
-  if (!isNonEmptyObject(json)) {
-    return { [key]: json }
+    for (const internalKey of keptInternalKeys) {
+      if (current[key] && current[key][internalKey]) {
+        result[key][internalKey] = current[key][internalKey]
+      }
+    }
   }
 
-  return Object.keys(json).reduce(
-    (result, current) => ({
-      ...result,
-      [`${key}/${current}`]: json[current],
-    }),
-    {},
-  )
-}
-
-function isNonEmptyObject(json: any): boolean {
-  return typeof json === 'object' && json !== null
-}
-
-export function deleteMissingKeysUpdateRequest(expected: any, current: any) {
-  if (!isNonEmptyObject(current)) return {}
-  const expectedKeys = new Set(Object.keys(expected))
-  const currentKeys = Object.keys(current)
-
-  return currentKeys
-    .filter((key) => !expectedKeys.has(key))
-    .reduce((result, key) => ({ ...result, [key]: null }), {})
+  return result
 }