Month Budget & Realtime with Better Signalr Status and Health check (#17

)

background-job/.env.sample

@@ -2,4 +2,8 @@ GSHEET_PRIVATE_KEY=
 GSHEET_CLIENT_EMAIL=
 GSHEET_SPREADSHEET_ID=
 
-GSHEET_SHEET_TRANSACTION_SHEET_ID=
+GSHEET_SHEET_TRANSACTION_SHEET_ID=
+GSHEET_SHEET_MONTHLY_BUDGET_SHEET_ID=
+GSHEET_SHEET_MONTHLY_BUDGET_SUMMARY_SHEET_ID=
+
+AzureSignalRConnectionString=

background-job/.gitignore

@@ -128,4 +128,6 @@ dist
 .yarn/build-state.yml
 .yarn/install-state.gz
 .pnp.*
-.azurite
+.azurite
+
+local.settings.json

background-job/local.settings.sample.json

@@ -0,0 +1,11 @@
+{
+  "IsEncrypted": false,
+  "Values": {
+    "FUNCTIONS_WORKER_RUNTIME": "node",
+    "AzureWebJobsFeatureFlags": "EnableWorkerIndexing",
+    "AzureWebJobsStorage": "UseDevelopmentStorage=true",
+    "AzureSignalRConnectionString": "Endpoint=https://<your-service>.service.signalr.net;AccessKey"
+  },
+  "ConnectionStrings": {},
+  "watchDirectories": ["node_modules", "dist"]
+}

background-job/package.json

@@ -5,9 +5,11 @@
   "main": "dist/src/main.js",
   "scripts": {
     "build": "tsc",
+    "build:watch": "tsc -w",
     "build:prod": "run-s build && npm prune --production",
     "lint": "tsc --noEmit",
-    "start": "tsc && func start",
+    "start": "func start",
+    "dev:func": "run-s build && run-p build:watch start",
     "dev": "cross-env NODE_ENV=development tsx watch src/main.ts",
     "azurite": "pnpx azurite --silent --location ./.azurite --debug ./.azurite/debug.log"
   },

background-job/src/bootstrap.ts

@@ -5,9 +5,8 @@ import { GoogleSheetRowClient } from './libs/google-sheet';
 import { TableClient } from '@azure/data-tables';
 import { AzureTable } from './libs/azure-table';
 import { TransactionCacheEntity } from './entities/transaction.entity';
-// import { TableCache } from './libs/table-cache';
 import { AzureTableCache } from './libs/azure-table-cache';
-import { CacheService } from './services/cache.service';
+import { MonthlyBudgetCacheEntity, MonthlyBudgetSummaryCacheEntity } from './entities/monthly-budget.entity';
 
 /**
  * Google Sheet Service
@@ -43,20 +42,98 @@ export const sheetClient = {
       UpdatedAt: 'date',
     },
   }),
+  monthlyBudget: new GoogleSheetRowClient(sheetDoc, env.GSHEET_SHEET_MONTHLY_BUDGET_SHEET_ID, {
+    ...commonOptions,
+    headers: {
+      Id: 'string',
+      Title: 'string',
+      Hide: 'boolean',
+      Type: 'string',
+      CategoryGroup: 'string',
+      CategoryGroupID: 'string',
+      Selectable: 'boolean',
+      BaseOrder: 'number',
+      Order: 'number',
+      /**
+       * for partition key, we use MonthKey
+       *
+       * pattern is 'YYYY-MM'
+       * e.g. '2021-01'
+       */
+      FilterMonth: 'string',
+      Assigned: 'number',
+      Activity: 'number',
+      CumulativeAssigned: 'number',
+      CumulativeActivity: 'number',
+      Available: 'number',
+    },
+  }),
+  monthlyBudgetSummary: new GoogleSheetRowClient(sheetDoc, env.GSHEET_SHEET_MONTHLY_BUDGET_SUMMARY_SHEET_ID, {
+    ...commonOptions,
+    headerRow: 2,
+    headers: {
+      LatestUpdate: 'date',
+      StartBudgetDate: 'date',
+      FilterMonth: 'date',
+      StartDate: 'date',
+      EndDate: 'date',
+      ReadyToAssign: 'number',
+      TotalIncome: 'number',
+      TotalAssigned: 'number',
+      TotalActivity: 'number',
+      TotalAvailable: 'number',
+    },
+  }),
 };
 
 /**
+ * ----------------------------------------
  * Azure Table Service
+ * ----------------------------------------
+ */
+/**
+ * Azure Table, Transaction Cache Table
  */
 const transactionCacheTableClient = TableClient.fromConnectionString(
   env.AzureWebJobsStorage,
   env.AZURE_STORAGE_TABLE_TRANSACTION_CACHE_TABLE_NAME
 );
-export const transactionCacheTable = new AzureTable<TransactionCacheEntity>(transactionCacheTableClient);
-
+const transactionCacheTable = new AzureTable<TransactionCacheEntity>(transactionCacheTableClient);
 export const transactionTableCache = new AzureTableCache(transactionCacheTable, {
   /**
    * Default field in Azure Table
    */
   lastUpdatedField: 'timestamp',
 });
+
+/**
+ * Azure Table, Monthly Budget Cache Table
+ */
+const monthlyBudgetCacheTableClient = TableClient.fromConnectionString(
+  env.AzureWebJobsStorage,
+  env.AZURE_STORAGE_TABLE_MONTHLY_BUDGET_CACHE_TABLE_NAME
+);
+const monthlyBudgetCacheTable = new AzureTable<MonthlyBudgetCacheEntity>(monthlyBudgetCacheTableClient);
+export const monthlyBudgetTableCache = new AzureTableCache(monthlyBudgetCacheTable, {
+  /**
+   * Default field in Azure Table
+   */
+  lastUpdatedField: 'timestamp',
+});
+
+/**
+ * Azure Table, Monthly Budget Summary Cache Table
+ */
+const monthlyBudgetSummaryCacheTableClient = TableClient.fromConnectionString(
+  env.AzureWebJobsStorage,
+  env.AZURE_STORAGE_TABLE_MONTHLY_BUDGET_SUMMARY_CACHE_TABLE_NAME
+);
+const monthlyBudgetSummaryCacheTable = new AzureTable<MonthlyBudgetSummaryCacheEntity>(
+  monthlyBudgetSummaryCacheTableClient
+);
+export const monthlyBudgetSummaryTableCache = new AzureTableCache(monthlyBudgetSummaryCacheTable, {
+  /**
+   * Default field in Azure Table
+   */
+  lastUpdatedField: 'timestamp',
+});

background-job/src/entities/monthly-budget.entity.ts

@@ -0,0 +1,59 @@
+import { AzureTableEntityBase } from '../libs/azure-table';
+
+/**
+ * Monthly Budget Cache Entity
+ *
+ * This entity is used to cache monthly budget data from Google Sheet
+ *
+ * Partition Key: MonthKey
+ * Row Key: id
+ */
+
+export interface MonthlyBudgetCacheEntity extends AzureTableEntityBase {
+  /**
+   * Rowkey is same value of `id`
+   */
+  id: string;
+  title: string;
+  hide: boolean;
+  type: string;
+  categoryGroup: string;
+  categoryGroupID: string;
+  selectable: boolean;
+  baseOrder: number;
+  order: number;
+  /**
+   * for partition key, we use MonthKey
+   *
+   * pattern is 'YYYY-MM'
+   * e.g. '2021-01'
+   */
+  filterMonth: Date;
+  assigned: number;
+  activity: number;
+  cumulativeAssigned: number;
+  cumulativeActivity: number;
+  available: number;
+}
+
+/**
+ * Monthly Budget Summary Cache Entity
+ *
+ * This entity is used to cache monthly budget summary data from Google Sheet
+ *
+ * Partition Key: year from filterMonth (e.g. '2021')
+ * Row Key: MonthKey from filterMonth (e.g. '2021-01')
+ */
+
+export interface MonthlyBudgetSummaryCacheEntity extends AzureTableEntityBase {
+  latestUpdate: Date;
+  startBudgetDate: Date;
+  filterMonth: Date;
+  startDate: Date;
+  endDate: Date;
+  readyToAssign: number;
+  totalIncome: number;
+  totalAssigned: number;
+  totalActivity: number;
+  totalAvailable: number;
+}

background-job/src/env.ts

@@ -24,6 +24,14 @@ export const envSchema = z.object({
    * Azure Storage Table Name, Transaction Cache Table
    */
   AZURE_STORAGE_TABLE_TRANSACTION_CACHE_TABLE_NAME: z.string().default('BudgetTransactionCache'),
+  /**
+   * Azure Storage Table Name, Monthly Budget Cache Table
+   */
+  AZURE_STORAGE_TABLE_MONTHLY_BUDGET_CACHE_TABLE_NAME: z.string().default('MonthlyBudgetCache'),
+  /**
+   * Azure Storage Table Name, Monthly Budget Summary Cache Table
+   */
+  AZURE_STORAGE_TABLE_MONTHLY_BUDGET_SUMMARY_CACHE_TABLE_NAME: z.string().default('MonthlyBudgetSummaryCache'),
   /**
    *  Google Sheet Private Key
    *
@@ -61,6 +69,20 @@ export const envSchema = z.object({
     value => parseGSheetId(value, 'GSHEET_SHEET_TRANSACTION_SHEET_ID'),
     z.number()
   ),
+  /**
+   * Google Sheet, Monthly Budget Sheet ID
+   */
+  GSHEET_SHEET_MONTHLY_BUDGET_SHEET_ID: z.preprocess(
+    value => parseGSheetId(value, 'GSHEET_SHEET_MONTHLY_BUDGET_SHEET_ID'),
+    z.number()
+  ),
+  /**
+   * Google Sheet, Monthly Budget Summary Sheet ID
+   */
+  GSHEET_SHEET_MONTHLY_BUDGET_SUMMARY_SHEET_ID: z.preprocess(
+    value => parseGSheetId(value, 'GSHEET_SHEET_MONTHLY_BUDGET_SUMMARY_SHEET_ID'),
+    z.number()
+  ),
   /**
    * Google Sheet, Required Field for Transaction Sheet ID
    *
@@ -73,6 +95,11 @@ export const envSchema = z.object({
    * Timezone
    */
   TIMEZONE: z.string().default('Asia/Bangkok'),
+
+  /**
+   * Azure SignalR Connection String
+   */
+  AzureSignalRConnectionString: z.string(),
 });
 
 function printSecretFields(data: Record<string, unknown>, secretFields: string[]) {

background-job/src/functions/cache-helper.ts

@@ -0,0 +1,47 @@
+import { InvocationContext } from '@azure/functions';
+import {
+  monthlyBudgetSummaryTableCache,
+  monthlyBudgetTableCache,
+  sheetClient,
+  transactionTableCache,
+} from '../bootstrap';
+import { MonthlyBudgetCacheService, MonthlyBudgetSummaryCacheService } from '../services/monthly-budget-cache.service';
+import { TransactionCacheService } from '../services/transaction-cache.service';
+
+export async function startCacheUpdate(context: InvocationContext, partial = false) {
+  const workers: Promise<void>[] = [];
+
+  if (!partial) {
+    workers.push(
+      new TransactionCacheService(context, sheetClient.transaction, transactionTableCache).updateWhenExpired()
+    );
+    workers.push(
+      new TransactionCacheService(context, sheetClient.transaction, transactionTableCache).deleteNonExistentRows()
+    );
+  }
+
+  workers.push(
+    new MonthlyBudgetSummaryCacheService(
+      context,
+      sheetClient.monthlyBudgetSummary,
+      monthlyBudgetSummaryTableCache
+    ).forceUpdate()
+  );
+
+  workers.push(
+    new MonthlyBudgetCacheService(context, sheetClient.monthlyBudget, monthlyBudgetTableCache).forceUpdate()
+  );
+
+  const result = await Promise.allSettled(workers);
+  let isError = false;
+  const errors: string[] = [];
+  for (const res of result) {
+    if (res.status === 'rejected') {
+      isError = true;
+      errors.push(res.reason);
+    }
+  }
+  if (isError) {
+    throw new Error(errors.join('\n'));
+  }
+}

background-job/src/functions/cache.ts

@@ -1,6 +1,5 @@
-import { sheetClient, transactionTableCache } from '../bootstrap';
 import { func } from '../nammatham';
-import { CacheService } from '../services/cache.service';
+import { startCacheUpdate } from './cache-helper';
 
 export default func
   .timer('updateCache', {
@@ -10,6 +9,5 @@ export default func
     schedule: '0 0 */6 * * *',
   })
   .handler(async c => {
-    await new CacheService(c.context, sheetClient, transactionTableCache).updateWhenExpired();
-    await new CacheService(c.context, sheetClient, transactionTableCache).deleteNonExistentRows();
+    await startCacheUpdate(c.context);
   });

background-job/src/functions/handle-long-queue.ts

@@ -0,0 +1,37 @@
+import { z } from 'zod';
+import { func } from '../nammatham';
+import { startCacheUpdate } from './cache-helper';
+import { output } from '@azure/functions';
+import { generateRealtimeMessage } from '../libs/signalr';
+
+const longQueueSchema = z.object({
+  type: z.enum(['update_monthly_budget']),
+});
+
+const signalrOutput = output.generic({
+  type: 'signalR',
+  hubName: 'serverless',
+  connectionStringSetting: 'AzureSignalRConnectionString',
+});
+
+export default func
+  .storageQueue('handleLongQueue', {
+    connection: 'AzureWebJobsStorage',
+    queueName: 'budgetlongqueue',
+    extraOutputs: [signalrOutput],
+  })
+  .handler(async c => {
+    const context = c.context;
+    context.log('Storage queue function processed work item:', c.trigger);
+    const triggerMetadata = c.context.triggerMetadata;
+    context.log('Queue metadata (dequeueCount):', triggerMetadata?.dequeueCount);
+    const data = longQueueSchema.parse(c.trigger);
+    context.log('data:', data);
+
+    if (data.type === 'update_monthly_budget') {
+      await startCacheUpdate(c.context, true);
+      context.extraOutputs.set(signalrOutput, [generateRealtimeMessage('monthlyBudgetUpdated')]);
+    } else {
+      throw new Error('Invalid type');
+    }
+  });