#3745 - Modify process that reads SIN & CRA verification response files

sources/packages/backend/apps/queue-consumers/src/processors/schedulers/cra-integration/_tests_/cra-receive-files/CRA_200_PBCSA00000.TXT

@@ -0,0 +1,8 @@
+7200                        20240101 BCSAP00000      00000001                                                                                     0
+720110000000120240001NAME                           JOHN                         1999010110000000000                                              0
+7201100000001202400025077 111 AA                  DUMMY                       OO V0Y 0V0                                                          0
+7201100000001202400220001010101010101010010000VERIFICATION_ID:00000000101BCSA                                                                     0
+720110000000120240150000050000                                                                                                              15000 0
+72011000000012024A010BC                                                                                                                           0
+72011000000012024A015BC                                                                                                                           0
+72011000000012024A030N00000000                                                                                                                    0

sources/packages/backend/apps/queue-consumers/src/processors/schedulers/cra-integration/_tests_/cra-response-integration.scheduler.e2e-spec.ts

@@ -0,0 +1,116 @@
+import { createMock, DeepMocked } from "@golevelup/ts-jest";
+import { INestApplication } from "@nestjs/common";
+import { QueueNames } from "@sims/utilities";
+import {
+  createTestingAppModule,
+  describeProcessorRootTest,
+} from "../../../../../test/helpers";
+import {
+  E2EDataSources,
+  createE2EDataSources,
+  createFakeCRAIncomeVerification,
+  saveFakeApplication,
+  saveFakeStudent,
+} from "@sims/test-utils";
+import * as Client from "ssh2-sftp-client";
+import * as path from "path";
+import { CRAResponseIntegrationScheduler } from "../cra-response-integration.scheduler";
+import {
+  createFileFromStructuredRecords,
+  getStructuredRecords,
+  mockDownloadFiles,
+} from "@sims/test-utils/mocks";
+import { Job } from "bull";
+import { ApplicationStatus } from "@sims/sims-db";
+
+const CRA_FILENAME = "CRA_200_PBCSA00000.TXT";
+
+const padWithLeadingZeros = (num: number): string => {
+  // Pad the number with leading zeros to make it 9 digits long
+  return num.toString().padStart(9, "0");
+};
+
+describe(describeProcessorRootTest(QueueNames.CRAResponseIntegration), () => {
+  let app: INestApplication;
+  let processor: CRAResponseIntegrationScheduler;
+  let db: E2EDataSources;
+  let sftpClientMock: DeepMocked<Client>;
+  let craResponseFolder: string;
+
+  beforeAll(async () => {
+    craResponseFolder = path.join(__dirname, "cra-receive-files");
+    process.env.CRA_RESPONSE_FOLDER = craResponseFolder;
+    const { nestApplication, dataSource, sshClientMock } =
+      await createTestingAppModule();
+    app = nestApplication;
+    db = createE2EDataSources(dataSource);
+    sftpClientMock = sshClientMock;
+    processor = app.get(CRAResponseIntegrationScheduler);
+  });
+
+  beforeEach(async () => {
+    jest.clearAllMocks();
+  });
+
+  it.only("should process CRA response file", async () => {
+    // Arrange
+    const student = await saveFakeStudent(db.dataSource);
+
+    const application = await saveFakeApplication(
+      db.dataSource,
+      { student },
+      { applicationStatus: ApplicationStatus.InProgress },
+    );
+
+    // Create CRA income verifications for student.
+    const studentCRAIncomeVerification = createFakeCRAIncomeVerification(
+      {
+        application,
+      },
+      { initialValues: { dateReceived: null } },
+    );
+    await db.craIncomeVerification.save([studentCRAIncomeVerification]);
+    // Queued job.
+    const job = createMock<Job<void>>();
+    mockDownloadFiles(sftpClientMock, [CRA_FILENAME]);
+
+    mockDownloadFiles(sftpClientMock, [CRA_FILENAME], (fileContent: string) => {
+      const file = getStructuredRecords(fileContent);
+      const line2 = file.records[2]; // Get the 4th item (index 3) in the array
+
+      // Split the record on colon and change 9 digits after colon
+      const [beforeColon, afterColon] = line2.split(":");
+      const newVerificationId = padWithLeadingZeros(
+        studentCRAIncomeVerification.id,
+      );
+      const updatedRecord4 = `${beforeColon}:${newVerificationId}${afterColon.substring(
+        9,
+      )}`;
+
+      // Update the fourth record with the modified content
+      file.records[2] = updatedRecord4;
+
+      return createFileFromStructuredRecords(file);
+    });
+
+    // Act
+    const processResult = await processor.processResponses(job);
+    // Assert
+    const downloadedFile = path.join(
+      process.env.CRA_RESPONSE_FOLDER,
+      CRA_FILENAME,
+    );
+
+    // Assert
+    expect(processResult).toStrictEqual([
+      {
+        processSummary: [
+          `Processing file ${downloadedFile}.`,
+          "File contains 1 verifications.",
+          "Processed income verification. Total income record line 5. Status record from line 4.",
+        ],
+        errorsSummary: [],
+      },
+    ]);
+  });
+});

sources/packages/backend/apps/queue-consumers/src/processors/schedulers/esdc-integration/sin-validation-integration/_tests_/sin-receive-files/PCSLP.PBC.BC0000.ISR

@@ -0,0 +1,4 @@
+00100500220220921BC1       
+0026000000014Y100000001YYNY
+0026000000025Y100000002YYNY
+999000014000000200000003

sources/packages/backend/apps/queue-consumers/src/processors/schedulers/esdc-integration/sin-validation-integration/_tests_/sin-validation-process-response-integration.scheduler.e2e-spec.ts

@@ -0,0 +1,119 @@
+import { createMock, DeepMocked } from "@golevelup/ts-jest";
+import { INestApplication } from "@nestjs/common";
+import { QueueNames } from "@sims/utilities";
+import {
+  createTestingAppModule,
+  describeProcessorRootTest,
+} from "../../../../../../test/helpers";
+import {
+  E2EDataSources,
+  createE2EDataSources,
+  saveFakeStudent,
+} from "@sims/test-utils";
+import * as Client from "ssh2-sftp-client";
+import * as path from "path";
+import { SINValidationResponseIntegrationScheduler } from "../sin-validation-process-response-integration.scheduler";
+import {
+  createFileFromStructuredRecords,
+  getStructuredRecords,
+  mockDownloadFiles,
+} from "@sims/test-utils/mocks";
+import { Job } from "bull";
+
+const SIN_VALIDATION_FILENAME = "PCSLP.PBC.BC0000.ISR";
+
+const padWithLeadingZeros = (num: number): string => {
+  // Pad the number with leading zeros to make it 9 digits long
+  return num.toString().padStart(9, "0");
+};
+
+describe(
+  describeProcessorRootTest(QueueNames.SINValidationResponseIntegration),
+  () => {
+    let app: INestApplication;
+    let processor: SINValidationResponseIntegrationScheduler;
+    let db: E2EDataSources;
+    let sftpClientMock: DeepMocked<Client>;
+    let sinValidationResponseFolder: string;
+
+    beforeAll(async () => {
+      sinValidationResponseFolder = path.join(__dirname, "sin-receive-files");
+      process.env.ESDC_RESPONSE_FOLDER = sinValidationResponseFolder;
+      const { nestApplication, dataSource, sshClientMock } =
+        await createTestingAppModule();
+      app = nestApplication;
+      db = createE2EDataSources(dataSource);
+      sftpClientMock = sshClientMock;
+      processor = app.get(SINValidationResponseIntegrationScheduler);
+    });
+
+    beforeEach(async () => {
+      jest.clearAllMocks();
+    });
+
+    it.only("should process SIN validation response file", async () => {
+      // Arrange
+      // Create a SIN record with REFERENCE_IDX = 600000001
+      const validSinStudent = await saveFakeStudent(db.dataSource, undefined, {
+        sinValidationInitialValue: {
+          sin: "100000001",
+          isValidSIN: true,
+        },
+      });
+      // Create a SIN record with REFERENCE_IDX = 600000002
+      const inValidSinStudent = await saveFakeStudent(
+        db.dataSource,
+        undefined,
+        {
+          sinValidationInitialValue: {
+            sin: "100000002",
+            isValidSIN: false,
+          },
+        },
+      );
+      await db.sinValidation.save([validSinStudent, inValidSinStudent]);
+
+      // Queued job.
+      const job = createMock<Job<void>>();
+      mockDownloadFiles(sftpClientMock, [SIN_VALIDATION_FILENAME]);
+
+      mockDownloadFiles(
+        sftpClientMock,
+        [SIN_VALIDATION_FILENAME],
+        (fileContent: string) => {
+          const file = getStructuredRecords(fileContent);
+          const [record1] = file.records;
+
+          // Update the first record with validSinStudent's padded ID
+          const paddedId1 = padWithLeadingZeros(
+            validSinStudent.sinValidation.id,
+          );
+          file.records[0] =
+            record1.substring(0, 3) + paddedId1 + record1.substring(12);
+          return createFileFromStructuredRecords(file);
+        },
+      );
+
+      // Act
+      const processResult = await processor.processSINValidationResponse(job);
+      // Assert
+      const downloadedFile = path.join(
+        process.env.ESDC_RESPONSE_FOLDER,
+        SIN_VALIDATION_FILENAME,
+      );
+
+      // Assert
+      expect(processResult).toStrictEqual([
+        {
+          processSummary: [
+            `Processing file ${downloadedFile}.`,
+            "File contains 2 SIN validations.",
+            "Processed SIN validation record from line 2: No SIN validation was updated because the record id is already present and this is not the most updated.",
+            "Processed SIN validation record from line 3: Not able to find the SIN validation on line number 3 to be updated with the ESDC response.",
+          ],
+          errorsSummary: [],
+        },
+      ]);
+    });
+  },
+);

sources/packages/backend/libs/integrations/src/cra-integration/cra-income-verification.processing.service.ts

@@ -232,7 +232,6 @@ export class CRAIncomeVerificationProcessingService {
       // allowing other response files to be processed.
       return result;
     }
-
     result.processSummary.push(
       `File contains ${responseFile.statusRecords.length} verifications.`,
     );

sources/packages/backend/libs/integrations/src/services/sin-validation/sin-validation.service.ts

@@ -121,9 +121,10 @@ export class SINValidationService extends RecordDataModelService<SINValidation>
           .getOne();
 
         if (!existingValidation) {
-          throw new Error(
-            `Not able to find the SIN validation id ${validationResponse.referenceIndex} to be updated with the ESDC response.`,
-          );
+          return {
+            operationDescription: `Not able to find the SIN validation on line number ${validationResponse.lineNumber} to be updated with the ESDC response.`,
+            record: null,
+          };
         }
 
         const sinValidationNeverUpdated = !existingValidation.dateReceived;