Adds long PAN functionality to send-pan task

packages/api/lib/pdrHelpers.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const pvl = require('@cumulus/pvl');
+const { getExecution } = require('@cumulus/api-client/executions');
 
 /**
  * Generate Short PAN message
@@ -17,6 +18,38 @@ function generateShortPAN(disposition) {
   );
 }
 
+async function getGranuleFromExecution(executionArn) {
+  const excObj = await getExecution({
+    prefix: process.env.stackName,
+    arn: executionArn,
+  });
+  return excObj.originalPayload.granules[0];
+}
+
+/**
+ * Generate Long PAN message
+ *
+ * @param {Object|string[]} executions - List of workflow executions
+ * @returns {string} the PAN message
+ */
+async function generateLongPAN(executions) {
+  const timeStamp = new Date();
+
+  const longPan = new pvl.models.PVLRoot()
+    .add('MESSAGE_TYPE', new pvl.models.PVLTextString('LONGPAN'))
+    .add('NO_OF_FILES', new pvl.models.PVLNumeric(executions.length));
+  /* eslint-disable no-await-in-loop */
+  for (const exc of executions) {
+    const granule = await getGranuleFromExecution(exc.arn || exc);
+    longPan.add('FILE_DIRECTORY', new pvl.models.PVLTextString(granule.files[0].path));
+    longPan.add('FILE_NAME', new pvl.models.PVLTextString(granule.granuleId));
+    longPan.add('DISPOSITION', new pvl.models.PVLTextString(exc.reason || 'SUCCESSFUL'));
+    longPan.add('TIME_STAMP', new pvl.models.PVLDateTime(timeStamp));
+  }
+  /* eslint-enable no-await-in-loop */
+  return pvl.jsToPVL(longPan);
+}
+
 /**
  * Generate a PDRD message with a given err
  *
@@ -33,5 +66,7 @@ function generatePDRD(err) {
 
 module.exports = {
   generateShortPAN,
+  generateLongPAN,
   generatePDRD,
+  getGranuleFromExecution,
 };

packages/api/tests/lib/test-pdrHelpers.js

@@ -1,12 +1,61 @@
 'use strict';
 
 const test = require('ava');
-const pdrHelpers = require('../../lib/pdrHelpers');
+const proxyquire = require('proxyquire');
+
+const fakeExecutionModule = {
+  getExecution: () => Promise.resolve({
+    originalPayload: {
+      granules: [
+        {
+          files: [
+            {
+              name: 'test_id.nc',
+              path: 'test',
+            },
+          ],
+          granuleId: 'test_id',
+        },
+      ],
+    },
+  }),
+};
+
+const pdrHelpers = proxyquire(
+  '../../lib/pdrHelpers',
+  {
+    '@cumulus/api-client/executions': fakeExecutionModule,
+  }
+);
 
 // eslint-disable-next-line max-len
 const regex = /MESSAGE_TYPE = "SHORTPAN";\nDISPOSITION = "SUCCESSFUL";\nTIME_STAMP = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z;\n/;
 // eslint-disable-next-line max-len
 const emptyRegex = /MESSAGE_TYPE = "SHORTPAN";\nDISPOSITION = "";\nTIME_STAMP = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z;\n/;
+const longPanRegex = new RegExp(
+  'MESSAGE_TYPE = "LONGPAN";\\n' +
+  'NO_OF_FILES = 5;\\n' +
+  'FILE_DIRECTORY = "test";\\n' +
+  'FILE_NAME = "test_id";\\n' +
+  'DISPOSITION = "FAILED A";\\n' +
+  'TIME_STAMP = \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}Z;\\n' +
+  'FILE_DIRECTORY = "test";\\n' +
+  'FILE_NAME = "test_id";\\n' +
+  'DISPOSITION = "FAILED B";\\n' +
+  'TIME_STAMP = \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}Z;\\n' +
+  'FILE_DIRECTORY = "test";\\n' +
+  'FILE_NAME = "test_id";\\n' +
+  'DISPOSITION = "FAILED C";\\n' +
+  'TIME_STAMP = \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}Z;\\n' +
+  'FILE_DIRECTORY = "test";\\n' +
+  'FILE_NAME = "test_id";\\n' +
+  'DISPOSITION = "SUCCESSFUL";\\n' +
+  'TIME_STAMP = \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}Z;\\n' +
+  'FILE_DIRECTORY = "test";\\n' +
+  'FILE_NAME = "test_id";\\n' +
+  'DISPOSITION = "SUCCESSFUL";\\n' +
+  'TIME_STAMP = \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}Z;\\n'
+);
 
 test('generateShortPAN with a disposition', (t) => {
   const disposition = 'SUCCESSFUL';
@@ -19,3 +68,15 @@ test('generateShortPAN with an empty disposition', (t) => {
   const pan = pdrHelpers.generateShortPAN(disposition);
   t.regex(pan, emptyRegex);
 });
+
+test('generateLongPAN', async (t) => {
+  const executions = [
+    { arn: 'arn:failed:execution', reason: 'FAILED A' },
+    { arn: 'arn:failed:execution', reason: 'FAILED B' },
+    { arn: 'arn:failed:execution', reason: 'FAILED C' },
+    'arn:completed:execution',
+    'arn:completed:execution',
+  ];
+  const pan = await pdrHelpers.generateLongPAN(executions);
+  t.regex(pan, longPanRegex);
+});

tasks/pdr-status-check/README.md

@@ -3,6 +3,15 @@
 Lambda function handler for checking the status of a workflow (step function) execution. Expects a payload object which includes the name of a PDR.
 The concurrency of SFN API calls is set to 10 by default, and it's configurable by setting the Lambda environment variable CONCURRENCY.
 
+Make sure the line: `"ErrorPath": "$.exception.Cause"` is added to your workflow failed task inside your ingest granule workflow to ensure the error is properly propagated to this task
+```json
+"WorkflowFailed": {
+    "Type": "Fail",
+    "Cause": "Workflow failed",
+    "ErrorPath": "$.exception.Cause"
+},
+```
+
 ## About Cumulus
 
 Cumulus is a cloud-based data ingest, archive, distribution and management prototype for NASA's future Earth science data streams.

tasks/pdr-status-check/index.js

@@ -91,7 +91,7 @@ function buildOutput(event, groupedExecutions) {
 
   const parseFailedExecution = (execution) => {
     let reason = 'Workflow Failed';
-    if (execution.output) reason = JSON.parse(execution.output).exception;
+    if (execution.error) reason = execution.error;
     return { arn: execution.executionArn, reason };
   };
 

tasks/send-pan/schemas/config.json

@@ -12,6 +12,11 @@
       "type": ["string", "null"],
       "description": "The path in the provider to upload the file to.",
       "default": "pans"
+    },
+    "panType": {
+      "type": ["string", "null"],
+      "description": "Determines which pan type to create: (shortPan, longPan, or longPanAlways)",
+      "default": "shortPan"
     }
   }
 }

tasks/send-pan/src/index.ts

@@ -41,16 +41,40 @@ async function sendPAN(event: HandlerEvent): Promise<HandlerOutput> {
   const { config, input } = event;
   const provider = config.provider;
   const remoteDir = config.remoteDir || 'pans';
+  const panType = config.panType || 'shortPan';
 
-  const panName = input.pdr.name.replace(/\.pdr/gi, '.pan');
+  const panName = input.pdr.name.replace(/\.pdr/gi, '.PAN');
   const uploadPath = path.join(remoteDir, panName);
 
   if (input.running.length !== 0) {
     throw new Error('Executions still running');
   }
-
-  const disposition = (input.failed.length > 0) ? 'FAILED' : 'SUCCESSFUL';
-  const pan = pdrHelpers.generateShortPAN(disposition);
+  let pan;
+  switch (panType) {
+    case 'longPanAlways':
+      pan = await pdrHelpers.generateLongPAN([...input.completed, ...input.failed]);
+      log.debug('Created long PAN');
+      break;
+    case 'shortPan': {
+      const disposition = (input.failed.length > 0) ? 'FAILED' : 'SUCCESSFUL';
+      pan = pdrHelpers.generateShortPAN(disposition);
+      log.debug('Created short PAN');
+      break;
+    }
+    case 'longPan': {
+      if (input.failed.length + input.completed.length <= 1) {
+        const disposition = (input.failed.length > 0) ? 'FAILED' : 'SUCCESSFUL';
+        pan = pdrHelpers.generateShortPAN(disposition);
+        log.debug('Created short PAN');
+      } else {
+        pan = await pdrHelpers.generateLongPAN([...input.completed, ...input.failed]);
+        log.debug('Created long PAN');
+      }
+      break;
+    }
+    default:
+      throw new Error(`Unknown panType: ${panType}, must be shortPan, longPan, or longPanAlways`);
+  }
 
   const localPath = path.join(tmpdir(), panName);
   fs.writeFileSync(localPath, pan);

tasks/send-pan/src/types.ts

@@ -32,6 +32,7 @@ export type HandlerEvent = {
       host: string,
     },
     remoteDir: string | null,
+    panType: string | null
   },
   input: HandlerInput,
 };

tasks/send-pan/tests/index.js

@@ -52,7 +52,7 @@ test('SendPan task calls upload', async (t) => {
   };
 
   const url = `http://${event.config.provider.host}:${port}`;
-  const remotePath = path.join(event.config.remoteDir, `${fileNameBase}.pan`);
+  const remotePath = path.join(event.config.remoteDir, `${fileNameBase}.PAN`);
   // Message should look like this:
   // MESSAGE_TYPE = "SHORTPAN";
   // DISPOSITION = "SUCCESSFUL";
@@ -103,7 +103,7 @@ test('SendPan task sends PAN to HTTP server', async (t) => {
 test('SendPan task sends PAN to s3', async (t) => {
   const remoteDir = 'pan/remote-dir';
   const fileNameBase = 'test-send-s3-pdr';
-  const uploadPath = path.join(remoteDir, `${fileNameBase}.pan`);
+  const uploadPath = path.join(remoteDir, `${fileNameBase}.PAN`);
   const event = {
     config: {
       provider: {
@@ -175,7 +175,7 @@ test('SendPan task throws error when provider protocol is not supported', async
 
 test('SendPan task sends PAN to default location when remoteDir is null', async (t) => {
   const fileNameBase = 'test-default-pan-path-pdr';
-  const uploadPath = `pans/${fileNameBase}.pan`;
+  const uploadPath = `pans/${fileNameBase}.PAN`;
   const event = {
     config: {
       provider: {
@@ -248,7 +248,7 @@ test('SendPan task fails with executions still running', async (t) => {
 
 test('SendPan task sends failed PAN to s3', async (t) => {
   const fileNameBase = 'test-failed-pan-path-pdr';
-  const uploadPath = `pans/${fileNameBase}.pan`;
+  const uploadPath = `pans/${fileNameBase}.PAN`;
   const event = {
     config: {
       provider: {