CUMULUS-3682 -- Update @cumulus/message.Granule to handle missing values in CMR metadata

CHANGELOG.md

@@ -40,6 +40,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ### Changed
 
+- **CUMULUS-3862**
+  - Updated `@cumulus/messages/Granules/convertDateToISOStringSettingNull` to handle empty string as null to address CMR metadata compatibility concern
 - **CUMULUS-3940**
   - Added 'dead_letter_recovery_cpu' and 'dead_letter_recovery_memory' to `cumulus` and `archive` module configuration to allow configuration of the dead_letter_recovery_operation task definition to better allow configuration of the tool's operating environment.
   - Updated the dead letter recovery tool to utilize it's own log group "${var.prefix}-DeadLetterRecoveryEcsLogs"

packages/cmrjs/package.json

@@ -49,5 +49,8 @@
     "public-ip": "^5.0.0",
     "url-join": "^1.0.0",
     "xml2js": "0.5.0"
+  },
+  "devDependencies": {
+    "@cumulus/types": "19.1.0"
   }
-}
+}

packages/cmrjs/src/cmr-utils.js

@@ -1084,7 +1084,8 @@ async function getUserAccessibleBuckets(edlUser, cmrProvider = process.env.cmr_p
  * Extract temporal information from granule object
  *
  * @param {Object} granule - granule object
- * @returns {Promise<Object>} - temporal information (beginningDateTime,
+ * @returns {Promise<import('@cumulus/types').PartialGranuleProcessingInfo>}
+ * - temporal information (beginningDateTime,
  *    endingDateTime, productionDateTime, lastUpdateDateTime) of the granule if
  *    available.
  */

packages/message/src/Granules.ts

@@ -9,6 +9,7 @@
  */
 
 import isEmpty from 'lodash/isEmpty';
+import isNil from 'lodash/isNil';
 import isInteger from 'lodash/isInteger';
 import isUndefined from 'lodash/isUndefined';
 import mapValues from 'lodash/mapValues';
@@ -22,6 +23,7 @@ import {
   ApiGranule,
   GranuleStatus,
   GranuleTemporalInfo,
+  PartialGranuleTemporalInfo,
   MessageGranule,
 } from '@cumulus/types/api/granules';
 import { ApiFile } from '@cumulus/types/api/files';
@@ -133,13 +135,15 @@ function isProcessingTimeInfo(
 
 /**
  * ** Private **
- * Convert date string to standard ISO format, retaining null values if they exist
+ * Convert date string to standard ISO format, retaining null/undefined values if they exist
+ * and converting '' to null
  *
  * @param {string} date - Date string, possibly in multiple formats
  * @returns {string} Standardized ISO date string
  */
-const convertDateToISOStringPreservingNull = (date: string | null) => {
-  if (date === null) return null;
+const convertDateToISOStringSettingNull = (date: string | null | undefined) => {
+  if (isNil(date)) return date;
+  if (date === '') return null;
   return convertDateToISOString(date);
 };
 
@@ -159,7 +163,7 @@ export const getGranuleProcessingTimeInfo = (
     : {};
   return mapValues(
     updatedProcessingTimeInfo,
-    convertDateToISOStringPreservingNull
+    convertDateToISOStringSettingNull
   );
 };
 
@@ -193,20 +197,20 @@ export const getGranuleCmrTemporalInfo = async ({
   granule: MessageGranule,
   cmrTemporalInfo?: GranuleTemporalInfo,
   cmrUtils: CmrUtilsClass
-}): Promise<GranuleTemporalInfo | {}> => {
+}): Promise<PartialGranuleTemporalInfo | {}> => {
   // Get CMR temporalInfo (beginningDateTime, endingDateTime,
   // productionDateTime, lastUpdateDateTime)
   const temporalInfo = isGranuleTemporalInfo(cmrTemporalInfo)
     ? { ...cmrTemporalInfo }
-    : await cmrUtils.getGranuleTemporalInfo(granule);
+    : await cmrUtils.getGranuleTemporalInfo(granule) as PartialGranuleTemporalInfo;
 
   if (isEmpty(temporalInfo)) {
     return pick(granule, ['beginningDateTime', 'endingDateTime', 'productionDateTime', 'lastUpdateDateTime']);
   }
 
   return mapValues(
     temporalInfo,
-    convertDateToISOStringPreservingNull
+    convertDateToISOStringSettingNull
   );
 };
 

packages/message/src/types.ts

@@ -1,5 +1,5 @@
 import { Message } from '@cumulus/types';
-import { GranuleTemporalInfo, MessageGranule } from '@cumulus/types/api/granules';
+import { PartialGranuleProcessingInfo, MessageGranule } from '@cumulus/types/api/granules';
 
 export interface WorkflowMessageTemplateCumulusMeta {
   queueExecutionLimits: Message.QueueExecutionLimits
@@ -18,5 +18,5 @@ export interface Workflow {
 }
 
 export interface CmrUtilsClass {
-  getGranuleTemporalInfo(granule: MessageGranule): Promise<GranuleTemporalInfo | {}>
+  getGranuleTemporalInfo(granule: MessageGranule): Promise<PartialGranuleProcessingInfo | {}>
 }

packages/message/tests/test-Granules.js

@@ -3,6 +3,7 @@
 const test = require('ava');
 const cryptoRandomString = require('crypto-random-string');
 const cloneDeep = require('lodash/cloneDeep');
+const mapValues = require('lodash/mapValues');
 
 const {
   getGranuleQueryFields,
@@ -335,6 +336,55 @@ test('getGranuleCmrTemporalInfo() handles empty return from CMR and gets tempora
   t.deepEqual(updatedCmrTemporalInfo, t.context.fakeCmrMetadata);
 });
 
+test('getGranuleCmrTemporalInfo() returns null for null values', async (t) => {
+  const granule = {
+    beginningDateTime: null,
+    endingDateTime: null,
+    productionDateTime: null,
+    lastUpdateDateTime: null,
+  };
+  const cmrUtils = {
+    getGranuleTemporalInfo: () => Promise.resolve(granule),
+  };
+
+  const result = await getGranuleCmrTemporalInfo({ cmrTemporalInfo: {}, granule, cmrUtils });
+  t.deepEqual(result, granule);
+});
+
+test('getGranuleCmrTemporalInfo() returns null for empty string', async (t) => {
+  const granule = {
+    beginningDateTime: '',
+    endingDateTime: '',
+    productionDateTime: '',
+    lastUpdateDateTime: '',
+  };
+  const cmrUtils = {
+    getGranuleTemporalInfo: () => Promise.resolve(granule),
+  };
+
+  const expected = mapValues(granule, () => null);
+
+  const result = await getGranuleCmrTemporalInfo({ cmrTemporalInfo: {}, granule, cmrUtils });
+  t.deepEqual(result, expected);
+});
+
+test('getGranuleCmrTemporalInfo() returns undefined for explicitly defined undefined keys', async (t) => {
+  const granule = {
+    beginningDateTime: undefined,
+    endingDateTime: undefined,
+    productionDateTime: undefined,
+    lastUpdateDateTime: undefined,
+  };
+  const cmrUtils = {
+    getGranuleTemporalInfo: () => Promise.resolve(granule),
+  };
+
+  const expected = mapValues(granule, () => undefined);
+
+  const result = await getGranuleCmrTemporalInfo({ cmrTemporalInfo: {}, granule, cmrUtils });
+  t.deepEqual(result, expected);
+});
+
 test('getGranuleProcessingTimeInfo() converts input timestamps to standardized format', (t) => {
   const { timestampExtraPrecision } = t.context;
 
@@ -351,6 +401,48 @@ test('getGranuleProcessingTimeInfo() converts input timestamps to standardized f
   });
 });
 
+test('(getGranuleProcessingTimeInfo() returns null for missing beginningDateTime', (t) => {
+  const granule = {
+    processingEndDateTime: '2023-01-01T01:00:00.000Z',
+    processingStartDateTime: null,
+  };
+  const result = getGranuleProcessingTimeInfo(granule);
+  t.deepEqual(result, granule);
+});
+
+test('getGranuleProcessingTimeInfo() returns null for missing endingDateTime', (t) => {
+  const granule = {
+    processingEndDateTime: null,
+    processingStartDateTime: '2023-01-01T01:00:00.000Z',
+  };
+  const result = getGranuleProcessingTimeInfo(granule);
+  t.deepEqual(result, granule);
+});
+
+test('(getGranuleProcessingTimeInfo() returns null for empty string endingDateTime', (t) => {
+  const granule = {
+    processingEndDateTime: '',
+    processingStartDateTime: '2023-01-01T01:00:00.000Z',
+  };
+  const result = getGranuleProcessingTimeInfo(granule);
+  t.deepEqual(result, {
+    processingEndDateTime: null,
+    processingStartDateTime: granule.processingStartDateTime,
+  });
+});
+
+test('getGranuleProcessingTimeInfo() returns null for empty beginningDateTime', (t) => {
+  const granule = {
+    processingStartDateTime: '',
+    processingEndDateTime: '2023-01-01T01:00:00.000Z',
+  };
+  const result = getGranuleProcessingTimeInfo(granule);
+  t.deepEqual(result, {
+    processingStartDateTime: null,
+    processingEndDateTime: granule.processingEndDateTime,
+  });
+});
+
 test('generateGranuleApiRecord() builds granule record with correct processing and temporal info', async (t) => {
   const {
     collectionId,

packages/types/api/granules.d.ts

@@ -25,7 +25,7 @@ export type NullablePartialType<T> = {
 };
 
 type PartialGranuleTemporalInfo = NullablePartialType<GranuleTemporalInfo>;
-type ParitalGranuleProcessingInfo = NullablePartialType<import('./executions').ExecutionProcessingTimes>;
+type PartialGranuleProcessingInfo = NullablePartialType<import('./executions').ExecutionProcessingTimes>;
 
 export type ApiGranuleRecord = {
   granuleId: string
@@ -46,7 +46,7 @@ export type ApiGranuleRecord = {
   timestamp?: number
   timeToArchive?: number
   timeToPreprocess?: number
-} & PartialGranuleTemporalInfo & ParitalGranuleProcessingInfo;
+} & PartialGranuleTemporalInfo & PartialGranuleProcessingInfo;
 
 export type ApiGranule = {
   granuleId: string
@@ -67,4 +67,4 @@ export type ApiGranule = {
   timestamp?: number | null
   timeToArchive?: number | null
   timeToPreprocess?: number | null
-} & PartialGranuleTemporalInfo & ParitalGranuleProcessingInfo;
+} & PartialGranuleTemporalInfo & PartialGranuleProcessingInfo;