diff --git a/CHANGELOG.md b/CHANGELOG.md index 619f319dece..9aaeed36fc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Changed +- **CUMULUS-3862** + - Updated `@cumulus/messages/Granules/convertDateToISOStringSettingNull` to handle empty string as null to address CMR metadata compatibility concern - **CUMULUS-3940** - Added 'dead_letter_recovery_cpu' and 'dead_letter_recovery_memory' to `cumulus` and `archive` module configuration to allow configuration of the dead_letter_recovery_operation task definition to better allow configuration of the tool's operating environment. - Updated the dead letter recovery tool to utilize it's own log group "${var.prefix}-DeadLetterRecoveryEcsLogs" diff --git a/packages/cmrjs/package.json b/packages/cmrjs/package.json index 251d3d2792e..ade2b67e920 100644 --- a/packages/cmrjs/package.json +++ b/packages/cmrjs/package.json @@ -49,5 +49,8 @@ "public-ip": "^5.0.0", "url-join": "^1.0.0", "xml2js": "0.5.0" + }, + "devDependencies": { + "@cumulus/types": "19.1.0" } -} +} \ No newline at end of file diff --git a/packages/cmrjs/src/cmr-utils.js b/packages/cmrjs/src/cmr-utils.js index 59fa5ff81b6..2951453c5eb 100644 --- a/packages/cmrjs/src/cmr-utils.js +++ b/packages/cmrjs/src/cmr-utils.js @@ -1084,7 +1084,8 @@ async function getUserAccessibleBuckets(edlUser, cmrProvider = process.env.cmr_p * Extract temporal information from granule object * * @param {Object} granule - granule object - * @returns {Promise} - temporal information (beginningDateTime, + * @returns {Promise} + * - temporal information (beginningDateTime, * endingDateTime, productionDateTime, lastUpdateDateTime) of the granule if * available. */ diff --git a/packages/message/src/Granules.ts b/packages/message/src/Granules.ts index 06850e177f4..21a64e53642 100644 --- a/packages/message/src/Granules.ts +++ b/packages/message/src/Granules.ts @@ -9,6 +9,7 @@ */ import isEmpty from 'lodash/isEmpty'; +import isNil from 'lodash/isNil'; import isInteger from 'lodash/isInteger'; import isUndefined from 'lodash/isUndefined'; import mapValues from 'lodash/mapValues'; @@ -22,6 +23,7 @@ import { ApiGranule, GranuleStatus, GranuleTemporalInfo, + PartialGranuleTemporalInfo, MessageGranule, } from '@cumulus/types/api/granules'; import { ApiFile } from '@cumulus/types/api/files'; @@ -133,13 +135,15 @@ function isProcessingTimeInfo( /** * ** Private ** - * Convert date string to standard ISO format, retaining null values if they exist + * Convert date string to standard ISO format, retaining null/undefined values if they exist + * and converting '' to null * * @param {string} date - Date string, possibly in multiple formats * @returns {string} Standardized ISO date string */ -const convertDateToISOStringPreservingNull = (date: string | null) => { - if (date === null) return null; +const convertDateToISOStringSettingNull = (date: string | null | undefined) => { + if (isNil(date)) return date; + if (date === '') return null; return convertDateToISOString(date); }; @@ -159,7 +163,7 @@ export const getGranuleProcessingTimeInfo = ( : {}; return mapValues( updatedProcessingTimeInfo, - convertDateToISOStringPreservingNull + convertDateToISOStringSettingNull ); }; @@ -193,12 +197,12 @@ export const getGranuleCmrTemporalInfo = async ({ granule: MessageGranule, cmrTemporalInfo?: GranuleTemporalInfo, cmrUtils: CmrUtilsClass -}): Promise => { +}): Promise => { // Get CMR temporalInfo (beginningDateTime, endingDateTime, // productionDateTime, lastUpdateDateTime) const temporalInfo = isGranuleTemporalInfo(cmrTemporalInfo) ? { ...cmrTemporalInfo } - : await cmrUtils.getGranuleTemporalInfo(granule); + : await cmrUtils.getGranuleTemporalInfo(granule) as PartialGranuleTemporalInfo; if (isEmpty(temporalInfo)) { return pick(granule, ['beginningDateTime', 'endingDateTime', 'productionDateTime', 'lastUpdateDateTime']); @@ -206,7 +210,7 @@ export const getGranuleCmrTemporalInfo = async ({ return mapValues( temporalInfo, - convertDateToISOStringPreservingNull + convertDateToISOStringSettingNull ); }; diff --git a/packages/message/src/types.ts b/packages/message/src/types.ts index ca90690f451..8cee16a8888 100644 --- a/packages/message/src/types.ts +++ b/packages/message/src/types.ts @@ -1,5 +1,5 @@ import { Message } from '@cumulus/types'; -import { GranuleTemporalInfo, MessageGranule } from '@cumulus/types/api/granules'; +import { PartialGranuleProcessingInfo, MessageGranule } from '@cumulus/types/api/granules'; export interface WorkflowMessageTemplateCumulusMeta { queueExecutionLimits: Message.QueueExecutionLimits @@ -18,5 +18,5 @@ export interface Workflow { } export interface CmrUtilsClass { - getGranuleTemporalInfo(granule: MessageGranule): Promise + getGranuleTemporalInfo(granule: MessageGranule): Promise } diff --git a/packages/message/tests/test-Granules.js b/packages/message/tests/test-Granules.js index 8d007e3edfa..fd747315764 100644 --- a/packages/message/tests/test-Granules.js +++ b/packages/message/tests/test-Granules.js @@ -3,6 +3,7 @@ const test = require('ava'); const cryptoRandomString = require('crypto-random-string'); const cloneDeep = require('lodash/cloneDeep'); +const mapValues = require('lodash/mapValues'); const { getGranuleQueryFields, @@ -335,6 +336,55 @@ test('getGranuleCmrTemporalInfo() handles empty return from CMR and gets tempora t.deepEqual(updatedCmrTemporalInfo, t.context.fakeCmrMetadata); }); +test('getGranuleCmrTemporalInfo() returns null for null values', async (t) => { + const granule = { + beginningDateTime: null, + endingDateTime: null, + productionDateTime: null, + lastUpdateDateTime: null, + }; + const cmrUtils = { + getGranuleTemporalInfo: () => Promise.resolve(granule), + }; + + const result = await getGranuleCmrTemporalInfo({ cmrTemporalInfo: {}, granule, cmrUtils }); + t.deepEqual(result, granule); +}); + +test('getGranuleCmrTemporalInfo() returns null for empty string', async (t) => { + const granule = { + beginningDateTime: '', + endingDateTime: '', + productionDateTime: '', + lastUpdateDateTime: '', + }; + const cmrUtils = { + getGranuleTemporalInfo: () => Promise.resolve(granule), + }; + + const expected = mapValues(granule, () => null); + + const result = await getGranuleCmrTemporalInfo({ cmrTemporalInfo: {}, granule, cmrUtils }); + t.deepEqual(result, expected); +}); + +test('getGranuleCmrTemporalInfo() returns undefined for explicitly defined undefined keys', async (t) => { + const granule = { + beginningDateTime: undefined, + endingDateTime: undefined, + productionDateTime: undefined, + lastUpdateDateTime: undefined, + }; + const cmrUtils = { + getGranuleTemporalInfo: () => Promise.resolve(granule), + }; + + const expected = mapValues(granule, () => undefined); + + const result = await getGranuleCmrTemporalInfo({ cmrTemporalInfo: {}, granule, cmrUtils }); + t.deepEqual(result, expected); +}); + test('getGranuleProcessingTimeInfo() converts input timestamps to standardized format', (t) => { const { timestampExtraPrecision } = t.context; @@ -351,6 +401,48 @@ test('getGranuleProcessingTimeInfo() converts input timestamps to standardized f }); }); +test('(getGranuleProcessingTimeInfo() returns null for missing beginningDateTime', (t) => { + const granule = { + processingEndDateTime: '2023-01-01T01:00:00.000Z', + processingStartDateTime: null, + }; + const result = getGranuleProcessingTimeInfo(granule); + t.deepEqual(result, granule); +}); + +test('getGranuleProcessingTimeInfo() returns null for missing endingDateTime', (t) => { + const granule = { + processingEndDateTime: null, + processingStartDateTime: '2023-01-01T01:00:00.000Z', + }; + const result = getGranuleProcessingTimeInfo(granule); + t.deepEqual(result, granule); +}); + +test('(getGranuleProcessingTimeInfo() returns null for empty string endingDateTime', (t) => { + const granule = { + processingEndDateTime: '', + processingStartDateTime: '2023-01-01T01:00:00.000Z', + }; + const result = getGranuleProcessingTimeInfo(granule); + t.deepEqual(result, { + processingEndDateTime: null, + processingStartDateTime: granule.processingStartDateTime, + }); +}); + +test('getGranuleProcessingTimeInfo() returns null for empty beginningDateTime', (t) => { + const granule = { + processingStartDateTime: '', + processingEndDateTime: '2023-01-01T01:00:00.000Z', + }; + const result = getGranuleProcessingTimeInfo(granule); + t.deepEqual(result, { + processingStartDateTime: null, + processingEndDateTime: granule.processingEndDateTime, + }); +}); + test('generateGranuleApiRecord() builds granule record with correct processing and temporal info', async (t) => { const { collectionId, diff --git a/packages/types/api/granules.d.ts b/packages/types/api/granules.d.ts index 45573e2c96e..00175a0bdb5 100644 --- a/packages/types/api/granules.d.ts +++ b/packages/types/api/granules.d.ts @@ -25,7 +25,7 @@ export type NullablePartialType = { }; type PartialGranuleTemporalInfo = NullablePartialType; -type ParitalGranuleProcessingInfo = NullablePartialType; +type PartialGranuleProcessingInfo = NullablePartialType; export type ApiGranuleRecord = { granuleId: string @@ -46,7 +46,7 @@ export type ApiGranuleRecord = { timestamp?: number timeToArchive?: number timeToPreprocess?: number -} & PartialGranuleTemporalInfo & ParitalGranuleProcessingInfo; +} & PartialGranuleTemporalInfo & PartialGranuleProcessingInfo; export type ApiGranule = { granuleId: string @@ -67,4 +67,4 @@ export type ApiGranule = { timestamp?: number | null timeToArchive?: number | null timeToPreprocess?: number | null -} & PartialGranuleTemporalInfo & ParitalGranuleProcessingInfo; +} & PartialGranuleTemporalInfo & PartialGranuleProcessingInfo;