diff --git a/packages/api/app-prod.yaml b/packages/api/app-prod.yaml index e312b94ea..1828e65ee 100644 --- a/packages/api/app-prod.yaml +++ b/packages/api/app-prod.yaml @@ -1,5 +1,9 @@ +service: production runtime: custom env: flex manual_scaling: instances: 2 + +resources: + memory_gb: 4 \ No newline at end of file diff --git a/packages/api/app-programize.yaml b/packages/api/app-programize.yaml index 920e11f56..fa161e44c 100644 --- a/packages/api/app-programize.yaml +++ b/packages/api/app-programize.yaml @@ -4,3 +4,6 @@ env: flex manual_scaling: instances: 1 + +resources: + memory_gb: 2 \ No newline at end of file diff --git a/packages/api/src/utils/uploads/upload-sheet-data.ts b/packages/api/src/utils/uploads/upload-sheet-data.ts index e82f8af2b..5a0988d9b 100644 --- a/packages/api/src/utils/uploads/upload-sheet-data.ts +++ b/packages/api/src/utils/uploads/upload-sheet-data.ts @@ -290,7 +290,13 @@ const groupBySitePointDepth = ({ headerToTokenMap, siteId, surveyPointId, -}: GroupBySitePointDepthProps): { [key: string]: string[][] } => { +}: GroupBySitePointDepthProps): { + data: string[][]; + siteId: number; + surveyPointId?: number; + depth?: number; +}[] => { + logger.log('Grouping data'); const siteIdIndex = headerToTokenMap.findIndex( (x) => x === 'aqualink_site_id', ); @@ -309,13 +315,22 @@ const groupBySitePointDepth = ({ const key = `${rowSiteId}_${rowSurveyPointId}_${rowDepth}`; const item = groupedByMap.get(key); if (item !== undefined) { - groupedByMap.set(key, [...item, val]); + // eslint-disable-next-line fp/no-mutating-methods -- mutating for performance, instead of spreading + item.push(val); } else { groupedByMap.set(key, [val]); } }); - return Object.fromEntries(Array.from(groupedByMap)); + return Array.from(groupedByMap).map(([key, data]) => { + const [rowSiteId, rowSurveyPointId, depth] = key.split('_'); + return { + data, + siteId: parseInt(rowSiteId, 10), + surveyPointId: parseInt(rowSurveyPointId, 10) || undefined, + depth: parseInt(depth, 10) || undefined, + }; + }); }; export const convertData = ( @@ -328,14 +343,24 @@ export const convertData = ( mimetype?: Mimetype, ) => { const timestampIndex = findTimeStampIndex(headerToTokenMap); + const timezone = + typeof timestampIndex === 'number' + ? first(headers[timestampIndex].match(TIMEZONE_REGEX)) + : undefined; + const metricHeadersMap = headerToTokenMap.reduce((acc, token, i) => { + if (token === undefined || nonMetric.includes(token as NonMetric)) { + return acc; + } + return { ...acc, [i]: token }; + }, {}); + const results = Array( + workSheetData.length * Object.keys(metricHeadersMap).length, + ); + let resultsIndex = 0; console.time(`Get data from sheet ${fileName}`); - const results = workSheetData.reduce((acc, row) => { - const timezone = - typeof timestampIndex === 'number' - ? first(headers[timestampIndex].match(TIMEZONE_REGEX)) - : undefined; + workSheetData.forEach((row) => { const timestampDate = getTimeStamp(timestampIndex, row, mimetype, timezone); // This need to be done for each row to take into account daylight savings @@ -349,22 +374,18 @@ export const convertData = ( timestampDate.valueOf() - offsetInMil, ).toISOString(); - const rowValues = row.map((cell, i) => { - const metric = headerToTokenMap[i]; - if (metric === undefined || nonMetric.includes(metric as NonMetric)) { - return undefined; + row.forEach((cell, i) => { + const metric = metricHeadersMap[i]; + if (metric) { + // eslint-disable-next-line fp/no-mutation -- mutating for performance + results[resultsIndex++] = { + timestamp, + value: parseFloat(cell), + metric: metric as Metric, + source: sourceEntity, + }; } - - return { - timestamp, - value: parseFloat(cell), - metric: metric as Metric, - source: sourceEntity, - }; }); - - const filtered = rowValues.filter((x): x is Data => x !== undefined); - return [...acc, ...filtered]; }, []); console.timeEnd(`Get data from sheet ${fileName}`); @@ -779,22 +800,15 @@ export const uploadTimeSeriesData = async ({ const trimmed = trimWorkSheetData(workSheetData, headers, headerIndex); - const uploadData = Object.entries( - groupBySitePointDepth({ - trimmedWorkSheetData: trimmed, - headerToTokenMap, - siteId, - surveyPointId, - }), - ).map(([key, data]) => ({ - data, - siteId: parseInt(key.split('_')[0], 10), - surveyPointId: parseInt(key.split('_')[1], 10) || undefined, - depth: parseInt(key.split('_')[2], 10) || undefined, - })); + const groupedData = groupBySitePointDepth({ + trimmedWorkSheetData: trimmed, + headerToTokenMap, + siteId, + surveyPointId, + }); const converted = await Promise.all( - uploadData.map((x) => { + groupedData.map((x) => { return createEntitiesAndConvert({ workSheetData: x.data, siteId: x.siteId,