From 90c55f78a2945bdbe5f8f781562d2a0cde93a958 Mon Sep 17 00:00:00 2001 From: Eleftherios Chaidemenos Date: Tue, 4 Jul 2023 17:50:21 +0300 Subject: [PATCH 01/11] Move csv creation to API --- packages/api/package.json | 2 + packages/api/src/sensors/sensors.service.ts | 16 +-- .../src/time-series/time-series.controller.ts | 33 +++++ .../src/time-series/time-series.service.ts | 77 +++++++++-- .../api/src/time-series/time-series.spec.ts | 14 ++ packages/api/src/utils/time-series.utils.ts | 34 +++-- packages/website/package.json | 1 - .../DownloadCSVButton.tsx | 124 +++--------------- .../Chart/MultipleSensorsCharts/index.tsx | 1 - packages/website/src/custom.d.ts | 12 -- packages/website/src/helpers/siteUtils.ts | 9 ++ .../routes/SiteRoutes/UploadData/Header.tsx | 22 +--- packages/website/src/utils/utils.ts | 22 ++++ yarn.lock | 15 ++- 14 files changed, 210 insertions(+), 172 deletions(-) create mode 100644 packages/website/src/utils/utils.ts diff --git a/packages/api/package.json b/packages/api/package.json index 69ec7e5a0..9f54bd681 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -121,6 +121,8 @@ "@types/passport-strategy": "^0.2.35", "@types/sharp": "^0.30.4", "@types/supertest": "^2.0.8", + "@types/ungap__structured-clone": "^0.3.0", + "@ungap/structured-clone": "^1.2.0", "faker": "^4.1.0", "firebase-admin": "^11.9.0", "firebase-functions": "^3.8.0", diff --git a/packages/api/src/sensors/sensors.service.ts b/packages/api/src/sensors/sensors.service.ts index d3c880840..c0f621459 100644 --- a/packages/api/src/sensors/sensors.service.ts +++ b/packages/api/src/sensors/sensors.service.ts @@ -111,14 +111,14 @@ export class SensorsService { const site = await getSiteFromSensorId(sensorId, this.siteRepository); - const data = await getDataQuery( - this.timeSeriesRepository, - site.id, - metrics as Metric[], - startDate, - endDate, - false, - ); + const data = await getDataQuery({ + timeSeriesRepository: this.timeSeriesRepository, + siteId: site.id, + metrics: metrics as Metric[], + start: startDate, + end: endDate, + hourly: false, + }); return groupByMetricAndSource(data); } diff --git a/packages/api/src/time-series/time-series.controller.ts b/packages/api/src/time-series/time-series.controller.ts index bff325804..e97a30c3e 100644 --- a/packages/api/src/time-series/time-series.controller.ts +++ b/packages/api/src/time-series/time-series.controller.ts @@ -173,4 +173,37 @@ export class TimeSeriesController { ); return file.pipe(res); } + + @ApiOperation({ + summary: 'Returns specified time series data for a specified site as csv', + }) + @ApiQuery({ name: 'start', example: '2021-05-18T10:20:28.017Z' }) + @ApiQuery({ name: 'end', example: '2021-05-18T10:20:28.017Z' }) + @ApiQuery({ + name: 'metrics', + example: [Metric.BOTTOM_TEMPERATURE, Metric.TOP_TEMPERATURE], + }) + @ApiQuery({ name: 'hourly', example: false, required: false }) + @Header('Content-Type', 'text/csv') + @Get('sites/:siteId/csv') + findSiteDataCsv( + @Param() siteDataDto: SiteDataDto, + @Query( + 'metrics', + new DefaultValuePipe(Object.values(Metric)), + ParseArrayPipe, + ) + metrics: Metric[], + @Query('start', ParseDatePipe) startDate?: string, + @Query('end', ParseDatePipe) endDate?: string, + @Query('hourly', ParseBoolPipe) hourly?: boolean, + ) { + return this.timeSeriesService.findSiteDataCsv( + siteDataDto, + metrics, + startDate, + endDate, + hourly, + ); + } } diff --git a/packages/api/src/time-series/time-series.service.ts b/packages/api/src/time-series/time-series.service.ts index 5afa01b43..10153e2ef 100644 --- a/packages/api/src/time-series/time-series.service.ts +++ b/packages/api/src/time-series/time-series.service.ts @@ -10,6 +10,7 @@ import { NotFoundException, } from '@nestjs/common'; import { join } from 'path'; +import { groupBy } from 'lodash'; import { SiteDataDto } from './dto/site-data.dto'; import { SurveyPointDataDto } from './dto/survey-point-data.dto'; import { TimeSeries } from './time-series.entity'; @@ -64,15 +65,15 @@ export class TimeSeriesService { ) { const { siteId, surveyPointId } = surveyPointDataDto; - const data: TimeSeriesData[] = await getDataQuery( - this.timeSeriesRepository, + const data: TimeSeriesData[] = await getDataQuery({ + timeSeriesRepository: this.timeSeriesRepository, siteId, metrics, - startDate, - endDate, + start: startDate, + end: endDate, hourly, surveyPointId, - ); + }); return groupByMetricAndSource(data); } @@ -86,18 +87,74 @@ export class TimeSeriesService { ) { const { siteId } = siteDataDto; - const data: TimeSeriesData[] = await getDataQuery( - this.timeSeriesRepository, + const data: TimeSeriesData[] = await getDataQuery({ + timeSeriesRepository: this.timeSeriesRepository, siteId, metrics, - startDate, - endDate, + start: startDate, + end: endDate, hourly, - ); + }); return groupByMetricAndSource(data); } + async findSiteDataCsv( + siteDataDto: SiteDataDto, + metrics: Metric[], + startDate?: string, + endDate?: string, + hourly?: boolean, + ) { + const { siteId } = siteDataDto; + + const data: TimeSeriesData[] = await getDataQuery({ + timeSeriesRepository: this.timeSeriesRepository, + siteId, + metrics, + start: startDate, + end: endDate, + hourly, + csv: true, + }); + + const metricSourceAsKey = data.map((x) => ({ + key: `${x.metric}_${x.source}`, + value: x.value, + timestamp: x.timestamp, + })); + + const allKeys = [ + 'timestamp', + ...new Map(metricSourceAsKey.map((x) => [x.key, x])).keys(), + ]; + + const emptyRow = Object.fromEntries(allKeys.map((x) => [x, undefined])) as { + [k: string]: any; + }; + + const groupedByTimestamp = groupBy(metricSourceAsKey, (x) => + x.timestamp.toISOString(), + ); + + const rows = Object.entries(groupedByTimestamp).map(([timestamp, values]) => + values.reduce((acc, curr) => { + // eslint-disable-next-line fp/no-mutation + acc[curr.key] = curr.value; + // eslint-disable-next-line fp/no-mutation + acc.timestamp = timestamp; + return acc; + }, structuredClone(emptyRow)), + ); + + const rowsAsStrings = [ + allKeys.join(','), + ...rows.map((row) => Object.values(row).join(',')), + ]; + + return rowsAsStrings.join('\n'); + } + async findSurveyPointDataRange( surveyPointDataRangeDto: SurveyPointDataRangeDto, ) { diff --git a/packages/api/src/time-series/time-series.spec.ts b/packages/api/src/time-series/time-series.spec.ts index a3784f436..29332a438 100644 --- a/packages/api/src/time-series/time-series.spec.ts +++ b/packages/api/src/time-series/time-series.spec.ts @@ -4,6 +4,7 @@ import { max, min, union } from 'lodash'; import moment from 'moment'; import { join } from 'path'; import { readFileSync } from 'fs'; +import * as structuredClone from '@ungap/structured-clone'; import { TestService } from '../../test/test.service'; import { athensSite, californiaSite } from '../../test/mock/site.mock'; import { athensSurveyPointPiraeus } from '../../test/mock/survey-point.mock'; @@ -14,6 +15,9 @@ import { spotterMetrics, } from '../../test/mock/time-series.mock'; +// https://github.com/jsdom/jsdom/issues/3363 +global.structuredClone = structuredClone.default as any; + type StringDateRange = [string, string]; export const timeSeriesTests = () => { @@ -159,4 +163,14 @@ export const timeSeriesTests = () => { expect(rsp.headers['content-type']).toMatch(/^text\/csv/); expect(rsp.text).toMatch(expectedData); }); + + it('GET sites/:siteId/csv fetch data as csv', async () => { + const rsp = await request(app.getHttpServer()) + .get(`/time-series/sites/${californiaSite.id}/csv`) + .query({ hourly: true }) + .set('Accept', 'text/csv'); + + expect(rsp.status).toBe(200); + expect(rsp.headers['content-type']).toMatch(/^text\/csv/); + }); }; diff --git a/packages/api/src/utils/time-series.utils.ts b/packages/api/src/utils/time-series.utils.ts index 4ae525c9e..fcfbd886e 100644 --- a/packages/api/src/utils/time-series.utils.ts +++ b/packages/api/src/utils/time-series.utils.ts @@ -78,16 +78,30 @@ export const groupByMetricAndSource = ( .toJSON(); }; -export const getDataQuery = ( - timeSeriesRepository: Repository, - siteId: number, - metrics: Metric[], - start?: string, - end?: string, - hourly?: boolean, - surveyPointId?: number, -): Promise => { - const { endDate, startDate } = getTimeSeriesDefaultDates(start, end); +interface GetDataQueryParams { + timeSeriesRepository: Repository; + siteId: number; + metrics: Metric[]; + start?: string; + end?: string; + hourly?: boolean; + surveyPointId?: number; + csv?: boolean; +} + +export const getDataQuery = ({ + timeSeriesRepository, + siteId, + metrics, + start, + end, + hourly, + surveyPointId, + csv = false, +}: GetDataQueryParams): Promise => { + const { endDate, startDate } = csv + ? { startDate: start, endDate: end } + : getTimeSeriesDefaultDates(start, end); const surveyPointCondition = surveyPointId ? `(source.survey_point_id = ${surveyPointId} OR source.survey_point_id is NULL)` diff --git a/packages/website/package.json b/packages/website/package.json index ca83e4b3e..ad0d8d79f 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -30,7 +30,6 @@ "chartjs-plugin-annotation": "^0.5.7", "classnames": "^2.2.6", "date-fns-tz": "^1.1.3", - "download-csv": "^1.1.1", "enzyme": "^3.11.0", "enzyme-adapter-react-16": "^1.15.2", "enzyme-to-json": "^3.4.4", diff --git a/packages/website/src/common/Chart/MultipleSensorsCharts/DownloadCSVButton.tsx b/packages/website/src/common/Chart/MultipleSensorsCharts/DownloadCSVButton.tsx index a72c6fd7c..f87188f9c 100644 --- a/packages/website/src/common/Chart/MultipleSensorsCharts/DownloadCSVButton.tsx +++ b/packages/website/src/common/Chart/MultipleSensorsCharts/DownloadCSVButton.tsx @@ -1,80 +1,23 @@ import React, { useState } from 'react'; -import downloadCsv from 'download-csv'; import { Button } from '@material-ui/core'; -import moment from 'moment'; import { useSelector } from 'react-redux'; import { useSnackbar } from 'notistack'; -import { ValueWithTimestamp, MetricsKeys } from 'store/Sites/types'; +import { MetricsKeys } from 'store/Sites/types'; import { spotterPositionSelector } from 'store/Sites/selectedSiteSlice'; -import siteServices from 'services/siteServices'; +import { downloadCsvFile } from 'utils/utils'; +import { constructTimeSeriesDataCsvRequestUrl } from 'helpers/siteUtils'; +import moment from 'moment'; import DownloadCSVDialog from './DownloadCSVDialog'; import { CSVColumnData } from './types'; -type CSVColumnNames = - | 'spotterBottomTemp' - | 'spotterTopTemp' - | 'hoboTemp' - | 'oceanSensePH' - | 'oceanSenseEC' - | 'oceanSensePRESS' - | 'oceanSenseDO' - | 'oceanSenseORP' - | 'dailySST'; - -interface CSVRow extends Partial> { - timestamp: string; -} - const DATE_FORMAT = 'YYYY_MM_DD'; -/** - * Construct CSV data to pass into download-csv. - * This function is designed with the 'builder' pattern, where a chain function and result function is returned. - * Call the chained() function to add a new column to the CSV, or the result() function to return the final csv object. - * @param columnName - The name of the new column. 'timestamp' is reserved. - * @param data - The data corresponding to the new column. Obj array of timestamp and value. - * @param existingData - Should only used by chained() and never directly. Stores the in-progress CSV object. - */ -function constructCSVData( - columnName: CSVColumnNames, - data: ValueWithTimestamp[] = [], - existingData: Record = {}, -) { - // writing this in an immutable fashion will be detrimental to performance. - /* eslint-disable no-param-reassign,fp/no-mutation,fp/no-mutating-methods */ - const result = data.reduce((obj, item) => { - // we are basically ensuring there's always a timestamp column in the final result. - if (obj[item.timestamp]) { - obj[item.timestamp][columnName] = item.value; - } else { - obj[item.timestamp] = { - timestamp: item.timestamp, - [columnName]: item.value, - }; - } - return obj; - }, existingData); - return { - result: () => - Object.values(result).sort( - (a, b) => - new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime(), - ), - chained: ( - chainedFinalKey: CSVColumnNames, - chainedData: ValueWithTimestamp[] = [], - ) => constructCSVData(chainedFinalKey, chainedData, result), - }; - /* eslint-enable no-param-reassign,fp/no-mutation */ -} - interface DownloadCSVButtonParams { data: CSVColumnData[]; startDate?: string; endDate?: string; className?: string; siteId?: number | string; - pointId?: number | string; defaultMetrics?: MetricsKeys[]; } @@ -83,7 +26,6 @@ function DownloadCSVButton({ startDate, endDate, className, - pointId, siteId, defaultMetrics, }: DownloadCSVButtonParams) { @@ -103,33 +45,24 @@ function DownloadCSVButton({ return; } - if (!additionalData && !allDates && hourly) { - downloadCsv(getCSVData(data), undefined, fileName); - setOpen(false); - return; - } + const fileName = `data_site_${siteId}_${moment(startDate).format( + DATE_FORMAT, + )}_${moment(endDate).format(DATE_FORMAT)}.csv`; setLoading(true); - try { - const response = await siteServices.getSiteTimeSeriesData({ - hourly, - start: allDates ? undefined : startDate, - end: allDates ? undefined : endDate, - metrics: additionalData ? undefined : defaultMetrics, - siteId: String(siteId), - }); - - const formattedData = Object.entries(response.data) - .map(([metric, sources]) => { - return Object.entries(sources).map(([type, values]) => ({ - name: `${metric}_${type}`, - values: values.data, - })); - }) - .flat(); - - downloadCsv(getCSVData(formattedData), undefined, fileName); + await downloadCsvFile( + `${ + process.env.REACT_APP_API_BASE_URL + }/${constructTimeSeriesDataCsvRequestUrl({ + hourly, + start: allDates ? undefined : startDate, + end: allDates ? undefined : endDate, + metrics: additionalData ? undefined : defaultMetrics, + siteId: String(siteId), + })}`, + fileName, + ); } catch (error) { console.error(error); enqueueSnackbar('There was an error downloading csv data', { @@ -141,24 +74,6 @@ function DownloadCSVButton({ setOpen(false); }; - const getCSVData = (selectedData: CSVColumnData[]) => { - const [head, ...tail] = selectedData; - - // TODO: Change either CSVDataColumn names type or make it generic string - const start = constructCSVData(head.name as CSVColumnNames, head.values); - const result = tail.reduce( - (prev, curr) => prev.chained(curr.name as CSVColumnNames, curr.values), - start, - ); - return result.result(); - }; - - const fileName = `data_site_${siteId}${ - pointId ? `_survey_point_${pointId}` : '' - }_${moment(startDate).format(DATE_FORMAT)}_${moment(endDate).format( - DATE_FORMAT, - )}.csv`; - return ( <> {i !== exampleFiles.length - 1 ? ', ' : ''} diff --git a/packages/website/src/utils/utils.ts b/packages/website/src/utils/utils.ts index fd5c9e740..eb81f6650 100644 --- a/packages/website/src/utils/utils.ts +++ b/packages/website/src/utils/utils.ts @@ -1,4 +1,4 @@ -export function downloadCsvFile(url: string, fileName: string) { +export function downloadCsvFile(url: string) { const link = document.createElement('a'); // eslint-disable-next-line fp/no-mutation link.href = url; @@ -6,9 +6,6 @@ export function downloadCsvFile(url: string, fileName: string) { link.target = '_blank'; // eslint-disable-next-line fp/no-mutation link.rel = 'noopener noreferrer'; - if (fileName) { - link.setAttribute('download', fileName); - } document.body.appendChild(link); link.click(); } From 62bf4c5057ff819ef90da3400c7c92851f4f3912 Mon Sep 17 00:00:00 2001 From: Eleftherios Chaidemenos Date: Mon, 17 Jul 2023 14:01:30 +0300 Subject: [PATCH 08/11] Add timout for queries and stard-end date validation --- .../api/migration/1689588980336-AddQueryTimeout.ts | 13 +++++++++++++ packages/api/src/pipes/parse-metric-array.pipe.ts | 9 ++------- .../api/src/time-series/time-series.controller.ts | 10 +++++++++- packages/api/src/time-series/time-series.service.ts | 6 +++++- 4 files changed, 29 insertions(+), 9 deletions(-) create mode 100644 packages/api/migration/1689588980336-AddQueryTimeout.ts diff --git a/packages/api/migration/1689588980336-AddQueryTimeout.ts b/packages/api/migration/1689588980336-AddQueryTimeout.ts new file mode 100644 index 000000000..118cb0413 --- /dev/null +++ b/packages/api/migration/1689588980336-AddQueryTimeout.ts @@ -0,0 +1,13 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddQueryTimeout1689588980336 implements MigrationInterface { + name = 'AddQueryTimeout1689588980336'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query('SET statement_timeout = 40000;'); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query('SET statement_timeout = 0;'); + } +} diff --git a/packages/api/src/pipes/parse-metric-array.pipe.ts b/packages/api/src/pipes/parse-metric-array.pipe.ts index f9d3b35ff..24cd460be 100644 --- a/packages/api/src/pipes/parse-metric-array.pipe.ts +++ b/packages/api/src/pipes/parse-metric-array.pipe.ts @@ -7,15 +7,10 @@ import { } from '@nestjs/common'; @Injectable() -export class MetricArrayPipe - implements PipeTransform> -{ +export class MetricArrayPipe implements PipeTransform> { constructor(private readonly options: UniqueSubsetArrayOptions) {} - async transform( - value: string, - metadata: ArgumentMetadata, - ): Promise { + async transform(value: any, metadata: ArgumentMetadata): Promise { if (!value) { return this.options.defaultArray; } diff --git a/packages/api/src/time-series/time-series.controller.ts b/packages/api/src/time-series/time-series.controller.ts index 6c491d932..fc7734d20 100644 --- a/packages/api/src/time-series/time-series.controller.ts +++ b/packages/api/src/time-series/time-series.controller.ts @@ -11,6 +11,7 @@ import { Body, Res, Header, + BadRequestException, } from '@nestjs/common'; import { FilesInterceptor } from '@nestjs/platform-express'; import { ApiOperation, ApiQuery, ApiTags } from '@nestjs/swagger'; @@ -175,7 +176,9 @@ export class TimeSeriesController { surveyPointDataRangeDto, ); res.set({ - 'Content-Disposition': `attachment; filename=${surveyPointDataRangeDto.source}_example.csv`, + 'Content-Disposition': `attachment; filename=${encodeURIComponent( + `${surveyPointDataRangeDto.source}_example.csv`, + )}`, }); return file.pipe(res); } @@ -207,6 +210,11 @@ export class TimeSeriesController { @Query('end', ParseDatePipe) endDate?: string, @Query('hourly', ParseBoolPipe) hourly?: boolean, ) { + if (startDate && endDate && startDate > endDate) { + throw new BadRequestException( + `Invalid Dates: start date can't be after end date`, + ); + } return this.timeSeriesService.findSiteDataCsv( res, siteDataDto, diff --git a/packages/api/src/time-series/time-series.service.ts b/packages/api/src/time-series/time-series.service.ts index ec81b1c01..4ed4bcb52 100644 --- a/packages/api/src/time-series/time-series.service.ts +++ b/packages/api/src/time-series/time-series.service.ts @@ -177,7 +177,11 @@ export class TimeSeriesService { )}_${moment(endDate).format(DATE_FORMAT)}.csv`; res - .set({ 'Content-Disposition': `attachment; filename=${fileName}` }) + .set({ + 'Content-Disposition': `attachment; filename=${encodeURIComponent( + fileName, + )}`, + }) .send(stringify(rows, { header: true })); } From f2db7d2c917607cb341c5d226115865b29d661be Mon Sep 17 00:00:00 2001 From: echaidemenos <80451946+echaidemenos@users.noreply.github.com> Date: Thu, 20 Jul 2023 12:15:55 +0300 Subject: [PATCH 09/11] Update time-series.controller.ts --- packages/api/src/time-series/time-series.controller.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/api/src/time-series/time-series.controller.ts b/packages/api/src/time-series/time-series.controller.ts index fc7734d20..7f7f88675 100644 --- a/packages/api/src/time-series/time-series.controller.ts +++ b/packages/api/src/time-series/time-series.controller.ts @@ -175,10 +175,9 @@ export class TimeSeriesController { const file = this.timeSeriesService.getSampleUploadFiles( surveyPointDataRangeDto, ); + const filename = `${surveyPointDataRangeDto.source}_example.csv`; res.set({ - 'Content-Disposition': `attachment; filename=${encodeURIComponent( - `${surveyPointDataRangeDto.source}_example.csv`, - )}`, + 'Content-Disposition': `attachment; filename=${encodeURIComponent(filename)}`, }); return file.pipe(res); } From 755ec3b0679ab0c46d11f39b8bc2f7aafff2e852 Mon Sep 17 00:00:00 2001 From: echaidemenos <80451946+echaidemenos@users.noreply.github.com> Date: Thu, 20 Jul 2023 12:21:00 +0300 Subject: [PATCH 10/11] Change timout to 1min --- packages/api/migration/1689588980336-AddQueryTimeout.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/migration/1689588980336-AddQueryTimeout.ts b/packages/api/migration/1689588980336-AddQueryTimeout.ts index 118cb0413..dda952a2b 100644 --- a/packages/api/migration/1689588980336-AddQueryTimeout.ts +++ b/packages/api/migration/1689588980336-AddQueryTimeout.ts @@ -4,7 +4,7 @@ export class AddQueryTimeout1689588980336 implements MigrationInterface { name = 'AddQueryTimeout1689588980336'; public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query('SET statement_timeout = 40000;'); + await queryRunner.query('SET statement_timeout = 60000;'); } public async down(queryRunner: QueryRunner): Promise { From 451e2b89255a6814cfc007f291c0666617a4bd5b Mon Sep 17 00:00:00 2001 From: Eleftherios Chaidemenos Date: Thu, 20 Jul 2023 13:05:23 +0300 Subject: [PATCH 11/11] Fix lint --- packages/api/src/time-series/time-series.controller.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/api/src/time-series/time-series.controller.ts b/packages/api/src/time-series/time-series.controller.ts index 7f7f88675..213ca493f 100644 --- a/packages/api/src/time-series/time-series.controller.ts +++ b/packages/api/src/time-series/time-series.controller.ts @@ -177,7 +177,9 @@ export class TimeSeriesController { ); const filename = `${surveyPointDataRangeDto.source}_example.csv`; res.set({ - 'Content-Disposition': `attachment; filename=${encodeURIComponent(filename)}`, + 'Content-Disposition': `attachment; filename=${encodeURIComponent( + filename, + )}`, }); return file.pipe(res); }