Fix missing data from CSV download

packages/api/migration/1689588980336-AddQueryTimeout.ts

@@ -0,0 +1,13 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddQueryTimeout1689588980336 implements MigrationInterface {
+  name = 'AddQueryTimeout1689588980336';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query('SET statement_timeout = 60000;');
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query('SET statement_timeout = 0;');
+  }
+}

packages/api/package.json

@@ -83,6 +83,7 @@
     "class-transformer": "^0.3.1",
     "class-validator": "^0.14.0",
     "csv-parse": "^4.15.1",
+    "csv-stringify": "^6.4.0",
     "express": "^4.17.1",
     "firebase-admin": "^11.9.0",
     "geo-tz": "^6.0.0",
@@ -121,6 +122,8 @@
     "@types/passport-strategy": "^0.2.35",
     "@types/sharp": "^0.30.4",
     "@types/supertest": "^2.0.8",
+    "@types/ungap__structured-clone": "^0.3.0",
+    "@ungap/structured-clone": "^1.2.0",
     "faker": "^4.1.0",
     "firebase-admin": "^11.9.0",
     "firebase-functions": "^3.8.0",

packages/api/src/pipes/parse-metric-array.pipe.ts

@@ -0,0 +1,46 @@
+import {
+  Injectable,
+  PipeTransform,
+  BadRequestException,
+  ParseArrayPipe,
+  ArgumentMetadata,
+} from '@nestjs/common';
+
+@Injectable()
+export class MetricArrayPipe implements PipeTransform<any, Promise<string[]>> {
+  constructor(private readonly options: UniqueSubsetArrayOptions) {}
+
+  async transform(value: any, metadata: ArgumentMetadata): Promise<string[]> {
+    if (!value) {
+      return this.options.defaultArray;
+    }
+
+    const parseArrayPipe = new ParseArrayPipe({
+      items: String,
+      separator: ',',
+    });
+    const parsedArray = (await parseArrayPipe.transform(
+      value,
+      metadata,
+    )) as string[];
+
+    const isSubset = parsedArray.every((item) =>
+      this.options.predefinedSet.includes(item),
+    );
+    if (!isSubset) {
+      throw new BadRequestException('Invalid array. Unknown elements');
+    }
+
+    const uniqueArray = [...new Set(parsedArray)];
+    if (uniqueArray.length !== parsedArray.length) {
+      throw new BadRequestException('Invalid array. Elements are not unique');
+    }
+
+    return uniqueArray;
+  }
+}
+
+interface UniqueSubsetArrayOptions {
+  predefinedSet: string[];
+  defaultArray: string[];
+}

packages/api/src/sensors/sensors.service.ts

@@ -111,14 +111,14 @@ export class SensorsService {
 
     const site = await getSiteFromSensorId(sensorId, this.siteRepository);
 
-    const data = await getDataQuery(
-      this.timeSeriesRepository,
-      site.id,
-      metrics as Metric[],
-      startDate,
-      endDate,
-      false,
-    );
+    const data = await getDataQuery({
+      timeSeriesRepository: this.timeSeriesRepository,
+      siteId: site.id,
+      metrics: metrics as Metric[],
+      start: startDate,
+      end: endDate,
+      hourly: false,
+    });
 
     return groupByMetricAndSource(data);
   }

packages/api/src/time-series/time-series.controller.ts

@@ -4,15 +4,14 @@ import {
   Param,
   Query,
   ParseBoolPipe,
-  ParseArrayPipe,
-  DefaultValuePipe,
   Post,
   UseInterceptors,
   UseGuards,
   UploadedFiles,
   Body,
   Res,
   Header,
+  BadRequestException,
 } from '@nestjs/common';
 import { FilesInterceptor } from '@nestjs/platform-express';
 import { ApiOperation, ApiQuery, ApiTags } from '@nestjs/swagger';
@@ -34,6 +33,7 @@ import { SourceType } from '../sites/schemas/source-type.enum';
 import { fileFilter } from '../utils/uploads/upload-sheet-data';
 import { SampleUploadFilesDto } from './dto/sample-upload-files.dto';
 import { Metric } from './metrics.enum';
+import { MetricArrayPipe } from '../pipes/parse-metric-array.pipe';
 
 const MAX_FILE_COUNT = 10;
 const MAX_FILE_SIZE_MB = 10;
@@ -60,8 +60,10 @@ export class TimeSeriesController {
     @Param() surveyPointDataDto: SurveyPointDataDto,
     @Query(
       'metrics',
-      new DefaultValuePipe(Object.values(Metric)),
-      ParseArrayPipe,
+      new MetricArrayPipe({
+        predefinedSet: Object.values(Metric),
+        defaultArray: Object.values(Metric),
+      }),
     )
     metrics: Metric[],
     @Query('start', ParseDatePipe) startDate?: string,
@@ -93,8 +95,10 @@ export class TimeSeriesController {
     @Param() siteDataDto: SiteDataDto,
     @Query(
       'metrics',
-      new DefaultValuePipe(Object.values(Metric)),
-      ParseArrayPipe,
+      new MetricArrayPipe({
+        predefinedSet: Object.values(Metric),
+        defaultArray: Object.values(Metric),
+      }),
     )
     metrics: Metric[],
     @Query('start', ParseDatePipe) startDate?: string,
@@ -171,6 +175,54 @@ export class TimeSeriesController {
     const file = this.timeSeriesService.getSampleUploadFiles(
       surveyPointDataRangeDto,
     );
+    const filename = `${surveyPointDataRangeDto.source}_example.csv`;
+    res.set({
+      'Content-Disposition': `attachment; filename=${encodeURIComponent(
+        filename,
+      )}`,
+    });
     return file.pipe(res);
   }
+
+  @ApiOperation({
+    summary: 'Returns specified time series data for a specified site as csv',
+  })
+  @ApiQuery({ name: 'start', example: '2021-05-18T10:20:28.017Z' })
+  @ApiQuery({ name: 'end', example: '2021-05-18T10:20:28.017Z' })
+  @ApiQuery({
+    name: 'metrics',
+    example: [Metric.BOTTOM_TEMPERATURE, Metric.TOP_TEMPERATURE],
+  })
+  @ApiQuery({ name: 'hourly', example: false, required: false })
+  @Header('Content-Type', 'text/csv')
+  @Get('sites/:siteId/csv')
+  findSiteDataCsv(
+    @Res() res: Response,
+    @Param() siteDataDto: SiteDataDto,
+    @Query(
+      'metrics',
+      new MetricArrayPipe({
+        predefinedSet: Object.values(Metric),
+        defaultArray: Object.values(Metric),
+      }),
+    )
+    metrics: Metric[],
+    @Query('start', ParseDatePipe) startDate?: string,
+    @Query('end', ParseDatePipe) endDate?: string,
+    @Query('hourly', ParseBoolPipe) hourly?: boolean,
+  ) {
+    if (startDate && endDate && startDate > endDate) {
+      throw new BadRequestException(
+        `Invalid Dates: start date can't be after end date`,
+      );
+    }
+    return this.timeSeriesService.findSiteDataCsv(
+      res,
+      siteDataDto,
+      metrics,
+      startDate,
+      endDate,
+      hourly,
+    );
+  }
 }

packages/api/src/time-series/time-series.service.ts

@@ -2,6 +2,8 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { createReadStream, unlinkSync } from 'fs';
 import { Repository } from 'typeorm';
 import Bluebird from 'bluebird';
+import type { Response } from 'express';
+import moment from 'moment';
 import {
   BadRequestException,
   HttpException,
@@ -10,6 +12,9 @@ import {
   NotFoundException,
 } from '@nestjs/common';
 import { join } from 'path';
+// https://github.com/adaltas/node-csv/issues/372
+// eslint-disable-next-line import/no-unresolved
+import { stringify } from 'csv-stringify/sync';
 import { SiteDataDto } from './dto/site-data.dto';
 import { SurveyPointDataDto } from './dto/survey-point-data.dto';
 import { TimeSeries } from './time-series.entity';
@@ -34,6 +39,8 @@ import { surveyPointBelongsToSite } from '../utils/site.utils';
 import { SampleUploadFilesDto } from './dto/sample-upload-files.dto';
 import { Metric } from './metrics.enum';
 
+const DATE_FORMAT = 'YYYY_MM_DD';
+
 @Injectable()
 export class TimeSeriesService {
   private logger = new Logger(TimeSeriesService.name);
@@ -64,15 +71,15 @@ export class TimeSeriesService {
   ) {
     const { siteId, surveyPointId } = surveyPointDataDto;
 
-    const data: TimeSeriesData[] = await getDataQuery(
-      this.timeSeriesRepository,
+    const data: TimeSeriesData[] = await getDataQuery({
+      timeSeriesRepository: this.timeSeriesRepository,
       siteId,
       metrics,
-      startDate,
-      endDate,
+      start: startDate,
+      end: endDate,
       hourly,
       surveyPointId,
-    );
+    });
 
     return groupByMetricAndSource(data);
   }
@@ -86,18 +93,98 @@ export class TimeSeriesService {
   ) {
     const { siteId } = siteDataDto;
 
-    const data: TimeSeriesData[] = await getDataQuery(
-      this.timeSeriesRepository,
+    const data: TimeSeriesData[] = await getDataQuery({
+      timeSeriesRepository: this.timeSeriesRepository,
       siteId,
       metrics,
-      startDate,
-      endDate,
+      start: startDate,
+      end: endDate,
       hourly,
-    );
+    });
 
     return groupByMetricAndSource(data);
   }
 
+  async findSiteDataCsv(
+    res: Response,
+    siteDataDto: SiteDataDto,
+    metrics: Metric[],
+    startDate?: string,
+    endDate?: string,
+    hourly?: boolean,
+  ) {
+    const { siteId } = siteDataDto;
+
+    const data: TimeSeriesData[] = await getDataQuery({
+      timeSeriesRepository: this.timeSeriesRepository,
+      siteId,
+      metrics,
+      start: startDate,
+      end: endDate,
+      hourly,
+      csv: true,
+    });
+
+    const metricSourceAsKey = data.map((x) => ({
+      key: `${x.metric}_${x.source}`,
+      value: x.value,
+      timestamp: x.timestamp,
+    }));
+
+    const allKeys = [
+      'timestamp',
+      ...new Map(metricSourceAsKey.map((x) => [x.key, x])).keys(),
+    ];
+
+    const emptyRow = Object.fromEntries(allKeys.map((x) => [x, undefined])) as {
+      [k: string]: any;
+    };
+
+    const groupedByTimestamp = metricSourceAsKey.reduce(
+      (acc, curr) => {
+        const key = curr.timestamp.toISOString();
+        const accValue = acc[key];
+        if (typeof accValue === 'object') {
+          // eslint-disable-next-line fp/no-mutating-methods
+          accValue.push(curr);
+        } else {
+          // eslint-disable-next-line fp/no-mutation
+          acc[key] = [curr];
+        }
+        return acc;
+      },
+      {} as {
+        [k: string]: {
+          key: string;
+          value: number;
+          timestamp: Date;
+        }[];
+      },
+    );
+
+    const rows = Object.entries(groupedByTimestamp).map(([timestamp, values]) =>
+      values.reduce((acc, curr) => {
+        // eslint-disable-next-line fp/no-mutation
+        acc[curr.key] = curr.value;
+        // eslint-disable-next-line fp/no-mutation
+        acc.timestamp = timestamp;
+        return acc;
+      }, structuredClone(emptyRow)),
+    );
+
+    const fileName = `data_site_${siteId}_${moment(startDate).format(
+      DATE_FORMAT,
+    )}_${moment(endDate).format(DATE_FORMAT)}.csv`;
+
+    res
+      .set({
+        'Content-Disposition': `attachment; filename=${encodeURIComponent(
+          fileName,
+        )}`,
+      })
+      .send(stringify(rows, { header: true }));
+  }
+
   async findSurveyPointDataRange(
     surveyPointDataRangeDto: SurveyPointDataRangeDto,
   ) {

packages/api/src/time-series/time-series.spec.ts

@@ -4,6 +4,7 @@ import { max, min, union } from 'lodash';
 import moment from 'moment';
 import { join } from 'path';
 import { readFileSync } from 'fs';
+import * as structuredClone from '@ungap/structured-clone';
 import { TestService } from '../../test/test.service';
 import { athensSite, californiaSite } from '../../test/mock/site.mock';
 import { athensSurveyPointPiraeus } from '../../test/mock/survey-point.mock';
@@ -14,6 +15,9 @@ import {
   spotterMetrics,
 } from '../../test/mock/time-series.mock';
 
+// https://github.com/jsdom/jsdom/issues/3363
+global.structuredClone = structuredClone.default as any;
+
 type StringDateRange = [string, string];
 
 export const timeSeriesTests = () => {
@@ -159,4 +163,14 @@ export const timeSeriesTests = () => {
     expect(rsp.headers['content-type']).toMatch(/^text\/csv/);
     expect(rsp.text).toMatch(expectedData);
   });
+
+  it('GET sites/:siteId/csv fetch data as csv', async () => {
+    const rsp = await request(app.getHttpServer())
+      .get(`/time-series/sites/${californiaSite.id}/csv`)
+      .query({ hourly: true })
+      .set('Accept', 'text/csv');
+
+    expect(rsp.status).toBe(200);
+    expect(rsp.headers['content-type']).toMatch(/^text\/csv/);
+  });
 };