Migrate to luxon

aqualinkorg · Aug 8, 2023 · 009f050 · 009f050
1 parent 9808d33
commit 009f050
Show file tree

Hide file tree

Showing 48 changed files with 484 additions and 403 deletions.
diff --git a/packages/api/package.json b/packages/api/package.json
@@ -93,8 +93,6 @@
     "lodash": "^4.17.15",
     "luxon": "^3.3.0",
     "md5-file": "^5.0.0",
-    "moment": "^2.27.0",
-    "moment-timezone": "^0.5.31",
     "multer": "^1.4.2",
     "node-xlsx": "^0.17.2",
     "objects-to-csv": "^1.3.6",
@@ -117,7 +115,6 @@
     "@types/jest": "24.9.0",
     "@types/lodash": "^4.14.156",
     "@types/luxon": "^3.3.1",
-    "@types/moment-timezone": "^0.5.13",
     "@types/multer": "^1.4.3",
     "@types/node": "^18.16.16",
     "@types/passport": "^1.0.4",

diff --git a/packages/api/scripts/backfill-daily-data.ts b/packages/api/scripts/backfill-daily-data.ts
@@ -1,6 +1,6 @@
 import Bluebird from 'bluebird';
 import yargs from 'yargs';
-import moment from 'moment';
+import { DateTime } from 'luxon';
 import { getSitesDailyData } from '../src/workers/dailyData';
 import AqualinkDataSource from '../ormconfig';
 
@@ -24,19 +24,13 @@ async function run() {
   const { d: days, s: sites } = argv;
   const backlogArray = Array.from(Array(days).keys());
   const siteIds = sites && sites.map((site) => parseInt(`${site}`, 10));
-  const today = moment()
-    .utc()
-    .hours(23)
-    .minutes(59)
-    .seconds(59)
-    .milliseconds(999);
+  const today = DateTime.utc().endOf('day');
   const connection = await AqualinkDataSource.initialize();
   // eslint-disable-next-line fp/no-mutating-methods
   await Bluebird.mapSeries(backlogArray.reverse(), async (past) => {
-    const date = moment(today);
-    date.day(today.day() - past - 1);
+    const date = today.set({ day: today.day - past - 1 });
     try {
-      await getSitesDailyData(connection, date.toDate(), siteIds);
+      await getSitesDailyData(connection, date.toJSDate(), siteIds);
     } catch (error) {
       console.error(error);
     }

diff --git a/packages/api/scripts/utils/netcdf.ts b/packages/api/scripts/utils/netcdf.ts
@@ -1,5 +1,5 @@
 import { times } from 'lodash';
-import moment from 'moment';
+import { DateTime } from 'luxon';
 import { Extent, pointToIndex } from '../../src/utils/coordinates';
 
 let netcdf4;
@@ -43,11 +43,10 @@ export function getNOAAData(year: number = 2020, long: number, lat: number) {
     height,
   );
 
-  const startDate = moment(new Date(year, 0, 1));
+  const startDate = DateTime.fromJSDate(new Date(year, 0, 1));
 
   return times(dateRange, (dateIndex) => {
-    const date = moment(startDate);
-    date.day(startDate.day() + dateIndex);
+    const date = startDate.set({ day: startDate.day + dateIndex });
     const data: number[] = variables.sst.readSlice(
       dateIndex,
       1,
@@ -57,6 +56,6 @@ export function getNOAAData(year: number = 2020, long: number, lat: number) {
       10,
     );
     const filteredData = data.filter((value) => value <= 9999999);
-    return { date: date.toDate(), satelliteTemperature: filteredData[0] };
+    return { date: date.toJSDate(), satelliteTemperature: filteredData[0] };
   });
 }
diff --git a/packages/api/src/sensors/sensors.spec.ts b/packages/api/src/sensors/sensors.spec.ts
@@ -1,5 +1,5 @@
 import { INestApplication } from '@nestjs/common';
-import moment from 'moment';
+import { DateTime } from 'luxon';
 import request from 'supertest';
 import { californiaSite } from '../../test/mock/site.mock';
 import { TestService } from '../../test/test.service';
@@ -25,8 +25,12 @@ export const sensorTests = () => {
     const rsp = await request(app.getHttpServer())
       .get(`/sensors/${californiaSite.sensorId}/data`)
       .query({
-        startDate: moment().subtract(5, 'days').startOf('day').toISOString(),
-        endDate: moment().endOf('day').toISOString(),
+        startDate: DateTime.now()
+          .minus({ days: 5 })
+          .startOf('day')
+          .toJSDate()
+          .toISOString(),
+        endDate: DateTime.now().endOf('day').toJSDate().toISOString(),
         metrics: Metric.TOP_TEMPERATURE,
       });
 
@@ -53,8 +57,12 @@ export const sensorTests = () => {
     const rsp = await request(app.getHttpServer())
       .get(`/sensors/${californiaSite.sensorId}/data`)
       .query({
-        startDate: moment().subtract(5, 'days').startOf('day').toISOString(),
-        endDate: moment().endOf('day').toISOString(),
+        startDate: DateTime.now()
+          .minus({ days: 5 })
+          .startOf('day')
+          .toJSDate()
+          .toISOString(),
+        endDate: DateTime.now().endOf('day').toJSDate().toISOString(),
         metrics: 'invalidMetric',
       });
 

diff --git a/packages/api/src/sites/sites.service.ts b/packages/api/src/sites/sites.service.ts
@@ -8,8 +8,8 @@ import {
 import { InjectRepository } from '@nestjs/typeorm';
 import { DataSource, Repository } from 'typeorm';
 import { omit } from 'lodash';
-import moment from 'moment';
 import Bluebird from 'bluebird';
+import { DateTime } from 'luxon';
 import { Site, SiteStatus } from './sites.entity';
 import { DailyData } from './daily-data.entity';
 import { FilterSiteDto } from './dto/filter-site.dto';
@@ -284,7 +284,10 @@ export class SitesService {
   ): Promise<DailyData[]> {
     await getSite(id, this.sitesRepository);
 
-    if (!moment(start).isValid() || !moment(end).isValid()) {
+    if (
+      (start && !DateTime.fromISO(start).isValid) ||
+      (end && !DateTime.fromISO(end).isValid)
+    ) {
       throw new BadRequestException('Start or end is not a valid date');
     }
 
@@ -455,7 +458,7 @@ export class SitesService {
     id: number,
     excludeSpotterDatesDto: ExcludeSpotterDatesDto,
   ) {
-    const dateFormat = 'MM/DD/YYYY HH:mm';
+    const dateFormat = 'LL/dd/yyyy HH:mm';
     const { startDate, endDate } = excludeSpotterDatesDto;
 
     const site = await getSite(id, this.sitesRepository);
@@ -494,11 +497,11 @@ export class SitesService {
 
       if (alreadyExists) {
         throw new ConflictException(
-          `Exclusion period [${moment(startDate).format(dateFormat)}, ${moment(
-            endDate,
-          ).format(dateFormat)}] already exists for spotter ${
-            source.sensorId
-          }.`,
+          `Exclusion period [${DateTime.fromJSDate(startDate).toFormat(
+            dateFormat,
+          )}, ${DateTime.fromJSDate(endDate).toFormat(
+            dateFormat,
+          )}] already exists for spotter ${source.sensorId}.`,
         );
       }
 

diff --git a/packages/api/src/sites/sites.spec.ts b/packages/api/src/sites/sites.spec.ts
@@ -2,7 +2,7 @@ import request from 'supertest';
 import { INestApplication } from '@nestjs/common';
 import { omit, sortBy } from 'lodash';
 import { DataSource } from 'typeorm';
-import moment from 'moment';
+import { DateTime } from 'luxon';
 import { TestService } from '../../test/test.service';
 import {
   mockBackfillSiteData,
@@ -34,11 +34,23 @@ export const siteTests = () => {
   let siteId: number;
   const firstExclusionPeriod = {
     startDate: null,
-    endDate: moment().subtract(8, 'days').endOf('day').toISOString(),
+    endDate: DateTime.now()
+      .minus({ days: 8 })
+      .endOf('day')
+      .toJSDate()
+      .toISOString(),
   };
   const secondExclusionPeriod = {
-    startDate: moment().subtract(6, 'days').startOf('day').toISOString(),
-    endDate: moment().subtract(4, 'days').endOf('day').toISOString(),
+    startDate: DateTime.now()
+      .minus({ days: 6 })
+      .startOf('day')
+      .toJSDate()
+      .toISOString(),
+    endDate: DateTime.now()
+      .minus({ days: 4 })
+      .endOf('day')
+      .toJSDate()
+      .toISOString(),
   };
   const siteDto = {
     site: {
@@ -117,8 +129,12 @@ export const siteTests = () => {
     const rsp = await request(app.getHttpServer())
       .get(`/sites/${californiaSite.id}/daily_data`)
       .query({
-        start: moment().subtract(5, 'days').startOf('day').toISOString(),
-        end: moment().endOf('day').toISOString(),
+        start: DateTime.now()
+          .minus({ days: 5 })
+          .startOf('day')
+          .toJSDate()
+          .toISOString(),
+        end: DateTime.now().endOf('day').toJSDate().toISOString(),
       });
 
     expect(rsp.status).toBe(200);
@@ -208,8 +224,12 @@ export const siteTests = () => {
       const rsp = await request(app.getHttpServer())
         .get(`/sites/${athensSite.id}/spotter_data`)
         .query({
-          startDate: moment().subtract(9, 'days').startOf('day').toISOString(),
-          endDate: moment().endOf('day').toISOString(),
+          startDate: DateTime.now()
+            .minus({ days: 9 })
+            .startOf('day')
+            .toJSDate()
+            .toISOString(),
+          endDate: DateTime.now().endOf('day').toJSDate().toISOString(),
         });
 
       expect(rsp.status).toBe(200);
@@ -299,8 +319,8 @@ export const siteTests = () => {
       const rsp = await request(app.getHttpServer())
         .get('/sites/0/daily_data')
         .query({
-          start: moment().subtract(1, 'days').toISOString(),
-          end: moment().toISOString(),
+          start: DateTime.now().minus({ days: 1 }).toJSDate().toISOString(),
+          end: DateTime.now().toJSDate().toISOString(),
         });
 
       expect(rsp.status).toBe(404);
@@ -380,8 +400,8 @@ export const siteTests = () => {
       const rsp = await request(app.getHttpServer())
         .post('/sites/0/exclusion_dates')
         .send({
-          startDate: moment().subtract(1, 'days').toISOString(),
-          endDate: moment().toISOString(),
+          startDate: DateTime.now().minus({ days: 1 }).toJSDate().toISOString(),
+          endDate: DateTime.now().toJSDate().toISOString(),
         });
 
       expect(rsp.status).toBe(404);
@@ -392,8 +412,8 @@ export const siteTests = () => {
       const rsp = await request(app.getHttpServer())
         .post(`/sites/${floridaSite.id}/exclusion_dates`)
         .send({
-          startDate: moment().subtract(1, 'days').toISOString(),
-          endDate: moment().toISOString(),
+          startDate: DateTime.now().minus({ days: 1 }).toJSDate().toISOString(),
+          endDate: DateTime.now().toJSDate().toISOString(),
         });
 
       expect(rsp.status).toBe(400);

diff --git a/packages/api/src/time-series/time-series.service.ts b/packages/api/src/time-series/time-series.service.ts
@@ -9,7 +9,6 @@ import {
 import { Repository } from 'typeorm';
 import Bluebird from 'bluebird';
 import type { Response } from 'express';
-import moment from 'moment';
 import {
   BadRequestException,
   HttpException,
@@ -194,8 +193,8 @@ export class TimeSeriesService {
           timeSeriesRepository: this.timeSeriesRepository,
           siteId,
           metrics,
-          start: chunks[i].start.toISO() as string,
-          end: chunks[i].end.toISO() as string,
+          start: chunks[i].start.toJSDate().toISOString(),
+          end: chunks[i].end.toJSDate().toISOString(),
           hourly,
           csv: true,
           order: 'DESC',
@@ -247,9 +246,9 @@ export class TimeSeriesService {
 
       closeSync(fd);
 
-      const fileName = `data_site_${siteId}_${moment(startDate).format(
+      const fileName = `data_site_${siteId}_${minDate.toFormat(
         DATE_FORMAT,
-      )}_${moment(endDate).format(DATE_FORMAT)}.csv`;
+      )}_${maxDate.toFormat(DATE_FORMAT)}.csv`;
 
       const readStream = createReadStream(tempFileName);
 

diff --git a/packages/api/src/time-series/time-series.spec.ts b/packages/api/src/time-series/time-series.spec.ts
@@ -1,10 +1,10 @@
 import request from 'supertest';
 import { INestApplication } from '@nestjs/common';
 import { max, min, union } from 'lodash';
-import moment from 'moment';
 import { join } from 'path';
 import { readFileSync } from 'fs';
 import * as structuredClone from '@ungap/structured-clone';
+import { DateTime } from 'luxon';
 import { TestService } from '../../test/test.service';
 import { athensSite, californiaSite } from '../../test/mock/site.mock';
 import { athensSurveyPointPiraeus } from '../../test/mock/survey-point.mock';
@@ -102,8 +102,14 @@ export const timeSeriesTests = () => {
       )
       .query({
         // Increase the search window to combat precision issues with the dates
-        start: moment(startDate).subtract(1, 'minute').toISOString(),
-        end: moment(endDate).add(1, 'day').toISOString(),
+        start: DateTime.fromISO(startDate)
+          .minus({ minutes: 1 })
+          .toJSDate()
+          .toISOString(),
+        end: DateTime.fromISO(endDate)
+          .plus({ days: 1 })
+          .toJSDate()
+          .toISOString(),
         metrics: hoboMetrics.concat(NOAAMetrics),
         hourly: false,
       });
@@ -129,8 +135,14 @@ export const timeSeriesTests = () => {
       .get(`/time-series/sites/${californiaSite.id}`)
       .query({
         // Increase the search window to combat precision issues with the dates
-        start: moment(startDate).subtract(1, 'minute').toISOString(),
-        end: moment(endDate).add(1, 'day').toISOString(),
+        start: DateTime.fromISO(startDate)
+          .minus({ minutes: 1 })
+          .toJSDate()
+          .toISOString(),
+        end: DateTime.fromISO(endDate)
+          .plus({ days: 1 })
+          .toJSDate()
+          .toISOString(),
         metrics: spotterMetrics.concat(NOAAMetrics),
         hourly: false,
       });

diff --git a/packages/api/src/utils/dates.ts b/packages/api/src/utils/dates.ts
@@ -1,9 +1,9 @@
-import moment from 'moment-timezone';
+import { DateTime } from 'luxon';
 
 export function getStartEndDate(endDate: Date, hours: number = 24) {
-  const endMoment = moment(endDate);
-  const startMoment = endMoment.clone().subtract(hours, 'hours');
-  return [startMoment.format(), endMoment.format()];
+  const endMoment = DateTime.fromJSDate(endDate);
+  const startMoment = endMoment.minus({ hours });
+  return [startMoment.toString(), endMoment.toString()];
 }
 
 // Util function to get the [startDate, endDate] time interval for time series data.

diff --git a/packages/api/src/utils/hindcast-wind-wave.ts b/packages/api/src/utils/hindcast-wind-wave.ts
@@ -2,7 +2,7 @@ import { Logger } from '@nestjs/common';
 import Bluebird from 'bluebird';
 import { Point } from 'geojson';
 import { isNil } from 'lodash';
-import moment from 'moment';
+import { DateTime } from 'luxon';
 import { In, Repository } from 'typeorm';
 import { SourceType } from '../sites/schemas/source-type.enum';
 import { Site } from '../sites/sites.entity';
@@ -160,9 +160,9 @@ export const addWindWaveData = async (
               .values([
                 {
                   site,
-                  timestamp: moment(sofarValue.timestamp)
+                  timestamp: DateTime.fromISO(sofarValue.timestamp)
                     .startOf('minute')
-                    .toDate(),
+                    .toJSDate(),
                   metric,
                   source,
                   value: sofarValue.value,