Add water quality data to site model and add site filter

.github/workflows/build_api.yml

@@ -47,9 +47,9 @@ jobs:
         with:
           path: |
             **/node_modules
-          key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }}
+          key: ${{ runner.os }}-yarn-api-${{ hashFiles('yarn.lock') }}
           restore-keys: |
-            ${{ runner.os }}-yarn-
+            ${{ runner.os }}-yarn-api-
 
       - name: Install dependencies if needed.
         id: install

packages/api/src/sites/sites.entity.ts

@@ -167,6 +167,8 @@ export class Site {
 
   maskedSpotterApiToken?: string;
 
+  waterQualitySources?: string[];
+
   @Expose()
   get applied(): boolean {
     return !!this.siteApplication?.permitRequirements;

packages/api/src/sites/sites.service.ts

@@ -25,6 +25,7 @@ import {
   filterMetricDataByDate,
   getConflictingExclusionDates,
   hasHoboDataSubQuery,
+  getWaterQualityDataSubQuery,
   getLatestData,
   getSite,
   createSite,
@@ -199,11 +200,16 @@ export class SitesService {
 
     const hasHoboDataSet = await hasHoboDataSubQuery(this.sourceRepository);
 
+    const waterQualityDataSet = await getWaterQualityDataSubQuery(
+      this.latestDataRepository,
+    );
+
     return res.map((site) => ({
       ...site,
       applied: site.applied,
       collectionData: mappedSiteData[site.id],
       hasHobo: hasHoboDataSet.has(site.id),
+      waterQualitySources: waterQualityDataSet.get(site.id),
     }));
   }
 

packages/api/src/utils/site.utils.ts

@@ -10,7 +10,7 @@ import {
   NotFoundException,
 } from '@nestjs/common';
 import { ObjectLiteral, Repository } from 'typeorm';
-import { mapValues, some } from 'lodash';
+import { groupBy, mapValues, some } from 'lodash';
 import geoTz from 'geo-tz';
 import { Region } from '../regions/regions.entity';
 import { ExclusionDates } from '../sites/exclusion-dates.entity';
@@ -23,6 +23,7 @@ import { LatestData } from '../time-series/latest-data.entity';
 import { SiteSurveyPoint } from '../site-survey-points/site-survey-points.entity';
 import { getHistoricalMonthlyMeans, getMMM } from './temperature';
 import { HistoricalMonthlyMean } from '../sites/historical-monthly-mean.entity';
+import { Metric } from '../time-series/metrics.enum';
 
 const googleMapsClient = new Client({});
 const logger = new Logger('Site Utils');
@@ -278,6 +279,52 @@ export const hasHoboDataSubQuery = async (
   return hasHoboDataSet;
 };
 
+export const getWaterQualityDataSubQuery = async (
+  latestDataRepository: Repository<LatestData>,
+): Promise<Map<number, string[]>> => {
+  const latestData: LatestData[] = await latestDataRepository
+    .createQueryBuilder('water_quality_data')
+    .select('site_id', 'siteId')
+    .addSelect('metric')
+    .addSelect('source')
+    .where(`source in ('${SourceType.HUI}', '${SourceType.SONDE}')`)
+    .getRawMany();
+
+  const sondeMetrics = [
+    Metric.ODO_CONCENTRATION,
+    Metric.CHOLOROPHYLL_CONCENTRATION,
+    Metric.PH,
+    Metric.SALINITY,
+    Metric.TURBIDITY,
+  ];
+
+  const waterQualityDataSet = new Map<number, string[]>();
+
+  Object.entries(groupBy(latestData, (o) => o.siteId)).forEach(
+    ([siteId, data]) => {
+      let sondeMetricsCount = 0;
+      const id = Number(siteId);
+      waterQualityDataSet.set(id, []);
+      data.forEach((siteData) => {
+        if (siteData.source === 'hui') {
+          // eslint-disable-next-line fp/no-mutating-methods
+          waterQualityDataSet.get(id)!.push('hui');
+        }
+        if (sondeMetrics.includes(siteData.metric)) {
+          // eslint-disable-next-line fp/no-mutation
+          sondeMetricsCount += 1;
+          if (sondeMetricsCount >= 3) {
+            // eslint-disable-next-line fp/no-mutating-methods
+            waterQualityDataSet.get(id)!.push('sonde');
+          }
+        }
+      });
+    },
+  );
+
+  return waterQualityDataSet;
+};
+
 export const getLatestData = async (
   site: Site,
   latestDataRepository: Repository<LatestData>,

packages/website/src/common/SiteDetails/WaterSampling/index.tsx

@@ -78,7 +78,11 @@ function WaterSamplingCard({ siteId, source }: WaterSamplingCardProps) {
       Object.entries(timeSeriesData || {})
         .map(([key, val]) => {
           const values = val
-            .find((x) => x.type === source)
+            .find(
+              (x) =>
+                // hui is specific type of sonde, look for hui as well when looking for sonde
+                x.type === source || (source === 'sonde' && x.type === 'hui'),
+            )
             ?.data.map((x) => x.value);
           if (!values) return [undefined, undefined];
           return [key, getMeanCalculationFunction(source)(values)];

packages/website/src/common/SiteDetails/WaterSampling/utils.ts

@@ -176,8 +176,12 @@ export async function getCardData(
     );
 
     const uploads =
-      uploadHistory.filter((x) => x.dataUpload.sensorTypes.includes(source)) ||
-      [];
+      uploadHistory.filter(
+        (x) =>
+          x.dataUpload.sensorTypes.includes(source) ||
+          // hui is specific type of sonde, look for hui as well when looking for sonde
+          (source === 'sonde' && x.dataUpload.sensorTypes.includes('hui')),
+      ) || [];
     if (uploads.length < 1) {
       return {};
     }
@@ -196,10 +200,13 @@ export async function getCardData(
           return currMin < min ? currMin : min;
         }, new Date().toISOString());
 
-        const maxDate = inLastYear.reduce((max, curr) => {
-          const currMax = curr.maxDate || curr.dataUpload.maxDate;
-          return currMax > max ? currMax : max;
-        }, new Date(0).toISOString());
+        const maxDate =
+          inLastYear.length > 0
+            ? inLastYear.reduce((max, curr) => {
+                const currMax = curr.maxDate || curr.dataUpload.maxDate;
+                return currMax > max ? currMax : max;
+              }, new Date(0).toISOString())
+            : new Date().toISOString();
 
         const [data] = await timeSeriesRequest({
           siteId,
@@ -209,7 +216,7 @@ export async function getCardData(
           hourly: true,
         });
 
-        const pointId = inLastYear[0].surveyPoint;
+        const pointId = inLastYear[0]?.surveyPoint;
         const samePoint =
           pointId !== null
             ? inLastYear.reduce(

packages/website/src/common/SiteDetails/index.tsx

@@ -51,8 +51,9 @@ import LoadingSkeleton from '../LoadingSkeleton';
 import playIcon from '../../assets/play-icon.svg';
 import { TemperatureChange } from './TemperatureChange';
 
+/**  Show only the last year of HUI data, should match with {@link getCardData} */
 const acceptHUIInterval = Interval.fromDateTimes(
-  DateTime.now().minus({ years: 2 }),
+  DateTime.now().minus({ years: 1 }),
   DateTime.now(),
 );
 
@@ -158,7 +159,11 @@ const SiteDetails = ({
         MINIMUM_SONDE_METRICS_TO_SHOW_CARD;
 
       const hasHUI =
-        latestData.some((x) => x.source === 'hui') ||
+        latestData.some(
+          (x) =>
+            x.source === 'hui' &&
+            acceptHUIInterval.contains(DateTime.fromISO(x.timestamp)),
+        ) ||
         sourceWithinDataRangeInterval(
           acceptHUIInterval,
           'hui',

packages/website/src/helpers/siteUtils.ts

@@ -152,6 +152,8 @@ export const sitesFilterFn = (
       return hasDeployedSpotter(s);
     case 'HOBO loggers':
       return s?.hasHobo;
+    case 'Water quality':
+      return s?.waterQualitySources?.length;
     case 'Reef Check':
       return !!s?.reefCheckSite;
     default:

packages/website/src/routes/HomeMap/SiteTable/__snapshots__/index.test.tsx.snap

@@ -146,6 +146,18 @@ exports[`SiteTable should render with given state from Redux store 1`] = `
               HOBO loggers
             </mock-typography>
           </mock-menuitem>
+          <mock-menuitem
+            aria-selected="false"
+            data-value="Water quality"
+            role="option"
+            selected="false"
+          >
+            <mock-typography
+              style="color: black;"
+            >
+              Water quality
+            </mock-typography>
+          </mock-menuitem>
           <mock-menuitem
             aria-selected="false"
             data-value="Reef Check"

packages/website/src/store/Sites/types.ts

@@ -288,6 +288,7 @@ export interface Site {
   contactInformation?: string;
   maskedSpotterApiToken?: string;
   iframe?: string | null;
+  waterQualitySources?: string[];
   reefCheckSurveys: ReefCheckSurvey[];
   reefCheckSite: ReefCheckSite | null;
 }
@@ -459,5 +460,6 @@ export const siteOptions = [
   'Live streams',
   '3D Models',
   'HOBO loggers',
+  'Water quality',
   'Reef Check',
 ] as const;