Endpoint to retrieve filtered EUDR Alerts

Vizzuality · Mar 4, 2024 · 2e48f45 · 2e48f45
1 parent b2c4e89
commit 2e48f45
Show file tree

Hide file tree

Showing 6 changed files with 195 additions and 57 deletions.
diff --git a/api/src/modules/eudr-alerts/alerts-query-builder/big-query-alerts-query.builder.ts b/api/src/modules/eudr-alerts/alerts-query-builder/big-query-alerts-query.builder.ts
@@ -0,0 +1,118 @@
+import { SelectQueryBuilder } from 'typeorm';
+import { AlertsOutput } from '../dto/alerts-output.dto';
+import { GetEUDRALertsDto } from '../dto/get-alerts.dto';
+import { Query } from '@google-cloud/bigquery';
+
+export class BigQueryAlertsQueryBuilder {
+  queryBuilder: SelectQueryBuilder<AlertsOutput>;
+  dto?: GetEUDRALertsDto;
+
+  constructor(
+    queryBuilder: SelectQueryBuilder<AlertsOutput>,
+    getAlertsDto?: GetEUDRALertsDto,
+  ) {
+    this.queryBuilder = queryBuilder;
+    this.dto = getAlertsDto;
+  }
+
+  buildQuery(): Query {
+    if (this.dto?.supplierIds) {
+      this.queryBuilder.andWhere('supplierid IN (:...supplierIds)', {
+        supplierIds: this.dto.supplierIds,
+      });
+    }
+    if (this.dto?.geoRegionIds) {
+      this.queryBuilder.andWhere('georegionid IN (:...geoRegionIds)', {
+        geoRegionIds: this.dto.geoRegionIds,
+      });
+    }
+    if (this.dto?.alertConfidence) {
+      this.queryBuilder.andWhere('alertConfidence = :alertConfidence', {
+        alertConfidence: this.dto.alertConfidence,
+      });
+    }
+
+    if (this.dto?.startYear && this.dto?.endYear) {
+      this.addYearRange();
+    } else if (this.dto?.startYear) {
+      this.addYearGreaterThanOrEqual();
+    } else if (this.dto?.endYear) {
+      this.addYearLessThanOrEqual();
+    }
+
+    if (this.dto?.startAlertDate && this.dto?.endAlertDate) {
+      this.addAlertDateRange();
+    } else if (this.dto?.startAlertDate) {
+      this.addAlertDateGreaterThanOrEqual();
+    } else if (this.dto?.endAlertDate) {
+      this.addAlertDateLessThanOrEqual();
+    }
+
+    this.queryBuilder.limit(this.dto?.limit);
+
+    const [query, params] = this.queryBuilder.getQueryAndParameters();
+
+    return this.parseToBigQuery(query, params);
+  }
+
+  addYearRange(): void {
+    this.queryBuilder.andWhere('year BETWEEN :startYear AND :endYear', {
+      startYear: this.dto?.startYear,
+      endYear: this.dto?.endYear,
+    });
+  }
+
+  addYearGreaterThanOrEqual(): void {
+    this.queryBuilder.andWhere('year >= :startYear', {
+      startYear: this.dto?.startYear,
+    });
+  }
+
+  addYearLessThanOrEqual(): void {
+    this.queryBuilder.andWhere('year <= :endYear', {
+      endYear: this.dto?.endYear,
+    });
+  }
+
+  addAlertDateRange(): void {
+    this.queryBuilder.andWhere(
+      'alertDate BETWEEN :startAlertDate AND :endAlertDate',
+      {
+        startAlertDate: this.dto?.startAlertDate,
+        endAlertDate: this.dto?.endAlertDate,
+      },
+    );
+  }
+
+  addAlertDateGreaterThanOrEqual(): void {
+    this.queryBuilder.andWhere('alertDate >= :startAlertDate', {
+      startAlertDate: this.dto?.startAlertDate,
+    });
+  }
+
+  addAlertDateLessThanOrEqual(): void {
+    this.queryBuilder.andWhere('alertDate <= :endAlertDate', {
+      endAlertDate: this.dto?.endAlertDate,
+    });
+  }
+
+  parseToBigQuery(query: string, params: any[]): Query {
+    return {
+      query: this.removeDoubleQuotesAndReplacePositionalArguments(query),
+      params,
+    };
+  }
+
+  /**
+   * @description: BigQuery does not allow double quotes and the positional argument symbol must be a "?".
+   * So there is a need to replace the way TypeORM handles the positional arguments, with $1, $2, etc.
+   */
+
+  private removeDoubleQuotesAndReplacePositionalArguments(
+    query: string,
+  ): string {
+    return query.replace(/\$\d+|"/g, (match: string) =>
+      match === '"' ? '' : '?',
+    );
+  }
+}
diff --git a/api/src/modules/eudr-alerts/alerts.repository.ts b/api/src/modules/eudr-alerts/alerts.repository.ts
@@ -1,16 +1,15 @@
-import { BigQuery } from '@google-cloud/bigquery';
+import { BigQuery, Query, QueryRowsResponse } from '@google-cloud/bigquery';
 import { Injectable } from '@nestjs/common';
 import { DataSource, SelectQueryBuilder } from 'typeorm';
 import { AlertsOutput } from './dto/alerts-output.dto';
-import { ResourceStream } from '@google-cloud/paginator';
 import { RowMetadata } from '@google-cloud/bigquery/build/src/table';
 import { IEUDRAlertsRepository } from './eudr.repositoty.interface';
 import { AppConfig } from '../../utils/app.config';
+import { GetEUDRALertsDto } from './dto/get-alerts.dto';
+import { BigQueryAlertsQueryBuilder } from './alerts-query-builder/big-query-alerts-query.builder';
 
 const projectId: string = 'carto-dw-ac-zk2uhih6';
 
-const limit: number = 1;
-
 @Injectable()
 export class AlertsRepository implements IEUDRAlertsRepository {
   bigQueryClient: BigQuery;
@@ -24,42 +23,25 @@ export class AlertsRepository implements IEUDRAlertsRepository {
     });
   }
 
-  select(dto?: any): ResourceStream<RowMetadata> {
-    const queryBuilder: SelectQueryBuilder<AlertsOutput> = this.dataSource
-      .createQueryBuilder()
-      .select('georegionid', 'geoRegionId')
-      .addSelect('supplierid', 'supplierId')
-      .addSelect('geometry', 'geometry')
-      .where('alertcount >= :alertCount', { alertCount: 2 })
-      .andWhere('supplierid IN (:...supplierIds)', {
-        supplierIds: [
-          '4132ab95-8b04-4438-b706-a82651f491bd',
-          '4132ab95-8b04-4438-b706-a82651f491bd',
-          '4132ab95-8b04-4438-b706-a82651f491bd',
-        ],
-      });
-    if (limit) {
-      queryBuilder.limit(limit);
-    }
-    // const [rows] = await this.bigQueryClient.query(
-    //   this.buildQuery(queryBuilder),
-    // );
-    return this.bigQueryClient.createQueryStream(this.buildQuery(queryBuilder));
+  async select(dto?: GetEUDRALertsDto): Promise<RowMetadata> {
+    const queryBuilder: SelectQueryBuilder<AlertsOutput> =
+      this.dataSource.createQueryBuilder();
+    // TODO: Make field selection dynamic
+    queryBuilder.from(this.BASE_DATASET, 'alerts');
+    queryBuilder.select('alertdate', 'alertDate');
+    queryBuilder.addSelect('alertconfidence', 'alertConfidence');
+    queryBuilder.addSelect('year', 'alertYear');
+    queryBuilder.addSelect('alertcount', 'alertCount');
+    return this.bigQueryClient.query(this.buildQuery(queryBuilder, dto));
   }
 
-  private buildQuery(queryBuilder: SelectQueryBuilder<AlertsOutput>): {
-    query: string;
-    params: any[];
-  } {
-    const [query, params] = queryBuilder
-      .from(this.BASE_DATASET, 'alerts')
-      .getQueryAndParameters();
-    const queryOptions = {
-      query: query.replace(/\$\d+|"/g, (match: string) =>
-        match === '"' ? '' : '?',
-      ),
-      params,
-    };
-    return queryOptions;
+  private buildQuery(
+    queryBuilder: SelectQueryBuilder<AlertsOutput>,
+    dto?: GetEUDRALertsDto,
+  ): Query {
+    const alertsQueryBuilder: BigQueryAlertsQueryBuilder =
+      new BigQueryAlertsQueryBuilder(queryBuilder, dto);
+
+    return alertsQueryBuilder.buildQuery();
   }
 }
diff --git a/api/src/modules/eudr-alerts/dto/alerts-output.dto.ts b/api/src/modules/eudr-alerts/dto/alerts-output.dto.ts
@@ -1,11 +1,14 @@
 import { GeoJSON } from 'geojson';
 
-export class AlertsOutput {
-  geoRegionId: string;
-  supplierId: string;
+export type AlertsOutput = {
   alertCount: boolean;
-  geometry: GeoJSON;
   date: Date;
   year: number;
   alertConfidence: 'low' | 'medium' | 'high' | 'very high';
-}
+};
+
+export type AlertGeometry = {
+  geometry: { value: string };
+};
+
+export type AlertsWithGeom = AlertsOutput & AlertGeometry;
diff --git a/api/src/modules/eudr-alerts/dto/get-alerts.dto.ts b/api/src/modules/eudr-alerts/dto/get-alerts.dto.ts
@@ -1,6 +1,43 @@
+import {
+  IsArray,
+  IsDate,
+  IsEnum,
+  IsInt,
+  IsNumber,
+  IsOptional,
+  IsUUID,
+} from 'class-validator';
+
 export class GetEUDRALertsDto {
-  supplierId: string;
-  geoRegionId: string;
-  geom: any;
-  year: number;
+  @IsOptional()
+  @IsArray()
+  @IsUUID('4', { each: true })
+  supplierIds: string[];
+
+  @IsOptional()
+  @IsArray()
+  @IsUUID('4', { each: true })
+  geoRegionIds: string[];
+
+  @IsOptional()
+  @IsNumber()
+  startYear: number;
+
+  @IsOptional()
+  @IsNumber()
+  endYear: number;
+
+  alertConfidence: 'high' | 'medium' | 'low';
+
+  @IsOptional()
+  @IsDate()
+  startAlertDate: Date;
+
+  @IsOptional()
+  @IsDate()
+  endAlertDate: Date;
+
+  @IsOptional()
+  @IsInt()
+  limit: number = 1000;
 }
diff --git a/api/src/modules/eudr-alerts/eudr.controller.ts b/api/src/modules/eudr-alerts/eudr.controller.ts
@@ -143,13 +143,8 @@ export class EudrController {
   }
 
   @Get('/alerts')
-  async getAlerts(
-    @Res() response: Response,
-    dto: GetEUDRALertsDto,
-  ): Promise<any> {
-    const stream: ResourceStream<AlertsOutput> =
-      this.eudrAlertsService.getAlerts();
-    this.streamResponse(response, stream);
+  async getAlerts(@Query(ValidationPipe) dto: GetEUDRALertsDto): Promise<any> {
+    return this.eudrAlertsService.getAlerts(dto);
   }
 
   streamResponse(response: Response, stream: Writable): any {

diff --git a/api/src/modules/eudr-alerts/eudr.service.ts b/api/src/modules/eudr-alerts/eudr.service.ts
@@ -2,12 +2,15 @@ import { Inject, Injectable } from '@nestjs/common';
 import { ResourceStream } from '@google-cloud/paginator';
 import { RowMetadata } from '@google-cloud/bigquery/build/src/table';
 import { AlertsRepository } from './alerts.repository';
+import { GetEUDRALertsDto } from './dto/get-alerts.dto';
 
 @Injectable()
 export class EudrService {
   constructor(private readonly alertsRepository: AlertsRepository) {}
 
-  getAlerts(): ResourceStream<RowMetadata> {
-    return this.alertsRepository.select();
+  async getAlerts(dto: GetEUDRALertsDto): Promise<RowMetadata> {
+    const [rows] = await this.alertsRepository.select(dto);
+    console.warn('DATA LENGTH:', rows.length);
+    return rows;
   }
 }