diff --git a/api/config/custom-environment-variables.json b/api/config/custom-environment-variables.json index 2699a221a..a9b0514db 100644 --- a/api/config/custom-environment-variables.json +++ b/api/config/custom-environment-variables.json @@ -73,9 +73,10 @@ "sendGridApiKey": "SENDGRID_API_KEY" } }, - "carto": { + "eudr": { "apiKey": "CARTO_API_KEY", "baseUrl": "CARTO_BASE_URL", - "credentials": "CARTO_CREDENTIALS" + "credentials": "EUDR_CREDENTIALS", + "dataset": "EUDR_DATASET" } } diff --git a/api/config/default.json b/api/config/default.json index 6069b6d9a..9f83e9a0d 100644 --- a/api/config/default.json +++ b/api/config/default.json @@ -82,9 +82,10 @@ "sendGridApiKey": null } }, - "carto": { + "eudr": { "apiKey": null, "baseUrl": "null", - "credentials": null + "credentials": null, + "dataset": null } } diff --git a/api/config/test.json b/api/config/test.json index 718a8f346..823d75b56 100644 --- a/api/config/test.json +++ b/api/config/test.json @@ -46,5 +46,11 @@ "email": { "sendGridApiKey": "SG.forSomeReasonSendGridApiKeysNeedToStartWithSG." } + }, + "eudr": { + "apiKey": null, + "baseUrl": "null", + "credentials": null, + "dataset": "test_dataset" } } diff --git a/api/src/modules/eudr-alerts/alerts-query-builder/big-query-alerts-query.builder.ts b/api/src/modules/eudr-alerts/alerts-query-builder/big-query-alerts-query.builder.ts new file mode 100644 index 000000000..0c622c9d6 --- /dev/null +++ b/api/src/modules/eudr-alerts/alerts-query-builder/big-query-alerts-query.builder.ts @@ -0,0 +1,118 @@ +import { SelectQueryBuilder } from 'typeorm'; +import { AlertsOutput } from 'modules/eudr-alerts/dto/alerts-output.dto'; +import { GetEUDRAlertsDto } from 'modules/eudr-alerts/dto/get-alerts.dto'; +import { Query } from '@google-cloud/bigquery'; + +export class BigQueryAlertsQueryBuilder { + queryBuilder: SelectQueryBuilder; + dto?: GetEUDRAlertsDto; + + constructor( + queryBuilder: SelectQueryBuilder, + getAlertsDto?: GetEUDRAlertsDto, + ) { + this.queryBuilder = queryBuilder; + this.dto = getAlertsDto; + } + + buildQuery(): Query { + if (this.dto?.supplierIds) { + this.queryBuilder.andWhere('supplierid IN (:...supplierIds)', { + supplierIds: this.dto.supplierIds, + }); + } + if (this.dto?.geoRegionIds) { + this.queryBuilder.andWhere('georegionid IN (:...geoRegionIds)', { + geoRegionIds: this.dto.geoRegionIds, + }); + } + if (this.dto?.alertConfidence) { + this.queryBuilder.andWhere('alertConfidence = :alertConfidence', { + alertConfidence: this.dto.alertConfidence, + }); + } + + if (this.dto?.startYear && this.dto?.endYear) { + this.addYearRange(); + } else if (this.dto?.startYear) { + this.addYearGreaterThanOrEqual(); + } else if (this.dto?.endYear) { + this.addYearLessThanOrEqual(); + } + + if (this.dto?.startAlertDate && this.dto?.endAlertDate) { + this.addAlertDateRange(); + } else if (this.dto?.startAlertDate) { + this.addAlertDateGreaterThanOrEqual(); + } else if (this.dto?.endAlertDate) { + this.addAlertDateLessThanOrEqual(); + } + + this.queryBuilder.limit(this.dto?.limit); + + const [query, params] = this.queryBuilder.getQueryAndParameters(); + + return this.parseToBigQuery(query, params); + } + + addYearRange(): void { + this.queryBuilder.andWhere('year BETWEEN :startYear AND :endYear', { + startYear: this.dto?.startYear, + endYear: this.dto?.endYear, + }); + } + + addYearGreaterThanOrEqual(): void { + this.queryBuilder.andWhere('year >= :startYear', { + startYear: this.dto?.startYear, + }); + } + + addYearLessThanOrEqual(): void { + this.queryBuilder.andWhere('year <= :endYear', { + endYear: this.dto?.endYear, + }); + } + + addAlertDateRange(): void { + this.queryBuilder.andWhere( + 'DATE(alertdate) BETWEEN :startAlertDate AND :endAlertDate', + { + startAlertDate: this.dto?.startAlertDate, + endAlertDate: this.dto?.endAlertDate, + }, + ); + } + + addAlertDateGreaterThanOrEqual(): void { + this.queryBuilder.andWhere('DATE(alertdate) >= DATE(:startAlertDate)', { + startAlertDate: this.dto?.startAlertDate, + }); + } + + addAlertDateLessThanOrEqual(): void { + this.queryBuilder.andWhere('DATE(alertDate) <= :DATE(endAlertDate)', { + endAlertDate: this.dto?.endAlertDate, + }); + } + + parseToBigQuery(query: string, params: any[]): Query { + return { + query: this.removeDoubleQuotesAndReplacePositionalArguments(query), + params, + }; + } + + /** + * @description: BigQuery does not allow double quotes and the positional argument symbol must be a "?". + * So there is a need to replace the way TypeORM handles the positional arguments, with $1, $2, etc. + */ + + private removeDoubleQuotesAndReplacePositionalArguments( + query: string, + ): string { + return query.replace(/\$\d+|"/g, (match: string) => + match === '"' ? '' : '?', + ); + } +} diff --git a/api/src/modules/eudr-alerts/alerts.repository.ts b/api/src/modules/eudr-alerts/alerts.repository.ts index 3dbaaf78f..ca611ebab 100644 --- a/api/src/modules/eudr-alerts/alerts.repository.ts +++ b/api/src/modules/eudr-alerts/alerts.repository.ts @@ -1,65 +1,85 @@ -import { BigQuery } from '@google-cloud/bigquery'; -import { Injectable } from '@nestjs/common'; +import { + BigQuery, + Query, + SimpleQueryRowsResponse, +} from '@google-cloud/bigquery'; +import { + Inject, + Injectable, + Logger, + ServiceUnavailableException, +} from '@nestjs/common'; import { DataSource, SelectQueryBuilder } from 'typeorm'; -import { AlertsOutput } from './dto/alerts-output.dto'; -import { ResourceStream } from '@google-cloud/paginator'; -import { RowMetadata } from '@google-cloud/bigquery/build/src/table'; -import { IEUDRAlertsRepository } from './eudr.repositoty.interface'; -import { AppConfig } from '../../utils/app.config'; +import { AlertsOutput } from 'modules/eudr-alerts/dto/alerts-output.dto'; +import { + EUDRAlertDates, + GetEUDRAlertDatesDto, + IEUDRAlertsRepository, +} from 'modules/eudr-alerts/eudr.repositoty.interface'; +import { GetEUDRAlertsDto } from 'modules/eudr-alerts/dto/get-alerts.dto'; +import { BigQueryAlertsQueryBuilder } from 'modules/eudr-alerts/alerts-query-builder/big-query-alerts-query.builder'; const projectId: string = 'carto-dw-ac-zk2uhih6'; -const limit: number = 1; - @Injectable() export class AlertsRepository implements IEUDRAlertsRepository { + logger: Logger = new Logger(AlertsRepository.name); bigQueryClient: BigQuery; - BASE_DATASET: string = 'cartobq.eudr.dev_mock_data_optimized'; - constructor(private readonly dataSource: DataSource) { - const { credentials } = AppConfig.get('carto'); + constructor( + private readonly dataSource: DataSource, + @Inject('EUDRCredentials') private credentials: string, + @Inject('EUDRDataset') private baseDataset: string, + ) { + // if (!credentials) { + // this.logger.error('BigQuery credentials are missing'); + // throw new ServiceUnavailableException( + // 'EUDR Module not available. Tearing down the application', + // ); + // } this.bigQueryClient = new BigQuery({ - credentials: JSON.parse(credentials), + credentials: JSON.parse(this.credentials), projectId, }); } - select(dto?: any): ResourceStream { - const queryBuilder: SelectQueryBuilder = this.dataSource - .createQueryBuilder() - .select('georegionid', 'geoRegionId') - .addSelect('supplierid', 'supplierId') - .addSelect('geometry', 'geometry') - .where('alertcount >= :alertCount', { alertCount: 2 }) - .andWhere('supplierid IN (:...supplierIds)', { - supplierIds: [ - '4132ab95-8b04-4438-b706-a82651f491bd', - '4132ab95-8b04-4438-b706-a82651f491bd', - '4132ab95-8b04-4438-b706-a82651f491bd', - ], - }); - if (limit) { - queryBuilder.limit(limit); + async getAlerts(dto?: GetEUDRAlertsDto): Promise { + const queryBuilder: SelectQueryBuilder = + this.dataSource.createQueryBuilder(); + // TODO: Make field selection dynamic + queryBuilder.from(this.baseDataset, 'alerts'); + queryBuilder.select('alertdate', 'alertDate'); + queryBuilder.addSelect('alertconfidence', 'alertConfidence'); + queryBuilder.addSelect('year', 'alertYear'); + queryBuilder.addSelect('alertcount', 'alertCount'); + try { + const response: SimpleQueryRowsResponse = await this.bigQueryClient.query( + this.buildQuery(queryBuilder, dto), + ); + if (!response.length || 'error' in response) { + this.logger.error('Error in query', response); + throw new Error(); + } + return response[0]; + } catch (e) { + this.logger.error('Error in query', e); + throw new ServiceUnavailableException( + 'Unable to retrieve EUDR Data. Please contact your administrator.', + ); } - // const [rows] = await this.bigQueryClient.query( - // this.buildQuery(queryBuilder), - // ); - return this.bigQueryClient.createQueryStream(this.buildQuery(queryBuilder)); } - private buildQuery(queryBuilder: SelectQueryBuilder): { - query: string; - params: any[]; - } { - const [query, params] = queryBuilder - .from(this.BASE_DATASET, 'alerts') - .getQueryAndParameters(); - const queryOptions = { - query: query.replace(/\$\d+|"/g, (match: string) => - match === '"' ? '' : '?', - ), - params, - }; - return queryOptions; + getDates(dto: GetEUDRAlertDatesDto): Promise { + return [] as any; + } + + private buildQuery( + queryBuilder: SelectQueryBuilder, + dto?: GetEUDRAlertsDto, + ): Query { + const alertsQueryBuilder: BigQueryAlertsQueryBuilder = + new BigQueryAlertsQueryBuilder(queryBuilder, dto); + + return alertsQueryBuilder.buildQuery(); } } diff --git a/api/src/modules/eudr-alerts/dto/alerts-output.dto.ts b/api/src/modules/eudr-alerts/dto/alerts-output.dto.ts index 78241b087..824e5785a 100644 --- a/api/src/modules/eudr-alerts/dto/alerts-output.dto.ts +++ b/api/src/modules/eudr-alerts/dto/alerts-output.dto.ts @@ -1,11 +1,14 @@ import { GeoJSON } from 'geojson'; -export class AlertsOutput { - geoRegionId: string; - supplierId: string; +export type AlertsOutput = { alertCount: boolean; - geometry: GeoJSON; date: Date; year: number; alertConfidence: 'low' | 'medium' | 'high' | 'very high'; -} +}; + +export type AlertGeometry = { + geometry: { value: string }; +}; + +export type AlertsWithGeom = AlertsOutput & AlertGeometry; diff --git a/api/src/modules/eudr-alerts/dto/get-alerts.dto.ts b/api/src/modules/eudr-alerts/dto/get-alerts.dto.ts index 5448ee413..77f3e295e 100644 --- a/api/src/modules/eudr-alerts/dto/get-alerts.dto.ts +++ b/api/src/modules/eudr-alerts/dto/get-alerts.dto.ts @@ -1,6 +1,48 @@ -export class GetEUDRALertsDto { - supplierId: string; - geoRegionId: string; - geom: any; - year: number; +import { Type } from 'class-transformer'; +import { + IsArray, + IsDate, + IsEnum, + IsInt, + IsNumber, + IsOptional, + IsUUID, +} from 'class-validator'; + +export class GetEUDRAlertsDto { + @IsOptional() + @IsArray() + @IsUUID('4', { each: true }) + supplierIds: string[]; + + @IsOptional() + @IsArray() + @IsUUID('4', { each: true }) + geoRegionIds: string[]; + + @IsOptional() + @IsNumber() + @Type(() => Number) + startYear: number; + + @IsOptional() + @IsNumber() + @Type(() => Number) + endYear: number; + + alertConfidence: 'high' | 'medium' | 'low'; + + @IsOptional() + @IsDate() + @Type(() => Date) + startAlertDate: Date; + + @IsOptional() + @IsDate() + @Type(() => Date) + endAlertDate: Date; + + @IsOptional() + @IsInt() + limit: number = 1000; } diff --git a/api/src/modules/eudr-alerts/eudr.controller.ts b/api/src/modules/eudr-alerts/eudr.controller.ts index d4f44b1ed..d45b8eeaa 100644 --- a/api/src/modules/eudr-alerts/eudr.controller.ts +++ b/api/src/modules/eudr-alerts/eudr.controller.ts @@ -32,10 +32,8 @@ import { } from 'modules/geo-regions/geo-region.entity'; import { JSONAPIQueryParams } from 'decorators/json-api-parameters.decorator'; import { GetEUDRGeoRegions } from 'modules/geo-regions/dto/get-geo-region.dto'; -import { AlertsOutput } from 'modules/eudr-alerts/dto/alerts-output.dto'; import { EudrService } from 'modules/eudr-alerts/eudr.service'; -import { ResourceStream } from '@google-cloud/paginator'; -import { GetEUDRALertsDto } from './dto/get-alerts.dto'; +import { GetEUDRAlertsDto } from 'modules/eudr-alerts/dto/get-alerts.dto'; @Controller('/api/v1/eudr') export class EudrController { @@ -143,13 +141,8 @@ export class EudrController { } @Get('/alerts') - async getAlerts( - @Res() response: Response, - dto: GetEUDRALertsDto, - ): Promise { - const stream: ResourceStream = - this.eudrAlertsService.getAlerts(); - this.streamResponse(response, stream); + async getAlerts(@Query(ValidationPipe) dto: GetEUDRAlertsDto): Promise { + return this.eudrAlertsService.getAlerts(dto); } streamResponse(response: Response, stream: Writable): any { diff --git a/api/src/modules/eudr-alerts/eudr.module.ts b/api/src/modules/eudr-alerts/eudr.module.ts index 9e65a3682..692323e5e 100644 --- a/api/src/modules/eudr-alerts/eudr.module.ts +++ b/api/src/modules/eudr-alerts/eudr.module.ts @@ -7,8 +7,15 @@ import { SuppliersModule } from 'modules/suppliers/suppliers.module'; import { GeoRegionsModule } from 'modules/geo-regions/geo-regions.module'; import { AdminRegionsModule } from 'modules/admin-regions/admin-regions.module'; import { AlertsRepository } from 'modules/eudr-alerts/alerts.repository'; +import { AppConfig } from 'utils/app.config'; -export const IEUDRAlertsRepositoryToken = Symbol('IEUDRAlertsRepository'); +export const IEUDRAlertsRepositoryToken: symbol = Symbol( + 'IEUDRAlertsRepository', +); +export const EUDRDataSetToken: symbol = Symbol('EUDRDataSet'); +export const EUDRCredentialsToken: symbol = Symbol('EUDRCredentials'); + +const { credentials, dataset } = AppConfig.get('eudr'); // TODO: Use token injection and refer to the interface, right now I am having a dependencv issue @Module({ @@ -21,10 +28,10 @@ export const IEUDRAlertsRepositoryToken = Symbol('IEUDRAlertsRepository'); ], providers: [ EudrService, - AlertsRepository, - { provide: IEUDRAlertsRepositoryToken, useClass: AlertsRepository }, + { provide: 'IEUDRAlertsRepository', useClass: AlertsRepository }, + { provide: 'EUDRDataset', useValue: dataset }, + { provide: 'EUDRCredentials', useValue: credentials }, ], controllers: [EudrController], - // exports: [IEUDRAlertsRepositoryToken], }) export class EudrModule {} diff --git a/api/src/modules/eudr-alerts/eudr.repositoty.interface.ts b/api/src/modules/eudr-alerts/eudr.repositoty.interface.ts index 12f3de272..dfd1c9662 100644 --- a/api/src/modules/eudr-alerts/eudr.repositoty.interface.ts +++ b/api/src/modules/eudr-alerts/eudr.repositoty.interface.ts @@ -1,3 +1,19 @@ +import { GetEUDRAlertsDto } from 'modules/eudr-alerts/dto/get-alerts.dto'; +import { AlertsOutput } from 'modules/eudr-alerts/dto/alerts-output.dto'; + +export class GetEUDRAlertDatesDto { + startDate: string; + endDate: string; +} + +export type EUDRAlertDates = { + alertDate: { + value: Date | string; + }; +}; + export interface IEUDRAlertsRepository { - select(): any; + getAlerts(dto?: GetEUDRAlertsDto): Promise; + + getDates(dto: GetEUDRAlertDatesDto): Promise; } diff --git a/api/src/modules/eudr-alerts/eudr.service.ts b/api/src/modules/eudr-alerts/eudr.service.ts index 01fb0686d..3d0a232f7 100644 --- a/api/src/modules/eudr-alerts/eudr.service.ts +++ b/api/src/modules/eudr-alerts/eudr.service.ts @@ -1,13 +1,16 @@ import { Inject, Injectable } from '@nestjs/common'; -import { ResourceStream } from '@google-cloud/paginator'; -import { RowMetadata } from '@google-cloud/bigquery/build/src/table'; -import { AlertsRepository } from './alerts.repository'; +import { GetEUDRAlertsDto } from 'modules/eudr-alerts/dto/get-alerts.dto'; +import { AlertsOutput } from 'modules/eudr-alerts/dto/alerts-output.dto'; +import { IEUDRAlertsRepository } from 'modules/eudr-alerts/eudr.repositoty.interface'; @Injectable() export class EudrService { - constructor(private readonly alertsRepository: AlertsRepository) {} + constructor( + @Inject('IEUDRAlertsRepository') + private readonly alertsRepository: IEUDRAlertsRepository, + ) {} - getAlerts(): ResourceStream { - return this.alertsRepository.select(); + async getAlerts(dto: GetEUDRAlertsDto): Promise { + return this.alertsRepository.getAlerts(dto); } } diff --git a/api/test/utils/service-mocks.ts b/api/test/utils/service-mocks.ts index 98626b7b8..1b393ae9b 100644 --- a/api/test/utils/service-mocks.ts +++ b/api/test/utils/service-mocks.ts @@ -3,7 +3,11 @@ import { SendMailDTO, } from '../../src/modules/notifications/email/email.service.interface'; import { Logger } from '@nestjs/common'; -import { IEUDRAlertsRepository } from 'modules/eudr-alerts/eudr.repositoty.interface'; +import { + EUDRAlertDates, + GetEUDRAlertDatesDto, + IEUDRAlertsRepository, +} from 'modules/eudr-alerts/eudr.repositoty.interface'; export class MockEmailService implements IEmailService { logger: Logger = new Logger(MockEmailService.name); @@ -17,10 +21,16 @@ export class MockEmailService implements IEmailService { export class MockAlertRepository implements IEUDRAlertsRepository { logger: Logger = new Logger(MockAlertRepository.name); - select(): any { + getAlerts(): any { this.logger.warn(`Alert Repository Mock called... `); return new Promise((resolve) => { resolve([]); }); } + + getDates(dto: GetEUDRAlertDatesDto): Promise { + return new Promise((resolve) => { + resolve([]); + }); + } }