From e4bf4becf001f065f2681ba17890b2c1cff5a723 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 17 Feb 2025 10:58:12 +0000 Subject: [PATCH] Delete application deadline reminder lambda This has now been reproduced in the pre-award repo as a scheduled job: https://github.com/communitiesuk/funding-service-pre-award/pull/243. Note: we have not yet reproduced the 'incomplete applications' email, which this lambda also did. There is an outstanding product question around whether we bother to reproduce this or not, as it hasn't been used since it was refactored to use the events table many months ago. --- .github/workflows/update-preaward-env.yml | 5 - .pre-commit-config.yaml | 2 +- .../addons/application-deadline-reminder.yml | 136 --------- apps/pre-award/lambdas/.gitignore | 3 - apps/pre-award/lambdas/README.md | 107 ------- .../application_reminder.py | 176 ------------ .../application-deadline-reminder/config.py | 47 ---- .../application-deadline-reminder/data.py | 102 ------- .../helpers/aws_extended_client.py | 155 ---------- .../incomplete_application.py | 265 ------------------ .../lambda_function.py | 35 --- 11 files changed, 1 insertion(+), 1032 deletions(-) delete mode 100644 apps/pre-award/copilot/environments/addons/application-deadline-reminder.yml delete mode 100644 apps/pre-award/lambdas/.gitignore delete mode 100644 apps/pre-award/lambdas/README.md delete mode 100644 apps/pre-award/lambdas/application-deadline-reminder/application_reminder.py delete mode 100644 apps/pre-award/lambdas/application-deadline-reminder/config.py delete mode 100644 apps/pre-award/lambdas/application-deadline-reminder/data.py delete mode 100644 apps/pre-award/lambdas/application-deadline-reminder/helpers/aws_extended_client.py delete mode 100644 apps/pre-award/lambdas/application-deadline-reminder/incomplete_application.py delete mode 100644 apps/pre-award/lambdas/application-deadline-reminder/lambda_function.py diff --git a/.github/workflows/update-preaward-env.yml b/.github/workflows/update-preaward-env.yml index 4e51f206..426d27e5 100644 --- a/.github/workflows/update-preaward-env.yml +++ b/.github/workflows/update-preaward-env.yml @@ -46,11 +46,6 @@ jobs: - name: Git clone the repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - name: Install requests library - working-directory: 'apps/pre-award/lambdas/application-deadline-reminder' - run: | - python -m pip install requests --target . - - name: Setup Copilot uses: ./.github/actions/copilot_setup with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5080ad1b..0e686e71 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -files: ^(?:application-deadline-reminder/|scripts/) +files: ^(?:scripts/) repos: - repo: https://github.com/ambv/black rev: 25.1.0 diff --git a/apps/pre-award/copilot/environments/addons/application-deadline-reminder.yml b/apps/pre-award/copilot/environments/addons/application-deadline-reminder.yml deleted file mode 100644 index 3a93bc36..00000000 --- a/apps/pre-award/copilot/environments/addons/application-deadline-reminder.yml +++ /dev/null @@ -1,136 +0,0 @@ -Parameters: - App: - Type: String - Description: Your application's name. - Env: - Type: String - Description: The environment name your service, job, or workflow is being deployed to. - -Resources: - ApplicationDeadlineReminderRole: - Type: AWS::IAM::Role - Properties: - Policies: - - PolicyName: !Sub ApplicationDeadlineReminderPolicy${Env} - PolicyDocument: - Version: 2012-10-17 - Statement: - - Action: - - 'logs:CreateLogGroup' - - 'logs:CreateLogStream' - - 'logs:PutLogEvents' - Resource: - - 'arn:aws:logs:*:*:*' - Effect: Allow - - Action: - - 'ec2:DescribeNetworkInterfaces' - - 'ec2:CreateNetworkInterface' - - 'ec2:DeleteNetworkInterface' - - 'ec2:DescribeInstances' - - 'ec2:AttachNetworkInterface' - Resource: - - '*' - Effect: Allow - - Action: - - 'sqs:SendMessage' - Resource: - - !ImportValue - 'Fn::Sub': "${App}-${Env}-NotificationQueueArn" - - !ImportValue - 'Fn::Sub': "${App}-${Env}-NotificationDeadLetterQueueARN" - Effect: Allow - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Action: - - 'sts:AssumeRole' - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - - edgelambda.amazonaws.com - - ApplicationDeadlineReminderLambdaFunction: - Type: AWS::Lambda::Function - Properties: - Code: lambdas/application-deadline-reminder/ - Handler: sentry_sdk.integrations.init_serverless_sdk.sentry_lambda_handler - Timeout: 900 - MemorySize: 512 - Role: !GetAtt ApplicationDeadlineReminderRole.Arn - Runtime: python3.11 - Environment: - Variables: - ACCOUNTS_ENDPOINT: /accounts - ACCOUNT_STORE_API_HOST: !Sub http://fsd-pre-award.${Env}.pre-award.local:8080/account - APPLICATIONS_ENDPOINT: /applications - APPLICATION_ENDPOINT: /applications/{application_id} - APPLICATION_REMINDER_STATUS: /funds/{round_id}/application_reminder_status?status=true - APPLICATION_STORE_API_HOST: !Sub http://fsd-pre-award.${Env}.pre-award.local:8080/application - FUND_ENDPOINT: /funds/{fund_id} - FUNDS_ENDPOINT: /funds - FUND_ROUNDS_ENDPOINT: /funds/{fund_id}/rounds - FUND_EVENTS_ENDPOINT: /funds/{fund_id}/rounds/{round_id}/events - FUND_EVENT_ENDPOINT: /funds/{fund_id}/rounds/{round_id}/event/{event_id} - FUND_STORE_API_HOST: !Sub http://fsd-pre-award.${Env}.pre-award.local:8080/fund - NOTIFICATION_SERVICE_API_HOST: !Sub http://fsd-notification.${Env}.pre-award.local:8080 - NOTIFY_TEMPLATE_APPLICATION_DEADLINE_REMINDER: APPLICATION_DEADLINE_REMINDER - NOTIFY_TEMPLATE_INCOMPLETE_APPLICATION: INCOMPLETE_APPLICATION_RECORDS - AWS_MSG_BUCKET_NAME: !ImportValue - 'Fn::Sub': "${App}-${Env}-MessageBucket" - AWS_SQS_NOTIF_APP_PRIMARY_QUEUE_URL: !ImportValue - 'Fn::Sub': "${App}-${Env}-NotificationQueueURL" - AWS_SQS_NOTIF_APP_SECONDARY_QUEUE_URL: !ImportValue - 'Fn::Sub': "${App}-${Env}-NotificationDeadLetterQueueURL" - SENTRY_DSN: https://80c7f65b54f0eff535777a66b375adf0@o1432034.ingest.us.sentry.io/4508324370317312 - SENTRY_ENVIRONMENT: !Sub ${Env} - SENTRY_INITIAL_HANDLER: lambda_function.lambda_handler - SENTRY_TRACES_SAMPLE_RATE: 1 - VpcConfig: - SecurityGroupIds: - - Fn::ImportValue: !Sub ${App}-${Env}-InternalLoadBalancerSecurityGroup - - Fn::ImportValue: !Sub ${App}-${Env}-EnvironmentSecurityGroup - SubnetIds: - !Split - - ',' - - Fn::ImportValue: !Sub ${App}-${Env}-PrivateSubnets - Layers: - - arn:aws:lambda:eu-west-2:943013980633:layer:SentryPythonServerlessSDK:138 - - - ApplicationDeadlineReminderLambdaVersion: - Type: AWS::Lambda::Version - Properties: - Description: Creation a version of the Application Deadline Reminder Lambda - FunctionName: !Ref ApplicationDeadlineReminderLambdaFunction - - ApplicationDeadlineReminderScheduledRule: - Type: AWS::Events::Rule - Properties: - Description: "Application Deadline Reminder Scheduled Rule" - ScheduleExpression: "cron(30 09 * * ? *)" - State: "ENABLED" - Targets: - - - Arn: !GetAtt ApplicationDeadlineReminderLambdaFunction.Arn - Id: "TargetApplicationDeadlineReminderFunctionV1" - - ApplicationDeadlineReminderPermissionForEventsToInvokeLambda: - Type: AWS::Lambda::Permission - Properties: - FunctionName: !Ref ApplicationDeadlineReminderLambdaFunction - Action: "lambda:InvokeFunction" - Principal: "events.amazonaws.com" - SourceArn: !GetAtt ApplicationDeadlineReminderScheduledRule.Arn - -Outputs: - ApplicationDeadlineReminderLambdaArn: - Description: The ARN of the Application Deadline Reminder Lambda - Value: !GetAtt ApplicationDeadlineReminderLambdaFunction.Arn - Export: - Name: !Join [ ':', [ !Ref 'AWS::StackName', 'ApplicationDeadlineReminderLambdaArn']] - ApplicationDeadlineReminderLambdaVersion: - Description: The version of the Application Deadline Reminder Lambda - Value: !GetAtt ApplicationDeadlineReminderLambdaVersion.Version - Export: - Name: !Join [ ':', [ !Ref 'AWS::StackName', 'ApplicationDeadlineReminderLambdaVersion']] diff --git a/apps/pre-award/lambdas/.gitignore b/apps/pre-award/lambdas/.gitignore deleted file mode 100644 index cbdbf540..00000000 --- a/apps/pre-award/lambdas/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# Environments -.env -.venv \ No newline at end of file diff --git a/apps/pre-award/lambdas/README.md b/apps/pre-award/lambdas/README.md deleted file mode 100644 index 2003cf85..00000000 --- a/apps/pre-award/lambdas/README.md +++ /dev/null @@ -1,107 +0,0 @@ -# funding-service-design-post-award-lambda - -## application-deadline-reminder - -## Prerequisites - -- Python 3.10.x or higher - -## Getting started -* Install Python 3.10 -(Instructions assume python 3 is installed on your PATH as `python` but may be `python3` on OSX) - -Check your python version starts with 3.11 i.e. -``` -python --version - -Python 3.10 -``` - -### Create the virtual environment - -From directory application-deadline-reminder: - -``` -python -m venv .venv -``` - -...or if using PyCharm **when importing project**, create venv and set local python interpreter to use it: - -In Pycharm: -1) File -> New Project... : -2) Select 'New environment using' -> Virtualenv -3) Set 'location' to project folder application-deadline-reminder -4) Base interpreter should be set to global Python install. - -### Enter the virtual environment - -...either macOS using bash/zsh: - - source .venv/bin/activate - -...or if on Windows using Command Prompt: - - .venv\Scripts\activate.bat - -...or if using Pycharm, if venv not set up during project import: - -1) settings -> project -> python interpreter -> add interpreter -> add local interpreter -2) **If not previously created** -> Environment -> New -> select path to top level of project -3) **If previously created** -> Environment -> Existing -> Select path to local venv/scripts/python.exe -4) Do not inherit global site packages - -To check if Pycharm is running local interpreter (rather than global): - - pip -V #check the resultant path points to virtual env folder in project - -Add pip tools: -``` -python -m pip install pip-tools -``` - -### Setup pre-commit checks - -* [Install pre-commit locally](https://pre-commit.com/#installation) -* Pre-commit hooks can either be installed using pip `pip install pre-commit` or homebrew (for Mac users)`brew install pre-commit` -* From your checkout directory run `pre-commit install` to set up the git hook scripts - -### Run app -To run the front-end app locally, you can run the following: - -01) Add python-lambda-local to run lambda locally - - ``` - pip install python-lambda-local - ``` - -02) To run locally in as a bash script - - ``` - python-lambda-local -f lambda_handler lambda_function.py event.json - ``` - - ...or if using Pycharm: In Edit configurations - - - script : - - ```/funding-service-design-workflows/apps/pre-award/lambdas/application-deadline-reminder/.venv/bin/python-lambda-local``` here check is the virtual environment folder is same as here - - script parameters : - - ```--timeout 3000 -f lambda_handler lambda_function.py event.json``` - - Working Directory - ```/funding-service-design-workflows/apps/pre-award/lambdas/application-deadline-reminder``` - - ---event.json--- - ```{}``` - - environment variables for local env: - - ```ACCOUNT_STORE_API_HOST=http://localhost:3003;ACCOUNTS_ENDPOINT=/accounts;APPLICATION_ENDPOINT=/applications /{application_id};APPLICATION_REMINDER_STATUS=/funds/{round_id}/application_reminder_status?status = true;APPLICATION_STORE_API_HOST=http://localhost:3002;APPLICATIONS_ENDPOINT=/applications;FUND_ENDPOINT=/funds/{fund_id};FUND_EVENT_ENDPOINT=/funds/{fund_id}/rounds/{round_id}/event/{event_id};FUND_EVENTS_ENDPOINT=/funds/{fund_id}/rounds/{round_id}/events;FUND_ROUNDS_ENDPOINT=/funds/{fund_id}/rounds;FUND_STORE_API_HOST=http://localhost:3001;FUNDS_ENDPOINT=/funds;NOTIFICATION_SERVICE_API_HOST=http://localhost:3006;NOTIFY_TEMPLATE_APPLICATION_DEADLINE_REMINDER=APPLICATION_DEADLINE_REMINDER;NOTIFY_TEMPLATE_INCOMPLETE_APPLICATION=INCOMPLETE_APPLICATION_RECORDS;PYTHONUNBUFFERED=1;``` - - path to env - ```/funding-service-design-docker-runner/.awslocal.env``` - -More information please refer : https://pypi.org/project/python-lambda-local/ diff --git a/apps/pre-award/lambdas/application-deadline-reminder/application_reminder.py b/apps/pre-award/lambdas/application-deadline-reminder/application_reminder.py deleted file mode 100644 index f5cbf6c1..00000000 --- a/apps/pre-award/lambdas/application-deadline-reminder/application_reminder.py +++ /dev/null @@ -1,176 +0,0 @@ -import logging -from datetime import datetime - -import requests -from config import Config -from data import get_account, get_data, send_notification -from dateutil import tz -from helpers.aws_extended_client import SQSExtendedClient - -# Logging to output to CloudWatch Logs -logging.getLogger("lambda_runtime").setLevel(logging.INFO) -logging.getLogger().setLevel(logging.DEBUG) - - -def application_deadline_reminder( - sqs_extended_client: SQSExtendedClient, fund_details: [] -): - logging.info("Application deadline reminder task is now running!") - uk_timezone = tz.gettz("Europe/London") - current_datetime = datetime.now(uk_timezone).replace(tzinfo=None) - - for fund_detail in fund_details: - fund_id = fund_detail["fund"]["id"] - fund_name = fund_detail["fund"]["name"] - - for round_detail in fund_detail["fund_round"]: - round_deadline_str = round_detail.get("deadline") - reminder_date_str = round_detail.get("reminder_date") - round_id = round_detail.get("id") - round_name = round_detail.get("title") - contact_email = round_detail.get("contact_email") - - if not reminder_date_str: - logging.info( - f"No reminder is set for the round {fund_name} {round_name}" - ) - continue - - application_reminder_sent = round_detail.get("application_reminder_sent") - - round_deadline = datetime.strptime(round_deadline_str, "%Y-%m-%dT%H:%M:%S") - - reminder_date = datetime.strptime(reminder_date_str, "%Y-%m-%dT%H:%M:%S") - - if ( - not application_reminder_sent - and reminder_date < current_datetime < round_deadline - ): - not_submitted_applications = _get_not_submitted_applications( - fund_id, round_id - ) - - all_applications = [] - for application in not_submitted_applications.json(): - application["round_name"] = round_name - application["fund_name"] = fund_name - application["contact_help_email"] = contact_email - account = get_account(account_id=application.get("account_id")) - - application["account_email"] = account.get("email_address") - application["deadline_date"] = round_deadline_str - all_applications.append({"application": application}) - - logging.info(f"Total unsubmitted applications: {len(all_applications)}") - # Only one email per account_email - unique_email_account = _get_unique_email_accounts(all_applications) - - logging.info( - f"Total unique email accounts: {len(unique_email_account)}" - ) - unique_application_email_addresses = list(unique_email_account.values()) - - if len(unique_application_email_addresses) > 0: - for count, application in enumerate( - unique_application_email_addresses, start=1 - ): - email = application["application"]["account_email"] - logging.info( - f"Sending reminder {count} of {len(unique_email_account)}" - f" for {fund_name} {round_name}" - f" to {email}" - ) - - _send_message_and_update_funds( - application, - count, - email, - fund_name, - round_id, - round_name, - sqs_extended_client, - unique_application_email_addresses, - ) - - else: - logging.info( - "Currently, there are no non-submitted applications" - f" for {fund_name} {round_name}" - ) - else: - if ( - current_datetime < reminder_date < round_deadline - and not application_reminder_sent - ): - days_to_reminder = reminder_date - current_datetime - logging.info( - "Application deadline reminder is due in " - f" {days_to_reminder.days} days" - f" for {fund_name} {round_name}." - ) - continue - continue - - -def _send_message_and_update_funds( - application, - count, - email, - fund_name, - round_id, - round_name, - sqs_extended_client, - unique_application_email_addresses, -): - try: - message_id = send_notification( - template_type=Config.NOTIFY_TEMPLATE_APPLICATION_DEADLINE_REMINDER, - to_email=email, - content=application, - application_id=application["application"]["id"], - sqs_extended_client=sqs_extended_client, - ) - - if message_id is not None and len(unique_application_email_addresses) == count: - logging.info( - "The application reminder has been" - " sent successfully for" - f" {fund_name} {round_name}" - ) - - app_reminder = ( - Config.FUND_STORE_API_HOST - + Config.APPLICATION_REMINDER_STATUS.format(round_id=round_id) - ) - response = requests.put(app_reminder) - if response.status_code == 200: - logging.info( - "The application_reminder_sent has been" - " set to True for" - f" {fund_name} {round_name}" - ) - - except Exception as e: - logging.info( - "There was a problem sending application(s)" - f" for {fund_name} {round_name}" - f" Error: {e}" - ) - - -def _get_unique_email_accounts(all_applications): - unique_email_account = {} - for application in all_applications: - unique_email_account[application["application"]["account_email"]] = application - return unique_email_account - - -def _get_not_submitted_applications(fund_id, round_id): - status = { - "status_only": ["IN_PROGRESS", "NOT_STARTED", "COMPLETED"], - "fund_id": fund_id, - "round_id": round_id, - } - endpoint = Config.APPLICATION_STORE_API_HOST + Config.APPLICATIONS_ENDPOINT - not_submitted_applications = requests.get(endpoint, params=status) - return not_submitted_applications diff --git a/apps/pre-award/lambdas/application-deadline-reminder/config.py b/apps/pre-award/lambdas/application-deadline-reminder/config.py deleted file mode 100644 index 7567183d..00000000 --- a/apps/pre-award/lambdas/application-deadline-reminder/config.py +++ /dev/null @@ -1,47 +0,0 @@ -from os import environ - - -class Config: - # fund store - FUND_STORE_API_HOST = environ.get("FUND_STORE_API_HOST") - FUNDS_ENDPOINT = environ.get("FUNDS_ENDPOINT") - FUND_ENDPOINT = environ.get("FUND_ENDPOINT") - FUND_ROUNDS_ENDPOINT = environ.get("FUND_ROUNDS_ENDPOINT") - FUND_EVENTS_ENDPOINT = environ.get("FUND_EVENTS_ENDPOINT") - FUND_EVENT_ENDPOINT = environ.get("FUND_EVENT_ENDPOINT") - - # account store - ACCOUNT_STORE_API_HOST = environ.get("ACCOUNT_STORE_API_HOST") - ACCOUNTS_ENDPOINT = environ.get("ACCOUNTS_ENDPOINT") - - # application store - APPLICATION_STORE_API_HOST = environ.get("APPLICATION_STORE_API_HOST") - APPLICATION_REMINDER_STATUS = environ.get("APPLICATION_REMINDER_STATUS") - APPLICATIONS_ENDPOINT = environ.get("APPLICATIONS_ENDPOINT") - APPLICATION_ENDPOINT = environ.get("APPLICATION_ENDPOINT") - - # notification service - NOTIFICATION_SERVICE_API_HOST = environ.get("NOTIFICATION_SERVICE_API_HOST") - NOTIFY_TEMPLATE_APPLICATION_DEADLINE_REMINDER = environ.get( - "NOTIFY_TEMPLATE_APPLICATION_DEADLINE_REMINDER" - ) - NOTIFY_TEMPLATE_INCOMPLETE_APPLICATION = environ.get( - "NOTIFY_TEMPLATE_INCOMPLETE_APPLICATION" - ) - - # --------------- - # AWS Overall Config # TODO after the refactoring test related configs will be moved - # --------------- - AWS_REGION = environ.get("AWS_REGION") - AWS_ENDPOINT_OVERRIDE = environ.get("AWS_ENDPOINT_OVERRIDE") - - # --------------- - # S3 Config - # --------------- - AWS_MSG_BUCKET_NAME = environ.get("AWS_MSG_BUCKET_NAME") - # --------------- - # SQS Config - # --------------- - AWS_SQS_NOTIF_APP_PRIMARY_QUEUE_URL = environ.get( - "AWS_SQS_NOTIF_APP_PRIMARY_QUEUE_URL" - ) diff --git a/apps/pre-award/lambdas/application-deadline-reminder/data.py b/apps/pre-award/lambdas/application-deadline-reminder/data.py deleted file mode 100644 index 1793f161..00000000 --- a/apps/pre-award/lambdas/application-deadline-reminder/data.py +++ /dev/null @@ -1,102 +0,0 @@ -import json -import logging -from typing import Optional -from urllib.parse import urlencode -from uuid import uuid4 - -import requests -from config import Config -from helpers.aws_extended_client import SQSExtendedClient - -# Logging to output to CloudWatch Logs -logging.getLogger("lambda_runtime").setLevel(logging.INFO) -logging.getLogger().setLevel(logging.DEBUG) - - -def get_data(endpoint, params: Optional[dict] = None): - query_string = "" - if params: - params = {k: v for k, v in params.items() if v is not None} - query_string = urlencode(params) - - endpoint = endpoint + "?" + query_string - response = requests.get(endpoint) - - if response.status_code == 200: - data = response.json() - return data - - logging.error( - "There was a problem retrieving response from" - f" {endpoint}. Status code: {response.status_code}" - ) - return None - - -def get_data_safe(endpoint, params: Optional[dict] = None): - try: - response = requests.get(endpoint, params=params) - response.raise_for_status() - return response.json() - except requests.HTTPError: - logging.info( - "No data retrieved from" f" {endpoint}. Status code {response.status_code}" - ) - except Exception as e: - logging.error("Unable to retrieve data from" f" {endpoint}. Exception {str(e)}") - - return None - - -def send_notification( - template_type: str, - to_email: str, - content, - application_id: str, - sqs_extended_client: SQSExtendedClient, -) -> str: - try: - json_payload = { - "type": template_type, - "to": to_email, - "content": content, - } - application_attributes = { - "application_id": { - "StringValue": application_id, - "DataType": "String", - }, - "S3Key": { - "StringValue": "notification/incomplete", - "DataType": "String", - }, - } - message_id = sqs_extended_client.submit_single_message( - Config.AWS_SQS_NOTIF_APP_PRIMARY_QUEUE_URL, - message=json.dumps(json_payload), - message_group_id="notification", - message_deduplication_id=str(uuid4()), # ensures message uniqueness - extra_attributes=application_attributes, - ) - logging.info( - f"Successfully added the message to queue for " - f"application id {application_id} and message id [{message_id}]." - ) - return str(message_id) - except Exception as e: - logging.error( - f"Unable to send message to sqs for {application_id}. Exception {str(e)}" - ) - raise e - - -def get_account(email: Optional[str] = None, account_id: Optional[str] = None): - if email is account_id is None: - raise TypeError("Requires an email address or account_id") - - url = Config.ACCOUNT_STORE_API_HOST + Config.ACCOUNTS_ENDPOINT - params = {"email_address": email, "account_id": account_id} - response = get_data(url, params) - - if response and "account_id" in response: - return response diff --git a/apps/pre-award/lambdas/application-deadline-reminder/helpers/aws_extended_client.py b/apps/pre-award/lambdas/application-deadline-reminder/helpers/aws_extended_client.py deleted file mode 100644 index 8cf2392f..00000000 --- a/apps/pre-award/lambdas/application-deadline-reminder/helpers/aws_extended_client.py +++ /dev/null @@ -1,155 +0,0 @@ -import json -import logging -from datetime import datetime -from uuid import uuid4 - -import boto3 - -logging.getLogger("lambda_runtime").setLevel(logging.INFO) -logging.getLogger().setLevel(logging.DEBUG) - -S3_KEY_ATTRIBUTE_NAME = "S3Key" -MAX_ALLOWED_ATTRIBUTES = 10 - 1 # 10 for SQS and 1 reserved attribute -DEFAULT_MESSAGE_SIZE_THRESHOLD = 262144 -RESERVED_ATTRIBUTE_NAME = "ExtendedPayloadSize" -MESSAGE_POINTER_CLASS = "software.amazon.payloadoffloading.PayloadS3Pointer" - - -class SQSExtendedClient: - def __init__( - self, - region_name="eu-west-2", - endpoint_url=None, - large_payload_support=None, - always_through_s3=None, - **kwargs, - ): - self.large_payload_support = large_payload_support - self.always_through_s3 = always_through_s3 - self.sqs_client = boto3.client( - "sqs", - region_name=region_name, - endpoint_url=endpoint_url, - **kwargs, - ) - self.s3_client = boto3.client( - "s3", - region_name=region_name, - endpoint_url=endpoint_url, - ) - logging.info("Create SQS and S3 client.") - - def submit_single_message( - self, - queue_url, - message, - extra_attributes: dict = None, - message_group_id=None, - message_deduplication_id=None, - ): - sqs_message_attributes = { - "message_created_at": { - "StringValue": str(datetime.now()), - "DataType": "String", - }, - } - message_body, message_attributes = self._store_message_in_s3( - message, sqs_message_attributes, extra_attributes - ) - # add extra message attributes (if provided) - if extra_attributes: - for key, value in extra_attributes.items(): - message_attributes[key] = value - - response = self.sqs_client.send_message( - QueueUrl=queue_url, - MessageBody=message_body, - MessageAttributes=message_attributes, - MessageGroupId=message_group_id, - MessageDeduplicationId=message_deduplication_id, - ) - # Check if the delete operation succeeded, if not raise an error? - status_code = response["ResponseMetadata"]["HTTPStatusCode"] - if status_code != 200: - logging.error( - f"submit_single_message failed with status code {status_code}." - ) - raise Exception( - f"submit_single_message failed with status code {status_code}." - ) - message_id = response["MessageId"] - logging.info(f"Called SQS and submitted the message and id [{message_id}]") - return message_id - - def _store_message_in_s3( - self, message_body: str, message_attributes: dict, extra_attributes: dict - ) -> (str, dict): - """ - Responsible for storing a message payload in a S3 Bucket - :message_body: A UTF-8 encoded version of the message body - :message_attributes: A dictionary consisting of message attributes - :extra_attributes: A dictionary consisting of message attributes - Each message attribute consists of the name (key) along with a - type and value of the message body. The following types are supported - for message attributes: StringValue, BinaryValue and DataType. - """ - if len(message_body) == 0: - # Message cannot be empty - logging.error("messageBody cannot be null or empty.") - - if self.large_payload_support and self.always_through_s3: - # Check message attributes for ExtendedClient related constraints - encoded_body = message_body.encode("utf-8") - - # Modifying the message attributes for storing it in the Queue - attribute_value = { - "DataType": "Number", - "StringValue": str(len(encoded_body)), - } - message_attributes[RESERVED_ATTRIBUTE_NAME] = attribute_value - - # S3 Key should either be a constant or be a random uuid4 string. - s3_key = SQSExtendedClient._get_s3_key(message_attributes, extra_attributes) - - # Adding the object into the bucket - response = self.s3_client.put_object( - Body=encoded_body, Bucket=self.large_payload_support, Key=s3_key - ) - # Check if the delete operation succeeded, if not raise an error? - status_code = response["ResponseMetadata"]["HTTPStatusCode"] - if status_code != 200: - logging.error( - f"submit_single_message failed with status code {status_code}." - ) - raise Exception( - f"submit_single_message failed with status code {status_code}." - ) - # Modifying the message body for storing it in the Queue - message_body = json.dumps( - [ - MESSAGE_POINTER_CLASS, - {"s3BucketName": self.large_payload_support, "s3Key": s3_key}, - ] - ) - return message_body, message_attributes - - @staticmethod - def _get_s3_key(message_attributes: dict, extra_attributes: dict) -> str: - """ - Responsible for checking if the S3 Key exists in the - message_attributes - :message_attributes: A dictionary consisting of message attributes - :extra_attributes: A dictionary consisting of message attributes - Each message attribute consists of the name (key) along with a - type and value of the message body. The following types are supported - for message attributes: StringValue, BinaryValue and DataType. - """ - if S3_KEY_ATTRIBUTE_NAME in message_attributes: - return message_attributes[S3_KEY_ATTRIBUTE_NAME]["StringValue"] - elif extra_attributes and S3_KEY_ATTRIBUTE_NAME in extra_attributes: - return ( - extra_attributes[S3_KEY_ATTRIBUTE_NAME]["StringValue"] - + "/" - + str(uuid4()) - ) - return str(uuid4()) diff --git a/apps/pre-award/lambdas/application-deadline-reminder/incomplete_application.py b/apps/pre-award/lambdas/application-deadline-reminder/incomplete_application.py deleted file mode 100644 index 8c90b697..00000000 --- a/apps/pre-award/lambdas/application-deadline-reminder/incomplete_application.py +++ /dev/null @@ -1,265 +0,0 @@ -import logging -from datetime import datetime - -import requests -from config import Config -from data import get_data_safe, send_notification -from dateutil import tz -from helpers.aws_extended_client import SQSExtendedClient - -logging.getLogger("lambda_runtime").setLevel(logging.INFO) -logging.getLogger().setLevel(logging.DEBUG) - - -def process_events(sqs_extended_client: SQSExtendedClient, fund_details: []): - """ - Pulls events from the fund store and checks if they need processing. If so, the relevant processor - will be called (determined by event type). If the processing was successful, the event is updated - and marked as processed. - - Return: - True if the function ran without issue. - """ - logging.info("Running event check") - uk_timezone = tz.gettz("Europe/London") - current_datetime = datetime.now(uk_timezone).replace(tzinfo=None) - - # Iterate over rounds and events. Note that failure to retrieve rounds / events should be non blocking so that - # the rest of the rounds / events can still be processed - for fund_detail in fund_details: - - fund_id = fund_detail["fund"]["id"] - rounds = fund_detail["fund_round"] - if not rounds: - continue - fund_name = fund_detail["fund"]["name"] - - for fund_round in rounds: - - round_id = fund_round["id"] - round_name = fund_round["title"] - round_contact_email = fund_round.get("contact_email") - events = _get_events(fund_id, round_id) - - if not events: - continue - - for event in events: - - event_type = event["type"] - event_activation_date = _get_formatted_activation_date(event) - event_id = event["id"] - event_processed = event["processed"] - - # Check if event needs to be processed and past the activation date - if event_processed or current_datetime < event_activation_date: - continue - - try: - event_processor = { - "SEND_INCOMPLETE_APPLICATIONS": _send_incomplete_applications_after_deadline - }[event_type] - except KeyError: - logging.error( - f"Incompatible event type found {event_type} for event {event_id}" - ) - continue - - # Process the event and mark it as processed. - result = event_processor( - fund_id=fund_id, - fund_name=fund_name, - round_id=round_id, - round_name=round_name, - round_contact_email=round_contact_email, - sqs_extended_client=sqs_extended_client, - ) - if not result: - continue - - try: - _update_events_for_fund(event_id, fund_id, round_id) - except Exception as e: - logging.error( - f"Failed to mark event {event_id}" - f" as processed for {fund_name} {round_name}" - f" an error {e}" - ) - - logging.info( - f"Event {event_id} has been" - " marked as processed for" - f" {fund_name} {round_name}" - ) - return True - - -def _get_formatted_activation_date(event): - event_activation_date = datetime.strptime( - event.get("activation_date"), "%Y-%m-%dT%H:%M:%S" - ) - return event_activation_date - - -def _update_events_for_fund(event_id, fund_id, round_id): - response = requests.put( - Config.FUND_STORE_API_HOST - + Config.FUND_EVENT_ENDPOINT.format( - fund_id=fund_id, - round_id=round_id, - event_id=event_id, - ), - params={"processed": True}, - ) - response.raise_for_status() - - -def _get_events(fund_id, round_id): - events = get_data_safe( - Config.FUND_STORE_API_HOST - + Config.FUND_EVENTS_ENDPOINT.format(fund_id=fund_id, round_id=round_id) - ) - return events - - -def _get_unsubmitted_applications(fund_id, round_id, fund_name, round_name): - try: - search_params = { - "status_only": ["NOT_STARTED", "IN_PROGRESS", "COMPLETED"], - "fund_id": fund_id, - "round_id": round_id, - } - response = requests.get( - Config.APPLICATION_STORE_API_HOST + Config.APPLICATIONS_ENDPOINT, - params=search_params, - ) - response.raise_for_status() - return response.json() - except Exception as e: - logging.error( - f"Unable to retrieve incomplete applications for fund {fund_name} and round {round_name}. Exception {str(e)}" - ) - return None - - -def _send_incomplete_applications_after_deadline( - fund_id, - fund_name, - round_id, - round_name, - round_contact_email, - sqs_extended_client: SQSExtendedClient, -): - """ - Retrieves a list of unsubmitted applications for the given fund and round. Then use the - notification service to email the account for each application. - - Args: - - fund_id (str): The ID of the fund. - - fund_name (str): The name of the fund. - - round_id (str): The ID of the funding round. - - round_name (str): The name of the round. - - round_contact_email (str): The email to contact for the round - - Return: - True if there were zero unsubmitted applications, or if at least one account was emailed. False otherwise. - """ - unsubmitted_applications = _get_unsubmitted_applications( - fund_id, round_id, fund_name, round_name - ) - if unsubmitted_applications is None: - return False - - logging.info( - f"Found {len(unsubmitted_applications)} unsubmitted applications for fund {fund_name} and round {round_name}" - ) - unsuccessful_notifications = 0 - - # Get all required information for applications - for application in unsubmitted_applications: - try: - account_info, application_to_send = _get_application_details( - application, fund_name, round_contact_email, round_name - ) - except Exception as e: - logging.error( - f"Unable to retrieve application or account information for application {application['id']}." - f" Exception {str(e)}" - ) - unsuccessful_notifications += 1 - continue - - unsuccessful_notifications = _send_messages( - account_info, - application, - application_to_send, - round_contact_email, - sqs_extended_client, - unsuccessful_notifications, - ) - - num_unsubmitted_applications = len(unsubmitted_applications) - logging.info( - f"Sent {num_unsubmitted_applications - unsuccessful_notifications} out of {num_unsubmitted_applications} incomplete application emails" - ) - - return ( - num_unsubmitted_applications > unsuccessful_notifications - if num_unsubmitted_applications > 0 - else True - ) - - -def _send_messages( - account_info, - application, - application_to_send, - round_contact_email, - sqs_extended_client, - unsuccessful_notifications, -): - # Send an email to the account associated with the application. - try: - message_id = send_notification( - template_type=Config.NOTIFY_TEMPLATE_INCOMPLETE_APPLICATION, - to_email=account_info["email_address"], - content={ - "application": application_to_send, - "contact_help_email": round_contact_email, - }, - application_id=application["id"], - sqs_extended_client=sqs_extended_client, - ) - logging.info(f"Successfully added the message into queue [{message_id}]") - except Exception as e: - logging.error( - f"Unable to send an incomplete application email for application {application['id']}. Exception {str(e)}" - ) - unsuccessful_notifications += 1 - return unsuccessful_notifications - - -def _get_application_details(application, fund_name, round_contact_email, round_name): - application_info_request = requests.get( - Config.APPLICATION_STORE_API_HOST - + Config.APPLICATION_ENDPOINT.format( - application_id=application["id"] + "?with_questions_file=true" - ) - ) - application_info_request.raise_for_status() - application_info = application_info_request.json() - account_info_request = requests.get( - Config.ACCOUNT_STORE_API_HOST + Config.ACCOUNTS_ENDPOINT, - params={"account_id": application["account_id"]}, - ) - account_info_request.raise_for_status() - account_info = account_info_request.json() - application_to_send = { - **application, - "fund_name": fund_name, - "questions_file": application_info["questions_file"], - "round_name": round_name, - "account_email": account_info["email_address"], - "contact_help_email": round_contact_email, - } - return account_info, application_to_send diff --git a/apps/pre-award/lambdas/application-deadline-reminder/lambda_function.py b/apps/pre-award/lambdas/application-deadline-reminder/lambda_function.py deleted file mode 100644 index a7c33cd8..00000000 --- a/apps/pre-award/lambdas/application-deadline-reminder/lambda_function.py +++ /dev/null @@ -1,35 +0,0 @@ -from application_reminder import application_deadline_reminder -from config import Config -from data import get_data -from helpers.aws_extended_client import SQSExtendedClient -from incomplete_application import process_events - - -def lambda_handler(event, context): - sqs_extended_client = SQSExtendedClient( - region_name=Config.AWS_REGION, - endpoint_url=Config.AWS_ENDPOINT_OVERRIDE, - large_payload_support=Config.AWS_MSG_BUCKET_NAME, - always_through_s3=True, - ) - - fund_details = [] - funds = get_data(Config.FUND_STORE_API_HOST + Config.FUNDS_ENDPOINT) - for fund in funds: - round_info = _get_round_details(fund["id"]) - fund_details.append({"fund": fund, "fund_round": round_info}) - - application_deadline_reminder(sqs_extended_client, fund_details) - result = process_events(sqs_extended_client, fund_details) - - return { - "statusCode": 200, - "body": result, - } - - -def _get_round_details(fund_id): - round_info = get_data( - Config.FUND_STORE_API_HOST + Config.FUND_ROUNDS_ENDPOINT.format(fund_id=fund_id) - ) - return round_info