diff --git a/.github/workflows/update-preaward-env.yml b/.github/workflows/update-preaward-env.yml index 4e51f206..426d27e5 100644 --- a/.github/workflows/update-preaward-env.yml +++ b/.github/workflows/update-preaward-env.yml @@ -46,11 +46,6 @@ jobs: - name: Git clone the repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - name: Install requests library - working-directory: 'apps/pre-award/lambdas/application-deadline-reminder' - run: | - python -m pip install requests --target . - - name: Setup Copilot uses: ./.github/actions/copilot_setup with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5080ad1b..0e686e71 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -files: ^(?:application-deadline-reminder/|scripts/) +files: ^(?:scripts/) repos: - repo: https://github.com/ambv/black rev: 25.1.0 diff --git a/apps/pre-award/copilot/environments/addons/application-deadline-reminder.yml b/apps/pre-award/copilot/environments/addons/application-deadline-reminder.yml deleted file mode 100644 index 3a93bc36..00000000 --- a/apps/pre-award/copilot/environments/addons/application-deadline-reminder.yml +++ /dev/null @@ -1,136 +0,0 @@ -Parameters: - App: - Type: String - Description: Your application's name. - Env: - Type: String - Description: The environment name your service, job, or workflow is being deployed to. - -Resources: - ApplicationDeadlineReminderRole: - Type: AWS::IAM::Role - Properties: - Policies: - - PolicyName: !Sub ApplicationDeadlineReminderPolicy${Env} - PolicyDocument: - Version: 2012-10-17 - Statement: - - Action: - - 'logs:CreateLogGroup' - - 'logs:CreateLogStream' - - 'logs:PutLogEvents' - Resource: - - 'arn:aws:logs:*:*:*' - Effect: Allow - - Action: - - 'ec2:DescribeNetworkInterfaces' - - 'ec2:CreateNetworkInterface' - - 'ec2:DeleteNetworkInterface' - - 'ec2:DescribeInstances' - - 'ec2:AttachNetworkInterface' - Resource: - - '*' - Effect: Allow - - Action: - - 'sqs:SendMessage' - Resource: - - !ImportValue - 'Fn::Sub': "${App}-${Env}-NotificationQueueArn" - - !ImportValue - 'Fn::Sub': "${App}-${Env}-NotificationDeadLetterQueueARN" - Effect: Allow - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Action: - - 'sts:AssumeRole' - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - - edgelambda.amazonaws.com - - ApplicationDeadlineReminderLambdaFunction: - Type: AWS::Lambda::Function - Properties: - Code: lambdas/application-deadline-reminder/ - Handler: sentry_sdk.integrations.init_serverless_sdk.sentry_lambda_handler - Timeout: 900 - MemorySize: 512 - Role: !GetAtt ApplicationDeadlineReminderRole.Arn - Runtime: python3.11 - Environment: - Variables: - ACCOUNTS_ENDPOINT: /accounts - ACCOUNT_STORE_API_HOST: !Sub http://fsd-pre-award.${Env}.pre-award.local:8080/account - APPLICATIONS_ENDPOINT: /applications - APPLICATION_ENDPOINT: /applications/{application_id} - APPLICATION_REMINDER_STATUS: /funds/{round_id}/application_reminder_status?status=true - APPLICATION_STORE_API_HOST: !Sub http://fsd-pre-award.${Env}.pre-award.local:8080/application - FUND_ENDPOINT: /funds/{fund_id} - FUNDS_ENDPOINT: /funds - FUND_ROUNDS_ENDPOINT: /funds/{fund_id}/rounds - FUND_EVENTS_ENDPOINT: /funds/{fund_id}/rounds/{round_id}/events - FUND_EVENT_ENDPOINT: /funds/{fund_id}/rounds/{round_id}/event/{event_id} - FUND_STORE_API_HOST: !Sub http://fsd-pre-award.${Env}.pre-award.local:8080/fund - NOTIFICATION_SERVICE_API_HOST: !Sub http://fsd-notification.${Env}.pre-award.local:8080 - NOTIFY_TEMPLATE_APPLICATION_DEADLINE_REMINDER: APPLICATION_DEADLINE_REMINDER - NOTIFY_TEMPLATE_INCOMPLETE_APPLICATION: INCOMPLETE_APPLICATION_RECORDS - AWS_MSG_BUCKET_NAME: !ImportValue - 'Fn::Sub': "${App}-${Env}-MessageBucket" - AWS_SQS_NOTIF_APP_PRIMARY_QUEUE_URL: !ImportValue - 'Fn::Sub': "${App}-${Env}-NotificationQueueURL" - AWS_SQS_NOTIF_APP_SECONDARY_QUEUE_URL: !ImportValue - 'Fn::Sub': "${App}-${Env}-NotificationDeadLetterQueueURL" - SENTRY_DSN: https://80c7f65b54f0eff535777a66b375adf0@o1432034.ingest.us.sentry.io/4508324370317312 - SENTRY_ENVIRONMENT: !Sub ${Env} - SENTRY_INITIAL_HANDLER: lambda_function.lambda_handler - SENTRY_TRACES_SAMPLE_RATE: 1 - VpcConfig: - SecurityGroupIds: - - Fn::ImportValue: !Sub ${App}-${Env}-InternalLoadBalancerSecurityGroup - - Fn::ImportValue: !Sub ${App}-${Env}-EnvironmentSecurityGroup - SubnetIds: - !Split - - ',' - - Fn::ImportValue: !Sub ${App}-${Env}-PrivateSubnets - Layers: - - arn:aws:lambda:eu-west-2:943013980633:layer:SentryPythonServerlessSDK:138 - - - ApplicationDeadlineReminderLambdaVersion: - Type: AWS::Lambda::Version - Properties: - Description: Creation a version of the Application Deadline Reminder Lambda - FunctionName: !Ref ApplicationDeadlineReminderLambdaFunction - - ApplicationDeadlineReminderScheduledRule: - Type: AWS::Events::Rule - Properties: - Description: "Application Deadline Reminder Scheduled Rule" - ScheduleExpression: "cron(30 09 * * ? *)" - State: "ENABLED" - Targets: - - - Arn: !GetAtt ApplicationDeadlineReminderLambdaFunction.Arn - Id: "TargetApplicationDeadlineReminderFunctionV1" - - ApplicationDeadlineReminderPermissionForEventsToInvokeLambda: - Type: AWS::Lambda::Permission - Properties: - FunctionName: !Ref ApplicationDeadlineReminderLambdaFunction - Action: "lambda:InvokeFunction" - Principal: "events.amazonaws.com" - SourceArn: !GetAtt ApplicationDeadlineReminderScheduledRule.Arn - -Outputs: - ApplicationDeadlineReminderLambdaArn: - Description: The ARN of the Application Deadline Reminder Lambda - Value: !GetAtt ApplicationDeadlineReminderLambdaFunction.Arn - Export: - Name: !Join [ ':', [ !Ref 'AWS::StackName', 'ApplicationDeadlineReminderLambdaArn']] - ApplicationDeadlineReminderLambdaVersion: - Description: The version of the Application Deadline Reminder Lambda - Value: !GetAtt ApplicationDeadlineReminderLambdaVersion.Version - Export: - Name: !Join [ ':', [ !Ref 'AWS::StackName', 'ApplicationDeadlineReminderLambdaVersion']] diff --git a/apps/pre-award/lambdas/.gitignore b/apps/pre-award/lambdas/.gitignore deleted file mode 100644 index cbdbf540..00000000 --- a/apps/pre-award/lambdas/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# Environments -.env -.venv \ No newline at end of file diff --git a/apps/pre-award/lambdas/README.md b/apps/pre-award/lambdas/README.md deleted file mode 100644 index 2003cf85..00000000 --- a/apps/pre-award/lambdas/README.md +++ /dev/null @@ -1,107 +0,0 @@ -# funding-service-design-post-award-lambda - -## application-deadline-reminder - -## Prerequisites - -- Python 3.10.x or higher - -## Getting started -* Install Python 3.10 -(Instructions assume python 3 is installed on your PATH as `python` but may be `python3` on OSX) - -Check your python version starts with 3.11 i.e. -``` -python --version - -Python 3.10 -``` - -### Create the virtual environment - -From directory application-deadline-reminder: - -``` -python -m venv .venv -``` - -...or if using PyCharm **when importing project**, create venv and set local python interpreter to use it: - -In Pycharm: -1) File -> New Project... : -2) Select 'New environment using' -> Virtualenv -3) Set 'location' to project folder application-deadline-reminder -4) Base interpreter should be set to global Python install. - -### Enter the virtual environment - -...either macOS using bash/zsh: - - source .venv/bin/activate - -...or if on Windows using Command Prompt: - - .venv\Scripts\activate.bat - -...or if using Pycharm, if venv not set up during project import: - -1) settings -> project -> python interpreter -> add interpreter -> add local interpreter -2) **If not previously created** -> Environment -> New -> select path to top level of project -3) **If previously created** -> Environment -> Existing -> Select path to local venv/scripts/python.exe -4) Do not inherit global site packages - -To check if Pycharm is running local interpreter (rather than global): - - pip -V #check the resultant path points to virtual env folder in project - -Add pip tools: -``` -python -m pip install pip-tools -``` - -### Setup pre-commit checks - -* [Install pre-commit locally](https://pre-commit.com/#installation) -* Pre-commit hooks can either be installed using pip `pip install pre-commit` or homebrew (for Mac users)`brew install pre-commit` -* From your checkout directory run `pre-commit install` to set up the git hook scripts - -### Run app -To run the front-end app locally, you can run the following: - -01) Add python-lambda-local to run lambda locally - - ``` - pip install python-lambda-local - ``` - -02) To run locally in as a bash script - - ``` - python-lambda-local -f lambda_handler lambda_function.py event.json - ``` - - ...or if using Pycharm: In Edit configurations - - - script : - - ```/funding-service-design-workflows/apps/pre-award/lambdas/application-deadline-reminder/.venv/bin/python-lambda-local``` here check is the virtual environment folder is same as here - - script parameters : - - ```--timeout 3000 -f lambda_handler lambda_function.py event.json``` - - Working Directory - ```/funding-service-design-workflows/apps/pre-award/lambdas/application-deadline-reminder``` - - ---event.json--- - ```{}``` - - environment variables for local env: - - ```ACCOUNT_STORE_API_HOST=http://localhost:3003;ACCOUNTS_ENDPOINT=/accounts;APPLICATION_ENDPOINT=/applications /{application_id};APPLICATION_REMINDER_STATUS=/funds/{round_id}/application_reminder_status?status = true;APPLICATION_STORE_API_HOST=http://localhost:3002;APPLICATIONS_ENDPOINT=/applications;FUND_ENDPOINT=/funds/{fund_id};FUND_EVENT_ENDPOINT=/funds/{fund_id}/rounds/{round_id}/event/{event_id};FUND_EVENTS_ENDPOINT=/funds/{fund_id}/rounds/{round_id}/events;FUND_ROUNDS_ENDPOINT=/funds/{fund_id}/rounds;FUND_STORE_API_HOST=http://localhost:3001;FUNDS_ENDPOINT=/funds;NOTIFICATION_SERVICE_API_HOST=http://localhost:3006;NOTIFY_TEMPLATE_APPLICATION_DEADLINE_REMINDER=APPLICATION_DEADLINE_REMINDER;NOTIFY_TEMPLATE_INCOMPLETE_APPLICATION=INCOMPLETE_APPLICATION_RECORDS;PYTHONUNBUFFERED=1;``` - - path to env - ```/funding-service-design-docker-runner/.awslocal.env``` - -More information please refer : https://pypi.org/project/python-lambda-local/ diff --git a/apps/pre-award/lambdas/application-deadline-reminder/application_reminder.py b/apps/pre-award/lambdas/application-deadline-reminder/application_reminder.py deleted file mode 100644 index f5cbf6c1..00000000 --- a/apps/pre-award/lambdas/application-deadline-reminder/application_reminder.py +++ /dev/null @@ -1,176 +0,0 @@ -import logging -from datetime import datetime - -import requests -from config import Config -from data import get_account, get_data, send_notification -from dateutil import tz -from helpers.aws_extended_client import SQSExtendedClient - -# Logging to output to CloudWatch Logs -logging.getLogger("lambda_runtime").setLevel(logging.INFO) -logging.getLogger().setLevel(logging.DEBUG) - - -def application_deadline_reminder( - sqs_extended_client: SQSExtendedClient, fund_details: [] -): - logging.info("Application deadline reminder task is now running!") - uk_timezone = tz.gettz("Europe/London") - current_datetime = datetime.now(uk_timezone).replace(tzinfo=None) - - for fund_detail in fund_details: - fund_id = fund_detail["fund"]["id"] - fund_name = fund_detail["fund"]["name"] - - for round_detail in fund_detail["fund_round"]: - round_deadline_str = round_detail.get("deadline") - reminder_date_str = round_detail.get("reminder_date") - round_id = round_detail.get("id") - round_name = round_detail.get("title") - contact_email = round_detail.get("contact_email") - - if not reminder_date_str: - logging.info( - f"No reminder is set for the round {fund_name} {round_name}" - ) - continue - - application_reminder_sent = round_detail.get("application_reminder_sent") - - round_deadline = datetime.strptime(round_deadline_str, "%Y-%m-%dT%H:%M:%S") - - reminder_date = datetime.strptime(reminder_date_str, "%Y-%m-%dT%H:%M:%S") - - if ( - not application_reminder_sent - and reminder_date < current_datetime < round_deadline - ): - not_submitted_applications = _get_not_submitted_applications( - fund_id, round_id - ) - - all_applications = [] - for application in not_submitted_applications.json(): - application["round_name"] = round_name - application["fund_name"] = fund_name - application["contact_help_email"] = contact_email - account = get_account(account_id=application.get("account_id")) - - application["account_email"] = account.get("email_address") - application["deadline_date"] = round_deadline_str - all_applications.append({"application": application}) - - logging.info(f"Total unsubmitted applications: {len(all_applications)}") - # Only one email per account_email - unique_email_account = _get_unique_email_accounts(all_applications) - - logging.info( - f"Total unique email accounts: {len(unique_email_account)}" - ) - unique_application_email_addresses = list(unique_email_account.values()) - - if len(unique_application_email_addresses) > 0: - for count, application in enumerate( - unique_application_email_addresses, start=1 - ): - email = application["application"]["account_email"] - logging.info( - f"Sending reminder {count} of {len(unique_email_account)}" - f" for {fund_name} {round_name}" - f" to {email}" - ) - - _send_message_and_update_funds( - application, - count, - email, - fund_name, - round_id, - round_name, - sqs_extended_client, - unique_application_email_addresses, - ) - - else: - logging.info( - "Currently, there are no non-submitted applications" - f" for {fund_name} {round_name}" - ) - else: - if ( - current_datetime < reminder_date < round_deadline - and not application_reminder_sent - ): - days_to_reminder = reminder_date - current_datetime - logging.info( - "Application deadline reminder is due in " - f" {days_to_reminder.days} days" - f" for {fund_name} {round_name}." - ) - continue - continue - - -def _send_message_and_update_funds( - application, - count, - email, - fund_name, - round_id, - round_name, - sqs_extended_client, - unique_application_email_addresses, -): - try: - message_id = send_notification( - template_type=Config.NOTIFY_TEMPLATE_APPLICATION_DEADLINE_REMINDER, - to_email=email, - content=application, - application_id=application["application"]["id"], - sqs_extended_client=sqs_extended_client, - ) - - if message_id is not None and len(unique_application_email_addresses) == count: - logging.info( - "The application reminder has been" - " sent successfully for" - f" {fund_name} {round_name}" - ) - - app_reminder = ( - Config.FUND_STORE_API_HOST - + Config.APPLICATION_REMINDER_STATUS.format(round_id=round_id) - ) - response = requests.put(app_reminder) - if response.status_code == 200: - logging.info( - "The application_reminder_sent has been" - " set to True for" - f" {fund_name} {round_name}" - ) - - except Exception as e: - logging.info( - "There was a problem sending application(s)" - f" for {fund_name} {round_name}" - f" Error: {e}" - ) - - -def _get_unique_email_accounts(all_applications): - unique_email_account = {} - for application in all_applications: - unique_email_account[application["application"]["account_email"]] = application - return unique_email_account - - -def _get_not_submitted_applications(fund_id, round_id): - status = { - "status_only": ["IN_PROGRESS", "NOT_STARTED", "COMPLETED"], - "fund_id": fund_id, - "round_id": round_id, - } - endpoint = Config.APPLICATION_STORE_API_HOST + Config.APPLICATIONS_ENDPOINT - not_submitted_applications = requests.get(endpoint, params=status) - return not_submitted_applications diff --git a/apps/pre-award/lambdas/application-deadline-reminder/config.py b/apps/pre-award/lambdas/application-deadline-reminder/config.py deleted file mode 100644 index 7567183d..00000000 --- a/apps/pre-award/lambdas/application-deadline-reminder/config.py +++ /dev/null @@ -1,47 +0,0 @@ -from os import environ - - -class Config: - # fund store - FUND_STORE_API_HOST = environ.get("FUND_STORE_API_HOST") - FUNDS_ENDPOINT = environ.get("FUNDS_ENDPOINT") - FUND_ENDPOINT = environ.get("FUND_ENDPOINT") - FUND_ROUNDS_ENDPOINT = environ.get("FUND_ROUNDS_ENDPOINT") - FUND_EVENTS_ENDPOINT = environ.get("FUND_EVENTS_ENDPOINT") - FUND_EVENT_ENDPOINT = environ.get("FUND_EVENT_ENDPOINT") - - # account store - ACCOUNT_STORE_API_HOST = environ.get("ACCOUNT_STORE_API_HOST") - ACCOUNTS_ENDPOINT = environ.get("ACCOUNTS_ENDPOINT") - - # application store - APPLICATION_STORE_API_HOST = environ.get("APPLICATION_STORE_API_HOST") - APPLICATION_REMINDER_STATUS = environ.get("APPLICATION_REMINDER_STATUS") - APPLICATIONS_ENDPOINT = environ.get("APPLICATIONS_ENDPOINT") - APPLICATION_ENDPOINT = environ.get("APPLICATION_ENDPOINT") - - # notification service - NOTIFICATION_SERVICE_API_HOST = environ.get("NOTIFICATION_SERVICE_API_HOST") - NOTIFY_TEMPLATE_APPLICATION_DEADLINE_REMINDER = environ.get( - "NOTIFY_TEMPLATE_APPLICATION_DEADLINE_REMINDER" - ) - NOTIFY_TEMPLATE_INCOMPLETE_APPLICATION = environ.get( - "NOTIFY_TEMPLATE_INCOMPLETE_APPLICATION" - ) - - # --------------- - # AWS Overall Config # TODO after the refactoring test related configs will be moved - # --------------- - AWS_REGION = environ.get("AWS_REGION") - AWS_ENDPOINT_OVERRIDE = environ.get("AWS_ENDPOINT_OVERRIDE") - - # --------------- - # S3 Config - # --------------- - AWS_MSG_BUCKET_NAME = environ.get("AWS_MSG_BUCKET_NAME") - # --------------- - # SQS Config - # --------------- - AWS_SQS_NOTIF_APP_PRIMARY_QUEUE_URL = environ.get( - "AWS_SQS_NOTIF_APP_PRIMARY_QUEUE_URL" - ) diff --git a/apps/pre-award/lambdas/application-deadline-reminder/data.py b/apps/pre-award/lambdas/application-deadline-reminder/data.py deleted file mode 100644 index 1793f161..00000000 --- a/apps/pre-award/lambdas/application-deadline-reminder/data.py +++ /dev/null @@ -1,102 +0,0 @@ -import json -import logging -from typing import Optional -from urllib.parse import urlencode -from uuid import uuid4 - -import requests -from config import Config -from helpers.aws_extended_client import SQSExtendedClient - -# Logging to output to CloudWatch Logs -logging.getLogger("lambda_runtime").setLevel(logging.INFO) -logging.getLogger().setLevel(logging.DEBUG) - - -def get_data(endpoint, params: Optional[dict] = None): - query_string = "" - if params: - params = {k: v for k, v in params.items() if v is not None} - query_string = urlencode(params) - - endpoint = endpoint + "?" + query_string - response = requests.get(endpoint) - - if response.status_code == 200: - data = response.json() - return data - - logging.error( - "There was a problem retrieving response from" - f" {endpoint}. Status code: {response.status_code}" - ) - return None - - -def get_data_safe(endpoint, params: Optional[dict] = None): - try: - response = requests.get(endpoint, params=params) - response.raise_for_status() - return response.json() - except requests.HTTPError: - logging.info( - "No data retrieved from" f" {endpoint}. Status code {response.status_code}" - ) - except Exception as e: - logging.error("Unable to retrieve data from" f" {endpoint}. Exception {str(e)}") - - return None - - -def send_notification( - template_type: str, - to_email: str, - content, - application_id: str, - sqs_extended_client: SQSExtendedClient, -) -> str: - try: - json_payload = { - "type": template_type, - "to": to_email, - "content": content, - } - application_attributes = { - "application_id": { - "StringValue": application_id, - "DataType": "String", - }, - "S3Key": { - "StringValue": "notification/incomplete", - "DataType": "String", - }, - } - message_id = sqs_extended_client.submit_single_message( - Config.AWS_SQS_NOTIF_APP_PRIMARY_QUEUE_URL, - message=json.dumps(json_payload), - message_group_id="notification", - message_deduplication_id=str(uuid4()), # ensures message uniqueness - extra_attributes=application_attributes, - ) - logging.info( - f"Successfully added the message to queue for " - f"application id {application_id} and message id [{message_id}]." - ) - return str(message_id) - except Exception as e: - logging.error( - f"Unable to send message to sqs for {application_id}. Exception {str(e)}" - ) - raise e - - -def get_account(email: Optional[str] = None, account_id: Optional[str] = None): - if email is account_id is None: - raise TypeError("Requires an email address or account_id") - - url = Config.ACCOUNT_STORE_API_HOST + Config.ACCOUNTS_ENDPOINT - params = {"email_address": email, "account_id": account_id} - response = get_data(url, params) - - if response and "account_id" in response: - return response diff --git a/apps/pre-award/lambdas/application-deadline-reminder/helpers/aws_extended_client.py b/apps/pre-award/lambdas/application-deadline-reminder/helpers/aws_extended_client.py deleted file mode 100644 index 8cf2392f..00000000 --- a/apps/pre-award/lambdas/application-deadline-reminder/helpers/aws_extended_client.py +++ /dev/null @@ -1,155 +0,0 @@ -import json -import logging -from datetime import datetime -from uuid import uuid4 - -import boto3 - -logging.getLogger("lambda_runtime").setLevel(logging.INFO) -logging.getLogger().setLevel(logging.DEBUG) - -S3_KEY_ATTRIBUTE_NAME = "S3Key" -MAX_ALLOWED_ATTRIBUTES = 10 - 1 # 10 for SQS and 1 reserved attribute -DEFAULT_MESSAGE_SIZE_THRESHOLD = 262144 -RESERVED_ATTRIBUTE_NAME = "ExtendedPayloadSize" -MESSAGE_POINTER_CLASS = "software.amazon.payloadoffloading.PayloadS3Pointer" - - -class SQSExtendedClient: - def __init__( - self, - region_name="eu-west-2", - endpoint_url=None, - large_payload_support=None, - always_through_s3=None, - **kwargs, - ): - self.large_payload_support = large_payload_support - self.always_through_s3 = always_through_s3 - self.sqs_client = boto3.client( - "sqs", - region_name=region_name, - endpoint_url=endpoint_url, - **kwargs, - ) - self.s3_client = boto3.client( - "s3", - region_name=region_name, - endpoint_url=endpoint_url, - ) - logging.info("Create SQS and S3 client.") - - def submit_single_message( - self, - queue_url, - message, - extra_attributes: dict = None, - message_group_id=None, - message_deduplication_id=None, - ): - sqs_message_attributes = { - "message_created_at": { - "StringValue": str(datetime.now()), - "DataType": "String", - }, - } - message_body, message_attributes = self._store_message_in_s3( - message, sqs_message_attributes, extra_attributes - ) - # add extra message attributes (if provided) - if extra_attributes: - for key, value in extra_attributes.items(): - message_attributes[key] = value - - response = self.sqs_client.send_message( - QueueUrl=queue_url, - MessageBody=message_body, - MessageAttributes=message_attributes, - MessageGroupId=message_group_id, - MessageDeduplicationId=message_deduplication_id, - ) - # Check if the delete operation succeeded, if not raise an error? - status_code = response["ResponseMetadata"]["HTTPStatusCode"] - if status_code != 200: - logging.error( - f"submit_single_message failed with status code {status_code}." - ) - raise Exception( - f"submit_single_message failed with status code {status_code}." - ) - message_id = response["MessageId"] - logging.info(f"Called SQS and submitted the message and id [{message_id}]") - return message_id - - def _store_message_in_s3( - self, message_body: str, message_attributes: dict, extra_attributes: dict - ) -> (str, dict): - """ - Responsible for storing a message payload in a S3 Bucket - :message_body: A UTF-8 encoded version of the message body - :message_attributes: A dictionary consisting of message attributes - :extra_attributes: A dictionary consisting of message attributes - Each message attribute consists of the name (key) along with a - type and value of the message body. The following types are supported - for message attributes: StringValue, BinaryValue and DataType. - """ - if len(message_body) == 0: - # Message cannot be empty - logging.error("messageBody cannot be null or empty.") - - if self.large_payload_support and self.always_through_s3: - # Check message attributes for ExtendedClient related constraints - encoded_body = message_body.encode("utf-8") - - # Modifying the message attributes for storing it in the Queue - attribute_value = { - "DataType": "Number", - "StringValue": str(len(encoded_body)), - } - message_attributes[RESERVED_ATTRIBUTE_NAME] = attribute_value - - # S3 Key should either be a constant or be a random uuid4 string. - s3_key = SQSExtendedClient._get_s3_key(message_attributes, extra_attributes) - - # Adding the object into the bucket - response = self.s3_client.put_object( - Body=encoded_body, Bucket=self.large_payload_support, Key=s3_key - ) - # Check if the delete operation succeeded, if not raise an error? - status_code = response["ResponseMetadata"]["HTTPStatusCode"] - if status_code != 200: - logging.error( - f"submit_single_message failed with status code {status_code}." - ) - raise Exception( - f"submit_single_message failed with status code {status_code}." - ) - # Modifying the message body for storing it in the Queue - message_body = json.dumps( - [ - MESSAGE_POINTER_CLASS, - {"s3BucketName": self.large_payload_support, "s3Key": s3_key}, - ] - ) - return message_body, message_attributes - - @staticmethod - def _get_s3_key(message_attributes: dict, extra_attributes: dict) -> str: - """ - Responsible for checking if the S3 Key exists in the - message_attributes - :message_attributes: A dictionary consisting of message attributes - :extra_attributes: A dictionary consisting of message attributes - Each message attribute consists of the name (key) along with a - type and value of the message body. The following types are supported - for message attributes: StringValue, BinaryValue and DataType. - """ - if S3_KEY_ATTRIBUTE_NAME in message_attributes: - return message_attributes[S3_KEY_ATTRIBUTE_NAME]["StringValue"] - elif extra_attributes and S3_KEY_ATTRIBUTE_NAME in extra_attributes: - return ( - extra_attributes[S3_KEY_ATTRIBUTE_NAME]["StringValue"] - + "/" - + str(uuid4()) - ) - return str(uuid4()) diff --git a/apps/pre-award/lambdas/application-deadline-reminder/incomplete_application.py b/apps/pre-award/lambdas/application-deadline-reminder/incomplete_application.py deleted file mode 100644 index 8c90b697..00000000 --- a/apps/pre-award/lambdas/application-deadline-reminder/incomplete_application.py +++ /dev/null @@ -1,265 +0,0 @@ -import logging -from datetime import datetime - -import requests -from config import Config -from data import get_data_safe, send_notification -from dateutil import tz -from helpers.aws_extended_client import SQSExtendedClient - -logging.getLogger("lambda_runtime").setLevel(logging.INFO) -logging.getLogger().setLevel(logging.DEBUG) - - -def process_events(sqs_extended_client: SQSExtendedClient, fund_details: []): - """ - Pulls events from the fund store and checks if they need processing. If so, the relevant processor - will be called (determined by event type). If the processing was successful, the event is updated - and marked as processed. - - Return: - True if the function ran without issue. - """ - logging.info("Running event check") - uk_timezone = tz.gettz("Europe/London") - current_datetime = datetime.now(uk_timezone).replace(tzinfo=None) - - # Iterate over rounds and events. Note that failure to retrieve rounds / events should be non blocking so that - # the rest of the rounds / events can still be processed - for fund_detail in fund_details: - - fund_id = fund_detail["fund"]["id"] - rounds = fund_detail["fund_round"] - if not rounds: - continue - fund_name = fund_detail["fund"]["name"] - - for fund_round in rounds: - - round_id = fund_round["id"] - round_name = fund_round["title"] - round_contact_email = fund_round.get("contact_email") - events = _get_events(fund_id, round_id) - - if not events: - continue - - for event in events: - - event_type = event["type"] - event_activation_date = _get_formatted_activation_date(event) - event_id = event["id"] - event_processed = event["processed"] - - # Check if event needs to be processed and past the activation date - if event_processed or current_datetime < event_activation_date: - continue - - try: - event_processor = { - "SEND_INCOMPLETE_APPLICATIONS": _send_incomplete_applications_after_deadline - }[event_type] - except KeyError: - logging.error( - f"Incompatible event type found {event_type} for event {event_id}" - ) - continue - - # Process the event and mark it as processed. - result = event_processor( - fund_id=fund_id, - fund_name=fund_name, - round_id=round_id, - round_name=round_name, - round_contact_email=round_contact_email, - sqs_extended_client=sqs_extended_client, - ) - if not result: - continue - - try: - _update_events_for_fund(event_id, fund_id, round_id) - except Exception as e: - logging.error( - f"Failed to mark event {event_id}" - f" as processed for {fund_name} {round_name}" - f" an error {e}" - ) - - logging.info( - f"Event {event_id} has been" - " marked as processed for" - f" {fund_name} {round_name}" - ) - return True - - -def _get_formatted_activation_date(event): - event_activation_date = datetime.strptime( - event.get("activation_date"), "%Y-%m-%dT%H:%M:%S" - ) - return event_activation_date - - -def _update_events_for_fund(event_id, fund_id, round_id): - response = requests.put( - Config.FUND_STORE_API_HOST - + Config.FUND_EVENT_ENDPOINT.format( - fund_id=fund_id, - round_id=round_id, - event_id=event_id, - ), - params={"processed": True}, - ) - response.raise_for_status() - - -def _get_events(fund_id, round_id): - events = get_data_safe( - Config.FUND_STORE_API_HOST - + Config.FUND_EVENTS_ENDPOINT.format(fund_id=fund_id, round_id=round_id) - ) - return events - - -def _get_unsubmitted_applications(fund_id, round_id, fund_name, round_name): - try: - search_params = { - "status_only": ["NOT_STARTED", "IN_PROGRESS", "COMPLETED"], - "fund_id": fund_id, - "round_id": round_id, - } - response = requests.get( - Config.APPLICATION_STORE_API_HOST + Config.APPLICATIONS_ENDPOINT, - params=search_params, - ) - response.raise_for_status() - return response.json() - except Exception as e: - logging.error( - f"Unable to retrieve incomplete applications for fund {fund_name} and round {round_name}. Exception {str(e)}" - ) - return None - - -def _send_incomplete_applications_after_deadline( - fund_id, - fund_name, - round_id, - round_name, - round_contact_email, - sqs_extended_client: SQSExtendedClient, -): - """ - Retrieves a list of unsubmitted applications for the given fund and round. Then use the - notification service to email the account for each application. - - Args: - - fund_id (str): The ID of the fund. - - fund_name (str): The name of the fund. - - round_id (str): The ID of the funding round. - - round_name (str): The name of the round. - - round_contact_email (str): The email to contact for the round - - Return: - True if there were zero unsubmitted applications, or if at least one account was emailed. False otherwise. - """ - unsubmitted_applications = _get_unsubmitted_applications( - fund_id, round_id, fund_name, round_name - ) - if unsubmitted_applications is None: - return False - - logging.info( - f"Found {len(unsubmitted_applications)} unsubmitted applications for fund {fund_name} and round {round_name}" - ) - unsuccessful_notifications = 0 - - # Get all required information for applications - for application in unsubmitted_applications: - try: - account_info, application_to_send = _get_application_details( - application, fund_name, round_contact_email, round_name - ) - except Exception as e: - logging.error( - f"Unable to retrieve application or account information for application {application['id']}." - f" Exception {str(e)}" - ) - unsuccessful_notifications += 1 - continue - - unsuccessful_notifications = _send_messages( - account_info, - application, - application_to_send, - round_contact_email, - sqs_extended_client, - unsuccessful_notifications, - ) - - num_unsubmitted_applications = len(unsubmitted_applications) - logging.info( - f"Sent {num_unsubmitted_applications - unsuccessful_notifications} out of {num_unsubmitted_applications} incomplete application emails" - ) - - return ( - num_unsubmitted_applications > unsuccessful_notifications - if num_unsubmitted_applications > 0 - else True - ) - - -def _send_messages( - account_info, - application, - application_to_send, - round_contact_email, - sqs_extended_client, - unsuccessful_notifications, -): - # Send an email to the account associated with the application. - try: - message_id = send_notification( - template_type=Config.NOTIFY_TEMPLATE_INCOMPLETE_APPLICATION, - to_email=account_info["email_address"], - content={ - "application": application_to_send, - "contact_help_email": round_contact_email, - }, - application_id=application["id"], - sqs_extended_client=sqs_extended_client, - ) - logging.info(f"Successfully added the message into queue [{message_id}]") - except Exception as e: - logging.error( - f"Unable to send an incomplete application email for application {application['id']}. Exception {str(e)}" - ) - unsuccessful_notifications += 1 - return unsuccessful_notifications - - -def _get_application_details(application, fund_name, round_contact_email, round_name): - application_info_request = requests.get( - Config.APPLICATION_STORE_API_HOST - + Config.APPLICATION_ENDPOINT.format( - application_id=application["id"] + "?with_questions_file=true" - ) - ) - application_info_request.raise_for_status() - application_info = application_info_request.json() - account_info_request = requests.get( - Config.ACCOUNT_STORE_API_HOST + Config.ACCOUNTS_ENDPOINT, - params={"account_id": application["account_id"]}, - ) - account_info_request.raise_for_status() - account_info = account_info_request.json() - application_to_send = { - **application, - "fund_name": fund_name, - "questions_file": application_info["questions_file"], - "round_name": round_name, - "account_email": account_info["email_address"], - "contact_help_email": round_contact_email, - } - return account_info, application_to_send diff --git a/apps/pre-award/lambdas/application-deadline-reminder/lambda_function.py b/apps/pre-award/lambdas/application-deadline-reminder/lambda_function.py deleted file mode 100644 index a7c33cd8..00000000 --- a/apps/pre-award/lambdas/application-deadline-reminder/lambda_function.py +++ /dev/null @@ -1,35 +0,0 @@ -from application_reminder import application_deadline_reminder -from config import Config -from data import get_data -from helpers.aws_extended_client import SQSExtendedClient -from incomplete_application import process_events - - -def lambda_handler(event, context): - sqs_extended_client = SQSExtendedClient( - region_name=Config.AWS_REGION, - endpoint_url=Config.AWS_ENDPOINT_OVERRIDE, - large_payload_support=Config.AWS_MSG_BUCKET_NAME, - always_through_s3=True, - ) - - fund_details = [] - funds = get_data(Config.FUND_STORE_API_HOST + Config.FUNDS_ENDPOINT) - for fund in funds: - round_info = _get_round_details(fund["id"]) - fund_details.append({"fund": fund, "fund_round": round_info}) - - application_deadline_reminder(sqs_extended_client, fund_details) - result = process_events(sqs_extended_client, fund_details) - - return { - "statusCode": 200, - "body": result, - } - - -def _get_round_details(fund_id): - round_info = get_data( - Config.FUND_STORE_API_HOST + Config.FUND_ROUNDS_ENDPOINT.format(fund_id=fund_id) - ) - return round_info