From 073174dd0f775f8ae19ecd0121e6bd9b1f97e4ad Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Tue, 24 Dec 2024 11:25:36 -0800 Subject: [PATCH] Add lambda layer resource cleanup. --- .github/workflows/resource-cleanup.yml | 46 +++++++++++++- .../clean/lambda_layer_cleanup/cleaner.py | 61 +++++++++++++++++++ .../lambda_layer_cleanup/requirements.txt | 1 + 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/util/clean/lambda_layer_cleanup/cleaner.py create mode 100644 .github/workflows/util/clean/lambda_layer_cleanup/requirements.txt diff --git a/.github/workflows/resource-cleanup.yml b/.github/workflows/resource-cleanup.yml index a6f022a3a..a612becf8 100644 --- a/.github/workflows/resource-cleanup.yml +++ b/.github/workflows/resource-cleanup.yml @@ -81,6 +81,50 @@ jobs: python -m pip install -r requirements.txt python cleaner.py + cleanup-lambda-layer: + strategy: + fail-fast: false + matrix: + aws-region: ['af-south-1','ap-east-1','ap-northeast-1','ap-northeast-2','ap-northeast-3','ap-south-1','ap-south-2','ap-southeast-1', + 'ap-southeast-2','ap-southeast-3','ap-southeast-4','ca-central-1','eu-central-1','eu-central-2','eu-north-1', + 'eu-south-1','eu-south-2','eu-west-1','eu-west-2','eu-west-3','il-central-1','me-central-1','me-south-1', 'sa-east-1', + 'us-east-1','us-east-2', 'us-west-1', 'us-west-2'] + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Configure AWS credentials for IAD account access + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.E2E_IAD_TEST_ACCOUNT_ARN }} + aws-region: us-east-1 + + - name: Retrieve account id for the region + uses: aws-actions/aws-secretsmanager-get-secrets@v1 + with: + secret-ids: + ACCOUNT_ID, region-account/${{ matrix.aws-region }} + + - name: Configure AWS credentials for the regional account access + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ env.ACCOUNT_ID }}:role/${{ secrets.RESOURCE_CLEANER_ROLE_NAME }} + aws-region: ${{ matrix.aws-region }} + + - name: Cleanup Lambda Layer + working-directory: .github/workflows/util/clean/lambda_layer_cleanup + env: + AWS_DEFAULT_REGION: ${{ matrix.aws-region }} + run: | + python -m pip install -r requirements.txt + python cleaner.py + publish-metric: needs: [ cleanup-ec2-instances, cleanup-k8s-cluster ] if: always() @@ -89,4 +133,4 @@ jobs: with: aws-region: 'us-east-1' caller-workflow-name: 'enablement-test-resource-cleanup' - validation-result: ${{ (needs.cleanup-ec2-instances.result == 'success' && needs.cleanup-k8s-cluster.result == 'success') && 'success' || 'failure' }} \ No newline at end of file + validation-result: ${{ (needs.cleanup-ec2-instances.result == 'success' && needs.cleanup-k8s-cluster.result == 'success' && needs.cleanup-lambda-layer.result == 'success') && 'success' || 'failure' }} \ No newline at end of file diff --git a/.github/workflows/util/clean/lambda_layer_cleanup/cleaner.py b/.github/workflows/util/clean/lambda_layer_cleanup/cleaner.py new file mode 100644 index 000000000..e79798419 --- /dev/null +++ b/.github/workflows/util/clean/lambda_layer_cleanup/cleaner.py @@ -0,0 +1,61 @@ +import boto3 +from botocore.exceptions import ClientError +from datetime import datetime, timezone, timedelta +import time + +client = boto3.client('apigateway') + +def delete_api_with_retries(client, api_id, retries=5): + """Deletes an API with retries and exponential backoff.""" + delay = 10 + for attempt in range(retries): + try: + client.delete_rest_api(restApiId=api_id) + print(f"API {api_id} deleted successfully.") + return + except ClientError as e: + if e.response['Error']['Code'] == 'TooManyRequestsException': + print(f"Rate limit exceeded. Retrying in {delay} seconds (Attempt {attempt + 1}/{retries})...") + time.sleep(delay) + delay *= 2 # Exponential backoff + else: + print(f"Error deleting API {api_id}: {e}") + raise # Re-raise other exceptions + print(f"Failed to delete API {api_id} after {retries} attempts.") + +def delete_old_api_gateways(hours_old=3, batch_size=5): + """Deletes API Gateways older than the specified hours in batches.""" + now = datetime.now(timezone.utc) # Ensure `now` is timezone-aware + cutoff_time = now - timedelta(hours=hours_old) + + print(f"Cutoff time: {cutoff_time}") + + # Fetch all APIs + apis = client.get_rest_apis() + batch_counter = 0 + + for api in apis.get('items', []): + created_date = api.get('createdDate') # This is usually UTC already + if created_date and isinstance(created_date, datetime): + # Ensure `created_date` is timezone-aware + created_date = created_date.astimezone(timezone.utc) + + if created_date < cutoff_time: + api_id = api['id'] + api_name = api.get('name', 'Unnamed API') + print(f"Preparing to delete API: {api_name} (ID: {api_id}), created at {created_date}") + + # Attempt to delete the API with retries + delete_api_with_retries(client, api_id) + + batch_counter += 1 + + # Pause after every batch + if batch_counter % batch_size == 0: + print("Pausing for 2 minutes to avoid rate-limiting...") + time.sleep(120) + else: + print("Invalid or missing createdDate for API:", api) + +if __name__ == "__main__": + delete_old_api_gateways() \ No newline at end of file diff --git a/.github/workflows/util/clean/lambda_layer_cleanup/requirements.txt b/.github/workflows/util/clean/lambda_layer_cleanup/requirements.txt new file mode 100644 index 000000000..1db657b6b --- /dev/null +++ b/.github/workflows/util/clean/lambda_layer_cleanup/requirements.txt @@ -0,0 +1 @@ +boto3 \ No newline at end of file