Add lambda layer resource cleanup. (#339)

*Description of changes:*
Add lambda layer resource cleanup script.

*Rollback procedure:*
Revert the commit.
<Can we safely revert this commit if needed? If not, detail what must be
done to safely revert and why it is needed.>

*Ensure you've run the following tests on your changes and include the
link below:*
Test workflow passed:
https://github.com/aws-observability/aws-application-signals-test-framework/actions/runs/12485707531/job/34844864812

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

.github/workflows/resource-cleanup.yml

@@ -81,6 +81,50 @@ jobs:
           python -m pip install -r requirements.txt
           python cleaner.py
 
+  cleanup-lambda-layer:
+    strategy:
+      fail-fast: false
+      matrix:
+        aws-region: ['af-south-1','ap-east-1','ap-northeast-1','ap-northeast-2','ap-northeast-3','ap-south-1','ap-south-2','ap-southeast-1',
+                     'ap-southeast-2','ap-southeast-3','ap-southeast-4','ca-central-1','eu-central-1','eu-central-2','eu-north-1',
+                     'eu-south-1','eu-south-2','eu-west-1','eu-west-2','eu-west-3','il-central-1','me-central-1','me-south-1', 'sa-east-1',
+                     'us-east-1','us-east-2', 'us-west-1', 'us-west-2']
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Configure AWS credentials for IAD account access
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.E2E_IAD_TEST_ACCOUNT_ARN }}
+          aws-region: us-east-1
+
+      - name: Retrieve account id for the region
+        uses: aws-actions/aws-secretsmanager-get-secrets@v1
+        with:
+          secret-ids:
+            ACCOUNT_ID, region-account/${{ matrix.aws-region }}
+
+      - name: Configure AWS credentials for the regional account access
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::${{ env.ACCOUNT_ID }}:role/${{ secrets.RESOURCE_CLEANER_ROLE_NAME }}
+          aws-region: ${{ matrix.aws-region }}
+
+      - name: Cleanup Lambda Layer
+        working-directory: .github/workflows/util/clean/lambda_layer_cleanup
+        env:
+          AWS_DEFAULT_REGION: ${{ matrix.aws-region }}
+        run: |
+          python -m pip install -r requirements.txt
+          python cleaner.py
+
   publish-metric:
     needs: [ cleanup-ec2-instances, cleanup-k8s-cluster ]
     if: always()
@@ -89,4 +133,4 @@ jobs:
     with:
       aws-region: 'us-east-1'
       caller-workflow-name: 'enablement-test-resource-cleanup'
-      validation-result: ${{ (needs.cleanup-ec2-instances.result == 'success' && needs.cleanup-k8s-cluster.result == 'success') && 'success' || 'failure' }}
+      validation-result: ${{ (needs.cleanup-ec2-instances.result == 'success' && needs.cleanup-k8s-cluster.result == 'success' && needs.cleanup-lambda-layer.result == 'success') && 'success' || 'failure' }}

.github/workflows/util/clean/lambda_layer_cleanup/cleaner.py

@@ -0,0 +1,62 @@
+import boto3
+from botocore.exceptions import ClientError
+from datetime import datetime, timezone, timedelta
+import time
+
+client = boto3.client('apigateway')
+
+def delete_api_with_retries(client, api_id, retries=5):
+    """Deletes an API with retries and exponential backoff."""
+    delay = 10
+    for attempt in range(retries):
+        try:
+            client.delete_rest_api(restApiId=api_id)
+            print(f"API {api_id} deleted successfully. Sleeping for 32 seconds...")
+            time.sleep(32)
+            return
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'TooManyRequestsException':
+                print(f"Rate limit exceeded. Retrying in {delay} seconds (Attempt {attempt + 1}/{retries})...")
+                time.sleep(delay)
+                delay *= 2  # Exponential backoff
+            else:
+                print(f"Error deleting API {api_id}: {e}")
+                raise  # Re-raise other exceptions
+    print(f"Failed to delete API {api_id} after {retries} attempts.")
+
+def delete_old_api_gateways(hours_old=3, batch_size=5):
+    """Deletes API Gateways older than the specified hours in batches."""
+    now = datetime.now(timezone.utc)  # Ensure `now` is timezone-aware
+    cutoff_time = now - timedelta(hours=hours_old)
+
+    print(f"Cutoff time: {cutoff_time}")
+
+    # Fetch all APIs
+    apis = client.get_rest_apis()
+    batch_counter = 0
+
+    for api in apis.get('items', []):
+        created_date = api.get('createdDate')  # This is usually UTC already
+        if created_date and isinstance(created_date, datetime):
+            # Ensure `created_date` is timezone-aware
+            created_date = created_date.astimezone(timezone.utc)
+
+            api_id = api['id']
+            api_name = api.get('name', 'Unnamed API')
+            if "AdotLambda" in api_name and "SampleApp" in api_name and created_date < cutoff_time:
+                print(f"Preparing to delete API: {api_name} (ID: {api_id}), created at {created_date}")
+
+                # Attempt to delete the API with retries
+                delete_api_with_retries(client, api_id)
+
+                batch_counter += 1
+
+                # Pause after every batch
+                if batch_counter % batch_size == 0:
+                    print("Pausing for 2 minutes to avoid rate-limiting...")
+                    time.sleep(120)
+        else:
+            print("Invalid or missing createdDate for API:", api)
+
+if __name__ == "__main__":
+    delete_old_api_gateways()

.github/workflows/util/clean/lambda_layer_cleanup/requirements.txt

@@ -0,0 +1 @@
+boto3