feat: add integration tests

.github/workflows/build.yaml

@@ -12,7 +12,7 @@ on:
         required: false
       DS_RELEASE_BOT_PRIVATE_KEY:
         required: false
-
+        
 jobs:
   build_and_package:
     name: Build and package
@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: 16
+          node-version: 18
           cache: "npm"
 
       - name: Install Dependencies
@@ -66,7 +66,7 @@ jobs:
           private_key: ${{ secrets.DS_RELEASE_BOT_PRIVATE_KEY }}
 
       - name: Maybe Release 🚀
-        if: ${{ inputs.release }}
+        if: "${{ inputs.release }}"
         run: |
           npm run semantic-release
         env:

.github/workflows/test.yaml → .github/workflows/build_and_release.yaml

@@ -1,4 +1,4 @@
-name: Test & Build
+name: Build & try to release
 
 on:
   push:

.github/workflows/deploy.yaml

@@ -0,0 +1,76 @@
+name: Test deployment
+
+on:
+  merge_group:
+    branches: [ main ]
+
+jobs:
+  build_package_and_deploy:
+    name: Build, package and deploy
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION_DEPLOY }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_DEPLOY }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEPLOY }}
+      AWS_DEFAULT_ACCOUNT: ${{ secrets.AWS_ACCOUNT_ID }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          cache: "npm"
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Compile project
+        run: npm run build
+
+      - name: Generate distribution packages
+        run: npm run package
+
+
+      - name: Install deployment environment
+        id: install_deploy_env
+        run: |
+          # install deployment environment with eoapi-cdk from build
+          python -m venv .deployment_venv
+          source .deployment_venv/bin/activate
+          pip install dist/python/*.gz
+          cd integration_tests/cdk
+          pip install -r requirements.txt
+          npm install   
+          deactivate
+          cd -
+
+
+      - name: Deploy test stack
+        id: deploy_step
+        run: |
+          source .deployment_venv/bin/activate
+
+          # synthesize the stack
+          cd integration_tests/cdk
+          npx cdk synth --debug --all --require-approval never
+
+          # deploy the stack
+          npx cdk deploy --ci --all --require-approval never
+          deactivate
+          cd -
+
+      - name: Tear down any infrastructure
+        if: always()
+        run: |
+          cd integration_tests/cdk
+          # run this only if we find a 'cdk.out' directory, which means there might be things to tear down
+          if [ -d "cdk.out" ]; then
+            cd -
+            source .deployment_venv/bin/activate
+            cd integration_tests/cdk
+            # see https://github.com/aws/aws-cdk/issues/24946
+            # this didn't work : rm -f cdk.out/synth.lock
+            # so we just duplicate the cdk output to cdk-destroy.out
+            npx cdk destroy --output cdk-destroy.out --ci --all --force
+          fi

.github/workflows/distribute.yaml

@@ -13,11 +13,12 @@ jobs:
     runs-on: ubuntu-latest
     needs: package
     steps:
+
       - uses: actions/download-artifact@v3
         with:
           name: python
           path: dist
-
+          
       - run: pip install twine
 
       - run: twine upload dist/*

.gitignore

@@ -8,3 +8,6 @@ docs
 __pycache__
 .venv
 .tox
+tests/*.egg*
+tests/*venv*
+tests/__pycache__

README.md

@@ -54,3 +54,6 @@ Versioning is automatically handled via [Conventional Commits](https://www.conve
 _Warning_: If you rebase `main`, you must ensure that the commits referenced by tags point to commits that are within the `main` branch. If a commit references a commit that is no longer on the `main` branch, Semantic Release will fail to detect the correct version of the project. [More information](https://github.com/semantic-release/semantic-release/issues/1121#issuecomment-517945233).
 
 
+## Tests
+
+Each pull request to `main` is added to a [merge queue](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue#triggering-merge-group-checks-with-github-actions) so that a "deployment test" workflow can run before the merge actually happens. If the deployment fails, the merge is cancelled. Here is [the definition of this workflow](https://github.com/developmentseed/eoapi-cdk/blob/main/.github/workflows/deploy.yaml) and the [tests definition](https://github.com/developmentseed/eoapi-cdk/blob/main/tests).

integration_tests/cdk/README.md

@@ -0,0 +1,55 @@
+
+# Deployment CDK code for eoapi-cdk deployment tests
+
+This is a wrapper CDK code that is used to test a deployment of the `eoapi-cdk` constructs before a release happens.
+
+## Requirements
+
+- python
+- docker
+- node
+- AWS credentials environment variables configured to point to an account. 
+
+## Installation
+
+Install python dependencies with 
+
+```
+python -m venv .venv
+source .venv/bin/activate
+python -m pip install -r requirements.txt
+```
+
+Install the latest `eoapi-cdk` either from PyPI:
+
+```
+pip install eoapi-cdk
+```
+
+Or alternatively, compile and package from the root of this repository to get the python version of the constructs locally.
+
+Also install node dependencies with 
+
+```
+npm install
+```
+
+Verify that the `cdk` CLI is available. Since `aws-cdk` is installed as a local dependency, you can use the `npx` node package runner tool, that comes with `npm`.
+
+```
+npx cdk --version
+```
+
+## Deployment
+
+First, synthesize the app 
+
+```
+npx cdk synth --all
+```
+
+Then, deploy
+
+```
+npx cdk deploy --all --require-approval never
+```

integration_tests/cdk/app.py

@@ -0,0 +1,17 @@
+from aws_cdk import App
+
+from config import build_app_config
+from eoapi_template import pgStacInfra, vpc
+
+app = App()
+
+app_config = build_app_config()
+
+vpc_stack = vpc.VpcStack(scope=app, app_config=app_config)
+
+pgstac_infra_stack = pgStacInfra.pgStacInfraStack(
+    scope=app,
+    vpc=vpc_stack.vpc,
+    app_config=app_config,
+)
+app.synth()

integration_tests/cdk/cdk.json

@@ -0,0 +1,32 @@
+{
+  "app": "python3 app.py",
+  "watch": {
+    "include": [
+      "**"
+    ],
+    "exclude": [
+      "README.md",
+      "cdk*.json",
+      "requirements*.txt",
+      "source.bat",
+      "**/*.pyc",
+      "**/*.tmp",
+      "**/__pycache__",
+      "tests",
+      "scripts",
+      "*venv"
+    ]
+  },
+  "context": {
+    "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true,
+    "@aws-cdk/core:stackRelativeExports": true,
+    "@aws-cdk/aws-rds:lowercaseDbIdentifier": true,
+    "@aws-cdk/aws-lambda:recognizeVersionProps": true,
+    "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true,
+    "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
+    "@aws-cdk/core:target-partitions": [
+      "aws",
+      "aws-cn"
+    ]
+  }
+}

integration_tests/cdk/config.py

@@ -0,0 +1,58 @@
+from typing import Dict
+
+import pydantic
+import yaml
+from pydantic_core.core_schema import FieldValidationInfo
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class AppConfig(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_file=".env"
+    )
+    aws_default_account: str = pydantic.Field(
+        description="AWS account ID"
+    )
+    project_id: str = pydantic.Field(
+        description="Project ID", default="eoapi-cdk"
+    )
+    stage: str = pydantic.Field(description="Stage of deployment", default="test")
+    # because of its validator, `tags` should always come after `project_id` and `stage`
+    tags: Dict[str, str] | None = pydantic.Field(
+        description="""Tags to apply to resources. If none provided,
+        will default to the defaults defined in `default_tags`.
+        Note that if tags are passed to the CDK CLI via `--tags`,
+        they will override any tags defined here.""",
+        default=None,
+    )
+    db_instance_type: str = pydantic.Field(
+        description="Database instance type", default="t3.micro"
+    )
+    db_allocated_storage: int = pydantic.Field(
+        description="Allocated storage for the database", default=5
+    )
+
+    @pydantic.field_validator("tags")
+    def default_tags(cls, v, info: FieldValidationInfo):
+        return v or {"project_id": info.data["project_id"], "stage": info.data["stage"]}
+
+    def build_service_name(self, service_id: str) -> str:
+        return f"{self.project_id}-{self.stage}-{service_id}"
+
+
+def build_app_config() -> AppConfig:
+    """Builds the AppConfig object from config.yaml file if exists,
+    otherwise use defaults"""
+    try:
+        with open("config.yaml") as f:
+            print("Loading config from config.yaml")
+            app_config = yaml.safe_load(f)
+            app_config = (
+                {} if app_config is None else app_config
+            )  # if config is empty, set it to an empty dict
+            app_config = AppConfig(**app_config)
+    except FileNotFoundError:
+        # if no config at the expected path, using defaults
+        app_config = AppConfig()
+
+    return app_config

integration_tests/cdk/eoapi_template/pgStacInfra.py

@@ -0,0 +1,71 @@
+from aws_cdk import (
+    Stack,
+    aws_ec2,
+    aws_rds
+)
+from constructs import Construct
+from eoapi_cdk import (
+    PgStacApiLambda,
+    PgStacDatabase,
+    TitilerPgstacApiLambda,
+)
+
+from config import AppConfig
+
+
+class pgStacInfraStack(Stack):
+    def __init__(
+        self,
+        scope: Construct,
+        vpc: aws_ec2.Vpc,
+        app_config: AppConfig,
+        **kwargs,
+    ) -> None:
+        super().__init__(
+            scope,
+            id=app_config.build_service_name("pgSTAC-infra"),
+            tags=app_config.tags,
+            **kwargs,
+        )
+
+        pgstac_db = PgStacDatabase(
+            self,
+            "pgstac-db",
+            vpc=vpc,
+            engine=aws_rds.DatabaseInstanceEngine.postgres(
+                version=aws_rds.PostgresEngineVersion.VER_14
+            ),
+            vpc_subnets=aws_ec2.SubnetSelection(
+                subnet_type=aws_ec2.SubnetType.PUBLIC,
+            ),
+            allocated_storage=app_config.db_allocated_storage,
+            instance_type=aws_ec2.InstanceType(app_config.db_instance_type)
+        )
+
+        pgstac_db.db.connections.allow_default_port_from_any_ipv4()
+
+        PgStacApiLambda(
+            self,
+            "pgstac-api",
+            api_env={
+                "NAME": app_config.build_service_name("STAC API"),
+                "description": f"{app_config.stage} STAC API",
+            },
+            db=pgstac_db.db,
+            db_secret=pgstac_db.pgstac_secret
+        )
+
+        TitilerPgstacApiLambda(
+            self,
+            "titiler-pgstac-api",
+            api_env={
+                "NAME": app_config.build_service_name("titiler pgSTAC API"),
+                "description": f"{app_config.stage} titiler pgstac API",
+            },
+            db=pgstac_db.db,
+            db_secret=pgstac_db.pgstac_secret,
+            buckets=[],
+            lambda_function_options={
+                "allow_public_subnet": True,
+            },
+        )