diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index fee9392..a817043 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -12,17 +12,30 @@ on: required: false DS_RELEASE_BOT_PRIVATE_KEY: required: false - + AWS_DEFAULT_REGION_DEPLOY: + required: false + AWS_ACCESS_KEY_ID_DEPLOY: + required: false + AWS_SECRET_ACCESS_KEY_DEPLOY: + required: false + AWS_ACCOUNT_ID: + required: false jobs: build_and_package: name: Build and package runs-on: ubuntu-latest + timeout-minutes: 60 + env: + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION_DEPLOY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_DEPLOY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEPLOY }} + AWS_DEFAULT_ACCOUNT: ${{ secrets.AWS_ACCOUNT_ID }} steps: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 cache: "npm" - name: Install Dependencies @@ -65,8 +78,84 @@ jobs: app_id: ${{ secrets.DS_RELEASE_BOT_ID }} private_key: ${{ secrets.DS_RELEASE_BOT_PRIVATE_KEY }} - - name: Maybe Release 🚀 + - name: Check release + id: check_release if: ${{ inputs.release }} + run: | + SHOULD_RELEASE=false + npm run semantic-release --dry-run > check_release_output.txt + if grep -q "Published release" check_release_output.txt; then + echo "SHOULD_RELEASE=true" >> $GITHUB_OUTPUT + else + echo "SHOULD_RELEASE=false" >> $GITHUB_OUTPUT + fi + + - name: Install deployment environment + if: "${{ inputs.release && steps.check_release.outputs.SHOULD_RELEASE }}" + id: install_deploy_env + run: | + # install deployment environment with eoapi-cdk from build + python -m venv .deployment_venv + source .deployment_venv/bin/activate + pip install dist/python/*.gz + cd integration_tests/cdk + pip install -r requirements.txt + npm install + deactivate + cd - + + + - name: Deploy test stack + if: "${{ inputs.release && steps.check_release.outputs.SHOULD_RELEASE }}" + id: deploy_step + run: | + source .deployment_venv/bin/activate + + # synthesize the stack + cd integration_tests/cdk + npx cdk synth --debug --all --require-approval never + + # deploy the stack and grab URLs for testing + npx cdk deploy --ci --all --require-approval never + echo "ingestor_url=$(aws cloudformation describe-stacks --stack-name eoapi-template-demo-test-pgSTAC-infra --query "Stacks[0].Outputs[?starts_with(OutputKey, 'stacingestor')].OutputValue | [0]" --output text)" >> $GITHUB_OUTPUT + echo "stac_api_url=$(aws cloudformation describe-stacks --stack-name eoapi-template-demo-test-pgSTAC-infra --query "Stacks[0].Outputs[?starts_with(OutputKey, 'pgstacapi')].OutputValue | [0]" --output text)" >> $GITHUB_OUTPUT + echo "titiler_pgstac_api_url=$(aws cloudformation describe-stacks --stack-name eoapi-template-demo-test-pgSTAC-infra --query "Stacks[0].Outputs[?starts_with(OutputKey, 'titilerpgstac')].OutputValue | [0]" --output text)" >> $GITHUB_OUTPUT + deactivate + cd - + + - name: Run integration tests + id: run_tests + if: "${{ inputs.release && steps.check_release.outputs.SHOULD_RELEASE }}" + env: + ingestor_url: ${{ steps.deploy_step.outputs.ingestor_url }} + stac_api_url: ${{ steps.deploy_step.outputs.stac_api_url }} + titiler_pgstac_api_url: ${{ steps.deploy_step.outputs.titiler_pgstac_api_url }} + run: | + cd integration_tests/tests + python -m venv .tests_venv + source .tests_venv/bin/activate + pip install -e . + pytest eoapi_tests + deactivate + cd - + + - name: Tear down any infrastructure + if: always() + run: | + cd integration_tests/cdk + # run this only if we find a 'cdk.out' directory, which means there might be things to tear down + if [ -d "cdk.out" ]; then + cd - + source .deployment_venv/bin/activate + cd integration_tests/cdk + npx cdk destroy --ci --all --force + fi + + + # run if the previous step set SHOULD_RELEASE to true + - name: Maybe Release 🚀 + # only run if the previous step set SHOULD_RELEASE to true + if: "${{ inputs.release && steps.check_release.outputs.SHOULD_RELEASE }}" run: | npm run semantic-release env: diff --git a/.github/workflows/build_and_release.yaml b/.github/workflows/build_and_release.yaml new file mode 100644 index 0000000..e252d72 --- /dev/null +++ b/.github/workflows/build_and_release.yaml @@ -0,0 +1,17 @@ +name: Build & try to release + +on: + push: + +jobs: + package: + uses: ./.github/workflows/build.yaml + with: + release: true + secrets: + DS_RELEASE_BOT_ID: ${{ secrets.DS_RELEASE_BOT_ID }} + DS_RELEASE_BOT_PRIVATE_KEY: ${{ secrets.DS_RELEASE_BOT_PRIVATE_KEY }} + AWS_DEFAULT_REGION_DEPLOY: ${{ secrets.AWS_DEFAULT_REGION_DEPLOY }} + AWS_ACCESS_KEY_ID_DEPLOY: ${{ secrets.AWS_ACCESS_KEY_ID_DEPLOY }} + AWS_SECRET_ACCESS_KEY_DEPLOY: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEPLOY }} + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} diff --git a/.github/workflows/distribute.yaml b/.github/workflows/distribute.yaml index e484b8f..fd7fe8a 100644 --- a/.github/workflows/distribute.yaml +++ b/.github/workflows/distribute.yaml @@ -9,10 +9,6 @@ jobs: package: uses: ./.github/workflows/build.yaml - integration-test: - uses: ./.github/workflows/integration-test.yaml - needs: package - distribute-python: runs-on: ubuntu-latest needs: package diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml deleted file mode 100644 index eb9fe79..0000000 --- a/.github/workflows/integration-test.yaml +++ /dev/null @@ -1,104 +0,0 @@ -name: Deploy & Integration Test - -permissions: - id-token: write # required for requesting the JWT - contents: read # required for actions/checkout - -on: - workflow_call: - - workflow_dispatch: - - # remove later - # push: - # branches: - # - "feat/add-integration-tests" - -jobs: - deploy-and-integration-test: - name: Deploy & Integration Test - runs-on: ubuntu-latest - env: - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION_DEPLOY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_DEPLOY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEPLOY }} - - - steps: - - - name: Checkout repository # for runs after `Distribute`, pulling from main will always give the latest eoapi-cdk. - uses: actions/checkout@v3 - - - name: Set up python - uses: actions/setup-python@v2 - with: - cache: pip - - - name: Set up node - uses: actions/setup-node@v3 - with: - node-version: '18' - - - name: Download compiled eoapi-cdk artifact - uses: actions/download-artifact@v3 - with: - name: python - path: dist - - - name: Install eoapi-cdk from artifact - run: pip install dist/python/*.gz - - - name: Install deployment environment - working-directory: integration_tests/cdk - run: | - python -m venv .deployment_venv - source .deployment_venv/bin/activate - pip install -r requirements.txt - npm install - deactivate - - - name: Synthesize the stack - working-directory: integration_tests/cdk - run: | - source .deployment_venv/bin/activate - npx cdk synth --debug --all --require-approval never - deactivate - - # deploys and grabs URLs from the output for later tests - - name: Deploy the stack - id: deploy_step - working-directory: integration_tests/cdk - run: | - source .deployment_venv/bin/activate - npx cdk deploy --ci --all --require-approval never - echo "ingestor_url=$(aws cloudformation describe-stacks --stack-name eoapi-template-demo-test-pgSTAC-infra --query "Stacks[0].Outputs[?starts_with(OutputKey, 'stacingestor')].OutputValue | [0]" --output text)" >> $GITHUB_OUTPUT - echo "stac_api_url=$(aws cloudformation describe-stacks --stack-name eoapi-template-demo-test-pgSTAC-infra --query "Stacks[0].Outputs[?starts_with(OutputKey, 'pgstacapi')].OutputValue | [0]" --output text)" >> $GITHUB_OUTPUT - echo "titiler_pgstac_api_url=$(aws cloudformation describe-stacks --stack-name eoapi-template-demo-test-pgSTAC-infra --query "Stacks[0].Outputs[?starts_with(OutputKey, 'titilerpgstac')].OutputValue | [0]" --output text)" >> $GITHUB_OUTPUT - deactivate - - - name: Install test environment - working-directory: integration_tests/tests - run: | - python -m venv .tests_venv - source .tests_venv/bin/activate - pip install -e . - deactivate - - - name: Test the stack - working-directory: integration_tests/tests - env: - ingestor_url: ${{ steps.deploy_step.outputs.ingestor_url }} - stac_api_url: ${{ steps.deploy_step.outputs.stac_api_url }} - titiler_pgstac_api_url: ${{ steps.deploy_step.outputs.titiler_pgstac_api_url }} - run: | - source .tests_venv/bin/activate - pytest eoapi_tests - deactivate - - - name: Always tear down the stack - if: always() - working-directory: integration_tests/cdk - run: | - source .deployment_venv/bin/activate - npx cdk destroy --ci --all --force - deactivate diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml deleted file mode 100644 index e543929..0000000 --- a/.github/workflows/test.yaml +++ /dev/null @@ -1,13 +0,0 @@ -name: Test & Build - -on: - push: - -jobs: - package: - uses: ./.github/workflows/build.yaml - with: - release: true - secrets: - DS_RELEASE_BOT_ID: ${{ secrets.DS_RELEASE_BOT_ID }} - DS_RELEASE_BOT_PRIVATE_KEY: ${{ secrets.DS_RELEASE_BOT_PRIVATE_KEY }} diff --git a/integration_tests/cdk/config.py b/integration_tests/cdk/config.py index 70457c5..624c424 100644 --- a/integration_tests/cdk/config.py +++ b/integration_tests/cdk/config.py @@ -4,10 +4,15 @@ import yaml from aws_cdk import aws_ec2 from pydantic_core.core_schema import FieldValidationInfo -from pydantic_settings import BaseSettings - +from pydantic_settings import BaseSettings, SettingsConfigDict class AppConfig(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env" + ) + aws_default_account: str = pydantic.Field( + description="AWS account ID" + ) project_id: str = pydantic.Field( description="Project ID", default="eoapi-template-demo" ) diff --git a/integration_tests/cdk/eoapi_template/pgStacInfra.py b/integration_tests/cdk/eoapi_template/pgStacInfra.py index 0f12e8d..fc1905c 100644 --- a/integration_tests/cdk/eoapi_template/pgStacInfra.py +++ b/integration_tests/cdk/eoapi_template/pgStacInfra.py @@ -232,7 +232,7 @@ def __init__( # must be already set up, or set up after this deployment. if not app_config.data_access_role_arn: data_access_role = self._grant_assume_role_with_principal_pattern( - data_access_role, stac_ingestor.handler_role.role_name + data_access_role, stac_ingestor.handler_role.role_name, app_config.aws_default_account ) def _create_data_access_role(self) -> aws_iam.Role: @@ -261,7 +261,7 @@ def _grant_assume_role_with_principal_pattern( self, role_to_assume: aws_iam.Role, principal_pattern: str, - account_id: str = boto3.client("sts").get_caller_identity().get("Account"), + account_id: str, ) -> aws_iam.Role: """ Grants assume role permissions to the role of the given diff --git a/integration_tests/cdk/package-lock.json b/integration_tests/cdk/package-lock.json index 9428ef3..5fa908a 100644 --- a/integration_tests/cdk/package-lock.json +++ b/integration_tests/cdk/package-lock.json @@ -8,13 +8,13 @@ "name": "eoapi-template", "version": "0.1.0", "dependencies": { - "aws-cdk": "^2.81.0" + "aws-cdk": "^2.99.1" } }, "node_modules/aws-cdk": { - "version": "2.96.2", - "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.96.2.tgz", - "integrity": "sha512-13ERpPV99OFAD75PLOtl0rRMXTWn6bCrmUPwYKkLwIMkj2xWCBiwo2Y9Qg+UzEszm5NMHA1N4ichSvuZ0mt2IQ==", + "version": "2.128.0", + "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.128.0.tgz", + "integrity": "sha512-epOAr/0WKqmyaKqBc7N0Ky5++93pu+v6yVN9jNOa4JYkAkGbeTS3vR9bj/W0o94jnlgWevG3HNHr83jtRvw/4A==", "bin": { "cdk": "bin/cdk" }, diff --git a/integration_tests/cdk/package.json b/integration_tests/cdk/package.json index 411cfc3..aaaa2d9 100644 --- a/integration_tests/cdk/package.json +++ b/integration_tests/cdk/package.json @@ -2,7 +2,7 @@ "name": "eoapi-template", "version": "0.1.0", "dependencies": { - "aws-cdk": "^2.81.0" + "aws-cdk": "^2.99.1" } } \ No newline at end of file diff --git a/integration_tests/cdk/requirements.txt b/integration_tests/cdk/requirements.txt index 53f55e7..3761dc1 100644 --- a/integration_tests/cdk/requirements.txt +++ b/integration_tests/cdk/requirements.txt @@ -1,6 +1,6 @@ -aws-cdk-lib>=2.75.0 -aws_cdk.aws_cognito_identitypool_alpha>=2.75.0a0 -aws-cdk.aws-apigatewayv2-alpha==2.95.1a0 +aws-cdk-lib>=2.99.1 +aws_cdk.aws_cognito_identitypool_alpha>=2.99.0a0 +aws-cdk.aws-apigatewayv2-alpha>=2.99.0a0 constructs>=10.0.0,<11.0.0 pydantic==2.0.2 pydantic-settings==2.0.1