From 4c69c962f5a926b11c4b5d6c89b6aea3654305a1 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Wed, 5 Feb 2025 09:11:52 -0600 Subject: [PATCH 1/2] fix: ensure db bootstrapper runs on each deploy fix: advertise secretBootstrapper so other resources can use it as a dependency --- integration_tests/cdk/app.py | 13 +++++++++---- lib/database/index.ts | 22 ++++++++++++++-------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/integration_tests/cdk/app.py b/integration_tests/cdk/app.py index ad70b21..ad2244b 100644 --- a/integration_tests/cdk/app.py +++ b/integration_tests/cdk/app.py @@ -81,18 +81,16 @@ def __init__( instance_type=aws_ec2.InstanceType(app_config.db_instance_type), add_pgbouncer=True, removal_policy=RemovalPolicy.DESTROY, + pgstac_version="0.9.2", ) assert pgstac_db.security_group - # make sure we can get the secret value! - assert pgstac_db.pgstac_secret.secret_value_from_json("host").to_string() - pgstac_db.security_group.add_ingress_rule( aws_ec2.Peer.any_ipv4(), aws_ec2.Port.tcp(5432) ) - PgStacApiLambda( + stac_api = PgStacApiLambda( self, "pgstac-api", db=pgstac_db.connection_target, @@ -100,9 +98,16 @@ def __init__( api_env={ "NAME": app_config.build_service_name("STAC API"), "description": f"{app_config.stage} STAC API", + # test that we can use the pgbouncer secret in downstream resources + "POSTGRES_HOST": pgstac_db.pgstac_secret.secret_value_from_json( + "host" + ).to_string(), }, ) + # make sure stac_api does not try to build before the secret has been boostrapped + stac_api.node.add_dependency(pgstac_db.secret_bootstrapper) + TitilerPgstacApiLambda( self, "titiler-pgstac-api", diff --git a/lib/database/index.ts b/lib/database/index.ts index 5e47df7..1510cb9 100644 --- a/lib/database/index.ts +++ b/lib/database/index.ts @@ -39,6 +39,7 @@ export class PgStacDatabase extends Construct { public readonly connectionTarget: rds.IDatabaseInstance | ec2.Instance; public readonly securityGroup?: ec2.SecurityGroup; + public readonly secretBootstrapper?: CustomResource; constructor(scope: Construct, id: string, props: PgStacDatabaseProps) { super(scope, id); @@ -79,7 +80,7 @@ export class PgStacDatabase extends Construct { code: aws_lambda.Code.fromDockerBuild(__dirname, { file: "bootstrapper_runtime/Dockerfile", buildArgs: { - PYTHON_VERSION: "3.11" + PYTHON_VERSION: "3.11", }, }), vpc: hasVpc(this.db) ? this.db.vpc : props.vpc, @@ -130,16 +131,20 @@ export class PgStacDatabase extends Construct { // if props.lambdaFunctionOptions doesn't have 'code' defined, update pgstac_version (needed for default runtime) if (!props.bootstrapperLambdaFunctionOptions?.code) { - customResourceProperties["pgstac_version"] = props.pgstacVersion || DEFAULT_PGSTAC_VERSION; + customResourceProperties["pgstac_version"] = + props.pgstacVersion || DEFAULT_PGSTAC_VERSION; } - // this.connections = props.database.connections; + + // add timestamp to properties to ensure the Lambda gets re-executed on each deploy + customResourceProperties["timestamp"] = new Date().toISOString(); + const bootstrapper = new CustomResource(this, "bootstrapper", { serviceToken: handler.functionArn, properties: customResourceProperties, removalPolicy: RemovalPolicy.RETAIN, // This retains the custom resource (which doesn't really exist), not the database }); - // PgBouncer: connection pooler + // PgBouncer: connection poolercustomresource trigger on redeploy const addPgbouncer = props.addPgbouncer ?? true; if (addPgbouncer) { this._pgBouncerServer = new PgBouncer(this, "pgbouncer", { @@ -172,6 +177,7 @@ export class PgStacDatabase extends Construct { this.pgstacSecret = this._pgBouncerServer.pgbouncerSecret; this.connectionTarget = this._pgBouncerServer.instance; this.securityGroup = this._pgBouncerServer.securityGroup; + this.secretBootstrapper = this._pgBouncerServer.secretUpdateComplete; } else { this.connectionTarget = this.db; } @@ -226,10 +232,10 @@ export interface PgStacDatabaseProps extends rds.DatabaseInstanceProps { readonly pgstacDbName?: string; /** - * Version of pgstac to install on the database - * - * @default 0.8.5 - */ + * Version of pgstac to install on the database + * + * @default 0.8.5 + */ readonly pgstacVersion?: string; /** From fe124d298915ff92fb275d89437514b04412372f Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Thu, 6 Feb 2025 08:44:14 +0100 Subject: [PATCH 2/2] set upper version for pydantic_ssm_settings --- lib/ingestor-api/runtime/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ingestor-api/runtime/requirements.txt b/lib/ingestor-api/runtime/requirements.txt index 187cd77..07d1813 100644 --- a/lib/ingestor-api/runtime/requirements.txt +++ b/lib/ingestor-api/runtime/requirements.txt @@ -3,7 +3,7 @@ cachetools==5.3.0 fastapi>=0.75.1 orjson>=3.6.8 psycopg[binary,pool]>=3.0.15 -pydantic_ssm_settings>=0.2.0 +pydantic_ssm_settings>=0.2.0,<1.0 pydantic>=1.9.0 pypgstac==0.8.5 requests>=2.27.1