diff --git a/sdlf-dataset/deployspec.yaml b/sdlf-dataset/deployspec.yaml index 6f35e8e8..634799c9 100644 --- a/sdlf-dataset/deployspec.yaml +++ b/sdlf-dataset/deployspec.yaml @@ -12,10 +12,7 @@ deploy: build: commands: - |- - cdk deploy --all --require-approval never --progress events --app "python src/app.py" --outputs-file ./cdk-exports.json \ - --no-previous-parameters \ - --parameters DatasetStack:pDatasetName="$SEEDFARMER_PARAMETER_DATASET" \ - --parameters DatasetStack:pS3Prefix="$SEEDFARMER_PARAMETER_S3_PREFIX" + cdk deploy --all --require-approval never --progress events --app "python src/app.py" --outputs-file ./cdk-exports.json - seedfarmer metadata convert -f cdk-exports.json || true destroy: phases: diff --git a/sdlf-dataset/src/app.py b/sdlf-dataset/src/app.py index 55a1eda3..ed0ec88f 100644 --- a/sdlf-dataset/src/app.py +++ b/sdlf-dataset/src/app.py @@ -12,8 +12,15 @@ if project_name: stack_name = f"{project_name}-{deployment_name}-{module_name}" + param_prefix = "SEEDFARMER_PARAMETER_" else: # app.py not used in a seedfarmer context somehow stack_name = "sdlf-dataset" + param_prefix = "" + + +def _param(name: str, default: str = None) -> str: + return os.getenv(f"{param_prefix}{name}", default) + app = cdk.App() lakeformation_cicd_stack = cdk.Stack(app, f"{stack_name}-lakeformation") @@ -33,7 +40,17 @@ stack = cdk.Stack(app, stack_name) stack.add_dependency(lakeformation_cicd_stack) -stack_dataset = Dataset(stack, "dataset") +stack_dataset = Dataset( + stack, + "dataset", + raw_bucket=_param("RAW_BUCKET"), + stage_bucket=_param("STAGE_BUCKET"), + analytics_bucket=_param("ANALYTICS_BUCKET"), + artifacts_bucket=_param("ARTIFACTS_BUCKET"), + lakeformation_dataaccess_role=_param("LAKEFORMATION_DATAACCESS_ROLE"), + dataset=_param("DATASET"), + s3_prefix=_param("S3_PREFIX"), +) cdk.CfnOutput( scope=stack, diff --git a/sdlf-dataset/src/dataset.py b/sdlf-dataset/src/dataset.py index ce49b939..0d5bf200 100644 --- a/sdlf-dataset/src/dataset.py +++ b/sdlf-dataset/src/dataset.py @@ -28,11 +28,37 @@ class Dataset(Construct): external_interface = {} - def __init__(self, scope: Construct, id: str, **kwargs) -> None: + def __init__( + self, + scope: Construct, + id: str, + dataset: str, + s3_prefix: str, + org: str = None, + data_domain: str = None, + raw_bucket: str = None, + stage_bucket: str = None, + analytics_bucket: str = None, + artifacts_bucket: str = None, + lakeformation_dataaccess_role: str = None, + **kwargs, + ) -> None: super().__init__(scope, id) - # using context values would be better(?) for CDK but we haven't decided yet what the story is around ServiceCatalog and CloudFormation modules - # perhaps both (context values feeding into CfnParameter) would be a nice-enough solution. Not sure though. TODO + # if arguments are passed to the constructor, their values are fed to CfnParameter() below as default + # if arguments aren't specified, standard SDLF SSM parameters are checked instead + # the jury is still out on the usage of CfnParameter(), + # and there is still work to get the latest SSM parameter values as well as using a prefix to allow multiple deployments TODO + raw_bucket = raw_bucket or "{{resolve:ssm:/sdlf/storage/rRawBucket:1}}" + stage_bucket = stage_bucket or "{{resolve:ssm:/sdlf/storage/rStageBucket:1}}" + analytics_bucket = analytics_bucket or "{{resolve:ssm:/sdlf/storage/rAnalyticsBucket:1}}" + artifacts_bucket = artifacts_bucket or "{{resolve:ssm:/sdlf/storage/rArtifactsBucket:1}}" + lakeformation_dataaccess_role = ( + lakeformation_dataaccess_role or "{{resolve:ssm:/sdlf/storage/rLakeFormationDataAccessRoleArn:1}}" + ) + org = org or "{{resolve:ssm:/sdlf/storage/rOrganization:1}}" + data_domain = data_domain or "{{resolve:ssm:/sdlf/storage/rDomain:1}}" + p_pipelinereference = CfnParameter( self, "pPipelineReference", @@ -45,7 +71,7 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None: "pOrg", description="Name of the organization owning the datalake", type="String", - default="{{resolve:ssm:/sdlf/storage/pOrg:1}}", + default="{{resolve:ssm:/sdlf/storage/rOrganization:1}}", ) p_org.override_logical_id("pOrg") p_domain = CfnParameter( @@ -53,7 +79,7 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None: "pDomain", description="Data domain name", type="String", - default="{{resolve:ssm:/sdlf/storage/pDomain:1}}", + default="{{resolve:ssm:/sdlf/storage/rDomain:1}}", ) p_domain.override_logical_id("pDomain") p_rawbucket = CfnParameter( @@ -61,7 +87,7 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None: "pRawBucket", description="The raw bucket for the solution", type="String", - default="{{resolve:ssm:/sdlf/storage/rRawBucket:1}}", + default=raw_bucket, ) p_rawbucket.override_logical_id("pRawBucket") p_stagebucket = CfnParameter( @@ -69,7 +95,7 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None: "pStageBucket", description="The stage bucket for the solution", type="String", - default="{{resolve:ssm:/sdlf/storage/rStageBucket:1}}", + default=stage_bucket, ) p_stagebucket.override_logical_id("pStageBucket") p_analyticsbucket = CfnParameter( @@ -77,7 +103,7 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None: "pAnalyticsBucket", description="The analytics bucket for the solution", type="String", - default="{{resolve:ssm:/sdlf/storage/rAnalyticsBucket:1}}", + default=analytics_bucket, ) p_analyticsbucket.override_logical_id("pAnalyticsBucket") p_artifactsbucket = CfnParameter( @@ -85,14 +111,14 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None: "pArtifactsBucket", description="The artifacts bucket used by CodeBuild and CodePipeline", type="String", - default="{{resolve:ssm:/sdlf/storage/rArtifactsBucket:1}}", + default=artifacts_bucket, ) p_artifactsbucket.override_logical_id("pArtifactsBucket") p_lakeformationdataaccessrole = CfnParameter( self, "pLakeFormationDataAccessRole", type="String", - default="{{resolve:ssm:/sdlf/storage/rLakeFormationDataAccessRoleArn:1}}", + default=lakeformation_dataaccess_role, ) p_lakeformationdataaccessrole.override_logical_id("pLakeFormationDataAccessRole") p_datasetname = CfnParameter( @@ -101,6 +127,7 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None: description="The name of the dataset (all lowercase, no symbols or spaces)", type="String", allowed_pattern="[a-z0-9]{2,14}", + default=dataset, ) p_datasetname.override_logical_id("pDatasetName") p_s3prefix = CfnParameter( @@ -109,6 +136,7 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None: description="S3 prefix or full bucket if empty/not provided", type="String", allowed_pattern="[a-z0-9]*", + default=s3_prefix, ) p_s3prefix.override_logical_id("pS3Prefix") p_cicdrole = CfnParameter(