Merge pull request #482 from awslabs/release/3.1.0

Release/3.1.0

CHANGELOG.md

@@ -12,6 +12,22 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Ch
 
 ### Fixes
 
+
+## v3.1.0 (2024-01-22)
+
+### New
+- adding AWS curated codebuild iamge override with opinionated runtimes
+
+### Changes
+- updating pydantic support from 1.X.X to 2.5.3
+- adding seedfarmer verions check support with `seedfarmer.yaml`
+- updating `aws-codeseeder` dependency top 0.11.0
+
+### Fixes
+- update `manifests/examples/` to point to an updated release branch
+- Docs - manifest name description (seed-farmer/docs/source/manifests.md) needed correction
+- Docs - added definition of `nameGenerator` for deployment manifest (seed-farmer/docs/source/manifests.md)
+
 ## v3.0.1 (2023-11-10)
 
 ### New

VERSION

@@ -1 +1 @@
-3.0.1
+3.1.0

docs/requirements-docs.txt

@@ -12,21 +12,23 @@ babel==2.12.1
     # via sphinx
 certifi==2023.7.22
     # via
-    #   -r docs/requirements-docs.in
+    #   -r requirements-docs.in
     #   requests
 charset-normalizer==3.1.0
     # via requests
 docutils==0.18.1
     # via
-    #   -r docs/requirements-docs.in
+    #   -r requirements-docs.in
     #   myst-parser
     #   sphinx
     #   sphinx-rtd-theme
 idna==3.4
     # via requests
 imagesize==1.4.1
     # via sphinx
-jinja2==3.1.2
+importlib-metadata==7.0.1
+    # via sphinx
+jinja2==3.1.3
     # via
     #   myst-parser
     #   sphinx
@@ -44,14 +46,14 @@ mdit-py-plugins==0.3.5
 mdurl==0.1.2
     # via markdown-it-py
 myst-parser==1.0.0
-    # via -r docs/requirements-docs.in
+    # via -r requirements-docs.in
 packaging==23.1
     # via sphinx
 pygments==2.15.1
     # via sphinx
 pyyaml==5.4
     # via
-    #   -r docs/requirements-docs.in
+    #   -r requirements-docs.in
     #   myst-parser
     #   sphinx-autoapi
 requests==2.31.0
@@ -65,46 +67,50 @@ sphinx==6.2.1
     #   sphinx-rtd-theme
     #   sphinxcontrib-jquery
 sphinx-autoapi==2.1.0
-    # via -r docs/requirements-docs.in
+    # via -r requirements-docs.in
 sphinx-rtd-theme==1.2.1
-    # via -r docs/requirements-docs.in
+    # via -r requirements-docs.in
 sphinxcontrib-applehelp==1.0.4
     # via
-    #   -r docs/requirements-docs.in
+    #   -r requirements-docs.in
     #   sphinx
 sphinxcontrib-devhelp==1.0.2
     # via
-    #   -r docs/requirements-docs.in
+    #   -r requirements-docs.in
     #   sphinx
 sphinxcontrib-htmlhelp==2.0.1
     # via
-    #   -r docs/requirements-docs.in
+    #   -r requirements-docs.in
     #   sphinx
 sphinxcontrib-jquery==4.1
     # via
-    #   -r docs/requirements-docs.in
+    #   -r requirements-docs.in
     #   sphinx-rtd-theme
 sphinxcontrib-jsmath==1.0.1
     # via
-    #   -r docs/requirements-docs.in
+    #   -r requirements-docs.in
     #   sphinx
 sphinxcontrib-qthelp==1.0.3
     # via
-    #   -r docs/requirements-docs.in
+    #   -r requirements-docs.in
     #   sphinx
 sphinxcontrib-serializinghtml==1.1.5
     # via
-    #   -r docs/requirements-docs.in
+    #   -r requirements-docs.in
     #   sphinx
 typing-extensions==4.5.0
-    # via -r docs/requirements-docs.in
+    # via
+    #   -r requirements-docs.in
+    #   astroid
 unidecode==1.3.6
     # via sphinx-autoapi
 urllib3==1.26.18
     # via
-    #   -r docs/requirements-docs.in
+    #   -r requirements-docs.in
     #   requests
 wheel==0.38.1
-    # via -r docs/requirements-docs.in
+    # via -r requirements-docs.in
 wrapt==1.15.0
     # via astroid
+zipp==3.17.0
+    # via importlib-metadata

docs/source/manifests.md

@@ -9,6 +9,11 @@ The deployment manifest is the top level manifest and resides in the `modules` d
 
 ```yaml
 name: examples
+nameGenerator:
+  prefix: myprefix
+  suffix:
+    valueFrom:
+        envVariable: SUFFIX_ENV_VARIABLE
 toolchainRegion: us-west-2
 forceDependencyRedeploy: False
 groups:
@@ -69,6 +74,12 @@ targetAccountMappings:
 ```
 
 - **name** : this is the name of your deployment.  There can be only one deployment with this name in a project.
+  - THIS CANNOT BE USED WITH `nameGenerator`
+- **nameGenerator** : this supports dynamically generating a deployment name by concatenation of the following fields:
+  - **prefix** - the prefix string of the name
+  - **suffix** - the suffix string of the name
+  - Both of these fields support the use of [Environment Variables](envVariable) (see example above)
+  - THIS CANNOT BE USED WITH `name`
 - **toolchainRegion** :the designated region that the `toolchain` is created in
 - **forceDependencyRedeploy**: this is a boolean that tells seedfarmer to redeploy ALL dependency modules (see [Force Dependency Redeploy](force-redeploy)) - Default is `False`
 - **groups** : the relative path to the [`module manifests`](module_manifest) that define each module in the group.  This sequential order is preserved in deployment, and reversed in destroy.
@@ -80,14 +91,14 @@ targetAccountMappings:
   - **alias** - the logical name for an account, referenced by [`module manifests`](module_manifest)
   - **account** - the account id tied to the alias.  This parameter also supports [Environment Variables](envVariable)
   - **default** - this designates this mapping as the default account for all modules unless otherwise specified.  This is primarily for supporting migrating from `seedfarmer v1` to the current version.
-  - **codebuildImage** - a custom build image to use (see [Custom Build Image](custombuildimage))
+  - **codebuildImage** - a custom build image to use (see [Build Image Override](buildimageoverride))
   - **parametersGlobal** - these are parameters that apply to all region mappings unless otherwise overridden at the region level
     - **dockerCredentialsSecret** - please see [Docker Credentials Secret](dockerCredentialsSecret)
     - **permissionsBoundaryName** - the name of the [permissions boundary](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_boundaries.html) policy to apply to all module-specific roles created
   - **regionMappings** - section to define region-specific configurations for the defined account, this is a list
     - **region** - the region name
     - **default** - this designates this mapping as the default region for all modules unless otherwise specified.  This is primarily for supporting migrating
-    - **codebuildImage** - a custom build image to use (see [Custom Build Image](custombuildimage))
+    - **codebuildImage** - a custom build image to use (see [Build Image Override](buildimageoverride))
     - **parametersRegional** - these are parameters that apply to all region mappings unless otherwise overridden at the region level
       - **dockerCredentialsSecret** - please see [Docker Credentials Secret](dockerCredentialsSecret)
         - This is a NAMED PARAMETER...in that `dockerCredentialsSecret` is recognized by `seed-farmer`
@@ -183,10 +194,10 @@ network:
       envVariable: SECURITYGROUPS
 ```
 The corresponding `.env` file would have the following defined (again, remember the lists!!):
-```code
+```bash
 VPCID="vpc-0c4cb9e06c9413222"
-PRIVATESUBNETS=["subnet-0c36d3d5808f67a02","subnet-00fa1e71cddcf57d3"]
-SECURITYGROUPS=["sg-049033188c114a3d2"]
+PRIVATESUBNETS='["subnet-0c36d3d5808f67a02","subnet-00fa1e71cddcf57d3"]'
+SECURITYGROUPS='["sg-049033188c114a3d2"]'
 ```
 (dependency-management)=
 ### Dependency Management
@@ -211,7 +222,7 @@ What does this mean?  Well, lets take the following module deployment order:
  **This is an important feature to understand: redeployment is not discriminant.**  SeedFarmer does not know how to assess what has changed in a module and its impact on downstream modules.  Nor does it have the ability to know if a module can incur a redeployment (as opposed to a destroy and deploy process).  That is up to you to determine with respect to the modules you are leveraging.  ANY change to the source code (deployspec, modulestack, comments in cdk code, etc.) will indicate to SeedFarmer that the module needs to be redeployed, even if the underlying logic / artifact has not changed.  
 
  Also, it is important to understand that this feature could put your deployment in an unusable state if the shared-responsibility model is not followed.
- For example: lets say a deployment has a module (called `networking`) that deploys a VPC with public and private subnets that are restricted to a particular CIDR (as input).  Then, downstream modules reference the metadata of `netowrking`.  If a user were to change the CIDR references and redeploy the `networking` module, this has the potential to render the deployment in an unusable state: the process to change the CIDR's would trigger a destroy of the existing subnets...which would fail due to resources from other modules leveraging those subnets.  The redeployment would fail, and the user would have to manually correct the state.
+ For example: lets say a deployment has a module (called `networking`) that deploys a VPC with public and private subnets that are restricted to a particular CIDR (as input).  Then, downstream modules reference the metadata of `networking`.  If a user were to change the CIDR references and redeploy the `networking` module, this has the potential to render the deployment in an unusable state: the process to change the CIDR's would trigger a destroy of the existing subnets...which would fail due to resources from other modules leveraging those subnets.  The redeployment would fail, and the user would have to manually correct the state.
 
 (module_manifest)=
 ## Module Manifest
@@ -244,13 +255,14 @@ dataFiles:
   - filePath: test1.txt
   - filePath: git::https://github.com/awslabs/idf-modules.git//modules/storage/buckets/deployspec.yaml?ref=release/1.0.0&depth=1
 ```
-- **name** - the name of the group
+- **name** - the name of the module
+  - this name must be unique in the group of the deployment
 - **path** - this element supports two sources of code:
-  - the relative path to the module code in the project
+  - the relative path to the module code in the project if deploying code from the local filesystem
   - a public Git Repository, leveraging the Terraform semantic as denoted [HERE](https://www.terraform.io/language/modules/sources#generic-git-repository)
 - **targetAccount** - the alias of the account from the [deployment manifest mappings](deployment_manifest)
 - **targetRegion** - the name of the region to deploy to - this overrides any mappings
-- **codebuildImage** - a custom build image to use (see [Custom Build Image](custombuildimage))
+- **codebuildImage** - a custom build image to use (see [Build Image Override](buildimageoverride))
 - **parameters** - the parameters section .... see [Parameters](parameters)
 - **dataFiles** - additional files to add to the bundle that are outside of the module code
   - this is LIST and EVERY element in the list must have the keyword **filePath**
@@ -276,13 +288,30 @@ When using this feature, any change to these file(s) (modifying, add to manifest
 ***Iceburg, dead ahead!*** Heres the rub: if you deploy with data files sourced from a local filesystem, you MUST provide those same files in order to destroy the module(s)...we are not keeping them stored anywhere (much like the module source code).  ***Iceburg  missed us! (why is everthing so wet??)***
 
 
-(custombuildimage)=
-## Custom Codebuild Image
-`seed-farmer` is preconfigued to use the optimal build image and we recommend using it as-is (no need to leverage the `codebuildImage` manifest named paramter).  But, we get it....no one wants to be boxed in.</br>
+(buildimageoverride)=
+## Codebuild Image Override
+An AWS Codebuild complaint image is provided for use with `seed-farmer` and we recommend using it as-is (no need to leverage the `codebuildImage` manifest named paramter).  But, we get it....no one wants to be boxed in.</br>
+
 <b>USER BEWARE</b> - this is a feature meant for advanced users...use at own risk!
+
+Users can override the default build image via one of the following:
+- an AWS Curated Build Image
+- a custom-built image 
+
+#### AWS Curated Build Images
+There are multiple [build images and available runtimes](https://docs.aws.amazon.com/codebuild/latest/userguide/available-runtimes.html) that are supported by AWS Codebuild.  For `seed-farmer`, we currently support the following AWS Curated Images with the default runtimes installed:
+
+| AWS Curated Build Image | Confgured Runtimes|
+| ----------- | ----------- |    
+|aws/codebuild/standard:6.0|nodejs:16|
+||python:3.10|
+||java:corretto17|
+|aws/codebuild/standard:7.0|nodejs:18|
+||python:3.11|
+||java:corretto21|
 
-### The Build Image
-An AWS Codebuild complaint image is provided for use with `seed-farmer` and the CLI is configured by default to use this image.  Advanced users have the option of building their own image and configuring their deployment to use it.  If an end user wants to build their own image, it is STRONGLY encouraged to use [this Dockerfile from AWS public repos](https://github.com/awslabs/aws-codeseeder/blob/main/images/code-build-image/Dockerfile) as the base layer.  `seed-farmer` leverages this as the base for its default image ([see HERE](https://github.com/awslabs/aws-codeseeder/blob/main/images/code-build-image/Dockerfile)).
+#### Custom Build Images
+If an end user wants to build their own image, it is STRONGLY encouraged to use [this Dockerfile from AWS public repos](https://github.com/awslabs/aws-codeseeder/blob/main/images/code-build-image/Dockerfile) as the base layer.  `seed-farmer` leverages this as the base for its default image ([see HERE](https://github.com/awslabs/aws-codeseeder/blob/main/images/code-build-image/Dockerfile)).  It is up to the module developer to verify all proper libraries are installed and available.
 
 ### Logic for Rules -- Application
 There are three (3) places to configure a custom build image:
@@ -420,7 +449,7 @@ parameters:
     valueFrom:
       parameterValue: mygreatkey
 ```
-`seed-farrmer` will first look in the Regional Parameters for a matching key, and return a string object (all json convert to a string) represening the value.  If not found, `seed-farrmer` will look in the Global Parameters for the same key and return that string-ified value.
+`seed-farmer` will first look in the Regional Parameters for a matching key, and return a string object (all json convert to a string) represening the value.  If not found, `seed-farrmer` will look in the Global Parameters for the same key and return that string-ified value.
 
 NOTE: the `network` section of the [deployment manifest](deployment_manifest) leverages Regional Parameters only!
 

docs/source/project_development.md

@@ -27,13 +27,15 @@ It is important to have the ```seedfarmer.yaml``` at the root of your project.
 project: <your project name>
 description: <your project description>
 projectPolicyPath: <relativepath/policyname.yaml>
+seedfarmer_version: <minimum required seedfermer version>
 ```
 - **project** (REQUIRED) - this is the name of the project that all deployments will reference 
 - **description** (OPTIONAL) - this is the description of the project
 - **projectPolicyPath** (OPTIONAL) - this allows advanced users change the project policy that has the basic minimim permissions seedfarmer needs
   - it consists of a path relative to the project root and MUST be a valid relative path
   - to synth the existing project policy, run `seedfarmer projectpolicy synth` 
-
+- **seedfarmer_version** (OPTIONAL) - this specifies what is the minimum allowable version of `seed-farmer` the project supports
+  - if this value is set AND the runtime version of seedfarmer is greater, `seed-farmer` will exit immediately
 
 
 (project_initalization)=

examples/exampleproject/manifests-isolated/examples/optional-modules-2.yaml

@@ -1,12 +1,12 @@
 # name: networking
-# path: git::https://github.com/awslabs/seedfarmer-modules.git//modules/network/basic-cdk/?ref=release/1.0.0&depth=1
+# path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk/?ref=release/1.2.0&depth=1
 # targetAccount: primary
 # parameters:
 #   - name: internet-accessible
 #     value: true
 # ---
 name: buckets
-path: git::https://github.com/awslabs/seedfarmer-modules.git//modules/storage/buckets/?ref=release/1.0.0&depth=1
+path: git::https://github.com/awslabs/idf-modules.git//modules/storage/buckets/?ref=release/1.2.0&depth=1
 targetAccount: secondary
 targetRegion: us-west-2
 parameters:

examples/exampleproject/manifests-isolated/examples/optional-modules.yaml

@@ -1,12 +1,12 @@
 # name: networking
-# path: git::https://github.com/awslabs/seedfarmer-modules.git//modules/network/basic-cdk/?ref=release/1.0.0&depth=1
+# path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk/?ref=release/1.2.0&depth=1
 # targetAccount: secondary
 # parameters:
 #   - name: internet-accessible
 #     value: true
 # ---
 name: buckets
-path: git::https://github.com/awslabs/seedfarmer-modules.git//modules/storage/buckets/?ref=release/1.0.0&depth=1
+path: git::https://github.com/awslabs/idf-modules.git//modules/storage/buckets/?ref=release/1.2.0&depth=1
 targetAccount: primary
 targetRegion: us-east-2
 parameters:

examples/exampleproject/manifests/examples/optional-modules.yaml

@@ -1,11 +1,11 @@
 name: networking
-path: git::https://github.com/awslabs/seedfarmer-modules.git//modules/network/basic-cdk/?ref=release/1.0.0&depth=1
+path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk/?ref=release/1.2.0&depth=1
 parameters:
   - name: internet-accessible
     value: true
 ---
 name: buckets
-path: git::https://github.com/awslabs/seedfarmer-modules.git//modules/storage/buckets/?ref=release/1.0.0&depth=1
+path: git::https://github.com/awslabs/idf-modules.git//modules/storage/buckets/?ref=release/1.2.0&depth=1
 parameters:
   - name: encryption-type
     value: SSE

pyproject.toml

@@ -74,6 +74,7 @@ markers = [
     "commands_parameters: marks all `commands_parameters` tests",
     "commands_modules: marks all `commands_modules` tests",
     "commands_deployment: marks all `commands_deployment` tests",
+    "commands_bootstrap: marks all `commands_bootstrap` tests",
 ]
 log_cli_level = "INFO"
 addopts = "-v --cov=. --cov-report=term --cov-report=html --cov-config=coverage.ini --cov-fail-under=80"

requirements-dev.txt

@@ -48,8 +48,10 @@ click==8.1.6
 colorama==0.4.4
     # via awscli
 coverage[toml]==7.2.7
-    # via pytest-cov
-cryptography==41.0.4
+    # via
+    #   coverage
+    #   pytest-cov
+cryptography==41.0.6
     # via
     #   moto
     #   secretstorage
@@ -61,6 +63,8 @@ docutils==0.16
     #   readme-renderer
     #   sphinx
     #   sphinx-rtd-theme
+exceptiongroup==1.2.0
+    # via pytest
 flake8==4.0.1
     # via -r requirements-dev.in
 idna==3.4
@@ -114,7 +118,9 @@ mdurl==0.1.2
 more-itertools==9.1.0
     # via jaraco-classes
 moto[codebuild,iam,s3,secretsmanager,ssm,sts]==4.0.13
-    # via -r requirements-dev.in
+    # via
+    #   -r requirements-dev.in
+    #   moto
 mypy==0.991
     # via -r requirements-dev.in
 mypy-extensions==1.0.0
@@ -239,6 +245,16 @@ sphinxcontrib-qthelp==1.0.3
     # via sphinx
 sphinxcontrib-serializinghtml==1.1.5
     # via sphinx
+tomli==2.0.1
+    # via
+    #   black
+    #   build
+    #   check-manifest
+    #   coverage
+    #   mypy
+    #   pip-tools
+    #   pyproject-hooks
+    #   pytest
 trove-classifiers==2023.7.6
     # via pyroma
 twine==4.0.2
@@ -251,6 +267,8 @@ types-setuptools==57.4.18
     # via -r requirements-dev.in
 typing-extensions==4.7.1
     # via
+    #   astroid
+    #   black
     #   mypy
     #   myst-parser
 unidecode==1.3.6