Doshitan/wip

bin/templatize-legacy-infra

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+# Convert a template-infra worktree into one that minimally supports the
+# Platform CLI tooling.
+#
+# Run from the root of a template-infra worktree.
+
+set -eu
+
+rename_file() {
+    local original_file=$1
+    local renamed_file=$2
+
+    mkdir -p "${renamed_file%/*}"
+    git mv "$original_file" "${renamed_file}"
+}
+
+rename_app_file() {
+    local original_file=$1
+    local renamed_file="${original_file/app/{{app_name\}\}}"
+
+    rename_file "${original_file}" "${renamed_file}"
+}
+
+if [[ -d .template-infra/ ]]; then
+    echo "Already templatized"
+    exit
+fi
+
+# get rid of example app junk
+find app -not -name Makefile -delete &> /dev/null || :
+git add -u
+
+# rename "app" files
+for f in $(git ls-files '*app*' | grep -v '^docs/' | grep -v 'template-only' | grep '\bapp\b'); do
+    rename_app_file "${f}"
+done
+
+# rather than copy over the upstream copier file:
+#
+#   curl -O https://raw.githubusercontent.com/navapbc/template-infra/refs/heads/main/copier.yml
+#
+# we'll use a simplified version, since the only variable we are using is
+# `app_name`
+cat <<EOF > copier.yml
+template:
+  type: str
+  choices:
+    - base
+    - app
+
+#
+# App vars
+#
+app_name:
+  type: str
+  help: The name of the app
+  validator: >-
+    {% if not (app_name | regex_search('^[a-z0-9\-_]+$')) %}
+    The app name can not be empty and should only contain lower case letters, digits, dashes, and underscores.
+    {% endif %}
+  when: &app
+    "{{ template == 'app' }}"
+
+_envops:
+  trim_blocks: true
+  lstrip_blocks: true
+
+_skip_if_exists:
+  - "/{{ app_name }}/"
+  - "/{{ app_name }}/Makefile"
+
+_exclude:
+  - /.git
+  - /copier.yml
+  - /CODEOWNERS
+  - /CONTRIBUTING.md
+  - /LICENSE.md
+  - /README.md
+EOF
+
+# create answers file
+mkdir .template-infra
+echo -e "# Changes here will be overwritten by Copier\n{{ _copier_answers|to_nice_yaml -}}" > '.template-infra/{{_copier_conf.answers_file}}.jinja'
+
+# fixup CI stuff, for file/path triggers
+for f in $(rg -l -e 'app/' -e '-app-' .github/ | grep -v 'template-only' | grep -v 'README.md'); do
+    renamed_file=$f.jinja
+    git mv "${f}" "${renamed_file}"
+    sed -ri 's|app/|{{ app_name }}/|g' "${renamed_file}"
+    # sed -ri 's|-app-|-{{ app_name }}-|g' "${renamed_file}"
+    sed -ri "s/-app(\b.*yml)/-{{ app_name}}\1/g" "${renamed_file}"
+    sed -ri "s/app_name: app/app_name: {{ app_name}}/g" "${renamed_file}"
+    sed -ri 's/app_name: "app"/app_name: {{ app_name}}/g' "${renamed_file}"
+    # sed -ri 's/\$\{\{(.*)\}\}/\$\{\{"\{\{"\}\}\1\{\{"\}\}"\}\}/g' "${renamed_file}"
+    perl -pi -e 's|\$\{\{(.*?)\}\}|\$\{\{"\{\{"\}\}\1\{\{"\}\}"\}\}|g' "${renamed_file}"
+    sed -ri "s/\bApp\b/{{ app_name}}/g" "${renamed_file}"
+done
+
+git add copier.yml .template-infra/

bin/templatize-legacy-infra-version

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Run from the root of a template-infra repo
+
+set -eux
+
+VERSION=$1
+
+# add any executables next to this script to the PATH for easy calling, like
+# templatize-legacy-infra
+SCRIPT_PATH=$(dirname "$(realpath -s "$0")")
+PATH=${SCRIPT_PATH}:${PATH}
+export PATH
+
+worktree_path=$(realpath "../template-infra-${VERSION}")
+
+git worktree add -d "${worktree_path}" "${VERSION}"
+
+pushd "${worktree_path}"
+
+templatize-legacy-infra
+
+msg="${VERSION} Platform CLI Migration Checkpoint"
+git commit --all -m "${msg}"
+git tag -a "platform-cli-migration/${VERSION}" -m "${msg}"
+
+# could push up the tag:
+#
+#   git push origin "platform-cli-migration/${VERSION}"
+#
+# but will do that manually
+
+popd
+
+git worktree remove "${worktree_path}"

docs/getting-started/migrating-from-legacy-template.md

@@ -8,14 +8,35 @@ Platform CLI you'll need to convert the old file into the new format.
 The Platform CLI provides commands for doing this migration, though the exact
 steps you need to take will vary depending on what templates you have installed.
 
-> [!IMPORTANT]
->
-> If you running a very old (pre-summer 2024) version of a template
-> (particularly `template-infra`), reach out to the platform team for some
-> guidance.
-
 ## template-infra
 
+The switch to Platform CLI happened with `v0.15.0`. If you are running a version
+earlier than this, you'll need to migrate things.
+
+One way to figure out what version of `template-infra` your project is using is
+to run, at the root of your project:
+
+```sh
+nava-platform infra info --template-uri gh:navapbc/template-infra .
+```
+
+Look for the "Closest upstream version" value. If it is "Unknown", reach out to
+the Platform team for guidance.
+
+If the value is pre-`v0.12.0`, you may want to approach the update in smaller
+steps than jumping directly to `v0.15.0`. You can use the Platform CLI to do
+these updates as well, see [Migrate in smaller
+steps](#migrate-in-smaller-steps).
+
+As always, read the [release
+notes](https://github.com/navapbc/template-infra/releases) for each version
+between your current one and your ultimate target. This process does not
+eliminate the need to apply the state changes/manual migration steps, it just
+updates the code. See [Version callouts](#version-callouts) below for some
+particular things to consider.
+
+### Migrate to latest
+
 To transform the old `.template-version` file into the new format, run:
 
 ```sh
@@ -26,7 +47,9 @@ This will result in a `.template-infra/` directory with a number of files inside
 of it. Check that the `app-<APP_NAME>.yml` files all correspond to proper
 applications. Remove any that don't and update the commit.
 
-Now perform the update, with:
+This gets your project into a state that Platform CLI can understand.
+
+Now perform the actual template update, with:
 
 ```sh
 nava-platform infra update .
@@ -46,6 +69,78 @@ nava-platform infra update-app --all .
 Likely you'll hit merge conflicts for each app as well, resolve those, commit,
 and move on to the next app, until you've done them all.
 
+See [the docs on updating in general](../updating.md) for more details on running
+updates.
+
+### Migrate in smaller steps
+
+This is similar to the previous section, so read that first.
+
+1. Run the `migrate-from-legacy` command as stated in previous section. This
+   gets you into the Platform CLI ecosystem.
+2. Then decide which version of `template-infra` you want to update to,
+   represented by `v0.x.x` in the following example:
+   ```sh
+   nava-platform infra update --version platform-cli-migration/v0.x.x .
+   ```
+3. Follow update guidance as discussed in previous section.
+4. Do steps 2-3 over and over, jumping versions as you see fit until you hit
+   `v0.15.0`.
+5. Once on `v0.15.0`, run a final update to get to the latest release (or to
+   whatever post-`v0.15.0` version you want):
+   ```sh
+   nava-platform infra update [--version vA.B.C] .
+   ```
+
+### Version callouts
+
+No substitute for reading the [release
+notes](https://github.com/navapbc/template-infra/releases), but here are a few
+points to consider when deciding what version to update to if you are
+significantly behind the latest:
+
+- A Feature Flags module, backed by AWS Evidently, was added in
+  [v0.5.0](https://github.com/navapbc/template-infra/releases/tag/v0.5.0) and
+  removed in
+  [v0.13.0](https://github.com/navapbc/template-infra/releases/tag/v0.13.0).
+    - If you are coming from pre-v0.5.0, you can delete the feature flag module
+      as you move past v0.5.0, or just ignore/don't change anything about it and
+      it will get cleaned up once you are post-v0.13.0.
+- [v0.9.0](https://github.com/navapbc/template-infra/releases/tag/v0.9.0) moved
+  account mapping to each environment config file, then
+  [v0.11.0](https://github.com/navapbc/template-infra/releases/tag/v0.11.0)
+  removed it from each environment config file and moved it to the network config.
+    - If you are pre-v0.9.0, you may want to consider jumping to v0.11.x+ to
+      avoid dealing with moving things multiple times.
+
+Misc. others:
+
+- [v0.11.0](https://github.com/navapbc/template-infra/releases/tag/v0.11.0)
+    - Starts pinning specific Terraform version in CI/CD
+- [v0.10.0](https://github.com/navapbc/template-infra/releases/tag/v0.10.0)
+    - DB changes: PostgreSQL version update to 16.2 and DB schema name hardcoded
+to `app`
+- [v0.9.0](https://github.com/navapbc/template-infra/releases/tag/v0.9.0)
+    - Requires Terraform 1.8.x (previous requirement was just >=1.4, more or less)
+    - Changes the way secrets are defined
+- [v0.7.0](https://github.com/navapbc/template-infra/releases/tag/v0.7.0)
+    - Minor state migration needed
+- [v0.6.0](https://github.com/navapbc/template-infra/releases/tag/v0.6.0)
+    - Networking changes likely requiring hours of downtime to apply
+
+### Post-migration
+
+After completing the migration, you may want to see what results from
+re-applying, more holistically, the latest (or your ultimate target) version of
+the template to the project:
+
+```sh
+nava-platform infra update --force [--version vA.B.C] .
+```
+
+This discards some of the "smart" logic of a regular update and might catch some
+things that were missed while trying to be smarter.
+
 ## Application templates
 
 These are historically less standard, so you'll have to provide a little more

docs/updating.md

@@ -24,10 +24,12 @@ Conflicts on Update](./avoiding-conflicts-on-update.md).
 nava-platform infra update .
 ```
 
-This can often run into merge conflicts that need resolved manually though. The
-tool will provide some guidance if this happens. But you can also approach the
-update in the separate pieces yourself, first updating the infrastructure base
-with:
+This will attempt to update the "base" template then each "app" instance in
+sequence. This can often run into merge conflicts that need resolved manually.
+The tool will provide some guidance if this happens.
+
+But you can also approach the update in the separate pieces yourself, first
+updating the infrastructure base with:
 
 ```sh
 nava-platform infra update-base .

nava/platform/cli/commands/infra/__init__.py

@@ -91,7 +91,7 @@ def add_app(
 def update(
     typer_context: typer.Context,
     project_dir: str,
-    template_uri: Annotated[str, opt_template_uri] = DEFAULT_TEMPLATE_URI,
+    template_uri: Annotated[str | None, opt_template_uri] = None,
     version: Annotated[str | None, opt_version] = DEFAULT_VERSION,
     data: Annotated[list[str] | None, opt_data] = None,
     answers_only: Annotated[bool, opt_answers_only] = False,
@@ -110,8 +110,8 @@ def update(
         try:
             update_command.update(
                 ctx,
-                template_uri,
                 project_dir,
+                template_uri=template_uri,
                 version=version if not answers_only else None,
                 data=dict_util.from_str_values(data),
                 answers_only=answers_only,
@@ -129,7 +129,7 @@ def update(
 def update_base(
     typer_context: typer.Context,
     project_dir: str,
-    template_uri: Annotated[str, opt_template_uri] = DEFAULT_TEMPLATE_URI,
+    template_uri: Annotated[str | None, opt_template_uri] = None,
     version: Annotated[str | None, opt_version] = DEFAULT_VERSION,
     data: Annotated[list[str] | None, opt_data] = None,
     commit: Annotated[bool, opt_commit] = True,
@@ -142,8 +142,8 @@ def update_base(
     with ctx.handle_exceptions():
         update_command.update_base(
             ctx,
-            template_uri,
             project_dir,
+            template_uri=template_uri,
             version=version if not answers_only else None,
             data=dict_util.from_str_values(data),
             commit=commit,
@@ -157,7 +157,7 @@ def update_app(
     typer_context: typer.Context,
     project_dir: str,
     app_name: Annotated[list[str] | None, typer.Argument()] = None,
-    template_uri: Annotated[str, opt_template_uri] = DEFAULT_TEMPLATE_URI,
+    template_uri: Annotated[str | None, opt_template_uri] = None,
     version: Annotated[str | None, opt_version] = DEFAULT_VERSION,
     data: Annotated[list[str] | None, opt_data] = None,
     commit: Annotated[bool, opt_commit] = True,
@@ -171,8 +171,8 @@ def update_app(
     with ctx.handle_exceptions():
         update_command.update_app(
             ctx,
-            template_uri,
             project_dir,
+            template_uri=template_uri,
             app_names=app_name,
             version=version if not answers_only else None,
             data=dict_util.from_str_values(data),

nava/platform/cli/commands/infra/migrate_from_legacy_command.py

@@ -35,7 +35,9 @@ def _migrate_from_legacy(
         new_version_answers_file_name="base.yml",
         extra_answers=lambda _: (base_project_config_answers | {"template": "base"}),
     )
-    base_migrate.migrate_from_legacy(preserve_legacy_file=True, commit=commit)
+    base_migrate.migrate_from_legacy(
+        preserve_legacy_file=True, commit=commit, use_migration_tags=True
+    )
 
     for app_name in infra_project.app_names_possible:
         app_answers = {"app_name": app_name, "template": "app"}
@@ -48,7 +50,9 @@ def _migrate_from_legacy(
             new_version_answers_file_name=f"app-{app_name}.yml",
             extra_answers=lambda _: app_answers,  # noqa: B023
         )
-        app_migrate.migrate_from_legacy(preserve_legacy_file=True, commit=commit)
+        app_migrate.migrate_from_legacy(
+            preserve_legacy_file=True, commit=commit, use_migration_tags=True
+        )
 
     # remove the old file once we are done with it
     ctx.console.print(f"Deleting legacy file ({base_migrate.legacy_version_file_path()})")