From 5c1005c9264645630c8c415d2e41eb48db260135 Mon Sep 17 00:00:00 2001 From: Daniel Sotirhos Date: Fri, 15 Nov 2024 13:56:38 -0800 Subject: [PATCH 1/4] Refactor load_env() to support deployments other than the active one --- scripts/export_environment.py | 55 ++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/scripts/export_environment.py b/scripts/export_environment.py index dfe4438c3..810e969c6 100644 --- a/scripts/export_environment.py +++ b/scripts/export_environment.py @@ -73,6 +73,14 @@ def env(self) -> DraftEnvironment: } +class InvalidDeployment(RuntimeError): + + def __init__(self, dir_: Path) -> None: + super().__init__( + f"{dir_} does not exist or is not a symbolic link to a directory." + ) + + class InvalidActiveDeployment(RuntimeError): def __init__(self, dir_: Path) -> None: @@ -91,35 +99,40 @@ def __init__(self, parent: Path, component: Path) -> None: ) -def load_env() -> Tuple[Environment, Optional[str]]: +def load_env(deployment: Optional[str] = None) -> Tuple[Environment, Optional[str]]: """ Load environment.py and environment.local.py modules from the project - root and the current active deployment directory, call their env() - function to obtain the environment dictionary and merge the dictionaries. - The entries from an environment.local.py take precedence over those from - a corresponding environment.py in the same directory. The modules from - the deployment directory take precedence over ones in the project root. + root and either the specified deployment or the current active deployment + directory, call their env() function to obtain the environment dictionary + and merge the dictionaries. The entries from an environment.local.py take + precedence over those from a corresponding environment.py in the same + directory. The modules from the deployment directory take precedence over + ones in the project root. """ deployments_dir = root_dir / 'deployments' active_deployment_dir = deployments_dir / '.active' - if active_deployment_dir.exists(): - if not active_deployment_dir.is_dir() or not active_deployment_dir.is_symlink(): - raise InvalidActiveDeployment(active_deployment_dir) - + if deployment is not None or active_deployment_dir.exists(): + if deployment is not None: + deployment_dir = deployments_dir / deployment + if not deployment_dir.is_dir(): + raise InvalidDeployment(deployment_dir) + else: + if not active_deployment_dir.is_dir() or not active_deployment_dir.is_symlink(): + raise InvalidActiveDeployment(active_deployment_dir) + deployment_dir = Path(os.readlink(str(active_deployment_dir))) + if not deployment_dir.is_absolute(): + deployment_dir = deployments_dir / deployment_dir + if not deployment_dir.is_dir(): + raise InvalidActiveDeployment(deployment_dir) # If active deployment is a component of another one, also load the parent # deployments (like dev.gitlab). - active_deployment_dir = Path(os.readlink(str(active_deployment_dir))) - if not active_deployment_dir.is_absolute(): - active_deployment_dir = deployments_dir / active_deployment_dir - if not active_deployment_dir.is_dir(): - raise InvalidActiveDeployment(active_deployment_dir) - relative_active_deployment_dir = active_deployment_dir.relative_to(deployments_dir) - prefix, _, suffix = str(relative_active_deployment_dir).partition('.') + relative_deployment_dir = deployment_dir.relative_to(deployments_dir) + prefix, _, suffix = str(relative_deployment_dir).partition('.') if suffix and suffix != 'local': parent_deployment_dir = deployments_dir / prefix if not parent_deployment_dir.exists(): - raise BadParentDeployment(parent_deployment_dir, active_deployment_dir) + raise BadParentDeployment(parent_deployment_dir, deployment_dir) else: parent_deployment_dir = None warning = None @@ -128,7 +141,7 @@ def load_env() -> Tuple[Environment, Optional[str]]: f'No active deployment (missing {str(active_deployment_dir)!r}). ' f'Loaded global defaults only.' ) - active_deployment_dir = None + deployment_dir = None parent_deployment_dir = None def _load(dir_path: Path, local: bool = False) -> Optional[EnvironmentModule]: @@ -153,10 +166,10 @@ def _load(dir_path: Path, local: bool = False) -> Optional[EnvironmentModule]: return None modules = [ - active_deployment_dir and _load(active_deployment_dir, local=True), + deployment_dir and _load(deployment_dir, local=True), parent_deployment_dir and _load(parent_deployment_dir, local=True), _load(root_dir, local=True), - active_deployment_dir and _load(active_deployment_dir), + deployment_dir and _load(deployment_dir), parent_deployment_dir and _load(parent_deployment_dir), _load(root_dir) ] From cebca548d5517f71c00468bc5af11514bfba62dc Mon Sep 17 00:00:00 2001 From: Daniel Sotirhos Date: Fri, 15 Nov 2024 14:01:51 -0800 Subject: [PATCH 2/4] fixup! Refactor load_env() to support deployments other than the active one --- scripts/export_environment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/export_environment.py b/scripts/export_environment.py index 810e969c6..8d7f3ca64 100644 --- a/scripts/export_environment.py +++ b/scripts/export_environment.py @@ -99,7 +99,8 @@ def __init__(self, parent: Path, component: Path) -> None: ) -def load_env(deployment: Optional[str] = None) -> Tuple[Environment, Optional[str]]: +def load_env(deployment: Optional[str] = None + ) -> Tuple[Environment, Optional[str]]: """ Load environment.py and environment.local.py modules from the project root and either the specified deployment or the current active deployment From 4736625531a7d4c321018b726e8cc8cd081a8743 Mon Sep 17 00:00:00 2001 From: Daniel Sotirhos Date: Thu, 14 Nov 2024 18:26:19 -0800 Subject: [PATCH 3/4] Add links to GitLab background migration pages (#6706) --- .../anvilprod-promotion.md | 2 +- .../PULL_REQUEST_TEMPLATE/prod-promotion.md | 2 +- .github/PULL_REQUEST_TEMPLATE/upgrade.md | 4 ++-- .github/pull_request_template.md | 4 ++-- .github/pull_request_template.md.template.py | 18 +++++++++++++++++- 5 files changed, 23 insertions(+), 7 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md b/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md index a3a3a0251..38ad93cff 100644 --- a/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md +++ b/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md @@ -53,7 +53,7 @@ Connected issue: #0000 ### System administrator -- [ ] Background migrations for `anvilprod.gitlab` are complete or this PR is not labeled `deploy:gitlab` +- [ ] Background migrations for [`anvilprod.gitlab`](https://gitlab.explore.anvilproject.org/admin/background_migrations) are complete or this PR is not labeled `deploy:gitlab` - [ ] PR is assigned to only the operator diff --git a/.github/PULL_REQUEST_TEMPLATE/prod-promotion.md b/.github/PULL_REQUEST_TEMPLATE/prod-promotion.md index c34b12777..33ec4a5d6 100644 --- a/.github/PULL_REQUEST_TEMPLATE/prod-promotion.md +++ b/.github/PULL_REQUEST_TEMPLATE/prod-promotion.md @@ -53,7 +53,7 @@ Connected issue: #0000 ### System administrator -- [ ] Background migrations for `prod.gitlab` are complete or this PR is not labeled `deploy:gitlab` +- [ ] Background migrations for [`prod.gitlab`](https://gitlab.azul.data.humancellatlas.org/admin/background_migrations) are complete or this PR is not labeled `deploy:gitlab` - [ ] PR is assigned to only the operator diff --git a/.github/PULL_REQUEST_TEMPLATE/upgrade.md b/.github/PULL_REQUEST_TEMPLATE/upgrade.md index 136e5b837..3ab7df433 100644 --- a/.github/PULL_REQUEST_TEMPLATE/upgrade.md +++ b/.github/PULL_REQUEST_TEMPLATE/upgrade.md @@ -65,8 +65,8 @@ Connected issue: #0000 ### System administrator -- [ ] Background migrations for `dev.gitlab` are complete or this PR is not labeled `deploy:gitlab` -- [ ] Background migrations for `anvildev.gitlab` are complete or this PR is not labeled `deploy:gitlab` +- [ ] Background migrations for [`dev.gitlab`](https://gitlab.dev.singlecell.gi.ucsc.edu/admin/background_migrations) are complete or this PR is not labeled `deploy:gitlab` +- [ ] Background migrations for [`anvildev.gitlab`](https://gitlab.anvil.gi.ucsc.edu/admin/background_migrations) are complete or this PR is not labeled `deploy:gitlab` - [ ] PR is assigned to only the operator diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index e60c198d3..bca1bc8f8 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -118,8 +118,8 @@ title is `Fix: ` followed by the issue title ### System administrator -- [ ] Background migrations for `dev.gitlab` are complete or this PR is not labeled `deploy:gitlab` -- [ ] Background migrations for `anvildev.gitlab` are complete or this PR is not labeled `deploy:gitlab` +- [ ] Background migrations for [`dev.gitlab`](https://gitlab.dev.singlecell.gi.ucsc.edu/admin/background_migrations) are complete or this PR is not labeled `deploy:gitlab` +- [ ] Background migrations for [`anvildev.gitlab`](https://gitlab.anvil.gi.ucsc.edu/admin/background_migrations) are complete or this PR is not labeled `deploy:gitlab` - [ ] PR is assigned to only the operator diff --git a/.github/pull_request_template.md.template.py b/.github/pull_request_template.md.template.py index 1fd2f544b..d671d0ea0 100644 --- a/.github/pull_request_template.md.template.py +++ b/.github/pull_request_template.md.template.py @@ -32,6 +32,9 @@ from azul.collections import ( OrderedSet, ) +from azul.modules import ( + load_module, +) from azul.strings import ( back_quote as bq, join_grammatically, @@ -237,6 +240,17 @@ def main(): emit(t, target_branch) +def env_var(deployment: str, variable: str) -> str: + """ + Return an environment variable value from the given deployment. + """ + module_path = f'{config.project_root}/scripts/export_environment.py' + export_environment = load_module(module_path, 'export_environment') + env, warning = export_environment.load_env(deployment) + resolved_env = export_environment.resolve_env(env) + return resolved_env[variable] + + def emit(t: T, target_branch: str): emit_checklist( [ @@ -714,7 +728,9 @@ def emit(t: T, target_branch: str): *[ { 'type': 'cli', - 'content': f'Background migrations for `{d}.gitlab` are complete', + 'content': f'Background migrations for ' + f'[`{d}.gitlab`](https://gitlab.{env_var(d, "AZUL_DOMAIN_NAME")}' + f'/admin/background_migrations) are complete', 'alt': 'or this PR is not labeled `deploy:gitlab`' } for d in t.target_deployments(target_branch) From 41b8f1239715daaf75031387187ff8b4d214ae4f Mon Sep 17 00:00:00 2001 From: Daniel Sotirhos Date: Thu, 14 Nov 2024 18:34:20 -0800 Subject: [PATCH 4/4] Make links to dockerhub images set to the tags page (#6706) --- .github/ISSUE_TEMPLATE/upgrade.md | 12 ++++++------ .github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md | 6 +++--- .github/PULL_REQUEST_TEMPLATE/prod-promotion.md | 6 +++--- .github/pull_request_template.md.template.py | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/upgrade.md b/.github/ISSUE_TEMPLATE/upgrade.md index 06d786361..d489492cf 100644 --- a/.github/ISSUE_TEMPLATE/upgrade.md +++ b/.github/ISSUE_TEMPLATE/upgrade.md @@ -13,27 +13,27 @@ _period: 14 days - [ ] Remove unused dependencies with high or critical CVEs - [ ] Push commit to GitHub (directly to `master` branch, no PR needed) - [ ] GH Action workflow succeeded - - [ ] Image is available on [DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-pycharm) + - [ ] Image is available on [DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-pycharm/tags) - [ ] Update [Elasticsearch image](https://github.com/DataBiosphere/azul-docker-elasticsearch) - [ ] Bump [base image](https://hub.docker.com/_/elasticsearch/tags) tag (only minor and patch versions), if possible - [ ] Bump internal version - [ ] Remove unused dependencies with high or critical CVEs - [ ] Push commit to GitHub (directly to `main` branch, no PR needed) - [ ] GH Action workflow succeeded - - [ ] Image is available on [DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-elasticsearch) + - [ ] Image is available on [DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-elasticsearch/tags) - [ ] Update [BigQuery Emulator image](https://github.com/DataBiosphere/azul-bigquery-emulator) - [ ] Bump [base image](https://hub.docker.com/_/debian/tags?name=bookworm) tag, if possible - [ ] Bump internal version - [ ] Push commit to GitHub (directly to `azul` branch, no PR needed) - [ ] GH Action workflow succeeded - - [ ] Image is available on [DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-bigquery-emulator) + - [ ] Image is available on [DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-bigquery-emulator/tags) - [ ] Create Azul PR, connected to this issue, with … - [ ] … changes to `requirements*.txt` from open Dependabot PRs, one commit per PR - [ ] … update to [Python](https://hub.docker.com/_/python/tags) (only patch versions) or no update available - [ ] … update to [Terraform](https://hub.docker.com/r/hashicorp/terraform/tags) (only patch versions) or no update available - - [ ] … new [PyCharm image](https://hub.docker.com/repository/docker/ucscgi/azul-pycharm) - - [ ] … new [Elasticsearch image](https://hub.docker.com/repository/docker/ucscgi/azul-elasticsearch) - - [ ] … new [BigQuery Emulator image](https://hub.docker.com/repository/docker/ucscgi/azul-bigquery-emulator) + - [ ] … new [PyCharm image](https://hub.docker.com/repository/docker/ucscgi/azul-pycharm/tags) + - [ ] … new [Elasticsearch image](https://hub.docker.com/repository/docker/ucscgi/azul-elasticsearch/tags) + - [ ] … new [BigQuery Emulator image](https://hub.docker.com/repository/docker/ucscgi/azul-bigquery-emulator/tags) - [ ] … update to [Docker images](https://hub.docker.com/_/docker/tags) (only minor and patch versions) or no update available - [ ] … update to [GitLab](https://hub.docker.com/r/gitlab/gitlab-ce/tags) & [GitLab runner images](https://hub.docker.com/r/gitlab/gitlab-runner/tags) or no update available - [ ] … update to [ClamAV image](https://hub.docker.com/r/clamav/clamav/tags) or no update available diff --git a/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md b/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md index 38ad93cff..c2e93937d 100644 --- a/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md +++ b/.github/PULL_REQUEST_TEMPLATE/anvilprod-promotion.md @@ -100,9 +100,9 @@ Connected issue: #0000 ### System administrator -- [ ] Removed unused image tags from [pycharm image on DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-pycharm) or this promotion does not alter references to that image -- [ ] Removed unused image tags from [elasticsearch image on DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-elasticsearch) or this promotion does not alter references to that image -- [ ] Removed unused image tags from [bigquery_emulator image on DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-bigquery-emulator) or this promotion does not alter references to that image +- [ ] Removed unused image tags from [pycharm image on DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-pycharm/tags) or this promotion does not alter references to that image +- [ ] Removed unused image tags from [elasticsearch image on DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-elasticsearch/tags) or this promotion does not alter references to that image +- [ ] Removed unused image tags from [bigquery_emulator image on DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-bigquery-emulator/tags) or this promotion does not alter references to that image - [ ] PR is assigned to no one diff --git a/.github/PULL_REQUEST_TEMPLATE/prod-promotion.md b/.github/PULL_REQUEST_TEMPLATE/prod-promotion.md index 33ec4a5d6..885da490e 100644 --- a/.github/PULL_REQUEST_TEMPLATE/prod-promotion.md +++ b/.github/PULL_REQUEST_TEMPLATE/prod-promotion.md @@ -94,9 +94,9 @@ Connected issue: #0000 ### System administrator -- [ ] Removed unused image tags from [pycharm image on DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-pycharm) or this promotion does not alter references to that image -- [ ] Removed unused image tags from [elasticsearch image on DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-elasticsearch) or this promotion does not alter references to that image -- [ ] Removed unused image tags from [bigquery_emulator image on DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-bigquery-emulator) or this promotion does not alter references to that image +- [ ] Removed unused image tags from [pycharm image on DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-pycharm/tags) or this promotion does not alter references to that image +- [ ] Removed unused image tags from [elasticsearch image on DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-elasticsearch/tags) or this promotion does not alter references to that image +- [ ] Removed unused image tags from [bigquery_emulator image on DockerHub](https://hub.docker.com/repository/docker/ucscgi/azul-bigquery-emulator/tags) or this promotion does not alter references to that image - [ ] PR is assigned to no one diff --git a/.github/pull_request_template.md.template.py b/.github/pull_request_template.md.template.py index d671d0ea0..5b5ee5872 100644 --- a/.github/pull_request_template.md.template.py +++ b/.github/pull_request_template.md.template.py @@ -1027,7 +1027,7 @@ def emit(t: T, target_branch: str): *[ { 'type': 'cli', - 'content': f'Removed unused image tags from [{name} image on DockerHub]({url})', + 'content': f'Removed unused image tags from [{name} image on DockerHub]({url}/tags)', 'alt': 'or this promotion does not alter references to that image' } for name, url in custom_images.items()