From f542db51fb9dbf508ca38550e89ed59f2fe6d7d4 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Tue, 21 Jan 2025 21:01:43 -0500 Subject: [PATCH] split me up: collection of fixes that will hopefully fix integration tests --- .../transforms/firefoxci_artifact.py | 17 +++---- .../transforms/integration_test.py | 49 +++++++++---------- taskcluster/kinds/firefoxci-artifact/kind.yml | 2 + taskcluster/kinds/integration-test/kind.yml | 1 + .../test/test_transforms_integration_test.py | 12 +++-- 5 files changed, 40 insertions(+), 41 deletions(-) diff --git a/taskcluster/fxci_config_taskgraph/transforms/firefoxci_artifact.py b/taskcluster/fxci_config_taskgraph/transforms/firefoxci_artifact.py index 331c0dcd..8d424c9d 100644 --- a/taskcluster/fxci_config_taskgraph/transforms/firefoxci_artifact.py +++ b/taskcluster/fxci_config_taskgraph/transforms/firefoxci_artifact.py @@ -4,6 +4,7 @@ import json import os +import re from collections import defaultdict from copy import deepcopy @@ -26,6 +27,7 @@ def make_firefoxci_artifact_tasks(config, tasks): include_attrs = task.pop("include-attrs", {}) exclude_attrs = task.pop("exclude-attrs", {}) include_deps = task.pop("include-deps", []) + mirror_public_artifacts = [re.compile(r) for r in task.pop("mirror-public-artifacts", [])] for decision_index_path in task.pop("decision-index-paths"): for _, task_def in find_tasks( decision_index_path, @@ -36,14 +38,10 @@ def make_firefoxci_artifact_tasks(config, tasks): # Add docker images if "image" in task_def["payload"]: image = task_def["payload"]["image"] - if not isinstance(image, dict) or "taskId" not in image: - continue - - task_id = image["taskId"] - if task_id in tasks_to_create: - continue - - tasks_to_create[task_id] = [image["path"]] + if isinstance(image, dict) and "taskId" in image: + task_id = image["taskId"] + if task_id not in tasks_to_create: + tasks_to_create[task_id] = [image["path"]] # Add private artifacts if "MOZ_FETCHES" in task_def["payload"].get("env", {}): @@ -52,7 +50,8 @@ def make_firefoxci_artifact_tasks(config, tasks): ) for fetch in fetches: if fetch["artifact"].startswith("public"): - continue + if not any([pat.match(task_def["metadata"]["name"]) for pat in mirror_public_artifacts]): + continue task_id = fetch["task"] tasks_to_create[task_id].append(fetch["artifact"]) diff --git a/taskcluster/fxci_config_taskgraph/transforms/integration_test.py b/taskcluster/fxci_config_taskgraph/transforms/integration_test.py index 701126be..fb68bb15 100644 --- a/taskcluster/fxci_config_taskgraph/transforms/integration_test.py +++ b/taskcluster/fxci_config_taskgraph/transforms/integration_test.py @@ -210,14 +210,17 @@ def rewrite_mirrored_dependencies( prefix: str, dependencies: dict[str, str], tasks: dict[str, Any], - patterns: list[re.Pattern], + include_deps: list[str], + artifact_tasks, ): """Re-write dependencies and fetches of tasks that are being re-run in the staging instance that are also being re-run in the staging instance. Without this, the downstream tasks will attempt to refer to firefoxci task ids that do not exist in the staging cluster, and task submission will fail. """ + patterns = [re.compile(p) for p in include_deps] modified_deps = set() + mirrored_deps = set() # First, update any dependencies that are also being run as part of this integration test for upstream_task_id in dependencies: if upstream_task_id in tasks: @@ -227,6 +230,15 @@ def rewrite_mirrored_dependencies( upstream_task_label = f"{prefix}-{name}" taskdesc["dependencies"][upstream_task_label] = upstream_task_label + artifact_task_label = f"firefoxci-artifact-{prefix}-{upstream_task_id}" + # TODO: shouldn't add docker images here; they get added already elsewhere + if artifact_task_label in artifact_tasks: + fetched_artifacts = artifact_tasks[artifact_task_label].task["payload"]["env"]["FETCH_FIREFOXCI_ARTIFACTS"] + if "image.tar.zst" in fetched_artifacts: + continue + mirrored_deps.add(upstream_task_id) + taskdesc["dependencies"][artifact_task_label] = artifact_task_label + # Second, update any fetches that point to dependencies that are also being run as part # of this integration test updated_fetches = [] @@ -241,6 +253,9 @@ def rewrite_mirrored_dependencies( fetch_task_label = tasks[fetch_task_id]["metadata"]["name"] fetch["task"] = f"<{prefix}-{fetch_task_label}>" + if fetch_task_id in mirrored_deps: + fetch["task"] = f"" + updated_fetches.append(fetch) taskdesc["task"]["payload"]["env"]["MOZ_FETCHES"] = { @@ -253,6 +268,8 @@ def make_integration_test_description( name_prefix: str, tasks: dict[str, Any], include_deps: list[str], + should_patch_root_url: bool, + artifact_tasks, ): """Schedule a task on the staging Taskcluster instance. @@ -274,30 +291,6 @@ def make_integration_test_description( if "treeherder" in task_def["extra"]: del task_def["extra"]["treeherder"] - include_patterns = [re.compile(p) for p in include_deps] - # Tasks may only have 1 root url set, which is primarily used to decide - # where to find `MOZ_FETCHES`. When we're including dependencies, we peek - # at upstream tasks to figure out what this should be set to. If any - # upstream task is present that doesn't match an include pattern, its - # dependencies must be pulled from the firefox ci cluster, and we must - # patch the root. If a task matches an include pattern, its dependencies - # will be in the staging cluster, and we must not patch the root. - # If we have a mix of tasks, we cannot proceed. - matches = set() - for upstream_task_id in orig_dependencies: - if upstream_task_id in tasks: - name = tasks[upstream_task_id]["metadata"]["name"] - matches.add(any([pat.match(name) for pat in include_patterns])) - - should_patch_root_url = True - if len(matches) == 2: - raise Exception( - f"task '{task_def['metadata']['name']}' has both mirrored and unmirrored dependencies; it will not work correctly" - ) - elif len(matches) == 1: - if matches.pop() is True: - should_patch_root_url = False - if should_patch_root_url: patch_root_url(task_def) rewrite_mounts(task_def) @@ -328,7 +321,7 @@ def make_integration_test_description( rewrite_docker_image(taskdesc) rewrite_private_fetches(taskdesc) rewrite_mirrored_dependencies( - taskdesc, name_prefix, orig_dependencies, tasks, include_patterns + taskdesc, name_prefix, orig_dependencies, tasks, include_deps, artifact_tasks, ) return taskdesc @@ -341,10 +334,12 @@ def schedule_tasks_at_index(config, tasks): if os.environ["TASKCLUSTER_ROOT_URL"] != STAGING_ROOT_URL: return + artifact_tasks = {k: v for k, v in config.kind_dependencies_tasks.items() if k.startswith("firefoxci-artifact")} for task in tasks: include_attrs = task.pop("include-attrs", {}) exclude_attrs = task.pop("exclude-attrs", {}) include_deps = task.pop("include-deps", []) + patch_root_url = task.pop("patch-root-url", True) for decision_index_path in task.pop("decision-index-paths"): found_tasks = find_tasks( decision_index_path, @@ -356,5 +351,5 @@ def schedule_tasks_at_index(config, tasks): # task_def is copied to avoid modifying the version in `tasks`, which # may be used to modify parts of the new task description yield make_integration_test_description( - copy.deepcopy(task_def), task["name"], found_tasks, include_deps + copy.deepcopy(task_def), task["name"], found_tasks, include_deps, patch_root_url, artifact_tasks ) diff --git a/taskcluster/kinds/firefoxci-artifact/kind.yml b/taskcluster/kinds/firefoxci-artifact/kind.yml index 7e64a1ee..7bb2013f 100644 --- a/taskcluster/kinds/firefoxci-artifact/kind.yml +++ b/taskcluster/kinds/firefoxci-artifact/kind.yml @@ -60,3 +60,5 @@ tasks: # expire, which causes us to try to mirror expired docker image tasks # into the staging cluster - ^(?!(Decision|Action|PR action|build-docker-image|fetch)).* + mirror-public-artifacts: + - ^toolchain.* diff --git a/taskcluster/kinds/integration-test/kind.yml b/taskcluster/kinds/integration-test/kind.yml index cf37d4e9..ce011396 100644 --- a/taskcluster/kinds/integration-test/kind.yml +++ b/taskcluster/kinds/integration-test/kind.yml @@ -42,3 +42,4 @@ tasks: # expire, which causes us to try to mirror expired docker image tasks # into the staging cluster - ^(?!(Decision|Action|PR action|build-docker-image|fetch)).* + patch-root-url: false diff --git a/taskcluster/test/test_transforms_integration_test.py b/taskcluster/test/test_transforms_integration_test.py index 69e1eea3..0b2b5ef3 100644 --- a/taskcluster/test/test_transforms_integration_test.py +++ b/taskcluster/test/test_transforms_integration_test.py @@ -379,7 +379,7 @@ def run_include_deps_test(run_test, *args, **kwargs): "taskId": "ghi", "path": "public/image.tar.zst", }, - "command": ["run-task", "build-a-thing"], + "command": ["run-task", "build-a-thing", "foo"], }, "tags": {}, }, @@ -394,7 +394,7 @@ def run_include_deps_test(run_test, *args, **kwargs): "taskId": "jkl", "path": "public/image.tar.zst", }, - "command": ["run-task", "test-a-thing"], + "command": ["run-task", "test-a-thing", "foo"], "env": { "MOZ_FETCHES": json.dumps( [ @@ -420,7 +420,7 @@ def run_include_deps_test(run_test, *args, **kwargs): "taskId": "mno", "path": "public/image.tar.zst", }, - "command": ["run-task", "sign-a-thing"], + "command": ["run-task", "sign-a-thing", "foo"], "env": { "MOZ_FETCHES": json.dumps( [ @@ -502,11 +502,11 @@ def test_include_some_deps(run_test): "taskId": task_id, "path": artifact, }, - "command": ["run-task", "foo"], + "command": ["run-task", "foo", "bar"], }, }, }, - include_deps=["^sign", "^test"], + include_deps=["^(?!(build))"], name="gecko", ) expected = set(["gecko-foo", "gecko-test-thing", "gecko-sign-thing"]) @@ -521,6 +521,8 @@ def test_include_some_deps(run_test): in sign_task["task"]["payload"]["env"]["MOZ_FETCHES"]["task-reference"] ) assert "TASKCLUSTER_ROOT_URL" in test_task["task"]["payload"]["command"][2] + assert "TASKCLUSTER_ROOT_URL" not in sign_task["task"]["payload"]["command"][2] + assert "TASKCLUSTER_ROOT_URL" not in foo_task["task"]["payload"]["command"][2] def test_no_deps(run_test):