diff --git a/taskcluster/fxci_config_taskgraph/transforms/integration_test.py b/taskcluster/fxci_config_taskgraph/transforms/integration_test.py index 1f12b31f..26da3b95 100644 --- a/taskcluster/fxci_config_taskgraph/transforms/integration_test.py +++ b/taskcluster/fxci_config_taskgraph/transforms/integration_test.py @@ -343,9 +343,13 @@ def schedule_tasks_at_index(config, tasks): exclude_attrs, include_deps, ) + created = set() for task_def in found_tasks.values(): - # task_def is copied to avoid modifying the version in `tasks`, which - # may be used to modify parts of the new task description - yield make_integration_test_description( - copy.deepcopy(task_def), task["name"], found_tasks, include_deps, patch_root_url, artifact_tasks - ) + if task_def["metadata"]["name"] not in created: + # task_def is copied to avoid modifying the version in `tasks`, which + # may be used to modify parts of the new task description + yield make_integration_test_description( + copy.deepcopy(task_def), task["name"], found_tasks, include_deps, patch_root_url, artifact_tasks + ) + + created.add(task_def["metadata"]["name"]) diff --git a/taskcluster/fxci_config_taskgraph/util/integration.py b/taskcluster/fxci_config_taskgraph/util/integration.py index a043cca0..3d5be55e 100644 --- a/taskcluster/fxci_config_taskgraph/util/integration.py +++ b/taskcluster/fxci_config_taskgraph/util/integration.py @@ -2,20 +2,70 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +import copy import os import re +import functools from functools import cache -from typing import Any +from typing import Any, Dict, List, Union import requests import taskcluster from taskgraph.util.attributes import attrmatch -from taskgraph.util.taskcluster import get_ancestors as taskgraph_get_ancestors -from taskgraph.util.taskcluster import get_root_url +from taskgraph.util.taskcluster import get_root_url, get_task_definition from fxci_config_taskgraph.util.constants import FIREFOXCI_ROOT_URL +@functools.lru_cache(maxsize=None) +def _get_deps(task_ids, use_proxy): + upstream_tasks = {} + for task_id in task_ids: + try: + task_def = get_task_definition(task_id, use_proxy) + except requests.HTTPError as e: + if e.response.status_code == 404: + continue + raise e + + upstream_tasks[task_id] = task_def["metadata"]["name"] + + upstream_tasks.update(_get_deps(tuple(task_def["dependencies"]), use_proxy)) + + return upstream_tasks + + +def taskgraph_get_ancestors( + task_ids: Union[List[str], str], use_proxy: bool = False +) -> Dict[str, str]: + """Gets the ancestor tasks of the given task_ids as a dictionary of taskid -> label. + + Args: + task_ids (str or [str]): A single task id or a list of task ids to find the ancestors of. + use_proxy (bool): See get_root_url. + + Returns: + dict: A dict whose keys are task ids and values are task labels. + """ + upstream_tasks: Dict[str, str] = {} + + if isinstance(task_ids, str): + task_ids = [task_ids] + + for task_id in task_ids: + try: + task_def = get_task_definition(task_id, use_proxy) + except requests.HTTPError as e: + # Task has most likely expired, which means it's no longer a + # dependency for the purposes of this function. + if e.response.status_code == 404: + continue + + raise e + + upstream_tasks.update(_get_deps(tuple(task_def["dependencies"]), use_proxy)) + + return copy.deepcopy(upstream_tasks) def get_ancestors(task_ids: list[str] | str) -> dict[str, str]: # This is not ideal, but at the moment we don't have a better way # to ensure that the upstream get_ancestors talks to the correct taskcluster @@ -149,7 +199,7 @@ def find_tasks( # use the results. if include_deps: patterns = [re.compile(p) for p in include_deps] - for label, upstream_task_id in get_ancestors(task_id).items(): + for upstream_task_id, label in get_ancestors(task_id).items(): if any([pat.match(label) for pat in patterns]): task_def = _queue_task(upstream_task_id) # The task definitions from `get_ancestors` are fully