work around bug in get_ancestors

taskcluster/fxci_config_taskgraph/transforms/integration_test.py

@@ -343,9 +343,13 @@ def schedule_tasks_at_index(config, tasks):
                 exclude_attrs,
                 include_deps,
             )
+            created = set()
             for task_def in found_tasks.values():
-                # task_def is copied to avoid modifying the version in `tasks`, which
-                # may be used to modify parts of the new task description
-                yield make_integration_test_description(
-                    copy.deepcopy(task_def), task["name"], found_tasks, include_deps, patch_root_url, artifact_tasks
-                )
+                if task_def["metadata"]["name"] not in created:
+                    # task_def is copied to avoid modifying the version in `tasks`, which
+                    # may be used to modify parts of the new task description
+                    yield make_integration_test_description(
+                        copy.deepcopy(task_def), task["name"], found_tasks, include_deps, patch_root_url, artifact_tasks
+                    )
+
+                created.add(task_def["metadata"]["name"])

taskcluster/fxci_config_taskgraph/util/integration.py

@@ -2,20 +2,70 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import copy
 import os
 import re
+import functools
 from functools import cache
-from typing import Any
+from typing import Any, Dict, List, Union
 
 import requests
 import taskcluster
 from taskgraph.util.attributes import attrmatch
-from taskgraph.util.taskcluster import get_ancestors as taskgraph_get_ancestors
-from taskgraph.util.taskcluster import get_root_url
+from taskgraph.util.taskcluster import get_root_url, get_task_definition
 
 from fxci_config_taskgraph.util.constants import FIREFOXCI_ROOT_URL
 
 
+@functools.lru_cache(maxsize=None)
+def _get_deps(task_ids, use_proxy):
+    upstream_tasks = {}
+    for task_id in task_ids:
+        try:
+            task_def = get_task_definition(task_id, use_proxy)
+        except requests.HTTPError as e:
+            if e.response.status_code == 404:
+                continue
+            raise e
+
+        upstream_tasks[task_id] = task_def["metadata"]["name"]
+
+        upstream_tasks.update(_get_deps(tuple(task_def["dependencies"]), use_proxy))
+
+    return upstream_tasks
+
+
+def taskgraph_get_ancestors(
+    task_ids: Union[List[str], str], use_proxy: bool = False
+) -> Dict[str, str]:
+    """Gets the ancestor tasks of the given task_ids as a dictionary of taskid -> label.
+
+    Args:
+        task_ids (str or [str]): A single task id or a list of task ids to find the ancestors of.
+        use_proxy (bool): See get_root_url.
+
+    Returns:
+        dict: A dict whose keys are task ids and values are task labels.
+    """
+    upstream_tasks: Dict[str, str] = {}
+
+    if isinstance(task_ids, str):
+        task_ids = [task_ids]
+
+    for task_id in task_ids:
+        try:
+            task_def = get_task_definition(task_id, use_proxy)
+        except requests.HTTPError as e:
+            # Task has most likely expired, which means it's no longer a
+            # dependency for the purposes of this function.
+            if e.response.status_code == 404:
+                continue
+
+            raise e
+
+        upstream_tasks.update(_get_deps(tuple(task_def["dependencies"]), use_proxy))
+
+    return copy.deepcopy(upstream_tasks)
 def get_ancestors(task_ids: list[str] | str) -> dict[str, str]:
     # This is not ideal, but at the moment we don't have a better way
     # to ensure that the upstream get_ancestors talks to the correct taskcluster
@@ -149,7 +199,7 @@ def find_tasks(
         # use the results.
         if include_deps:
             patterns = [re.compile(p) for p in include_deps]
-            for label, upstream_task_id in get_ancestors(task_id).items():
+            for upstream_task_id, label in get_ancestors(task_id).items():
                 if any([pat.match(label) for pat in patterns]):
                     task_def = _queue_task(upstream_task_id)
                     # The task definitions from `get_ancestors` are fully