Fix bug in how we collected runs

Kiln-AI · Feb 26, 2025 · f6dec21 · f6dec21
1 parent a8bb4db
commit f6dec21
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/libs/core/kiln_ai/adapters/eval/eval_runner.py b/libs/core/kiln_ai/adapters/eval/eval_runner.py
@@ -133,8 +133,11 @@ def collect_tasks_for_task_run_eval(self) -> List[EvalJob]:
             already_run[eval_config.id] = {}
             for run_config in self.run_configs or []:
                 already_run[eval_config.id][run_config.id] = set()
-                for run in eval_config.runs(readonly=True):
-                    already_run[eval_config.id][run_config.id].add(run.dataset_id)
+            for run in eval_config.runs(readonly=True):
+                if run.task_run_config_id is not None:
+                    already_run[eval_config.id][run.task_run_config_id].add(
+                        run.dataset_id
+                    )
 
         return [
             EvalJob(