From f6dec21b234682cea371ffbcef5f276e06bd3919 Mon Sep 17 00:00:00 2001 From: scosman Date: Tue, 25 Feb 2025 21:14:10 -0500 Subject: [PATCH] Fix bug in how we collected runs --- libs/core/kiln_ai/adapters/eval/eval_runner.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libs/core/kiln_ai/adapters/eval/eval_runner.py b/libs/core/kiln_ai/adapters/eval/eval_runner.py index 3b4f0a6f..11d8b9f1 100644 --- a/libs/core/kiln_ai/adapters/eval/eval_runner.py +++ b/libs/core/kiln_ai/adapters/eval/eval_runner.py @@ -133,8 +133,11 @@ def collect_tasks_for_task_run_eval(self) -> List[EvalJob]: already_run[eval_config.id] = {} for run_config in self.run_configs or []: already_run[eval_config.id][run_config.id] = set() - for run in eval_config.runs(readonly=True): - already_run[eval_config.id][run_config.id].add(run.dataset_id) + for run in eval_config.runs(readonly=True): + if run.task_run_config_id is not None: + already_run[eval_config.id][run.task_run_config_id].add( + run.dataset_id + ) return [ EvalJob(