run black

deepeval/experimental/harness/task.py

@@ -6,7 +6,7 @@
 
 from lm_eval.api.registry import ALL_TASKS
 from lm_eval import utils
-from lm_eval.tasks import initialize_tasks, get_task_dict 
+from lm_eval.tasks import initialize_tasks, get_task_dict
 
 initialize_tasks()
 
@@ -16,13 +16,10 @@
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
 # Todo:
-# For everytask, we need to save a task descriptor. 
+# For everytask, we need to save a task descriptor.
 # This can be done by going through each task and use gpt to summarize
 # and store it.
 
-# Todo:
-# For every sub task or so, we should have an option to download that very task and 
-# use it under deepeval seperately or may be outside of deepeval.
 
 class HarnessTasks:
     @classmethod
@@ -79,20 +76,26 @@ def group_tasks(task_names: Union[str, List[str]]) -> None:
         raise NotImplementedError()
 
     @classmethod
-    def get_dataset_from_task(cls, task_names: Union[str, List[str]], task_limit: int, rank: int=0, world_size:int = 1) -> None:
+    def get_dataset_from_task(
+        cls,
+        task_names: Union[str, List[str]],
+        task_limit: int,
+        rank: int = 0,
+        world_size: int = 1,
+    ) -> None:
         """Loads a single task and download the tasks in the form of JSON"""
-        # rank and world_size is 0 and 1 by default to a single process 
+        # rank and world_size is 0 and 1 by default to a single process
         task_names = cls.load_task(tasks=[task_names])
         task_dict = get_task_dict(task_names)
         all_task_data = {}
-        
+
         for task_name, task in task_dict.items():
             if type(task) == tuple:
                 _, task = task
-                
+
             if task is None:
                 continue
-            
+
             if task_limit is not None:
                 if task.has_test_docs():
                     task_docs = task.test_docs()
@@ -101,16 +104,22 @@ def get_dataset_from_task(cls, task_names: Union[str, List[str]], task_limit: in
                 else:
                     print("Task has neither test_docs nor validation docs")
                     continue
-                limit = int(len(task_docs) * task_limit) if task_limit < 1.0 else int(task_limit)
-            task.build_all_requests(limit=limit, rank=rank, world_size=world_size)
-            task_wise_data = {
-                'doc_id': [], 'prompt': [], 'target': []
-            }
-
+                limit = (
+                    int(len(task_docs) * task_limit)
+                    if task_limit < 1.0
+                    else int(task_limit)
+                )
+            task.build_all_requests(
+                limit=limit, rank=rank, world_size=world_size
+            )
+            task_wise_data = {"doc_id": [], "prompt": [], "target": []}
+
             for instance in task.instances:
                 task_wise_data["doc_id"].append(instance.doc_id)
                 # FIXME: instance.args[0] is a bit explicit and prompt does not makes sense for tasks like hellaswag.
                 task_wise_data["prompt"].append(instance.args[0])
-                task_wise_data["target"].append(task.doc_to_target(instance.doc))
+                task_wise_data["target"].append(
+                    task.doc_to_target(instance.doc)
+                )
             all_task_data[task_name] = task_wise_data
-        return all_task_data
+        return all_task_data