diff --git a/.gitignore b/.gitignore index 159278a98..6e23ceb31 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ /examples/nli/scone/compiled_program.dspy /examples/qa/hotpot/compiled_program.dspy /ScoNe/ +# testing/playbook.ipynb # Byte-compiled / optimized / DLL files __pycache__/ @@ -58,4 +59,4 @@ assertion.log .mypy_cache dummy.csv docs/docs/**/*.json* -*.index \ No newline at end of file +*.index diff --git a/dspy/__init__.py b/dspy/__init__.py index 28e0a352b..32a14781d 100644 --- a/dspy/__init__.py +++ b/dspy/__init__.py @@ -7,10 +7,10 @@ from .signatures import * # Functional must be imported after primitives, predict and signatures -from .functional import * # isort: skip -from dspy.evaluate import Evaluate # isort: skip -from dspy.clients import * # isort: skip -from dspy.adapters import * # isort: skip +from .functional import * # isort: skip +from dspy.evaluate import Evaluate # isort: skip +from dspy.clients import * # isort: skip +from dspy.adapters import * # isort: skip from dspy.utils.logging_utils import configure_dspy_loggers, disable_logging, enable_logging from dspy.utils.asyncify import asyncify @@ -68,6 +68,8 @@ LabeledFewShot = dspy.teleprompt.LabeledFewShot BootstrapFewShot = dspy.teleprompt.BootstrapFewShot BootstrapFewShotWithRandomSearch = dspy.teleprompt.BootstrapFewShotWithRandomSearch +BootstrapKNN = dspy.teleprompt.BootstrapKNN +BootstrapKNNWithRandomSearch = dspy.teleprompt.BootstrapKNNWithRandomSearch BootstrapRS = dspy.teleprompt.BootstrapFewShotWithRandomSearch BootstrapFinetune = dspy.teleprompt.BootstrapFinetune BetterTogether = dspy.teleprompt.BetterTogether diff --git a/dspy/predict/knn.py b/dspy/predict/knn.py index 12e2183e0..c07d5e8c4 100644 --- a/dspy/predict/knn.py +++ b/dspy/predict/knn.py @@ -40,3 +40,6 @@ def __call__(self, **kwargs) -> List["dspy.Example"]: nearest_samples_idxs = scores.argsort()[-self.k :][::-1] train_sampled = [self.trainset[cur_idx] for cur_idx in nearest_samples_idxs] return train_sampled + + def demo_selector(self, predict: "dspy.Predict", inputs): + return self(**inputs) diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py index 5ffb0d2c0..1bf0cb019 100644 --- a/dspy/predict/predict.py +++ b/dspy/predict/predict.py @@ -1,7 +1,6 @@ import logging import random from functools import lru_cache -from typing import Optional from pydantic import BaseModel @@ -191,7 +190,7 @@ def forward(self, **kwargs): import dspy if hasattr(self, "knn"): - demos = self.knn(**inputs) + demos += self.knn(**inputs) if isinstance(lm, dspy.LM): completions = v2_5_generate(lm, config, signature, demos, inputs, _parse_values=self._parse_values) diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.py index a89ec0ed3..9cef9276d 100644 --- a/dspy/teleprompt/bootstrap.py +++ b/dspy/teleprompt/bootstrap.py @@ -278,15 +278,18 @@ class BootstrapKNN(BootstrapFewShot): def __init__( self, embedding: Optional[Callable[[list[str]], np.ndarray]] = None, # dspy.Embedding - k: int = 16, metric=None, metric_threshold=None, teacher_settings: Optional[Dict] = None, - max_bootstrapped_demos=10_000, # fill the predictor .demos - max_labeled_demos=16, # for the teacher model only + max_bootstrapped_demos=10_000, + num_static_demos=0, + max_labeled_demos=16, max_rounds=1, max_errors=2000, + random_seed=0, ): + assert num_static_demos < max_labeled_demos, "static demos must be less than max labeled demos." + super().__init__( metric=metric, metric_threshold=metric_threshold, @@ -296,17 +299,23 @@ def __init__( max_rounds=max_rounds, max_errors=max_errors, ) - self.k = k + self.num_static_demos = num_static_demos self.embedding = embedding + self.random_seed = random_seed def _train(self): + rng = random.Random(self.random_seed) + for name, predictor in self.student.named_predictors(): - predictor.demos = self.name2traces[name][: self.max_labeled_demos] + augmented_demos = self.name2traces[name] + + static_demos = rng.sample(augmented_demos, self.num_static_demos) + dynamic_demos = list(set(augmented_demos) - set(static_demos)) + + predictor.demos = static_demos + # TODO: Make this dump/load-able - predictor.knn = dspy.KNN( - k=self.k, - trainset=predictor.demos, - vectorizer=self.embedding, - ) + k = self.max_labeled_demos - self.num_static_demos + predictor.knn = dspy.KNN(k, trainset=dynamic_demos, vectorizer=self.embedding) return self.student diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.py index 4fa8ba23f..85eeb674a 100644 --- a/dspy/teleprompt/random_search.py +++ b/dspy/teleprompt/random_search.py @@ -3,7 +3,7 @@ from dspy.evaluate.evaluate import Evaluate from dspy.teleprompt.teleprompt import Teleprompter -from .bootstrap import BootstrapFewShot +from .bootstrap import BootstrapFewShot, BootstrapKNN from .vanilla import LabeledFewShot # TODO: Don't forget dealing with the raw demos. @@ -27,7 +27,7 @@ class BootstrapFewShotWithRandomSearch(Teleprompter): def __init__( self, metric, - teacher_settings={}, + teacher_settings=None, max_bootstrapped_demos=4, max_labeled_demos=16, max_rounds=1, @@ -38,7 +38,7 @@ def __init__( metric_threshold=None, ): self.metric = metric - self.teacher_settings = teacher_settings + self.teacher_settings = teacher_settings or {} self.max_rounds = max_rounds self.num_threads = num_threads @@ -150,6 +150,143 @@ def compile(self, student, *, teacher=None, trainset, valset=None, restrict=None return best_program +class BootstrapKNNWithRandomSearch(Teleprompter): + def __init__( + self, + metric, + embedding, + teacher_settings=None, + max_bootstrapped_demos=10_000, + max_labeled_demos=16, + max_rounds=1, + num_candidate_programs=16, + num_threads=6, + max_errors=10, + stop_at_score=None, + metric_threshold=None, + ): + self.metric = metric + self.embedding = embedding + self.teacher_settings = teacher_settings or {} + self.max_rounds = max_rounds + + self.num_threads = num_threads + self.stop_at_score = stop_at_score + self.metric_threshold = metric_threshold + self.min_static_demos = 1 + self.max_static_demos = max_labeled_demos - 1 + self.max_errors = max_errors + self.num_candidate_sets = num_candidate_programs + self.max_labeled_demos = max_labeled_demos + self.max_bootstrapped_demos = max_bootstrapped_demos + + print( + f"Going to sample between {self.min_static_demos} and {self.max_static_demos} static demos per predictor." + ) + print(f"Will attempt to bootstrap {self.num_candidate_sets} candidate sets.") + + def compile(self, student, *, teacher=None, trainset, valset=None, restrict=None, labeled_sample=True): + self.trainset = trainset + self.valset = valset or trainset # TODO: FIXME: Note this choice. + + scores = [] + all_subscores = [] + score_data = [] + + for seed in range(-3, self.num_candidate_sets): + if (restrict is not None) and (seed not in restrict): + continue + + trainset_copy = list(self.trainset) + + if seed == -3: + "Zero Shot" + program = student.reset_copy() + + elif seed == -2: + "Labeled Few Shot" + teleprompter = LabeledFewShot(k=self.max_labeled_demos) + program = teleprompter.compile(student, trainset=trainset_copy, sample=labeled_sample) + + elif seed == -1: + print("BootstrapKNN with 0 static demos") + optimizer = BootstrapKNN( + metric=self.metric, + embedding=self.embedding, + metric_threshold=self.metric_threshold, + max_bootstrapped_demos=self.max_bootstrapped_demos, + max_labeled_demos=self.max_labeled_demos, + num_static_demos=0, + teacher_settings=self.teacher_settings, + max_rounds=self.max_rounds, + max_errors=self.max_errors, + ) + program = optimizer.compile(student, teacher=teacher, trainset=trainset_copy) + + else: + num_static_demos = random.Random(seed).randint( + self.min_static_demos, + self.max_static_demos, + ) + print(f"BootstrapKNN with {num_static_demos} static demos") + + optimizer = BootstrapKNN( + metric=self.metric, + metric_threshold=self.metric_threshold, + max_bootstrapped_demos=self.max_bootstrapped_demos, + max_labeled_demos=self.max_labeled_demos, + teacher_settings=self.teacher_settings, + max_rounds=self.max_rounds, + max_errors=self.max_errors, + num_static_demos=num_static_demos, + random_seed=seed, + ) + + program = optimizer.compile(student, teacher=teacher, trainset=trainset_copy) + + evaluate = Evaluate( + devset=self.valset, + metric=self.metric, + num_threads=self.num_threads, + max_errors=self.max_errors, + display_table=False, + display_progress=True, + ) + + score, subscores = evaluate(program, return_all_scores=True) + + all_subscores.append(subscores) + + ############ Assertion-aware Optimization ############ + if hasattr(program, "_suggest_failures"): + score = score - program._suggest_failures * 0.2 + if hasattr(program, "_assert_failures"): + score = 0 if program._assert_failures > 0 else score + ###################################################### + + if len(scores) == 0 or score > max(scores): + print("New best score:", score, "for seed", seed) + best_program = program + + scores.append(score) + print(f"Scores so far: {scores}") + print(f"Best score so far: {max(scores)}") + + score_data.append((score, subscores, seed, program)) + + if self.stop_at_score is not None and score >= self.stop_at_score: + print(f"Stopping early because score {score} is >= stop_at_score {self.stop_at_score}") + break + + # To best program, attach all program candidates in decreasing average score + best_program.candidate_programs = score_data + best_program.candidate_programs = sorted(best_program.candidate_programs, key=lambda x: x[0], reverse=True) + + print(f"{len(best_program.candidate_programs)} candidate programs found.") + + return best_program + + # sample between 4 and 10 examples from traces # TODO: FIXME: The max number of demos should be determined in part by the LM's tokenizer + max_length. # This does require executing the program, or at least the predictor. diff --git a/examples/multi-input-output/beginner-multi-input-output.ipynb b/examples/multi-input-output/beginner-multi-input-output.ipynb index 72a1c51b8..8c8eab2c2 100644 --- a/examples/multi-input-output/beginner-multi-input-output.ipynb +++ b/examples/multi-input-output/beginner-multi-input-output.ipynb @@ -74,9 +74,19 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "import os\n", "import re\n", "\n", "import dspy\n", @@ -87,26 +97,13 @@ " BootstrapFewShot,\n", " BootstrapFewShotWithRandomSearch,\n", " BootstrapKNN,\n", + " BootstrapKNNWithRandomSearch,\n", " KNNFewShot,\n", " MIPROv2,\n", ")\n", "\n", "# load your environment variables from .env file\n", - "load_dotenv()\n", - "\n", - "# azure-openai model deployment\n", - "AZURE_OPENAI_KEY = os.getenv(\"AZURE_OPENAI_KEY\")\n", - "AZURE_OPENAI_ENDPOINT = os.getenv(\"AZURE_OPENAI_ENDPOINT\")\n", - "AZURE_OPENAI_DEPLOYMENT = os.getenv(\"AZURE_OPENAI_DEPLOYMENT\")\n", - "AZURE_OPENAI_VERSION = os.getenv(\"AZURE_OPENAI_VERSION\")\n", - "\n", - "# openai model deployment\n", - "\n", - "OPENAI_MODEL = os.getenv(\"OPENAI_MODEL\")\n", - "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n", - "\n", - "# ollama deployment\n", - "OLLAMA_URL = os.getenv(\"OLLAMA_URL\")" + "load_dotenv()" ] }, { @@ -310,7 +307,7 @@ "source": [ "import litellm\n", "\n", - "# litellm.api_key = \"...\"\n", + "litellm.api_key = \"...\"\n", "lm = dspy.LM(model=\"gpt-4o-mini\")\n", "\n", "dspy.settings.configure(lm=lm)" @@ -404,7 +401,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -412,7 +409,7 @@ "output_type": "stream", "text": [ "plagiarized : Yes\n", - "explanation : The two code samples are identical, with the same class name, method structure, and repeated output statements. This indicates that one code sample is likely copied from the other without any modification.\n" + "explanation : The two code samples are identical, with the same class name, method structure, and repeated output statements. This indicates that one code sample is a direct copy of the other, which constitutes plagiarism.\n" ] } ], @@ -527,13 +524,28 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "Average Metric: 26.00 / 63 (41.3%): 100%|██████████| 63/63 [00:00<00:00, 941.06it/s]" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024/11/19 15:07:36 INFO dspy.evaluate.evaluate: Average Metric: 26 / 63 (41.3%)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", "Zero-shot score: 41.27\n" ] } @@ -559,7 +571,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -572,14 +584,6 @@ " similar = dspy.OutputField(desc=\"Yes if inputs are similar or No if not\")\n", "\n", "\n", - "# gpt_judge = dspy.AzureOpenAI(\n", - "# api_base=AZURE_OPENAI_ENDPOINT,\n", - "# api_version=AZURE_OPENAI_VERSION,\n", - "# deployment_id=AZURE_OPENAI_DEPLOYMENT,\n", - "# api_key=AZURE_OPENAI_KEY,\n", - "# )\n", - "\n", - "\n", "def validate_answer_with_explanation(\n", " example: dspy.Example, pred: PlagiarismSignature, trace: object = None\n", ") -> bool:\n", @@ -683,21 +687,21 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - " 79%|███████▉ | 50/63 [00:00<00:00, 429.21it/s]" + " 81%|████████ | 51/63 [01:38<00:23, 1.92s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Bootstrapped 32 full traces after 50 examples for up to 1 rounds, amounting to 50 attempts.\n" + "Bootstrapped 32 full traces after 51 examples for up to 1 rounds, amounting to 51 attempts.\n" ] }, { @@ -709,7 +713,7 @@ } ], "source": [ - "config = {\"max_bootstrapped_demos\": 32, \"max_labeled_demos\": 16}\n", + "config = {\"max_bootstrapped_demos\": 32, \"max_labeled_demos\": 8}\n", "\n", "optimizer = BootstrapFewShot(metric=validate_answer, **config)\n", "optimized_program = optimizer.compile(PlagiarismCoT(), trainset=examples)\n", @@ -718,21 +722,28 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Average Metric: 41.00 / 63 (65.1%): 100%|██████████| 63/63 [01:20<00:00, 1.28s/it]" + " 0%| | 0/63 [00:00 9\u001b[0m optimized_program \u001b[38;5;241m=\u001b[39m \u001b[43moptimizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcompile\u001b[49m\u001b[43m(\u001b[49m\u001b[43mPlagiarismCoT\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrainset\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mexamples\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 10\u001b[0m optimized_program\u001b[38;5;241m.\u001b[39msave(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mBootstrapFewShotWithRandomSearch_program.json\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/dspy/teleprompt/random_search.py:119\u001b[0m, in \u001b[0;36mBootstrapFewShotWithRandomSearch.compile\u001b[0;34m(self, student, teacher, trainset, valset, restrict, labeled_sample)\u001b[0m\n\u001b[1;32m 108\u001b[0m program \u001b[38;5;241m=\u001b[39m optimizer\u001b[38;5;241m.\u001b[39mcompile(student, teacher\u001b[38;5;241m=\u001b[39mteacher, trainset\u001b[38;5;241m=\u001b[39mtrainset_copy)\n\u001b[1;32m 110\u001b[0m evaluate \u001b[38;5;241m=\u001b[39m Evaluate(\n\u001b[1;32m 111\u001b[0m devset\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mvalset,\n\u001b[1;32m 112\u001b[0m metric\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetric,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 116\u001b[0m display_progress\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 117\u001b[0m )\n\u001b[0;32m--> 119\u001b[0m score, subscores \u001b[38;5;241m=\u001b[39m \u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprogram\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreturn_all_scores\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 121\u001b[0m all_subscores\u001b[38;5;241m.\u001b[39mappend(subscores)\n\u001b[1;32m 123\u001b[0m \u001b[38;5;66;03m############ Assertion-aware Optimization ############\u001b[39;00m\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/dspy/evaluate/evaluate.py:115\u001b[0m, in \u001b[0;36mEvaluate.__call__\u001b[0;34m(self, program, metric, devset, num_threads, display_progress, display_table, return_all_scores, return_outputs)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n\u001b[1;32m 113\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m example_idx, example, {}, \u001b[38;5;241m0.0\u001b[39m\n\u001b[0;32m--> 115\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[43mexecutor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprocess_item\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevset\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 116\u001b[0m reordered_devset \u001b[38;5;241m=\u001b[39m [r \u001b[38;5;28;01mfor\u001b[39;00m r \u001b[38;5;129;01min\u001b[39;00m results \u001b[38;5;28;01mif\u001b[39;00m r \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m]\n\u001b[1;32m 118\u001b[0m ncorrect \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msum\u001b[39m(score \u001b[38;5;28;01mfor\u001b[39;00m _, _, _, score \u001b[38;5;129;01min\u001b[39;00m reordered_devset)\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/dspy/utils/parallelizer.py:42\u001b[0m, in \u001b[0;36mParallelExecutor.execute\u001b[0;34m(self, function, data)\u001b[0m\n\u001b[1;32m 40\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_execute_single_thread(wrapped_function, data)\n\u001b[1;32m 41\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 42\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execute_multi_thread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mwrapped_function\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/dspy/utils/parallelizer.py:163\u001b[0m, in \u001b[0;36mParallelExecutor._execute_multi_thread\u001b[0;34m(self, function, data)\u001b[0m\n\u001b[1;32m 155\u001b[0m futures \u001b[38;5;241m=\u001b[39m {executor\u001b[38;5;241m.\u001b[39msubmit(cancellable_function, pair, parent_id): pair \u001b[38;5;28;01mfor\u001b[39;00m pair \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(data)}\n\u001b[1;32m 156\u001b[0m pbar \u001b[38;5;241m=\u001b[39m tqdm\u001b[38;5;241m.\u001b[39mtqdm(\n\u001b[1;32m 157\u001b[0m total\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(data),\n\u001b[1;32m 158\u001b[0m dynamic_ncols\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 159\u001b[0m disable\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdisable_progress_bar,\n\u001b[1;32m 160\u001b[0m file\u001b[38;5;241m=\u001b[39msys\u001b[38;5;241m.\u001b[39mstdout,\n\u001b[1;32m 161\u001b[0m )\n\u001b[0;32m--> 163\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m future \u001b[38;5;129;01min\u001b[39;00m as_completed(futures):\n\u001b[1;32m 164\u001b[0m index, result \u001b[38;5;241m=\u001b[39m future\u001b[38;5;241m.\u001b[39mresult()\n\u001b[1;32m 166\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m job_cancelled:\n", - "File \u001b[0;32m~/.rye/py/cpython@3.9.18/lib/python3.9/concurrent/futures/_base.py:245\u001b[0m, in \u001b[0;36mas_completed\u001b[0;34m(fs, timeout)\u001b[0m\n\u001b[1;32m 240\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m wait_timeout \u001b[38;5;241m<\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 241\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTimeoutError\u001b[39;00m(\n\u001b[1;32m 242\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%d\u001b[39;00m\u001b[38;5;124m (of \u001b[39m\u001b[38;5;132;01m%d\u001b[39;00m\u001b[38;5;124m) futures unfinished\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m%\u001b[39m (\n\u001b[1;32m 243\u001b[0m \u001b[38;5;28mlen\u001b[39m(pending), total_futures))\n\u001b[0;32m--> 245\u001b[0m \u001b[43mwaiter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mevent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43mwait_timeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 247\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m waiter\u001b[38;5;241m.\u001b[39mlock:\n\u001b[1;32m 248\u001b[0m finished \u001b[38;5;241m=\u001b[39m waiter\u001b[38;5;241m.\u001b[39mfinished_futures\n", - "File \u001b[0;32m~/.rye/py/cpython@3.9.18/lib/python3.9/threading.py:581\u001b[0m, in \u001b[0;36mEvent.wait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 579\u001b[0m signaled \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_flag\n\u001b[1;32m 580\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m signaled:\n\u001b[0;32m--> 581\u001b[0m signaled \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_cond\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 582\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m signaled\n", - "File \u001b[0;32m~/.rye/py/cpython@3.9.18/lib/python3.9/threading.py:312\u001b[0m, in \u001b[0;36mCondition.wait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 310\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m: \u001b[38;5;66;03m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[39;00m\n\u001b[1;32m 311\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 312\u001b[0m \u001b[43mwaiter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43macquire\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 313\u001b[0m gotit \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m 314\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/dspy/utils/parallelizer.py:135\u001b[0m, in \u001b[0;36mParallelExecutor._execute_multi_thread..interrupt_handler_manager..interrupt_handler\u001b[0;34m(sig, frame)\u001b[0m\n\u001b[1;32m 133\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcancel_jobs\u001b[38;5;241m.\u001b[39mset()\n\u001b[1;32m 134\u001b[0m logger\u001b[38;5;241m.\u001b[39mwarning(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mReceived SIGINT. Cancelling execution.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 135\u001b[0m \u001b[43mdefault_handler\u001b[49m\u001b[43m(\u001b[49m\u001b[43msig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mframe\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + "name": "stdout", + "output_type": "stream", + "text": [ + "Bootstrapped 9 full traces after 13 examples for up to 1 rounds, amounting to 13 attempts.\n", + "Average Metric: 39.00 / 63 (61.9%): 100%|██████████| 63/63 [00:30<00:00, 2.05it/s]" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024/11/19 14:25:52 INFO dspy.evaluate.evaluate: Average Metric: 39 / 63 (61.9%)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Scores so far: [41.27, 46.03, 58.73, 61.9, 61.9]\n", + "Best score so far: 61.9\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 10%|▉ | 6/63 [00:07<01:14, 1.31s/it]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Bootstrapped 4 full traces after 6 examples for up to 1 rounds, amounting to 6 attempts.\n", + "Average Metric: 41.00 / 63 (65.1%): 100%|██████████| 63/63 [00:27<00:00, 2.26it/s]" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024/11/19 14:26:28 INFO dspy.evaluate.evaluate: Average Metric: 41 / 63 (65.1%)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "New best score: 65.08 for seed 2\n", + "Scores so far: [41.27, 46.03, 58.73, 61.9, 61.9, 65.08]\n", + "Best score so far: 65.08\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 43%|████▎ | 27/63 [00:46<01:01, 1.72s/it]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Bootstrapped 16 full traces after 27 examples for up to 1 rounds, amounting to 27 attempts.\n", + "Average Metric: 47.00 / 63 (74.6%): 100%|██████████| 63/63 [00:29<00:00, 2.15it/s]" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024/11/19 14:27:43 INFO dspy.evaluate.evaluate: Average Metric: 47 / 63 (74.6%)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "New best score: 74.6 for seed 3\n", + "Scores so far: [41.27, 46.03, 58.73, 61.9, 61.9, 65.08, 74.6]\n", + "Best score so far: 74.6\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 52%|█████▏ | 33/63 [01:16<01:09, 2.32s/it]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Bootstrapped 16 full traces after 33 examples for up to 1 rounds, amounting to 33 attempts.\n", + "Average Metric: 32.00 / 63 (50.8%): 100%|██████████| 63/63 [01:38<00:00, 1.57s/it]" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024/11/19 14:30:39 INFO dspy.evaluate.evaluate: Average Metric: 32 / 63 (50.8%)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Scores so far: [41.27, 46.03, 58.73, 61.9, 61.9, 65.08, 74.6, 50.79]\n", + "Best score so far: 74.6\n", + "8 candidate programs found.\n" ] } ], @@ -833,14 +1026,29 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 29, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "BootstrapFewShotWithRandomSearch score for validate_answer : 76.19\n" + "Average Metric: 47.00 / 63 (74.6%): 100%|██████████| 63/63 [00:00<00:00, 2293.66it/s]" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024/11/19 14:30:39 INFO dspy.evaluate.evaluate: Average Metric: 47 / 63 (74.6%)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "BootstrapFewShotWithRandomSearch score : 74.6\n" ] } ], @@ -861,48 +1069,9 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "metadata": {}, - "outputs": [ - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[8], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m k \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m3\u001b[39m\n\u001b[0;32m----> 2\u001b[0m optimizer \u001b[38;5;241m=\u001b[39m \u001b[43mKNNFewShot\u001b[49m\u001b[43m(\u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mexamples\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3\u001b[0m optimized_program \u001b[38;5;241m=\u001b[39m optimizer\u001b[38;5;241m.\u001b[39mcompile(PlagiarismCoT(), trainset\u001b[38;5;241m=\u001b[39mexamples)\n\u001b[1;32m 4\u001b[0m score \u001b[38;5;241m=\u001b[39m evaluate(optimized_program)\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/dspy/teleprompt/knn_fewshot.py:13\u001b[0m, in \u001b[0;36mKNNFewShot.__init__\u001b[0;34m(self, k, trainset, vectorizer, **few_shot_bootstrap_args)\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, k: \u001b[38;5;28mint\u001b[39m, trainset: List[dsp\u001b[38;5;241m.\u001b[39mExample], vectorizer\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mfew_shot_bootstrap_args):\n\u001b[0;32m---> 13\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mKNN \u001b[38;5;241m=\u001b[39m \u001b[43mKNN\u001b[49m\u001b[43m(\u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrainset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvectorizer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvectorizer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 14\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfew_shot_bootstrap_args \u001b[38;5;241m=\u001b[39m few_shot_bootstrap_args\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/dspy/predict/knn.py:30\u001b[0m, in \u001b[0;36mKNN.__init__\u001b[0;34m(self, k, trainset, vectorizer)\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mk \u001b[38;5;241m=\u001b[39m k\n\u001b[1;32m 29\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtrainset \u001b[38;5;241m=\u001b[39m trainset\n\u001b[0;32m---> 30\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membedding \u001b[38;5;241m=\u001b[39m vectorizer \u001b[38;5;129;01mor\u001b[39;00m dspy\u001b[38;5;241m.\u001b[39mEmbedding(\u001b[43mdsp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mSentenceTransformersVectorizer\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m 31\u001b[0m trainset_casted_to_vectorize \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m 32\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m | \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin([\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mkey\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvalue\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m example\u001b[38;5;241m.\u001b[39mitems() \u001b[38;5;28;01mif\u001b[39;00m key \u001b[38;5;129;01min\u001b[39;00m example\u001b[38;5;241m.\u001b[39minputs()])\n\u001b[1;32m 33\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m example \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtrainset\n\u001b[1;32m 34\u001b[0m ]\n\u001b[1;32m 35\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtrainset_vectors \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membedding(trainset_casted_to_vectorize)\u001b[38;5;241m.\u001b[39mastype(np\u001b[38;5;241m.\u001b[39mfloat32)\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/dsp/modules/sentence_vectorizer.py:66\u001b[0m, in \u001b[0;36mSentenceTransformersVectorizer.__init__\u001b[0;34m(self, model_name_or_path, vectorize_bs, max_gpu_devices, normalize_embeddings)\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_devices, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_gpu \u001b[38;5;241m=\u001b[39m determine_devices(max_gpu_devices)\n\u001b[1;32m 64\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mproxy_device \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcuda\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_gpu \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcpu\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m---> 66\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel \u001b[38;5;241m=\u001b[39m \u001b[43mSentenceTransformer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel_name_or_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mproxy_device\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 68\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel_name_or_path \u001b[38;5;241m=\u001b[39m model_name_or_path\n\u001b[1;32m 69\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mvectorize_bs \u001b[38;5;241m=\u001b[39m vectorize_bs\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/sentence_transformers/SentenceTransformer.py:301\u001b[0m, in \u001b[0;36mSentenceTransformer.__init__\u001b[0;34m(self, model_name_or_path, modules, device, prompts, default_prompt_name, similarity_fn_name, cache_folder, trust_remote_code, revision, local_files_only, token, use_auth_token, truncate_dim, model_kwargs, tokenizer_kwargs, config_kwargs, model_card_data, backend)\u001b[0m\n\u001b[1;32m 297\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m model_name_or_path \u001b[38;5;129;01mand\u001b[39;00m model_name_or_path\u001b[38;5;241m.\u001b[39mlower() \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m basic_transformer_models:\n\u001b[1;32m 298\u001b[0m \u001b[38;5;66;03m# A model from sentence-transformers\u001b[39;00m\n\u001b[1;32m 299\u001b[0m model_name_or_path \u001b[38;5;241m=\u001b[39m __MODEL_HUB_ORGANIZATION__ \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m model_name_or_path\n\u001b[0;32m--> 301\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[43mis_sentence_transformer_model\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 302\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel_name_or_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 303\u001b[0m \u001b[43m \u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 304\u001b[0m \u001b[43m \u001b[49m\u001b[43mcache_folder\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcache_folder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 305\u001b[0m \u001b[43m \u001b[49m\u001b[43mrevision\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 306\u001b[0m \u001b[43m \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 307\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[1;32m 308\u001b[0m modules, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodule_kwargs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_load_sbert_model(\n\u001b[1;32m 309\u001b[0m model_name_or_path,\n\u001b[1;32m 310\u001b[0m token\u001b[38;5;241m=\u001b[39mtoken,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 317\u001b[0m config_kwargs\u001b[38;5;241m=\u001b[39mconfig_kwargs,\n\u001b[1;32m 318\u001b[0m )\n\u001b[1;32m 319\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/sentence_transformers/util.py:1303\u001b[0m, in \u001b[0;36mis_sentence_transformer_model\u001b[0;34m(model_name_or_path, token, cache_folder, revision, local_files_only)\u001b[0m\n\u001b[1;32m 1282\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mis_sentence_transformer_model\u001b[39m(\n\u001b[1;32m 1283\u001b[0m model_name_or_path: \u001b[38;5;28mstr\u001b[39m,\n\u001b[1;32m 1284\u001b[0m token: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m|\u001b[39m \u001b[38;5;28mstr\u001b[39m \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1287\u001b[0m local_files_only: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 1288\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mbool\u001b[39m:\n\u001b[1;32m 1289\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1290\u001b[0m \u001b[38;5;124;03m Checks if the given model name or path corresponds to a SentenceTransformer model.\u001b[39;00m\n\u001b[1;32m 1291\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1300\u001b[0m \u001b[38;5;124;03m bool: True if the model is a SentenceTransformer model, False otherwise.\u001b[39;00m\n\u001b[1;32m 1301\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[1;32m 1302\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mbool\u001b[39m(\n\u001b[0;32m-> 1303\u001b[0m \u001b[43mload_file_path\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1304\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel_name_or_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1305\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mmodules.json\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1306\u001b[0m \u001b[43m \u001b[49m\u001b[43mtoken\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1307\u001b[0m \u001b[43m \u001b[49m\u001b[43mcache_folder\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcache_folder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1308\u001b[0m \u001b[43m \u001b[49m\u001b[43mrevision\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1309\u001b[0m \u001b[43m \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1310\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1311\u001b[0m )\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/sentence_transformers/util.py:1343\u001b[0m, in \u001b[0;36mload_file_path\u001b[0;34m(model_name_or_path, filename, token, cache_folder, revision, local_files_only)\u001b[0m\n\u001b[1;32m 1341\u001b[0m \u001b[38;5;66;03m# If file is remote\u001b[39;00m\n\u001b[1;32m 1342\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1343\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mhf_hub_download\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1344\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel_name_or_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1345\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfilename\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1346\u001b[0m \u001b[43m \u001b[49m\u001b[43mrevision\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1347\u001b[0m \u001b[43m \u001b[49m\u001b[43mlibrary_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43msentence-transformers\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1348\u001b[0m \u001b[43m \u001b[49m\u001b[43mtoken\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1349\u001b[0m \u001b[43m \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcache_folder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1350\u001b[0m \u001b[43m \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1351\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1352\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n\u001b[1;32m 1353\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py:114\u001b[0m, in \u001b[0;36mvalidate_hf_hub_args.._inner_fn\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 111\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m check_use_auth_token:\n\u001b[1;32m 112\u001b[0m kwargs \u001b[38;5;241m=\u001b[39m smoothly_deprecate_use_auth_token(fn_name\u001b[38;5;241m=\u001b[39mfn\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, has_token\u001b[38;5;241m=\u001b[39mhas_token, kwargs\u001b[38;5;241m=\u001b[39mkwargs)\n\u001b[0;32m--> 114\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/huggingface_hub/file_download.py:862\u001b[0m, in \u001b[0;36mhf_hub_download\u001b[0;34m(repo_id, filename, subfolder, repo_type, revision, library_name, library_version, cache_dir, local_dir, user_agent, force_download, proxies, etag_timeout, token, local_files_only, headers, endpoint, resume_download, force_filename, local_dir_use_symlinks)\u001b[0m\n\u001b[1;32m 842\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _hf_hub_download_to_local_dir(\n\u001b[1;32m 843\u001b[0m \u001b[38;5;66;03m# Destination\u001b[39;00m\n\u001b[1;32m 844\u001b[0m local_dir\u001b[38;5;241m=\u001b[39mlocal_dir,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 859\u001b[0m local_files_only\u001b[38;5;241m=\u001b[39mlocal_files_only,\n\u001b[1;32m 860\u001b[0m )\n\u001b[1;32m 861\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 862\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_hf_hub_download_to_cache_dir\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 863\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# Destination\u001b[39;49;00m\n\u001b[1;32m 864\u001b[0m \u001b[43m \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 865\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# File info\u001b[39;49;00m\n\u001b[1;32m 866\u001b[0m \u001b[43m \u001b[49m\u001b[43mrepo_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrepo_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 867\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfilename\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 868\u001b[0m \u001b[43m \u001b[49m\u001b[43mrepo_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrepo_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 869\u001b[0m \u001b[43m \u001b[49m\u001b[43mrevision\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 870\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# HTTP info\u001b[39;49;00m\n\u001b[1;32m 871\u001b[0m \u001b[43m \u001b[49m\u001b[43mendpoint\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mendpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 872\u001b[0m \u001b[43m \u001b[49m\u001b[43metag_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43metag_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 873\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 874\u001b[0m \u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 875\u001b[0m \u001b[43m \u001b[49m\u001b[43mtoken\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 876\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# Additional options\u001b[39;49;00m\n\u001b[1;32m 877\u001b[0m \u001b[43m \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 878\u001b[0m \u001b[43m \u001b[49m\u001b[43mforce_download\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 879\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/huggingface_hub/file_download.py:925\u001b[0m, in \u001b[0;36m_hf_hub_download_to_cache_dir\u001b[0;34m(cache_dir, repo_id, filename, repo_type, revision, endpoint, etag_timeout, headers, proxies, token, local_files_only, force_download)\u001b[0m\n\u001b[1;32m 921\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m pointer_path\n\u001b[1;32m 923\u001b[0m \u001b[38;5;66;03m# Try to get metadata (etag, commit_hash, url, size) from the server.\u001b[39;00m\n\u001b[1;32m 924\u001b[0m \u001b[38;5;66;03m# If we can't, a HEAD request error is returned.\u001b[39;00m\n\u001b[0;32m--> 925\u001b[0m (url_to_download, etag, commit_hash, expected_size, head_call_error) \u001b[38;5;241m=\u001b[39m \u001b[43m_get_metadata_or_catch_error\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 926\u001b[0m \u001b[43m \u001b[49m\u001b[43mrepo_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrepo_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 927\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfilename\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 928\u001b[0m \u001b[43m \u001b[49m\u001b[43mrepo_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrepo_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 929\u001b[0m \u001b[43m \u001b[49m\u001b[43mrevision\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 930\u001b[0m \u001b[43m \u001b[49m\u001b[43mendpoint\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mendpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 931\u001b[0m \u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 932\u001b[0m \u001b[43m \u001b[49m\u001b[43metag_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43metag_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 933\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 934\u001b[0m \u001b[43m \u001b[49m\u001b[43mtoken\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 935\u001b[0m \u001b[43m \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 936\u001b[0m \u001b[43m \u001b[49m\u001b[43mstorage_folder\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstorage_folder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 937\u001b[0m \u001b[43m \u001b[49m\u001b[43mrelative_filename\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrelative_filename\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 938\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 940\u001b[0m \u001b[38;5;66;03m# etag can be None for several reasons:\u001b[39;00m\n\u001b[1;32m 941\u001b[0m \u001b[38;5;66;03m# 1. we passed local_files_only.\u001b[39;00m\n\u001b[1;32m 942\u001b[0m \u001b[38;5;66;03m# 2. we don't have a connection\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 948\u001b[0m \u001b[38;5;66;03m# If the specified revision is a commit hash, look inside \"snapshots\".\u001b[39;00m\n\u001b[1;32m 949\u001b[0m \u001b[38;5;66;03m# If the specified revision is a branch or tag, look inside \"refs\".\u001b[39;00m\n\u001b[1;32m 950\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m head_call_error \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 951\u001b[0m \u001b[38;5;66;03m# Couldn't make a HEAD call => let's try to find a local file\u001b[39;00m\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/huggingface_hub/file_download.py:1376\u001b[0m, in \u001b[0;36m_get_metadata_or_catch_error\u001b[0;34m(repo_id, filename, repo_type, revision, endpoint, proxies, etag_timeout, headers, token, local_files_only, relative_filename, storage_folder)\u001b[0m\n\u001b[1;32m 1374\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1375\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1376\u001b[0m metadata \u001b[38;5;241m=\u001b[39m \u001b[43mget_hf_file_metadata\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1377\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43metag_timeout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtoken\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken\u001b[49m\n\u001b[1;32m 1378\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1379\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m EntryNotFoundError \u001b[38;5;28;01mas\u001b[39;00m http_error:\n\u001b[1;32m 1380\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m storage_folder \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m relative_filename \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 1381\u001b[0m \u001b[38;5;66;03m# Cache the non-existence of the file\u001b[39;00m\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py:114\u001b[0m, in \u001b[0;36mvalidate_hf_hub_args.._inner_fn\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 111\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m check_use_auth_token:\n\u001b[1;32m 112\u001b[0m kwargs \u001b[38;5;241m=\u001b[39m smoothly_deprecate_use_auth_token(fn_name\u001b[38;5;241m=\u001b[39mfn\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, has_token\u001b[38;5;241m=\u001b[39mhas_token, kwargs\u001b[38;5;241m=\u001b[39mkwargs)\n\u001b[0;32m--> 114\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/huggingface_hub/file_download.py:1296\u001b[0m, in \u001b[0;36mget_hf_file_metadata\u001b[0;34m(url, token, proxies, timeout, library_name, library_version, user_agent, headers)\u001b[0m\n\u001b[1;32m 1293\u001b[0m headers[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAccept-Encoding\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124midentity\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;66;03m# prevent any compression => we want to know the real size of the file\u001b[39;00m\n\u001b[1;32m 1295\u001b[0m \u001b[38;5;66;03m# Retrieve metadata\u001b[39;00m\n\u001b[0;32m-> 1296\u001b[0m r \u001b[38;5;241m=\u001b[39m \u001b[43m_request_wrapper\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1297\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mHEAD\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1298\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1299\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1300\u001b[0m \u001b[43m \u001b[49m\u001b[43mallow_redirects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 1301\u001b[0m \u001b[43m \u001b[49m\u001b[43mfollow_relative_redirects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 1302\u001b[0m \u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1303\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1304\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1305\u001b[0m hf_raise_for_status(r)\n\u001b[1;32m 1307\u001b[0m \u001b[38;5;66;03m# Return\u001b[39;00m\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/huggingface_hub/file_download.py:277\u001b[0m, in \u001b[0;36m_request_wrapper\u001b[0;34m(method, url, follow_relative_redirects, **params)\u001b[0m\n\u001b[1;32m 275\u001b[0m \u001b[38;5;66;03m# Recursively follow relative redirects\u001b[39;00m\n\u001b[1;32m 276\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m follow_relative_redirects:\n\u001b[0;32m--> 277\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43m_request_wrapper\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 278\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 279\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 280\u001b[0m \u001b[43m \u001b[49m\u001b[43mfollow_relative_redirects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 281\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 282\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 284\u001b[0m \u001b[38;5;66;03m# If redirection, we redirect only relative paths.\u001b[39;00m\n\u001b[1;32m 285\u001b[0m \u001b[38;5;66;03m# This is useful in case of a renamed repository.\u001b[39;00m\n\u001b[1;32m 286\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;241m300\u001b[39m \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m response\u001b[38;5;241m.\u001b[39mstatus_code \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m399\u001b[39m:\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/huggingface_hub/file_download.py:300\u001b[0m, in \u001b[0;36m_request_wrapper\u001b[0;34m(method, url, follow_relative_redirects, **params)\u001b[0m\n\u001b[1;32m 297\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n\u001b[1;32m 299\u001b[0m \u001b[38;5;66;03m# Perform request and return if status_code is not in the retry list.\u001b[39;00m\n\u001b[0;32m--> 300\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mget_session\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 301\u001b[0m hf_raise_for_status(response)\n\u001b[1;32m 302\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/requests/sessions.py:589\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m 584\u001b[0m send_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 585\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtimeout\u001b[39m\u001b[38;5;124m\"\u001b[39m: timeout,\n\u001b[1;32m 586\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mallow_redirects\u001b[39m\u001b[38;5;124m\"\u001b[39m: allow_redirects,\n\u001b[1;32m 587\u001b[0m }\n\u001b[1;32m 588\u001b[0m send_kwargs\u001b[38;5;241m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 589\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 591\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/requests/sessions.py:703\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m 700\u001b[0m start \u001b[38;5;241m=\u001b[39m preferred_clock()\n\u001b[1;32m 702\u001b[0m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[0;32m--> 703\u001b[0m r \u001b[38;5;241m=\u001b[39m \u001b[43madapter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 705\u001b[0m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[1;32m 706\u001b[0m elapsed \u001b[38;5;241m=\u001b[39m preferred_clock() \u001b[38;5;241m-\u001b[39m start\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/huggingface_hub/utils/_http.py:93\u001b[0m, in \u001b[0;36mUniqueRequestIdAdapter.send\u001b[0;34m(self, request, *args, **kwargs)\u001b[0m\n\u001b[1;32m 91\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Catch any RequestException to append request id to the error message for debugging.\"\"\"\u001b[39;00m\n\u001b[1;32m 92\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 93\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 94\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mRequestException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 95\u001b[0m request_id \u001b[38;5;241m=\u001b[39m request\u001b[38;5;241m.\u001b[39mheaders\u001b[38;5;241m.\u001b[39mget(X_AMZN_TRACE_ID)\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/requests/adapters.py:667\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m 664\u001b[0m timeout \u001b[38;5;241m=\u001b[39m TimeoutSauce(connect\u001b[38;5;241m=\u001b[39mtimeout, read\u001b[38;5;241m=\u001b[39mtimeout)\n\u001b[1;32m 666\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 667\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 668\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 669\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 670\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 671\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 672\u001b[0m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 673\u001b[0m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 674\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 675\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 676\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 677\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 678\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 679\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 681\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[1;32m 682\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(err, request\u001b[38;5;241m=\u001b[39mrequest)\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/urllib3/connectionpool.py:716\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m 713\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_prepare_proxy(conn)\n\u001b[1;32m 715\u001b[0m \u001b[38;5;66;03m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[0;32m--> 716\u001b[0m httplib_response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 717\u001b[0m \u001b[43m \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 718\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 719\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 720\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 721\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 722\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 723\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 724\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 726\u001b[0m \u001b[38;5;66;03m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[1;32m 727\u001b[0m \u001b[38;5;66;03m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[1;32m 728\u001b[0m \u001b[38;5;66;03m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[1;32m 729\u001b[0m \u001b[38;5;66;03m# mess.\u001b[39;00m\n\u001b[1;32m 730\u001b[0m response_conn \u001b[38;5;241m=\u001b[39m conn \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m release_conn \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/urllib3/connectionpool.py:404\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m 402\u001b[0m \u001b[38;5;66;03m# Trigger any extra validation we need to do.\u001b[39;00m\n\u001b[1;32m 403\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 404\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconn\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 405\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (SocketTimeout, BaseSSLError) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 406\u001b[0m \u001b[38;5;66;03m# Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.\u001b[39;00m\n\u001b[1;32m 407\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_raise_timeout(err\u001b[38;5;241m=\u001b[39me, url\u001b[38;5;241m=\u001b[39murl, timeout_value\u001b[38;5;241m=\u001b[39mconn\u001b[38;5;241m.\u001b[39mtimeout)\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/urllib3/connectionpool.py:1061\u001b[0m, in \u001b[0;36mHTTPSConnectionPool._validate_conn\u001b[0;34m(self, conn)\u001b[0m\n\u001b[1;32m 1059\u001b[0m \u001b[38;5;66;03m# Force connect early to allow us to validate the connection.\u001b[39;00m\n\u001b[1;32m 1060\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mgetattr\u001b[39m(conn, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msock\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m): \u001b[38;5;66;03m# AppEngine might not have `.sock`\u001b[39;00m\n\u001b[0;32m-> 1061\u001b[0m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1063\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m conn\u001b[38;5;241m.\u001b[39mis_verified:\n\u001b[1;32m 1064\u001b[0m warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[1;32m 1065\u001b[0m (\n\u001b[1;32m 1066\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUnverified HTTPS request is being made to host \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1071\u001b[0m InsecureRequestWarning,\n\u001b[1;32m 1072\u001b[0m )\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/urllib3/connection.py:419\u001b[0m, in \u001b[0;36mHTTPSConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 410\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m 411\u001b[0m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mca_certs\n\u001b[1;32m 412\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mca_cert_dir\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 415\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(context, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mload_default_certs\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 416\u001b[0m ):\n\u001b[1;32m 417\u001b[0m context\u001b[38;5;241m.\u001b[39mload_default_certs()\n\u001b[0;32m--> 419\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock \u001b[38;5;241m=\u001b[39m \u001b[43mssl_wrap_socket\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 420\u001b[0m \u001b[43m \u001b[49m\u001b[43msock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 421\u001b[0m \u001b[43m \u001b[49m\u001b[43mkeyfile\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkey_file\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 422\u001b[0m \u001b[43m \u001b[49m\u001b[43mcertfile\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcert_file\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 423\u001b[0m \u001b[43m \u001b[49m\u001b[43mkey_password\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkey_password\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 424\u001b[0m \u001b[43m \u001b[49m\u001b[43mca_certs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mca_certs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 425\u001b[0m \u001b[43m \u001b[49m\u001b[43mca_cert_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mca_cert_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 426\u001b[0m \u001b[43m \u001b[49m\u001b[43mca_cert_data\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mca_cert_data\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 427\u001b[0m \u001b[43m \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 428\u001b[0m \u001b[43m \u001b[49m\u001b[43mssl_context\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontext\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 429\u001b[0m \u001b[43m \u001b[49m\u001b[43mtls_in_tls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtls_in_tls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 430\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 432\u001b[0m \u001b[38;5;66;03m# If we're using all defaults and the connection\u001b[39;00m\n\u001b[1;32m 433\u001b[0m \u001b[38;5;66;03m# is TLSv1 or TLSv1.1 we throw a DeprecationWarning\u001b[39;00m\n\u001b[1;32m 434\u001b[0m \u001b[38;5;66;03m# for the host.\u001b[39;00m\n\u001b[1;32m 435\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m 436\u001b[0m default_ssl_context\n\u001b[1;32m 437\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mssl_version \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 438\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mversion\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 439\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock\u001b[38;5;241m.\u001b[39mversion() \u001b[38;5;129;01min\u001b[39;00m {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTLSv1\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTLSv1.1\u001b[39m\u001b[38;5;124m\"\u001b[39m}\n\u001b[1;32m 440\u001b[0m ): \u001b[38;5;66;03m# Defensive:\u001b[39;00m\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/urllib3/util/ssl_.py:458\u001b[0m, in \u001b[0;36mssl_wrap_socket\u001b[0;34m(sock, keyfile, certfile, cert_reqs, ca_certs, server_hostname, ssl_version, ciphers, ssl_context, ca_cert_dir, key_password, ca_cert_data, tls_in_tls)\u001b[0m\n\u001b[1;32m 446\u001b[0m warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[1;32m 447\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAn HTTPS request has been made, but the SNI (Server Name \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 448\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIndication) extension to TLS is not available on this platform. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 454\u001b[0m SNIMissingWarning,\n\u001b[1;32m 455\u001b[0m )\n\u001b[1;32m 457\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m send_sni:\n\u001b[0;32m--> 458\u001b[0m ssl_sock \u001b[38;5;241m=\u001b[39m \u001b[43m_ssl_wrap_socket_impl\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 459\u001b[0m \u001b[43m \u001b[49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtls_in_tls\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\n\u001b[1;32m 460\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 461\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 462\u001b[0m ssl_sock \u001b[38;5;241m=\u001b[39m _ssl_wrap_socket_impl(sock, context, tls_in_tls)\n", - "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/urllib3/util/ssl_.py:502\u001b[0m, in \u001b[0;36m_ssl_wrap_socket_impl\u001b[0;34m(sock, ssl_context, tls_in_tls, server_hostname)\u001b[0m\n\u001b[1;32m 499\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m SSLTransport(sock, ssl_context, server_hostname)\n\u001b[1;32m 501\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m server_hostname:\n\u001b[0;32m--> 502\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mssl_context\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrap_socket\u001b[49m\u001b[43m(\u001b[49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 503\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 504\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ssl_context\u001b[38;5;241m.\u001b[39mwrap_socket(sock)\n", - "File \u001b[0;32m~/.rye/py/cpython@3.9.18/lib/python3.9/ssl.py:501\u001b[0m, in \u001b[0;36mSSLContext.wrap_socket\u001b[0;34m(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)\u001b[0m\n\u001b[1;32m 495\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrap_socket\u001b[39m(\u001b[38;5;28mself\u001b[39m, sock, server_side\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 496\u001b[0m do_handshake_on_connect\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 497\u001b[0m suppress_ragged_eofs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 498\u001b[0m server_hostname\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, session\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[1;32m 499\u001b[0m \u001b[38;5;66;03m# SSLSocket class handles server_hostname encoding before it calls\u001b[39;00m\n\u001b[1;32m 500\u001b[0m \u001b[38;5;66;03m# ctx._wrap_socket()\u001b[39;00m\n\u001b[0;32m--> 501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msslsocket_class\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_create\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 502\u001b[0m \u001b[43m \u001b[49m\u001b[43msock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 503\u001b[0m \u001b[43m \u001b[49m\u001b[43mserver_side\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_side\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 504\u001b[0m \u001b[43m \u001b[49m\u001b[43mdo_handshake_on_connect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdo_handshake_on_connect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 505\u001b[0m \u001b[43m \u001b[49m\u001b[43msuppress_ragged_eofs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msuppress_ragged_eofs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 506\u001b[0m \u001b[43m \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 507\u001b[0m \u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 508\u001b[0m \u001b[43m \u001b[49m\u001b[43msession\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msession\u001b[49m\n\u001b[1;32m 509\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/.rye/py/cpython@3.9.18/lib/python3.9/ssl.py:1074\u001b[0m, in \u001b[0;36mSSLSocket._create\u001b[0;34m(cls, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, context, session)\u001b[0m\n\u001b[1;32m 1071\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0.0\u001b[39m:\n\u001b[1;32m 1072\u001b[0m \u001b[38;5;66;03m# non-blocking\u001b[39;00m\n\u001b[1;32m 1073\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdo_handshake_on_connect should not be specified for non-blocking sockets\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1074\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_handshake\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1075\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (\u001b[38;5;167;01mOSError\u001b[39;00m, \u001b[38;5;167;01mValueError\u001b[39;00m):\n\u001b[1;32m 1076\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclose()\n", - "File \u001b[0;32m~/.rye/py/cpython@3.9.18/lib/python3.9/ssl.py:1343\u001b[0m, in \u001b[0;36mSSLSocket.do_handshake\u001b[0;34m(self, block)\u001b[0m\n\u001b[1;32m 1341\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0.0\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m block:\n\u001b[1;32m 1342\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msettimeout(\u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m-> 1343\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_handshake\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1344\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 1345\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msettimeout(timeout)\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], + "outputs": [], "source": [ "k = 3\n", "optimizer = KNNFewShot(k, examples)\n", @@ -912,17 +1081,9 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "KNNFewShot score : 65.08\n" - ] - } - ], + "outputs": [], "source": [ "# score for KNNFewShot evaluation with `validate_answer` function\n", "score = evaluate(optimized_program)\n", @@ -938,59 +1099,111 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - " 79%|███████▉ | 50/63 [00:00<00:00, 1373.78it/s]" + " 81%|████████ | 51/63 [00:00<00:00, 520.45it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Bootstrapped 32 full traces after 50 examples for up to 1 rounds, amounting to 50 attempts.\n" + "Bootstrapped 32 full traces after 51 examples for up to 1 rounds, amounting to 51 attempts.\n" + ] + } + ], + "source": [ + "optimizer = BootstrapKNN(\n", + " metric=validate_answer,\n", + " embedding=dspy.Embedding(\"openai/text-embedding-3-small\"),\n", + " max_labeled_demos=8,\n", + " max_bootstrapped_demos=32,\n", + ")\n", + "optimized_program = optimizer.compile(PlagiarismCoT(), trainset=examples)\n", + "optimized_program.save(\"BootstrapKNN_program.json\")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 27.00 / 63 (42.9%): 100%|██████████| 63/63 [02:00<00:00, 1.91s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\n" + "2024/11/19 15:34:59 INFO dspy.evaluate.evaluate: Average Metric: 27.0 / 63 (42.9%)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "BootstrapKNN score : 42.86\n" ] } ], "source": [ - "optimizer = BootstrapKNN(\n", + "# score for BootstrapKNN evaluation with `validate_answer` function\n", + "score = evaluate(optimized_program)\n", + "print(f\"BootstrapKNN score : {score}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5.5 Example of BootstrapKNNWithRandomSearch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = BootstrapKNNWithRandomSearch(\n", " metric=validate_answer,\n", " embedding=dspy.Embedding(\"openai/text-embedding-3-small\"),\n", - " k=4,\n", + " max_labeled_demos=8,\n", " max_bootstrapped_demos=32,\n", + " num_candidate_programs=5,\n", + " num_threads=4,\n", ")\n", "optimized_program = optimizer.compile(PlagiarismCoT(), trainset=examples)\n", - "optimized_program.save(\"BootstrapKNNFewShot_program.json\")" + "optimized_program.save(\"BootstrapKNNWithRandomSearch_program.json\")" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Average Metric: 38.00 / 63 (60.3%): 100%|██████████| 63/63 [00:00<00:00, 223.77it/s]" + "Average Metric: 43.00 / 63 (68.3%): 100%|██████████| 63/63 [00:33<00:00, 1.86it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2024/11/18 18:46:42 INFO dspy.evaluate.evaluate: Average Metric: 38 / 63 (60.3%)\n" + "2024/11/19 14:47:31 INFO dspy.evaluate.evaluate: Average Metric: 43 / 63 (68.3%)\n" ] }, { @@ -998,21 +1211,21 @@ "output_type": "stream", "text": [ "\n", - "BootstrapKNNFewShotWithRandomSearch score : 60.32\n" + "BootstrapKNN score : 68.25\n" ] } ], "source": [ - "# score for BootstrapKNN evaluation with `validate_answer` function\n", + "# score for BootstrapKNNWithRandomSearch evaluation with `validate_answer` function\n", "score = evaluate(optimized_program)\n", - "print(f\"BootstrapKNN score : {score}\")" + "print(f\"BootstrapKNNWithRandomSearch score : {score}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 5.5 Example of MIPROv2\n", + "## 5.6 Example of MIPROv2\n", "\n", "[MIPROv2 paper](https://arxiv.org/abs/2406.11695), [MIPROv2 class](https://github.com/stanfordnlp/dspy/blob/main/dspy/teleprompt/mipro_optimizer_v2.py) " ] diff --git a/testing/playbook.ipynb b/testing/playbook.ipynb new file mode 100644 index 000000000..2f4cda14d --- /dev/null +++ b/testing/playbook.ipynb @@ -0,0 +1,148 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "import dspy\n", + "import litellm\n", + "from optimizer_tester import OptimizerTester\n", + "\n", + "litellm.api_key = \"...\"\n", + "lm = dspy.LM(model=\"gpt-4o-mini\")\n", + "\n", + "dspy.settings.configure(lm=lm)\n", + "tester = OptimizerTester(task_model=lm, prompt_model=lm)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using the latest cached version of the module from /Users/knrz/.cache/huggingface/modules/datasets_modules/datasets/hotpot_qa/133b9501f892e5193babbad937bee3b4899deb4691ef4d791e6ac0111c875bb5 (last modified on Tue Nov 19 15:48:51 2024) since it couldn't be found locally at hotpot_qa, or remotely on the Hugging Face Hub.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8c75bdab7fb54cc898e264209086de08", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Downloading data: 0%| | 0.00/566M [00:00 21\u001b[0m \u001b[43mtester\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtest_optimizer_default\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 22\u001b[0m \u001b[43m \u001b[49m\u001b[43mtest_bootstrap_knn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 23\u001b[0m \u001b[43m \u001b[49m\u001b[43mdatasets\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\n\u001b[1;32m 24\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# \"hotpotqa\",\u001b[39;49;00m\n\u001b[1;32m 25\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# \"hotpotqa_conditional\",\u001b[39;49;00m\n\u001b[1;32m 26\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtweet\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 27\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtweet_metric\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 28\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# \"heart_disease\",\u001b[39;49;00m\n\u001b[1;32m 29\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;66;43;03m# \"hover_retrieve_discrete\",\u001b[39;49;00m\n\u001b[1;32m 30\u001b[0m \u001b[43m \u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 31\u001b[0m \u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/testing/optimizer_tester.py:254\u001b[0m, in \u001b[0;36mOptimizerTester.test_optimizer_default\u001b[0;34m(self, optimizer_function, datasets, test_name)\u001b[0m\n\u001b[1;32m 249\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mtest_optimizer_default\u001b[39m(\n\u001b[1;32m 250\u001b[0m \u001b[38;5;28mself\u001b[39m, optimizer_function, datasets\u001b[38;5;241m=\u001b[39mdatasets, test_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdefault\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 251\u001b[0m ):\n\u001b[1;32m 253\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m dataset \u001b[38;5;129;01min\u001b[39;00m datasets:\n\u001b[0;32m--> 254\u001b[0m task \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_dataset\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdataset\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 255\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTesting Optimizers on \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdataset\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m ...\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 256\u001b[0m dspy\u001b[38;5;241m.\u001b[39msettings\u001b[38;5;241m.\u001b[39mlm\u001b[38;5;241m.\u001b[39mmax_tokens \u001b[38;5;241m=\u001b[39m task\u001b[38;5;241m.\u001b[39mget_max_tokens()\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/testing/optimizer_tester.py:160\u001b[0m, in \u001b[0;36mOptimizerTester.load_dataset\u001b[0;34m(self, dataset)\u001b[0m\n\u001b[1;32m 158\u001b[0m ds \u001b[38;5;241m=\u001b[39m GSM8KTask()\n\u001b[1;32m 159\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m dataset \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtweet\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m--> 160\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mTweetTask\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 161\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m dataset \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mheart_disease\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 162\u001b[0m ds \u001b[38;5;241m=\u001b[39m HeartDiseaseTask()\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/testing/tasks/tweet.py:120\u001b[0m, in \u001b[0;36mTweetTask.__init__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 118\u001b[0m \n\u001b[1;32m 119\u001b[0m \u001b[38;5;66;03m# Load the dataset.\u001b[39;00m\n\u001b[0;32m--> 120\u001b[0m dataset \u001b[38;5;241m=\u001b[39m \u001b[43mHotPotQA\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 121\u001b[0m \u001b[43m \u001b[49m\u001b[43mtrain_seed\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 122\u001b[0m \u001b[43m \u001b[49m\u001b[43mtrain_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m500\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 123\u001b[0m \u001b[43m \u001b[49m\u001b[43meval_seed\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m2023\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 124\u001b[0m \u001b[43m \u001b[49m\u001b[43mdev_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m200\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 125\u001b[0m \u001b[43m \u001b[49m\u001b[43mtest_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 126\u001b[0m \u001b[43m \u001b[49m\u001b[43mkeep_details\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 127\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 129\u001b[0m \u001b[38;5;66;03m# Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.\u001b[39;00m\n\u001b[1;32m 130\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtrainset \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m 131\u001b[0m x\u001b[38;5;241m.\u001b[39mwithout(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtype\u001b[39m\u001b[38;5;124m\"\u001b[39m)\u001b[38;5;241m.\u001b[39mwith_inputs(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mquestion\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m dataset\u001b[38;5;241m.\u001b[39mtrain\n\u001b[1;32m 132\u001b[0m ]\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/dspy/datasets/hotpotqa.py:14\u001b[0m, in \u001b[0;36mHotPotQA.__init__\u001b[0;34m(self, only_hard_examples, keep_details, unofficial_dev, *args, **kwargs)\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 11\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m only_hard_examples, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCare must be taken when adding support for easy examples.\u001b[39m\u001b[38;5;124m\"\u001b[39m \\\n\u001b[1;32m 12\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDev must be all hard to match official dev, but training can be flexible.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m---> 14\u001b[0m hf_official_train \u001b[38;5;241m=\u001b[39m \u001b[43mload_dataset\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mhotpot_qa\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mfullwiki\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msplit\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mtrain\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrust_remote_code\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 15\u001b[0m hf_official_dev \u001b[38;5;241m=\u001b[39m load_dataset(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhotpot_qa\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfullwiki\u001b[39m\u001b[38;5;124m'\u001b[39m, split\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mvalidation\u001b[39m\u001b[38;5;124m'\u001b[39m, trust_remote_code\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 17\u001b[0m official_train \u001b[38;5;241m=\u001b[39m []\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/datasets/load.py:2628\u001b[0m, in \u001b[0;36mload_dataset\u001b[0;34m(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, verification_mode, ignore_verifications, keep_in_memory, save_infos, revision, token, use_auth_token, task, streaming, num_proc, storage_options, trust_remote_code, **config_kwargs)\u001b[0m\n\u001b[1;32m 2625\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m builder_instance\u001b[38;5;241m.\u001b[39mas_streaming_dataset(split\u001b[38;5;241m=\u001b[39msplit)\n\u001b[1;32m 2627\u001b[0m \u001b[38;5;66;03m# Download and prepare data\u001b[39;00m\n\u001b[0;32m-> 2628\u001b[0m \u001b[43mbuilder_instance\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdownload_and_prepare\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2629\u001b[0m \u001b[43m \u001b[49m\u001b[43mdownload_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_config\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2630\u001b[0m \u001b[43m \u001b[49m\u001b[43mdownload_mode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_mode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2631\u001b[0m \u001b[43m \u001b[49m\u001b[43mverification_mode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mverification_mode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2632\u001b[0m \u001b[43m \u001b[49m\u001b[43mnum_proc\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnum_proc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2633\u001b[0m \u001b[43m \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstorage_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2634\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2636\u001b[0m \u001b[38;5;66;03m# Build dataset for splits\u001b[39;00m\n\u001b[1;32m 2637\u001b[0m keep_in_memory \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 2638\u001b[0m keep_in_memory \u001b[38;5;28;01mif\u001b[39;00m keep_in_memory \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m is_small_dataset(builder_instance\u001b[38;5;241m.\u001b[39minfo\u001b[38;5;241m.\u001b[39mdataset_size)\n\u001b[1;32m 2639\u001b[0m )\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/datasets/builder.py:1029\u001b[0m, in \u001b[0;36mDatasetBuilder.download_and_prepare\u001b[0;34m(self, output_dir, download_config, download_mode, verification_mode, ignore_verifications, try_from_hf_gcs, dl_manager, base_path, use_auth_token, file_format, max_shard_size, num_proc, storage_options, **download_and_prepare_kwargs)\u001b[0m\n\u001b[1;32m 1027\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m num_proc \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 1028\u001b[0m prepare_split_kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnum_proc\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m num_proc\n\u001b[0;32m-> 1029\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_download_and_prepare\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1030\u001b[0m \u001b[43m \u001b[49m\u001b[43mdl_manager\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdl_manager\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1031\u001b[0m \u001b[43m \u001b[49m\u001b[43mverification_mode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mverification_mode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1032\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mprepare_split_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1033\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mdownload_and_prepare_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1034\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1035\u001b[0m \u001b[38;5;66;03m# Sync info\u001b[39;00m\n\u001b[1;32m 1036\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minfo\u001b[38;5;241m.\u001b[39mdataset_size \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msum\u001b[39m(split\u001b[38;5;241m.\u001b[39mnum_bytes \u001b[38;5;28;01mfor\u001b[39;00m split \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minfo\u001b[38;5;241m.\u001b[39msplits\u001b[38;5;241m.\u001b[39mvalues())\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/datasets/builder.py:1791\u001b[0m, in \u001b[0;36mGeneratorBasedBuilder._download_and_prepare\u001b[0;34m(self, dl_manager, verification_mode, **prepare_splits_kwargs)\u001b[0m\n\u001b[1;32m 1790\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_download_and_prepare\u001b[39m(\u001b[38;5;28mself\u001b[39m, dl_manager, verification_mode, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mprepare_splits_kwargs):\n\u001b[0;32m-> 1791\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_download_and_prepare\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1792\u001b[0m \u001b[43m \u001b[49m\u001b[43mdl_manager\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1793\u001b[0m \u001b[43m \u001b[49m\u001b[43mverification_mode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1794\u001b[0m \u001b[43m \u001b[49m\u001b[43mcheck_duplicate_keys\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mverification_mode\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m==\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mVerificationMode\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mBASIC_CHECKS\u001b[49m\n\u001b[1;32m 1795\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mverification_mode\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m==\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mVerificationMode\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mALL_CHECKS\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1796\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mprepare_splits_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1797\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/datasets/builder.py:1102\u001b[0m, in \u001b[0;36mDatasetBuilder._download_and_prepare\u001b[0;34m(self, dl_manager, verification_mode, **prepare_split_kwargs)\u001b[0m\n\u001b[1;32m 1100\u001b[0m split_dict \u001b[38;5;241m=\u001b[39m SplitDict(dataset_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset_name)\n\u001b[1;32m 1101\u001b[0m split_generators_kwargs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_make_split_generators_kwargs(prepare_split_kwargs)\n\u001b[0;32m-> 1102\u001b[0m split_generators \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_split_generators\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdl_manager\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msplit_generators_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1104\u001b[0m \u001b[38;5;66;03m# Checksums verification\u001b[39;00m\n\u001b[1;32m 1105\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m verification_mode \u001b[38;5;241m==\u001b[39m VerificationMode\u001b[38;5;241m.\u001b[39mALL_CHECKS \u001b[38;5;129;01mand\u001b[39;00m dl_manager\u001b[38;5;241m.\u001b[39mrecord_checksums:\n", + "File \u001b[0;32m~/.cache/huggingface/modules/datasets_modules/datasets/hotpot_qa/133b9501f892e5193babbad937bee3b4899deb4691ef4d791e6ac0111c875bb5/hotpot_qa.py:116\u001b[0m, in \u001b[0;36mHotpotQA._split_generators\u001b[0;34m(self, dl_manager)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mname \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfullwiki\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 114\u001b[0m paths[datasets\u001b[38;5;241m.\u001b[39mSplit\u001b[38;5;241m.\u001b[39mTEST] \u001b[38;5;241m=\u001b[39m _URL_BASE \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhotpot_test_fullwiki_v1.json\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 116\u001b[0m files \u001b[38;5;241m=\u001b[39m \u001b[43mdl_manager\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdownload\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpaths\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 118\u001b[0m split_generators \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 119\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m split \u001b[38;5;129;01min\u001b[39;00m files:\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/datasets/download/download_manager.py:257\u001b[0m, in \u001b[0;36mDownloadManager.download\u001b[0;34m(self, url_or_urls)\u001b[0m\n\u001b[1;32m 255\u001b[0m start_time \u001b[38;5;241m=\u001b[39m datetime\u001b[38;5;241m.\u001b[39mnow()\n\u001b[1;32m 256\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m stack_multiprocessing_download_progress_bars():\n\u001b[0;32m--> 257\u001b[0m downloaded_path_or_paths \u001b[38;5;241m=\u001b[39m \u001b[43mmap_nested\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 258\u001b[0m \u001b[43m \u001b[49m\u001b[43mdownload_func\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 259\u001b[0m \u001b[43m \u001b[49m\u001b[43murl_or_urls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 260\u001b[0m \u001b[43m \u001b[49m\u001b[43mmap_tuple\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 261\u001b[0m \u001b[43m \u001b[49m\u001b[43mnum_proc\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnum_proc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 262\u001b[0m \u001b[43m \u001b[49m\u001b[43mdesc\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mDownloading data files\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 263\u001b[0m \u001b[43m \u001b[49m\u001b[43mbatched\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 264\u001b[0m \u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 265\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 266\u001b[0m duration \u001b[38;5;241m=\u001b[39m datetime\u001b[38;5;241m.\u001b[39mnow() \u001b[38;5;241m-\u001b[39m start_time\n\u001b[1;32m 267\u001b[0m logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDownloading took \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mduration\u001b[38;5;241m.\u001b[39mtotal_seconds()\u001b[38;5;250m \u001b[39m\u001b[38;5;241m/\u001b[39m\u001b[38;5;241m/\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;241m60\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m min\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/datasets/utils/py_utils.py:511\u001b[0m, in \u001b[0;36mmap_nested\u001b[0;34m(function, data_struct, dict_only, map_list, map_tuple, map_numpy, num_proc, parallel_min_length, batched, batch_size, types, disable_tqdm, desc)\u001b[0m\n\u001b[1;32m 509\u001b[0m batch_size \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mmax\u001b[39m(\u001b[38;5;28mlen\u001b[39m(iterable) \u001b[38;5;241m/\u001b[39m\u001b[38;5;241m/\u001b[39m num_proc \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mint\u001b[39m(\u001b[38;5;28mlen\u001b[39m(iterable) \u001b[38;5;241m%\u001b[39m num_proc \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m), \u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 510\u001b[0m iterable \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(iter_batched(iterable, batch_size))\n\u001b[0;32m--> 511\u001b[0m mapped \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m 512\u001b[0m _single_map_nested((function, obj, batched, batch_size, types, \u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[38;5;28;01mNone\u001b[39;00m))\n\u001b[1;32m 513\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m obj \u001b[38;5;129;01min\u001b[39;00m hf_tqdm(iterable, disable\u001b[38;5;241m=\u001b[39mdisable_tqdm, desc\u001b[38;5;241m=\u001b[39mdesc)\n\u001b[1;32m 514\u001b[0m ]\n\u001b[1;32m 515\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m batched:\n\u001b[1;32m 516\u001b[0m mapped \u001b[38;5;241m=\u001b[39m [mapped_item \u001b[38;5;28;01mfor\u001b[39;00m mapped_batch \u001b[38;5;129;01min\u001b[39;00m mapped \u001b[38;5;28;01mfor\u001b[39;00m mapped_item \u001b[38;5;129;01min\u001b[39;00m mapped_batch]\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/datasets/utils/py_utils.py:512\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 509\u001b[0m batch_size \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mmax\u001b[39m(\u001b[38;5;28mlen\u001b[39m(iterable) \u001b[38;5;241m/\u001b[39m\u001b[38;5;241m/\u001b[39m num_proc \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mint\u001b[39m(\u001b[38;5;28mlen\u001b[39m(iterable) \u001b[38;5;241m%\u001b[39m num_proc \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m), \u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 510\u001b[0m iterable \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(iter_batched(iterable, batch_size))\n\u001b[1;32m 511\u001b[0m mapped \u001b[38;5;241m=\u001b[39m [\n\u001b[0;32m--> 512\u001b[0m \u001b[43m_single_map_nested\u001b[49m\u001b[43m(\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfunction\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mobj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatched\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtypes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 513\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m obj \u001b[38;5;129;01min\u001b[39;00m hf_tqdm(iterable, disable\u001b[38;5;241m=\u001b[39mdisable_tqdm, desc\u001b[38;5;241m=\u001b[39mdesc)\n\u001b[1;32m 514\u001b[0m ]\n\u001b[1;32m 515\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m batched:\n\u001b[1;32m 516\u001b[0m mapped \u001b[38;5;241m=\u001b[39m [mapped_item \u001b[38;5;28;01mfor\u001b[39;00m mapped_batch \u001b[38;5;129;01min\u001b[39;00m mapped \u001b[38;5;28;01mfor\u001b[39;00m mapped_item \u001b[38;5;129;01min\u001b[39;00m mapped_batch]\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/datasets/utils/py_utils.py:380\u001b[0m, in \u001b[0;36m_single_map_nested\u001b[0;34m(args)\u001b[0m\n\u001b[1;32m 373\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m function(data_struct)\n\u001b[1;32m 374\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m 375\u001b[0m batched\n\u001b[1;32m 376\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(data_struct, \u001b[38;5;28mdict\u001b[39m)\n\u001b[1;32m 377\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(data_struct, types)\n\u001b[1;32m 378\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mall\u001b[39m(\u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(v, (\u001b[38;5;28mdict\u001b[39m, types)) \u001b[38;5;28;01mfor\u001b[39;00m v \u001b[38;5;129;01min\u001b[39;00m data_struct)\n\u001b[1;32m 379\u001b[0m ):\n\u001b[0;32m--> 380\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m [mapped_item \u001b[38;5;28;01mfor\u001b[39;00m batch \u001b[38;5;129;01min\u001b[39;00m iter_batched(data_struct, batch_size) \u001b[38;5;28;01mfor\u001b[39;00m mapped_item \u001b[38;5;129;01min\u001b[39;00m function(batch)]\n\u001b[1;32m 382\u001b[0m \u001b[38;5;66;03m# Reduce logging to keep things readable in multiprocessing with tqdm\u001b[39;00m\n\u001b[1;32m 383\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m rank \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m logging\u001b[38;5;241m.\u001b[39mget_verbosity() \u001b[38;5;241m<\u001b[39m logging\u001b[38;5;241m.\u001b[39mWARNING:\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/datasets/utils/py_utils.py:380\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 373\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m function(data_struct)\n\u001b[1;32m 374\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m 375\u001b[0m batched\n\u001b[1;32m 376\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(data_struct, \u001b[38;5;28mdict\u001b[39m)\n\u001b[1;32m 377\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(data_struct, types)\n\u001b[1;32m 378\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mall\u001b[39m(\u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(v, (\u001b[38;5;28mdict\u001b[39m, types)) \u001b[38;5;28;01mfor\u001b[39;00m v \u001b[38;5;129;01min\u001b[39;00m data_struct)\n\u001b[1;32m 379\u001b[0m ):\n\u001b[0;32m--> 380\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m [mapped_item \u001b[38;5;28;01mfor\u001b[39;00m batch \u001b[38;5;129;01min\u001b[39;00m iter_batched(data_struct, batch_size) \u001b[38;5;28;01mfor\u001b[39;00m mapped_item \u001b[38;5;129;01min\u001b[39;00m \u001b[43mfunction\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch\u001b[49m\u001b[43m)\u001b[49m]\n\u001b[1;32m 382\u001b[0m \u001b[38;5;66;03m# Reduce logging to keep things readable in multiprocessing with tqdm\u001b[39;00m\n\u001b[1;32m 383\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m rank \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m logging\u001b[38;5;241m.\u001b[39mget_verbosity() \u001b[38;5;241m<\u001b[39m logging\u001b[38;5;241m.\u001b[39mWARNING:\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/datasets/download/download_manager.py:313\u001b[0m, in \u001b[0;36mDownloadManager._download_batched\u001b[0;34m(self, url_or_filenames, download_config)\u001b[0m\n\u001b[1;32m 300\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m thread_map(\n\u001b[1;32m 301\u001b[0m download_func,\n\u001b[1;32m 302\u001b[0m url_or_filenames,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 310\u001b[0m tqdm_class\u001b[38;5;241m=\u001b[39mtqdm,\n\u001b[1;32m 311\u001b[0m )\n\u001b[1;32m 312\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 313\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m [\n\u001b[1;32m 314\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_download_single(url_or_filename, download_config\u001b[38;5;241m=\u001b[39mdownload_config)\n\u001b[1;32m 315\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m url_or_filename \u001b[38;5;129;01min\u001b[39;00m url_or_filenames\n\u001b[1;32m 316\u001b[0m ]\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/datasets/download/download_manager.py:314\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 300\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m thread_map(\n\u001b[1;32m 301\u001b[0m download_func,\n\u001b[1;32m 302\u001b[0m url_or_filenames,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 310\u001b[0m tqdm_class\u001b[38;5;241m=\u001b[39mtqdm,\n\u001b[1;32m 311\u001b[0m )\n\u001b[1;32m 312\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 313\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m [\n\u001b[0;32m--> 314\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_download_single\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl_or_filename\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdownload_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_config\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 315\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m url_or_filename \u001b[38;5;129;01min\u001b[39;00m url_or_filenames\n\u001b[1;32m 316\u001b[0m ]\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/datasets/download/download_manager.py:323\u001b[0m, in \u001b[0;36mDownloadManager._download_single\u001b[0;34m(self, url_or_filename, download_config)\u001b[0m\n\u001b[1;32m 320\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_relative_path(url_or_filename):\n\u001b[1;32m 321\u001b[0m \u001b[38;5;66;03m# append the relative path to the base_path\u001b[39;00m\n\u001b[1;32m 322\u001b[0m url_or_filename \u001b[38;5;241m=\u001b[39m url_or_path_join(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_base_path, url_or_filename)\n\u001b[0;32m--> 323\u001b[0m out \u001b[38;5;241m=\u001b[39m \u001b[43mcached_path\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl_or_filename\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdownload_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_config\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 324\u001b[0m out \u001b[38;5;241m=\u001b[39m tracked_str(out)\n\u001b[1;32m 325\u001b[0m out\u001b[38;5;241m.\u001b[39mset_origin(url_or_filename)\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/datasets/utils/file_utils.py:211\u001b[0m, in \u001b[0;36mcached_path\u001b[0;34m(url_or_filename, download_config, **download_kwargs)\u001b[0m\n\u001b[1;32m 205\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m 206\u001b[0m storage_options\n\u001b[1;32m 207\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m storage_options\u001b[38;5;241m.\u001b[39mkeys() \u001b[38;5;241m<\u001b[39m {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttp\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttps\u001b[39m\u001b[38;5;124m\"\u001b[39m}\n\u001b[1;32m 208\u001b[0m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (download_config\u001b[38;5;241m.\u001b[39mstorage_options \u001b[38;5;129;01mand\u001b[39;00m download_config\u001b[38;5;241m.\u001b[39mstorage_options\u001b[38;5;241m.\u001b[39mkeys() \u001b[38;5;241m<\u001b[39m {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttp\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttps\u001b[39m\u001b[38;5;124m\"\u001b[39m})\n\u001b[1;32m 209\u001b[0m ):\n\u001b[1;32m 210\u001b[0m storage_options \u001b[38;5;241m=\u001b[39m {}\n\u001b[0;32m--> 211\u001b[0m output_path \u001b[38;5;241m=\u001b[39m \u001b[43mget_from_cache\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 212\u001b[0m \u001b[43m \u001b[49m\u001b[43murl_or_filename\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 213\u001b[0m \u001b[43m \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 214\u001b[0m \u001b[43m \u001b[49m\u001b[43mforce_download\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforce_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 215\u001b[0m \u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 216\u001b[0m \u001b[43m \u001b[49m\u001b[43mresume_download\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresume_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 217\u001b[0m \u001b[43m \u001b[49m\u001b[43muser_agent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43muser_agent\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 218\u001b[0m \u001b[43m \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 219\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_etag\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43muse_etag\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 220\u001b[0m \u001b[43m \u001b[49m\u001b[43mmax_retries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 221\u001b[0m \u001b[43m \u001b[49m\u001b[43mtoken\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 222\u001b[0m \u001b[43m \u001b[49m\u001b[43mignore_url_params\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mignore_url_params\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 223\u001b[0m \u001b[43m \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstorage_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 224\u001b[0m \u001b[43m \u001b[49m\u001b[43mdownload_desc\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdownload_desc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 225\u001b[0m \u001b[43m \u001b[49m\u001b[43mdisable_tqdm\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdisable_tqdm\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 226\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 227\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(url_or_filename):\n\u001b[1;32m 228\u001b[0m \u001b[38;5;66;03m# File, and it exists.\u001b[39;00m\n\u001b[1;32m 229\u001b[0m output_path \u001b[38;5;241m=\u001b[39m url_or_filename\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/datasets/utils/file_utils.py:693\u001b[0m, in \u001b[0;36mget_from_cache\u001b[0;34m(url, cache_dir, force_download, proxies, etag_timeout, resume_download, user_agent, local_files_only, use_etag, max_retries, token, use_auth_token, ignore_url_params, storage_options, download_desc, disable_tqdm)\u001b[0m\n\u001b[1;32m 689\u001b[0m fsspec_get(\n\u001b[1;32m 690\u001b[0m url, temp_file, storage_options\u001b[38;5;241m=\u001b[39mstorage_options, desc\u001b[38;5;241m=\u001b[39mdownload_desc, disable_tqdm\u001b[38;5;241m=\u001b[39mdisable_tqdm\n\u001b[1;32m 691\u001b[0m )\n\u001b[1;32m 692\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 693\u001b[0m \u001b[43mhttp_get\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 694\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 695\u001b[0m \u001b[43m \u001b[49m\u001b[43mtemp_file\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtemp_file\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 696\u001b[0m \u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 697\u001b[0m \u001b[43m \u001b[49m\u001b[43mresume_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 698\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 699\u001b[0m \u001b[43m \u001b[49m\u001b[43mcookies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcookies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 700\u001b[0m \u001b[43m \u001b[49m\u001b[43mmax_retries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 701\u001b[0m \u001b[43m \u001b[49m\u001b[43mdesc\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdownload_desc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 702\u001b[0m \u001b[43m \u001b[49m\u001b[43mdisable_tqdm\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdisable_tqdm\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 703\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 705\u001b[0m logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstoring \u001b[39m\u001b[38;5;132;01m{\u001b[39;00murl\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m in cache at \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcache_path\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 706\u001b[0m shutil\u001b[38;5;241m.\u001b[39mmove(temp_file\u001b[38;5;241m.\u001b[39mname, cache_path)\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/datasets/utils/file_utils.py:462\u001b[0m, in \u001b[0;36mhttp_get\u001b[0;34m(url, temp_file, proxies, resume_size, headers, cookies, timeout, max_retries, desc, disable_tqdm)\u001b[0m\n\u001b[1;32m 449\u001b[0m total \u001b[38;5;241m=\u001b[39m resume_size \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mint\u001b[39m(content_length) \u001b[38;5;28;01mif\u001b[39;00m content_length \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 450\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m hf_tqdm(\n\u001b[1;32m 451\u001b[0m unit\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mB\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 452\u001b[0m unit_scale\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 460\u001b[0m disable\u001b[38;5;241m=\u001b[39mdisable_tqdm,\n\u001b[1;32m 461\u001b[0m ) \u001b[38;5;28;01mas\u001b[39;00m progress:\n\u001b[0;32m--> 462\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk \u001b[38;5;129;01min\u001b[39;00m response\u001b[38;5;241m.\u001b[39miter_content(chunk_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1024\u001b[39m):\n\u001b[1;32m 463\u001b[0m progress\u001b[38;5;241m.\u001b[39mupdate(\u001b[38;5;28mlen\u001b[39m(chunk))\n\u001b[1;32m 464\u001b[0m temp_file\u001b[38;5;241m.\u001b[39mwrite(chunk)\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/requests/models.py:820\u001b[0m, in \u001b[0;36mResponse.iter_content..generate\u001b[0;34m()\u001b[0m\n\u001b[1;32m 818\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mraw, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstream\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[1;32m 819\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 820\u001b[0m \u001b[38;5;28;01myield from\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mraw\u001b[38;5;241m.\u001b[39mstream(chunk_size, decode_content\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 821\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ProtocolError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 822\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m ChunkedEncodingError(e)\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/urllib3/response.py:628\u001b[0m, in \u001b[0;36mHTTPResponse.stream\u001b[0;34m(self, amt, decode_content)\u001b[0m\n\u001b[1;32m 626\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 627\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_fp_closed(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp):\n\u001b[0;32m--> 628\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mamt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_content\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 630\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m data:\n\u001b[1;32m 631\u001b[0m \u001b[38;5;28;01myield\u001b[39;00m data\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/urllib3/response.py:567\u001b[0m, in \u001b[0;36mHTTPResponse.read\u001b[0;34m(self, amt, decode_content, cache_content)\u001b[0m\n\u001b[1;32m 564\u001b[0m fp_closed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mclosed\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m 566\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_error_catcher():\n\u001b[0;32m--> 567\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fp_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m fp_closed \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 568\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 569\u001b[0m flush_decoder \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n", + "File \u001b[0;32m~/Git/stanfordnlp/dspy/.venv/lib/python3.9/site-packages/urllib3/response.py:533\u001b[0m, in \u001b[0;36mHTTPResponse._fp_read\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m 530\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m buffer\u001b[38;5;241m.\u001b[39mgetvalue()\n\u001b[1;32m 531\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 532\u001b[0m \u001b[38;5;66;03m# StringIO doesn't like amt=None\u001b[39;00m\n\u001b[0;32m--> 533\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mamt\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fp\u001b[38;5;241m.\u001b[39mread()\n", + "File \u001b[0;32m~/.rye/py/cpython@3.9.18/lib/python3.9/http/client.py:463\u001b[0m, in \u001b[0;36mHTTPResponse.read\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m 460\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m amt \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 461\u001b[0m \u001b[38;5;66;03m# Amount is given, implement using readinto\u001b[39;00m\n\u001b[1;32m 462\u001b[0m b \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mbytearray\u001b[39m(amt)\n\u001b[0;32m--> 463\u001b[0m n \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreadinto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 464\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mmemoryview\u001b[39m(b)[:n]\u001b[38;5;241m.\u001b[39mtobytes()\n\u001b[1;32m 465\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 466\u001b[0m \u001b[38;5;66;03m# Amount is not given (unbounded read) so we must check self.length\u001b[39;00m\n\u001b[1;32m 467\u001b[0m \u001b[38;5;66;03m# and self.chunked\u001b[39;00m\n", + "File \u001b[0;32m~/.rye/py/cpython@3.9.18/lib/python3.9/http/client.py:507\u001b[0m, in \u001b[0;36mHTTPResponse.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m 502\u001b[0m b \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mmemoryview\u001b[39m(b)[\u001b[38;5;241m0\u001b[39m:\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlength]\n\u001b[1;32m 504\u001b[0m \u001b[38;5;66;03m# we do not use _safe_read() here because this may be a .will_close\u001b[39;00m\n\u001b[1;32m 505\u001b[0m \u001b[38;5;66;03m# connection, and the user is reading more bytes than will be provided\u001b[39;00m\n\u001b[1;32m 506\u001b[0m \u001b[38;5;66;03m# (for example, reading in 1k chunks)\u001b[39;00m\n\u001b[0;32m--> 507\u001b[0m n \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreadinto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 508\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m n \u001b[38;5;129;01mand\u001b[39;00m b:\n\u001b[1;32m 509\u001b[0m \u001b[38;5;66;03m# Ideally, we would raise IncompleteRead if the content-length\u001b[39;00m\n\u001b[1;32m 510\u001b[0m \u001b[38;5;66;03m# wasn't satisfied, but it might break compatibility.\u001b[39;00m\n\u001b[1;32m 511\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_close_conn()\n", + "File \u001b[0;32m~/.rye/py/cpython@3.9.18/lib/python3.9/socket.py:704\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m 702\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m 703\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 704\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 705\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[1;32m 706\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_timeout_occurred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "def test_bootstrap_knn(default_program, trainset, devset, test_name, dataset_name, kwargs):\n", + " # initialize your teleprompter (optimizer) here!\n", + " teleprompter = dspy.BootstrapKNN(\n", + " metric=kwargs[\"metric\"],\n", + " embedding=dspy.Embedding(model=\"openai/text-embedding-3-small\"),\n", + " max_bootstrapped_demos=64,\n", + " max_labeled_demos=16,\n", + " )\n", + "\n", + " # call your optimizer on the default_program here!\n", + " compiled_program = teleprompter.compile(default_program.deepcopy(), trainset=trainset)\n", + "\n", + " # if you wish to tweak any of the outputs to the csv file you can do that here\n", + " output = {\n", + " \"test_name\": f\"bootstrap-knn-{dataset_name}-{test_name}\",\n", + " }\n", + "\n", + " # return the compiled program and modified output (or empty dict if no changes made)\n", + " return compiled_program, output\n", + "\n", + "tester.test_optimizer_default(\n", + " test_bootstrap_knn,\n", + " datasets=[\n", + " # \"hotpotqa\",\n", + " # \"hotpotqa_conditional\",\n", + " \"tweet\",\n", + " \"tweet_metric\",\n", + " # \"heart_disease\",\n", + " # \"hover_retrieve_discrete\",\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}