{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports & Env Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/justinai/.conda/envs/prompt-migration/lib/python3.10/site-packages/pydantic/_internal/_config.py:345: UserWarning: Valid config keys have changed in V2:\n",
      "* 'fields' has been removed\n",
      "  warnings.warn(message, UserWarning)\n"
     ]
    }
   ],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "import sys\n",
    "import os\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv()\n",
    "\n",
    "import dspy\n",
    "sys.path.append(os.path.abspath('../'))\n",
    "from benchmarks import llama_mmlu_pro, leaderboard_mmlu_pro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_THREADS = 16\n",
    "\n",
    "FEW_SHOTS = 5\n",
    "\n",
    "# See https://docs.litellm.ai/docs/providers/vllm for details\n",
    "TASK_MODEL = dspy.LM(\n",
    "    \"hosted_vllm/meta-llama/Llama-3.3-70B-Instruct\",\n",
    "    api_base = 'http://localhost:8000/v1' , # or api_base ?\n",
    "    # api_version: Optional[str] = None,\n",
    "    api_key = \"\",\n",
    "    # seed: Optional[int] = None,\n",
    "    # max_tokens: Optional[int] = None,\n",
    "    # timeout: Optional[Union[float, int]] = None,\n",
    ")\n",
    "PROMPT_MODEL = dspy.LM(\n",
    "    \"hosted_vllm/meta-llama/Llama-3.3-70B-Instruct\",\n",
    "    api_base = 'http://localhost:8000/v1', # or api_base ?\n",
    "    # api_version: Optional[str] = None,\n",
    "    api_key = \"\",\n",
    "    # seed: Optional[int] = None,\n",
    "    # max_tokens: Optional[int] = None,\n",
    "    # timeout: Optional[Union[float, int]] = None,\n",
    ")\n",
    "\n",
    "dspy.configure(lm=TASK_MODEL)\n",
    "\n",
    "# replace this with llama_mmlu_pro or whatever\n",
    "benchmark = llama_mmlu_pro\n",
    "\n",
    "# Without chain of thought:\n",
    "# program = dspy.Predict(\n",
    "#     benchmark.signature(\"\")\n",
    "# )\n",
    "\n",
    "# With chain of thought:\n",
    "program = dspy.ChainOfThought(\n",
    "    benchmark.signature(\"You are a helpful assistant designed to help with multiple choice question.\") # put your initial system prompt here, or leave blank\n",
    ")\n",
    "\n",
    "evaluate = dspy.Evaluate(\n",
    "    devset=[],\n",
    "    metric=benchmark.metric,\n",
    "    num_threads=NUM_THREADS,\n",
    "    display_progress=True,\n",
    "    display_table=True,\n",
    "    return_all_scores=True,\n",
    "    return_outputs=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1197, 2156, 8626)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainset, valset, testset = benchmark.datasets(\n",
    "    train_size=0.1,\n",
    "    validation_size=0.2,\n",
    ")\n",
    "\n",
    "len(trainset), len(valset), len(testset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Baseline Benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BASE PROMPT:\n",
      " You are a helpful assistant designed to help with multiple choice question.\n"
     ]
    }
   ],
   "source": [
    "print(\"BASE PROMPT:\\n\", program.predict.signature.instructions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "!export HOSTED_VLLM_API_KEY=\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os \n",
    "\n",
    "os.environ[\"HOSTED_VLLM_API_KEY\"]=\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025/01/21 15:44:46 ERROR dspy.utils.parallelizer: Error processing item Example({'question': 'The spontaneous fission activity rate of U-238 is 6.7 fissions/kg s. A sample of shale contains 0.055% U-238 by weight. Calculate the number of spontaneous fissions in one day in a 106-kg pile of the shale by determining the number of fissions.', 'options': {'A': '400000000.0', 'B': '600000000.0', 'C': '50000000.0', 'D': '250000000.0', 'E': '100000000.0', 'F': '200000000.0', 'G': '700000000.0', 'H': '450000000.0', 'I': '150000000.0', 'J': '320000000.0'}, 'answer': 'J'}) (input_keys={'options', 'question'}): Expected dict_keys(['reasoning', 'answer']) but got dict_keys(['reasoning']). Set `provide_traceback=True` to see the stack trace.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Metric: 638.00 / 848 (75.2%):  10%|████████                                                                         | 852/8626 [19:30<2:57:55,  1.37s/it]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025/01/21 15:44:46 ERROR dspy.utils.parallelizer: Error processing item Example({'question': 'Compute $\\\\dbinom{85}{82}$.', 'options': {'A': '102560', 'B': '252', 'C': '101170', 'D': '100890', 'E': '88440', 'F': '78960', 'G': '98770', 'H': '110870', 'I': '4680', 'J': '1254'}, 'answer': ''}) (input_keys={'options', 'question'}): 'list' object has no attribute 'items'. Set `provide_traceback=True` to see the stack trace.\n",
      "2025/01/21 15:44:46 ERROR dspy.utils.parallelizer: Error processing item Example({'question': 'A sleeve-coupling is used on a shaft 1(1/2) in. in diameter, delivering a torque of 500 lbs.-in. Calculate the diameter of the pin required to hold the coupling if the design stress for the pin material in shear is 15,000 psi.', 'options': {'A': '0.145 in.', 'B': '0.125 in.', 'C': '0.210 in.', 'D': '0.155 in.', 'E': '0.162 in.', 'F': '0.168 in.', 'G': '0.190 in.', 'H': '0.158 in.', 'I': '0.175 in.', 'J': '0.182 in.'}, 'answer': ''}) (input_keys={'options', 'question'}): Expected dict_keys(['reasoning', 'answer']) but got dict_keys(['reasoning']). Set `provide_traceback=True` to see the stack trace.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025/01/21 15:44:46 ERROR dspy.utils.parallelizer: Error processing item Example({'question': 'A 10 foot long chain is placed on a 4 foot high, frictionless table so that one end just reaches the floor. With what velocity will the other end slide off the table?', 'options': {'A': '13.6 ft. / sec.', 'B': '9.8 ft. / sec.', 'C': '12.8 ft. / sec.', 'D': '10.2 ft. / sec.', 'E': '6.7 ft. / sec.', 'F': '17.1 ft. / sec.', 'G': '18.3 ft. / sec.', 'H': '15.4 ft. / sec.', 'I': '20.4 ft. / sec.', 'J': '14.7 ft. / sec.'}, 'answer': 'B'}) (input_keys={'options', 'question'}): Expected dict_keys(['reasoning', 'answer']) but got dict_keys(['reasoning']). Set `provide_traceback=True` to see the stack trace.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Metric: 251.00 / 329 (76.3%):   4%|███                                                                              | 328/8626 [00:00<00:05, 1592.43it/s]"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'list' object has no attribute 'items'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/adapters/base.py:30\u001b[0m, in \u001b[0;36mAdapter.__call__\u001b[0;34m(self, lm, lm_kwargs, signature, demos, inputs)\u001b[0m\n\u001b[1;32m     28\u001b[0m     output, output_logprobs \u001b[38;5;241m=\u001b[39m output[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m\"\u001b[39m], output[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlogprobs\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[0;32m---> 30\u001b[0m value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparse\u001b[49m\u001b[43m(\u001b[49m\u001b[43msignature\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moutput\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     32\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mset\u001b[39m(value\u001b[38;5;241m.\u001b[39mkeys()) \u001b[38;5;241m==\u001b[39m \u001b[38;5;28mset\u001b[39m(signature\u001b[38;5;241m.\u001b[39moutput_fields\u001b[38;5;241m.\u001b[39mkeys()), \\\n\u001b[1;32m     33\u001b[0m     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExpected \u001b[39m\u001b[38;5;132;01m{\u001b[39;00msignature\u001b[38;5;241m.\u001b[39moutput_fields\u001b[38;5;241m.\u001b[39mkeys()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m but got \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvalue\u001b[38;5;241m.\u001b[39mkeys()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/utils/callback.py:234\u001b[0m, in \u001b[0;36mwith_callbacks.<locals>.wrapper\u001b[0;34m(instance, *args, **kwargs)\u001b[0m\n\u001b[1;32m    233\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m callbacks:\n\u001b[0;32m--> 234\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43minstance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    236\u001b[0m \u001b[38;5;66;03m# Generate call ID as the unique identifier for the call, this is useful for instrumentation.\u001b[39;00m\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/adapters/chat_adapter.py:84\u001b[0m, in \u001b[0;36mChatAdapter.parse\u001b[0;34m(self, signature, completion)\u001b[0m\n\u001b[1;32m     83\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m fields\u001b[38;5;241m.\u001b[39mkeys() \u001b[38;5;241m!=\u001b[39m signature\u001b[38;5;241m.\u001b[39moutput_fields\u001b[38;5;241m.\u001b[39mkeys():\n\u001b[0;32m---> 84\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExpected \u001b[39m\u001b[38;5;132;01m{\u001b[39;00msignature\u001b[38;5;241m.\u001b[39moutput_fields\u001b[38;5;241m.\u001b[39mkeys()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m but got \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfields\u001b[38;5;241m.\u001b[39mkeys()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     86\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m fields\n",
      "\u001b[0;31mValueError\u001b[0m: Expected dict_keys(['reasoning', 'answer']) but got dict_keys(['reasoning'])",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[8], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m eval_subset_size \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(testset)\n\u001b[0;32m----> 2\u001b[0m \u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m      3\u001b[0m \u001b[43m    \u001b[49m\u001b[43mprogram\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      4\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdevset\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtestset\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43meval_subset_size\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      5\u001b[0m \u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/evaluate/evaluate.py:112\u001b[0m, in \u001b[0;36mEvaluate.__call__\u001b[0;34m(self, program, metric, devset, num_threads, display_progress, display_table, return_all_scores, return_outputs)\u001b[0m\n\u001b[1;32m    108\u001b[0m         program\u001b[38;5;241m.\u001b[39m_suggest_failures \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m dspy\u001b[38;5;241m.\u001b[39msettings\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msuggest_failures\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    110\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m prediction, score\n\u001b[0;32m--> 112\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[43mexecutor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprocess_item\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevset\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    113\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(devset) \u001b[38;5;241m==\u001b[39m \u001b[38;5;28mlen\u001b[39m(results)\n\u001b[1;32m    115\u001b[0m results \u001b[38;5;241m=\u001b[39m [((dspy\u001b[38;5;241m.\u001b[39mPrediction(), \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfailure_score) \u001b[38;5;28;01mif\u001b[39;00m r \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m r) \u001b[38;5;28;01mfor\u001b[39;00m r \u001b[38;5;129;01min\u001b[39;00m results]\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/utils/parallelizer.py:39\u001b[0m, in \u001b[0;36mParallelExecutor.execute\u001b[0;34m(self, function, data)\u001b[0m\n\u001b[1;32m     37\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_execute_isolated_single_thread(wrapped_function, data)\n\u001b[1;32m     38\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 39\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execute_multi_thread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mwrapped_function\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/utils/parallelizer.py:180\u001b[0m, in \u001b[0;36mParallelExecutor._execute_multi_thread\u001b[0;34m(self, function, data)\u001b[0m\n\u001b[1;32m    172\u001b[0m pbar \u001b[38;5;241m=\u001b[39m tqdm\u001b[38;5;241m.\u001b[39mtqdm(\n\u001b[1;32m    173\u001b[0m     total\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(data),\n\u001b[1;32m    174\u001b[0m     dynamic_ncols\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m    175\u001b[0m     disable\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdisable_progress_bar,\n\u001b[1;32m    176\u001b[0m     file\u001b[38;5;241m=\u001b[39msys\u001b[38;5;241m.\u001b[39mstdout\n\u001b[1;32m    177\u001b[0m )\n\u001b[1;32m    179\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m future \u001b[38;5;129;01min\u001b[39;00m as_completed(futures):\n\u001b[0;32m--> 180\u001b[0m     index, result \u001b[38;5;241m=\u001b[39m \u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    182\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m job_cancelled:\n\u001b[1;32m    183\u001b[0m         \u001b[38;5;28;01mcontinue\u001b[39;00m\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/concurrent/futures/_base.py:451\u001b[0m, in \u001b[0;36mFuture.result\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    449\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m CancelledError()\n\u001b[1;32m    450\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;241m==\u001b[39m FINISHED:\n\u001b[0;32m--> 451\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m__get_result\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    453\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_condition\u001b[38;5;241m.\u001b[39mwait(timeout)\n\u001b[1;32m    455\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;129;01min\u001b[39;00m [CANCELLED, CANCELLED_AND_NOTIFIED]:\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/concurrent/futures/_base.py:403\u001b[0m, in \u001b[0;36mFuture.__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    401\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception:\n\u001b[1;32m    402\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 403\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception\n\u001b[1;32m    404\u001b[0m     \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m    405\u001b[0m         \u001b[38;5;66;03m# Break a reference cycle with the exception in self._exception\u001b[39;00m\n\u001b[1;32m    406\u001b[0m         \u001b[38;5;28mself\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/concurrent/futures/thread.py:58\u001b[0m, in \u001b[0;36m_WorkItem.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     55\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[1;32m     57\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 58\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     59\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m     60\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfuture\u001b[38;5;241m.\u001b[39mset_exception(exc)\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/utils/parallelizer.py:158\u001b[0m, in \u001b[0;36mParallelExecutor._execute_multi_thread.<locals>.cancellable_function\u001b[0;34m(parent_overrides, index_item)\u001b[0m\n\u001b[1;32m    155\u001b[0m thread_local_overrides\u001b[38;5;241m.\u001b[39moverrides \u001b[38;5;241m=\u001b[39m parent_overrides\u001b[38;5;241m.\u001b[39mcopy()\n\u001b[1;32m    157\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 158\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m index, \u001b[43mfunction\u001b[49m\u001b[43m(\u001b[49m\u001b[43mitem\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    159\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m    160\u001b[0m     thread_local_overrides\u001b[38;5;241m.\u001b[39moverrides \u001b[38;5;241m=\u001b[39m original_overrides\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/utils/parallelizer.py:54\u001b[0m, in \u001b[0;36mParallelExecutor._wrap_function.<locals>.wrapped\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m     52\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m current_error_count \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmax_errors:\n\u001b[1;32m     53\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcancel_jobs\u001b[38;5;241m.\u001b[39mset()\n\u001b[0;32m---> 54\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\n\u001b[1;32m     55\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprovide_traceback:\n\u001b[1;32m     56\u001b[0m     logger\u001b[38;5;241m.\u001b[39merror(\n\u001b[1;32m     57\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mError processing item \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mitem\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mStack trace:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mtraceback\u001b[38;5;241m.\u001b[39mformat_exc()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     58\u001b[0m     )\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/utils/parallelizer.py:47\u001b[0m, in \u001b[0;36mParallelExecutor._wrap_function.<locals>.wrapped\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m     45\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     46\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 47\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunction\u001b[49m\u001b[43m(\u001b[49m\u001b[43mitem\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     48\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     49\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39merror_lock:\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/evaluate/evaluate.py:101\u001b[0m, in \u001b[0;36mEvaluate.__call__.<locals>.process_item\u001b[0;34m(example)\u001b[0m\n\u001b[1;32m    100\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mprocess_item\u001b[39m(example):\n\u001b[0;32m--> 101\u001b[0m     prediction \u001b[38;5;241m=\u001b[39m \u001b[43mprogram\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mexample\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    102\u001b[0m     score \u001b[38;5;241m=\u001b[39m metric(example, prediction)\n\u001b[1;32m    104\u001b[0m     \u001b[38;5;66;03m# Increment assert and suggest failures to program's attributes\u001b[39;00m\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/utils/callback.py:234\u001b[0m, in \u001b[0;36mwith_callbacks.<locals>.wrapper\u001b[0;34m(instance, *args, **kwargs)\u001b[0m\n\u001b[1;32m    232\u001b[0m \u001b[38;5;66;03m# If no callbacks are provided, just call the function\u001b[39;00m\n\u001b[1;32m    233\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m callbacks:\n\u001b[0;32m--> 234\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43minstance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    236\u001b[0m \u001b[38;5;66;03m# Generate call ID as the unique identifier for the call, this is useful for instrumentation.\u001b[39;00m\n\u001b[1;32m    237\u001b[0m call_id \u001b[38;5;241m=\u001b[39m uuid\u001b[38;5;241m.\u001b[39muuid4()\u001b[38;5;241m.\u001b[39mhex\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/primitives/program.py:22\u001b[0m, in \u001b[0;36mModule.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[38;5;129m@with_callbacks\u001b[39m\n\u001b[1;32m     21\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m---> 22\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/predict/chain_of_thought.py:20\u001b[0m, in \u001b[0;36mChainOfThought.forward\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m---> 20\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpredict\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/utils/callback.py:234\u001b[0m, in \u001b[0;36mwith_callbacks.<locals>.wrapper\u001b[0;34m(instance, *args, **kwargs)\u001b[0m\n\u001b[1;32m    232\u001b[0m \u001b[38;5;66;03m# If no callbacks are provided, just call the function\u001b[39;00m\n\u001b[1;32m    233\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m callbacks:\n\u001b[0;32m--> 234\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43minstance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    236\u001b[0m \u001b[38;5;66;03m# Generate call ID as the unique identifier for the call, this is useful for instrumentation.\u001b[39;00m\n\u001b[1;32m    237\u001b[0m call_id \u001b[38;5;241m=\u001b[39m uuid\u001b[38;5;241m.\u001b[39muuid4()\u001b[38;5;241m.\u001b[39mhex\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/predict/predict.py:81\u001b[0m, in \u001b[0;36mPredict.__call__\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m     79\u001b[0m \u001b[38;5;129m@with_callbacks\u001b[39m\n\u001b[1;32m     80\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m---> 81\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/predict/predict.py:111\u001b[0m, in \u001b[0;36mPredict.forward\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m    109\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mdspy\u001b[39;00m\n\u001b[1;32m    110\u001b[0m adapter \u001b[38;5;241m=\u001b[39m dspy\u001b[38;5;241m.\u001b[39msettings\u001b[38;5;241m.\u001b[39madapter \u001b[38;5;129;01mor\u001b[39;00m dspy\u001b[38;5;241m.\u001b[39mChatAdapter()\n\u001b[0;32m--> 111\u001b[0m completions \u001b[38;5;241m=\u001b[39m \u001b[43madapter\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlm\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlm_kwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msignature\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msignature\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdemos\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdemos\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    113\u001b[0m pred \u001b[38;5;241m=\u001b[39m Prediction\u001b[38;5;241m.\u001b[39mfrom_completions(completions, signature\u001b[38;5;241m=\u001b[39msignature)\n\u001b[1;32m    115\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_trace\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mTrue\u001b[39;00m) \u001b[38;5;129;01mand\u001b[39;00m dspy\u001b[38;5;241m.\u001b[39msettings\u001b[38;5;241m.\u001b[39mtrace \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/adapters/base.py:45\u001b[0m, in \u001b[0;36mAdapter.__call__\u001b[0;34m(self, lm, lm_kwargs, signature, demos, inputs)\u001b[0m\n\u001b[1;32m     43\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mjson_adapter\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m JSONAdapter\n\u001b[1;32m     44\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m, JSONAdapter):\n\u001b[0;32m---> 45\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mJSONAdapter\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlm\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlm_kwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msignature\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdemos\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     46\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m e\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/adapters/json_adapter.py:60\u001b[0m, in \u001b[0;36mJSONAdapter.__call__\u001b[0;34m(self, lm, lm_kwargs, signature, demos, inputs)\u001b[0m\n\u001b[1;32m     57\u001b[0m values \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m     59\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m output \u001b[38;5;129;01min\u001b[39;00m outputs:\n\u001b[0;32m---> 60\u001b[0m     value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparse\u001b[49m\u001b[43m(\u001b[49m\u001b[43msignature\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moutput\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     61\u001b[0m     \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mset\u001b[39m(value\u001b[38;5;241m.\u001b[39mkeys()) \u001b[38;5;241m==\u001b[39m \u001b[38;5;28mset\u001b[39m(\n\u001b[1;32m     62\u001b[0m         signature\u001b[38;5;241m.\u001b[39moutput_fields\u001b[38;5;241m.\u001b[39mkeys()\n\u001b[1;32m     63\u001b[0m     ), \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExpected \u001b[39m\u001b[38;5;132;01m{\u001b[39;00msignature\u001b[38;5;241m.\u001b[39moutput_fields\u001b[38;5;241m.\u001b[39mkeys()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m but got \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvalue\u001b[38;5;241m.\u001b[39mkeys()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     64\u001b[0m     values\u001b[38;5;241m.\u001b[39mappend(value)\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/utils/callback.py:234\u001b[0m, in \u001b[0;36mwith_callbacks.<locals>.wrapper\u001b[0;34m(instance, *args, **kwargs)\u001b[0m\n\u001b[1;32m    232\u001b[0m \u001b[38;5;66;03m# If no callbacks are provided, just call the function\u001b[39;00m\n\u001b[1;32m    233\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m callbacks:\n\u001b[0;32m--> 234\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43minstance\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    236\u001b[0m \u001b[38;5;66;03m# Generate call ID as the unique identifier for the call, this is useful for instrumentation.\u001b[39;00m\n\u001b[1;32m    237\u001b[0m call_id \u001b[38;5;241m=\u001b[39m uuid\u001b[38;5;241m.\u001b[39muuid4()\u001b[38;5;241m.\u001b[39mhex\n",
      "File \u001b[0;32m~/.conda/envs/prompt-migration/lib/python3.10/site-packages/dspy/adapters/json_adapter.py:94\u001b[0m, in \u001b[0;36mJSONAdapter.parse\u001b[0;34m(self, signature, completion)\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mparse\u001b[39m(\u001b[38;5;28mself\u001b[39m, signature, completion):\n\u001b[1;32m     93\u001b[0m     fields \u001b[38;5;241m=\u001b[39m json_repair\u001b[38;5;241m.\u001b[39mloads(completion)\n\u001b[0;32m---> 94\u001b[0m     fields \u001b[38;5;241m=\u001b[39m {k: v \u001b[38;5;28;01mfor\u001b[39;00m k, v \u001b[38;5;129;01min\u001b[39;00m \u001b[43mfields\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mitems\u001b[49m() \u001b[38;5;28;01mif\u001b[39;00m k \u001b[38;5;129;01min\u001b[39;00m signature\u001b[38;5;241m.\u001b[39moutput_fields}\n\u001b[1;32m     96\u001b[0m     \u001b[38;5;66;03m# attempt to cast each value to type signature.output_fields[k].annotation\u001b[39;00m\n\u001b[1;32m     97\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m k, v \u001b[38;5;129;01min\u001b[39;00m fields\u001b[38;5;241m.\u001b[39mitems():\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'list' object has no attribute 'items'"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Metric: 251.00 / 329 (76.3%):   4%|███                                                                              | 329/8626 [00:10<00:05, 1592.43it/s]"
     ]
    }
   ],
   "source": [
    "eval_subset_size = len(testset)\n",
    "evaluate(\n",
    "    program,\n",
    "    devset=testset[:eval_subset_size],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optimize Subset + Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subset_size = 20\n",
    "optimizer = dspy.MIPROv2(\n",
    "    metric=benchmark.metric,\n",
    "    auto=\"light\",\n",
    "    num_threads=NUM_THREADS,\n",
    "    task_model=TASK_MODEL,\n",
    "    prompt_model=PROMPT_MODEL,\n",
    "    max_labeled_demos=FEW_SHOTS,\n",
    ")\n",
    "\n",
    "optimized_program = optimizer.compile(\n",
    "    program,\n",
    "    trainset=trainset[:subset_size],\n",
    "    valset=valset[:subset_size],\n",
    "    requires_permission_to_run=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"BEST PROMPT:\\n\", optimized_program.predict.signature.instructions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_subset_size = 200\n",
    "score, results, all_scores = evaluate(\n",
    "    optimized_program,\n",
    "    devset=testset[:eval_subset_size],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Medium Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subset_size = 500\n",
    "optimizer = dspy.MIPROv2(\n",
    "    metric=benchmark.metric,\n",
    "    auto=\"medium\",\n",
    "    num_threads=NUM_THREADS,\n",
    "    task_model=TASK_MODEL,\n",
    "    prompt_model=PROMPT_MODEL,\n",
    "    max_labeled_demos=FEW_SHOTS,\n",
    ")\n",
    "\n",
    "optimized_program = optimizer.compile(\n",
    "    program,\n",
    "    trainset=trainset[:subset_size],\n",
    "    valset=valset[:subset_size],\n",
    "    requires_permission_to_run=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"BEST PROMPT:\\n\", optimized_program.predict.signature.instructions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"BEST PROMPT:\\n\", optimized_program.predict.signature.instructions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"BEST EXAMPLES:\\n\", optimized_program.predict.demos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "score, results, all_scores = evaluate(\n",
    "    optimized_program,\n",
    "    devset=testset[:subset_size],\n",
    "    display_table=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_medium_subset_size = 300\n",
    "score, results, all_scores = evaluate(\n",
    "    optimized_program,\n",
    "    devset=testset[:eval_medium_subset_size],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Heavy Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = dspy.MIPROv2(\n",
    "    metric=benchmark.metric,\n",
    "    auto=\"heavy\",\n",
    "    num_threads=NUM_THREADS,\n",
    "    task_model=TASK_MODEL,\n",
    "    prompt_model=PROMPT_MODEL,\n",
    "    max_labeled_demos=FEW_SHOTS,\n",
    ")\n",
    "\n",
    "optimized_program = optimizer.compile(\n",
    "    program,\n",
    "    trainset=trainset,\n",
    "    valset=valset,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"BEST PROMPT:\\n\", optimized_program.predict.signature.instructions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "score, results, all_scores = evaluate(\n",
    "    optimized_program,\n",
    "    devset=testset,\n",
    "    display_table=False,\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}