Delete trec_pointwise.ipynb

trec_pointwise.ipynb

@@ -1,964 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Setup"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "First, we will import any dependencies, initialize GPT-3.5 Turbo model from OpenAI, and configure settings for the DSPy library."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "fatal: destination path 'pointwise_dspy' already exists and is not an empty directory.\n",
-      "/future/u/sdsam/home/projects/pointwise_dspy\n",
-      "Already on 'main'\n",
-      "Your branch is up-to-date with 'origin/main'.\n",
-      "/future/u/sdsam/home/projects\n"
-     ]
-    }
-   ],
-   "source": [
-    "!git clone https://huggingface.co/sdsam/pointwise_dspy\n",
-    "%cd pointwise_dspy/\n",
-    "!git checkout main\n",
-    "%cd ..\n",
-    "import os\n",
-    "repo_clone_path = './pointwise_dspy'\n",
-    "\n",
-    "# Set up the cache for this notebook\n",
-    "os.environ[\"DSP_NOTEBOOK_CACHEDIR\"] = repo_clone_path"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "%load_ext autoreload\n",
-    "%autoreload 2\n",
-    "\n",
-    "import sys; sys.path.append('/future/u/okhattab/repos/public/stanfordnlp/dspy')\n",
-    "import pytrec_eval\n",
-    "import openai\n",
-    "import dspy\n",
-    "import tqdm\n",
-    "import random\n",
-    "import pytrec_eval\n",
-    "import math\n",
-    "import ujson\n",
-    "import re\n",
-    "from dspy.evaluate.evaluate import Evaluate\n",
-    "\n",
-    "# Initialize GPT-3.5 Turbo model from OpenAI\n",
-    "turbo = dspy.OpenAI(model='gpt-3.5-turbo-1106', max_tokens=250, model_type='chat')\n",
-    "\n",
-    "# Configure settings for dspy library\n",
-    "dspy.settings.configure(lm=turbo)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "This loads and organizes query relevance judgments from the TREC 2019 DL track and randomly selects queries for development and training sets. You can use any dataset you want, but this notebook will use the TREC 2019 DL dataset for demonstration purposes."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "query2qrels = {}\n",
-    "\n",
-    "with open('/future/u/sdsam/home/projects/rank_dspy/formatted_qrels/final_updated_merged_ranking.jsonl') as f:\n",
-    "    for line in f:\n",
-    "        example = dspy.Example(ujson.loads(line))\n",
-    "        query2qrels[example.query] = query2qrels.get(example.query, []) + [example]\n",
-    "\n",
-    "# there are a total of 43 queries in the dataset. \n",
-    "# you can change the number that you want\n",
-    "num_dev_queries = 43\n",
-    "\n",
-    "dev_queries = random.sample(list(query2qrels.keys()), num_dev_queries)\n",
-    "train_queries = list(set(query2qrels.keys()) - set(dev_queries))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Now, we processes the training queries to create a trainset with relevant information, including query, passages, relevance scores, and ColBERT ranks (initial ranked list ranking), then shuffles the dataset. We do the same process for the devset."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "trainset = []\n",
-    "for query in train_queries:\n",
-    "    passages = [example.passage for example in query2qrels[query]]\n",
-    "    rates_list = [int(example.qrel_score) for example in query2qrels[query]]\n",
-    "    colbert_rank = [int(example.rank) for example in query2qrels[query]]\n",
-    "    \n",
-    "    example = dspy.Example({\n",
-    "        'query': query,\n",
-    "        'passages': passages,\n",
-    "        'rates_list': rates_list,\n",
-    "        'colbert_rank': colbert_rank\n",
-    "    })\n",
-    "    \n",
-    "    trainset.append(example)\n",
-    "\n",
-    "random.Random(0).shuffle(trainset)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "devset = []\n",
-    "for query in dev_queries:\n",
-    "    passages = [example.passage for example in query2qrels[query]]\n",
-    "    rates_list = [int(example.qrel_score) for example in query2qrels[query]]\n",
-    "    colbert_rank = [int(example.rank) for example in query2qrels[query]]\n",
-    "    \n",
-    "    example = dspy.Example({\n",
-    "        'query': query,\n",
-    "        'passages': passages,\n",
-    "        'rates_list': rates_list,\n",
-    "        'colbert_rank': colbert_rank\n",
-    "    })\n",
-    "    \n",
-    "    devset.append(example)\n",
-    "\n",
-    "random.Random(0).shuffle(devset)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Here, we specifically assign 'query' and 'passages' as input fields for each example."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "trainset = [example.with_inputs('query', 'passages') for example in trainset]\n",
-    "devset = [example.with_inputs('query', 'passages') for example in devset]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# DSPy"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Here we define a function that attempts to convert a given string to an integer or 0 if it fails. This helps us process the LM output."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def try_to_int(x: str):\n",
-    "    try: return int(re.findall(r'\\d+', x)[0]) - 1\n",
-    "    except Exception: return 0"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Here we define the module `RatePassage` that scores passages given a query using a `ChainOfThought` model. It limits the number of passages considered and sorts them based on scores, returning the result as a prediction."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "class RatePassage(dspy.Module):\n",
-    "    def __init__(self, depth=10):\n",
-    "        super().__init__()\n",
-    "        self.score_passage = dspy.ChainOfThought(\"query, passages -> score\")\n",
-    "        self.depth = depth\n",
-    "\n",
-    "    def forward(self, query, passages):\n",
-    "        passages = passages[:self.depth]\n",
-    "        scored_passages = []\n",
-    "\n",
-    "        for passage in passages:\n",
-    "            score = self.score_passage(query=query, passages=passage).score\n",
-    "            score = try_to_int(score)\n",
-    "\n",
-    "            scored_passages.append((score, passage))\n",
-    "\n",
-    "        scored_passages.sort(key=lambda x: x[0], reverse=True)\n",
-    "\n",
-    "        final_pred = {query: {passage: score for score, passage in scored_passages}}\n",
-    "        return dspy.Prediction(output=final_pred)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Evaluation"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "This defines a validation function `validate_rating_v4` that evaluates the performance of a rating prediction output using the NDCG metric, comparing predicted scores to ground truth ratings for a given query and set of passages."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def validate_rating_v4(query, output, trace=None):\n",
-    "    output = output.output\n",
-    "\n",
-    "    rates_list = query.rates_list\n",
-    "    passages = query.passages\n",
-    "\n",
-    "    gt_dict = {passages[i]: int(rates_list[i]) for i in range(len(passages))}\n",
-    "\n",
-    "    evaluator = pytrec_eval.RelevanceEvaluator({query.query: gt_dict}, {'ndcg_cut'})\n",
-    "    val = evaluator.evaluate({query.query: output[query.query]})\n",
-    "\n",
-    "    available_metrics = val[query.query].keys()\n",
-    "\n",
-    "    if 'ndcg_cut_10' in available_metrics:\n",
-    "        return val[query.query]['ndcg_cut_10']\n",
-    "    elif 'ndcg' in available_metrics:\n",
-    "        return val[query.query]['ndcg']\n",
-    "    else:\n",
-    "        raise KeyError(\"Expected NDCG metric not found in evaluation result.\")\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Zeroshot and Compile"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Now we can instantiate a `RatePassage` model with a specified depth, set up evaluation on a subset of the development dataset (`devset`), and evaluate the model's performance using the `validate_rating_v4` metric!"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "First, we will start with a zeroshot test"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "  0%|          | 0/43 [00:00<?, ?it/s]"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 24.274700567599893 / 43  (56.5): 100%|██████████| 43/43 [00:03<00:00, 12.13it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 24.274700567599893 / 43  (56.5%)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "56.45"
-      ]
-     },
-     "execution_count": 10,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "rate_passage_model = RatePassage(depth=40)\n",
-    "\n",
-    "\n",
-    "kwargs = dict(num_threads=9, display_progress=True, display_table=0)\n",
-    "evaluate = Evaluate(devset=devset[:43], metric=validate_rating_v4, **kwargs)\n",
-    "\n",
-    "# evaluate zero-shot\n",
-    "evaluate(rate_passage_model)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We can inspect the most recent conversation history for the GPT-3.5 Turbo model using the `inspect_history` function with a history depth of 1."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 14,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "\n",
-      "\n",
-      "\n",
-      "Given the fields `query`, `passages`, produce the fields `score`.\n",
-      "\n",
-      "---\n",
-      "\n",
-      "Follow the following format.\n",
-      "\n",
-      "Query: ${query}\n",
-      "\n",
-      "Passages: ${passages}\n",
-      "\n",
-      "Reasoning: Let's think step by step in order to ${produce the score}. We ...\n",
-      "\n",
-      "Score: ${score}\n",
-      "\n",
-      "---\n",
-      "\n",
-      "Query: is cdg airport in main paris\n",
-      "\n",
-      "Passages: cdg, Paris Charles de Gaulle Airport, Paris, France weather text for Thu 20th April. The cdg, Paris Charles de Gaulle Airport, Paris, France weather is going to be sunny. cdg, Paris Charles de Gaulle Airport, Paris, France visibility is going to be around 20 km i.e. 12 miles and an atmospheric pressure of 1035 mb.\n",
-      "\n",
-      "Reasoning: Let's think step by step in order to\u001b[32m produce the score. We will first need to check if the passages contain the information related to the query, such as the location of CDG airport in Paris. Then, we can assign a score based on the relevance and accuracy of the information provided in the passages.\n",
-      "\n",
-      "Score: 8.5\u001b[0m\n",
-      "\n",
-      "\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "turbo.inspect_history(n=1)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Compiled"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Now, we initialize a `SignatureOptimizer` from the `dspy` library, utilizing the `validate_rating_v4` metric, a history depth of 2, and the GPT-3.5 Turbo model as the prompt model. \n",
-    "\n",
-    "This time, we are compiling the optimizer with the `RatePassage` model and a subset of the `devset` for evaluation."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "./pointwise_dspy/compiler\n",
-      "Starting iteration 0/2.\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: Given the field `query` and the fields `passages`, calculate the relevancy score for each passage related to the query.\n",
-      "p: Relevancy scores:\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 8.402444304089686 / 20  (42.0): 100%|██████████| 20/20 [00:02<00:00,  8.99it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 8.402444304089686 / 20  (42.0%)\n",
-      "----------------\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: Identify and match the most relevant passage to the query based on semantic similarity and produce a score representing the degree of relevance.\n",
-      "p: Relevant passage scoring for query:\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 8.62779048156313 / 20  (43.1): 100%|██████████| 20/20 [00:02<00:00,  8.66it/s]  \n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 8.62779048156313 / 20  (43.1%)\n",
-      "----------------\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: Feed the language model with the query and passages and generate a score based on the relevance of each passage to the query.\n",
-      "p: Scoring for query and passages.\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 10.069514300349399 / 20  (50.3): 100%|██████████| 20/20 [00:04<00:00,  4.45it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 10.069514300349399 / 20  (50.3%)\n",
-      "----------------\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: Before you start computing the scores, carefully analyze the query and compare it to the passages. Take into consideration both relevance and potential keywords that might influence the score calculation.\n",
-      "p: Scores for query analysis:\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 11.351022284735192 / 20  (56.8): 100%|██████████| 20/20 [00:02<00:00,  8.22it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 11.351022284735192 / 20  (56.8%)\n",
-      "----------------\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: Use the query to compare and evaluate the relevance of each passage.\n",
-      "p: Relevance_scores_\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 11.48445319834673 / 20  (57.4): 100%|██████████| 20/20 [00:02<00:00,  7.84it/s] \n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 11.48445319834673 / 20  (57.4%)\n",
-      "----------------\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: First, perform an initial keyword search of the query within the passages. Then, compare the search results to the original query to identify relevant passages.\n",
-      "p: Scored results:\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 11.605294980491518 / 20  (58.0): 100%|██████████| 20/20 [00:02<00:00,  8.20it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 11.605294980491518 / 20  (58.0%)\n",
-      "----------------\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: First, identify the most relevant passages to the given query. Then, calculate a score for each passage based on its relevance to the query.\n",
-      "p: Relevant passages and their scores:\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 10.581135809365799 / 20  (52.9): 100%|██████████| 20/20 [00:02<00:00,  8.08it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 10.581135809365799 / 20  (52.9%)\n",
-      "----------------\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: Before generating `score`, carefully analyze each `passage` in relation to the `query` and assign a numeric value based on relevance and overall quality of the `passage`.\n",
-      "p: Calculation of relevance score for passages in relation to query:\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 12.76273405727381 / 20  (63.8): 100%|██████████| 20/20 [00:02<00:00,  8.03it/s] \n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 12.76273405727381 / 20  (63.8%)\n",
-      "----------------\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: Given the fields `query`, `passages`, produce the fields `score`.\n",
-      "p: Score:\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 10.615206363813893 / 20  (53.1): 100%|██████████| 20/20 [00:00<00:00, 957.64it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 10.615206363813893 / 20  (53.1%)\n",
-      "----------------\n",
-      "Updating Predictor 140321429486800 to:\n",
-      "i: Before generating `score`, carefully analyze each `passage` in relation to the `query` and assign a numeric value based on relevance and overall quality of the `passage`.\n",
-      "p: Calculation of relevance score for passages in relation to query:\n",
-      "Full predictor with update: \n",
-      "Predictor 0\n",
-      "i: Before generating `score`, carefully analyze each `passage` in relation to the `query` and assign a numeric value based on relevance and overall quality of the `passage`.\n",
-      "p: Calculation of relevance score for passages in relation to query:\n",
-      "\n",
-      "\n",
-      "\n",
-      "\n",
-      "\n",
-      "You are an experienced instruction optimizer for large language models. I will give some task instructions I've tried, along with their corresponding validation scores. The instructions are arranged in increasing order based on their scores, where higher scores indicate better quality. Your task is to propose a new instruction that will lead a good language model to perform the task even better. Don't be afraid to be creative.\n",
-      "\n",
-      "---\n",
-      "\n",
-      "Follow the following format.\n",
-      "\n",
-      "Attempted Instructions: ${attempted_instructions}\n",
-      "Proposed Instruction: The improved instructions for the language model\n",
-      "Proposed Prefix For Output Field: The string at the end of the prompt, which will help the model start solving the task\n",
-      "\n",
-      "---\n",
-      "\n",
-      "Attempted Instructions:\n",
-      "[1] «Instruction #1: Given the field `query` and the fields `passages`, calculate the relevancy score for each passage related to the query.»\n",
-      "[2] «Prefix #1: Relevancy scores:»\n",
-      "[3] «Resulting Score #1: 42.01»\n",
-      "[4] «Instruction #2: Identify and match the most relevant passage to the query based on semantic similarity and produce a score representing the degree of relevance.»\n",
-      "[5] «Prefix #2: Relevant passage scoring for query:»\n",
-      "[6] «Resulting Score #2: 43.14»\n",
-      "[7] «Instruction #3: Feed the language model with the query and passages and generate a score based on the relevance of each passage to the query.»\n",
-      "[8] «Prefix #3: Scoring for query and passages.»\n",
-      "[9] «Resulting Score #3: 50.35»\n",
-      "[10] «Instruction #4: First, identify the most relevant passages to the given query. Then, calculate a score for each passage based on its relevance to the query.»\n",
-      "[11] «Prefix #4: Relevant passages and their scores:»\n",
-      "[12] «Resulting Score #4: 52.91»\n",
-      "[13] «Instruction #5: Given the fields `query`, `passages`, produce the fields `score`.»\n",
-      "[14] «Prefix #5: Score:»\n",
-      "[15] «Resulting Score #5: 53.08»\n",
-      "[16] «Instruction #6: Before you start computing the scores, carefully analyze the query and compare it to the passages. Take into consideration both relevance and potential keywords that might influence the score calculation.»\n",
-      "[17] «Prefix #6: Scores for query analysis:»\n",
-      "[18] «Resulting Score #6: 56.76»\n",
-      "[19] «Instruction #7: Use the query to compare and evaluate the relevance of each passage.»\n",
-      "[20] «Prefix #7: Relevance_scores_»\n",
-      "[21] «Resulting Score #7: 57.42»\n",
-      "[22] «Instruction #8: First, perform an initial keyword search of the query within the passages. Then, compare the search results to the original query to identify relevant passages.»\n",
-      "[23] «Prefix #8: Scored results:»\n",
-      "[24] «Resulting Score #8: 58.03»\n",
-      "[25] «Instruction #9: Before generating `score`, carefully analyze each `passage` in relation to the `query` and assign a numeric value based on relevance and overall quality of the `passage`.»\n",
-      "[26] «Prefix #9: Calculation of relevance score for passages in relation to query:»\n",
-      "[27] «Resulting Score #9: 63.81»\n",
-      "Proposed Instruction:\u001b[32m Given the fields `query`, and `passages`, analyze the semantic similarity of each passage to the query and generate a relevancy score for each passage based on the analysis.\n",
-      "Proposed Prefix For Output Field: Improved relevancy scores:\u001b[0m\u001b[31m \t (and 9 other completions)\u001b[0m\n",
-      "\n",
-      "\n",
-      "\n",
-      "None\n",
-      "Starting iteration 1/2.\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: Given the fields `query`, and `passages`, analyze the semantic similarity of each passage to the query and generate a relevancy score for each passage based on the analysis.\n",
-      "p: Improved relevancy scores:\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 8.854505551688998 / 20  (44.3): 100%|██████████| 20/20 [03:06<00:00,  9.32s/it] \n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 8.854505551688998 / 20  (44.3%)\n",
-      "----------------\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: Given the fields 'query' and 'passages', apply a natural language processing model to analyze semantic similarity and contextual relevance in order to calculate and assign a relevancy score to each passage corresponding to the query.\n",
-      "p: Relevance scores computation for query:\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 9.305273086417204 / 20  (46.5): 100%|██████████| 20/20 [03:12<00:00,  9.60s/it]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 9.305273086417204 / 20  (46.5%)\n",
-      "----------------\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: Proposed Instruction: Utilize semantic analysis and keyword extraction to identify the most relevant passages to the given query. Calculate a relevancy score for each passage based on its semantic similarity and keyword matches with the query. Consider the overall quality of the passage when assigning a numeric value to its relevance to the query.\n",
-      "p: Relevancy scores for query and passages:\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 8.46655847511304 / 20  (42.3): 100%|██████████| 20/20 [03:28<00:00, 10.44s/it]  \n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 8.46655847511304 / 20  (42.3%)\n",
-      "----------------\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: Given the fields `query`, `passages`, and `keywords`, utilize a multi-faceted approach that includes semantic similarity, keyword analysis, and contextual relevance to calculate a comprehensive score for each passage in relation to the query.\n",
-      "p: Comprehensive passage relevance scores based on query analysis and keyword relevance:\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 11.722319635627931 / 20  (58.6): 100%|██████████| 20/20 [03:30<00:00, 10.54s/it]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 11.722319635627931 / 20  (58.6%)\n",
-      "----------------\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: Proposed Instruction: Train the model to identify not only the semantic similarity between the query and passages but also the contextual relevance and significance of the information within the passages in relation to the query.\n",
-      "p: Improved relevancy scores:\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 10.353255566525437 / 20  (51.8): 100%|██████████| 20/20 [03:19<00:00,  9.99s/it]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 10.353255566525437 / 20  (51.8%)\n",
-      "----------------\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: Calculate the relevancy score for each passage related to the query by taking into consideration the semantic similarity, potential keywords, and overall quality of the passage.\n",
-      "p: Enhanced relevancy scores:\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 12.415814266014621 / 20  (62.1): 100%|██████████| 20/20 [03:14<00:00,  9.71s/it]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 12.415814266014621 / 20  (62.1%)\n",
-      "----------------\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: Instruction #10: Utilize semantic embedding to analyze the query and passages, then calculate a relevance score for each passage based on semantic similarity and key keyword analysis.\n",
-      "p: Score and relevance rankings based on semantic analysis.\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 8.25174783595295 / 20  (41.3): 100%|██████████| 20/20 [06:30<00:00, 19.54s/it]  \n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 8.25174783595295 / 20  (41.3%)\n",
-      "----------------\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: Proposed Instruction:\n",
-      "Given the input fields `query` and `passages`, analyze the semantic relevance of each passage to the query, taking into account key contextual information within the query and passages to generate a comprehensive relevance score.\n",
-      "p: Semantic relevance scores for passages to query:\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 11.49267383394411 / 20  (57.5): 100%|██████████| 20/20 [03:25<00:00, 10.26s/it] \n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 11.49267383394411 / 20  (57.5%)\n",
-      "----------------\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: Instruction #10: Utilize contextual understanding and semantic analysis to assess the relevancy and coherence of each passage with respect to the given query, and generate a score that encapsulates both aspects.\n",
-      "p: Semantic relevancy assessments:\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 10.245880431686144 / 20  (51.2): 100%|██████████| 20/20 [03:30<00:00, 10.52s/it]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 10.245880431686144 / 20  (51.2%)\n",
-      "----------------\n",
-      "----------------\n",
-      "Predictor 0\n",
-      "i: The improved instructions for the language model could be: \"Examine each passage to scrutinize semantic congruity with the provided query and derive a relevance score for each passage.\n",
-      "p: Semantic Relevancy:\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 12.53923996601208 / 20  (62.7): 100%|██████████| 20/20 [03:19<00:00,  9.96s/it] "
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 12.53923996601208 / 20  (62.7%)\n",
-      "----------------\n",
-      "Updating Predictor 140321429486800 to:\n",
-      "i: Before generating `score`, carefully analyze each `passage` in relation to the `query` and assign a numeric value based on relevance and overall quality of the `passage`.\n",
-      "p: Calculation of relevance score for passages in relation to query:\n",
-      "Full predictor with update: \n",
-      "Predictor 0\n",
-      "i: Before generating `score`, carefully analyze each `passage` in relation to the `query` and assign a numeric value based on relevance and overall quality of the `passage`.\n",
-      "p: Calculation of relevance score for passages in relation to query:\n",
-      "\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "from dspy.teleprompt import SignatureOptimizer\n",
-    "\n",
-    "\n",
-    "teleprompter = SignatureOptimizer(metric=validate_rating_v4, depth=2, verbose=True, prompt_model=turbo)\n",
-    "compiled_prompt_opt = teleprompter.compile(rate_passage_model.deepcopy(), devset=devset[:20], eval_kwargs=kwargs)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Finnally, let's evaluate the compiled `SignatureOptimizer` model, `compiled_prompt_opt`, using the defined evaluation settings."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 28.321976567062965 / 43  (65.9): 100%|██████████| 43/43 [00:31<00:00,  1.36it/s] "
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Average Metric: 28.321976567062965 / 43  (65.9%)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "eval_score = evaluate(compiled_prompt_opt)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "base",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.16"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}