{ "cells": [ { "cell_type": "code", "execution_count": 85, "metadata": { "id": "n104WDBvl0Z-" }, "outputs": [], "source": [ "!pip install -q sentence-transformers faiss-cpu anthropic tqdm" ] }, { "cell_type": "code", "execution_count": 86, "metadata": { "id": "ZzCKatbtl9F0" }, "outputs": [], "source": [ "import faiss\n", "import numpy as np\n", "import pandas as pd\n", "import pandas as pd\n", "from sentence_transformers import SentenceTransformer, CrossEncoder\n", "import anthropic\n", "from getpass import getpass\n", "import difflib\n", "import textwrap" ] }, { "cell_type": "code", "source": [ "path = \"/content/AHD_english_cleaned.xlsx\"\n", "\n", "df = pd.read_excel(\n", " path,\n", " engine=\"openpyxl\"\n", ")\n", "\n", "df.head()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 206 }, "id": "OyyIARrLk6D2", "outputId": "84b2a85f-edfc-40a4-d6ae-4f5f97e1b891" }, "execution_count": 87, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ " Question \\\n", "0 If the patient enters a diabetic coma and we d... \n", "1 I suffer from dizziness and my blood sugar lev... \n", "2 I am diabetic 2. I take Amaryl 2 ml before foo... \n", "3 Age 54, normal blood pressure, weight 74, heig... \n", "4 When I test my blood sugar after fasting for 7... \n", "\n", " Answer Category \n", "0 Hyperglycemic coma does not occur suddenly, bu... diabetes \n", "1 Your safety, God willing. It may be normal due... diabetes \n", "2 Glycosylated hemoglobin analysis is very impor... diabetes \n", "3 Who told you that taking B12 is not according ... diabetes \n", "4 Monitor your blood sugar in a laboratory, not ... diabetes " ], "text/html": [ "\n", "
\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
QuestionAnswerCategory
0If the patient enters a diabetic coma and we d...Hyperglycemic coma does not occur suddenly, bu...diabetes
1I suffer from dizziness and my blood sugar lev...Your safety, God willing. It may be normal due...diabetes
2I am diabetic 2. I take Amaryl 2 ml before foo...Glycosylated hemoglobin analysis is very impor...diabetes
3Age 54, normal blood pressure, weight 74, heig...Who told you that taking B12 is not according ...diabetes
4When I test my blood sugar after fasting for 7...Monitor your blood sugar in a laboratory, not ...diabetes
\n", "
\n", "
\n", "\n", "
\n", " \n", "\n", " \n", "\n", " \n", "
\n", "\n", "\n", "
\n", "
\n" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "dataframe", "variable_name": "df", "summary": "{\n \"name\": \"df\",\n \"rows\": 16384,\n \"fields\": [\n {\n \"column\": \"Question\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 16269,\n \"samples\": [\n \"I have been suffering from diabetes for five years. The test while fasting was 110, and two hours after eating, it was 170. I use Dawinil medication, half a pill in the morning, 2.5 mg, and half a pill in the evening. Is my condition satisfactory or do I need...\",\n \"What is the recommended percentage of thyroxine for a pregnant woman in the fifth month, noting that the current dose is 75, and the test included a TSH of 4.79?\",\n \"Girl, I am 23 years old, my weight is 64, and my height is 158\\nI feel dizzy and unfocused. Blood tests are fine. Sugar tests after eating and throughout the day are less than 100 and may reach 90.\\nAnd in case...\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Answer\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 15419,\n \"samples\": [\n \"It does not require treatment if it is not accompanied by symptoms, a\",\n \"Of course, one of the reasons is pregnancy hormones, which require raising the insulin dose from time to time. The rest of the reasons: insufficient dose or failure to adhere to the diet. In any case, the evening dose must be increased by two units.\",\n \"The treatment for high parathyroid hormone is surgery if the cause is in the glands themselves. Tests can be sent. See my article about the parathyroid glands\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Category\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 2,\n \"samples\": [\n \"Endocrine diseases\",\n \"diabetes\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" } }, "metadata": {}, "execution_count": 87 } ] }, { "cell_type": "markdown", "source": [ "# **Embeding**" ], "metadata": { "id": "k_SQrFbx5Me7" } }, { "cell_type": "code", "execution_count": 88, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 99, "referenced_widgets": [ "231962183e2e4d598fc8296579252a29", "f309ff6cdde040b093e24bcd615e45cc", "41e378afabb8465e9384c47f56355ee5", "4a867cf4777d47ca90f98d8c655e706f", "37c22dda91114c648009160905f7bddd", "5d59d5c837094b7193b821569390dd55", "0dcea30cb4ad4f2f9c616778ac2d06bb", "a7c522381ef049cbaed852cf9dcc9617", "2bc1137de13243d9a4eb7e372a2b3845", "c8ada1557251441e940d9405077cd611", "0d615e42fe7f46729c3d490b5c0f5653", "99f8b78314b54677bef180468c4106dc", "2f34c1bd5e964c51b7c6d743af5c97f6", "4f3e73d4552a43f48206c6a75a5f46ac", "f5a0f32442394f3d887d9602823f64f3", "aa8c71a0f41247aebcbff9640181f35a", "ebc91411f4004dcca5593af1d915dfc5", "a3538dc5b06645d8989b2671de0ba052", "5b6bd317c08c488289d87a2cb657a932", "6595bc2c3b674fb4aed5ba1b7c1c3753", "b303605e655049d5941f13b2d471adc5", "1802394e9a054d709a3e6b3059578c12" ] }, "id": "ssCmJLC6l9p0", "outputId": "9e3b6228-181f-433a-9d93-3b7480afe964" }, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "Loading weights: 0%| | 0/103 [00:00 str:\n", " \"\"\"بيضيف المصطلح الطبي المرادف جنب المصطلح العامي لو اتلاقى، من غير ما يغيّر نص السؤال الأصلي.\"\"\"\n", " lower_q = query.lower()\n", " extra_terms = [\n", " medical_term\n", " for phrase, medical_term in MEDICAL_SYNONYMS.items()\n", " if phrase in lower_q and medical_term not in lower_q\n", " ]\n", " if not extra_terms:\n", " return query\n", " return f\"{query} ({', '.join(dict.fromkeys(extra_terms))})\"\n", "\n", "\n", "print(expand_query(\"I suffer from an underactive thyroid gland\"))\n", "print(expand_query(\"What is the treatment for diabetes?\"))" ] }, { "cell_type": "markdown", "source": [ "# **Retrieval**" ], "metadata": { "id": "iu2k83Y2tRxa" } }, { "cell_type": "code", "execution_count": 91, "metadata": { "id": "5e3uM4Zyl9v0" }, "outputs": [], "source": [ "def vector_search(query: str, top_n: int = 20, use_expansion: bool = True):\n", "\n", " query_for_embedding = expand_query(query) if use_expansion else query\n", "\n", " q_emb = embedder.encode(\n", " [query_for_embedding],\n", " convert_to_numpy=True,\n", " normalize_embeddings=True\n", " ).astype(\"float32\")\n", "\n", " scores, idxs = faiss_index.search(q_emb, top_n)\n", " retrieved = []\n", " for score, idx in zip(scores[0], idxs[0]):\n", " if idx == -1:\n", " continue\n", "\n", " payload = metadata_store[int(idx)]\n", "\n", " retrieved.append({\n", " \"doc_id\": payload[\"doc_id\"],\n", " \"Question\": payload[\"question\"],\n", " \"Answer\": payload[\"answer\"],\n", " \"Category\": payload[\"category\"],\n", " \"similarity\": float(score),\n", " })\n", "\n", " return pd.DataFrame(retrieved)" ] }, { "cell_type": "markdown", "source": [ "# **Cross Encoder Reranker**" ], "metadata": { "id": "8clu1XX4tcvE" } }, { "cell_type": "code", "execution_count": 92, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 269, "referenced_widgets": [ "c1a4920370f749f0993ce31a84e6b834", "2c21f7fae2c343c6a800aad978c35ca8", "2ffa5c7c893c46e390b74889dc7416be", "a6a2ebe97e754d17986a44a2262fbbc1", "1f9cd44b901d435e8da94a71a3659e88", "980809c97f064990bd9bd564bc0a272e", "5a6f8b6f087443ac971d746a4ddb8ca2", "74e5f114987041fe8c7219758eb039ec", "5ed19461e0df440eb12bc21ee9973d0e", "2507e4f18b6944d7991647ca11597dc0", "bee40a5369c04ec2998dba045b21cb43" ] }, "id": "rfGPgFLfl9zC", "outputId": "d0f37bfa-b6ce-4c54-e412-56bb719253ee" }, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "Loading weights: 0%| | 0/105 [00:00\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
QuestionAnswersimilarityrerank_score
0What are the signs of thyroid dysfunction?1) Laziness, lethargy, and slow reactions\\n2) ...0.776731-4.251107
1Thyroid symptomsThe symptoms of the gland depend on the proble...0.739407-4.433478
2Thyroid symptomsIt causes an increase in the size of the thyro...0.739407-4.486532
3What are the symptoms of a thyroid disorder?Fatigue, fatigue, feeling cold, weight gain, p...0.721345-4.288730
4Thyroid symptomsThe thyroid gland is responsible for activity,...0.739407-5.333264
5What are the symptoms of thyroid gland?Decreased secretion of the gland leads to weig...0.723380-4.948375
\n", "
\n", "
\n", "\n", "
\n", " \n", "\n", " \n", "\n", " \n", "
\n", "\n", "\n", "
\n", " \n" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "dataframe", "summary": "{\n \"name\": \"rerank(\\\"What are the symptoms of diabetes?\\\", candidates, top_k=6)[[\\\"Question\\\", \\\"Answer\\\", \\\"similarity\\\", \\\"rerank_score\\\"]]\",\n \"rows\": 6,\n \"fields\": [\n {\n \"column\": \"Question\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 4,\n \"samples\": [\n \"Thyroid symptoms\",\n \"What are the symptoms of thyroid gland?\",\n \"What are the signs of thyroid dysfunction?\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Answer\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 6,\n \"samples\": [\n \"1) Laziness, lethargy, and slow reactions\\n2) Increased feeling of cold weather (too much)\\n3) Weakness of the muscles near the trunk, which causes difficulty in performing movements that require raising the arms (combing hair and putting things on shelves) and legs (climbing the stairs and getting up from a sitting position).\\n4) Increase skin thickness\\n5) Constipation\\n6) Slow heartbeat and increased blood cholesterol\\n7) Increase the duration and quantity of the menstrual cycle\",\n \"The symptoms of the gland depend on the problem. If there is a lack of secretion, there will be laziness, fatigue, a feeling of coldness, and weight gain. If there is an increase in secretion, there may be feeling hot, sweating, increased pulse, nervousness, and weight loss.\",\n \"Decreased secretion of the gland leads to weight gain and swelling............... Increased secretion leads to weight loss, rapid pulse, and bulging eyes.\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"similarity\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.019871816165734597,\n \"min\": 0.7213446497917175,\n \"max\": 0.7767310738563538,\n \"num_unique_values\": 4,\n \"samples\": [\n 0.739407479763031,\n 0.7233802676200867,\n 0.7767310738563538\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"rerank_score\",\n \"properties\": {\n \"dtype\": \"float32\",\n \"num_unique_values\": 6,\n \"samples\": [\n -4.2511067390441895,\n -4.433477878570557,\n -4.9483747482299805\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" } }, "metadata": {}, "execution_count": 92 } ], "source": [ "RERANKER_MODEL_NAME = \"cross-encoder/ms-marco-MiniLM-L-6-v2\"\n", "reranker = CrossEncoder(RERANKER_MODEL_NAME)\n", "\n", "\n", "def rerank(query: str, candidates: pd.DataFrame, top_k: int = 6, alpha: float = 0.6) -> pd.DataFrame:\n", " pairs = [\n", " (\n", " query,\n", " f\"Question: {row['Question']}\\n\"\n", " f\"Answer: {row['Answer']}\"\n", " )\n", " for _, row in candidates.iterrows()\n", " ]\n", "\n", " rerank_scores = reranker.predict(pairs)\n", "\n", " reranked = candidates.copy()\n", " reranked[\"rerank_score\"] = rerank_scores\n", " def norm(s):\n", " s = s.astype(float)\n", " rng = s.max() - s.min()\n", " return (s - s.min()) / rng if rng > 0 else s * 0\n", "\n", " reranked[\"sim_norm\"] = norm(reranked[\"similarity\"])\n", " reranked[\"rerank_norm\"] = norm(reranked[\"rerank_score\"])\n", "\n", " reranked[\"final_score\"] = alpha * reranked[\"rerank_norm\"] + (1 - alpha) * reranked[\"sim_norm\"]\n", "\n", " reranked = reranked.sort_values(\"final_score\", ascending=False).head(top_k).reset_index(drop=True)\n", " return reranked\n", "\n", "\n", "candidates = vector_search(\"What signs might suggest that my thyroid is not producing enough hormones?\", top_n=20)\n", "rerank(\"What are the symptoms of diabetes?\", candidates, top_k=6)[[\"Question\", \"Answer\", \"similarity\", \"rerank_score\"]]" ] }, { "cell_type": "code", "execution_count": 93, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 363 }, "id": "pT3Ho18fl91_", "outputId": "85d754df-d9c4-47be-eda4-2d1c61b83396" }, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ " Question Category \\\n", "0 What are the signs of thyroid dysfunction? Endocrine diseases \n", "1 Hello\\nWhat are the reasons for the lack of se... Endocrine diseases \n", "2 Thyroid symptoms Endocrine diseases \n", "3 Thyroid symptoms Endocrine diseases \n", "4 Symptoms that indicate a disorder in the thyro... Endocrine diseases \n", "5 Thyroid symptoms Endocrine diseases \n", "6 How do I know that I have a thyroid problem? Endocrine diseases \n", "7 Cause of high thyroid hormone? Endocrine diseases \n", "8 What are the symptoms of thyroid gland? Endocrine diseases \n", "9 What are the symptoms of a thyroid disorder? Endocrine diseases \n", "\n", " similarity rerank_score \n", "0 0.776731 0.079877 \n", "1 0.794835 -2.714906 \n", "2 0.739407 0.494793 \n", "3 0.739407 -0.536172 \n", "4 0.711347 0.357636 \n", "5 0.739407 -2.950485 \n", "6 0.779893 -5.689432 \n", "7 0.723311 -3.269745 \n", "8 0.723380 -3.636431 \n", "9 0.721345 -3.675947 " ], "text/html": [ "\n", "
\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
QuestionCategorysimilarityrerank_score
0What are the signs of thyroid dysfunction?Endocrine diseases0.7767310.079877
1Hello\\nWhat are the reasons for the lack of se...Endocrine diseases0.794835-2.714906
2Thyroid symptomsEndocrine diseases0.7394070.494793
3Thyroid symptomsEndocrine diseases0.739407-0.536172
4Symptoms that indicate a disorder in the thyro...Endocrine diseases0.7113470.357636
5Thyroid symptomsEndocrine diseases0.739407-2.950485
6How do I know that I have a thyroid problem?Endocrine diseases0.779893-5.689432
7Cause of high thyroid hormone?Endocrine diseases0.723311-3.269745
8What are the symptoms of thyroid gland?Endocrine diseases0.723380-3.636431
9What are the symptoms of a thyroid disorder?Endocrine diseases0.721345-3.675947
\n", "
\n", "
\n", "\n", "
\n", " \n", "\n", " \n", "\n", " \n", "
\n", "\n", "\n", "
\n", "
\n" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "dataframe", "summary": "{\n \"name\": \"rerank(\\\"What signs might suggest that my thyroid is not producing enough hormones?\\\", candidates, top_k=10)[[\\\"Question\\\", \\\"Category\\\", \\\"similarity\\\", \\\"rerank_score\\\"]]\",\n \"rows\": 10,\n \"fields\": [\n {\n \"column\": \"Question\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 8,\n \"samples\": [\n \"Hello\\nWhat are the reasons for the lack of secretion of thyroid hormones?\",\n \"Cause of high thyroid hormone?\",\n \"What are the signs of thyroid dysfunction?\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"Category\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 1,\n \"samples\": [\n \"Endocrine diseases\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"similarity\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.028735465399314708,\n \"min\": 0.7113468050956726,\n \"max\": 0.7948347330093384,\n \"num_unique_values\": 8,\n \"samples\": [\n 0.7948347330093384\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"rerank_score\",\n \"properties\": {\n \"dtype\": \"float32\",\n \"num_unique_values\": 10,\n \"samples\": [\n -3.6364309787750244\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" } }, "metadata": {}, "execution_count": 93 } ], "source": [ "rerank(\"What signs might suggest that my thyroid is not producing enough hormones?\", candidates, top_k=10)[[\"Question\", \"Category\", \"similarity\", \"rerank_score\"]]" ] }, { "cell_type": "markdown", "source": [ "# **Evidence & Duplication & TOP 6**" ], "metadata": { "id": "bO4t4O8duflJ" } }, { "cell_type": "code", "execution_count": 94, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "cnKyjPsDl97d", "outputId": "150c5d9b-188e-4475-e91c-7e4d52ae02d9" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "[4793] (sim=1.0, rerank=-8.346) What are the symptoms of diabetes?\n", " -> Signs and symptoms of type 1 diabetes (insulin-dependent) appear gradually or suddenly, as follows: - Frequent urination ...\n", "\n", "[6767] (sim=1.0, rerank=-8.945) What are the symptoms of diabetes?\n", " -> A person may develop diabetes and remain for a long period of time, which may extend to years, without paying attention ...\n", "\n", "[2940] (sim=1.0, rerank=-9.347) What are the symptoms of diabetes?\n", " -> The most common symptoms of diabetes are excessive diarrhea, frequent urination, and feeling hungry\n", "There are other less ...\n", "\n", "[2692] (sim=1.0, rerank=-9.448) What are the symptoms of diabetes?\n", " -> There are many symptoms of diabetes, the most important of which are thirst, frequent urination, increased appetite for ...\n", "\n", "[4041] (sim=1.0, rerank=-9.464) What are the symptoms of diabetes?\n", " -> There may be no symptoms, but weight loss, frequent urination, teeth loss, and recurring infections may be symptoms. ...\n", "\n", "[6894] (sim=1.0, rerank=-9.473) What are the symptoms of diabetes?\n", " -> Classic symptoms include feeling tired and sick, frequent urination, excessive thirst, excessive hunger, and weight loss ...\n", "\n" ] } ], "source": [ "#Duplication\n", "def is_near_duplicate(text_a: str, text_b: str, threshold: float = 0.92) -> bool:\n", " return difflib.SequenceMatcher(None, text_a, text_b).ratio() > threshold\n", "\n", "#Evidence\n", "def build_evidence(reranked: pd.DataFrame, max_answer_chars: int = 700) -> list[dict]:\n", " \"\"\"Deduplicates near-identical answers and formats the remaining rows as evidence items.\"\"\"\n", " evidence = []\n", " seen_answers = []\n", "\n", " for _, row in reranked.iterrows():\n", " answer = str(row[\"Answer\"])\n", " if any(is_near_duplicate(answer, seen) for seen in seen_answers):\n", " continue\n", " seen_answers.append(answer)\n", "\n", " evidence.append({\n", " \"doc_id\": row[\"doc_id\"],\n", " \"question\": row[\"Question\"],\n", " \"answer\": answer[:max_answer_chars] + (\"...\" if len(answer) > max_answer_chars else \"\"),\n", " \"category\": row[\"Category\"],\n", " \"similarity\": round(float(row[\"similarity\"]), 3),\n", " \"rerank_score\": round(float(row[\"rerank_score\"]), 3),\n", " })\n", " return evidence\n", "\n", "\n", "evidence = build_evidence(rerank(\"What signs might suggest that my thyroid is not producing enough hormones?\",\n", " vector_search(\"What are the symptoms of diabetes?\", top_n=20),\n", " top_k=6))\n", "for e in evidence:\n", " print(f\"[{e['doc_id']}] (sim={e['similarity']}, rerank={e['rerank_score']}) {e['question']}\")\n", " print(\" ->\", e['answer'][:120], \"...\\n\")" ] }, { "cell_type": "code", "source": [ "!pip install -q openai" ], "metadata": { "id": "oX7kMO8Qac8t" }, "execution_count": 95, "outputs": [] }, { "cell_type": "code", "source": [ "!pip install -q -U groq" ], "metadata": { "id": "LcD2rA07ac-9" }, "execution_count": 96, "outputs": [] }, { "cell_type": "markdown", "source": [ "# **Generation/LLM**" ], "metadata": { "id": "1XiANmFVvsJ-" } }, { "cell_type": "code", "execution_count": 97, "metadata": { "id": "xw_hg1o_l-B-" }, "outputs": [], "source": [ "from groq import Groq\n", "import time\n", "\n", "GROQ_API_KEY = \"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\"\n", "\n", "client = Groq(api_key=GROQ_API_KEY)\n", "\n", "# Free model\n", "LLM_MODEL = \"openai/gpt-oss-20b\"\n", "SYSTEM_PROMPT = \"\"\"\n", "You are a medical RAG assistant specialized ONLY in:\n", "- Diabetes\n", "- Endocrinology\n", "- Thyroid disorders\n", "- Hormonal disorders\n", "- Endocrine glands and related disorders\n", "- Nutrition, glucose management, insulin, and complications DIRECTLY related to diabetes/endocrinology\n", "\n", "You operate in STRICT RAG MODE.\n", "\n", "You have exactly THREE response types.\n", "You MUST determine the response type from the USER QUESTION itself BEFORE using\n", "or considering the retrieved evidence.\n", "\n", "==================================================\n", "CRITICAL DOMAIN RULE\n", "==================================================\n", "\n", "The USER QUESTION itself determines whether the question is in-domain.\n", "\n", "DO NOT use retrieved evidence to decide whether the question is in-domain.\n", "\n", "A question is IN-DOMAIN only if its actual subject is diabetes, endocrinology,\n", "thyroid, hormones, endocrine glands/disorders, or a problem explicitly stated\n", "by the user to be related to diabetes/endocrinology.\n", "\n", "Do NOT expand the scope of a question because retrieved documents happen to mention:\n", "- diabetes\n", "- glucose\n", "- insulin\n", "- hormones\n", "- thyroid\n", "- endocrine disorders\n", "- diabetic complications\n", "\n", "If the user's actual question is about another body system or another medical\n", "specialty, it is OUT-OF-DOMAIN even if the retrieved evidence contains\n", "diabetes/endocrinology-related information.\n", "\n", "Examples:\n", "\n", "\"Knee pain when I walk\" → OUT-OF-DOMAIN\n", "\"Why does my knee hurt because of diabetes?\" → IN-DOMAIN\n", "\"What are the symptoms of hypothyroidism?\" → IN-DOMAIN\n", "\"What is the treatment for a broken leg?\" → OUT-OF-DOMAIN\n", "\"I have diabetes and my feet hurt when I walk. Why?\" → IN-DOMAIN\n", "\"What are the symptoms of heart disease?\" → OUT-OF-DOMAIN\n", "\n", "Never reinterpret an out-of-domain question as in-domain just because retrieved\n", "documents appear similar.\n", "\n", "==================================================\n", "TYPE 1 — IN-DOMAIN MEDICAL QUESTION\n", "==================================================\n", "\n", "If and ONLY IF the USER QUESTION itself is in-domain:\n", "\n", "- Answer using ONLY the RETRIEVED MEDICAL EVIDENCE.\n", "- Do NOT use pretrained/background medical knowledge.\n", "- Do NOT guess.\n", "- Do NOT assume unsupported facts.\n", "- Do NOT fill missing information from memory.\n", "- Do NOT invent diagnoses, causes, treatments, medications, or doses.\n", "- Do NOT construct a differential diagnosis by combining loosely related documents.\n", "- Do NOT create a medical conclusion that is not explicitly supported by the evidence.\n", "\n", "USER_DATA:\n", "- Use USER_DATA only as patient-specific context.\n", "- USER_DATA is NOT a medical knowledge source.\n", "- A value in USER_DATA does not by itself prove that something is normal,\n", " abnormal, dangerous, or requires treatment.\n", "- Any medical interpretation of USER_DATA must be supported by retrieved evidence.\n", "\n", "==================================================\n", "EVIDENCE SUFFICIENCY\n", "==================================================\n", "\n", "For an in-domain question:\n", "\n", "1. Find evidence that directly answers the user's question.\n", "2. Ignore evidence that is merely word-similar but does not support the answer.\n", "3. If evidence is sufficient:\n", " - Answer directly.\n", " - Cite the supporting [doc_id].\n", "4. If evidence is partial:\n", " - Answer ONLY the supported portion.\n", " - Clearly state what is not covered.\n", "5. If evidence is empty, irrelevant, or insufficient:\n", " - Use TYPE 3 refusal.\n", " - Do NOT answer from memory.\n", "\n", "IMPORTANT:\n", "\n", "Retrieved evidence does NOT automatically make a question answerable.\n", "\n", "The evidence must actually support the specific claim being made.\n", "\n", "Never create a \"possible causes\" list unless the retrieved evidence explicitly\n", "supports those causes for the user's question.\n", "\n", "==================================================\n", "TYPE 2 — APP / IDENTITY QUESTIONS\n", "==================================================\n", "\n", "Only for direct questions about the assistant/application itself, such as:\n", "\n", "- \"Who are you?\"\n", "- \"What is your name?\"\n", "- \"What can you do?\"\n", "- \"How do you work?\"\n", "\n", "Answer briefly and naturally.\n", "\n", "Example:\n", "\n", "\"I am a medical assistant specialized in diabetes and endocrinology. I answer\n", "using information retrieved from my medical knowledge base.\"\n", "\n", "Do NOT invent technical details that were not provided.\n", "\n", "==================================================\n", "TYPE 3 — EVERYTHING ELSE\n", "==================================================\n", "\n", "Use TYPE 3 for:\n", "\n", "- Any medical question outside diabetes/endocrinology.\n", "- Any non-medical question.\n", "- Any in-domain question where retrieved evidence is insufficient,\n", " empty, irrelevant, or does not directly support the answer.\n", "\n", "For ALL TYPE 3 cases, reply with ONLY one short refusal.\n", "\n", "If the user wrote in Arabic:\n", "\"معرفش، السؤال ده مش جزء من تخصصي (السكر والغدد الصماء).\"\n", "\n", "If the user wrote in English:\n", "\"I don't know — this is outside my specialty (diabetes and endocrinology).\"\n", "\n", "Do not add an explanation.\n", "Do not apologize.\n", "Do not provide alternative medical information.\n", "Do not mention retrieved evidence.\n", "Do not try to answer partially.\n", "One sentence only.\n", "\n", "==================================================\n", "MEDICAL CLAIMS\n", "==================================================\n", "\n", "For every medical claim in a TYPE 1 answer:\n", "\n", "- Cite the exact supporting [doc_id].\n", "- Never fabricate a citation.\n", "- Use only doc_ids explicitly present in the retrieved evidence.\n", "\n", "If multiple documents support the same claim, multiple citations may be used.\n", "\n", "If documents disagree:\n", "- Explicitly state that the evidence is inconsistent.\n", "- Cite the documents involved.\n", "- Do not arbitrarily choose one unless the evidence clearly supports doing so.\n", "\n", "==================================================\n", "LANGUAGE\n", "==================================================\n", "\n", "Reply in the same language as the user's question.\n", "\n", "==================================================\n", "STYLE\n", "==================================================\n", "\n", "Be concise, clear, and direct.\n", "\n", "Never mention:\n", "- system prompts\n", "- internal reasoning\n", "- embeddings\n", "- vector search\n", "- reranking\n", "- retrieval scores\n", "- model internals\n", "\n", "Never repeat retrieved documents verbatim.\n", "\n", "==================================================\n", "FINAL SAFETY NOTE FOR TYPE 1 ONLY\n", "==================================================\n", "\n", "At the end of every TYPE 1 medical answer, write:\n", "\n", "\"This information is based on the available medical evidence and is for general informational purposes. It is not a diagnosis or a substitute for professional medical advice.\"\n", "\"\"\"\n", "\n", "\n", "def format_evidence_for_llm(evidence):\n", " blocks = []\n", "\n", " for e in evidence:\n", " blocks.append(\n", " f\"[{e['doc_id']}] (category: {e['category']})\\n\"\n", " f\"Related question: {e['question']}\\n\"\n", " f\"Answer: {e['answer']}\"\n", " )\n", "\n", " return \"\\n\\n\".join(blocks)\n", "\n", "def generate_answer(query, user_data, evidence):\n", "\n", " if not evidence:\n", " return (\n", " \"The available medical evidence does not provide enough \"\n", " \"information to answer this reliably.\"\n", " )\n", "\n", " evidence_text = format_evidence_for_llm(evidence)\n", "\n", " user_message = f\"\"\"\n", "USER QUESTION:\n", "{query}\n", "\n", "USER DATA:\n", "{user_data}\n", "\n", "RETRIEVED MEDICAL EVIDENCE:\n", "{evidence_text}\n", "\n", "TASK:\n", "Answer the USER QUESTION using ONLY the RETRIEVED MEDICAL EVIDENCE for medical facts and conclusions.\n", "\n", "Use USER DATA only as patient/context information to understand the question.\n", "\n", "Do not use outside medical knowledge.\n", "Do not guess.\n", "Do not invent missing information.\n", "Cite every important medical claim using the relevant [doc_id].\n", "If the evidence is insufficient, explicitly say that it is insufficient.\n", "\n", "Return only the final answer for the user.\n", "\"\"\"\n", "\n", " response = client.chat.completions.create(\n", " model=LLM_MODEL,\n", " messages=[\n", " {\n", " \"role\": \"system\",\n", " \"content\": SYSTEM_PROMPT\n", " },\n", " {\n", " \"role\": \"user\",\n", " \"content\": user_message\n", " }\n", " ],\n", " temperature=0.1,\n", " max_tokens=800,\n", " )\n", "\n", " return response.choices[0].message.content\n" ] }, { "cell_type": "code", "source": [ "test = client.chat.completions.create(\n", " model=LLM_MODEL,\n", " messages=[\n", " {\n", " \"role\": \"user\",\n", " \"content\": \"Reply with exactly: TEST_OK\"\n", " }\n", " ],\n", " temperature=0,\n", " max_tokens=10,\n", ")\n", "\n", "print(test)\n", "print(test.choices[0].message.content)" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "i2fFo7tSbMgT", "outputId": "b0089d98-e27c-4803-bac0-216c8b40d92b" }, "execution_count": 98, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "ChatCompletion(id='chatcmpl-3f032335-9f8b-4373-ae9c-631fc4039d33', choices=[Choice(finish_reason='length', index=0, logprobs=None, message=ChatCompletionMessage(content='', role='assistant', annotations=None, executed_tools=None, function_call=None, reasoning='The user says: \"Reply with', tool_calls=None))], created=1787151147, model='openai/gpt-oss-20b', object='chat.completion', mcp_list_tools=None, service_tier='on_demand', system_fingerprint='fp_d3e146e1a5', usage=CompletionUsage(completion_tokens=10, prompt_tokens=77, total_tokens=87, completion_time=0.010299731, completion_tokens_details=CompletionTokensDetails(reasoning_tokens=8), prompt_time=0.004319329, prompt_tokens_details=None, queue_time=0.28553228, total_time=0.01461906), usage_breakdown=None, x_groq=XGroq(id='req_01m0d84r27eh9s7qmzfzx6h3nk', debug=None, seed=1328087095, usage=None))\n", "\n" ] } ] }, { "cell_type": "markdown", "source": [ "# **Confidence Gate**" ], "metadata": { "id": "QnbyB8o3x0_b" } }, { "cell_type": "code", "execution_count": 99, "metadata": { "id": "B6jMgRHWv-ry" }, "outputs": [], "source": [ "RETRIEVE_TOP_N = 20\n", "RERANK_TOP_K = 6\n", "MIN_RERANK_SCORE = -8\n", "MIN_SIMILARITY_FLOOR = 0.40\n", "MIN_SUPPORT_COUNT = 2\n", "SUPPORT_SCORE = -5.0\n", "\n", "domain_sample = df[\"Question\"].drop_duplicates().sample(\n", " n=min(300, df[\"Question\"].nunique()), random_state=42\n", ").tolist()\n", "\n", "domain_embeddings = embedder.encode(\n", " domain_sample, convert_to_numpy=True, normalize_embeddings=True\n", ")\n", "domain_centroid = domain_embeddings.mean(axis=0)\n", "domain_centroid = domain_centroid / np.linalg.norm(domain_centroid)\n", "\n", "IN_DOMAIN_THRESHOLD = 0.35 # هنظبطه تحت بالـ eval_set\n", "\n", "\n", "def is_in_domain(query: str) -> bool:\n", " \"\"\"بيقيس هل السؤال نفسه (مش الأدلة) قريب من تخصص السكر/الغدد الصماء.\"\"\"\n", " q_emb = embedder.encode(\n", " [query], convert_to_numpy=True, normalize_embeddings=True\n", " )[0]\n", " sim = float(np.dot(q_emb, domain_centroid))\n", " return sim >= IN_DOMAIN_THRESHOLD\n", "\n", "def passes_gate(query: str, reranked: pd.DataFrame) -> bool:\n", " \"\"\"يرجّع True لو السؤال داخل التخصص والأدلة كافية، وFalse غير كده.\"\"\"\n", " if not is_in_domain(query):\n", " return False\n", "\n", " if len(reranked) == 0:\n", " return False\n", "\n", " top = reranked.iloc[0]\n", "\n", " if float(top[\"rerank_score\"]) < MIN_RERANK_SCORE:\n", " return False\n", "\n", " if float(top[\"similarity\"]) < MIN_SIMILARITY_FLOOR:\n", " return False\n", "\n", " support = int((reranked[\"rerank_score\"] >= SUPPORT_SCORE).sum())\n", "\n", " if support < MIN_SUPPORT_COUNT:\n", " return False\n", "\n", " return True\n", "\n", "\n", "def medical_rag_answer(\n", " query: str,\n", " user_data: str = \"\",\n", " verbose: bool = True\n", ") -> dict:\n", "\n", " candidates = vector_search(query, top_n=RETRIEVE_TOP_N)\n", "\n", " reranked = rerank(\n", " query,\n", " candidates,\n", " top_k=RERANK_TOP_K\n", " )\n", "\n", " top_score = (\n", " float(reranked[\"rerank_score\"].max())\n", " if len(reranked)\n", " else -999\n", " )\n", "\n", " if passes_gate(query, reranked): # <-- اتغيرت هنا: بقى بياخد query\n", " evidence = build_evidence(reranked)\n", " else:\n", " evidence = []\n", "\n", " answer = generate_answer(\n", " query,\n", " user_data,\n", " evidence\n", " )\n", "\n", " result = {\n", " \"query\": query,\n", " \"user_data\": user_data,\n", " \"answer\": answer,\n", " \"evidence\": evidence,\n", " \"top_rerank_score\": top_score,\n", " }\n", "\n", " if verbose:\n", " print(\"Q:\", query)\n", "\n", " print(\"\\n--- User Data ---\")\n", " print(user_data if user_data else \"(none)\")\n", "\n", " print(\"\\n--- Evidence used ---\")\n", "\n", " if not evidence:\n", " print(\"(none — did not pass the confidence gate)\")\n", "\n", " for e in evidence:\n", " print(\n", " f\"[{e['doc_id']}] \"\n", " f\"sim={e['similarity']} \"\n", " f\"rerank={e['rerank_score']} | \"\n", " f\"{e['question']}\"\n", " )\n", "\n", " print(\"\\n--- Final Answer ---\")\n", " print(textwrap.fill(answer, width=100))\n", "\n", " return result" ] }, { "cell_type": "code", "execution_count": 100, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "kFfmjaO2xlqB", "outputId": "ad8ac5cd-717d-4b39-e868-8cf7f85cae8e" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Q: What are the common symptoms of hypothyroidism?\n", "\n", "--- User Data ---\n", "(none)\n", "\n", "--- Evidence used ---\n", "[10924] sim=0.99 rerank=9.309 | What are the symptoms of hypothyroidism?\n", "[12883] sim=0.99 rerank=8.24 | What are the symptoms of hypothyroidism?\n", "[12438] sim=0.981 rerank=8.693 | Hello\n", "What are the most prominent symptoms of hypothyroidism?\n", "[15321] sim=0.957 rerank=7.477 | I want to know the symptoms of hypothyroidism\n", "[14391] sim=0.949 rerank=7.562 | Hello, my question is: What are the symptoms that indicate hypothyroidism?\n", "[14226] sim=0.937 rerank=7.942 | What are the symptoms of hypothyroidism\n", "\n", "--- Final Answer ---\n", "Common symptoms of hypothyroidism include: - Extreme fatigue or general laziness and\n", "lethargy [10924][12438][15321][14391][14226] - Sensitivity to cold / feeling\n", "cold [10924][12883][12438][14391][14226] - Weight gain [10924][12883][12438][15321][14391][14226]\n", "- Constipation [10924][12438] - Depression or mood changes [10924] - Muscle pain, weakness,\n", "spasms, or tension [10924][12438] - Dry skin and increased skin thickness [10924][12438][15321]\n", "- Hair loss or breakage of hair and nails [10924][15321][14391][14226] - Pain and numbness in the\n", "hands and fingers [10924] - Menstrual disorders, heavy or prolonged periods [10924][12438] -\n", "Elevated blood cholesterol [10924][12438] - Forgetfulness or cognitive slowing [10924] - Joint\n", "pain [10924] - Decreased sweating [10924] - Reproductive issues in women [10924] This\n", "information is based on the available medical evidence and is for general informational purposes. It\n", "is not a diagnosis or a substitute for professional medical advice.\n" ] } ], "source": [ "_ = medical_rag_answer(\"What are the common symptoms of hypothyroidism?\")" ] }, { "cell_type": "code", "execution_count": 101, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "EhXLgXaaxnMd", "outputId": "647533b9-4132-42df-a347-e2b17e9793c4" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Q: How much insulin should I take?\n", "\n", "--- User Data ---\n", "(none)\n", "\n", "--- Evidence used ---\n", "[2890] sim=0.767 rerank=7.13 | The amount of insulin I take is as follows: 25 units in the morning, 35 units in the afternoon, 25 units at night, and the daily blood sugar average is 300. Can these doses be increased, and do I need more than that?\n", "[4794] sim=0.773 rerank=6.172 | I don't know why insulin doses are appropriate for me or not. I take 70 units a day, 40 units in the morning and 30 units in the evening.\n", "[541] sim=0.726 rerank=5.837 | How much insulin should be in the blood? What is insulin A?\n", "[2622] sim=0.726 rerank=3.524 | How much insulin should a diabetic patient inject if the sugar level rises above 300 ml?\n", "[9334] sim=0.712 rerank=4.169 | I use 10 units of insulin 70/30 in the morning, but surprisingly, it does not exceed 99 or less, and after four or three hours, 45, and before breakfast at most, 89, I stopped eating all...\n", "[3848] sim=0.742 rerank=1.425 | What is the recommended dose of insulin for a diabetic patient whose blood sugar level reaches 300 after eating?\n", "\n", "--- Final Answer ---\n", "The evidence indicates that insulin dosing should be individualized and adjusted under a physician’s\n", "supervision. If your blood sugar rises above 300 mg/dL, a typical adjustment might be 4–8 units of\n", "insulin, but you should not change your dose without consulting your treating physician and\n", "considering your meal plan and blood‑sugar monitoring [2622][2890][4794][3848][9334]. This\n", "information is based on the available medical evidence and is for general informational purposes. It\n", "is not a diagnosis or a substitute for professional medical advice.\n" ] } ], "source": [ "_ = medical_rag_answer(\"How much insulin should I take?\")" ] }, { "cell_type": "code", "execution_count": 102, "metadata": { "id": "YXUHPbtVxyo-", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "9dee7569-4bac-4f70-ff53-949f2d20724c" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Q: What are the best treatments for a broken wrist?\n", "\n", "--- User Data ---\n", "(none)\n", "\n", "--- Evidence used ---\n", "(none — did not pass the confidence gate)\n", "\n", "--- Final Answer ---\n", "The available medical evidence does not provide enough information to answer this reliably.\n" ] } ], "source": [ "_ = medical_rag_answer(\"What are the best treatments for a broken wrist?\")" ] }, { "cell_type": "code", "source": [ "_ = medical_rag_answer(\"I have knee pain when I walk, what could this be?\")" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "sfmxFQaGbA1j", "outputId": "52f785d6-d2cb-4b7b-92d5-7e01d85f8fad" }, "execution_count": 103, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Q: I have knee pain when I walk, what could this be?\n", "\n", "--- User Data ---\n", "(none)\n", "\n", "--- Evidence used ---\n", "(none — did not pass the confidence gate)\n", "\n", "--- Final Answer ---\n", "The available medical evidence does not provide enough information to answer this reliably.\n" ] } ] }, { "cell_type": "code", "source": [ "_ = medical_rag_answer(\"Why do I feel short of breath and tired when I climb stairs?\")" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "svhP9rKUbA5E", "outputId": "9cc62224-58b9-4cc3-c262-39ece358a4d2" }, "execution_count": 104, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Q: Why do I feel short of breath and tired when I climb stairs?\n", "\n", "--- User Data ---\n", "(none)\n", "\n", "--- Evidence used ---\n", "(none — did not pass the confidence gate)\n", "\n", "--- Final Answer ---\n", "The available medical evidence does not provide enough information to answer this reliably.\n" ] } ] }, { "cell_type": "markdown", "source": [ "# **Evaluation**" ], "metadata": { "id": "zGZepRpMzm0s" } }, { "cell_type": "code", "execution_count": 105, "metadata": { "id": "X6_lPsDex2O6", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "f7cefce3-2cf3-4700-9cde-47e76fcea796" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Self-retrieval accuracy (Top-1): 50/50 = 100.0%\n" ] } ], "source": [ "sample = df.sample(n=50, random_state=42)\n", "self_hit = 0\n", "\n", "for _, row in sample.iterrows():\n", " top1 = vector_search(row[\"Question\"], top_n=1)\n", " if top1.iloc[0][\"doc_id\"] == row[\"doc_id\"]:\n", " self_hit += 1\n", "\n", "print(f\"Self-retrieval accuracy (Top-1): {self_hit}/{len(sample)} = {self_hit/len(sample):.1%}\")" ] }, { "cell_type": "code", "execution_count": 106, "metadata": { "id": "WcOSGIdBycgE", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "129126ad-c4f8-4360-b83b-7ede176e3046" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Questions with duplicates usable for evaluation: 67\n", "Groups with consistent (answer-aligned) duplicates: 13\n", "Retrieval only:\n", "{'Recall@1': np.float64(1.0), 'Precision@1': np.float64(1.0), 'F1@1': np.float64(1.0), 'Recall@3': np.float64(1.0), 'Precision@3': np.float64(0.333), 'F1@3': np.float64(0.5), 'Recall@5': np.float64(1.0), 'Precision@5': np.float64(0.2), 'F1@5': np.float64(0.333), 'Recall@10': np.float64(1.0), 'Precision@10': np.float64(0.1), 'F1@10': np.float64(0.182), 'MRR': np.float64(1.0), 'n_queries': 26}\n", "\n", "Retrieval + Cross-Encoder Reranker:\n", "{'Recall@1': np.float64(0.769), 'Precision@1': np.float64(0.769), 'F1@1': np.float64(0.769), 'Recall@3': np.float64(1.0), 'Precision@3': np.float64(0.333), 'F1@3': np.float64(0.5), 'Recall@5': np.float64(1.0), 'Precision@5': np.float64(0.2), 'F1@5': np.float64(0.333), 'Recall@10': np.float64(1.0), 'Precision@10': np.float64(0.1), 'F1@10': np.float64(0.182), 'MRR': np.float64(0.865), 'n_queries': 26}\n" ] } ], "source": [ "import numpy as np\n", "\n", "question_groups = df.groupby(\"Question\")[\"doc_id\"].apply(set)\n", "duplicate_questions = question_groups[question_groups.apply(len) >= 2]\n", "\n", "print(f\"Questions with duplicates usable for evaluation: {len(duplicate_questions)}\")\n", "\n", "def group_has_consistent_answers(doc_ids, threshold: float = 0.5) -> bool:\n", " answers = df[df[\"doc_id\"].isin(doc_ids)][\"Answer\"].astype(str).tolist()\n", " if len(answers) < 2:\n", " return False\n", " ref = answers[0]\n", " return all(is_near_duplicate(ref, a, threshold=threshold) for a in answers[1:])\n", "\n", "consistent_groups = duplicate_questions[\n", " duplicate_questions.apply(lambda ids: group_has_consistent_answers(ids))\n", "]\n", "\n", "print(f\"Groups with consistent (answer-aligned) duplicates: {len(consistent_groups)}\")\n", "\n", "eligible_doc_ids = [\n", " doc_id\n", " for group in consistent_groups\n", " for doc_id in group\n", "]\n", "\n", "def evaluate_retrieval(\n", " top_k_values=(1, 3, 5, 10),\n", " n_queries=150,\n", " seed=42,\n", " use_reranker=False\n", "):\n", " rng = np.random.RandomState(seed)\n", " n = min(n_queries, len(eligible_doc_ids))\n", " sample_ids = rng.choice(\n", " eligible_doc_ids,\n", " size=n,\n", " replace=False\n", " )\n", "\n", " max_k = max(top_k_values)\n", "\n", " recalls = {k: [] for k in top_k_values}\n", " precisions = {k: [] for k in top_k_values}\n", " f1_scores = {k: [] for k in top_k_values}\n", "\n", " reciprocal_ranks = []\n", "\n", " for doc_id in sample_ids:\n", " row = df[df[\"doc_id\"] == doc_id].iloc[0]\n", " query_text = row[\"Question\"]\n", "\n", " relevant_ids = consistent_groups[query_text] - {doc_id}\n", "\n", " if not relevant_ids:\n", " continue\n", "\n", " candidates = vector_search(\n", " query_text,\n", " top_n=max_k * 3\n", " )\n", "\n", " candidates = candidates[\n", " candidates[\"doc_id\"] != doc_id\n", " ]\n", "\n", " if use_reranker:\n", " candidates = rerank(\n", " query_text,\n", " candidates,\n", " top_k=max_k\n", " )\n", "\n", " retrieved_ids = candidates[\"doc_id\"].tolist()[:max_k]\n", "\n", " total_relevant = len(relevant_ids)\n", "\n", " for k in top_k_values:\n", " retrieved_at_k = retrieved_ids[:k]\n", "\n", " relevant_retrieved = sum(\n", " rid in relevant_ids\n", " for rid in retrieved_at_k\n", " )\n", "\n", " recall = (\n", " relevant_retrieved / total_relevant\n", " if total_relevant > 0\n", " else 0.0\n", " )\n", "\n", " precision = (\n", " relevant_retrieved / k\n", " if k > 0\n", " else 0.0\n", " )\n", "\n", " if recall + precision > 0:\n", " f1 = (\n", " 2 * precision * recall\n", " / (precision + recall)\n", " )\n", " else:\n", " f1 = 0.0\n", "\n", " recalls[k].append(recall)\n", " precisions[k].append(precision)\n", " f1_scores[k].append(f1)\n", "\n", " rr = 0.0\n", "\n", " for rank, rid in enumerate(retrieved_ids, start=1):\n", " if rid in relevant_ids:\n", " rr = 1.0 / rank\n", " break\n", "\n", " reciprocal_ranks.append(rr)\n", "\n", " results = {}\n", "\n", " for k in top_k_values:\n", " results[f\"Recall@{k}\"] = round(\n", " np.mean(recalls[k]),\n", " 3\n", " )\n", "\n", " results[f\"Precision@{k}\"] = round(\n", " np.mean(precisions[k]),\n", " 3\n", " )\n", "\n", " results[f\"F1@{k}\"] = round(\n", " np.mean(f1_scores[k]),\n", " 3\n", " )\n", "\n", " results[\"MRR\"] = round(\n", " np.mean(reciprocal_ranks),\n", " 3\n", " )\n", "\n", " results[\"n_queries\"] = len(reciprocal_ranks)\n", "\n", " return results\n", "\n", "\n", "print(\"Retrieval only:\")\n", "print(\n", " evaluate_retrieval(\n", " use_reranker=False\n", " )\n", ")\n", "\n", "print(\"\\nRetrieval + Cross-Encoder Reranker:\")\n", "print(\n", " evaluate_retrieval(\n", " use_reranker=True\n", " )\n", ")" ] }, { "cell_type": "code", "execution_count": 107, "metadata": { "id": "guzkmPZh1M4H", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "327302f7-da72-48c9-916f-f06327897684" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Built a starter gold set for 100 questions.\n", "3 of them have extra semantic candidates that need manual review.\n", "Saved 16 candidate pairs to /content/gold_set_for_manual_review.xlsx for manual labeling.\n" ] } ], "source": [ "SEMANTIC_SIM_THRESHOLD = 0.90 # عتبة عالية عمدًا عشان نقلل False Positives\n", "\n", "def build_semi_automatic_gold_set(n_questions: int = 100, seed: int = 42) -> dict:\n", " \"\"\"\n", " بيرجع dict:\n", " query -> {\n", " \"relevant_ids\": set(...), # exact duplicates -> نثق فيها 100%\n", " \"candidate_ids\": set(...), # قريبة معنويًا (مش نفس النص) -> تحتاج مراجعة يدوية\n", " }\n", " \"\"\"\n", " rng = np.random.RandomState(seed)\n", " unique_questions = df[\"Question\"].drop_duplicates().tolist()\n", " sample_questions = rng.choice(\n", " unique_questions,\n", " size=min(n_questions, len(unique_questions)),\n", " replace=False,\n", " )\n", "\n", " gold = {}\n", " for q in sample_questions:\n", " own_ids = set(df[df[\"Question\"] == q][\"doc_id\"])\n", "\n", " candidates = vector_search(q, top_n=15)\n", " semantic_candidates = candidates[\n", " (candidates[\"similarity\"] >= SEMANTIC_SIM_THRESHOLD)\n", " & (~candidates[\"doc_id\"].isin(own_ids))\n", " ]\n", "\n", " gold[q] = {\n", " \"relevant_ids\": own_ids,\n", " \"candidate_ids\": set(semantic_candidates[\"doc_id\"]),\n", " }\n", " return gold\n", "\n", "\n", "gold_set = build_semi_automatic_gold_set(n_questions=100)\n", "\n", "n_with_candidates = sum(1 for v in gold_set.values() if v[\"candidate_ids\"])\n", "print(f\"Built a starter gold set for {len(gold_set)} questions.\")\n", "print(f\"{n_with_candidates} of them have extra semantic candidates that need manual review.\")\n", "\n", "review_rows = []\n", "for q, v in gold_set.items():\n", " for cid in v[\"candidate_ids\"]:\n", " cand_row = df[df[\"doc_id\"] == cid].iloc[0]\n", " review_rows.append({\n", " \"query\": q,\n", " \"candidate_doc_id\": cid,\n", " \"candidate_question\": cand_row[\"Question\"],\n", " \"candidate_answer\": str(cand_row[\"Answer\"])[:200],\n", " \"is_relevant (fill manually: 1/0)\": \"\",\n", " })\n", "\n", "review_df = pd.DataFrame(review_rows)\n", "review_path = \"/content/gold_set_for_manual_review.xlsx\"\n", "review_df.to_excel(review_path, index=False)\n", "print(f\"Saved {len(review_df)} candidate pairs to {review_path} for manual labeling.\")" ] }, { "cell_type": "code", "execution_count": 108, "metadata": { "id": "hIS79V-ABUML", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "1fc39843-ac28-4703-f062-6b4d4ba6d908" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Total questions in gold set: 100\n", "Questions with at least one manually-approved candidate: 0\n", "\n", "⚠️ لسه معملتش تعديل في ملف gold_set_for_manual_review.xlsx (أو رفعته من غير حفظ).\n", "افتح الملف، حط 1 في الصفوف اللي فعلاً relevant، ارفعه تاني على /content بنفس الاسم، وشغّل الخلية دي تاني.\n" ] } ], "source": [ "def recall_at_k(retrieved_ids, relevant_ids, k):\n", " retrieved = retrieved_ids[:k]\n", " return int(len(set(retrieved) & set(relevant_ids)) > 0)\n", "\n", "\n", "def reciprocal_rank(retrieved_ids, relevant_ids):\n", " relevant_ids = set(relevant_ids)\n", " for rank, doc_id in enumerate(retrieved_ids, start=1):\n", " if doc_id in relevant_ids:\n", " return 1.0 / rank\n", " return 0.0\n", "\n", "\n", "def ndcg_at_k(retrieved_ids, relevant_ids, k):\n", " relevant_ids = set(relevant_ids)\n", " dcg = 0.0\n", " for i, doc_id in enumerate(retrieved_ids[:k], start=1):\n", " if doc_id in relevant_ids:\n", " dcg += 1.0 / np.log2(i + 1)\n", " ideal_hits = min(len(relevant_ids), k)\n", " idcg = sum(1.0 / np.log2(i + 1) for i in range(1, ideal_hits + 1))\n", " return dcg / idcg if idcg > 0 else 0.0\n", "\n", "\n", "def evaluate_gold_retrieval(gold_set: dict, relevant_ids_map: dict, top_k: int = 5, retrieval_n: int = 30, use_reranker: bool = False):\n", " \"\"\"\n", " gold_set: الـ dict الأصلي (query -> {\"relevant_ids\": own_ids, ...}) --\n", " بنستخدمه بس عشان نعرف نستبعد الصف/النسخ المطابقة حرفيًا من البحث.\n", " relevant_ids_map: query -> مجموعة الصح الكاملة (الصف نفسه + الـ candidates المعتمدة بـ 1) --\n", " بنستخدمه في حساب الـ score.\n", " \"\"\"\n", " recalls, mrrs, ndcgs = [], [], []\n", " skipped_no_new_evidence = 0\n", "\n", " for query, relevant_ids in relevant_ids_map.items():\n", " own_ids = gold_set[query][\"relevant_ids\"]\n", " extra_relevant = relevant_ids - own_ids\n", "\n", " if not extra_relevant:\n", " skipped_no_new_evidence += 1\n", " continue\n", "\n", " candidates = vector_search(query, top_n=retrieval_n)\n", " candidates = candidates[~candidates[\"doc_id\"].isin(own_ids)] # استبعد الصف نفسه بس\n", " if use_reranker:\n", " candidates = rerank(query, candidates, top_k=top_k)\n", " else:\n", " candidates = candidates.head(top_k)\n", " retrieved_ids = candidates[\"doc_id\"].tolist()[:top_k]\n", "\n", " recalls.append(recall_at_k(retrieved_ids, extra_relevant, top_k))\n", " mrrs.append(reciprocal_rank(retrieved_ids, extra_relevant))\n", " ndcgs.append(ndcg_at_k(retrieved_ids, extra_relevant, top_k))\n", "\n", " print(f\"(تم تخطي {skipped_no_new_evidence} سؤال مفيهمش candidates معتمدة)\")\n", "\n", " if not recalls:\n", " return {\"note\": \"مفيش ولا سؤال عنده candidate معتمد بـ 1 -- راجع ملف الإكسل وتأكد إنك حطيت 1 في صفوف فعلاً relevant\"}\n", "\n", " return {\n", " f\"Recall@{top_k}\": round(float(np.mean(recalls)), 3),\n", " \"MRR\": round(float(np.mean(mrrs)), 3),\n", " f\"NDCG@{top_k}\": round(float(np.mean(ndcgs)), 3),\n", " \"n_queries\": len(recalls),\n", " }\n", "\n", "\n", "def merge_reviewed_gold_set(gold_set: dict, reviewed_path: str = \"/content/gold_set_for_manual_review.xlsx\") -> dict:\n", " \"\"\"بعد ما تراجع ملف gold_set_for_manual_review.xlsx وتحط 1/0، شغّل الدالة دي\n", " عشان تضيف الـ candidates اللي عليهم 1 لمجموعة الـ relevant_ids النهائية.\n", "\n", " لو الملف لسه مترفعش (مثلاً وقت Run All كامل من غير توقف للمراجعة اليدوية)،\n", " بترجع الـ gold set الأساسي زي ما هو (exact duplicates بس) بدل ما تكسر الـ notebook\n", " بـ FileNotFoundError مش واضحة.\n", " \"\"\"\n", " try:\n", " reviewed = pd.read_excel(reviewed_path)\n", " except FileNotFoundError:\n", " print(f\"⚠️ {reviewed_path} مش موجود لسه. راجع الملف اللي اتصدّر فوق وحطه هنا الأول.\")\n", " print(\"هرجع الـ gold set الأساسي (exact duplicates فقط) لحد ما تعمل المراجعة.\")\n", " return {q: set(v[\"relevant_ids\"]) for q, v in gold_set.items()}\n", "\n", " merged = {q: set(v[\"relevant_ids\"]) for q, v in gold_set.items()}\n", " for _, r in reviewed.iterrows():\n", " if str(r[\"is_relevant (fill manually: 1/0)\"]).strip() == \"1\":\n", " merged.setdefault(r[\"query\"], set()).add(r[\"candidate_doc_id\"])\n", " return merged\n", "\n", "\n", "gold_evidence_final = merge_reviewed_gold_set(gold_set)\n", "\n", "n_with_approved = sum(1 for q, ids in gold_evidence_final.items() if ids - gold_set[q][\"relevant_ids\"])\n", "print(f\"Total questions in gold set: {len(gold_evidence_final)}\")\n", "print(f\"Questions with at least one manually-approved candidate: {n_with_approved}\")\n", "\n", "if n_with_approved == 0:\n", " print(\"\\n⚠️ لسه معملتش تعديل في ملف gold_set_for_manual_review.xlsx (أو رفعته من غير حفظ).\")\n", " print(\"افتح الملف، حط 1 في الصفوف اللي فعلاً relevant، ارفعه تاني على /content بنفس الاسم، وشغّل الخلية دي تاني.\")\n", "else:\n", " print(\"\\nFinal Gold Retrieval only (bi-encoder / FAISS):\")\n", " print(evaluate_gold_retrieval(gold_set, gold_evidence_final, top_k=5, use_reranker=False))\n", "\n", " print(\"\\nFinal Gold Retrieval + Cross-Encoder Reranker:\")\n", " print(evaluate_gold_retrieval(gold_set, gold_evidence_final, top_k=5, use_reranker=True))\n" ] }, { "cell_type": "code", "execution_count": 109, "metadata": { "id": "jC8rtDO3BUP_", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "f8ece522-ead0-43b7-f156-158271b6b5a0" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Category sanity check (retriever + reranker):\n", "{'category_recall': 0.97, 'n': 100}\n", "\n", "Category sanity check (retriever only):\n", "{'category_recall': 0.97, 'n': 100}\n" ] } ], "source": [ "def category_match_recall(top_k: int = 5, n_queries: int = 100, seed: int = 42, use_reranker: bool = True):\n", " sample = df.sample(n=min(n_queries, len(df)), random_state=seed)\n", " hits = []\n", "\n", " for _, row in sample.iterrows():\n", " query = row[\"Question\"]\n", " true_category = row[\"Category\"]\n", "\n", " candidates = vector_search(query, top_n=20)\n", " candidates = candidates[candidates[\"doc_id\"] != row[\"doc_id\"]] # leave-one-out بالـ id بدل نص الـ Question\n", "\n", " if use_reranker:\n", " candidates = rerank(query, candidates, top_k=top_k)\n", " else:\n", " candidates = candidates.head(top_k)\n", "\n", " hit = candidates[\"Category\"].astype(str).eq(str(true_category)).any()\n", " hits.append(hit)\n", "\n", " return {\"category_recall\": round(float(np.mean(hits)), 3), \"n\": len(hits)}\n", "\n", "\n", "print(\"Category sanity check (retriever + reranker):\")\n", "print(category_match_recall(use_reranker=True))\n", "print(\"\\nCategory sanity check (retriever only):\")\n", "print(category_match_recall(use_reranker=False))" ] }, { "cell_type": "code", "execution_count": 116, "metadata": { "id": "JLiiv958BUTF", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "8ae4ad39-a3ee-4d2c-be83-a3a37181c178" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " query expected_answer system_answered top_rerank_score correct_decision\n", " What are the symptoms of diabetes? True True 9.837 True\n", " What is the treatment for diabetes? True True 9.100 True\n", " Is there a cure for diabetes? True True 9.930 True\n", " What is the normal blood sugar level? True True 8.873 True\n", " What are the causes of diabetes? True True 9.455 True\n", " What foods are beneficial for diabetics? True True 9.858 True\n", " What is diabetic foot? True True 9.154 True\n", " What are the symptoms of thyroid disease? True True 9.468 True\n", " What is the treatment for hypothyroidism? True True 9.041 True\n", " I suffer from an underactive thyroid gland True True 9.566 True\n", " What are the symptoms of high blood sugar? True True 10.173 True\n", " What is the treatment for low blood sugar? True True 8.904 True\n", " What are the symptoms of gestational diabetes? True True 8.832 True\n", " What are the risk factors for type 1 diabetes? True True 9.436 True\n", " What is the treatment for hyperthyroidism? True True 8.256 True\n", " Can diabetes be cured permanently? True True 10.189 True\n", " What is a normal HbA1c level? True False 7.678 False\n", " What are the complications of untreated diabetes? True True 4.662 True\n", "Is it normal for TSH to be slightly high after thyroid surgery? True True 3.788 True\n", " What is the relationship between diabetes and weight loss? True True 3.504 True\n", " How is hypothyroidism diagnosed? True True 3.590 True\n", " What are the symptoms of hyperthyroidism? True True 9.664 True\n", " What is the treatment for diabetic foot? True True 8.185 True\n", " What causes weight gain in hypothyroidism? True True 6.112 True\n", " Is thyroid disease hereditary? True True 9.274 True\n", " What is the best treatment for a broken leg? False False 5.989 True\n", " How do I fix a slow laptop? False False -9.115 True\n", " What is the capital of France? False False -10.812 True\n", " Best recipe for chocolate cake? False False -9.082 True\n", " How to change a car tire? False False -10.981 True\n", " What is the treatment for a migraine headache? False False -2.041 True\n", " How do I treat a common cold? False False 1.136 True\n", " What are the symptoms of appendicitis? False False -2.848 True\n", " How to remove a stain from a carpet? False False -7.555 True\n", " What is the best programming language to learn? False False -9.665 True\n", " How do I train for a marathon? False False -9.465 True\n", " What is the treatment for a skin rash from poison ivy? False False -6.862 True\n", " How do I apply for a passport? False False -10.861 True\n", " What are the symptoms of a kidney stone? False False -3.246 True\n", " Best way to whiten teeth at home? False False -8.814 True\n", " How do I treat a sprained ankle? False False -1.468 True\n", " What is the weather like today? False False -3.724 True\n", " How to fix a leaking faucet? False False -10.968 True\n", " What is the treatment for asthma? False False -4.835 True\n", " How do I improve my credit score? False False -7.260 True\n", " What is the treatment for a broken arm? False False -2.200 True\n", " How do I treat a sunburn? False False -3.939 True\n", " What's the best laptop for gaming? False False -10.349 True\n", " How do I remove a splinter? False False -7.584 True\n", " What is the treatment for food poisoning? False False -0.960 True\n", "\n", "Gate decision accuracy: 98.0%\n", "\n", "top_rerank_score by expected label:\n", " mean min max\n", "expected_answer \n", "False -5.90036 -10.981 5.989\n", "True 8.26116 3.504 10.189\n" ] } ], "source": [ "eval_set = [\n", " #Positive\n", " (\"What are the symptoms of diabetes?\", True),\n", " (\"What is the treatment for diabetes?\", True),\n", " (\"Is there a cure for diabetes?\", True),\n", " (\"What is the normal blood sugar level?\", True),\n", " (\"What are the causes of diabetes?\", True),\n", " (\"What foods are beneficial for diabetics?\", True),\n", " (\"What is diabetic foot?\", True),\n", " (\"What are the symptoms of thyroid disease?\", True),\n", " (\"What is the treatment for hypothyroidism?\", True),\n", " (\"I suffer from an underactive thyroid gland\", True),\n", " (\"What are the symptoms of high blood sugar?\", True),\n", " (\"What is the treatment for low blood sugar?\", True),\n", " (\"What are the symptoms of gestational diabetes?\", True),\n", " (\"What are the risk factors for type 1 diabetes?\", True),\n", " (\"What is the treatment for hyperthyroidism?\", True),\n", " (\"Can diabetes be cured permanently?\", True),\n", " (\"What is a normal HbA1c level?\", True),\n", " (\"What are the complications of untreated diabetes?\", True),\n", " (\"Is it normal for TSH to be slightly high after thyroid surgery?\", True),\n", " (\"What is the relationship between diabetes and weight loss?\", True),\n", " (\"How is hypothyroidism diagnosed?\", True),\n", " (\"What are the symptoms of hyperthyroidism?\", True),\n", " (\"What is the treatment for diabetic foot?\", True),\n", " (\"What causes weight gain in hypothyroidism?\", True),\n", " (\"Is thyroid disease hereditary?\", True),\n", "\n", " #Negative\n", " (\"What is the best treatment for a broken leg?\", False),\n", " (\"How do I fix a slow laptop?\", False),\n", " (\"What is the capital of France?\", False),\n", " (\"Best recipe for chocolate cake?\", False),\n", " (\"How to change a car tire?\", False),\n", " (\"What is the treatment for a migraine headache?\", False),\n", " (\"How do I treat a common cold?\", False),\n", " (\"What are the symptoms of appendicitis?\", False),\n", " (\"How to remove a stain from a carpet?\", False),\n", " (\"What is the best programming language to learn?\", False),\n", " (\"How do I train for a marathon?\", False),\n", " (\"What is the treatment for a skin rash from poison ivy?\", False),\n", " (\"How do I apply for a passport?\", False),\n", " (\"What are the symptoms of a kidney stone?\", False),\n", " (\"Best way to whiten teeth at home?\", False),\n", " (\"How do I treat a sprained ankle?\", False),\n", " (\"What is the weather like today?\", False),\n", " (\"How to fix a leaking faucet?\", False),\n", " (\"What is the treatment for asthma?\", False),\n", " (\"How do I improve my credit score?\", False),\n", " (\"What is the treatment for a broken arm?\", False),\n", " (\"How do I treat a sunburn?\", False),\n", " (\"What's the best laptop for gaming?\", False),\n", " (\"How do I remove a splinter?\", False),\n", " (\"What is the treatment for food poisoning?\", False),\n", "]\n", "\n", "rows = []\n", "for query, should_answer in eval_set:\n", " candidates = vector_search(query, top_n=RETRIEVE_TOP_N)\n", " reranked = rerank(query, candidates, top_k=RERANK_TOP_K)\n", " top_score = float(reranked[\"rerank_score\"].max()) if len(reranked) else -999\n", " system_answered = passes_gate(query, reranked)\n", " rows.append({\n", " \"query\": query,\n", " \"expected_answer\": should_answer,\n", " \"system_answered\": system_answered,\n", " \"top_rerank_score\": round(top_score, 3),\n", " \"correct_decision\": should_answer == system_answered,\n", " })\n", "\n", "gate_df = pd.DataFrame(rows)\n", "print(gate_df.to_string(index=False))\n", "print(f\"\\nGate decision accuracy: {gate_df['correct_decision'].mean():.1%}\")\n", "\n", "print(\"\\ntop_rerank_score by expected label:\")\n", "print(gate_df.groupby(\"expected_answer\")[\"top_rerank_score\"].describe()[[\"mean\", \"min\", \"max\"]])" ] }, { "cell_type": "code", "execution_count": 117, "metadata": { "id": "TO_pOqOLBUWR", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "41fd3aaa-7080-4d88-e8bf-4b753de351ff" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Cross-validated threshold tuning:\n", " fold_thresholds: [1.147, 1.147, -0.875, 1.147, 1.147]\n", " mean_threshold: 0.742\n", " std_threshold: 0.809\n", " fold_test_accuracies: [0.9, 1.0, 0.9, 1.0, 1.0]\n", " mean_test_accuracy: 0.96\n", " std_test_accuracy: 0.049\n", "\n", "متوسط الـ threshold المقترح: 0.742 (± 0.809), بمتوسط accuracy على test folds = 96.0% (± 4.9%)\n", "لو الـ std كبير (يعني القيمة بتتغير كتير بين fold وfold)، ده معناه العينة لسه صغيرة ومحتاجة توسّع أكتر قبل ما تثق في threshold نهائي.\n" ] } ], "source": [ "def find_best_threshold_cv(scores, labels, n_folds: int = 5, n_steps: int = 200, seed: int = 42):\n", " scores = np.array(scores, dtype=float)\n", " labels = np.array(labels, dtype=int)\n", " n = len(scores)\n", " n_folds = min(n_folds, n)\n", "\n", " rng = np.random.RandomState(seed)\n", " idx = rng.permutation(n)\n", " folds = np.array_split(idx, n_folds)\n", "\n", " fold_thresholds, fold_test_accuracies = [], []\n", "\n", " for i in range(n_folds):\n", " test_idx = folds[i]\n", " train_idx = np.concatenate([folds[j] for j in range(n_folds) if j != i])\n", " if len(train_idx) == 0 or len(test_idx) == 0:\n", " continue\n", "\n", " train_scores, train_labels = scores[train_idx], labels[train_idx]\n", " test_scores, test_labels = scores[test_idx], labels[test_idx]\n", "\n", " best_threshold, best_acc = None, -1.0\n", " for t in np.linspace(scores.min(), scores.max(), n_steps):\n", " preds = (train_scores >= t).astype(int)\n", " acc = float(np.mean(preds == train_labels))\n", " if acc > best_acc:\n", " best_acc, best_threshold = acc, float(t)\n", "\n", " test_preds = (test_scores >= best_threshold).astype(int)\n", " test_acc = float(np.mean(test_preds == test_labels))\n", "\n", " fold_thresholds.append(best_threshold)\n", " fold_test_accuracies.append(test_acc)\n", "\n", " return {\n", " \"fold_thresholds\": [round(t, 3) for t in fold_thresholds],\n", " \"mean_threshold\": round(float(np.mean(fold_thresholds)), 3),\n", " \"std_threshold\": round(float(np.std(fold_thresholds)), 3),\n", " \"fold_test_accuracies\": [round(a, 3) for a in fold_test_accuracies],\n", " \"mean_test_accuracy\": round(float(np.mean(fold_test_accuracies)), 3),\n", " \"std_test_accuracy\": round(float(np.std(fold_test_accuracies)), 3),\n", " }\n", "\n", "\n", "cv_result = find_best_threshold_cv(\n", " scores=gate_df[\"top_rerank_score\"].tolist(),\n", " labels=gate_df[\"expected_answer\"].astype(int).tolist(),\n", " n_folds=5,\n", ")\n", "print(\"Cross-validated threshold tuning:\")\n", "for k, v in cv_result.items():\n", " print(f\" {k}: {v}\")\n", "\n", "print(\n", " f\"\\nمتوسط الـ threshold المقترح: {cv_result['mean_threshold']} \"\n", " f\"(± {cv_result['std_threshold']}), بمتوسط accuracy على test folds = \"\n", " f\"{cv_result['mean_test_accuracy']:.1%} (± {cv_result['std_test_accuracy']:.1%})\"\n", ")\n", "print(\n", " \"لو الـ std كبير (يعني القيمة بتتغير كتير بين fold وfold)، ده معناه العينة \"\n", " \"لسه صغيرة ومحتاجة توسّع أكتر قبل ما تثق في threshold نهائي.\"\n", ")\n" ] }, { "cell_type": "code", "execution_count": 118, "metadata": { "id": "mEKfECUI1M7h", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "4bb51a7a-8352-427e-bb69-3a4b6e1f6fe8" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "{'retrieval': 0.015, 'rerank': 0.097, 'evidence_build': 0.02, 'llm_generation': 1.432, 'total': 1.565}\n" ] } ], "source": [ "import time\n", "\n", "def medical_rag_answer_timed(query: str) -> dict:\n", " t0 = time.perf_counter()\n", " candidates = vector_search(query, top_n=RETRIEVE_TOP_N)\n", " t1 = time.perf_counter()\n", "\n", " reranked = rerank(query, candidates, top_k=RERANK_TOP_K)\n", " t2 = time.perf_counter()\n", "\n", " top_score = float(reranked[\"rerank_score\"].max()) if len(reranked) else -999\n", " evidence = build_evidence(reranked) if passes_gate(query, reranked) else [] # <-- اتغيرت هنا\n", " t3 = time.perf_counter()\n", "\n", " answer = generate_answer(query, \"\", evidence)\n", " t4 = time.perf_counter()\n", "\n", " return {\n", " \"query\": query, \"answer\": answer, \"evidence\": evidence,\n", " \"timing_sec\": {\n", " \"retrieval\": round(t1 - t0, 3),\n", " \"rerank\": round(t2 - t1, 3),\n", " \"evidence_build\": round(t3 - t2, 3),\n", " \"llm_generation\": round(t4 - t3, 3),\n", " \"total\": round(t4 - t0, 3),\n", " },\n", " }\n", "\n", "\n", "result = medical_rag_answer_timed(\"What are the symptoms of diabetes?\")\n", "print(result[\"timing_sec\"])" ] }, { "cell_type": "markdown", "source": [ "# **SAVE**" ], "metadata": { "id": "fRPbr7SJIgpw" } }, { "cell_type": "code", "source": [ "import os\n", "import json\n", "import shutil\n", "\n", "SAVE_DIR = \"/content/rag_model\"\n", "\n", "os.makedirs(SAVE_DIR, exist_ok=True)\n", "os.makedirs(f\"{SAVE_DIR}/faiss\", exist_ok=True)\n", "os.makedirs(f\"{SAVE_DIR}/models\", exist_ok=True)\n", "os.makedirs(f\"{SAVE_DIR}/data\", exist_ok=True)\n", "\n", "faiss.write_index(\n", " faiss_index,\n", " f\"{SAVE_DIR}/faiss/questions.index\"\n", ")\n", "\n", "print(\"✓ FAISS index saved\")\n", "\n", "\n", "# 2) Save embeddings\n", "np.save(\n", " f\"{SAVE_DIR}/faiss/question_embeddings.npy\",\n", " question_embeddings\n", ")\n", "\n", "print(\"✓ Embeddings saved\")\n", "\n", "\n", "# 3) Save dataset\n", "df.to_pickle(\n", " f\"{SAVE_DIR}/data/knowledge_base.pkl\"\n", ")\n", "\n", "print(\"✓ Knowledge base saved\")\n", "\n", "\n", "# 4) Save embedding model name\n", "with open(\n", " f\"{SAVE_DIR}/models/embedding_model.txt\",\n", " \"w\"\n", ") as f:\n", " f.write(EMBEDDING_MODEL_NAME)\n", "\n", "print(\"✓ Embedding model config saved\")\n", "\n", "\n", "# 5) Save reranker model name\n", "with open(\n", " f\"{SAVE_DIR}/models/reranker_model.txt\",\n", " \"w\"\n", ") as f:\n", " f.write(RERANKER_MODEL_NAME)\n", "\n", "print(\"✓ Reranker model config saved\")\n", "\n", "\n", "# 6) Save medical synonyms\n", "with open(\n", " f\"{SAVE_DIR}/models/medical_synonyms.json\",\n", " \"w\",\n", " encoding=\"utf-8\"\n", ") as f:\n", " json.dump(\n", " MEDICAL_SYNONYMS,\n", " f,\n", " ensure_ascii=False,\n", " indent=2\n", " )\n", "\n", "print(\"✓ Medical synonyms saved\")\n", "\n", "# 7) Save RAG configuration\n", "rag_config = {\n", " \"embedding_model\": EMBEDDING_MODEL_NAME,\n", " \"reranker_model\": RERANKER_MODEL_NAME,\n", "\n", " \"retrieve_top_n\": RETRIEVE_TOP_N,\n", " \"rerank_top_k\": RERANK_TOP_K,\n", "\n", " \"min_rerank_score\": MIN_RERANK_SCORE,\n", " \"min_similarity_floor\": MIN_SIMILARITY_FLOOR,\n", " \"min_support_count\": MIN_SUPPORT_COUNT,\n", " \"support_score\": SUPPORT_SCORE,\n", "\n", " \"semantic_similarity_threshold\": SEMANTIC_SIM_THRESHOLD,\n", "\n", " \"embedding_dimension\": int(embedding_dim),\n", "\n", " \"dataset_size\": int(len(df))\n", "}\n", "\n", "with open(\n", " f\"{SAVE_DIR}/rag_config.json\",\n", " \"w\",\n", " encoding=\"utf-8\"\n", ") as f:\n", " json.dump(\n", " rag_config,\n", " f,\n", " indent=2\n", " )\n", "\n", "print(\"✓ RAG configuration saved\")\n", "\n", "\n", "# 8) Save requirements\n", "requirements = \"\"\"sentence-transformers\n", "faiss-cpu\n", "pandas\n", "numpy\n", "openpyxl\n", "groq\n", "tqdm\n", "\"\"\"\n", "\n", "with open(\n", " f\"{SAVE_DIR}/requirements.txt\",\n", " \"w\"\n", ") as f:\n", " f.write(requirements)\n", "\n", "print(\"✓ Requirements saved\")\n", "\n", "\n", "# 9) Summary\n", "print(\"RAG PIPELINE SAVED SUCCESSFULLY\")\n", "\n", "for root, dirs, files in os.walk(SAVE_DIR):\n", " level = root.replace(SAVE_DIR, \"\").count(os.sep)\n", " indent = \" \" * level\n", " print(f\"{indent}{os.path.basename(root)}/\")\n", "\n", " for file in files:\n", " print(f\"{indent} └── {file}\")" ], "metadata": { "id": "AiRqpjHLOoDX", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "b3de8a83-0298-4afc-9ece-9df8bc145b4d" }, "execution_count": 119, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "✓ FAISS index saved\n", "✓ Embeddings saved\n", "✓ Knowledge base saved\n", "✓ Embedding model config saved\n", "✓ Reranker model config saved\n", "✓ Medical synonyms saved\n", "✓ RAG configuration saved\n", "✓ Requirements saved\n", "RAG PIPELINE SAVED SUCCESSFULLY\n", "rag_model/\n", " └── requirements.txt\n", " └── rag_config.json\n", " models/\n", " └── embedding_model.txt\n", " └── medical_synonyms.json\n", " └── reranker_model.txt\n", " faiss/\n", " └── question_embeddings.npy\n", " └── questions.index\n", " data/\n", " └── knowledge_base.pkl\n", " └── metadata_store.json\n" ] } ] }, { "cell_type": "code", "source": [ "# Save metadata_store\n", "with open(f\"{SAVE_DIR}/data/metadata_store.json\", \"w\", encoding=\"utf-8\") as f:\n", " json.dump(metadata_store, f, ensure_ascii=False, indent=2)\n", "\n", "print(\"✓ Metadata store saved\")" ], "metadata": { "id": "qV2cTA4lx_1L", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "3ba3243e-b60e-482c-b662-07c438107ec8" }, "execution_count": 120, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "✓ Metadata store saved\n" ] } ] }, { "cell_type": "code", "source": [ "from google.colab import files\n", "\n", "files.download(\"/content/questions.index\")" ], "metadata": { "id": "NymF4KADJgzk", "colab": { "base_uri": "https://localhost:8080/", "height": 17 }, "outputId": "bc0352ed-c73f-49ab-e94c-c65dae2d623e" }, "execution_count": 121, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "" ], "application/javascript": [ "\n", " async function download(id, filename, size) {\n", " if (!google.colab.kernel.accessAllowed) {\n", " return;\n", " }\n", " const div = document.createElement('div');\n", " const label = document.createElement('label');\n", " label.textContent = `Downloading \"${filename}\": `;\n", " div.appendChild(label);\n", " const progress = document.createElement('progress');\n", " progress.max = size;\n", " div.appendChild(progress);\n", " document.body.appendChild(div);\n", "\n", " const buffers = [];\n", " let downloaded = 0;\n", "\n", " const channel = await google.colab.kernel.comms.open(id);\n", " // Send a message to notify the kernel that we're ready.\n", " channel.send({})\n", "\n", " for await (const message of channel.messages) {\n", " // Send a message to notify the kernel that we're ready.\n", " channel.send({})\n", " if (message.buffers) {\n", " for (const buffer of message.buffers) {\n", " buffers.push(buffer);\n", " downloaded += buffer.byteLength;\n", " progress.value = downloaded;\n", " }\n", " }\n", " }\n", " const blob = new Blob(buffers, {type: 'application/binary'});\n", " const a = document.createElement('a');\n", " a.href = window.URL.createObjectURL(blob);\n", " a.download = filename;\n", " div.appendChild(a);\n", " a.click();\n", " div.remove();\n", " }\n", " " ] }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": [ "" ], "application/javascript": [ "download(\"download_ee55f828-362e-49ad-a05e-2b480eb9545a\", \"questions.index\", 25165869)" ] }, "metadata": {} } ] }, { "cell_type": "code", "source": [], "metadata": { "id": "V8nXNzm7JnuL" }, "execution_count": 121, "outputs": [] } ], "metadata": { "accelerator": "GPU", "colab": { "gpuType": "T4", "provenance": [] }, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python" }, "widgets": { "application/vnd.jupyter.widget-state+json": { "state": { "231962183e2e4d598fc8296579252a29": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_f309ff6cdde040b093e24bcd615e45cc", "IPY_MODEL_41e378afabb8465e9384c47f56355ee5", "IPY_MODEL_4a867cf4777d47ca90f98d8c655e706f" ], "layout": "IPY_MODEL_37c22dda91114c648009160905f7bddd" } }, "f309ff6cdde040b093e24bcd615e45cc": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_5d59d5c837094b7193b821569390dd55", "placeholder": "​", "style": "IPY_MODEL_0dcea30cb4ad4f2f9c616778ac2d06bb", "value": "Loading weights: 100%" } }, "41e378afabb8465e9384c47f56355ee5": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_a7c522381ef049cbaed852cf9dcc9617", "max": 103, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_2bc1137de13243d9a4eb7e372a2b3845", "value": 103 } }, "4a867cf4777d47ca90f98d8c655e706f": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_c8ada1557251441e940d9405077cd611", "placeholder": "​", "style": "IPY_MODEL_0d615e42fe7f46729c3d490b5c0f5653", "value": " 103/103 [00:00<00:00, 1502.89it/s]" } }, "37c22dda91114c648009160905f7bddd": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "5d59d5c837094b7193b821569390dd55": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "0dcea30cb4ad4f2f9c616778ac2d06bb": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "a7c522381ef049cbaed852cf9dcc9617": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "2bc1137de13243d9a4eb7e372a2b3845": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "c8ada1557251441e940d9405077cd611": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "0d615e42fe7f46729c3d490b5c0f5653": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "99f8b78314b54677bef180468c4106dc": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_2f34c1bd5e964c51b7c6d743af5c97f6", "IPY_MODEL_4f3e73d4552a43f48206c6a75a5f46ac", "IPY_MODEL_f5a0f32442394f3d887d9602823f64f3" ], "layout": "IPY_MODEL_aa8c71a0f41247aebcbff9640181f35a" } }, "2f34c1bd5e964c51b7c6d743af5c97f6": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_ebc91411f4004dcca5593af1d915dfc5", "placeholder": "​", "style": "IPY_MODEL_a3538dc5b06645d8989b2671de0ba052", "value": "Batches: 100%" } }, "4f3e73d4552a43f48206c6a75a5f46ac": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_5b6bd317c08c488289d87a2cb657a932", "max": 256, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_6595bc2c3b674fb4aed5ba1b7c1c3753", "value": 256 } }, "f5a0f32442394f3d887d9602823f64f3": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_b303605e655049d5941f13b2d471adc5", "placeholder": "​", "style": "IPY_MODEL_1802394e9a054d709a3e6b3059578c12", "value": " 256/256 [00:17<00:00, 45.66it/s]" } }, "aa8c71a0f41247aebcbff9640181f35a": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ebc91411f4004dcca5593af1d915dfc5": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "a3538dc5b06645d8989b2671de0ba052": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "5b6bd317c08c488289d87a2cb657a932": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6595bc2c3b674fb4aed5ba1b7c1c3753": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "b303605e655049d5941f13b2d471adc5": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "1802394e9a054d709a3e6b3059578c12": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "c1a4920370f749f0993ce31a84e6b834": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_2c21f7fae2c343c6a800aad978c35ca8", "IPY_MODEL_2ffa5c7c893c46e390b74889dc7416be", "IPY_MODEL_a6a2ebe97e754d17986a44a2262fbbc1" ], "layout": "IPY_MODEL_1f9cd44b901d435e8da94a71a3659e88" } }, "2c21f7fae2c343c6a800aad978c35ca8": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_980809c97f064990bd9bd564bc0a272e", "placeholder": "​", "style": "IPY_MODEL_5a6f8b6f087443ac971d746a4ddb8ca2", "value": "Loading weights: 100%" } }, "2ffa5c7c893c46e390b74889dc7416be": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_74e5f114987041fe8c7219758eb039ec", "max": 105, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_5ed19461e0df440eb12bc21ee9973d0e", "value": 105 } }, "a6a2ebe97e754d17986a44a2262fbbc1": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_2507e4f18b6944d7991647ca11597dc0", "placeholder": "​", "style": "IPY_MODEL_bee40a5369c04ec2998dba045b21cb43", "value": " 105/105 [00:00<00:00, 2661.26it/s]" } }, "1f9cd44b901d435e8da94a71a3659e88": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "980809c97f064990bd9bd564bc0a272e": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "5a6f8b6f087443ac971d746a4ddb8ca2": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "74e5f114987041fe8c7219758eb039ec": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "5ed19461e0df440eb12bc21ee9973d0e": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "2507e4f18b6944d7991647ca11597dc0": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "bee40a5369c04ec2998dba045b21cb43": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } }, "version_major": 2, "version_minor": 0 } } }, "nbformat": 4, "nbformat_minor": 0 }