Upload Arabic_QG_Pipeline.ipynb

Arabic_QG_Pipeline.ipynb

@@ -0,0 +1,810 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "a4080da7",
+      "metadata": {
+        "id": "a4080da7"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install -q stanza transformers sentencepiece torch sentence-transformers arabert pyarabic yake bert-score python-bidi"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "0b9b3227",
+      "metadata": {
+        "id": "0b9b3227"
+      },
+      "outputs": [],
+      "source": [
+        "import re\n",
+        "import difflib\n",
+        "import numpy as np\n",
+        "import torch\n",
+        "import pyarabic.araby as araby\n",
+        "import stanza\n",
+        "from transformers import AutoTokenizer, AutoModel\n",
+        "from sentence_transformers import SentenceTransformer, util\n",
+        "import arabert.preprocess\n",
+        "import yake\n",
+        "from bert_score import score as bertscore\n",
+        "from sentence_transformers import util\n",
+        "\n",
+        "torch.set_grad_enabled(False)\n",
+        "\n",
+        "ARAELECTRA_NAME = \"aubmindlab/araelectra-base-discriminator\"\n",
+        "SBERT_MODEL      = \"sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2\"\n",
+        "QG_MODEL         = \"Mihakram/AraT5-base-question-generation\"\n",
+        "print(\"ARAELECTRA_NAME:\", ARAELECTRA_NAME)\n",
+        "print(\"SBERT_MODEL:\", SBERT_MODEL)\n",
+        "print(\"QG_MODEL:\", QG_MODEL)\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "1ecb47e2",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "1ecb47e2",
+        "outputId": "0e68e4f5-e34e-482b-d77b-c8cfd165db31"
+      },
+      "outputs": [
+        {
+          "data": {
+            "application/vnd.jupyter.widget-view+json": {
+              "model_id": "49432146f2b74114be2ae0548515d574",
+              "version_major": 2,
+              "version_minor": 0
+            },
+            "text/plain": [
+              "Downloading https://raw.githubusercontent.com/stanfordnlp/stanza-resources/main/resources_1.10.0.json:   0%|  …"
+            ]
+          },
+          "metadata": {},
+          "output_type": "display_data"
+        },
+        {
+          "metadata": {
+            "tags": null
+          },
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "INFO:stanza:Downloaded file to /root/stanza_resources/resources.json\n",
+            "INFO:stanza:Downloading default packages for language: ar (Arabic) ...\n"
+          ]
+        },
+        {
+          "data": {
+            "application/vnd.jupyter.widget-view+json": {
+              "model_id": "b2db9589812540deb72baf8c80312dbf",
+              "version_major": 2,
+              "version_minor": 0
+            },
+            "text/plain": [
+              "Downloading https://huggingface.co/stanfordnlp/stanza-ar/resolve/v1.10.0/models/default.zip:   0%|          | …"
+            ]
+          },
+          "metadata": {},
+          "output_type": "display_data"
+        },
+        {
+          "metadata": {
+            "tags": null
+          },
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "INFO:stanza:Downloaded file to /root/stanza_resources/ar/default.zip\n",
+            "INFO:stanza:Finished downloading models and saved to /root/stanza_resources\n",
+            "INFO:stanza:Checking for updates to resources.json in case models have been updated.  Note: this behavior can be turned off with download_method=None or download_method=DownloadMethod.REUSE_RESOURCES\n"
+          ]
+        },
+        {
+          "data": {
+            "application/vnd.jupyter.widget-view+json": {
+              "model_id": "a7acd3cf34cc4fe8aa89fcfe87adbd66",
+              "version_major": 2,
+              "version_minor": 0
+            },
+            "text/plain": [
+              "Downloading https://raw.githubusercontent.com/stanfordnlp/stanza-resources/main/resources_1.10.0.json:   0%|  …"
+            ]
+          },
+          "metadata": {},
+          "output_type": "display_data"
+        },
+        {
+          "metadata": {
+            "tags": null
+          },
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "INFO:stanza:Downloaded file to /root/stanza_resources/resources.json\n",
+            "WARNING:stanza:Language ar package default expects mwt, which has been added\n"
+          ]
+        }
+      ],
+      "source": [
+        "stanza.download('ar')\n",
+        "nlp = stanza.Pipeline(lang='ar', processors='tokenize,pos,lemma,depparse', tokenize_no_ssplit=False)\n",
+        "arabert_prep = arabert.preprocess.ArabertPreprocessor(ARAELECTRA_NAME)\n",
+        "\n",
+        "tokenizer_electra = AutoTokenizer.from_pretrained(ARAELECTRA_NAME)\n",
+        "model_electra     = AutoModel.from_pretrained(ARAELECTRA_NAME)\n",
+        "\n",
+        "sbert = SentenceTransformer(SBERT_MODEL)\n",
+        "\n",
+        "from transformers import AutoTokenizer as HFTokenizer, AutoModelForSeq2SeqLM\n",
+        "qg_tokenizer = HFTokenizer.from_pretrained(QG_MODEL)\n",
+        "qg_model     = AutoModelForSeq2SeqLM.from_pretrained(QG_MODEL)\n",
+        "\n",
+        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+        "qg_model = qg_model.to(device)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 33,
+      "id": "9c2da7d8",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "9c2da7d8",
+        "outputId": "7a362049-8853-408d-ba7a-07f0086fb174"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "The Original:\n",
+            " يتشكل غمد النخاعين في الجهاز العصبي المركزي بدءاً من خلايا الدبق قليلة الاستطالات وفي الجهاز العصبي المحيطي من خلايا شوان\n"
+          ]
+        }
+      ],
+      "source": [
+        "# text = \"\"\"المحور العصبي هو نتوء طويل ونحيل، يحمل النبضات الكهربائية بعيدًا عن جسم الخلية إلى الخلايا العصبية الأخرى، أو العضلات، أو الغدد.\n",
+        "# يُغطى العديد من المحاور العصبية بمادة دهنية تُسمى غمد الميالين، والتي تعمل كعازل وتُسرّع من نقل الإشارات.\n",
+        "# ينتهي المحور العصبي عند نهايات المحور العصبي، حيث يتم إطلاق النواقل العصبية للتواصل مع الخلايا العصبية الأخرى، أو الخلايا المستهدفة.\"\"\"\n",
+        "# text = \"\"\"الجهاز العصبي المركزي:\n",
+        "# كثُر في الآونة الأخيرة انتشار حالات السكتة الدماغية، وهي حالة تحدث نتيجة عدم وصول الدم المحمّل بالأكسجين إلى الدماغ؛ كحالة طبية طارئة تبدأ فيها خلايا الدماغ بالموت بعد بضع دقائق من عدم وصول الأكسجين. وهناك نوعان رئيسان من السكتة هما السكتة الدماغية التي تحدث بسبب الجلطات الدموية، وتشكل\n",
+        "# 87% من الحالات، والسكتة الدماغية التي تحدث بسبب النزيف في الدماغ أو حوله.\n",
+        "# وتختلف أعراضها، إذ تشمل: الخدر المفاجئ، وعدم القدرة على تحريك الوجه أو الذراع أو الساق (لاسيما في أحد جانبي الجسم)، والارتباك، ومشاكل في التحدث والرؤية والدوخة، وصعوبة في المشي، وفقدان التوازن، والصداع المفاجئ والشديد، ومشاكل في التنفس، وفقدان الوعي.\"\"\"\n",
+        "text = \"يتشكل غمد النخاعين في الجهاز العصبي المركزي بدءاً من خلايا الدبق قليلة الاستطالات وفي الجهاز العصبي المحيطي من خلايا شوان\"\n",
+        "print(\"The Original:\\n\", text)\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 34,
+      "id": "0f2a3f6a",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "0f2a3f6a",
+        "outputId": "39092572-679c-4a58-bfdc-ba47492227de"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Text after normalization:\n",
+            " يتشكل غمد النخاعين في الجهاز العصبي المركزي بدءا من خلايا الدبق قليلة الاستطالات وفي الجهاز العصبي المحيطي من خلايا شوان\n"
+          ]
+        }
+      ],
+      "source": [
+        "def normalize(s: str) -> str:\n",
+        "    t = araby.strip_tashkeel(s)\n",
+        "    t = t.replace('آ','ا').replace('أ','ا').replace('إ','ا').replace('ى','ي')\n",
+        "    t = t.replace('ـ','')\n",
+        "    t = ' '.join(t.split())\n",
+        "    return t\n",
+        "text_norm = normalize(text)\n",
+        "print(\"Text after normalization:\\n\", text_norm)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 35,
+      "id": "b7db97d3",
+      "metadata": {
+        "id": "b7db97d3"
+      },
+      "outputs": [],
+      "source": [
+        "def build_char_map(src: str, tgt: str):\n",
+        "    sm = difflib.SequenceMatcher(a=src, b=tgt)\n",
+        "    src2tgt = [-1] * len(src)\n",
+        "    for tag, i1, i2, j1, j2 in sm.get_opcodes():\n",
+        "        if tag == 'equal':\n",
+        "            for k in range(i2 - i1):\n",
+        "                src2tgt[i1 + k] = j1 + k\n",
+        "        elif tag in ('replace', 'delete'):\n",
+        "            for k in range(i2 - i1):\n",
+        "                src2tgt[i1 + k] = j1\n",
+        "        elif tag == 'insert':\n",
+        "            pass\n",
+        "    last = 0\n",
+        "    for i in range(len(src2tgt)):\n",
+        "        if src2tgt[i] == -1:\n",
+        "            src2tgt[i] = last\n",
+        "        else:\n",
+        "            last = src2tgt[i]\n",
+        "    return src2tgt\n",
+        "\n",
+        "def map_span_src_to_tgt(src2tgt, start, end, tgt_len):\n",
+        "    if start >= len(src2tgt): start = max(0, len(src2tgt)-1)\n",
+        "    if end == 0: end = 1\n",
+        "    if end-1 >= len(src2tgt): end = len(src2tgt)\n",
+        "    ts = src2tgt[start]\n",
+        "    te = src2tgt[end-1] + 1\n",
+        "    ts = max(0, min(ts, max(0, tgt_len-1)))\n",
+        "    te = max(ts+1, min(te, tgt_len))\n",
+        "    return ts, te\n",
+        "\n",
+        "def token_indices_overlapping_span(offsets, span_start, span_end):\n",
+        "    idxs = []\n",
+        "    for i, (s, e) in enumerate(offsets):\n",
+        "        if e > span_start and s < span_end:\n",
+        "            idxs.append(i)\n",
+        "    return idxs\n",
+        "\n",
+        "def electra_hidden_states(prep_text):\n",
+        "    encoded = tokenizer_electra(prep_text, return_tensors=\"pt\", return_offsets_mapping=True, padding=False, truncation=True)\n",
+        "    offsets = encoded.pop('offset_mapping')[0].tolist()\n",
+        "    with torch.no_grad():\n",
+        "        out = model_electra(**encoded)\n",
+        "    H = out.last_hidden_state.squeeze(0)\n",
+        "    return offsets, H\n",
+        "\n",
+        "def word_span_list_from_stanza(doc):\n",
+        "    spans = []\n",
+        "    for si, sent in enumerate(doc.sentences):\n",
+        "        for ti, tok in enumerate(sent.tokens):\n",
+        "            for w in tok.words:\n",
+        "                spans.append({\n",
+        "                    \"text\": w.text,\n",
+        "                    \"start\": tok.start_char,\n",
+        "                    \"end\": tok.end_char,\n",
+        "                    \"upos\": w.upos,\n",
+        "                    \"feats\": getattr(w, \"feats\", None),\n",
+        "                    \"deprel\": w.deprel,\n",
+        "                    \"head\": w.head,\n",
+        "                    \"sent_idx\": si,\n",
+        "                    \"tok_idx\": ti\n",
+        "                })\n",
+        "    return spans\n",
+        "\n",
+        "def electra_phrase_vec_via_offsets(span_start, span_end, src2tgt, prep_text, offsets, H):\n",
+        "    ts, te = map_span_src_to_tgt(src2tgt, span_start, span_end, len(prep_text))\n",
+        "    tok_ids = token_indices_overlapping_span(offsets, ts, te)\n",
+        "    if not tok_ids:\n",
+        "        return None\n",
+        "    vecs = [H[i] for i in tok_ids]\n",
+        "    return torch.stack(vecs, dim=0).mean(dim=0)\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 36,
+      "id": "c6a6d4ff",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "c6a6d4ff",
+        "outputId": "a132a016-6217-4050-f798-5d8d501b48db"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "\n",
+            "=== Sentence 1 ===\n",
+            "Word: يتشكل           UPOS: VERB   Dep: root      Head: ROOT         Feats: Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Person=3|VerbForm=Fin|Voice=Act\n",
+            "Word: الماضي          UPOS: ADJ    Dep: nsubj     Head: يتشكل        Feats: Case=Nom|Definite=Def|Gender=Masc|Number=Sing\n",
+            "Word: غمد             UPOS: NOUN   Dep: nsubj     Head: يتشكل        Feats: Case=Nom|Definite=Cons|Number=Sing\n",
+            "Word: النخاعين        UPOS: NOUN   Dep: nmod      Head: غمد          Feats: Case=Gen|Definite=Def|Number=Dual\n",
+            "Word: في              UPOS: ADP    Dep: case      Head: الجهاز       Feats: AdpType=Prep\n",
+            "Word: الجهاز          UPOS: NOUN   Dep: nmod      Head: غمد          Feats: Case=Gen|Definite=Def|Number=Sing\n",
+            "Word: العصبي          UPOS: ADJ    Dep: amod      Head: الجهاز       Feats: Case=Gen|Definite=Def|Gender=Masc|Number=Sing\n",
+            "Word: المركزي         UPOS: ADJ    Dep: amod      Head: الجهاز       Feats: Case=Gen|Definite=Def|Gender=Masc|Number=Sing\n",
+            "Word: بدءا            UPOS: NOUN   Dep: obl       Head: يتشكل        Feats: Case=Acc|Definite=Ind|Number=Sing\n",
+            "Word: من              UPOS: ADP    Dep: fixed     Head: بدءا         Feats: AdpType=Prep\n",
+            "Word: خلايا           UPOS: NOUN   Dep: nmod      Head: بدءا         Feats: Case=Gen|Definite=Cons|Number=Plur\n",
+            "Word: الدبق           UPOS: NOUN   Dep: nmod      Head: خلايا        Feats: Case=Gen|Definite=Def|Number=Sing\n",
+            "Word: قليلة           UPOS: ADJ    Dep: amod      Head: الدبق        Feats: Case=Gen|Definite=Cons|Gender=Fem|Number=Sing\n",
+            "Word: الاستطالات      UPOS: NOUN   Dep: nmod      Head: قليلة        Feats: Case=Gen|Definite=Def|Number=Plur\n",
+            "Word: و               UPOS: CCONJ  Dep: cc        Head: الجهاز       Feats: _\n",
+            "Word: في              UPOS: ADP    Dep: case      Head: الجهاز       Feats: AdpType=Prep\n",
+            "Word: الجهاز          UPOS: NOUN   Dep: conj      Head: خلايا        Feats: Case=Gen|Definite=Def|Number=Sing\n",
+            "Word: العصبي          UPOS: ADJ    Dep: amod      Head: الجهاز       Feats: Case=Gen|Definite=Def|Gender=Masc|Number=Sing\n",
+            "Word: المحيطي         UPOS: ADJ    Dep: amod      Head: الجهاز       Feats: Case=Gen|Definite=Def|Gender=Masc|Number=Sing\n",
+            "Word: من              UPOS: ADP    Dep: case      Head: خلايا        Feats: AdpType=Prep\n",
+            "Word: خلايا           UPOS: X      Dep: obl       Head: المحيطي      Feats: _\n",
+            "Word: شوان            UPOS: X      Dep: nmod      Head: خلايا        Feats: _\n"
+          ]
+        }
+      ],
+      "source": [
+        "doc = nlp(text_norm)\n",
+        "for si, sentence in enumerate(doc.sentences, start=1):\n",
+        "    print(f\"\\n=== Sentence {si} ===\")\n",
+        "    for w in sentence.words:\n",
+        "        feats = w.feats if w.feats else \"_\"\n",
+        "        head_text = sentence.words[w.head-1].text if w.head and w.head-1 < len(sentence.words) else \"ROOT\"\n",
+        "        print(f\"Word: {w.text:<15} UPOS: {w.upos:<6} Dep: {w.deprel:<9} Head: {head_text:<12} Feats: {feats}\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 37,
+      "id": "bbde14e6",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "bbde14e6",
+        "outputId": "05fb00d4-262c-4c1e-acb2-c6c1d347ce2d"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Number of nominal phrases : 6\n",
+            " 1. غمد النخاعين في الجهاز  (span=6:28, head=غمد)\n",
+            " 2. الجهاز العصبي المركزي  (span=22:43, head=الجهاز)\n",
+            " 3. بدءا من خلايا  (span=44:57, head=بدءا)\n",
+            " 4. خلايا الدبق  (span=52:63, head=خلايا)\n",
+            " 5. الدبق قليلة  (span=58:69, head=الدبق)\n",
+            " 6. الجهاز العصبي المحيطي  (span=85:106, head=الجهاز)\n"
+          ]
+        }
+      ],
+      "source": [
+        "def build_noun_phrases(doc, text_norm):\n",
+        "    noun_phrases = []\n",
+        "    for si, sent in enumerate(doc.sentences):\n",
+        "        words_info = []\n",
+        "        for ti, tok in enumerate(sent.tokens):\n",
+        "            for w in tok.words:\n",
+        "                words_info.append({\n",
+        "                    \"id\": w.id,\n",
+        "                    \"text\": w.text,\n",
+        "                    \"upos\": w.upos,\n",
+        "                    \"deprel\": w.deprel,\n",
+        "                    \"head\": w.head,\n",
+        "                    \"start\": tok.start_char,\n",
+        "                    \"end\": tok.end_char,\n",
+        "                    \"tok_idx\": ti\n",
+        "                })\n",
+        "        id2info = {wi[\"id\"]: wi for wi in words_info}\n",
+        "\n",
+        "        for wi in words_info:\n",
+        "            if wi[\"upos\"] not in {\"NOUN\",\"PROPN\"}:\n",
+        "                continue\n",
+        "            head = wi\n",
+        "            left_mods, right_mods = [], []\n",
+        "            for cj in words_info:\n",
+        "                if cj[\"head\"] == head[\"id\"] and cj[\"deprel\"] in {\"amod\",\"compound\",\"nmod\"}:\n",
+        "                    if cj[\"start\"] <= head[\"start\"]:\n",
+        "                        left_mods.append(cj)\n",
+        "                    else:\n",
+        "                        right_mods.append(cj)\n",
+        "            left_mods  = sorted(left_mods,  key=lambda x: x[\"start\"])\n",
+        "            right_mods = sorted(right_mods, key=lambda x: x[\"start\"])\n",
+        "            phrase_tokens = left_mods + [head] + right_mods\n",
+        "            if not phrase_tokens:\n",
+        "                continue\n",
+        "            if len(phrase_tokens) < 2 and head[\"upos\"] != \"PROPN\":\n",
+        "                continue\n",
+        "            span_start = min(t[\"start\"] for t in phrase_tokens)\n",
+        "            span_end   = max(t[\"end\"]   for t in phrase_tokens)\n",
+        "            phrase_text = text_norm[span_start:span_end].strip()\n",
+        "            phrase_text = re.sub(r\"\\s+\",\" \", phrase_text)\n",
+        "            if len(phrase_text) < 2:\n",
+        "                continue\n",
+        "            noun_phrases.append({\n",
+        "                \"text\": phrase_text,\n",
+        "                \"start\": span_start,\n",
+        "                \"end\": span_end,\n",
+        "                \"head_text\": head[\"text\"],\n",
+        "                \"sent_idx\": si,\n",
+        "                \"token_indices\": [t[\"tok_idx\"] for t in phrase_tokens]\n",
+        "            })\n",
+        "    uniq = {}\n",
+        "    for np_item in noun_phrases:\n",
+        "        key = np_item[\"text\"]\n",
+        "        if key not in uniq or (np_item[\"end\"] - np_item[\"start\"]) > (uniq[key][\"end\"] - uniq[key][\"start\"]):\n",
+        "            uniq[key] = np_item\n",
+        "    return list(uniq.values())\n",
+        "\n",
+        "nps = build_noun_phrases(doc, text_norm)\n",
+        "print(\"Number of nominal phrases :\", len(nps))\n",
+        "for i, p in enumerate(nps[:20], 1):\n",
+        "    print(f\"{i:>2}. {p['text']}  (span={p['start']}:{p['end']}, head={p['head_text']})\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 38,
+      "id": "47771183",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "47771183",
+        "outputId": "7cf0361c-d250-4c85-bcb2-07856fb752a4"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "MMR selection (various): ['الجهاز العصبي المركزي', 'الجهاز العصبي المحيطي', 'بدءا من خلايا', 'خلايا الدبق', 'غمد النخاعين في الجهاز', 'الدبق قليلة']\n",
+            "\n",
+            "Top 15 by blended (phrase, blended, sBERT, ELECTRA_ctx):\n",
+            "الجهاز العصبي المركزي                    -> 0.8126 | sBERT=0.7804 | ELECTRA=0.9415\n",
+            "الجهاز العصبي المحيطي                    -> 0.7930 | sBERT=0.7494 | ELECTRA=0.9674\n",
+            "خلايا الدبق                              -> 0.6421 | sBERT=0.5611 | ELECTRA=0.9663\n",
+            "بدءا من خلايا                            -> 0.6291 | sBERT=0.5483 | ELECTRA=0.9524\n",
+            "الدبق قليلة                              -> 0.3127 | sBERT=0.1528 | ELECTRA=0.9525\n",
+            "غمد النخاعين في الجهاز                   -> 0.2822 | sBERT=0.1064 | ELECTRA=0.9855\n"
+          ]
+        }
+      ],
+      "source": [
+        "def mmr_select(doc_emb, cand_embs, candidates, k=10, lam=0.7):\n",
+        "    if len(candidates) == 0: return []\n",
+        "    chosen_idx, cand_idx = [], list(range(len(candidates)))\n",
+        "    sim_doc = util.cos_sim(doc_emb, cand_embs)[0]\n",
+        "    first = int(np.argmax(sim_doc.cpu().numpy()))\n",
+        "    chosen_idx.append(first); cand_idx.remove(first)\n",
+        "    if len(candidates) == 1 or k == 1:\n",
+        "        return [candidates[first]]\n",
+        "    sim_between = util.cos_sim(cand_embs, cand_embs)\n",
+        "    for _ in range(min(k, len(candidates)) - 1):\n",
+        "        best_i, best_score = None, -1e9\n",
+        "        for i in cand_idx:\n",
+        "            redundancy = max(sim_between[i, j].item() for j in chosen_idx) if chosen_idx else 0.0\n",
+        "            score = lam*sim_doc[i].item() - (1-lam)*redundancy\n",
+        "            if score > best_score:\n",
+        "                best_score, best_i = score, i\n",
+        "        chosen_idx.append(best_i); cand_idx.remove(best_i)\n",
+        "    return [candidates[i] for i in chosen_idx]\n",
+        "\n",
+        "def rank_keyphrases_with_mmr(text_norm, nps, arabert_prep, sbert, TOP_K=12, alpha=0.8, lam=0.7):\n",
+        "    if not nps: return [], []\n",
+        "    phrases = [p[\"text\"] for p in nps]\n",
+        "    text_prep = arabert_prep.preprocess(text_norm)\n",
+        "    src2tgt = build_char_map(text_norm, text_prep)\n",
+        "    # sBERT\n",
+        "    doc_emb_sbert  = sbert.encode([text_prep], convert_to_tensor=True)\n",
+        "    phr_embs_sbert = sbert.encode(phrases, convert_to_tensor=True)\n",
+        "    sims_sbert = util.cos_sim(doc_emb_sbert, phr_embs_sbert).cpu().numpy()[0]\n",
+        "    # ELECTRA doc vec\n",
+        "    prep_offsets, prep_H = electra_hidden_states(text_prep)\n",
+        "    with torch.no_grad():\n",
+        "        doc_vec_electra = prep_H.mean(dim=0)\n",
+        "    # ELECTRA phrase sims via span\n",
+        "    sims_electra = []\n",
+        "    for p in nps:\n",
+        "        v = electra_phrase_vec_via_offsets(p[\"start\"], p[\"end\"], src2tgt, text_prep, prep_offsets, prep_H)\n",
+        "        if v is None:\n",
+        "            sims_electra.append(0.0)\n",
+        "        else:\n",
+        "            num = torch.dot(doc_vec_electra, v).item()\n",
+        "            den = (doc_vec_electra.norm().item() * v.norm().item() + 1e-9)\n",
+        "            sims_electra.append(num/den)\n",
+        "    sims_electra = np.array(sims_electra)\n",
+        "    final_scores = alpha * sims_sbert + (1 - alpha) * sims_electra\n",
+        "    order = np.argsort(-final_scores)\n",
+        "    ranked = [(phrases[i], float(final_scores[i]), float(sims_sbert[i]), float(sims_electra[i])) for i in order]\n",
+        "    top_diverse = mmr_select(doc_emb_sbert, phr_embs_sbert, phrases, k=min(TOP_K, len(phrases)), lam=lam)\n",
+        "    return ranked, top_diverse\n",
+        "\n",
+        "ranked, top_diverse = rank_keyphrases_with_mmr(text_norm, nps, arabert_prep, sbert, TOP_K=12, alpha=0.8, lam=0.7)\n",
+        "print(\"MMR selection (various):\", top_diverse[:10])\n",
+        "\n",
+        "print(\"\\nTop 15 by blended (phrase, blended, sBERT, ELECTRA_ctx):\")\n",
+        "for phr, sc, sb, el in ranked[:15]:\n",
+        "    print(f\"{phr:<40s} -> {sc:.4f} | sBERT={sb:.4f} | ELECTRA={el:.4f}\")\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 39,
+      "id": "7e47f31f",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "7e47f31f",
+        "outputId": "6235fb61-57de-4f36-849d-c900ee119610"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Top 15 (Blended SEM+YAKE):\n",
+            "الجهاز العصبي المركزي                    -> final=0.8688 | sem=0.8126 | yake=1.0000 | sBERT=0.7804 | ELECTRA=0.9415\n",
+            "الجهاز العصبي المحيطي                    -> final=0.8551 | sem=0.7930 | yake=1.0000 | sBERT=0.7494 | ELECTRA=0.9674\n",
+            "خلايا الدبق                              -> final=0.4495 | sem=0.6421 | yake=0.0000 | sBERT=0.5611 | ELECTRA=0.9663\n",
+            "بدءا من خلايا                            -> final=0.4404 | sem=0.6291 | yake=0.0000 | sBERT=0.5483 | ELECTRA=0.9524\n",
+            "غمد النخاعين في الجهاز                   -> final=0.4177 | sem=0.2822 | yake=0.7339 | sBERT=0.1064 | ELECTRA=0.9855\n",
+            "الدبق قليلة                              -> final=0.2565 | sem=0.3127 | yake=0.1254 | sBERT=0.1528 | ELECTRA=0.9525\n"
+          ]
+        }
+      ],
+      "source": [
+        "def yake_scores_for_phrases(text_norm, phrases, max_ngram_size=5, lan='ar'):\n",
+        "    kw_extractor = yake.KeywordExtractor(lan=lan, n=max_ngram_size, dedupLim=0.9, top=1000)\n",
+        "    scored = kw_extractor.extract_keywords(text_norm)\n",
+        "    norm = lambda s: re.sub(r\"\\s+\",\" \", s).strip().lower()\n",
+        "    scored_norm = {norm(k): v for k, v in scored}\n",
+        "    score_map = {}\n",
+        "    for p in phrases:\n",
+        "        pn = norm(p)\n",
+        "        score_map[p] = scored_norm.get(pn, None)\n",
+        "    return score_map\n",
+        "\n",
+        "def invert_and_minmax_yake(score_map):\n",
+        "    vals = []\n",
+        "    for v in score_map.values():\n",
+        "        vals.append(None if v is None else 1.0/(1.0+v))\n",
+        "    finite_vals = [x for x in vals if x is not None]\n",
+        "    if not finite_vals:\n",
+        "        return {k: 0.0 for k in score_map.keys()}\n",
+        "    vmin, vmax = min(finite_vals), max(finite_vals)\n",
+        "    rng = (vmax - vmin) if vmax > vmin else 1.0\n",
+        "    out = {}\n",
+        "    for (k, v), pos in zip(score_map.items(), vals):\n",
+        "        out[k] = 0.0 if pos is None else (pos - vmin)/rng\n",
+        "    return out\n",
+        "\n",
+        "def blend_semantic_with_yake(ranked_semantic, yake_norm_map, w_sem=0.7, w_yake=0.3):\n",
+        "    merged = []\n",
+        "    for phr, sem_sc, sb, el in ranked_semantic:\n",
+        "        y = yake_norm_map.get(phr, 0.0)\n",
+        "        final = w_sem*sem_sc + w_yake*y\n",
+        "        merged.append((phr, final, sem_sc, y, sb, el))\n",
+        "    merged.sort(key=lambda x: -x[1])\n",
+        "    return merged\n",
+        "\n",
+        "phrases = [r[0] for r in ranked]\n",
+        "yake_raw  = yake_scores_for_phrases(text_norm, phrases, max_ngram_size=5, lan='ar')\n",
+        "yake_norm = invert_and_minmax_yake(yake_raw)\n",
+        "\n",
+        "ranked_blended = blend_semantic_with_yake(ranked, yake_norm, w_sem=0.7, w_yake=0.3)\n",
+        "\n",
+        "print(\"Top 15 (Blended SEM+YAKE):\")\n",
+        "for phr, final, sem_sc, yake_sc, sb, el in ranked_blended[:15]:\n",
+        "    print(f\"{phr:<40s} -> final={final:.4f} | sem={sem_sc:.4f} | yake={yake_sc:.4f} | sBERT={sb:.4f} | ELECTRA={el:.4f}\")\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "def split_by_dots(text: str):\n",
+        "    parts = re.split(r\"\\.{1,}\\s*\", text)\n",
+        "    sentences = [p.strip() for p in parts if p.strip()]\n",
+        "    return sentences\n",
+        "\n",
+        "def sentence_kind_from_root(stanza_sentence):\n",
+        "    root = next((w for w in stanza_sentence.words if w.deprel == \"root\"), None)\n",
+        "    if not root:\n",
+        "        return \"unknown\"\n",
+        "    return \"verbal\" if root.upos == \"VERB\" else \"nominal\"\n",
+        "\n",
+        "def split_and_tag_nominal_verbal_by_dots(text_norm, nlp):\n",
+        "    sents = split_by_dots(text_norm)\n",
+        "    tagged = []\n",
+        "    for s in sents:\n",
+        "        doc_s = nlp(s)\n",
+        "        if not doc_s.sentences:\n",
+        "            tagged.append({\"text\": s, \"kind\": \"unknown\"})\n",
+        "            continue\n",
+        "        kind = sentence_kind_from_root(doc_s.sentences[0])\n",
+        "        tagged.append({\"text\": s, \"kind\": kind})\n",
+        "    return tagged\n",
+        "def link_phrases_to_sentences_by_dots(text_norm, phrases, nlp, sbert, top_k_per_phrase=2):\n",
+        "    sentences_tagged = split_and_tag_nominal_verbal_by_dots(text_norm, nlp)\n",
+        "    if not sentences_tagged:\n",
+        "        return [], {p: [] for p in phrases}\n",
+        "\n",
+        "    sent_texts = [m[\"text\"] for m in sentences_tagged]\n",
+        "    sent_embs  = sbert.encode(sent_texts, convert_to_tensor=True)\n",
+        "\n",
+        "    phrase_links = {}\n",
+        "    for p in phrases:\n",
+        "        p_emb = sbert.encode([p], convert_to_tensor=True)\n",
+        "        sims = util.cos_sim(p_emb, sent_embs)[0].cpu().numpy()\n",
+        "        order = np.argsort(-sims)\n",
+        "        links = []\n",
+        "        for idx in order[:min(top_k_per_phrase, len(order))]:\n",
+        "            links.append({\n",
+        "                \"sent\": sent_texts[idx],\n",
+        "                \"sim\": float(sims[idx]),\n",
+        "                \"kind\": sentences_tagged[idx][\"kind\"]\n",
+        "            })\n",
+        "        phrase_links[p] = links\n",
+        "\n",
+        "    return sentences_tagged, phrase_links\n",
+        "\n",
+        "\n",
+        "tagged_sents = split_and_tag_nominal_verbal_by_dots(text_norm, nlp)\n",
+        "\n",
+        "print(\"\\nSentences (divided by points only) and their classification:\")\n",
+        "for i, it in enumerate(tagged_sents, 1):\n",
+        "    print(f\"{i:>2}. ({it['kind']}) {it['text']}\")\n",
+        "\n",
+        "topK_for_support = 1\n",
+        "phr_top = [x[0] for x in ranked_blended[:5]]\n",
+        "sentences_tagged, phrase_links = link_phrases_to_sentences_by_dots(\n",
+        "    text_norm, phr_top, nlp, sbert, top_k_per_phrase=topK_for_support\n",
+        ")\n",
+        "\n",
+        "print(\"\\nLinking phrases to supporting sentences (highest similarity):\")\n",
+        "for p in phr_top:\n",
+        "    print(f\"- عبارة: {p}\")\n",
+        "    for l in phrase_links.get(p, []):\n",
+        "        print(f\"   • ({l['kind']}) sim={l['sim']:.3f} | {l['sent']}\")"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "msca6UdcjU09",
+        "outputId": "39e28ae0-d184-455d-c2f2-b014bbb3c5ee"
+      },
+      "id": "msca6UdcjU09",
+      "execution_count": 40,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "\n",
+            "Sentences (divided by points only) and their classification:\n",
+            " 1. (verbal) يتشكل غمد النخاعين في الجهاز العصبي المركزي بدءا من خلايا الدبق قليلة الاستطالات وفي الجهاز العصبي المحيطي من خلايا شوان\n",
+            "\n",
+            "Linking phrases to supporting sentences (highest similarity):\n",
+            "- عبارة: الجهاز العصبي المركزي\n",
+            "   • (verbal) sim=0.780 | يتشكل غمد النخاعين في الجهاز العصبي المركزي بدءا من خلايا الدبق قليلة الاستطالات وفي الجهاز العصبي المحيطي من خلايا شوان\n",
+            "- عبارة: الجهاز العصبي المحيطي\n",
+            "   • (verbal) sim=0.749 | يتشكل غمد النخاعين في الجهاز العصبي المركزي بدءا من خلايا الدبق قليلة الاستطالات وفي الجهاز العصبي المحيطي من خلايا شوان\n",
+            "- عبارة: خلايا الدبق\n",
+            "   • (verbal) sim=0.561 | يتشكل غمد النخاعين في الجهاز العصبي المركزي بدءا من خلايا الدبق قليلة الاستطالات وفي الجهاز العصبي المحيطي من خلايا شوان\n",
+            "- عبارة: بدءا من خلايا\n",
+            "   • (verbal) sim=0.548 | يتشكل غمد النخاعين في الجهاز العصبي المركزي بدءا من خلايا الدبق قليلة الاستطالات وفي الجهاز العصبي المحيطي من خلايا شوان\n",
+            "- عبارة: غمد النخاعين في الجهاز\n",
+            "   • (verbal) sim=0.106 | يتشكل غمد النخاعين في الجهاز العصبي المركزي بدءا من خلايا الدبق قليلة الاستطالات وفي الجهاز العصبي المحيطي من خلايا شوان\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "def gen_unified_question_freeform(phrases, supports, context_text, max_len=96, num_beams=5):\n",
+        "    context_short = context_text.strip()[:600]\n",
+        "    items_block = \"\\n\".join(\n",
+        "        [f\"- العبارة: {p}\\n  جملة داعمة: {s}\" for p, s in zip(phrases, supports)]\n",
+        "    )\n",
+        "    prompt = (\n",
+        "        \"حوّل العبارات التالية إلى سؤال واحد شامل بالعربية يعتمد على السياق. \"\n",
+        "        \"يجب أن يغطي جميع العبارات بشكل موجز وواضح.\\n\"\n",
+        "        f\"{items_block}\\n\"\n",
+        "        f\"سياق: {context_short}\\n\"\n",
+        "        \"السؤال الموحد:\"\n",
+        "    )\n",
+        "\n",
+        "    inputs = qg_tokenizer(prompt, return_tensors=\"pt\", truncation=True).to(device)\n",
+        "    outputs = qg_model.generate(\n",
+        "        **inputs,\n",
+        "        max_length=max_len,\n",
+        "        num_beams=num_beams,\n",
+        "        early_stopping=True,\n",
+        "        no_repeat_ngram_size=3\n",
+        "    )\n",
+        "    q = qg_tokenizer.decode(outputs[0], skip_special_tokens=True).strip()\n",
+        "    q = q.rstrip(\"?.؟\")\n",
+        "    if q and not q.endswith(\"؟\"):\n",
+        "        q += \"؟\"\n",
+        "    return q\n",
+        "\n",
+        "def unified_question_from_top5_phrases(text_norm, ranked_blended, nlp, sbert, top_k=5):\n",
+        "    if not ranked_blended:\n",
+        "        print(\"لا توجد عبارات.\")\n",
+        "        return {\"phrases\": [], \"supports\": [], \"question\": \"\"}\n",
+        "    top_n = min(top_k, len(ranked_blended))\n",
+        "    phrases = [ranked_blended[i][0] for i in range(top_n)]\n",
+        "\n",
+        "    supports = []\n",
+        "    for p in phrases:\n",
+        "        s = best_support_sentence_by_dots(text_norm, p, nlp, sbert)\n",
+        "        supports.append(s)\n",
+        "\n",
+        "    unified_q = gen_unified_question_freeform(phrases, supports, context_text=text_norm)\n",
+        "    print(\"The context : :\\n\", text_norm, \"\\n\")\n",
+        "    print(\"The selected phrase (Top):\")\n",
+        "    for i, p in enumerate(phrases, 1):\n",
+        "        print(f\"{i}. {p}\")\n",
+        "    print(\"\\The supporting sentences :\")\n",
+        "    for i, s in enumerate(supports, 1):\n",
+        "        print(f\"{i}. {s}\")\n",
+        "    print(\"\\nUnified Generated Question:\")\n",
+        "    print(unified_q)\n",
+        "\n",
+        "    return {\"phrases\": phrases, \"supports\": supports, \"question\": unified_q}\n",
+        "unified_result = unified_question_from_top5_phrases(text_norm, ranked_blended, nlp, sbert, top_k=5)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "mGtgQgSAwv8H",
+        "outputId": "a901de5f-ba37-49ee-8eb9-15750e1b3ce8"
+      },
+      "id": "mGtgQgSAwv8H",
+      "execution_count": 42,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "السياق:\n",
+            " يتشكل غمد النخاعين في الجهاز العصبي المركزي بدءا من خلايا الدبق قليلة الاستطالات وفي الجهاز العصبي المحيطي من خلايا شوان \n",
+            "\n",
+            "العبارات المختارة (Top):\n",
+            "1. الجهاز العصبي المركزي\n",
+            "2. الجهاز العصبي المحيطي\n",
+            "3. خلايا الدبق\n",
+            "4. بدءا من خلايا\n",
+            "5. غمد النخاعين في الجهاز\n",
+            "\n",
+            "الجمل الداعمة:\n",
+            "1. يتشكل غمد النخاعين في الجهاز العصبي المركزي بدءا من خلايا الدبق قليلة الاستطالات وفي الجهاز العصبي المحيطي من خلايا شوان\n",
+            "2. يتشكل غمد النخاعين في الجهاز العصبي المركزي بدءا من خلايا الدبق قليلة الاستطالات وفي الجهاز العصبي المحيطي من خلايا شوان\n",
+            "3. يتشكل غمد النخاعين في الجهاز العصبي المركزي بدءا من خلايا الدبق قليلة الاستطالات وفي الجهاز العصبي المحيطي من خلايا شوان\n",
+            "4. يتشكل غمد النخاعين في الجهاز العصبي المركزي بدءا من خلايا الدبق قليلة الاستطالات وفي الجهاز العصبي المحيطي من خلايا شوان\n",
+            "5. يتشكل غمد النخاعين في الجهاز العصبي المركزي بدءا من خلايا الدبق قليلة الاستطالات وفي الجهاز العصبي المحيطي من خلايا شوان\n",
+            "\n",
+            "السؤال الموحد المولّد:\n",
+            "question: من أين يتكون غمد النخاعين؟\n"
+          ]
+        }
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 5
+}