Upload Skewness_paper_last.ipynb

Skewness_paper_last.ipynb

@@ -0,0 +1,1340 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "gpuType": "T4"
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU"
+  },
+  "cells": [
+    {
+      "cell_type": "code",
+      "source": [
+        "!pip install -q x-transformers"
+      ],
+      "metadata": {
+        "id": "tBfQX92lxEfP"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title 🛠️ Setup & Model Loading\n",
+        "# ==============================================================================\n",
+        "# 1. INSTALL DEPENDENCIES\n",
+        "# ==============================================================================\n",
+        "!pip install -q numpy torch pandas scipy transformers huggingface_hub\n",
+        "\n",
+        "import torch\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "from scipy.stats import skew\n",
+        "import sys\n",
+        "import os\n",
+        "from huggingface_hub import hf_hub_download\n",
+        "from transformers import AutoTokenizer\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 2. LOAD PRISM ARCHITECTURE\n",
+        "# ==============================================================================\n",
+        "REPO_ID = \"prism-lab/prism-shimmer-100k\"\n",
+        "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+        "\n",
+        "print(f\"⚙️ Hardware: {DEVICE}\")\n",
+        "print(f\"📥 Downloading Architecture from {REPO_ID}...\")\n",
+        "\n",
+        "# Download the model code to a local folder\n",
+        "os.makedirs(\"shimmer_code\", exist_ok=True)\n",
+        "hf_hub_download(repo_id=REPO_ID, filename=\"modeling_prism_gated.py\", local_dir=\"shimmer_code\")\n",
+        "sys.path.append(\"shimmer_code\")\n",
+        "\n",
+        "# Now we can import the class\n",
+        "from modeling_prism_gated import PRISMHybrid_RoPE\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 3. LOAD WEIGHTS\n",
+        "# ==============================================================================\n",
+        "print(\"📚 Loading Tokenizer...\")\n",
+        "tokenizer = AutoTokenizer.from_pretrained(REPO_ID)\n",
+        "\n",
+        "print(\"🏗️ Constructing PRISM Model...\")\n",
+        "CONFIG = {\n",
+        "    \"vocab_size\": 58101,\n",
+        "    \"d_model\": 512,\n",
+        "    \"num_heads\": 8,\n",
+        "    \"dff\": 2048,\n",
+        "    \"dropout\": 0.1,\n",
+        "    \"max_length\": 128,\n",
+        "    \"num_encoder_layers\": 6,\n",
+        "    \"num_refining_layers\": 0,\n",
+        "    \"num_decoder_layers\": 6\n",
+        "}\n",
+        "model = PRISMHybrid_RoPE(**CONFIG)\n",
+        "\n",
+        "print(\"📥 Loading Checkpoint...\")\n",
+        "weights_path = hf_hub_download(repo_id=REPO_ID, filename=\"pytorch_model.bin\")\n",
+        "state_dict = torch.load(weights_path, map_location=DEVICE)\n",
+        "model.load_state_dict(state_dict)\n",
+        "model.to(DEVICE)\n",
+        "model.eval()\n",
+        "\n",
+        "print(\"✅ Model Ready for Probing.\")"
+      ],
+      "metadata": {
+        "id": "fn7A70MZxV1U"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title 🧪 The Probe & Datasets\n",
+        "# ==============================================================================\n",
+        "# 1. DATASETS (The \"76 + 70\" Split)\n",
+        "# ==============================================================================\n",
+        "\n",
+        "# A. HARD MODE (Polysemous / Ambiguous)\n",
+        "# Words that require context to resolve (High Entropy)\n",
+        "raw_poly_candidates = [\n",
+        "    # --- ORIGINAL SET ---\n",
+        "    (\"Ich gehe zur Bank um Geld zu holen\", \"Bank\"), (\"Die Bank hat hohe Zinsen\", \"Bank\"),\n",
+        "    (\"Wir saßen auf einer Bank im Park\", \"Bank\"), (\"Die Bank aus Holz war bequem\", \"Bank\"),\n",
+        "    (\"Das Schloss hat viele Türme\", \"Schloss\"), (\"Der König wohnt im Schloss\", \"Schloss\"),\n",
+        "    (\"Der Schlüssel steckt im Schloss\", \"Schloss\"), (\"Das Schloss an der Tür klemmt\", \"Schloss\"),\n",
+        "    (\"Der Leiter der Firma ist streng\", \"Leiter\"), (\"Unser Leiter plant das Projekt\", \"Leiter\"),\n",
+        "    (\"Ich steige auf die Leiter\", \"Leiter\"), (\"Die Leiter ist aus Aluminium\", \"Leiter\"),\n",
+        "    (\"Die Lampe hängt an der Decke\", \"Decke\"), (\"Die Decke ist weiß gestrichen\", \"Decke\"),\n",
+        "    (\"Mir ist kalt gib mir eine Decke\", \"Decke\"), (\"Die Decke aus Wolle ist warm\", \"Decke\"),\n",
+        "    (\"Der Kiefer ist ein Nadelbaum\", \"Kiefer\"), (\"Das Holz der Kiefer ist weich\", \"Kiefer\"),\n",
+        "    (\"Der Arzt röntgt meinen Kiefer\", \"Kiefer\"), (\"Er hat Schmerzen im Kiefer\", \"Kiefer\"),\n",
+        "    (\"Der Strauß ist ein schneller Vogel\", \"Strauß\"), (\"Dieser Strauß kann nicht fliegen\", \"Strauß\"),\n",
+        "    (\"Sie kaufte einen bunten Strauß\", \"Strauß\"), (\"Der Strauß Blumen duftet gut\", \"Strauß\"),\n",
+        "    (\"Er schoss ein schönes Tor\", \"Tor\"), (\"Der Ball flog ins Tor\", \"Tor\"),\n",
+        "    (\"Das eiserne Tor war verschlossen\", \"Tor\"), (\"Sie öffneten das große Tor\", \"Tor\"),\n",
+        "    (\"Wir tanzen auf dem Ball\", \"Ball\"), (\"Der Maskenball war elegant\", \"Ball\"),\n",
+        "    (\"Er warf den Ball weit weg\", \"Ball\"), (\"Der Ball ist rund und rot\", \"Ball\"),\n",
+        "    (\"Die Schlange im Zoo ist giftig\", \"Schlange\"), (\"Die Schlange zischte laut\", \"Schlange\"),\n",
+        "    (\"Wir stehen in einer langen Schlange\", \"Schlange\"), (\"Die Schlange an der Kasse war lang\", \"Schlange\"),\n",
+        "    (\"Der Strom ist ausgefallen\", \"Strom\"), (\"Strom kostet viel Geld\", \"Strom\"),\n",
+        "    (\"Der Strom fließt ins Meer\", \"Strom\"), (\"Wir schwammen gegen den Strom\", \"Strom\"),\n",
+        "    (\"Seine Mutter ist sehr nett\", \"Mutter\"), (\"Die Mutter kocht das Essen\", \"Mutter\"),\n",
+        "    (\"Die Mutter passt auf die Schraube\", \"Mutter\"), (\"Ich brauche eine neue Mutter\", \"Mutter\"),\n",
+        "    (\"Die Birne schmeckt süß\", \"Birne\"), (\"Ich esse gerne eine Birne\", \"Birne\"),\n",
+        "    (\"Die Birne in der Lampe ist kaputt\", \"Birne\"), (\"Wir müssen die Birne wechseln\", \"Birne\"),\n",
+        "    # --- EXPANSION SET ---\n",
+        "    (\"Das Gericht hat ihn verurteilt\", \"Gericht\"), (\"Der Anwalt geht zum Gericht\", \"Gericht\"),\n",
+        "    (\"Mein Lieblingsessen ist ein Gericht aus Reis\", \"Gericht\"), (\"Das Gericht schmeckt sehr salzig\", \"Gericht\"),\n",
+        "    (\"Der Ton war sehr laut\", \"Ton\"), (\"Ich hörte einen hohen Ton\", \"Ton\"),\n",
+        "    (\"Die Vase ist aus Ton\", \"Ton\"), (\"Wir formen Figuren aus Ton\", \"Ton\"),\n",
+        "    (\"Das Blatt fällt vom Baum\", \"Blatt\"), (\"Im Herbst werden die Blätter braun\", \"Blatt\"),\n",
+        "    (\"Ich schreibe auf ein Blatt Papier\", \"Blatt\"), (\"Gib mir bitte ein leeres Blatt\", \"Blatt\"),\n",
+        "    (\"Der Nagel steckt in der Wand\", \"Nagel\"), (\"Ich schlage den Nagel mit dem Hammer\", \"Nagel\"),\n",
+        "    (\"Mein Nagel ist abgebrochen\", \"Nagel\"), (\"Sie lackiert sich den Nagel rot\", \"Nagel\"),\n",
+        "    (\"Die Maus frisst den Käse\", \"Maus\"), (\"Die Katze jagt die Maus\", \"Maus\"),\n",
+        "    (\"Ich klicke mit der Maus\", \"Maus\"), (\"Der Computer braucht eine neue Maus\", \"Maus\"),\n",
+        "    (\"Die Erde dreht sich um die Sonne\", \"Erde\"), (\"Der Astronaut schaut auf die Erde\", \"Erde\"),\n",
+        "    (\"Die Blume braucht frische Erde\", \"Erde\"), (\"Er gräbt ein Loch in die Erde\", \"Erde\"),\n",
+        "    (\"Der Hahn kräht am Morgen\", \"Hahn\"), (\"Der Hahn hat bunte Federn\", \"Hahn\"),\n",
+        "    (\"Der Wasserhahn tropft\", \"Hahn\"), (\"Dreh bitte den Hahn zu\", \"Hahn\"),\n",
+        "    (\"Die Schale der Orange ist bitter\", \"Schale\"), (\"Er wirft die Schale weg\", \"Schale\"),\n",
+        "    (\"Die Schale steht auf dem Tisch\", \"Schale\"), (\"Ich esse Müsli aus der Schale\", \"Schale\"),\n",
+        "    (\"Der Bauer melkt die Kühe\", \"Bauer\"), (\"Der Bauer fährt auf dem Traktor\", \"Bauer\"),\n",
+        "    (\"Ich ziehe den Bauer auf E4\", \"Bauer\"), (\"Der Bauer schlägt den Turm\", \"Bauer\"),\n",
+        "]\n",
+        "\n",
+        "# B. EASY MODE (Casual)\n",
+        "raw_casual_candidates = [\n",
+        "    (\"Die Katze schläft\", \"Katze\"), (\"Der Hund bellt\", \"Hund\"), (\"Das Auto fährt\", \"Auto\"),\n",
+        "    (\"Wasser ist nass\", \"Wasser\"), (\"Das Brot schmeckt gut\", \"Brot\"), (\"Die Sonne scheint\", \"Sonne\"),\n",
+        "    (\"Der Mond leuchtet\", \"Mond\"), (\"Das Buch ist spannend\", \"Buch\"), (\"Der Tisch ist rund\", \"Tisch\"),\n",
+        "    (\"Der Stuhl ist bequem\", \"Stuhl\"), (\"Der Apfel ist rot\", \"Apfel\"), (\"Meine Hand ist kalt\", \"Hand\"),\n",
+        "    (\"Das Herz klopft\", \"Herz\"), (\"Wir haben Zeit\", \"Zeit\"), (\"Geld ist wichtig\", \"Geld\"),\n",
+        "    (\"Musik ist schön\", \"Musik\"), (\"Der Film ist zu Ende\", \"Film\"), (\"Das Spiel beginnt\", \"Spiel\"),\n",
+        "    (\"Die Schule ist aus\", \"Schule\"), (\"Die Stadt ist laut\", \"Stadt\"), (\"Der Fluss fließt\", \"Fluss\"),\n",
+        "    (\"Das Meer ist tief\", \"Meer\"), (\"Kaffee ist schwarz\", \"Kaffee\"), (\"Milch ist weiß\", \"Milch\"),\n",
+        "    (\"Der Bruder lacht\", \"Bruder\"), (\"Die Schwester weint\", \"Schwester\"), (\"Das Haus ist groß\", \"Haus\"),\n",
+        "    (\"Der Garten ist grün\", \"Garten\"), (\"Der Sommer ist heiß\", \"Sommer\"), (\"Der Winter ist kalt\", \"Winter\"),\n",
+        "    (\"Das Fenster ist offen\", \"Fenster\"), (\"Die Tür ist zu\", \"Tür\"), (\"Der Boden ist sauber\", \"Boden\"),\n",
+        "    (\"Die Wand ist weiß\", \"Wand\"), (\"Das Dach ist rot\", \"Dach\"), (\"Der Wald ist dunkel\", \"Wald\"),\n",
+        "    (\"Der Berg ist hoch\", \"Berg\"), (\"Der See ist ruhig\", \"See\"), (\"Das Tier ist wild\", \"Tier\"),\n",
+        "    (\"Der Mensch denkt\", \"Mensch\"), (\"Das Kind spielt\", \"Kind\"), (\"Die Frau arbeitet\", \"Frau\"),\n",
+        "    (\"Der Mann schläft\", \"Mann\"), (\"Das Auge sieht\", \"Auge\"), (\"Das Ohr hört\", \"Ohr\"),\n",
+        "    (\"Die Nase riecht\", \"Nase\"), (\"Der Mund spricht\", \"Mund\"), (\"Der Arm ist stark\", \"Arm\"),\n",
+        "    (\"Das Bein tut weh\", \"Bein\"), (\"Der Fuß ist groß\", \"Fuß\"), (\"Der Tee ist heiß\", \"Tee\"),\n",
+        "    (\"Das Bier ist kalt\", \"Bier\"), (\"Der Wein ist rot\", \"Wein\"), (\"Das Glas ist voll\", \"Glas\"),\n",
+        "    (\"Die Tasse ist leer\", \"Tasse\"), (\"Der Teller ist blau\", \"Teller\"), (\"Die Gabel ist spitz\", \"Gabel\"),\n",
+        "    (\"Der Löffel ist rund\", \"Löffel\"), (\"Das Messer ist scharf\", \"Messer\"), (\"Der Stift schreibt\", \"Stift\"),\n",
+        "    (\"Der Brief ist lang\", \"Brief\"), (\"Das Bild ist schön\", \"Bild\"), (\"Die Uhr tickt\", \"Uhr\"),\n",
+        "    (\"Das Bett ist weich\", \"Bett\"), (\"Der Schrank ist voll\", \"Schrank\"), (\"Das Sofa ist neu\", \"Sofa\"),\n",
+        "    (\"Das Radio spielt\", \"Radio\"), (\"Das Jahr ist um\", \"Jahr\"), (\"Der Tag war lang\", \"Tag\"),\n",
+        "    (\"Die Nacht ist kurz\", \"Nacht\")\n",
+        "]\n",
+        "# ==============================================================================\n",
+        "# 2. HELPER FUNCTIONS\n",
+        "# ==============================================================================\n",
+        "def find_token_index(input_ids, target_word, tokenizer):\n",
+        "    tokens = tokenizer.convert_ids_to_tokens(input_ids)\n",
+        "    for i, t in enumerate(tokens):\n",
+        "        # Clean BPE artifacts\n",
+        "        clean = t.replace('Ġ', '').replace('▁', '').replace(' ', '')\n",
+        "        if target_word.lower() == clean.lower():\n",
+        "            return i\n",
+        "    # Fallback\n",
+        "    for i, t in enumerate(tokens):\n",
+        "        clean = t.replace('Ġ', '').replace('▁', '').replace(' ', '')\n",
+        "        if target_word.lower() in clean.lower():\n",
+        "            return i\n",
+        "    return 1\n",
+        "\n",
+        "def filter_dataset(candidates, tokenizer, label):\n",
+        "    \"\"\"Ensures we only test single-token words to keep phase metrics valid.\"\"\"\n",
+        "    valid_data = []\n",
+        "    rejected_count = 0\n",
+        "    print(f\"\\n🔍 Validating {label} Candidates...\")\n",
+        "\n",
+        "    for context, target in candidates:\n",
+        "        # Check standard and space-prefixed tokenization\n",
+        "        t1 = tokenizer.encode(target, add_special_tokens=False)\n",
+        "        t2 = tokenizer.encode(\" \" + target, add_special_tokens=False)\n",
+        "\n",
+        "        if len(t1) == 1 or len(t2) == 1:\n",
+        "            valid_data.append((context, target))\n",
+        "        else:\n",
+        "            rejected_count += 1\n",
+        "\n",
+        "    print(f\"   ✅ Accepted: {len(valid_data)} examples.\")\n",
+        "    print(f\"   ❌ Rejected: {rejected_count} multi-token words.\")\n",
+        "    return valid_data\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 3. UNIFIED PROBE CLASS\n",
+        "# ==============================================================================\n",
+        "def run_unified_probe(model, tokenizer, dataset, label, device):\n",
+        "    num_layers = len(model.prism_encoder.layers)\n",
+        "    rotation_stats = {i: [] for i in range(num_layers)}\n",
+        "\n",
+        "    # Hooks\n",
+        "    hook_data = {}\n",
+        "\n",
+        "    def physics_hook(layer_idx):\n",
+        "        def hook(module, input, output):\n",
+        "            x, y = input[0].detach(), output.detach()\n",
+        "\n",
+        "            # --- PHASE ROTATION CALCULATION ---\n",
+        "            # 1. Norms\n",
+        "            norm_x = torch.norm(x, p=2, dim=-1)\n",
+        "            norm_y = torch.norm(y, p=2, dim=-1)\n",
+        "\n",
+        "            # 2. Flatten Complex to 2D Real [Batch, Seq, Dim*2]\n",
+        "            # This allows us to calculate geometric angle\n",
+        "            x_f = x.view(x.shape[0], x.shape[1], -1)\n",
+        "            y_f = y.view(y.shape[0], y.shape[1], -1)\n",
+        "\n",
+        "            # 3. Dot Product\n",
+        "            dot = (x_f.real * y_f.real + x_f.imag * y_f.imag).sum(dim=-1)\n",
+        "\n",
+        "            # 4. Angle (Arccos)\n",
+        "            cosine = torch.clamp(dot / (norm_x * norm_y + 1e-9), -1.0, 1.0)\n",
+        "            angle = torch.rad2deg(torch.acos(cosine)).cpu()\n",
+        "\n",
+        "            hook_data[f'rot_{layer_idx}'] = angle\n",
+        "        return hook\n",
+        "\n",
+        "    # Register\n",
+        "    model.prism_encoder.apply(lambda m: m._forward_hooks.clear())\n",
+        "    for i, layer in enumerate(model.prism_encoder.layers):\n",
+        "        layer.register_forward_hook(physics_hook(i))\n",
+        "\n",
+        "    # Execute\n",
+        "    model.eval()\n",
+        "    print(f\"🔬 Running Probe on {len(dataset)} {label} examples...\")\n",
+        "\n",
+        "    for context, target in dataset:\n",
+        "        hook_data = {}\n",
+        "        inputs = tokenizer(context, return_tensors=\"pt\").to(device)\n",
+        "\n",
+        "        with torch.no_grad():\n",
+        "            x = model.harmonic_embedding(inputs.input_ids)\n",
+        "            src_mask = (inputs.input_ids == tokenizer.pad_token_id)\n",
+        "            model.prism_encoder(x, src_mask)\n",
+        "\n",
+        "        idx = find_token_index(inputs.input_ids[0], target, tokenizer)\n",
+        "\n",
+        "        for i in range(num_layers):\n",
+        "            if f'rot_{i}' in hook_data:\n",
+        "                batch = hook_data[f'rot_{i}']\n",
+        "                # Handle batch dimension if present\n",
+        "                val = batch[0, idx].item() if batch.dim() > 1 else batch[idx].item()\n",
+        "                rotation_stats[i].append(val)\n",
+        "\n",
+        "    model.prism_encoder.apply(lambda m: m._forward_hooks.clear())\n",
+        "    return pd.DataFrame(rotation_stats)\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 4. EXECUTION & REPORTING\n",
+        "# ==============================================================================\n",
+        "# A. Filter Data\n",
+        "ds_hard = filter_dataset(raw_poly_candidates, tokenizer, \"HARD (Polysemous)\")\n",
+        "ds_easy = filter_dataset(raw_casual_candidates, tokenizer, \"EASY (Casual)\")\n",
+        "\n",
+        "# B. Run Analysis\n",
+        "DEVICE = next(model.parameters()).device # Robust device check\n",
+        "df_hard = run_unified_probe(model, tokenizer, ds_hard, \"HARD\", DEVICE)\n",
+        "df_easy = run_unified_probe(model, tokenizer, ds_easy, \"EASY\", DEVICE)\n",
+        "\n",
+        "# C. Generate ASCII Tables\n",
+        "def print_stats(df, title):\n",
+        "    print(f\"\\n📊 {title} (N={len(df)})\")\n",
+        "    print(\"=\"*90)\n",
+        "    print(f\"{'Lyr':<3} | {'Mean (°)':<10} | {'Median (°)':<10} | {'Max (°)':<10} | {'Skewness':<10} | {'Regime'}\")\n",
+        "    print(\"-\" * 90)\n",
+        "\n",
+        "    total_skew = 0\n",
+        "    for col in df.columns:\n",
+        "        d = df[col]\n",
+        "        skew_val = d.skew()\n",
+        "        total_skew += skew_val\n",
+        "\n",
+        "        # Interpret Regime\n",
+        "        if skew_val > 1.5: regime = \"⚡ STEERING (Heavy Tail)\"\n",
+        "        elif skew_val > 0.5: regime = \"⚖️  HYBRID\"\n",
+        "        else: regime = \"💤 INERTIAL\"\n",
+        "\n",
+        "        print(f\"{col:<3} | {d.mean():6.2f}     | {d.median():6.2f}     | {d.max():6.2f}     | {skew_val:6.2f}     | {regime}\")\n",
+        "\n",
+        "    print(\"-\" * 90)\n",
+        "    print(f\"∑ INTEGRATED SKEWNESS (Metabolic Load): {total_skew:.2f}\")\n",
+        "\n",
+        "print_stats(df_hard, \"TABLE 3A: POLYSEMOUS TOKENS (Ambiguous)\")\n",
+        "print_stats(df_easy, \"TABLE 3B: CASUAL TOKENS (Unambiguous)\")"
+      ],
+      "metadata": {
+        "id": "hvHJcuqmxDkt"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import matplotlib.pyplot as plt\n",
+        "import seaborn as sns\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# FIGURE 3 GENERATOR (Robust N=76 Version)\n",
+        "# ==============================================================================\n",
+        "def plot_figure_3_robust(df_hard, df_easy, save_path=\"fig3_phase_steering_robust.png\"):\n",
+        "    \"\"\"\n",
+        "    Generates the \"Dual Regime\" Violin Plot matching the paper style.\n",
+        "    Visualizes the 'Mid-Network Resolution' (Skew spike at Layer 3).\n",
+        "    \"\"\"\n",
+        "    # Setup the canvas (Two panels, shared Y-axis)\n",
+        "    fig, axes = plt.subplots(1, 2, figsize=(10, 4), sharey=True, dpi=300)\n",
+        "\n",
+        "    # 1. AMBIGUOUS PANEL (The Steering Regime)\n",
+        "    # We use a Red palette to signify \"Metabolic Work\"\n",
+        "    sns.violinplot(\n",
+        "        data=df_hard,\n",
+        "        palette=\"Reds\",\n",
+        "        ax=axes[0],\n",
+        "        inner=\"quartile\",\n",
+        "        linewidth=1.2,\n",
+        "        cut=0 # Don't extend past data range\n",
+        "    )\n",
+        "    axes[0].set_title(\"(A) Ambiguous (Steering)\", fontweight='bold', fontsize=12, color='darkred')\n",
+        "    axes[0].set_ylabel(\"Phase Rotation (°)\", fontsize=11, fontweight='bold')\n",
+        "    axes[0].set_xlabel(\"Layer Depth\", fontsize=10)\n",
+        "    axes[0].grid(axis='y', linestyle='--', alpha=0.3)\n",
+        "\n",
+        "    # 2. UNAMBIGUOUS PANEL (The Inertial Regime)\n",
+        "    # We use a Blue/Green palette to signify \"Coasting\"\n",
+        "    sns.violinplot(\n",
+        "        data=df_easy,\n",
+        "        palette=\"mako\",\n",
+        "        ax=axes[1],\n",
+        "        inner=\"quartile\",\n",
+        "        linewidth=1.2,\n",
+        "        cut=0\n",
+        "    )\n",
+        "    axes[1].set_title(\"(B) Unambiguous (Inertial)\", fontweight='bold', fontsize=12, color='darkgreen')\n",
+        "    axes[1].set_xlabel(\"Layer Depth\", fontsize=10)\n",
+        "    axes[1].grid(axis='y', linestyle='--', alpha=0.3)\n",
+        "    axes[1].set_ylabel(\"\") # Remove redundant y-label\n",
+        "\n",
+        "    # Formatting\n",
+        "    plt.ylim(0, 30) # Focus on the active range (Max is ~22 deg)\n",
+        "    sns.despine(trim=True, offset=5)\n",
+        "    plt.tight_layout()\n",
+        "\n",
+        "    # Save & Show\n",
+        "    plt.savefig(save_path, bbox_inches='tight')\n",
+        "    plt.show()\n",
+        "    print(f\"✅ Updated Figure 3 saved to: {save_path}\")\n",
+        "\n",
+        "# Run with your existing dataframes\n",
+        "plot_figure_3_robust(df_hard, df_easy)"
+      ],
+      "metadata": {
+        "id": "bG4t_QIyycpD"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title 🛠️ Fixed Stress Test (Using Custom Generation API)\n",
+        "import torch\n",
+        "import pandas as pd\n",
+        "from tqdm import tqdm\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 1. SETUP & DATA\n",
+        "# ==============================================================================\n",
+        "test_cases = [\n",
+        "    # (German Word, Context Helper, Expected English)\n",
+        "    (\"Bank\", \"Die Bank.\", \"bench\"),          # Ambiguous: Bench vs Bank\n",
+        "    (\"Schloss\", \"Das Schloss.\", \"castle\"),   # Ambiguous: Castle vs Lock\n",
+        "    (\"Leiter\", \"Der Leiter.\", \"leader\"),     # Ambiguous: Ladder vs Leader\n",
+        "    (\"Decke\", \"Die Decke.\", \"ceiling\"),      # Ambiguous: Blanket vs Ceiling\n",
+        "    (\"Kiefer\", \"Der Kiefer.\", \"jaw\"),        # Ambiguous: Pine vs Jaw\n",
+        "    (\"Gericht\", \"Das Gericht.\", \"court\"),    # Ambiguous: Dish vs Court\n",
+        "    (\"Steuer\", \"Das Steuer.\", \"helm\"),       # Ambiguous: Tax vs Helm\n",
+        "    (\"Hahn\", \"Der Hahn.\", \"rooster\"),        # Ambiguous: Tap vs Rooster\n",
+        "    (\"Tau\", \"Das Tau.\", \"rope\"),             # Ambiguous: Dew vs Rope\n",
+        "    (\"Strauß\", \"Der Strauß.\", \"bouquet\"),    # Ambiguous: Ostrich vs Bouquet\n",
+        "]\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 2. THE EXPERIMENT LOOP\n",
+        "# ==============================================================================\n",
+        "results = []\n",
+        "print(f\"📉 Running Phase Interference Test on {len(test_cases)} ambiguous terms...\\n\")\n",
+        "\n",
+        "model.eval()\n",
+        "\n",
+        "# Helper to run your custom generate function\n",
+        "def run_prism_gen(text_input):\n",
+        "    # 1. Tokenize (Get IDs only)\n",
+        "    input_tensor = tokenizer(text_input, return_tensors=\"pt\", add_special_tokens=False).input_ids.to(DEVICE)\n",
+        "\n",
+        "    # 2. Call YOUR custom generate method\n",
+        "    # Signature: generate(self, src, max_length, num_beams=5)\n",
+        "    with torch.no_grad():\n",
+        "        out_ids = model.generate(src=input_tensor, max_length=10, num_beams=1)\n",
+        "\n",
+        "    # 3. Decode\n",
+        "    return tokenizer.decode(out_ids[0], skip_special_tokens=True).strip()\n",
+        "\n",
+        "for word, context_phrase, target in tqdm(test_cases):\n",
+        "    try:\n",
+        "        # --- PASS 1: ISOLATION (Single Token) ---\n",
+        "        trans_iso = run_prism_gen(word)\n",
+        "\n",
+        "        # --- PASS 2: INTERFERENCE (Context) ---\n",
+        "        trans_int = run_prism_gen(context_phrase)\n",
+        "\n",
+        "        # Log Result\n",
+        "        # We flag it as \"FAIL\" (bug) if the isolation translation is WRONG.\n",
+        "        # BUT for your paper, an \"Isolation Fail\" + \"Context Pass\" is actually a SUCCESSFUL scientific result.\n",
+        "\n",
+        "        status = \"✅ Context Fix\" if (target.lower() not in trans_iso.lower() and target.lower() in trans_int.lower()) else \"Neutral\"\n",
+        "\n",
+        "        results.append({\n",
+        "            \"Source\": word,\n",
+        "            \"Target\": target,\n",
+        "            \"⛔ Isolation\": trans_iso,\n",
+        "            \"✅ Context\": trans_int,\n",
+        "            \"Outcome\": status\n",
+        "        })\n",
+        "    except Exception as e:\n",
+        "        print(f\"Error on {word}: {e}\")\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 3. ANALYSIS & REPORT\n",
+        "# ==============================================================================\n",
+        "df_res = pd.DataFrame(results)\n",
+        "\n",
+        "print(\"\\n\\n🌊 EXPERIMENTAL RESULTS: The Single-Token Paradox\")\n",
+        "print(\"=\"*110)\n",
+        "print(f\"{'Input':<10} | {'Target':<10} | {'⛔ Isolation (No Wave)':<30} | {'✅ Context (Interference)':<30}\")\n",
+        "print(\"-\" * 110)\n",
+        "\n",
+        "success_scientific_count = 0\n",
+        "\n",
+        "for _, row in df_res.iterrows():\n",
+        "    iso_text = row['⛔ Isolation']\n",
+        "    ctx_text = row['✅ Context']\n",
+        "\n",
+        "    # Visual Logic: If Isolation failed to find target, but Context found it -> Highlight\n",
+        "    if row['Outcome'] == \"✅ Context Fix\":\n",
+        "        iso_text = f\"--> {iso_text} <--\" # Shows the \"Inertial\" failure\n",
+        "        success_scientific_count += 1\n",
+        "\n",
+        "    print(f\"{row['Source']:<10} | {row['Target']:<10} | {iso_text:<30} | {ctx_text:<30}\")\n",
+        "\n",
+        "print(\"=\"*110)\n",
+        "print(f\"\\n🧪 SCIENTIFIC CONCLUSION:\")\n",
+        "if success_scientific_count > 0:\n",
+        "    print(f\"🎉 OBSERVED: {success_scientific_count}/{len(test_cases)} cases showed the 'Physical Abstractor' effect!\")\n",
+        "    print(\"   The model failed to resolve ambiguity in isolation (as predicted) but resolved it with context.\")\n",
+        "else:\n",
+        "    print(\"🤔 OBSERVED: The model resolved isolation perfectly. Rate-Coding (Magnitude) might be leaking.\")"
+      ],
+      "metadata": {
+        "id": "llU4wmT67ZRm"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title 🧪 Sanity Check: Full Wave Packets (Sentences)\n",
+        "# ==============================================================================\n",
+        "# HYPOTHESIS:\n",
+        "# If the \"Single-Token Paradox\" is real, then full sentences (rich interference)\n",
+        "# should translate fluently, unlike the \"broken\" isolated tokens.\n",
+        "# ==============================================================================\n",
+        "\n",
+        "sanity_sentences = [\n",
+        "    # 1. Standard Fluency (Control)\n",
+        "    \"Das Haus ist groß und schön.\",\n",
+        "    \"Die Katze schläft auf dem Sofa.\",\n",
+        "    \"Wir gehen heute in den Park.\",\n",
+        "    \"Das Wetter ist sehr gut.\",\n",
+        "\n",
+        "    # 2. Contextual Resolution (The ambiguous words from before)\n",
+        "    \"Der Lehrer schreibt an die Tafel.\",      # \"Leiter\" implies leader/head, but checking context\n",
+        "    \"Ich sitze auf einer Bank im Garten.\",    # Should lock to \"Bench\"\n",
+        "    \"Ich bringe mein Geld zur Bank.\",         # Should lock to \"Bank\" (Financial)\n",
+        "    \"Das Schloss ist alt und aus Stein.\",     # Should lock to \"Castle\"\n",
+        "    \"Der Schlüssel steckt im Schloss.\",       # Should lock to \"Lock\"\n",
+        "\n",
+        "    # 3. Complex Grammar (Phase coherence test)\n",
+        "    \"Obwohl es regnet, gehe ich spazieren.\",\n",
+        "    \"Wenn du Zeit hast, komm bitte vorbei.\"\n",
+        "]\n",
+        "\n",
+        "print(f\"🌊 Running Sanity Check on {len(sanity_sentences)} sentences...\\n\")\n",
+        "print(\"=\"*100)\n",
+        "print(f\"{'German Source':<40} | {'🇬🇧 PRISM Translation'}\")\n",
+        "print(\"-\" * 100)\n",
+        "\n",
+        "model.eval()\n",
+        "\n",
+        "# Re-using the helper from before\n",
+        "def run_prism_gen(text_input):\n",
+        "    input_tensor = tokenizer(text_input, return_tensors=\"pt\", add_special_tokens=False).input_ids.to(DEVICE)\n",
+        "    with torch.no_grad():\n",
+        "        # Increased max_length for full sentences\n",
+        "        out_ids = model.generate(src=input_tensor, max_length=40, num_beams=1)\n",
+        "    return tokenizer.decode(out_ids[0], skip_special_tokens=True).strip()\n",
+        "\n",
+        "for sent in sanity_sentences:\n",
+        "    try:\n",
+        "        translation = run_prism_gen(sent)\n",
+        "        print(f\"{sent:<40} | {translation}\")\n",
+        "    except Exception as e:\n",
+        "        print(f\"{sent:<40} | ❌ Error: {e}\")\n",
+        "\n",
+        "print(\"=\"*100)"
+      ],
+      "metadata": {
+        "id": "71sdvAAn8O63"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title 🧪 Control Experiment: Unambiguous Single Tokens\n",
+        "# ==============================================================================\n",
+        "# HYPOTHESIS:\n",
+        "# If the single-token collapse is due to AMBIGUITY (phase can't resolve meaning),\n",
+        "# then UNAMBIGUOUS tokens should translate correctly even in isolation.\n",
+        "# If they ALSO collapse, the issue is purely L=1 (no interference at all).\n",
+        "# ==============================================================================\n",
+        "\n",
+        "import torch\n",
+        "import pandas as pd\n",
+        "from tqdm import tqdm\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 1. UNAMBIGUOUS TEST SET\n",
+        "# ==============================================================================\n",
+        "# These words have ONE clear meaning - no context needed for humans\n",
+        "unambiguous_cases = [\n",
+        "    # (German Word, Expected English)\n",
+        "    # --- Animals (No ambiguity) ---\n",
+        "    (\"Katze\", \"cat\"),\n",
+        "    (\"Hund\", \"dog\"),\n",
+        "    (\"Pferd\", \"horse\"),\n",
+        "    (\"Vogel\", \"bird\"),\n",
+        "    (\"Fisch\", \"fish\"),\n",
+        "    (\"Elefant\", \"elephant\"),\n",
+        "    (\"Löwe\", \"lion\"),\n",
+        "    (\"Bär\", \"bear\"),\n",
+        "\n",
+        "    # --- Objects (No ambiguity) ---\n",
+        "    (\"Tisch\", \"table\"),\n",
+        "    (\"Stuhl\", \"chair\"),\n",
+        "    (\"Buch\", \"book\"),\n",
+        "    (\"Auto\", \"car\"),\n",
+        "    (\"Haus\", \"house\"),\n",
+        "    (\"Fenster\", \"window\"),\n",
+        "    (\"Lampe\", \"lamp\"),\n",
+        "    (\"Telefon\", \"phone\"),\n",
+        "\n",
+        "    # --- Nature (No ambiguity) ---\n",
+        "    (\"Baum\", \"tree\"),\n",
+        "    (\"Blume\", \"flower\"),\n",
+        "    (\"Wolke\", \"cloud\"),\n",
+        "    (\"Regen\", \"rain\"),\n",
+        "    (\"Schnee\", \"snow\"),\n",
+        "    (\"Feuer\", \"fire\"),\n",
+        "\n",
+        "    # --- Body Parts (No ambiguity) ---\n",
+        "    (\"Kopf\", \"head\"),\n",
+        "    (\"Auge\", \"eye\"),\n",
+        "    (\"Ohr\", \"ear\"),\n",
+        "    (\"Nase\", \"nose\"),\n",
+        "    (\"Finger\", \"finger\"),\n",
+        "\n",
+        "    # --- Food (No ambiguity) ---\n",
+        "    (\"Brot\", \"bread\"),\n",
+        "    (\"Käse\", \"cheese\"),\n",
+        "    (\"Apfel\", \"apple\"),\n",
+        "    (\"Wasser\", \"water\"),\n",
+        "    (\"Milch\", \"milk\"),\n",
+        "]\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 2. THE EXPERIMENT\n",
+        "# ==============================================================================\n",
+        "results_unambig = []\n",
+        "print(f\"🔬 Running UNAMBIGUOUS Single-Token Test on {len(unambiguous_cases)} words...\\n\")\n",
+        "\n",
+        "model.eval()\n",
+        "\n",
+        "def run_prism_gen(text_input, max_len=10):\n",
+        "    \"\"\"Helper to run generation\"\"\"\n",
+        "    input_tensor = tokenizer(text_input, return_tensors=\"pt\", add_special_tokens=False).input_ids.to(DEVICE)\n",
+        "    with torch.no_grad():\n",
+        "        out_ids = model.generate(src=input_tensor, max_length=max_len, num_beams=1)\n",
+        "    return tokenizer.decode(out_ids[0], skip_special_tokens=True).strip()\n",
+        "\n",
+        "def is_repetition_collapse(text):\n",
+        "    \"\"\"Detect if output is repetitive garbage\"\"\"\n",
+        "    words = text.split()\n",
+        "    if len(words) < 2:\n",
+        "        return False\n",
+        "    # Check if first word repeats\n",
+        "    first_word = words[0].lower().strip('.,!?')\n",
+        "    repeat_count = sum(1 for w in words if w.lower().strip('.,!?') == first_word)\n",
+        "    return repeat_count >= len(words) * 0.5  # 50%+ repetition = collapse\n",
+        "\n",
+        "def check_correct(output, target):\n",
+        "    \"\"\"Check if target word appears in output\"\"\"\n",
+        "    return target.lower() in output.lower()\n",
+        "\n",
+        "# Run the test\n",
+        "for word, target in tqdm(unambiguous_cases):\n",
+        "    try:\n",
+        "        # Single token translation\n",
+        "        translation = run_prism_gen(word)\n",
+        "\n",
+        "        # Analyze\n",
+        "        collapsed = is_repetition_collapse(translation)\n",
+        "        correct = check_correct(translation, target)\n",
+        "\n",
+        "        # Also test with minimal context (article)\n",
+        "        # German articles: der/die/das\n",
+        "        context_translation = run_prism_gen(f\"Das {word}.\")\n",
+        "        context_correct = check_correct(context_translation, target)\n",
+        "\n",
+        "        results_unambig.append({\n",
+        "            \"German\": word,\n",
+        "            \"Target\": target,\n",
+        "            \"Isolation\": translation,\n",
+        "            \"Collapsed\": collapsed,\n",
+        "            \"Correct\": correct,\n",
+        "            \"With Article\": context_translation,\n",
+        "            \"Context Correct\": context_correct\n",
+        "        })\n",
+        "    except Exception as e:\n",
+        "        print(f\"Error on {word}: {e}\")\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 3. ANALYSIS & REPORT\n",
+        "# ==============================================================================\n",
+        "df_unambig = pd.DataFrame(results_unambig)\n",
+        "\n",
+        "# Stats\n",
+        "total = len(df_unambig)\n",
+        "collapsed_count = df_unambig['Collapsed'].sum()\n",
+        "correct_iso = df_unambig['Correct'].sum()\n",
+        "correct_ctx = df_unambig['Context Correct'].sum()\n",
+        "\n",
+        "print(\"\\n\" + \"=\"*120)\n",
+        "print(\"🧪 CONTROL EXPERIMENT: UNAMBIGUOUS SINGLE TOKENS\")\n",
+        "print(\"=\"*120)\n",
+        "print(f\"{'German':<12} | {'Target':<10} | {'⛔ Isolation (L=1)':<35} | {'Collapse?':<10} | {'✅ With Article':<30}\")\n",
+        "print(\"-\" * 120)\n",
+        "\n",
+        "for _, row in df_unambig.iterrows():\n",
+        "    collapse_marker = \"💥 YES\" if row['Collapsed'] else \"No\"\n",
+        "    iso_display = row['Isolation'][:33] + \"..\" if len(row['Isolation']) > 35 else row['Isolation']\n",
+        "    ctx_display = row['With Article'][:28] + \"..\" if len(row['With Article']) > 30 else row['With Article']\n",
+        "\n",
+        "    print(f\"{row['German']:<12} | {row['Target']:<10} | {iso_display:<35} | {collapse_marker:<10} | {ctx_display:<30}\")\n",
+        "\n",
+        "print(\"=\"*120)\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 4. SCIENTIFIC SUMMARY\n",
+        "# ==============================================================================\n",
+        "print(\"\\n📊 STATISTICAL SUMMARY\")\n",
+        "print(\"-\"*60)\n",
+        "print(f\"Total test cases:                    {total}\")\n",
+        "print(f\"Repetition collapses (L=1):          {collapsed_count} ({100*collapsed_count/total:.1f}%)\")\n",
+        "print(f\"Correct in isolation:                {correct_iso} ({100*correct_iso/total:.1f}%)\")\n",
+        "print(f\"Correct with article context:        {correct_ctx} ({100*correct_ctx/total:.1f}%)\")\n",
+        "print(\"-\"*60)\n",
+        "\n",
+        "print(\"\\n🔬 INTERPRETATION:\")\n",
+        "if collapsed_count > total * 0.5:\n",
+        "    print(\"   💥 FINDING: Unambiguous tokens ALSO collapse!\")\n",
+        "    print(\"   → This suggests L=1 provides insufficient interference for ANY semantic encoding.\")\n",
+        "    print(\"   → The decoder needs wave structure, not just meaning clarity.\")\n",
+        "    print(\"\\n   📝 PAPER IMPLICATION: Phase interference is necessary for GENERATION,\")\n",
+        "    print(\"      not just disambiguation. Single tokens lack the spectral 'carrier wave'\")\n",
+        "    print(\"      needed to bootstrap autoregressive decoding.\")\n",
+        "elif collapsed_count > 0:\n",
+        "    print(\"   ⚖️ FINDING: Mixed results - some collapse, some don't.\")\n",
+        "    print(\"   → Suggests a threshold effect based on embedding quality or token frequency.\")\n",
+        "else:\n",
+        "    print(\"   ✅ FINDING: Unambiguous tokens translate correctly!\")\n",
+        "    print(\"   → This confirms the hypothesis: ambiguity requires interference to resolve,\")\n",
+        "    print(\"      but clear semantics can propagate through the encoder even at L=1.\")\n",
+        "    print(\"\\n   📝 PAPER IMPLICATION: The single-token collapse is SPECIFIC to polysemy.\")\n",
+        "    print(\"      PRISM's phase encoding captures unambiguous semantics in static embeddings,\")\n",
+        "    print(\"      but requires interference patterns to perform 'semantic selection'.\")\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 5. COMPARISON TABLE (Side by Side)\n",
+        "# ==============================================================================\n",
+        "print(\"\\n\\n\" + \"=\"*80)\n",
+        "print(\"📈 AMBIGUOUS vs UNAMBIGUOUS COMPARISON\")\n",
+        "print(\"=\"*80)\n",
+        "print(f\"{'Metric':<40} | {'Ambiguous':<15} | {'Unambiguous':<15}\")\n",
+        "print(\"-\"*80)\n",
+        "print(f\"{'Repetition Collapse Rate':<40} | {'~100%':<15} | {f'{100*collapsed_count/total:.0f}%':<15}\")\n",
+        "print(f\"{'Correct in Isolation':<40} | {'~0%':<15} | {f'{100*correct_iso/total:.0f}%':<15}\")\n",
+        "print(f\"{'Correct with Minimal Context':<40} | {'Partial':<15} | {f'{100*correct_ctx/total:.0f}%':<15}\")\n",
+        "print(\"=\"*80)"
+      ],
+      "metadata": {
+        "id": "k6rH5YJWauTc"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Test with num_beams=1 (greedy) vs num_beams=5 (beam search)\n",
+        "word = \"Hund\"\n",
+        "input_tensor = tokenizer(word, return_tensors=\"pt\", add_special_tokens=False).input_ids.to(DEVICE)\n",
+        "\n",
+        "with torch.no_grad():\n",
+        "    out_greedy = model.generate(src=input_tensor, max_length=10, num_beams=1)\n",
+        "    out_beam = model.generate(src=input_tensor, max_length=10, num_beams=5)\n",
+        "\n",
+        "print(f\"Greedy (beams=1): {tokenizer.decode(out_greedy[0], skip_special_tokens=True)}\")\n",
+        "print(f\"Beam (beams=5):   {tokenizer.decode(out_beam[0], skip_special_tokens=True)}\")"
+      ],
+      "metadata": {
+        "id": "E0WRta7Qdjcg"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title 🧪 Large-Scale Single-Token Analysis (N >> 32)\n",
+        "# ==============================================================================\n",
+        "# GOAL: Increase sample size with automatic single-token validation\n",
+        "# ==============================================================================\n",
+        "\n",
+        "import torch\n",
+        "import pandas as pd\n",
+        "from tqdm import tqdm\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 1. LARGE CANDIDATE POOLS\n",
+        "# ==============================================================================\n",
+        "\n",
+        "# A. AMBIGUOUS CANDIDATES (German words with multiple meanings)\n",
+        "ambiguous_candidates = [\n",
+        "    # Word, Meaning1, Meaning2\n",
+        "    (\"Bank\", \"bench\", \"bank\"),\n",
+        "    (\"Schloss\", \"castle\", \"lock\"),\n",
+        "    (\"Leiter\", \"ladder\", \"leader\"),\n",
+        "    (\"Decke\", \"ceiling\", \"blanket\"),\n",
+        "    (\"Kiefer\", \"pine\", \"jaw\"),\n",
+        "    (\"Strauß\", \"ostrich\", \"bouquet\"),\n",
+        "    (\"Tor\", \"gate\", \"goal\"),\n",
+        "    (\"Ball\", \"ball\", \"dance\"),\n",
+        "    (\"Schlange\", \"snake\", \"queue\"),\n",
+        "    (\"Strom\", \"electricity\", \"river\"),\n",
+        "    (\"Mutter\", \"mother\", \"nut\"),\n",
+        "    (\"Birne\", \"pear\", \"lightbulb\"),\n",
+        "    (\"Gericht\", \"court\", \"dish\"),\n",
+        "    (\"Ton\", \"sound\", \"clay\"),\n",
+        "    (\"Blatt\", \"leaf\", \"sheet\"),\n",
+        "    (\"Nagel\", \"nail\", \"fingernail\"),\n",
+        "    (\"Maus\", \"mouse\", \"computer mouse\"),\n",
+        "    (\"Erde\", \"earth\", \"soil\"),\n",
+        "    (\"Hahn\", \"rooster\", \"tap\"),\n",
+        "    (\"Schale\", \"shell\", \"bowl\"),\n",
+        "    (\"Bauer\", \"farmer\", \"pawn\"),\n",
+        "    (\"Steuer\", \"tax\", \"steering wheel\"),\n",
+        "    (\"Tau\", \"dew\", \"rope\"),\n",
+        "    (\"Feder\", \"feather\", \"spring\"),\n",
+        "    (\"Absatz\", \"heel\", \"paragraph\"),\n",
+        "    (\"Band\", \"ribbon\", \"volume\"),\n",
+        "    (\"Brücke\", \"bridge\", \"dental bridge\"),\n",
+        "    (\"Flügel\", \"wing\", \"grand piano\"),\n",
+        "    (\"Golf\", \"golf\", \"gulf\"),\n",
+        "    (\"Grund\", \"reason\", \"ground\"),\n",
+        "    (\"Hut\", \"hat\", \"guard\"),\n",
+        "    (\"Kette\", \"chain\", \"necklace\"),\n",
+        "    (\"Kran\", \"crane bird\", \"crane machine\"),\n",
+        "    (\"Lauf\", \"run\", \"barrel\"),\n",
+        "    (\"Linse\", \"lens\", \"lentil\"),\n",
+        "    (\"Mark\", \"marrow\", \"mark currency\"),\n",
+        "    (\"Masse\", \"mass\", \"crowd\"),\n",
+        "    (\"Netz\", \"net\", \"network\"),\n",
+        "    (\"Pony\", \"pony\", \"bangs\"),\n",
+        "    (\"Raum\", \"room\", \"space\"),\n",
+        "    (\"Reif\", \"hoop\", \"frost\"),\n",
+        "    (\"Rock\", \"skirt\", \"rock music\"),\n",
+        "    (\"Schalter\", \"switch\", \"counter\"),\n",
+        "    (\"Schild\", \"sign\", \"shield\"),\n",
+        "    (\"See\", \"lake\", \"sea\"),\n",
+        "    (\"Seite\", \"side\", \"page\"),\n",
+        "    (\"Star\", \"starling\", \"celebrity\"),\n",
+        "    (\"Stock\", \"stick\", \"floor\"),\n",
+        "    (\"Wahl\", \"choice\", \"election\"),\n",
+        "    (\"Welle\", \"wave\", \"shaft\"),\n",
+        "    (\"Zug\", \"train\", \"pull\"),\n",
+        "]\n",
+        "\n",
+        "# B. UNAMBIGUOUS CANDIDATES (German words with single clear meaning)\n",
+        "unambiguous_candidates = [\n",
+        "    # Animals\n",
+        "    (\"Katze\", \"cat\"), (\"Hund\", \"dog\"), (\"Pferd\", \"horse\"), (\"Vogel\", \"bird\"),\n",
+        "    (\"Fisch\", \"fish\"), (\"Elefant\", \"elephant\"), (\"Löwe\", \"lion\"), (\"Bär\", \"bear\"),\n",
+        "    (\"Tiger\", \"tiger\"), (\"Affe\", \"monkey\"), (\"Schaf\", \"sheep\"), (\"Kuh\", \"cow\"),\n",
+        "    (\"Schwein\", \"pig\"), (\"Huhn\", \"chicken\"), (\"Ente\", \"duck\"), (\"Gans\", \"goose\"),\n",
+        "    (\"Wolf\", \"wolf\"), (\"Fuchs\", \"fox\"), (\"Hase\", \"rabbit\"), (\"Hirsch\", \"deer\"),\n",
+        "    (\"Frosch\", \"frog\"), (\"Spinne\", \"spider\"), (\"Biene\", \"bee\"), (\"Käfer\", \"beetle\"),\n",
+        "\n",
+        "    # Objects\n",
+        "    (\"Tisch\", \"table\"), (\"Stuhl\", \"chair\"), (\"Buch\", \"book\"), (\"Auto\", \"car\"),\n",
+        "    (\"Haus\", \"house\"), (\"Fenster\", \"window\"), (\"Lampe\", \"lamp\"), (\"Telefon\", \"phone\"),\n",
+        "    (\"Computer\", \"computer\"), (\"Uhr\", \"clock\"), (\"Brille\", \"glasses\"), (\"Schlüssel\", \"key\"),\n",
+        "    (\"Tasche\", \"bag\"), (\"Schuh\", \"shoe\"), (\"Hemd\", \"shirt\"), (\"Hose\", \"pants\"),\n",
+        "    (\"Kleid\", \"dress\"), (\"Jacke\", \"jacket\"), (\"Tür\", \"door\"), (\"Bett\", \"bed\"),\n",
+        "    (\"Schrank\", \"closet\"), (\"Sofa\", \"sofa\"), (\"Spiegel\", \"mirror\"), (\"Teppich\", \"carpet\"),\n",
+        "\n",
+        "    # Nature\n",
+        "    (\"Baum\", \"tree\"), (\"Blume\", \"flower\"), (\"Wolke\", \"cloud\"), (\"Regen\", \"rain\"),\n",
+        "    (\"Schnee\", \"snow\"), (\"Feuer\", \"fire\"), (\"Berg\", \"mountain\"), (\"Wald\", \"forest\"),\n",
+        "    (\"Fluss\", \"river\"), (\"Himmel\", \"sky\"), (\"Stern\", \"star\"), (\"Mond\", \"moon\"),\n",
+        "    (\"Sonne\", \"sun\"), (\"Gras\", \"grass\"), (\"Stein\", \"stone\"), (\"Sand\", \"sand\"),\n",
+        "\n",
+        "    # Body parts\n",
+        "    (\"Kopf\", \"head\"), (\"Auge\", \"eye\"), (\"Ohr\", \"ear\"), (\"Nase\", \"nose\"),\n",
+        "    (\"Mund\", \"mouth\"), (\"Zahn\", \"tooth\"), (\"Zunge\", \"tongue\"), (\"Hals\", \"neck\"),\n",
+        "    (\"Arm\", \"arm\"), (\"Bein\", \"leg\"), (\"Fuß\", \"foot\"), (\"Knie\", \"knee\"),\n",
+        "    (\"Finger\", \"finger\"), (\"Herz\", \"heart\"), (\"Lunge\", \"lung\"), (\"Magen\", \"stomach\"),\n",
+        "\n",
+        "    # Food & Drink\n",
+        "    (\"Brot\", \"bread\"), (\"Käse\", \"cheese\"), (\"Apfel\", \"apple\"), (\"Wasser\", \"water\"),\n",
+        "    (\"Milch\", \"milk\"), (\"Ei\", \"egg\"), (\"Fleisch\", \"meat\"), (\"Reis\", \"rice\"),\n",
+        "    (\"Nudel\", \"noodle\"), (\"Suppe\", \"soup\"), (\"Salat\", \"salad\"), (\"Kuchen\", \"cake\"),\n",
+        "    (\"Kaffee\", \"coffee\"), (\"Tee\", \"tea\"), (\"Bier\", \"beer\"), (\"Wein\", \"wine\"),\n",
+        "    (\"Saft\", \"juice\"), (\"Zucker\", \"sugar\"), (\"Salz\", \"salt\"), (\"Butter\", \"butter\"),\n",
+        "\n",
+        "    # Colors (as nouns)\n",
+        "    (\"Rot\", \"red\"), (\"Blau\", \"blue\"), (\"Grün\", \"green\"), (\"Gelb\", \"yellow\"),\n",
+        "    (\"Schwarz\", \"black\"), (\"Weiß\", \"white\"), (\"Braun\", \"brown\"), (\"Grau\", \"gray\"),\n",
+        "\n",
+        "    # Numbers (as nouns)\n",
+        "    (\"Eins\", \"one\"), (\"Zwei\", \"two\"), (\"Drei\", \"three\"), (\"Vier\", \"four\"),\n",
+        "    (\"Fünf\", \"five\"), (\"Sechs\", \"six\"), (\"Sieben\", \"seven\"), (\"Acht\", \"eight\"),\n",
+        "\n",
+        "    # Family\n",
+        "    (\"Vater\", \"father\"), (\"Bruder\", \"brother\"), (\"Schwester\", \"sister\"),\n",
+        "    (\"Onkel\", \"uncle\"), (\"Tante\", \"aunt\"), (\"Oma\", \"grandma\"), (\"Opa\", \"grandpa\"),\n",
+        "\n",
+        "    # Professions\n",
+        "    (\"Arzt\", \"doctor\"), (\"Lehrer\", \"teacher\"), (\"Koch\", \"cook\"), (\"Pilot\", \"pilot\"),\n",
+        "    (\"Polizist\", \"policeman\"), (\"Bäcker\", \"baker\"), (\"Maler\", \"painter\"),\n",
+        "]\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 2. SINGLE-TOKEN VALIDATION FUNCTION\n",
+        "# ==============================================================================\n",
+        "\n",
+        "def is_single_token(word, tokenizer):\n",
+        "    \"\"\"\n",
+        "    Check if a word is represented as a single token.\n",
+        "    Tests both with and without space prefix (BPE behavior varies).\n",
+        "    \"\"\"\n",
+        "    # Test 1: Raw word\n",
+        "    tokens_raw = tokenizer.encode(word, add_special_tokens=False)\n",
+        "\n",
+        "    # Test 2: With space prefix (common in BPE)\n",
+        "    tokens_space = tokenizer.encode(\" \" + word, add_special_tokens=False)\n",
+        "\n",
+        "    # Test 3: With article (might help with German nouns)\n",
+        "    tokens_article = tokenizer.encode(\"Das \" + word, add_special_tokens=False)\n",
+        "\n",
+        "    # Accept if ANY encoding is single token (or 2 tokens for article version)\n",
+        "    is_single = (len(tokens_raw) == 1) or (len(tokens_space) == 1)\n",
+        "\n",
+        "    return is_single, len(tokens_raw), len(tokens_space)\n",
+        "\n",
+        "def filter_single_tokens(candidates, tokenizer, is_ambiguous=True):\n",
+        "    \"\"\"\n",
+        "    Filter candidates to only include single-token words.\n",
+        "    Returns validated list with token info.\n",
+        "    \"\"\"\n",
+        "    valid = []\n",
+        "    rejected = []\n",
+        "\n",
+        "    for item in candidates:\n",
+        "        if is_ambiguous:\n",
+        "            word = item[0]\n",
+        "        else:\n",
+        "            word = item[0]\n",
+        "\n",
+        "        is_single, n_raw, n_space = is_single_token(word, tokenizer)\n",
+        "\n",
+        "        if is_single:\n",
+        "            valid.append(item)\n",
+        "        else:\n",
+        "            rejected.append((word, n_raw, n_space))\n",
+        "\n",
+        "    return valid, rejected\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 3. GENERATION & ANALYSIS FUNCTIONS\n",
+        "# ==============================================================================\n",
+        "\n",
+        "def run_prism_gen(text_input, model, tokenizer, device, max_len=10):\n",
+        "    \"\"\"Run PRISM generation\"\"\"\n",
+        "    input_tensor = tokenizer(text_input, return_tensors=\"pt\", add_special_tokens=False).input_ids.to(device)\n",
+        "    with torch.no_grad():\n",
+        "        out_ids = model.generate(src=input_tensor, max_length=max_len, num_beams=1)\n",
+        "    return tokenizer.decode(out_ids[0], skip_special_tokens=True).strip()\n",
+        "\n",
+        "def is_repetition_collapse(text):\n",
+        "    \"\"\"Detect if output shows repetition collapse\"\"\"\n",
+        "    if not text or len(text.split()) < 2:\n",
+        "        return False\n",
+        "\n",
+        "    words = text.lower().split()\n",
+        "    # Clean punctuation\n",
+        "    words = [w.strip('.,!?;:') for w in words]\n",
+        "\n",
+        "    if len(words) < 2:\n",
+        "        return False\n",
+        "\n",
+        "    # Check various collapse patterns\n",
+        "    # Pattern 1: Same word repeats\n",
+        "    first_word = words[0]\n",
+        "    same_count = sum(1 for w in words if w == first_word or first_word.startswith(w) or w.startswith(first_word))\n",
+        "\n",
+        "    # Pattern 2: Substring repetition (e.g., \"dogdogdog\")\n",
+        "    joined = ''.join(words)\n",
+        "    if len(joined) > 3:\n",
+        "        chunk = joined[:3]\n",
+        "        if joined.count(chunk) >= 3:\n",
+        "            return True\n",
+        "\n",
+        "    return same_count >= len(words) * 0.5\n",
+        "\n",
+        "def check_correct(output, targets):\n",
+        "    \"\"\"Check if any target word appears in output\"\"\"\n",
+        "    output_lower = output.lower()\n",
+        "    if isinstance(targets, str):\n",
+        "        targets = [targets]\n",
+        "    return any(t.lower() in output_lower for t in targets)\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 4. MAIN EXPERIMENT\n",
+        "# ==============================================================================\n",
+        "\n",
+        "print(\"=\" * 100)\n",
+        "print(\"🔬 LARGE-SCALE SINGLE-TOKEN CARRIER WAVE ANALYSIS\")\n",
+        "print(\"=\" * 100)\n",
+        "\n",
+        "# A. Filter to single tokens only\n",
+        "print(\"\\n📋 STEP 1: Validating Single-Token Candidates...\")\n",
+        "print(\"-\" * 60)\n",
+        "\n",
+        "ambig_valid, ambig_rejected = filter_single_tokens(ambiguous_candidates, tokenizer, is_ambiguous=True)\n",
+        "unambig_valid, unambig_rejected = filter_single_tokens(unambiguous_candidates, tokenizer, is_ambiguous=False)\n",
+        "\n",
+        "print(f\"AMBIGUOUS:   {len(ambig_valid)} valid / {len(ambig_rejected)} rejected (multi-token)\")\n",
+        "print(f\"UNAMBIGUOUS: {len(unambig_valid)} valid / {len(unambig_rejected)} rejected (multi-token)\")\n",
+        "\n",
+        "# Show some rejected examples\n",
+        "if ambig_rejected:\n",
+        "    print(f\"\\n   Rejected ambiguous (examples): {ambig_rejected[:5]}\")\n",
+        "if unambig_rejected:\n",
+        "    print(f\"   Rejected unambiguous (examples): {unambig_rejected[:5]}\")\n",
+        "\n",
+        "# B. Run experiments\n",
+        "print(f\"\\n📋 STEP 2: Running Generation Tests...\")\n",
+        "print(\"-\" * 60)\n",
+        "\n",
+        "# Storage\n",
+        "results_ambig = []\n",
+        "results_unambig = []\n",
+        "\n",
+        "# Test AMBIGUOUS tokens\n",
+        "print(f\"\\n🔴 Testing {len(ambig_valid)} AMBIGUOUS single tokens...\")\n",
+        "for word, meaning1, meaning2 in tqdm(ambig_valid):\n",
+        "    try:\n",
+        "        output = run_prism_gen(word, model, tokenizer, DEVICE)\n",
+        "        collapsed = is_repetition_collapse(output)\n",
+        "        # For ambiguous, \"correct\" is undefined - check if either meaning appears\n",
+        "        has_meaning = check_correct(output, [meaning1, meaning2])\n",
+        "\n",
+        "        results_ambig.append({\n",
+        "            \"word\": word,\n",
+        "            \"meanings\": f\"{meaning1}/{meaning2}\",\n",
+        "            \"output\": output,\n",
+        "            \"collapsed\": collapsed,\n",
+        "            \"has_any_meaning\": has_meaning\n",
+        "        })\n",
+        "    except Exception as e:\n",
+        "        print(f\"Error on {word}: {e}\")\n",
+        "\n",
+        "# Test UNAMBIGUOUS tokens\n",
+        "print(f\"\\n🟢 Testing {len(unambig_valid)} UNAMBIGUOUS single tokens...\")\n",
+        "for word, meaning in tqdm(unambig_valid):\n",
+        "    try:\n",
+        "        output = run_prism_gen(word, model, tokenizer, DEVICE)\n",
+        "        collapsed = is_repetition_collapse(output)\n",
+        "        correct = check_correct(output, meaning)\n",
+        "\n",
+        "        results_unambig.append({\n",
+        "            \"word\": word,\n",
+        "            \"meaning\": meaning,\n",
+        "            \"output\": output,\n",
+        "            \"collapsed\": collapsed,\n",
+        "            \"correct\": correct\n",
+        "        })\n",
+        "    except Exception as e:\n",
+        "        print(f\"Error on {word}: {e}\")\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 5. STATISTICAL ANALYSIS\n",
+        "# ==============================================================================\n",
+        "\n",
+        "df_ambig = pd.DataFrame(results_ambig)\n",
+        "df_unambig = pd.DataFrame(results_unambig)\n",
+        "\n",
+        "print(\"\\n\" + \"=\" * 100)\n",
+        "print(\"📊 RESULTS: AMBIGUOUS TOKENS (L=1)\")\n",
+        "print(\"=\" * 100)\n",
+        "print(f\"{'Word':<15} | {'Meanings':<25} | {'Output':<40} | {'Collapse?'}\")\n",
+        "print(\"-\" * 100)\n",
+        "\n",
+        "for _, row in df_ambig.iterrows():\n",
+        "    collapse_mark = \"💥 YES\" if row['collapsed'] else \"No\"\n",
+        "    output_display = row['output'][:38] + \"..\" if len(row['output']) > 40 else row['output']\n",
+        "    print(f\"{row['word']:<15} | {row['meanings']:<25} | {output_display:<40} | {collapse_mark}\")\n",
+        "\n",
+        "print(\"\\n\" + \"=\" * 100)\n",
+        "print(\"📊 RESULTS: UNAMBIGUOUS TOKENS (L=1)\")\n",
+        "print(\"=\" * 100)\n",
+        "print(f\"{'Word':<15} | {'Target':<15} | {'Output':<40} | {'Collapse?':<10} | {'Correct?'}\")\n",
+        "print(\"-\" * 100)\n",
+        "\n",
+        "for _, row in df_unambig.iterrows():\n",
+        "    collapse_mark = \"💥 YES\" if row['collapsed'] else \"No\"\n",
+        "    correct_mark = \"✅\" if row['correct'] else \"❌\"\n",
+        "    output_display = row['output'][:38] + \"..\" if len(row['output']) > 40 else row['output']\n",
+        "    print(f\"{row['word']:<15} | {row['meaning']:<15} | {output_display:<40} | {collapse_mark:<10} | {correct_mark}\")\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 6. SUMMARY STATISTICS\n",
+        "# ==============================================================================\n",
+        "\n",
+        "print(\"\\n\" + \"=\" * 100)\n",
+        "print(\"📈 SUMMARY STATISTICS\")\n",
+        "print(\"=\" * 100)\n",
+        "\n",
+        "n_ambig = len(df_ambig)\n",
+        "n_unambig = len(df_unambig)\n",
+        "\n",
+        "ambig_collapse_rate = df_ambig['collapsed'].sum() / n_ambig * 100 if n_ambig > 0 else 0\n",
+        "unambig_collapse_rate = df_unambig['collapsed'].sum() / n_unambig * 100 if n_unambig > 0 else 0\n",
+        "unambig_correct_rate = df_unambig['correct'].sum() / n_unambig * 100 if n_unambig > 0 else 0\n",
+        "\n",
+        "print(f\"\"\"\n",
+        "┌─────────────────────────────────────────────────────────────┐\n",
+        "│                    CARRIER WAVE THRESHOLD                   │\n",
+        "├──��──────────────────────────────────────────────────────────┤\n",
+        "│  Condition          │  N      │  Collapse Rate │  Correct  │\n",
+        "├─────────────────────────────────────────────────────────────┤\n",
+        "│  AMBIGUOUS (L=1)    │  {n_ambig:<5}  │  {ambig_collapse_rate:>6.1f}%       │   N/A     │\n",
+        "│  UNAMBIGUOUS (L=1)  │  {n_unambig:<5}  │  {unambig_collapse_rate:>6.1f}%       │  {unambig_correct_rate:>5.1f}%   │\n",
+        "└─────────────────────────────────────────────────────────────┘\n",
+        "\"\"\")\n",
+        "\n",
+        "# Statistical comparison\n",
+        "print(\"🔬 STATISTICAL INTERPRETATION:\")\n",
+        "print(\"-\" * 60)\n",
+        "\n",
+        "if ambig_collapse_rate > unambig_collapse_rate + 10:\n",
+        "    print(f\"   → Ambiguous tokens collapse MORE ({ambig_collapse_rate:.1f}% vs {unambig_collapse_rate:.1f}%)\")\n",
+        "    print(f\"   → Difference: {ambig_collapse_rate - unambig_collapse_rate:.1f} percentage points\")\n",
+        "    print(f\"   → SUPPORTS: Ambiguity exacerbates L=1 failure\")\n",
+        "elif abs(ambig_collapse_rate - unambig_collapse_rate) <= 10:\n",
+        "    print(f\"   → Both collapse at similar rates ({ambig_collapse_rate:.1f}% vs {unambig_collapse_rate:.1f}%)\")\n",
+        "    print(f\"   → SUPPORTS: L=1 failure is about sequence length, not ambiguity\")\n",
+        "else:\n",
+        "    print(f\"   → Unexpected pattern - investigate further\")\n",
+        "\n",
+        "print(f\"\\n   → Unambiguous tokens that ARE correct despite collapse: {unambig_correct_rate:.1f}%\")\n",
+        "print(f\"   → This suggests embeddings DO encode meaning, but decoder loops anyway\")\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 7. EXPORT FOR PAPER\n",
+        "# ==============================================================================\n",
+        "\n",
+        "print(\"\\n\" + \"=\" * 100)\n",
+        "print(\"📝 LATEX-READY TABLE\")\n",
+        "print(\"=\" * 100)\n",
+        "\n",
+        "print(f\"\"\"\n",
+        "\\\\begin{{table}}[h]\n",
+        "\\\\centering\n",
+        "\\\\caption{{Carrier Wave Threshold Analysis (Extended). Large-scale validation confirms\n",
+        "that repetition collapse at $L=1$ affects both ambiguous and unambiguous tokens.}}\n",
+        "\\\\label{{tab:carrier_wave_extended}}\n",
+        "\\\\begin{{tabular}}{{lccc}}\n",
+        "\\\\toprule\n",
+        "\\\\textbf{{Condition}} & \\\\textbf{{N}} & \\\\textbf{{Collapse Rate}} & \\\\textbf{{Correct (if applicable)}} \\\\\\\\\n",
+        "\\\\midrule\n",
+        "Ambiguous ($L=1$) & {n_ambig} & {ambig_collapse_rate:.1f}\\\\% & N/A \\\\\\\\\n",
+        "Unambiguous ($L=1$) & {n_unambig} & {unambig_collapse_rate:.1f}\\\\% & {unambig_correct_rate:.1f}\\\\% \\\\\\\\\n",
+        "\\\\bottomrule\n",
+        "\\\\end{{tabular}}\n",
+        "\\\\end{{table}}\n",
+        "\"\"\")"
+      ],
+      "metadata": {
+        "id": "A1Ta_6di5s7E"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import torch\n",
+        "import pandas as pd\n",
+        "from tqdm import tqdm\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 1. SETUP & HELPERS\n",
+        "# ==============================================================================\n",
+        "print(\"=\" * 100)\n",
+        "print(\"🌊 PHASE 2: THE CARRIER WAVE RESURRECTION (L=2)\")\n",
+        "print(\"=\" * 100)\n",
+        "\n",
+        "def add_minimal_context(word):\n",
+        "    \"\"\"Prepends a neutral article to force L >= 2\"\"\"\n",
+        "    return f\"Das {word}\"\n",
+        "\n",
+        "def run_prism_gen_exact(text_input, max_len=10):\n",
+        "    \"\"\"\n",
+        "    Exact generation wrapper matching previous usage.\n",
+        "    \"\"\"\n",
+        "    # 1. Tokenize (Get IDs only, no special tokens)\n",
+        "    input_tensor = tokenizer(text_input, return_tensors=\"pt\", add_special_tokens=False).input_ids.to(DEVICE)\n",
+        "\n",
+        "    # 2. Call custom generate method exactly as before\n",
+        "    with torch.no_grad():\n",
+        "        out_ids = model.generate(src=input_tensor, max_length=max_len, num_beams=1)\n",
+        "\n",
+        "    # 3. Decode\n",
+        "    return tokenizer.decode(out_ids[0], skip_special_tokens=True).strip()\n",
+        "\n",
+        "results_recovery = []\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 2. RUN THE COMPARISON LOOP\n",
+        "# ==============================================================================\n",
+        "print(f\"🔄 Retesting {len(ambig_valid)} Ambiguous & {len(unambig_valid)} Unambiguous tokens with 'Das [X]'...\")\n",
+        "\n",
+        "# --- A. AMBIGUOUS RECOVERY ---\n",
+        "# Expected format: (word, meaning1, meaning2)\n",
+        "for item in tqdm(ambig_valid, desc=\"Ambiguous L=2\"):\n",
+        "    word = item[0]\n",
+        "    meanings = item[1:] # Capture all meanings provided in tuple\n",
+        "\n",
+        "    try:\n",
+        "        # Run L=1 (Single Token)\n",
+        "        out_L1 = run_prism_gen_exact(word)\n",
+        "        is_collapsed_L1 = is_repetition_collapse(out_L1)\n",
+        "\n",
+        "        # Run L=2 (Minimal Context)\n",
+        "        input_L2 = add_minimal_context(word)\n",
+        "        out_L2 = run_prism_gen_exact(input_L2)\n",
+        "        is_collapsed_L2 = is_repetition_collapse(out_L2)\n",
+        "\n",
+        "        # Check Meaning Recovery: Does L=2 output contain any valid meaning?\n",
+        "        # We assume check_correct handles a list of valid targets\n",
+        "        has_meaning_L2 = check_correct(out_L2, meanings)\n",
+        "\n",
+        "        results_recovery.append({\n",
+        "            \"Type\": \"Ambiguous\",\n",
+        "            \"Word\": word,\n",
+        "            \"L1_Output\": out_L1,\n",
+        "            \"L1_Collapse\": is_collapsed_L1,\n",
+        "            \"L2_Input\": input_L2,\n",
+        "            \"L2_Output\": out_L2,\n",
+        "            \"L2_Collapse\": is_collapsed_L2,\n",
+        "            \"L2_Success\": has_meaning_L2\n",
+        "        })\n",
+        "    except Exception as e:\n",
+        "        print(f\"Error on {word}: {e}\")\n",
+        "\n",
+        "# --- B. UNAMBIGUOUS RECOVERY ---\n",
+        "# Expected format: (word, target)\n",
+        "for item in tqdm(unambig_valid, desc=\"Unambiguous L=2\"):\n",
+        "    word = item[0]\n",
+        "    target = item[1]\n",
+        "\n",
+        "    try:\n",
+        "        # Run L=1\n",
+        "        out_L1 = run_prism_gen_exact(word)\n",
+        "        is_collapsed_L1 = is_repetition_collapse(out_L1)\n",
+        "\n",
+        "        # Run L=2\n",
+        "        input_L2 = add_minimal_context(word)\n",
+        "        out_L2 = run_prism_gen_exact(input_L2)\n",
+        "        is_collapsed_L2 = is_repetition_collapse(out_L2)\n",
+        "\n",
+        "        # Check Accuracy\n",
+        "        is_correct_L2 = check_correct(out_L2, [target])\n",
+        "\n",
+        "        results_recovery.append({\n",
+        "            \"Type\": \"Unambiguous\",\n",
+        "            \"Word\": word,\n",
+        "            \"L1_Output\": out_L1,\n",
+        "            \"L1_Collapse\": is_collapsed_L1,\n",
+        "            \"L2_Input\": input_L2,\n",
+        "            \"L2_Output\": out_L2,\n",
+        "            \"L2_Collapse\": is_collapsed_L2,\n",
+        "            \"L2_Success\": is_correct_L2\n",
+        "        })\n",
+        "    except Exception as e:\n",
+        "        print(f\"Error on {word}: {e}\")\n",
+        "\n",
+        "# ==============================================================================\n",
+        "# 3. ANALYSIS & VISUALIZATION\n",
+        "# ==============================================================================\n",
+        "df_rec = pd.DataFrame(results_recovery)\n",
+        "\n",
+        "# Metrics Calculation\n",
+        "total_ambig = len(df_rec[df_rec[\"Type\"] == \"Ambiguous\"])\n",
+        "total_unambig = len(df_rec[df_rec[\"Type\"] == \"Unambiguous\"])\n",
+        "\n",
+        "# L1 Collapse counts\n",
+        "collapse_L1_ambig = df_rec[(df_rec[\"Type\"] == \"Ambiguous\") & (df_rec[\"L1_Collapse\"])].shape[0]\n",
+        "collapse_L1_unambig = df_rec[(df_rec[\"Type\"] == \"Unambiguous\") & (df_rec[\"L1_Collapse\"])].shape[0]\n",
+        "\n",
+        "# L2 Collapse counts\n",
+        "collapse_L2_ambig = df_rec[(df_rec[\"Type\"] == \"Ambiguous\") & (df_rec[\"L2_Collapse\"])].shape[0]\n",
+        "collapse_L2_unambig = df_rec[(df_rec[\"Type\"] == \"Unambiguous\") & (df_rec[\"L2_Collapse\"])].shape[0]\n",
+        "\n",
+        "# Resurrection counts (Collapsed at L1 AND Succeeded at L2)\n",
+        "resurrected_ambig = df_rec[\n",
+        "    (df_rec[\"Type\"] == \"Ambiguous\") &\n",
+        "    (df_rec[\"L1_Collapse\"] == True) &\n",
+        "    (df_rec[\"L2_Success\"] == True)\n",
+        "].shape[0]\n",
+        "\n",
+        "resurrected_unambig = df_rec[\n",
+        "    (df_rec[\"Type\"] == \"Unambiguous\") &\n",
+        "    (df_rec[\"L1_Collapse\"] == True) &\n",
+        "    (df_rec[\"L2_Success\"] == True)\n",
+        "].shape[0]\n",
+        "\n",
+        "# Recovery Rates (Percentage of collapsed L1 tokens that were fixed)\n",
+        "recov_rate_ambig = (resurrected_ambig / collapse_L1_ambig * 100) if collapse_L1_ambig > 0 else 0\n",
+        "recov_rate_unambig = (resurrected_unambig / collapse_L1_unambig * 100) if collapse_L1_unambig > 0 else 0\n",
+        "\n",
+        "# --- ASCII TABLE OUTPUT ---\n",
+        "print(\"\\n\" + \"=\"*110)\n",
+        "print(\"🏥 THE RECOVERY WARD: L=1 vs L=2 COMPARISON\")\n",
+        "print(\"=\"*110)\n",
+        "print(f\"{'Word':<12} | {'L=1 Output (Collapsed)':<30} | {'➡️'} | {'L=2 Output (Recovered)':<30} | {'Status'}\")\n",
+        "print(\"-\" * 110)\n",
+        "\n",
+        "# Show examples of successful recoveries\n",
+        "recoveries = df_rec[(df_rec[\"L1_Collapse\"] == True) & (df_rec[\"L2_Success\"] == True)]\n",
+        "for _, row in recoveries.head(15).iterrows():\n",
+        "    l1_short = row['L1_Output'][:28] + \"..\" if len(row['L1_Output']) > 30 else row['L1_Output']\n",
+        "    l2_short = row['L2_Output'][:28] + \"..\" if len(row['L2_Output']) > 30 else row['L2_Output']\n",
+        "    print(f\"{row['Word']:<12} | {l1_short:<30} | {'➡️'} | {l2_short:<30} | {'✅ FIXED'}\")\n",
+        "\n",
+        "print(\"=\"*110)\n",
+        "\n",
+        "# --- SUMMARY STATISTICS ---\n",
+        "print(\"\\n📊 CARRIER WAVE RECOVERY STATISTICS\")\n",
+        "print(\"-\" * 80)\n",
+        "print(f\"{'Metric':<30} | {'Ambiguous':<15} | {'Unambiguous':<15}\")\n",
+        "print(\"-\" * 80)\n",
+        "print(f\"{'Total Samples':<30} | {total_ambig:<15} | {total_unambig:<15}\")\n",
+        "print(f\"{'L=1 Collapse Rate':<30} | {collapse_L1_ambig/total_ambig*100:5.1f}%          | {collapse_L1_unambig/total_unambig*100:5.1f}%\")\n",
+        "print(f\"{'L=2 Collapse Rate':<30} | {collapse_L2_ambig/total_ambig*100:5.1f}%          | {collapse_L2_unambig/total_unambig*100:5.1f}%\")\n",
+        "print(f\"{'Resurrection Rate':<30} | {recov_rate_ambig:5.1f}%          | {recov_rate_unambig:5.1f}%\")\n",
+        "print(\"-\" * 80)\n",
+        "\n",
+        "\n",
+        "print(\"DO NOT MENTION 'FIXED' TAGS. RESULTS ARE CORRECTLY INTERPRETED ON PAPER, NOT HERE.\" )"
+      ],
+      "metadata": {
+        "id": "9EemBN675ZLh"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}