corrected llm code

jupyter_notebooks/Section_2-3-4_Figure_8_deepfake_adapters.ipynb

@@ -954,7 +954,168 @@
    "id": "qwen_code",
    "metadata": {},
    "outputs": [],
-   "source": "import pandas as pd\nimport json\nimport time\nimport re\nfrom pathlib import Path\nfrom tqdm import tqdm\nfrom openai import OpenAI\n\ncurrent_dir = Path.cwd()\ninput_file = current_dir.parent / \"data/CSV/NER_POI_step02_annotated.csv\"\napi_key_file = current_dir.parent / \"misc/credentials/qwen_api_key.txt\"\n\n# === CONFIGURATION ===\nTEST_MODE = True\nTEST_SIZE = 10\nMAX_ROWS = 20000\nSAVE_INTERVAL = 10\n\nPROFESSION_CATEGORIES = [\n    \"actor\", \"adult performer\", \"singer/musician\", \"model\",\n    \"online personality\", \"public figure\", \"voice actor/ASMR\",\n    \"sports professional\", \"tv personality\"\n]\n\n# === LOAD API KEY ===\ntry:\n    with open(api_key_file) as f:\n        api_key = f.read().strip()\n    print(\"✅ Qwen API key loaded\")\n    print(f\"   Key starts with: {api_key[:10]}...\")\nexcept FileNotFoundError:\n    print(\"❌ Error: qwen_api_key.txt not found\")\n    raise\n\n# Initialize OpenAI client for Qwen (matching dashboard example)\nclient = OpenAI(\n    api_key=api_key,\n    base_url=\"https://dashscope-intl.aliyuncs.com/compatible-mode/v1\"\n)\n\n# === LOAD DATA ===\ndf = pd.read_csv(input_file)\nprint(f\"Loaded {len(df)} rows\")\n\nif TEST_MODE:\n    print(f\"Running in TEST MODE with {TEST_SIZE} samples\")\n    df = df.head(TEST_SIZE).copy()\nelif MAX_ROWS:\n    df = df.head(MAX_ROWS).copy()\n\n# === CREATE PROMPT ===\ndef create_prompt(name: str, tags: list, country: str) -> str:\n    tags_str = \", \".join(tags[:5]) if tags else \"none\"\n    country_str = country if country else \"unknown\"\n    \n    return f\"\"\"You are an expert at identifying public figures.\n\nGiven: Name: {name}, Tags: {tags_str}, Country: {country_str}\n\nProvide in JSON format:\n{{\n  \"full_name\": \"<full name>\",\n  \"gender\": \"<Male/Female/Other/Unknown>\",\n  \"professions\": [\"<profession 1>\", \"<profession 2>\", \"<profession 3>\"],\n  \"country\": \"<country>\"\n}}\n\nChoose professions ONLY from: {\", \".join(PROFESSION_CATEGORIES)}\n- Max 3 professions, ordered by relevance\n- \"online personality\" includes streamers, cosplayers, influencers\n- Return \"Unknown\" if uncertain\n\nReturn ONLY JSON, no extra text.\"\"\"\n\n# === QUERY QWEN ===\ndef query_qwen(prompt: str) -> dict:\n    \"\"\"Query Qwen using OpenAI-compatible API (matching dashboard example).\"\"\"\n    for attempt in range(3):\n        try:\n            completion = client.chat.completions.create(\n                model=\"qwen-max\",  # Note: dashboard shows \"qwen3-max\" but \"qwen-max\" works too\n                messages=[\n                    {\"role\": \"user\", \"content\": prompt}\n                ],\n                temperature=0.1,\n                max_tokens=500,\n                stream=False  # Disable streaming for easier parsing\n            )\n            \n            content = completion.choices[0].message.content\n            \n            # Parse JSON from response\n            json_match = re.search(r'\\{[^}]+\\}', content, re.DOTALL)\n            if json_match:\n                return json.loads(json_match.group())\n            \n            return None\n            \n        except Exception as e:\n            print(f\"Qwen error (attempt {attempt+1}): {e}\")\n            if attempt < 2:\n                time.sleep(2 ** attempt)\n    \n    return None\n\n# === PROCESS ===\noutput_file = current_dir.parent / f\"data/CSV/qwen_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\nindex_file = current_dir.parent / \"misc/qwen_query_index.txt\"\n\ncurrent_index = 0\nif index_file.exists():\n    with open(index_file) as f:\n        current_index = int(f.read().strip())\n\nfor col in ['full_name', 'gender', 'profession_llm', 'country']:\n    if col not in df.columns:\n        df[col] = 'Unknown'\n\nprint(f\"\\nAnnotating with QWEN-MAX (OpenAI-compatible API) - rows {current_index} to {len(df)}...\")\n\ntry:\n    for i in tqdm(range(current_index, len(df)), desc=\"Qwen\"):\n        row = df.iloc[i]\n        \n        tags = [str(row[f'tag_{j}']).lower() for j in range(1, 8)\n                if f'tag_{j}' in row and pd.notna(row[f'tag_{j}'])]\n        \n        prompt = create_prompt(\n            row.get('real_name', row.get('name', '')),\n            tags,\n            row.get('likely_country', '')\n        )\n        \n        result = query_qwen(prompt)\n        \n        if result:\n            df.at[i, 'full_name'] = result.get('full_name', 'Unknown')\n            df.at[i, 'gender'] = result.get('gender', 'Unknown')\n            profs = result.get('professions', [])\n            df.at[i, 'profession_llm'] = \", \".join(profs[:3]) if isinstance(profs, list) else str(profs)\n            df.at[i, 'country'] = result.get('country', 'Unknown')\n        \n        current_index = i + 1\n        \n        if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n            df.to_csv(output_file, index=False)\n            with open(index_file, 'w') as f:\n                f.write(str(current_index))\n            print(f\"✅ Saved after {i+1} rows\")\n        \n        time.sleep(0.5)\n    \n    print(f\"\\n✅ Done! Results: {output_file}\")\n    print(f\"\\nTop professions:\\n{df['profession_llm'].value_counts().head(10)}\")\n    \nexcept Exception as e:\n    print(f\"⚠️ Error: {e}\")\n    df.to_csv(output_file, index=False)\n    with open(index_file, 'w') as f:\n        f.write(str(current_index))"
+   "source": [
+    "import pandas as pd\n",
+    "import json\n",
+    "import time\n",
+    "import re\n",
+    "from pathlib import Path\n",
+    "from tqdm import tqdm\n",
+    "from openai import OpenAI\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "input_file = current_dir.parent / \"data/CSV/NER_POI_step02_annotated.csv\"\n",
+    "api_key_file = current_dir.parent / \"misc/credentials/qwen_api_key.txt\"\n",
+    "\n",
+    "# === CONFIGURATION ===\n",
+    "TEST_MODE = True\n",
+    "TEST_SIZE = 10\n",
+    "MAX_ROWS = 20000\n",
+    "SAVE_INTERVAL = 10\n",
+    "\n",
+    "PROFESSION_CATEGORIES = [\n",
+    "    \"actor\", \"adult performer\", \"singer/musician\", \"model\",\n",
+    "    \"online personality\", \"public figure\", \"voice actor/ASMR\",\n",
+    "    \"sports professional\", \"tv personality\"\n",
+    "]\n",
+    "\n",
+    "# === LOAD API KEY ===\n",
+    "try:\n",
+    "    with open(api_key_file) as f:\n",
+    "        api_key = f.read().strip()\n",
+    "    print(\"✅ Qwen API key loaded\")\n",
+    "    print(f\"   Key starts with: {api_key[:10]}...\")\n",
+    "except FileNotFoundError:\n",
+    "    print(\"❌ Error: qwen_api_key.txt not found\")\n",
+    "    raise\n",
+    "\n",
+    "# Initialize OpenAI client for Qwen (matching dashboard example)\n",
+    "client = OpenAI(\n",
+    "    api_key=api_key,\n",
+    "    base_url=\"https://dashscope-intl.aliyuncs.com/compatible-mode/v1\"\n",
+    ")\n",
+    "\n",
+    "# === LOAD DATA ===\n",
+    "df = pd.read_csv(input_file)\n",
+    "print(f\"Loaded {len(df)} rows\")\n",
+    "\n",
+    "if TEST_MODE:\n",
+    "    print(f\"Running in TEST MODE with {TEST_SIZE} samples\")\n",
+    "    df = df.head(TEST_SIZE).copy()\n",
+    "elif MAX_ROWS:\n",
+    "    df = df.head(MAX_ROWS).copy()\n",
+    "\n",
+    "# === CREATE PROMPT ===\n",
+    "def create_prompt(name: str, tags: list, country: str) -> str:\n",
+    "    tags_str = \", \".join(tags[:5]) if tags else \"none\"\n",
+    "    country_str = country if country else \"unknown\"\n",
+    "    \n",
+    "    return f\"\"\"You are an expert at identifying public figures.\n",
+    "\n",
+    "Given: Name: {name}, Tags: {tags_str}, Country: {country_str}\n",
+    "\n",
+    "Provide in JSON format:\n",
+    "{{\n",
+    "  \"full_name\": \"<full name>\",\n",
+    "  \"gender\": \"<Male/Female/Other/Unknown>\",\n",
+    "  \"professions\": [\"<profession 1>\", \"<profession 2>\", \"<profession 3>\"],\n",
+    "  \"country\": \"<country>\"\n",
+    "}}\n",
+    "\n",
+    "Choose professions ONLY from: {\", \".join(PROFESSION_CATEGORIES)}\n",
+    "- Max 3 professions, ordered by relevance\n",
+    "- \"online personality\" includes streamers, cosplayers, influencers\n",
+    "- Return \"Unknown\" if uncertain\n",
+    "\n",
+    "Return ONLY JSON, no extra text.\"\"\"\n",
+    "\n",
+    "# === QUERY QWEN ===\n",
+    "def query_qwen(prompt: str) -> dict:\n",
+    "    \"\"\"Query Qwen using OpenAI-compatible API (matching dashboard example).\"\"\"\n",
+    "    for attempt in range(3):\n",
+    "        try:\n",
+    "            completion = client.chat.completions.create(\n",
+    "                model=\"qwen-max\",  # Note: dashboard shows \"qwen3-max\" but \"qwen-max\" works too\n",
+    "                messages=[\n",
+    "                    {\"role\": \"user\", \"content\": prompt}\n",
+    "                ],\n",
+    "                temperature=0.1,\n",
+    "                max_tokens=500,\n",
+    "                stream=False  # Disable streaming for easier parsing\n",
+    "            )\n",
+    "            \n",
+    "            content = completion.choices[0].message.content\n",
+    "            \n",
+    "            # Parse JSON from response\n",
+    "            json_match = re.search(r'\\{[^}]+\\}', content, re.DOTALL)\n",
+    "            if json_match:\n",
+    "                return json.loads(json_match.group())\n",
+    "            \n",
+    "            return None\n",
+    "            \n",
+    "        except Exception as e:\n",
+    "            print(f\"Qwen error (attempt {attempt+1}): {e}\")\n",
+    "            if attempt < 2:\n",
+    "                time.sleep(2 ** attempt)\n",
+    "    \n",
+    "    return None\n",
+    "\n",
+    "# === PROCESS ===\n",
+    "output_file = current_dir.parent / f\"data/CSV/qwen_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
+    "index_file = current_dir.parent / \"misc/qwen_query_index.txt\"\n",
+    "\n",
+    "current_index = 0\n",
+    "if index_file.exists():\n",
+    "    with open(index_file) as f:\n",
+    "        current_index = int(f.read().strip())\n",
+    "\n",
+    "for col in ['full_name', 'gender', 'profession_llm', 'country']:\n",
+    "    if col not in df.columns:\n",
+    "        df[col] = 'Unknown'\n",
+    "\n",
+    "print(f\"\\nAnnotating with QWEN-MAX (OpenAI-compatible API) - rows {current_index} to {len(df)}...\")\n",
+    "\n",
+    "try:\n",
+    "    for i in tqdm(range(current_index, len(df)), desc=\"Qwen\"):\n",
+    "        row = df.iloc[i]\n",
+    "        \n",
+    "        tags = [str(row[f'tag_{j}']).lower() for j in range(1, 8)\n",
+    "                if f'tag_{j}' in row and pd.notna(row[f'tag_{j}'])]\n",
+    "        \n",
+    "        prompt = create_prompt(\n",
+    "            row.get('real_name', row.get('name', '')),\n",
+    "            tags,\n",
+    "            row.get('likely_country', '')\n",
+    "        )\n",
+    "        \n",
+    "        result = query_qwen(prompt)\n",
+    "        \n",
+    "        if result:\n",
+    "            df.at[i, 'full_name'] = result.get('full_name', 'Unknown')\n",
+    "            df.at[i, 'gender'] = result.get('gender', 'Unknown')\n",
+    "            profs = result.get('professions', [])\n",
+    "            df.at[i, 'profession_llm'] = \", \".join(profs[:3]) if isinstance(profs, list) else str(profs)\n",
+    "            df.at[i, 'country'] = result.get('country', 'Unknown')\n",
+    "        \n",
+    "        current_index = i + 1\n",
+    "        \n",
+    "        if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n",
+    "            df.to_csv(output_file, index=False)\n",
+    "            with open(index_file, 'w') as f:\n",
+    "                f.write(str(current_index))\n",
+    "            print(f\"✅ Saved after {i+1} rows\")\n",
+    "        \n",
+    "        time.sleep(0.5)\n",
+    "    \n",
+    "    print(f\"\\n✅ Done! Results: {output_file}\")\n",
+    "    print(f\"\\nTop professions:\\n{df['profession_llm'].value_counts().head(10)}\")\n",
+    "    \n",
+    "except Exception as e:\n",
+    "    print(f\"⚠️ Error: {e}\")\n",
+    "    df.to_csv(output_file, index=False)\n",
+    "    with open(index_file, 'w') as f:\n",
+    "        f.write(str(current_index))"
+   ]
   },
   {
    "cell_type": "markdown",
@@ -963,47 +1124,361 @@
    "source": [
     "## Llama Annotation (Together AI)\n",
     "\n",
-    "Uses **Llama-3.1-70B-Instruct-Turbo** via Together AI.\n",
+    "Uses **Llama-3.1-70B-Instruct-Turbo** via Together AI.\n",
+    "\n",
+    "**Requirements:**\n",
+    "- API Key: `misc/credentials/together_api_key.txt`\n",
+    "- Sign up: https://www.together.ai/\n",
+    "- Model: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo\n",
+    "- Cost: ~$0.90 per 1M tokens (~$5-10 for 10k entries)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "llama_code",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import json\n",
+    "import time\n",
+    "import requests\n",
+    "import re\n",
+    "from pathlib import Path\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "input_file = current_dir.parent / \"data/CSV/NER_POI_step02_annotated.csv\"\n",
+    "api_key_file = current_dir.parent / \"misc/credentials/together_api_key.txt\"\n",
+    "\n",
+    "# === CONFIGURATION ===\n",
+    "TEST_MODE = True\n",
+    "TEST_SIZE = 10\n",
+    "MAX_ROWS = 20000\n",
+    "SAVE_INTERVAL = 10\n",
+    "\n",
+    "PROFESSION_CATEGORIES = [\n",
+    "    \"actor\", \"adult performer\", \"singer/musician\", \"model\",\n",
+    "    \"online personality\", \"public figure\", \"voice actor/ASMR\",\n",
+    "    \"sports professional\", \"tv personality\"\n",
+    "]\n",
+    "\n",
+    "# === LOAD API KEY ===\n",
+    "try:\n",
+    "    with open(api_key_file) as f:\n",
+    "        api_key = f.read().strip()\n",
+    "    print(\"✅ Llama API key loaded\")\n",
+    "except FileNotFoundError:\n",
+    "    print(\"❌ Error: together_api_key.txt not found\")\n",
+    "    raise\n",
+    "\n",
+    "# === LOAD DATA ===\n",
+    "df = pd.read_csv(input_file)\n",
+    "print(f\"Loaded {len(df)} rows\")\n",
+    "\n",
+    "if TEST_MODE:\n",
+    "    print(f\"Running in TEST MODE with {TEST_SIZE} samples\")\n",
+    "    df = df.head(TEST_SIZE).copy()\n",
+    "elif MAX_ROWS:\n",
+    "    df = df.head(MAX_ROWS).copy()\n",
+    "\n",
+    "# === CREATE PROMPT ===\n",
+    "def create_prompt(name: str, tags: list, country: str) -> str:\n",
+    "    tags_str = \", \".join(tags[:5]) if tags else \"none\"\n",
+    "    country_str = country if country else \"unknown\"\n",
+    "    \n",
+    "    return f\"\"\"You are an expert at identifying public figures.\n",
+    "\n",
+    "Given: Name: {name}, Tags: {tags_str}, Country: {country_str}\n",
+    "\n",
+    "Provide in JSON format:\n",
+    "{{\n",
+    "  \"full_name\": \"<full name>\",\n",
+    "  \"gender\": \"<Male/Female/Other/Unknown>\",\n",
+    "  \"professions\": [\"<profession 1>\", \"<profession 2>\", \"<profession 3>\"],\n",
+    "  \"country\": \"<country>\"\n",
+    "}}\n",
+    "\n",
+    "Choose professions ONLY from: {\", \".join(PROFESSION_CATEGORIES)}\n",
+    "- Max 3 professions, ordered by relevance\n",
+    "- \"online personality\" includes streamers, cosplayers, influencers\n",
+    "- Return \"Unknown\" if uncertain\n",
+    "\n",
+    "Return ONLY JSON, no extra text.\"\"\"\n",
+    "\n",
+    "# === QUERY LLAMA ===\n",
+    "def query_llama(prompt: str) -> dict:\n",
+    "    url = \"https://api.together.xyz/v1/chat/completions\"\n",
+    "    \n",
+    "    headers = {\n",
+    "        \"Authorization\": f\"Bearer {api_key}\",\n",
+    "        \"Content-Type\": \"application/json\"\n",
+    "    }\n",
+    "    \n",
+    "    data = {\n",
+    "        \"model\": \"meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo\",\n",
+    "        \"messages\": [{\"role\": \"user\", \"content\": prompt}],\n",
+    "        \"temperature\": 0.1,\n",
+    "        \"max_tokens\": 500\n",
+    "    }\n",
+    "    \n",
+    "    for attempt in range(3):\n",
+    "        try:\n",
+    "            response = requests.post(url, headers=headers, json=data, timeout=30)\n",
+    "            response.raise_for_status()\n",
+    "            result = response.json()\n",
+    "            \n",
+    "            if 'choices' in result:\n",
+    "                content = result['choices'][0]['message']['content']\n",
+    "                json_match = re.search(r'\\{[^}]+\\}', content, re.DOTALL)\n",
+    "                if json_match:\n",
+    "                    return json.loads(json_match.group())\n",
+    "            return None\n",
+    "        except Exception as e:\n",
+    "            print(f\"Llama error (attempt {attempt+1}): {e}\")\n",
+    "            if attempt < 2:\n",
+    "                time.sleep(2 ** attempt)\n",
+    "    return None\n",
+    "\n",
+    "# === PROCESS ===\n",
+    "output_file = current_dir.parent / f\"data/CSV/llama_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
+    "index_file = current_dir.parent / \"misc/llama_query_index.txt\"\n",
+    "\n",
+    "current_index = 0\n",
+    "if index_file.exists():\n",
+    "    with open(index_file) as f:\n",
+    "        current_index = int(f.read().strip())\n",
+    "\n",
+    "for col in ['full_name', 'gender', 'profession_llm', 'country']:\n",
+    "    if col not in df.columns:\n",
+    "        df[col] = 'Unknown'\n",
+    "\n",
+    "print(f\"\\nAnnotating with LLAMA 3.1 70B - rows {current_index} to {len(df)}...\")\n",
+    "\n",
+    "try:\n",
+    "    for i in tqdm(range(current_index, len(df)), desc=\"Llama\"):\n",
+    "        row = df.iloc[i]\n",
+    "        \n",
+    "        tags = [str(row[f'tag_{j}']).lower() for j in range(1, 8)\n",
+    "                if f'tag_{j}' in row and pd.notna(row[f'tag_{j}'])]\n",
+    "        \n",
+    "        prompt = create_prompt(\n",
+    "            row.get('real_name', row.get('name', '')),\n",
+    "            tags,\n",
+    "            row.get('likely_country', '')\n",
+    "        )\n",
+    "        \n",
+    "        result = query_llama(prompt)\n",
+    "        \n",
+    "        if result:\n",
+    "            df.at[i, 'full_name'] = result.get('full_name', 'Unknown')\n",
+    "            df.at[i, 'gender'] = result.get('gender', 'Unknown')\n",
+    "            profs = result.get('professions', [])\n",
+    "            df.at[i, 'profession_llm'] = \", \".join(profs[:3]) if isinstance(profs, list) else str(profs)\n",
+    "            df.at[i, 'country'] = result.get('country', 'Unknown')\n",
+    "        \n",
+    "        current_index = i + 1\n",
+    "        \n",
+    "        if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n",
+    "            df.to_csv(output_file, index=False)\n",
+    "            with open(index_file, 'w') as f:\n",
+    "                f.write(str(current_index))\n",
+    "            print(f\"✅ Saved after {i+1} rows\")\n",
+    "        \n",
+    "        time.sleep(0.5)\n",
+    "    \n",
+    "    print(f\"\\n✅ Done! Results: {output_file}\")\n",
+    "    print(f\"\\nTop professions:\\n{df['profession_llm'].value_counts().head(10)}\")\n",
+    "    \n",
+    "except Exception as e:\n",
+    "    print(f\"⚠️ Error: {e}\")\n",
+    "    df.to_csv(output_file, index=False)\n",
+    "    with open(index_file, 'w') as f:\n",
+    "        f.write(str(current_index))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "mistral_header",
+   "metadata": {},
+   "source": [
+    "## Mixtral Annotation (Mistral AI)\n",
+    "\n",
+    "Uses **Open-Mixtral-8x22B** via Mistral AI API.\n",
+    "\n",
+    "**Requirements:**\n",
+    "- API Key: `misc/credentials/mistral_api_key.txt`\n",
+    "- Sign up: https://mistral.ai/\n",
+    "- Model: open-mixtral-8x22b\n",
+    "- Cost: ~$2 per 1M tokens (~$10-20 for 10k entries)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "mixtral_code",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import json\n",
+    "import time\n",
+    "import requests\n",
+    "import re\n",
+    "from pathlib import Path\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "input_file = current_dir.parent / \"data/CSV/NER_POI_step02_annotated.csv\"\n",
+    "api_key_file = current_dir.parent / \"misc/credentials/mistral_api_key.txt\"\n",
+    "\n",
+    "# === CONFIGURATION ===\n",
+    "TEST_MODE = True\n",
+    "TEST_SIZE = 10\n",
+    "MAX_ROWS = 20000\n",
+    "SAVE_INTERVAL = 10\n",
+    "\n",
+    "PROFESSION_CATEGORIES = [\n",
+    "    \"actor\", \"adult performer\", \"singer/musician\", \"model\",\n",
+    "    \"online personality\", \"public figure\", \"voice actor/ASMR\",\n",
+    "    \"sports professional\", \"tv personality\"\n",
+    "]\n",
+    "\n",
+    "# === LOAD API KEY ===\n",
+    "try:\n",
+    "    with open(api_key_file) as f:\n",
+    "        api_key = f.read().strip()\n",
+    "    print(\"✅ Mixtral API key loaded\")\n",
+    "except FileNotFoundError:\n",
+    "    print(\"❌ Error: mistral_api_key.txt not found\")\n",
+    "    raise\n",
+    "\n",
+    "# === LOAD DATA ===\n",
+    "df = pd.read_csv(input_file)\n",
+    "print(f\"Loaded {len(df)} rows\")\n",
+    "\n",
+    "if TEST_MODE:\n",
+    "    print(f\"Running in TEST MODE with {TEST_SIZE} samples\")\n",
+    "    df = df.head(TEST_SIZE).copy()\n",
+    "elif MAX_ROWS:\n",
+    "    df = df.head(MAX_ROWS).copy()\n",
     "\n",
-    "**Requirements:**\n",
-    "- API Key: `misc/credentials/together_api_key.txt`\n",
-    "- Sign up: https://www.together.ai/\n",
-    "- Model: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo\n",
-    "- Cost: ~$0.90 per 1M tokens (~$5-10 for 10k entries)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "llama_code",
-   "metadata": {},
-   "outputs": [],
-   "source": "import pandas as pd\nimport json\nimport time\nimport requests\nimport re\nfrom pathlib import Path\nfrom tqdm import tqdm\n\ncurrent_dir = Path.cwd()\ninput_file = current_dir.parent / \"data/CSV/NER_POI_step02_annotated.csv\"\napi_key_file = current_dir.parent / \"misc/credentials/together_api_key.txt\"\n\n# === CONFIGURATION ===\nTEST_MODE = True\nTEST_SIZE = 10\nMAX_ROWS = 20000\nSAVE_INTERVAL = 10\n\nPROFESSION_CATEGORIES = [\n    \"actor\", \"adult performer\", \"singer/musician\", \"model\",\n    \"online personality\", \"public figure\", \"voice actor/ASMR\",\n    \"sports professional\", \"tv personality\"\n]\n\n# === LOAD API KEY ===\ntry:\n    with open(api_key_file) as f:\n        api_key = f.read().strip()\n    print(\"✅ Llama API key loaded\")\nexcept FileNotFoundError:\n    print(\"❌ Error: together_api_key.txt not found\")\n    raise\n\n# === LOAD DATA ===\ndf = pd.read_csv(input_file)\nprint(f\"Loaded {len(df)} rows\")\n\nif TEST_MODE:\n    print(f\"Running in TEST MODE with {TEST_SIZE} samples\")\n    df = df.head(TEST_SIZE).copy()\nelif MAX_ROWS:\n    df = df.head(MAX_ROWS).copy()\n\n# === CREATE PROMPT ===\ndef create_prompt(name: str, tags: list, country: str) -> str:\n    tags_str = \", \".join(tags[:5]) if tags else \"none\"\n    country_str = country if country else \"unknown\"\n    \n    return f\"\"\"You are an expert at identifying public figures.\n\nGiven: Name: {name}, Tags: {tags_str}, Country: {country_str}\n\nProvide in JSON format:\n{{\n  \"full_name\": \"<full name>\",\n  \"gender\": \"<Male/Female/Other/Unknown>\",\n  \"professions\": [\"<profession 1>\", \"<profession 2>\", \"<profession 3>\"],\n  \"country\": \"<country>\"\n}}\n\nChoose professions ONLY from: {\", \".join(PROFESSION_CATEGORIES)}\n- Max 3 professions, ordered by relevance\n- \"online personality\" includes streamers, cosplayers, influencers\n- Return \"Unknown\" if uncertain\n\nReturn ONLY JSON, no extra text.\"\"\"\n\n# === QUERY LLAMA ===\ndef query_llama(prompt: str) -> dict:\n    url = \"https://api.together.xyz/v1/chat/completions\"\n    \n    headers = {\n        \"Authorization\": f\"Bearer {api_key}\",\n        \"Content-Type\": \"application/json\"\n    }\n    \n    data = {\n        \"model\": \"meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo\",\n        \"messages\": [{\"role\": \"user\", \"content\": prompt}],\n        \"temperature\": 0.1,\n        \"max_tokens\": 500\n    }\n    \n    for attempt in range(3):\n        try:\n            response = requests.post(url, headers=headers, json=data, timeout=30)\n            response.raise_for_status()\n            result = response.json()\n            \n            if 'choices' in result:\n                content = result['choices'][0]['message']['content']\n                json_match = re.search(r'\\{[^}]+\\}', content, re.DOTALL)\n                if json_match:\n                    return json.loads(json_match.group())\n            return None\n        except Exception as e:\n            print(f\"Llama error (attempt {attempt+1}): {e}\")\n            if attempt < 2:\n                time.sleep(2 ** attempt)\n    return None\n\n# === PROCESS ===\noutput_file = current_dir.parent / f\"data/CSV/llama_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\nindex_file = current_dir.parent / \"misc/llama_query_index.txt\"\n\ncurrent_index = 0\nif index_file.exists():\n    with open(index_file) as f:\n        current_index = int(f.read().strip())\n\nfor col in ['full_name', 'gender', 'profession_llm', 'country']:\n    if col not in df.columns:\n        df[col] = 'Unknown'\n\nprint(f\"\\nAnnotating with LLAMA 3.1 70B - rows {current_index} to {len(df)}...\")\n\ntry:\n    for i in tqdm(range(current_index, len(df)), desc=\"Llama\"):\n        row = df.iloc[i]\n        \n        tags = [str(row[f'tag_{j}']).lower() for j in range(1, 8)\n                if f'tag_{j}' in row and pd.notna(row[f'tag_{j}'])]\n        \n        prompt = create_prompt(\n            row.get('real_name', row.get('name', '')),\n            tags,\n            row.get('likely_country', '')\n        )\n        \n        result = query_llama(prompt)\n        \n        if result:\n            df.at[i, 'full_name'] = result.get('full_name', 'Unknown')\n            df.at[i, 'gender'] = result.get('gender', 'Unknown')\n            profs = result.get('professions', [])\n            df.at[i, 'profession_llm'] = \", \".join(profs[:3]) if isinstance(profs, list) else str(profs)\n            df.at[i, 'country'] = result.get('country', 'Unknown')\n        \n        current_index = i + 1\n        \n        if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n            df.to_csv(output_file, index=False)\n            with open(index_file, 'w') as f:\n                f.write(str(current_index))\n            print(f\"✅ Saved after {i+1} rows\")\n        \n        time.sleep(0.5)\n    \n    print(f\"\\n✅ Done! Results: {output_file}\")\n    print(f\"\\nTop professions:\\n{df['profession_llm'].value_counts().head(10)}\")\n    \nexcept Exception as e:\n    print(f\"⚠️ Error: {e}\")\n    df.to_csv(output_file, index=False)\n    with open(index_file, 'w') as f:\n        f.write(str(current_index))"
-  },
-  {
-   "cell_type": "markdown",
-   "id": "mistral_header",
-   "metadata": {},
-   "source": [
-    "## Mixtral Annotation (Mistral AI)\n",
+    "# === CREATE PROMPT ===\n",
+    "def create_prompt(name: str, tags: list, country: str) -> str:\n",
+    "    tags_str = \", \".join(tags[:5]) if tags else \"none\"\n",
+    "    country_str = country if country else \"unknown\"\n",
+    "    \n",
+    "    return f\"\"\"You are an expert at identifying public figures.\n",
     "\n",
-    "Uses **Open-Mixtral-8x22B** via Mistral AI API.\n",
+    "Given: Name: {name}, Tags: {tags_str}, Country: {country_str}\n",
     "\n",
-    "**Requirements:**\n",
-    "- API Key: `misc/credentials/mistral_api_key.txt`\n",
-    "- Sign up: https://mistral.ai/\n",
-    "- Model: open-mixtral-8x22b\n",
-    "- Cost: ~$2 per 1M tokens (~$10-20 for 10k entries)"
+    "Provide in JSON format:\n",
+    "{{\n",
+    "  \"full_name\": \"<full name>\",\n",
+    "  \"gender\": \"<Male/Female/Other/Unknown>\",\n",
+    "  \"professions\": [\"<profession 1>\", \"<profession 2>\", \"<profession 3>\"],\n",
+    "  \"country\": \"<country>\"\n",
+    "}}\n",
+    "\n",
+    "Choose professions ONLY from: {\", \".join(PROFESSION_CATEGORIES)}\n",
+    "- Max 3 professions, ordered by relevance\n",
+    "- \"online personality\" includes streamers, cosplayers, influencers\n",
+    "- Return \"Unknown\" if uncertain\n",
+    "\n",
+    "Return ONLY JSON, no extra text.\"\"\"\n",
+    "\n",
+    "# === QUERY MIXTRAL ===\n",
+    "def query_mixtral(prompt: str) -> dict:\n",
+    "    url = \"https://api.mistral.ai/v1/chat/completions\"\n",
+    "    \n",
+    "    headers = {\n",
+    "        \"Authorization\": f\"Bearer {api_key}\",\n",
+    "        \"Content-Type\": \"application/json\"\n",
+    "    }\n",
+    "    \n",
+    "    data = {\n",
+    "        \"model\": \"open-mixtral-8x22b\",\n",
+    "        \"messages\": [{\"role\": \"user\", \"content\": prompt}],\n",
+    "        \"temperature\": 0.1,\n",
+    "        \"max_tokens\": 500\n",
+    "    }\n",
+    "    \n",
+    "    for attempt in range(3):\n",
+    "        try:\n",
+    "            response = requests.post(url, headers=headers, json=data, timeout=30)\n",
+    "            response.raise_for_status()\n",
+    "            result = response.json()\n",
+    "            \n",
+    "            if 'choices' in result:\n",
+    "                content = result['choices'][0]['message']['content']\n",
+    "                json_match = re.search(r'\\{[^}]+\\}', content, re.DOTALL)\n",
+    "                if json_match:\n",
+    "                    return json.loads(json_match.group())\n",
+    "            return None\n",
+    "        except Exception as e:\n",
+    "            print(f\"Mixtral error (attempt {attempt+1}): {e}\")\n",
+    "            if attempt < 2:\n",
+    "                time.sleep(2 ** attempt)\n",
+    "    return None\n",
+    "\n",
+    "# === PROCESS ===\n",
+    "output_file = current_dir.parent / f\"data/CSV/mixtral_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
+    "index_file = current_dir.parent / \"misc/mixtral_query_index.txt\"\n",
+    "\n",
+    "current_index = 0\n",
+    "if index_file.exists():\n",
+    "    with open(index_file) as f:\n",
+    "        current_index = int(f.read().strip())\n",
+    "\n",
+    "for col in ['full_name', 'gender', 'profession_llm', 'country']:\n",
+    "    if col not in df.columns:\n",
+    "        df[col] = 'Unknown'\n",
+    "\n",
+    "print(f\"\\nAnnotating with MIXTRAL 8x22B - rows {current_index} to {len(df)}...\")\n",
+    "\n",
+    "try:\n",
+    "    for i in tqdm(range(current_index, len(df)), desc=\"Mixtral\"):\n",
+    "        row = df.iloc[i]\n",
+    "        \n",
+    "        tags = [str(row[f'tag_{j}']).lower() for j in range(1, 8)\n",
+    "                if f'tag_{j}' in row and pd.notna(row[f'tag_{j}'])]\n",
+    "        \n",
+    "        prompt = create_prompt(\n",
+    "            row.get('real_name', row.get('name', '')),\n",
+    "            tags,\n",
+    "            row.get('likely_country', '')\n",
+    "        )\n",
+    "        \n",
+    "        result = query_mixtral(prompt)\n",
+    "        \n",
+    "        if result:\n",
+    "            df.at[i, 'full_name'] = result.get('full_name', 'Unknown')\n",
+    "            df.at[i, 'gender'] = result.get('gender', 'Unknown')\n",
+    "            profs = result.get('professions', [])\n",
+    "            df.at[i, 'profession_llm'] = \", \".join(profs[:3]) if isinstance(profs, list) else str(profs)\n",
+    "            df.at[i, 'country'] = result.get('country', 'Unknown')\n",
+    "        \n",
+    "        current_index = i + 1\n",
+    "        \n",
+    "        if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n",
+    "            df.to_csv(output_file, index=False)\n",
+    "            with open(index_file, 'w') as f:\n",
+    "                f.write(str(current_index))\n",
+    "            print(f\"✅ Saved after {i+1} rows\")\n",
+    "        \n",
+    "        time.sleep(0.5)\n",
+    "    \n",
+    "    print(f\"\\n✅ Done! Results: {output_file}\")\n",
+    "    print(f\"\\nTop professions:\\n{df['profession_llm'].value_counts().head(10)}\")\n",
+    "    \n",
+    "except Exception as e:\n",
+    "    print(f\"⚠️ Error: {e}\")\n",
+    "    df.to_csv(output_file, index=False)\n",
+    "    with open(index_file, 'w') as f:\n",
+    "        f.write(str(current_index))"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "mixtral_code",
-   "metadata": {},
-   "outputs": [],
-   "source": "import pandas as pd\nimport json\nimport time\nimport requests\nimport re\nfrom pathlib import Path\nfrom tqdm import tqdm\n\ncurrent_dir = Path.cwd()\ninput_file = current_dir.parent / \"data/CSV/NER_POI_step02_annotated.csv\"\napi_key_file = current_dir.parent / \"misc/credentials/mistral_api_key.txt\"\n\n# === CONFIGURATION ===\nTEST_MODE = True\nTEST_SIZE = 10\nMAX_ROWS = 20000\nSAVE_INTERVAL = 10\n\nPROFESSION_CATEGORIES = [\n    \"actor\", \"adult performer\", \"singer/musician\", \"model\",\n    \"online personality\", \"public figure\", \"voice actor/ASMR\",\n    \"sports professional\", \"tv personality\"\n]\n\n# === LOAD API KEY ===\ntry:\n    with open(api_key_file) as f:\n        api_key = f.read().strip()\n    print(\"✅ Mixtral API key loaded\")\nexcept FileNotFoundError:\n    print(\"❌ Error: mistral_api_key.txt not found\")\n    raise\n\n# === LOAD DATA ===\ndf = pd.read_csv(input_file)\nprint(f\"Loaded {len(df)} rows\")\n\nif TEST_MODE:\n    print(f\"Running in TEST MODE with {TEST_SIZE} samples\")\n    df = df.head(TEST_SIZE).copy()\nelif MAX_ROWS:\n    df = df.head(MAX_ROWS).copy()\n\n# === CREATE PROMPT ===\ndef create_prompt(name: str, tags: list, country: str) -> str:\n    tags_str = \", \".join(tags[:5]) if tags else \"none\"\n    country_str = country if country else \"unknown\"\n    \n    return f\"\"\"You are an expert at identifying public figures.\n\nGiven: Name: {name}, Tags: {tags_str}, Country: {country_str}\n\nProvide in JSON format:\n{{\n  \"full_name\": \"<full name>\",\n  \"gender\": \"<Male/Female/Other/Unknown>\",\n  \"professions\": [\"<profession 1>\", \"<profession 2>\", \"<profession 3>\"],\n  \"country\": \"<country>\"\n}}\n\nChoose professions ONLY from: {\", \".join(PROFESSION_CATEGORIES)}\n- Max 3 professions, ordered by relevance\n- \"online personality\" includes streamers, cosplayers, influencers\n- Return \"Unknown\" if uncertain\n\nReturn ONLY JSON, no extra text.\"\"\"\n\n# === QUERY MIXTRAL ===\ndef query_mixtral(prompt: str) -> dict:\n    url = \"https://api.mistral.ai/v1/chat/completions\"\n    \n    headers = {\n        \"Authorization\": f\"Bearer {api_key}\",\n        \"Content-Type\": \"application/json\"\n    }\n    \n    data = {\n        \"model\": \"open-mixtral-8x22b\",\n        \"messages\": [{\"role\": \"user\", \"content\": prompt}],\n        \"temperature\": 0.1,\n        \"max_tokens\": 500\n    }\n    \n    for attempt in range(3):\n        try:\n            response = requests.post(url, headers=headers, json=data, timeout=30)\n            response.raise_for_status()\n            result = response.json()\n            \n            if 'choices' in result:\n                content = result['choices'][0]['message']['content']\n                json_match = re.search(r'\\{[^}]+\\}', content, re.DOTALL)\n                if json_match:\n                    return json.loads(json_match.group())\n            return None\n        except Exception as e:\n            print(f\"Mixtral error (attempt {attempt+1}): {e}\")\n            if attempt < 2:\n                time.sleep(2 ** attempt)\n    return None\n\n# === PROCESS ===\noutput_file = current_dir.parent / f\"data/CSV/mixtral_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\nindex_file = current_dir.parent / \"misc/mixtral_query_index.txt\"\n\ncurrent_index = 0\nif index_file.exists():\n    with open(index_file) as f:\n        current_index = int(f.read().strip())\n\nfor col in ['full_name', 'gender', 'profession_llm', 'country']:\n    if col not in df.columns:\n        df[col] = 'Unknown'\n\nprint(f\"\\nAnnotating with MIXTRAL 8x22B - rows {current_index} to {len(df)}...\")\n\ntry:\n    for i in tqdm(range(current_index, len(df)), desc=\"Mixtral\"):\n        row = df.iloc[i]\n        \n        tags = [str(row[f'tag_{j}']).lower() for j in range(1, 8)\n                if f'tag_{j}' in row and pd.notna(row[f'tag_{j}'])]\n        \n        prompt = create_prompt(\n            row.get('real_name', row.get('name', '')),\n            tags,\n            row.get('likely_country', '')\n        )\n        \n        result = query_mixtral(prompt)\n        \n        if result:\n            df.at[i, 'full_name'] = result.get('full_name', 'Unknown')\n            df.at[i, 'gender'] = result.get('gender', 'Unknown')\n            profs = result.get('professions', [])\n            df.at[i, 'profession_llm'] = \", \".join(profs[:3]) if isinstance(profs, list) else str(profs)\n            df.at[i, 'country'] = result.get('country', 'Unknown')\n        \n        current_index = i + 1\n        \n        if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n            df.to_csv(output_file, index=False)\n            with open(index_file, 'w') as f:\n                f.write(str(current_index))\n            print(f\"✅ Saved after {i+1} rows\")\n        \n        time.sleep(0.5)\n    \n    print(f\"\\n✅ Done! Results: {output_file}\")\n    print(f\"\\nTop professions:\\n{df['profession_llm'].value_counts().head(10)}\")\n    \nexcept Exception as e:\n    print(f\"⚠️ Error: {e}\")\n    df.to_csv(output_file, index=False)\n    with open(index_file, 'w') as f:\n        f.write(str(current_index))"
-  },
   {
    "cell_type": "markdown",
    "id": "gemma_header",
@@ -1027,10 +1502,168 @@
    "id": "gemma_code",
    "metadata": {},
    "outputs": [],
-   "source": "import pandas as pd\nimport json\nimport time\nimport requests\nimport re\nfrom pathlib import Path\nfrom tqdm import tqdm\n\ncurrent_dir = Path.cwd()\ninput_file = current_dir.parent / \"data/CSV/NER_POI_step02_annotated.csv\"\napi_key_file = current_dir.parent / \"misc/credentials/together_api_key.txt\"\n\n# === CONFIGURATION ===\nTEST_MODE = True\nTEST_SIZE = 10\nMAX_ROWS = 20000\nSAVE_INTERVAL = 10\n\nPROFESSION_CATEGORIES = [\n    \"actor\", \"adult performer\", \"singer/musician\", \"model\",\n    \"online personality\", \"public figure\", \"voice actor/ASMR\",\n    \"sports professional\", \"tv personality\"\n]\n\n# === LOAD API KEY ===\ntry:\n    with open(api_key_file) as f:\n        api_key = f.read().strip()\n    print(\"✅ Gemma API key loaded\")\nexcept FileNotFoundError:\n    print(\"❌ Error: together_api_key.txt not found\")\n    raise\n\n# === LOAD DATA ===\ndf = pd.read_csv(input_file)\nprint(f\"Loaded {len(df)} rows\")\n\nif TEST_MODE:\n    print(f\"Running in TEST MODE with {TEST_SIZE} samples\")\n    df = df.head(TEST_SIZE).copy()\nelif MAX_ROWS:\n    df = df.head(MAX_ROWS).copy()\n\n# === CREATE PROMPT ===\ndef create_prompt(name: str, tags: list, country: str) -> str:\n    tags_str = \", \".join(tags[:5]) if tags else \"none\"\n    country_str = country if country else \"unknown\"\n    \n    return f\"\"\"You are an expert at identifying public figures.\n\nGiven: Name: {name}, Tags: {tags_str}, Country: {country_str}\n\nProvide in JSON format:\n{{\n  \"full_name\": \"<full name>\",\n  \"gender\": \"<Male/Female/Other/Unknown>\",\n  \"professions\": [\"<profession 1>\", \"<profession 2>\", \"<profession 3>\"],\n  \"country\": \"<country>\"\n}}\n\nChoose professions ONLY from: {\", \".join(PROFESSION_CATEGORIES)}\n- Max 3 professions, ordered by relevance\n- \"online personality\" includes streamers, cosplayers, influencers\n- Return \"Unknown\" if uncertain\n\nReturn ONLY JSON, no extra text.\"\"\"\n\n# === QUERY GEMMA ===\ndef query_gemma(prompt: str) -> dict:\n    url = \"https://api.together.xyz/v1/chat/completions\"\n    \n    headers = {\n        \"Authorization\": f\"Bearer {api_key}\",\n        \"Content-Type\": \"application/json\"\n    }\n    \n    data = {\n        \"model\": \"google/gemma-2-27b-it\",\n        \"messages\": [{\"role\": \"user\", \"content\": prompt}],\n        \"temperature\": 0.1,\n        \"max_tokens\": 500\n    }\n    \n    for attempt in range(3):\n        try:\n            response = requests.post(url, headers=headers, json=data, timeout=30)\n            response.raise_for_status()\n            result = response.json()\n            \n            if 'choices' in result:\n                content = result['choices'][0]['message']['content']\n                json_match = re.search(r'\\{[^}]+\\}', content, re.DOTALL)\n                if json_match:\n                    return json.loads(json_match.group())\n            return None\n        except Exception as e:\n            print(f\"Gemma error (attempt {attempt+1}): {e}\")\n            if attempt < 2:\n                time.sleep(2 ** attempt)\n    return None\n\n# === PROCESS ===\noutput_file = current_dir.parent / f\"data/CSV/gemma_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\nindex_file = current_dir.parent / \"misc/gemma_query_index.txt\"\n\ncurrent_index = 0\nif index_file.exists():\n    with open(index_file) as f:\n        current_index = int(f.read().strip())\n\nfor col in ['full_name', 'gender', 'profession_llm', 'country']:\n    if col not in df.columns:\n        df[col] = 'Unknown'\n\nprint(f\"\\nAnnotating with GEMMA 2 27B (Google) - rows {current_index} to {len(df)}...\")\n\ntry:\n    for i in tqdm(range(current_index, len(df)), desc=\"Gemma\"):\n        row = df.iloc[i]\n        \n        tags = [str(row[f'tag_{j}']).lower() for j in range(1, 8)\n                if f'tag_{j}' in row and pd.notna(row[f'tag_{j}'])]\n        \n        prompt = create_prompt(\n            row.get('real_name', row.get('name', '')),\n            tags,\n            row.get('likely_country', '')\n        )\n        \n        result = query_gemma(prompt)\n        \n        if result:\n            df.at[i, 'full_name'] = result.get('full_name', 'Unknown')\n            df.at[i, 'gender'] = result.get('gender', 'Unknown')\n            profs = result.get('professions', [])\n            df.at[i, 'profession_llm'] = \", \".join(profs[:3]) if isinstance(profs, list) else str(profs)\n            df.at[i, 'country'] = result.get('country', 'Unknown')\n        \n        current_index = i + 1\n        \n        if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n            df.to_csv(output_file, index=False)\n            with open(index_file, 'w') as f:\n                f.write(str(current_index))\n            print(f\"✅ Saved after {i+1} rows\")\n        \n        time.sleep(0.5)\n    \n    print(f\"\\n✅ Done! Results: {output_file}\")\n    print(f\"\\nTop professions:\\n{df['profession_llm'].value_counts().head(10)}\")\n    \nexcept Exception as e:\n    print(f\"⚠️ Error: {e}\")\n    df.to_csv(output_file, index=False)\n    with open(index_file, 'w') as f:\n        f.write(str(current_index))"
+   "source": [
+    "import pandas as pd\n",
+    "import json\n",
+    "import time\n",
+    "import requests\n",
+    "import re\n",
+    "from pathlib import Path\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "input_file = current_dir.parent / \"data/CSV/NER_POI_step02_annotated.csv\"\n",
+    "api_key_file = current_dir.parent / \"misc/credentials/together_api_key.txt\"\n",
+    "\n",
+    "# === CONFIGURATION ===\n",
+    "TEST_MODE = True\n",
+    "TEST_SIZE = 10\n",
+    "MAX_ROWS = 20000\n",
+    "SAVE_INTERVAL = 10\n",
+    "\n",
+    "PROFESSION_CATEGORIES = [\n",
+    "    \"actor\", \"adult performer\", \"singer/musician\", \"model\",\n",
+    "    \"online personality\", \"public figure\", \"voice actor/ASMR\",\n",
+    "    \"sports professional\", \"tv personality\"\n",
+    "]\n",
+    "\n",
+    "# === LOAD API KEY ===\n",
+    "try:\n",
+    "    with open(api_key_file) as f:\n",
+    "        api_key = f.read().strip()\n",
+    "    print(\"✅ Gemma API key loaded\")\n",
+    "except FileNotFoundError:\n",
+    "    print(\"❌ Error: together_api_key.txt not found\")\n",
+    "    raise\n",
+    "\n",
+    "# === LOAD DATA ===\n",
+    "df = pd.read_csv(input_file)\n",
+    "print(f\"Loaded {len(df)} rows\")\n",
+    "\n",
+    "if TEST_MODE:\n",
+    "    print(f\"Running in TEST MODE with {TEST_SIZE} samples\")\n",
+    "    df = df.head(TEST_SIZE).copy()\n",
+    "elif MAX_ROWS:\n",
+    "    df = df.head(MAX_ROWS).copy()\n",
+    "\n",
+    "# === CREATE PROMPT ===\n",
+    "def create_prompt(name: str, tags: list, country: str) -> str:\n",
+    "    tags_str = \", \".join(tags[:5]) if tags else \"none\"\n",
+    "    country_str = country if country else \"unknown\"\n",
+    "    \n",
+    "    return f\"\"\"You are an expert at identifying public figures.\n",
+    "\n",
+    "Given: Name: {name}, Tags: {tags_str}, Country: {country_str}\n",
+    "\n",
+    "Provide in JSON format:\n",
+    "{{\n",
+    "  \"full_name\": \"<full name>\",\n",
+    "  \"gender\": \"<Male/Female/Other/Unknown>\",\n",
+    "  \"professions\": [\"<profession 1>\", \"<profession 2>\", \"<profession 3>\"],\n",
+    "  \"country\": \"<country>\"\n",
+    "}}\n",
+    "\n",
+    "Choose professions ONLY from: {\", \".join(PROFESSION_CATEGORIES)}\n",
+    "- Max 3 professions, ordered by relevance\n",
+    "- \"online personality\" includes streamers, cosplayers, influencers\n",
+    "- Return \"Unknown\" if uncertain\n",
+    "\n",
+    "Return ONLY JSON, no extra text.\"\"\"\n",
+    "\n",
+    "# === QUERY GEMMA ===\n",
+    "def query_gemma(prompt: str) -> dict:\n",
+    "    url = \"https://api.together.xyz/v1/chat/completions\"\n",
+    "    \n",
+    "    headers = {\n",
+    "        \"Authorization\": f\"Bearer {api_key}\",\n",
+    "        \"Content-Type\": \"application/json\"\n",
+    "    }\n",
+    "    \n",
+    "    data = {\n",
+    "        \"model\": \"google/gemma-2-27b-it\",\n",
+    "        \"messages\": [{\"role\": \"user\", \"content\": prompt}],\n",
+    "        \"temperature\": 0.1,\n",
+    "        \"max_tokens\": 500\n",
+    "    }\n",
+    "    \n",
+    "    for attempt in range(3):\n",
+    "        try:\n",
+    "            response = requests.post(url, headers=headers, json=data, timeout=30)\n",
+    "            response.raise_for_status()\n",
+    "            result = response.json()\n",
+    "            \n",
+    "            if 'choices' in result:\n",
+    "                content = result['choices'][0]['message']['content']\n",
+    "                json_match = re.search(r'\\{[^}]+\\}', content, re.DOTALL)\n",
+    "                if json_match:\n",
+    "                    return json.loads(json_match.group())\n",
+    "            return None\n",
+    "        except Exception as e:\n",
+    "            print(f\"Gemma error (attempt {attempt+1}): {e}\")\n",
+    "            if attempt < 2:\n",
+    "                time.sleep(2 ** attempt)\n",
+    "    return None\n",
+    "\n",
+    "# === PROCESS ===\n",
+    "output_file = current_dir.parent / f\"data/CSV/gemma_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
+    "index_file = current_dir.parent / \"misc/gemma_query_index.txt\"\n",
+    "\n",
+    "current_index = 0\n",
+    "if index_file.exists():\n",
+    "    with open(index_file) as f:\n",
+    "        current_index = int(f.read().strip())\n",
+    "\n",
+    "for col in ['full_name', 'gender', 'profession_llm', 'country']:\n",
+    "    if col not in df.columns:\n",
+    "        df[col] = 'Unknown'\n",
+    "\n",
+    "print(f\"\\nAnnotating with GEMMA 2 27B (Google) - rows {current_index} to {len(df)}...\")\n",
+    "\n",
+    "try:\n",
+    "    for i in tqdm(range(current_index, len(df)), desc=\"Gemma\"):\n",
+    "        row = df.iloc[i]\n",
+    "        \n",
+    "        tags = [str(row[f'tag_{j}']).lower() for j in range(1, 8)\n",
+    "                if f'tag_{j}' in row and pd.notna(row[f'tag_{j}'])]\n",
+    "        \n",
+    "        prompt = create_prompt(\n",
+    "            row.get('real_name', row.get('name', '')),\n",
+    "            tags,\n",
+    "            row.get('likely_country', '')\n",
+    "        )\n",
+    "        \n",
+    "        result = query_gemma(prompt)\n",
+    "        \n",
+    "        if result:\n",
+    "            df.at[i, 'full_name'] = result.get('full_name', 'Unknown')\n",
+    "            df.at[i, 'gender'] = result.get('gender', 'Unknown')\n",
+    "            profs = result.get('professions', [])\n",
+    "            df.at[i, 'profession_llm'] = \", \".join(profs[:3]) if isinstance(profs, list) else str(profs)\n",
+    "            df.at[i, 'country'] = result.get('country', 'Unknown')\n",
+    "        \n",
+    "        current_index = i + 1\n",
+    "        \n",
+    "        if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n",
+    "            df.to_csv(output_file, index=False)\n",
+    "            with open(index_file, 'w') as f:\n",
+    "                f.write(str(current_index))\n",
+    "            print(f\"✅ Saved after {i+1} rows\")\n",
+    "        \n",
+    "        time.sleep(0.5)\n",
+    "    \n",
+    "    print(f\"\\n✅ Done! Results: {output_file}\")\n",
+    "    print(f\"\\nTop professions:\\n{df['profession_llm'].value_counts().head(10)}\")\n",
+    "    \n",
+    "except Exception as e:\n",
+    "    print(f\"⚠️ Error: {e}\")\n",
+    "    df.to_csv(output_file, index=False)\n",
+    "    with open(index_file, 'w') as f:\n",
+    "        f.write(str(current_index))"
+   ]
   },
   {
    "cell_type": "markdown",
+   "id": "01086b35",
    "metadata": {},
    "source": [
     "## Qwen-2.5-32B Local Annotation (Ollama)\n",
@@ -1073,6 +1706,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "5dd7a803",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1304,4 +1938,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}