added local inference code

jupyter_notebooks/Section_2-3-4_Figure_8_deepfake_adapters.ipynb

@@ -1665,43 +1665,7 @@
    "cell_type": "markdown",
    "id": "01086b35",
    "metadata": {},
-   "source": [
-    "## Qwen-2.5-32B Local Annotation (Ollama)\n",
-    "\n",
-    "Run **Qwen-2.5-32B-Instruct** locally using Ollama - NO API COSTS!\n",
-    "\n",
-    "**Requirements:**\n",
-    "- Ollama installed: https://ollama.com/\n",
-    "- GPU: NVIDIA A100 80GB (recommended)\n",
-    "- RAM: 265GB system RAM\n",
-    "- Model cache: `data/models/` (auto-downloaded on first run)\n",
-    "- Model size: ~20GB download\n",
-    "- VRAM usage: ~22-25GB during inference\n",
-    "\n",
-    "**Setup:**\n",
-    "```bash\n",
-    "# 1. Install Ollama (if not installed)\n",
-    "curl -fsSL https://ollama.com/install.sh | sh\n",
-    "\n",
-    "# 2. Pull Qwen-2.5-32B-Instruct\n",
-    "ollama pull qwen2.5:32b-instruct\n",
-    "\n",
-    "# 3. Verify model is available\n",
-    "ollama list\n",
-    "```\n",
-    "\n",
-    "**Advantages:**\n",
-    "- ✅ FREE - No API costs\n",
-    "- ✅ Fast - Local inference on A100\n",
-    "- ✅ Privacy - Data never leaves your machine\n",
-    "- ✅ Offline - Works without internet\n",
-    "- ✅ High quality - 32B parameter model\n",
-    "\n",
-    "**Performance:**\n",
-    "- Speed: ~5-10 tokens/sec on A100\n",
-    "- Throughput: ~100-200 samples/hour (depends on prompt length)\n",
-    "- Cost: $0 (after model download)"
-   ]
+   "source": "## Qwen-2.5-32B Local Annotation (Hugging Face Transformers)\n\nRun **Qwen-2.5-32B-Instruct** locally using Hugging Face Transformers - NO API COSTS!\n\n**Requirements:**\n- GPU: NVIDIA A100 80GB or similar (32B model needs ~60-70GB VRAM)\n- RAM: 128GB+ system RAM recommended\n- Model cache: `data/models/` (auto-downloaded on first run)\n- Model size: ~65GB download (BF16 weights)\n\n**Installation:**\n```bash\n# Install required packages\npip install torch transformers accelerate\n```\n\n**Advantages:**\n- ✅ FREE - No API costs\n- ✅ Fast - Local inference with GPU acceleration\n- ✅ Privacy - Data never leaves your machine\n- ✅ Offline - Works without internet (after download)\n- ✅ High quality - 32B parameter instruction-tuned model\n\n**Performance:**\n- Speed: ~10-30 tokens/sec on A100 (depends on batch size)\n- Throughput: ~200-500 samples/hour\n- Cost: $0 (after model download)\n\n**Model Info:**\n- HuggingFace: `Qwen/Qwen2.5-32B-Instruct`\n- License: Apache 2.0 (commercial use allowed)\n- Context: 32K tokens"
   },
   {
    "cell_type": "code",
@@ -1709,226 +1673,7 @@
    "id": "5dd7a803",
    "metadata": {},
    "outputs": [],
-   "source": [
-    "import pandas as pd\n",
-    "import json\n",
-    "import time\n",
-    "import re\n",
-    "import subprocess\n",
-    "from pathlib import Path\n",
-    "from tqdm import tqdm\n",
-    "\n",
-    "current_dir = Path.cwd()\n",
-    "input_file = current_dir.parent / \"data/CSV/NER_POI_step02_annotated.csv\"\n",
-    "\n",
-    "# === CONFIGURATION ===\n",
-    "TEST_MODE = True\n",
-    "TEST_SIZE = 10\n",
-    "MAX_ROWS = 20000\n",
-    "SAVE_INTERVAL = 10\n",
-    "\n",
-    "# Ollama settings\n",
-    "MODEL_NAME = \"qwen2.5:32b-instruct\"  # or \"qwen2.5:32b\" for base model\n",
-    "OLLAMA_HOST = \"http://localhost:11434\"  # Default Ollama port\n",
-    "\n",
-    "PROFESSION_CATEGORIES = [\n",
-    "    \"actor\", \"adult performer\", \"singer/musician\", \"model\",\n",
-    "    \"online personality\", \"public figure\", \"voice actor/ASMR\",\n",
-    "    \"sports professional\", \"tv personality\"\n",
-    "]\n",
-    "\n",
-    "# === CHECK OLLAMA ===\n",
-    "def check_ollama():\n",
-    "    \"\"\"Check if Ollama is installed and model is available.\"\"\"\n",
-    "    try:\n",
-    "        result = subprocess.run(\n",
-    "            [\"ollama\", \"list\"],\n",
-    "            capture_output=True,\n",
-    "            text=True,\n",
-    "            timeout=5\n",
-    "        )\n",
-    "        \n",
-    "        if result.returncode == 0:\n",
-    "            print(\"✅ Ollama is installed\")\n",
-    "            \n",
-    "            # Check if model is available\n",
-    "            if MODEL_NAME in result.stdout or \"qwen2.5\" in result.stdout:\n",
-    "                print(f\"✅ Model {MODEL_NAME} is available\")\n",
-    "                return True\n",
-    "            else:\n",
-    "                print(f\"⚠️  Model {MODEL_NAME} not found\")\n",
-    "                print(f\"   Run: ollama pull {MODEL_NAME}\")\n",
-    "                return False\n",
-    "        else:\n",
-    "            print(\"❌ Ollama command failed\")\n",
-    "            return False\n",
-    "    except FileNotFoundError:\n",
-    "        print(\"❌ Ollama not installed\")\n",
-    "        print(\"   Install: curl -fsSL https://ollama.com/install.sh | sh\")\n",
-    "        return False\n",
-    "    except Exception as e:\n",
-    "        print(f\"❌ Error checking Ollama: {e}\")\n",
-    "        return False\n",
-    "\n",
-    "# Check Ollama before proceeding\n",
-    "if not check_ollama():\n",
-    "    raise RuntimeError(\"Ollama not ready. Please install Ollama and pull the model.\")\n",
-    "\n",
-    "# === LOAD DATA ===\n",
-    "df = pd.read_csv(input_file)\n",
-    "print(f\"\\nLoaded {len(df)} rows\")\n",
-    "\n",
-    "if TEST_MODE:\n",
-    "    print(f\"Running in TEST MODE with {TEST_SIZE} samples\")\n",
-    "    df = df.head(TEST_SIZE).copy()\n",
-    "elif MAX_ROWS:\n",
-    "    df = df.head(MAX_ROWS).copy()\n",
-    "\n",
-    "# === CREATE PROMPT ===\n",
-    "def create_prompt(name: str, tags: list, country: str) -> str:\n",
-    "    tags_str = \", \".join(tags[:5]) if tags else \"none\"\n",
-    "    country_str = country if country else \"unknown\"\n",
-    "    \n",
-    "    return f\"\"\"You are an expert at identifying public figures.\n",
-    "\n",
-    "Given: Name: {name}, Tags: {tags_str}, Country: {country_str}\n",
-    "\n",
-    "Provide in JSON format:\n",
-    "{{\n",
-    "  \"full_name\": \"<full name>\",\n",
-    "  \"gender\": \"<Male/Female/Other/Unknown>\",\n",
-    "  \"professions\": [\"<profession 1>\", \"<profession 2>\", \"<profession 3>\"],\n",
-    "  \"country\": \"<country>\"\n",
-    "}}\n",
-    "\n",
-    "Choose professions ONLY from: {\", \".join(PROFESSION_CATEGORIES)}\n",
-    "- Max 3 professions, ordered by relevance\n",
-    "- \"online personality\" includes streamers, cosplayers, influencers\n",
-    "- Return \"Unknown\" if uncertain\n",
-    "\n",
-    "Return ONLY JSON, no extra text.\"\"\"\n",
-    "\n",
-    "# === QUERY QWEN LOCAL ===\n",
-    "def query_qwen_local(prompt: str) -> dict:\n",
-    "    \"\"\"Query Qwen-2.5-32B via Ollama.\"\"\"\n",
-    "    for attempt in range(3):\n",
-    "        try:\n",
-    "            result = subprocess.run(\n",
-    "                [\n",
-    "                    \"ollama\", \"run\", MODEL_NAME,\n",
-    "                    prompt\n",
-    "                ],\n",
-    "                capture_output=True,\n",
-    "                text=True,\n",
-    "                timeout=60  # 60 second timeout per query\n",
-    "            )\n",
-    "            \n",
-    "            if result.returncode == 0:\n",
-    "                content = result.stdout.strip()\n",
-    "                \n",
-    "                # Parse JSON from response\n",
-    "                json_match = re.search(r'\\{[^}]+\\}', content, re.DOTALL)\n",
-    "                if json_match:\n",
-    "                    return json.loads(json_match.group())\n",
-    "                \n",
-    "                # If no JSON found, try to extract from markdown code block\n",
-    "                code_block = re.search(r'```(?:json)?\\s*({[^}]+})\\s*```', content, re.DOTALL)\n",
-    "                if code_block:\n",
-    "                    return json.loads(code_block.group(1))\n",
-    "            \n",
-    "            return None\n",
-    "        \n",
-    "        except subprocess.TimeoutExpired:\n",
-    "            print(f\"Qwen local timeout (attempt {attempt+1})\")\n",
-    "            if attempt < 2:\n",
-    "                time.sleep(2)\n",
-    "        except json.JSONDecodeError as e:\n",
-    "            print(f\"Qwen local JSON error (attempt {attempt+1}): {e}\")\n",
-    "            if attempt < 2:\n",
-    "                time.sleep(1)\n",
-    "        except Exception as e:\n",
-    "            print(f\"Qwen local error (attempt {attempt+1}): {e}\")\n",
-    "            if attempt < 2:\n",
-    "                time.sleep(2)\n",
-    "    \n",
-    "    return None\n",
-    "\n",
-    "# === PROCESS ===\n",
-    "output_file = current_dir.parent / f\"data/CSV/qwen_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
-    "index_file = current_dir.parent / \"misc/qwen_local_query_index.txt\"\n",
-    "\n",
-    "current_index = 0\n",
-    "if index_file.exists():\n",
-    "    with open(index_file) as f:\n",
-    "        current_index = int(f.read().strip())\n",
-    "\n",
-    "for col in ['full_name', 'gender', 'profession_llm', 'country']:\n",
-    "    if col not in df.columns:\n",
-    "        df[col] = 'Unknown'\n",
-    "\n",
-    "print(f\"\\nAnnotating with QWEN-2.5-32B LOCAL - rows {current_index} to {len(df)}...\")\n",
-    "print(f\"Model: {MODEL_NAME}\")\n",
-    "print(f\"This may take a while (~100-200 samples/hour on A100)\\n\")\n",
-    "\n",
-    "try:\n",
-    "    start_time = time.time()\n",
-    "    \n",
-    "    for i in tqdm(range(current_index, len(df)), desc=\"Qwen Local\"):\n",
-    "        row = df.iloc[i]\n",
-    "        \n",
-    "        tags = [str(row[f'tag_{j}']).lower() for j in range(1, 8)\n",
-    "                if f'tag_{j}' in row and pd.notna(row[f'tag_{j}'])]\n",
-    "        \n",
-    "        prompt = create_prompt(\n",
-    "            row.get('real_name', row.get('name', '')),\n",
-    "            tags,\n",
-    "            row.get('likely_country', '')\n",
-    "        )\n",
-    "        \n",
-    "        result = query_qwen_local(prompt)\n",
-    "        \n",
-    "        if result:\n",
-    "            df.at[i, 'full_name'] = result.get('full_name', 'Unknown')\n",
-    "            df.at[i, 'gender'] = result.get('gender', 'Unknown')\n",
-    "            profs = result.get('professions', [])\n",
-    "            df.at[i, 'profession_llm'] = \", \".join(profs[:3]) if isinstance(profs, list) else str(profs)\n",
-    "            df.at[i, 'country'] = result.get('country', 'Unknown')\n",
-    "        \n",
-    "        current_index = i + 1\n",
-    "        \n",
-    "        if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n",
-    "            df.to_csv(output_file, index=False)\n",
-    "            with open(index_file, 'w') as f:\n",
-    "                f.write(str(current_index))\n",
-    "            \n",
-    "            # Calculate stats\n",
-    "            elapsed = time.time() - start_time\n",
-    "            samples_done = i + 1 - (current_index - (i + 1))\n",
-    "            samples_per_hour = (samples_done / elapsed) * 3600 if elapsed > 0 else 0\n",
-    "            \n",
-    "            print(f\"✅ Saved after {i+1} rows (~{samples_per_hour:.1f} samples/hour)\")\n",
-    "    \n",
-    "    elapsed_total = time.time() - start_time\n",
-    "    print(f\"\\n✅ Done! Results: {output_file}\")\n",
-    "    print(f\"Total time: {elapsed_total/60:.1f} minutes\")\n",
-    "    print(f\"Average speed: {len(df)/(elapsed_total/3600):.1f} samples/hour\")\n",
-    "    \n",
-    "    print(f\"\\nTop professions:\\n{df['profession_llm'].value_counts().head(10)}\")\n",
-    "\n",
-    "except KeyboardInterrupt:\n",
-    "    print(f\"\\n⚠️ Interrupted by user\")\n",
-    "    df.to_csv(output_file, index=False)\n",
-    "    with open(index_file, 'w') as f:\n",
-    "        f.write(str(current_index))\n",
-    "    print(f\"Progress saved up to row {current_index}\")\n",
-    "\n",
-    "except Exception as e:\n",
-    "    print(f\"⚠️ Error: {e}\")\n",
-    "    df.to_csv(output_file, index=False)\n",
-    "    with open(index_file, 'w') as f:\n",
-    "        f.write(str(current_index))\n",
-    "    print(f\"Progress saved up to row {current_index}\")"
-   ]
+   "source": "import pandas as pd\nimport json\nimport time\nimport re\nfrom pathlib import Path\nfrom tqdm import tqdm\nimport torch\nfrom transformers import AutoModelForCausalLM, AutoTokenizer\n\ncurrent_dir = Path.cwd()\ninput_file = current_dir.parent / \"data/CSV/NER_POI_step02_annotated.csv\"\n\n# === CONFIGURATION ===\nTEST_MODE = True\nTEST_SIZE = 10\nMAX_ROWS = 20000\nSAVE_INTERVAL = 10\n\n# Model settings\nMODEL_NAME = \"Qwen/Qwen2.5-32B-Instruct\"\nCACHE_DIR = current_dir.parent / \"data/models\"\nCACHE_DIR.mkdir(parents=True, exist_ok=True)\n\nPROFESSION_CATEGORIES = [\n    \"actor\", \"adult performer\", \"singer/musician\", \"model\",\n    \"online personality\", \"public figure\", \"voice actor/ASMR\",\n    \"sports professional\", \"tv personality\"\n]\n\n# === LOAD MODEL ===\nprint(f\"Loading model: {MODEL_NAME}\")\nprint(f\"Cache directory: {CACHE_DIR}\")\nprint(f\"This may take a while on first run (~65GB download)...\\n\")\n\n# Check GPU availability\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(f\"Device: {device}\")\n\nif device == \"cpu\":\n    print(\"⚠️  WARNING: No GPU detected! Inference will be VERY slow.\")\n    print(\"   Consider using a GPU or reducing model size.\")\n\n# Load tokenizer\nprint(\"Loading tokenizer...\")\ntokenizer = AutoTokenizer.from_pretrained(\n    MODEL_NAME,\n    cache_dir=str(CACHE_DIR),\n    trust_remote_code=True\n)\nprint(\"✅ Tokenizer loaded\")\n\n# Load model with optimizations\nprint(\"Loading model (this may take several minutes)...\")\nmodel = AutoModelForCausalLM.from_pretrained(\n    MODEL_NAME,\n    cache_dir=str(CACHE_DIR),\n    torch_dtype=torch.bfloat16,  # Use BF16 for efficiency\n    device_map=\"auto\",  # Automatically distribute across GPUs\n    trust_remote_code=True\n)\nmodel.eval()  # Set to evaluation mode\nprint(\"✅ Model loaded\")\n\n# Check VRAM usage\nif torch.cuda.is_available():\n    vram_gb = torch.cuda.max_memory_allocated() / 1024**3\n    print(f\"VRAM used: {vram_gb:.2f} GB\\n\")\n\n# === LOAD DATA ===\ndf = pd.read_csv(input_file)\nprint(f\"Loaded {len(df)} rows\")\n\nif TEST_MODE:\n    print(f\"Running in TEST MODE with {TEST_SIZE} samples\")\n    df = df.head(TEST_SIZE).copy()\nelif MAX_ROWS:\n    df = df.head(MAX_ROWS).copy()\n\n# === CREATE PROMPT ===\ndef create_prompt(name: str, tags: list, country: str) -> str:\n    tags_str = \", \".join(tags[:5]) if tags else \"none\"\n    country_str = country if country else \"unknown\"\n    \n    return f\"\"\"You are an expert at identifying public figures.\n\nGiven: Name: {name}, Tags: {tags_str}, Country: {country_str}\n\nProvide in JSON format:\n{{\n  \"full_name\": \"<full name>\",\n  \"gender\": \"<Male/Female/Other/Unknown>\",\n  \"professions\": [\"<profession 1>\", \"<profession 2>\", \"<profession 3>\"],\n  \"country\": \"<country>\"\n}}\n\nChoose professions ONLY from: {\", \".join(PROFESSION_CATEGORIES)}\n- Max 3 professions, ordered by relevance\n- \"online personality\" includes streamers, cosplayers, influencers\n- Return \"Unknown\" if uncertain\n\nReturn ONLY JSON, no extra text.\"\"\"\n\n# === QUERY QWEN LOCAL ===\ndef query_qwen_local(prompt: str) -> dict:\n    \"\"\"Query Qwen-2.5-32B locally via transformers.\"\"\"\n    try:\n        # Format as chat message\n        messages = [\n            {\"role\": \"user\", \"content\": prompt}\n        ]\n        \n        # Tokenize\n        text = tokenizer.apply_chat_template(\n            messages,\n            tokenize=False,\n            add_generation_prompt=True\n        )\n        \n        inputs = tokenizer([text], return_tensors=\"pt\").to(device)\n        \n        # Generate\n        with torch.no_grad():\n            outputs = model.generate(\n                **inputs,\n                max_new_tokens=512,\n                temperature=0.1,\n                do_sample=True,\n                top_p=0.9,\n                pad_token_id=tokenizer.eos_token_id\n            )\n        \n        # Decode\n        generated_ids = outputs[0][inputs['input_ids'].shape[1]:]\n        response = tokenizer.decode(generated_ids, skip_special_tokens=True)\n        \n        # Parse JSON from response\n        json_match = re.search(r'\\{[^}]+\\}', response, re.DOTALL)\n        if json_match:\n            return json.loads(json_match.group())\n        \n        # Try markdown code block\n        code_block = re.search(r'```(?:json)?\\s*(\\{[^}]+\\})\\s*```', response, re.DOTALL)\n        if code_block:\n            return json.loads(code_block.group(1))\n        \n        return None\n        \n    except json.JSONDecodeError as e:\n        print(f\"JSON parse error: {e}\")\n        return None\n    except Exception as e:\n        print(f\"Generation error: {e}\")\n        return None\n\n# === PROCESS ===\noutput_file = current_dir.parent / f\"data/CSV/qwen_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\nindex_file = current_dir.parent / \"misc/qwen_local_query_index.txt\"\n\ncurrent_index = 0\nif index_file.exists():\n    with open(index_file) as f:\n        current_index = int(f.read().strip())\n    print(f\"Resuming from index {current_index}\")\n\nfor col in ['full_name', 'gender', 'profession_llm', 'country']:\n    if col not in df.columns:\n        df[col] = 'Unknown'\n\nprint(f\"\\nAnnotating with QWEN-2.5-32B LOCAL - rows {current_index} to {len(df)}...\")\nprint(f\"Model: {MODEL_NAME}\")\nprint(f\"This may take a while (~200-500 samples/hour on A100)\\n\")\n\ntry:\n    start_time = time.time()\n    \n    for i in tqdm(range(current_index, len(df)), desc=\"Qwen Local\"):\n        row = df.iloc[i]\n        \n        tags = [str(row[f'tag_{j}']).lower() for j in range(1, 8)\n                if f'tag_{j}' in row and pd.notna(row[f'tag_{j}'])]\n        \n        prompt = create_prompt(\n            row.get('real_name', row.get('name', '')),\n            tags,\n            row.get('likely_country', '')\n        )\n        \n        result = query_qwen_local(prompt)\n        \n        if result:\n            df.at[i, 'full_name'] = result.get('full_name', 'Unknown')\n            df.at[i, 'gender'] = result.get('gender', 'Unknown')\n            profs = result.get('professions', [])\n            df.at[i, 'profession_llm'] = \", \".join(profs[:3]) if isinstance(profs, list) else str(profs)\n            df.at[i, 'country'] = result.get('country', 'Unknown')\n        \n        current_index = i + 1\n        \n        if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n            df.to_csv(output_file, index=False)\n            with open(index_file, 'w') as f:\n                f.write(str(current_index))\n            \n            # Calculate stats\n            elapsed = time.time() - start_time\n            samples_done = current_index - (0 if not index_file.exists() else int(open(index_file).read().strip()))\n            samples_per_hour = (samples_done / elapsed) * 3600 if elapsed > 0 else 0\n            \n            print(f\"✅ Saved after {i+1} rows (~{samples_per_hour:.1f} samples/hour)\")\n    \n    elapsed_total = time.time() - start_time\n    print(f\"\\n✅ Done! Results: {output_file}\")\n    print(f\"Total time: {elapsed_total/60:.1f} minutes\")\n    print(f\"Average speed: {len(df)/(elapsed_total/3600):.1f} samples/hour\")\n    \n    print(f\"\\nTop professions:\\n{df['profession_llm'].value_counts().head(10)}\")\n    \n    # Show VRAM stats\n    if torch.cuda.is_available():\n        print(f\"\\nFinal VRAM usage: {torch.cuda.max_memory_allocated() / 1024**3:.2f} GB\")\n\nexcept KeyboardInterrupt:\n    print(f\"\\n⚠️ Interrupted by user\")\n    df.to_csv(output_file, index=False)\n    with open(index_file, 'w') as f:\n        f.write(str(current_index))\n    print(f\"Progress saved up to row {current_index}\")\n\nexcept Exception as e:\n    print(f\"⚠️ Error: {e}\")\n    df.to_csv(output_file, index=False)\n    with open(index_file, 'w') as f:\n        f.write(str(current_index))\n    print(f\"Progress saved up to row {current_index}\")"
   }
  ],
  "metadata": {
@@ -1938,4 +1683,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}