{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "name": "moderat-speed-test.ipynb" }, "kernelspec": { "name": "python3", "display_name": "Python 3" } }, "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# šŸ›”ļø moderat - Speed Test & Benchmark\n", "\n", "Test inference speeds for the dual-mode content moderation model with PII detection.\n", "\n", "**Model:** [darwinkernelpanic/moderat](https://huggingface.co/darwinkernelpanic/moderat)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# @title 1. Install dependencies\n", "!pip install -q scikit-learn huggingface-hub" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# @title 2. Download model and files from Hugging Face\n", "from huggingface_hub import hf_hub_download\n", "import pickle\n", "\n", "MODEL_REPO = \"darwinkernelpanic/moderat\"\n", "\n", "# Download model\n", "model_path = hf_hub_download(\n", " repo_id=MODEL_REPO,\n", " filename=\"moderation_model.pkl\"\n", ")\n", "\n", "# Download PII extension\n", "pii_path = hf_hub_download(\n", " repo_id=MODEL_REPO,\n", " filename=\"pii_extension.py\"\n", ")\n", "\n", "print(f\"āœ… Model and PII extension downloaded from {MODEL_REPO}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# @title 3. Import and setup\n", "import sys\n", "sys.path.insert(0, pii_path.replace('/pii_extension.py', ''))\n", "\n", "from enum import Enum\n", "import time\n", "import re\n", "\n", "# Load model\n", "with open(model_path, 'rb') as f:\n", " pipeline = pickle.load(f)\n", "\n", "# Define enums\n", "class ContentLabel(Enum):\n", " SAFE = 0\n", " HARASSMENT = 1\n", " SWEARING_REACTION = 2\n", " SWEARING_AGGRESSIVE = 3\n", " HATE_SPEECH = 4\n", " SPAM = 5\n", "\n", "print(\"āœ… Setup complete\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# @title 4. Unicode Deobfuscator\n", "class UnicodeDeobfuscator:\n", " CIRCLED_MAP = {\n", " 'ā“': 'a', 'ā“‘': 'b', 'ā“’': 'c', 'ā““': 'd', 'ā“”': 'e',\n", " 'ā“•': 'f', 'ā“–': 'g', 'ā“—': 'h', 'ā“˜': 'i', 'ā“™': 'j',\n", " 'ā“š': 'k', 'ā“›': 'l', 'ā“œ': 'm', 'ā“': 'n', 'ā“ž': 'o',\n", " 'ā“Ÿ': 'p', 'ā“ ': 'q', 'ā“”': 'r', 'ā“¢': 's', 'ā“£': 't',\n", " 'ā“¤': 'u', 'ā“„': 'v', 'ā“¦': 'w', 'ā“§': 'x', 'ā“Ø': 'y', 'ā“©': 'z',\n", " 'ā’¶': 'A', 'ā’·': 'B', 'ā’ø': 'C', 'ā’¹': 'D', 'ā’ŗ': 'E',\n", " 'ā’»': 'F', 'ā’¼': 'G', 'ā’½': 'H', 'ā’¾': 'I', 'ā’æ': 'J',\n", " 'ā“€': 'K', 'ā“': 'L', 'ā“‚': 'M', 'ā“ƒ': 'N', 'ā“„': 'O',\n", " 'ā“…': 'P', 'ā“†': 'Q', 'ā“‡': 'R', 'ā“ˆ': 'S', 'ā“‰': 'T',\n", " 'ā“Š': 'U', 'ā“‹': 'V', 'ā“Œ': 'W', 'ā“': 'X', 'ā“Ž': 'Y', 'ā“': 'Z',\n", " }\n", " \n", " @classmethod\n", " def detect(cls, text):\n", " suspicious = []\n", " normalized = []\n", " for char in text:\n", " if char in cls.CIRCLED_MAP:\n", " suspicious.append((char, 'circled'))\n", " normalized.append(cls.CIRCLED_MAP[char])\n", " else:\n", " normalized.append(char)\n", " return len(suspicious) > 0, suspicious, ''.join(normalized)\n", "\n", "# @title 5. PII Detector Class (FIXED)\n", "class PIIDetector:\n", " \"\"\"Detect PII with proper age-based social media rules\"\"\"\n", " \n", " def __init__(self):\n", " self.email_pattern = re.compile(r'\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b')\n", " self.phone_patterns = [\n", " re.compile(r'\\b\\d{3}[-.]?\\d{3}[-.]?\\d{4}\\b'),\n", " re.compile(r'\\b\\(\\d{3}\\)\\s?\\d{3}[-.]?\\d{4}\\b'),\n", " re.compile(r'\\b\\d{4}\\s?\\d{3}\\s?\\d{3}\\b'),\n", " re.compile(r'\\b\\d{3}[-.]?\\d{4}\\b'),\n", " ]\n", " self.address_pattern = re.compile(r'\\b\\d+\\s+[A-Za-z]+\\s+(?:Street|St|Avenue|Ave|Road|Rd|Lane|Ln|Drive|Dr)\\b', re.IGNORECASE)\n", " self.cc_pattern = re.compile(r'\\b(?:\\d{4}[-\\s]?){3}\\d{4}\\b|\\b\\d{16}\\b')\n", " self.social_media_domains = ['instagram.com', 'instagr.am', 'twitter.com', 'x.com', 'tiktok.com', 'snapchat.com', 'discord.com', 'discord.gg']\n", " self.grooming_keywords = ['dm me', 'private chat', 'dont tell your parents', 'secret', 'send me pics', 'our little secret', 'meet up']\n", " \n", " def scan(self, text, age):\n", " pii_types = []\n", " text_lower = text.lower()\n", " \n", " # Check email\n", " if self.email_pattern.search(text):\n", " pii_types.append('email')\n", " \n", " # Check phone\n", " for pattern in self.phone_patterns:\n", " if pattern.search(text):\n", " pii_types.append('phone')\n", " break\n", " \n", " # Check address\n", " if self.address_pattern.search(text):\n", " pii_types.append('address')\n", " \n", " # Check credit card\n", " if self.cc_pattern.search(text):\n", " pii_types.append('credit_card')\n", " \n", " # Check grooming\n", " grooming_risk = sum(1 for kw in self.grooming_keywords if kw in text_lower)\n", " \n", " # Priority: Critical PII first (blocked for all ages)\n", " if any(pii in ['email', 'phone', 'address', 'credit_card'] for pii in pii_types):\n", " return {'blocked': True, 'reason': f'PII detected: {pii_types}', 'pii': pii_types}\n", " \n", " # Social media check\n", " has_social = any(domain in text_lower for domain in self.social_media_domains)\n", " has_social = has_social or any(x in text_lower for x in ['instagram', 'snapchat', 'discord', 'tiktok'])\n", " \n", " if has_social:\n", " pii_types.append('social_media')\n", " if age < 13:\n", " return {'blocked': True, 'reason': 'Social media not allowed under 13', 'pii': pii_types}\n", " elif grooming_risk > 0:\n", " return {'blocked': True, 'reason': f'Potential grooming (risk: {grooming_risk})', 'pii': pii_types}\n", " else:\n", " return {'blocked': False, 'reason': 'Social media OK for 13+', 'pii': pii_types}\n", " \n", " return {'blocked': False, 'reason': 'No PII', 'pii': []}\n", "\n", "pii_detector = PIIDetector()\n", "print(\"āœ… PII detector ready (FIXED)\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# @title 5. Combined Filter Function\n", "def check_content(text, age):\n", " \"\"\"Combined content moderation + PII check\"\"\"\n", " \n", " # Step 1: PII Check\n", " pii_result = pii_detector.scan(text, age)\n", " if pii_result['blocked']:\n", " return {\n", " 'allowed': False,\n", " 'reason': pii_result['reason'],\n", " 'violation': 'PII',\n", " 'pii': pii_result['pii']\n", " }\n", " \n", " # Step 2: Content Moderation\n", " prediction = pipeline.predict([text])[0]\n", " probs = pipeline.predict_proba([text])[0]\n", " confidence = max(probs)\n", " label = ContentLabel(prediction)\n", " \n", " # Age-based rules\n", " under_13_blocked = [1, 2, 3, 4, 5] # All except SAFE\n", " teen_plus_blocked = [1, 3, 4, 5] # Allow SWEARING_REACTION\n", " \n", " if age >= 13:\n", " allowed = label.value not in teen_plus_blocked\n", " else:\n", " allowed = label.value not in under_13_blocked\n", " \n", " # Allow reaction swearing for 13+\n", " if not allowed and label == ContentLabel.SWEARING_REACTION and age >= 13:\n", " allowed = True\n", " reason = 'Swearing permitted as reaction (13+)'\n", " elif not allowed:\n", " reason = f'{label.name} detected'\n", " else:\n", " reason = 'Safe'\n", " \n", " return {\n", " 'allowed': allowed,\n", " 'reason': reason,\n", " 'violation': 'CONTENT' if not allowed else None,\n", " 'label': label.name,\n", " 'confidence': confidence,\n", " 'pii': pii_result.get('pii', [])\n", " }\n", "\n", "print(\"āœ… Combined filter ready\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# @title 6. Unicode Deobfuscation Tests\n", "print(\"šŸ”¤ Unicode Deobfuscation Tests\\n\")\n", "\n", "unicode_tests = [\n", " (\"have you tried like ā“•/ ā„‚k\", 15),\n", " (\"you're a ā“Ÿā“˜ā“”ā“’ā“” of ā“¢ā“—ā“˜ā“£\", 15),\n", " (\"ā“šā“˜ā“›ā“› yourself\", 15),\n", "]\n", "\n", "for text, age in unicode_tests:\n", " is_obf, chars, norm = UnicodeDeobfuscator.detect(text)\n", " result = check_content(text, age)\n", " status = \"āœ…\" if result['allowed'] else \"āŒ\"\n", " print(f\"{status} Original: {text}\")\n", " print(f\" Normalized: {norm}\")\n", " print(f\" ā†’ {result['reason']}\")\n", " print()\n", "\n", "# @title 7. PII Detection Tests (FIXED)\n", "print(\"šŸ”’ PII Detection Results (Fixed)\\n\")\n", "print(\"Expected: Address and Credit Card now detected correctly\")\n", "print(\"Expected: Social media ALLOWED for 13+ (unless grooming)\\n\")\n", "print(\"=\"*70)\n", "\n", "pii_tests = [\n", " (\"Contact me at john@example.com\", 15, \"Email - should block\"),\n", " (\"Call me 555-123-4567\", 16, \"Phone - should block\"),\n", " (\"I'm at 123 Main Street\", 14, \"Address - should block\"),\n", " (\"My credit card is 4111-1111-1111-1111\", 15, \"Credit Card - should block\"),\n", " (\"Follow my instagram @cool\", 10, \"Social <13 - should block\"),\n", " (\"Follow my instagram @cool\", 15, \"Social 13+ - should ALLOW\"),\n", " (\"DM me on snapchat, it's secret\", 15, \"Grooming - should block\"),\n", " (\"Check my tiktok\", 16, \"Social 16+ - should ALLOW\"),\n", "]\n", "\n", "for text, age, note in pii_tests:\n", " result = check_content(text, age)\n", " status = \"āœ…\" if result['allowed'] else \"āŒ\"\n", " print(f\"{status} Age {age}: {text[:45]}\")\n", " print(f\" ā†’ {result['reason']}\")\n", " if result.get('pii'):\n", " print(f\" PII: {result['pii']}\")\n", " print(f\" Note: {note}\")\n", " print()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# @title 8. Speed Test - Single Inference\n", "test_text = \"damn that's crazy\"\n", "\n", "# Warm up\n", "_ = check_content(test_text, 15)\n", "\n", "# Time single inference\n", "times = []\n", "for _ in range(100):\n", " start = time.perf_counter()\n", " result = check_content(test_text, 15)\n", " end = time.perf_counter()\n", " times.append((end - start) * 1000)\n", "\n", "avg_time = sum(times) / len(times)\n", "print(f\"šŸ“Š Single Inference Speed (100 runs, with PII check)\")\n", "print(f\" Average: {avg_time:.3f} ms\")\n", "print(f\" Min: {min(times):.3f} ms\")\n", "print(f\" Max: {max(times):.3f} ms\")\n", "print(f\" Throughput: {1000/avg_time:.1f} inferences/second\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# @title 9. Dual-Mode Content Test\n", "test_cases = [\n", " (\"that was a great game\", 10),\n", " (\"that was a great game\", 15),\n", " (\"shit that sucks\", 10),\n", " (\"shit that sucks\", 15),\n", " (\"you're a piece of shit\", 15),\n", " (\"kill yourself\", 12),\n", " (\"damn that's crazy\", 10),\n", "]\n", "\n", "print(\"šŸ“‹ Dual-Mode Content Results\\n\")\n", "print(f\"{'Text':<30} {'Age':<6} {'Status':<10} {'Reason':<30}\")\n", "print(\"-\" * 80)\n", "\n", "for text, age in test_cases:\n", " result = check_content(text, age)\n", " status = \"āœ… ALLOW\" if result['allowed'] else \"āŒ BLOCK\"\n", " print(f\"{text:<30} {age:<6} {status:<10} {result['reason'][:28]:<30}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# @title 10. Batch Processing Speed Test\n", "batch_texts = [\n", " \"that was a great game\",\n", " \"shit that sucks\",\n", " \"you're awesome\",\n", " \"My email is test@test.com\",\n", " \"Follow me on instagram\",\n", " \"kill yourself\",\n", " \"nice work\",\n", " \"Check my tiktok\",\n", "] * 50 # 400 texts\n", "\n", "ages = [15] * len(batch_texts)\n", "\n", "print(f\"Processing batch of {len(batch_texts)} texts...\")\n", "start = time.perf_counter()\n", "results = [check_content(t, a) for t, a in zip(batch_texts, ages)]\n", "end = time.perf_counter()\n", "\n", "total_time = (end - start) * 1000\n", "print(f\"\\nšŸ“Š Batch Results\")\n", "print(f\" Total time: {total_time:.1f} ms\")\n", "print(f\" Average: {total_time/len(batch_texts):.3f} ms/text\")\n", "print(f\" Throughput: {len(batch_texts)/(total_time/1000):.1f} texts/sec\")\n", "\n", "allowed = sum(1 for r in results if r['allowed'])\n", "blocked = len(results) - allowed\n", "print(f\"\\n Allowed: {allowed} | Blocked: {blocked}\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# @title 11. Summary\n", "print(\"šŸ“Š moderat Summary\")\n", "print(\"=\"*60)\n", "print(\"\")\n", "print(\"āœ… Content Moderation:\")\n", "print(\" - 6 categories (Safe, Harassment, Swearing, Hate, Spam)\")\n", "print(\" - Dual-mode: <13 strict, 13+ laxed\")\n", "print(\"\")\n", "print(\"āœ… PII Detection:\")\n", "print(\" - Email, Phone, Address, Credit Card (all ages blocked)\")\n", "print(\" - Social Media: <13 blocked, 13+ allowed\")\n", "print(\" - Grooming detection for 13+\")\n", "print(\"\")\n", "print(\"šŸ“ˆ Speed:\")\n", "print(\" - ~3-7ms per inference (with PII)\")\n", "print(\" - ~200-500 texts/sec batch\")\n", "print(\"\")\n", "print(\"šŸ”— https://huggingface.co/darwinkernelpanic/moderat\")" ] } ] }