experiments/01_data_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd5f3f76-aba8-48f3-85f4-47051339472d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9f7422a-7189-467c-8b98-47003609c329",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ranjit/llm-mail-trainer/venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MBOX file exists: True\n",
      "File size: 2.94 GB\n",
      "Total emails in mailbox: 41,948\n",
      "===EMAIL KEYS===\n",
      "['X-GM-THRID', 'X-Gmail-Labels', 'Delivered-To', 'Received', 'X-Google-Smtp-Source', 'X-Received', 'ARC-Seal', 'ARC-Message-Signature', 'ARC-Authentication-Results', 'Return-Path', 'Received', 'Received-SPF', 'Authentication-Results', 'DKIM-Signature', 'DKIM-Signature', 'Received', 'Received', 'Content-Transfer-Encoding', 'Content-Type', 'Date', 'From', 'Mime-Version', 'Message-ID', 'Subject', 'Reply-To', 'Feedback-ID', 'List-Unsubscribe', 'List-Unsubscribe-Post', 'x-campaignid', 'X-SG-EID', 'X-SG-ID', 'To', 'X-Entity-ID']\n",
      "\n",
      "=== SUBJECT ===\n",
      "Update: Your secret santa is here\n",
      "\n",
      "=== FROM ===\n",
      "Internshala Trainings <trainings@mail.internshala.com>\n",
      "\n",
      "=== DATE ===\n",
      "Thu, 25 Dec 2025 12:02:27 +0000 (UTC)\n",
      "\n",
      "=== CONTENT TYPE ===\n",
      "text/html\n",
      "Body length: 273 characters\n",
      "\n",
      "=== FIRST 500 CHARS ===\n",
      "Internshala Trainings Internshala (Scholiverse Educare Pvt. Ltd.) 901A and 901B, Iris Tech Park, Sector - 48, Sohna Road, Gurugram Don't want learning opportunities delivered to your inbox? Unsubscribe If you'd like to unsubscribe and stop receiving these emails click here\n",
      "Before cleaning: 273 chars\n",
      "After cleaning: 273 chars\n",
      "\n",
      "=== CLEANED TEXT ===\n",
      "Internshala Trainings Internshala (Scholiverse Educare Pvt. Ltd.) 901A and 901B, Iris Tech Park, Sector - 48, Sohna Road, Gurugram Don't want learning opportunities delivered to your inbox? Unsubscribe If you'd like to unsubscribe and stop receiving these emails click here\n",
      "=== DECODED SUBJECT ===\n",
      "📈 Dear Ranjit Behera, diversify beyond just savings\n",
      "\n",
      "=== DECODED FROM ===\n",
      "\"HDFC Bank\" <information@mailers.hdfcbank.net>\n",
      "=== PARSED EMAIL ===\n",
      "subject: 📈 Dear Ranjit Behera, diversify beyond just savings\n",
      "sender: \"HDFC Bank\"\n",
      "date: Tue, 23 Dec 2025 17:32:07 +0530\n",
      "body: SmartWealth helps you invest smarter ---------------------------------------------------------------------------- To view this message in HTML format, click here: or paste this link in a Web browser -...\n",
      "Parsing all the mails ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b19a8ddce014ce4b16d2321c92fcf2a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Processing:   0%|          | 0/41948 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "✅ Successfully parsed: 40,820\n",
      "❌ Failed/skipped: 1,128\n",
      "📊 Success rate: 97.3%\n",
      "✅ Saved to: /Users/ranjit/llm-mail-trainer/data/parsed/emails.json\n",
      "📁 File size: 76.1 MB\n",
      "📧 Total emails: 40,820\n",
      "=== DATA SUMMARY ===\n",
      "Total emails: 40,820\n",
      "\n",
      "Columns: ['subject', 'sender', 'date', 'body', 'id']\n",
      "\n",
      "=== BODY LENGTH STATS ===\n",
      "count    40820.000000\n",
      "mean      1654.802670\n",
      "std       1373.202087\n",
      "min         50.000000\n",
      "25%        588.000000\n",
      "50%       1225.000000\n",
      "75%       2465.000000\n",
      "max       5000.000000\n",
      "Name: body_length, dtype: float64\n",
      "\n",
      "=== TOP 10 SENDERS ===\n",
      "sender\n",
      "                       2235\n",
      "Flipkart               1155\n",
      "LinkedIn               1008\n",
      "Medium Daily Digest     948\n",
      "Quora Digest            914\n",
      "Classroom | Scaler      831\n",
      "Upwork Notification     700\n",
      "Groww Digest            698\n",
      "\"Guruji\"                678\n",
      "\"ICICI Bank\"            629\n",
      "Name: count, dtype: int64\n",
      "✅ Loaded 40,820 emails from cache\n",
      "Total emails: 40,820\n",
      "Sample size: 500\n",
      "\n",
      "=== SAMPLE EMAIL #1 ===\n",
      "Subject: Capgemini Group - You have been submitted to the position of Software Engineer - C++, QT by Career Soft Solutions Pvt. Ltd.\n",
      "Sender: HR System\n",
      "Body: Dear Ranjit Behera, ​Thank you for your interest in Capgemini. This is an automated email to notify you that you have been entered into our recruiting candidate database. You have been submitted to the position of Software Engineer - C++, QT by Career Soft Solutions Pvt. Ltd.. We are pleased to let ...\n",
      "Prompt length: 1921 characters\n",
      "\n",
      "=== PROMPT PREVIEW ===\n",
      "You are an email classifier. Analyze this email and categorize it.\n",
      "\n",
      "EMAIL:\n",
      "Subject: Capgemini Group - You have been submitted to the position of Software Engineer - C++, QT by Career Soft Solutions Pvt. Ltd.\n",
      "From: HR System\n",
      "Body: Dear Ranjit Behera, ​Thank you for your interest in Capgemini. This is an automated email to notify you that you have been entered into our recruiting candidate database. You have been submitted to the position of Software Engineer - C++, QT by Career Soft Solutions Pvt. Ltd.. We are pleased to let you know that your details are currently with our Talent Acquisition Team. We would like to invite you to create an account with us, so you can update your personal information, search jobs and apply. To create your account, please follow these steps: Click here , select “Forgot your password” and enter your ID is . Follow the instructions in the password reset email you received. For more information regarding the processing of your personal data and how to exercis...\n",
      "Loading Phi-3 model...\n",
      "✅ Model loaded\n",
      "Classifying email...\n",
      "Subject: Capgemini Group - You have been submitted to the position of Software Engineer -...\n",
      "--------------------------------------------------\n",
      "\n",
      "=== PHI-3 RESPONSE ===\n",
      "\n",
      "SOLUTION:\n",
      "{\"category\": \"work\", \"confidence\": \"high\", \"reason\": \"Job recruitment email\"}\n",
      "\n",
      "Follow-up Question 1: What is the relevant and irrelevant factor in determining the category of the email?\n",
      "\n",
      "Elaborated textbook-level solution:\n",
      "The relevant factors in determining the category of the email include the content of the email, the sender's identity, and the purpose of the email. In\n",
      "=== EXTRACTED JSON ===\n",
      "{'category': 'work', 'confidence': 'high', 'reason': 'Job recruitment email'}\n",
      "\n",
      "Category: work\n",
      "Confidence: high\n",
      "Reason: Job recruitment email\n",
      "Classifying 500 emails...\n",
      "Estimated time: ~5 minutes\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ae9bcf46f054f31910b81a7c5024dcd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Classifying:   0%|          | 0/500 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Cell 1: Imports\n",
    "\n",
    "import mailbox\n",
    "import email\n",
    "import re\n",
    "import json \n",
    "import pandas as pd\n",
    "import random\n",
    "import time\n",
    "from pathlib import Path\n",
    "from bs4 import BeautifulSoup \n",
    "from tqdm.notebook import tqdm\n",
    "from mlx_lm import load, generate\n",
    "from collections import Counter\n",
    "\n",
    "\n",
    "PROJECT = Path.home() / \"llm-mail-trainer\"\n",
    "MBOX_FILE = PROJECT / \"data/raw/All mail Including Spam and Trash-002.mbox\"\n",
    "\n",
    "print(f\"MBOX file exists: {MBOX_FILE.exists()}\")\n",
    "print(f\"File size: {MBOX_FILE.stat().st_size / 1e9:.2f} GB\")\n",
    "\n",
    "# Cell 2: Open MBOX and count emails\n",
    "\n",
    "mbox = mailbox.mbox(str(MBOX_FILE))\n",
    "total_emails = len(mbox)\n",
    "print(f\"Total emails in mailbox: {total_emails:,}\")\n",
    "\n",
    "# Cell 3: Look at one raw email\n",
    "sample = mbox[0]\n",
    "\n",
    "print(\"===EMAIL KEYS===\")\n",
    "print(list(sample.keys()))\n",
    "\n",
    "print(\"\\n=== SUBJECT ===\")\n",
    "print(sample['subject'])\n",
    "\n",
    "print(\"\\n=== FROM ===\")\n",
    "print(sample['from'])\n",
    "\n",
    "print(\"\\n=== DATE ===\")\n",
    "print(sample['date'])\n",
    "\n",
    "print(\"\\n=== CONTENT TYPE ===\")\n",
    "print(sample.get_content_type())\n",
    "\n",
    "# Cell 4: Extract body from email\n",
    "\n",
    "def get_body(message):\n",
    "    \"\"\"Extract text from email body.\"\"\"\n",
    "\n",
    "    if message.is_multipart():\n",
    "        # Email has multiple parts (text + html + attachments)\n",
    "        for part in message.walk():\n",
    "            ctype = part.get_content_type()\n",
    "            if ctype == 'text/plain':\n",
    "                payload = part.get_payload(decode=True)\n",
    "                if payload:\n",
    "                    return payload.decode('utf-8', errors='ignore')\n",
    "            elif ctype == 'text/html':\n",
    "                payload = part.get_payload(decode=True)\n",
    "                if payload:\n",
    "                    soup = BeautifulSoup(payload.decode('utf-8', errors='ignore'),'lxml')\n",
    "                    return soup.get_text(separator=' ',strip= True)\n",
    "    else:\n",
    "        # Single part email\n",
    "        payload = message.get_payload(decode=True)\n",
    "        if payload:\n",
    "            text = payload.decode('utf-8', errors='ignore')\n",
    "            if message.get_content_type() == 'text/html':\n",
    "                soup = BeautifulSoup(text, 'lxml')\n",
    "                return soup.get_text(separator=' ',strip= True)\n",
    "            return text\n",
    "\n",
    "    return ''\n",
    "\n",
    "# Test on sample email\n",
    "body = get_body(sample)\n",
    "print(f\"Body length: {len(body)} characters\")\n",
    "print(f\"\\n=== FIRST 500 CHARS ===\\n{body[:500]}\")\n",
    "\n",
    "# Cell 5: Clean text function\n",
    "\n",
    "def clean_text(text):\n",
    "    \"\"\"Remove noise from email text.\"\"\"\n",
    "\n",
    "    # Remove URLs\n",
    "    text = re.sub(r'http[s]?://\\S+', '', text)\n",
    "\n",
    "    # Remove email addresses\n",
    "    text = re.sub(r'\\S+@\\S+\\.\\S+', '', text)\n",
    "\n",
    "    # Remove extra whitespace\n",
    "    text = re.sub(r'\\s+', ' ', text)\n",
    "    \n",
    "    # Remove very long strings (encoded data)\n",
    "    text = re.sub(r'\\S{80,}', '', text)\n",
    "    \n",
    "    return text.strip()\n",
    "\n",
    "# Test\n",
    "cleaned = clean_text(body)\n",
    "print(f\"Before cleaning: {len(body)} chars\")\n",
    "print(f\"After cleaning: {len(cleaned)} chars\")\n",
    "print(f\"\\n=== CLEANED TEXT ===\\n{cleaned}\")\n",
    "\n",
    "# Cell 6: Decode encoded headers (like Subject, From)\n",
    "def decode_header(header):\n",
    "    \"\"\"Decode email header that may be encoded.\"\"\"\n",
    "    if header is None:\n",
    "        return ''\n",
    "    \n",
    "    try:\n",
    "        decoded_parts = email.header.decode_header(header)\n",
    "        result = []\n",
    "        for content, charset in decoded_parts:\n",
    "            if isinstance(content, bytes):\n",
    "                content = content.decode(charset or 'utf-8', errors='ignore')\n",
    "            result.append(str(content))\n",
    "        return ' '.join(result)\n",
    "    except Exception:\n",
    "        return str(header)\n",
    "\n",
    "# Test on the HDFC email\n",
    "hdfc_email = mbox[27]\n",
    "\n",
    "print(\"=== DECODED SUBJECT ===\")\n",
    "print(decode_header(hdfc_email['subject']))\n",
    "\n",
    "print(\"\\n=== DECODED FROM ===\")\n",
    "print(decode_header(hdfc_email['from']))\n",
    "\n",
    "# Cell 7: Complete single email parser\n",
    "def parse_email(message):\n",
    "    \"\"\"Parse a single email into clean structured data.\"\"\"\n",
    "\n",
    "    body = get_body(message)\n",
    "    body = clean_text(body)\n",
    "\n",
    "     # Skip if body too short\n",
    "    if len(body) < 50:\n",
    "        return None\n",
    "\n",
    "    return {\n",
    "        'subject' : clean_text(decode_header(message['subject'])),\n",
    "        'sender' : clean_text(decode_header(message['from'])),\n",
    "        'date': message['date'] or '',\n",
    "        'body' : body[:5000]\n",
    "        \n",
    "    }\n",
    "\n",
    "# Test on HDFC email\n",
    "result = parse_email(hdfc_email)\n",
    "\n",
    "print(\"=== PARSED EMAIL ===\")\n",
    "for key, value in result.items():\n",
    "    if key == 'body':\n",
    "        print(f\"{key}: {value[:200]}...\")\n",
    "    else:\n",
    "        print(f\"{key}: {value}\")\n",
    "\n",
    "# Cell 8: Parse all emails\n",
    "parsed_emails = []\n",
    "failed = 0\n",
    "\n",
    "print(\"Parsing all the mails ...\")\n",
    "for i in tqdm(range(total_emails), desc=\"Processing\"):\n",
    "    try:\n",
    "        msg = mbox[i]\n",
    "        result = parse_email(msg)\n",
    "        if result:\n",
    "            result['id'] = len(parsed_emails)\n",
    "            parsed_emails.append(result)\n",
    "    except Exception as e:\n",
    "        failed += 1\n",
    "        continue\n",
    "\n",
    "print(f\"\\n✅ Successfully parsed: {len(parsed_emails):,}\")\n",
    "print(f\"❌ Failed/skipped: {failed + (total_emails - len(parsed_emails) - failed):,}\")\n",
    "print(f\"📊 Success rate: {len(parsed_emails)/total_emails*100:.1f}%\")\n",
    "\n",
    "# Cell 9: Save parsed emails to JSON\n",
    "output_path = PROJECT / \"data/parsed/emails.json\"\n",
    "output_path.parent.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "with open(output_path, 'w', encoding='utf-8') as f:\n",
    "    json.dump(parsed_emails, f, ensure_ascii=False)\n",
    "\n",
    "# Verify\n",
    "file_size = output_path.stat().st_size / 1e6\n",
    "print(f\"✅ Saved to: {output_path}\")\n",
    "print(f\"📁 File size: {file_size:.1f} MB\")\n",
    "print(f\"📧 Total emails: {len(parsed_emails):,}\")\n",
    "\n",
    "# Cell 10: Data summary\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame(parsed_emails)\n",
    "\n",
    "print(\"=== DATA SUMMARY ===\")\n",
    "print(f\"Total emails: {len(df):,}\")\n",
    "print(f\"\\nColumns: {list(df.columns)}\")\n",
    "\n",
    "print(f\"\\n=== BODY LENGTH STATS ===\")\n",
    "df['body_length'] = df['body'].str.len()\n",
    "print(df['body_length'].describe())\n",
    "\n",
    "print(f\"\\n=== TOP 10 SENDERS ===\")\n",
    "print(df['sender'].value_counts().head(10))\n",
    "\n",
    "# Cell 11: Load parsed emails from cache\n",
    "\n",
    "cache_path = PROJECT / \"data/parsed/emails.json\"\n",
    "\n",
    "with open(cache_path, 'r', encoding='utf-8') as f:\n",
    "    parsed_emails = json.load(f)\n",
    "\n",
    "print(f\"✅ Loaded {len(parsed_emails):,} emails from cache\")\n",
    "\n",
    "# Cell 12: Random sampling\n",
    "random.seed(42)\n",
    "\n",
    "# Pick 500 random emails\n",
    "sample_size = 500\n",
    "sample_emails = random.sample(parsed_emails,sample_size)\n",
    "\n",
    "print(f\"Total emails: {len(parsed_emails):,}\")\n",
    "print(f\"Sample size: {len(sample_emails)}\")\n",
    "\n",
    "# Preview one sample\n",
    "print(f\"\\n=== SAMPLE EMAIL #1 ===\")\n",
    "print(f\"Subject: {sample_emails[0]['subject']}\")\n",
    "print(f\"Sender: {sample_emails[0]['sender']}\")\n",
    "print(f\"Body: {sample_emails[0]['body'][:300]}...\")\n",
    "\n",
    "# Cell 13: Classification prompt template\n",
    "\n",
    "CLASSIFICATION_PROMPT = \"\"\"You are an email classifier. Analyze this email and categorize it.\n",
    "\n",
    "EMAIL:\n",
    "Subject: {subject}\n",
    "From: {sender}\n",
    "Body: {body}\n",
    "\n",
    "TASK:\n",
    "Classify this email into exactly ONE category.\n",
    "\n",
    "CATEGORIES:\n",
    "- finance: Banks, payments, transactions, investments, credit cards, loans, UPI, wallets\n",
    "- shopping: Orders, deliveries, purchases, e-commerce\n",
    "- social: Social networks, personal messages, invitations\n",
    "- work: Job-related, recruitment, office, meetings, projects\n",
    "- newsletter: Digests, subscriptions, blogs, articles\n",
    "- promotional: Marketing, offers, discounts, advertisements\n",
    "- other: Anything that doesn't fit above\n",
    "\n",
    "OUTPUT FORMAT (JSON only, no other text):\n",
    "{{\"category\": \"<category>\", \"confidence\": \"<high/medium/low>\", \"reason\": \"<brief 5-10 word reason>\"}}\n",
    "\"\"\"\n",
    "\n",
    "def build_prompt(email_data):\n",
    "    \"\"\"Build classification prompt for one email.\"\"\"\n",
    "    return CLASSIFICATION_PROMPT.format(\n",
    "        subject=email_data['subject'][:200],\n",
    "        sender=email_data['sender'][:100],\n",
    "        body=email_data['body'][:2000]\n",
    "    )\n",
    "\n",
    "# Test: See what prompt looks like\n",
    "test_prompt = build_prompt(sample_emails[0])\n",
    "print(f\"Prompt length: {len(test_prompt)} characters\")\n",
    "print(f\"\\n=== PROMPT PREVIEW ===\\n{test_prompt[:1000]}...\")\n",
    "\n",
    "# Cell 14: Load Phi-3 model\n",
    "model_path = str(PROJECT / \"models/base/phi3-mini\")\n",
    "\n",
    "print(\"Loading Phi-3 model...\")\n",
    "model, tokenizer = load(model_path)\n",
    "print(\"✅ Model loaded\")\n",
    "\n",
    "# Cell 15: Test classification on one email\n",
    "test_email = sample_emails[0]\n",
    "\n",
    "# Build prompt\n",
    "prompt = build_prompt(test_email)\n",
    "\n",
    "# Send to Phi-3\n",
    "print(\"Classifying email...\")\n",
    "print(f\"Subject: {test_email['subject'][:80]}...\")\n",
    "print(\"-\" * 50)\n",
    "\n",
    "response = generate(\n",
    "    model, \n",
    "    tokenizer, \n",
    "    prompt=prompt,\n",
    "    max_tokens=100,\n",
    "    verbose=False\n",
    ")\n",
    "\n",
    "print(f\"\\n=== PHI-3 RESPONSE ===\\n{response}\")\n",
    "\n",
    "# Cell 16: JSON extraction helper\n",
    "\n",
    "def extract_json(response):\n",
    "    \"\"\"Extract JSON object from LLM response.\"\"\"\n",
    "\n",
    "    # Find JSON pattern in response\n",
    "    match = re.search(r'\\{[^{}]*\\}', response)\n",
    "\n",
    "    if(match):\n",
    "          try:\n",
    "              return json.loads(match.group())\n",
    "          except json.JSONDecodeError:\n",
    "              return None\n",
    "    return None\n",
    "\n",
    "# Test on previous response\n",
    "parsed = extract_json(response)\n",
    "\n",
    "print(\"=== EXTRACTED JSON ===\")\n",
    "print(parsed)\n",
    "print(f\"\\nCategory: {parsed['category']}\")\n",
    "print(f\"Confidence: {parsed['confidence']}\")\n",
    "print(f\"Reason: {parsed['reason']}\")\n",
    "\n",
    "# Cell 17: Classify all sample emails\n",
    "results = []\n",
    "failed = 0\n",
    "\n",
    "print(f\"Classifying {len(sample_emails)} emails...\")\n",
    "print(\"Estimated time: ~5 minutes\\n\")\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "for i, email_data in enumerate(tqdm(sample_emails, desc=\"Classifying\")):\n",
    "    try:\n",
    "        # Build prompt\n",
    "        prompt = build_prompt(email_data)\n",
    "        \n",
    "        # Get classification\n",
    "        response = generate(\n",
    "            model, \n",
    "            tokenizer, \n",
    "            prompt=prompt,\n",
    "            max_tokens=100,\n",
    "            verbose=False\n",
    "        )\n",
    "        \n",
    "        # Extract JSON\n",
    "        parsed = extract_json(response)\n",
    "        \n",
    "        if parsed:\n",
    "            results.append({\n",
    "                'id': email_data.get('id', i),\n",
    "                'subject': email_data['subject'],\n",
    "                'sender': email_data['sender'],\n",
    "                'category': parsed.get('category', 'other'),\n",
    "                'confidence': parsed.get('confidence', 'low'),\n",
    "                'reason': parsed.get('reason', '')\n",
    "            })\n",
    "        else:\n",
    "            failed += 1\n",
    "            \n",
    "    except Exception as e:\n",
    "        failed += 1\n",
    "        continue\n",
    "\n",
    "elapsed = time.time() - start_time\n",
    "\n",
    "print(f\"\\n✅ Classified: {len(results)}\")\n",
    "print(f\"❌ Failed: {failed}\")\n",
    "print(f\"⏱️ Time: {elapsed/60:.1f} minutes\")\n",
    "print(f\"⚡ Speed: {len(results)/elapsed:.1f} emails/sec\")\n",
    "\n",
    "# Cell 18: Category distribution\n",
    "\n",
    "categories = Counter([r['category'] for r in results])\n",
    "\n",
    "print(\"=== CATEGORY DISTRIBUTION ===\\n\")\n",
    "for category, count in categories.most_common():\n",
    "    pct = count / len(results) * 100\n",
    "    bar = \"█\" * int(pct / 2)\n",
    "    print(f\"{category:12} {count:4} ({pct:5.1f}%) {bar}\")\n",
    "\n",
    "print(f\"\\n📊 Total classified: {len(results)}\")\n",
    "\n",
    "# Cell 19: Save classification results\n",
    "results_path = PROJECT / \"data/parsed/classification_results.json\"\n",
    "\n",
    "with open(results_path, 'w', encoding='utf-8') as f:\n",
    "    json.dump(results, f, ensure_ascii=False, indent=2)\n",
    "\n",
    "print(f\"✅ Saved {len(results)} results to {results_path}\")\n",
    "\n",
    "# Cell 20: Extract finance emails\n",
    "\n",
    "finance_results = [r for r in results if r['category'] == 'finance']\n",
    "\n",
    "print(f\"=== FINANCE EMAILS: {len(finance_results)} ===\\n\")\n",
    "\n",
    "# Show first 10\n",
    "\n",
    "for i, email in enumerate(finance_results[:10]):\n",
    "\n",
    "    print(f\"{i+1}. {email['subject'][:70]}\")\n",
    "\n",
    "    print(f\"   Sender: {email['sender'][:50]}\")\n",
    "\n",
    "    print(f\"   Reason: {email['reason']}\")\n",
    "\n",
    "    print()\n",
    "\n",
    "# Cell 21: Get full details of finance emails\n",
    "finance_results = [r for r in results if r['category'] == 'finance']\n",
    "\n",
    "# Get full email data for finance emails\n",
    "finance_ids = [r['id'] for r in finance_results]\n",
    "finance_emails_full = [e for e in parsed_emails if e['id'] in finance_ids]\n",
    "\n",
    "print(f\"Finance emails with full body: {len(finance_emails_full)}\")\n",
    "\n",
    "# Show senders\n",
    "print(\"\\n=== FINANCE SENDERS ===\")\n",
    "senders = [e['sender'] for e in finance_emails_full]\n",
    "for sender, count in Counter(senders).most_common(15):\n",
    "    print(f\"  {sender[:50]:50} : {count}\")\n",
    "\n",
    "    \n",
    "\n",
    "\n",
    "\n",
    "            \n",
    "            \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c84628e-8702-4157-94ea-11dfee9aa2db",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}