"""Seed realistic human feedback data for demo purposes. Simulates a reviewer going through 15 cases: approving some, correcting others. Creates the feedback.jsonl that powers the Human Feedback analytics page. """ import json import sqlite3 import time import random from pathlib import Path FEEDBACK_PATH = Path("data/processed/feedback.jsonl") DB_PATH = Path("data/processed/results.db") random.seed(42) # reproducible def seed_feedback(): """Generate 15 realistic feedback entries from actual pipeline extractions.""" conn = sqlite3.connect(DB_PATH) conn.row_factory = sqlite3.Row extractions = [dict(r) for r in conn.execute("SELECT * FROM extractions").fetchall()] cases = {dict(r)["case_id"]: dict(r) for r in conn.execute("SELECT * FROM cases").fetchall()} conn.close() if not extractions: print("No extractions in DB. Run pipeline first.") return # Pick 15 cases: mix of review-routed and auto-routed review_cases = [e for e in extractions if e["gate_route"] == "review"] auto_cases = [e for e in extractions if e["gate_route"] == "auto"] selected = random.sample(review_cases, min(10, len(review_cases))) selected += random.sample(auto_cases, min(5, len(auto_cases))) random.shuffle(selected) FEEDBACK_PATH.parent.mkdir(parents=True, exist_ok=True) # Clear existing FEEDBACK_PATH.write_text("") reviewable_fields = [ "root_cause_l1", "root_cause_l2", "sentiment_score", "risk_level", "confidence", "churn_risk", "review_required", ] # Correction scenarios — realistic reviewer behavior correction_templates = [ { "corrected": {"root_cause_l1": "billing", "root_cause_l2": "billing_dispute"}, "notes": "AI classified as data_loss but this is clearly a billing dispute about charges", }, { "corrected": {"risk_level": "high"}, "notes": "Risk underestimated — customer mentioned legal action, should be high", }, { "corrected": {"confidence": 0.5}, "notes": "Text is too short and ambiguous for this confidence level. Lowered.", }, { "corrected": {"root_cause_l1": "network", "risk_level": "medium"}, "notes": "Misclassified as outage but it's a network performance issue, not full outage", }, { "corrected": {"churn_risk": 0.8, "review_required": True}, "notes": "VIP customer explicitly threatening to leave. Churn risk should be much higher.", }, ] entries = [] base_time = time.time() - 86400 * 3 # start 3 days ago for i, ext in enumerate(selected): ts = base_time + i * 3600 * random.uniform(2, 8) # spread across days case_id = ext["case_id"] case_meta = cases.get(case_id, {}) if i < 8: # First 8: approve (good AI output) entry = { "timestamp": ts, "case_id": case_id, "action": "approval", "original": {}, "corrected": {}, "reviewer_notes": random.choice([ "Classification looks correct.", "Agree with AI assessment. Evidence is solid.", "Good extraction. Risk level matches my read.", "Verified — root cause and sentiment are accurate.", "Approved. Evidence quotes are all grounded in source text.", "Correct classification. No changes needed.", "AI got this right. Good confidence calibration.", "Looks good. Next actions are appropriate.", ]), "agreement": { "fields_reviewed": reviewable_fields, "fields_agreed": reviewable_fields, "agreement_rate": 1.0, }, } else: # Last 7: correct (AI made mistakes) template = correction_templates[(i - 8) % len(correction_templates)] corrected = template["corrected"] original = {k: ext.get(k) for k in corrected} agreed = [f for f in reviewable_fields if f not in corrected] entry = { "timestamp": ts, "case_id": case_id, "action": "correction", "original": original, "corrected": corrected, "reviewer_notes": template["notes"], "agreement": { "fields_reviewed": reviewable_fields, "fields_agreed": agreed, "fields_corrected": list(corrected.keys()), "agreement_rate": len(agreed) / len(reviewable_fields), }, } entries.append(entry) with open(FEEDBACK_PATH, "w") as f: for entry in entries: f.write(json.dumps(entry, ensure_ascii=False) + "\n") approvals = sum(1 for e in entries if e["action"] == "approval") corrections = len(entries) - approvals total_fields = len(entries) * len(reviewable_fields) agreed_fields = sum( len(e["agreement"].get("fields_agreed", reviewable_fields)) for e in entries ) rate = agreed_fields / total_fields if total_fields else 0 print(f"Seeded {len(entries)} feedback entries to {FEEDBACK_PATH}") print(f" Approvals: {approvals}") print(f" Corrections: {corrections}") print(f" Overall agreement rate: {rate:.0%}") if __name__ == "__main__": seed_feedback()