Upload app.py

app.py

@@ -33,6 +33,7 @@ RATING_CHOICES = ["Bad (0)", "Borderline (1)", "Perfect (2)"]
 RATING_MAP = {"Bad (0)": 0, "Borderline (1)": 1, "Perfect (2)": 2}
 LABEL_NAMES = ["Bad", "Borderline", "Perfect"]
 BAR_COLORS = ["#ef4444", "#f59e0b", "#22c55e"]
+FAST_TRACK_THRESHOLD = 0.90  # Min confidence for 1-vote completion
 
 # ═════════════════════════════════════════════════════════════════════════
 # Globals
@@ -289,7 +290,19 @@ def check_consensus_and_route(doc_id):
         MONGO_COL.update_one({'_id': doc_id}, {'$set': {'status': 'conflict'}})
 
 
+def _max_deviation(consensus, model_classes):
+    """Calculate the maximum abs difference between human consensus and model prediction."""
+    max_diff = 0
+    for key in ('data_quality', 'regular_fit_quality', 'ai_fit_quality'):
+        h = consensus.get(key)
+        m = model_classes.get(key)
+        if h is not None and m is not None:
+            max_diff = max(max_diff, abs(h - m))
+    return max_diff
+
+
 def route_completed(doc_id):
+    """Route completed doc to granular collections based on human-model deviation."""
     if MONGO_DB is None:
         return
     doc = MONGO_COL.find_one({'_id': doc_id})
@@ -301,22 +314,28 @@ def route_completed(doc_id):
     routed = {k: v for k, v in doc.items() if k != '_id'}
     routed['original_id'] = doc['_id']
 
-    hd, hr, ha = consensus.get('data_quality'), consensus.get('regular_fit_quality'), consensus.get('ai_fit_quality')
-    md, mr, ma = mc.get('data_quality'), mc.get('regular_fit_quality'), mc.get('ai_fit_quality')
-
-    clf_match = (hd == md and hr == mr and ha == ma)
+    # --- Classification routing (granular) ---
+    dev = _max_deviation(consensus, mc)
     try:
-        tgt = 'experiments_clf_success' if clf_match else 'experiments_clf_failed'
+        if dev == 0:
+            tgt = 'experiments_clf_success'
+        elif dev >= 2:
+            tgt = 'experiments_clf_catastrophic_failure'
+        else:
+            tgt = 'experiments_clf_failed'
         MONGO_DB[tgt].update_one({'original_id': doc['_id']}, {'$set': routed}, upsert=True)
     except Exception as e:
         print(f"Route clf err: {e}")
 
+    # --- Seed vs Regular edge-case routing ---
     if not overlap:
+        hd = consensus.get('regular_fit_quality')
+        ha = consensus.get('ai_fit_quality')
         try:
-            if ha == 2 and hr is not None and hr <= 1:
+            if ha == 2 and hd is not None and hd <= 1:
                 MONGO_DB['experiments_seed_success_reg_failed'].update_one(
                     {'original_id': doc['_id']}, {'$set': routed}, upsert=True)
-            if hr == 2 and ha is not None and ha <= 1:
+            if hd == 2 and ha is not None and ha <= 1:
                 MONGO_DB['experiments_reg_success_seed_failed'].update_one(
                     {'original_id': doc['_id']}, {'$set': routed}, upsert=True)
         except Exception as e:
@@ -397,9 +416,10 @@ def fetch_next(username):
         return _empty_ws("❌ MongoDB not connected.")
 
     try:
+        # Smart queue: prioritize experiments flagged needs_second_opinion
         doc = MONGO_COL.find_one(
             {'status': 'pending', 'tagged_by': {'$nin': [username]}},
-            sort=[('num_votes', ASCENDING)])
+            sort=[('needs_second_opinion', -1), ('num_votes', ASCENDING)])
     except Exception as e:
         return _empty_ws(f"❌ DB query failed: {e}")
 
@@ -477,16 +497,58 @@ def submit_feedback(username, doc_id, overlap, model_preds,
         eval_doc['fit_quality'] = {'regular_fit': fqr, 'ai_fit': fqa}
         eval_doc['fit_comment'] = {'regular_fit': c_reg or '', 'ai_fit': c_ai or ''}
 
+    # Determine if this evaluation contains any manual corrections
+    all_sources = [dq['source']]
+    if overlap:
+        all_sources.append(eval_doc['fit_quality']['combined']['source'])
+    else:
+        all_sources.append(eval_doc['fit_quality']['regular_fit']['source'])
+        all_sources.append(eval_doc['fit_quality']['ai_fit']['source'])
+    has_correction = any(s == 'manual_correction' for s in all_sources)
+    all_implicit = all(s == 'implicit_agreement' for s in all_sources)
+
     try:
-        MONGO_COL.update_one({'_id': doc_id}, {
+        update_ops = {
             '$push': {'evaluations': eval_doc},
             '$inc': {'num_votes': 1},
             '$addToSet': {'tagged_by': username},
-        })
+        }
+        # Flag for smart queue prioritization if user corrected the model
+        if has_correction:
+            update_ops['$set'] = {'needs_second_opinion': True}
+
+        MONGO_COL.update_one({'_id': doc_id}, update_ops)
         updated = MONGO_COL.find_one({'_id': doc_id})
-        if updated and updated.get('num_votes', 0) >= VOTES_REQUIRED:
+        votes = updated.get('num_votes', 0) if updated else 0
+
+        # --- Fast-Track: 1-vote completion ---
+        if votes == 1 and all_implicit and model_preds:
+            probs = model_preds or {}
+            dp = probs.get('data_probs') or []
+            rfp = probs.get('regular_fit_probs') or []
+            afp = probs.get('ai_fit_probs') or rfp  # overlap → same as regular
+
+            high_conf = (max(dp, default=0) >= FAST_TRACK_THRESHOLD
+                         and max(rfp, default=0) >= FAST_TRACK_THRESHOLD
+                         and max(afp, default=0) >= FAST_TRACK_THRESHOLD)
+
+            if high_conf:
+                scores = _extract_scores(eval_doc)
+                consensus = {'data_quality': scores['data'],
+                             'regular_fit_quality': scores['reg_fit'],
+                             'ai_fit_quality': scores['ai_fit']}
+                MONGO_COL.update_one({'_id': doc_id},
+                    {'$set': {'status': 'completed', 'final_consensus': consensus,
+                              'fast_tracked': True}})
+                route_completed(doc_id)
+                status = "⚡ Fast-tracked! High-confidence implicit agreement."
+            else:
+                status = "✅ Saved. Awaiting 2nd vote."
+        elif votes >= VOTES_REQUIRED:
             check_consensus_and_route(doc_id)
-        status = "✅ Feedback saved!"
+            status = "✅ Feedback saved!"
+        else:
+            status = "✅ Saved. Awaiting 2nd vote."
     except Exception as e:
         status = f"❌ Save error: {e}"
 
@@ -910,6 +972,234 @@ def build_ui():
     return demo
 
 
+# ═════════════════════════════════════════════════════════════════════════
+# Validation Tests (run with: python app.py --test)
+# ═════════════════════════════════════════════════════════════════════════
+
+def run_validation_tests():
+    """Verify MLOps logic: fast-track, catastrophic routing, smart queue."""
+    print("=" * 60)
+    print("VALIDATION TESTS")
+    print("=" * 60)
+    passed = 0
+    failed = 0
+
+    def check(name, condition):
+        nonlocal passed, failed
+        if condition:
+            print(f"  ✅ {name}")
+            passed += 1
+        else:
+            print(f"  ❌ {name}")
+            failed += 1
+
+    # --- 1. _max_deviation ---
+    print("\n1. Catastrophic Failure Detection (_max_deviation)")
+    check("Perfect match = 0",
+          _max_deviation({'data_quality': 2, 'regular_fit_quality': 2, 'ai_fit_quality': 2},
+                         {'data_quality': 2, 'regular_fit_quality': 2, 'ai_fit_quality': 2}) == 0)
+    check("Off-by-1 = 1",
+          _max_deviation({'data_quality': 2, 'regular_fit_quality': 1, 'ai_fit_quality': 2},
+                         {'data_quality': 2, 'regular_fit_quality': 2, 'ai_fit_quality': 2}) == 1)
+    check("Catastrophic (2→0) = 2",
+          _max_deviation({'data_quality': 0, 'regular_fit_quality': 2, 'ai_fit_quality': 2},
+                         {'data_quality': 2, 'regular_fit_quality': 2, 'ai_fit_quality': 2}) == 2)
+    check("Catastrophic (0→2) = 2",
+          _max_deviation({'data_quality': 2, 'regular_fit_quality': 2, 'ai_fit_quality': 2},
+                         {'data_quality': 0, 'regular_fit_quality': 2, 'ai_fit_quality': 2}) == 2)
+    check("Mixed: worst wins (data diff=2, fit diff=1) = 2",
+          _max_deviation({'data_quality': 0, 'regular_fit_quality': 1, 'ai_fit_quality': 2},
+                         {'data_quality': 2, 'regular_fit_quality': 2, 'ai_fit_quality': 2}) == 2)
+    check("Missing keys handled gracefully",
+          _max_deviation({'data_quality': 2}, {'regular_fit_quality': 0}) == 0)
+
+    # --- 2. Fast-Track confidence check ---
+    print("\n2. Fast-Track Confidence Logic")
+    high_probs = {'data_probs': [0.02, 0.03, 0.95],
+                  'regular_fit_probs': [0.01, 0.05, 0.94],
+                  'ai_fit_probs': [0.03, 0.02, 0.95]}
+    dp = high_probs['data_probs']
+    rfp = high_probs['regular_fit_probs']
+    afp = high_probs['ai_fit_probs']
+    check("All ≥ 0.90 → fast-track eligible",
+          max(dp) >= FAST_TRACK_THRESHOLD and max(rfp) >= FAST_TRACK_THRESHOLD and max(afp) >= FAST_TRACK_THRESHOLD)
+
+    low_probs = {'data_probs': [0.1, 0.3, 0.6],
+                 'regular_fit_probs': [0.01, 0.05, 0.94],
+                 'ai_fit_probs': [0.03, 0.02, 0.95]}
+    dp2 = low_probs['data_probs']
+    check("Data conf < 0.90 → NOT eligible",
+          not (max(dp2) >= FAST_TRACK_THRESHOLD))
+
+    # --- 3. _resolve_rating implicit vs manual ---
+    print("\n3. Implicit Agreement vs Manual Correction")
+    mp = {'model_classes': {'data_quality': 2, 'regular_fit_quality': 1, 'ai_fit_quality': 0}}
+    r1 = _resolve_rating(None, mp, 'data_quality')
+    check("None input → implicit_agreement, score=2",
+          r1 == {'score': 2, 'source': 'implicit_agreement'})
+    r2 = _resolve_rating("Bad (0)", mp, 'data_quality')
+    check("'Bad (0)' input → manual_correction, score=0",
+          r2 == {'score': 0, 'source': 'manual_correction'})
+
+    # --- 4. Smart Queue needs_second_opinion ---
+    print("\n4. Smart Queue Prioritization")
+    sources_all_implicit = ['implicit_agreement', 'implicit_agreement']
+    sources_with_correction = ['implicit_agreement', 'manual_correction']
+    check("All implicit → no needs_second_opinion flag",
+          not any(s == 'manual_correction' for s in sources_all_implicit))
+    check("Has correction → needs_second_opinion = True",
+          any(s == 'manual_correction' for s in sources_with_correction))
+
+    # --- 5. Username normalization ---
+    print("\n5. Username Normalization")
+    check("'aDI mASKIL' → 'Adi Maskil'", 'aDI mASKIL'.strip().title() == 'Adi Maskil')
+    check("'bob' → 'Bob'", 'bob'.strip().title() == 'Bob')
+    check("'ALICE SMITH' → 'Alice Smith'", 'ALICE SMITH'.strip().title() == 'Alice Smith')
+
+    print(f"\n{'=' * 60}")
+    print(f"RESULTS: {passed} passed, {failed} failed")
+    print(f"{'=' * 60}")
+    return failed == 0
+
+
+def run_mongodb_routing_test():
+    """E2E test: inject mock data → route → verify collections exist in Atlas."""
+    print("=" * 60)
+    print("MONGODB E2E ROUTING TEST")
+    print("=" * 60)
+
+    # 1. Connect using same logic as init_globals
+    client = None
+    for kw in [dict(tlsCAFile=certifi.where()),
+               dict(tls=True, tlsAllowInvalidCertificates=True)]:
+        try:
+            client = MongoClient(MONGO_URI, serverSelectionTimeoutMS=5000, **kw)
+            client.admin.command('ping')
+            break
+        except Exception:
+            client = None
+    if not client:
+        print("❌ Cannot connect to MongoDB. Aborting.")
+        return False
+
+    db = client[DB_NAME]
+    test_col = db['experiments_test_mock']
+    print(f"✓ Connected to {DB_NAME}")
+
+    # 2. Clean up previous test data
+    test_col.delete_many({})
+    for col_name in ['experiments_clf_success', 'experiments_clf_failed',
+                     'experiments_clf_catastrophic_failure',
+                     'experiments_seed_success_reg_failed',
+                     'experiments_reg_success_seed_failed']:
+        db[col_name].delete_many({'original_id': {'$regex': '^e2e_test_'}})
+    print("✓ Cleaned previous test data")
+
+    # 3. Insert mock documents for each routing scenario
+    test_cases = [
+        {
+            'name': 'Catastrophic Failure (model=2, human=0)',
+            '_id': 'e2e_test_catastrophic',
+            'model_predictions': {
+                'model_classes': {'data_quality': 2, 'regular_fit_quality': 2, 'ai_fit_quality': 2},
+                'data_probs': [0.01, 0.02, 0.97],
+                'regular_fit_probs': [0.02, 0.03, 0.95],
+                'ai_fit_probs': [0.01, 0.04, 0.95],
+            },
+            'final_consensus': {'data_quality': 0, 'regular_fit_quality': 0, 'ai_fit_quality': 0},
+            'curves_overlap': True,
+            'status': 'completed',
+            'expected_collection': 'experiments_clf_catastrophic_failure',
+        },
+        {
+            'name': 'Minor Failure (model=2, human=1)',
+            '_id': 'e2e_test_minor_fail',
+            'model_predictions': {
+                'model_classes': {'data_quality': 2, 'regular_fit_quality': 2, 'ai_fit_quality': 2},
+                'data_probs': [0.01, 0.02, 0.97],
+                'regular_fit_probs': [0.02, 0.03, 0.95],
+                'ai_fit_probs': [0.01, 0.04, 0.95],
+            },
+            'final_consensus': {'data_quality': 1, 'regular_fit_quality': 2, 'ai_fit_quality': 2},
+            'curves_overlap': True,
+            'status': 'completed',
+            'expected_collection': 'experiments_clf_failed',
+        },
+        {
+            'name': 'Perfect Match (model=human)',
+            '_id': 'e2e_test_success',
+            'model_predictions': {
+                'model_classes': {'data_quality': 2, 'regular_fit_quality': 2, 'ai_fit_quality': 2},
+                'data_probs': [0.01, 0.02, 0.97],
+                'regular_fit_probs': [0.02, 0.03, 0.95],
+                'ai_fit_probs': [0.01, 0.04, 0.95],
+            },
+            'final_consensus': {'data_quality': 2, 'regular_fit_quality': 2, 'ai_fit_quality': 2},
+            'curves_overlap': True,
+            'status': 'completed',
+            'expected_collection': 'experiments_clf_success',
+        },
+    ]
+
+    # Insert all test docs into the test collection
+    for tc in test_cases:
+        doc = {k: v for k, v in tc.items() if k not in ('name', 'expected_collection')}
+        doc['raw_data'] = {'measured_data': {'x_values': [0, 1], 'y_values': [0, 1]}}
+        doc['evaluations'] = []
+        doc['tagged_by'] = []
+        doc['num_votes'] = 2
+        test_col.insert_one(doc)
+    print(f"✓ Inserted {len(test_cases)} mock documents into experiments_test_mock")
+
+    # 4. Run routing on each mock doc using the REAL route_completed logic
+    # We temporarily point MONGO_COL and MONGO_DB to our test setup
+    global MONGO_COL, MONGO_DB
+    original_col = MONGO_COL
+    original_db = MONGO_DB
+    MONGO_COL = test_col
+    MONGO_DB = db
+
+    print("\nRouting documents...")
+    for tc in test_cases:
+        route_completed(tc['_id'])
+        print(f"  → Routed: {tc['name']}")
+
+    # 5. Verify each document landed in the correct collection
+    print("\nChecking for collection creation...")
+    all_passed = True
+    for tc in test_cases:
+        target = tc['expected_collection']
+        found = db[target].find_one({'original_id': tc['_id']})
+        if found:
+            print(f"  ✅ {target} contains '{tc['name']}'")
+        else:
+            print(f"  ❌ {target} MISSING '{tc['name']}'")
+            all_passed = False
+
+    # 6. List all collections to confirm they exist in Atlas
+    print("\nCollections in database:")
+    for name in sorted(db.list_collection_names()):
+        count = db[name].count_documents({})
+        print(f"  📁 {name} ({count} docs)")
+
+    # Restore globals
+    MONGO_COL = original_col
+    MONGO_DB = original_db
+
+    print(f"\n{'=' * 60}")
+    print(f"E2E RESULT: {'ALL PASSED ✅' if all_passed else 'FAILURES DETECTED ❌'}")
+    print(f"{'=' * 60}")
+    return all_passed
+
+
 if __name__ == '__main__':
-    demo = build_ui()
-    demo.launch(css=CSS, js=TOGGLE_JS)
+    import sys
+    if '--test' in sys.argv:
+        success = run_validation_tests()
+        sys.exit(0 if success else 1)
+    elif '--e2e' in sys.argv:
+        success = run_mongodb_routing_test()
+        sys.exit(0 if success else 1)
+    else:
+        demo = build_ui()
+        demo.launch(css=CSS, js=TOGGLE_JS)