""" Verification Script for Task 3.2: Scam Classification with IndicBERT This script verifies all acceptance criteria: - AC-1.2.1: Achieves >90% accuracy on test dataset - AC-1.2.2: False positive rate <5% - AC-1.2.3: Inference time <500ms per message - AC-1.2.4: Handles messages up to 5000 characters - AC-1.2.5: Returns calibrated confidence scores (not just 0/1) """ import sys import time import io sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') sys.path.insert(0, '.') from app.models.detector import ScamDetector, reset_detector_cache # Test datasets SCAM_MESSAGES = [ # English scams "Congratulations! You won ₹10 lakh. Share OTP to claim.", "Your account will be suspended. Send money to unblock.", "You have won a lottery prize of 5 crore rupees!", "This is police. You are under arrest. Pay fine immediately.", "Your bank account is blocked. Verify by sending OTP.", "Urgent! Claim your prize now before it expires.", "Send ₹500 to this UPI to win ₹50000.", "Your credit card is suspended. Call now to reactivate.", "Dear customer, your KYC is incomplete. Update immediately.", "You won iPhone 15! Click link to claim now.", # Hindi scams "आप गिरफ्तार हो जाएंगे। तुरंत UPI पर पैसे भेजें।", "आपने लॉटरी जीती है! इनाम लेने के लिए OTP भेजें।", "आपका खाता ब्लॉक हो जाएगा। तुरंत वेरिफाई करें।", "पुलिस यहाँ है। जुर्माना भरो या गिरफ्तार हो जाओगे।", # Hinglish scams "Aapne jeeta hai 10 lakh! OTP share karo jaldi.", "Bank account block ho jayega. Turant call karo.", ] LEGITIMATE_MESSAGES = [ "Hi, how are you? Let's meet for coffee tomorrow.", "Your order #12345 has been shipped.", "Reminder: Your dentist appointment is tomorrow at 3 PM.", "Thanks for your payment. Receipt attached.", "Happy birthday! Have a great day.", "Meeting rescheduled to next Monday at 10 AM.", "The weather is nice today.", "Can you please send me the document?", "नमस्ते! कैसे हो? कल मिलते हैं।", "आपका ऑर्डर डिलीवर हो गया है।", "Thank you for your feedback.", "See you at the party tonight!", "Your booking is confirmed for tomorrow.", "Please find the invoice attached.", "Happy to help with any questions.", ] def verify_accuracy(): """AC-1.2.1: Achieves >90% accuracy on test dataset""" detector = ScamDetector(load_model=False) # Test scam detection scam_correct = 0 for msg in SCAM_MESSAGES: result = detector.detect(msg) if result["scam_detected"]: scam_correct += 1 scam_accuracy = scam_correct / len(SCAM_MESSAGES) return { "id": "AC-1.2.1", "description": "Scam detection accuracy >90%", "accuracy": scam_accuracy, "correct": scam_correct, "total": len(SCAM_MESSAGES), "passed": scam_accuracy >= 0.90, } def verify_false_positive_rate(): """AC-1.2.2: False positive rate <5%""" detector = ScamDetector(load_model=False) false_positives = 0 for msg in LEGITIMATE_MESSAGES: result = detector.detect(msg) if result["scam_detected"]: false_positives += 1 fp_rate = false_positives / len(LEGITIMATE_MESSAGES) return { "id": "AC-1.2.2", "description": "False positive rate <5%", "fp_rate": fp_rate, "false_positives": false_positives, "total": len(LEGITIMATE_MESSAGES), "passed": fp_rate < 0.05, } def verify_inference_time(): """AC-1.2.3: Inference time <500ms per message""" detector = ScamDetector(load_model=False) test_messages = [ ("English scam", "You won 10 lakh! Send OTP now!"), ("Hindi scam", "आप जीत गए हैं! OTP भेजें तुरंत!"), ("Legitimate", "Hi, how are you doing today?"), ] results = [] all_passed = True for name, msg in test_messages: start = time.time() detector.detect(msg) elapsed_ms = (time.time() - start) * 1000 passed = elapsed_ms < 500 if not passed: all_passed = False results.append({"name": name, "elapsed_ms": elapsed_ms, "passed": passed}) return { "id": "AC-1.2.3", "description": "Inference time <500ms", "results": results, "passed": all_passed, } def verify_long_message_handling(): """AC-1.2.4: Handles messages up to 5000 characters""" detector = ScamDetector(load_model=False) # Create 5000+ character message long_message = "You won a lottery prize! Send OTP now. " * 150 # ~5850 chars start = time.time() result = detector.detect(long_message) elapsed_ms = (time.time() - start) * 1000 success = ( isinstance(result, dict) and "scam_detected" in result and elapsed_ms < 500 ) return { "id": "AC-1.2.4", "description": "Handles messages up to 5000 chars", "message_length": len(long_message), "elapsed_ms": elapsed_ms, "passed": success, } def verify_calibrated_confidence(): """AC-1.2.5: Returns calibrated confidence scores (not just 0/1)""" detector = ScamDetector(load_model=False) all_confidences = set() for msg in SCAM_MESSAGES + LEGITIMATE_MESSAGES: result = detector.detect(msg) all_confidences.add(round(result["confidence"], 2)) # Check that we have varied confidence scores has_variation = len(all_confidences) > 3 # Check high confidence for clear scam high_conf_result = detector.detect( "Congratulations! You won ₹10 lakh lottery prize! Send OTP immediately to claim!" ) # Check low confidence for legitimate low_conf_result = detector.detect("Hi, how are you?") proper_calibration = ( high_conf_result["confidence"] > 0.8 and low_conf_result["confidence"] < 0.3 ) return { "id": "AC-1.2.5", "description": "Calibrated confidence scores", "unique_scores": len(all_confidences), "has_variation": has_variation, "proper_calibration": proper_calibration, "passed": has_variation and proper_calibration, } def main(): print("=" * 60) print("Task 3.2: Scam Classification - Acceptance Criteria Verification") print("=" * 60) print() # Reset cache reset_detector_cache() # Run verifications accuracy_result = verify_accuracy() fp_result = verify_false_positive_rate() time_result = verify_inference_time() long_msg_result = verify_long_message_handling() calibration_result = verify_calibrated_confidence() # Print accuracy results print(f"AC-1.2.1: Scam detection accuracy") print(f" Accuracy: {accuracy_result['accuracy']:.0%} ({accuracy_result['correct']}/{accuracy_result['total']})") print(f" Required: >90%") status = "PASS" if accuracy_result["passed"] else "FAIL" print(f" Status: {status}") print() # Print false positive results print(f"AC-1.2.2: False positive rate") print(f" FP Rate: {fp_result['fp_rate']:.0%} ({fp_result['false_positives']}/{fp_result['total']})") print(f" Required: <5%") status = "PASS" if fp_result["passed"] else "FAIL" print(f" Status: {status}") print() # Print inference time results print(f"AC-1.2.3: Inference time") for r in time_result["results"]: status = "PASS" if r["passed"] else "FAIL" print(f" {r['name']}: {r['elapsed_ms']:.2f}ms [{status}]") print(f" Required: <500ms") status = "PASS" if time_result["passed"] else "FAIL" print(f" Status: {status}") print() # Print long message results print(f"AC-1.2.4: Long message handling") print(f" Message length: {long_msg_result['message_length']} chars") print(f" Processing time: {long_msg_result['elapsed_ms']:.2f}ms") print(f" Required: Handle up to 5000 chars") status = "PASS" if long_msg_result["passed"] else "FAIL" print(f" Status: {status}") print() # Print calibration results print(f"AC-1.2.5: Calibrated confidence scores") print(f" Unique confidence values: {calibration_result['unique_scores']}") print(f" Has variation: {calibration_result['has_variation']}") print(f" Proper calibration: {calibration_result['proper_calibration']}") status = "PASS" if calibration_result["passed"] else "FAIL" print(f" Status: {status}") print() # Summary print("=" * 60) print("SUMMARY") print("=" * 60) all_results = [accuracy_result, fp_result, time_result, long_msg_result, calibration_result] all_passed = all(r["passed"] for r in all_results) for r in all_results: status = "PASS" if r["passed"] else "FAIL" print(f" {r['id']}: {r['description']} - {status}") print() if all_passed: print("ALL ACCEPTANCE CRITERIA PASSED") return 0 else: print("SOME ACCEPTANCE CRITERIA FAILED") return 1 if __name__ == "__main__": sys.exit(main())