| { |
| "acceptance": { |
| "artifact_at_most_2_mb": true, |
| "empty_skops_untrusted_type_set": true, |
| "exact_reproduction": true, |
| "lockbox_error_recall_at_least_0.50": true, |
| "validation_error_recall_margin_delta_at_least_0.02": true, |
| "validation_routed_accuracy_gain_at_least_0.03": true |
| }, |
| "acceptance_decision": "release_work_pending", |
| "artifacts": { |
| "candidate": { |
| "bytes": 1104573, |
| "path": "model.skops", |
| "sha256": "1d5a785c69b01135981c9052aa8d124cf3d423f7d15e558820502d37a18686b0", |
| "skops_untrusted_types": [] |
| }, |
| "primary_baseline": { |
| "bytes": 28966090, |
| "path": "primary_baseline.skops", |
| "sha256": "f735dfa1e498fef3c6a0d0a87664cd2e77acaadc892af7875c7ab30148a0e020", |
| "skops_untrusted_types": [] |
| } |
| }, |
| "base_model_revision": null, |
| "campaign_id": "banking77-intent-error-predictor-v1", |
| "candidate": "histogram_gradient_boosting_score_error_predictor", |
| "code_sha256": "852699c3ee6b8e545f9f0fd6c4cf8ccce0bccdea3d67ae4ceceefcdda1404bd7", |
| "configuration": { |
| "architecture": "histogram gradient boosting over probabilities, predicted-intent one-hot, confidence geometry, and five text-shape features", |
| "campaign_id": "banking77-intent-error-predictor-v1", |
| "candidate": "histogram_gradient_boosting_score_error_predictor", |
| "code_sha256": "852699c3ee6b8e545f9f0fd6c4cf8ccce0bccdea3d67ae4ceceefcdda1404bd7", |
| "consumer": "banking-support intent router with a human review queue", |
| "dataset_hashes": { |
| "categories.json": "53261da888122daf2d120d925458631d9619e15d82e56052e7a42e535ce32b63", |
| "test.csv": "d12d6e3bc4c3103966ae786dc435913c0c563dfa328f5a3646d0e62cfeeb474d", |
| "train.csv": "b06e26ac675513959a63135f11b94ea7786ed02da65db93a5650d8838cbc664b" |
| }, |
| "dataset_hub_revision": "90d4e2ee5521c04fc1488f065b8b083658768c57", |
| "dataset_source_revision": "57ec275d8078af65b7731c2a98be812d844a6d6b", |
| "hyperparameters": { |
| "l2_regularization": 1.0, |
| "learning_rate": 0.06, |
| "max_iter": 140, |
| "max_leaf_nodes": 15, |
| "min_samples_leaf": 25 |
| }, |
| "input_contract": "77 probabilities in sorted Banking77 label order plus bounded text-shape features", |
| "objective": "balanced binary log loss for primary-router error prediction", |
| "output_contract": "error probability and advisory review decision", |
| "preprocessing": "NFKC casefold whitespace normalization for group hashing; TF-IDF primary; score and text-shape candidate features", |
| "primary": "24k word plus 36k character TF-IDF with multinomial logistic regression C=4", |
| "seed": 20260811, |
| "split": "normalized-text SHA-256 grouped 60/20/20 development partitions; official test untouched lockbox with overlap removal", |
| "task": "predict whether a fixed BANKING77 primary router is wrong" |
| }, |
| "dataset_revision": "57ec275d8078af65b7731c2a98be812d844a6d6b", |
| "experiment_spec_hash": "8a4c7e16a4a0ef3615735c7214e429df5268dc1610612ce6e833aa9e3829ccdc", |
| "hypothesis": "learned score-shape and intent features improve error capture over margin-only triage at equal review rate", |
| "lockbox": { |
| "candidate": { |
| "confusion": { |
| "fn": 72, |
| "fp": 326, |
| "tn": 2412, |
| "tp": 263 |
| }, |
| "coverage": 0.8083306215424666, |
| "error_f1": 0.5692640692640693, |
| "error_precision": 0.4465195246179966, |
| "error_prevalence": 0.10901399284087211, |
| "error_recall": 0.7850746268656716, |
| "pr_auc": 0.5199652272918066, |
| "review_rate": 0.19166937845753335, |
| "roc_auc": 0.9035912475605902, |
| "routed_accuracy": 0.9710144927536232, |
| "threshold": 0.27443790545050933 |
| }, |
| "margin_baseline": { |
| "confusion": { |
| "fn": 78, |
| "fp": 366, |
| "tn": 2372, |
| "tp": 257 |
| }, |
| "coverage": 0.7972665148063781, |
| "error_f1": 0.5365344467640919, |
| "error_precision": 0.41252006420545745, |
| "error_prevalence": 0.10901399284087211, |
| "error_recall": 0.7671641791044777, |
| "pr_auc": 0.5863979952872228, |
| "review_rate": 0.20273348519362186, |
| "roc_auc": 0.9092310543702234, |
| "routed_accuracy": 0.9681632653061224, |
| "threshold": 0.6936486958535751 |
| } |
| }, |
| "partitions": { |
| "complement_train": 1951, |
| "lockbox": 3073, |
| "lockbox_overlap_removed": 7, |
| "primary_train": 6050, |
| "validation": 2002 |
| }, |
| "primary": { |
| "lockbox_accuracy": 0.8909860071591279, |
| "lockbox_macro_f1": 0.8910051474654282, |
| "validation_accuracy": 0.8951048951048951, |
| "validation_macro_f1": 0.8919223884704242 |
| }, |
| "record_type": "fit", |
| "release_validation": { |
| "bounded_inference_api": true, |
| "lockbox_scores_persisted": true, |
| "reference_consumer": true, |
| "representative_example_passed": true |
| }, |
| "reproduction": { |
| "clean_refit_predictions_exact": true, |
| "clean_refit_scores_exact": true, |
| "serialization_scores_exact": true |
| }, |
| "telemetry": { |
| "cache_growth_bytes": 1081070, |
| "cpu_seconds": 14.953067, |
| "minor_page_fault_delta": 37952, |
| "peak_rss_bytes": 743817216, |
| "single_training_process": true, |
| "temporary_disk_growth_bytes": 30076757, |
| "thread_limit": 8, |
| "wall_seconds": 6.962122584000099 |
| }, |
| "validation": { |
| "candidate": { |
| "confusion": { |
| "fn": 42, |
| "fp": 233, |
| "tn": 1559, |
| "tp": 168 |
| }, |
| "coverage": 0.7997002997002997, |
| "error_f1": 0.5499181669394435, |
| "error_precision": 0.41895261845386533, |
| "error_prevalence": 0.1048951048951049, |
| "error_recall": 0.8, |
| "pr_auc": 0.48498408831658685, |
| "review_rate": 0.2002997002997003, |
| "roc_auc": 0.8968856292517007, |
| "routed_accuracy": 0.9737663960024985, |
| "threshold": 0.27443790545050933 |
| }, |
| "margin_baseline": { |
| "confusion": { |
| "fn": 52, |
| "fp": 243, |
| "tn": 1549, |
| "tp": 158 |
| }, |
| "coverage": 0.7997002997002997, |
| "error_f1": 0.5171849427168577, |
| "error_precision": 0.3940149625935162, |
| "error_prevalence": 0.1048951048951049, |
| "error_recall": 0.7523809523809524, |
| "pr_auc": 0.5163906717576305, |
| "review_rate": 0.2002997002997003, |
| "roc_auc": 0.8986208545918366, |
| "routed_accuracy": 0.9675202998126171, |
| "threshold": 0.6936486958535751 |
| } |
| } |
| } |
|
|
