Spaces:
Running
Running
| { | |
| "project": "EvalFlow CI Lab", | |
| "type": "fixed-snapshot-research-gate", | |
| "status": "PASS", | |
| "baseline": "nb", | |
| "candidate": "hybrid", | |
| "split": "challenge", | |
| "policy": { | |
| "minN": 50, | |
| "minAccuracy": 0.75, | |
| "minMacroF1": 0.72, | |
| "maxRegression": 0.03, | |
| "maxTypoFlip": 0.05, | |
| "maxContextFlip": 0.15, | |
| "minLanguageN": 15, | |
| "minLanguageAccuracy": 0.65, | |
| "requireComplete": true | |
| }, | |
| "baselineMetrics": { | |
| "method": "nb", | |
| "split": "challenge", | |
| "n": 54, | |
| "accuracy": 0.7407407407407407, | |
| "macroF1": 0.7362809928289805, | |
| "classes": [ | |
| { | |
| "label": "background", | |
| "n": 9, | |
| "tp": 4, | |
| "fp": 0, | |
| "fn": 5, | |
| "precision": 1, | |
| "recall": 0.4444444444444444, | |
| "f1": 0.6153846153846153 | |
| }, | |
| { | |
| "label": "objective", | |
| "n": 9, | |
| "tp": 7, | |
| "fp": 1, | |
| "fn": 2, | |
| "precision": 0.875, | |
| "recall": 0.7777777777777778, | |
| "f1": 0.823529411764706 | |
| }, | |
| { | |
| "label": "method", | |
| "n": 9, | |
| "tp": 8, | |
| "fp": 2, | |
| "fn": 1, | |
| "precision": 0.8, | |
| "recall": 0.8888888888888888, | |
| "f1": 0.8421052631578948 | |
| }, | |
| { | |
| "label": "result", | |
| "n": 9, | |
| "tp": 6, | |
| "fp": 1, | |
| "fn": 3, | |
| "precision": 0.8571428571428571, | |
| "recall": 0.6666666666666666, | |
| "f1": 0.75 | |
| }, | |
| { | |
| "label": "limitation", | |
| "n": 9, | |
| "tp": 9, | |
| "fp": 7, | |
| "fn": 0, | |
| "precision": 0.5625, | |
| "recall": 1, | |
| "f1": 0.72 | |
| }, | |
| { | |
| "label": "implication", | |
| "n": 9, | |
| "tp": 6, | |
| "fp": 3, | |
| "fn": 3, | |
| "precision": 0.6666666666666666, | |
| "recall": 0.6666666666666666, | |
| "f1": 0.6666666666666666 | |
| } | |
| ], | |
| "groups": { | |
| "language": [ | |
| { | |
| "label": "en", | |
| "n": 18, | |
| "accuracy": 0.8333333333333334, | |
| "errors": 3 | |
| }, | |
| { | |
| "label": "es", | |
| "n": 18, | |
| "accuracy": 0.6111111111111112, | |
| "errors": 7 | |
| }, | |
| { | |
| "label": "pt", | |
| "n": 18, | |
| "accuracy": 0.7777777777777778, | |
| "errors": 4 | |
| } | |
| ], | |
| "domain": [ | |
| { | |
| "label": "mixed", | |
| "n": 54, | |
| "accuracy": 0.7407407407407407, | |
| "errors": 14 | |
| } | |
| ], | |
| "truth": [ | |
| { | |
| "label": "background", | |
| "n": 9, | |
| "accuracy": 0.4444444444444444, | |
| "errors": 5 | |
| }, | |
| { | |
| "label": "implication", | |
| "n": 9, | |
| "accuracy": 0.6666666666666666, | |
| "errors": 3 | |
| }, | |
| { | |
| "label": "limitation", | |
| "n": 9, | |
| "accuracy": 1, | |
| "errors": 0 | |
| }, | |
| { | |
| "label": "method", | |
| "n": 9, | |
| "accuracy": 0.8888888888888888, | |
| "errors": 1 | |
| }, | |
| { | |
| "label": "objective", | |
| "n": 9, | |
| "accuracy": 0.7777777777777778, | |
| "errors": 2 | |
| }, | |
| { | |
| "label": "result", | |
| "n": 9, | |
| "accuracy": 0.6666666666666666, | |
| "errors": 3 | |
| } | |
| ] | |
| }, | |
| "attacks": [ | |
| { | |
| "id": "original", | |
| "n": 54, | |
| "matched": 0, | |
| "coverage": 1, | |
| "changedShare": 0, | |
| "flipRate": 0, | |
| "flipN": 0, | |
| "matchedAccuracy": null, | |
| "matchedOriginalAccuracy": null, | |
| "destructive": false | |
| }, | |
| { | |
| "id": "format", | |
| "n": 54, | |
| "matched": 54, | |
| "coverage": 1, | |
| "changedShare": 1, | |
| "flipRate": 0, | |
| "flipN": 0, | |
| "matchedAccuracy": 0.7407407407407407, | |
| "matchedOriginalAccuracy": 0.7407407407407407, | |
| "destructive": false | |
| }, | |
| { | |
| "id": "typo", | |
| "n": 54, | |
| "matched": 54, | |
| "coverage": 1, | |
| "changedShare": 1, | |
| "flipRate": 0.05555555555555555, | |
| "flipN": 3, | |
| "matchedAccuracy": 0.6851851851851852, | |
| "matchedOriginalAccuracy": 0.7407407407407407, | |
| "destructive": false | |
| }, | |
| { | |
| "id": "synonym", | |
| "n": 54, | |
| "matched": 9, | |
| "coverage": 1, | |
| "changedShare": 0.16666666666666666, | |
| "flipRate": 0, | |
| "flipN": 0, | |
| "matchedAccuracy": 0.8888888888888888, | |
| "matchedOriginalAccuracy": 0.8888888888888888, | |
| "destructive": false | |
| }, | |
| { | |
| "id": "context", | |
| "n": 54, | |
| "matched": 54, | |
| "coverage": 1, | |
| "changedShare": 1, | |
| "flipRate": 0.2962962962962963, | |
| "flipN": 16, | |
| "matchedAccuracy": 0.6296296296296297, | |
| "matchedOriginalAccuracy": 0.7407407407407407, | |
| "destructive": false | |
| }, | |
| { | |
| "id": "truncate", | |
| "n": 54, | |
| "matched": 54, | |
| "coverage": 1, | |
| "changedShare": 1, | |
| "flipRate": 0.12962962962962962, | |
| "flipN": 7, | |
| "matchedAccuracy": null, | |
| "matchedOriginalAccuracy": null, | |
| "destructive": true | |
| } | |
| ] | |
| }, | |
| "candidateMetrics": { | |
| "method": "hybrid", | |
| "split": "challenge", | |
| "n": 54, | |
| "accuracy": 0.7777777777777778, | |
| "macroF1": 0.7617571707045391, | |
| "classes": [ | |
| { | |
| "label": "background", | |
| "n": 9, | |
| "tp": 3, | |
| "fp": 0, | |
| "fn": 6, | |
| "precision": 1, | |
| "recall": 0.3333333333333333, | |
| "f1": 0.5 | |
| }, | |
| { | |
| "label": "objective", | |
| "n": 9, | |
| "tp": 8, | |
| "fp": 1, | |
| "fn": 1, | |
| "precision": 0.8888888888888888, | |
| "recall": 0.8888888888888888, | |
| "f1": 0.8888888888888888 | |
| }, | |
| { | |
| "label": "method", | |
| "n": 9, | |
| "tp": 9, | |
| "fp": 1, | |
| "fn": 0, | |
| "precision": 0.9, | |
| "recall": 1, | |
| "f1": 0.9473684210526316 | |
| }, | |
| { | |
| "label": "result", | |
| "n": 9, | |
| "tp": 5, | |
| "fp": 0, | |
| "fn": 4, | |
| "precision": 1, | |
| "recall": 0.5555555555555556, | |
| "f1": 0.7142857142857143 | |
| }, | |
| { | |
| "label": "limitation", | |
| "n": 9, | |
| "tp": 9, | |
| "fp": 7, | |
| "fn": 0, | |
| "precision": 0.5625, | |
| "recall": 1, | |
| "f1": 0.72 | |
| }, | |
| { | |
| "label": "implication", | |
| "n": 9, | |
| "tp": 8, | |
| "fp": 3, | |
| "fn": 1, | |
| "precision": 0.7272727272727273, | |
| "recall": 0.8888888888888888, | |
| "f1": 0.7999999999999999 | |
| } | |
| ], | |
| "groups": { | |
| "language": [ | |
| { | |
| "label": "en", | |
| "n": 18, | |
| "accuracy": 0.7777777777777778, | |
| "errors": 4 | |
| }, | |
| { | |
| "label": "es", | |
| "n": 18, | |
| "accuracy": 0.7222222222222222, | |
| "errors": 5 | |
| }, | |
| { | |
| "label": "pt", | |
| "n": 18, | |
| "accuracy": 0.8333333333333334, | |
| "errors": 3 | |
| } | |
| ], | |
| "domain": [ | |
| { | |
| "label": "mixed", | |
| "n": 54, | |
| "accuracy": 0.7777777777777778, | |
| "errors": 12 | |
| } | |
| ], | |
| "truth": [ | |
| { | |
| "label": "background", | |
| "n": 9, | |
| "accuracy": 0.3333333333333333, | |
| "errors": 6 | |
| }, | |
| { | |
| "label": "implication", | |
| "n": 9, | |
| "accuracy": 0.8888888888888888, | |
| "errors": 1 | |
| }, | |
| { | |
| "label": "limitation", | |
| "n": 9, | |
| "accuracy": 1, | |
| "errors": 0 | |
| }, | |
| { | |
| "label": "method", | |
| "n": 9, | |
| "accuracy": 1, | |
| "errors": 0 | |
| }, | |
| { | |
| "label": "objective", | |
| "n": 9, | |
| "accuracy": 0.8888888888888888, | |
| "errors": 1 | |
| }, | |
| { | |
| "label": "result", | |
| "n": 9, | |
| "accuracy": 0.5555555555555556, | |
| "errors": 4 | |
| } | |
| ] | |
| }, | |
| "attacks": [ | |
| { | |
| "id": "original", | |
| "n": 54, | |
| "matched": 0, | |
| "coverage": 1, | |
| "changedShare": 0, | |
| "flipRate": 0, | |
| "flipN": 0, | |
| "matchedAccuracy": null, | |
| "matchedOriginalAccuracy": null, | |
| "destructive": false | |
| }, | |
| { | |
| "id": "format", | |
| "n": 54, | |
| "matched": 54, | |
| "coverage": 1, | |
| "changedShare": 1, | |
| "flipRate": 0, | |
| "flipN": 0, | |
| "matchedAccuracy": 0.7777777777777778, | |
| "matchedOriginalAccuracy": 0.7777777777777778, | |
| "destructive": false | |
| }, | |
| { | |
| "id": "typo", | |
| "n": 54, | |
| "matched": 54, | |
| "coverage": 1, | |
| "changedShare": 1, | |
| "flipRate": 0.018518518518518517, | |
| "flipN": 1, | |
| "matchedAccuracy": 0.7592592592592593, | |
| "matchedOriginalAccuracy": 0.7777777777777778, | |
| "destructive": false | |
| }, | |
| { | |
| "id": "synonym", | |
| "n": 54, | |
| "matched": 9, | |
| "coverage": 1, | |
| "changedShare": 0.16666666666666666, | |
| "flipRate": 0.018518518518518517, | |
| "flipN": 1, | |
| "matchedAccuracy": 1, | |
| "matchedOriginalAccuracy": 0.8888888888888888, | |
| "destructive": false | |
| }, | |
| { | |
| "id": "context", | |
| "n": 54, | |
| "matched": 54, | |
| "coverage": 1, | |
| "changedShare": 1, | |
| "flipRate": 0.09259259259259259, | |
| "flipN": 5, | |
| "matchedAccuracy": 0.7222222222222222, | |
| "matchedOriginalAccuracy": 0.7777777777777778, | |
| "destructive": false | |
| }, | |
| { | |
| "id": "truncate", | |
| "n": 54, | |
| "matched": 54, | |
| "coverage": 1, | |
| "changedShare": 1, | |
| "flipRate": 0.1111111111111111, | |
| "flipN": 6, | |
| "matchedAccuracy": null, | |
| "matchedOriginalAccuracy": null, | |
| "destructive": true | |
| } | |
| ] | |
| }, | |
| "pair": { | |
| "n": 54, | |
| "delta": 0.037037037037037035, | |
| "improved": 4, | |
| "regressed": 2, | |
| "onlyBase": [], | |
| "onlyCandidate": [] | |
| }, | |
| "bootstrap": { | |
| "point": 0.037037037037037035, | |
| "low": -0.037037037037037035, | |
| "high": 0.12962962962962962, | |
| "iterations": 500, | |
| "seed": 241, | |
| "n": 54, | |
| "assumptions": "Paired example bootstrap with fixed model predictions; excludes retraining and data-label uncertainty." | |
| }, | |
| "gates": [ | |
| { | |
| "id": "n", | |
| "label": "Minimum held-out examples", | |
| "actual": 54, | |
| "limit": 50, | |
| "comparator": "≥", | |
| "pass": true, | |
| "unit": "count" | |
| }, | |
| { | |
| "id": "accuracy", | |
| "label": "Candidate Accuracy", | |
| "actual": 0.7777777777777778, | |
| "limit": 0.75, | |
| "comparator": "≥", | |
| "pass": true, | |
| "unit": "percent" | |
| }, | |
| { | |
| "id": "macroF1", | |
| "label": "Candidate macro-F1", | |
| "actual": 0.7617571707045391, | |
| "limit": 0.72, | |
| "comparator": "≥", | |
| "pass": true, | |
| "unit": "percent" | |
| }, | |
| { | |
| "id": "regression", | |
| "label": "Maximum allowed regression", | |
| "actual": 0.037037037037037035, | |
| "limit": -0.03, | |
| "comparator": "≥", | |
| "pass": true, | |
| "unit": "delta" | |
| }, | |
| { | |
| "id": "typo", | |
| "label": "Typo prediction-flip rate", | |
| "actual": 0.018518518518518517, | |
| "limit": 0.05, | |
| "comparator": "≤", | |
| "pass": true, | |
| "unit": "percent" | |
| }, | |
| { | |
| "id": "context", | |
| "label": "Context prediction-flip rate", | |
| "actual": 0.09259259259259259, | |
| "limit": 0.15, | |
| "comparator": "≤", | |
| "pass": true, | |
| "unit": "percent" | |
| }, | |
| { | |
| "id": "sliceN", | |
| "label": "Minimum per-language examples", | |
| "actual": 18, | |
| "limit": 15, | |
| "comparator": "≥", | |
| "pass": true, | |
| "unit": "count" | |
| }, | |
| { | |
| "id": "sliceAcc", | |
| "label": "Minimum language Accuracy", | |
| "actual": 0.7222222222222222, | |
| "limit": 0.65, | |
| "comparator": "≥", | |
| "pass": true, | |
| "unit": "percent" | |
| }, | |
| { | |
| "id": "coverage", | |
| "label": "Paired evaluation / stress coverage", | |
| "actual": 54, | |
| "limit": 54, | |
| "comparator": "=", | |
| "pass": true, | |
| "unit": "count" | |
| } | |
| ], | |
| "limitations": "This is a local research replay, not production CI execution, a trained model update, deployment authorization, or a fairness certification." | |
| } | |