| { |
| "development": { |
| "baseline kev": { |
| "auroc": 0.9595338983050847, |
| "acc": 0.9420289855072463, |
| "brier": 0.051967926948896535, |
| "ece": 0.08224694059061195, |
| "cov@2%": 0.8188405797101449, |
| "cov@5%": 0.8731884057971014, |
| "cov@10%": 0.9492753623188406 |
| }, |
| "fine-tuned kev": { |
| "auroc": 0.9582627118644068, |
| "acc": 0.967391304347826, |
| "brier": 0.03196338159102184, |
| "ece": 0.04480786530262329, |
| "cov@2%": 0.7391304347826086, |
| "cov@5%": 0.8985507246376812, |
| "cov@10%": 0.9492753623188406 |
| }, |
| "evidence score": { |
| "auroc": 0.9599576271186441, |
| "acc": 0.9710144927536232, |
| "brier": 0.03740658967391304, |
| "ece": 0.07269021739130432, |
| "cov@2%": 0.7898550724637681, |
| "cov@5%": 0.8985507246376812, |
| "cov@10%": 0.9239130434782609 |
| }, |
| "mean(evidence, baseline)": { |
| "auroc": 0.9653601694915255, |
| "acc": 0.9492753623188406, |
| "brier": 0.03681774037976611, |
| "ece": 0.07842630691187563, |
| "cov@2%": 0.7934782608695652, |
| "cov@5%": 0.8985507246376812, |
| "cov@10%": 0.9492753623188406 |
| }, |
| "mean(evidence, fine-tuned)": { |
| "auroc": 0.9679025423728813, |
| "acc": 0.967391304347826, |
| "brier": 0.030796113996718024, |
| "ece": 0.054936653462283394, |
| "cov@2%": 0.8007246376811594, |
| "cov@5%": 0.8985507246376812, |
| "cov@10%": 0.9492753623188406 |
| } |
| }, |
| "test_unseen_coder": { |
| "baseline kev": { |
| "auroc": 0.9663382594417077, |
| "acc": 0.9295774647887324, |
| "brier": 0.06354770562384918, |
| "ece": 0.04240858525824696, |
| "cov@2%": 0.28169014084507044, |
| "cov@5%": 0.4788732394366197, |
| "cov@10%": 0.647887323943662 |
| }, |
| "fine-tuned kev": { |
| "auroc": 0.9860426929392446, |
| "acc": 0.9154929577464789, |
| "brier": 0.06591741550271953, |
| "ece": 0.04675058358510263, |
| "cov@2%": 0.43661971830985913, |
| "cov@5%": 0.5915492957746479, |
| "cov@10%": 0.647887323943662 |
| }, |
| "evidence score": { |
| "auroc": 0.9433497536945813, |
| "acc": 0.9154929577464789, |
| "brier": 0.08483714788732394, |
| "ece": 0.1338028169014084, |
| "cov@2%": 0.15492957746478872, |
| "cov@5%": 0.15492957746478872, |
| "cov@10%": 0.5915492957746479 |
| }, |
| "mean(evidence, baseline)": { |
| "auroc": 0.9794745484400657, |
| "acc": 0.9154929577464789, |
| "brier": 0.06687451167408419, |
| "ece": 0.08106238257021522, |
| "cov@2%": 0.4225352112676056, |
| "cov@5%": 0.5774647887323944, |
| "cov@10%": 0.6197183098591549 |
| }, |
| "mean(evidence, fine-tuned)": { |
| "auroc": 0.9712643678160919, |
| "acc": 0.9154929577464789, |
| "brier": 0.06803778906539469, |
| "ece": 0.07175734922897826, |
| "cov@2%": 0.38028169014084506, |
| "cov@5%": 0.5774647887323944, |
| "cov@10%": 0.6056338028169014 |
| } |
| }, |
| "test_unseen_coder_all_tasks": { |
| "baseline kev": { |
| "auroc": 0.9704155219780219, |
| "acc": 0.9241803278688525, |
| "brier": 0.05993135831653411, |
| "ece": 0.031035312001956816, |
| "cov@2%": 0.3114754098360656, |
| "cov@5%": 0.514344262295082, |
| "cov@10%": 0.6229508196721312 |
| }, |
| "fine-tuned kev": { |
| "auroc": 0.9739182692307692, |
| "acc": 0.9385245901639344, |
| "brier": 0.04889003537440916, |
| "ece": 0.028302458894132104, |
| "cov@2%": 0.3442622950819672, |
| "cov@5%": 0.5430327868852459, |
| "cov@10%": 0.6352459016393442 |
| }, |
| "evidence score": { |
| "auroc": 0.943543956043956, |
| "acc": 0.9057377049180327, |
| "brier": 0.09111320255251247, |
| "ece": 0.09400301997638061, |
| "cov@2%": 0.0, |
| "cov@5%": 0.20901639344262296, |
| "cov@10%": 0.6188524590163934 |
| }, |
| "mean(evidence, baseline)": { |
| "auroc": 0.9705013736263737, |
| "acc": 0.9262295081967213, |
| "brier": 0.06611740777073485, |
| "ece": 0.06348054507231347, |
| "cov@2%": 0.3094262295081967, |
| "cov@5%": 0.5, |
| "cov@10%": 0.6270491803278688 |
| }, |
| "mean(evidence, fine-tuned)": { |
| "auroc": 0.9726304945054945, |
| "acc": 0.9364754098360656, |
| "brier": 0.05954172687246182, |
| "ece": 0.05905205255287546, |
| "cov@2%": 0.3463114754098361, |
| "cov@5%": 0.5061475409836066, |
| "cov@10%": 0.6352459016393442 |
| } |
| } |
| } |
