| { |
| "development": { |
| "baseline kev": { |
| "auroc": 0.8941919191919192, |
| "acc": 0.9553571428571429, |
| "brier": 0.04663790840267551, |
| "ece": 0.054507326705475256, |
| "cov@2%": 0.12946428571428573, |
| "cov@5%": 0.84375, |
| "cov@10%": 0.8928571428571429 |
| }, |
| "fine-tuned kev": { |
| "auroc": 0.9127209595959596, |
| "acc": 0.9553571428571429, |
| "brier": 0.04322990868327309, |
| "ece": 0.026368809832356274, |
| "cov@2%": 0.22321428571428573, |
| "cov@5%": 0.8258928571428571, |
| "cov@10%": 0.8928571428571429 |
| }, |
| "evidence score": { |
| "auroc": 0.9590277777777778, |
| "acc": 0.9553571428571429, |
| "brier": 0.04669189453125, |
| "ece": 0.07882254464285716, |
| "cov@2%": 0.6852678571428571, |
| "cov@5%": 0.8058035714285714, |
| "cov@10%": 0.8794642857142857 |
| }, |
| "mean(evidence, baseline)": { |
| "auroc": 0.9642992424242425, |
| "acc": 0.9553571428571429, |
| "brier": 0.04325954805619029, |
| "ece": 0.06962798473367474, |
| "cov@2%": 0.7433035714285714, |
| "cov@5%": 0.8125, |
| "cov@10%": 0.8928571428571429 |
| }, |
| "mean(evidence, fine-tuned)": { |
| "auroc": 0.9620580808080809, |
| "acc": 0.953125, |
| "brier": 0.041460332654909955, |
| "ece": 0.05010392415948695, |
| "cov@2%": 0.7232142857142857, |
| "cov@5%": 0.8125, |
| "cov@10%": 0.8928571428571429 |
| } |
| }, |
| "test_unseen_coder": { |
| "baseline kev": { |
| "auroc": 0.8957307060755336, |
| "acc": 0.9154929577464789, |
| "brier": 0.08782003066740127, |
| "ece": 0.10255376683858186, |
| "cov@2%": 0.09859154929577464, |
| "cov@5%": 0.09859154929577464, |
| "cov@10%": 0.30985915492957744 |
| }, |
| "fine-tuned kev": { |
| "auroc": 0.9490968801313628, |
| "acc": 0.9154929577464789, |
| "brier": 0.07603090115173004, |
| "ece": 0.06278563419018489, |
| "cov@2%": 0.36619718309859156, |
| "cov@5%": 0.4084507042253521, |
| "cov@10%": 0.5774647887323944 |
| }, |
| "evidence score": { |
| "auroc": 0.9433497536945813, |
| "acc": 0.9154929577464789, |
| "brier": 0.08483714788732394, |
| "ece": 0.1338028169014084, |
| "cov@2%": 0.15492957746478872, |
| "cov@5%": 0.15492957746478872, |
| "cov@10%": 0.5915492957746479 |
| }, |
| "mean(evidence, baseline)": { |
| "auroc": 0.9417077175697866, |
| "acc": 0.9154929577464789, |
| "brier": 0.08107140750327455, |
| "ece": 0.11410879479547414, |
| "cov@2%": 0.2535211267605634, |
| "cov@5%": 0.2535211267605634, |
| "cov@10%": 0.6056338028169014 |
| }, |
| "mean(evidence, fine-tuned)": { |
| "auroc": 0.9482758620689655, |
| "acc": 0.9154929577464789, |
| "brier": 0.07402588517908092, |
| "ece": 0.1054580941083675, |
| "cov@2%": 0.30985915492957744, |
| "cov@5%": 0.323943661971831, |
| "cov@10%": 0.6056338028169014 |
| } |
| }, |
| "test_unseen_coder_all_tasks": { |
| "baseline kev": { |
| "auroc": 0.9353021978021978, |
| "acc": 0.8995901639344263, |
| "brier": 0.09627491548962074, |
| "ece": 0.11438050809440908, |
| "cov@2%": 0.22540983606557377, |
| "cov@5%": 0.2766393442622951, |
| "cov@10%": 0.47950819672131145 |
| }, |
| "fine-tuned kev": { |
| "auroc": 0.9695741758241758, |
| "acc": 0.9364754098360656, |
| "brier": 0.05493458662718413, |
| "ece": 0.04307024866475148, |
| "cov@2%": 0.2192622950819672, |
| "cov@5%": 0.5061475409836066, |
| "cov@10%": 0.6372950819672131 |
| }, |
| "evidence score": { |
| "auroc": 0.943543956043956, |
| "acc": 0.9057377049180327, |
| "brier": 0.09111320255251247, |
| "ece": 0.09400301997638061, |
| "cov@2%": 0.0, |
| "cov@5%": 0.20901639344262296, |
| "cov@10%": 0.6188524590163934 |
| }, |
| "mean(evidence, baseline)": { |
| "auroc": 0.9518028846153846, |
| "acc": 0.9057377049180327, |
| "brier": 0.08850434302950522, |
| "ece": 0.09898749653604542, |
| "cov@2%": 0.11475409836065574, |
| "cov@5%": 0.375, |
| "cov@10%": 0.6086065573770492 |
| }, |
| "mean(evidence, fine-tuned)": { |
| "auroc": 0.9614354395604395, |
| "acc": 0.9344262295081968, |
| "brier": 0.06332625202060693, |
| "ece": 0.07127569574009193, |
| "cov@2%": 0.18032786885245902, |
| "cov@5%": 0.38729508196721313, |
| "cov@10%": 0.6372950819672131 |
| } |
| } |
| } |
