| { |
| "schema_version": 1, |
| "model": "maximerivest/paper-classifier-0.8B-v2", |
| "expert_200": { |
| "expert_flat": { |
| "disciplinary": [ |
| 57.0, |
| 47.2, |
| 66.3 |
| ], |
| "multidisciplinary": [ |
| 55.0, |
| 45.2, |
| 64.4 |
| ], |
| "top3": 85.5, |
| "ece_vs_experts": 0.1067, |
| "mean_confidence": 0.63 |
| }, |
| "expert_flat_compact": { |
| "disciplinary": [ |
| 57.0, |
| 47.2, |
| 66.3 |
| ], |
| "multidisciplinary": [ |
| 50.0, |
| 40.4, |
| 59.6 |
| ], |
| "top3": 82.0 |
| }, |
| "expert_shuffled": { |
| "disciplinary": [ |
| 55.0, |
| 45.2, |
| 64.4 |
| ], |
| "multidisciplinary": [ |
| 54.0, |
| 44.3, |
| 63.4 |
| ], |
| "same_pick_as_unshuffled": 75.5 |
| }, |
| "expert_field": { |
| "field_agreement": [ |
| 48.5, |
| 41.7, |
| 55.4 |
| ] |
| }, |
| "expert_tree": { |
| "disciplinary": [ |
| 42.0, |
| 32.8, |
| 51.8 |
| ], |
| "multidisciplinary": [ |
| 29.0, |
| 21.0, |
| 38.5 |
| ] |
| }, |
| "yesno_heldout_topics": { |
| "macro_average_precision": 0.7811, |
| "macro_auc": 0.9901, |
| "topics": 8 |
| }, |
| "unseen_invented": { |
| "reference_a": { |
| "agree_top": 79.0, |
| "log_loss": 0.7, |
| "ece": 0.0162, |
| "n": 800 |
| }, |
| "reference_b": { |
| "agree_top": 78.6, |
| "log_loss": 0.787, |
| "ece": 0.0353, |
| "n": 760 |
| }, |
| "reference_agreement": 89.9 |
| }, |
| "unseen_jel": { |
| "reference_a": { |
| "agree_top": 79.2, |
| "log_loss": 0.671, |
| "ece": 0.0388, |
| "n": 600 |
| }, |
| "reference_b": { |
| "agree_top": 79.0, |
| "log_loss": 0.908, |
| "ece": 0.0349, |
| "n": 600 |
| }, |
| "reference_agreement": 91.8 |
| }, |
| "unseen_papers": { |
| "reference_a": { |
| "agree_top": 85.3, |
| "log_loss": 0.518, |
| "ece": 0.0245, |
| "n": 1200 |
| }, |
| "reference_b": { |
| "agree_top": 85.5, |
| "log_loss": 0.634, |
| "ece": 0.0257, |
| "n": 1180 |
| }, |
| "reference_agreement": 91.5 |
| } |
| }, |
| "arxiv_and_mesh": { |
| "arxiv": { |
| "flat": { |
| "primary": [ |
| 54.1, |
| 52.3, |
| 55.8 |
| ], |
| "any_category": [ |
| 66.8, |
| 65.1, |
| 68.5 |
| ], |
| "top3": [ |
| 79.2, |
| 77.7, |
| 80.6 |
| ], |
| "group_from_pick": [ |
| 84.9, |
| 83.5, |
| 86.1 |
| ], |
| "ece": 0.0753, |
| "mean_confidence": 0.615, |
| "n": 2980 |
| }, |
| "about": { |
| "primary": [ |
| 52.9, |
| 51.1, |
| 54.7 |
| ], |
| "any_category": [ |
| 65.4, |
| 63.6, |
| 67.1 |
| ], |
| "top3": [ |
| 78.6, |
| 77.0, |
| 80.0 |
| ], |
| "group_from_pick": [ |
| 84.4, |
| 83.1, |
| 85.7 |
| ], |
| "ece": 0.0892, |
| "mean_confidence": 0.618, |
| "n": 2980 |
| }, |
| "title": { |
| "primary": [ |
| 42.2, |
| 40.5, |
| 44.0 |
| ], |
| "any_category": [ |
| 54.2, |
| 52.4, |
| 56.0 |
| ], |
| "top3": [ |
| 64.3, |
| 62.5, |
| 66.0 |
| ], |
| "group_from_pick": [ |
| 80.3, |
| 78.9, |
| 81.7 |
| ], |
| "ece": 0.1515, |
| "mean_confidence": 0.574, |
| "n": 2980 |
| }, |
| "shuffled": { |
| "primary": [ |
| 53.0, |
| 51.2, |
| 54.7 |
| ], |
| "any_category": [ |
| 65.9, |
| 64.2, |
| 67.6 |
| ], |
| "top3": [ |
| 78.6, |
| 77.0, |
| 80.0 |
| ], |
| "group_from_pick": [ |
| 84.5, |
| 83.2, |
| 85.8 |
| ], |
| "ece": 0.0825, |
| "mean_confidence": 0.609, |
| "n": 2980 |
| }, |
| "group_direct": { |
| "accuracy": [ |
| 81.1, |
| 79.7, |
| 82.5 |
| ], |
| "ece": 0.1091 |
| }, |
| "order": { |
| "same_pick_shuffled_vs_flat": [ |
| 78.2, |
| 76.7, |
| 79.6 |
| ], |
| "conf_0-0.4": { |
| "n": 508, |
| "same_pick": 0.484, |
| "accuracy": 0.293 |
| }, |
| "conf_0.4-0.6": { |
| "n": 978, |
| "same_pick": 0.692, |
| "accuracy": 0.412 |
| }, |
| "conf_0.6-0.8": { |
| "n": 808, |
| "same_pick": 0.901, |
| "accuracy": 0.611 |
| }, |
| "conf_0.8-1.01": { |
| "n": 686, |
| "same_pick": 0.99, |
| "accuracy": 0.824 |
| } |
| }, |
| "flat_by_group": { |
| "Computer Science": [ |
| 48.0, |
| 44.5, |
| 51.6 |
| ], |
| "Economics": [ |
| 61.7, |
| 49.0, |
| 72.9 |
| ], |
| "Electrical Engineering and Systems Science": [ |
| 63.7, |
| 52.8, |
| 73.4 |
| ], |
| "Mathematics": [ |
| 61.5, |
| 57.5, |
| 65.3 |
| ], |
| "Physics": [ |
| 59.6, |
| 56.6, |
| 62.6 |
| ], |
| "Quantitative Biology": [ |
| 35.5, |
| 29.2, |
| 42.3 |
| ], |
| "Quantitative Finance": [ |
| 44.4, |
| 36.9, |
| 52.1 |
| ], |
| "Statistics": [ |
| 39.0, |
| 30.0, |
| 48.8 |
| ] |
| }, |
| "flat_by_year": { |
| "2008-2011": [ |
| 50.4, |
| 45.2, |
| 55.6 |
| ], |
| "2012-2015": [ |
| 54.3, |
| 50.6, |
| 57.8 |
| ], |
| "2016-2019": [ |
| 55.4, |
| 51.8, |
| 58.9 |
| ], |
| "2020-2023": [ |
| 55.4, |
| 51.8, |
| 58.8 |
| ], |
| "2024-2027": [ |
| 51.8, |
| 46.7, |
| 56.8 |
| ] |
| }, |
| "flat_by_in_journal": { |
| "False": [ |
| 52.9, |
| 50.8, |
| 55.0 |
| ], |
| "True": [ |
| 57.3, |
| 53.8, |
| 60.7 |
| ] |
| }, |
| "flat_by_cited_10plus": { |
| "False": [ |
| 54.1, |
| 52.3, |
| 55.9 |
| ], |
| "True": [ |
| 53.3, |
| 43.8, |
| 62.6 |
| ] |
| }, |
| "flat_by_confidence": { |
| "0.0": [ |
| 11.1, |
| 3.9, |
| 28.1 |
| ], |
| "0.2": [ |
| 30.4, |
| 26.4, |
| 34.6 |
| ], |
| "0.4": [ |
| 41.2, |
| 38.2, |
| 44.3 |
| ], |
| "0.6": [ |
| 61.1, |
| 57.7, |
| 64.4 |
| ], |
| "0.8": [ |
| 82.4, |
| 79.3, |
| 85.0 |
| ] |
| }, |
| "flat_worst_categories": { |
| "cs.GL": [ |
| 0.0, |
| -0.0, |
| 16.1 |
| ], |
| "cs.OH": [ |
| 0.0, |
| -0.0, |
| 16.1 |
| ], |
| "q-bio.QM": [ |
| 0.0, |
| -0.0, |
| 16.1 |
| ], |
| "q-bio.SC": [ |
| 5.0, |
| 0.9, |
| 23.6 |
| ], |
| "cond-mat.other": [ |
| 10.0, |
| 2.8, |
| 30.1 |
| ], |
| "cs.CE": [ |
| 10.0, |
| 2.8, |
| 30.1 |
| ], |
| "cs.DM": [ |
| 10.0, |
| 2.8, |
| 30.1 |
| ], |
| "cs.PF": [ |
| 10.0, |
| 2.8, |
| 30.1 |
| ], |
| "math-ph": [ |
| 10.0, |
| 2.8, |
| 30.1 |
| ], |
| "physics.data-an": [ |
| 10.0, |
| 2.8, |
| 30.1 |
| ], |
| "physics.gen-ph": [ |
| 10.0, |
| 2.8, |
| 30.1 |
| ], |
| "q-fin.GN": [ |
| 10.0, |
| 2.8, |
| 30.1 |
| ], |
| "cs.MM": [ |
| 15.0, |
| 5.2, |
| 36.0 |
| ], |
| "physics.comp-ph": [ |
| 15.0, |
| 5.2, |
| 36.0 |
| ], |
| "q-bio.OT": [ |
| 15.0, |
| 5.2, |
| 36.0 |
| ] |
| }, |
| "flat_confusions": [ |
| "q-bio.SC -> physics.bio-ph: 12", |
| "q-bio.CB -> q-bio.QM: 9", |
| "cs.DM -> math.CO: 8", |
| "cs.OS -> cs.DC: 8", |
| "cs.PL -> cs.LO: 8", |
| "stat.ML -> cs.LG: 7", |
| "math.GM -> math.NT: 7", |
| "cs.SI -> physics.soc-ph: 7", |
| "q-bio.BM -> q-bio.QM: 7", |
| "physics.space-ph -> astro-ph.EP: 6", |
| "cs.PF -> cs.NI: 6", |
| "physics.gen-ph -> hep-th: 6", |
| "q-fin.CP -> q-fin.PR: 6", |
| "physics.hist-ph -> physics.gen-ph: 6", |
| "cs.FL -> cs.LO: 6", |
| "q-bio.TO -> physics.bio-ph: 6", |
| "physics.atm-clus -> physics.chem-ph: 6", |
| "q-fin.PM -> q-fin.MF: 6", |
| "cs.DM -> cs.DS: 5", |
| "cond-mat.dis-nn -> cond-mat.stat-mech: 5" |
| ], |
| "flat_cross_group_errors": { |
| "same_group": 939, |
| "other_group": 430 |
| } |
| }, |
| "mesh": { |
| "yesno_all": { |
| "auc": 0.8242, |
| "average_precision": 0.5238, |
| "accuracy_at_0.5": 0.7652, |
| "recall_major_at_0.5": 0.6304, |
| "ece": 0.0318, |
| "n": 9826, |
| "positives": 2500 |
| }, |
| "yesno_major_vs_minor": { |
| "auc": 0.589, |
| "average_precision": 0.5552, |
| "false_positive_rate_at_0.5": 0.4924, |
| "n_negatives": 2500 |
| }, |
| "yesno_major_vs_sibling": { |
| "auc": 0.9171, |
| "average_precision": 0.9233, |
| "false_positive_rate_at_0.5": 0.0509, |
| "n_negatives": 2375 |
| }, |
| "yesno_major_vs_branch": { |
| "auc": 0.974, |
| "average_precision": 0.9707, |
| "false_positive_rate_at_0.5": 0.0126, |
| "n_negatives": 2451 |
| }, |
| "yesno_auc_by_category": { |
| "A": 0.858, |
| "B": 0.887, |
| "C": 0.878, |
| "D": 0.826, |
| "E": 0.829, |
| "F": 0.816, |
| "G": 0.855, |
| "H": 0.87, |
| "I": 0.81, |
| "J": 0.778, |
| "K": 0.816, |
| "L": 0.755, |
| "M": 0.791, |
| "N": 0.807 |
| }, |
| "yesno_recall_by_n_majors": { |
| "1": [ |
| 64.4, |
| 61.6, |
| 67.1 |
| ], |
| "2": [ |
| 64.0, |
| 60.7, |
| 67.2 |
| ], |
| "3": [ |
| 60.3, |
| 55.1, |
| 65.2 |
| ], |
| "4": [ |
| 55.6, |
| 48.5, |
| 62.6 |
| ] |
| }, |
| "pick": { |
| "accuracy_any_major": [ |
| 44.7, |
| 42.6, |
| 46.9 |
| ], |
| "by_n_options": { |
| "10": [ |
| 32.4, |
| 25.1, |
| 40.6 |
| ], |
| "11": [ |
| 34.7, |
| 25.9, |
| 44.7 |
| ], |
| "12": [ |
| 36.8, |
| 26.9, |
| 48.1 |
| ], |
| "4": [ |
| 57.3, |
| 51.8, |
| 62.6 |
| ], |
| "5": [ |
| 51.7, |
| 46.7, |
| 56.7 |
| ], |
| "6": [ |
| 45.6, |
| 40.4, |
| 50.8 |
| ], |
| "7": [ |
| 34.6, |
| 29.4, |
| 40.2 |
| ], |
| "8": [ |
| 40.6, |
| 34.4, |
| 47.2 |
| ], |
| "9": [ |
| 45.3, |
| 38.2, |
| 52.6 |
| ] |
| }, |
| "chance": 0.307, |
| "ece": 0.1898 |
| } |
| } |
| }, |
| "original_topic_judging": { |
| "frag": { |
| "population": { |
| "auc": 0.9577, |
| "agree": 0.9555, |
| "agree_sample": 0.906, |
| "precision": 0.788, |
| "recall": 0.566, |
| "f1": 0.659, |
| "average_precision": 0.7725964404504749 |
| }, |
| "cited50": { |
| "auc": 0.9742, |
| "agree": 0.9761, |
| "agree_sample": 0.8528, |
| "precision": 0.798, |
| "recall": 0.666, |
| "f1": 0.726 |
| }, |
| "meta_refs": { |
| "found": 0.73, |
| "n": 640, |
| "found_title_only": 0.697 |
| } |
| }, |
| "llm": { |
| "population": { |
| "auc": 0.973, |
| "agree": 0.9734, |
| "agree_sample": 0.8299, |
| "precision": 0.82, |
| "recall": 0.557, |
| "f1": 0.663, |
| "average_precision": 0.7437275909250752 |
| }, |
| "landmarks": { |
| "found": 0.886, |
| "n": 44 |
| } |
| } |
| }, |
| "notes": [ |
| "Reference-labelled topic tests are not human ground truth.", |
| "Original two topics are reused development diagnostics.", |
| "Paired comparisons in arxiv_and_mesh refer to a private previous list-first checkpoint, not the earlier public paper-first release.", |
| "Benchmarks use native model prompts and restricted answer-code softmax; no text generation." |
| ] |
| } |
|
|
