| { |
| "schema_version": 1, |
| "model": "maximerivest/paper-classifier-2B-v2", |
| "expert_200": { |
| "expert_flat": { |
| "disciplinary": [ |
| 57.0, |
| 47.2, |
| 66.3 |
| ], |
| "multidisciplinary": [ |
| 66.0, |
| 56.3, |
| 74.5 |
| ], |
| "top3": 86.0, |
| "ece_vs_experts": 0.046, |
| "mean_confidence": 0.649 |
| }, |
| "expert_flat_compact": { |
| "disciplinary": [ |
| 61.0, |
| 51.2, |
| 70.0 |
| ], |
| "multidisciplinary": [ |
| 65.0, |
| 55.3, |
| 73.6 |
| ], |
| "top3": 85.5 |
| }, |
| "expert_shuffled": { |
| "disciplinary": [ |
| 57.0, |
| 47.2, |
| 66.3 |
| ], |
| "multidisciplinary": [ |
| 64.0, |
| 54.2, |
| 72.7 |
| ], |
| "same_pick_as_unshuffled": 76.5 |
| }, |
| "expert_field": { |
| "field_agreement": [ |
| 51.5, |
| 44.6, |
| 58.3 |
| ] |
| }, |
| "expert_tree": { |
| "disciplinary": [ |
| 47.0, |
| 37.5, |
| 56.7 |
| ], |
| "multidisciplinary": [ |
| 33.0, |
| 24.6, |
| 42.7 |
| ] |
| }, |
| "yesno_heldout_topics": { |
| "macro_average_precision": 0.8196, |
| "macro_auc": 0.9939, |
| "topics": 8 |
| }, |
| "unseen_invented": { |
| "reference_a": { |
| "agree_top": 81.6, |
| "log_loss": 0.614, |
| "ece": 0.0206, |
| "n": 800 |
| }, |
| "reference_b": { |
| "agree_top": 81.1, |
| "log_loss": 0.702, |
| "ece": 0.0313, |
| "n": 760 |
| }, |
| "reference_agreement": 89.9 |
| }, |
| "unseen_jel": { |
| "reference_a": { |
| "agree_top": 82.8, |
| "log_loss": 0.558, |
| "ece": 0.0233, |
| "n": 600 |
| }, |
| "reference_b": { |
| "agree_top": 83.7, |
| "log_loss": 0.809, |
| "ece": 0.0217, |
| "n": 600 |
| }, |
| "reference_agreement": 91.8 |
| }, |
| "unseen_papers": { |
| "reference_a": { |
| "agree_top": 87.4, |
| "log_loss": 0.476, |
| "ece": 0.0337, |
| "n": 1200 |
| }, |
| "reference_b": { |
| "agree_top": 87.7, |
| "log_loss": 0.589, |
| "ece": 0.0371, |
| "n": 1180 |
| }, |
| "reference_agreement": 91.5 |
| } |
| }, |
| "arxiv_and_mesh": { |
| "arxiv": { |
| "flat": { |
| "primary": [ |
| 57.3, |
| 55.6, |
| 59.1 |
| ], |
| "any_category": [ |
| 70.4, |
| 68.8, |
| 72.0 |
| ], |
| "top3": [ |
| 82.5, |
| 81.1, |
| 83.8 |
| ], |
| "group_from_pick": [ |
| 85.4, |
| 84.1, |
| 86.6 |
| ], |
| "ece": 0.0793, |
| "mean_confidence": 0.652, |
| "n": 2980 |
| }, |
| "about": { |
| "primary": [ |
| 55.8, |
| 54.0, |
| 57.5 |
| ], |
| "any_category": [ |
| 69.3, |
| 67.6, |
| 70.9 |
| ], |
| "top3": [ |
| 82.2, |
| 80.8, |
| 83.5 |
| ], |
| "group_from_pick": [ |
| 84.6, |
| 83.3, |
| 85.9 |
| ], |
| "ece": 0.0935, |
| "mean_confidence": 0.651, |
| "n": 2980 |
| }, |
| "title": { |
| "primary": [ |
| 46.9, |
| 45.1, |
| 48.7 |
| ], |
| "any_category": [ |
| 59.2, |
| 57.5, |
| 61.0 |
| ], |
| "top3": [ |
| 69.9, |
| 68.2, |
| 71.5 |
| ], |
| "group_from_pick": [ |
| 81.2, |
| 79.8, |
| 82.6 |
| ], |
| "ece": 0.1444, |
| "mean_confidence": 0.613, |
| "n": 2980 |
| }, |
| "shuffled": { |
| "primary": [ |
| 57.3, |
| 55.6, |
| 59.1 |
| ], |
| "any_category": [ |
| 70.0, |
| 68.3, |
| 71.6 |
| ], |
| "top3": [ |
| 82.3, |
| 80.9, |
| 83.6 |
| ], |
| "group_from_pick": [ |
| 86.4, |
| 85.1, |
| 87.6 |
| ], |
| "ece": 0.0836, |
| "mean_confidence": 0.656, |
| "n": 2980 |
| }, |
| "group_direct": { |
| "accuracy": [ |
| 80.6, |
| 79.1, |
| 82.0 |
| ], |
| "ece": 0.1108 |
| }, |
| "order": { |
| "same_pick_shuffled_vs_flat": [ |
| 81.8, |
| 80.4, |
| 83.1 |
| ], |
| "conf_0-0.4": { |
| "n": 429, |
| "same_pick": 0.522, |
| "accuracy": 0.315 |
| }, |
| "conf_0.4-0.6": { |
| "n": 863, |
| "same_pick": 0.694, |
| "accuracy": 0.417 |
| }, |
| "conf_0.6-0.8": { |
| "n": 782, |
| "same_pick": 0.914, |
| "accuracy": 0.587 |
| }, |
| "conf_0.8-1.01": { |
| "n": 906, |
| "same_pick": 0.992, |
| "accuracy": 0.833 |
| } |
| }, |
| "flat_by_group": { |
| "Computer Science": [ |
| 51.1, |
| 47.5, |
| 54.6 |
| ], |
| "Economics": [ |
| 56.7, |
| 44.1, |
| 68.4 |
| ], |
| "Electrical Engineering and Systems Science": [ |
| 61.3, |
| 50.3, |
| 71.2 |
| ], |
| "Mathematics": [ |
| 66.5, |
| 62.6, |
| 70.2 |
| ], |
| "Physics": [ |
| 63.2, |
| 60.2, |
| 66.1 |
| ], |
| "Quantitative Biology": [ |
| 34.0, |
| 27.8, |
| 40.8 |
| ], |
| "Quantitative Finance": [ |
| 52.5, |
| 44.8, |
| 60.1 |
| ], |
| "Statistics": [ |
| 42.0, |
| 32.8, |
| 51.8 |
| ] |
| }, |
| "flat_by_year": { |
| "2008-2011": [ |
| 54.7, |
| 49.5, |
| 59.8 |
| ], |
| "2012-2015": [ |
| 57.3, |
| 53.7, |
| 60.8 |
| ], |
| "2016-2019": [ |
| 57.7, |
| 54.2, |
| 61.2 |
| ], |
| "2020-2023": [ |
| 58.3, |
| 54.8, |
| 61.7 |
| ], |
| "2024-2027": [ |
| 57.2, |
| 52.1, |
| 62.1 |
| ] |
| }, |
| "flat_by_in_journal": { |
| "False": [ |
| 55.9, |
| 53.8, |
| 57.9 |
| ], |
| "True": [ |
| 61.5, |
| 58.0, |
| 64.8 |
| ] |
| }, |
| "flat_by_cited_10plus": { |
| "False": [ |
| 57.3, |
| 55.5, |
| 59.1 |
| ], |
| "True": [ |
| 59.0, |
| 49.5, |
| 68.0 |
| ] |
| }, |
| "flat_by_confidence": { |
| "0.0": [ |
| 28.6, |
| 11.7, |
| 54.6 |
| ], |
| "0.2": [ |
| 31.6, |
| 27.3, |
| 36.2 |
| ], |
| "0.4": [ |
| 41.7, |
| 38.5, |
| 45.0 |
| ], |
| "0.6": [ |
| 58.7, |
| 55.2, |
| 62.1 |
| ], |
| "0.8": [ |
| 83.3, |
| 80.8, |
| 85.6 |
| ] |
| }, |
| "flat_worst_categories": { |
| "cs.OH": [ |
| 0.0, |
| -0.0, |
| 16.1 |
| ], |
| "q-bio.SC": [ |
| 0.0, |
| -0.0, |
| 16.1 |
| ], |
| "cs.GL": [ |
| 5.0, |
| 0.9, |
| 23.6 |
| ], |
| "q-bio.BM": [ |
| 5.0, |
| 0.9, |
| 23.6 |
| ], |
| "q-bio.OT": [ |
| 5.0, |
| 0.9, |
| 23.6 |
| ], |
| "cs.DM": [ |
| 10.0, |
| 2.8, |
| 30.1 |
| ], |
| "cs.ET": [ |
| 10.0, |
| 2.8, |
| 30.1 |
| ], |
| "cs.PF": [ |
| 10.0, |
| 2.8, |
| 30.1 |
| ], |
| "physics.data-an": [ |
| 10.0, |
| 2.8, |
| 30.1 |
| ], |
| "cond-mat.other": [ |
| 15.0, |
| 5.2, |
| 36.0 |
| ], |
| "cs.CE": [ |
| 15.0, |
| 5.2, |
| 36.0 |
| ], |
| "q-bio.QM": [ |
| 15.0, |
| 5.2, |
| 36.0 |
| ], |
| "cs.MM": [ |
| 20.0, |
| 8.1, |
| 41.6 |
| ], |
| "math-ph": [ |
| 25.0, |
| 11.2, |
| 46.9 |
| ], |
| "physics.pop-ph": [ |
| 25.0, |
| 11.2, |
| 46.9 |
| ] |
| }, |
| "flat_confusions": [ |
| "q-bio.SC -> physics.bio-ph: 14", |
| "stat.ML -> cs.LG: 10", |
| "q-bio.BM -> physics.bio-ph: 9", |
| "cs.DM -> math.CO: 7", |
| "cs.OS -> cs.DC: 7", |
| "q-bio.CB -> q-bio.QM: 7", |
| "cs.PF -> cs.DC: 7", |
| "q-bio.TO -> physics.bio-ph: 6", |
| "physics.atm-clus -> physics.chem-ph: 6", |
| "cs.PL -> cs.LO: 6", |
| "cs.SI -> physics.soc-ph: 6", |
| "q-bio.BM -> q-bio.QM: 6", |
| "cs.PF -> cs.NI: 5", |
| "math.GN -> math.LO: 5", |
| "econ.EM -> stat.AP: 5", |
| "math.GM -> math.NT: 5", |
| "cs.NI -> cs.DC: 5", |
| "econ.GN -> econ.TH: 5", |
| "physics.space-ph -> astro-ph.EP: 4", |
| "cs.GR -> cs.HC: 4" |
| ], |
| "flat_cross_group_errors": { |
| "same_group": 848, |
| "other_group": 423 |
| } |
| }, |
| "mesh": { |
| "yesno_all": { |
| "auc": 0.8372, |
| "average_precision": 0.5485, |
| "accuracy_at_0.5": 0.7715, |
| "recall_major_at_0.5": 0.68, |
| "ece": 0.0266, |
| "n": 9826, |
| "positives": 2500 |
| }, |
| "yesno_major_vs_minor": { |
| "auc": 0.6069, |
| "average_precision": 0.575, |
| "false_positive_rate_at_0.5": 0.52, |
| "n_negatives": 2500 |
| }, |
| "yesno_major_vs_sibling": { |
| "auc": 0.9324, |
| "average_precision": 0.9357, |
| "false_positive_rate_at_0.5": 0.0505, |
| "n_negatives": 2375 |
| }, |
| "yesno_major_vs_branch": { |
| "auc": 0.9799, |
| "average_precision": 0.9769, |
| "false_positive_rate_at_0.5": 0.0102, |
| "n_negatives": 2451 |
| }, |
| "yesno_auc_by_category": { |
| "A": 0.864, |
| "B": 0.861, |
| "C": 0.883, |
| "D": 0.836, |
| "E": 0.841, |
| "F": 0.828, |
| "G": 0.856, |
| "H": 0.886, |
| "I": 0.823, |
| "J": 0.8, |
| "K": 0.832, |
| "L": 0.791, |
| "M": 0.807, |
| "N": 0.82 |
| }, |
| "yesno_recall_by_n_majors": { |
| "1": [ |
| 69.2, |
| 66.4, |
| 71.8 |
| ], |
| "2": [ |
| 69.4, |
| 66.1, |
| 72.4 |
| ], |
| "3": [ |
| 65.9, |
| 60.8, |
| 70.7 |
| ], |
| "4": [ |
| 58.8, |
| 51.7, |
| 65.6 |
| ] |
| }, |
| "pick": { |
| "accuracy_any_major": [ |
| 45.0, |
| 42.9, |
| 47.2 |
| ], |
| "by_n_options": { |
| "10": [ |
| 28.7, |
| 21.7, |
| 36.8 |
| ], |
| "11": [ |
| 33.7, |
| 25.0, |
| 43.7 |
| ], |
| "12": [ |
| 40.8, |
| 30.4, |
| 52.0 |
| ], |
| "4": [ |
| 58.5, |
| 53.1, |
| 63.8 |
| ], |
| "5": [ |
| 49.1, |
| 44.0, |
| 54.1 |
| ], |
| "6": [ |
| 48.1, |
| 42.9, |
| 53.4 |
| ], |
| "7": [ |
| 38.7, |
| 33.3, |
| 44.4 |
| ], |
| "8": [ |
| 39.3, |
| 33.1, |
| 45.8 |
| ], |
| "9": [ |
| 43.6, |
| 36.6, |
| 50.9 |
| ] |
| }, |
| "chance": 0.307, |
| "ece": 0.1792 |
| } |
| } |
| }, |
| "original_topic_judging": { |
| "frag": { |
| "population": { |
| "auc": 0.9702, |
| "agree": 0.9602, |
| "agree_sample": 0.9054, |
| "precision": 0.708, |
| "recall": 0.808, |
| "f1": 0.755, |
| "average_precision": 0.8342379000300797 |
| }, |
| "cited50": { |
| "auc": 0.9909, |
| "agree": 0.9798, |
| "agree_sample": 0.8917, |
| "precision": 0.732, |
| "recall": 0.906, |
| "f1": 0.81 |
| }, |
| "meta_refs": { |
| "found": 0.912, |
| "n": 640, |
| "found_title_only": 0.915 |
| } |
| }, |
| "llm": { |
| "population": { |
| "auc": 0.9882, |
| "agree": 0.9548, |
| "agree_sample": 0.7907, |
| "precision": 0.51, |
| "recall": 0.94, |
| "f1": 0.662, |
| "average_precision": 0.8262713035326974 |
| }, |
| "landmarks": { |
| "found": 0.932, |
| "n": 44 |
| } |
| } |
| }, |
| "notes": [ |
| "Reference-labelled topic tests are not human ground truth.", |
| "Original two topics are reused development diagnostics.", |
| "Paired comparisons in arxiv_and_mesh refer to a private previous list-first checkpoint, not the earlier public paper-first release.", |
| "Benchmarks use native model prompts and restricted answer-code softmax; no text generation." |
| ] |
| } |
|
|
