{
  "schema_version": 1,
  "model": "maximerivest/paper-classifier-2B-v2",
  "expert_200": {
    "expert_flat": {
      "disciplinary": [
        57.0,
        47.2,
        66.3
      ],
      "multidisciplinary": [
        66.0,
        56.3,
        74.5
      ],
      "top3": 86.0,
      "ece_vs_experts": 0.046,
      "mean_confidence": 0.649
    },
    "expert_flat_compact": {
      "disciplinary": [
        61.0,
        51.2,
        70.0
      ],
      "multidisciplinary": [
        65.0,
        55.3,
        73.6
      ],
      "top3": 85.5
    },
    "expert_shuffled": {
      "disciplinary": [
        57.0,
        47.2,
        66.3
      ],
      "multidisciplinary": [
        64.0,
        54.2,
        72.7
      ],
      "same_pick_as_unshuffled": 76.5
    },
    "expert_field": {
      "field_agreement": [
        51.5,
        44.6,
        58.3
      ]
    },
    "expert_tree": {
      "disciplinary": [
        47.0,
        37.5,
        56.7
      ],
      "multidisciplinary": [
        33.0,
        24.6,
        42.7
      ]
    },
    "yesno_heldout_topics": {
      "macro_average_precision": 0.8196,
      "macro_auc": 0.9939,
      "topics": 8
    },
    "unseen_invented": {
      "reference_a": {
        "agree_top": 81.6,
        "log_loss": 0.614,
        "ece": 0.0206,
        "n": 800
      },
      "reference_b": {
        "agree_top": 81.1,
        "log_loss": 0.702,
        "ece": 0.0313,
        "n": 760
      },
      "reference_agreement": 89.9
    },
    "unseen_jel": {
      "reference_a": {
        "agree_top": 82.8,
        "log_loss": 0.558,
        "ece": 0.0233,
        "n": 600
      },
      "reference_b": {
        "agree_top": 83.7,
        "log_loss": 0.809,
        "ece": 0.0217,
        "n": 600
      },
      "reference_agreement": 91.8
    },
    "unseen_papers": {
      "reference_a": {
        "agree_top": 87.4,
        "log_loss": 0.476,
        "ece": 0.0337,
        "n": 1200
      },
      "reference_b": {
        "agree_top": 87.7,
        "log_loss": 0.589,
        "ece": 0.0371,
        "n": 1180
      },
      "reference_agreement": 91.5
    }
  },
  "arxiv_and_mesh": {
    "arxiv": {
      "flat": {
        "primary": [
          57.3,
          55.6,
          59.1
        ],
        "any_category": [
          70.4,
          68.8,
          72.0
        ],
        "top3": [
          82.5,
          81.1,
          83.8
        ],
        "group_from_pick": [
          85.4,
          84.1,
          86.6
        ],
        "ece": 0.0793,
        "mean_confidence": 0.652,
        "n": 2980
      },
      "about": {
        "primary": [
          55.8,
          54.0,
          57.5
        ],
        "any_category": [
          69.3,
          67.6,
          70.9
        ],
        "top3": [
          82.2,
          80.8,
          83.5
        ],
        "group_from_pick": [
          84.6,
          83.3,
          85.9
        ],
        "ece": 0.0935,
        "mean_confidence": 0.651,
        "n": 2980
      },
      "title": {
        "primary": [
          46.9,
          45.1,
          48.7
        ],
        "any_category": [
          59.2,
          57.5,
          61.0
        ],
        "top3": [
          69.9,
          68.2,
          71.5
        ],
        "group_from_pick": [
          81.2,
          79.8,
          82.6
        ],
        "ece": 0.1444,
        "mean_confidence": 0.613,
        "n": 2980
      },
      "shuffled": {
        "primary": [
          57.3,
          55.6,
          59.1
        ],
        "any_category": [
          70.0,
          68.3,
          71.6
        ],
        "top3": [
          82.3,
          80.9,
          83.6
        ],
        "group_from_pick": [
          86.4,
          85.1,
          87.6
        ],
        "ece": 0.0836,
        "mean_confidence": 0.656,
        "n": 2980
      },
      "group_direct": {
        "accuracy": [
          80.6,
          79.1,
          82.0
        ],
        "ece": 0.1108
      },
      "order": {
        "same_pick_shuffled_vs_flat": [
          81.8,
          80.4,
          83.1
        ],
        "conf_0-0.4": {
          "n": 429,
          "same_pick": 0.522,
          "accuracy": 0.315
        },
        "conf_0.4-0.6": {
          "n": 863,
          "same_pick": 0.694,
          "accuracy": 0.417
        },
        "conf_0.6-0.8": {
          "n": 782,
          "same_pick": 0.914,
          "accuracy": 0.587
        },
        "conf_0.8-1.01": {
          "n": 906,
          "same_pick": 0.992,
          "accuracy": 0.833
        }
      },
      "flat_by_group": {
        "Computer Science": [
          51.1,
          47.5,
          54.6
        ],
        "Economics": [
          56.7,
          44.1,
          68.4
        ],
        "Electrical Engineering and Systems Science": [
          61.3,
          50.3,
          71.2
        ],
        "Mathematics": [
          66.5,
          62.6,
          70.2
        ],
        "Physics": [
          63.2,
          60.2,
          66.1
        ],
        "Quantitative Biology": [
          34.0,
          27.8,
          40.8
        ],
        "Quantitative Finance": [
          52.5,
          44.8,
          60.1
        ],
        "Statistics": [
          42.0,
          32.8,
          51.8
        ]
      },
      "flat_by_year": {
        "2008-2011": [
          54.7,
          49.5,
          59.8
        ],
        "2012-2015": [
          57.3,
          53.7,
          60.8
        ],
        "2016-2019": [
          57.7,
          54.2,
          61.2
        ],
        "2020-2023": [
          58.3,
          54.8,
          61.7
        ],
        "2024-2027": [
          57.2,
          52.1,
          62.1
        ]
      },
      "flat_by_in_journal": {
        "False": [
          55.9,
          53.8,
          57.9
        ],
        "True": [
          61.5,
          58.0,
          64.8
        ]
      },
      "flat_by_cited_10plus": {
        "False": [
          57.3,
          55.5,
          59.1
        ],
        "True": [
          59.0,
          49.5,
          68.0
        ]
      },
      "flat_by_confidence": {
        "0.0": [
          28.6,
          11.7,
          54.6
        ],
        "0.2": [
          31.6,
          27.3,
          36.2
        ],
        "0.4": [
          41.7,
          38.5,
          45.0
        ],
        "0.6": [
          58.7,
          55.2,
          62.1
        ],
        "0.8": [
          83.3,
          80.8,
          85.6
        ]
      },
      "flat_worst_categories": {
        "cs.OH": [
          0.0,
          -0.0,
          16.1
        ],
        "q-bio.SC": [
          0.0,
          -0.0,
          16.1
        ],
        "cs.GL": [
          5.0,
          0.9,
          23.6
        ],
        "q-bio.BM": [
          5.0,
          0.9,
          23.6
        ],
        "q-bio.OT": [
          5.0,
          0.9,
          23.6
        ],
        "cs.DM": [
          10.0,
          2.8,
          30.1
        ],
        "cs.ET": [
          10.0,
          2.8,
          30.1
        ],
        "cs.PF": [
          10.0,
          2.8,
          30.1
        ],
        "physics.data-an": [
          10.0,
          2.8,
          30.1
        ],
        "cond-mat.other": [
          15.0,
          5.2,
          36.0
        ],
        "cs.CE": [
          15.0,
          5.2,
          36.0
        ],
        "q-bio.QM": [
          15.0,
          5.2,
          36.0
        ],
        "cs.MM": [
          20.0,
          8.1,
          41.6
        ],
        "math-ph": [
          25.0,
          11.2,
          46.9
        ],
        "physics.pop-ph": [
          25.0,
          11.2,
          46.9
        ]
      },
      "flat_confusions": [
        "q-bio.SC -> physics.bio-ph: 14",
        "stat.ML -> cs.LG: 10",
        "q-bio.BM -> physics.bio-ph: 9",
        "cs.DM -> math.CO: 7",
        "cs.OS -> cs.DC: 7",
        "q-bio.CB -> q-bio.QM: 7",
        "cs.PF -> cs.DC: 7",
        "q-bio.TO -> physics.bio-ph: 6",
        "physics.atm-clus -> physics.chem-ph: 6",
        "cs.PL -> cs.LO: 6",
        "cs.SI -> physics.soc-ph: 6",
        "q-bio.BM -> q-bio.QM: 6",
        "cs.PF -> cs.NI: 5",
        "math.GN -> math.LO: 5",
        "econ.EM -> stat.AP: 5",
        "math.GM -> math.NT: 5",
        "cs.NI -> cs.DC: 5",
        "econ.GN -> econ.TH: 5",
        "physics.space-ph -> astro-ph.EP: 4",
        "cs.GR -> cs.HC: 4"
      ],
      "flat_cross_group_errors": {
        "same_group": 848,
        "other_group": 423
      }
    },
    "mesh": {
      "yesno_all": {
        "auc": 0.8372,
        "average_precision": 0.5485,
        "accuracy_at_0.5": 0.7715,
        "recall_major_at_0.5": 0.68,
        "ece": 0.0266,
        "n": 9826,
        "positives": 2500
      },
      "yesno_major_vs_minor": {
        "auc": 0.6069,
        "average_precision": 0.575,
        "false_positive_rate_at_0.5": 0.52,
        "n_negatives": 2500
      },
      "yesno_major_vs_sibling": {
        "auc": 0.9324,
        "average_precision": 0.9357,
        "false_positive_rate_at_0.5": 0.0505,
        "n_negatives": 2375
      },
      "yesno_major_vs_branch": {
        "auc": 0.9799,
        "average_precision": 0.9769,
        "false_positive_rate_at_0.5": 0.0102,
        "n_negatives": 2451
      },
      "yesno_auc_by_category": {
        "A": 0.864,
        "B": 0.861,
        "C": 0.883,
        "D": 0.836,
        "E": 0.841,
        "F": 0.828,
        "G": 0.856,
        "H": 0.886,
        "I": 0.823,
        "J": 0.8,
        "K": 0.832,
        "L": 0.791,
        "M": 0.807,
        "N": 0.82
      },
      "yesno_recall_by_n_majors": {
        "1": [
          69.2,
          66.4,
          71.8
        ],
        "2": [
          69.4,
          66.1,
          72.4
        ],
        "3": [
          65.9,
          60.8,
          70.7
        ],
        "4": [
          58.8,
          51.7,
          65.6
        ]
      },
      "pick": {
        "accuracy_any_major": [
          45.0,
          42.9,
          47.2
        ],
        "by_n_options": {
          "10": [
            28.7,
            21.7,
            36.8
          ],
          "11": [
            33.7,
            25.0,
            43.7
          ],
          "12": [
            40.8,
            30.4,
            52.0
          ],
          "4": [
            58.5,
            53.1,
            63.8
          ],
          "5": [
            49.1,
            44.0,
            54.1
          ],
          "6": [
            48.1,
            42.9,
            53.4
          ],
          "7": [
            38.7,
            33.3,
            44.4
          ],
          "8": [
            39.3,
            33.1,
            45.8
          ],
          "9": [
            43.6,
            36.6,
            50.9
          ]
        },
        "chance": 0.307,
        "ece": 0.1792
      }
    }
  },
  "original_topic_judging": {
    "frag": {
      "population": {
        "auc": 0.9702,
        "agree": 0.9602,
        "agree_sample": 0.9054,
        "precision": 0.708,
        "recall": 0.808,
        "f1": 0.755,
        "average_precision": 0.8342379000300797
      },
      "cited50": {
        "auc": 0.9909,
        "agree": 0.9798,
        "agree_sample": 0.8917,
        "precision": 0.732,
        "recall": 0.906,
        "f1": 0.81
      },
      "meta_refs": {
        "found": 0.912,
        "n": 640,
        "found_title_only": 0.915
      }
    },
    "llm": {
      "population": {
        "auc": 0.9882,
        "agree": 0.9548,
        "agree_sample": 0.7907,
        "precision": 0.51,
        "recall": 0.94,
        "f1": 0.662,
        "average_precision": 0.8262713035326974
      },
      "landmarks": {
        "found": 0.932,
        "n": 44
      }
    }
  },
  "notes": [
    "Reference-labelled topic tests are not human ground truth.",
    "Original two topics are reused development diagnostics.",
    "Paired comparisons in arxiv_and_mesh refer to a private previous list-first checkpoint, not the earlier public paper-first release.",
    "Benchmarks use native model prompts and restricted answer-code softmax; no text generation."
  ]
}