{
  "schema_version": 1,
  "model": "maximerivest/paper-classifier-0.8B-v2",
  "expert_200": {
    "expert_flat": {
      "disciplinary": [
        57.0,
        47.2,
        66.3
      ],
      "multidisciplinary": [
        55.0,
        45.2,
        64.4
      ],
      "top3": 85.5,
      "ece_vs_experts": 0.1067,
      "mean_confidence": 0.63
    },
    "expert_flat_compact": {
      "disciplinary": [
        57.0,
        47.2,
        66.3
      ],
      "multidisciplinary": [
        50.0,
        40.4,
        59.6
      ],
      "top3": 82.0
    },
    "expert_shuffled": {
      "disciplinary": [
        55.0,
        45.2,
        64.4
      ],
      "multidisciplinary": [
        54.0,
        44.3,
        63.4
      ],
      "same_pick_as_unshuffled": 75.5
    },
    "expert_field": {
      "field_agreement": [
        48.5,
        41.7,
        55.4
      ]
    },
    "expert_tree": {
      "disciplinary": [
        42.0,
        32.8,
        51.8
      ],
      "multidisciplinary": [
        29.0,
        21.0,
        38.5
      ]
    },
    "yesno_heldout_topics": {
      "macro_average_precision": 0.7811,
      "macro_auc": 0.9901,
      "topics": 8
    },
    "unseen_invented": {
      "reference_a": {
        "agree_top": 79.0,
        "log_loss": 0.7,
        "ece": 0.0162,
        "n": 800
      },
      "reference_b": {
        "agree_top": 78.6,
        "log_loss": 0.787,
        "ece": 0.0353,
        "n": 760
      },
      "reference_agreement": 89.9
    },
    "unseen_jel": {
      "reference_a": {
        "agree_top": 79.2,
        "log_loss": 0.671,
        "ece": 0.0388,
        "n": 600
      },
      "reference_b": {
        "agree_top": 79.0,
        "log_loss": 0.908,
        "ece": 0.0349,
        "n": 600
      },
      "reference_agreement": 91.8
    },
    "unseen_papers": {
      "reference_a": {
        "agree_top": 85.3,
        "log_loss": 0.518,
        "ece": 0.0245,
        "n": 1200
      },
      "reference_b": {
        "agree_top": 85.5,
        "log_loss": 0.634,
        "ece": 0.0257,
        "n": 1180
      },
      "reference_agreement": 91.5
    }
  },
  "arxiv_and_mesh": {
    "arxiv": {
      "flat": {
        "primary": [
          54.1,
          52.3,
          55.8
        ],
        "any_category": [
          66.8,
          65.1,
          68.5
        ],
        "top3": [
          79.2,
          77.7,
          80.6
        ],
        "group_from_pick": [
          84.9,
          83.5,
          86.1
        ],
        "ece": 0.0753,
        "mean_confidence": 0.615,
        "n": 2980
      },
      "about": {
        "primary": [
          52.9,
          51.1,
          54.7
        ],
        "any_category": [
          65.4,
          63.6,
          67.1
        ],
        "top3": [
          78.6,
          77.0,
          80.0
        ],
        "group_from_pick": [
          84.4,
          83.1,
          85.7
        ],
        "ece": 0.0892,
        "mean_confidence": 0.618,
        "n": 2980
      },
      "title": {
        "primary": [
          42.2,
          40.5,
          44.0
        ],
        "any_category": [
          54.2,
          52.4,
          56.0
        ],
        "top3": [
          64.3,
          62.5,
          66.0
        ],
        "group_from_pick": [
          80.3,
          78.9,
          81.7
        ],
        "ece": 0.1515,
        "mean_confidence": 0.574,
        "n": 2980
      },
      "shuffled": {
        "primary": [
          53.0,
          51.2,
          54.7
        ],
        "any_category": [
          65.9,
          64.2,
          67.6
        ],
        "top3": [
          78.6,
          77.0,
          80.0
        ],
        "group_from_pick": [
          84.5,
          83.2,
          85.8
        ],
        "ece": 0.0825,
        "mean_confidence": 0.609,
        "n": 2980
      },
      "group_direct": {
        "accuracy": [
          81.1,
          79.7,
          82.5
        ],
        "ece": 0.1091
      },
      "order": {
        "same_pick_shuffled_vs_flat": [
          78.2,
          76.7,
          79.6
        ],
        "conf_0-0.4": {
          "n": 508,
          "same_pick": 0.484,
          "accuracy": 0.293
        },
        "conf_0.4-0.6": {
          "n": 978,
          "same_pick": 0.692,
          "accuracy": 0.412
        },
        "conf_0.6-0.8": {
          "n": 808,
          "same_pick": 0.901,
          "accuracy": 0.611
        },
        "conf_0.8-1.01": {
          "n": 686,
          "same_pick": 0.99,
          "accuracy": 0.824
        }
      },
      "flat_by_group": {
        "Computer Science": [
          48.0,
          44.5,
          51.6
        ],
        "Economics": [
          61.7,
          49.0,
          72.9
        ],
        "Electrical Engineering and Systems Science": [
          63.7,
          52.8,
          73.4
        ],
        "Mathematics": [
          61.5,
          57.5,
          65.3
        ],
        "Physics": [
          59.6,
          56.6,
          62.6
        ],
        "Quantitative Biology": [
          35.5,
          29.2,
          42.3
        ],
        "Quantitative Finance": [
          44.4,
          36.9,
          52.1
        ],
        "Statistics": [
          39.0,
          30.0,
          48.8
        ]
      },
      "flat_by_year": {
        "2008-2011": [
          50.4,
          45.2,
          55.6
        ],
        "2012-2015": [
          54.3,
          50.6,
          57.8
        ],
        "2016-2019": [
          55.4,
          51.8,
          58.9
        ],
        "2020-2023": [
          55.4,
          51.8,
          58.8
        ],
        "2024-2027": [
          51.8,
          46.7,
          56.8
        ]
      },
      "flat_by_in_journal": {
        "False": [
          52.9,
          50.8,
          55.0
        ],
        "True": [
          57.3,
          53.8,
          60.7
        ]
      },
      "flat_by_cited_10plus": {
        "False": [
          54.1,
          52.3,
          55.9
        ],
        "True": [
          53.3,
          43.8,
          62.6
        ]
      },
      "flat_by_confidence": {
        "0.0": [
          11.1,
          3.9,
          28.1
        ],
        "0.2": [
          30.4,
          26.4,
          34.6
        ],
        "0.4": [
          41.2,
          38.2,
          44.3
        ],
        "0.6": [
          61.1,
          57.7,
          64.4
        ],
        "0.8": [
          82.4,
          79.3,
          85.0
        ]
      },
      "flat_worst_categories": {
        "cs.GL": [
          0.0,
          -0.0,
          16.1
        ],
        "cs.OH": [
          0.0,
          -0.0,
          16.1
        ],
        "q-bio.QM": [
          0.0,
          -0.0,
          16.1
        ],
        "q-bio.SC": [
          5.0,
          0.9,
          23.6
        ],
        "cond-mat.other": [
          10.0,
          2.8,
          30.1
        ],
        "cs.CE": [
          10.0,
          2.8,
          30.1
        ],
        "cs.DM": [
          10.0,
          2.8,
          30.1
        ],
        "cs.PF": [
          10.0,
          2.8,
          30.1
        ],
        "math-ph": [
          10.0,
          2.8,
          30.1
        ],
        "physics.data-an": [
          10.0,
          2.8,
          30.1
        ],
        "physics.gen-ph": [
          10.0,
          2.8,
          30.1
        ],
        "q-fin.GN": [
          10.0,
          2.8,
          30.1
        ],
        "cs.MM": [
          15.0,
          5.2,
          36.0
        ],
        "physics.comp-ph": [
          15.0,
          5.2,
          36.0
        ],
        "q-bio.OT": [
          15.0,
          5.2,
          36.0
        ]
      },
      "flat_confusions": [
        "q-bio.SC -> physics.bio-ph: 12",
        "q-bio.CB -> q-bio.QM: 9",
        "cs.DM -> math.CO: 8",
        "cs.OS -> cs.DC: 8",
        "cs.PL -> cs.LO: 8",
        "stat.ML -> cs.LG: 7",
        "math.GM -> math.NT: 7",
        "cs.SI -> physics.soc-ph: 7",
        "q-bio.BM -> q-bio.QM: 7",
        "physics.space-ph -> astro-ph.EP: 6",
        "cs.PF -> cs.NI: 6",
        "physics.gen-ph -> hep-th: 6",
        "q-fin.CP -> q-fin.PR: 6",
        "physics.hist-ph -> physics.gen-ph: 6",
        "cs.FL -> cs.LO: 6",
        "q-bio.TO -> physics.bio-ph: 6",
        "physics.atm-clus -> physics.chem-ph: 6",
        "q-fin.PM -> q-fin.MF: 6",
        "cs.DM -> cs.DS: 5",
        "cond-mat.dis-nn -> cond-mat.stat-mech: 5"
      ],
      "flat_cross_group_errors": {
        "same_group": 939,
        "other_group": 430
      }
    },
    "mesh": {
      "yesno_all": {
        "auc": 0.8242,
        "average_precision": 0.5238,
        "accuracy_at_0.5": 0.7652,
        "recall_major_at_0.5": 0.6304,
        "ece": 0.0318,
        "n": 9826,
        "positives": 2500
      },
      "yesno_major_vs_minor": {
        "auc": 0.589,
        "average_precision": 0.5552,
        "false_positive_rate_at_0.5": 0.4924,
        "n_negatives": 2500
      },
      "yesno_major_vs_sibling": {
        "auc": 0.9171,
        "average_precision": 0.9233,
        "false_positive_rate_at_0.5": 0.0509,
        "n_negatives": 2375
      },
      "yesno_major_vs_branch": {
        "auc": 0.974,
        "average_precision": 0.9707,
        "false_positive_rate_at_0.5": 0.0126,
        "n_negatives": 2451
      },
      "yesno_auc_by_category": {
        "A": 0.858,
        "B": 0.887,
        "C": 0.878,
        "D": 0.826,
        "E": 0.829,
        "F": 0.816,
        "G": 0.855,
        "H": 0.87,
        "I": 0.81,
        "J": 0.778,
        "K": 0.816,
        "L": 0.755,
        "M": 0.791,
        "N": 0.807
      },
      "yesno_recall_by_n_majors": {
        "1": [
          64.4,
          61.6,
          67.1
        ],
        "2": [
          64.0,
          60.7,
          67.2
        ],
        "3": [
          60.3,
          55.1,
          65.2
        ],
        "4": [
          55.6,
          48.5,
          62.6
        ]
      },
      "pick": {
        "accuracy_any_major": [
          44.7,
          42.6,
          46.9
        ],
        "by_n_options": {
          "10": [
            32.4,
            25.1,
            40.6
          ],
          "11": [
            34.7,
            25.9,
            44.7
          ],
          "12": [
            36.8,
            26.9,
            48.1
          ],
          "4": [
            57.3,
            51.8,
            62.6
          ],
          "5": [
            51.7,
            46.7,
            56.7
          ],
          "6": [
            45.6,
            40.4,
            50.8
          ],
          "7": [
            34.6,
            29.4,
            40.2
          ],
          "8": [
            40.6,
            34.4,
            47.2
          ],
          "9": [
            45.3,
            38.2,
            52.6
          ]
        },
        "chance": 0.307,
        "ece": 0.1898
      }
    }
  },
  "original_topic_judging": {
    "frag": {
      "population": {
        "auc": 0.9577,
        "agree": 0.9555,
        "agree_sample": 0.906,
        "precision": 0.788,
        "recall": 0.566,
        "f1": 0.659,
        "average_precision": 0.7725964404504749
      },
      "cited50": {
        "auc": 0.9742,
        "agree": 0.9761,
        "agree_sample": 0.8528,
        "precision": 0.798,
        "recall": 0.666,
        "f1": 0.726
      },
      "meta_refs": {
        "found": 0.73,
        "n": 640,
        "found_title_only": 0.697
      }
    },
    "llm": {
      "population": {
        "auc": 0.973,
        "agree": 0.9734,
        "agree_sample": 0.8299,
        "precision": 0.82,
        "recall": 0.557,
        "f1": 0.663,
        "average_precision": 0.7437275909250752
      },
      "landmarks": {
        "found": 0.886,
        "n": 44
      }
    }
  },
  "notes": [
    "Reference-labelled topic tests are not human ground truth.",
    "Original two topics are reused development diagnostics.",
    "Paired comparisons in arxiv_and_mesh refer to a private previous list-first checkpoint, not the earlier public paper-first release.",
    "Benchmarks use native model prompts and restricted answer-code softmax; no text generation."
  ]
}