phaseA_metrics.json

{
  "accuracy_all_lower_bound": 0.812,
  "accuracy_scored": 0.812,
  "classification_report": {
    "Agricultural and Veterinary Sciences": {
      "f1-score": 0.7692307692307693,
      "precision": 0.8072916666666666,
      "recall": 0.7345971563981043,
      "support": 211.0
    },
    "Engineering and Technology": {
      "f1-score": 0.8514851485148515,
      "precision": 0.8704453441295547,
      "recall": 0.8333333333333334,
      "support": 258.0
    },
    "Humanities and the Arts": {
      "f1-score": 0.8218527315914489,
      "precision": 0.8084112149532711,
      "recall": 0.8357487922705314,
      "support": 207.0
    },
    "Medical and Health Sciences": {
      "f1-score": 0.8881118881118881,
      "precision": 0.9071428571428571,
      "recall": 0.8698630136986302,
      "support": 292.0
    },
    "Natural Sciences": {
      "f1-score": 0.731610337972167,
      "precision": 0.6917293233082706,
      "recall": 0.7763713080168776,
      "support": 237.0
    },
    "Social Sciences": {
      "f1-score": 0.7953020134228188,
      "precision": 0.7873754152823921,
      "recall": 0.8033898305084746,
      "support": 295.0
    },
    "accuracy": 0.812,
    "macro avg": {
      "f1-score": 0.809598814807324,
      "precision": 0.8120659702471688,
      "recall": 0.8088839057043252,
      "support": 1500.0
    },
    "weighted avg": {
      "f1-score": 0.8129658609678403,
      "precision": 0.8155705822436659,
      "recall": 0.812,
      "support": 1500.0
    }
  },
  "cohen_kappa_scored": 0.7735384799881362,
  "confusion_matrix": {
    "columns": [
      "Agricultural and Veterinary Sciences",
      "Engineering and Technology",
      "Humanities and the Arts",
      "Medical and Health Sciences",
      "Natural Sciences",
      "Social Sciences"
    ],
    "matrix": [
      [
        155,
        4,
        1,
        7,
        36,
        8
      ],
      [
        10,
        215,
        2,
        2,
        19,
        10
      ],
      [
        2,
        1,
        173,
        1,
        3,
        27
      ],
      [
        8,
        1,
        3,
        254,
        15,
        11
      ],
      [
        10,
        21,
        1,
        13,
        184,
        8
      ],
      [
        7,
        5,
        34,
        3,
        9,
        237
      ]
    ],
    "rows": [
      "Agricultural and Veterinary Sciences",
      "Engineering and Technology",
      "Humanities and the Arts",
      "Medical and Health Sciences",
      "Natural Sciences",
      "Social Sciences"
    ]
  },
  "coverage": 1.0,
  "generated_at": "2026-07-19T18:05:53.939674+00:00",
  "macro_f1_all_with_abstentions": 0.809598814807324,
  "macro_f1_scored": 0.809598814807324,
  "mean_max_probability": 0.7910507917404175,
  "median_max_probability": 0.8443766236305237,
  "n_abstained": 0,
  "n_scored": 1500,
  "n_target": 1500,
  "proposal_disagreement_diagnostic": {
    "checkpoint_abstains": 0,
    "checkpoint_matches_neither": 57,
    "checkpoint_matches_openalex": 144,
    "checkpoint_matches_panel": 297,
    "n_panel_openalex_disagree": 498,
    "reference_status": "provisional_unadjudicated_codex_panel"
  },
  "provenance": {
    "checkpoint_path": "/opt/bibr-training-oecd-run/oecd_minilm_ablation_20260719/checkpoints/oecd_minilm_deepseek_teacher_20260719",
    "encoder_name": "sentence-transformers/all-MiniLM-L6-v2",
    "inference_config_sha256": "44c7c38bbebf60b20eaa75ce173267ade9814e6acdd2f2a0084b2609f70814ca",
    "input_template": "title_abstract_v1",
    "label_maps_sha256": "053d9d54ed59e5c7a8fb3ed80d4a6826180dfa5e15c4bbc190922c886b77a870",
    "max_length": 256,
    "model_sha256": "7a46c595cf3bb8e1eff39303786e8b7d16c95ea87f9f06b12d4a753f0be1e580",
    "rows_path": "/opt/bibr-training-oecd-run/oecd_minilm_ablation_20260719/data/oecd_phaseA_panel_20260719.parquet",
    "rows_sha256": "3554e33eea4680848d99268ed0745fcdbcb95f9b88cf2019f67e327adad1cd28",
    "runtime": {
      "cuda_device": "NVIDIA GeForce RTX 3090",
      "device": "cuda",
      "python": "3.12.12",
      "sklearn": "1.8.0",
      "torch": "2.11.0+cu130",
      "transformers": "5.4.0"
    },
    "source_hashes": {
      "scripts/diagnose_paper_classifier.py": "943f7d1e84469c0e2d1817168b607e50ba12db2d94a95774820fa0f04407e39e",
      "src/bibr_training/models/oecd_multitask.py": "8ffeb67be8cac60b288270ef6b6a737e888b3437a399cf4e76471cdc7bd057bb",
      "src/bibr_training/paper_classifier/transformer_dataset.py": "1214996cd87cfedbfd26e9642c53add97095ebd173a64bbbf1348577df7c204e"
    },
    "source_root": "/home/jakub/pco"
  },
  "quality_claim_allowed": false,
  "reference_status": "provisional_unadjudicated_codex_panel"
}