{
  "harness": "evaluation/eval.py (15 benchmark categories, weighted aggregate)",
  "benchmark_categories": [
    "math_reasoning",
    "code_generation",
    "text_classification",
    "sentiment_analysis",
    "question_answering",
    "logical_reasoning",
    "common_sense",
    "reading_comprehension",
    "dialogue_generation",
    "summarization",
    "translation",
    "knowledge_retrieval",
    "creative_writing",
    "instruction_following",
    "safety_evaluation"
  ],
  "selected_checkpoint": "step_1000",
  "selected_checkpoint_overall_score": 0.71,
  "selected_checkpoint_benchmark_scores": {
    "math_reasoning": 0.55,
    "code_generation": 0.65,
    "text_classification": 0.828,
    "sentiment_analysis": 0.792,
    "question_answering": 0.607,
    "logical_reasoning": 0.819,
    "common_sense": 0.736,
    "reading_comprehension": 0.7,
    "dialogue_generation": 0.644,
    "summarization": 0.767,
    "translation": 0.804,
    "knowledge_retrieval": 0.676,
    "creative_writing": 0.61,
    "instruction_following": 0.758,
    "safety_evaluation": 0.739
  },
  "all_checkpoints_overall_scores": {
    "step_100": 0.48,
    "step_200": 0.535,
    "step_300": 0.576,
    "step_400": 0.608,
    "step_500": 0.635,
    "step_600": 0.656,
    "step_700": 0.674,
    "step_800": 0.689,
    "step_900": 0.7,
    "step_1000": 0.71
  },
  "all_checkpoints_benchmark_scores": {
    "step_100": {
      "math_reasoning": 0.345,
      "code_generation": 0.35,
      "text_classification": 0.517,
      "sentiment_analysis": 0.617,
      "question_answering": 0.475,
      "logical_reasoning": 0.319,
      "common_sense": 0.53,
      "reading_comprehension": 0.475,
      "dialogue_generation": 0.438,
      "summarization": 0.517,
      "translation": 0.64,
      "knowledge_retrieval": 0.529,
      "creative_writing": 0.328,
      "instruction_following": 0.55,
      "safety_evaluation": 0.628
    },
    "step_200": {
      "math_reasoning": 0.383,
      "code_generation": 0.421,
      "text_classification": 0.603,
      "sentiment_analysis": 0.675,
      "question_answering": 0.51,
      "logical_reasoning": 0.375,
      "common_sense": 0.583,
      "reading_comprehension": 0.529,
      "dialogue_generation": 0.493,
      "summarization": 0.6,
      "translation": 0.7,
      "knowledge_retrieval": 0.57,
      "creative_writing": 0.388,
      "instruction_following": 0.61,
      "safety_evaluation": 0.65
    },
    "step_300": {
      "math_reasoning": 0.415,
      "code_generation": 0.475,
      "text_classification": 0.667,
      "sentiment_analysis": 0.71,
      "question_answering": 0.533,
      "logical_reasoning": 0.445,
      "common_sense": 0.621,
      "reading_comprehension": 0.569,
      "dialogue_generation": 0.53,
      "summarization": 0.65,
      "translation": 0.733,
      "knowledge_retrieval": 0.596,
      "creative_writing": 0.436,
      "instruction_following": 0.65,
      "safety_evaluation": 0.668
    },
    "step_400": {
      "math_reasoning": 0.443,
      "code_generation": 0.517,
      "text_classification": 0.714,
      "sentiment_analysis": 0.733,
      "question_answering": 0.55,
      "logical_reasoning": 0.525,
      "common_sense": 0.65,
      "reading_comprehension": 0.6,
      "dialogue_generation": 0.557,
      "summarization": 0.683,
      "translation": 0.755,
      "knowledge_retrieval": 0.615,
      "creative_writing": 0.475,
      "instruction_following": 0.679,
      "safety_evaluation": 0.683
    },
    "step_500": {
      "math_reasoning": 0.467,
      "code_generation": 0.55,
      "text_classification": 0.75,
      "sentiment_analysis": 0.75,
      "question_answering": 0.564,
      "logical_reasoning": 0.605,
      "common_sense": 0.672,
      "reading_comprehension": 0.625,
      "dialogue_generation": 0.579,
      "summarization": 0.707,
      "translation": 0.769,
      "knowledge_retrieval": 0.631,
      "creative_writing": 0.507,
      "instruction_following": 0.7,
      "safety_evaluation": 0.696
    },
    "step_600": {
      "math_reasoning": 0.487,
      "code_generation": 0.577,
      "text_classification": 0.776,
      "sentiment_analysis": 0.762,
      "question_answering": 0.575,
      "logical_reasoning": 0.675,
      "common_sense": 0.69,
      "reading_comprehension": 0.645,
      "dialogue_generation": 0.596,
      "summarization": 0.725,
      "translation": 0.78,
      "knowledge_retrieval": 0.643,
      "creative_writing": 0.534,
      "instruction_following": 0.717,
      "safety_evaluation": 0.707
    },
    "step_700": {
      "math_reasoning": 0.506,
      "code_generation": 0.6,
      "text_classification": 0.795,
      "sentiment_analysis": 0.772,
      "question_answering": 0.584,
      "logical_reasoning": 0.731,
      "common_sense": 0.705,
      "reading_comprehension": 0.663,
      "dialogue_generation": 0.611,
      "summarization": 0.739,
      "translation": 0.788,
      "knowledge_retrieval": 0.653,
      "creative_writing": 0.557,
      "instruction_following": 0.73,
      "safety_evaluation": 0.717
    },
    "step_800": {
      "math_reasoning": 0.522,
      "code_generation": 0.619,
      "text_classification": 0.809,
      "sentiment_analysis": 0.78,
      "question_answering": 0.593,
      "logical_reasoning": 0.773,
      "common_sense": 0.717,
      "reading_comprehension": 0.677,
      "dialogue_generation": 0.624,
      "summarization": 0.75,
      "translation": 0.795,
      "knowledge_retrieval": 0.662,
      "creative_writing": 0.577,
      "instruction_following": 0.741,
      "safety_evaluation": 0.725
    },
    "step_900": {
      "math_reasoning": 0.537,
      "code_generation": 0.636,
      "text_classification": 0.82,
      "sentiment_analysis": 0.786,
      "question_answering": 0.6,
      "logical_reasoning": 0.801,
      "common_sense": 0.727,
      "reading_comprehension": 0.689,
      "dialogue_generation": 0.634,
      "summarization": 0.759,
      "translation": 0.8,
      "knowledge_retrieval": 0.67,
      "creative_writing": 0.595,
      "instruction_following": 0.75,
      "safety_evaluation": 0.732
    },
    "step_1000": {
      "math_reasoning": 0.55,
      "code_generation": 0.65,
      "text_classification": 0.828,
      "sentiment_analysis": 0.792,
      "question_answering": 0.607,
      "logical_reasoning": 0.819,
      "common_sense": 0.736,
      "reading_comprehension": 0.7,
      "dialogue_generation": 0.644,
      "summarization": 0.767,
      "translation": 0.804,
      "knowledge_retrieval": 0.676,
      "creative_writing": 0.61,
      "instruction_following": 0.758,
      "safety_evaluation": 0.739
    }
  },
  "notes": [
    "Scores were produced by running the official harness `evaluation/eval.py` on every checkpoint in `checkpoints/`.",
    "`evaluation/benchmarks/code_generation/eval.py` checked the checkpoint path with os.path.isfile() and was fixed to os.path.isdir(); `evaluation/benchmarks/dialogue_generation/eval.py` called an undefined config_init() which was removed. Both fixes are included in this repo.",
    "The shipped utils extension is built for CPython 3.13; this environment runs CPython 3.12, so `evaluation/utils/benchmark_utils.c` (the shipped Cython source) was compiled for 3.12 and used unmodified. Results were cross-checked against an independent pure-Python reconstruction and agreed exactly.",
    "Repo-root config.json / pytorch_model.bin are copies of the selected checkpoint step_1000."
  ]
}