| { |
| "harness": "evaluation/eval.py (15 benchmark categories, weighted aggregate)", |
| "selected_checkpoint": "step_1000", |
| "selected_checkpoint_overall_score": 0.71, |
| "selected_checkpoint_benchmark_scores": { |
| "math_reasoning": 0.55, |
| "code_generation": 0.65, |
| "text_classification": 0.828, |
| "sentiment_analysis": 0.792, |
| "question_answering": 0.607, |
| "logical_reasoning": 0.819, |
| "common_sense": 0.736, |
| "reading_comprehension": 0.7, |
| "dialogue_generation": 0.644, |
| "summarization": 0.767, |
| "translation": 0.804, |
| "knowledge_retrieval": 0.676, |
| "creative_writing": 0.61, |
| "instruction_following": 0.758, |
| "safety_evaluation": 0.739 |
| }, |
| "all_checkpoints_overall_scores": { |
| "step_100": 0.48, |
| "step_200": 0.535, |
| "step_300": 0.576, |
| "step_400": 0.608, |
| "step_500": 0.635, |
| "step_600": 0.656, |
| "step_700": 0.674, |
| "step_800": 0.689, |
| "step_900": 0.7, |
| "step_1000": 0.71 |
| }, |
| "all_checkpoints_benchmark_scores": { |
| "step_100": { |
| "math_reasoning": 0.345, |
| "code_generation": 0.35, |
| "text_classification": 0.517, |
| "sentiment_analysis": 0.617, |
| "question_answering": 0.475, |
| "logical_reasoning": 0.319, |
| "common_sense": 0.53, |
| "reading_comprehension": 0.475, |
| "dialogue_generation": 0.438, |
| "summarization": 0.517, |
| "translation": 0.64, |
| "knowledge_retrieval": 0.529, |
| "creative_writing": 0.328, |
| "instruction_following": 0.55, |
| "safety_evaluation": 0.628 |
| }, |
| "step_200": { |
| "math_reasoning": 0.383, |
| "code_generation": 0.421, |
| "text_classification": 0.603, |
| "sentiment_analysis": 0.675, |
| "question_answering": 0.51, |
| "logical_reasoning": 0.375, |
| "common_sense": 0.583, |
| "reading_comprehension": 0.529, |
| "dialogue_generation": 0.493, |
| "summarization": 0.6, |
| "translation": 0.7, |
| "knowledge_retrieval": 0.57, |
| "creative_writing": 0.388, |
| "instruction_following": 0.61, |
| "safety_evaluation": 0.65 |
| }, |
| "step_300": { |
| "math_reasoning": 0.415, |
| "code_generation": 0.475, |
| "text_classification": 0.667, |
| "sentiment_analysis": 0.71, |
| "question_answering": 0.533, |
| "logical_reasoning": 0.445, |
| "common_sense": 0.621, |
| "reading_comprehension": 0.569, |
| "dialogue_generation": 0.53, |
| "summarization": 0.65, |
| "translation": 0.733, |
| "knowledge_retrieval": 0.596, |
| "creative_writing": 0.436, |
| "instruction_following": 0.65, |
| "safety_evaluation": 0.668 |
| }, |
| "step_400": { |
| "math_reasoning": 0.443, |
| "code_generation": 0.517, |
| "text_classification": 0.714, |
| "sentiment_analysis": 0.733, |
| "question_answering": 0.55, |
| "logical_reasoning": 0.525, |
| "common_sense": 0.65, |
| "reading_comprehension": 0.6, |
| "dialogue_generation": 0.557, |
| "summarization": 0.683, |
| "translation": 0.755, |
| "knowledge_retrieval": 0.615, |
| "creative_writing": 0.475, |
| "instruction_following": 0.679, |
| "safety_evaluation": 0.683 |
| }, |
| "step_500": { |
| "math_reasoning": 0.467, |
| "code_generation": 0.55, |
| "text_classification": 0.75, |
| "sentiment_analysis": 0.75, |
| "question_answering": 0.564, |
| "logical_reasoning": 0.605, |
| "common_sense": 0.672, |
| "reading_comprehension": 0.625, |
| "dialogue_generation": 0.579, |
| "summarization": 0.707, |
| "translation": 0.769, |
| "knowledge_retrieval": 0.631, |
| "creative_writing": 0.507, |
| "instruction_following": 0.7, |
| "safety_evaluation": 0.696 |
| }, |
| "step_600": { |
| "math_reasoning": 0.487, |
| "code_generation": 0.577, |
| "text_classification": 0.776, |
| "sentiment_analysis": 0.762, |
| "question_answering": 0.575, |
| "logical_reasoning": 0.675, |
| "common_sense": 0.69, |
| "reading_comprehension": 0.645, |
| "dialogue_generation": 0.596, |
| "summarization": 0.725, |
| "translation": 0.78, |
| "knowledge_retrieval": 0.643, |
| "creative_writing": 0.534, |
| "instruction_following": 0.717, |
| "safety_evaluation": 0.707 |
| }, |
| "step_700": { |
| "math_reasoning": 0.506, |
| "code_generation": 0.6, |
| "text_classification": 0.795, |
| "sentiment_analysis": 0.772, |
| "question_answering": 0.584, |
| "logical_reasoning": 0.731, |
| "common_sense": 0.705, |
| "reading_comprehension": 0.663, |
| "dialogue_generation": 0.611, |
| "summarization": 0.739, |
| "translation": 0.788, |
| "knowledge_retrieval": 0.653, |
| "creative_writing": 0.557, |
| "instruction_following": 0.73, |
| "safety_evaluation": 0.717 |
| }, |
| "step_800": { |
| "math_reasoning": 0.522, |
| "code_generation": 0.619, |
| "text_classification": 0.809, |
| "sentiment_analysis": 0.78, |
| "question_answering": 0.593, |
| "logical_reasoning": 0.773, |
| "common_sense": 0.717, |
| "reading_comprehension": 0.677, |
| "dialogue_generation": 0.624, |
| "summarization": 0.75, |
| "translation": 0.795, |
| "knowledge_retrieval": 0.662, |
| "creative_writing": 0.577, |
| "instruction_following": 0.741, |
| "safety_evaluation": 0.725 |
| }, |
| "step_900": { |
| "math_reasoning": 0.537, |
| "code_generation": 0.636, |
| "text_classification": 0.82, |
| "sentiment_analysis": 0.786, |
| "question_answering": 0.6, |
| "logical_reasoning": 0.801, |
| "common_sense": 0.727, |
| "reading_comprehension": 0.689, |
| "dialogue_generation": 0.634, |
| "summarization": 0.759, |
| "translation": 0.8, |
| "knowledge_retrieval": 0.67, |
| "creative_writing": 0.595, |
| "instruction_following": 0.75, |
| "safety_evaluation": 0.732 |
| }, |
| "step_1000": { |
| "math_reasoning": 0.55, |
| "code_generation": 0.65, |
| "text_classification": 0.828, |
| "sentiment_analysis": 0.792, |
| "question_answering": 0.607, |
| "logical_reasoning": 0.819, |
| "common_sense": 0.736, |
| "reading_comprehension": 0.7, |
| "dialogue_generation": 0.644, |
| "summarization": 0.767, |
| "translation": 0.804, |
| "knowledge_retrieval": 0.676, |
| "creative_writing": 0.61, |
| "instruction_following": 0.758, |
| "safety_evaluation": 0.739 |
| } |
| } |
| } |
