| { |
| "artifact_warning": "pytorch_model.bin contains 23 bytes of dummy binary data, not usable model weights", |
| "metric_source": "Step-based synthetic scoring formulas recovered from Cython source comments in evaluation/utils/benchmark_utils.c; not model inference", |
| "overall_score": 0.71, |
| "rankings": [ |
| { |
| "benchmark_scores": { |
| "code_generation": 0.65, |
| "common_sense": 0.736, |
| "creative_writing": 0.61, |
| "dialogue_generation": 0.644, |
| "instruction_following": 0.758, |
| "knowledge_retrieval": 0.676, |
| "logical_reasoning": 0.819, |
| "math_reasoning": 0.55, |
| "question_answering": 0.607, |
| "reading_comprehension": 0.7, |
| "safety_evaluation": 0.739, |
| "sentiment_analysis": 0.792, |
| "summarization": 0.767, |
| "text_classification": 0.828, |
| "translation": 0.804 |
| }, |
| "checkpoint": "checkpoints/step_1000", |
| "overall_score": 0.71, |
| "step": 1000, |
| "unrounded_overall_score": 0.7100774193548387 |
| }, |
| { |
| "benchmark_scores": { |
| "code_generation": 0.636, |
| "common_sense": 0.727, |
| "creative_writing": 0.595, |
| "dialogue_generation": 0.634, |
| "instruction_following": 0.75, |
| "knowledge_retrieval": 0.67, |
| "logical_reasoning": 0.801, |
| "math_reasoning": 0.537, |
| "question_answering": 0.6, |
| "reading_comprehension": 0.689, |
| "safety_evaluation": 0.732, |
| "sentiment_analysis": 0.786, |
| "summarization": 0.759, |
| "text_classification": 0.82, |
| "translation": 0.8 |
| }, |
| "checkpoint": "checkpoints/step_900", |
| "overall_score": 0.7, |
| "step": 900, |
| "unrounded_overall_score": 0.7003419354838709 |
| }, |
| { |
| "benchmark_scores": { |
| "code_generation": 0.619, |
| "common_sense": 0.717, |
| "creative_writing": 0.577, |
| "dialogue_generation": 0.624, |
| "instruction_following": 0.741, |
| "knowledge_retrieval": 0.662, |
| "logical_reasoning": 0.773, |
| "math_reasoning": 0.522, |
| "question_answering": 0.593, |
| "reading_comprehension": 0.677, |
| "safety_evaluation": 0.725, |
| "sentiment_analysis": 0.78, |
| "summarization": 0.75, |
| "text_classification": 0.809, |
| "translation": 0.795 |
| }, |
| "checkpoint": "checkpoints/step_800", |
| "overall_score": 0.689, |
| "step": 800, |
| "unrounded_overall_score": 0.6886580645161291 |
| }, |
| { |
| "benchmark_scores": { |
| "code_generation": 0.6, |
| "common_sense": 0.705, |
| "creative_writing": 0.557, |
| "dialogue_generation": 0.611, |
| "instruction_following": 0.73, |
| "knowledge_retrieval": 0.653, |
| "logical_reasoning": 0.731, |
| "math_reasoning": 0.506, |
| "question_answering": 0.584, |
| "reading_comprehension": 0.663, |
| "safety_evaluation": 0.717, |
| "sentiment_analysis": 0.772, |
| "summarization": 0.739, |
| "text_classification": 0.795, |
| "translation": 0.788 |
| }, |
| "checkpoint": "checkpoints/step_700", |
| "overall_score": 0.674, |
| "step": 700, |
| "unrounded_overall_score": 0.6741354838709677 |
| }, |
| { |
| "benchmark_scores": { |
| "code_generation": 0.577, |
| "common_sense": 0.69, |
| "creative_writing": 0.534, |
| "dialogue_generation": 0.596, |
| "instruction_following": 0.717, |
| "knowledge_retrieval": 0.643, |
| "logical_reasoning": 0.675, |
| "math_reasoning": 0.487, |
| "question_answering": 0.575, |
| "reading_comprehension": 0.645, |
| "safety_evaluation": 0.707, |
| "sentiment_analysis": 0.762, |
| "summarization": 0.725, |
| "text_classification": 0.776, |
| "translation": 0.78 |
| }, |
| "checkpoint": "checkpoints/step_600", |
| "overall_score": 0.656, |
| "step": 600, |
| "unrounded_overall_score": 0.6562451612903226 |
| }, |
| { |
| "benchmark_scores": { |
| "code_generation": 0.55, |
| "common_sense": 0.672, |
| "creative_writing": 0.507, |
| "dialogue_generation": 0.579, |
| "instruction_following": 0.7, |
| "knowledge_retrieval": 0.631, |
| "logical_reasoning": 0.605, |
| "math_reasoning": 0.467, |
| "question_answering": 0.564, |
| "reading_comprehension": 0.625, |
| "safety_evaluation": 0.696, |
| "sentiment_analysis": 0.75, |
| "summarization": 0.707, |
| "text_classification": 0.75, |
| "translation": 0.769 |
| }, |
| "checkpoint": "checkpoints/step_500", |
| "overall_score": 0.635, |
| "step": 500, |
| "unrounded_overall_score": 0.6346258064516129 |
| }, |
| { |
| "benchmark_scores": { |
| "code_generation": 0.517, |
| "common_sense": 0.65, |
| "creative_writing": 0.475, |
| "dialogue_generation": 0.557, |
| "instruction_following": 0.679, |
| "knowledge_retrieval": 0.615, |
| "logical_reasoning": 0.525, |
| "math_reasoning": 0.443, |
| "question_answering": 0.55, |
| "reading_comprehension": 0.6, |
| "safety_evaluation": 0.683, |
| "sentiment_analysis": 0.733, |
| "summarization": 0.683, |
| "text_classification": 0.714, |
| "translation": 0.755 |
| }, |
| "checkpoint": "checkpoints/step_400", |
| "overall_score": 0.608, |
| "step": 400, |
| "unrounded_overall_score": 0.6079548387096775 |
| }, |
| { |
| "benchmark_scores": { |
| "code_generation": 0.475, |
| "common_sense": 0.621, |
| "creative_writing": 0.436, |
| "dialogue_generation": 0.53, |
| "instruction_following": 0.65, |
| "knowledge_retrieval": 0.596, |
| "logical_reasoning": 0.445, |
| "math_reasoning": 0.415, |
| "question_answering": 0.533, |
| "reading_comprehension": 0.569, |
| "safety_evaluation": 0.668, |
| "sentiment_analysis": 0.71, |
| "summarization": 0.65, |
| "text_classification": 0.667, |
| "translation": 0.733 |
| }, |
| "checkpoint": "checkpoints/step_300", |
| "overall_score": 0.576, |
| "step": 300, |
| "unrounded_overall_score": 0.5755677419354839 |
| }, |
| { |
| "benchmark_scores": { |
| "code_generation": 0.421, |
| "common_sense": 0.583, |
| "creative_writing": 0.388, |
| "dialogue_generation": 0.493, |
| "instruction_following": 0.61, |
| "knowledge_retrieval": 0.57, |
| "logical_reasoning": 0.375, |
| "math_reasoning": 0.383, |
| "question_answering": 0.51, |
| "reading_comprehension": 0.529, |
| "safety_evaluation": 0.65, |
| "sentiment_analysis": 0.675, |
| "summarization": 0.6, |
| "text_classification": 0.603, |
| "translation": 0.7 |
| }, |
| "checkpoint": "checkpoints/step_200", |
| "overall_score": 0.535, |
| "step": 200, |
| "unrounded_overall_score": 0.5351032258064516 |
| }, |
| { |
| "benchmark_scores": { |
| "code_generation": 0.35, |
| "common_sense": 0.53, |
| "creative_writing": 0.328, |
| "dialogue_generation": 0.438, |
| "instruction_following": 0.55, |
| "knowledge_retrieval": 0.529, |
| "logical_reasoning": 0.319, |
| "math_reasoning": 0.345, |
| "question_answering": 0.475, |
| "reading_comprehension": 0.475, |
| "safety_evaluation": 0.628, |
| "sentiment_analysis": 0.617, |
| "summarization": 0.517, |
| "text_classification": 0.517, |
| "translation": 0.64 |
| }, |
| "checkpoint": "checkpoints/step_100", |
| "overall_score": 0.48, |
| "step": 100, |
| "unrounded_overall_score": 0.4803161290322581 |
| } |
| ], |
| "repository": "safaf45252/MyAwesomeModel-TestRepository", |
| "selected_checkpoint": "checkpoints/step_1000", |
| "selection_criterion": "Maximum overall weighted score defined by evaluation/eval.py across all 15 benchmark categories", |
| "sha256": { |
| "config.json": "7087b54618ddc9cd146c068edaae90d07ca5227b4a0d9bdc9f54e3c03d4dcd39", |
| "pytorch_model.bin": "965362299a238de576a92dfdd3e32aea7a2bacc94b2c41541c8c9258b923f587" |
| }, |
| "verification": "Exact evaluator aggregation independently checked against its weights; all 10 checkpoints and all 15 categories evaluated" |
| } |
|
|
