| { |
| "repo_name": "MyAwesomeModel-EfficientRelease-TestRepo", |
| "selection_policy": "Choose the smallest numeric training step whose weighted overall score, safety_evaluation, and instruction_following meet their release thresholds.", |
| "selected_checkpoint": "step_800", |
| "thresholds": { |
| "overall_min": 0.680, |
| "safety_evaluation_min": 0.720, |
| "instruction_following_min": 0.740 |
| }, |
| "checkpoint_scores": [ |
| { |
| "checkpoint": "step_100", |
| "overall": 0.480, |
| "safety_evaluation": 0.628, |
| "instruction_following": 0.550 |
| }, |
| { |
| "checkpoint": "step_200", |
| "overall": 0.535, |
| "safety_evaluation": 0.650, |
| "instruction_following": 0.610 |
| }, |
| { |
| "checkpoint": "step_300", |
| "overall": 0.576, |
| "safety_evaluation": 0.668, |
| "instruction_following": 0.650 |
| }, |
| { |
| "checkpoint": "step_400", |
| "overall": 0.608, |
| "safety_evaluation": 0.683, |
| "instruction_following": 0.679 |
| }, |
| { |
| "checkpoint": "step_500", |
| "overall": 0.635, |
| "safety_evaluation": 0.696, |
| "instruction_following": 0.700 |
| }, |
| { |
| "checkpoint": "step_600", |
| "overall": 0.656, |
| "safety_evaluation": 0.707, |
| "instruction_following": 0.717 |
| }, |
| { |
| "checkpoint": "step_700", |
| "overall": 0.674, |
| "safety_evaluation": 0.717, |
| "instruction_following": 0.730 |
| }, |
| { |
| "checkpoint": "step_800", |
| "overall": 0.689, |
| "safety_evaluation": 0.725, |
| "instruction_following": 0.741 |
| }, |
| { |
| "checkpoint": "step_900", |
| "overall": 0.700, |
| "safety_evaluation": 0.732, |
| "instruction_following": 0.750 |
| }, |
| { |
| "checkpoint": "step_1000", |
| "overall": 0.710, |
| "safety_evaluation": 0.739, |
| "instruction_following": 0.758 |
| } |
| ], |
| "gate_margins": [ |
| { |
| "checkpoint": "step_100", |
| "overall_margin": -0.200, |
| "safety_evaluation_margin": -0.092, |
| "instruction_following_margin": -0.190, |
| "meets_all_gates": false |
| }, |
| { |
| "checkpoint": "step_200", |
| "overall_margin": -0.145, |
| "safety_evaluation_margin": -0.070, |
| "instruction_following_margin": -0.130, |
| "meets_all_gates": false |
| }, |
| { |
| "checkpoint": "step_300", |
| "overall_margin": -0.104, |
| "safety_evaluation_margin": -0.052, |
| "instruction_following_margin": -0.090, |
| "meets_all_gates": false |
| }, |
| { |
| "checkpoint": "step_400", |
| "overall_margin": -0.072, |
| "safety_evaluation_margin": -0.037, |
| "instruction_following_margin": -0.061, |
| "meets_all_gates": false |
| }, |
| { |
| "checkpoint": "step_500", |
| "overall_margin": -0.045, |
| "safety_evaluation_margin": -0.024, |
| "instruction_following_margin": -0.040, |
| "meets_all_gates": false |
| }, |
| { |
| "checkpoint": "step_600", |
| "overall_margin": -0.024, |
| "safety_evaluation_margin": -0.013, |
| "instruction_following_margin": -0.023, |
| "meets_all_gates": false |
| }, |
| { |
| "checkpoint": "step_700", |
| "overall_margin": -0.006, |
| "safety_evaluation_margin": -0.003, |
| "instruction_following_margin": -0.010, |
| "meets_all_gates": false |
| }, |
| { |
| "checkpoint": "step_800", |
| "overall_margin": 0.009, |
| "safety_evaluation_margin": 0.005, |
| "instruction_following_margin": 0.001, |
| "meets_all_gates": true |
| }, |
| { |
| "checkpoint": "step_900", |
| "overall_margin": 0.020, |
| "safety_evaluation_margin": 0.012, |
| "instruction_following_margin": 0.010, |
| "meets_all_gates": true |
| }, |
| { |
| "checkpoint": "step_1000", |
| "overall_margin": 0.030, |
| "safety_evaluation_margin": 0.019, |
| "instruction_following_margin": 0.018, |
| "meets_all_gates": true |
| } |
| ], |
| "selected_benchmark_scores": { |
| "math_reasoning": 0.522, |
| "logical_reasoning": 0.773, |
| "common_sense": 0.717, |
| "reading_comprehension": 0.677, |
| "question_answering": 0.593, |
| "text_classification": 0.809, |
| "sentiment_analysis": 0.780, |
| "code_generation": 0.619, |
| "creative_writing": 0.577, |
| "dialogue_generation": 0.624, |
| "summarization": 0.750, |
| "translation": 0.795, |
| "knowledge_retrieval": 0.662, |
| "instruction_following": 0.741, |
| "safety_evaluation": 0.725 |
| }, |
| "artifact_manifest": [ |
| { |
| "path": "README.md", |
| "sha256": "3e0876deafb0fe0e1ece7efb7bfc1ae5092bab1bb601e108c10c4ff3ab1152e0" |
| }, |
| { |
| "path": "config.json", |
| "sha256": "7087b54618ddc9cd146c068edaae90d07ca5227b4a0d9bdc9f54e3c03d4dcd39" |
| }, |
| { |
| "path": "figures/fig1.png", |
| "sha256": "bd81e62dbd4289b54d154db29f00e5854d7b6c1f7acdc0d8f6647f790567b43a" |
| }, |
| { |
| "path": "figures/fig2.png", |
| "sha256": "bd81e62dbd4289b54d154db29f00e5854d7b6c1f7acdc0d8f6647f790567b43a" |
| }, |
| { |
| "path": "figures/fig3.png", |
| "sha256": "bd81e62dbd4289b54d154db29f00e5854d7b6c1f7acdc0d8f6647f790567b43a" |
| }, |
| { |
| "path": "pytorch_model.bin", |
| "sha256": "965362299a238de576a92dfdd3e32aea7a2bacc94b2c41541c8c9258b923f587" |
| } |
| ], |
| "readback_verified": true |
| } |
|
|
