| { |
| "actual_model_predictions_all_stages": 29106, |
| "benchmark_predictions_including_allocation_probe": 12012, |
| "calibration": { |
| "offline": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2252, |
| "initial_complete_functions": 106, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 548, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 604, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 106, |
| 96, |
| 94, |
| 88 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 106, |
| 202, |
| 296, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2252, |
| 3858, |
| 5120, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "family_language": { |
| "assignment_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 204, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 204, |
| 332, |
| 454, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "assignment_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 204, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 204, |
| 332, |
| 454, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "assignment_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 164, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 164, |
| 288, |
| 402, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "assignment_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 164, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 164, |
| 288, |
| 402, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "branch_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 164, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 164, |
| 298, |
| 403, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "branch_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 164, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 164, |
| 298, |
| 403, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "predicate_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 276, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 36, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 60, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 8, |
| 8, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 20, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 276, |
| 440, |
| 507, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "predicate_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 276, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 36, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 60, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 8, |
| 8, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 20, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 276, |
| 440, |
| 507, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "reference_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 174, |
| "initial_complete_functions": 9, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 46, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 50, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 9, |
| 8, |
| 7, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 9, |
| 17, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 174, |
| 294, |
| 399, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "reference_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 174, |
| "initial_complete_functions": 9, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 46, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 50, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 9, |
| 8, |
| 7, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 9, |
| 17, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 174, |
| 294, |
| 399, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "reference_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 144, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 144, |
| 277, |
| 395, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "reference_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 144, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 144, |
| 277, |
| 395, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| } |
| } |
| }, |
| "v2-fp32": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2737, |
| "initial_complete_functions": 145, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 397, |
| "actual_model_predictions": 1358, |
| "actual_feedback_predictions": 590, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 755, |
| "bilingual_initial_mask_disagreements": 104, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 145, |
| 127, |
| 66, |
| 46 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 145, |
| 272, |
| 338, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2737, |
| 4797, |
| 5671, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 14155551 |
| }, |
| "family_language": { |
| "assignment_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 263, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 16, |
| "actual_model_predictions": 96, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 80, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 16, |
| 0, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 32, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 263, |
| 512, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 994546 |
| }, |
| "assignment_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 181, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 181, |
| 394, |
| 476, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1346630 |
| }, |
| "assignment_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 272, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 16, |
| "actual_model_predictions": 96, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 80, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 16, |
| 0, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 32, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 272, |
| 512, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 993585 |
| }, |
| "assignment_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 221, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 40, |
| "actual_model_predictions": 120, |
| "actual_feedback_predictions": 56, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 56, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 4, |
| 12, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 16, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 221, |
| 292, |
| 458, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1285084 |
| }, |
| "branch_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 155, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 155, |
| 277, |
| 407, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1314708 |
| }, |
| "branch_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 151, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 47, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 49, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 9, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 151, |
| 287, |
| 410, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1321879 |
| }, |
| "predicate_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 276, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 112, |
| "actual_feedback_predictions": 48, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 276, |
| 502, |
| 507, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1173662 |
| }, |
| "predicate_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 276, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 112, |
| "actual_feedback_predictions": 48, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 276, |
| 447, |
| 507, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1165500 |
| }, |
| "reference_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 293, |
| "initial_complete_functions": 17, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 15, |
| "actual_model_predictions": 94, |
| "actual_feedback_predictions": 30, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 81, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 17, |
| 15, |
| 0, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 17, |
| 32, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 293, |
| 512, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1006668 |
| }, |
| "reference_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 169, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 47, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 49, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 9, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 169, |
| 276, |
| 407, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1340206 |
| }, |
| "reference_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 271, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 16, |
| "actual_model_predictions": 96, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 80, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 16, |
| 0, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 32, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 271, |
| 512, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 974540 |
| }, |
| "reference_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 209, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 40, |
| "actual_model_predictions": 120, |
| "actual_feedback_predictions": 56, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 56, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 4, |
| 12, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 16, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 209, |
| 274, |
| 451, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1238543 |
| } |
| } |
| }, |
| "v2-ptq_ternary": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2451, |
| "initial_complete_functions": 121, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 470, |
| "actual_model_predictions": 1430, |
| "actual_feedback_predictions": 662, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 682, |
| "bilingual_initial_mask_disagreements": 96, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 121, |
| 127, |
| 65, |
| 71 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 121, |
| 248, |
| 313, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2451, |
| 4379, |
| 5417, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 16698471 |
| }, |
| "family_language": { |
| "assignment_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 181, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 41, |
| "actual_model_predictions": 121, |
| "actual_feedback_predictions": 57, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 55, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 15, |
| 1, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 23, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 181, |
| 387, |
| 476, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1382458 |
| }, |
| "assignment_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 181, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 47, |
| "actual_model_predictions": 127, |
| "actual_feedback_predictions": 63, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 49, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 9, |
| 7, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 17, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 181, |
| 340, |
| 476, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1489584 |
| }, |
| "assignment_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 159, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 43, |
| "actual_model_predictions": 123, |
| "actual_feedback_predictions": 59, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 53, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 13, |
| 3, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 21, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 159, |
| 357, |
| 406, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1415000 |
| }, |
| "assignment_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 156, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 43, |
| "actual_model_predictions": 123, |
| "actual_feedback_predictions": 59, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 53, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 13, |
| 3, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 21, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 156, |
| 348, |
| 405, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1415173 |
| }, |
| "branch_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 169, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 47, |
| "actual_model_predictions": 127, |
| "actual_feedback_predictions": 63, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 49, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 9, |
| 7, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 17, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 169, |
| 296, |
| 407, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1466792 |
| }, |
| "branch_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 169, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 169, |
| 288, |
| 407, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1483583 |
| }, |
| "predicate_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 276, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 112, |
| "actual_feedback_predictions": 48, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 276, |
| 502, |
| 507, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1304085 |
| }, |
| "predicate_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 276, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 112, |
| "actual_feedback_predictions": 48, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 276, |
| 447, |
| 507, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1304670 |
| }, |
| "reference_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 293, |
| "initial_complete_functions": 17, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 15, |
| "actual_model_predictions": 94, |
| "actual_feedback_predictions": 30, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 81, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 17, |
| 15, |
| 0, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 17, |
| 32, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 293, |
| 512, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1064163 |
| }, |
| "reference_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 169, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 51, |
| "actual_model_predictions": 132, |
| "actual_feedback_predictions": 68, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 45, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 4, |
| 13, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 12, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 169, |
| 222, |
| 407, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1544548 |
| }, |
| "reference_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 271, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 16, |
| "actual_model_predictions": 96, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 80, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 16, |
| 0, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 32, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 271, |
| 512, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1113538 |
| }, |
| "reference_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 151, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 55, |
| "actual_model_predictions": 135, |
| "actual_feedback_predictions": 71, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 41, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 1, |
| 15, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 9, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 151, |
| 168, |
| 395, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1714877 |
| } |
| } |
| }, |
| "v2-qat_ternary": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2555, |
| "initial_complete_functions": 136, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 444, |
| "actual_model_predictions": 1392, |
| "actual_feedback_predictions": 624, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 708, |
| "bilingual_initial_mask_disagreements": 98, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 136, |
| 120, |
| 60, |
| 68 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 136, |
| 256, |
| 316, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2555, |
| 4552, |
| 5395, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 16389180 |
| }, |
| "family_language": { |
| "assignment_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 263, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 104, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 8, |
| 0, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 24, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 263, |
| 387, |
| 476, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1226217 |
| }, |
| "assignment_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 181, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 181, |
| 394, |
| 476, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1537046 |
| }, |
| "assignment_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 201, |
| "initial_complete_functions": 11, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 37, |
| "actual_model_predictions": 114, |
| "actual_feedback_predictions": 50, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 59, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 11, |
| 13, |
| 0, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 11, |
| 24, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 201, |
| 396, |
| 397, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1295466 |
| }, |
| "assignment_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 175, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 175, |
| 292, |
| 399, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1483995 |
| }, |
| "branch_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 155, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 155, |
| 294, |
| 407, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1536835 |
| }, |
| "branch_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 155, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 155, |
| 287, |
| 407, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1557835 |
| }, |
| "predicate_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 276, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 112, |
| "actual_feedback_predictions": 48, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 276, |
| 502, |
| 507, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1343711 |
| }, |
| "predicate_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 276, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 20, |
| "actual_model_predictions": 100, |
| "actual_feedback_predictions": 36, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 76, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 12, |
| 4, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 28, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 276, |
| 480, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1157713 |
| }, |
| "reference_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 293, |
| "initial_complete_functions": 17, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 15, |
| "actual_model_predictions": 94, |
| "actual_feedback_predictions": 30, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 81, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 17, |
| 15, |
| 0, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 17, |
| 32, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 293, |
| 512, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1072622 |
| }, |
| "reference_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 168, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 52, |
| "actual_model_predictions": 132, |
| "actual_feedback_predictions": 68, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 44, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 4, |
| 12, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 12, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 168, |
| 222, |
| 392, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1538745 |
| }, |
| "reference_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 271, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 16, |
| "actual_model_predictions": 96, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 80, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 16, |
| 0, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 32, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 271, |
| 512, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1100251 |
| }, |
| "reference_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 141, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 141, |
| 274, |
| 398, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1538744 |
| } |
| } |
| }, |
| "v3-fp32": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 3238, |
| "initial_complete_functions": 179, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 377, |
| "actual_model_predictions": 1337, |
| "actual_feedback_predictions": 569, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 775, |
| "bilingual_initial_mask_disagreements": 124, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 179, |
| 75, |
| 88, |
| 42 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 179, |
| 254, |
| 342, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 3238, |
| 4409, |
| 5686, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 12776346 |
| }, |
| "family_language": { |
| "assignment_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 263, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 33, |
| "actual_model_predictions": 108, |
| "actual_feedback_predictions": 44, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 63, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 7, |
| 1, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 23, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 263, |
| 391, |
| 452, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 995003 |
| }, |
| "assignment_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 291, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 30, |
| "actual_model_predictions": 117, |
| "actual_feedback_predictions": 53, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 66, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 11, |
| 8, |
| 1 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 23, |
| 31, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 291, |
| 394, |
| 505, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1096214 |
| }, |
| "assignment_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 272, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 104, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 8, |
| 0, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 24, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 272, |
| 393, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1015583 |
| }, |
| "assignment_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 230, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 31, |
| "actual_model_predictions": 122, |
| "actual_feedback_predictions": 58, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 65, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 9, |
| 11, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 21, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 230, |
| 358, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1111084 |
| }, |
| "branch_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 164, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 44, |
| "actual_model_predictions": 127, |
| "actual_feedback_predictions": 63, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 52, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 9, |
| 10, |
| 5 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 17, |
| 27, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 164, |
| 295, |
| 448, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1282459 |
| }, |
| "branch_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 305, |
| "initial_complete_functions": 18, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 31, |
| "actual_model_predictions": 104, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 65, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 18, |
| 2, |
| 7, |
| 5 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 18, |
| 20, |
| 27, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 305, |
| 340, |
| 449, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1031215 |
| }, |
| "predicate_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 276, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 33, |
| "actual_model_predictions": 114, |
| "actual_feedback_predictions": 50, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 63, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 10, |
| 7, |
| 3 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 22, |
| 29, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 276, |
| 442, |
| 509, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1107040 |
| }, |
| "predicate_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 394, |
| "initial_complete_functions": 24, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 18, |
| "actual_model_predictions": 88, |
| "actual_feedback_predictions": 24, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 78, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 24, |
| 0, |
| 6, |
| 2 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 24, |
| 24, |
| 30, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 394, |
| 446, |
| 510, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 846336 |
| }, |
| "reference_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 293, |
| "initial_complete_functions": 17, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 36, |
| "actual_model_predictions": 109, |
| "actual_feedback_predictions": 45, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 60, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 17, |
| 0, |
| 9, |
| 6 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 17, |
| 17, |
| 26, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 293, |
| 300, |
| 424, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1030670 |
| }, |
| "reference_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 276, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 23, |
| "actual_model_predictions": 110, |
| "actual_feedback_predictions": 46, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 73, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 9, |
| 7, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 25, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 276, |
| 407, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1050498 |
| }, |
| "reference_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 271, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 34, |
| "actual_model_predictions": 110, |
| "actual_feedback_predictions": 46, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 62, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 2, |
| 10, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 18, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 271, |
| 305, |
| 452, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 998042 |
| }, |
| "reference_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 203, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 124, |
| "actual_feedback_predictions": 60, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 8, |
| 12, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 20, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 203, |
| 338, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1212202 |
| } |
| } |
| }, |
| "v3-ptq_ternary": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2689, |
| "initial_complete_functions": 145, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 430, |
| "actual_model_predictions": 1387, |
| "actual_feedback_predictions": 619, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 722, |
| "bilingual_initial_mask_disagreements": 124, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 145, |
| 109, |
| 69, |
| 61 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 145, |
| 254, |
| 323, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2689, |
| 4438, |
| 5460, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 14982908 |
| }, |
| "family_language": { |
| "assignment_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 263, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 33, |
| "actual_model_predictions": 109, |
| "actual_feedback_predictions": 45, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 63, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 7, |
| 1, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 23, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 263, |
| 394, |
| 452, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1141952 |
| }, |
| "assignment_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 181, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 34, |
| "actual_model_predictions": 120, |
| "actual_feedback_predictions": 56, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 62, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 16, |
| 6, |
| 2 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 24, |
| 30, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 181, |
| 388, |
| 497, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1290461 |
| }, |
| "assignment_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 272, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 104, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 8, |
| 0, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 24, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 272, |
| 398, |
| 399, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1096538 |
| }, |
| "assignment_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 175, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 36, |
| "actual_model_predictions": 120, |
| "actual_feedback_predictions": 56, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 60, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 16, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 175, |
| 398, |
| 455, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1323954 |
| }, |
| "branch_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 164, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 164, |
| 284, |
| 407, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1377375 |
| }, |
| "branch_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 155, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 155, |
| 284, |
| 407, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1471245 |
| }, |
| "predicate_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 276, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 36, |
| "actual_model_predictions": 116, |
| "actual_feedback_predictions": 52, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 60, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 8, |
| 8, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 20, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 276, |
| 440, |
| 507, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1214040 |
| }, |
| "predicate_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 276, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 34, |
| "actual_model_predictions": 110, |
| "actual_feedback_predictions": 46, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 62, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 2, |
| 10, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 18, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 276, |
| 385, |
| 507, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1153959 |
| }, |
| "reference_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 293, |
| "initial_complete_functions": 17, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 30, |
| "actual_model_predictions": 102, |
| "actual_feedback_predictions": 38, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 66, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 17, |
| 7, |
| 1, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 17, |
| 24, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 293, |
| 394, |
| 407, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1114875 |
| }, |
| "reference_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 222, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 27, |
| "actual_model_predictions": 118, |
| "actual_feedback_predictions": 54, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 69, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 13, |
| 7, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 25, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 222, |
| 407, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1267290 |
| }, |
| "reference_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 271, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 40, |
| "actual_model_predictions": 112, |
| "actual_feedback_predictions": 48, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 56, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 0, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 271, |
| 273, |
| 398, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1245088 |
| }, |
| "reference_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 141, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 120, |
| "actual_feedback_predictions": 56, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 16, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 141, |
| 393, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1286131 |
| } |
| } |
| }, |
| "v3-qat_ternary": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 3289, |
| "initial_complete_functions": 183, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 365, |
| "actual_model_predictions": 1318, |
| "actual_feedback_predictions": 550, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 787, |
| "bilingual_initial_mask_disagreements": 138, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 183, |
| 89, |
| 60, |
| 52 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 183, |
| 272, |
| 332, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 3289, |
| 4723, |
| 5557, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 14719928 |
| }, |
| "family_language": { |
| "assignment_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 263, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 110, |
| "actual_feedback_predictions": 46, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 8, |
| 0, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 24, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 263, |
| 415, |
| 428, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1221541 |
| }, |
| "assignment_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 241, |
| "initial_complete_functions": 10, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 28, |
| "actual_model_predictions": 114, |
| "actual_feedback_predictions": 50, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 68, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 10, |
| 16, |
| 6, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 10, |
| 26, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 241, |
| 420, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1283007 |
| }, |
| "assignment_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 272, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 108, |
| "actual_feedback_predictions": 44, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 8, |
| 0, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 24, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 272, |
| 394, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1128752 |
| }, |
| "assignment_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 225, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 28, |
| "actual_model_predictions": 116, |
| "actual_feedback_predictions": 52, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 68, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 225, |
| 398, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1285662 |
| }, |
| "branch_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 164, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 132, |
| "actual_feedback_predictions": 68, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 164, |
| 286, |
| 409, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1476296 |
| }, |
| "branch_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 398, |
| "initial_complete_functions": 24, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 20, |
| "actual_model_predictions": 88, |
| "actual_feedback_predictions": 24, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 76, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 24, |
| 0, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 24, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 398, |
| 400, |
| 463, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 948798 |
| }, |
| "predicate_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 338, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 28, |
| "actual_model_predictions": 104, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 68, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 8, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 338, |
| 502, |
| 507, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1113997 |
| }, |
| "predicate_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 334, |
| "initial_complete_functions": 20, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 26, |
| "actual_model_predictions": 98, |
| "actual_feedback_predictions": 34, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 70, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 20, |
| 2, |
| 6, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 20, |
| 22, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 334, |
| 443, |
| 507, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1045915 |
| }, |
| "reference_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 293, |
| "initial_complete_functions": 17, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 34, |
| "actual_model_predictions": 106, |
| "actual_feedback_predictions": 42, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 62, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 17, |
| 4, |
| 3, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 17, |
| 21, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 293, |
| 355, |
| 397, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1167788 |
| }, |
| "reference_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 276, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 23, |
| "actual_model_predictions": 110, |
| "actual_feedback_predictions": 46, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 73, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 9, |
| 7, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 25, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 276, |
| 407, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1343751 |
| }, |
| "reference_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 271, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 38, |
| "actual_model_predictions": 116, |
| "actual_feedback_predictions": 52, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 58, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 2, |
| 6, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 18, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 271, |
| 310, |
| 397, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1328047 |
| }, |
| "reference_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 214, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 28, |
| "actual_model_predictions": 116, |
| "actual_feedback_predictions": 52, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 68, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 214, |
| 393, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1376374 |
| } |
| } |
| } |
| }, |
| "dataset_sha256": "44393fb6ae27d1dc75c9706ea50077fab8e2573928e4f59422e3e3a65aa79620", |
| "default_model_promoted": false, |
| "development": { |
| "offline": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2200, |
| "initial_complete_functions": 104, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 548, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 604, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 104, |
| 98, |
| 96, |
| 86 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 104, |
| 202, |
| 298, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2200, |
| 3844, |
| 5124, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "family_language": { |
| "assignment_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 203, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 203, |
| 330, |
| 450, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "assignment_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 203, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 203, |
| 330, |
| 450, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "assignment_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 164, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 164, |
| 291, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "assignment_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 164, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 164, |
| 291, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "branch_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 166, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 166, |
| 290, |
| 395, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "branch_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 166, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 166, |
| 290, |
| 395, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "predicate_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 268, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 36, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 60, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 8, |
| 8, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 20, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 268, |
| 446, |
| 508, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "predicate_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 268, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 36, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 60, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 8, |
| 8, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 20, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 268, |
| 446, |
| 508, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "reference_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 151, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 46, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 50, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 9, |
| 8, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 17, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 151, |
| 290, |
| 410, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "reference_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 151, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 46, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 50, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 9, |
| 8, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 17, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 151, |
| 290, |
| 410, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "reference_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 148, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 148, |
| 275, |
| 398, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| }, |
| "reference_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 148, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 148, |
| 275, |
| 398, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0 |
| } |
| } |
| }, |
| "v2-fp32": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2240, |
| "initial_complete_functions": 110, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 524, |
| "actual_model_predictions": 1487, |
| "actual_feedback_predictions": 719, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 628, |
| "bilingual_initial_mask_disagreements": 126, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 110, |
| 103, |
| 92, |
| 79 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 110, |
| 213, |
| 305, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2240, |
| 3788, |
| 5290, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 15672695 |
| }, |
| "family_language": { |
| "assignment_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 263, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 24, |
| "actual_model_predictions": 104, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 72, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 8, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 263, |
| 388, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1110579 |
| }, |
| "assignment_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 93, |
| "initial_complete_functions": 0, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 64, |
| "actual_model_predictions": 144, |
| "actual_feedback_predictions": 80, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 32, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 0, |
| 16, |
| 0, |
| 16 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 0, |
| 16, |
| 16, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 93, |
| 264, |
| 438, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1512085 |
| }, |
| "assignment_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 277, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 20, |
| "actual_model_predictions": 100, |
| "actual_feedback_predictions": 36, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 76, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 12, |
| 4, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 28, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 277, |
| 456, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1038084 |
| }, |
| "assignment_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 78, |
| "initial_complete_functions": 1, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 67, |
| "actual_model_predictions": 153, |
| "actual_feedback_predictions": 89, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 29, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 1, |
| 4, |
| 18, |
| 9 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 1, |
| 5, |
| 23, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 78, |
| 143, |
| 389, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1612992 |
| }, |
| "branch_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 161, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 161, |
| 274, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1336287 |
| }, |
| "branch_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 161, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 161, |
| 274, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1390174 |
| }, |
| "predicate_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 268, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 34, |
| "actual_model_predictions": 114, |
| "actual_feedback_predictions": 50, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 62, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 10, |
| 6, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 22, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 268, |
| 475, |
| 508, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1187170 |
| }, |
| "predicate_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 268, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 112, |
| "actual_feedback_predictions": 48, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 268, |
| 449, |
| 508, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1183251 |
| }, |
| "reference_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 157, |
| "initial_complete_functions": 9, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 47, |
| "actual_model_predictions": 126, |
| "actual_feedback_predictions": 62, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 49, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 9, |
| 7, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 9, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 157, |
| 270, |
| 402, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1275712 |
| }, |
| "reference_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 159, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 49, |
| "actual_model_predictions": 130, |
| "actual_feedback_predictions": 66, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 47, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 6, |
| 11, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 14, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 159, |
| 242, |
| 404, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1433781 |
| }, |
| "reference_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 145, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 145, |
| 276, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1343036 |
| }, |
| "reference_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 210, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 43, |
| "actual_model_predictions": 120, |
| "actual_feedback_predictions": 56, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 53, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 4, |
| 9, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 16, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 210, |
| 277, |
| 414, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1249544 |
| } |
| } |
| }, |
| "v2-ptq_ternary": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2367, |
| "initial_complete_functions": 121, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 504, |
| "actual_model_predictions": 1464, |
| "actual_feedback_predictions": 696, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 648, |
| "bilingual_initial_mask_disagreements": 96, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 121, |
| 93, |
| 99, |
| 71 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 121, |
| 214, |
| 313, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2367, |
| 3801, |
| 5306, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 17030920 |
| }, |
| "family_language": { |
| "assignment_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 263, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 112, |
| "actual_feedback_predictions": 48, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 0, |
| 16, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 16, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 263, |
| 264, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1282464 |
| }, |
| "assignment_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 179, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 179, |
| 264, |
| 475, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1519960 |
| }, |
| "assignment_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 277, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 26, |
| "actual_model_predictions": 106, |
| "actual_feedback_predictions": 42, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 70, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 6, |
| 10, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 22, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 277, |
| 370, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1212251 |
| }, |
| "assignment_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 168, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 168, |
| 280, |
| 404, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1472496 |
| }, |
| "branch_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 161, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 161, |
| 274, |
| 399, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1498912 |
| }, |
| "branch_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 161, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 161, |
| 290, |
| 399, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1516006 |
| }, |
| "predicate_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 268, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 112, |
| "actual_feedback_predictions": 48, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 268, |
| 504, |
| 508, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1332002 |
| }, |
| "predicate_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 268, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 112, |
| "actual_feedback_predictions": 48, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 268, |
| 449, |
| 508, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1303917 |
| }, |
| "reference_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 157, |
| "initial_complete_functions": 9, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 39, |
| "actual_model_predictions": 118, |
| "actual_feedback_predictions": 54, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 57, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 9, |
| 15, |
| 0, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 9, |
| 24, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 157, |
| 402, |
| 402, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1350245 |
| }, |
| "reference_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 159, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 47, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 49, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 9, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 159, |
| 275, |
| 404, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1474294 |
| }, |
| "reference_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 154, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 154, |
| 276, |
| 389, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1489036 |
| }, |
| "reference_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 152, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 56, |
| "actual_model_predictions": 136, |
| "actual_feedback_predictions": 72, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 40, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 0, |
| 16, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 8, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 152, |
| 153, |
| 394, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1579337 |
| } |
| } |
| }, |
| "v2-qat_ternary": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2353, |
| "initial_complete_functions": 125, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 491, |
| "actual_model_predictions": 1450, |
| "actual_feedback_predictions": 682, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 661, |
| "bilingual_initial_mask_disagreements": 112, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 125, |
| 95, |
| 96, |
| 68 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 125, |
| 220, |
| 316, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2353, |
| 3918, |
| 5319, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 17671623 |
| }, |
| "family_language": { |
| "assignment_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 263, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 27, |
| "actual_model_predictions": 107, |
| "actual_feedback_predictions": 43, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 69, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 5, |
| 11, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 21, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 263, |
| 340, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1273708 |
| }, |
| "assignment_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 177, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 50, |
| "actual_model_predictions": 130, |
| "actual_feedback_predictions": 66, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 46, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 6, |
| 10, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 14, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 177, |
| 258, |
| 475, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1553668 |
| }, |
| "assignment_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 277, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 24, |
| "actual_model_predictions": 104, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 72, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 8, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 277, |
| 402, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1260371 |
| }, |
| "assignment_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 179, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 179, |
| 289, |
| 403, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1508922 |
| }, |
| "branch_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 161, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 161, |
| 274, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1565956 |
| }, |
| "branch_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 161, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 161, |
| 294, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1570958 |
| }, |
| "predicate_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 268, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 38, |
| "actual_model_predictions": 118, |
| "actual_feedback_predictions": 54, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 58, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 6, |
| 10, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 18, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 268, |
| 443, |
| 508, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1339334 |
| }, |
| "predicate_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 268, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 20, |
| "actual_model_predictions": 100, |
| "actual_feedback_predictions": 36, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 76, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 12, |
| 4, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 28, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 268, |
| 480, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1154835 |
| }, |
| "reference_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 157, |
| "initial_complete_functions": 9, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 47, |
| "actual_model_predictions": 126, |
| "actual_feedback_predictions": 62, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 49, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 9, |
| 7, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 9, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 157, |
| 270, |
| 402, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1455868 |
| }, |
| "reference_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 158, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 158, |
| 275, |
| 392, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1631584 |
| }, |
| "reference_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 145, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 45, |
| "actual_model_predictions": 125, |
| "actual_feedback_predictions": 61, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 51, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 11, |
| 5, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 19, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 145, |
| 316, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1602373 |
| }, |
| "reference_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 139, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 139, |
| 277, |
| 400, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1754046 |
| } |
| } |
| }, |
| "v3-fp32": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2931, |
| "initial_complete_functions": 163, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 397, |
| "actual_model_predictions": 1360, |
| "actual_feedback_predictions": 592, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 755, |
| "bilingual_initial_mask_disagreements": 144, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 163, |
| 96, |
| 74, |
| 51 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 163, |
| 259, |
| 333, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2931, |
| 4471, |
| 5560, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 13122380 |
| }, |
| "family_language": { |
| "assignment_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 263, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 105, |
| "actual_feedback_predictions": 41, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 8, |
| 0, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 24, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 263, |
| 399, |
| 452, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 985335 |
| }, |
| "assignment_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 288, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 121, |
| "actual_feedback_predictions": 57, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 8, |
| 12, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 20, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 288, |
| 363, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1152877 |
| }, |
| "assignment_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 277, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 106, |
| "actual_feedback_predictions": 42, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 8, |
| 0, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 24, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 277, |
| 400, |
| 403, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1095836 |
| }, |
| "assignment_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 218, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 29, |
| "actual_model_predictions": 118, |
| "actual_feedback_predictions": 54, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 67, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 11, |
| 9, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 23, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 218, |
| 386, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1126918 |
| }, |
| "branch_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 152, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 44, |
| "actual_model_predictions": 124, |
| "actual_feedback_predictions": 60, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 52, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 12, |
| 4, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 20, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 152, |
| 346, |
| 411, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1216379 |
| }, |
| "branch_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 296, |
| "initial_complete_functions": 18, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 104, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 18, |
| 2, |
| 6, |
| 6 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 18, |
| 20, |
| 26, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 296, |
| 332, |
| 428, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 980461 |
| }, |
| "predicate_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 268, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 36, |
| "actual_model_predictions": 116, |
| "actual_feedback_predictions": 52, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 60, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 8, |
| 8, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 20, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 268, |
| 418, |
| 508, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1092914 |
| }, |
| "predicate_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 390, |
| "initial_complete_functions": 24, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 17, |
| "actual_model_predictions": 87, |
| "actual_feedback_predictions": 23, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 79, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 24, |
| 1, |
| 5, |
| 2 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 24, |
| 25, |
| 30, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 390, |
| 451, |
| 510, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 881791 |
| }, |
| "reference_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 157, |
| "initial_complete_functions": 9, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 45, |
| "actual_model_predictions": 125, |
| "actual_feedback_predictions": 61, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 51, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 9, |
| 8, |
| 8, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 9, |
| 17, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 157, |
| 294, |
| 404, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1220584 |
| }, |
| "reference_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 268, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 23, |
| "actual_model_predictions": 110, |
| "actual_feedback_predictions": 46, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 73, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 9, |
| 7, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 25, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 268, |
| 403, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1068956 |
| }, |
| "reference_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 150, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 46, |
| "actual_model_predictions": 126, |
| "actual_feedback_predictions": 62, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 50, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 10, |
| 6, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 18, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 150, |
| 302, |
| 396, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1140374 |
| }, |
| "reference_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 204, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 29, |
| "actual_model_predictions": 118, |
| "actual_feedback_predictions": 54, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 67, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 11, |
| 9, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 23, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 204, |
| 377, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1159955 |
| } |
| } |
| }, |
| "v3-ptq_ternary": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2519, |
| "initial_complete_functions": 127, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 465, |
| "actual_model_predictions": 1440, |
| "actual_feedback_predictions": 672, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 687, |
| "bilingual_initial_mask_disagreements": 100, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 127, |
| 104, |
| 98, |
| 55 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 127, |
| 231, |
| 329, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2519, |
| 4241, |
| 5548, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 15688552 |
| }, |
| "family_language": { |
| "assignment_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 238, |
| "initial_complete_functions": 10, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 46, |
| "actual_model_predictions": 124, |
| "actual_feedback_predictions": 60, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 50, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 10, |
| 6, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 10, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 238, |
| 390, |
| 475, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1308996 |
| }, |
| "assignment_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 179, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 37, |
| "actual_model_predictions": 125, |
| "actual_feedback_predictions": 61, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 59, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 11, |
| 13, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 19, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 179, |
| 312, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1427874 |
| }, |
| "assignment_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 179, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 179, |
| 293, |
| 403, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1380166 |
| }, |
| "assignment_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 155, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 40, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 56, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 16, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 155, |
| 289, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1354787 |
| }, |
| "branch_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 161, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 161, |
| 289, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1375877 |
| }, |
| "branch_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 161, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 161, |
| 289, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1372426 |
| }, |
| "predicate_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 268, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 38, |
| "actual_model_predictions": 118, |
| "actual_feedback_predictions": 54, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 58, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 6, |
| 10, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 18, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 268, |
| 401, |
| 508, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1316837 |
| }, |
| "predicate_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 268, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 40, |
| "actual_model_predictions": 120, |
| "actual_feedback_predictions": 56, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 56, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 4, |
| 12, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 16, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 268, |
| 386, |
| 508, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1300833 |
| }, |
| "reference_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 290, |
| "initial_complete_functions": 17, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 29, |
| "actual_model_predictions": 103, |
| "actual_feedback_predictions": 39, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 67, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 17, |
| 8, |
| 0, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 17, |
| 25, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 290, |
| 404, |
| 404, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1109421 |
| }, |
| "reference_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 213, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 27, |
| "actual_model_predictions": 114, |
| "actual_feedback_predictions": 50, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 69, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 13, |
| 7, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 25, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 213, |
| 404, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1222630 |
| }, |
| "reference_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 268, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 104, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 8, |
| 0, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 24, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 268, |
| 392, |
| 400, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1183372 |
| }, |
| "reference_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 139, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 120, |
| "actual_feedback_predictions": 56, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 16, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 139, |
| 392, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1335333 |
| } |
| } |
| }, |
| "v3-qat_ternary": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2269, |
| "initial_complete_functions": 115, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 470, |
| "actual_model_predictions": 1479, |
| "actual_feedback_predictions": 711, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 682, |
| "bilingual_initial_mask_disagreements": 166, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 115, |
| 126, |
| 85, |
| 58 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 115, |
| 241, |
| 326, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2269, |
| 4348, |
| 5482, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 16228448 |
| }, |
| "family_language": { |
| "assignment_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 201, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 46, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 50, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 10, |
| 6 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 26, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 201, |
| 390, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1405413 |
| }, |
| "assignment_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 240, |
| "initial_complete_functions": 10, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 30, |
| "actual_model_predictions": 118, |
| "actual_feedback_predictions": 54, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 66, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 10, |
| 14, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 10, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 240, |
| 404, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1272663 |
| }, |
| "assignment_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 105, |
| "initial_complete_functions": 4, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 52, |
| "actual_model_predictions": 136, |
| "actual_feedback_predictions": 72, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 44, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 4, |
| 12, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 4, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 105, |
| 293, |
| 402, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1500620 |
| }, |
| "assignment_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 160, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 124, |
| "actual_feedback_predictions": 60, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 16, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 160, |
| 402, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1331831 |
| }, |
| "branch_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 89, |
| "initial_complete_functions": 4, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 54, |
| "actual_model_predictions": 139, |
| "actual_feedback_predictions": 75, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 42, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 4, |
| 11, |
| 8, |
| 9 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 4, |
| 15, |
| 23, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 89, |
| 272, |
| 398, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1506342 |
| }, |
| "branch_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 221, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 41, |
| "actual_model_predictions": 119, |
| "actual_feedback_predictions": 55, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 55, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 7, |
| 5, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 19, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 221, |
| 334, |
| 399, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1294867 |
| }, |
| "predicate_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 268, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 112, |
| "actual_feedback_predictions": 48, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 268, |
| 450, |
| 508, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1193248 |
| }, |
| "predicate_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 328, |
| "initial_complete_functions": 20, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 27, |
| "actual_model_predictions": 99, |
| "actual_feedback_predictions": 35, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 69, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 20, |
| 1, |
| 7, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 20, |
| 21, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 328, |
| 447, |
| 508, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1127786 |
| }, |
| "reference_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 157, |
| "initial_complete_functions": 9, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 41, |
| "actual_model_predictions": 126, |
| "actual_feedback_predictions": 62, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 55, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 9, |
| 12, |
| 4, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 9, |
| 21, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 157, |
| 348, |
| 404, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1364718 |
| }, |
| "reference_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 268, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 23, |
| "actual_model_predictions": 110, |
| "actual_feedback_predictions": 46, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 73, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 9, |
| 7, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 25, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 268, |
| 403, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1259041 |
| }, |
| "reference_schedule/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 88, |
| "initial_complete_functions": 4, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 60, |
| "actual_model_predictions": 144, |
| "actual_feedback_predictions": 80, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 36, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 4, |
| 8, |
| 8, |
| 12 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 4, |
| 12, |
| 20, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 88, |
| 213, |
| 337, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1598959 |
| }, |
| "reference_schedule/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 144, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 124, |
| "actual_feedback_predictions": 60, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 16, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 144, |
| 392, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1372960 |
| } |
| } |
| } |
| }, |
| "go_python_parity_max_absolute_error": { |
| "v2": 1.4901161193847656e-7, |
| "v3": 2.682209014892578e-7 |
| }, |
| "native_calls": 0, |
| "new_optimizer_updates": 0, |
| "parity_predictions": 192, |
| "runtime": { |
| "v2-fp32": { |
| "warm_benchmark_predictions": 1001, |
| "warm_ns_per_prediction": 9445.75, |
| "predict_into_heap_allocations_per_call": 0, |
| "allocation_probe_predictions": 1001, |
| "resident_tensor_bytes": 50912, |
| "matrix_scale_bytes": 0, |
| "workspace_bytes": 1248 |
| }, |
| "v2-ptq_ternary": { |
| "warm_benchmark_predictions": 1001, |
| "warm_ns_per_prediction": 10591.542, |
| "predict_into_heap_allocations_per_call": 0, |
| "allocation_probe_predictions": 1001, |
| "resident_tensor_bytes": 12896, |
| "matrix_scale_bytes": 8, |
| "workspace_bytes": 1248 |
| }, |
| "v2-qat_ternary": { |
| "warm_benchmark_predictions": 1001, |
| "warm_ns_per_prediction": 10927.917, |
| "predict_into_heap_allocations_per_call": 0, |
| "allocation_probe_predictions": 1001, |
| "resident_tensor_bytes": 12896, |
| "matrix_scale_bytes": 8, |
| "workspace_bytes": 1248 |
| }, |
| "v3-fp32": { |
| "warm_benchmark_predictions": 1001, |
| "warm_ns_per_prediction": 8280.458, |
| "predict_into_heap_allocations_per_call": 0, |
| "allocation_probe_predictions": 1001, |
| "resident_tensor_bytes": 50912, |
| "matrix_scale_bytes": 0, |
| "workspace_bytes": 1248 |
| }, |
| "v3-ptq_ternary": { |
| "warm_benchmark_predictions": 1001, |
| "warm_ns_per_prediction": 9686.167, |
| "predict_into_heap_allocations_per_call": 0, |
| "allocation_probe_predictions": 1001, |
| "resident_tensor_bytes": 12896, |
| "matrix_scale_bytes": 8, |
| "workspace_bytes": 1248 |
| }, |
| "v3-qat_ternary": { |
| "warm_benchmark_predictions": 1001, |
| "warm_ns_per_prediction": 9674.833, |
| "predict_into_heap_allocations_per_call": 0, |
| "allocation_probe_predictions": 1001, |
| "resident_tensor_bytes": 12896, |
| "matrix_scale_bytes": 8, |
| "workspace_bytes": 1248 |
| } |
| }, |
| "schema": "gooo/fresh-composition-study/v1", |
| "scope": "Six compositions, four goals with bilingual/parameter variants; all 16 ordered independent cases retained. Model ranks legal compiler-owned fragments and observed feedback only. Calibration selector frozen before development sessions. Four-path completeness is finite enumerative evidence, not general language understanding. Warm inference allocations exclude loading/context/session and decoded ternary matrices remain int8.", |
| "selected_candidate": "v3-qat_ternary", |
| "source_revision": "15962b584c4d7d0841171323374c66b19a01cb9f", |
| "status": "PASS" |
| } |
|
|
