| { |
| "model_id": "m25", |
| "checkpoint": "results/m25/final.pt", |
| "trained_step": 15258, |
| "num_fewshot": 0, |
| "limit": null, |
| "results": { |
| "lambada_openai": { |
| "name": "lambada_openai", |
| "alias": "lambada_openai", |
| "sample_len": 5153, |
| "perplexity,none": 569.0693925302556, |
| "perplexity_stderr,none": 31.078854946186166, |
| "acc,none": 0.17058024451775664, |
| "acc_stderr,none": 0.00524039021223409 |
| }, |
| "hellaswag": { |
| "name": "hellaswag", |
| "alias": "hellaswag", |
| "sample_len": 10042, |
| "acc,none": 0.2741485759808803, |
| "acc_stderr,none": 0.004451725530626685, |
| "acc_norm,none": 0.2802230631348337, |
| "acc_norm_stderr,none": 0.004481902637505966 |
| }, |
| "piqa": { |
| "name": "piqa", |
| "alias": "piqa", |
| "sample_len": 1838, |
| "acc,none": 0.5946681175190425, |
| "acc_stderr,none": 0.011454816387346781, |
| "acc_norm,none": 0.5772578890097932, |
| "acc_norm_stderr,none": 0.01152572136157213 |
| }, |
| "sciq": { |
| "name": "sciq", |
| "alias": "sciq", |
| "sample_len": 1000, |
| "acc,none": 0.682, |
| "acc_stderr,none": 0.014734079309311903, |
| "acc_norm,none": 0.597, |
| "acc_norm_stderr,none": 0.015518757419066512 |
| }, |
| "arc_easy": { |
| "name": "arc_easy", |
| "alias": "arc_easy", |
| "sample_len": 2376, |
| "acc,none": 0.45286195286195285, |
| "acc_stderr,none": 0.010214087372211118, |
| "acc_norm,none": 0.4053030303030303, |
| "acc_norm_stderr,none": 0.010074093589739265 |
| }, |
| "winogrande": { |
| "name": "winogrande", |
| "alias": "winogrande", |
| "sample_len": 1267, |
| "acc,none": 0.5256511444356748, |
| "acc_stderr,none": 0.014033980956108428 |
| }, |
| "blimp_adjunct_island": { |
| "name": "blimp_adjunct_island", |
| "alias": "blimp_adjunct_island", |
| "sample_len": 1000, |
| "acc,none": 0.747, |
| "acc_stderr,none": 0.013754278613587126 |
| }, |
| "blimp_anaphor_gender_agreement": { |
| "name": "blimp_anaphor_gender_agreement", |
| "alias": "blimp_anaphor_gender_agreement", |
| "sample_len": 1000, |
| "acc,none": 0.858, |
| "acc_stderr,none": 0.011043457699378305 |
| }, |
| "blimp_anaphor_number_agreement": { |
| "name": "blimp_anaphor_number_agreement", |
| "alias": "blimp_anaphor_number_agreement", |
| "sample_len": 1000, |
| "acc,none": 0.972, |
| "acc_stderr,none": 0.005219506034410081 |
| }, |
| "blimp_animate_subject_passive": { |
| "name": "blimp_animate_subject_passive", |
| "alias": "blimp_animate_subject_passive", |
| "sample_len": 1000, |
| "acc,none": 0.683, |
| "acc_stderr,none": 0.014721675438880174 |
| }, |
| "blimp_animate_subject_trans": { |
| "name": "blimp_animate_subject_trans", |
| "alias": "blimp_animate_subject_trans", |
| "sample_len": 1000, |
| "acc,none": 0.904, |
| "acc_stderr,none": 0.009320454434783257 |
| }, |
| "blimp_causative": { |
| "name": "blimp_causative", |
| "alias": "blimp_causative", |
| "sample_len": 1000, |
| "acc,none": 0.656, |
| "acc_stderr,none": 0.015029633724408919 |
| }, |
| "blimp_complex_NP_island": { |
| "name": "blimp_complex_NP_island", |
| "alias": "blimp_complex_NP_island", |
| "sample_len": 1000, |
| "acc,none": 0.473, |
| "acc_stderr,none": 0.01579621855130273 |
| }, |
| "blimp_coordinate_structure_constraint_complex_left_branch": { |
| "name": "blimp_coordinate_structure_constraint_complex_left_branch", |
| "alias": "blimp_coordinate_structure_constraint_complex_left_branch", |
| "sample_len": 1000, |
| "acc,none": 0.459, |
| "acc_stderr,none": 0.015766025737882314 |
| }, |
| "blimp_coordinate_structure_constraint_object_extraction": { |
| "name": "blimp_coordinate_structure_constraint_object_extraction", |
| "alias": "blimp_coordinate_structure_constraint_object_extraction", |
| "sample_len": 1000, |
| "acc,none": 0.788, |
| "acc_stderr,none": 0.01293148186493811 |
| }, |
| "blimp_determiner_noun_agreement_1": { |
| "name": "blimp_determiner_noun_agreement_1", |
| "alias": "blimp_determiner_noun_agreement_1", |
| "sample_len": 1000, |
| "acc,none": 0.968, |
| "acc_stderr,none": 0.005568393575081328 |
| }, |
| "blimp_determiner_noun_agreement_2": { |
| "name": "blimp_determiner_noun_agreement_2", |
| "alias": "blimp_determiner_noun_agreement_2", |
| "sample_len": 1000, |
| "acc,none": 0.946, |
| "acc_stderr,none": 0.007150883521295473 |
| }, |
| "blimp_determiner_noun_agreement_irregular_1": { |
| "name": "blimp_determiner_noun_agreement_irregular_1", |
| "alias": "blimp_determiner_noun_agreement_irregular_1", |
| "sample_len": 1000, |
| "acc,none": 0.886, |
| "acc_stderr,none": 0.010055103435823278 |
| }, |
| "blimp_determiner_noun_agreement_irregular_2": { |
| "name": "blimp_determiner_noun_agreement_irregular_2", |
| "alias": "blimp_determiner_noun_agreement_irregular_2", |
| "sample_len": 1000, |
| "acc,none": 0.917, |
| "acc_stderr,none": 0.008728527206074756 |
| }, |
| "blimp_determiner_noun_agreement_with_adj_2": { |
| "name": "blimp_determiner_noun_agreement_with_adj_2", |
| "alias": "blimp_determiner_noun_agreement_with_adj_2", |
| "sample_len": 1000, |
| "acc,none": 0.916, |
| "acc_stderr,none": 0.008776162089491113 |
| }, |
| "blimp_determiner_noun_agreement_with_adj_irregular_1": { |
| "name": "blimp_determiner_noun_agreement_with_adj_irregular_1", |
| "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1", |
| "sample_len": 1000, |
| "acc,none": 0.882, |
| "acc_stderr,none": 0.010206869264381718 |
| }, |
| "blimp_determiner_noun_agreement_with_adj_irregular_2": { |
| "name": "blimp_determiner_noun_agreement_with_adj_irregular_2", |
| "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2", |
| "sample_len": 1000, |
| "acc,none": 0.923, |
| "acc_stderr,none": 0.00843458014024066 |
| }, |
| "blimp_determiner_noun_agreement_with_adjective_1": { |
| "name": "blimp_determiner_noun_agreement_with_adjective_1", |
| "alias": "blimp_determiner_noun_agreement_with_adjective_1", |
| "sample_len": 1000, |
| "acc,none": 0.936, |
| "acc_stderr,none": 0.007743640226919376 |
| }, |
| "blimp_distractor_agreement_relational_noun": { |
| "name": "blimp_distractor_agreement_relational_noun", |
| "alias": "blimp_distractor_agreement_relational_noun", |
| "sample_len": 1000, |
| "acc,none": 0.841, |
| "acc_stderr,none": 0.011569479368271195 |
| }, |
| "blimp_distractor_agreement_relative_clause": { |
| "name": "blimp_distractor_agreement_relative_clause", |
| "alias": "blimp_distractor_agreement_relative_clause", |
| "sample_len": 1000, |
| "acc,none": 0.705, |
| "acc_stderr,none": 0.014428554438445425 |
| }, |
| "blimp_drop_argument": { |
| "name": "blimp_drop_argument", |
| "alias": "blimp_drop_argument", |
| "sample_len": 1000, |
| "acc,none": 0.724, |
| "acc_stderr,none": 0.01414298497574059 |
| }, |
| "blimp_ellipsis_n_bar_1": { |
| "name": "blimp_ellipsis_n_bar_1", |
| "alias": "blimp_ellipsis_n_bar_1", |
| "sample_len": 1000, |
| "acc,none": 0.773, |
| "acc_stderr,none": 0.013253174964763977 |
| }, |
| "blimp_ellipsis_n_bar_2": { |
| "name": "blimp_ellipsis_n_bar_2", |
| "alias": "blimp_ellipsis_n_bar_2", |
| "sample_len": 1000, |
| "acc,none": 0.834, |
| "acc_stderr,none": 0.011772110370812133 |
| }, |
| "blimp_existential_there_object_raising": { |
| "name": "blimp_existential_there_object_raising", |
| "alias": "blimp_existential_there_object_raising", |
| "sample_len": 1000, |
| "acc,none": 0.81, |
| "acc_stderr,none": 0.012411851354816254 |
| }, |
| "blimp_existential_there_quantifiers_1": { |
| "name": "blimp_existential_there_quantifiers_1", |
| "alias": "blimp_existential_there_quantifiers_1", |
| "sample_len": 1000, |
| "acc,none": 0.973, |
| "acc_stderr,none": 0.005128089049275305 |
| }, |
| "blimp_existential_there_quantifiers_2": { |
| "name": "blimp_existential_there_quantifiers_2", |
| "alias": "blimp_existential_there_quantifiers_2", |
| "sample_len": 1000, |
| "acc,none": 0.339, |
| "acc_stderr,none": 0.014976758771620224 |
| }, |
| "blimp_existential_there_subject_raising": { |
| "name": "blimp_existential_there_subject_raising", |
| "alias": "blimp_existential_there_subject_raising", |
| "sample_len": 1000, |
| "acc,none": 0.852, |
| "acc_stderr,none": 0.011234866364235145 |
| }, |
| "blimp_expletive_it_object_raising": { |
| "name": "blimp_expletive_it_object_raising", |
| "alias": "blimp_expletive_it_object_raising", |
| "sample_len": 1000, |
| "acc,none": 0.739, |
| "acc_stderr,none": 0.013895037677965183 |
| }, |
| "blimp_inchoative": { |
| "name": "blimp_inchoative", |
| "alias": "blimp_inchoative", |
| "sample_len": 1000, |
| "acc,none": 0.66, |
| "acc_stderr,none": 0.014987482264363972 |
| }, |
| "blimp_intransitive": { |
| "name": "blimp_intransitive", |
| "alias": "blimp_intransitive", |
| "sample_len": 1000, |
| "acc,none": 0.743, |
| "acc_stderr,none": 0.013825416526895026 |
| }, |
| "blimp_irregular_past_participle_adjectives": { |
| "name": "blimp_irregular_past_participle_adjectives", |
| "alias": "blimp_irregular_past_participle_adjectives", |
| "sample_len": 1000, |
| "acc,none": 0.769, |
| "acc_stderr,none": 0.013334797216936478 |
| }, |
| "blimp_irregular_past_participle_verbs": { |
| "name": "blimp_irregular_past_participle_verbs", |
| "alias": "blimp_irregular_past_participle_verbs", |
| "sample_len": 1000, |
| "acc,none": 0.848, |
| "acc_stderr,none": 0.01135891830347525 |
| }, |
| "blimp_irregular_plural_subject_verb_agreement_1": { |
| "name": "blimp_irregular_plural_subject_verb_agreement_1", |
| "alias": "blimp_irregular_plural_subject_verb_agreement_1", |
| "sample_len": 1000, |
| "acc,none": 0.896, |
| "acc_stderr,none": 0.009658016218524242 |
| }, |
| "blimp_irregular_plural_subject_verb_agreement_2": { |
| "name": "blimp_irregular_plural_subject_verb_agreement_2", |
| "alias": "blimp_irregular_plural_subject_verb_agreement_2", |
| "sample_len": 1000, |
| "acc,none": 0.881, |
| "acc_stderr,none": 0.01024421514533661 |
| }, |
| "blimp_left_branch_island_echo_question": { |
| "name": "blimp_left_branch_island_echo_question", |
| "alias": "blimp_left_branch_island_echo_question", |
| "sample_len": 1000, |
| "acc,none": 0.585, |
| "acc_stderr,none": 0.015589035185604594 |
| }, |
| "blimp_left_branch_island_simple_question": { |
| "name": "blimp_left_branch_island_simple_question", |
| "alias": "blimp_left_branch_island_simple_question", |
| "sample_len": 1000, |
| "acc,none": 0.485, |
| "acc_stderr,none": 0.01581217964181488 |
| }, |
| "blimp_matrix_question_npi_licensor_present": { |
| "name": "blimp_matrix_question_npi_licensor_present", |
| "alias": "blimp_matrix_question_npi_licensor_present", |
| "sample_len": 1000, |
| "acc,none": 0.492, |
| "acc_stderr,none": 0.015817274929209084 |
| }, |
| "blimp_npi_present_1": { |
| "name": "blimp_npi_present_1", |
| "alias": "blimp_npi_present_1", |
| "sample_len": 1000, |
| "acc,none": 0.474, |
| "acc_stderr,none": 0.015797897758042797 |
| }, |
| "blimp_npi_present_2": { |
| "name": "blimp_npi_present_2", |
| "alias": "blimp_npi_present_2", |
| "sample_len": 1000, |
| "acc,none": 0.619, |
| "acc_stderr,none": 0.015364734787007391 |
| }, |
| "blimp_only_npi_licensor_present": { |
| "name": "blimp_only_npi_licensor_present", |
| "alias": "blimp_only_npi_licensor_present", |
| "sample_len": 1000, |
| "acc,none": 0.983, |
| "acc_stderr,none": 0.004089954489689063 |
| }, |
| "blimp_only_npi_scope": { |
| "name": "blimp_only_npi_scope", |
| "alias": "blimp_only_npi_scope", |
| "sample_len": 1000, |
| "acc,none": 0.756, |
| "acc_stderr,none": 0.013588548437881386 |
| }, |
| "blimp_passive_1": { |
| "name": "blimp_passive_1", |
| "alias": "blimp_passive_1", |
| "sample_len": 1000, |
| "acc,none": 0.877, |
| "acc_stderr,none": 0.010391293421849803 |
| }, |
| "blimp_passive_2": { |
| "name": "blimp_passive_2", |
| "alias": "blimp_passive_2", |
| "sample_len": 1000, |
| "acc,none": 0.85, |
| "acc_stderr,none": 0.011297239823409418 |
| }, |
| "blimp_principle_A_c_command": { |
| "name": "blimp_principle_A_c_command", |
| "alias": "blimp_principle_A_c_command", |
| "sample_len": 1000, |
| "acc,none": 0.617, |
| "acc_stderr,none": 0.0153801023256527 |
| }, |
| "blimp_principle_A_case_1": { |
| "name": "blimp_principle_A_case_1", |
| "alias": "blimp_principle_A_case_1", |
| "sample_len": 1000, |
| "acc,none": 1.0, |
| "acc_stderr,none": 0.0 |
| }, |
| "blimp_principle_A_case_2": { |
| "name": "blimp_principle_A_case_2", |
| "alias": "blimp_principle_A_case_2", |
| "sample_len": 1000, |
| "acc,none": 0.909, |
| "acc_stderr,none": 0.009099549538400337 |
| }, |
| "blimp_principle_A_domain_1": { |
| "name": "blimp_principle_A_domain_1", |
| "alias": "blimp_principle_A_domain_1", |
| "sample_len": 1000, |
| "acc,none": 0.922, |
| "acc_stderr,none": 0.00848457353011853 |
| }, |
| "blimp_principle_A_domain_2": { |
| "name": "blimp_principle_A_domain_2", |
| "alias": "blimp_principle_A_domain_2", |
| "sample_len": 1000, |
| "acc,none": 0.774, |
| "acc_stderr,none": 0.013232501619085452 |
| }, |
| "blimp_principle_A_domain_3": { |
| "name": "blimp_principle_A_domain_3", |
| "alias": "blimp_principle_A_domain_3", |
| "sample_len": 1000, |
| "acc,none": 0.621, |
| "acc_stderr,none": 0.015349091002225332 |
| }, |
| "blimp_principle_A_reconstruction": { |
| "name": "blimp_principle_A_reconstruction", |
| "alias": "blimp_principle_A_reconstruction", |
| "sample_len": 1000, |
| "acc,none": 0.38, |
| "acc_stderr,none": 0.015356947477797658 |
| }, |
| "blimp_regular_plural_subject_verb_agreement_1": { |
| "name": "blimp_regular_plural_subject_verb_agreement_1", |
| "alias": "blimp_regular_plural_subject_verb_agreement_1", |
| "sample_len": 1000, |
| "acc,none": 0.862, |
| "acc_stderr,none": 0.010912152632504508 |
| }, |
| "blimp_regular_plural_subject_verb_agreement_2": { |
| "name": "blimp_regular_plural_subject_verb_agreement_2", |
| "alias": "blimp_regular_plural_subject_verb_agreement_2", |
| "sample_len": 1000, |
| "acc,none": 0.87, |
| "acc_stderr,none": 0.010640169792499236 |
| }, |
| "blimp_sentential_negation_npi_licensor_present": { |
| "name": "blimp_sentential_negation_npi_licensor_present", |
| "alias": "blimp_sentential_negation_npi_licensor_present", |
| "sample_len": 1000, |
| "acc,none": 0.99, |
| "acc_stderr,none": 0.003148000938676784 |
| }, |
| "blimp_sentential_negation_npi_scope": { |
| "name": "blimp_sentential_negation_npi_scope", |
| "alias": "blimp_sentential_negation_npi_scope", |
| "sample_len": 1000, |
| "acc,none": 0.699, |
| "acc_stderr,none": 0.01451239503354321 |
| }, |
| "blimp_sentential_subject_island": { |
| "name": "blimp_sentential_subject_island", |
| "alias": "blimp_sentential_subject_island", |
| "sample_len": 1000, |
| "acc,none": 0.508, |
| "acc_stderr,none": 0.015817274929209084 |
| }, |
| "blimp_superlative_quantifiers_1": { |
| "name": "blimp_superlative_quantifiers_1", |
| "alias": "blimp_superlative_quantifiers_1", |
| "sample_len": 1000, |
| "acc,none": 0.72, |
| "acc_stderr,none": 0.014205696104091548 |
| }, |
| "blimp_superlative_quantifiers_2": { |
| "name": "blimp_superlative_quantifiers_2", |
| "alias": "blimp_superlative_quantifiers_2", |
| "sample_len": 1000, |
| "acc,none": 0.702, |
| "acc_stderr,none": 0.014470846741134585 |
| }, |
| "blimp_tough_vs_raising_1": { |
| "name": "blimp_tough_vs_raising_1", |
| "alias": "blimp_tough_vs_raising_1", |
| "sample_len": 1000, |
| "acc,none": 0.55, |
| "acc_stderr,none": 0.015740004693383918 |
| }, |
| "blimp_tough_vs_raising_2": { |
| "name": "blimp_tough_vs_raising_2", |
| "alias": "blimp_tough_vs_raising_2", |
| "sample_len": 1000, |
| "acc,none": 0.875, |
| "acc_stderr,none": 0.010463483381956722 |
| }, |
| "blimp_transitive": { |
| "name": "blimp_transitive", |
| "alias": "blimp_transitive", |
| "sample_len": 1000, |
| "acc,none": 0.806, |
| "acc_stderr,none": 0.012510816141264347 |
| }, |
| "blimp_wh_island": { |
| "name": "blimp_wh_island", |
| "alias": "blimp_wh_island", |
| "sample_len": 1000, |
| "acc,none": 0.802, |
| "acc_stderr,none": 0.012607733934175283 |
| }, |
| "blimp_wh_questions_object_gap": { |
| "name": "blimp_wh_questions_object_gap", |
| "alias": "blimp_wh_questions_object_gap", |
| "sample_len": 1000, |
| "acc,none": 0.777, |
| "acc_stderr,none": 0.013169830843425608 |
| }, |
| "blimp_wh_questions_subject_gap": { |
| "name": "blimp_wh_questions_subject_gap", |
| "alias": "blimp_wh_questions_subject_gap", |
| "sample_len": 1000, |
| "acc,none": 0.919, |
| "acc_stderr,none": 0.008632121032139986 |
| }, |
| "blimp_wh_questions_subject_gap_long_distance": { |
| "name": "blimp_wh_questions_subject_gap_long_distance", |
| "alias": "blimp_wh_questions_subject_gap_long_distance", |
| "sample_len": 1000, |
| "acc,none": 0.9, |
| "acc_stderr,none": 0.00949157995752499 |
| }, |
| "blimp_wh_vs_that_no_gap": { |
| "name": "blimp_wh_vs_that_no_gap", |
| "alias": "blimp_wh_vs_that_no_gap", |
| "sample_len": 1000, |
| "acc,none": 0.96, |
| "acc_stderr,none": 0.006199874066337028 |
| }, |
| "blimp_wh_vs_that_no_gap_long_distance": { |
| "name": "blimp_wh_vs_that_no_gap_long_distance", |
| "alias": "blimp_wh_vs_that_no_gap_long_distance", |
| "sample_len": 1000, |
| "acc,none": 0.982, |
| "acc_stderr,none": 0.00420638724961144 |
| }, |
| "blimp_wh_vs_that_with_gap": { |
| "name": "blimp_wh_vs_that_with_gap", |
| "alias": "blimp_wh_vs_that_with_gap", |
| "sample_len": 1000, |
| "acc,none": 0.551, |
| "acc_stderr,none": 0.015736792768752044 |
| }, |
| "blimp_wh_vs_that_with_gap_long_distance": { |
| "name": "blimp_wh_vs_that_with_gap_long_distance", |
| "alias": "blimp_wh_vs_that_with_gap_long_distance", |
| "sample_len": 1000, |
| "acc,none": 0.215, |
| "acc_stderr,none": 0.01299784381903183 |
| }, |
| "blimp": { |
| "alias": "blimp", |
| "name": "blimp", |
| "sample_len": 67000, |
| "acc,none": 0.7661641791044775, |
| "acc_stderr,none": 0.0014856106734045368, |
| "sample_count": { |
| "acc,none": 67000 |
| } |
| } |
| } |
| } |
