| { |
| "model_id": "m29", |
| "checkpoint": "results/m29/final.pt", |
| "trained_step": 15258, |
| "num_fewshot": 0, |
| "limit": null, |
| "results": { |
| "lambada_openai": { |
| "name": "lambada_openai", |
| "alias": "lambada_openai", |
| "sample_len": 5153, |
| "perplexity,none": 580.781945907576, |
| "perplexity_stderr,none": 32.536453548312494, |
| "acc,none": 0.1841645643314574, |
| "acc_stderr,none": 0.005400281393204285 |
| }, |
| "hellaswag": { |
| "name": "hellaswag", |
| "alias": "hellaswag", |
| "sample_len": 10042, |
| "acc,none": 0.2727544313881697, |
| "acc_stderr,none": 0.004444654076551038, |
| "acc_norm,none": 0.28201553475403307, |
| "acc_norm_stderr,none": 0.004490612245335525 |
| }, |
| "piqa": { |
| "name": "piqa", |
| "alias": "piqa", |
| "sample_len": 1838, |
| "acc,none": 0.5919477693144722, |
| "acc_stderr,none": 0.011466872778651143, |
| "acc_norm,none": 0.5919477693144722, |
| "acc_norm_stderr,none": 0.011466872778651143 |
| }, |
| "sciq": { |
| "name": "sciq", |
| "alias": "sciq", |
| "sample_len": 1000, |
| "acc,none": 0.694, |
| "acc_stderr,none": 0.014580006055436898, |
| "acc_norm,none": 0.599, |
| "acc_norm_stderr,none": 0.015506109745498368 |
| }, |
| "arc_easy": { |
| "name": "arc_easy", |
| "alias": "arc_easy", |
| "sample_len": 2376, |
| "acc,none": 0.4701178451178451, |
| "acc_stderr,none": 0.010241444322886225, |
| "acc_norm,none": 0.4158249158249158, |
| "acc_norm_stderr,none": 0.01011334824464792 |
| }, |
| "winogrande": { |
| "name": "winogrande", |
| "alias": "winogrande", |
| "sample_len": 1267, |
| "acc,none": 0.5272296764009471, |
| "acc_stderr,none": 0.01403163162982771 |
| }, |
| "blimp_adjunct_island": { |
| "name": "blimp_adjunct_island", |
| "alias": "blimp_adjunct_island", |
| "sample_len": 1000, |
| "acc,none": 0.768, |
| "acc_stderr,none": 0.013354937452281623 |
| }, |
| "blimp_anaphor_gender_agreement": { |
| "name": "blimp_anaphor_gender_agreement", |
| "alias": "blimp_anaphor_gender_agreement", |
| "sample_len": 1000, |
| "acc,none": 0.887, |
| "acc_stderr,none": 0.010016552866696773 |
| }, |
| "blimp_anaphor_number_agreement": { |
| "name": "blimp_anaphor_number_agreement", |
| "alias": "blimp_anaphor_number_agreement", |
| "sample_len": 1000, |
| "acc,none": 0.978, |
| "acc_stderr,none": 0.004640855259274689 |
| }, |
| "blimp_animate_subject_passive": { |
| "name": "blimp_animate_subject_passive", |
| "alias": "blimp_animate_subject_passive", |
| "sample_len": 1000, |
| "acc,none": 0.715, |
| "acc_stderr,none": 0.014282120955200562 |
| }, |
| "blimp_animate_subject_trans": { |
| "name": "blimp_animate_subject_trans", |
| "alias": "blimp_animate_subject_trans", |
| "sample_len": 1000, |
| "acc,none": 0.901, |
| "acc_stderr,none": 0.009449248027662746 |
| }, |
| "blimp_causative": { |
| "name": "blimp_causative", |
| "alias": "blimp_causative", |
| "sample_len": 1000, |
| "acc,none": 0.708, |
| "acc_stderr,none": 0.014385511563477433 |
| }, |
| "blimp_complex_NP_island": { |
| "name": "blimp_complex_NP_island", |
| "alias": "blimp_complex_NP_island", |
| "sample_len": 1000, |
| "acc,none": 0.506, |
| "acc_stderr,none": 0.015818160898606836 |
| }, |
| "blimp_coordinate_structure_constraint_complex_left_branch": { |
| "name": "blimp_coordinate_structure_constraint_complex_left_branch", |
| "alias": "blimp_coordinate_structure_constraint_complex_left_branch", |
| "sample_len": 1000, |
| "acc,none": 0.393, |
| "acc_stderr,none": 0.015452824654081525 |
| }, |
| "blimp_coordinate_structure_constraint_object_extraction": { |
| "name": "blimp_coordinate_structure_constraint_object_extraction", |
| "alias": "blimp_coordinate_structure_constraint_object_extraction", |
| "sample_len": 1000, |
| "acc,none": 0.786, |
| "acc_stderr,none": 0.012975838021968847 |
| }, |
| "blimp_determiner_noun_agreement_1": { |
| "name": "blimp_determiner_noun_agreement_1", |
| "alias": "blimp_determiner_noun_agreement_1", |
| "sample_len": 1000, |
| "acc,none": 0.973, |
| "acc_stderr,none": 0.005128089049275305 |
| }, |
| "blimp_determiner_noun_agreement_2": { |
| "name": "blimp_determiner_noun_agreement_2", |
| "alias": "blimp_determiner_noun_agreement_2", |
| "sample_len": 1000, |
| "acc,none": 0.97, |
| "acc_stderr,none": 0.0053971408290991564 |
| }, |
| "blimp_determiner_noun_agreement_irregular_1": { |
| "name": "blimp_determiner_noun_agreement_irregular_1", |
| "alias": "blimp_determiner_noun_agreement_irregular_1", |
| "sample_len": 1000, |
| "acc,none": 0.916, |
| "acc_stderr,none": 0.008776162089491113 |
| }, |
| "blimp_determiner_noun_agreement_irregular_2": { |
| "name": "blimp_determiner_noun_agreement_irregular_2", |
| "alias": "blimp_determiner_noun_agreement_irregular_2", |
| "sample_len": 1000, |
| "acc,none": 0.953, |
| "acc_stderr,none": 0.006695956678163013 |
| }, |
| "blimp_determiner_noun_agreement_with_adj_2": { |
| "name": "blimp_determiner_noun_agreement_with_adj_2", |
| "alias": "blimp_determiner_noun_agreement_with_adj_2", |
| "sample_len": 1000, |
| "acc,none": 0.932, |
| "acc_stderr,none": 0.007964887911291624 |
| }, |
| "blimp_determiner_noun_agreement_with_adj_irregular_1": { |
| "name": "blimp_determiner_noun_agreement_with_adj_irregular_1", |
| "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1", |
| "sample_len": 1000, |
| "acc,none": 0.885, |
| "acc_stderr,none": 0.010093407594904551 |
| }, |
| "blimp_determiner_noun_agreement_with_adj_irregular_2": { |
| "name": "blimp_determiner_noun_agreement_with_adj_irregular_2", |
| "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2", |
| "sample_len": 1000, |
| "acc,none": 0.909, |
| "acc_stderr,none": 0.009099549538400337 |
| }, |
| "blimp_determiner_noun_agreement_with_adjective_1": { |
| "name": "blimp_determiner_noun_agreement_with_adjective_1", |
| "alias": "blimp_determiner_noun_agreement_with_adjective_1", |
| "sample_len": 1000, |
| "acc,none": 0.943, |
| "acc_stderr,none": 0.007335175853706905 |
| }, |
| "blimp_distractor_agreement_relational_noun": { |
| "name": "blimp_distractor_agreement_relational_noun", |
| "alias": "blimp_distractor_agreement_relational_noun", |
| "sample_len": 1000, |
| "acc,none": 0.863, |
| "acc_stderr,none": 0.01087884871433321 |
| }, |
| "blimp_distractor_agreement_relative_clause": { |
| "name": "blimp_distractor_agreement_relative_clause", |
| "alias": "blimp_distractor_agreement_relative_clause", |
| "sample_len": 1000, |
| "acc,none": 0.713, |
| "acc_stderr,none": 0.014312087053810001 |
| }, |
| "blimp_drop_argument": { |
| "name": "blimp_drop_argument", |
| "alias": "blimp_drop_argument", |
| "sample_len": 1000, |
| "acc,none": 0.766, |
| "acc_stderr,none": 0.01339490288966006 |
| }, |
| "blimp_ellipsis_n_bar_1": { |
| "name": "blimp_ellipsis_n_bar_1", |
| "alias": "blimp_ellipsis_n_bar_1", |
| "sample_len": 1000, |
| "acc,none": 0.709, |
| "acc_stderr,none": 0.014370995982378032 |
| }, |
| "blimp_ellipsis_n_bar_2": { |
| "name": "blimp_ellipsis_n_bar_2", |
| "alias": "blimp_ellipsis_n_bar_2", |
| "sample_len": 1000, |
| "acc,none": 0.853, |
| "acc_stderr,none": 0.011203415395160376 |
| }, |
| "blimp_existential_there_object_raising": { |
| "name": "blimp_existential_there_object_raising", |
| "alias": "blimp_existential_there_object_raising", |
| "sample_len": 1000, |
| "acc,none": 0.703, |
| "acc_stderr,none": 0.01445683229480096 |
| }, |
| "blimp_existential_there_quantifiers_1": { |
| "name": "blimp_existential_there_quantifiers_1", |
| "alias": "blimp_existential_there_quantifiers_1", |
| "sample_len": 1000, |
| "acc,none": 0.973, |
| "acc_stderr,none": 0.005128089049275305 |
| }, |
| "blimp_existential_there_quantifiers_2": { |
| "name": "blimp_existential_there_quantifiers_2", |
| "alias": "blimp_existential_there_quantifiers_2", |
| "sample_len": 1000, |
| "acc,none": 0.308, |
| "acc_stderr,none": 0.01460648312734278 |
| }, |
| "blimp_existential_there_subject_raising": { |
| "name": "blimp_existential_there_subject_raising", |
| "alias": "blimp_existential_there_subject_raising", |
| "sample_len": 1000, |
| "acc,none": 0.866, |
| "acc_stderr,none": 0.010777762298369738 |
| }, |
| "blimp_expletive_it_object_raising": { |
| "name": "blimp_expletive_it_object_raising", |
| "alias": "blimp_expletive_it_object_raising", |
| "sample_len": 1000, |
| "acc,none": 0.714, |
| "acc_stderr,none": 0.014297146862518003 |
| }, |
| "blimp_inchoative": { |
| "name": "blimp_inchoative", |
| "alias": "blimp_inchoative", |
| "sample_len": 1000, |
| "acc,none": 0.69, |
| "acc_stderr,none": 0.014632638658632766 |
| }, |
| "blimp_intransitive": { |
| "name": "blimp_intransitive", |
| "alias": "blimp_intransitive", |
| "sample_len": 1000, |
| "acc,none": 0.782, |
| "acc_stderr,none": 0.013063179040595235 |
| }, |
| "blimp_irregular_past_participle_adjectives": { |
| "name": "blimp_irregular_past_participle_adjectives", |
| "alias": "blimp_irregular_past_participle_adjectives", |
| "sample_len": 1000, |
| "acc,none": 0.746, |
| "acc_stderr,none": 0.013772206565168622 |
| }, |
| "blimp_irregular_past_participle_verbs": { |
| "name": "blimp_irregular_past_participle_verbs", |
| "alias": "blimp_irregular_past_participle_verbs", |
| "sample_len": 1000, |
| "acc,none": 0.867, |
| "acc_stderr,none": 0.010743669132397295 |
| }, |
| "blimp_irregular_plural_subject_verb_agreement_1": { |
| "name": "blimp_irregular_plural_subject_verb_agreement_1", |
| "alias": "blimp_irregular_plural_subject_verb_agreement_1", |
| "sample_len": 1000, |
| "acc,none": 0.913, |
| "acc_stderr,none": 0.008916866630745944 |
| }, |
| "blimp_irregular_plural_subject_verb_agreement_2": { |
| "name": "blimp_irregular_plural_subject_verb_agreement_2", |
| "alias": "blimp_irregular_plural_subject_verb_agreement_2", |
| "sample_len": 1000, |
| "acc,none": 0.903, |
| "acc_stderr,none": 0.009363689373248062 |
| }, |
| "blimp_left_branch_island_echo_question": { |
| "name": "blimp_left_branch_island_echo_question", |
| "alias": "blimp_left_branch_island_echo_question", |
| "sample_len": 1000, |
| "acc,none": 0.42, |
| "acc_stderr,none": 0.015615500115072968 |
| }, |
| "blimp_left_branch_island_simple_question": { |
| "name": "blimp_left_branch_island_simple_question", |
| "alias": "blimp_left_branch_island_simple_question", |
| "sample_len": 1000, |
| "acc,none": 0.502, |
| "acc_stderr,none": 0.01581917337430266 |
| }, |
| "blimp_matrix_question_npi_licensor_present": { |
| "name": "blimp_matrix_question_npi_licensor_present", |
| "alias": "blimp_matrix_question_npi_licensor_present", |
| "sample_len": 1000, |
| "acc,none": 0.52, |
| "acc_stderr,none": 0.015806639423035177 |
| }, |
| "blimp_npi_present_1": { |
| "name": "blimp_npi_present_1", |
| "alias": "blimp_npi_present_1", |
| "sample_len": 1000, |
| "acc,none": 0.499, |
| "acc_stderr,none": 0.015819268290576817 |
| }, |
| "blimp_npi_present_2": { |
| "name": "blimp_npi_present_2", |
| "alias": "blimp_npi_present_2", |
| "sample_len": 1000, |
| "acc,none": 0.616, |
| "acc_stderr,none": 0.015387682761896958 |
| }, |
| "blimp_only_npi_licensor_present": { |
| "name": "blimp_only_npi_licensor_present", |
| "alias": "blimp_only_npi_licensor_present", |
| "sample_len": 1000, |
| "acc,none": 0.913, |
| "acc_stderr,none": 0.008916866630745944 |
| }, |
| "blimp_only_npi_scope": { |
| "name": "blimp_only_npi_scope", |
| "alias": "blimp_only_npi_scope", |
| "sample_len": 1000, |
| "acc,none": 0.627, |
| "acc_stderr,none": 0.015300493622922927 |
| }, |
| "blimp_passive_1": { |
| "name": "blimp_passive_1", |
| "alias": "blimp_passive_1", |
| "sample_len": 1000, |
| "acc,none": 0.871, |
| "acc_stderr,none": 0.010605256784796466 |
| }, |
| "blimp_passive_2": { |
| "name": "blimp_passive_2", |
| "alias": "blimp_passive_2", |
| "sample_len": 1000, |
| "acc,none": 0.869, |
| "acc_stderr,none": 0.0106748748448379 |
| }, |
| "blimp_principle_A_c_command": { |
| "name": "blimp_principle_A_c_command", |
| "alias": "blimp_principle_A_c_command", |
| "sample_len": 1000, |
| "acc,none": 0.618, |
| "acc_stderr,none": 0.015372453034968502 |
| }, |
| "blimp_principle_A_case_1": { |
| "name": "blimp_principle_A_case_1", |
| "alias": "blimp_principle_A_case_1", |
| "sample_len": 1000, |
| "acc,none": 1.0, |
| "acc_stderr,none": 0.0 |
| }, |
| "blimp_principle_A_case_2": { |
| "name": "blimp_principle_A_case_2", |
| "alias": "blimp_principle_A_case_2", |
| "sample_len": 1000, |
| "acc,none": 0.925, |
| "acc_stderr,none": 0.008333333333333333 |
| }, |
| "blimp_principle_A_domain_1": { |
| "name": "blimp_principle_A_domain_1", |
| "alias": "blimp_principle_A_domain_1", |
| "sample_len": 1000, |
| "acc,none": 0.956, |
| "acc_stderr,none": 0.006488921798427387 |
| }, |
| "blimp_principle_A_domain_2": { |
| "name": "blimp_principle_A_domain_2", |
| "alias": "blimp_principle_A_domain_2", |
| "sample_len": 1000, |
| "acc,none": 0.765, |
| "acc_stderr,none": 0.013414729030247124 |
| }, |
| "blimp_principle_A_domain_3": { |
| "name": "blimp_principle_A_domain_3", |
| "alias": "blimp_principle_A_domain_3", |
| "sample_len": 1000, |
| "acc,none": 0.635, |
| "acc_stderr,none": 0.015231776226264848 |
| }, |
| "blimp_principle_A_reconstruction": { |
| "name": "blimp_principle_A_reconstruction", |
| "alias": "blimp_principle_A_reconstruction", |
| "sample_len": 1000, |
| "acc,none": 0.299, |
| "acc_stderr,none": 0.014484778521220435 |
| }, |
| "blimp_regular_plural_subject_verb_agreement_1": { |
| "name": "blimp_regular_plural_subject_verb_agreement_1", |
| "alias": "blimp_regular_plural_subject_verb_agreement_1", |
| "sample_len": 1000, |
| "acc,none": 0.9, |
| "acc_stderr,none": 0.00949157995752499 |
| }, |
| "blimp_regular_plural_subject_verb_agreement_2": { |
| "name": "blimp_regular_plural_subject_verb_agreement_2", |
| "alias": "blimp_regular_plural_subject_verb_agreement_2", |
| "sample_len": 1000, |
| "acc,none": 0.902, |
| "acc_stderr,none": 0.009406619184621247 |
| }, |
| "blimp_sentential_negation_npi_licensor_present": { |
| "name": "blimp_sentential_negation_npi_licensor_present", |
| "alias": "blimp_sentential_negation_npi_licensor_present", |
| "sample_len": 1000, |
| "acc,none": 0.98, |
| "acc_stderr,none": 0.004429403980178329 |
| }, |
| "blimp_sentential_negation_npi_scope": { |
| "name": "blimp_sentential_negation_npi_scope", |
| "alias": "blimp_sentential_negation_npi_scope", |
| "sample_len": 1000, |
| "acc,none": 0.555, |
| "acc_stderr,none": 0.015723301886761007 |
| }, |
| "blimp_sentential_subject_island": { |
| "name": "blimp_sentential_subject_island", |
| "alias": "blimp_sentential_subject_island", |
| "sample_len": 1000, |
| "acc,none": 0.453, |
| "acc_stderr,none": 0.015749255189977683 |
| }, |
| "blimp_superlative_quantifiers_1": { |
| "name": "blimp_superlative_quantifiers_1", |
| "alias": "blimp_superlative_quantifiers_1", |
| "sample_len": 1000, |
| "acc,none": 0.783, |
| "acc_stderr,none": 0.013041513757270706 |
| }, |
| "blimp_superlative_quantifiers_2": { |
| "name": "blimp_superlative_quantifiers_2", |
| "alias": "blimp_superlative_quantifiers_2", |
| "sample_len": 1000, |
| "acc,none": 0.693, |
| "acc_stderr,none": 0.01459328489285274 |
| }, |
| "blimp_tough_vs_raising_1": { |
| "name": "blimp_tough_vs_raising_1", |
| "alias": "blimp_tough_vs_raising_1", |
| "sample_len": 1000, |
| "acc,none": 0.6, |
| "acc_stderr,none": 0.015499685165842571 |
| }, |
| "blimp_tough_vs_raising_2": { |
| "name": "blimp_tough_vs_raising_2", |
| "alias": "blimp_tough_vs_raising_2", |
| "sample_len": 1000, |
| "acc,none": 0.879, |
| "acc_stderr,none": 0.010318210380946179 |
| }, |
| "blimp_transitive": { |
| "name": "blimp_transitive", |
| "alias": "blimp_transitive", |
| "sample_len": 1000, |
| "acc,none": 0.818, |
| "acc_stderr,none": 0.0122075806376622 |
| }, |
| "blimp_wh_island": { |
| "name": "blimp_wh_island", |
| "alias": "blimp_wh_island", |
| "sample_len": 1000, |
| "acc,none": 0.724, |
| "acc_stderr,none": 0.01414298497574059 |
| }, |
| "blimp_wh_questions_object_gap": { |
| "name": "blimp_wh_questions_object_gap", |
| "alias": "blimp_wh_questions_object_gap", |
| "sample_len": 1000, |
| "acc,none": 0.769, |
| "acc_stderr,none": 0.013334797216936478 |
| }, |
| "blimp_wh_questions_subject_gap": { |
| "name": "blimp_wh_questions_subject_gap", |
| "alias": "blimp_wh_questions_subject_gap", |
| "sample_len": 1000, |
| "acc,none": 0.945, |
| "acc_stderr,none": 0.0072129762946392 |
| }, |
| "blimp_wh_questions_subject_gap_long_distance": { |
| "name": "blimp_wh_questions_subject_gap_long_distance", |
| "alias": "blimp_wh_questions_subject_gap_long_distance", |
| "sample_len": 1000, |
| "acc,none": 0.909, |
| "acc_stderr,none": 0.009099549538400337 |
| }, |
| "blimp_wh_vs_that_no_gap": { |
| "name": "blimp_wh_vs_that_no_gap", |
| "alias": "blimp_wh_vs_that_no_gap", |
| "sample_len": 1000, |
| "acc,none": 0.967, |
| "acc_stderr,none": 0.005651808820452346 |
| }, |
| "blimp_wh_vs_that_no_gap_long_distance": { |
| "name": "blimp_wh_vs_that_no_gap_long_distance", |
| "alias": "blimp_wh_vs_that_no_gap_long_distance", |
| "sample_len": 1000, |
| "acc,none": 0.984, |
| "acc_stderr,none": 0.003969856390319398 |
| }, |
| "blimp_wh_vs_that_with_gap": { |
| "name": "blimp_wh_vs_that_with_gap", |
| "alias": "blimp_wh_vs_that_with_gap", |
| "sample_len": 1000, |
| "acc,none": 0.533, |
| "acc_stderr,none": 0.015784807891138876 |
| }, |
| "blimp_wh_vs_that_with_gap_long_distance": { |
| "name": "blimp_wh_vs_that_with_gap_long_distance", |
| "alias": "blimp_wh_vs_that_with_gap_long_distance", |
| "sample_len": 1000, |
| "acc,none": 0.177, |
| "acc_stderr,none": 0.012075463420375087 |
| }, |
| "blimp": { |
| "alias": "blimp", |
| "name": "blimp", |
| "sample_len": 67000, |
| "acc,none": 0.7626567164179104, |
| "acc_stderr,none": 0.0014674121164425231, |
| "sample_count": { |
| "acc,none": 67000 |
| } |
| } |
| } |
| } |
