| { |
| "model_id": "m27", |
| "checkpoint": "results/m27/final.pt", |
| "trained_step": 15258, |
| "num_fewshot": 0, |
| "limit": null, |
| "results": { |
| "lambada_openai": { |
| "name": "lambada_openai", |
| "alias": "lambada_openai", |
| "sample_len": 5153, |
| "perplexity,none": 570.3424875824041, |
| "perplexity_stderr,none": 30.24159359562431, |
| "acc,none": 0.16553464001552493, |
| "acc_stderr,none": 0.005177983637644952 |
| }, |
| "hellaswag": { |
| "name": "hellaswag", |
| "alias": "hellaswag", |
| "sample_len": 10042, |
| "acc,none": 0.27365066719776937, |
| "acc_stderr,none": 0.0044492062959226135, |
| "acc_norm,none": 0.2861979685321649, |
| "acc_norm_stderr,none": 0.004510593395290067 |
| }, |
| "piqa": { |
| "name": "piqa", |
| "alias": "piqa", |
| "sample_len": 1838, |
| "acc,none": 0.6055495103373232, |
| "acc_stderr,none": 0.011402931101558359, |
| "acc_norm,none": 0.5963003264417845, |
| "acc_norm_stderr,none": 0.011447407541749126 |
| }, |
| "sciq": { |
| "name": "sciq", |
| "alias": "sciq", |
| "sample_len": 1000, |
| "acc,none": 0.695, |
| "acc_stderr,none": 0.014566646394664325, |
| "acc_norm,none": 0.608, |
| "acc_norm_stderr,none": 0.015445859463771338 |
| }, |
| "arc_easy": { |
| "name": "arc_easy", |
| "alias": "arc_easy", |
| "sample_len": 2376, |
| "acc,none": 0.44654882154882153, |
| "acc_stderr,none": 0.010200990076245286, |
| "acc_norm,none": 0.4074074074074074, |
| "acc_norm_stderr,none": 0.010082326627832896 |
| }, |
| "winogrande": { |
| "name": "winogrande", |
| "alias": "winogrande", |
| "sample_len": 1267, |
| "acc,none": 0.5169692186266772, |
| "acc_stderr,none": 0.014044390401613116 |
| }, |
| "blimp_adjunct_island": { |
| "name": "blimp_adjunct_island", |
| "alias": "blimp_adjunct_island", |
| "sample_len": 1000, |
| "acc,none": 0.812, |
| "acc_stderr,none": 0.012361586015103758 |
| }, |
| "blimp_anaphor_gender_agreement": { |
| "name": "blimp_anaphor_gender_agreement", |
| "alias": "blimp_anaphor_gender_agreement", |
| "sample_len": 1000, |
| "acc,none": 0.892, |
| "acc_stderr,none": 0.009820001651345708 |
| }, |
| "blimp_anaphor_number_agreement": { |
| "name": "blimp_anaphor_number_agreement", |
| "alias": "blimp_anaphor_number_agreement", |
| "sample_len": 1000, |
| "acc,none": 0.983, |
| "acc_stderr,none": 0.004089954489689063 |
| }, |
| "blimp_animate_subject_passive": { |
| "name": "blimp_animate_subject_passive", |
| "alias": "blimp_animate_subject_passive", |
| "sample_len": 1000, |
| "acc,none": 0.735, |
| "acc_stderr,none": 0.013963164754810064 |
| }, |
| "blimp_animate_subject_trans": { |
| "name": "blimp_animate_subject_trans", |
| "alias": "blimp_animate_subject_trans", |
| "sample_len": 1000, |
| "acc,none": 0.891, |
| "acc_stderr,none": 0.009859828407037124 |
| }, |
| "blimp_causative": { |
| "name": "blimp_causative", |
| "alias": "blimp_causative", |
| "sample_len": 1000, |
| "acc,none": 0.713, |
| "acc_stderr,none": 0.014312087053810001 |
| }, |
| "blimp_complex_NP_island": { |
| "name": "blimp_complex_NP_island", |
| "alias": "blimp_complex_NP_island", |
| "sample_len": 1000, |
| "acc,none": 0.553, |
| "acc_stderr,none": 0.015730176046009098 |
| }, |
| "blimp_coordinate_structure_constraint_complex_left_branch": { |
| "name": "blimp_coordinate_structure_constraint_complex_left_branch", |
| "alias": "blimp_coordinate_structure_constraint_complex_left_branch", |
| "sample_len": 1000, |
| "acc,none": 0.515, |
| "acc_stderr,none": 0.01581217964181488 |
| }, |
| "blimp_coordinate_structure_constraint_object_extraction": { |
| "name": "blimp_coordinate_structure_constraint_object_extraction", |
| "alias": "blimp_coordinate_structure_constraint_object_extraction", |
| "sample_len": 1000, |
| "acc,none": 0.742, |
| "acc_stderr,none": 0.013842963108656601 |
| }, |
| "blimp_determiner_noun_agreement_1": { |
| "name": "blimp_determiner_noun_agreement_1", |
| "alias": "blimp_determiner_noun_agreement_1", |
| "sample_len": 1000, |
| "acc,none": 0.962, |
| "acc_stderr,none": 0.0060491811505849775 |
| }, |
| "blimp_determiner_noun_agreement_2": { |
| "name": "blimp_determiner_noun_agreement_2", |
| "alias": "blimp_determiner_noun_agreement_2", |
| "sample_len": 1000, |
| "acc,none": 0.944, |
| "acc_stderr,none": 0.007274401481697097 |
| }, |
| "blimp_determiner_noun_agreement_irregular_1": { |
| "name": "blimp_determiner_noun_agreement_irregular_1", |
| "alias": "blimp_determiner_noun_agreement_irregular_1", |
| "sample_len": 1000, |
| "acc,none": 0.889, |
| "acc_stderr,none": 0.009938701010583716 |
| }, |
| "blimp_determiner_noun_agreement_irregular_2": { |
| "name": "blimp_determiner_noun_agreement_irregular_2", |
| "alias": "blimp_determiner_noun_agreement_irregular_2", |
| "sample_len": 1000, |
| "acc,none": 0.919, |
| "acc_stderr,none": 0.008632121032139986 |
| }, |
| "blimp_determiner_noun_agreement_with_adj_2": { |
| "name": "blimp_determiner_noun_agreement_with_adj_2", |
| "alias": "blimp_determiner_noun_agreement_with_adj_2", |
| "sample_len": 1000, |
| "acc,none": 0.896, |
| "acc_stderr,none": 0.009658016218524242 |
| }, |
| "blimp_determiner_noun_agreement_with_adj_irregular_1": { |
| "name": "blimp_determiner_noun_agreement_with_adj_irregular_1", |
| "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1", |
| "sample_len": 1000, |
| "acc,none": 0.855, |
| "acc_stderr,none": 0.0111399775178902 |
| }, |
| "blimp_determiner_noun_agreement_with_adj_irregular_2": { |
| "name": "blimp_determiner_noun_agreement_with_adj_irregular_2", |
| "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2", |
| "sample_len": 1000, |
| "acc,none": 0.89, |
| "acc_stderr,none": 0.00989939381972442 |
| }, |
| "blimp_determiner_noun_agreement_with_adjective_1": { |
| "name": "blimp_determiner_noun_agreement_with_adjective_1", |
| "alias": "blimp_determiner_noun_agreement_with_adjective_1", |
| "sample_len": 1000, |
| "acc,none": 0.9, |
| "acc_stderr,none": 0.00949157995752499 |
| }, |
| "blimp_distractor_agreement_relational_noun": { |
| "name": "blimp_distractor_agreement_relational_noun", |
| "alias": "blimp_distractor_agreement_relational_noun", |
| "sample_len": 1000, |
| "acc,none": 0.793, |
| "acc_stderr,none": 0.012818553557844009 |
| }, |
| "blimp_distractor_agreement_relative_clause": { |
| "name": "blimp_distractor_agreement_relative_clause", |
| "alias": "blimp_distractor_agreement_relative_clause", |
| "sample_len": 1000, |
| "acc,none": 0.602, |
| "acc_stderr,none": 0.015486634102859016 |
| }, |
| "blimp_drop_argument": { |
| "name": "blimp_drop_argument", |
| "alias": "blimp_drop_argument", |
| "sample_len": 1000, |
| "acc,none": 0.78, |
| "acc_stderr,none": 0.013106173040661862 |
| }, |
| "blimp_ellipsis_n_bar_1": { |
| "name": "blimp_ellipsis_n_bar_1", |
| "alias": "blimp_ellipsis_n_bar_1", |
| "sample_len": 1000, |
| "acc,none": 0.73, |
| "acc_stderr,none": 0.01404625563263382 |
| }, |
| "blimp_ellipsis_n_bar_2": { |
| "name": "blimp_ellipsis_n_bar_2", |
| "alias": "blimp_ellipsis_n_bar_2", |
| "sample_len": 1000, |
| "acc,none": 0.854, |
| "acc_stderr,none": 0.011171786285496551 |
| }, |
| "blimp_existential_there_object_raising": { |
| "name": "blimp_existential_there_object_raising", |
| "alias": "blimp_existential_there_object_raising", |
| "sample_len": 1000, |
| "acc,none": 0.769, |
| "acc_stderr,none": 0.013334797216936478 |
| }, |
| "blimp_existential_there_quantifiers_1": { |
| "name": "blimp_existential_there_quantifiers_1", |
| "alias": "blimp_existential_there_quantifiers_1", |
| "sample_len": 1000, |
| "acc,none": 0.972, |
| "acc_stderr,none": 0.005219506034410081 |
| }, |
| "blimp_existential_there_quantifiers_2": { |
| "name": "blimp_existential_there_quantifiers_2", |
| "alias": "blimp_existential_there_quantifiers_2", |
| "sample_len": 1000, |
| "acc,none": 0.417, |
| "acc_stderr,none": 0.015599819048769583 |
| }, |
| "blimp_existential_there_subject_raising": { |
| "name": "blimp_existential_there_subject_raising", |
| "alias": "blimp_existential_there_subject_raising", |
| "sample_len": 1000, |
| "acc,none": 0.857, |
| "acc_stderr,none": 0.011075814808567074 |
| }, |
| "blimp_expletive_it_object_raising": { |
| "name": "blimp_expletive_it_object_raising", |
| "alias": "blimp_expletive_it_object_raising", |
| "sample_len": 1000, |
| "acc,none": 0.76, |
| "acc_stderr,none": 0.013512312258920847 |
| }, |
| "blimp_inchoative": { |
| "name": "blimp_inchoative", |
| "alias": "blimp_inchoative", |
| "sample_len": 1000, |
| "acc,none": 0.668, |
| "acc_stderr,none": 0.014899597242811567 |
| }, |
| "blimp_intransitive": { |
| "name": "blimp_intransitive", |
| "alias": "blimp_intransitive", |
| "sample_len": 1000, |
| "acc,none": 0.763, |
| "acc_stderr,none": 0.013454070462577935 |
| }, |
| "blimp_irregular_past_participle_adjectives": { |
| "name": "blimp_irregular_past_participle_adjectives", |
| "alias": "blimp_irregular_past_participle_adjectives", |
| "sample_len": 1000, |
| "acc,none": 0.896, |
| "acc_stderr,none": 0.009658016218524242 |
| }, |
| "blimp_irregular_past_participle_verbs": { |
| "name": "blimp_irregular_past_participle_verbs", |
| "alias": "blimp_irregular_past_participle_verbs", |
| "sample_len": 1000, |
| "acc,none": 0.823, |
| "acc_stderr,none": 0.012075463420375087 |
| }, |
| "blimp_irregular_plural_subject_verb_agreement_1": { |
| "name": "blimp_irregular_plural_subject_verb_agreement_1", |
| "alias": "blimp_irregular_plural_subject_verb_agreement_1", |
| "sample_len": 1000, |
| "acc,none": 0.887, |
| "acc_stderr,none": 0.010016552866696773 |
| }, |
| "blimp_irregular_plural_subject_verb_agreement_2": { |
| "name": "blimp_irregular_plural_subject_verb_agreement_2", |
| "alias": "blimp_irregular_plural_subject_verb_agreement_2", |
| "sample_len": 1000, |
| "acc,none": 0.879, |
| "acc_stderr,none": 0.010318210380946179 |
| }, |
| "blimp_left_branch_island_echo_question": { |
| "name": "blimp_left_branch_island_echo_question", |
| "alias": "blimp_left_branch_island_echo_question", |
| "sample_len": 1000, |
| "acc,none": 0.4, |
| "acc_stderr,none": 0.015499685165842571 |
| }, |
| "blimp_left_branch_island_simple_question": { |
| "name": "blimp_left_branch_island_simple_question", |
| "alias": "blimp_left_branch_island_simple_question", |
| "sample_len": 1000, |
| "acc,none": 0.561, |
| "acc_stderr,none": 0.015701131345400736 |
| }, |
| "blimp_matrix_question_npi_licensor_present": { |
| "name": "blimp_matrix_question_npi_licensor_present", |
| "alias": "blimp_matrix_question_npi_licensor_present", |
| "sample_len": 1000, |
| "acc,none": 0.513, |
| "acc_stderr,none": 0.015813952101896636 |
| }, |
| "blimp_npi_present_1": { |
| "name": "blimp_npi_present_1", |
| "alias": "blimp_npi_present_1", |
| "sample_len": 1000, |
| "acc,none": 0.631, |
| "acc_stderr,none": 0.015266698139154782 |
| }, |
| "blimp_npi_present_2": { |
| "name": "blimp_npi_present_2", |
| "alias": "blimp_npi_present_2", |
| "sample_len": 1000, |
| "acc,none": 0.705, |
| "acc_stderr,none": 0.014428554438445425 |
| }, |
| "blimp_only_npi_licensor_present": { |
| "name": "blimp_only_npi_licensor_present", |
| "alias": "blimp_only_npi_licensor_present", |
| "sample_len": 1000, |
| "acc,none": 0.806, |
| "acc_stderr,none": 0.012510816141264347 |
| }, |
| "blimp_only_npi_scope": { |
| "name": "blimp_only_npi_scope", |
| "alias": "blimp_only_npi_scope", |
| "sample_len": 1000, |
| "acc,none": 0.491, |
| "acc_stderr,none": 0.0158167369950053 |
| }, |
| "blimp_passive_1": { |
| "name": "blimp_passive_1", |
| "alias": "blimp_passive_1", |
| "sample_len": 1000, |
| "acc,none": 0.895, |
| "acc_stderr,none": 0.00969892102602496 |
| }, |
| "blimp_passive_2": { |
| "name": "blimp_passive_2", |
| "alias": "blimp_passive_2", |
| "sample_len": 1000, |
| "acc,none": 0.877, |
| "acc_stderr,none": 0.010391293421849803 |
| }, |
| "blimp_principle_A_c_command": { |
| "name": "blimp_principle_A_c_command", |
| "alias": "blimp_principle_A_c_command", |
| "sample_len": 1000, |
| "acc,none": 0.547, |
| "acc_stderr,none": 0.015749255189977687 |
| }, |
| "blimp_principle_A_case_1": { |
| "name": "blimp_principle_A_case_1", |
| "alias": "blimp_principle_A_case_1", |
| "sample_len": 1000, |
| "acc,none": 0.999, |
| "acc_stderr,none": 0.001 |
| }, |
| "blimp_principle_A_case_2": { |
| "name": "blimp_principle_A_case_2", |
| "alias": "blimp_principle_A_case_2", |
| "sample_len": 1000, |
| "acc,none": 0.923, |
| "acc_stderr,none": 0.00843458014024066 |
| }, |
| "blimp_principle_A_domain_1": { |
| "name": "blimp_principle_A_domain_1", |
| "alias": "blimp_principle_A_domain_1", |
| "sample_len": 1000, |
| "acc,none": 0.9, |
| "acc_stderr,none": 0.00949157995752499 |
| }, |
| "blimp_principle_A_domain_2": { |
| "name": "blimp_principle_A_domain_2", |
| "alias": "blimp_principle_A_domain_2", |
| "sample_len": 1000, |
| "acc,none": 0.809, |
| "acc_stderr,none": 0.012436787112179482 |
| }, |
| "blimp_principle_A_domain_3": { |
| "name": "blimp_principle_A_domain_3", |
| "alias": "blimp_principle_A_domain_3", |
| "sample_len": 1000, |
| "acc,none": 0.658, |
| "acc_stderr,none": 0.015008706182121804 |
| }, |
| "blimp_principle_A_reconstruction": { |
| "name": "blimp_principle_A_reconstruction", |
| "alias": "blimp_principle_A_reconstruction", |
| "sample_len": 1000, |
| "acc,none": 0.408, |
| "acc_stderr,none": 0.015549205052920803 |
| }, |
| "blimp_regular_plural_subject_verb_agreement_1": { |
| "name": "blimp_regular_plural_subject_verb_agreement_1", |
| "alias": "blimp_regular_plural_subject_verb_agreement_1", |
| "sample_len": 1000, |
| "acc,none": 0.882, |
| "acc_stderr,none": 0.010206869264381718 |
| }, |
| "blimp_regular_plural_subject_verb_agreement_2": { |
| "name": "blimp_regular_plural_subject_verb_agreement_2", |
| "alias": "blimp_regular_plural_subject_verb_agreement_2", |
| "sample_len": 1000, |
| "acc,none": 0.862, |
| "acc_stderr,none": 0.010912152632504508 |
| }, |
| "blimp_sentential_negation_npi_licensor_present": { |
| "name": "blimp_sentential_negation_npi_licensor_present", |
| "alias": "blimp_sentential_negation_npi_licensor_present", |
| "sample_len": 1000, |
| "acc,none": 0.998, |
| "acc_stderr,none": 0.0014135055705578234 |
| }, |
| "blimp_sentential_negation_npi_scope": { |
| "name": "blimp_sentential_negation_npi_scope", |
| "alias": "blimp_sentential_negation_npi_scope", |
| "sample_len": 1000, |
| "acc,none": 0.599, |
| "acc_stderr,none": 0.015506109745498368 |
| }, |
| "blimp_sentential_subject_island": { |
| "name": "blimp_sentential_subject_island", |
| "alias": "blimp_sentential_subject_island", |
| "sample_len": 1000, |
| "acc,none": 0.519, |
| "acc_stderr,none": 0.015807874268505835 |
| }, |
| "blimp_superlative_quantifiers_1": { |
| "name": "blimp_superlative_quantifiers_1", |
| "alias": "blimp_superlative_quantifiers_1", |
| "sample_len": 1000, |
| "acc,none": 0.923, |
| "acc_stderr,none": 0.00843458014024066 |
| }, |
| "blimp_superlative_quantifiers_2": { |
| "name": "blimp_superlative_quantifiers_2", |
| "alias": "blimp_superlative_quantifiers_2", |
| "sample_len": 1000, |
| "acc,none": 0.878, |
| "acc_stderr,none": 0.010354864712936776 |
| }, |
| "blimp_tough_vs_raising_1": { |
| "name": "blimp_tough_vs_raising_1", |
| "alias": "blimp_tough_vs_raising_1", |
| "sample_len": 1000, |
| "acc,none": 0.557, |
| "acc_stderr,none": 0.015716169953204184 |
| }, |
| "blimp_tough_vs_raising_2": { |
| "name": "blimp_tough_vs_raising_2", |
| "alias": "blimp_tough_vs_raising_2", |
| "sample_len": 1000, |
| "acc,none": 0.888, |
| "acc_stderr,none": 0.009977753031397177 |
| }, |
| "blimp_transitive": { |
| "name": "blimp_transitive", |
| "alias": "blimp_transitive", |
| "sample_len": 1000, |
| "acc,none": 0.815, |
| "acc_stderr,none": 0.012285191326386656 |
| }, |
| "blimp_wh_island": { |
| "name": "blimp_wh_island", |
| "alias": "blimp_wh_island", |
| "sample_len": 1000, |
| "acc,none": 0.767, |
| "acc_stderr,none": 0.01337497251921999 |
| }, |
| "blimp_wh_questions_object_gap": { |
| "name": "blimp_wh_questions_object_gap", |
| "alias": "blimp_wh_questions_object_gap", |
| "sample_len": 1000, |
| "acc,none": 0.708, |
| "acc_stderr,none": 0.014385511563477433 |
| }, |
| "blimp_wh_questions_subject_gap": { |
| "name": "blimp_wh_questions_subject_gap", |
| "alias": "blimp_wh_questions_subject_gap", |
| "sample_len": 1000, |
| "acc,none": 0.867, |
| "acc_stderr,none": 0.010743669132397295 |
| }, |
| "blimp_wh_questions_subject_gap_long_distance": { |
| "name": "blimp_wh_questions_subject_gap_long_distance", |
| "alias": "blimp_wh_questions_subject_gap_long_distance", |
| "sample_len": 1000, |
| "acc,none": 0.782, |
| "acc_stderr,none": 0.013063179040595235 |
| }, |
| "blimp_wh_vs_that_no_gap": { |
| "name": "blimp_wh_vs_that_no_gap", |
| "alias": "blimp_wh_vs_that_no_gap", |
| "sample_len": 1000, |
| "acc,none": 0.953, |
| "acc_stderr,none": 0.006695956678163013 |
| }, |
| "blimp_wh_vs_that_no_gap_long_distance": { |
| "name": "blimp_wh_vs_that_no_gap_long_distance", |
| "alias": "blimp_wh_vs_that_no_gap_long_distance", |
| "sample_len": 1000, |
| "acc,none": 0.96, |
| "acc_stderr,none": 0.006199874066337028 |
| }, |
| "blimp_wh_vs_that_with_gap": { |
| "name": "blimp_wh_vs_that_with_gap", |
| "alias": "blimp_wh_vs_that_with_gap", |
| "sample_len": 1000, |
| "acc,none": 0.471, |
| "acc_stderr,none": 0.015792669451628764 |
| }, |
| "blimp_wh_vs_that_with_gap_long_distance": { |
| "name": "blimp_wh_vs_that_with_gap_long_distance", |
| "alias": "blimp_wh_vs_that_with_gap_long_distance", |
| "sample_len": 1000, |
| "acc,none": 0.184, |
| "acc_stderr,none": 0.01225945734093864 |
| }, |
| "blimp": { |
| "alias": "blimp", |
| "name": "blimp", |
| "sample_len": 67000, |
| "acc,none": 0.7653283582089553, |
| "acc_stderr,none": 0.0014914982264402836, |
| "sample_count": { |
| "acc,none": 67000 |
| } |
| } |
| } |
| } |
