| { |
| "model_id": "m19", |
| "checkpoint": "results/m19/final.pt", |
| "trained_step": 15258, |
| "num_fewshot": 0, |
| "limit": null, |
| "results": { |
| "lambada_openai": { |
| "name": "lambada_openai", |
| "alias": "lambada_openai", |
| "sample_len": 5153, |
| "perplexity,none": 920.0575260263694, |
| "perplexity_stderr,none": 49.12608073743694, |
| "acc,none": 0.11643702697457792, |
| "acc_stderr,none": 0.004468650413126095 |
| }, |
| "hellaswag": { |
| "name": "hellaswag", |
| "alias": "hellaswag", |
| "sample_len": 10042, |
| "acc,none": 0.27185819557857, |
| "acc_stderr,none": 0.0044400791732768724, |
| "acc_norm,none": 0.27454690300736906, |
| "acc_norm_stderr,none": 0.004453735900948104 |
| }, |
| "piqa": { |
| "name": "piqa", |
| "alias": "piqa", |
| "sample_len": 1838, |
| "acc,none": 0.5870511425462459, |
| "acc_stderr,none": 0.011487658725078972, |
| "acc_norm,none": 0.6011969532100109, |
| "acc_norm_stderr,none": 0.011424390545037256 |
| }, |
| "sciq": { |
| "name": "sciq", |
| "alias": "sciq", |
| "sample_len": 1000, |
| "acc,none": 0.64, |
| "acc_stderr,none": 0.015186527932039985, |
| "acc_norm,none": 0.578, |
| "acc_norm_stderr,none": 0.015625625112620622 |
| }, |
| "arc_easy": { |
| "name": "arc_easy", |
| "alias": "arc_easy", |
| "sample_len": 2376, |
| "acc,none": 0.44234006734006737, |
| "acc_stderr,none": 0.010191334444220676, |
| "acc_norm,none": 0.4065656565656566, |
| "acc_norm_stderr,none": 0.01007905641922357 |
| }, |
| "winogrande": { |
| "name": "winogrande", |
| "alias": "winogrande", |
| "sample_len": 1267, |
| "acc,none": 0.5232833464877664, |
| "acc_stderr,none": 0.0140372413095736 |
| }, |
| "blimp_adjunct_island": { |
| "name": "blimp_adjunct_island", |
| "alias": "blimp_adjunct_island", |
| "sample_len": 1000, |
| "acc,none": 0.83, |
| "acc_stderr,none": 0.011884495834541793 |
| }, |
| "blimp_anaphor_gender_agreement": { |
| "name": "blimp_anaphor_gender_agreement", |
| "alias": "blimp_anaphor_gender_agreement", |
| "sample_len": 1000, |
| "acc,none": 0.952, |
| "acc_stderr,none": 0.006763264133666652 |
| }, |
| "blimp_anaphor_number_agreement": { |
| "name": "blimp_anaphor_number_agreement", |
| "alias": "blimp_anaphor_number_agreement", |
| "sample_len": 1000, |
| "acc,none": 0.984, |
| "acc_stderr,none": 0.003969856390319398 |
| }, |
| "blimp_animate_subject_passive": { |
| "name": "blimp_animate_subject_passive", |
| "alias": "blimp_animate_subject_passive", |
| "sample_len": 1000, |
| "acc,none": 0.733, |
| "acc_stderr,none": 0.013996674851796299 |
| }, |
| "blimp_animate_subject_trans": { |
| "name": "blimp_animate_subject_trans", |
| "alias": "blimp_animate_subject_trans", |
| "sample_len": 1000, |
| "acc,none": 0.913, |
| "acc_stderr,none": 0.008916866630745944 |
| }, |
| "blimp_causative": { |
| "name": "blimp_causative", |
| "alias": "blimp_causative", |
| "sample_len": 1000, |
| "acc,none": 0.705, |
| "acc_stderr,none": 0.014428554438445425 |
| }, |
| "blimp_complex_NP_island": { |
| "name": "blimp_complex_NP_island", |
| "alias": "blimp_complex_NP_island", |
| "sample_len": 1000, |
| "acc,none": 0.559, |
| "acc_stderr,none": 0.015708779894242766 |
| }, |
| "blimp_coordinate_structure_constraint_complex_left_branch": { |
| "name": "blimp_coordinate_structure_constraint_complex_left_branch", |
| "alias": "blimp_coordinate_structure_constraint_complex_left_branch", |
| "sample_len": 1000, |
| "acc,none": 0.476, |
| "acc_stderr,none": 0.01580106558665186 |
| }, |
| "blimp_coordinate_structure_constraint_object_extraction": { |
| "name": "blimp_coordinate_structure_constraint_object_extraction", |
| "alias": "blimp_coordinate_structure_constraint_object_extraction", |
| "sample_len": 1000, |
| "acc,none": 0.821, |
| "acc_stderr,none": 0.012128730605719056 |
| }, |
| "blimp_determiner_noun_agreement_1": { |
| "name": "blimp_determiner_noun_agreement_1", |
| "alias": "blimp_determiner_noun_agreement_1", |
| "sample_len": 1000, |
| "acc,none": 0.967, |
| "acc_stderr,none": 0.005651808820452346 |
| }, |
| "blimp_determiner_noun_agreement_2": { |
| "name": "blimp_determiner_noun_agreement_2", |
| "alias": "blimp_determiner_noun_agreement_2", |
| "sample_len": 1000, |
| "acc,none": 0.944, |
| "acc_stderr,none": 0.007274401481697097 |
| }, |
| "blimp_determiner_noun_agreement_irregular_1": { |
| "name": "blimp_determiner_noun_agreement_irregular_1", |
| "alias": "blimp_determiner_noun_agreement_irregular_1", |
| "sample_len": 1000, |
| "acc,none": 0.909, |
| "acc_stderr,none": 0.009099549538400337 |
| }, |
| "blimp_determiner_noun_agreement_irregular_2": { |
| "name": "blimp_determiner_noun_agreement_irregular_2", |
| "alias": "blimp_determiner_noun_agreement_irregular_2", |
| "sample_len": 1000, |
| "acc,none": 0.937, |
| "acc_stderr,none": 0.007687007876286461 |
| }, |
| "blimp_determiner_noun_agreement_with_adj_2": { |
| "name": "blimp_determiner_noun_agreement_with_adj_2", |
| "alias": "blimp_determiner_noun_agreement_with_adj_2", |
| "sample_len": 1000, |
| "acc,none": 0.902, |
| "acc_stderr,none": 0.009406619184621247 |
| }, |
| "blimp_determiner_noun_agreement_with_adj_irregular_1": { |
| "name": "blimp_determiner_noun_agreement_with_adj_irregular_1", |
| "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1", |
| "sample_len": 1000, |
| "acc,none": 0.895, |
| "acc_stderr,none": 0.00969892102602496 |
| }, |
| "blimp_determiner_noun_agreement_with_adj_irregular_2": { |
| "name": "blimp_determiner_noun_agreement_with_adj_irregular_2", |
| "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2", |
| "sample_len": 1000, |
| "acc,none": 0.898, |
| "acc_stderr,none": 0.009575368801653944 |
| }, |
| "blimp_determiner_noun_agreement_with_adjective_1": { |
| "name": "blimp_determiner_noun_agreement_with_adjective_1", |
| "alias": "blimp_determiner_noun_agreement_with_adjective_1", |
| "sample_len": 1000, |
| "acc,none": 0.933, |
| "acc_stderr,none": 0.007910345983177575 |
| }, |
| "blimp_distractor_agreement_relational_noun": { |
| "name": "blimp_distractor_agreement_relational_noun", |
| "alias": "blimp_distractor_agreement_relational_noun", |
| "sample_len": 1000, |
| "acc,none": 0.781, |
| "acc_stderr,none": 0.013084731950262116 |
| }, |
| "blimp_distractor_agreement_relative_clause": { |
| "name": "blimp_distractor_agreement_relative_clause", |
| "alias": "blimp_distractor_agreement_relative_clause", |
| "sample_len": 1000, |
| "acc,none": 0.68, |
| "acc_stderr,none": 0.014758652303574938 |
| }, |
| "blimp_drop_argument": { |
| "name": "blimp_drop_argument", |
| "alias": "blimp_drop_argument", |
| "sample_len": 1000, |
| "acc,none": 0.772, |
| "acc_stderr,none": 0.013273740700804434 |
| }, |
| "blimp_ellipsis_n_bar_1": { |
| "name": "blimp_ellipsis_n_bar_1", |
| "alias": "blimp_ellipsis_n_bar_1", |
| "sample_len": 1000, |
| "acc,none": 0.799, |
| "acc_stderr,none": 0.012679107214617274 |
| }, |
| "blimp_ellipsis_n_bar_2": { |
| "name": "blimp_ellipsis_n_bar_2", |
| "alias": "blimp_ellipsis_n_bar_2", |
| "sample_len": 1000, |
| "acc,none": 0.804, |
| "acc_stderr,none": 0.012559527926707347 |
| }, |
| "blimp_existential_there_object_raising": { |
| "name": "blimp_existential_there_object_raising", |
| "alias": "blimp_existential_there_object_raising", |
| "sample_len": 1000, |
| "acc,none": 0.775, |
| "acc_stderr,none": 0.013211720158614833 |
| }, |
| "blimp_existential_there_quantifiers_1": { |
| "name": "blimp_existential_there_quantifiers_1", |
| "alias": "blimp_existential_there_quantifiers_1", |
| "sample_len": 1000, |
| "acc,none": 0.981, |
| "acc_stderr,none": 0.004319451082910613 |
| }, |
| "blimp_existential_there_quantifiers_2": { |
| "name": "blimp_existential_there_quantifiers_2", |
| "alias": "blimp_existential_there_quantifiers_2", |
| "sample_len": 1000, |
| "acc,none": 0.318, |
| "acc_stderr,none": 0.014734079309311901 |
| }, |
| "blimp_existential_there_subject_raising": { |
| "name": "blimp_existential_there_subject_raising", |
| "alias": "blimp_existential_there_subject_raising", |
| "sample_len": 1000, |
| "acc,none": 0.865, |
| "acc_stderr,none": 0.010811655372416006 |
| }, |
| "blimp_expletive_it_object_raising": { |
| "name": "blimp_expletive_it_object_raising", |
| "alias": "blimp_expletive_it_object_raising", |
| "sample_len": 1000, |
| "acc,none": 0.714, |
| "acc_stderr,none": 0.014297146862518003 |
| }, |
| "blimp_inchoative": { |
| "name": "blimp_inchoative", |
| "alias": "blimp_inchoative", |
| "sample_len": 1000, |
| "acc,none": 0.717, |
| "acc_stderr,none": 0.01425181090648182 |
| }, |
| "blimp_intransitive": { |
| "name": "blimp_intransitive", |
| "alias": "blimp_intransitive", |
| "sample_len": 1000, |
| "acc,none": 0.787, |
| "acc_stderr,none": 0.012953717566737312 |
| }, |
| "blimp_irregular_past_participle_adjectives": { |
| "name": "blimp_irregular_past_participle_adjectives", |
| "alias": "blimp_irregular_past_participle_adjectives", |
| "sample_len": 1000, |
| "acc,none": 0.861, |
| "acc_stderr,none": 0.010945263761042892 |
| }, |
| "blimp_irregular_past_participle_verbs": { |
| "name": "blimp_irregular_past_participle_verbs", |
| "alias": "blimp_irregular_past_participle_verbs", |
| "sample_len": 1000, |
| "acc,none": 0.776, |
| "acc_stderr,none": 0.013190830072364589 |
| }, |
| "blimp_irregular_plural_subject_verb_agreement_1": { |
| "name": "blimp_irregular_plural_subject_verb_agreement_1", |
| "alias": "blimp_irregular_plural_subject_verb_agreement_1", |
| "sample_len": 1000, |
| "acc,none": 0.915, |
| "acc_stderr,none": 0.008823426366942316 |
| }, |
| "blimp_irregular_plural_subject_verb_agreement_2": { |
| "name": "blimp_irregular_plural_subject_verb_agreement_2", |
| "alias": "blimp_irregular_plural_subject_verb_agreement_2", |
| "sample_len": 1000, |
| "acc,none": 0.894, |
| "acc_stderr,none": 0.00973955126578524 |
| }, |
| "blimp_left_branch_island_echo_question": { |
| "name": "blimp_left_branch_island_echo_question", |
| "alias": "blimp_left_branch_island_echo_question", |
| "sample_len": 1000, |
| "acc,none": 0.498, |
| "acc_stderr,none": 0.01581917337430266 |
| }, |
| "blimp_left_branch_island_simple_question": { |
| "name": "blimp_left_branch_island_simple_question", |
| "alias": "blimp_left_branch_island_simple_question", |
| "sample_len": 1000, |
| "acc,none": 0.62, |
| "acc_stderr,none": 0.015356947477797658 |
| }, |
| "blimp_matrix_question_npi_licensor_present": { |
| "name": "blimp_matrix_question_npi_licensor_present", |
| "alias": "blimp_matrix_question_npi_licensor_present", |
| "sample_len": 1000, |
| "acc,none": 0.595, |
| "acc_stderr,none": 0.01553113699045296 |
| }, |
| "blimp_npi_present_1": { |
| "name": "blimp_npi_present_1", |
| "alias": "blimp_npi_present_1", |
| "sample_len": 1000, |
| "acc,none": 0.602, |
| "acc_stderr,none": 0.015486634102859016 |
| }, |
| "blimp_npi_present_2": { |
| "name": "blimp_npi_present_2", |
| "alias": "blimp_npi_present_2", |
| "sample_len": 1000, |
| "acc,none": 0.597, |
| "acc_stderr,none": 0.015518757419066512 |
| }, |
| "blimp_only_npi_licensor_present": { |
| "name": "blimp_only_npi_licensor_present", |
| "alias": "blimp_only_npi_licensor_present", |
| "sample_len": 1000, |
| "acc,none": 0.996, |
| "acc_stderr,none": 0.001996994739098726 |
| }, |
| "blimp_only_npi_scope": { |
| "name": "blimp_only_npi_scope", |
| "alias": "blimp_only_npi_scope", |
| "sample_len": 1000, |
| "acc,none": 0.706, |
| "acc_stderr,none": 0.014414290540008265 |
| }, |
| "blimp_passive_1": { |
| "name": "blimp_passive_1", |
| "alias": "blimp_passive_1", |
| "sample_len": 1000, |
| "acc,none": 0.892, |
| "acc_stderr,none": 0.009820001651345708 |
| }, |
| "blimp_passive_2": { |
| "name": "blimp_passive_2", |
| "alias": "blimp_passive_2", |
| "sample_len": 1000, |
| "acc,none": 0.88, |
| "acc_stderr,none": 0.010281328012747462 |
| }, |
| "blimp_principle_A_c_command": { |
| "name": "blimp_principle_A_c_command", |
| "alias": "blimp_principle_A_c_command", |
| "sample_len": 1000, |
| "acc,none": 0.625, |
| "acc_stderr,none": 0.015316971293620996 |
| }, |
| "blimp_principle_A_case_1": { |
| "name": "blimp_principle_A_case_1", |
| "alias": "blimp_principle_A_case_1", |
| "sample_len": 1000, |
| "acc,none": 1.0, |
| "acc_stderr,none": 0.0 |
| }, |
| "blimp_principle_A_case_2": { |
| "name": "blimp_principle_A_case_2", |
| "alias": "blimp_principle_A_case_2", |
| "sample_len": 1000, |
| "acc,none": 0.953, |
| "acc_stderr,none": 0.006695956678163013 |
| }, |
| "blimp_principle_A_domain_1": { |
| "name": "blimp_principle_A_domain_1", |
| "alias": "blimp_principle_A_domain_1", |
| "sample_len": 1000, |
| "acc,none": 0.9, |
| "acc_stderr,none": 0.00949157995752499 |
| }, |
| "blimp_principle_A_domain_2": { |
| "name": "blimp_principle_A_domain_2", |
| "alias": "blimp_principle_A_domain_2", |
| "sample_len": 1000, |
| "acc,none": 0.757, |
| "acc_stderr,none": 0.013569640199177543 |
| }, |
| "blimp_principle_A_domain_3": { |
| "name": "blimp_principle_A_domain_3", |
| "alias": "blimp_principle_A_domain_3", |
| "sample_len": 1000, |
| "acc,none": 0.632, |
| "acc_stderr,none": 0.015258073561521743 |
| }, |
| "blimp_principle_A_reconstruction": { |
| "name": "blimp_principle_A_reconstruction", |
| "alias": "blimp_principle_A_reconstruction", |
| "sample_len": 1000, |
| "acc,none": 0.386, |
| "acc_stderr,none": 0.015402637476784335 |
| }, |
| "blimp_regular_plural_subject_verb_agreement_1": { |
| "name": "blimp_regular_plural_subject_verb_agreement_1", |
| "alias": "blimp_regular_plural_subject_verb_agreement_1", |
| "sample_len": 1000, |
| "acc,none": 0.893, |
| "acc_stderr,none": 0.009779910359847203 |
| }, |
| "blimp_regular_plural_subject_verb_agreement_2": { |
| "name": "blimp_regular_plural_subject_verb_agreement_2", |
| "alias": "blimp_regular_plural_subject_verb_agreement_2", |
| "sample_len": 1000, |
| "acc,none": 0.893, |
| "acc_stderr,none": 0.009779910359847203 |
| }, |
| "blimp_sentential_negation_npi_licensor_present": { |
| "name": "blimp_sentential_negation_npi_licensor_present", |
| "alias": "blimp_sentential_negation_npi_licensor_present", |
| "sample_len": 1000, |
| "acc,none": 0.99, |
| "acc_stderr,none": 0.003148000938676784 |
| }, |
| "blimp_sentential_negation_npi_scope": { |
| "name": "blimp_sentential_negation_npi_scope", |
| "alias": "blimp_sentential_negation_npi_scope", |
| "sample_len": 1000, |
| "acc,none": 0.682, |
| "acc_stderr,none": 0.014734079309311903 |
| }, |
| "blimp_sentential_subject_island": { |
| "name": "blimp_sentential_subject_island", |
| "alias": "blimp_sentential_subject_island", |
| "sample_len": 1000, |
| "acc,none": 0.538, |
| "acc_stderr,none": 0.015773547629015002 |
| }, |
| "blimp_superlative_quantifiers_1": { |
| "name": "blimp_superlative_quantifiers_1", |
| "alias": "blimp_superlative_quantifiers_1", |
| "sample_len": 1000, |
| "acc,none": 0.871, |
| "acc_stderr,none": 0.010605256784796466 |
| }, |
| "blimp_superlative_quantifiers_2": { |
| "name": "blimp_superlative_quantifiers_2", |
| "alias": "blimp_superlative_quantifiers_2", |
| "sample_len": 1000, |
| "acc,none": 0.659, |
| "acc_stderr,none": 0.014998131348402837 |
| }, |
| "blimp_tough_vs_raising_1": { |
| "name": "blimp_tough_vs_raising_1", |
| "alias": "blimp_tough_vs_raising_1", |
| "sample_len": 1000, |
| "acc,none": 0.661, |
| "acc_stderr,none": 0.014976758771620224 |
| }, |
| "blimp_tough_vs_raising_2": { |
| "name": "blimp_tough_vs_raising_2", |
| "alias": "blimp_tough_vs_raising_2", |
| "sample_len": 1000, |
| "acc,none": 0.869, |
| "acc_stderr,none": 0.0106748748448379 |
| }, |
| "blimp_transitive": { |
| "name": "blimp_transitive", |
| "alias": "blimp_transitive", |
| "sample_len": 1000, |
| "acc,none": 0.81, |
| "acc_stderr,none": 0.012411851354816254 |
| }, |
| "blimp_wh_island": { |
| "name": "blimp_wh_island", |
| "alias": "blimp_wh_island", |
| "sample_len": 1000, |
| "acc,none": 0.782, |
| "acc_stderr,none": 0.013063179040595235 |
| }, |
| "blimp_wh_questions_object_gap": { |
| "name": "blimp_wh_questions_object_gap", |
| "alias": "blimp_wh_questions_object_gap", |
| "sample_len": 1000, |
| "acc,none": 0.809, |
| "acc_stderr,none": 0.012436787112179482 |
| }, |
| "blimp_wh_questions_subject_gap": { |
| "name": "blimp_wh_questions_subject_gap", |
| "alias": "blimp_wh_questions_subject_gap", |
| "sample_len": 1000, |
| "acc,none": 0.943, |
| "acc_stderr,none": 0.007335175853706905 |
| }, |
| "blimp_wh_questions_subject_gap_long_distance": { |
| "name": "blimp_wh_questions_subject_gap_long_distance", |
| "alias": "blimp_wh_questions_subject_gap_long_distance", |
| "sample_len": 1000, |
| "acc,none": 0.857, |
| "acc_stderr,none": 0.011075814808567074 |
| }, |
| "blimp_wh_vs_that_no_gap": { |
| "name": "blimp_wh_vs_that_no_gap", |
| "alias": "blimp_wh_vs_that_no_gap", |
| "sample_len": 1000, |
| "acc,none": 0.98, |
| "acc_stderr,none": 0.004429403980178329 |
| }, |
| "blimp_wh_vs_that_no_gap_long_distance": { |
| "name": "blimp_wh_vs_that_no_gap_long_distance", |
| "alias": "blimp_wh_vs_that_no_gap_long_distance", |
| "sample_len": 1000, |
| "acc,none": 0.974, |
| "acc_stderr,none": 0.005034813735318254 |
| }, |
| "blimp_wh_vs_that_with_gap": { |
| "name": "blimp_wh_vs_that_with_gap", |
| "alias": "blimp_wh_vs_that_with_gap", |
| "sample_len": 1000, |
| "acc,none": 0.506, |
| "acc_stderr,none": 0.015818160898606836 |
| }, |
| "blimp_wh_vs_that_with_gap_long_distance": { |
| "name": "blimp_wh_vs_that_with_gap_long_distance", |
| "alias": "blimp_wh_vs_that_with_gap_long_distance", |
| "sample_len": 1000, |
| "acc,none": 0.274, |
| "acc_stderr,none": 0.01411109928825969 |
| }, |
| "blimp": { |
| "alias": "blimp", |
| "name": "blimp", |
| "sample_len": 67000, |
| "acc,none": 0.7824925373134328, |
| "acc_stderr,none": 0.0014535747039117115, |
| "sample_count": { |
| "acc,none": 67000 |
| } |
| } |
| } |
| } |
