| { |
| "model_id": "m15", |
| "checkpoint": "results/m15/final.pt", |
| "trained_step": 15258, |
| "num_fewshot": 0, |
| "limit": null, |
| "results": { |
| "lambada_openai": { |
| "name": "lambada_openai", |
| "alias": "lambada_openai", |
| "sample_len": 5153, |
| "perplexity,none": 413.4500055271363, |
| "perplexity_stderr,none": 23.383529548584082, |
| "acc,none": 0.19483795847079372, |
| "acc_stderr,none": 0.005518111913121999 |
| }, |
| "hellaswag": { |
| "name": "hellaswag", |
| "alias": "hellaswag", |
| "sample_len": 10042, |
| "acc,none": 0.2808205536745668, |
| "acc_stderr,none": 0.004484815647064684, |
| "acc_norm,none": 0.2944632543318064, |
| "acc_norm_stderr,none": 0.004548695749620993 |
| }, |
| "piqa": { |
| "name": "piqa", |
| "alias": "piqa", |
| "sample_len": 1838, |
| "acc,none": 0.6088139281828074, |
| "acc_stderr,none": 0.011386215606728629, |
| "acc_norm,none": 0.6077257889009793, |
| "acc_norm_stderr,none": 0.011391846744072072 |
| }, |
| "sciq": { |
| "name": "sciq", |
| "alias": "sciq", |
| "sample_len": 1000, |
| "acc,none": 0.679, |
| "acc_stderr,none": 0.01477082181793475, |
| "acc_norm,none": 0.583, |
| "acc_norm_stderr,none": 0.015599819048769583 |
| }, |
| "arc_easy": { |
| "name": "arc_easy", |
| "alias": "arc_easy", |
| "sample_len": 2376, |
| "acc,none": 0.4591750841750842, |
| "acc_stderr,none": 0.010225526906982679, |
| "acc_norm,none": 0.4078282828282828, |
| "acc_norm_stderr,none": 0.01008395024004126 |
| }, |
| "winogrande": { |
| "name": "winogrande", |
| "alias": "winogrande", |
| "sample_len": 1267, |
| "acc,none": 0.4988161010260458, |
| "acc_stderr,none": 0.014052446290528871 |
| }, |
| "blimp_adjunct_island": { |
| "name": "blimp_adjunct_island", |
| "alias": "blimp_adjunct_island", |
| "sample_len": 1000, |
| "acc,none": 0.836, |
| "acc_stderr,none": 0.011715000693181425 |
| }, |
| "blimp_anaphor_gender_agreement": { |
| "name": "blimp_anaphor_gender_agreement", |
| "alias": "blimp_anaphor_gender_agreement", |
| "sample_len": 1000, |
| "acc,none": 0.862, |
| "acc_stderr,none": 0.010912152632504508 |
| }, |
| "blimp_anaphor_number_agreement": { |
| "name": "blimp_anaphor_number_agreement", |
| "alias": "blimp_anaphor_number_agreement", |
| "sample_len": 1000, |
| "acc,none": 0.966, |
| "acc_stderr,none": 0.0057338361396953854 |
| }, |
| "blimp_animate_subject_passive": { |
| "name": "blimp_animate_subject_passive", |
| "alias": "blimp_animate_subject_passive", |
| "sample_len": 1000, |
| "acc,none": 0.706, |
| "acc_stderr,none": 0.014414290540008265 |
| }, |
| "blimp_animate_subject_trans": { |
| "name": "blimp_animate_subject_trans", |
| "alias": "blimp_animate_subject_trans", |
| "sample_len": 1000, |
| "acc,none": 0.898, |
| "acc_stderr,none": 0.009575368801653944 |
| }, |
| "blimp_causative": { |
| "name": "blimp_causative", |
| "alias": "blimp_causative", |
| "sample_len": 1000, |
| "acc,none": 0.702, |
| "acc_stderr,none": 0.014470846741134585 |
| }, |
| "blimp_complex_NP_island": { |
| "name": "blimp_complex_NP_island", |
| "alias": "blimp_complex_NP_island", |
| "sample_len": 1000, |
| "acc,none": 0.528, |
| "acc_stderr,none": 0.015794475789511517 |
| }, |
| "blimp_coordinate_structure_constraint_complex_left_branch": { |
| "name": "blimp_coordinate_structure_constraint_complex_left_branch", |
| "alias": "blimp_coordinate_structure_constraint_complex_left_branch", |
| "sample_len": 1000, |
| "acc,none": 0.43, |
| "acc_stderr,none": 0.01566350361015521 |
| }, |
| "blimp_coordinate_structure_constraint_object_extraction": { |
| "name": "blimp_coordinate_structure_constraint_object_extraction", |
| "alias": "blimp_coordinate_structure_constraint_object_extraction", |
| "sample_len": 1000, |
| "acc,none": 0.783, |
| "acc_stderr,none": 0.013041513757270706 |
| }, |
| "blimp_determiner_noun_agreement_1": { |
| "name": "blimp_determiner_noun_agreement_1", |
| "alias": "blimp_determiner_noun_agreement_1", |
| "sample_len": 1000, |
| "acc,none": 0.973, |
| "acc_stderr,none": 0.005128089049275305 |
| }, |
| "blimp_determiner_noun_agreement_2": { |
| "name": "blimp_determiner_noun_agreement_2", |
| "alias": "blimp_determiner_noun_agreement_2", |
| "sample_len": 1000, |
| "acc,none": 0.942, |
| "acc_stderr,none": 0.0073953154557929906 |
| }, |
| "blimp_determiner_noun_agreement_irregular_1": { |
| "name": "blimp_determiner_noun_agreement_irregular_1", |
| "alias": "blimp_determiner_noun_agreement_irregular_1", |
| "sample_len": 1000, |
| "acc,none": 0.878, |
| "acc_stderr,none": 0.010354864712936776 |
| }, |
| "blimp_determiner_noun_agreement_irregular_2": { |
| "name": "blimp_determiner_noun_agreement_irregular_2", |
| "alias": "blimp_determiner_noun_agreement_irregular_2", |
| "sample_len": 1000, |
| "acc,none": 0.923, |
| "acc_stderr,none": 0.00843458014024066 |
| }, |
| "blimp_determiner_noun_agreement_with_adj_2": { |
| "name": "blimp_determiner_noun_agreement_with_adj_2", |
| "alias": "blimp_determiner_noun_agreement_with_adj_2", |
| "sample_len": 1000, |
| "acc,none": 0.921, |
| "acc_stderr,none": 0.00853415677333337 |
| }, |
| "blimp_determiner_noun_agreement_with_adj_irregular_1": { |
| "name": "blimp_determiner_noun_agreement_with_adj_irregular_1", |
| "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1", |
| "sample_len": 1000, |
| "acc,none": 0.862, |
| "acc_stderr,none": 0.010912152632504508 |
| }, |
| "blimp_determiner_noun_agreement_with_adj_irregular_2": { |
| "name": "blimp_determiner_noun_agreement_with_adj_irregular_2", |
| "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2", |
| "sample_len": 1000, |
| "acc,none": 0.892, |
| "acc_stderr,none": 0.009820001651345708 |
| }, |
| "blimp_determiner_noun_agreement_with_adjective_1": { |
| "name": "blimp_determiner_noun_agreement_with_adjective_1", |
| "alias": "blimp_determiner_noun_agreement_with_adjective_1", |
| "sample_len": 1000, |
| "acc,none": 0.93, |
| "acc_stderr,none": 0.008072494358323525 |
| }, |
| "blimp_distractor_agreement_relational_noun": { |
| "name": "blimp_distractor_agreement_relational_noun", |
| "alias": "blimp_distractor_agreement_relational_noun", |
| "sample_len": 1000, |
| "acc,none": 0.829, |
| "acc_stderr,none": 0.011912216456264588 |
| }, |
| "blimp_distractor_agreement_relative_clause": { |
| "name": "blimp_distractor_agreement_relative_clause", |
| "alias": "blimp_distractor_agreement_relative_clause", |
| "sample_len": 1000, |
| "acc,none": 0.737, |
| "acc_stderr,none": 0.013929286594259776 |
| }, |
| "blimp_drop_argument": { |
| "name": "blimp_drop_argument", |
| "alias": "blimp_drop_argument", |
| "sample_len": 1000, |
| "acc,none": 0.744, |
| "acc_stderr,none": 0.013807775152234145 |
| }, |
| "blimp_ellipsis_n_bar_1": { |
| "name": "blimp_ellipsis_n_bar_1", |
| "alias": "blimp_ellipsis_n_bar_1", |
| "sample_len": 1000, |
| "acc,none": 0.816, |
| "acc_stderr,none": 0.01225945734093864 |
| }, |
| "blimp_ellipsis_n_bar_2": { |
| "name": "blimp_ellipsis_n_bar_2", |
| "alias": "blimp_ellipsis_n_bar_2", |
| "sample_len": 1000, |
| "acc,none": 0.839, |
| "acc_stderr,none": 0.011628164696727263 |
| }, |
| "blimp_existential_there_object_raising": { |
| "name": "blimp_existential_there_object_raising", |
| "alias": "blimp_existential_there_object_raising", |
| "sample_len": 1000, |
| "acc,none": 0.726, |
| "acc_stderr,none": 0.01411109928825969 |
| }, |
| "blimp_existential_there_quantifiers_1": { |
| "name": "blimp_existential_there_quantifiers_1", |
| "alias": "blimp_existential_there_quantifiers_1", |
| "sample_len": 1000, |
| "acc,none": 0.969, |
| "acc_stderr,none": 0.005483527064679181 |
| }, |
| "blimp_existential_there_quantifiers_2": { |
| "name": "blimp_existential_there_quantifiers_2", |
| "alias": "blimp_existential_there_quantifiers_2", |
| "sample_len": 1000, |
| "acc,none": 0.187, |
| "acc_stderr,none": 0.012336254828074166 |
| }, |
| "blimp_existential_there_subject_raising": { |
| "name": "blimp_existential_there_subject_raising", |
| "alias": "blimp_existential_there_subject_raising", |
| "sample_len": 1000, |
| "acc,none": 0.872, |
| "acc_stderr,none": 0.010570133761108595 |
| }, |
| "blimp_expletive_it_object_raising": { |
| "name": "blimp_expletive_it_object_raising", |
| "alias": "blimp_expletive_it_object_raising", |
| "sample_len": 1000, |
| "acc,none": 0.685, |
| "acc_stderr,none": 0.014696631960792617 |
| }, |
| "blimp_inchoative": { |
| "name": "blimp_inchoative", |
| "alias": "blimp_inchoative", |
| "sample_len": 1000, |
| "acc,none": 0.665, |
| "acc_stderr,none": 0.014933117490932629 |
| }, |
| "blimp_intransitive": { |
| "name": "blimp_intransitive", |
| "alias": "blimp_intransitive", |
| "sample_len": 1000, |
| "acc,none": 0.748, |
| "acc_stderr,none": 0.013736254390651213 |
| }, |
| "blimp_irregular_past_participle_adjectives": { |
| "name": "blimp_irregular_past_participle_adjectives", |
| "alias": "blimp_irregular_past_participle_adjectives", |
| "sample_len": 1000, |
| "acc,none": 0.723, |
| "acc_stderr,none": 0.014158794845306273 |
| }, |
| "blimp_irregular_past_participle_verbs": { |
| "name": "blimp_irregular_past_participle_verbs", |
| "alias": "blimp_irregular_past_participle_verbs", |
| "sample_len": 1000, |
| "acc,none": 0.839, |
| "acc_stderr,none": 0.011628164696727263 |
| }, |
| "blimp_irregular_plural_subject_verb_agreement_1": { |
| "name": "blimp_irregular_plural_subject_verb_agreement_1", |
| "alias": "blimp_irregular_plural_subject_verb_agreement_1", |
| "sample_len": 1000, |
| "acc,none": 0.905, |
| "acc_stderr,none": 0.009276910103103353 |
| }, |
| "blimp_irregular_plural_subject_verb_agreement_2": { |
| "name": "blimp_irregular_plural_subject_verb_agreement_2", |
| "alias": "blimp_irregular_plural_subject_verb_agreement_2", |
| "sample_len": 1000, |
| "acc,none": 0.82, |
| "acc_stderr,none": 0.012155153135512022 |
| }, |
| "blimp_left_branch_island_echo_question": { |
| "name": "blimp_left_branch_island_echo_question", |
| "alias": "blimp_left_branch_island_echo_question", |
| "sample_len": 1000, |
| "acc,none": 0.618, |
| "acc_stderr,none": 0.015372453034968502 |
| }, |
| "blimp_left_branch_island_simple_question": { |
| "name": "blimp_left_branch_island_simple_question", |
| "alias": "blimp_left_branch_island_simple_question", |
| "sample_len": 1000, |
| "acc,none": 0.595, |
| "acc_stderr,none": 0.01553113699045296 |
| }, |
| "blimp_matrix_question_npi_licensor_present": { |
| "name": "blimp_matrix_question_npi_licensor_present", |
| "alias": "blimp_matrix_question_npi_licensor_present", |
| "sample_len": 1000, |
| "acc,none": 0.532, |
| "acc_stderr,none": 0.015786868759359002 |
| }, |
| "blimp_npi_present_1": { |
| "name": "blimp_npi_present_1", |
| "alias": "blimp_npi_present_1", |
| "sample_len": 1000, |
| "acc,none": 0.665, |
| "acc_stderr,none": 0.014933117490932629 |
| }, |
| "blimp_npi_present_2": { |
| "name": "blimp_npi_present_2", |
| "alias": "blimp_npi_present_2", |
| "sample_len": 1000, |
| "acc,none": 0.723, |
| "acc_stderr,none": 0.014158794845306273 |
| }, |
| "blimp_only_npi_licensor_present": { |
| "name": "blimp_only_npi_licensor_present", |
| "alias": "blimp_only_npi_licensor_present", |
| "sample_len": 1000, |
| "acc,none": 0.993, |
| "acc_stderr,none": 0.002637794146243781 |
| }, |
| "blimp_only_npi_scope": { |
| "name": "blimp_only_npi_scope", |
| "alias": "blimp_only_npi_scope", |
| "sample_len": 1000, |
| "acc,none": 0.712, |
| "acc_stderr,none": 0.01432694179723158 |
| }, |
| "blimp_passive_1": { |
| "name": "blimp_passive_1", |
| "alias": "blimp_passive_1", |
| "sample_len": 1000, |
| "acc,none": 0.862, |
| "acc_stderr,none": 0.010912152632504508 |
| }, |
| "blimp_passive_2": { |
| "name": "blimp_passive_2", |
| "alias": "blimp_passive_2", |
| "sample_len": 1000, |
| "acc,none": 0.842, |
| "acc_stderr,none": 0.011539894677559635 |
| }, |
| "blimp_principle_A_c_command": { |
| "name": "blimp_principle_A_c_command", |
| "alias": "blimp_principle_A_c_command", |
| "sample_len": 1000, |
| "acc,none": 0.706, |
| "acc_stderr,none": 0.014414290540008265 |
| }, |
| "blimp_principle_A_case_1": { |
| "name": "blimp_principle_A_case_1", |
| "alias": "blimp_principle_A_case_1", |
| "sample_len": 1000, |
| "acc,none": 0.998, |
| "acc_stderr,none": 0.0014135055705578234 |
| }, |
| "blimp_principle_A_case_2": { |
| "name": "blimp_principle_A_case_2", |
| "alias": "blimp_principle_A_case_2", |
| "sample_len": 1000, |
| "acc,none": 0.908, |
| "acc_stderr,none": 0.009144376393151129 |
| }, |
| "blimp_principle_A_domain_1": { |
| "name": "blimp_principle_A_domain_1", |
| "alias": "blimp_principle_A_domain_1", |
| "sample_len": 1000, |
| "acc,none": 0.905, |
| "acc_stderr,none": 0.009276910103103353 |
| }, |
| "blimp_principle_A_domain_2": { |
| "name": "blimp_principle_A_domain_2", |
| "alias": "blimp_principle_A_domain_2", |
| "sample_len": 1000, |
| "acc,none": 0.704, |
| "acc_stderr,none": 0.01444273494157504 |
| }, |
| "blimp_principle_A_domain_3": { |
| "name": "blimp_principle_A_domain_3", |
| "alias": "blimp_principle_A_domain_3", |
| "sample_len": 1000, |
| "acc,none": 0.647, |
| "acc_stderr,none": 0.015120172605483696 |
| }, |
| "blimp_principle_A_reconstruction": { |
| "name": "blimp_principle_A_reconstruction", |
| "alias": "blimp_principle_A_reconstruction", |
| "sample_len": 1000, |
| "acc,none": 0.425, |
| "acc_stderr,none": 0.01564032031704017 |
| }, |
| "blimp_regular_plural_subject_verb_agreement_1": { |
| "name": "blimp_regular_plural_subject_verb_agreement_1", |
| "alias": "blimp_regular_plural_subject_verb_agreement_1", |
| "sample_len": 1000, |
| "acc,none": 0.864, |
| "acc_stderr,none": 0.01084535023047304 |
| }, |
| "blimp_regular_plural_subject_verb_agreement_2": { |
| "name": "blimp_regular_plural_subject_verb_agreement_2", |
| "alias": "blimp_regular_plural_subject_verb_agreement_2", |
| "sample_len": 1000, |
| "acc,none": 0.869, |
| "acc_stderr,none": 0.0106748748448379 |
| }, |
| "blimp_sentential_negation_npi_licensor_present": { |
| "name": "blimp_sentential_negation_npi_licensor_present", |
| "alias": "blimp_sentential_negation_npi_licensor_present", |
| "sample_len": 1000, |
| "acc,none": 0.985, |
| "acc_stderr,none": 0.003845749574502972 |
| }, |
| "blimp_sentential_negation_npi_scope": { |
| "name": "blimp_sentential_negation_npi_scope", |
| "alias": "blimp_sentential_negation_npi_scope", |
| "sample_len": 1000, |
| "acc,none": 0.632, |
| "acc_stderr,none": 0.015258073561521743 |
| }, |
| "blimp_sentential_subject_island": { |
| "name": "blimp_sentential_subject_island", |
| "alias": "blimp_sentential_subject_island", |
| "sample_len": 1000, |
| "acc,none": 0.528, |
| "acc_stderr,none": 0.015794475789511517 |
| }, |
| "blimp_superlative_quantifiers_1": { |
| "name": "blimp_superlative_quantifiers_1", |
| "alias": "blimp_superlative_quantifiers_1", |
| "sample_len": 1000, |
| "acc,none": 0.832, |
| "acc_stderr,none": 0.011828605831454352 |
| }, |
| "blimp_superlative_quantifiers_2": { |
| "name": "blimp_superlative_quantifiers_2", |
| "alias": "blimp_superlative_quantifiers_2", |
| "sample_len": 1000, |
| "acc,none": 0.663, |
| "acc_stderr,none": 0.01495508791865346 |
| }, |
| "blimp_tough_vs_raising_1": { |
| "name": "blimp_tough_vs_raising_1", |
| "alias": "blimp_tough_vs_raising_1", |
| "sample_len": 1000, |
| "acc,none": 0.66, |
| "acc_stderr,none": 0.014987482264363972 |
| }, |
| "blimp_tough_vs_raising_2": { |
| "name": "blimp_tough_vs_raising_2", |
| "alias": "blimp_tough_vs_raising_2", |
| "sample_len": 1000, |
| "acc,none": 0.848, |
| "acc_stderr,none": 0.01135891830347525 |
| }, |
| "blimp_transitive": { |
| "name": "blimp_transitive", |
| "alias": "blimp_transitive", |
| "sample_len": 1000, |
| "acc,none": 0.797, |
| "acc_stderr,none": 0.01272607374459825 |
| }, |
| "blimp_wh_island": { |
| "name": "blimp_wh_island", |
| "alias": "blimp_wh_island", |
| "sample_len": 1000, |
| "acc,none": 0.685, |
| "acc_stderr,none": 0.014696631960792617 |
| }, |
| "blimp_wh_questions_object_gap": { |
| "name": "blimp_wh_questions_object_gap", |
| "alias": "blimp_wh_questions_object_gap", |
| "sample_len": 1000, |
| "acc,none": 0.795, |
| "acc_stderr,none": 0.012772554096113201 |
| }, |
| "blimp_wh_questions_subject_gap": { |
| "name": "blimp_wh_questions_subject_gap", |
| "alias": "blimp_wh_questions_subject_gap", |
| "sample_len": 1000, |
| "acc,none": 0.931, |
| "acc_stderr,none": 0.008018934050315124 |
| }, |
| "blimp_wh_questions_subject_gap_long_distance": { |
| "name": "blimp_wh_questions_subject_gap_long_distance", |
| "alias": "blimp_wh_questions_subject_gap_long_distance", |
| "sample_len": 1000, |
| "acc,none": 0.828, |
| "acc_stderr,none": 0.011939788882495308 |
| }, |
| "blimp_wh_vs_that_no_gap": { |
| "name": "blimp_wh_vs_that_no_gap", |
| "alias": "blimp_wh_vs_that_no_gap", |
| "sample_len": 1000, |
| "acc,none": 0.968, |
| "acc_stderr,none": 0.005568393575081328 |
| }, |
| "blimp_wh_vs_that_no_gap_long_distance": { |
| "name": "blimp_wh_vs_that_no_gap_long_distance", |
| "alias": "blimp_wh_vs_that_no_gap_long_distance", |
| "sample_len": 1000, |
| "acc,none": 0.969, |
| "acc_stderr,none": 0.005483527064679181 |
| }, |
| "blimp_wh_vs_that_with_gap": { |
| "name": "blimp_wh_vs_that_with_gap", |
| "alias": "blimp_wh_vs_that_with_gap", |
| "sample_len": 1000, |
| "acc,none": 0.486, |
| "acc_stderr,none": 0.01581309754773093 |
| }, |
| "blimp_wh_vs_that_with_gap_long_distance": { |
| "name": "blimp_wh_vs_that_with_gap_long_distance", |
| "alias": "blimp_wh_vs_that_with_gap_long_distance", |
| "sample_len": 1000, |
| "acc,none": 0.258, |
| "acc_stderr,none": 0.013842963108656601 |
| }, |
| "blimp": { |
| "alias": "blimp", |
| "name": "blimp", |
| "sample_len": 67000, |
| "acc,none": 0.7696865671641792, |
| "acc_stderr,none": 0.0014888534910081543, |
| "sample_count": { |
| "acc,none": 67000 |
| } |
| } |
| } |
| } |
