{
  "model_id": "m10",
  "checkpoint": "results/m10/final.pt",
  "trained_step": 15258,
  "num_fewshot": 0,
  "limit": null,
  "results": {
    "lambada_openai": {
      "name": "lambada_openai",
      "alias": "lambada_openai",
      "sample_len": 5153,
      "perplexity,none": 465.64225464950346,
      "perplexity_stderr,none": 24.94192578220801,
      "acc,none": 0.1956142053172909,
      "acc_stderr,none": 0.005526427333044867
    },
    "hellaswag": {
      "name": "hellaswag",
      "alias": "hellaswag",
      "sample_len": 10042,
      "acc,none": 0.2733519219279028,
      "acc_stderr,none": 0.004447691405592456,
      "acc_norm,none": 0.27853017327225654,
      "acc_norm_stderr,none": 0.004473595650807741
    },
    "piqa": {
      "name": "piqa",
      "alias": "piqa",
      "sample_len": 1838,
      "acc,none": 0.5919477693144722,
      "acc_stderr,none": 0.011466872778651143,
      "acc_norm,none": 0.5924918389553863,
      "acc_norm_stderr,none": 0.01146449062565553
    },
    "sciq": {
      "name": "sciq",
      "alias": "sciq",
      "sample_len": 1000,
      "acc,none": 0.722,
      "acc_stderr,none": 0.014174516461485272,
      "acc_norm,none": 0.629,
      "acc_norm_stderr,none": 0.015283736211823096
    },
    "arc_easy": {
      "name": "arc_easy",
      "alias": "arc_easy",
      "sample_len": 2376,
      "acc,none": 0.4654882154882155,
      "acc_stderr,none": 0.010235314238969277,
      "acc_norm,none": 0.4086700336700337,
      "acc_norm_stderr,none": 0.010087174498762844
    },
    "winogrande": {
      "name": "winogrande",
      "alias": "winogrande",
      "sample_len": 1267,
      "acc,none": 0.5272296764009471,
      "acc_stderr,none": 0.01403163162982771
    },
    "blimp_adjunct_island": {
      "name": "blimp_adjunct_island",
      "alias": "blimp_adjunct_island",
      "sample_len": 1000,
      "acc,none": 0.824,
      "acc_stderr,none": 0.012048616898597498
    },
    "blimp_anaphor_gender_agreement": {
      "name": "blimp_anaphor_gender_agreement",
      "alias": "blimp_anaphor_gender_agreement",
      "sample_len": 1000,
      "acc,none": 0.955,
      "acc_stderr,none": 0.006558812241406063
    },
    "blimp_anaphor_number_agreement": {
      "name": "blimp_anaphor_number_agreement",
      "alias": "blimp_anaphor_number_agreement",
      "sample_len": 1000,
      "acc,none": 0.984,
      "acc_stderr,none": 0.003969856390319398
    },
    "blimp_animate_subject_passive": {
      "name": "blimp_animate_subject_passive",
      "alias": "blimp_animate_subject_passive",
      "sample_len": 1000,
      "acc,none": 0.731,
      "acc_stderr,none": 0.014029819522568203
    },
    "blimp_animate_subject_trans": {
      "name": "blimp_animate_subject_trans",
      "alias": "blimp_animate_subject_trans",
      "sample_len": 1000,
      "acc,none": 0.909,
      "acc_stderr,none": 0.009099549538400337
    },
    "blimp_causative": {
      "name": "blimp_causative",
      "alias": "blimp_causative",
      "sample_len": 1000,
      "acc,none": 0.722,
      "acc_stderr,none": 0.014174516461485272
    },
    "blimp_complex_NP_island": {
      "name": "blimp_complex_NP_island",
      "alias": "blimp_complex_NP_island",
      "sample_len": 1000,
      "acc,none": 0.488,
      "acc_stderr,none": 0.015814743314581686
    },
    "blimp_coordinate_structure_constraint_complex_left_branch": {
      "name": "blimp_coordinate_structure_constraint_complex_left_branch",
      "alias": "blimp_coordinate_structure_constraint_complex_left_branch",
      "sample_len": 1000,
      "acc,none": 0.576,
      "acc_stderr,none": 0.015635487471405186
    },
    "blimp_coordinate_structure_constraint_object_extraction": {
      "name": "blimp_coordinate_structure_constraint_object_extraction",
      "alias": "blimp_coordinate_structure_constraint_object_extraction",
      "sample_len": 1000,
      "acc,none": 0.791,
      "acc_stderr,none": 0.012864077288499337
    },
    "blimp_determiner_noun_agreement_1": {
      "name": "blimp_determiner_noun_agreement_1",
      "alias": "blimp_determiner_noun_agreement_1",
      "sample_len": 1000,
      "acc,none": 0.967,
      "acc_stderr,none": 0.005651808820452346
    },
    "blimp_determiner_noun_agreement_2": {
      "name": "blimp_determiner_noun_agreement_2",
      "alias": "blimp_determiner_noun_agreement_2",
      "sample_len": 1000,
      "acc,none": 0.964,
      "acc_stderr,none": 0.00589395781616553
    },
    "blimp_determiner_noun_agreement_irregular_1": {
      "name": "blimp_determiner_noun_agreement_irregular_1",
      "alias": "blimp_determiner_noun_agreement_irregular_1",
      "sample_len": 1000,
      "acc,none": 0.918,
      "acc_stderr,none": 0.008680515615523732
    },
    "blimp_determiner_noun_agreement_irregular_2": {
      "name": "blimp_determiner_noun_agreement_irregular_2",
      "alias": "blimp_determiner_noun_agreement_irregular_2",
      "sample_len": 1000,
      "acc,none": 0.924,
      "acc_stderr,none": 0.008384169266796462
    },
    "blimp_determiner_noun_agreement_with_adj_2": {
      "name": "blimp_determiner_noun_agreement_with_adj_2",
      "alias": "blimp_determiner_noun_agreement_with_adj_2",
      "sample_len": 1000,
      "acc,none": 0.93,
      "acc_stderr,none": 0.008072494358323525
    },
    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
      "name": "blimp_determiner_noun_agreement_with_adj_irregular_1",
      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1",
      "sample_len": 1000,
      "acc,none": 0.891,
      "acc_stderr,none": 0.009859828407037124
    },
    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
      "name": "blimp_determiner_noun_agreement_with_adj_irregular_2",
      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2",
      "sample_len": 1000,
      "acc,none": 0.91,
      "acc_stderr,none": 0.009054390204866477
    },
    "blimp_determiner_noun_agreement_with_adjective_1": {
      "name": "blimp_determiner_noun_agreement_with_adjective_1",
      "alias": "blimp_determiner_noun_agreement_with_adjective_1",
      "sample_len": 1000,
      "acc,none": 0.937,
      "acc_stderr,none": 0.007687007876286461
    },
    "blimp_distractor_agreement_relational_noun": {
      "name": "blimp_distractor_agreement_relational_noun",
      "alias": "blimp_distractor_agreement_relational_noun",
      "sample_len": 1000,
      "acc,none": 0.87,
      "acc_stderr,none": 0.010640169792499236
    },
    "blimp_distractor_agreement_relative_clause": {
      "name": "blimp_distractor_agreement_relative_clause",
      "alias": "blimp_distractor_agreement_relative_clause",
      "sample_len": 1000,
      "acc,none": 0.794,
      "acc_stderr,none": 0.012795613612786522
    },
    "blimp_drop_argument": {
      "name": "blimp_drop_argument",
      "alias": "blimp_drop_argument",
      "sample_len": 1000,
      "acc,none": 0.759,
      "acc_stderr,none": 0.01353152253451555
    },
    "blimp_ellipsis_n_bar_1": {
      "name": "blimp_ellipsis_n_bar_1",
      "alias": "blimp_ellipsis_n_bar_1",
      "sample_len": 1000,
      "acc,none": 0.756,
      "acc_stderr,none": 0.013588548437881386
    },
    "blimp_ellipsis_n_bar_2": {
      "name": "blimp_ellipsis_n_bar_2",
      "alias": "blimp_ellipsis_n_bar_2",
      "sample_len": 1000,
      "acc,none": 0.863,
      "acc_stderr,none": 0.01087884871433321
    },
    "blimp_existential_there_object_raising": {
      "name": "blimp_existential_there_object_raising",
      "alias": "blimp_existential_there_object_raising",
      "sample_len": 1000,
      "acc,none": 0.831,
      "acc_stderr,none": 0.011856625977890152
    },
    "blimp_existential_there_quantifiers_1": {
      "name": "blimp_existential_there_quantifiers_1",
      "alias": "blimp_existential_there_quantifiers_1",
      "sample_len": 1000,
      "acc,none": 0.977,
      "acc_stderr,none": 0.004742730594656784
    },
    "blimp_existential_there_quantifiers_2": {
      "name": "blimp_existential_there_quantifiers_2",
      "alias": "blimp_existential_there_quantifiers_2",
      "sample_len": 1000,
      "acc,none": 0.275,
      "acc_stderr,none": 0.014127086556490498
    },
    "blimp_existential_there_subject_raising": {
      "name": "blimp_existential_there_subject_raising",
      "alias": "blimp_existential_there_subject_raising",
      "sample_len": 1000,
      "acc,none": 0.826,
      "acc_stderr,none": 0.011994493230973447
    },
    "blimp_expletive_it_object_raising": {
      "name": "blimp_expletive_it_object_raising",
      "alias": "blimp_expletive_it_object_raising",
      "sample_len": 1000,
      "acc,none": 0.767,
      "acc_stderr,none": 0.01337497251921999
    },
    "blimp_inchoative": {
      "name": "blimp_inchoative",
      "alias": "blimp_inchoative",
      "sample_len": 1000,
      "acc,none": 0.645,
      "acc_stderr,none": 0.015139491543780598
    },
    "blimp_intransitive": {
      "name": "blimp_intransitive",
      "alias": "blimp_intransitive",
      "sample_len": 1000,
      "acc,none": 0.751,
      "acc_stderr,none": 0.013681600278702275
    },
    "blimp_irregular_past_participle_adjectives": {
      "name": "blimp_irregular_past_participle_adjectives",
      "alias": "blimp_irregular_past_participle_adjectives",
      "sample_len": 1000,
      "acc,none": 0.933,
      "acc_stderr,none": 0.007910345983177575
    },
    "blimp_irregular_past_participle_verbs": {
      "name": "blimp_irregular_past_participle_verbs",
      "alias": "blimp_irregular_past_participle_verbs",
      "sample_len": 1000,
      "acc,none": 0.916,
      "acc_stderr,none": 0.008776162089491113
    },
    "blimp_irregular_plural_subject_verb_agreement_1": {
      "name": "blimp_irregular_plural_subject_verb_agreement_1",
      "alias": "blimp_irregular_plural_subject_verb_agreement_1",
      "sample_len": 1000,
      "acc,none": 0.9,
      "acc_stderr,none": 0.00949157995752499
    },
    "blimp_irregular_plural_subject_verb_agreement_2": {
      "name": "blimp_irregular_plural_subject_verb_agreement_2",
      "alias": "blimp_irregular_plural_subject_verb_agreement_2",
      "sample_len": 1000,
      "acc,none": 0.895,
      "acc_stderr,none": 0.00969892102602496
    },
    "blimp_left_branch_island_echo_question": {
      "name": "blimp_left_branch_island_echo_question",
      "alias": "blimp_left_branch_island_echo_question",
      "sample_len": 1000,
      "acc,none": 0.579,
      "acc_stderr,none": 0.015620595475301287
    },
    "blimp_left_branch_island_simple_question": {
      "name": "blimp_left_branch_island_simple_question",
      "alias": "blimp_left_branch_island_simple_question",
      "sample_len": 1000,
      "acc,none": 0.629,
      "acc_stderr,none": 0.015283736211823096
    },
    "blimp_matrix_question_npi_licensor_present": {
      "name": "blimp_matrix_question_npi_licensor_present",
      "alias": "blimp_matrix_question_npi_licensor_present",
      "sample_len": 1000,
      "acc,none": 0.446,
      "acc_stderr,none": 0.01572677116675039
    },
    "blimp_npi_present_1": {
      "name": "blimp_npi_present_1",
      "alias": "blimp_npi_present_1",
      "sample_len": 1000,
      "acc,none": 0.48,
      "acc_stderr,none": 0.015806639423035177
    },
    "blimp_npi_present_2": {
      "name": "blimp_npi_present_2",
      "alias": "blimp_npi_present_2",
      "sample_len": 1000,
      "acc,none": 0.607,
      "acc_stderr,none": 0.015452824654081525
    },
    "blimp_only_npi_licensor_present": {
      "name": "blimp_only_npi_licensor_present",
      "alias": "blimp_only_npi_licensor_present",
      "sample_len": 1000,
      "acc,none": 0.99,
      "acc_stderr,none": 0.003148000938676784
    },
    "blimp_only_npi_scope": {
      "name": "blimp_only_npi_scope",
      "alias": "blimp_only_npi_scope",
      "sample_len": 1000,
      "acc,none": 0.726,
      "acc_stderr,none": 0.01411109928825969
    },
    "blimp_passive_1": {
      "name": "blimp_passive_1",
      "alias": "blimp_passive_1",
      "sample_len": 1000,
      "acc,none": 0.881,
      "acc_stderr,none": 0.01024421514533661
    },
    "blimp_passive_2": {
      "name": "blimp_passive_2",
      "alias": "blimp_passive_2",
      "sample_len": 1000,
      "acc,none": 0.873,
      "acc_stderr,none": 0.010534798620855644
    },
    "blimp_principle_A_c_command": {
      "name": "blimp_principle_A_c_command",
      "alias": "blimp_principle_A_c_command",
      "sample_len": 1000,
      "acc,none": 0.621,
      "acc_stderr,none": 0.015349091002225332
    },
    "blimp_principle_A_case_1": {
      "name": "blimp_principle_A_case_1",
      "alias": "blimp_principle_A_case_1",
      "sample_len": 1000,
      "acc,none": 1.0,
      "acc_stderr,none": 0.0
    },
    "blimp_principle_A_case_2": {
      "name": "blimp_principle_A_case_2",
      "alias": "blimp_principle_A_case_2",
      "sample_len": 1000,
      "acc,none": 0.932,
      "acc_stderr,none": 0.007964887911291624
    },
    "blimp_principle_A_domain_1": {
      "name": "blimp_principle_A_domain_1",
      "alias": "blimp_principle_A_domain_1",
      "sample_len": 1000,
      "acc,none": 0.953,
      "acc_stderr,none": 0.006695956678163013
    },
    "blimp_principle_A_domain_2": {
      "name": "blimp_principle_A_domain_2",
      "alias": "blimp_principle_A_domain_2",
      "sample_len": 1000,
      "acc,none": 0.661,
      "acc_stderr,none": 0.014976758771620224
    },
    "blimp_principle_A_domain_3": {
      "name": "blimp_principle_A_domain_3",
      "alias": "blimp_principle_A_domain_3",
      "sample_len": 1000,
      "acc,none": 0.608,
      "acc_stderr,none": 0.015445859463771338
    },
    "blimp_principle_A_reconstruction": {
      "name": "blimp_principle_A_reconstruction",
      "alias": "blimp_principle_A_reconstruction",
      "sample_len": 1000,
      "acc,none": 0.519,
      "acc_stderr,none": 0.015807874268505835
    },
    "blimp_regular_plural_subject_verb_agreement_1": {
      "name": "blimp_regular_plural_subject_verb_agreement_1",
      "alias": "blimp_regular_plural_subject_verb_agreement_1",
      "sample_len": 1000,
      "acc,none": 0.934,
      "acc_stderr,none": 0.007855297938697532
    },
    "blimp_regular_plural_subject_verb_agreement_2": {
      "name": "blimp_regular_plural_subject_verb_agreement_2",
      "alias": "blimp_regular_plural_subject_verb_agreement_2",
      "sample_len": 1000,
      "acc,none": 0.888,
      "acc_stderr,none": 0.009977753031397177
    },
    "blimp_sentential_negation_npi_licensor_present": {
      "name": "blimp_sentential_negation_npi_licensor_present",
      "alias": "blimp_sentential_negation_npi_licensor_present",
      "sample_len": 1000,
      "acc,none": 0.988,
      "acc_stderr,none": 0.00344497719409982
    },
    "blimp_sentential_negation_npi_scope": {
      "name": "blimp_sentential_negation_npi_scope",
      "alias": "blimp_sentential_negation_npi_scope",
      "sample_len": 1000,
      "acc,none": 0.504,
      "acc_stderr,none": 0.01581879370351084
    },
    "blimp_sentential_subject_island": {
      "name": "blimp_sentential_subject_island",
      "alias": "blimp_sentential_subject_island",
      "sample_len": 1000,
      "acc,none": 0.458,
      "acc_stderr,none": 0.015763390640483554
    },
    "blimp_superlative_quantifiers_1": {
      "name": "blimp_superlative_quantifiers_1",
      "alias": "blimp_superlative_quantifiers_1",
      "sample_len": 1000,
      "acc,none": 0.741,
      "acc_stderr,none": 0.013860415257527861
    },
    "blimp_superlative_quantifiers_2": {
      "name": "blimp_superlative_quantifiers_2",
      "alias": "blimp_superlative_quantifiers_2",
      "sample_len": 1000,
      "acc,none": 0.806,
      "acc_stderr,none": 0.012510816141264347
    },
    "blimp_tough_vs_raising_1": {
      "name": "blimp_tough_vs_raising_1",
      "alias": "blimp_tough_vs_raising_1",
      "sample_len": 1000,
      "acc,none": 0.559,
      "acc_stderr,none": 0.015708779894242766
    },
    "blimp_tough_vs_raising_2": {
      "name": "blimp_tough_vs_raising_2",
      "alias": "blimp_tough_vs_raising_2",
      "sample_len": 1000,
      "acc,none": 0.904,
      "acc_stderr,none": 0.009320454434783257
    },
    "blimp_transitive": {
      "name": "blimp_transitive",
      "alias": "blimp_transitive",
      "sample_len": 1000,
      "acc,none": 0.838,
      "acc_stderr,none": 0.011657267771304384
    },
    "blimp_wh_island": {
      "name": "blimp_wh_island",
      "alias": "blimp_wh_island",
      "sample_len": 1000,
      "acc,none": 0.817,
      "acc_stderr,none": 0.012233587399477745
    },
    "blimp_wh_questions_object_gap": {
      "name": "blimp_wh_questions_object_gap",
      "alias": "blimp_wh_questions_object_gap",
      "sample_len": 1000,
      "acc,none": 0.745,
      "acc_stderr,none": 0.013790038620872863
    },
    "blimp_wh_questions_subject_gap": {
      "name": "blimp_wh_questions_subject_gap",
      "alias": "blimp_wh_questions_subject_gap",
      "sample_len": 1000,
      "acc,none": 0.908,
      "acc_stderr,none": 0.009144376393151129
    },
    "blimp_wh_questions_subject_gap_long_distance": {
      "name": "blimp_wh_questions_subject_gap_long_distance",
      "alias": "blimp_wh_questions_subject_gap_long_distance",
      "sample_len": 1000,
      "acc,none": 0.863,
      "acc_stderr,none": 0.01087884871433321
    },
    "blimp_wh_vs_that_no_gap": {
      "name": "blimp_wh_vs_that_no_gap",
      "alias": "blimp_wh_vs_that_no_gap",
      "sample_len": 1000,
      "acc,none": 0.963,
      "acc_stderr,none": 0.0059721576223896395
    },
    "blimp_wh_vs_that_no_gap_long_distance": {
      "name": "blimp_wh_vs_that_no_gap_long_distance",
      "alias": "blimp_wh_vs_that_no_gap_long_distance",
      "sample_len": 1000,
      "acc,none": 0.983,
      "acc_stderr,none": 0.004089954489689063
    },
    "blimp_wh_vs_that_with_gap": {
      "name": "blimp_wh_vs_that_with_gap",
      "alias": "blimp_wh_vs_that_with_gap",
      "sample_len": 1000,
      "acc,none": 0.522,
      "acc_stderr,none": 0.01580397942816194
    },
    "blimp_wh_vs_that_with_gap_long_distance": {
      "name": "blimp_wh_vs_that_with_gap_long_distance",
      "alias": "blimp_wh_vs_that_with_gap_long_distance",
      "sample_len": 1000,
      "acc,none": 0.203,
      "acc_stderr,none": 0.01272607374459825
    },
    "blimp": {
      "alias": "blimp",
      "name": "blimp",
      "sample_len": 67000,
      "acc,none": 0.7807164179104478,
      "acc_stderr,none": 0.001434701056616203,
      "sample_count": {
        "acc,none": 67000
      }
    }
  }
}