{
  "model_id": "m13",
  "checkpoint": "results/m13/final.pt",
  "trained_step": 15258,
  "num_fewshot": 0,
  "limit": null,
  "results": {
    "lambada_openai": {
      "name": "lambada_openai",
      "alias": "lambada_openai",
      "sample_len": 5153,
      "perplexity,none": 1062.7518294979589,
      "perplexity_stderr,none": 57.21521745469535,
      "acc,none": 0.15272656704832135,
      "acc_stderr,none": 0.005011654948166229
    },
    "hellaswag": {
      "name": "hellaswag",
      "alias": "hellaswag",
      "sample_len": 10042,
      "acc,none": 0.2678749253136825,
      "acc_stderr,none": 0.004419469983938969,
      "acc_norm,none": 0.2731527584146584,
      "acc_norm_stderr,none": 0.004446680081493255
    },
    "piqa": {
      "name": "piqa",
      "alias": "piqa",
      "sample_len": 1838,
      "acc,none": 0.5903155603917302,
      "acc_stderr,none": 0.011473932007187755,
      "acc_norm,none": 0.5718171926006529,
      "acc_norm_stderr,none": 0.011544859155318712
    },
    "sciq": {
      "name": "sciq",
      "alias": "sciq",
      "sample_len": 1000,
      "acc,none": 0.68,
      "acc_stderr,none": 0.014758652303574938,
      "acc_norm,none": 0.609,
      "acc_norm_stderr,none": 0.015438826294681775
    },
    "arc_easy": {
      "name": "arc_easy",
      "alias": "arc_easy",
      "sample_len": 2376,
      "acc,none": 0.43897306397306396,
      "acc_stderr,none": 0.01018307601297191,
      "acc_norm,none": 0.390993265993266,
      "acc_norm_stderr,none": 0.010012992232540683
    },
    "winogrande": {
      "name": "winogrande",
      "alias": "winogrande",
      "sample_len": 1267,
      "acc,none": 0.4996053670086819,
      "acc_stderr,none": 0.01405248130604961
    },
    "blimp_adjunct_island": {
      "name": "blimp_adjunct_island",
      "alias": "blimp_adjunct_island",
      "sample_len": 1000,
      "acc,none": 0.717,
      "acc_stderr,none": 0.01425181090648182
    },
    "blimp_anaphor_gender_agreement": {
      "name": "blimp_anaphor_gender_agreement",
      "alias": "blimp_anaphor_gender_agreement",
      "sample_len": 1000,
      "acc,none": 0.89,
      "acc_stderr,none": 0.00989939381972442
    },
    "blimp_anaphor_number_agreement": {
      "name": "blimp_anaphor_number_agreement",
      "alias": "blimp_anaphor_number_agreement",
      "sample_len": 1000,
      "acc,none": 0.982,
      "acc_stderr,none": 0.00420638724961144
    },
    "blimp_animate_subject_passive": {
      "name": "blimp_animate_subject_passive",
      "alias": "blimp_animate_subject_passive",
      "sample_len": 1000,
      "acc,none": 0.745,
      "acc_stderr,none": 0.013790038620872863
    },
    "blimp_animate_subject_trans": {
      "name": "blimp_animate_subject_trans",
      "alias": "blimp_animate_subject_trans",
      "sample_len": 1000,
      "acc,none": 0.902,
      "acc_stderr,none": 0.009406619184621247
    },
    "blimp_causative": {
      "name": "blimp_causative",
      "alias": "blimp_causative",
      "sample_len": 1000,
      "acc,none": 0.691,
      "acc_stderr,none": 0.014619600977206347
    },
    "blimp_complex_NP_island": {
      "name": "blimp_complex_NP_island",
      "alias": "blimp_complex_NP_island",
      "sample_len": 1000,
      "acc,none": 0.449,
      "acc_stderr,none": 0.015736792768752044
    },
    "blimp_coordinate_structure_constraint_complex_left_branch": {
      "name": "blimp_coordinate_structure_constraint_complex_left_branch",
      "alias": "blimp_coordinate_structure_constraint_complex_left_branch",
      "sample_len": 1000,
      "acc,none": 0.428,
      "acc_stderr,none": 0.015654426245029267
    },
    "blimp_coordinate_structure_constraint_object_extraction": {
      "name": "blimp_coordinate_structure_constraint_object_extraction",
      "alias": "blimp_coordinate_structure_constraint_object_extraction",
      "sample_len": 1000,
      "acc,none": 0.746,
      "acc_stderr,none": 0.013772206565168622
    },
    "blimp_determiner_noun_agreement_1": {
      "name": "blimp_determiner_noun_agreement_1",
      "alias": "blimp_determiner_noun_agreement_1",
      "sample_len": 1000,
      "acc,none": 0.946,
      "acc_stderr,none": 0.007150883521295473
    },
    "blimp_determiner_noun_agreement_2": {
      "name": "blimp_determiner_noun_agreement_2",
      "alias": "blimp_determiner_noun_agreement_2",
      "sample_len": 1000,
      "acc,none": 0.905,
      "acc_stderr,none": 0.009276910103103353
    },
    "blimp_determiner_noun_agreement_irregular_1": {
      "name": "blimp_determiner_noun_agreement_irregular_1",
      "alias": "blimp_determiner_noun_agreement_irregular_1",
      "sample_len": 1000,
      "acc,none": 0.854,
      "acc_stderr,none": 0.011171786285496551
    },
    "blimp_determiner_noun_agreement_irregular_2": {
      "name": "blimp_determiner_noun_agreement_irregular_2",
      "alias": "blimp_determiner_noun_agreement_irregular_2",
      "sample_len": 1000,
      "acc,none": 0.897,
      "acc_stderr,none": 0.009616833339695739
    },
    "blimp_determiner_noun_agreement_with_adj_2": {
      "name": "blimp_determiner_noun_agreement_with_adj_2",
      "alias": "blimp_determiner_noun_agreement_with_adj_2",
      "sample_len": 1000,
      "acc,none": 0.873,
      "acc_stderr,none": 0.010534798620855644
    },
    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
      "name": "blimp_determiner_noun_agreement_with_adj_irregular_1",
      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1",
      "sample_len": 1000,
      "acc,none": 0.79,
      "acc_stderr,none": 0.012886662332274642
    },
    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
      "name": "blimp_determiner_noun_agreement_with_adj_irregular_2",
      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2",
      "sample_len": 1000,
      "acc,none": 0.839,
      "acc_stderr,none": 0.011628164696727263
    },
    "blimp_determiner_noun_agreement_with_adjective_1": {
      "name": "blimp_determiner_noun_agreement_with_adjective_1",
      "alias": "blimp_determiner_noun_agreement_with_adjective_1",
      "sample_len": 1000,
      "acc,none": 0.891,
      "acc_stderr,none": 0.009859828407037124
    },
    "blimp_distractor_agreement_relational_noun": {
      "name": "blimp_distractor_agreement_relational_noun",
      "alias": "blimp_distractor_agreement_relational_noun",
      "sample_len": 1000,
      "acc,none": 0.834,
      "acc_stderr,none": 0.011772110370812133
    },
    "blimp_distractor_agreement_relative_clause": {
      "name": "blimp_distractor_agreement_relative_clause",
      "alias": "blimp_distractor_agreement_relative_clause",
      "sample_len": 1000,
      "acc,none": 0.666,
      "acc_stderr,none": 0.014922019523732928
    },
    "blimp_drop_argument": {
      "name": "blimp_drop_argument",
      "alias": "blimp_drop_argument",
      "sample_len": 1000,
      "acc,none": 0.732,
      "acc_stderr,none": 0.014013292702729519
    },
    "blimp_ellipsis_n_bar_1": {
      "name": "blimp_ellipsis_n_bar_1",
      "alias": "blimp_ellipsis_n_bar_1",
      "sample_len": 1000,
      "acc,none": 0.718,
      "acc_stderr,none": 0.01423652621529138
    },
    "blimp_ellipsis_n_bar_2": {
      "name": "blimp_ellipsis_n_bar_2",
      "alias": "blimp_ellipsis_n_bar_2",
      "sample_len": 1000,
      "acc,none": 0.82,
      "acc_stderr,none": 0.012155153135512022
    },
    "blimp_existential_there_object_raising": {
      "name": "blimp_existential_there_object_raising",
      "alias": "blimp_existential_there_object_raising",
      "sample_len": 1000,
      "acc,none": 0.792,
      "acc_stderr,none": 0.012841374572097058
    },
    "blimp_existential_there_quantifiers_1": {
      "name": "blimp_existential_there_quantifiers_1",
      "alias": "blimp_existential_there_quantifiers_1",
      "sample_len": 1000,
      "acc,none": 0.969,
      "acc_stderr,none": 0.005483527064679181
    },
    "blimp_existential_there_quantifiers_2": {
      "name": "blimp_existential_there_quantifiers_2",
      "alias": "blimp_existential_there_quantifiers_2",
      "sample_len": 1000,
      "acc,none": 0.244,
      "acc_stderr,none": 0.013588548437881386
    },
    "blimp_existential_there_subject_raising": {
      "name": "blimp_existential_there_subject_raising",
      "alias": "blimp_existential_there_subject_raising",
      "sample_len": 1000,
      "acc,none": 0.862,
      "acc_stderr,none": 0.010912152632504508
    },
    "blimp_expletive_it_object_raising": {
      "name": "blimp_expletive_it_object_raising",
      "alias": "blimp_expletive_it_object_raising",
      "sample_len": 1000,
      "acc,none": 0.717,
      "acc_stderr,none": 0.01425181090648182
    },
    "blimp_inchoative": {
      "name": "blimp_inchoative",
      "alias": "blimp_inchoative",
      "sample_len": 1000,
      "acc,none": 0.604,
      "acc_stderr,none": 0.015473313265859495
    },
    "blimp_intransitive": {
      "name": "blimp_intransitive",
      "alias": "blimp_intransitive",
      "sample_len": 1000,
      "acc,none": 0.75,
      "acc_stderr,none": 0.013699915608779773
    },
    "blimp_irregular_past_participle_adjectives": {
      "name": "blimp_irregular_past_participle_adjectives",
      "alias": "blimp_irregular_past_participle_adjectives",
      "sample_len": 1000,
      "acc,none": 0.833,
      "acc_stderr,none": 0.011800434324644593
    },
    "blimp_irregular_past_participle_verbs": {
      "name": "blimp_irregular_past_participle_verbs",
      "alias": "blimp_irregular_past_participle_verbs",
      "sample_len": 1000,
      "acc,none": 0.878,
      "acc_stderr,none": 0.010354864712936776
    },
    "blimp_irregular_plural_subject_verb_agreement_1": {
      "name": "blimp_irregular_plural_subject_verb_agreement_1",
      "alias": "blimp_irregular_plural_subject_verb_agreement_1",
      "sample_len": 1000,
      "acc,none": 0.881,
      "acc_stderr,none": 0.01024421514533661
    },
    "blimp_irregular_plural_subject_verb_agreement_2": {
      "name": "blimp_irregular_plural_subject_verb_agreement_2",
      "alias": "blimp_irregular_plural_subject_verb_agreement_2",
      "sample_len": 1000,
      "acc,none": 0.902,
      "acc_stderr,none": 0.009406619184621247
    },
    "blimp_left_branch_island_echo_question": {
      "name": "blimp_left_branch_island_echo_question",
      "alias": "blimp_left_branch_island_echo_question",
      "sample_len": 1000,
      "acc,none": 0.549,
      "acc_stderr,none": 0.015743152379585443
    },
    "blimp_left_branch_island_simple_question": {
      "name": "blimp_left_branch_island_simple_question",
      "alias": "blimp_left_branch_island_simple_question",
      "sample_len": 1000,
      "acc,none": 0.467,
      "acc_stderr,none": 0.015784807891138876
    },
    "blimp_matrix_question_npi_licensor_present": {
      "name": "blimp_matrix_question_npi_licensor_present",
      "alias": "blimp_matrix_question_npi_licensor_present",
      "sample_len": 1000,
      "acc,none": 0.35,
      "acc_stderr,none": 0.015090650341444127
    },
    "blimp_npi_present_1": {
      "name": "blimp_npi_present_1",
      "alias": "blimp_npi_present_1",
      "sample_len": 1000,
      "acc,none": 0.428,
      "acc_stderr,none": 0.015654426245029267
    },
    "blimp_npi_present_2": {
      "name": "blimp_npi_present_2",
      "alias": "blimp_npi_present_2",
      "sample_len": 1000,
      "acc,none": 0.488,
      "acc_stderr,none": 0.015814743314581686
    },
    "blimp_only_npi_licensor_present": {
      "name": "blimp_only_npi_licensor_present",
      "alias": "blimp_only_npi_licensor_present",
      "sample_len": 1000,
      "acc,none": 0.984,
      "acc_stderr,none": 0.003969856390319398
    },
    "blimp_only_npi_scope": {
      "name": "blimp_only_npi_scope",
      "alias": "blimp_only_npi_scope",
      "sample_len": 1000,
      "acc,none": 0.712,
      "acc_stderr,none": 0.01432694179723158
    },
    "blimp_passive_1": {
      "name": "blimp_passive_1",
      "alias": "blimp_passive_1",
      "sample_len": 1000,
      "acc,none": 0.864,
      "acc_stderr,none": 0.01084535023047304
    },
    "blimp_passive_2": {
      "name": "blimp_passive_2",
      "alias": "blimp_passive_2",
      "sample_len": 1000,
      "acc,none": 0.855,
      "acc_stderr,none": 0.0111399775178902
    },
    "blimp_principle_A_c_command": {
      "name": "blimp_principle_A_c_command",
      "alias": "blimp_principle_A_c_command",
      "sample_len": 1000,
      "acc,none": 0.592,
      "acc_stderr,none": 0.015549205052920803
    },
    "blimp_principle_A_case_1": {
      "name": "blimp_principle_A_case_1",
      "alias": "blimp_principle_A_case_1",
      "sample_len": 1000,
      "acc,none": 1.0,
      "acc_stderr,none": 0.0
    },
    "blimp_principle_A_case_2": {
      "name": "blimp_principle_A_case_2",
      "alias": "blimp_principle_A_case_2",
      "sample_len": 1000,
      "acc,none": 0.915,
      "acc_stderr,none": 0.008823426366942316
    },
    "blimp_principle_A_domain_1": {
      "name": "blimp_principle_A_domain_1",
      "alias": "blimp_principle_A_domain_1",
      "sample_len": 1000,
      "acc,none": 0.918,
      "acc_stderr,none": 0.008680515615523732
    },
    "blimp_principle_A_domain_2": {
      "name": "blimp_principle_A_domain_2",
      "alias": "blimp_principle_A_domain_2",
      "sample_len": 1000,
      "acc,none": 0.74,
      "acc_stderr,none": 0.013877773329774218
    },
    "blimp_principle_A_domain_3": {
      "name": "blimp_principle_A_domain_3",
      "alias": "blimp_principle_A_domain_3",
      "sample_len": 1000,
      "acc,none": 0.629,
      "acc_stderr,none": 0.015283736211823096
    },
    "blimp_principle_A_reconstruction": {
      "name": "blimp_principle_A_reconstruction",
      "alias": "blimp_principle_A_reconstruction",
      "sample_len": 1000,
      "acc,none": 0.45,
      "acc_stderr,none": 0.015740004693383918
    },
    "blimp_regular_plural_subject_verb_agreement_1": {
      "name": "blimp_regular_plural_subject_verb_agreement_1",
      "alias": "blimp_regular_plural_subject_verb_agreement_1",
      "sample_len": 1000,
      "acc,none": 0.898,
      "acc_stderr,none": 0.009575368801653944
    },
    "blimp_regular_plural_subject_verb_agreement_2": {
      "name": "blimp_regular_plural_subject_verb_agreement_2",
      "alias": "blimp_regular_plural_subject_verb_agreement_2",
      "sample_len": 1000,
      "acc,none": 0.922,
      "acc_stderr,none": 0.00848457353011853
    },
    "blimp_sentential_negation_npi_licensor_present": {
      "name": "blimp_sentential_negation_npi_licensor_present",
      "alias": "blimp_sentential_negation_npi_licensor_present",
      "sample_len": 1000,
      "acc,none": 0.986,
      "acc_stderr,none": 0.003717232548256541
    },
    "blimp_sentential_negation_npi_scope": {
      "name": "blimp_sentential_negation_npi_scope",
      "alias": "blimp_sentential_negation_npi_scope",
      "sample_len": 1000,
      "acc,none": 0.576,
      "acc_stderr,none": 0.015635487471405186
    },
    "blimp_sentential_subject_island": {
      "name": "blimp_sentential_subject_island",
      "alias": "blimp_sentential_subject_island",
      "sample_len": 1000,
      "acc,none": 0.38,
      "acc_stderr,none": 0.015356947477797658
    },
    "blimp_superlative_quantifiers_1": {
      "name": "blimp_superlative_quantifiers_1",
      "alias": "blimp_superlative_quantifiers_1",
      "sample_len": 1000,
      "acc,none": 0.823,
      "acc_stderr,none": 0.012075463420375087
    },
    "blimp_superlative_quantifiers_2": {
      "name": "blimp_superlative_quantifiers_2",
      "alias": "blimp_superlative_quantifiers_2",
      "sample_len": 1000,
      "acc,none": 0.815,
      "acc_stderr,none": 0.012285191326386656
    },
    "blimp_tough_vs_raising_1": {
      "name": "blimp_tough_vs_raising_1",
      "alias": "blimp_tough_vs_raising_1",
      "sample_len": 1000,
      "acc,none": 0.555,
      "acc_stderr,none": 0.015723301886761007
    },
    "blimp_tough_vs_raising_2": {
      "name": "blimp_tough_vs_raising_2",
      "alias": "blimp_tough_vs_raising_2",
      "sample_len": 1000,
      "acc,none": 0.871,
      "acc_stderr,none": 0.010605256784796466
    },
    "blimp_transitive": {
      "name": "blimp_transitive",
      "alias": "blimp_transitive",
      "sample_len": 1000,
      "acc,none": 0.793,
      "acc_stderr,none": 0.012818553557844009
    },
    "blimp_wh_island": {
      "name": "blimp_wh_island",
      "alias": "blimp_wh_island",
      "sample_len": 1000,
      "acc,none": 0.809,
      "acc_stderr,none": 0.012436787112179482
    },
    "blimp_wh_questions_object_gap": {
      "name": "blimp_wh_questions_object_gap",
      "alias": "blimp_wh_questions_object_gap",
      "sample_len": 1000,
      "acc,none": 0.724,
      "acc_stderr,none": 0.01414298497574059
    },
    "blimp_wh_questions_subject_gap": {
      "name": "blimp_wh_questions_subject_gap",
      "alias": "blimp_wh_questions_subject_gap",
      "sample_len": 1000,
      "acc,none": 0.931,
      "acc_stderr,none": 0.008018934050315124
    },
    "blimp_wh_questions_subject_gap_long_distance": {
      "name": "blimp_wh_questions_subject_gap_long_distance",
      "alias": "blimp_wh_questions_subject_gap_long_distance",
      "sample_len": 1000,
      "acc,none": 0.964,
      "acc_stderr,none": 0.00589395781616553
    },
    "blimp_wh_vs_that_no_gap": {
      "name": "blimp_wh_vs_that_no_gap",
      "alias": "blimp_wh_vs_that_no_gap",
      "sample_len": 1000,
      "acc,none": 0.977,
      "acc_stderr,none": 0.004742730594656784
    },
    "blimp_wh_vs_that_no_gap_long_distance": {
      "name": "blimp_wh_vs_that_no_gap_long_distance",
      "alias": "blimp_wh_vs_that_no_gap_long_distance",
      "sample_len": 1000,
      "acc,none": 0.99,
      "acc_stderr,none": 0.003148000938676784
    },
    "blimp_wh_vs_that_with_gap": {
      "name": "blimp_wh_vs_that_with_gap",
      "alias": "blimp_wh_vs_that_with_gap",
      "sample_len": 1000,
      "acc,none": 0.392,
      "acc_stderr,none": 0.015445859463771338
    },
    "blimp_wh_vs_that_with_gap_long_distance": {
      "name": "blimp_wh_vs_that_with_gap_long_distance",
      "alias": "blimp_wh_vs_that_with_gap_long_distance",
      "sample_len": 1000,
      "acc,none": 0.114,
      "acc_stderr,none": 0.010055103435823278
    },
    "blimp": {
      "alias": "blimp",
      "name": "blimp",
      "sample_len": 67000,
      "acc,none": 0.7493731343283582,
      "acc_stderr,none": 0.0014843308972713653,
      "sample_count": {
        "acc,none": 67000
      }
    }
  }
}