{
  "model_id": "m14",
  "checkpoint": "results/m14/final.pt",
  "trained_step": 15258,
  "num_fewshot": 0,
  "limit": null,
  "results": {
    "lambada_openai": {
      "name": "lambada_openai",
      "alias": "lambada_openai",
      "sample_len": 5153,
      "perplexity,none": 404.5905237231641,
      "perplexity_stderr,none": 21.19147790393626,
      "acc,none": 0.18105957694546865,
      "acc_stderr,none": 0.005364743691197477
    },
    "hellaswag": {
      "name": "hellaswag",
      "alias": "hellaswag",
      "sample_len": 10042,
      "acc,none": 0.2801234813782115,
      "acc_stderr,none": 0.004481416180499928,
      "acc_norm,none": 0.293168691495718,
      "acc_norm_stderr,none": 0.0045428479356060215
    },
    "piqa": {
      "name": "piqa",
      "alias": "piqa",
      "sample_len": 1838,
      "acc,none": 0.6093579978237215,
      "acc_stderr,none": 0.011383377760053567,
      "acc_norm,none": 0.6066376496191512,
      "acc_norm_stderr,none": 0.011397418546689315
    },
    "sciq": {
      "name": "sciq",
      "alias": "sciq",
      "sample_len": 1000,
      "acc,none": 0.685,
      "acc_stderr,none": 0.014696631960792617,
      "acc_norm,none": 0.594,
      "acc_norm_stderr,none": 0.01553722643863458
    },
    "arc_easy": {
      "name": "arc_easy",
      "alias": "arc_easy",
      "sample_len": 2376,
      "acc,none": 0.4654882154882155,
      "acc_stderr,none": 0.010235314238969277,
      "acc_norm,none": 0.4313973063973064,
      "acc_norm_stderr,none": 0.010162752847747605
    },
    "winogrande": {
      "name": "winogrande",
      "alias": "winogrande",
      "sample_len": 1267,
      "acc,none": 0.5122336227308603,
      "acc_stderr,none": 0.014048278820405729
    },
    "blimp_adjunct_island": {
      "name": "blimp_adjunct_island",
      "alias": "blimp_adjunct_island",
      "sample_len": 1000,
      "acc,none": 0.804,
      "acc_stderr,none": 0.012559527926707347
    },
    "blimp_anaphor_gender_agreement": {
      "name": "blimp_anaphor_gender_agreement",
      "alias": "blimp_anaphor_gender_agreement",
      "sample_len": 1000,
      "acc,none": 0.962,
      "acc_stderr,none": 0.0060491811505849775
    },
    "blimp_anaphor_number_agreement": {
      "name": "blimp_anaphor_number_agreement",
      "alias": "blimp_anaphor_number_agreement",
      "sample_len": 1000,
      "acc,none": 0.984,
      "acc_stderr,none": 0.003969856390319398
    },
    "blimp_animate_subject_passive": {
      "name": "blimp_animate_subject_passive",
      "alias": "blimp_animate_subject_passive",
      "sample_len": 1000,
      "acc,none": 0.719,
      "acc_stderr,none": 0.01422115470843494
    },
    "blimp_animate_subject_trans": {
      "name": "blimp_animate_subject_trans",
      "alias": "blimp_animate_subject_trans",
      "sample_len": 1000,
      "acc,none": 0.907,
      "acc_stderr,none": 0.009188875634996669
    },
    "blimp_causative": {
      "name": "blimp_causative",
      "alias": "blimp_causative",
      "sample_len": 1000,
      "acc,none": 0.749,
      "acc_stderr,none": 0.013718133516888775
    },
    "blimp_complex_NP_island": {
      "name": "blimp_complex_NP_island",
      "alias": "blimp_complex_NP_island",
      "sample_len": 1000,
      "acc,none": 0.522,
      "acc_stderr,none": 0.01580397942816194
    },
    "blimp_coordinate_structure_constraint_complex_left_branch": {
      "name": "blimp_coordinate_structure_constraint_complex_left_branch",
      "alias": "blimp_coordinate_structure_constraint_complex_left_branch",
      "sample_len": 1000,
      "acc,none": 0.511,
      "acc_stderr,none": 0.01581547119529257
    },
    "blimp_coordinate_structure_constraint_object_extraction": {
      "name": "blimp_coordinate_structure_constraint_object_extraction",
      "alias": "blimp_coordinate_structure_constraint_object_extraction",
      "sample_len": 1000,
      "acc,none": 0.816,
      "acc_stderr,none": 0.01225945734093864
    },
    "blimp_determiner_noun_agreement_1": {
      "name": "blimp_determiner_noun_agreement_1",
      "alias": "blimp_determiner_noun_agreement_1",
      "sample_len": 1000,
      "acc,none": 0.976,
      "acc_stderr,none": 0.004842256441727084
    },
    "blimp_determiner_noun_agreement_2": {
      "name": "blimp_determiner_noun_agreement_2",
      "alias": "blimp_determiner_noun_agreement_2",
      "sample_len": 1000,
      "acc,none": 0.954,
      "acc_stderr,none": 0.006627814717380679
    },
    "blimp_determiner_noun_agreement_irregular_1": {
      "name": "blimp_determiner_noun_agreement_irregular_1",
      "alias": "blimp_determiner_noun_agreement_irregular_1",
      "sample_len": 1000,
      "acc,none": 0.903,
      "acc_stderr,none": 0.009363689373248062
    },
    "blimp_determiner_noun_agreement_irregular_2": {
      "name": "blimp_determiner_noun_agreement_irregular_2",
      "alias": "blimp_determiner_noun_agreement_irregular_2",
      "sample_len": 1000,
      "acc,none": 0.931,
      "acc_stderr,none": 0.008018934050315124
    },
    "blimp_determiner_noun_agreement_with_adj_2": {
      "name": "blimp_determiner_noun_agreement_with_adj_2",
      "alias": "blimp_determiner_noun_agreement_with_adj_2",
      "sample_len": 1000,
      "acc,none": 0.927,
      "acc_stderr,none": 0.008230354715243976
    },
    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
      "name": "blimp_determiner_noun_agreement_with_adj_irregular_1",
      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1",
      "sample_len": 1000,
      "acc,none": 0.878,
      "acc_stderr,none": 0.010354864712936776
    },
    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
      "name": "blimp_determiner_noun_agreement_with_adj_irregular_2",
      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2",
      "sample_len": 1000,
      "acc,none": 0.891,
      "acc_stderr,none": 0.009859828407037124
    },
    "blimp_determiner_noun_agreement_with_adjective_1": {
      "name": "blimp_determiner_noun_agreement_with_adjective_1",
      "alias": "blimp_determiner_noun_agreement_with_adjective_1",
      "sample_len": 1000,
      "acc,none": 0.934,
      "acc_stderr,none": 0.007855297938697532
    },
    "blimp_distractor_agreement_relational_noun": {
      "name": "blimp_distractor_agreement_relational_noun",
      "alias": "blimp_distractor_agreement_relational_noun",
      "sample_len": 1000,
      "acc,none": 0.757,
      "acc_stderr,none": 0.013569640199177543
    },
    "blimp_distractor_agreement_relative_clause": {
      "name": "blimp_distractor_agreement_relative_clause",
      "alias": "blimp_distractor_agreement_relative_clause",
      "sample_len": 1000,
      "acc,none": 0.617,
      "acc_stderr,none": 0.0153801023256527
    },
    "blimp_drop_argument": {
      "name": "blimp_drop_argument",
      "alias": "blimp_drop_argument",
      "sample_len": 1000,
      "acc,none": 0.771,
      "acc_stderr,none": 0.013294199326613684
    },
    "blimp_ellipsis_n_bar_1": {
      "name": "blimp_ellipsis_n_bar_1",
      "alias": "blimp_ellipsis_n_bar_1",
      "sample_len": 1000,
      "acc,none": 0.801,
      "acc_stderr,none": 0.01263164908309911
    },
    "blimp_ellipsis_n_bar_2": {
      "name": "blimp_ellipsis_n_bar_2",
      "alias": "blimp_ellipsis_n_bar_2",
      "sample_len": 1000,
      "acc,none": 0.838,
      "acc_stderr,none": 0.011657267771304384
    },
    "blimp_existential_there_object_raising": {
      "name": "blimp_existential_there_object_raising",
      "alias": "blimp_existential_there_object_raising",
      "sample_len": 1000,
      "acc,none": 0.774,
      "acc_stderr,none": 0.013232501619085452
    },
    "blimp_existential_there_quantifiers_1": {
      "name": "blimp_existential_there_quantifiers_1",
      "alias": "blimp_existential_there_quantifiers_1",
      "sample_len": 1000,
      "acc,none": 0.97,
      "acc_stderr,none": 0.0053971408290991564
    },
    "blimp_existential_there_quantifiers_2": {
      "name": "blimp_existential_there_quantifiers_2",
      "alias": "blimp_existential_there_quantifiers_2",
      "sample_len": 1000,
      "acc,none": 0.326,
      "acc_stderr,none": 0.014830507204541047
    },
    "blimp_existential_there_subject_raising": {
      "name": "blimp_existential_there_subject_raising",
      "alias": "blimp_existential_there_subject_raising",
      "sample_len": 1000,
      "acc,none": 0.845,
      "acc_stderr,none": 0.011450157470799522
    },
    "blimp_expletive_it_object_raising": {
      "name": "blimp_expletive_it_object_raising",
      "alias": "blimp_expletive_it_object_raising",
      "sample_len": 1000,
      "acc,none": 0.791,
      "acc_stderr,none": 0.012864077288499337
    },
    "blimp_inchoative": {
      "name": "blimp_inchoative",
      "alias": "blimp_inchoative",
      "sample_len": 1000,
      "acc,none": 0.691,
      "acc_stderr,none": 0.014619600977206347
    },
    "blimp_intransitive": {
      "name": "blimp_intransitive",
      "alias": "blimp_intransitive",
      "sample_len": 1000,
      "acc,none": 0.782,
      "acc_stderr,none": 0.013063179040595235
    },
    "blimp_irregular_past_participle_adjectives": {
      "name": "blimp_irregular_past_participle_adjectives",
      "alias": "blimp_irregular_past_participle_adjectives",
      "sample_len": 1000,
      "acc,none": 0.934,
      "acc_stderr,none": 0.007855297938697532
    },
    "blimp_irregular_past_participle_verbs": {
      "name": "blimp_irregular_past_participle_verbs",
      "alias": "blimp_irregular_past_participle_verbs",
      "sample_len": 1000,
      "acc,none": 0.9,
      "acc_stderr,none": 0.00949157995752499
    },
    "blimp_irregular_plural_subject_verb_agreement_1": {
      "name": "blimp_irregular_plural_subject_verb_agreement_1",
      "alias": "blimp_irregular_plural_subject_verb_agreement_1",
      "sample_len": 1000,
      "acc,none": 0.911,
      "acc_stderr,none": 0.009008893392651516
    },
    "blimp_irregular_plural_subject_verb_agreement_2": {
      "name": "blimp_irregular_plural_subject_verb_agreement_2",
      "alias": "blimp_irregular_plural_subject_verb_agreement_2",
      "sample_len": 1000,
      "acc,none": 0.896,
      "acc_stderr,none": 0.009658016218524242
    },
    "blimp_left_branch_island_echo_question": {
      "name": "blimp_left_branch_island_echo_question",
      "alias": "blimp_left_branch_island_echo_question",
      "sample_len": 1000,
      "acc,none": 0.576,
      "acc_stderr,none": 0.015635487471405186
    },
    "blimp_left_branch_island_simple_question": {
      "name": "blimp_left_branch_island_simple_question",
      "alias": "blimp_left_branch_island_simple_question",
      "sample_len": 1000,
      "acc,none": 0.615,
      "acc_stderr,none": 0.015395194445410945
    },
    "blimp_matrix_question_npi_licensor_present": {
      "name": "blimp_matrix_question_npi_licensor_present",
      "alias": "blimp_matrix_question_npi_licensor_present",
      "sample_len": 1000,
      "acc,none": 0.521,
      "acc_stderr,none": 0.015805341148131185
    },
    "blimp_npi_present_1": {
      "name": "blimp_npi_present_1",
      "alias": "blimp_npi_present_1",
      "sample_len": 1000,
      "acc,none": 0.614,
      "acc_stderr,none": 0.015402637476784335
    },
    "blimp_npi_present_2": {
      "name": "blimp_npi_present_2",
      "alias": "blimp_npi_present_2",
      "sample_len": 1000,
      "acc,none": 0.691,
      "acc_stderr,none": 0.014619600977206347
    },
    "blimp_only_npi_licensor_present": {
      "name": "blimp_only_npi_licensor_present",
      "alias": "blimp_only_npi_licensor_present",
      "sample_len": 1000,
      "acc,none": 0.976,
      "acc_stderr,none": 0.004842256441727084
    },
    "blimp_only_npi_scope": {
      "name": "blimp_only_npi_scope",
      "alias": "blimp_only_npi_scope",
      "sample_len": 1000,
      "acc,none": 0.582,
      "acc_stderr,none": 0.015605111967541904
    },
    "blimp_passive_1": {
      "name": "blimp_passive_1",
      "alias": "blimp_passive_1",
      "sample_len": 1000,
      "acc,none": 0.87,
      "acc_stderr,none": 0.010640169792499236
    },
    "blimp_passive_2": {
      "name": "blimp_passive_2",
      "alias": "blimp_passive_2",
      "sample_len": 1000,
      "acc,none": 0.825,
      "acc_stderr,none": 0.012021627157731998
    },
    "blimp_principle_A_c_command": {
      "name": "blimp_principle_A_c_command",
      "alias": "blimp_principle_A_c_command",
      "sample_len": 1000,
      "acc,none": 0.58,
      "acc_stderr,none": 0.015615500115072966
    },
    "blimp_principle_A_case_1": {
      "name": "blimp_principle_A_case_1",
      "alias": "blimp_principle_A_case_1",
      "sample_len": 1000,
      "acc,none": 1.0,
      "acc_stderr,none": 0.0
    },
    "blimp_principle_A_case_2": {
      "name": "blimp_principle_A_case_2",
      "alias": "blimp_principle_A_case_2",
      "sample_len": 1000,
      "acc,none": 0.915,
      "acc_stderr,none": 0.008823426366942316
    },
    "blimp_principle_A_domain_1": {
      "name": "blimp_principle_A_domain_1",
      "alias": "blimp_principle_A_domain_1",
      "sample_len": 1000,
      "acc,none": 0.877,
      "acc_stderr,none": 0.010391293421849803
    },
    "blimp_principle_A_domain_2": {
      "name": "blimp_principle_A_domain_2",
      "alias": "blimp_principle_A_domain_2",
      "sample_len": 1000,
      "acc,none": 0.764,
      "acc_stderr,none": 0.013434451402438602
    },
    "blimp_principle_A_domain_3": {
      "name": "blimp_principle_A_domain_3",
      "alias": "blimp_principle_A_domain_3",
      "sample_len": 1000,
      "acc,none": 0.637,
      "acc_stderr,none": 0.01521389044467136
    },
    "blimp_principle_A_reconstruction": {
      "name": "blimp_principle_A_reconstruction",
      "alias": "blimp_principle_A_reconstruction",
      "sample_len": 1000,
      "acc,none": 0.51,
      "acc_stderr,none": 0.015816135752773183
    },
    "blimp_regular_plural_subject_verb_agreement_1": {
      "name": "blimp_regular_plural_subject_verb_agreement_1",
      "alias": "blimp_regular_plural_subject_verb_agreement_1",
      "sample_len": 1000,
      "acc,none": 0.92,
      "acc_stderr,none": 0.00858333697775368
    },
    "blimp_regular_plural_subject_verb_agreement_2": {
      "name": "blimp_regular_plural_subject_verb_agreement_2",
      "alias": "blimp_regular_plural_subject_verb_agreement_2",
      "sample_len": 1000,
      "acc,none": 0.878,
      "acc_stderr,none": 0.010354864712936776
    },
    "blimp_sentential_negation_npi_licensor_present": {
      "name": "blimp_sentential_negation_npi_licensor_present",
      "alias": "blimp_sentential_negation_npi_licensor_present",
      "sample_len": 1000,
      "acc,none": 0.987,
      "acc_stderr,none": 0.003583830889403662
    },
    "blimp_sentential_negation_npi_scope": {
      "name": "blimp_sentential_negation_npi_scope",
      "alias": "blimp_sentential_negation_npi_scope",
      "sample_len": 1000,
      "acc,none": 0.652,
      "acc_stderr,none": 0.015070604603768328
    },
    "blimp_sentential_subject_island": {
      "name": "blimp_sentential_subject_island",
      "alias": "blimp_sentential_subject_island",
      "sample_len": 1000,
      "acc,none": 0.506,
      "acc_stderr,none": 0.015818160898606836
    },
    "blimp_superlative_quantifiers_1": {
      "name": "blimp_superlative_quantifiers_1",
      "alias": "blimp_superlative_quantifiers_1",
      "sample_len": 1000,
      "acc,none": 0.934,
      "acc_stderr,none": 0.007855297938697532
    },
    "blimp_superlative_quantifiers_2": {
      "name": "blimp_superlative_quantifiers_2",
      "alias": "blimp_superlative_quantifiers_2",
      "sample_len": 1000,
      "acc,none": 0.683,
      "acc_stderr,none": 0.014721675438880174
    },
    "blimp_tough_vs_raising_1": {
      "name": "blimp_tough_vs_raising_1",
      "alias": "blimp_tough_vs_raising_1",
      "sample_len": 1000,
      "acc,none": 0.673,
      "acc_stderr,none": 0.014842213153411162
    },
    "blimp_tough_vs_raising_2": {
      "name": "blimp_tough_vs_raising_2",
      "alias": "blimp_tough_vs_raising_2",
      "sample_len": 1000,
      "acc,none": 0.876,
      "acc_stderr,none": 0.010427498872343873
    },
    "blimp_transitive": {
      "name": "blimp_transitive",
      "alias": "blimp_transitive",
      "sample_len": 1000,
      "acc,none": 0.809,
      "acc_stderr,none": 0.012436787112179482
    },
    "blimp_wh_island": {
      "name": "blimp_wh_island",
      "alias": "blimp_wh_island",
      "sample_len": 1000,
      "acc,none": 0.855,
      "acc_stderr,none": 0.0111399775178902
    },
    "blimp_wh_questions_object_gap": {
      "name": "blimp_wh_questions_object_gap",
      "alias": "blimp_wh_questions_object_gap",
      "sample_len": 1000,
      "acc,none": 0.767,
      "acc_stderr,none": 0.01337497251921999
    },
    "blimp_wh_questions_subject_gap": {
      "name": "blimp_wh_questions_subject_gap",
      "alias": "blimp_wh_questions_subject_gap",
      "sample_len": 1000,
      "acc,none": 0.917,
      "acc_stderr,none": 0.008728527206074756
    },
    "blimp_wh_questions_subject_gap_long_distance": {
      "name": "blimp_wh_questions_subject_gap_long_distance",
      "alias": "blimp_wh_questions_subject_gap_long_distance",
      "sample_len": 1000,
      "acc,none": 0.847,
      "acc_stderr,none": 0.011389500459665504
    },
    "blimp_wh_vs_that_no_gap": {
      "name": "blimp_wh_vs_that_no_gap",
      "alias": "blimp_wh_vs_that_no_gap",
      "sample_len": 1000,
      "acc,none": 0.969,
      "acc_stderr,none": 0.005483527064679181
    },
    "blimp_wh_vs_that_no_gap_long_distance": {
      "name": "blimp_wh_vs_that_no_gap_long_distance",
      "alias": "blimp_wh_vs_that_no_gap_long_distance",
      "sample_len": 1000,
      "acc,none": 0.95,
      "acc_stderr,none": 0.006895472974897918
    },
    "blimp_wh_vs_that_with_gap": {
      "name": "blimp_wh_vs_that_with_gap",
      "alias": "blimp_wh_vs_that_with_gap",
      "sample_len": 1000,
      "acc,none": 0.528,
      "acc_stderr,none": 0.015794475789511517
    },
    "blimp_wh_vs_that_with_gap_long_distance": {
      "name": "blimp_wh_vs_that_with_gap_long_distance",
      "alias": "blimp_wh_vs_that_with_gap_long_distance",
      "sample_len": 1000,
      "acc,none": 0.284,
      "acc_stderr,none": 0.014267009061031241
    },
    "blimp": {
      "alias": "blimp",
      "name": "blimp",
      "sample_len": 67000,
      "acc,none": 0.7844776119402985,
      "acc_stderr,none": 0.0014535278370615976,
      "sample_count": {
        "acc,none": 67000
      }
    }
  }
}