{
  "model_id": "m17",
  "checkpoint": "results/m17/final.pt",
  "trained_step": 15258,
  "num_fewshot": 0,
  "limit": null,
  "results": {
    "lambada_openai": {
      "name": "lambada_openai",
      "alias": "lambada_openai",
      "sample_len": 5153,
      "perplexity,none": 885.3734741187526,
      "perplexity_stderr,none": 47.5428730543591,
      "acc,none": 0.15990685037842034,
      "acc_stderr,none": 0.005106335304761839
    },
    "hellaswag": {
      "name": "hellaswag",
      "alias": "hellaswag",
      "sample_len": 10042,
      "acc,none": 0.2693686516630153,
      "acc_stderr,none": 0.0044272514992366,
      "acc_norm,none": 0.2725552678749253,
      "acc_norm_stderr,none": 0.004443639394177571
    },
    "piqa": {
      "name": "piqa",
      "alias": "piqa",
      "sample_len": 1838,
      "acc,none": 0.5930359085963003,
      "acc_stderr,none": 0.011462093919190149,
      "acc_norm,none": 0.588139281828074,
      "acc_norm_stderr,none": 0.011483141106304523
    },
    "sciq": {
      "name": "sciq",
      "alias": "sciq",
      "sample_len": 1000,
      "acc,none": 0.681,
      "acc_stderr,none": 0.014746404865473363,
      "acc_norm,none": 0.589,
      "acc_norm_stderr,none": 0.01556667341859915
    },
    "arc_easy": {
      "name": "arc_easy",
      "alias": "arc_easy",
      "sample_len": 2376,
      "acc,none": 0.4457070707070707,
      "acc_stderr,none": 0.010199118183322836,
      "acc_norm,none": 0.4010942760942761,
      "acc_norm_stderr,none": 0.010057051106534466
    },
    "winogrande": {
      "name": "winogrande",
      "alias": "winogrande",
      "sample_len": 1267,
      "acc,none": 0.5027624309392266,
      "acc_stderr,none": 0.014052271211616401
    },
    "blimp_adjunct_island": {
      "name": "blimp_adjunct_island",
      "alias": "blimp_adjunct_island",
      "sample_len": 1000,
      "acc,none": 0.71,
      "acc_stderr,none": 0.014356395999905621
    },
    "blimp_anaphor_gender_agreement": {
      "name": "blimp_anaphor_gender_agreement",
      "alias": "blimp_anaphor_gender_agreement",
      "sample_len": 1000,
      "acc,none": 0.911,
      "acc_stderr,none": 0.009008893392651516
    },
    "blimp_anaphor_number_agreement": {
      "name": "blimp_anaphor_number_agreement",
      "alias": "blimp_anaphor_number_agreement",
      "sample_len": 1000,
      "acc,none": 0.98,
      "acc_stderr,none": 0.004429403980178329
    },
    "blimp_animate_subject_passive": {
      "name": "blimp_animate_subject_passive",
      "alias": "blimp_animate_subject_passive",
      "sample_len": 1000,
      "acc,none": 0.713,
      "acc_stderr,none": 0.014312087053810001
    },
    "blimp_animate_subject_trans": {
      "name": "blimp_animate_subject_trans",
      "alias": "blimp_animate_subject_trans",
      "sample_len": 1000,
      "acc,none": 0.902,
      "acc_stderr,none": 0.009406619184621247
    },
    "blimp_causative": {
      "name": "blimp_causative",
      "alias": "blimp_causative",
      "sample_len": 1000,
      "acc,none": 0.684,
      "acc_stderr,none": 0.014709193056057166
    },
    "blimp_complex_NP_island": {
      "name": "blimp_complex_NP_island",
      "alias": "blimp_complex_NP_island",
      "sample_len": 1000,
      "acc,none": 0.497,
      "acc_stderr,none": 0.01581901517924682
    },
    "blimp_coordinate_structure_constraint_complex_left_branch": {
      "name": "blimp_coordinate_structure_constraint_complex_left_branch",
      "alias": "blimp_coordinate_structure_constraint_complex_left_branch",
      "sample_len": 1000,
      "acc,none": 0.397,
      "acc_stderr,none": 0.015480007449307982
    },
    "blimp_coordinate_structure_constraint_object_extraction": {
      "name": "blimp_coordinate_structure_constraint_object_extraction",
      "alias": "blimp_coordinate_structure_constraint_object_extraction",
      "sample_len": 1000,
      "acc,none": 0.774,
      "acc_stderr,none": 0.013232501619085452
    },
    "blimp_determiner_noun_agreement_1": {
      "name": "blimp_determiner_noun_agreement_1",
      "alias": "blimp_determiner_noun_agreement_1",
      "sample_len": 1000,
      "acc,none": 0.974,
      "acc_stderr,none": 0.005034813735318254
    },
    "blimp_determiner_noun_agreement_2": {
      "name": "blimp_determiner_noun_agreement_2",
      "alias": "blimp_determiner_noun_agreement_2",
      "sample_len": 1000,
      "acc,none": 0.947,
      "acc_stderr,none": 0.00708810561724649
    },
    "blimp_determiner_noun_agreement_irregular_1": {
      "name": "blimp_determiner_noun_agreement_irregular_1",
      "alias": "blimp_determiner_noun_agreement_irregular_1",
      "sample_len": 1000,
      "acc,none": 0.882,
      "acc_stderr,none": 0.010206869264381718
    },
    "blimp_determiner_noun_agreement_irregular_2": {
      "name": "blimp_determiner_noun_agreement_irregular_2",
      "alias": "blimp_determiner_noun_agreement_irregular_2",
      "sample_len": 1000,
      "acc,none": 0.897,
      "acc_stderr,none": 0.009616833339695739
    },
    "blimp_determiner_noun_agreement_with_adj_2": {
      "name": "blimp_determiner_noun_agreement_with_adj_2",
      "alias": "blimp_determiner_noun_agreement_with_adj_2",
      "sample_len": 1000,
      "acc,none": 0.918,
      "acc_stderr,none": 0.008680515615523732
    },
    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
      "name": "blimp_determiner_noun_agreement_with_adj_irregular_1",
      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1",
      "sample_len": 1000,
      "acc,none": 0.875,
      "acc_stderr,none": 0.010463483381956722
    },
    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
      "name": "blimp_determiner_noun_agreement_with_adj_irregular_2",
      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2",
      "sample_len": 1000,
      "acc,none": 0.865,
      "acc_stderr,none": 0.010811655372416006
    },
    "blimp_determiner_noun_agreement_with_adjective_1": {
      "name": "blimp_determiner_noun_agreement_with_adjective_1",
      "alias": "blimp_determiner_noun_agreement_with_adjective_1",
      "sample_len": 1000,
      "acc,none": 0.935,
      "acc_stderr,none": 0.007799733061832043
    },
    "blimp_distractor_agreement_relational_noun": {
      "name": "blimp_distractor_agreement_relational_noun",
      "alias": "blimp_distractor_agreement_relational_noun",
      "sample_len": 1000,
      "acc,none": 0.807,
      "acc_stderr,none": 0.012486268734370044
    },
    "blimp_distractor_agreement_relative_clause": {
      "name": "blimp_distractor_agreement_relative_clause",
      "alias": "blimp_distractor_agreement_relative_clause",
      "sample_len": 1000,
      "acc,none": 0.61,
      "acc_stderr,none": 0.01543172505386673
    },
    "blimp_drop_argument": {
      "name": "blimp_drop_argument",
      "alias": "blimp_drop_argument",
      "sample_len": 1000,
      "acc,none": 0.746,
      "acc_stderr,none": 0.013772206565168622
    },
    "blimp_ellipsis_n_bar_1": {
      "name": "blimp_ellipsis_n_bar_1",
      "alias": "blimp_ellipsis_n_bar_1",
      "sample_len": 1000,
      "acc,none": 0.76,
      "acc_stderr,none": 0.013512312258920847
    },
    "blimp_ellipsis_n_bar_2": {
      "name": "blimp_ellipsis_n_bar_2",
      "alias": "blimp_ellipsis_n_bar_2",
      "sample_len": 1000,
      "acc,none": 0.824,
      "acc_stderr,none": 0.012048616898597498
    },
    "blimp_existential_there_object_raising": {
      "name": "blimp_existential_there_object_raising",
      "alias": "blimp_existential_there_object_raising",
      "sample_len": 1000,
      "acc,none": 0.72,
      "acc_stderr,none": 0.014205696104091548
    },
    "blimp_existential_there_quantifiers_1": {
      "name": "blimp_existential_there_quantifiers_1",
      "alias": "blimp_existential_there_quantifiers_1",
      "sample_len": 1000,
      "acc,none": 0.953,
      "acc_stderr,none": 0.006695956678163013
    },
    "blimp_existential_there_quantifiers_2": {
      "name": "blimp_existential_there_quantifiers_2",
      "alias": "blimp_existential_there_quantifiers_2",
      "sample_len": 1000,
      "acc,none": 0.328,
      "acc_stderr,none": 0.01485384248727031
    },
    "blimp_existential_there_subject_raising": {
      "name": "blimp_existential_there_subject_raising",
      "alias": "blimp_existential_there_subject_raising",
      "sample_len": 1000,
      "acc,none": 0.837,
      "acc_stderr,none": 0.011686212712746913
    },
    "blimp_expletive_it_object_raising": {
      "name": "blimp_expletive_it_object_raising",
      "alias": "blimp_expletive_it_object_raising",
      "sample_len": 1000,
      "acc,none": 0.709,
      "acc_stderr,none": 0.014370995982378032
    },
    "blimp_inchoative": {
      "name": "blimp_inchoative",
      "alias": "blimp_inchoative",
      "sample_len": 1000,
      "acc,none": 0.582,
      "acc_stderr,none": 0.015605111967541904
    },
    "blimp_intransitive": {
      "name": "blimp_intransitive",
      "alias": "blimp_intransitive",
      "sample_len": 1000,
      "acc,none": 0.693,
      "acc_stderr,none": 0.01459328489285274
    },
    "blimp_irregular_past_participle_adjectives": {
      "name": "blimp_irregular_past_participle_adjectives",
      "alias": "blimp_irregular_past_participle_adjectives",
      "sample_len": 1000,
      "acc,none": 0.886,
      "acc_stderr,none": 0.010055103435823278
    },
    "blimp_irregular_past_participle_verbs": {
      "name": "blimp_irregular_past_participle_verbs",
      "alias": "blimp_irregular_past_participle_verbs",
      "sample_len": 1000,
      "acc,none": 0.841,
      "acc_stderr,none": 0.011569479368271195
    },
    "blimp_irregular_plural_subject_verb_agreement_1": {
      "name": "blimp_irregular_plural_subject_verb_agreement_1",
      "alias": "blimp_irregular_plural_subject_verb_agreement_1",
      "sample_len": 1000,
      "acc,none": 0.858,
      "acc_stderr,none": 0.011043457699378305
    },
    "blimp_irregular_plural_subject_verb_agreement_2": {
      "name": "blimp_irregular_plural_subject_verb_agreement_2",
      "alias": "blimp_irregular_plural_subject_verb_agreement_2",
      "sample_len": 1000,
      "acc,none": 0.898,
      "acc_stderr,none": 0.009575368801653944
    },
    "blimp_left_branch_island_echo_question": {
      "name": "blimp_left_branch_island_echo_question",
      "alias": "blimp_left_branch_island_echo_question",
      "sample_len": 1000,
      "acc,none": 0.505,
      "acc_stderr,none": 0.015818508944436743
    },
    "blimp_left_branch_island_simple_question": {
      "name": "blimp_left_branch_island_simple_question",
      "alias": "blimp_left_branch_island_simple_question",
      "sample_len": 1000,
      "acc,none": 0.459,
      "acc_stderr,none": 0.015766025737882314
    },
    "blimp_matrix_question_npi_licensor_present": {
      "name": "blimp_matrix_question_npi_licensor_present",
      "alias": "blimp_matrix_question_npi_licensor_present",
      "sample_len": 1000,
      "acc,none": 0.429,
      "acc_stderr,none": 0.015658997547870177
    },
    "blimp_npi_present_1": {
      "name": "blimp_npi_present_1",
      "alias": "blimp_npi_present_1",
      "sample_len": 1000,
      "acc,none": 0.367,
      "acc_stderr,none": 0.015249378464171699
    },
    "blimp_npi_present_2": {
      "name": "blimp_npi_present_2",
      "alias": "blimp_npi_present_2",
      "sample_len": 1000,
      "acc,none": 0.6,
      "acc_stderr,none": 0.015499685165842571
    },
    "blimp_only_npi_licensor_present": {
      "name": "blimp_only_npi_licensor_present",
      "alias": "blimp_only_npi_licensor_present",
      "sample_len": 1000,
      "acc,none": 0.876,
      "acc_stderr,none": 0.010427498872343873
    },
    "blimp_only_npi_scope": {
      "name": "blimp_only_npi_scope",
      "alias": "blimp_only_npi_scope",
      "sample_len": 1000,
      "acc,none": 0.467,
      "acc_stderr,none": 0.015784807891138876
    },
    "blimp_passive_1": {
      "name": "blimp_passive_1",
      "alias": "blimp_passive_1",
      "sample_len": 1000,
      "acc,none": 0.867,
      "acc_stderr,none": 0.010743669132397295
    },
    "blimp_passive_2": {
      "name": "blimp_passive_2",
      "alias": "blimp_passive_2",
      "sample_len": 1000,
      "acc,none": 0.886,
      "acc_stderr,none": 0.010055103435823278
    },
    "blimp_principle_A_c_command": {
      "name": "blimp_principle_A_c_command",
      "alias": "blimp_principle_A_c_command",
      "sample_len": 1000,
      "acc,none": 0.606,
      "acc_stderr,none": 0.015459721957493382
    },
    "blimp_principle_A_case_1": {
      "name": "blimp_principle_A_case_1",
      "alias": "blimp_principle_A_case_1",
      "sample_len": 1000,
      "acc,none": 1.0,
      "acc_stderr,none": 0.0
    },
    "blimp_principle_A_case_2": {
      "name": "blimp_principle_A_case_2",
      "alias": "blimp_principle_A_case_2",
      "sample_len": 1000,
      "acc,none": 0.902,
      "acc_stderr,none": 0.009406619184621247
    },
    "blimp_principle_A_domain_1": {
      "name": "blimp_principle_A_domain_1",
      "alias": "blimp_principle_A_domain_1",
      "sample_len": 1000,
      "acc,none": 0.946,
      "acc_stderr,none": 0.007150883521295473
    },
    "blimp_principle_A_domain_2": {
      "name": "blimp_principle_A_domain_2",
      "alias": "blimp_principle_A_domain_2",
      "sample_len": 1000,
      "acc,none": 0.804,
      "acc_stderr,none": 0.012559527926707347
    },
    "blimp_principle_A_domain_3": {
      "name": "blimp_principle_A_domain_3",
      "alias": "blimp_principle_A_domain_3",
      "sample_len": 1000,
      "acc,none": 0.627,
      "acc_stderr,none": 0.015300493622922927
    },
    "blimp_principle_A_reconstruction": {
      "name": "blimp_principle_A_reconstruction",
      "alias": "blimp_principle_A_reconstruction",
      "sample_len": 1000,
      "acc,none": 0.649,
      "acc_stderr,none": 0.015100563798316508
    },
    "blimp_regular_plural_subject_verb_agreement_1": {
      "name": "blimp_regular_plural_subject_verb_agreement_1",
      "alias": "blimp_regular_plural_subject_verb_agreement_1",
      "sample_len": 1000,
      "acc,none": 0.834,
      "acc_stderr,none": 0.011772110370812133
    },
    "blimp_regular_plural_subject_verb_agreement_2": {
      "name": "blimp_regular_plural_subject_verb_agreement_2",
      "alias": "blimp_regular_plural_subject_verb_agreement_2",
      "sample_len": 1000,
      "acc,none": 0.918,
      "acc_stderr,none": 0.008680515615523732
    },
    "blimp_sentential_negation_npi_licensor_present": {
      "name": "blimp_sentential_negation_npi_licensor_present",
      "alias": "blimp_sentential_negation_npi_licensor_present",
      "sample_len": 1000,
      "acc,none": 0.999,
      "acc_stderr,none": 0.001
    },
    "blimp_sentential_negation_npi_scope": {
      "name": "blimp_sentential_negation_npi_scope",
      "alias": "blimp_sentential_negation_npi_scope",
      "sample_len": 1000,
      "acc,none": 0.556,
      "acc_stderr,none": 0.015719768163402127
    },
    "blimp_sentential_subject_island": {
      "name": "blimp_sentential_subject_island",
      "alias": "blimp_sentential_subject_island",
      "sample_len": 1000,
      "acc,none": 0.446,
      "acc_stderr,none": 0.01572677116675039
    },
    "blimp_superlative_quantifiers_1": {
      "name": "blimp_superlative_quantifiers_1",
      "alias": "blimp_superlative_quantifiers_1",
      "sample_len": 1000,
      "acc,none": 0.853,
      "acc_stderr,none": 0.011203415395160376
    },
    "blimp_superlative_quantifiers_2": {
      "name": "blimp_superlative_quantifiers_2",
      "alias": "blimp_superlative_quantifiers_2",
      "sample_len": 1000,
      "acc,none": 0.839,
      "acc_stderr,none": 0.011628164696727263
    },
    "blimp_tough_vs_raising_1": {
      "name": "blimp_tough_vs_raising_1",
      "alias": "blimp_tough_vs_raising_1",
      "sample_len": 1000,
      "acc,none": 0.551,
      "acc_stderr,none": 0.015736792768752044
    },
    "blimp_tough_vs_raising_2": {
      "name": "blimp_tough_vs_raising_2",
      "alias": "blimp_tough_vs_raising_2",
      "sample_len": 1000,
      "acc,none": 0.868,
      "acc_stderr,none": 0.010709373963527965
    },
    "blimp_transitive": {
      "name": "blimp_transitive",
      "alias": "blimp_transitive",
      "sample_len": 1000,
      "acc,none": 0.803,
      "acc_stderr,none": 0.012583693787968196
    },
    "blimp_wh_island": {
      "name": "blimp_wh_island",
      "alias": "blimp_wh_island",
      "sample_len": 1000,
      "acc,none": 0.607,
      "acc_stderr,none": 0.015452824654081525
    },
    "blimp_wh_questions_object_gap": {
      "name": "blimp_wh_questions_object_gap",
      "alias": "blimp_wh_questions_object_gap",
      "sample_len": 1000,
      "acc,none": 0.731,
      "acc_stderr,none": 0.014029819522568203
    },
    "blimp_wh_questions_subject_gap": {
      "name": "blimp_wh_questions_subject_gap",
      "alias": "blimp_wh_questions_subject_gap",
      "sample_len": 1000,
      "acc,none": 0.904,
      "acc_stderr,none": 0.009320454434783257
    },
    "blimp_wh_questions_subject_gap_long_distance": {
      "name": "blimp_wh_questions_subject_gap_long_distance",
      "alias": "blimp_wh_questions_subject_gap_long_distance",
      "sample_len": 1000,
      "acc,none": 0.888,
      "acc_stderr,none": 0.009977753031397177
    },
    "blimp_wh_vs_that_no_gap": {
      "name": "blimp_wh_vs_that_no_gap",
      "alias": "blimp_wh_vs_that_no_gap",
      "sample_len": 1000,
      "acc,none": 0.959,
      "acc_stderr,none": 0.006273624021118764
    },
    "blimp_wh_vs_that_no_gap_long_distance": {
      "name": "blimp_wh_vs_that_no_gap_long_distance",
      "alias": "blimp_wh_vs_that_no_gap_long_distance",
      "sample_len": 1000,
      "acc,none": 0.984,
      "acc_stderr,none": 0.003969856390319398
    },
    "blimp_wh_vs_that_with_gap": {
      "name": "blimp_wh_vs_that_with_gap",
      "alias": "blimp_wh_vs_that_with_gap",
      "sample_len": 1000,
      "acc,none": 0.431,
      "acc_stderr,none": 0.01566794448817344
    },
    "blimp_wh_vs_that_with_gap_long_distance": {
      "name": "blimp_wh_vs_that_with_gap_long_distance",
      "alias": "blimp_wh_vs_that_with_gap_long_distance",
      "sample_len": 1000,
      "acc,none": 0.144,
      "acc_stderr,none": 0.01110798754893916
    },
    "blimp": {
      "alias": "blimp",
      "name": "blimp",
      "sample_len": 67000,
      "acc,none": 0.7495223880597014,
      "acc_stderr,none": 0.0014977385356857837,
      "sample_count": {
        "acc,none": 67000
      }
    }
  }
}