{
  "model_id": "m04",
  "checkpoint": "results/m04/final.pt",
  "trained_step": 15258,
  "num_fewshot": 0,
  "limit": null,
  "results": {
    "lambada_openai": {
      "name": "lambada_openai",
      "alias": "lambada_openai",
      "sample_len": 5153,
      "perplexity,none": 887.1387056167104,
      "perplexity_stderr,none": 48.06586665128127,
      "acc,none": 0.15796623326217737,
      "acc_stderr,none": 0.005081114222421306
    },
    "hellaswag": {
      "name": "hellaswag",
      "alias": "hellaswag",
      "sample_len": 10042,
      "acc,none": 0.2724556861183031,
      "acc_stderr,none": 0.0044431316326797,
      "acc_norm,none": 0.27165903206532566,
      "acc_norm_stderr,none": 0.004439059440526545
    },
    "piqa": {
      "name": "piqa",
      "alias": "piqa",
      "sample_len": 1838,
      "acc,none": 0.588683351468988,
      "acc_stderr,none": 0.01148086057719274,
      "acc_norm,none": 0.5908596300326442,
      "acc_norm_stderr,none": 0.011471593460443175
    },
    "sciq": {
      "name": "sciq",
      "alias": "sciq",
      "sample_len": 1000,
      "acc,none": 0.685,
      "acc_stderr,none": 0.014696631960792617,
      "acc_norm,none": 0.605,
      "acc_norm_stderr,none": 0.015466551464829328
    },
    "arc_easy": {
      "name": "arc_easy",
      "alias": "arc_easy",
      "sample_len": 2376,
      "acc,none": 0.4494949494949495,
      "acc_stderr,none": 0.010207308833916014,
      "acc_norm,none": 0.4036195286195286,
      "acc_norm_stderr,none": 0.010067368960348095
    },
    "winogrande": {
      "name": "winogrande",
      "alias": "winogrande",
      "sample_len": 1267,
      "acc,none": 0.5201262825572218,
      "acc_stderr,none": 0.014041096664344348
    },
    "blimp_adjunct_island": {
      "name": "blimp_adjunct_island",
      "alias": "blimp_adjunct_island",
      "sample_len": 1000,
      "acc,none": 0.774,
      "acc_stderr,none": 0.013232501619085452
    },
    "blimp_anaphor_gender_agreement": {
      "name": "blimp_anaphor_gender_agreement",
      "alias": "blimp_anaphor_gender_agreement",
      "sample_len": 1000,
      "acc,none": 0.902,
      "acc_stderr,none": 0.009406619184621247
    },
    "blimp_anaphor_number_agreement": {
      "name": "blimp_anaphor_number_agreement",
      "alias": "blimp_anaphor_number_agreement",
      "sample_len": 1000,
      "acc,none": 0.971,
      "acc_stderr,none": 0.005309160685756948
    },
    "blimp_animate_subject_passive": {
      "name": "blimp_animate_subject_passive",
      "alias": "blimp_animate_subject_passive",
      "sample_len": 1000,
      "acc,none": 0.696,
      "acc_stderr,none": 0.014553205687950535
    },
    "blimp_animate_subject_trans": {
      "name": "blimp_animate_subject_trans",
      "alias": "blimp_animate_subject_trans",
      "sample_len": 1000,
      "acc,none": 0.897,
      "acc_stderr,none": 0.009616833339695739
    },
    "blimp_causative": {
      "name": "blimp_causative",
      "alias": "blimp_causative",
      "sample_len": 1000,
      "acc,none": 0.694,
      "acc_stderr,none": 0.014580006055436898
    },
    "blimp_complex_NP_island": {
      "name": "blimp_complex_NP_island",
      "alias": "blimp_complex_NP_island",
      "sample_len": 1000,
      "acc,none": 0.606,
      "acc_stderr,none": 0.015459721957493382
    },
    "blimp_coordinate_structure_constraint_complex_left_branch": {
      "name": "blimp_coordinate_structure_constraint_complex_left_branch",
      "alias": "blimp_coordinate_structure_constraint_complex_left_branch",
      "sample_len": 1000,
      "acc,none": 0.413,
      "acc_stderr,none": 0.015577986829936455
    },
    "blimp_coordinate_structure_constraint_object_extraction": {
      "name": "blimp_coordinate_structure_constraint_object_extraction",
      "alias": "blimp_coordinate_structure_constraint_object_extraction",
      "sample_len": 1000,
      "acc,none": 0.82,
      "acc_stderr,none": 0.012155153135512022
    },
    "blimp_determiner_noun_agreement_1": {
      "name": "blimp_determiner_noun_agreement_1",
      "alias": "blimp_determiner_noun_agreement_1",
      "sample_len": 1000,
      "acc,none": 0.961,
      "acc_stderr,none": 0.006125072776426131
    },
    "blimp_determiner_noun_agreement_2": {
      "name": "blimp_determiner_noun_agreement_2",
      "alias": "blimp_determiner_noun_agreement_2",
      "sample_len": 1000,
      "acc,none": 0.955,
      "acc_stderr,none": 0.006558812241406063
    },
    "blimp_determiner_noun_agreement_irregular_1": {
      "name": "blimp_determiner_noun_agreement_irregular_1",
      "alias": "blimp_determiner_noun_agreement_irregular_1",
      "sample_len": 1000,
      "acc,none": 0.859,
      "acc_stderr,none": 0.01101091459599248
    },
    "blimp_determiner_noun_agreement_irregular_2": {
      "name": "blimp_determiner_noun_agreement_irregular_2",
      "alias": "blimp_determiner_noun_agreement_irregular_2",
      "sample_len": 1000,
      "acc,none": 0.939,
      "acc_stderr,none": 0.007572076091557476
    },
    "blimp_determiner_noun_agreement_with_adj_2": {
      "name": "blimp_determiner_noun_agreement_with_adj_2",
      "alias": "blimp_determiner_noun_agreement_with_adj_2",
      "sample_len": 1000,
      "acc,none": 0.918,
      "acc_stderr,none": 0.008680515615523732
    },
    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
      "name": "blimp_determiner_noun_agreement_with_adj_irregular_1",
      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1",
      "sample_len": 1000,
      "acc,none": 0.763,
      "acc_stderr,none": 0.013454070462577935
    },
    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
      "name": "blimp_determiner_noun_agreement_with_adj_irregular_2",
      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2",
      "sample_len": 1000,
      "acc,none": 0.878,
      "acc_stderr,none": 0.010354864712936776
    },
    "blimp_determiner_noun_agreement_with_adjective_1": {
      "name": "blimp_determiner_noun_agreement_with_adjective_1",
      "alias": "blimp_determiner_noun_agreement_with_adjective_1",
      "sample_len": 1000,
      "acc,none": 0.905,
      "acc_stderr,none": 0.009276910103103353
    },
    "blimp_distractor_agreement_relational_noun": {
      "name": "blimp_distractor_agreement_relational_noun",
      "alias": "blimp_distractor_agreement_relational_noun",
      "sample_len": 1000,
      "acc,none": 0.847,
      "acc_stderr,none": 0.011389500459665504
    },
    "blimp_distractor_agreement_relative_clause": {
      "name": "blimp_distractor_agreement_relative_clause",
      "alias": "blimp_distractor_agreement_relative_clause",
      "sample_len": 1000,
      "acc,none": 0.726,
      "acc_stderr,none": 0.01411109928825969
    },
    "blimp_drop_argument": {
      "name": "blimp_drop_argument",
      "alias": "blimp_drop_argument",
      "sample_len": 1000,
      "acc,none": 0.722,
      "acc_stderr,none": 0.014174516461485272
    },
    "blimp_ellipsis_n_bar_1": {
      "name": "blimp_ellipsis_n_bar_1",
      "alias": "blimp_ellipsis_n_bar_1",
      "sample_len": 1000,
      "acc,none": 0.775,
      "acc_stderr,none": 0.013211720158614833
    },
    "blimp_ellipsis_n_bar_2": {
      "name": "blimp_ellipsis_n_bar_2",
      "alias": "blimp_ellipsis_n_bar_2",
      "sample_len": 1000,
      "acc,none": 0.825,
      "acc_stderr,none": 0.012021627157731998
    },
    "blimp_existential_there_object_raising": {
      "name": "blimp_existential_there_object_raising",
      "alias": "blimp_existential_there_object_raising",
      "sample_len": 1000,
      "acc,none": 0.742,
      "acc_stderr,none": 0.013842963108656601
    },
    "blimp_existential_there_quantifiers_1": {
      "name": "blimp_existential_there_quantifiers_1",
      "alias": "blimp_existential_there_quantifiers_1",
      "sample_len": 1000,
      "acc,none": 0.965,
      "acc_stderr,none": 0.005814534272734903
    },
    "blimp_existential_there_quantifiers_2": {
      "name": "blimp_existential_there_quantifiers_2",
      "alias": "blimp_existential_there_quantifiers_2",
      "sample_len": 1000,
      "acc,none": 0.187,
      "acc_stderr,none": 0.012336254828074166
    },
    "blimp_existential_there_subject_raising": {
      "name": "blimp_existential_there_subject_raising",
      "alias": "blimp_existential_there_subject_raising",
      "sample_len": 1000,
      "acc,none": 0.869,
      "acc_stderr,none": 0.0106748748448379
    },
    "blimp_expletive_it_object_raising": {
      "name": "blimp_expletive_it_object_raising",
      "alias": "blimp_expletive_it_object_raising",
      "sample_len": 1000,
      "acc,none": 0.727,
      "acc_stderr,none": 0.014095022868717513
    },
    "blimp_inchoative": {
      "name": "blimp_inchoative",
      "alias": "blimp_inchoative",
      "sample_len": 1000,
      "acc,none": 0.654,
      "acc_stderr,none": 0.01505026612756434
    },
    "blimp_intransitive": {
      "name": "blimp_intransitive",
      "alias": "blimp_intransitive",
      "sample_len": 1000,
      "acc,none": 0.736,
      "acc_stderr,none": 0.01394627184944057
    },
    "blimp_irregular_past_participle_adjectives": {
      "name": "blimp_irregular_past_participle_adjectives",
      "alias": "blimp_irregular_past_participle_adjectives",
      "sample_len": 1000,
      "acc,none": 0.856,
      "acc_stderr,none": 0.01110798754893916
    },
    "blimp_irregular_past_participle_verbs": {
      "name": "blimp_irregular_past_participle_verbs",
      "alias": "blimp_irregular_past_participle_verbs",
      "sample_len": 1000,
      "acc,none": 0.895,
      "acc_stderr,none": 0.00969892102602496
    },
    "blimp_irregular_plural_subject_verb_agreement_1": {
      "name": "blimp_irregular_plural_subject_verb_agreement_1",
      "alias": "blimp_irregular_plural_subject_verb_agreement_1",
      "sample_len": 1000,
      "acc,none": 0.867,
      "acc_stderr,none": 0.010743669132397295
    },
    "blimp_irregular_plural_subject_verb_agreement_2": {
      "name": "blimp_irregular_plural_subject_verb_agreement_2",
      "alias": "blimp_irregular_plural_subject_verb_agreement_2",
      "sample_len": 1000,
      "acc,none": 0.875,
      "acc_stderr,none": 0.010463483381956722
    },
    "blimp_left_branch_island_echo_question": {
      "name": "blimp_left_branch_island_echo_question",
      "alias": "blimp_left_branch_island_echo_question",
      "sample_len": 1000,
      "acc,none": 0.588,
      "acc_stderr,none": 0.015572363292015072
    },
    "blimp_left_branch_island_simple_question": {
      "name": "blimp_left_branch_island_simple_question",
      "alias": "blimp_left_branch_island_simple_question",
      "sample_len": 1000,
      "acc,none": 0.535,
      "acc_stderr,none": 0.015780495050030086
    },
    "blimp_matrix_question_npi_licensor_present": {
      "name": "blimp_matrix_question_npi_licensor_present",
      "alias": "blimp_matrix_question_npi_licensor_present",
      "sample_len": 1000,
      "acc,none": 0.491,
      "acc_stderr,none": 0.0158167369950053
    },
    "blimp_npi_present_1": {
      "name": "blimp_npi_present_1",
      "alias": "blimp_npi_present_1",
      "sample_len": 1000,
      "acc,none": 0.59,
      "acc_stderr,none": 0.015560917136921659
    },
    "blimp_npi_present_2": {
      "name": "blimp_npi_present_2",
      "alias": "blimp_npi_present_2",
      "sample_len": 1000,
      "acc,none": 0.66,
      "acc_stderr,none": 0.014987482264363972
    },
    "blimp_only_npi_licensor_present": {
      "name": "blimp_only_npi_licensor_present",
      "alias": "blimp_only_npi_licensor_present",
      "sample_len": 1000,
      "acc,none": 0.988,
      "acc_stderr,none": 0.00344497719409982
    },
    "blimp_only_npi_scope": {
      "name": "blimp_only_npi_scope",
      "alias": "blimp_only_npi_scope",
      "sample_len": 1000,
      "acc,none": 0.78,
      "acc_stderr,none": 0.013106173040661862
    },
    "blimp_passive_1": {
      "name": "blimp_passive_1",
      "alias": "blimp_passive_1",
      "sample_len": 1000,
      "acc,none": 0.873,
      "acc_stderr,none": 0.010534798620855644
    },
    "blimp_passive_2": {
      "name": "blimp_passive_2",
      "alias": "blimp_passive_2",
      "sample_len": 1000,
      "acc,none": 0.872,
      "acc_stderr,none": 0.010570133761108595
    },
    "blimp_principle_A_c_command": {
      "name": "blimp_principle_A_c_command",
      "alias": "blimp_principle_A_c_command",
      "sample_len": 1000,
      "acc,none": 0.557,
      "acc_stderr,none": 0.015716169953204184
    },
    "blimp_principle_A_case_1": {
      "name": "blimp_principle_A_case_1",
      "alias": "blimp_principle_A_case_1",
      "sample_len": 1000,
      "acc,none": 1.0,
      "acc_stderr,none": 0.0
    },
    "blimp_principle_A_case_2": {
      "name": "blimp_principle_A_case_2",
      "alias": "blimp_principle_A_case_2",
      "sample_len": 1000,
      "acc,none": 0.867,
      "acc_stderr,none": 0.010743669132397295
    },
    "blimp_principle_A_domain_1": {
      "name": "blimp_principle_A_domain_1",
      "alias": "blimp_principle_A_domain_1",
      "sample_len": 1000,
      "acc,none": 0.936,
      "acc_stderr,none": 0.007743640226919376
    },
    "blimp_principle_A_domain_2": {
      "name": "blimp_principle_A_domain_2",
      "alias": "blimp_principle_A_domain_2",
      "sample_len": 1000,
      "acc,none": 0.72,
      "acc_stderr,none": 0.014205696104091548
    },
    "blimp_principle_A_domain_3": {
      "name": "blimp_principle_A_domain_3",
      "alias": "blimp_principle_A_domain_3",
      "sample_len": 1000,
      "acc,none": 0.616,
      "acc_stderr,none": 0.015387682761896958
    },
    "blimp_principle_A_reconstruction": {
      "name": "blimp_principle_A_reconstruction",
      "alias": "blimp_principle_A_reconstruction",
      "sample_len": 1000,
      "acc,none": 0.489,
      "acc_stderr,none": 0.01581547119529257
    },
    "blimp_regular_plural_subject_verb_agreement_1": {
      "name": "blimp_regular_plural_subject_verb_agreement_1",
      "alias": "blimp_regular_plural_subject_verb_agreement_1",
      "sample_len": 1000,
      "acc,none": 0.9,
      "acc_stderr,none": 0.00949157995752499
    },
    "blimp_regular_plural_subject_verb_agreement_2": {
      "name": "blimp_regular_plural_subject_verb_agreement_2",
      "alias": "blimp_regular_plural_subject_verb_agreement_2",
      "sample_len": 1000,
      "acc,none": 0.888,
      "acc_stderr,none": 0.009977753031397177
    },
    "blimp_sentential_negation_npi_licensor_present": {
      "name": "blimp_sentential_negation_npi_licensor_present",
      "alias": "blimp_sentential_negation_npi_licensor_present",
      "sample_len": 1000,
      "acc,none": 0.999,
      "acc_stderr,none": 0.001
    },
    "blimp_sentential_negation_npi_scope": {
      "name": "blimp_sentential_negation_npi_scope",
      "alias": "blimp_sentential_negation_npi_scope",
      "sample_len": 1000,
      "acc,none": 0.345,
      "acc_stderr,none": 0.015039986742055365
    },
    "blimp_sentential_subject_island": {
      "name": "blimp_sentential_subject_island",
      "alias": "blimp_sentential_subject_island",
      "sample_len": 1000,
      "acc,none": 0.404,
      "acc_stderr,none": 0.015524980677122437
    },
    "blimp_superlative_quantifiers_1": {
      "name": "blimp_superlative_quantifiers_1",
      "alias": "blimp_superlative_quantifiers_1",
      "sample_len": 1000,
      "acc,none": 0.805,
      "acc_stderr,none": 0.012535235623319256
    },
    "blimp_superlative_quantifiers_2": {
      "name": "blimp_superlative_quantifiers_2",
      "alias": "blimp_superlative_quantifiers_2",
      "sample_len": 1000,
      "acc,none": 0.841,
      "acc_stderr,none": 0.011569479368271195
    },
    "blimp_tough_vs_raising_1": {
      "name": "blimp_tough_vs_raising_1",
      "alias": "blimp_tough_vs_raising_1",
      "sample_len": 1000,
      "acc,none": 0.515,
      "acc_stderr,none": 0.01581217964181488
    },
    "blimp_tough_vs_raising_2": {
      "name": "blimp_tough_vs_raising_2",
      "alias": "blimp_tough_vs_raising_2",
      "sample_len": 1000,
      "acc,none": 0.895,
      "acc_stderr,none": 0.00969892102602496
    },
    "blimp_transitive": {
      "name": "blimp_transitive",
      "alias": "blimp_transitive",
      "sample_len": 1000,
      "acc,none": 0.8,
      "acc_stderr,none": 0.012655439943366653
    },
    "blimp_wh_island": {
      "name": "blimp_wh_island",
      "alias": "blimp_wh_island",
      "sample_len": 1000,
      "acc,none": 0.786,
      "acc_stderr,none": 0.012975838021968847
    },
    "blimp_wh_questions_object_gap": {
      "name": "blimp_wh_questions_object_gap",
      "alias": "blimp_wh_questions_object_gap",
      "sample_len": 1000,
      "acc,none": 0.669,
      "acc_stderr,none": 0.0148882725882038
    },
    "blimp_wh_questions_subject_gap": {
      "name": "blimp_wh_questions_subject_gap",
      "alias": "blimp_wh_questions_subject_gap",
      "sample_len": 1000,
      "acc,none": 0.869,
      "acc_stderr,none": 0.0106748748448379
    },
    "blimp_wh_questions_subject_gap_long_distance": {
      "name": "blimp_wh_questions_subject_gap_long_distance",
      "alias": "blimp_wh_questions_subject_gap_long_distance",
      "sample_len": 1000,
      "acc,none": 0.868,
      "acc_stderr,none": 0.010709373963527965
    },
    "blimp_wh_vs_that_no_gap": {
      "name": "blimp_wh_vs_that_no_gap",
      "alias": "blimp_wh_vs_that_no_gap",
      "sample_len": 1000,
      "acc,none": 0.944,
      "acc_stderr,none": 0.007274401481697097
    },
    "blimp_wh_vs_that_no_gap_long_distance": {
      "name": "blimp_wh_vs_that_no_gap_long_distance",
      "alias": "blimp_wh_vs_that_no_gap_long_distance",
      "sample_len": 1000,
      "acc,none": 0.975,
      "acc_stderr,none": 0.004939574819698493
    },
    "blimp_wh_vs_that_with_gap": {
      "name": "blimp_wh_vs_that_with_gap",
      "alias": "blimp_wh_vs_that_with_gap",
      "sample_len": 1000,
      "acc,none": 0.572,
      "acc_stderr,none": 0.015654426245029267
    },
    "blimp_wh_vs_that_with_gap_long_distance": {
      "name": "blimp_wh_vs_that_with_gap_long_distance",
      "alias": "blimp_wh_vs_that_with_gap_long_distance",
      "sample_len": 1000,
      "acc,none": 0.176,
      "acc_stderr,none": 0.012048616898597498
    },
    "blimp": {
      "alias": "blimp",
      "name": "blimp",
      "sample_len": 67000,
      "acc,none": 0.7616716417910447,
      "acc_stderr,none": 0.0014786292668924085,
      "sample_count": {
        "acc,none": 67000
      }
    }
  }
}