Add m22 weights and results

m22/benchmark_results.json

@@ -0,0 +1,540 @@
+{
+  "model_id": "m22",
+  "checkpoint": "results/m22/final.pt",
+  "trained_step": 15258,
+  "num_fewshot": 0,
+  "limit": null,
+  "results": {
+    "lambada_openai": {
+      "name": "lambada_openai",
+      "alias": "lambada_openai",
+      "sample_len": 5153,
+      "perplexity,none": 785.2879050413346,
+      "perplexity_stderr,none": 43.50157604743723,
+      "acc,none": 0.17019212109450804,
+      "acc_stderr,none": 0.005235649623696734
+    },
+    "hellaswag": {
+      "name": "hellaswag",
+      "alias": "hellaswag",
+      "sample_len": 10042,
+      "acc,none": 0.27145986855208126,
+      "acc_stderr,none": 0.004438038583345241,
+      "acc_norm,none": 0.2756423023302131,
+      "acc_norm_stderr,none": 0.004459241474518529
+    },
+    "piqa": {
+      "name": "piqa",
+      "alias": "piqa",
+      "sample_len": 1838,
+      "acc,none": 0.5946681175190425,
+      "acc_stderr,none": 0.011454816387346781,
+      "acc_norm,none": 0.5778019586507073,
+      "acc_norm_stderr,none": 0.01152372892677764
+    },
+    "sciq": {
+      "name": "sciq",
+      "alias": "sciq",
+      "sample_len": 1000,
+      "acc,none": 0.691,
+      "acc_stderr,none": 0.014619600977206347,
+      "acc_norm,none": 0.619,
+      "acc_norm_stderr,none": 0.015364734787007391
+    },
+    "arc_easy": {
+      "name": "arc_easy",
+      "alias": "arc_easy",
+      "sample_len": 2376,
+      "acc,none": 0.43392255892255893,
+      "acc_stderr,none": 0.010169795770462075,
+      "acc_norm,none": 0.38762626262626265,
+      "acc_norm_stderr,none": 0.009997307914447627
+    },
+    "winogrande": {
+      "name": "winogrande",
+      "alias": "winogrande",
+      "sample_len": 1267,
+      "acc,none": 0.5169692186266772,
+      "acc_stderr,none": 0.014044390401613116
+    },
+    "blimp_adjunct_island": {
+      "name": "blimp_adjunct_island",
+      "alias": "blimp_adjunct_island",
+      "sample_len": 1000,
+      "acc,none": 0.796,
+      "acc_stderr,none": 0.012749374359024311
+    },
+    "blimp_anaphor_gender_agreement": {
+      "name": "blimp_anaphor_gender_agreement",
+      "alias": "blimp_anaphor_gender_agreement",
+      "sample_len": 1000,
+      "acc,none": 0.911,
+      "acc_stderr,none": 0.009008893392651516
+    },
+    "blimp_anaphor_number_agreement": {
+      "name": "blimp_anaphor_number_agreement",
+      "alias": "blimp_anaphor_number_agreement",
+      "sample_len": 1000,
+      "acc,none": 0.975,
+      "acc_stderr,none": 0.004939574819698493
+    },
+    "blimp_animate_subject_passive": {
+      "name": "blimp_animate_subject_passive",
+      "alias": "blimp_animate_subject_passive",
+      "sample_len": 1000,
+      "acc,none": 0.739,
+      "acc_stderr,none": 0.013895037677965183
+    },
+    "blimp_animate_subject_trans": {
+      "name": "blimp_animate_subject_trans",
+      "alias": "blimp_animate_subject_trans",
+      "sample_len": 1000,
+      "acc,none": 0.894,
+      "acc_stderr,none": 0.00973955126578524
+    },
+    "blimp_causative": {
+      "name": "blimp_causative",
+      "alias": "blimp_causative",
+      "sample_len": 1000,
+      "acc,none": 0.718,
+      "acc_stderr,none": 0.01423652621529138
+    },
+    "blimp_complex_NP_island": {
+      "name": "blimp_complex_NP_island",
+      "alias": "blimp_complex_NP_island",
+      "sample_len": 1000,
+      "acc,none": 0.481,
+      "acc_stderr,none": 0.015807874268505835
+    },
+    "blimp_coordinate_structure_constraint_complex_left_branch": {
+      "name": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "alias": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "sample_len": 1000,
+      "acc,none": 0.415,
+      "acc_stderr,none": 0.015589035185604594
+    },
+    "blimp_coordinate_structure_constraint_object_extraction": {
+      "name": "blimp_coordinate_structure_constraint_object_extraction",
+      "alias": "blimp_coordinate_structure_constraint_object_extraction",
+      "sample_len": 1000,
+      "acc,none": 0.801,
+      "acc_stderr,none": 0.01263164908309911
+    },
+    "blimp_determiner_noun_agreement_1": {
+      "name": "blimp_determiner_noun_agreement_1",
+      "alias": "blimp_determiner_noun_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.972,
+      "acc_stderr,none": 0.005219506034410081
+    },
+    "blimp_determiner_noun_agreement_2": {
+      "name": "blimp_determiner_noun_agreement_2",
+      "alias": "blimp_determiner_noun_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.942,
+      "acc_stderr,none": 0.0073953154557929906
+    },
+    "blimp_determiner_noun_agreement_irregular_1": {
+      "name": "blimp_determiner_noun_agreement_irregular_1",
+      "alias": "blimp_determiner_noun_agreement_irregular_1",
+      "sample_len": 1000,
+      "acc,none": 0.88,
+      "acc_stderr,none": 0.010281328012747462
+    },
+    "blimp_determiner_noun_agreement_irregular_2": {
+      "name": "blimp_determiner_noun_agreement_irregular_2",
+      "alias": "blimp_determiner_noun_agreement_irregular_2",
+      "sample_len": 1000,
+      "acc,none": 0.924,
+      "acc_stderr,none": 0.008384169266796462
+    },
+    "blimp_determiner_noun_agreement_with_adj_2": {
+      "name": "blimp_determiner_noun_agreement_with_adj_2",
+      "alias": "blimp_determiner_noun_agreement_with_adj_2",
+      "sample_len": 1000,
+      "acc,none": 0.889,
+      "acc_stderr,none": 0.009938701010583716
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
+      "name": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "sample_len": 1000,
+      "acc,none": 0.827,
+      "acc_stderr,none": 0.011967214137559927
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
+      "name": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "sample_len": 1000,
+      "acc,none": 0.87,
+      "acc_stderr,none": 0.010640169792499236
+    },
+    "blimp_determiner_noun_agreement_with_adjective_1": {
+      "name": "blimp_determiner_noun_agreement_with_adjective_1",
+      "alias": "blimp_determiner_noun_agreement_with_adjective_1",
+      "sample_len": 1000,
+      "acc,none": 0.916,
+      "acc_stderr,none": 0.008776162089491113
+    },
+    "blimp_distractor_agreement_relational_noun": {
+      "name": "blimp_distractor_agreement_relational_noun",
+      "alias": "blimp_distractor_agreement_relational_noun",
+      "sample_len": 1000,
+      "acc,none": 0.853,
+      "acc_stderr,none": 0.011203415395160376
+    },
+    "blimp_distractor_agreement_relative_clause": {
+      "name": "blimp_distractor_agreement_relative_clause",
+      "alias": "blimp_distractor_agreement_relative_clause",
+      "sample_len": 1000,
+      "acc,none": 0.684,
+      "acc_stderr,none": 0.014709193056057166
+    },
+    "blimp_drop_argument": {
+      "name": "blimp_drop_argument",
+      "alias": "blimp_drop_argument",
+      "sample_len": 1000,
+      "acc,none": 0.713,
+      "acc_stderr,none": 0.014312087053810001
+    },
+    "blimp_ellipsis_n_bar_1": {
+      "name": "blimp_ellipsis_n_bar_1",
+      "alias": "blimp_ellipsis_n_bar_1",
+      "sample_len": 1000,
+      "acc,none": 0.689,
+      "acc_stderr,none": 0.01464559638572269
+    },
+    "blimp_ellipsis_n_bar_2": {
+      "name": "blimp_ellipsis_n_bar_2",
+      "alias": "blimp_ellipsis_n_bar_2",
+      "sample_len": 1000,
+      "acc,none": 0.866,
+      "acc_stderr,none": 0.010777762298369738
+    },
+    "blimp_existential_there_object_raising": {
+      "name": "blimp_existential_there_object_raising",
+      "alias": "blimp_existential_there_object_raising",
+      "sample_len": 1000,
+      "acc,none": 0.696,
+      "acc_stderr,none": 0.014553205687950535
+    },
+    "blimp_existential_there_quantifiers_1": {
+      "name": "blimp_existential_there_quantifiers_1",
+      "alias": "blimp_existential_there_quantifiers_1",
+      "sample_len": 1000,
+      "acc,none": 0.972,
+      "acc_stderr,none": 0.005219506034410081
+    },
+    "blimp_existential_there_quantifiers_2": {
+      "name": "blimp_existential_there_quantifiers_2",
+      "alias": "blimp_existential_there_quantifiers_2",
+      "sample_len": 1000,
+      "acc,none": 0.368,
+      "acc_stderr,none": 0.015258073561521743
+    },
+    "blimp_existential_there_subject_raising": {
+      "name": "blimp_existential_there_subject_raising",
+      "alias": "blimp_existential_there_subject_raising",
+      "sample_len": 1000,
+      "acc,none": 0.834,
+      "acc_stderr,none": 0.011772110370812133
+    },
+    "blimp_expletive_it_object_raising": {
+      "name": "blimp_expletive_it_object_raising",
+      "alias": "blimp_expletive_it_object_raising",
+      "sample_len": 1000,
+      "acc,none": 0.732,
+      "acc_stderr,none": 0.014013292702729519
+    },
+    "blimp_inchoative": {
+      "name": "blimp_inchoative",
+      "alias": "blimp_inchoative",
+      "sample_len": 1000,
+      "acc,none": 0.611,
+      "acc_stderr,none": 0.015424555647308514
+    },
+    "blimp_intransitive": {
+      "name": "blimp_intransitive",
+      "alias": "blimp_intransitive",
+      "sample_len": 1000,
+      "acc,none": 0.72,
+      "acc_stderr,none": 0.014205696104091548
+    },
+    "blimp_irregular_past_participle_adjectives": {
+      "name": "blimp_irregular_past_participle_adjectives",
+      "alias": "blimp_irregular_past_participle_adjectives",
+      "sample_len": 1000,
+      "acc,none": 0.816,
+      "acc_stderr,none": 0.01225945734093864
+    },
+    "blimp_irregular_past_participle_verbs": {
+      "name": "blimp_irregular_past_participle_verbs",
+      "alias": "blimp_irregular_past_participle_verbs",
+      "sample_len": 1000,
+      "acc,none": 0.78,
+      "acc_stderr,none": 0.013106173040661862
+    },
+    "blimp_irregular_plural_subject_verb_agreement_1": {
+      "name": "blimp_irregular_plural_subject_verb_agreement_1",
+      "alias": "blimp_irregular_plural_subject_verb_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.857,
+      "acc_stderr,none": 0.011075814808567074
+    },
+    "blimp_irregular_plural_subject_verb_agreement_2": {
+      "name": "blimp_irregular_plural_subject_verb_agreement_2",
+      "alias": "blimp_irregular_plural_subject_verb_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.87,
+      "acc_stderr,none": 0.010640169792499236
+    },
+    "blimp_left_branch_island_echo_question": {
+      "name": "blimp_left_branch_island_echo_question",
+      "alias": "blimp_left_branch_island_echo_question",
+      "sample_len": 1000,
+      "acc,none": 0.536,
+      "acc_stderr,none": 0.015778243024904673
+    },
+    "blimp_left_branch_island_simple_question": {
+      "name": "blimp_left_branch_island_simple_question",
+      "alias": "blimp_left_branch_island_simple_question",
+      "sample_len": 1000,
+      "acc,none": 0.514,
+      "acc_stderr,none": 0.01581309754773093
+    },
+    "blimp_matrix_question_npi_licensor_present": {
+      "name": "blimp_matrix_question_npi_licensor_present",
+      "alias": "blimp_matrix_question_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.414,
+      "acc_stderr,none": 0.015583544104177552
+    },
+    "blimp_npi_present_1": {
+      "name": "blimp_npi_present_1",
+      "alias": "blimp_npi_present_1",
+      "sample_len": 1000,
+      "acc,none": 0.573,
+      "acc_stderr,none": 0.015649789644462186
+    },
+    "blimp_npi_present_2": {
+      "name": "blimp_npi_present_2",
+      "alias": "blimp_npi_present_2",
+      "sample_len": 1000,
+      "acc,none": 0.563,
+      "acc_stderr,none": 0.01569322392873043
+    },
+    "blimp_only_npi_licensor_present": {
+      "name": "blimp_only_npi_licensor_present",
+      "alias": "blimp_only_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.738,
+      "acc_stderr,none": 0.013912208651021217
+    },
+    "blimp_only_npi_scope": {
+      "name": "blimp_only_npi_scope",
+      "alias": "blimp_only_npi_scope",
+      "sample_len": 1000,
+      "acc,none": 0.486,
+      "acc_stderr,none": 0.01581309754773093
+    },
+    "blimp_passive_1": {
+      "name": "blimp_passive_1",
+      "alias": "blimp_passive_1",
+      "sample_len": 1000,
+      "acc,none": 0.88,
+      "acc_stderr,none": 0.010281328012747462
+    },
+    "blimp_passive_2": {
+      "name": "blimp_passive_2",
+      "alias": "blimp_passive_2",
+      "sample_len": 1000,
+      "acc,none": 0.865,
+      "acc_stderr,none": 0.010811655372416006
+    },
+    "blimp_principle_A_c_command": {
+      "name": "blimp_principle_A_c_command",
+      "alias": "blimp_principle_A_c_command",
+      "sample_len": 1000,
+      "acc,none": 0.623,
+      "acc_stderr,none": 0.0153331701257798
+    },
+    "blimp_principle_A_case_1": {
+      "name": "blimp_principle_A_case_1",
+      "alias": "blimp_principle_A_case_1",
+      "sample_len": 1000,
+      "acc,none": 1.0,
+      "acc_stderr,none": 0.0
+    },
+    "blimp_principle_A_case_2": {
+      "name": "blimp_principle_A_case_2",
+      "alias": "blimp_principle_A_case_2",
+      "sample_len": 1000,
+      "acc,none": 0.943,
+      "acc_stderr,none": 0.007335175853706905
+    },
+    "blimp_principle_A_domain_1": {
+      "name": "blimp_principle_A_domain_1",
+      "alias": "blimp_principle_A_domain_1",
+      "sample_len": 1000,
+      "acc,none": 0.9,
+      "acc_stderr,none": 0.00949157995752499
+    },
+    "blimp_principle_A_domain_2": {
+      "name": "blimp_principle_A_domain_2",
+      "alias": "blimp_principle_A_domain_2",
+      "sample_len": 1000,
+      "acc,none": 0.745,
+      "acc_stderr,none": 0.013790038620872863
+    },
+    "blimp_principle_A_domain_3": {
+      "name": "blimp_principle_A_domain_3",
+      "alias": "blimp_principle_A_domain_3",
+      "sample_len": 1000,
+      "acc,none": 0.613,
+      "acc_stderr,none": 0.015410011955494015
+    },
+    "blimp_principle_A_reconstruction": {
+      "name": "blimp_principle_A_reconstruction",
+      "alias": "blimp_principle_A_reconstruction",
+      "sample_len": 1000,
+      "acc,none": 0.488,
+      "acc_stderr,none": 0.015814743314581686
+    },
+    "blimp_regular_plural_subject_verb_agreement_1": {
+      "name": "blimp_regular_plural_subject_verb_agreement_1",
+      "alias": "blimp_regular_plural_subject_verb_agreement_1",
+      "sample_len": 1000,
+      "acc,none": 0.834,
+      "acc_stderr,none": 0.011772110370812133
+    },
+    "blimp_regular_plural_subject_verb_agreement_2": {
+      "name": "blimp_regular_plural_subject_verb_agreement_2",
+      "alias": "blimp_regular_plural_subject_verb_agreement_2",
+      "sample_len": 1000,
+      "acc,none": 0.881,
+      "acc_stderr,none": 0.01024421514533661
+    },
+    "blimp_sentential_negation_npi_licensor_present": {
+      "name": "blimp_sentential_negation_npi_licensor_present",
+      "alias": "blimp_sentential_negation_npi_licensor_present",
+      "sample_len": 1000,
+      "acc,none": 0.98,
+      "acc_stderr,none": 0.004429403980178329
+    },
+    "blimp_sentential_negation_npi_scope": {
+      "name": "blimp_sentential_negation_npi_scope",
+      "alias": "blimp_sentential_negation_npi_scope",
+      "sample_len": 1000,
+      "acc,none": 0.388,
+      "acc_stderr,none": 0.015417317979911216
+    },
+    "blimp_sentential_subject_island": {
+      "name": "blimp_sentential_subject_island",
+      "alias": "blimp_sentential_subject_island",
+      "sample_len": 1000,
+      "acc,none": 0.546,
+      "acc_stderr,none": 0.01575221038877186
+    },
+    "blimp_superlative_quantifiers_1": {
+      "name": "blimp_superlative_quantifiers_1",
+      "alias": "blimp_superlative_quantifiers_1",
+      "sample_len": 1000,
+      "acc,none": 0.82,
+      "acc_stderr,none": 0.012155153135512022
+    },
+    "blimp_superlative_quantifiers_2": {
+      "name": "blimp_superlative_quantifiers_2",
+      "alias": "blimp_superlative_quantifiers_2",
+      "sample_len": 1000,
+      "acc,none": 0.797,
+      "acc_stderr,none": 0.01272607374459825
+    },
+    "blimp_tough_vs_raising_1": {
+      "name": "blimp_tough_vs_raising_1",
+      "alias": "blimp_tough_vs_raising_1",
+      "sample_len": 1000,
+      "acc,none": 0.523,
+      "acc_stderr,none": 0.015802554246726087
+    },
+    "blimp_tough_vs_raising_2": {
+      "name": "blimp_tough_vs_raising_2",
+      "alias": "blimp_tough_vs_raising_2",
+      "sample_len": 1000,
+      "acc,none": 0.881,
+      "acc_stderr,none": 0.01024421514533661
+    },
+    "blimp_transitive": {
+      "name": "blimp_transitive",
+      "alias": "blimp_transitive",
+      "sample_len": 1000,
+      "acc,none": 0.822,
+      "acc_stderr,none": 0.012102167676183571
+    },
+    "blimp_wh_island": {
+      "name": "blimp_wh_island",
+      "alias": "blimp_wh_island",
+      "sample_len": 1000,
+      "acc,none": 0.611,
+      "acc_stderr,none": 0.015424555647308514
+    },
+    "blimp_wh_questions_object_gap": {
+      "name": "blimp_wh_questions_object_gap",
+      "alias": "blimp_wh_questions_object_gap",
+      "sample_len": 1000,
+      "acc,none": 0.734,
+      "acc_stderr,none": 0.013979965645145179
+    },
+    "blimp_wh_questions_subject_gap": {
+      "name": "blimp_wh_questions_subject_gap",
+      "alias": "blimp_wh_questions_subject_gap",
+      "sample_len": 1000,
+      "acc,none": 0.919,
+      "acc_stderr,none": 0.008632121032139986
+    },
+    "blimp_wh_questions_subject_gap_long_distance": {
+      "name": "blimp_wh_questions_subject_gap_long_distance",
+      "alias": "blimp_wh_questions_subject_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.889,
+      "acc_stderr,none": 0.009938701010583716
+    },
+    "blimp_wh_vs_that_no_gap": {
+      "name": "blimp_wh_vs_that_no_gap",
+      "alias": "blimp_wh_vs_that_no_gap",
+      "sample_len": 1000,
+      "acc,none": 0.968,
+      "acc_stderr,none": 0.005568393575081328
+    },
+    "blimp_wh_vs_that_no_gap_long_distance": {
+      "name": "blimp_wh_vs_that_no_gap_long_distance",
+      "alias": "blimp_wh_vs_that_no_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.986,
+      "acc_stderr,none": 0.003717232548256541
+    },
+    "blimp_wh_vs_that_with_gap": {
+      "name": "blimp_wh_vs_that_with_gap",
+      "alias": "blimp_wh_vs_that_with_gap",
+      "sample_len": 1000,
+      "acc,none": 0.52,
+      "acc_stderr,none": 0.015806639423035177
+    },
+    "blimp_wh_vs_that_with_gap_long_distance": {
+      "name": "blimp_wh_vs_that_with_gap_long_distance",
+      "alias": "blimp_wh_vs_that_with_gap_long_distance",
+      "sample_len": 1000,
+      "acc,none": 0.213,
+      "acc_stderr,none": 0.012953717566737312
+    },
+    "blimp": {
+      "alias": "blimp",
+      "name": "blimp",
+      "sample_len": 67000,
+      "acc,none": 0.7493134328358209,
+      "acc_stderr,none": 0.0015197586373841688,
+      "sample_count": {
+        "acc,none": 67000
+      }
+    }
+  }
+}

m22/config.json

@@ -0,0 +1,50 @@
+{
+  "model_id": "m22",
+  "run_name": "m22-mha-learned-dense-adamw-seed1338",
+  "data_dir": "data/fineweb-edu",
+  "vocab_size": 50257,
+  "block_size": 1024,
+  "d_model": 512,
+  "num_layers": 12,
+  "num_heads": 8,
+  "ffn_mult": 4,
+  "dropout": 0.0,
+  "bias": false,
+  "norm_type": "rmsnorm",
+  "tie_embeddings": true,
+  "attention_type": "mha",
+  "gqa_num_kv_heads": 2,
+  "mla_latent_dim": 64,
+  "pos_type": "learned",
+  "rope_base": 10000,
+  "ffn_type": "dense",
+  "moe_num_experts": 8,
+  "moe_top_k": 2,
+  "optimizer": "adamw",
+  "lr": 0.0006,
+  "min_lr": 6e-05,
+  "weight_decay": 0.1,
+  "beta1": 0.9,
+  "beta2": 0.95,
+  "grad_clip": 1.0,
+  "muon_lr": 0.02,
+  "muon_momentum": 0.95,
+  "training_tokens": 1000000000,
+  "warmup_tokens": 6000000,
+  "lr_schedule": "cosine",
+  "batch_size": 4,
+  "grad_accum_steps": 16,
+  "device": "cuda",
+  "dtype": "bfloat16",
+  "compile": true,
+  "seed": 1338,
+  "eval_interval": 250,
+  "eval_iters": 100,
+  "log_interval": 10,
+  "checkpoint_interval": 1000,
+  "results_dir": "results",
+  "mlflow_experiment": "slm-ablation",
+  "mlflow_tracking_uri": "sqlite:///mlflow.db",
+  "mlflow_artifact_location": "./mlartifacts",
+  "seed_group": "m01"
+}

m22/eval_results.json

@@ -0,0 +1,212 @@
+{
+  "model_id": "m22",
+  "checkpoint": "/home/akshat/codemindv2/results/m22/final.pt",
+  "trained_step": 15258,
+  "total_params": 64017408,
+  "active_params": 64017408,
+  "flops_per_forward": 128034816.0,
+  "val_loss": 3.5585316568613052,
+  "val_perplexity": 35.11160339377922,
+  "bits_per_token": 5.133875974200191,
+  "ppl_tokens_evaluated": 655360,
+  "prefill_tok_per_sec": 195315.3982182166,
+  "inference_tok_per_sec": 326.0861900573855,
+  "inference_tok_per_sec_kvcache": null,
+  "kvcache_speedup": null,
+  "gen_tokens": 128,
+  "peak_infer_vram_gb": 0.394,
+  "peak_train_vram_gb": 4.093,
+  "closeness_answer_ppl": 33.765,
+  "closeness_hit_rate": 0.45,
+  "closeness_token_f1": 0.093,
+  "closeness_n": 20,
+  "closeness_items": [
+    {
+      "prompt": "The capital of France is",
+      "answer": " Paris",
+      "generated": " the capital of France.\n",
+      "answer_nll": 3.2356,
+      "answer_ppl": 25.421,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The capital of Japan is",
+      "answer": " Tokyo",
+      "generated": " the capital of the United States",
+      "answer_nll": 2.6915,
+      "answer_ppl": 14.754,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The largest planet in the solar system is",
+      "answer": " Jupiter",
+      "generated": " the Sun, which is the",
+      "answer_nll": 3.1711,
+      "answer_ppl": 23.834,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Water is made of hydrogen and",
+      "answer": " oxygen",
+      "generated": " oxygen.\n- The oxygen",
+      "answer_nll": 0.9604,
+      "answer_ppl": 2.613,
+      "hit": true,
+      "token_f1": 0.4
+    },
+    {
+      "prompt": "The opposite of hot is",
+      "answer": " cold",
+      "generated": " the coldest, the cold",
+      "answer_nll": 3.1722,
+      "answer_ppl": 23.86,
+      "hit": true,
+      "token_f1": 0.4
+    },
+    {
+      "prompt": "The opposite of up is",
+      "answer": " down",
+      "generated": " the fact that the two are",
+      "answer_nll": 6.9194,
+      "answer_ppl": 1011.664,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Two plus two equals",
+      "answer": " four",
+      "generated": ", and the two equals,",
+      "answer_nll": 5.168,
+      "answer_ppl": 175.56,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The sun rises in the",
+      "answer": " east",
+      "generated": " sky, and the sun rises",
+      "answer_nll": 3.1618,
+      "answer_ppl": 23.613,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The first president of the United States was George",
+      "answer": " Washington",
+      "generated": " Washington, who was the first",
+      "answer_nll": 0.9015,
+      "answer_ppl": 2.463,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The speed of light is approximately 300,000 kilometers per",
+      "answer": " second",
+      "generated": " second.\nThe speed of",
+      "answer_nll": 0.2886,
+      "answer_ppl": 1.335,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "A triangle has three",
+      "answer": " sides",
+      "generated": " sides, and the sides are",
+      "answer_nll": 2.1492,
+      "answer_ppl": 8.578,
+      "hit": true,
+      "token_f1": 0.333
+    },
+    {
+      "prompt": "The chemical symbol for gold is",
+      "answer": " Au",
+      "generated": " the gold of the world.",
+      "answer_nll": 9.4926,
+      "answer_ppl": 13261.109,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The freezing point of water in Celsius is",
+      "answer": " zero",
+      "generated": " about 1.5 degrees Celsius",
+      "answer_nll": 5.8433,
+      "answer_ppl": 344.908,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The author of Romeo and Juliet is William",
+      "answer": " Shakespeare",
+      "generated": " Shakespeare.\nThe author of",
+      "answer_nll": 1.3143,
+      "answer_ppl": 3.722,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The Earth revolves around the",
+      "answer": " Sun",
+      "generated": " Sun, and the Sun revolves",
+      "answer_nll": 0.9692,
+      "answer_ppl": 2.636,
+      "hit": true,
+      "token_f1": 0.333
+    },
+    {
+      "prompt": "The plural of mouse is",
+      "answer": " mice",
+      "generated": " the plural of mouse.\n",
+      "answer_nll": 10.0879,
+      "answer_ppl": 24050.054,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "Photosynthesis occurs in the",
+      "answer": " leaves",
+      "generated": " chlorophyll-rich waters",
+      "answer_nll": 3.7037,
+      "answer_ppl": 40.596,
+      "hit": false,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The human heart pumps",
+      "answer": " blood",
+      "generated": " blood to the heart.\n",
+      "answer_nll": 1.0486,
+      "answer_ppl": 2.854,
+      "hit": true,
+      "token_f1": 0.4
+    },
+    {
+      "prompt": "The currency used in the United States is the",
+      "answer": " dollar",
+      "generated": " United States Dollar.\nThe",
+      "answer_nll": 3.1815,
+      "answer_ppl": 24.084,
+      "hit": true,
+      "token_f1": 0.0
+    },
+    {
+      "prompt": "The tallest animal in the world is the",
+      "answer": " giraffe",
+      "generated": " giant elephant.\nThe elephant",
+      "answer_nll": 2.928,
+      "answer_ppl": 18.69,
+      "hit": false,
+      "token_f1": 0.0
+    }
+  ],
+  "generation_samples": {
+    "The most important thing about sleep is": "The most important thing about sleep is waking up. It can be easy to go back to sleep for a while. So, it is important to give proper care to your sleep.\nIn some cases sleep is disrupted because of the lack of a proper sleep environment. If the body doesn\u2019t get enough sleep, your body can",
+    "Scientists have discovered that": "Scientists have discovered that when their muscles contract, it can trigger a reaction. This is how the muscles contract when they're stressed.\nTrying to work on the muscles in these situations is the best way to help your heart and brain function.\nYour body is only a little bit smaller than any other organs.\n",
+    "In the history of mathematics,": "In the history of mathematics, we had very much in this field of science a year before, we have only two problems which have always proved to us quite difficult.\nI shall attempt to describe our theory of an elementary school which we have only to discuss in terms of mathematics, and we shall say that the mathematics that we will",
+    "Python code for prime number": "Python code for prime number, and also the P8 and C, respectively.\nIn fact, the same rules apply to the P8 range, which is the same in both cases. In either case, we may make a higher number, with the P8 higher; in the case of the P8 lower one,",
+    "The best way to learn a new language is": "The best way to learn a new language is to read our language, and then get over to the basics. If you're ready to learn a new language, it's easy!\nWhat Do I Need to Know?\nYou are speaking to a native speaker every day, not on the same day, at the same time. That's right"
+  }
+}

m22/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7b240badfbb7ac04b19cec038fa0728e85377114cd524c4d7cc3a8b4d17c6b4
+size 256079264