File size: 7,176 Bytes
ae1dbd2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 | {
"model_id": "m18",
"checkpoint": "/home/akshat/codemindv2/results/m18/final.pt",
"trained_step": 15258,
"total_params": 58774528,
"active_params": 58774528,
"flops_per_forward": 117549056.0,
"val_loss": 3.5828016698360443,
"val_perplexity": 35.97418758800085,
"bits_per_token": 5.168890201561158,
"ppl_tokens_evaluated": 655360,
"prefill_tok_per_sec": 192856.71674094364,
"inference_tok_per_sec": 286.03230651282206,
"inference_tok_per_sec_kvcache": 283.4770307274281,
"kvcache_speedup": 0.9910664784109643,
"gen_tokens": 128,
"peak_infer_vram_gb": 0.363,
"peak_train_vram_gb": 3.987,
"closeness_answer_ppl": 36.553,
"closeness_hit_rate": 0.4,
"closeness_token_f1": 0.033,
"closeness_n": 20,
"closeness_items": [
{
"prompt": "The capital of France is",
"answer": " Paris",
"generated": " the French capital of France.",
"answer_nll": 3.9908,
"answer_ppl": 54.098,
"hit": false,
"token_f1": 0.0
},
{
"prompt": "The capital of Japan is",
"answer": " Tokyo",
"generated": " Japan.\nThe Japanese government",
"answer_nll": 3.3975,
"answer_ppl": 29.888,
"hit": false,
"token_f1": 0.0
},
{
"prompt": "The largest planet in the solar system is",
"answer": " Jupiter",
"generated": " the sun. The sun is",
"answer_nll": 4.4833,
"answer_ppl": 88.53,
"hit": false,
"token_f1": 0.0
},
{
"prompt": "Water is made of hydrogen and",
"answer": " oxygen",
"generated": " oxygen.\nThe hydrogen and",
"answer_nll": 1.0029,
"answer_ppl": 2.726,
"hit": true,
"token_f1": 0.0
},
{
"prompt": "The opposite of hot is",
"answer": " cold",
"generated": " the case with the hot water",
"answer_nll": 5.5526,
"answer_ppl": 257.904,
"hit": false,
"token_f1": 0.0
},
{
"prompt": "The opposite of up is",
"answer": " down",
"generated": " the fact that the two are",
"answer_nll": 6.6943,
"answer_ppl": 807.812,
"hit": false,
"token_f1": 0.0
},
{
"prompt": "Two plus two equals",
"answer": " four",
"generated": ", and the sum of the",
"answer_nll": 5.2257,
"answer_ppl": 185.998,
"hit": false,
"token_f1": 0.0
},
{
"prompt": "The sun rises in the",
"answer": " east",
"generated": " sky, and the sun rises",
"answer_nll": 4.1287,
"answer_ppl": 62.099,
"hit": false,
"token_f1": 0.0
},
{
"prompt": "The first president of the United States was George",
"answer": " Washington",
"generated": " Washington, who was a member",
"answer_nll": 0.4225,
"answer_ppl": 1.526,
"hit": true,
"token_f1": 0.0
},
{
"prompt": "The speed of light is approximately 300,000 kilometers per",
"answer": " second",
"generated": " second, and the speed of",
"answer_nll": 0.4998,
"answer_ppl": 1.648,
"hit": true,
"token_f1": 0.0
},
{
"prompt": "A triangle has three",
"answer": " sides",
"generated": " sides, one of which is",
"answer_nll": 1.2318,
"answer_ppl": 3.427,
"hit": true,
"token_f1": 0.0
},
{
"prompt": "The chemical symbol for gold is",
"answer": " Au",
"generated": " the gold symbol.\nThe",
"answer_nll": 8.984,
"answer_ppl": 7974.202,
"hit": false,
"token_f1": 0.0
},
{
"prompt": "The freezing point of water in Celsius is",
"answer": " zero",
"generated": " the melting point of water in",
"answer_nll": 5.2682,
"answer_ppl": 194.07,
"hit": false,
"token_f1": 0.0
},
{
"prompt": "The author of Romeo and Juliet is William",
"answer": " Shakespeare",
"generated": " Shakespeare.\nThe author of",
"answer_nll": 0.8406,
"answer_ppl": 2.318,
"hit": true,
"token_f1": 0.0
},
{
"prompt": "The Earth revolves around the",
"answer": " Sun",
"generated": " Sun, and the Sun revolves",
"answer_nll": 1.3375,
"answer_ppl": 3.809,
"hit": true,
"token_f1": 0.333
},
{
"prompt": "The plural of mouse is",
"answer": " mice",
"generated": " the word for the word \"",
"answer_nll": 7.2894,
"answer_ppl": 1464.741,
"hit": false,
"token_f1": 0.0
},
{
"prompt": "Photosynthesis occurs in the",
"answer": " leaves",
"generated": " plant and is a major source",
"answer_nll": 3.6544,
"answer_ppl": 38.645,
"hit": false,
"token_f1": 0.0
},
{
"prompt": "The human heart pumps",
"answer": " blood",
"generated": " blood to the heart, and",
"answer_nll": 1.629,
"answer_ppl": 5.099,
"hit": true,
"token_f1": 0.333
},
{
"prompt": "The currency used in the United States is the",
"answer": " dollar",
"generated": " United States dollar.\nThe",
"answer_nll": 3.4756,
"answer_ppl": 32.318,
"hit": true,
"token_f1": 0.0
},
{
"prompt": "The tallest animal in the world is the",
"answer": " giraffe",
"generated": " giant, the largest, and",
"answer_nll": 2.8665,
"answer_ppl": 17.575,
"hit": false,
"token_f1": 0.0
}
],
"generation_samples": {
"The most important thing about sleep is": "The most important thing about sleep is to create a schedule that doesn\u2019t require you to eat,\u201d says Dr. Martin. \u201cIt\u2019s important to focus on eating before, and not on just those things that you can\u2019t burn.\u201d\nDr. Andrew Walker, director of the Sleep",
"Scientists have discovered that": "Scientists have discovered that, in short, the amount of CO2 released by the sun causes serious problems in the environment.\n\u201cWe do not know how damaging the global warming is; the warming is caused by the sun\u2019s rays, rather than by the sun\u2019s rays,\u201d said John",
"In the history of mathematics,": "In the history of mathematics, the term \"curved' is a synonym for the common expression \"curved\" which can be found in many different ways. For example, the common expression \"curved\" (equation) can be found in two different places: a curving place, a table or a table?",
"Python code for prime number": "Python code for prime number of 5 decimal numbers. However, the string is not a string itself, and as such it is not a string. It does not have a string of 6 decimal numbers with a string of 6 decimal numbers. The string is not an integer, and the length\nnumber of 5 decimal numbers is,",
"The best way to learn a new language is": "The best way to learn a new language is to learn about the language you\u2019re looking for. If you are learning something that you have noticed, such as the following sentence, \u201cI have heard someone say, \u2018I heard this,\u2019 and I heard that.\u201d\nIf you were to learn an old language"
}
} |