| { |
| "model_id": "m06", |
| "checkpoint": "results/m06/final.pt", |
| "trained_step": 15258, |
| "total_params": 63493120, |
| "active_params": 63493120, |
| "flops_per_forward": 126986240.0, |
| "val_loss": 3.4427546560764313, |
| "val_perplexity": 31.272985946204606, |
| "bits_per_token": 4.966845069318857, |
| "ppl_tokens_evaluated": 655360, |
| "prefill_tok_per_sec": 153440.495550482, |
| "inference_tok_per_sec": 196.12124010573368, |
| "inference_tok_per_sec_kvcache": 190.99119877394725, |
| "kvcache_speedup": 0.9738425000320174, |
| "gen_tokens": 128, |
| "peak_infer_vram_gb": 0.395, |
| "peak_train_vram_gb": 4.097, |
| "closeness_answer_ppl": 15.475, |
| "closeness_hit_rate": 0.55, |
| "closeness_token_f1": 0.083, |
| "closeness_n": 20, |
| "closeness_items": [ |
| { |
| "prompt": "The capital of France is", |
| "answer": " Paris", |
| "generated": " Paris. The capital is Paris", |
| "answer_nll": 1.351, |
| "answer_ppl": 3.861, |
| "hit": true, |
| "token_f1": 0.333 |
| }, |
| { |
| "prompt": "The capital of Japan is", |
| "answer": " Tokyo", |
| "generated": " Tokyo. The capital is Tokyo", |
| "answer_nll": 1.5316, |
| "answer_ppl": 4.625, |
| "hit": true, |
| "token_f1": 0.333 |
| }, |
| { |
| "prompt": "The largest planet in the solar system is", |
| "answer": " Jupiter", |
| "generated": " the Sun, which is the", |
| "answer_nll": 3.301, |
| "answer_ppl": 27.139, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "Water is made of hydrogen and", |
| "answer": " oxygen", |
| "generated": " oxygen.\nThe hydrogen is", |
| "answer_nll": 1.3272, |
| "answer_ppl": 3.77, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The opposite of hot is", |
| "answer": " cold", |
| "generated": " the heat of the sun.", |
| "answer_nll": 2.3498, |
| "answer_ppl": 10.483, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The opposite of up is", |
| "answer": " down", |
| "generated": " the \u201cdown\u201d", |
| "answer_nll": 2.0964, |
| "answer_ppl": 8.137, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "Two plus two equals", |
| "answer": " four", |
| "generated": " the number of times the number", |
| "answer_nll": 4.6418, |
| "answer_ppl": 103.733, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The sun rises in the", |
| "answer": " east", |
| "generated": " sky, and the sun rises", |
| "answer_nll": 3.7532, |
| "answer_ppl": 42.656, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The first president of the United States was George", |
| "answer": " Washington", |
| "generated": " Washington. He was a member", |
| "answer_nll": 0.3467, |
| "answer_ppl": 1.414, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The speed of light is approximately 300,000 kilometers per", |
| "answer": " second", |
| "generated": " second.\nThe speed of", |
| "answer_nll": 0.0454, |
| "answer_ppl": 1.046, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "A triangle has three", |
| "answer": " sides", |
| "generated": " sides, and the triangle has", |
| "answer_nll": 0.9853, |
| "answer_ppl": 2.679, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The chemical symbol for gold is", |
| "answer": " Au", |
| "generated": " the symbol of the sun.", |
| "answer_nll": 8.9973, |
| "answer_ppl": 8081.313, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The freezing point of water in Celsius is", |
| "answer": " zero", |
| "generated": " 1.5\u00b0C.", |
| "answer_nll": 6.5868, |
| "answer_ppl": 725.456, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The author of Romeo and Juliet is William", |
| "answer": " Shakespeare", |
| "generated": " Shakespeare. He is the son", |
| "answer_nll": 0.5215, |
| "answer_ppl": 1.684, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The Earth revolves around the", |
| "answer": " Sun", |
| "generated": " Sun, the Sun is the", |
| "answer_nll": 0.7619, |
| "answer_ppl": 2.142, |
| "hit": true, |
| "token_f1": 0.333 |
| }, |
| { |
| "prompt": "The plural of mouse is", |
| "answer": " mice", |
| "generated": " the mouse. The plural of", |
| "answer_nll": 5.464, |
| "answer_ppl": 236.051, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "Photosynthesis occurs in the", |
| "answer": " leaves", |
| "generated": " body, but it is not", |
| "answer_nll": 4.3326, |
| "answer_ppl": 76.143, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The human heart pumps", |
| "answer": " blood", |
| "generated": " blood to the lungs. The", |
| "answer_nll": 1.4275, |
| "answer_ppl": 4.168, |
| "hit": true, |
| "token_f1": 0.333 |
| }, |
| { |
| "prompt": "The currency used in the United States is the", |
| "answer": " dollar", |
| "generated": " US dollar. The US dollar", |
| "answer_nll": 2.7287, |
| "answer_ppl": 15.313, |
| "hit": true, |
| "token_f1": 0.333 |
| }, |
| { |
| "prompt": "The tallest animal in the world is the", |
| "answer": " giraffe", |
| "generated": " giant, the giant, the", |
| "answer_nll": 2.2349, |
| "answer_ppl": 9.346, |
| "hit": false, |
| "token_f1": 0.0 |
| } |
| ], |
| "generation_samples": { |
| "The most important thing about sleep is": "The most important thing about sleep is to create a routine that is consistent, consistent and meaningful. It\u2019s important to keep a record of what you\u2019re doing, what you\u2019re doing, and what you\u2019re doing.\nThe most important thing about sleep is to keep a routine that is consistent,", |
| "Scientists have discovered that": "Scientists have discovered that a large volume of water is collected in the river than the Earth\u2019s surface. Water is a \u201cgreen\u201d gas, with a pH that\u2019s higher than 1.6.\nBut why is the water so important?\nThere are many reasons why scientists think this type", |
| "In the history of mathematics,": "In the history of mathematics, the term has been used to describe the practice of using the term \u201cknowledge\u201d to describe a mathematical concept. The term \u201cknowledge\u201d is used as a synonym for knowledge, but can be used in two different ways. One way is the word \u201cknowledge\u201d,", |
| "Python code for prime number": "Python code for prime number tables, and the common name for the string 'A' to be used.\nNote that the C++ program is not designed to store large numbers of numbers, so the C++ library must store large numbers of numbers.\n- Note that the C++\n- the C++\n- the", |
| "The best way to learn a new language is": "The best way to learn a new language is to learn it in a fun and fun way. When you do some fun learning, you can start to improve your learning technique and become proficient in the new language.\n- Don\u2019t assume the new language is a language of the past. Most people are not familiar with new language, it" |
| } |
| } |
