| { |
| "model_id": "m23", |
| "checkpoint": "/home/akshat/codemindv2/results/m23/final.pt", |
| "trained_step": 15258, |
| "total_params": 58774528, |
| "active_params": 58774528, |
| "flops_per_forward": 117549056.0, |
| "val_loss": 3.4765728563070297, |
| "val_perplexity": 32.3486683513085, |
| "bits_per_token": 5.015634419083331, |
| "ppl_tokens_evaluated": 655360, |
| "prefill_tok_per_sec": 189550.8386998898, |
| "inference_tok_per_sec": 201.23427577598775, |
| "inference_tok_per_sec_kvcache": 210.77698908250312, |
| "kvcache_speedup": 1.047420914104803, |
| "gen_tokens": 128, |
| "peak_infer_vram_gb": 0.366, |
| "peak_train_vram_gb": 3.993, |
| "closeness_answer_ppl": 24.479, |
| "closeness_hit_rate": 0.4, |
| "closeness_token_f1": 0.051, |
| "closeness_n": 20, |
| "closeness_items": [ |
| { |
| "prompt": "The capital of France is", |
| "answer": " Paris", |
| "generated": " the French capital of the province", |
| "answer_nll": 2.3204, |
| "answer_ppl": 10.179, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The capital of Japan is", |
| "answer": " Tokyo", |
| "generated": " the K\u014dsh\u014d,", |
| "answer_nll": 2.3939, |
| "answer_ppl": 10.956, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The largest planet in the solar system is", |
| "answer": " Jupiter", |
| "generated": " the Sun.\nThe Sun", |
| "answer_nll": 4.4133, |
| "answer_ppl": 82.538, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "Water is made of hydrogen and", |
| "answer": " oxygen", |
| "generated": " oxygen.\nThe hydrogen is", |
| "answer_nll": 1.1928, |
| "answer_ppl": 3.296, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The opposite of hot is", |
| "answer": " cold", |
| "generated": " the opposite of hot.\n", |
| "answer_nll": 3.8481, |
| "answer_ppl": 46.903, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The opposite of up is", |
| "answer": " down", |
| "generated": " the same as up.\n", |
| "answer_nll": 4.1841, |
| "answer_ppl": 65.637, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "Two plus two equals", |
| "answer": " four", |
| "generated": " two.\n- The first", |
| "answer_nll": 2.8029, |
| "answer_ppl": 16.493, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The sun rises in the", |
| "answer": " east", |
| "generated": " middle of the night sky and", |
| "answer_nll": 3.0683, |
| "answer_ppl": 21.506, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The first president of the United States was George", |
| "answer": " Washington", |
| "generated": " Washington, who was the first", |
| "answer_nll": 0.5824, |
| "answer_ppl": 1.79, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The speed of light is approximately 300,000 kilometers per", |
| "answer": " second", |
| "generated": " second.\nThe speed of", |
| "answer_nll": 0.1998, |
| "answer_ppl": 1.221, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "A triangle has three", |
| "answer": " sides", |
| "generated": " sides and a triangle has four", |
| "answer_nll": 1.2028, |
| "answer_ppl": 3.329, |
| "hit": true, |
| "token_f1": 0.286 |
| }, |
| { |
| "prompt": "The chemical symbol for gold is", |
| "answer": " Au", |
| "generated": " the gold-silver-silver", |
| "answer_nll": 8.5195, |
| "answer_ppl": 5011.537, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The freezing point of water in Celsius is", |
| "answer": " zero", |
| "generated": " about 1.5 degrees Celsius", |
| "answer_nll": 5.5563, |
| "answer_ppl": 258.868, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The author of Romeo and Juliet is William", |
| "answer": " Shakespeare", |
| "generated": " Shakespeare.\nThe author of", |
| "answer_nll": 0.3232, |
| "answer_ppl": 1.382, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The Earth revolves around the", |
| "answer": " Sun", |
| "generated": " sun.\nThe sun is", |
| "answer_nll": 1.018, |
| "answer_ppl": 2.768, |
| "hit": true, |
| "token_f1": 0.4 |
| }, |
| { |
| "prompt": "The plural of mouse is", |
| "answer": " mice", |
| "generated": " the plural of mouse.\n", |
| "answer_nll": 9.8359, |
| "answer_ppl": 18693.672, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "Photosynthesis occurs in the", |
| "answer": " leaves", |
| "generated": " same way as the production of", |
| "answer_nll": 5.8299, |
| "answer_ppl": 340.327, |
| "hit": false, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The human heart pumps", |
| "answer": " blood", |
| "generated": " blood to the brain, which", |
| "answer_nll": 1.6457, |
| "answer_ppl": 5.185, |
| "hit": true, |
| "token_f1": 0.333 |
| }, |
| { |
| "prompt": "The currency used in the United States is the", |
| "answer": " dollar", |
| "generated": " US dollar.\nThe US", |
| "answer_nll": 2.3366, |
| "answer_ppl": 10.346, |
| "hit": true, |
| "token_f1": 0.0 |
| }, |
| { |
| "prompt": "The tallest animal in the world is the", |
| "answer": " giraffe", |
| "generated": " giant tortoise.\nThe", |
| "answer_nll": 2.6826, |
| "answer_ppl": 14.623, |
| "hit": false, |
| "token_f1": 0.0 |
| } |
| ], |
| "generation_samples": { |
| "The most important thing about sleep is": "The most important thing about sleep is knowing what\u2019s going on. However, some people may notice:\n- Sleep difficulties:\n- Dizziness or lightheadedness\n- Sleep apnea\n- Trouble concentrating\nMost people will also notice:\n- Swelling on their head, neck, or spine\n- Increased", |
| "Scientists have discovered that": "Scientists have discovered that, in a recent study published in the British journal Nature, an Australian geneticist has uncovered a new genetic signature of a rare parasite called Listeria monocytogenes. Using a computer simulation of the gene, the scientists found that one of the parasite\u2019s genes, E2,", |
| "In the history of mathematics,": "In the history of mathematics, the term \"curntage\" is often used to describe a series of fields known as fields or sets of fields. The term \"curntage\" is often used to describe a collection of fields or sets of fields. In the history of mathematics, the term \"curntage\" is often", |
| "Python code for prime number": "Python code for prime number numbers, decimal numbers, and expressions. In addition, for example, \u201c2x4\u201d or \u201c2x4.9\u201d is used in the following form:\n- (x1) 1 = (2x3)\n- (x1) 1 = (", |
| "The best way to learn a new language is": "The best way to learn a new language is to learn about it. So if you have an understanding of a language, you will feel confident to learn it!\nBefore you begin learning a language, you should be aware of the terms and terms of the language, and you should understand them carefully. We\u2019ve discussed everything with the language" |
| } |
| } |
