Import experiment archive from bart (batch 3)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- experiments/clean1930s-d12-r11.25-muoneq-muonplus/run.json +10 -0
- experiments/clean1930s-d12-r11.25-muoneq-muonplus/summary.json +93 -0
- experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer/experiment_tokenizer.json +18 -0
- experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer/token_bytes.pt +3 -0
- experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer/tokenizer.pkl +3 -0
- experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_000500.json +142 -0
- experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_001000.json +142 -0
- experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_001500.json +142 -0
- experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_002000.json +142 -0
- experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_002362.json +142 -0
- experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_000500.pt +3 -0
- experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_001000.pt +3 -0
- experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_001500.pt +3 -0
- experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_002000.pt +3 -0
- experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_002362.pt +3 -0
- experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_000500_rank0.pt +3 -0
- experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_001000_rank0.pt +3 -0
- experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_001500_rank0.pt +3 -0
- experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_002000_rank0.pt +3 -0
- experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_002362_rank0.pt +3 -0
- experiments/clean1930s-d12-r11.25-randtok/config.json +59 -0
- experiments/clean1930s-d12-r11.25-randtok/evals/core.json +56 -0
- experiments/clean1930s-d12-r11.25-randtok/evals/samples.json +48 -0
- experiments/clean1930s-d12-r11.25-randtok/evals/val_bpb.json +12 -0
- experiments/clean1930s-d12-r11.25-randtok/run.json +10 -0
- experiments/clean1930s-d12-r11.25-randtok/summary.json +93 -0
- experiments/clean1930s-d12-r11.25-randtok/tokenizer/experiment_tokenizer.json +20 -0
- experiments/clean1930s-d12-r11.25-randtok/tokenizer/token_bytes.pt +3 -0
- experiments/clean1930s-d12-r11.25-randtok/tokenizer/tokenizer.pkl +3 -0
- experiments/clean1930s-d12-r11.25/base_checkpoints/meta_000500.json +139 -0
- experiments/clean1930s-d12-r11.25/base_checkpoints/meta_001000.json +139 -0
- experiments/clean1930s-d12-r11.25/base_checkpoints/meta_001500.json +139 -0
- experiments/clean1930s-d12-r11.25/base_checkpoints/meta_002000.json +139 -0
- experiments/clean1930s-d12-r11.25/base_checkpoints/meta_002362.json +139 -0
- experiments/clean1930s-d12-r11.25/base_checkpoints/model_000500.pt +3 -0
- experiments/clean1930s-d12-r11.25/base_checkpoints/model_001000.pt +3 -0
- experiments/clean1930s-d12-r11.25/base_checkpoints/model_001500.pt +3 -0
- experiments/clean1930s-d12-r11.25/base_checkpoints/model_002000.pt +3 -0
- experiments/clean1930s-d12-r11.25/base_checkpoints/model_002362.pt +3 -0
- experiments/clean1930s-d12-r11.25/base_checkpoints/optim_000500_rank0.pt +3 -0
- experiments/clean1930s-d12-r11.25/base_checkpoints/optim_001000_rank0.pt +3 -0
- experiments/clean1930s-d12-r11.25/base_checkpoints/optim_001500_rank0.pt +3 -0
- experiments/clean1930s-d12-r11.25/base_checkpoints/optim_002000_rank0.pt +3 -0
- experiments/clean1930s-d12-r11.25/base_checkpoints/optim_002362_rank0.pt +3 -0
- experiments/clean1930s-d12-r11.25/config.json +56 -0
- experiments/clean1930s-d12-r11.25/evals/core.json +56 -0
- experiments/clean1930s-d12-r11.25/evals/samples.json +48 -0
- experiments/clean1930s-d12-r11.25/evals/val_bpb.json +12 -0
- experiments/clean1930s-d12-r11.25/evals/val_bpb_on_think-dataset-clean.json +12 -0
- experiments/clean1930s-d12-r11.25/evals/val_bpb_on_think-dataset.json +12 -0
experiments/clean1930s-d12-r11.25-muoneq-muonplus/run.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus",
|
| 3 |
+
"stage": "base",
|
| 4 |
+
"base_experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus",
|
| 5 |
+
"parent_experiment_id": null,
|
| 6 |
+
"parent_checkpoint_step": null,
|
| 7 |
+
"config_fingerprint": "4f01f653a56d0daf",
|
| 8 |
+
"wandb_run_id": "95708491",
|
| 9 |
+
"created_at": 1783965749
|
| 10 |
+
}
|
experiments/clean1930s-d12-r11.25-muoneq-muonplus/summary.json
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus",
|
| 3 |
+
"stage": "base",
|
| 4 |
+
"base_experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus",
|
| 5 |
+
"parent_experiment_id": null,
|
| 6 |
+
"parent_checkpoint_step": null,
|
| 7 |
+
"dataset": "jbduran/think-dataset-clean-1930s",
|
| 8 |
+
"dataset_revision": "main",
|
| 9 |
+
"step": 2362,
|
| 10 |
+
"depth": 12,
|
| 11 |
+
"target_param_data_ratio": 11.25,
|
| 12 |
+
"training_tokens": 1238368256,
|
| 13 |
+
"final_sampled_val_bpb": 1.1162452479490097,
|
| 14 |
+
"minimum_sampled_val_bpb": 1.1162452479490097,
|
| 15 |
+
"full_val_bpb": 1.0583351355174524,
|
| 16 |
+
"core_metric": 0.06305335515020447,
|
| 17 |
+
"centered_results": {
|
| 18 |
+
"hellaswag_zeroshot": 0.036181370417277016,
|
| 19 |
+
"jeopardy": 0.0004723665479104966,
|
| 20 |
+
"bigbench_qa_wikidata": 0.08601938933134079,
|
| 21 |
+
"arc_easy": 0.0953984260559082,
|
| 22 |
+
"arc_challenge": -0.04550625880559286,
|
| 23 |
+
"copa": 0.0,
|
| 24 |
+
"commonsense_qa": 0.10421784967184065,
|
| 25 |
+
"piqa": 0.08487486839294434,
|
| 26 |
+
"openbook_qa": -0.02399998903274536,
|
| 27 |
+
"lambada_openai": 0.21191538870334625,
|
| 28 |
+
"hellaswag": 0.03910243511199951,
|
| 29 |
+
"winograd": 0.040293097496032715,
|
| 30 |
+
"winogrande": 0.0007892847061157227,
|
| 31 |
+
"bigbench_dyck_languages": 0.07200000435113907,
|
| 32 |
+
"agi_eval_lsat_ar": 0.07608695328235625,
|
| 33 |
+
"bigbench_cs_algorithms": 0.4371212124824524,
|
| 34 |
+
"bigbench_operators": 0.10476190596818924,
|
| 35 |
+
"bigbench_repeat_copy_logic": 0.0,
|
| 36 |
+
"squad": 0.020813623443245888,
|
| 37 |
+
"coqa": 0.07303018867969513,
|
| 38 |
+
"boolq": -0.2071463622544941,
|
| 39 |
+
"bigbench_language_identification": 0.18074805875553682
|
| 40 |
+
},
|
| 41 |
+
"conditioned_samples": [
|
| 42 |
+
{
|
| 43 |
+
"prompt": "The capital of France is",
|
| 44 |
+
"text": "<|bos|>The capital of France is the capital of the kingdom of France, and the capital of the kingdom of England"
|
| 45 |
+
},
|
| 46 |
+
{
|
| 47 |
+
"prompt": "The chemical symbol of gold is",
|
| 48 |
+
"text": "<|bos|>The chemical symbol of gold is the gold of the earth, and the gold of the earth is the gold of"
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"prompt": "If yesterday was Friday, then tomorrow will be",
|
| 52 |
+
"text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday, and the day will be Sunday. The day will be Sunday, and"
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"prompt": "The opposite of hot is",
|
| 56 |
+
"text": "<|bos|>The opposite of hot is the same as that of cold, and the same as that of hot. The"
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"prompt": "The planets of the solar system are:",
|
| 60 |
+
"text": "<|bos|>The planets of the solar system are: the sun, the moon, the stars, the planets, the sun, the"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"prompt": "My favorite color is",
|
| 64 |
+
"text": "<|bos|>My favorite color is the color of the sky, and the color of the earth. The color of"
|
| 65 |
+
},
|
| 66 |
+
{
|
| 67 |
+
"prompt": "If 5*x + 3 = 13, then x is",
|
| 68 |
+
"text": "<|bos|>If 5*x + 3 = 13, then x is the sum of the sum of the sum of the sum of the sum of the"
|
| 69 |
+
}
|
| 70 |
+
],
|
| 71 |
+
"unconditioned_samples": [
|
| 72 |
+
"<|bos|>CHAPTER VS. CREV. A. It has already been mentioned that Roger North and the Anglo- Saxon Chronicle and the reigns of Henry and Edward III. tell us that Eden was situated on England's Apple within the government of Richard II., and it must now be premised that England brought over men from Normandy and Anjou, and maintained them there. The folly and swiftness of our pilgrims were its source.\n\nProvidence was no idler in being driven over this island by the rivers of Canterbury and Canterbury, and thence overland to Rome; nor in being conveyed to Jerusalem, to meet the friendlessness of an ignorant man",
|
| 73 |
+
"<|bos|>PREFACE.\n\nTHE portions of this work, which, without modification of time, would be impossible to be within the little compass of the compass, are so valuable, that it must be judged more as a class; and all men, of every class, are judged by the same standards of skill, observed by the same standard of practice.\n\nThe first part of the Cloaca Maxima, which was written in print, is too exuberant and exuberant, to allow of annotation. \"But the question came up whilst we were considering the books,\" he adds, \"of Ministers, And the gentlemen that were about me, joined in the",
|
| 74 |
+
"<|bos|>PREFACE.\n\nix emphatic word is given to such relief; and in the language of filial affection or maternal emotion or affection, the language he employs is never \"the language of all true repentance and deep sorrow, but a resort\" of sighs or complaints to a Father to whom all are alike reconciled, whose infinite attributes are majesty, loving-kindness, mercy, and justice.\n\nIf the incipient man was a material object or an object, he would \"stalk;\" by indolence, by vice, by attraction, he would throw himself into a stagnant pool, throw into it little basins and merely onions, drenching",
|
| 75 |
+
"<|bos|> Dreams and Rejoice over my Lute and Pledge I've Soon Began to\n\nWish never the pains of keep as Oblation all my Steps I took to Narrow the bounds I had to set;\n\nExtanced at this Stage my Branding spowl yet\n\nSeemed it a rather\n\nSpeech and gn\n\nA considerable Quantity Flapping in my 80 whiteness forbid the Onely False ity did afford me;\n\nAnd whoever Fidelity's Man bring in all South about 100\n\nEtched upon Conscience an Evoluble Sport Gold upon there a Tremb Billail he Built I had little faith till",
|
| 76 |
+
"<|bos|>Drops, with apples nourishing them and lilies suiting them. It is a custom with corn-growers to mow loaves. It is thus with the rusted corn, which we call wheat because it is hard. If a may be properly branched out into layers and used in raising and other purposes, it is sometimes described as branched out simply for bed, and germinated like a cap or knife, into one or other of the ultimate layers. It sometimes happens that pipes are left, which being laid on the ground and decayed or damped in time as they decompose them, remain",
|
| 77 |
+
"<|bos|>VE ZABE. [Heir of the Superinto- FOW is a gener- Founding of the Morality | Sociessie, by the Origin of Lan- Turks Previous to his elevation strength of Language, very spe- ties, been endowed with this essential business, ago there are only three distinct untitutional faculties. But the reconstruction of modern sciences in the Shinto variant is a very daring one, and treats the sublimity of the Fourteenth century as such. Storia in three skins parapped off byila- Strumeton, Nev., drawn him away and merrie (Wat v. Ortsboth",
|
| 78 |
+
"<|bos|>PREFACE Ruth and Robin Hood suggested the latter an asylum in the outhouses at Oakum Orbeek, where the late Charles Lee, the publisher of 1 A little instalment of this and other historical works is printed at the end of 1872.\n\nPREFACE.\n\nto secure a place for retirement should he be ever called on to join the rebels. This was left the further task of keeping all the patent offices in the selection of Warwick the publisher of aegicide Liber Quintale. \"Master Pashan and the vae variarent,\" added Francis Gorton after the interview above to Pashan's title,",
|
| 79 |
+
"<|bos|>opers.\n\n CarthABELLA LUBI.\n\n221 however, still perseveres (1251 n. d.), and succeeded in breaking down the walls and overwhelming the rest with stones. 66 It appears,\" says Lubio (d.) I, \"that the free city is secure in its centre; and that I, who have interfered on behalf of my people at all times during my rule, can easily turn the safety of Smyrna when made famous by the massacre of the Amir and the surrender of the city on the appointed day.\" This is somewhat like about Cortez: \"Franciscus qui desitunt\n\n"
|
| 80 |
+
],
|
| 81 |
+
"training_time_seconds": 6341.746661424637,
|
| 82 |
+
"stage_training_flops": 1.0985538644638433e+18,
|
| 83 |
+
"inherited_parent_flops": 0.0,
|
| 84 |
+
"cumulative_pipeline_training_flops": 1.0985538644638433e+18,
|
| 85 |
+
"config_fingerprint": "4f01f653a56d0daf",
|
| 86 |
+
"git_commit_sha": "a47442614b208fd6df1549f6ca130504ca8975ed",
|
| 87 |
+
"wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/95708491",
|
| 88 |
+
"huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/clean1930s-d12-r11.25-muoneq-muonplus",
|
| 89 |
+
"dataset_fingerprint": "4af5759948d15ff0",
|
| 90 |
+
"tokenizer_fingerprint": "803338edbf3458e6",
|
| 91 |
+
"unique_train_tokens": 1275519304,
|
| 92 |
+
"effective_epochs": 0.970873786164196
|
| 93 |
+
}
|
experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer/experiment_tokenizer.json
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus",
|
| 3 |
+
"dataset": {
|
| 4 |
+
"adapter": "parquet_shards",
|
| 5 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 6 |
+
"revision": "main",
|
| 7 |
+
"validation_shard": 472,
|
| 8 |
+
"num_train_shards": 26,
|
| 9 |
+
"download_workers": 4
|
| 10 |
+
},
|
| 11 |
+
"tokenizer": {
|
| 12 |
+
"mode": "train",
|
| 13 |
+
"max_chars": 2000000000,
|
| 14 |
+
"doc_cap": 10000,
|
| 15 |
+
"vocab_size": 32768
|
| 16 |
+
},
|
| 17 |
+
"created_at": 1783965799
|
| 18 |
+
}
|
experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer/token_bytes.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4d25a126690db557828b34dc3bd3e8ebd615058016c1870e76b32b31266805ed
|
| 3 |
+
size 132649
|
experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer/tokenizer.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4932125f789267dbd571cc4bd19339ecfd7160c9b9cc0441ba6635756e4a8b0e
|
| 3 |
+
size 407747
|
experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_000500.json
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 500,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 5 |
+
"val_bpb": 1.3561541455317183,
|
| 6 |
+
"model_config": {
|
| 7 |
+
"sequence_len": 2048,
|
| 8 |
+
"vocab_size": 32768,
|
| 9 |
+
"n_layer": 12,
|
| 10 |
+
"n_head": 6,
|
| 11 |
+
"n_kv_head": 6,
|
| 12 |
+
"n_embd": 768,
|
| 13 |
+
"window_pattern": "L"
|
| 14 |
+
},
|
| 15 |
+
"user_config": {
|
| 16 |
+
"run": "clean1930s-d12-r11.25-randtok",
|
| 17 |
+
"wandb_run_id": "6b8f19a5",
|
| 18 |
+
"wandb_group": "clean1930s-d12",
|
| 19 |
+
"wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,randtok",
|
| 20 |
+
"device_type": "",
|
| 21 |
+
"fp8": false,
|
| 22 |
+
"fp8_recipe": "tensorwise",
|
| 23 |
+
"depth": 12,
|
| 24 |
+
"aspect_ratio": 64,
|
| 25 |
+
"head_dim": 128,
|
| 26 |
+
"max_seq_len": 2048,
|
| 27 |
+
"window_pattern": "L",
|
| 28 |
+
"num_iterations": -1,
|
| 29 |
+
"target_flops": -1.0,
|
| 30 |
+
"target_param_data_ratio": 11.25,
|
| 31 |
+
"device_batch_size": 16,
|
| 32 |
+
"total_batch_size": 524288,
|
| 33 |
+
"embedding_lr": 0.3,
|
| 34 |
+
"unembedding_lr": 0.008,
|
| 35 |
+
"weight_decay": 0.28,
|
| 36 |
+
"matrix_lr": 0.02,
|
| 37 |
+
"scalar_lr": 0.5,
|
| 38 |
+
"warmup_steps": 40,
|
| 39 |
+
"warmdown_ratio": 0.65,
|
| 40 |
+
"final_lr_frac": 0.05,
|
| 41 |
+
"resume_from_step": -1,
|
| 42 |
+
"pretokenized": true,
|
| 43 |
+
"data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/data",
|
| 44 |
+
"tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/tokenizer",
|
| 45 |
+
"pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/pretok",
|
| 46 |
+
"checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints",
|
| 47 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 48 |
+
"experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/config.json",
|
| 49 |
+
"tokenizer_fingerprint": "f01549dbe6e2aaa9",
|
| 50 |
+
"git_commit_sha": "118adeb2fc3350cf27ad635af372bd9b9410c0ef",
|
| 51 |
+
"seed": 42,
|
| 52 |
+
"eval_every": 250,
|
| 53 |
+
"eval_tokens": 2097152,
|
| 54 |
+
"core_metric_every": -1,
|
| 55 |
+
"core_metric_max_per_task": 500,
|
| 56 |
+
"sample_every": -1,
|
| 57 |
+
"save_every": 500,
|
| 58 |
+
"model_tag": "clean1930s-d12-r11.25-randtok",
|
| 59 |
+
"experiment": {
|
| 60 |
+
"schema_version": 1,
|
| 61 |
+
"stage": "base",
|
| 62 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 63 |
+
"dataset": {
|
| 64 |
+
"adapter": "parquet_shards",
|
| 65 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 66 |
+
"revision": "main",
|
| 67 |
+
"validation_shard": 472,
|
| 68 |
+
"num_train_shards": 26,
|
| 69 |
+
"download_workers": 4
|
| 70 |
+
},
|
| 71 |
+
"tokenizer": {
|
| 72 |
+
"mode": "train",
|
| 73 |
+
"max_chars": 2000000000,
|
| 74 |
+
"doc_cap": 10000,
|
| 75 |
+
"vocab_size": 32768,
|
| 76 |
+
"sampling": "random",
|
| 77 |
+
"sampling_seed": 42
|
| 78 |
+
},
|
| 79 |
+
"pretokenize": {
|
| 80 |
+
"enabled": true,
|
| 81 |
+
"slack": 1.03,
|
| 82 |
+
"val_tokens": 20971520,
|
| 83 |
+
"shard_tokens": 100000000,
|
| 84 |
+
"tokenizer_threads": 8
|
| 85 |
+
},
|
| 86 |
+
"training": {
|
| 87 |
+
"depth": 12,
|
| 88 |
+
"scaling_params": 110100912,
|
| 89 |
+
"target_param_data_ratio": 11.25,
|
| 90 |
+
"window_pattern": "L",
|
| 91 |
+
"device_batch_size": 16,
|
| 92 |
+
"total_batch_size": 524288,
|
| 93 |
+
"seed": 42,
|
| 94 |
+
"save_every": 500,
|
| 95 |
+
"eval_every": 250,
|
| 96 |
+
"eval_tokens": 2097152,
|
| 97 |
+
"core_metric_every": -1,
|
| 98 |
+
"sample_every": -1
|
| 99 |
+
},
|
| 100 |
+
"artifacts": {
|
| 101 |
+
"repo": "jbduran/think.nano"
|
| 102 |
+
},
|
| 103 |
+
"wandb": {
|
| 104 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 105 |
+
"project": "think.nano",
|
| 106 |
+
"name": "clean1930s-d12-r11.25-randtok",
|
| 107 |
+
"group": "clean1930s-d12",
|
| 108 |
+
"tags": [
|
| 109 |
+
"think-dataset-clean-1930s",
|
| 110 |
+
"d12",
|
| 111 |
+
"ratio11.25",
|
| 112 |
+
"randtok"
|
| 113 |
+
]
|
| 114 |
+
},
|
| 115 |
+
"config_fingerprint": "175184ba2ee0f5d3",
|
| 116 |
+
"artifact_path": "experiments/clean1930s-d12-r11.25-randtok"
|
| 117 |
+
},
|
| 118 |
+
"stage": "base",
|
| 119 |
+
"base_experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 120 |
+
"parent_experiment_id": null,
|
| 121 |
+
"parent_checkpoint_step": null,
|
| 122 |
+
"config_fingerprint": "175184ba2ee0f5d3"
|
| 123 |
+
},
|
| 124 |
+
"device_batch_size": 16,
|
| 125 |
+
"max_seq_len": 2048,
|
| 126 |
+
"total_batch_size": 524288,
|
| 127 |
+
"dataloader_state_dict": {
|
| 128 |
+
"file_idx": 2,
|
| 129 |
+
"pos": 62184769,
|
| 130 |
+
"epoch": 1,
|
| 131 |
+
"pq_idx": 2,
|
| 132 |
+
"rg_idx": 62184769
|
| 133 |
+
},
|
| 134 |
+
"loop_state": {
|
| 135 |
+
"min_val_bpb": 1.3561541455317183,
|
| 136 |
+
"smooth_train_loss": 3.8651697814606103,
|
| 137 |
+
"total_training_time": 1305.9748463630676,
|
| 138 |
+
"stage_training_flops": 232547388751872000,
|
| 139 |
+
"inherited_parent_flops": 0.0,
|
| 140 |
+
"cumulative_pipeline_training_flops": 232547388751872000
|
| 141 |
+
}
|
| 142 |
+
}
|
experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_001000.json
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 1000,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 5 |
+
"val_bpb": 1.2521959168139378,
|
| 6 |
+
"model_config": {
|
| 7 |
+
"sequence_len": 2048,
|
| 8 |
+
"vocab_size": 32768,
|
| 9 |
+
"n_layer": 12,
|
| 10 |
+
"n_head": 6,
|
| 11 |
+
"n_kv_head": 6,
|
| 12 |
+
"n_embd": 768,
|
| 13 |
+
"window_pattern": "L"
|
| 14 |
+
},
|
| 15 |
+
"user_config": {
|
| 16 |
+
"run": "clean1930s-d12-r11.25-randtok",
|
| 17 |
+
"wandb_run_id": "6b8f19a5",
|
| 18 |
+
"wandb_group": "clean1930s-d12",
|
| 19 |
+
"wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,randtok",
|
| 20 |
+
"device_type": "",
|
| 21 |
+
"fp8": false,
|
| 22 |
+
"fp8_recipe": "tensorwise",
|
| 23 |
+
"depth": 12,
|
| 24 |
+
"aspect_ratio": 64,
|
| 25 |
+
"head_dim": 128,
|
| 26 |
+
"max_seq_len": 2048,
|
| 27 |
+
"window_pattern": "L",
|
| 28 |
+
"num_iterations": -1,
|
| 29 |
+
"target_flops": -1.0,
|
| 30 |
+
"target_param_data_ratio": 11.25,
|
| 31 |
+
"device_batch_size": 16,
|
| 32 |
+
"total_batch_size": 524288,
|
| 33 |
+
"embedding_lr": 0.3,
|
| 34 |
+
"unembedding_lr": 0.008,
|
| 35 |
+
"weight_decay": 0.28,
|
| 36 |
+
"matrix_lr": 0.02,
|
| 37 |
+
"scalar_lr": 0.5,
|
| 38 |
+
"warmup_steps": 40,
|
| 39 |
+
"warmdown_ratio": 0.65,
|
| 40 |
+
"final_lr_frac": 0.05,
|
| 41 |
+
"resume_from_step": -1,
|
| 42 |
+
"pretokenized": true,
|
| 43 |
+
"data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/data",
|
| 44 |
+
"tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/tokenizer",
|
| 45 |
+
"pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/pretok",
|
| 46 |
+
"checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints",
|
| 47 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 48 |
+
"experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/config.json",
|
| 49 |
+
"tokenizer_fingerprint": "f01549dbe6e2aaa9",
|
| 50 |
+
"git_commit_sha": "118adeb2fc3350cf27ad635af372bd9b9410c0ef",
|
| 51 |
+
"seed": 42,
|
| 52 |
+
"eval_every": 250,
|
| 53 |
+
"eval_tokens": 2097152,
|
| 54 |
+
"core_metric_every": -1,
|
| 55 |
+
"core_metric_max_per_task": 500,
|
| 56 |
+
"sample_every": -1,
|
| 57 |
+
"save_every": 500,
|
| 58 |
+
"model_tag": "clean1930s-d12-r11.25-randtok",
|
| 59 |
+
"experiment": {
|
| 60 |
+
"schema_version": 1,
|
| 61 |
+
"stage": "base",
|
| 62 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 63 |
+
"dataset": {
|
| 64 |
+
"adapter": "parquet_shards",
|
| 65 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 66 |
+
"revision": "main",
|
| 67 |
+
"validation_shard": 472,
|
| 68 |
+
"num_train_shards": 26,
|
| 69 |
+
"download_workers": 4
|
| 70 |
+
},
|
| 71 |
+
"tokenizer": {
|
| 72 |
+
"mode": "train",
|
| 73 |
+
"max_chars": 2000000000,
|
| 74 |
+
"doc_cap": 10000,
|
| 75 |
+
"vocab_size": 32768,
|
| 76 |
+
"sampling": "random",
|
| 77 |
+
"sampling_seed": 42
|
| 78 |
+
},
|
| 79 |
+
"pretokenize": {
|
| 80 |
+
"enabled": true,
|
| 81 |
+
"slack": 1.03,
|
| 82 |
+
"val_tokens": 20971520,
|
| 83 |
+
"shard_tokens": 100000000,
|
| 84 |
+
"tokenizer_threads": 8
|
| 85 |
+
},
|
| 86 |
+
"training": {
|
| 87 |
+
"depth": 12,
|
| 88 |
+
"scaling_params": 110100912,
|
| 89 |
+
"target_param_data_ratio": 11.25,
|
| 90 |
+
"window_pattern": "L",
|
| 91 |
+
"device_batch_size": 16,
|
| 92 |
+
"total_batch_size": 524288,
|
| 93 |
+
"seed": 42,
|
| 94 |
+
"save_every": 500,
|
| 95 |
+
"eval_every": 250,
|
| 96 |
+
"eval_tokens": 2097152,
|
| 97 |
+
"core_metric_every": -1,
|
| 98 |
+
"sample_every": -1
|
| 99 |
+
},
|
| 100 |
+
"artifacts": {
|
| 101 |
+
"repo": "jbduran/think.nano"
|
| 102 |
+
},
|
| 103 |
+
"wandb": {
|
| 104 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 105 |
+
"project": "think.nano",
|
| 106 |
+
"name": "clean1930s-d12-r11.25-randtok",
|
| 107 |
+
"group": "clean1930s-d12",
|
| 108 |
+
"tags": [
|
| 109 |
+
"think-dataset-clean-1930s",
|
| 110 |
+
"d12",
|
| 111 |
+
"ratio11.25",
|
| 112 |
+
"randtok"
|
| 113 |
+
]
|
| 114 |
+
},
|
| 115 |
+
"config_fingerprint": "175184ba2ee0f5d3",
|
| 116 |
+
"artifact_path": "experiments/clean1930s-d12-r11.25-randtok"
|
| 117 |
+
},
|
| 118 |
+
"stage": "base",
|
| 119 |
+
"base_experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 120 |
+
"parent_experiment_id": null,
|
| 121 |
+
"parent_checkpoint_step": null,
|
| 122 |
+
"config_fingerprint": "175184ba2ee0f5d3"
|
| 123 |
+
},
|
| 124 |
+
"device_batch_size": 16,
|
| 125 |
+
"max_seq_len": 2048,
|
| 126 |
+
"total_batch_size": 524288,
|
| 127 |
+
"dataloader_state_dict": {
|
| 128 |
+
"file_idx": 5,
|
| 129 |
+
"pos": 24336769,
|
| 130 |
+
"epoch": 1,
|
| 131 |
+
"pq_idx": 5,
|
| 132 |
+
"rg_idx": 24336769
|
| 133 |
+
},
|
| 134 |
+
"loop_state": {
|
| 135 |
+
"min_val_bpb": 1.2521959168139378,
|
| 136 |
+
"smooth_train_loss": 3.5352641959529243,
|
| 137 |
+
"total_training_time": 2640.971613883972,
|
| 138 |
+
"stage_training_flops": 465094777503744000,
|
| 139 |
+
"inherited_parent_flops": 0.0,
|
| 140 |
+
"cumulative_pipeline_training_flops": 465094777503744000
|
| 141 |
+
}
|
| 142 |
+
}
|
experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_001500.json
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 1500,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 5 |
+
"val_bpb": 1.171838291812204,
|
| 6 |
+
"model_config": {
|
| 7 |
+
"sequence_len": 2048,
|
| 8 |
+
"vocab_size": 32768,
|
| 9 |
+
"n_layer": 12,
|
| 10 |
+
"n_head": 6,
|
| 11 |
+
"n_kv_head": 6,
|
| 12 |
+
"n_embd": 768,
|
| 13 |
+
"window_pattern": "L"
|
| 14 |
+
},
|
| 15 |
+
"user_config": {
|
| 16 |
+
"run": "clean1930s-d12-r11.25-randtok",
|
| 17 |
+
"wandb_run_id": "6b8f19a5",
|
| 18 |
+
"wandb_group": "clean1930s-d12",
|
| 19 |
+
"wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,randtok",
|
| 20 |
+
"device_type": "",
|
| 21 |
+
"fp8": false,
|
| 22 |
+
"fp8_recipe": "tensorwise",
|
| 23 |
+
"depth": 12,
|
| 24 |
+
"aspect_ratio": 64,
|
| 25 |
+
"head_dim": 128,
|
| 26 |
+
"max_seq_len": 2048,
|
| 27 |
+
"window_pattern": "L",
|
| 28 |
+
"num_iterations": -1,
|
| 29 |
+
"target_flops": -1.0,
|
| 30 |
+
"target_param_data_ratio": 11.25,
|
| 31 |
+
"device_batch_size": 16,
|
| 32 |
+
"total_batch_size": 524288,
|
| 33 |
+
"embedding_lr": 0.3,
|
| 34 |
+
"unembedding_lr": 0.008,
|
| 35 |
+
"weight_decay": 0.28,
|
| 36 |
+
"matrix_lr": 0.02,
|
| 37 |
+
"scalar_lr": 0.5,
|
| 38 |
+
"warmup_steps": 40,
|
| 39 |
+
"warmdown_ratio": 0.65,
|
| 40 |
+
"final_lr_frac": 0.05,
|
| 41 |
+
"resume_from_step": -1,
|
| 42 |
+
"pretokenized": true,
|
| 43 |
+
"data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/data",
|
| 44 |
+
"tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/tokenizer",
|
| 45 |
+
"pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/pretok",
|
| 46 |
+
"checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints",
|
| 47 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 48 |
+
"experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/config.json",
|
| 49 |
+
"tokenizer_fingerprint": "f01549dbe6e2aaa9",
|
| 50 |
+
"git_commit_sha": "118adeb2fc3350cf27ad635af372bd9b9410c0ef",
|
| 51 |
+
"seed": 42,
|
| 52 |
+
"eval_every": 250,
|
| 53 |
+
"eval_tokens": 2097152,
|
| 54 |
+
"core_metric_every": -1,
|
| 55 |
+
"core_metric_max_per_task": 500,
|
| 56 |
+
"sample_every": -1,
|
| 57 |
+
"save_every": 500,
|
| 58 |
+
"model_tag": "clean1930s-d12-r11.25-randtok",
|
| 59 |
+
"experiment": {
|
| 60 |
+
"schema_version": 1,
|
| 61 |
+
"stage": "base",
|
| 62 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 63 |
+
"dataset": {
|
| 64 |
+
"adapter": "parquet_shards",
|
| 65 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 66 |
+
"revision": "main",
|
| 67 |
+
"validation_shard": 472,
|
| 68 |
+
"num_train_shards": 26,
|
| 69 |
+
"download_workers": 4
|
| 70 |
+
},
|
| 71 |
+
"tokenizer": {
|
| 72 |
+
"mode": "train",
|
| 73 |
+
"max_chars": 2000000000,
|
| 74 |
+
"doc_cap": 10000,
|
| 75 |
+
"vocab_size": 32768,
|
| 76 |
+
"sampling": "random",
|
| 77 |
+
"sampling_seed": 42
|
| 78 |
+
},
|
| 79 |
+
"pretokenize": {
|
| 80 |
+
"enabled": true,
|
| 81 |
+
"slack": 1.03,
|
| 82 |
+
"val_tokens": 20971520,
|
| 83 |
+
"shard_tokens": 100000000,
|
| 84 |
+
"tokenizer_threads": 8
|
| 85 |
+
},
|
| 86 |
+
"training": {
|
| 87 |
+
"depth": 12,
|
| 88 |
+
"scaling_params": 110100912,
|
| 89 |
+
"target_param_data_ratio": 11.25,
|
| 90 |
+
"window_pattern": "L",
|
| 91 |
+
"device_batch_size": 16,
|
| 92 |
+
"total_batch_size": 524288,
|
| 93 |
+
"seed": 42,
|
| 94 |
+
"save_every": 500,
|
| 95 |
+
"eval_every": 250,
|
| 96 |
+
"eval_tokens": 2097152,
|
| 97 |
+
"core_metric_every": -1,
|
| 98 |
+
"sample_every": -1
|
| 99 |
+
},
|
| 100 |
+
"artifacts": {
|
| 101 |
+
"repo": "jbduran/think.nano"
|
| 102 |
+
},
|
| 103 |
+
"wandb": {
|
| 104 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 105 |
+
"project": "think.nano",
|
| 106 |
+
"name": "clean1930s-d12-r11.25-randtok",
|
| 107 |
+
"group": "clean1930s-d12",
|
| 108 |
+
"tags": [
|
| 109 |
+
"think-dataset-clean-1930s",
|
| 110 |
+
"d12",
|
| 111 |
+
"ratio11.25",
|
| 112 |
+
"randtok"
|
| 113 |
+
]
|
| 114 |
+
},
|
| 115 |
+
"config_fingerprint": "175184ba2ee0f5d3",
|
| 116 |
+
"artifact_path": "experiments/clean1930s-d12-r11.25-randtok"
|
| 117 |
+
},
|
| 118 |
+
"stage": "base",
|
| 119 |
+
"base_experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 120 |
+
"parent_experiment_id": null,
|
| 121 |
+
"parent_checkpoint_step": null,
|
| 122 |
+
"config_fingerprint": "175184ba2ee0f5d3"
|
| 123 |
+
},
|
| 124 |
+
"device_batch_size": 16,
|
| 125 |
+
"max_seq_len": 2048,
|
| 126 |
+
"total_batch_size": 524288,
|
| 127 |
+
"dataloader_state_dict": {
|
| 128 |
+
"file_idx": 7,
|
| 129 |
+
"pos": 86488769,
|
| 130 |
+
"epoch": 1,
|
| 131 |
+
"pq_idx": 7,
|
| 132 |
+
"rg_idx": 86488769
|
| 133 |
+
},
|
| 134 |
+
"loop_state": {
|
| 135 |
+
"min_val_bpb": 1.171838291812204,
|
| 136 |
+
"smooth_train_loss": 3.4159228083544537,
|
| 137 |
+
"total_training_time": 3976.167183637619,
|
| 138 |
+
"stage_training_flops": 697642166255616000,
|
| 139 |
+
"inherited_parent_flops": 0.0,
|
| 140 |
+
"cumulative_pipeline_training_flops": 697642166255616000
|
| 141 |
+
}
|
| 142 |
+
}
|
experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_002000.json
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 2000,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 5 |
+
"val_bpb": 1.1368966310332098,
|
| 6 |
+
"model_config": {
|
| 7 |
+
"sequence_len": 2048,
|
| 8 |
+
"vocab_size": 32768,
|
| 9 |
+
"n_layer": 12,
|
| 10 |
+
"n_head": 6,
|
| 11 |
+
"n_kv_head": 6,
|
| 12 |
+
"n_embd": 768,
|
| 13 |
+
"window_pattern": "L"
|
| 14 |
+
},
|
| 15 |
+
"user_config": {
|
| 16 |
+
"run": "clean1930s-d12-r11.25-randtok",
|
| 17 |
+
"wandb_run_id": "6b8f19a5",
|
| 18 |
+
"wandb_group": "clean1930s-d12",
|
| 19 |
+
"wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,randtok",
|
| 20 |
+
"device_type": "",
|
| 21 |
+
"fp8": false,
|
| 22 |
+
"fp8_recipe": "tensorwise",
|
| 23 |
+
"depth": 12,
|
| 24 |
+
"aspect_ratio": 64,
|
| 25 |
+
"head_dim": 128,
|
| 26 |
+
"max_seq_len": 2048,
|
| 27 |
+
"window_pattern": "L",
|
| 28 |
+
"num_iterations": -1,
|
| 29 |
+
"target_flops": -1.0,
|
| 30 |
+
"target_param_data_ratio": 11.25,
|
| 31 |
+
"device_batch_size": 16,
|
| 32 |
+
"total_batch_size": 524288,
|
| 33 |
+
"embedding_lr": 0.3,
|
| 34 |
+
"unembedding_lr": 0.008,
|
| 35 |
+
"weight_decay": 0.28,
|
| 36 |
+
"matrix_lr": 0.02,
|
| 37 |
+
"scalar_lr": 0.5,
|
| 38 |
+
"warmup_steps": 40,
|
| 39 |
+
"warmdown_ratio": 0.65,
|
| 40 |
+
"final_lr_frac": 0.05,
|
| 41 |
+
"resume_from_step": -1,
|
| 42 |
+
"pretokenized": true,
|
| 43 |
+
"data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/data",
|
| 44 |
+
"tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/tokenizer",
|
| 45 |
+
"pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/pretok",
|
| 46 |
+
"checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints",
|
| 47 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 48 |
+
"experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/config.json",
|
| 49 |
+
"tokenizer_fingerprint": "f01549dbe6e2aaa9",
|
| 50 |
+
"git_commit_sha": "118adeb2fc3350cf27ad635af372bd9b9410c0ef",
|
| 51 |
+
"seed": 42,
|
| 52 |
+
"eval_every": 250,
|
| 53 |
+
"eval_tokens": 2097152,
|
| 54 |
+
"core_metric_every": -1,
|
| 55 |
+
"core_metric_max_per_task": 500,
|
| 56 |
+
"sample_every": -1,
|
| 57 |
+
"save_every": 500,
|
| 58 |
+
"model_tag": "clean1930s-d12-r11.25-randtok",
|
| 59 |
+
"experiment": {
|
| 60 |
+
"schema_version": 1,
|
| 61 |
+
"stage": "base",
|
| 62 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 63 |
+
"dataset": {
|
| 64 |
+
"adapter": "parquet_shards",
|
| 65 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 66 |
+
"revision": "main",
|
| 67 |
+
"validation_shard": 472,
|
| 68 |
+
"num_train_shards": 26,
|
| 69 |
+
"download_workers": 4
|
| 70 |
+
},
|
| 71 |
+
"tokenizer": {
|
| 72 |
+
"mode": "train",
|
| 73 |
+
"max_chars": 2000000000,
|
| 74 |
+
"doc_cap": 10000,
|
| 75 |
+
"vocab_size": 32768,
|
| 76 |
+
"sampling": "random",
|
| 77 |
+
"sampling_seed": 42
|
| 78 |
+
},
|
| 79 |
+
"pretokenize": {
|
| 80 |
+
"enabled": true,
|
| 81 |
+
"slack": 1.03,
|
| 82 |
+
"val_tokens": 20971520,
|
| 83 |
+
"shard_tokens": 100000000,
|
| 84 |
+
"tokenizer_threads": 8
|
| 85 |
+
},
|
| 86 |
+
"training": {
|
| 87 |
+
"depth": 12,
|
| 88 |
+
"scaling_params": 110100912,
|
| 89 |
+
"target_param_data_ratio": 11.25,
|
| 90 |
+
"window_pattern": "L",
|
| 91 |
+
"device_batch_size": 16,
|
| 92 |
+
"total_batch_size": 524288,
|
| 93 |
+
"seed": 42,
|
| 94 |
+
"save_every": 500,
|
| 95 |
+
"eval_every": 250,
|
| 96 |
+
"eval_tokens": 2097152,
|
| 97 |
+
"core_metric_every": -1,
|
| 98 |
+
"sample_every": -1
|
| 99 |
+
},
|
| 100 |
+
"artifacts": {
|
| 101 |
+
"repo": "jbduran/think.nano"
|
| 102 |
+
},
|
| 103 |
+
"wandb": {
|
| 104 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 105 |
+
"project": "think.nano",
|
| 106 |
+
"name": "clean1930s-d12-r11.25-randtok",
|
| 107 |
+
"group": "clean1930s-d12",
|
| 108 |
+
"tags": [
|
| 109 |
+
"think-dataset-clean-1930s",
|
| 110 |
+
"d12",
|
| 111 |
+
"ratio11.25",
|
| 112 |
+
"randtok"
|
| 113 |
+
]
|
| 114 |
+
},
|
| 115 |
+
"config_fingerprint": "175184ba2ee0f5d3",
|
| 116 |
+
"artifact_path": "experiments/clean1930s-d12-r11.25-randtok"
|
| 117 |
+
},
|
| 118 |
+
"stage": "base",
|
| 119 |
+
"base_experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 120 |
+
"parent_experiment_id": null,
|
| 121 |
+
"parent_checkpoint_step": null,
|
| 122 |
+
"config_fingerprint": "175184ba2ee0f5d3"
|
| 123 |
+
},
|
| 124 |
+
"device_batch_size": 16,
|
| 125 |
+
"max_seq_len": 2048,
|
| 126 |
+
"total_batch_size": 524288,
|
| 127 |
+
"dataloader_state_dict": {
|
| 128 |
+
"file_idx": 10,
|
| 129 |
+
"pos": 48640769,
|
| 130 |
+
"epoch": 1,
|
| 131 |
+
"pq_idx": 10,
|
| 132 |
+
"rg_idx": 48640769
|
| 133 |
+
},
|
| 134 |
+
"loop_state": {
|
| 135 |
+
"min_val_bpb": 1.1368966310332098,
|
| 136 |
+
"smooth_train_loss": 3.136166264429775,
|
| 137 |
+
"total_training_time": 5310.99071097374,
|
| 138 |
+
"stage_training_flops": 930189555007488000,
|
| 139 |
+
"inherited_parent_flops": 0.0,
|
| 140 |
+
"cumulative_pipeline_training_flops": 930189555007488000
|
| 141 |
+
}
|
| 142 |
+
}
|
experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/meta_002362.json
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 2362,
|
| 3 |
+
"training_complete": true,
|
| 4 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 5 |
+
"val_bpb": 1.1128382305031206,
|
| 6 |
+
"model_config": {
|
| 7 |
+
"sequence_len": 2048,
|
| 8 |
+
"vocab_size": 32768,
|
| 9 |
+
"n_layer": 12,
|
| 10 |
+
"n_head": 6,
|
| 11 |
+
"n_kv_head": 6,
|
| 12 |
+
"n_embd": 768,
|
| 13 |
+
"window_pattern": "L"
|
| 14 |
+
},
|
| 15 |
+
"user_config": {
|
| 16 |
+
"run": "clean1930s-d12-r11.25-randtok",
|
| 17 |
+
"wandb_run_id": "6b8f19a5",
|
| 18 |
+
"wandb_group": "clean1930s-d12",
|
| 19 |
+
"wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,randtok",
|
| 20 |
+
"device_type": "",
|
| 21 |
+
"fp8": false,
|
| 22 |
+
"fp8_recipe": "tensorwise",
|
| 23 |
+
"depth": 12,
|
| 24 |
+
"aspect_ratio": 64,
|
| 25 |
+
"head_dim": 128,
|
| 26 |
+
"max_seq_len": 2048,
|
| 27 |
+
"window_pattern": "L",
|
| 28 |
+
"num_iterations": -1,
|
| 29 |
+
"target_flops": -1.0,
|
| 30 |
+
"target_param_data_ratio": 11.25,
|
| 31 |
+
"device_batch_size": 16,
|
| 32 |
+
"total_batch_size": 524288,
|
| 33 |
+
"embedding_lr": 0.3,
|
| 34 |
+
"unembedding_lr": 0.008,
|
| 35 |
+
"weight_decay": 0.28,
|
| 36 |
+
"matrix_lr": 0.02,
|
| 37 |
+
"scalar_lr": 0.5,
|
| 38 |
+
"warmup_steps": 40,
|
| 39 |
+
"warmdown_ratio": 0.65,
|
| 40 |
+
"final_lr_frac": 0.05,
|
| 41 |
+
"resume_from_step": -1,
|
| 42 |
+
"pretokenized": true,
|
| 43 |
+
"data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/data",
|
| 44 |
+
"tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/tokenizer",
|
| 45 |
+
"pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/pretok",
|
| 46 |
+
"checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/base_checkpoints",
|
| 47 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 48 |
+
"experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-randtok/config.json",
|
| 49 |
+
"tokenizer_fingerprint": "f01549dbe6e2aaa9",
|
| 50 |
+
"git_commit_sha": "118adeb2fc3350cf27ad635af372bd9b9410c0ef",
|
| 51 |
+
"seed": 42,
|
| 52 |
+
"eval_every": 250,
|
| 53 |
+
"eval_tokens": 2097152,
|
| 54 |
+
"core_metric_every": -1,
|
| 55 |
+
"core_metric_max_per_task": 500,
|
| 56 |
+
"sample_every": -1,
|
| 57 |
+
"save_every": 500,
|
| 58 |
+
"model_tag": "clean1930s-d12-r11.25-randtok",
|
| 59 |
+
"experiment": {
|
| 60 |
+
"schema_version": 1,
|
| 61 |
+
"stage": "base",
|
| 62 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 63 |
+
"dataset": {
|
| 64 |
+
"adapter": "parquet_shards",
|
| 65 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 66 |
+
"revision": "main",
|
| 67 |
+
"validation_shard": 472,
|
| 68 |
+
"num_train_shards": 26,
|
| 69 |
+
"download_workers": 4
|
| 70 |
+
},
|
| 71 |
+
"tokenizer": {
|
| 72 |
+
"mode": "train",
|
| 73 |
+
"max_chars": 2000000000,
|
| 74 |
+
"doc_cap": 10000,
|
| 75 |
+
"vocab_size": 32768,
|
| 76 |
+
"sampling": "random",
|
| 77 |
+
"sampling_seed": 42
|
| 78 |
+
},
|
| 79 |
+
"pretokenize": {
|
| 80 |
+
"enabled": true,
|
| 81 |
+
"slack": 1.03,
|
| 82 |
+
"val_tokens": 20971520,
|
| 83 |
+
"shard_tokens": 100000000,
|
| 84 |
+
"tokenizer_threads": 8
|
| 85 |
+
},
|
| 86 |
+
"training": {
|
| 87 |
+
"depth": 12,
|
| 88 |
+
"scaling_params": 110100912,
|
| 89 |
+
"target_param_data_ratio": 11.25,
|
| 90 |
+
"window_pattern": "L",
|
| 91 |
+
"device_batch_size": 16,
|
| 92 |
+
"total_batch_size": 524288,
|
| 93 |
+
"seed": 42,
|
| 94 |
+
"save_every": 500,
|
| 95 |
+
"eval_every": 250,
|
| 96 |
+
"eval_tokens": 2097152,
|
| 97 |
+
"core_metric_every": -1,
|
| 98 |
+
"sample_every": -1
|
| 99 |
+
},
|
| 100 |
+
"artifacts": {
|
| 101 |
+
"repo": "jbduran/think.nano"
|
| 102 |
+
},
|
| 103 |
+
"wandb": {
|
| 104 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 105 |
+
"project": "think.nano",
|
| 106 |
+
"name": "clean1930s-d12-r11.25-randtok",
|
| 107 |
+
"group": "clean1930s-d12",
|
| 108 |
+
"tags": [
|
| 109 |
+
"think-dataset-clean-1930s",
|
| 110 |
+
"d12",
|
| 111 |
+
"ratio11.25",
|
| 112 |
+
"randtok"
|
| 113 |
+
]
|
| 114 |
+
},
|
| 115 |
+
"config_fingerprint": "175184ba2ee0f5d3",
|
| 116 |
+
"artifact_path": "experiments/clean1930s-d12-r11.25-randtok"
|
| 117 |
+
},
|
| 118 |
+
"stage": "base",
|
| 119 |
+
"base_experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 120 |
+
"parent_experiment_id": null,
|
| 121 |
+
"parent_checkpoint_step": null,
|
| 122 |
+
"config_fingerprint": "175184ba2ee0f5d3"
|
| 123 |
+
},
|
| 124 |
+
"device_batch_size": 16,
|
| 125 |
+
"max_seq_len": 2048,
|
| 126 |
+
"total_batch_size": 524288,
|
| 127 |
+
"dataloader_state_dict": {
|
| 128 |
+
"file_idx": 12,
|
| 129 |
+
"pos": 38438817,
|
| 130 |
+
"epoch": 1,
|
| 131 |
+
"pq_idx": 12,
|
| 132 |
+
"rg_idx": 38438817
|
| 133 |
+
},
|
| 134 |
+
"loop_state": {
|
| 135 |
+
"min_val_bpb": 1.1128382305031206,
|
| 136 |
+
"smooth_train_loss": 3.185300296294561,
|
| 137 |
+
"total_training_time": 6277.177423000336,
|
| 138 |
+
"stage_training_flops": 1098553864463843328,
|
| 139 |
+
"inherited_parent_flops": 0.0,
|
| 140 |
+
"cumulative_pipeline_training_flops": 1098553864463843328
|
| 141 |
+
}
|
| 142 |
+
}
|
experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_000500.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ae8a604031071534c2cd16d6086cd495012f38121b53e2072da33e084a8b3ba0
|
| 3 |
+
size 792761690
|
experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_001000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:48630f071b47f0d89ea2572c606d32e32b1c143d345176add9aeaf81483ea490
|
| 3 |
+
size 792761690
|
experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_001500.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1359363c9b03f7dd9ae84fb5a606e06dfaaaeabf612168817daa388780ca4ed4
|
| 3 |
+
size 792761690
|
experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_002000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5cd9a38e9ecae268247a5a41d1710293cf7c9176b74660d0d3a36a86a9770302
|
| 3 |
+
size 792761690
|
experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/model_002362.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a72dbad509f7969cc0c8825a3078a402e31a466476b39573b8c638ac3d37a519
|
| 3 |
+
size 792761690
|
experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_000500_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:443ccb4c4d76adf1c33e0d0b3cacad3f29f522f07643ee180e6f461178593f51
|
| 3 |
+
size 1246165357
|
experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_001000_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:36325a3bba2b4d7461ae926bf79112585c2e52c5bf1c9f5847f5198713794add
|
| 3 |
+
size 1246165357
|
experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_001500_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7bb11ab9d0b1d54523b190dea5f37893ebd91858694403d047606196aa16887d
|
| 3 |
+
size 1246165357
|
experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_002000_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f6941a6e8678fc86b1041e974eed1173410cca4fea06167c76e96a64d5260f6a
|
| 3 |
+
size 1246165357
|
experiments/clean1930s-d12-r11.25-randtok/base_checkpoints/optim_002362_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0648d159d91067bff35675cba8b5828e792b5ef7eb442e6ec5ea883179e65a12
|
| 3 |
+
size 1246165357
|
experiments/clean1930s-d12-r11.25-randtok/config.json
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"stage": "base",
|
| 4 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 5 |
+
"dataset": {
|
| 6 |
+
"adapter": "parquet_shards",
|
| 7 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 8 |
+
"revision": "main",
|
| 9 |
+
"validation_shard": 472,
|
| 10 |
+
"num_train_shards": 26,
|
| 11 |
+
"download_workers": 4
|
| 12 |
+
},
|
| 13 |
+
"tokenizer": {
|
| 14 |
+
"mode": "train",
|
| 15 |
+
"max_chars": 2000000000,
|
| 16 |
+
"doc_cap": 10000,
|
| 17 |
+
"vocab_size": 32768,
|
| 18 |
+
"sampling": "random",
|
| 19 |
+
"sampling_seed": 42
|
| 20 |
+
},
|
| 21 |
+
"pretokenize": {
|
| 22 |
+
"enabled": true,
|
| 23 |
+
"slack": 1.03,
|
| 24 |
+
"val_tokens": 20971520,
|
| 25 |
+
"shard_tokens": 100000000,
|
| 26 |
+
"tokenizer_threads": 8
|
| 27 |
+
},
|
| 28 |
+
"training": {
|
| 29 |
+
"depth": 12,
|
| 30 |
+
"scaling_params": 110100912,
|
| 31 |
+
"target_param_data_ratio": 11.25,
|
| 32 |
+
"window_pattern": "L",
|
| 33 |
+
"device_batch_size": 16,
|
| 34 |
+
"total_batch_size": 524288,
|
| 35 |
+
"seed": 42,
|
| 36 |
+
"save_every": 500,
|
| 37 |
+
"eval_every": 250,
|
| 38 |
+
"eval_tokens": 2097152,
|
| 39 |
+
"core_metric_every": -1,
|
| 40 |
+
"sample_every": -1
|
| 41 |
+
},
|
| 42 |
+
"artifacts": {
|
| 43 |
+
"repo": "jbduran/think.nano"
|
| 44 |
+
},
|
| 45 |
+
"wandb": {
|
| 46 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 47 |
+
"project": "think.nano",
|
| 48 |
+
"name": "clean1930s-d12-r11.25-randtok",
|
| 49 |
+
"group": "clean1930s-d12",
|
| 50 |
+
"tags": [
|
| 51 |
+
"think-dataset-clean-1930s",
|
| 52 |
+
"d12",
|
| 53 |
+
"ratio11.25",
|
| 54 |
+
"randtok"
|
| 55 |
+
]
|
| 56 |
+
},
|
| 57 |
+
"config_fingerprint": "175184ba2ee0f5d3",
|
| 58 |
+
"artifact_path": "experiments/clean1930s-d12-r11.25-randtok"
|
| 59 |
+
}
|
experiments/clean1930s-d12-r11.25-randtok/evals/core.json
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "base_model (step 2362)",
|
| 3 |
+
"step": 2362,
|
| 4 |
+
"bpb": {},
|
| 5 |
+
"core_metric": 0.05120713426476866,
|
| 6 |
+
"core_results": {
|
| 7 |
+
"hellaswag_zeroshot": 0.2790280878543854,
|
| 8 |
+
"jeopardy": 0.0004723665479104966,
|
| 9 |
+
"bigbench_qa_wikidata": 0.07145317643880844,
|
| 10 |
+
"arc_easy": 0.31607744097709656,
|
| 11 |
+
"arc_challenge": 0.20563139021396637,
|
| 12 |
+
"copa": 0.4899999797344208,
|
| 13 |
+
"commonsense_qa": 0.3054873049259186,
|
| 14 |
+
"piqa": 0.5429815053939819,
|
| 15 |
+
"openbook_qa": 0.23200000822544098,
|
| 16 |
+
"lambada_openai": 0.19153890013694763,
|
| 17 |
+
"hellaswag": 0.27853015065193176,
|
| 18 |
+
"winograd": 0.5604395866394043,
|
| 19 |
+
"winogrande": 0.49486976861953735,
|
| 20 |
+
"bigbench_dyck_languages": 0.08400000631809235,
|
| 21 |
+
"agi_eval_lsat_ar": 0.30869564414024353,
|
| 22 |
+
"bigbench_cs_algorithms": 0.4204545319080353,
|
| 23 |
+
"bigbench_operators": 0.07619047909975052,
|
| 24 |
+
"bigbench_repeat_copy_logic": 0.0,
|
| 25 |
+
"squad": 0.021948911249637604,
|
| 26 |
+
"coqa": 0.05273706465959549,
|
| 27 |
+
"boolq": 0.430886834859848,
|
| 28 |
+
"bigbench_language_identification": 0.2540999948978424
|
| 29 |
+
},
|
| 30 |
+
"centered_results": {
|
| 31 |
+
"hellaswag_zeroshot": 0.038704117139180504,
|
| 32 |
+
"jeopardy": 0.0004723665479104966,
|
| 33 |
+
"bigbench_qa_wikidata": 0.07145317643880844,
|
| 34 |
+
"arc_easy": 0.08810325463612874,
|
| 35 |
+
"arc_challenge": -0.059158146381378174,
|
| 36 |
+
"copa": -0.020000040531158447,
|
| 37 |
+
"commonsense_qa": 0.1318591311573982,
|
| 38 |
+
"piqa": 0.08596301078796387,
|
| 39 |
+
"openbook_qa": -0.02399998903274536,
|
| 40 |
+
"lambada_openai": 0.19153890013694763,
|
| 41 |
+
"hellaswag": 0.03804020086924235,
|
| 42 |
+
"winograd": 0.1208791732788086,
|
| 43 |
+
"winogrande": -0.010260462760925293,
|
| 44 |
+
"bigbench_dyck_languages": 0.08400000631809235,
|
| 45 |
+
"agi_eval_lsat_ar": 0.13586955517530439,
|
| 46 |
+
"bigbench_cs_algorithms": 0.4204545319080353,
|
| 47 |
+
"bigbench_operators": 0.07619047909975052,
|
| 48 |
+
"bigbench_repeat_copy_logic": 0.0,
|
| 49 |
+
"squad": 0.021948911249637604,
|
| 50 |
+
"coqa": 0.05273706465959549,
|
| 51 |
+
"boolq": -0.4976662240530315,
|
| 52 |
+
"bigbench_language_identification": 0.17942793718134478
|
| 53 |
+
},
|
| 54 |
+
"conditioned_samples": [],
|
| 55 |
+
"unconditioned_samples": []
|
| 56 |
+
}
|
experiments/clean1930s-d12-r11.25-randtok/evals/samples.json
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "base_model (step 2362)",
|
| 3 |
+
"step": 2362,
|
| 4 |
+
"bpb": {},
|
| 5 |
+
"core_metric": null,
|
| 6 |
+
"core_results": null,
|
| 7 |
+
"centered_results": null,
|
| 8 |
+
"conditioned_samples": [
|
| 9 |
+
{
|
| 10 |
+
"prompt": "The capital of France is",
|
| 11 |
+
"text": "<|bos|>The capital of France is the capital of the nation, and the capital of the nation is the capital of"
|
| 12 |
+
},
|
| 13 |
+
{
|
| 14 |
+
"prompt": "The chemical symbol of gold is",
|
| 15 |
+
"text": "<|bos|>The chemical symbol of gold is the same as that of silver, and the latter is the same as the former"
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"prompt": "If yesterday was Friday, then tomorrow will be",
|
| 19 |
+
"text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday, and the day of the week will be Sunday. The day of the"
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"prompt": "The opposite of hot is",
|
| 23 |
+
"text": "<|bos|>The opposite of hot is the most common form of the word. The word is used in the sense of"
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"prompt": "The planets of the solar system are:",
|
| 27 |
+
"text": "<|bos|>The planets of the solar system are: the sun, the moon, the stars, the planets, the sun, the"
|
| 28 |
+
},
|
| 29 |
+
{
|
| 30 |
+
"prompt": "My favorite color is",
|
| 31 |
+
"text": "<|bos|>My favorite color is a very fine one, and I have never seen it in my life. I"
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"prompt": "If 5*x + 3 = 13, then x is",
|
| 35 |
+
"text": "<|bos|>If 5*x + 3 = 13, then x is the number of the parts of the body, and x is the number of the"
|
| 36 |
+
}
|
| 37 |
+
],
|
| 38 |
+
"unconditioned_samples": [
|
| 39 |
+
"<|bos|>ccliary on a certain phase of life-into which the crews of vessels of war have their chief and are fortunate in holding the chief place at the point of the stroke-what second seeks life artillery fortification in the future, the young shoot; and resisting a longer bullet-wounded tale in order to conform to the demands of business, declare openly, what is the duty of these young officers. 339\n\nINTRODUCTION\n\nAn indefatigable fellow who has attained middle life experiences abundant satisfaction no less keen on such occasions than when a fighter's son with ardour in the fray will, having crowned his freshness with laurels of strong individuality and shouting the",
|
| 40 |
+
"<|bos|>CHAPTER XVIII.\n\nPersonal Voyages in Shropshire.- Voyage to Lourdes.-Inland Voyages to Madison County. - Impressments upon the northern New\n\nHampshire Army.-The King entertained Chancellor Kent.-The French Mail-Scope.\n\n-Comparison of Narrative given by Prince Hoofloy. - Encounters and villages of the Church of Lourdes.-No Severance of Early Days.-The\n\nEnglish secure coal from Pilgrim Hall.--Mr. De Chaumier's\"-Public Windows.- Cholera of the Country.-(Robert Hall's)-Return to London before Alvare",
|
| 41 |
+
"<|bos|>PREFACE.\n\nix it in the seventh book to the lines full and true. ritie Mc-\n\nMullen, receiving theory kinde of his essay on \"The Heart and the Heart.\"\n\nSomebody has written of my scholarship, and mine ability. But no apology cately based on facts has beamed from the lips of Shobaugh: 'Tis ake win' and 'tis ake ake win\n\nBy early discovery eke of the common 'tall Hhor.'\n\nYoung Horace, who stands as fellow-musical with the immigration of\n\nClarkes and Iredellians; was born in the Pacific Alps",
|
| 42 |
+
"<|bos|>The company will draw on its rent at Five Pence.\n\nTail & Doors set up for Assoc. J. Glasgow Modern Greek\n\nSons, Parts 16 and 17.\n\nTECHNICAL PICTURES.\n\nIntroductory.\n\nThe writer of the foregoing points out two systems of pillaging as the result: one for intent and one for specialty, regardless of cost and charges; but as elements in analysis, the aims of the permisi Ph. and 3 years\n\nTail & Dooors loud. $ength. 61 a. 50 asses emmen. 25 phied ",
|
| 43 |
+
"<|bos|>ENDORION COURTISEMENT\n\nCOPYRIGHT, 1901, BY\n\nPHILIP S. ROGERS Esquire to the Honorable the Honorable the Board of Admiralty, Admiralty, &c.,:\n\n(MARY\n\nN. ROGERS Mary Heptam. The Hon. H. H.\n\nMackintosh.\n\nThe Honorable the Honorable the Honorable the Honorable the Queen Margot, and the Countess of\n\nPoultney, do certify that the following are the names of the persons named in the above-mentioned proposals, and the quantities of gold the said persons have Ibrahim the Harriethbout King etc., which",
|
| 44 |
+
"<|bos|>onica, 1. 3. 22 and weissen, 233, 4 Supr. 455.\n\nYAWARD, See BARPAIN ON.\n\n1 On my official report of my public duties in chief, Nays Lincoln, Boston, To-day, 31st December, 1861. [28th June, 1862] in\n\nWAR DEPARTMENT,\n\nWASHINGTON, June 10, 1863. \"MY DEAR BROTHER: By letter I received yesterday the letter written to you, father, father, and uncle, and sent you because you do not belong to them who wrote",
|
| 45 |
+
"<|bos|>AV-Government of the Six Nations, so long as it exists, is materially weak, and may be ultimately superseded, by the Government of Modern India itself; and a ministry in England may turn it in its administration into a formidable force, which it cannot cope with in India, as a subordinate agency, by extraordinary administrative talents, required for its focus; and in default may render futile any further offensive or nugatory effort to the natives of India. As a temporary agent, the Council put itself under the protection of\n\nJacob Grimm, an Edinburgh man, who, like him, was under orders.\n\nThe senior partner of the same",
|
| 46 |
+
"<|bos|>PREFACE.\n\nI scruple not to reiterate this impression, and to acquiesce, with grateful emotions, in the misfortunes which late, my dear Mr. Ballantyne, have Providence permitted for my own sake.\n\nIt is impossible that the author of a set of printed memoirs relating to Scotland could not find an excuse for having discovered his friend's address on a paper called undoubtedly \"The Popular Musick Services,\" published in the year-book of the year 1822 --a paper wholly in blank, for sale in Kensington Gardens, where it happened to have that particular merit, and on whose reception he thought it only right to use his name whenever"
|
| 47 |
+
]
|
| 48 |
+
}
|
experiments/clean1930s-d12-r11.25-randtok/evals/val_bpb.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "base_model (step 2362)",
|
| 3 |
+
"step": 2362,
|
| 4 |
+
"bpb": {
|
| 5 |
+
"val": 1.053600922794229
|
| 6 |
+
},
|
| 7 |
+
"core_metric": null,
|
| 8 |
+
"core_results": null,
|
| 9 |
+
"centered_results": null,
|
| 10 |
+
"conditioned_samples": [],
|
| 11 |
+
"unconditioned_samples": []
|
| 12 |
+
}
|
experiments/clean1930s-d12-r11.25-randtok/run.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 3 |
+
"stage": "base",
|
| 4 |
+
"base_experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 5 |
+
"parent_experiment_id": null,
|
| 6 |
+
"parent_checkpoint_step": null,
|
| 7 |
+
"config_fingerprint": "175184ba2ee0f5d3",
|
| 8 |
+
"wandb_run_id": "6b8f19a5",
|
| 9 |
+
"created_at": 1784310529
|
| 10 |
+
}
|
experiments/clean1930s-d12-r11.25-randtok/summary.json
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 3 |
+
"stage": "base",
|
| 4 |
+
"base_experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 5 |
+
"parent_experiment_id": null,
|
| 6 |
+
"parent_checkpoint_step": null,
|
| 7 |
+
"dataset": "jbduran/think-dataset-clean-1930s",
|
| 8 |
+
"dataset_revision": "main",
|
| 9 |
+
"step": 2362,
|
| 10 |
+
"depth": 12,
|
| 11 |
+
"target_param_data_ratio": 11.25,
|
| 12 |
+
"training_tokens": 1238368256,
|
| 13 |
+
"final_sampled_val_bpb": 1.1128382305031206,
|
| 14 |
+
"minimum_sampled_val_bpb": 1.1128382305031206,
|
| 15 |
+
"full_val_bpb": 1.053600922794229,
|
| 16 |
+
"core_metric": 0.05120713426476866,
|
| 17 |
+
"centered_results": {
|
| 18 |
+
"hellaswag_zeroshot": 0.038704117139180504,
|
| 19 |
+
"jeopardy": 0.0004723665479104966,
|
| 20 |
+
"bigbench_qa_wikidata": 0.07145317643880844,
|
| 21 |
+
"arc_easy": 0.08810325463612874,
|
| 22 |
+
"arc_challenge": -0.059158146381378174,
|
| 23 |
+
"copa": -0.020000040531158447,
|
| 24 |
+
"commonsense_qa": 0.1318591311573982,
|
| 25 |
+
"piqa": 0.08596301078796387,
|
| 26 |
+
"openbook_qa": -0.02399998903274536,
|
| 27 |
+
"lambada_openai": 0.19153890013694763,
|
| 28 |
+
"hellaswag": 0.03804020086924235,
|
| 29 |
+
"winograd": 0.1208791732788086,
|
| 30 |
+
"winogrande": -0.010260462760925293,
|
| 31 |
+
"bigbench_dyck_languages": 0.08400000631809235,
|
| 32 |
+
"agi_eval_lsat_ar": 0.13586955517530439,
|
| 33 |
+
"bigbench_cs_algorithms": 0.4204545319080353,
|
| 34 |
+
"bigbench_operators": 0.07619047909975052,
|
| 35 |
+
"bigbench_repeat_copy_logic": 0.0,
|
| 36 |
+
"squad": 0.021948911249637604,
|
| 37 |
+
"coqa": 0.05273706465959549,
|
| 38 |
+
"boolq": -0.4976662240530315,
|
| 39 |
+
"bigbench_language_identification": 0.17942793718134478
|
| 40 |
+
},
|
| 41 |
+
"conditioned_samples": [
|
| 42 |
+
{
|
| 43 |
+
"prompt": "The capital of France is",
|
| 44 |
+
"text": "<|bos|>The capital of France is the capital of the nation, and the capital of the nation is the capital of"
|
| 45 |
+
},
|
| 46 |
+
{
|
| 47 |
+
"prompt": "The chemical symbol of gold is",
|
| 48 |
+
"text": "<|bos|>The chemical symbol of gold is the same as that of silver, and the latter is the same as the former"
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"prompt": "If yesterday was Friday, then tomorrow will be",
|
| 52 |
+
"text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday, and the day of the week will be Sunday. The day of the"
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"prompt": "The opposite of hot is",
|
| 56 |
+
"text": "<|bos|>The opposite of hot is the most common form of the word. The word is used in the sense of"
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"prompt": "The planets of the solar system are:",
|
| 60 |
+
"text": "<|bos|>The planets of the solar system are: the sun, the moon, the stars, the planets, the sun, the"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"prompt": "My favorite color is",
|
| 64 |
+
"text": "<|bos|>My favorite color is a very fine one, and I have never seen it in my life. I"
|
| 65 |
+
},
|
| 66 |
+
{
|
| 67 |
+
"prompt": "If 5*x + 3 = 13, then x is",
|
| 68 |
+
"text": "<|bos|>If 5*x + 3 = 13, then x is the number of the parts of the body, and x is the number of the"
|
| 69 |
+
}
|
| 70 |
+
],
|
| 71 |
+
"unconditioned_samples": [
|
| 72 |
+
"<|bos|>ccliary on a certain phase of life-into which the crews of vessels of war have their chief and are fortunate in holding the chief place at the point of the stroke-what second seeks life artillery fortification in the future, the young shoot; and resisting a longer bullet-wounded tale in order to conform to the demands of business, declare openly, what is the duty of these young officers. 339\n\nINTRODUCTION\n\nAn indefatigable fellow who has attained middle life experiences abundant satisfaction no less keen on such occasions than when a fighter's son with ardour in the fray will, having crowned his freshness with laurels of strong individuality and shouting the",
|
| 73 |
+
"<|bos|>CHAPTER XVIII.\n\nPersonal Voyages in Shropshire.- Voyage to Lourdes.-Inland Voyages to Madison County. - Impressments upon the northern New\n\nHampshire Army.-The King entertained Chancellor Kent.-The French Mail-Scope.\n\n-Comparison of Narrative given by Prince Hoofloy. - Encounters and villages of the Church of Lourdes.-No Severance of Early Days.-The\n\nEnglish secure coal from Pilgrim Hall.--Mr. De Chaumier's\"-Public Windows.- Cholera of the Country.-(Robert Hall's)-Return to London before Alvare",
|
| 74 |
+
"<|bos|>PREFACE.\n\nix it in the seventh book to the lines full and true. ritie Mc-\n\nMullen, receiving theory kinde of his essay on \"The Heart and the Heart.\"\n\nSomebody has written of my scholarship, and mine ability. But no apology cately based on facts has beamed from the lips of Shobaugh: 'Tis ake win' and 'tis ake ake win\n\nBy early discovery eke of the common 'tall Hhor.'\n\nYoung Horace, who stands as fellow-musical with the immigration of\n\nClarkes and Iredellians; was born in the Pacific Alps",
|
| 75 |
+
"<|bos|>The company will draw on its rent at Five Pence.\n\nTail & Doors set up for Assoc. J. Glasgow Modern Greek\n\nSons, Parts 16 and 17.\n\nTECHNICAL PICTURES.\n\nIntroductory.\n\nThe writer of the foregoing points out two systems of pillaging as the result: one for intent and one for specialty, regardless of cost and charges; but as elements in analysis, the aims of the permisi Ph. and 3 years\n\nTail & Dooors loud. $ength. 61 a. 50 asses emmen. 25 phied ",
|
| 76 |
+
"<|bos|>ENDORION COURTISEMENT\n\nCOPYRIGHT, 1901, BY\n\nPHILIP S. ROGERS Esquire to the Honorable the Honorable the Board of Admiralty, Admiralty, &c.,:\n\n(MARY\n\nN. ROGERS Mary Heptam. The Hon. H. H.\n\nMackintosh.\n\nThe Honorable the Honorable the Honorable the Honorable the Queen Margot, and the Countess of\n\nPoultney, do certify that the following are the names of the persons named in the above-mentioned proposals, and the quantities of gold the said persons have Ibrahim the Harriethbout King etc., which",
|
| 77 |
+
"<|bos|>onica, 1. 3. 22 and weissen, 233, 4 Supr. 455.\n\nYAWARD, See BARPAIN ON.\n\n1 On my official report of my public duties in chief, Nays Lincoln, Boston, To-day, 31st December, 1861. [28th June, 1862] in\n\nWAR DEPARTMENT,\n\nWASHINGTON, June 10, 1863. \"MY DEAR BROTHER: By letter I received yesterday the letter written to you, father, father, and uncle, and sent you because you do not belong to them who wrote",
|
| 78 |
+
"<|bos|>AV-Government of the Six Nations, so long as it exists, is materially weak, and may be ultimately superseded, by the Government of Modern India itself; and a ministry in England may turn it in its administration into a formidable force, which it cannot cope with in India, as a subordinate agency, by extraordinary administrative talents, required for its focus; and in default may render futile any further offensive or nugatory effort to the natives of India. As a temporary agent, the Council put itself under the protection of\n\nJacob Grimm, an Edinburgh man, who, like him, was under orders.\n\nThe senior partner of the same",
|
| 79 |
+
"<|bos|>PREFACE.\n\nI scruple not to reiterate this impression, and to acquiesce, with grateful emotions, in the misfortunes which late, my dear Mr. Ballantyne, have Providence permitted for my own sake.\n\nIt is impossible that the author of a set of printed memoirs relating to Scotland could not find an excuse for having discovered his friend's address on a paper called undoubtedly \"The Popular Musick Services,\" published in the year-book of the year 1822 --a paper wholly in blank, for sale in Kensington Gardens, where it happened to have that particular merit, and on whose reception he thought it only right to use his name whenever"
|
| 80 |
+
],
|
| 81 |
+
"training_time_seconds": 6277.177423000336,
|
| 82 |
+
"stage_training_flops": 1.0985538644638433e+18,
|
| 83 |
+
"inherited_parent_flops": 0.0,
|
| 84 |
+
"cumulative_pipeline_training_flops": 1.0985538644638433e+18,
|
| 85 |
+
"config_fingerprint": "175184ba2ee0f5d3",
|
| 86 |
+
"git_commit_sha": "118adeb2fc3350cf27ad635af372bd9b9410c0ef",
|
| 87 |
+
"wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/6b8f19a5",
|
| 88 |
+
"huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/clean1930s-d12-r11.25-randtok",
|
| 89 |
+
"dataset_fingerprint": "4af5759948d15ff0",
|
| 90 |
+
"tokenizer_fingerprint": "f01549dbe6e2aaa9",
|
| 91 |
+
"unique_train_tokens": 1275519304,
|
| 92 |
+
"effective_epochs": 0.970873786164196
|
| 93 |
+
}
|
experiments/clean1930s-d12-r11.25-randtok/tokenizer/experiment_tokenizer.json
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment_id": "clean1930s-d12-r11.25-randtok",
|
| 3 |
+
"dataset": {
|
| 4 |
+
"adapter": "parquet_shards",
|
| 5 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 6 |
+
"revision": "main",
|
| 7 |
+
"validation_shard": 472,
|
| 8 |
+
"num_train_shards": 26,
|
| 9 |
+
"download_workers": 4
|
| 10 |
+
},
|
| 11 |
+
"tokenizer": {
|
| 12 |
+
"mode": "train",
|
| 13 |
+
"max_chars": 2000000000,
|
| 14 |
+
"doc_cap": 10000,
|
| 15 |
+
"vocab_size": 32768,
|
| 16 |
+
"sampling": "random",
|
| 17 |
+
"sampling_seed": 42
|
| 18 |
+
},
|
| 19 |
+
"created_at": 1784310546
|
| 20 |
+
}
|
experiments/clean1930s-d12-r11.25-randtok/tokenizer/token_bytes.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b9de5ef70a8805ae1e23ed607d05ab79cf92582f123294be962ca714ff0664ec
|
| 3 |
+
size 132649
|
experiments/clean1930s-d12-r11.25-randtok/tokenizer/tokenizer.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:21d846dc85121d501a10b42cacd77297a6358e43203f03c320f64c2c395ba894
|
| 3 |
+
size 411592
|
experiments/clean1930s-d12-r11.25/base_checkpoints/meta_000500.json
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 500,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "clean1930s-d12-r11.25",
|
| 5 |
+
"val_bpb": 1.3406276768636045,
|
| 6 |
+
"model_config": {
|
| 7 |
+
"sequence_len": 2048,
|
| 8 |
+
"vocab_size": 32768,
|
| 9 |
+
"n_layer": 12,
|
| 10 |
+
"n_head": 6,
|
| 11 |
+
"n_kv_head": 6,
|
| 12 |
+
"n_embd": 768,
|
| 13 |
+
"window_pattern": "L"
|
| 14 |
+
},
|
| 15 |
+
"user_config": {
|
| 16 |
+
"run": "clean1930s-d12-r11.25",
|
| 17 |
+
"wandb_run_id": "764ab08e",
|
| 18 |
+
"wandb_group": "clean1930s-d12",
|
| 19 |
+
"wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25",
|
| 20 |
+
"device_type": "",
|
| 21 |
+
"fp8": false,
|
| 22 |
+
"fp8_recipe": "tensorwise",
|
| 23 |
+
"depth": 12,
|
| 24 |
+
"aspect_ratio": 64,
|
| 25 |
+
"head_dim": 128,
|
| 26 |
+
"max_seq_len": 2048,
|
| 27 |
+
"window_pattern": "L",
|
| 28 |
+
"num_iterations": -1,
|
| 29 |
+
"target_flops": -1.0,
|
| 30 |
+
"target_param_data_ratio": 11.25,
|
| 31 |
+
"device_batch_size": 16,
|
| 32 |
+
"total_batch_size": 524288,
|
| 33 |
+
"embedding_lr": 0.3,
|
| 34 |
+
"unembedding_lr": 0.008,
|
| 35 |
+
"weight_decay": 0.28,
|
| 36 |
+
"matrix_lr": 0.02,
|
| 37 |
+
"scalar_lr": 0.5,
|
| 38 |
+
"warmup_steps": 40,
|
| 39 |
+
"warmdown_ratio": 0.65,
|
| 40 |
+
"final_lr_frac": 0.05,
|
| 41 |
+
"resume_from_step": -1,
|
| 42 |
+
"pretokenized": true,
|
| 43 |
+
"data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/data",
|
| 44 |
+
"tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/tokenizer",
|
| 45 |
+
"pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/pretok",
|
| 46 |
+
"checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/base_checkpoints",
|
| 47 |
+
"experiment_id": "clean1930s-d12-r11.25",
|
| 48 |
+
"experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/config.json",
|
| 49 |
+
"tokenizer_fingerprint": "fa7b147cb0e4b5db",
|
| 50 |
+
"git_commit_sha": "083cd7f99484b5e894a23e4a093de6d339c412ea",
|
| 51 |
+
"seed": 42,
|
| 52 |
+
"eval_every": 250,
|
| 53 |
+
"eval_tokens": 2097152,
|
| 54 |
+
"core_metric_every": -1,
|
| 55 |
+
"core_metric_max_per_task": 500,
|
| 56 |
+
"sample_every": -1,
|
| 57 |
+
"save_every": 500,
|
| 58 |
+
"model_tag": "clean1930s-d12-r11.25",
|
| 59 |
+
"experiment": {
|
| 60 |
+
"schema_version": 1,
|
| 61 |
+
"stage": "base",
|
| 62 |
+
"experiment_id": "clean1930s-d12-r11.25",
|
| 63 |
+
"dataset": {
|
| 64 |
+
"adapter": "parquet_shards",
|
| 65 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 66 |
+
"revision": "main",
|
| 67 |
+
"validation_shard": 472,
|
| 68 |
+
"num_train_shards": 26,
|
| 69 |
+
"download_workers": 4
|
| 70 |
+
},
|
| 71 |
+
"tokenizer": {
|
| 72 |
+
"mode": "train",
|
| 73 |
+
"max_chars": 2000000000,
|
| 74 |
+
"doc_cap": 10000,
|
| 75 |
+
"vocab_size": 32768
|
| 76 |
+
},
|
| 77 |
+
"pretokenize": {
|
| 78 |
+
"enabled": true,
|
| 79 |
+
"slack": 1.03,
|
| 80 |
+
"val_tokens": 20971520,
|
| 81 |
+
"shard_tokens": 100000000,
|
| 82 |
+
"tokenizer_threads": 8
|
| 83 |
+
},
|
| 84 |
+
"training": {
|
| 85 |
+
"depth": 12,
|
| 86 |
+
"scaling_params": 110100912,
|
| 87 |
+
"target_param_data_ratio": 11.25,
|
| 88 |
+
"window_pattern": "L",
|
| 89 |
+
"device_batch_size": 16,
|
| 90 |
+
"total_batch_size": 524288,
|
| 91 |
+
"seed": 42,
|
| 92 |
+
"save_every": 500,
|
| 93 |
+
"eval_every": 250,
|
| 94 |
+
"eval_tokens": 2097152,
|
| 95 |
+
"core_metric_every": -1,
|
| 96 |
+
"sample_every": -1
|
| 97 |
+
},
|
| 98 |
+
"artifacts": {
|
| 99 |
+
"repo": "jbduran/think.nano"
|
| 100 |
+
},
|
| 101 |
+
"wandb": {
|
| 102 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 103 |
+
"project": "think.nano",
|
| 104 |
+
"name": "clean1930s-d12-r11.25",
|
| 105 |
+
"group": "clean1930s-d12",
|
| 106 |
+
"tags": [
|
| 107 |
+
"think-dataset-clean-1930s",
|
| 108 |
+
"d12",
|
| 109 |
+
"ratio11.25"
|
| 110 |
+
]
|
| 111 |
+
},
|
| 112 |
+
"config_fingerprint": "00e7ae6b9109c167",
|
| 113 |
+
"artifact_path": "experiments/clean1930s-d12-r11.25"
|
| 114 |
+
},
|
| 115 |
+
"stage": "base",
|
| 116 |
+
"base_experiment_id": "clean1930s-d12-r11.25",
|
| 117 |
+
"parent_experiment_id": null,
|
| 118 |
+
"parent_checkpoint_step": null,
|
| 119 |
+
"config_fingerprint": "00e7ae6b9109c167"
|
| 120 |
+
},
|
| 121 |
+
"device_batch_size": 16,
|
| 122 |
+
"max_seq_len": 2048,
|
| 123 |
+
"total_batch_size": 524288,
|
| 124 |
+
"dataloader_state_dict": {
|
| 125 |
+
"file_idx": 2,
|
| 126 |
+
"pos": 62184769,
|
| 127 |
+
"epoch": 1,
|
| 128 |
+
"pq_idx": 2,
|
| 129 |
+
"rg_idx": 62184769
|
| 130 |
+
},
|
| 131 |
+
"loop_state": {
|
| 132 |
+
"min_val_bpb": 1.3406276768636045,
|
| 133 |
+
"smooth_train_loss": 3.8475896185653875,
|
| 134 |
+
"total_training_time": 1308.1742749214172,
|
| 135 |
+
"stage_training_flops": 232547388751872000,
|
| 136 |
+
"inherited_parent_flops": 0.0,
|
| 137 |
+
"cumulative_pipeline_training_flops": 232547388751872000
|
| 138 |
+
}
|
| 139 |
+
}
|
experiments/clean1930s-d12-r11.25/base_checkpoints/meta_001000.json
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 1000,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "clean1930s-d12-r11.25",
|
| 5 |
+
"val_bpb": 1.2453749523249924,
|
| 6 |
+
"model_config": {
|
| 7 |
+
"sequence_len": 2048,
|
| 8 |
+
"vocab_size": 32768,
|
| 9 |
+
"n_layer": 12,
|
| 10 |
+
"n_head": 6,
|
| 11 |
+
"n_kv_head": 6,
|
| 12 |
+
"n_embd": 768,
|
| 13 |
+
"window_pattern": "L"
|
| 14 |
+
},
|
| 15 |
+
"user_config": {
|
| 16 |
+
"run": "clean1930s-d12-r11.25",
|
| 17 |
+
"wandb_run_id": "764ab08e",
|
| 18 |
+
"wandb_group": "clean1930s-d12",
|
| 19 |
+
"wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25",
|
| 20 |
+
"device_type": "",
|
| 21 |
+
"fp8": false,
|
| 22 |
+
"fp8_recipe": "tensorwise",
|
| 23 |
+
"depth": 12,
|
| 24 |
+
"aspect_ratio": 64,
|
| 25 |
+
"head_dim": 128,
|
| 26 |
+
"max_seq_len": 2048,
|
| 27 |
+
"window_pattern": "L",
|
| 28 |
+
"num_iterations": -1,
|
| 29 |
+
"target_flops": -1.0,
|
| 30 |
+
"target_param_data_ratio": 11.25,
|
| 31 |
+
"device_batch_size": 16,
|
| 32 |
+
"total_batch_size": 524288,
|
| 33 |
+
"embedding_lr": 0.3,
|
| 34 |
+
"unembedding_lr": 0.008,
|
| 35 |
+
"weight_decay": 0.28,
|
| 36 |
+
"matrix_lr": 0.02,
|
| 37 |
+
"scalar_lr": 0.5,
|
| 38 |
+
"warmup_steps": 40,
|
| 39 |
+
"warmdown_ratio": 0.65,
|
| 40 |
+
"final_lr_frac": 0.05,
|
| 41 |
+
"resume_from_step": -1,
|
| 42 |
+
"pretokenized": true,
|
| 43 |
+
"data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/data",
|
| 44 |
+
"tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/tokenizer",
|
| 45 |
+
"pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/pretok",
|
| 46 |
+
"checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/base_checkpoints",
|
| 47 |
+
"experiment_id": "clean1930s-d12-r11.25",
|
| 48 |
+
"experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/config.json",
|
| 49 |
+
"tokenizer_fingerprint": "fa7b147cb0e4b5db",
|
| 50 |
+
"git_commit_sha": "083cd7f99484b5e894a23e4a093de6d339c412ea",
|
| 51 |
+
"seed": 42,
|
| 52 |
+
"eval_every": 250,
|
| 53 |
+
"eval_tokens": 2097152,
|
| 54 |
+
"core_metric_every": -1,
|
| 55 |
+
"core_metric_max_per_task": 500,
|
| 56 |
+
"sample_every": -1,
|
| 57 |
+
"save_every": 500,
|
| 58 |
+
"model_tag": "clean1930s-d12-r11.25",
|
| 59 |
+
"experiment": {
|
| 60 |
+
"schema_version": 1,
|
| 61 |
+
"stage": "base",
|
| 62 |
+
"experiment_id": "clean1930s-d12-r11.25",
|
| 63 |
+
"dataset": {
|
| 64 |
+
"adapter": "parquet_shards",
|
| 65 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 66 |
+
"revision": "main",
|
| 67 |
+
"validation_shard": 472,
|
| 68 |
+
"num_train_shards": 26,
|
| 69 |
+
"download_workers": 4
|
| 70 |
+
},
|
| 71 |
+
"tokenizer": {
|
| 72 |
+
"mode": "train",
|
| 73 |
+
"max_chars": 2000000000,
|
| 74 |
+
"doc_cap": 10000,
|
| 75 |
+
"vocab_size": 32768
|
| 76 |
+
},
|
| 77 |
+
"pretokenize": {
|
| 78 |
+
"enabled": true,
|
| 79 |
+
"slack": 1.03,
|
| 80 |
+
"val_tokens": 20971520,
|
| 81 |
+
"shard_tokens": 100000000,
|
| 82 |
+
"tokenizer_threads": 8
|
| 83 |
+
},
|
| 84 |
+
"training": {
|
| 85 |
+
"depth": 12,
|
| 86 |
+
"scaling_params": 110100912,
|
| 87 |
+
"target_param_data_ratio": 11.25,
|
| 88 |
+
"window_pattern": "L",
|
| 89 |
+
"device_batch_size": 16,
|
| 90 |
+
"total_batch_size": 524288,
|
| 91 |
+
"seed": 42,
|
| 92 |
+
"save_every": 500,
|
| 93 |
+
"eval_every": 250,
|
| 94 |
+
"eval_tokens": 2097152,
|
| 95 |
+
"core_metric_every": -1,
|
| 96 |
+
"sample_every": -1
|
| 97 |
+
},
|
| 98 |
+
"artifacts": {
|
| 99 |
+
"repo": "jbduran/think.nano"
|
| 100 |
+
},
|
| 101 |
+
"wandb": {
|
| 102 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 103 |
+
"project": "think.nano",
|
| 104 |
+
"name": "clean1930s-d12-r11.25",
|
| 105 |
+
"group": "clean1930s-d12",
|
| 106 |
+
"tags": [
|
| 107 |
+
"think-dataset-clean-1930s",
|
| 108 |
+
"d12",
|
| 109 |
+
"ratio11.25"
|
| 110 |
+
]
|
| 111 |
+
},
|
| 112 |
+
"config_fingerprint": "00e7ae6b9109c167",
|
| 113 |
+
"artifact_path": "experiments/clean1930s-d12-r11.25"
|
| 114 |
+
},
|
| 115 |
+
"stage": "base",
|
| 116 |
+
"base_experiment_id": "clean1930s-d12-r11.25",
|
| 117 |
+
"parent_experiment_id": null,
|
| 118 |
+
"parent_checkpoint_step": null,
|
| 119 |
+
"config_fingerprint": "00e7ae6b9109c167"
|
| 120 |
+
},
|
| 121 |
+
"device_batch_size": 16,
|
| 122 |
+
"max_seq_len": 2048,
|
| 123 |
+
"total_batch_size": 524288,
|
| 124 |
+
"dataloader_state_dict": {
|
| 125 |
+
"file_idx": 5,
|
| 126 |
+
"pos": 24336769,
|
| 127 |
+
"epoch": 1,
|
| 128 |
+
"pq_idx": 5,
|
| 129 |
+
"rg_idx": 24336769
|
| 130 |
+
},
|
| 131 |
+
"loop_state": {
|
| 132 |
+
"min_val_bpb": 1.2453749523249924,
|
| 133 |
+
"smooth_train_loss": 3.5828240195598844,
|
| 134 |
+
"total_training_time": 2642.554317712784,
|
| 135 |
+
"stage_training_flops": 465094777503744000,
|
| 136 |
+
"inherited_parent_flops": 0.0,
|
| 137 |
+
"cumulative_pipeline_training_flops": 465094777503744000
|
| 138 |
+
}
|
| 139 |
+
}
|
experiments/clean1930s-d12-r11.25/base_checkpoints/meta_001500.json
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 1500,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "clean1930s-d12-r11.25",
|
| 5 |
+
"val_bpb": 1.1784689301618998,
|
| 6 |
+
"model_config": {
|
| 7 |
+
"sequence_len": 2048,
|
| 8 |
+
"vocab_size": 32768,
|
| 9 |
+
"n_layer": 12,
|
| 10 |
+
"n_head": 6,
|
| 11 |
+
"n_kv_head": 6,
|
| 12 |
+
"n_embd": 768,
|
| 13 |
+
"window_pattern": "L"
|
| 14 |
+
},
|
| 15 |
+
"user_config": {
|
| 16 |
+
"run": "clean1930s-d12-r11.25",
|
| 17 |
+
"wandb_run_id": "764ab08e",
|
| 18 |
+
"wandb_group": "clean1930s-d12",
|
| 19 |
+
"wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25",
|
| 20 |
+
"device_type": "",
|
| 21 |
+
"fp8": false,
|
| 22 |
+
"fp8_recipe": "tensorwise",
|
| 23 |
+
"depth": 12,
|
| 24 |
+
"aspect_ratio": 64,
|
| 25 |
+
"head_dim": 128,
|
| 26 |
+
"max_seq_len": 2048,
|
| 27 |
+
"window_pattern": "L",
|
| 28 |
+
"num_iterations": -1,
|
| 29 |
+
"target_flops": -1.0,
|
| 30 |
+
"target_param_data_ratio": 11.25,
|
| 31 |
+
"device_batch_size": 16,
|
| 32 |
+
"total_batch_size": 524288,
|
| 33 |
+
"embedding_lr": 0.3,
|
| 34 |
+
"unembedding_lr": 0.008,
|
| 35 |
+
"weight_decay": 0.28,
|
| 36 |
+
"matrix_lr": 0.02,
|
| 37 |
+
"scalar_lr": 0.5,
|
| 38 |
+
"warmup_steps": 40,
|
| 39 |
+
"warmdown_ratio": 0.65,
|
| 40 |
+
"final_lr_frac": 0.05,
|
| 41 |
+
"resume_from_step": -1,
|
| 42 |
+
"pretokenized": true,
|
| 43 |
+
"data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/data",
|
| 44 |
+
"tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/tokenizer",
|
| 45 |
+
"pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/pretok",
|
| 46 |
+
"checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/base_checkpoints",
|
| 47 |
+
"experiment_id": "clean1930s-d12-r11.25",
|
| 48 |
+
"experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/config.json",
|
| 49 |
+
"tokenizer_fingerprint": "fa7b147cb0e4b5db",
|
| 50 |
+
"git_commit_sha": "083cd7f99484b5e894a23e4a093de6d339c412ea",
|
| 51 |
+
"seed": 42,
|
| 52 |
+
"eval_every": 250,
|
| 53 |
+
"eval_tokens": 2097152,
|
| 54 |
+
"core_metric_every": -1,
|
| 55 |
+
"core_metric_max_per_task": 500,
|
| 56 |
+
"sample_every": -1,
|
| 57 |
+
"save_every": 500,
|
| 58 |
+
"model_tag": "clean1930s-d12-r11.25",
|
| 59 |
+
"experiment": {
|
| 60 |
+
"schema_version": 1,
|
| 61 |
+
"stage": "base",
|
| 62 |
+
"experiment_id": "clean1930s-d12-r11.25",
|
| 63 |
+
"dataset": {
|
| 64 |
+
"adapter": "parquet_shards",
|
| 65 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 66 |
+
"revision": "main",
|
| 67 |
+
"validation_shard": 472,
|
| 68 |
+
"num_train_shards": 26,
|
| 69 |
+
"download_workers": 4
|
| 70 |
+
},
|
| 71 |
+
"tokenizer": {
|
| 72 |
+
"mode": "train",
|
| 73 |
+
"max_chars": 2000000000,
|
| 74 |
+
"doc_cap": 10000,
|
| 75 |
+
"vocab_size": 32768
|
| 76 |
+
},
|
| 77 |
+
"pretokenize": {
|
| 78 |
+
"enabled": true,
|
| 79 |
+
"slack": 1.03,
|
| 80 |
+
"val_tokens": 20971520,
|
| 81 |
+
"shard_tokens": 100000000,
|
| 82 |
+
"tokenizer_threads": 8
|
| 83 |
+
},
|
| 84 |
+
"training": {
|
| 85 |
+
"depth": 12,
|
| 86 |
+
"scaling_params": 110100912,
|
| 87 |
+
"target_param_data_ratio": 11.25,
|
| 88 |
+
"window_pattern": "L",
|
| 89 |
+
"device_batch_size": 16,
|
| 90 |
+
"total_batch_size": 524288,
|
| 91 |
+
"seed": 42,
|
| 92 |
+
"save_every": 500,
|
| 93 |
+
"eval_every": 250,
|
| 94 |
+
"eval_tokens": 2097152,
|
| 95 |
+
"core_metric_every": -1,
|
| 96 |
+
"sample_every": -1
|
| 97 |
+
},
|
| 98 |
+
"artifacts": {
|
| 99 |
+
"repo": "jbduran/think.nano"
|
| 100 |
+
},
|
| 101 |
+
"wandb": {
|
| 102 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 103 |
+
"project": "think.nano",
|
| 104 |
+
"name": "clean1930s-d12-r11.25",
|
| 105 |
+
"group": "clean1930s-d12",
|
| 106 |
+
"tags": [
|
| 107 |
+
"think-dataset-clean-1930s",
|
| 108 |
+
"d12",
|
| 109 |
+
"ratio11.25"
|
| 110 |
+
]
|
| 111 |
+
},
|
| 112 |
+
"config_fingerprint": "00e7ae6b9109c167",
|
| 113 |
+
"artifact_path": "experiments/clean1930s-d12-r11.25"
|
| 114 |
+
},
|
| 115 |
+
"stage": "base",
|
| 116 |
+
"base_experiment_id": "clean1930s-d12-r11.25",
|
| 117 |
+
"parent_experiment_id": null,
|
| 118 |
+
"parent_checkpoint_step": null,
|
| 119 |
+
"config_fingerprint": "00e7ae6b9109c167"
|
| 120 |
+
},
|
| 121 |
+
"device_batch_size": 16,
|
| 122 |
+
"max_seq_len": 2048,
|
| 123 |
+
"total_batch_size": 524288,
|
| 124 |
+
"dataloader_state_dict": {
|
| 125 |
+
"file_idx": 7,
|
| 126 |
+
"pos": 86488769,
|
| 127 |
+
"epoch": 1,
|
| 128 |
+
"pq_idx": 7,
|
| 129 |
+
"rg_idx": 86488769
|
| 130 |
+
},
|
| 131 |
+
"loop_state": {
|
| 132 |
+
"min_val_bpb": 1.1784689301618998,
|
| 133 |
+
"smooth_train_loss": 3.3444808841561646,
|
| 134 |
+
"total_training_time": 3977.2887001037598,
|
| 135 |
+
"stage_training_flops": 697642166255616000,
|
| 136 |
+
"inherited_parent_flops": 0.0,
|
| 137 |
+
"cumulative_pipeline_training_flops": 697642166255616000
|
| 138 |
+
}
|
| 139 |
+
}
|
experiments/clean1930s-d12-r11.25/base_checkpoints/meta_002000.json
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 2000,
|
| 3 |
+
"training_complete": false,
|
| 4 |
+
"experiment_id": "clean1930s-d12-r11.25",
|
| 5 |
+
"val_bpb": 1.1385790516794956,
|
| 6 |
+
"model_config": {
|
| 7 |
+
"sequence_len": 2048,
|
| 8 |
+
"vocab_size": 32768,
|
| 9 |
+
"n_layer": 12,
|
| 10 |
+
"n_head": 6,
|
| 11 |
+
"n_kv_head": 6,
|
| 12 |
+
"n_embd": 768,
|
| 13 |
+
"window_pattern": "L"
|
| 14 |
+
},
|
| 15 |
+
"user_config": {
|
| 16 |
+
"run": "clean1930s-d12-r11.25",
|
| 17 |
+
"wandb_run_id": "764ab08e",
|
| 18 |
+
"wandb_group": "clean1930s-d12",
|
| 19 |
+
"wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25",
|
| 20 |
+
"device_type": "",
|
| 21 |
+
"fp8": false,
|
| 22 |
+
"fp8_recipe": "tensorwise",
|
| 23 |
+
"depth": 12,
|
| 24 |
+
"aspect_ratio": 64,
|
| 25 |
+
"head_dim": 128,
|
| 26 |
+
"max_seq_len": 2048,
|
| 27 |
+
"window_pattern": "L",
|
| 28 |
+
"num_iterations": -1,
|
| 29 |
+
"target_flops": -1.0,
|
| 30 |
+
"target_param_data_ratio": 11.25,
|
| 31 |
+
"device_batch_size": 16,
|
| 32 |
+
"total_batch_size": 524288,
|
| 33 |
+
"embedding_lr": 0.3,
|
| 34 |
+
"unembedding_lr": 0.008,
|
| 35 |
+
"weight_decay": 0.28,
|
| 36 |
+
"matrix_lr": 0.02,
|
| 37 |
+
"scalar_lr": 0.5,
|
| 38 |
+
"warmup_steps": 40,
|
| 39 |
+
"warmdown_ratio": 0.65,
|
| 40 |
+
"final_lr_frac": 0.05,
|
| 41 |
+
"resume_from_step": -1,
|
| 42 |
+
"pretokenized": true,
|
| 43 |
+
"data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/data",
|
| 44 |
+
"tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/tokenizer",
|
| 45 |
+
"pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/pretok",
|
| 46 |
+
"checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/base_checkpoints",
|
| 47 |
+
"experiment_id": "clean1930s-d12-r11.25",
|
| 48 |
+
"experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/config.json",
|
| 49 |
+
"tokenizer_fingerprint": "fa7b147cb0e4b5db",
|
| 50 |
+
"git_commit_sha": "083cd7f99484b5e894a23e4a093de6d339c412ea",
|
| 51 |
+
"seed": 42,
|
| 52 |
+
"eval_every": 250,
|
| 53 |
+
"eval_tokens": 2097152,
|
| 54 |
+
"core_metric_every": -1,
|
| 55 |
+
"core_metric_max_per_task": 500,
|
| 56 |
+
"sample_every": -1,
|
| 57 |
+
"save_every": 500,
|
| 58 |
+
"model_tag": "clean1930s-d12-r11.25",
|
| 59 |
+
"experiment": {
|
| 60 |
+
"schema_version": 1,
|
| 61 |
+
"stage": "base",
|
| 62 |
+
"experiment_id": "clean1930s-d12-r11.25",
|
| 63 |
+
"dataset": {
|
| 64 |
+
"adapter": "parquet_shards",
|
| 65 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 66 |
+
"revision": "main",
|
| 67 |
+
"validation_shard": 472,
|
| 68 |
+
"num_train_shards": 26,
|
| 69 |
+
"download_workers": 4
|
| 70 |
+
},
|
| 71 |
+
"tokenizer": {
|
| 72 |
+
"mode": "train",
|
| 73 |
+
"max_chars": 2000000000,
|
| 74 |
+
"doc_cap": 10000,
|
| 75 |
+
"vocab_size": 32768
|
| 76 |
+
},
|
| 77 |
+
"pretokenize": {
|
| 78 |
+
"enabled": true,
|
| 79 |
+
"slack": 1.03,
|
| 80 |
+
"val_tokens": 20971520,
|
| 81 |
+
"shard_tokens": 100000000,
|
| 82 |
+
"tokenizer_threads": 8
|
| 83 |
+
},
|
| 84 |
+
"training": {
|
| 85 |
+
"depth": 12,
|
| 86 |
+
"scaling_params": 110100912,
|
| 87 |
+
"target_param_data_ratio": 11.25,
|
| 88 |
+
"window_pattern": "L",
|
| 89 |
+
"device_batch_size": 16,
|
| 90 |
+
"total_batch_size": 524288,
|
| 91 |
+
"seed": 42,
|
| 92 |
+
"save_every": 500,
|
| 93 |
+
"eval_every": 250,
|
| 94 |
+
"eval_tokens": 2097152,
|
| 95 |
+
"core_metric_every": -1,
|
| 96 |
+
"sample_every": -1
|
| 97 |
+
},
|
| 98 |
+
"artifacts": {
|
| 99 |
+
"repo": "jbduran/think.nano"
|
| 100 |
+
},
|
| 101 |
+
"wandb": {
|
| 102 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 103 |
+
"project": "think.nano",
|
| 104 |
+
"name": "clean1930s-d12-r11.25",
|
| 105 |
+
"group": "clean1930s-d12",
|
| 106 |
+
"tags": [
|
| 107 |
+
"think-dataset-clean-1930s",
|
| 108 |
+
"d12",
|
| 109 |
+
"ratio11.25"
|
| 110 |
+
]
|
| 111 |
+
},
|
| 112 |
+
"config_fingerprint": "00e7ae6b9109c167",
|
| 113 |
+
"artifact_path": "experiments/clean1930s-d12-r11.25"
|
| 114 |
+
},
|
| 115 |
+
"stage": "base",
|
| 116 |
+
"base_experiment_id": "clean1930s-d12-r11.25",
|
| 117 |
+
"parent_experiment_id": null,
|
| 118 |
+
"parent_checkpoint_step": null,
|
| 119 |
+
"config_fingerprint": "00e7ae6b9109c167"
|
| 120 |
+
},
|
| 121 |
+
"device_batch_size": 16,
|
| 122 |
+
"max_seq_len": 2048,
|
| 123 |
+
"total_batch_size": 524288,
|
| 124 |
+
"dataloader_state_dict": {
|
| 125 |
+
"file_idx": 10,
|
| 126 |
+
"pos": 48640769,
|
| 127 |
+
"epoch": 1,
|
| 128 |
+
"pq_idx": 10,
|
| 129 |
+
"rg_idx": 48640769
|
| 130 |
+
},
|
| 131 |
+
"loop_state": {
|
| 132 |
+
"min_val_bpb": 1.1385790516794956,
|
| 133 |
+
"smooth_train_loss": 3.19688287657793,
|
| 134 |
+
"total_training_time": 5312.674092292786,
|
| 135 |
+
"stage_training_flops": 930189555007488000,
|
| 136 |
+
"inherited_parent_flops": 0.0,
|
| 137 |
+
"cumulative_pipeline_training_flops": 930189555007488000
|
| 138 |
+
}
|
| 139 |
+
}
|
experiments/clean1930s-d12-r11.25/base_checkpoints/meta_002362.json
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"step": 2362,
|
| 3 |
+
"training_complete": true,
|
| 4 |
+
"experiment_id": "clean1930s-d12-r11.25",
|
| 5 |
+
"val_bpb": 1.1175529448105965,
|
| 6 |
+
"model_config": {
|
| 7 |
+
"sequence_len": 2048,
|
| 8 |
+
"vocab_size": 32768,
|
| 9 |
+
"n_layer": 12,
|
| 10 |
+
"n_head": 6,
|
| 11 |
+
"n_kv_head": 6,
|
| 12 |
+
"n_embd": 768,
|
| 13 |
+
"window_pattern": "L"
|
| 14 |
+
},
|
| 15 |
+
"user_config": {
|
| 16 |
+
"run": "clean1930s-d12-r11.25",
|
| 17 |
+
"wandb_run_id": "764ab08e",
|
| 18 |
+
"wandb_group": "clean1930s-d12",
|
| 19 |
+
"wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25",
|
| 20 |
+
"device_type": "",
|
| 21 |
+
"fp8": false,
|
| 22 |
+
"fp8_recipe": "tensorwise",
|
| 23 |
+
"depth": 12,
|
| 24 |
+
"aspect_ratio": 64,
|
| 25 |
+
"head_dim": 128,
|
| 26 |
+
"max_seq_len": 2048,
|
| 27 |
+
"window_pattern": "L",
|
| 28 |
+
"num_iterations": -1,
|
| 29 |
+
"target_flops": -1.0,
|
| 30 |
+
"target_param_data_ratio": 11.25,
|
| 31 |
+
"device_batch_size": 16,
|
| 32 |
+
"total_batch_size": 524288,
|
| 33 |
+
"embedding_lr": 0.3,
|
| 34 |
+
"unembedding_lr": 0.008,
|
| 35 |
+
"weight_decay": 0.28,
|
| 36 |
+
"matrix_lr": 0.02,
|
| 37 |
+
"scalar_lr": 0.5,
|
| 38 |
+
"warmup_steps": 40,
|
| 39 |
+
"warmdown_ratio": 0.65,
|
| 40 |
+
"final_lr_frac": 0.05,
|
| 41 |
+
"resume_from_step": -1,
|
| 42 |
+
"pretokenized": true,
|
| 43 |
+
"data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/data",
|
| 44 |
+
"tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/tokenizer",
|
| 45 |
+
"pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/pretok",
|
| 46 |
+
"checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/base_checkpoints",
|
| 47 |
+
"experiment_id": "clean1930s-d12-r11.25",
|
| 48 |
+
"experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25/config.json",
|
| 49 |
+
"tokenizer_fingerprint": "fa7b147cb0e4b5db",
|
| 50 |
+
"git_commit_sha": "083cd7f99484b5e894a23e4a093de6d339c412ea",
|
| 51 |
+
"seed": 42,
|
| 52 |
+
"eval_every": 250,
|
| 53 |
+
"eval_tokens": 2097152,
|
| 54 |
+
"core_metric_every": -1,
|
| 55 |
+
"core_metric_max_per_task": 500,
|
| 56 |
+
"sample_every": -1,
|
| 57 |
+
"save_every": 500,
|
| 58 |
+
"model_tag": "clean1930s-d12-r11.25",
|
| 59 |
+
"experiment": {
|
| 60 |
+
"schema_version": 1,
|
| 61 |
+
"stage": "base",
|
| 62 |
+
"experiment_id": "clean1930s-d12-r11.25",
|
| 63 |
+
"dataset": {
|
| 64 |
+
"adapter": "parquet_shards",
|
| 65 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 66 |
+
"revision": "main",
|
| 67 |
+
"validation_shard": 472,
|
| 68 |
+
"num_train_shards": 26,
|
| 69 |
+
"download_workers": 4
|
| 70 |
+
},
|
| 71 |
+
"tokenizer": {
|
| 72 |
+
"mode": "train",
|
| 73 |
+
"max_chars": 2000000000,
|
| 74 |
+
"doc_cap": 10000,
|
| 75 |
+
"vocab_size": 32768
|
| 76 |
+
},
|
| 77 |
+
"pretokenize": {
|
| 78 |
+
"enabled": true,
|
| 79 |
+
"slack": 1.03,
|
| 80 |
+
"val_tokens": 20971520,
|
| 81 |
+
"shard_tokens": 100000000,
|
| 82 |
+
"tokenizer_threads": 8
|
| 83 |
+
},
|
| 84 |
+
"training": {
|
| 85 |
+
"depth": 12,
|
| 86 |
+
"scaling_params": 110100912,
|
| 87 |
+
"target_param_data_ratio": 11.25,
|
| 88 |
+
"window_pattern": "L",
|
| 89 |
+
"device_batch_size": 16,
|
| 90 |
+
"total_batch_size": 524288,
|
| 91 |
+
"seed": 42,
|
| 92 |
+
"save_every": 500,
|
| 93 |
+
"eval_every": 250,
|
| 94 |
+
"eval_tokens": 2097152,
|
| 95 |
+
"core_metric_every": -1,
|
| 96 |
+
"sample_every": -1
|
| 97 |
+
},
|
| 98 |
+
"artifacts": {
|
| 99 |
+
"repo": "jbduran/think.nano"
|
| 100 |
+
},
|
| 101 |
+
"wandb": {
|
| 102 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 103 |
+
"project": "think.nano",
|
| 104 |
+
"name": "clean1930s-d12-r11.25",
|
| 105 |
+
"group": "clean1930s-d12",
|
| 106 |
+
"tags": [
|
| 107 |
+
"think-dataset-clean-1930s",
|
| 108 |
+
"d12",
|
| 109 |
+
"ratio11.25"
|
| 110 |
+
]
|
| 111 |
+
},
|
| 112 |
+
"config_fingerprint": "00e7ae6b9109c167",
|
| 113 |
+
"artifact_path": "experiments/clean1930s-d12-r11.25"
|
| 114 |
+
},
|
| 115 |
+
"stage": "base",
|
| 116 |
+
"base_experiment_id": "clean1930s-d12-r11.25",
|
| 117 |
+
"parent_experiment_id": null,
|
| 118 |
+
"parent_checkpoint_step": null,
|
| 119 |
+
"config_fingerprint": "00e7ae6b9109c167"
|
| 120 |
+
},
|
| 121 |
+
"device_batch_size": 16,
|
| 122 |
+
"max_seq_len": 2048,
|
| 123 |
+
"total_batch_size": 524288,
|
| 124 |
+
"dataloader_state_dict": {
|
| 125 |
+
"file_idx": 12,
|
| 126 |
+
"pos": 38438817,
|
| 127 |
+
"epoch": 1,
|
| 128 |
+
"pq_idx": 12,
|
| 129 |
+
"rg_idx": 38438817
|
| 130 |
+
},
|
| 131 |
+
"loop_state": {
|
| 132 |
+
"min_val_bpb": 1.1175529448105965,
|
| 133 |
+
"smooth_train_loss": 3.1313785378510484,
|
| 134 |
+
"total_training_time": 6279.889491558075,
|
| 135 |
+
"stage_training_flops": 1098553864463843328,
|
| 136 |
+
"inherited_parent_flops": 0.0,
|
| 137 |
+
"cumulative_pipeline_training_flops": 1098553864463843328
|
| 138 |
+
}
|
| 139 |
+
}
|
experiments/clean1930s-d12-r11.25/base_checkpoints/model_000500.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4e818606e95665b2d93328ebac939eff1721dfddd5bc633c670d05644ec02b09
|
| 3 |
+
size 792761690
|
experiments/clean1930s-d12-r11.25/base_checkpoints/model_001000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6afac069374a34152cb01eff7142a0e285d70880bee793b10c0de073805a3864
|
| 3 |
+
size 792761690
|
experiments/clean1930s-d12-r11.25/base_checkpoints/model_001500.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1da723ae288e3aedd7142082491758bd1a6be3791cb4b881f32909c63e22ec68
|
| 3 |
+
size 792761690
|
experiments/clean1930s-d12-r11.25/base_checkpoints/model_002000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:108865e4d197f8c5f893fc9e0ef7568bf33258fa908fd843002e4033436e6169
|
| 3 |
+
size 792761690
|
experiments/clean1930s-d12-r11.25/base_checkpoints/model_002362.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:66478807ab543dc52a021b0989d39b1f59fdb9978bc679dc23781c97b0136a23
|
| 3 |
+
size 792761690
|
experiments/clean1930s-d12-r11.25/base_checkpoints/optim_000500_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a57bc43f0fc66aa44a67432373a76db9ee4be1b9b425ec037327188641df180d
|
| 3 |
+
size 1246165357
|
experiments/clean1930s-d12-r11.25/base_checkpoints/optim_001000_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5876a82a2083edec62f32471e1e8cd87749d59d9663905c7762447601e480bb7
|
| 3 |
+
size 1246165357
|
experiments/clean1930s-d12-r11.25/base_checkpoints/optim_001500_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:46edf74481ffc6508a1abdf1ad0837b2b1d1fefb6d09328fed8067b7819df19a
|
| 3 |
+
size 1246165357
|
experiments/clean1930s-d12-r11.25/base_checkpoints/optim_002000_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c13b9d6a8e454e20f62c5f1ea69c9076a2baffe739b3af61569b029c3203fe33
|
| 3 |
+
size 1246165357
|
experiments/clean1930s-d12-r11.25/base_checkpoints/optim_002362_rank0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a09823e14e88be9d6acaea7b9d3a2e5c88b435d1d40bbcbe656f780c874cdb32
|
| 3 |
+
size 1246165357
|
experiments/clean1930s-d12-r11.25/config.json
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"stage": "base",
|
| 4 |
+
"experiment_id": "clean1930s-d12-r11.25",
|
| 5 |
+
"dataset": {
|
| 6 |
+
"adapter": "parquet_shards",
|
| 7 |
+
"repo": "jbduran/think-dataset-clean-1930s",
|
| 8 |
+
"revision": "main",
|
| 9 |
+
"validation_shard": 472,
|
| 10 |
+
"num_train_shards": 26,
|
| 11 |
+
"download_workers": 4
|
| 12 |
+
},
|
| 13 |
+
"tokenizer": {
|
| 14 |
+
"mode": "train",
|
| 15 |
+
"max_chars": 2000000000,
|
| 16 |
+
"doc_cap": 10000,
|
| 17 |
+
"vocab_size": 32768
|
| 18 |
+
},
|
| 19 |
+
"pretokenize": {
|
| 20 |
+
"enabled": true,
|
| 21 |
+
"slack": 1.03,
|
| 22 |
+
"val_tokens": 20971520,
|
| 23 |
+
"shard_tokens": 100000000,
|
| 24 |
+
"tokenizer_threads": 8
|
| 25 |
+
},
|
| 26 |
+
"training": {
|
| 27 |
+
"depth": 12,
|
| 28 |
+
"scaling_params": 110100912,
|
| 29 |
+
"target_param_data_ratio": 11.25,
|
| 30 |
+
"window_pattern": "L",
|
| 31 |
+
"device_batch_size": 16,
|
| 32 |
+
"total_batch_size": 524288,
|
| 33 |
+
"seed": 42,
|
| 34 |
+
"save_every": 500,
|
| 35 |
+
"eval_every": 250,
|
| 36 |
+
"eval_tokens": 2097152,
|
| 37 |
+
"core_metric_every": -1,
|
| 38 |
+
"sample_every": -1
|
| 39 |
+
},
|
| 40 |
+
"artifacts": {
|
| 41 |
+
"repo": "jbduran/think.nano"
|
| 42 |
+
},
|
| 43 |
+
"wandb": {
|
| 44 |
+
"entity": "jbduran-thinkingmachinesncsu",
|
| 45 |
+
"project": "think.nano",
|
| 46 |
+
"name": "clean1930s-d12-r11.25",
|
| 47 |
+
"group": "clean1930s-d12",
|
| 48 |
+
"tags": [
|
| 49 |
+
"think-dataset-clean-1930s",
|
| 50 |
+
"d12",
|
| 51 |
+
"ratio11.25"
|
| 52 |
+
]
|
| 53 |
+
},
|
| 54 |
+
"config_fingerprint": "00e7ae6b9109c167",
|
| 55 |
+
"artifact_path": "experiments/clean1930s-d12-r11.25"
|
| 56 |
+
}
|
experiments/clean1930s-d12-r11.25/evals/core.json
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "base_model (step 2362)",
|
| 3 |
+
"step": 2362,
|
| 4 |
+
"bpb": {},
|
| 5 |
+
"core_metric": 0.06238580602029931,
|
| 6 |
+
"core_results": {
|
| 7 |
+
"hellaswag_zeroshot": 0.2756423056125641,
|
| 8 |
+
"jeopardy": 0.0,
|
| 9 |
+
"bigbench_qa_wikidata": 0.07450420409440994,
|
| 10 |
+
"arc_easy": 0.32281145453453064,
|
| 11 |
+
"arc_challenge": 0.21587030589580536,
|
| 12 |
+
"copa": 0.5399999618530273,
|
| 13 |
+
"commonsense_qa": 0.26535627245903015,
|
| 14 |
+
"piqa": 0.5402611494064331,
|
| 15 |
+
"openbook_qa": 0.242000013589859,
|
| 16 |
+
"lambada_openai": 0.2115272581577301,
|
| 17 |
+
"hellaswag": 0.2757418751716614,
|
| 18 |
+
"winograd": 0.5641025900840759,
|
| 19 |
+
"winogrande": 0.49723756313323975,
|
| 20 |
+
"bigbench_dyck_languages": 0.1120000034570694,
|
| 21 |
+
"agi_eval_lsat_ar": 0.2913043200969696,
|
| 22 |
+
"bigbench_cs_algorithms": 0.41969695687294006,
|
| 23 |
+
"bigbench_operators": 0.06190476566553116,
|
| 24 |
+
"bigbench_repeat_copy_logic": 0.0,
|
| 25 |
+
"squad": 0.014758750796318054,
|
| 26 |
+
"coqa": 0.05574345588684082,
|
| 27 |
+
"boolq": 0.4880733788013458,
|
| 28 |
+
"bigbench_language_identification": 0.2556000053882599
|
| 29 |
+
},
|
| 30 |
+
"centered_results": {
|
| 31 |
+
"hellaswag_zeroshot": 0.034189740816752114,
|
| 32 |
+
"jeopardy": 0.0,
|
| 33 |
+
"bigbench_qa_wikidata": 0.07450420409440994,
|
| 34 |
+
"arc_easy": 0.09708193937937419,
|
| 35 |
+
"arc_challenge": -0.04550625880559286,
|
| 36 |
+
"copa": 0.07999992370605469,
|
| 37 |
+
"commonsense_qa": 0.08169534057378768,
|
| 38 |
+
"piqa": 0.08052229881286621,
|
| 39 |
+
"openbook_qa": -0.010666648546854654,
|
| 40 |
+
"lambada_openai": 0.2115272581577301,
|
| 41 |
+
"hellaswag": 0.034322500228881836,
|
| 42 |
+
"winograd": 0.12820518016815186,
|
| 43 |
+
"winogrande": -0.005524873733520508,
|
| 44 |
+
"bigbench_dyck_languages": 0.1120000034570694,
|
| 45 |
+
"agi_eval_lsat_ar": 0.11413040012121199,
|
| 46 |
+
"bigbench_cs_algorithms": 0.41969695687294006,
|
| 47 |
+
"bigbench_operators": 0.06190476566553116,
|
| 48 |
+
"bigbench_repeat_copy_logic": 0.0,
|
| 49 |
+
"squad": 0.014758750796318054,
|
| 50 |
+
"coqa": 0.05574345588684082,
|
| 51 |
+
"boolq": -0.34717531894382675,
|
| 52 |
+
"bigbench_language_identification": 0.18107811373845972
|
| 53 |
+
},
|
| 54 |
+
"conditioned_samples": [],
|
| 55 |
+
"unconditioned_samples": []
|
| 56 |
+
}
|
experiments/clean1930s-d12-r11.25/evals/samples.json
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "base_model (step 2362)",
|
| 3 |
+
"step": 2362,
|
| 4 |
+
"bpb": {},
|
| 5 |
+
"core_metric": null,
|
| 6 |
+
"core_results": null,
|
| 7 |
+
"centered_results": null,
|
| 8 |
+
"conditioned_samples": [
|
| 9 |
+
{
|
| 10 |
+
"prompt": "The capital of France is",
|
| 11 |
+
"text": "<|bos|>The capital of France is the capital of the Empire, and the capital of the Empire is the capital of"
|
| 12 |
+
},
|
| 13 |
+
{
|
| 14 |
+
"prompt": "The chemical symbol of gold is",
|
| 15 |
+
"text": "<|bos|>The chemical symbol of gold is the \"milk of the earth,\" and the \"milk of"
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"prompt": "If yesterday was Friday, then tomorrow will be",
|
| 19 |
+
"text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. If yesterday was Saturday, then tomorrow will be Saturday. If yesterday was"
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"prompt": "The opposite of hot is",
|
| 23 |
+
"text": "<|bos|>The opposite of hot is the hot, and the hot is the cold. The hot is the hot,"
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"prompt": "The planets of the solar system are:",
|
| 27 |
+
"text": "<|bos|>The planets of the solar system are: 1. The sun, 2. The moon, 3. The"
|
| 28 |
+
},
|
| 29 |
+
{
|
| 30 |
+
"prompt": "My favorite color is",
|
| 31 |
+
"text": "<|bos|>My favorite color is the red of the red of the red of the red of the red of the"
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"prompt": "If 5*x + 3 = 13, then x is",
|
| 35 |
+
"text": "<|bos|>If 5*x + 3 = 13, then x is the number of days in which the earth is in motion, and the number of"
|
| 36 |
+
}
|
| 37 |
+
],
|
| 38 |
+
"unconditioned_samples": [
|
| 39 |
+
"<|bos|>CHAPTER V FORMS ON STEAM MILLS, It has already been pointed out that the very good Anglo- Saxon capital and its effects are of recent introduction in the East. The consciousness of modernity in architecture is rapidly \"tasking.\" In Europe and America, no single innovation, no single change in the least tenuous architecture, would be tolerated. On the contrary, so many hardy pioneers of civilization now lie dead in the mud or are idly rotting in stones; so general an application would still be to a period prior to 1870, just as we now to a century since, are inclined to reckon an ignorant man",
|
| 40 |
+
"<|bos|>PREFACE.\n\nso often quoted and described, and revealed the fact of average, almost unknowable, within the little knowable field of opinion of his.\n\nLet me hope that we may go as far as the narrative will admit into the explanation of such expressions as The Genealogies of Defendant, The Book-hunter, The Wise Man, etc.\n\nIt is often said, and so I deem it, that the gentleman who draws the following is not moving on his account, and does not design the question.\n\nLet us say, to the reader, This pantheistic text furnishes a full and complete explanation of the about the differences which make",
|
| 41 |
+
"<|bos|>PREFACE.\n\nix emphatic word is my wish. with one or two reasonable exceptions, and who experiences any emotion or fear on hearing of a narrative of her \"Franks.\"\n\nA strong and authoritative book, on a vouchsafed subject, or novelty or tedium to human beliefes careles of this life of yours shall be the common wish of every man, and it will make the incipient man that child alone that compilers, with the fore-power of vision, have by indolence, or vice, or other powerful means, precipitated the history ofori- this society.\"MI- MENTS FROM THE JUDGmeign. I",
|
| 42 |
+
"<|bos|> Dreams and Reverses, by Rev. P. I. Sheard.\n\n(ACT IV.) WALL RUN RECREATIONS\n\nIII Last Summer Evening, I took a Narrow Way to Princeton, Indiana, the night after Shamokoka fired their last shot, in their settlement and near Spring Hill, near\n\nAkosta Flats. Throughout the day whitening the trees around, tinged the spires of farms seated on clumps of trees and in all directions about them, and we had a fine time. Evidently to begin with, there came a large party of our men out of the lines, among",
|
| 43 |
+
"<|bos|>Dieta, i. 653, 686, 692.\n\nCook's ear, i. 76-80, 167, 235, 236.\n\nDoor-watchage, i. 82, 83, 113.\n\nDoolittle's reading, i. 548.\n\nDoomer's simply court notice, i. 355.\n\nEhambra parting teeth, i. 700, 702.. Ehambra weeping, i. 146. e.\n\nEast Bangalore, ii. 374.\n\nEcclesiastics",
|
| 44 |
+
"<|bos|>orms, while the moral principle is allowable in ordinary life. The boys belonging to a tradesman's home are let alone after spending a few weeks, during which he does not receive any pay; this exhausts their labors. Since there are no other means for carrying through business, few will be willing to undertake it; but few display the ability that can sustain sustained excitement by the prompt bestowal of a generous payment.\n\nThe third day is occupied in operaucb Reading. Sticks have been hired for over six months for delivery at a drug store, but this payment has been withheld because otherwise (Wat v. Ortsville",
|
| 45 |
+
"<|bos|>) Ruth and Robin lick'll look an' put up o' Tal-ledge's our followlin' lar, Iween, a 're luther Tol a little afraid 'll stay 'ead a-dandy at a P'ock harness an' es 'to Awlfe Cl Proper should slavery me sent, Weritan To! Cane flax can Gither it ble at Cutnish Maalock the times of aeething, Some lady misses to be able to Pane the vaeuce ground, some lady rod theows hast' a to Pate downt wi the",
|
| 46 |
+
"<|bos|> shares ought to be limited by the terms of a deed and however they may be in the hands of numerous individuals, the sum can easily be made large by even a few executed persons alone. Let the number of such persons gathered and the results produced justify the conclusion that the shares were free from all liabilities in their hands, and that their share had a legal amount sufficient to cover them at all times.\n\nThe rule adopted by the American Accountant General for Improving the Powers of the President of the United States Government in regard to the use of legal instruments for the transaction of business in the execution of contracts, applies as well to such legal"
|
| 47 |
+
]
|
| 48 |
+
}
|
experiments/clean1930s-d12-r11.25/evals/val_bpb.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "base_model (step 2362)",
|
| 3 |
+
"step": 2362,
|
| 4 |
+
"bpb": {
|
| 5 |
+
"val": 1.0596072239127994
|
| 6 |
+
},
|
| 7 |
+
"core_metric": null,
|
| 8 |
+
"core_results": null,
|
| 9 |
+
"centered_results": null,
|
| 10 |
+
"conditioned_samples": [],
|
| 11 |
+
"unconditioned_samples": []
|
| 12 |
+
}
|
experiments/clean1930s-d12-r11.25/evals/val_bpb_on_think-dataset-clean.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "base_model (step 2362)",
|
| 3 |
+
"step": 2362,
|
| 4 |
+
"bpb": {
|
| 5 |
+
"val": 1.06282022394172
|
| 6 |
+
},
|
| 7 |
+
"core_metric": null,
|
| 8 |
+
"core_results": null,
|
| 9 |
+
"centered_results": null,
|
| 10 |
+
"conditioned_samples": [],
|
| 11 |
+
"unconditioned_samples": []
|
| 12 |
+
}
|
experiments/clean1930s-d12-r11.25/evals/val_bpb_on_think-dataset.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "base_model (step 2362)",
|
| 3 |
+
"step": 2362,
|
| 4 |
+
"bpb": {
|
| 5 |
+
"val": 1.1058237779626336
|
| 6 |
+
},
|
| 7 |
+
"core_metric": null,
|
| 8 |
+
"core_results": null,
|
| 9 |
+
"centered_results": null,
|
| 10 |
+
"conditioned_samples": [],
|
| 11 |
+
"unconditioned_samples": []
|
| 12 |
+
}
|