Hugging Face's logo

jbduran
/
bartholomew-experiments

PyTorch
English
nanochat
language-model
pre-1930
historical
vintage
research
Model card Files
xet
 Community
bartholomew-experiments
File size: 2,781 Bytes
6951e66
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
{
  "schema_version": 1,
  "stage": "base",
  "experiment_id": "Think.Unbounded-d32",
  "datasets": {
    "original": {
      "adapter": "parquet_shards",
      "repo": "jbduran/think-dataset-clean-1930s",
      "revision": "45225d95bc15f942be3b4b344738cea1f66e3de8",
      "validation_shard": 472,
      "num_train_shards": 330,
      "download_workers": 4
    },
    "midtrain_r30": {
      "adapter": "parquet_shards",
      "repo": "zachnorton03/think-midtrain",
      "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
      "subfolder": "mixed/ratio_30/data",
      "validation_shard": 1262,
      "num_train_shards": 391,
      "download_workers": 4
    },
    "midtrain_r60": {
      "adapter": "parquet_shards",
      "repo": "zachnorton03/think-midtrain",
      "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc",
      "subfolder": "mixed/ratio_60/data",
      "validation_shard": 523,
      "num_train_shards": 207,
      "download_workers": 4
    }
  },
  "mixture_schedule": {
    "total_tokens": 20132659200,
    "seed_data": 1234,
    "max_epochs": 4,
    "stages": [
      {
        "name": "base",
        "start_tokens": 0,
        "source": "original"
      },
      {
        "name": "injection",
        "start_tokens": 14092861440,
        "source": "midtrain_r30"
      },
      {
        "name": "decay_mix",
        "start_tokens": 18119393280,
        "source": "midtrain_r60"
      }
    ]
  },
  "tokenizer": {
    "mode": "reuse",
    "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
    "vocab_size": 32768
  },
  "pretokenize": {
    "enabled": true,
    "slack": 1.03,
    "require_no_wrap": true,
    "val_tokens": 20971520,
    "shard_tokens": 100000000,
    "tokenizer_threads": 8
  },
  "training": {
    "depth": 32,
    "scaling_params": 1677724672,
    "target_param_data_ratio": 12,
    "max_seq_len": 4096,
    "window_pattern": "SSSL",
    "muon_momentum": 0.9,
    "device_batch_size": 2,
    "total_batch_size": 2097152,
    "fp8": true,
    "fp8_recipe": "tensorwise",
    "seed": 42,
    "save_every": 500,
    "eval_every": 250,
    "eval_tokens": 2097152,
    "core_metric_every": -1,
    "sample_every": -1
  },
  "artifacts": {
    "repo": "jbduran/think.nano",
    "keep_local_checkpoints": 1
  },
  "wandb": {
    "entity": "jbduran-thinkingmachinesncsu",
    "project": "think.nano",
    "name": "Think.Unbounded-d32",
    "group": "clean1930s-d32",
    "tags": [
      "think-dataset-clean-1930s",
      "d32",
      "ratio12",
      "ctx4096",
      "sssl",
      "fp8",
      "muon-momentum-constant",
      "muon-momentum-0.90",
      "midtrain-schedule",
      "mix-og",
      "0-30-60"
    ]
  },
  "config_fingerprint": "6f8ae461e0eb9ee2",
  "artifact_path": "experiments/Think.Unbounded-d32"
}