tokenizer/experiment_tokenizer.json

{
  "experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1",
  "dataset": {
    "adapter": "parquet_shards",
    "repo": "jbduran/think-dataset-clean-1930s",
    "revision": "main",
    "validation_shard": 472,
    "num_train_shards": 200,
    "download_workers": 4
  },
  "tokenizer": {
    "mode": "train",
    "max_chars": 1000000000000,
    "doc_cap": 1000000000,
    "vocab_size": 32768
  },
  "created_at": 1784129374
}