{
  "format": "bananamind2_pico_tokenizer_v1",
  "stage": "tokenizer_complete",
  "vocab_size": 384,
  "model_max_length": 8192,
  "target_gib": 1.0,
  "sources": [
    {
      "name": "fineweb_hq",
      "dataset_id": "epfml/FineWeb-HQ",
      "config": null,
      "text_field": "text",
      "byte_share": 0.75,
      "revision": "e58199cdd52438d94405df1a4d8630cc5f13bf84",
      "target_bytes": 805306368,
      "consumed_bytes": 805307468
    },
    {
      "name": "cosmopedia_v2",
      "dataset_id": "HuggingFaceTB/smollm-corpus",
      "config": "cosmopedia-v2",
      "text_field": "text",
      "byte_share": 0.25,
      "revision": "3ba9d605774198c5868892d7a8deda78031a781f",
      "target_bytes": 268435456,
      "consumed_bytes": 268438475
    }
  ]
}