recipe/corpus_stats.json

{
  "recipe_version": "hf-bytelevel-bpe-v1",
  "test_run": true,
  "weights_target": {
    "exotic": 0.7,
    "eng": 0.1,
    "spa": 0.1,
    "code": 0.1
  },
  "weights_actual": {
    "modern": 0.22659476422696032,
    "old": 0.47387660725836755,
    "eng": 0.10006682589867377,
    "spa": 0.10003697198211248,
    "code": 0.09942483063388591,
    "exotic": 0.7004713714853279
  },
  "bytes": {
    "modern": 25813993,
    "old": 53984687,
    "eng": 11399753,
    "spa": 11396352,
    "code": 11326616,
    "exotic": 79798680
  },
  "total_bytes": 113921401,
  "vocab_target": 256000,
  "min_frequency": 2,
  "special_token_count": 126,
  "language_token_count": 92,
  "external_to_exotic_ratio": 0.14285714285714288
}