inputs/length-audit.json

{
  "corpus": {
    "bytes": 1262158944,
    "path": "/mnt/donto-data/alpha-corpora/sft-text-v2/sft-v2.txt",
    "rows": 511428,
    "sha256": "ffad0a376c7eac2e0ec91f0901ec1ff87cba67cc298222828ce3df1a3e60b3fb"
  },
  "overall": {
    "max": 1024,
    "mean": 589.1191018090523,
    "min": 16,
    "p50": 657,
    "p95": 978,
    "p99": 1014,
    "rows": 511428
  },
  "result": "PASS",
  "rows_at_bound": 470,
  "rows_over_bound": 0,
  "schema": "alpha-sft-length-audit-v1",
  "sources": [
    {
      "max": 1024,
      "mean": 615.071214159795,
      "min": 22,
      "p50": 689,
      "p95": 980,
      "p99": 1015,
      "rows": 450402,
      "source": "smol-smoltalk"
    },
    {
      "max": 277,
      "mean": 185.38842975206612,
      "min": 86,
      "p50": 188,
      "p95": 241,
      "p99": 276,
      "rows": 121,
      "source": "smoltalk2:smoltalk-smollm3_everyday-conversations"
    },
    {
      "max": 1024,
      "mean": 561.594733951672,
      "min": 34,
      "p50": 556,
      "p95": 981,
      "p99": 1016,
      "rows": 32776,
      "source": "smoltalk2:smoltalk-smollm3_systemchats-30k"
    },
    {
      "max": 1023,
      "mean": 447.0511776679267,
      "min": 16,
      "p50": 423,
      "p95": 901,
      "p99": 991,
      "rows": 3439,
      "source": "oasst2"
    },
    {
      "max": 916,
      "mean": 173.9987444309437,
      "min": 30,
      "p50": 159,
      "p95": 319,
      "p99": 428,
      "rows": 24690,
      "source": "soda"
    }
  ],
  "token_bound": 1024,
  "tokenizer": {
    "path": "/mnt/donto-data/alpha-corpora/tokenizers/hf-bpe-byte-12k-20260722/tokenizer.json",
    "vocab_size": 12288
  }
}