scripts/yans/lm-evaluation-harness/pile_statistics.json

{
    "Data": "Pile statistics",
    "Document Count": 210607728,
    "Total Pile Characters": 421215456,
    "File Start Offsets": [
        0,
        7021438,
        14042822,
        21066113,
        28086515,
        35106072,
        42123306,
        49145091,
        56165817,
        63185587,
        70211208,
        77234322,
        84249267,
        91267634,
        98285983,
        105305110,
        112322489,
        119342491,
        126367373,
        133389153,
        140412039,
        147432373,
        154452516,
        161470190,
        168492733,
        175512521,
        182526939,
        189547478,
        196565318,
        203583306
    ]
}