check_length.py

import json
import numpy as np
from tqdm import tqdm

INPUT_JSONL = "lenfiltered_fix_done_dupcleaned_data_turn_gpt_oss_20b_pretokenized.jsonl"

lengths = []

print("🔍 開始讀取資料...")

with open(INPUT_JSONL, "r", encoding="utf-8") as f:
    for line in tqdm(f):
        line = line.strip()
        if not line:
            continue
        
        data = json.loads(line)
        l = len(data["input_ids"])
        lengths.append(l)

lengths = np.array(lengths)

print("\n📊 統計結果：")
print(f"count: {len(lengths)}")
print(f"mean:  {lengths.mean():.2f}")
print(f"min:   {lengths.min()}")
print(f"max:   {lengths.max()}")

print("\n📈 Percentiles：")
for p in [50, 75, 90, 95, 98, 99, 99.5,99.9,99.99,99.999]:
    print(f"p{p}: {np.percentile(lengths, p):.0f}")