File size: 4,402 Bytes
29f25be | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 | """Inspect local raw datasets without cleaning or modifying them."""
import json
import random
from collections import Counter
from pathlib import Path
import pyarrow.parquet as pq
ROOT = Path(__file__).resolve().parents[3]
DATASETS = ROOT / "datasets"
OUTPUT = ROOT / "outputs" / "inspection"
SEED = 42
def inspect_fineweb():
files = sorted(
(DATASETS / "fineweb-2-edu-japanese").glob("*.parquet")
)
if not files:
raise FileNotFoundError("No FineWeb2 parquet files found.")
rng = random.Random(SEED)
reports = []
for path in files:
with pq.ParquetFile(path) as parquet:
names = parquet.schema_arrow.names
if "text" not in names:
raise ValueError(f"{path.name}: missing text column")
wanted = [
"text", "id", "url", "language", "language_score",
"score", "is_cleaned", "token_count",
]
columns = [name for name in wanted if name in names]
samples = []
sampled_group = None
if parquet.metadata.num_row_groups:
sampled_group = rng.randrange(
parquet.metadata.num_row_groups
)
# Read one row group, rather than the entire shard.
table = parquet.read_row_group(
sampled_group, columns=columns
)
indices = rng.sample(
range(table.num_rows), min(3, table.num_rows)
)
samples = table.take(indices).to_pylist()
reports.append({
"file": path.relative_to(ROOT).as_posix(),
"bytes": path.stat().st_size,
"documents": parquet.metadata.num_rows,
"row_groups": parquet.metadata.num_row_groups,
"schema": str(parquet.schema_arrow),
"sampled_row_group": sampled_group,
"samples": samples,
})
return reports
def inspect_tatoeba():
path = DATASETS / "Tatoeba" / "jpn_sentences.tsv"
rng = random.Random(SEED)
languages = Counter()
samples = []
rows = valid_rows = malformed_rows = characters = 0
with path.open(encoding="utf-8") as stream:
for line in stream:
rows += 1
parts = line.rstrip("\r\n").split("\t", 2)
if len(parts) != 3:
malformed_rows += 1
continue
sentence_id, language, text = parts
valid_rows += 1
languages[language] += 1
characters += len(text)
record = {
"line": rows,
"id": sentence_id,
"language": language,
"text": text,
}
# Reservoir sampling: a uniform sample without loading all rows.
if len(samples) < 10:
samples.append(record)
else:
index = rng.randrange(valid_rows)
if index < 10:
samples[index] = record
return {
"file": path.relative_to(ROOT).as_posix(),
"rows": rows,
"malformed_rows": malformed_rows,
"languages": dict(languages),
"raw_text_characters": characters,
"samples": samples,
}
def main():
fineweb = inspect_fineweb()
tatoeba = inspect_tatoeba()
summary = {
"fineweb_shards": len(fineweb),
"fineweb_documents": sum(item["documents"] for item in fineweb),
"fineweb_bytes": sum(item["bytes"] for item in fineweb),
"tatoeba_rows": tatoeba["rows"],
"tatoeba_malformed_rows": tatoeba["malformed_rows"],
"tatoeba_raw_text_characters": tatoeba["raw_text_characters"],
}
report = {
"seed": SEED,
"summary": summary,
"fineweb": fineweb,
"tatoeba": tatoeba,
}
OUTPUT.mkdir(parents=True, exist_ok=True)
destination = OUTPUT / "report.json"
destination.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
print(json.dumps(summary, ensure_ascii=False, indent=2))
print(f"Report: {destination}")
if __name__ == "__main__":
main() |