| { | |
| "generated_at": "2026-09-11T07:23:51.120123+00:00", | |
| "lang": "snd_Arab", | |
| "map_url": "https://data.hplt-project.org/three/sorted/snd_Arab.map", | |
| "bins_requested": [ | |
| 10, | |
| 9 | |
| ], | |
| "prob_threshold": 0.8, | |
| "min_chars": 20, | |
| "val_fraction": 0.005, | |
| "max_docs_per_shard": 50000, | |
| "max_chars_per_shard": 500000000, | |
| "max_shards": null, | |
| "stream_download": true, | |
| "shards": [ | |
| { | |
| "url": "https://data.hplt-project.org/three/sorted/snd_Arab/10_1.jsonl.zst", | |
| "bin": 10, | |
| "raw_docs": 167, | |
| "dropped_low_prob": 0, | |
| "dropped_empty": 0, | |
| "dropped_exact_dup": 0, | |
| "pii_spans_redacted": 17, | |
| "kept_train": 167, | |
| "kept_val": 0, | |
| "chars_kept": 3752256 | |
| }, | |
| { | |
| "url": "https://data.hplt-project.org/three/sorted/snd_Arab/9_1.jsonl.zst", | |
| "bin": 9, | |
| "raw_docs": 32166, | |
| "dropped_low_prob": 9, | |
| "dropped_empty": 0, | |
| "dropped_exact_dup": 0, | |
| "pii_spans_redacted": 1356, | |
| "kept_train": 32004, | |
| "kept_val": 153, | |
| "chars_kept": 247251424 | |
| } | |
| ], | |
| "totals": { | |
| "raw_docs": 32333, | |
| "kept_train": 32171, | |
| "kept_val": 153, | |
| "dropped_low_prob": 9, | |
| "dropped_empty": 0, | |
| "dropped_exact_dup": 0, | |
| "pii_spans_redacted": 1373, | |
| "chars_kept": 251003680 | |
| }, | |
| "consolidated_counts": { | |
| "train": 32171, | |
| "val": 153 | |
| } | |
| } |