File size: 1,935 Bytes
a360597 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | """Randomly sample original corpus rows, without using the cleaned subset."""
import gzip
import json
from pathlib import Path
import random
def main():
root = Path(__file__).resolve().parents[1]
source = root / "offers_corpus_all_v2_non_norm.json.gz"
manifest = json.loads((root / "artifacts/prepared/manifest.json").read_text())
row_count = manifest["rows_scanned"]
# Random order provides an unbiased order of candidate rows. Extra candidates
# allow skipping records without a usable title while keeping 50 products.
indices = random.SystemRandom().sample(range(row_count), 100)
wanted = set(indices)
found = {}
observed = 0
with gzip.open(source, "rb") as handle:
for index, line in enumerate(handle):
observed = index + 1
if index in wanted:
record = json.loads(line)
title = record.get("title")
if isinstance(title, str) and title.strip():
found[index] = {"source_row": index + 1, "record": record}
if observed != row_count:
raise ValueError("Original row count differs from the manifest")
samples = [found[index] for index in indices if index in found][:50]
if len(samples) != 50:
raise ValueError("Insufficient titled rows in the random candidates; retry")
output = root / "artifacts"
# Preserve raw source records as provenance; only collapse whitespace in the
# text export so each title fits on one line in the web interface.
(output / "random-products-50.json").write_text(json.dumps(samples, ensure_ascii=False, indent=2), encoding="utf-8")
titles = [" ".join(item["record"]["title"].split()) for item in samples]
(output / "random-products-50.txt").write_text("\n".join(titles) + "\n", encoding="utf-8")
print(f"Sampled {len(samples)} titles from {observed:,} original rows.")
if __name__ == "__main__":
main()
|