Publish WDC product embedding checkpoint, training source, and evaluation

a360597 verified 6 days ago

1.94 kB

"""Randomly sample original corpus rows, without using the cleaned subset."""

import gzip
import json
from pathlib import Path
import random


def main():
    root = Path(__file__).resolve().parents[1]
    source = root / "offers_corpus_all_v2_non_norm.json.gz"
    manifest = json.loads((root / "artifacts/prepared/manifest.json").read_text())
    row_count = manifest["rows_scanned"]
    # Random order provides an unbiased order of candidate rows. Extra candidates
    # allow skipping records without a usable title while keeping 50 products.
    indices = random.SystemRandom().sample(range(row_count), 100)
    wanted = set(indices)
    found = {}
    observed = 0
    with gzip.open(source, "rb") as handle:
        for index, line in enumerate(handle):
            observed = index + 1
            if index in wanted:
                record = json.loads(line)
                title = record.get("title")
                if isinstance(title, str) and title.strip():
                    found[index] = {"source_row": index + 1, "record": record}
    if observed != row_count:
        raise ValueError("Original row count differs from the manifest")
    samples = [found[index] for index in indices if index in found][:50]
    if len(samples) != 50:
        raise ValueError("Insufficient titled rows in the random candidates; retry")
    output = root / "artifacts"
    # Preserve raw source records as provenance; only collapse whitespace in the
    # text export so each title fits on one line in the web interface.
    (output / "random-products-50.json").write_text(json.dumps(samples, ensure_ascii=False, indent=2), encoding="utf-8")
    titles = [" ".join(item["record"]["title"].split()) for item in samples]
    (output / "random-products-50.txt").write_text("\n".join(titles) + "\n", encoding="utf-8")
    print(f"Sampled {len(samples)} titles from {observed:,} original rows.")


if __name__ == "__main__":
    main()

	"""Randomly sample original corpus rows, without using the cleaned subset."""

	import gzip
	import json
	from pathlib import Path
	import random


	def main():
	root = Path(__file__).resolve().parents[1]
	source = root / "offers_corpus_all_v2_non_norm.json.gz"
	manifest = json.loads((root / "artifacts/prepared/manifest.json").read_text())
	row_count = manifest["rows_scanned"]
	# Random order provides an unbiased order of candidate rows. Extra candidates
	# allow skipping records without a usable title while keeping 50 products.
	indices = random.SystemRandom().sample(range(row_count), 100)
	wanted = set(indices)
	found = {}
	observed = 0
	with gzip.open(source, "rb") as handle:
	for index, line in enumerate(handle):
	observed = index + 1
	if index in wanted:
	record = json.loads(line)
	title = record.get("title")
	if isinstance(title, str) and title.strip():
	found[index] = {"source_row": index + 1, "record": record}
	if observed != row_count:
	raise ValueError("Original row count differs from the manifest")
	samples = [found[index] for index in indices if index in found][:50]
	if len(samples) != 50:
	raise ValueError("Insufficient titled rows in the random candidates; retry")
	output = root / "artifacts"
	# Preserve raw source records as provenance; only collapse whitespace in the
	# text export so each title fits on one line in the web interface.
	(output / "random-products-50.json").write_text(json.dumps(samples, ensure_ascii=False, indent=2), encoding="utf-8")
	titles = [" ".join(item["record"]["title"].split()) for item in samples]
	(output / "random-products-50.txt").write_text("\n".join(titles) + "\n", encoding="utf-8")
	print(f"Sampled {len(samples)} titles from {observed:,} original rows.")


	if __name__ == "__main__":
	main()