Spaces:
Sleeping
Sleeping
File size: 3,558 Bytes
15eb4b9 8eaca44 15eb4b9 8eaca44 15eb4b9 8eaca44 15eb4b9 8eaca44 15eb4b9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | """Build a toy CC bucket layout for local verification.
Creates, under <root> (default tests/fixtures/cc), the same two subtrees the real
commoncrawl bucket has:
crawl-data/CC-MAIN-2026-25/segments/0001/warc/toy.warc.gz (a few WARC records)
cc-index/table/cc-main/warc/crawl=CC-MAIN-2026-25/subset=warc/part-0.parquet
The parquet index rows carry the exact gzip-member byte ranges of the WARC records,
so `cdxt repackage` can fetch them by range. Three records cover two domains plus a
non-matching control host.
"""
from __future__ import annotations
import os
import sys
from io import BytesIO
import pyarrow as pa
import pyarrow.parquet as pq
from warcio.statusandheaders import StatusAndHeaders
from warcio.warcwriter import WARCWriter
CRAWL = "CC-MAIN-2026-25"
WARC_RELPATH = f"crawl-data/{CRAWL}/segments/0001/warc/toy.warc.gz"
# (url, host, registered_domain, tld, content_languages, payload)
# content_languages mirrors the real index: comma-separated ISO-639-3, most confident
# first — so "deu,eng" must match a filter for either deu or eng.
RECORDS = [
("https://example.com/", "example.com", "example.com", "com", "eng", b"<html>example.com homepage</html>"),
("https://blog.example.org/post", "blog.example.org", "example.org", "org", "deu,eng", b"<html>blog post on example.org with more bytes " + b"x" * 200 + b"</html>"),
("https://www.other.net/", "www.other.net", "other.net", "net", "fra", b"<html>unrelated control host</html>"),
]
def _write_warc(path: str):
os.makedirs(os.path.dirname(path), exist_ok=True)
rows = []
with open(path, "wb") as out:
writer = WARCWriter(out, gzip=True)
writer.write_record(
writer.create_warcinfo_record(os.path.basename(path), {"software": "cc-repackage-fixture"})
)
for url, host, regd, tld, langs, payload in RECORDS:
offset = out.tell()
http_headers = StatusAndHeaders(
"200 OK", [("Content-Type", "text/html")], protocol="HTTP/1.1"
)
rec = writer.create_warc_record(
url, "response", payload=BytesIO(payload), http_headers=http_headers
)
writer.write_record(rec)
length = out.tell() - offset
rows.append(
{
"url": url,
"url_host_name": host,
"url_host_registered_domain": regd,
"url_host_tld": tld,
"warc_filename": WARC_RELPATH,
"warc_record_offset": offset,
"warc_record_length": length,
"content_languages": langs,
"content_mime_type": "text/html",
"fetch_status": 200,
}
)
return rows
def _write_index(root: str, rows):
part_dir = os.path.join(
root, "cc-index", "table", "cc-main", "warc", f"crawl={CRAWL}", "subset=warc"
)
os.makedirs(part_dir, exist_ok=True)
# crawl/subset come from the hive partition path, so they are NOT data columns.
table = pa.Table.from_pylist(rows)
pq.write_table(table, os.path.join(part_dir, "part-0.parquet"))
def build(root: str = "tests/fixtures/cc") -> str:
warc_path = os.path.join(root, WARC_RELPATH)
rows = _write_warc(warc_path)
_write_index(root, rows)
print(f"fixtures built at {root}: {len(rows)} index rows, warc={warc_path}")
return root
if __name__ == "__main__":
build(sys.argv[1] if len(sys.argv) > 1 else "tests/fixtures/cc")
|