jaivial's picture
Upload folder using huggingface_hub
a8a96bf verified
Raw
History Blame Contribute Delete
2 kB
#!/usr/bin/env python3
"""Create deterministic V25PG01 fixed-slot page artifacts for tests/benchmarks."""
import argparse
import struct
import zlib
from pathlib import Path
HEADER_BYTES = 4096
PAGE_HEADER_BYTES = 16
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("output", type=Path)
parser.add_argument("--pages", type=int, default=128)
parser.add_argument("--page-bytes", type=int, default=4096)
parser.add_argument("--useful-bytes", type=int, default=1000)
parser.add_argument("--logical-params", type=int, default=0)
args = parser.parse_args()
if not 1 <= args.pages <= 1_000_000:
parser.error("--pages must be between 1 and 1000000")
if not 4096 <= args.page_bytes <= 16 * 1024 * 1024 or args.page_bytes % 4096:
parser.error("--page-bytes must be a 4096-byte multiple up to 16 MiB")
if not 0 <= args.useful_bytes <= args.page_bytes - PAGE_HEADER_BYTES:
parser.error("--useful-bytes exceeds page payload")
if args.logical_params < 0:
parser.error("--logical-params must be nonnegative")
header = bytearray(HEADER_BYTES)
struct.pack_into(
"<8sIIIIQQQIII",
header,
0,
b"V25PG01\0",
1,
HEADER_BYTES,
args.page_bytes,
PAGE_HEADER_BYTES,
args.pages,
HEADER_BYTES,
args.logical_params,
1,
0,
args.useful_bytes,
)
struct.pack_into("<I", header, 60, zlib.crc32(header[:60]))
with args.output.open("wb") as output:
output.write(header)
padding = bytes(args.page_bytes - PAGE_HEADER_BYTES - args.useful_bytes)
for page_id in range(args.pages):
payload = bytes(((page_id * 131 + i * 17) & 0xFF) for i in range(args.useful_bytes))
output.write(struct.pack("<QII", page_id, len(payload), zlib.crc32(payload)))
output.write(payload)
output.write(padding)
if __name__ == "__main__":
main()