sharktide/tools / buckets_large_e2e.py
sharktide's picture
download
raw
4.25 kB
"""5 GiB+ multipart test on the isolated real Azure backend, with bounded memory.
BUCKETS_E2E_CONFIG=work/buckets-azure-e2e/config.json python tools/buckets_large_e2e.py
Refuses the emulator configuration. Uploads one 32 MiB in-memory block repeatedly;
no 5 GiB scratch file or full-object buffer is created. Azure retention is billable.
"""
import hashlib
import json
import os
from pathlib import Path
import resource
import time
from uuid import uuid4
import boto3
from botocore.config import Config
import httpx
def main():
config_path = Path(os.environ["BUCKETS_E2E_CONFIG"])
directory = config_path.parent
config = json.loads(config_path.read_text())
if not config.get("BUCKETS_AZURE_ACCOUNT_URL", "").startswith("https://"):
raise RuntimeError("The large test requires the explicitly configured isolated Azure backend")
tokens = json.loads((directory / "credentials.json").read_text())
endpoint = config["BUCKETS_PUBLIC_ENDPOINT"]
native = httpx.Client(
base_url=endpoint.removesuffix("/s3/"),
verify=str(directory / "localhost.pem"),
trust_env=False,
headers={"Authorization": "Bearer " + tokens["user1_personal"]},
timeout=120,
)
response = native.post(
"/v1/buckets/s3-credentials",
json={"permissions": ["storage.read", "storage.write", "storage.admin"], "rate_limit_rpm": 600},
)
response.raise_for_status()
credential = response.json()
client = boto3.client(
"s3",
endpoint_url=endpoint,
region_name="us-east-1",
verify=str(directory / "localhost.pem"),
aws_access_key_id=credential["access_key_id"],
aws_secret_access_key=credential["secret_access_key"],
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=30, read_timeout=600),
)
name = "large-e2e-" + uuid4().hex[:12]
identifier = None
completed = False
started = time.monotonic()
try:
client.create_bucket(Bucket=name)
identifier = client.create_multipart_upload(Bucket=name, Key="model-5gib.bin")["UploadId"]
part = b"W" * (32 * 1024**2)
total = 5 * 1024**3 + 1
sent = 0
parts = []
while sent < total:
body = part if total - sent >= len(part) else b"Z" * (total - sent)
number = len(parts) + 1
result = client.upload_part(
Bucket=name, Key="model-5gib.bin", UploadId=identifier, PartNumber=number, Body=body
)
parts.append({"PartNumber": number, "ETag": result["ETag"]})
sent += len(body)
if number % 8 == 0:
print(
f"Uploaded {sent / (1024**3):.2f} GiB; part {number}; elapsed {time.monotonic() - started:.0f}s",
flush=True,
)
result = client.complete_multipart_upload(
Bucket=name, Key="model-5gib.bin", UploadId=identifier, MultipartUpload={"Parts": parts}
)
completed = True
assert client.head_object(Bucket=name, Key="model-5gib.bin")["ContentLength"] == total
assert (
client.get_object(Bucket=name, Key="model-5gib.bin", Range=f"bytes={total - 1}-{total - 1}")["Body"].read()
== b"Z"
)
expected = (
hashlib.md5(b"".join(bytes.fromhex(p["ETag"].strip('"')) for p in parts), usedforsecurity=False).hexdigest()
+ f"-{len(parts)}"
)
assert result["ETag"].strip('"') == expected
print(
f"PASS real Azure 5 GiB+ multipart: {total} bytes, {len(parts)} parts, correct length/ETag/range; client RSS {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024:.0f} MiB",
flush=True,
)
finally:
if identifier and not completed:
client.abort_multipart_upload(Bucket=name, Key="model-5gib.bin", UploadId=identifier)
if completed:
client.delete_object(Bucket=name, Key="model-5gib.bin")
client.delete_bucket(Bucket=name)
native.delete("/v1/buckets/s3-credentials/" + credential["id"]).raise_for_status()
native.close()
client.close()
if __name__ == "__main__":
main()

Xet Storage Details

Size:
4.25 kB
·
Xet hash:
06e62caddb657e3b2f5ff635682915f3a14b0fa201f99742ac2577d9234d5593

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.