| """5 GiB+ multipart test on the isolated real Azure backend, with bounded memory. | |
| BUCKETS_E2E_CONFIG=work/buckets-azure-e2e/config.json python tools/buckets_large_e2e.py | |
| Refuses the emulator configuration. Uploads one 32 MiB in-memory block repeatedly; | |
| no 5 GiB scratch file or full-object buffer is created. Azure retention is billable. | |
| """ | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import resource | |
| import time | |
| from uuid import uuid4 | |
| import boto3 | |
| from botocore.config import Config | |
| import httpx | |
| def main(): | |
| config_path = Path(os.environ["BUCKETS_E2E_CONFIG"]) | |
| directory = config_path.parent | |
| config = json.loads(config_path.read_text()) | |
| if not config.get("BUCKETS_AZURE_ACCOUNT_URL", "").startswith("https://"): | |
| raise RuntimeError("The large test requires the explicitly configured isolated Azure backend") | |
| tokens = json.loads((directory / "credentials.json").read_text()) | |
| endpoint = config["BUCKETS_PUBLIC_ENDPOINT"] | |
| native = httpx.Client( | |
| base_url=endpoint.removesuffix("/s3/"), | |
| verify=str(directory / "localhost.pem"), | |
| trust_env=False, | |
| headers={"Authorization": "Bearer " + tokens["user1_personal"]}, | |
| timeout=120, | |
| ) | |
| response = native.post( | |
| "/v1/buckets/s3-credentials", | |
| json={"permissions": ["storage.read", "storage.write", "storage.admin"], "rate_limit_rpm": 600}, | |
| ) | |
| response.raise_for_status() | |
| credential = response.json() | |
| client = boto3.client( | |
| "s3", | |
| endpoint_url=endpoint, | |
| region_name="us-east-1", | |
| verify=str(directory / "localhost.pem"), | |
| aws_access_key_id=credential["access_key_id"], | |
| aws_secret_access_key=credential["secret_access_key"], | |
| config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=30, read_timeout=600), | |
| ) | |
| name = "large-e2e-" + uuid4().hex[:12] | |
| identifier = None | |
| completed = False | |
| started = time.monotonic() | |
| try: | |
| client.create_bucket(Bucket=name) | |
| identifier = client.create_multipart_upload(Bucket=name, Key="model-5gib.bin")["UploadId"] | |
| part = b"W" * (32 * 1024**2) | |
| total = 5 * 1024**3 + 1 | |
| sent = 0 | |
| parts = [] | |
| while sent < total: | |
| body = part if total - sent >= len(part) else b"Z" * (total - sent) | |
| number = len(parts) + 1 | |
| result = client.upload_part( | |
| Bucket=name, Key="model-5gib.bin", UploadId=identifier, PartNumber=number, Body=body | |
| ) | |
| parts.append({"PartNumber": number, "ETag": result["ETag"]}) | |
| sent += len(body) | |
| if number % 8 == 0: | |
| print( | |
| f"Uploaded {sent / (1024**3):.2f} GiB; part {number}; elapsed {time.monotonic() - started:.0f}s", | |
| flush=True, | |
| ) | |
| result = client.complete_multipart_upload( | |
| Bucket=name, Key="model-5gib.bin", UploadId=identifier, MultipartUpload={"Parts": parts} | |
| ) | |
| completed = True | |
| assert client.head_object(Bucket=name, Key="model-5gib.bin")["ContentLength"] == total | |
| assert ( | |
| client.get_object(Bucket=name, Key="model-5gib.bin", Range=f"bytes={total - 1}-{total - 1}")["Body"].read() | |
| == b"Z" | |
| ) | |
| expected = ( | |
| hashlib.md5(b"".join(bytes.fromhex(p["ETag"].strip('"')) for p in parts), usedforsecurity=False).hexdigest() | |
| + f"-{len(parts)}" | |
| ) | |
| assert result["ETag"].strip('"') == expected | |
| print( | |
| f"PASS real Azure 5 GiB+ multipart: {total} bytes, {len(parts)} parts, correct length/ETag/range; client RSS {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024:.0f} MiB", | |
| flush=True, | |
| ) | |
| finally: | |
| if identifier and not completed: | |
| client.abort_multipart_upload(Bucket=name, Key="model-5gib.bin", UploadId=identifier) | |
| if completed: | |
| client.delete_object(Bucket=name, Key="model-5gib.bin") | |
| client.delete_bucket(Bucket=name) | |
| native.delete("/v1/buckets/s3-credentials/" + credential["id"]).raise_for_status() | |
| native.close() | |
| client.close() | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 4.25 kB
- Xet hash:
- 06e62caddb657e3b2f5ff635682915f3a14b0fa201f99742ac2577d9234d5593
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.