| """Run real HTTP SDK/management conformance against the isolated sandbox. | |
| Set BUCKETS_E2E_CONFIG to an alternative private config.json to test real Azure. | |
| """ | |
| import json | |
| import os | |
| from pathlib import Path | |
| import sys | |
| import uuid | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT / "src")) | |
| WORK = Path(os.environ.get("BUCKETS_E2E_CONFIG", str(ROOT / "work/buckets-sandbox/config.json"))).parent | |
| config = json.loads((WORK / "config.json").read_text()) | |
| tokens = json.loads((WORK / "credentials.json").read_text()) | |
| endpoint = config["BUCKETS_PUBLIC_ENDPOINT"] | |
| ca = str(WORK / "localhost.pem") | |
| def main(): | |
| import boto3 | |
| import httpx | |
| from botocore.config import Config | |
| from botocore.exceptions import ClientError | |
| base = endpoint.removesuffix("/s3/") | |
| native = httpx.Client( | |
| base_url=base, verify=ca, headers={"Authorization": "Bearer " + tokens["user1_personal"]}, trust_env=False | |
| ) | |
| def api(method, path, **kwargs): | |
| r = native.request(method, path, **kwargs) | |
| if r.status_code >= 400: | |
| raise AssertionError(f"{method} {path}: {r.status_code} {r.text}") | |
| return r.json() if r.content else None | |
| credential = api( | |
| "POST", "/v1/buckets/s3-credentials", json={"permissions": ["storage.read", "storage.write", "storage.admin"]} | |
| ) | |
| def sdk(credential): | |
| return boto3.client( | |
| "s3", | |
| endpoint_url=endpoint, | |
| region_name="us-east-1", | |
| aws_access_key_id=credential["access_key_id"], | |
| aws_secret_access_key=credential["secret_access_key"], | |
| verify=ca, | |
| config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, retries={"max_attempts": 0}), | |
| ) | |
| client = sdk(credential) | |
| name = "e2e-" + uuid.uuid4().hex[:12] | |
| print("Testing real SDK against", endpoint) | |
| client.create_bucket(Bucket=name) | |
| client.head_bucket(Bucket=name) | |
| assert name in [v["Name"] for v in client.list_buckets()["Buckets"]] | |
| assert client.get_bucket_location(Bucket=name)["LocationConstraint"] == "us-east-1" | |
| print("PASS create/head/list/location") | |
| data = b"model-weights\x00" * 1000 | |
| keys = ["weights.bin", "models/模型//../a%2Fb", "models/nested/b", "models/nested/c"] | |
| for key in keys: | |
| result = client.put_object( | |
| Bucket=name, Key=key, Body=data, ContentType="application/octet-stream", Metadata={"test": "e2e"} | |
| ) | |
| assert result["ETag"] | |
| response = client.get_object(Bucket=name, Key=key) | |
| assert response["Body"].read() == data | |
| assert client.head_object(Bucket=name, Key=key)["Metadata"]["test"] == "e2e" | |
| assert client.get_object(Bucket=name, Key="weights.bin", Range="bytes=2-7")["Body"].read() == data[2:8] | |
| print("PASS streaming put/get/head, opaque keys, checksums, range") | |
| first = client.list_objects_v2(Bucket=name, MaxKeys=2) | |
| second = client.list_objects_v2(Bucket=name, MaxKeys=2, ContinuationToken=first["NextContinuationToken"]) | |
| assert len(first["Contents"]) + len(second["Contents"]) == 4 | |
| grouped = client.list_objects_v2(Bucket=name, Prefix="models/", Delimiter="/") | |
| assert grouped.get("CommonPrefixes") | |
| print("PASS list pagination and delimiter") | |
| url = client.generate_presigned_url("get_object", Params={"Bucket": name, "Key": "weights.bin"}, ExpiresIn=60) | |
| assert httpx.get(url, verify=ca, trust_env=False).content == data | |
| print("PASS presigned URL") | |
| upload = client.create_multipart_upload(Bucket=name, Key="multipart.bin")["UploadId"] | |
| parts = [] | |
| for number, body in [(1, b"A" * (5 * 1024 * 1024)), (2, b"B" * 100)]: | |
| value = client.upload_part(Bucket=name, Key="multipart.bin", UploadId=upload, PartNumber=number, Body=body) | |
| parts.append({"PartNumber": number, "ETag": value["ETag"]}) | |
| assert len(client.list_parts(Bucket=name, Key="multipart.bin", UploadId=upload)["Parts"]) == 2 | |
| client.complete_multipart_upload( | |
| Bucket=name, Key="multipart.bin", UploadId=upload, MultipartUpload={"Parts": parts} | |
| ) | |
| assert client.head_object(Bucket=name, Key="multipart.bin")["ContentLength"] == 5 * 1024 * 1024 + 100 | |
| aborted = client.create_multipart_upload(Bucket=name, Key="aborted")["UploadId"] | |
| client.abort_multipart_upload(Bucket=name, Key="aborted", UploadId=aborted) | |
| print("PASS durable multipart/list/complete/abort") | |
| other = httpx.Client( | |
| base_url=base, verify=ca, headers={"Authorization": "Bearer " + tokens["user2_personal"]}, trust_env=False | |
| ) | |
| other_credential = other.post("/v1/buckets/s3-credentials", json={"permissions": ["storage.read"]}).json() | |
| try: | |
| sdk(other_credential).get_object(Bucket=name, Key="weights.bin") | |
| raise AssertionError("Cross tenant read succeeded") | |
| except ClientError as exc: | |
| assert exc.response["ResponseMetadata"]["HTTPStatusCode"] == 404 | |
| native.delete("/v1/buckets/s3-credentials/" + credential["id"]).raise_for_status() | |
| try: | |
| client.list_buckets() | |
| raise AssertionError("Revoked key succeeded") | |
| except ClientError as exc: | |
| assert exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403 | |
| print("PASS two-user isolation and credential revocation") | |
| # Clean up via the still-authorized native identity. | |
| bucket = next(v for v in api("GET", "/v1/buckets")["buckets"] if v["name"] == name) | |
| from urllib.parse import quote | |
| for key in keys + ["multipart.bin"]: | |
| api("DELETE", "/s3/" + name + "/" + quote(key, safe="").replace(".", "%2E")) | |
| api("DELETE", "/v1/buckets/" + bucket["id"]) | |
| print("ALL HTTP E2E CHECKS PASSED") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 5.7 kB
- Xet hash:
- 622f9f8cd985b2796908085eba3a0e11c5b75b79b85356479eff51a62bfcb2d7
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.