sharktide/tools / buckets_e2e.py
sharktide's picture
download
raw
5.7 kB
"""Run real HTTP SDK/management conformance against the isolated sandbox.
Set BUCKETS_E2E_CONFIG to an alternative private config.json to test real Azure.
"""
import json
import os
from pathlib import Path
import sys
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
WORK = Path(os.environ.get("BUCKETS_E2E_CONFIG", str(ROOT / "work/buckets-sandbox/config.json"))).parent
config = json.loads((WORK / "config.json").read_text())
tokens = json.loads((WORK / "credentials.json").read_text())
endpoint = config["BUCKETS_PUBLIC_ENDPOINT"]
ca = str(WORK / "localhost.pem")
def main():
import boto3
import httpx
from botocore.config import Config
from botocore.exceptions import ClientError
base = endpoint.removesuffix("/s3/")
native = httpx.Client(
base_url=base, verify=ca, headers={"Authorization": "Bearer " + tokens["user1_personal"]}, trust_env=False
)
def api(method, path, **kwargs):
r = native.request(method, path, **kwargs)
if r.status_code >= 400:
raise AssertionError(f"{method} {path}: {r.status_code} {r.text}")
return r.json() if r.content else None
credential = api(
"POST", "/v1/buckets/s3-credentials", json={"permissions": ["storage.read", "storage.write", "storage.admin"]}
)
def sdk(credential):
return boto3.client(
"s3",
endpoint_url=endpoint,
region_name="us-east-1",
aws_access_key_id=credential["access_key_id"],
aws_secret_access_key=credential["secret_access_key"],
verify=ca,
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, retries={"max_attempts": 0}),
)
client = sdk(credential)
name = "e2e-" + uuid.uuid4().hex[:12]
print("Testing real SDK against", endpoint)
client.create_bucket(Bucket=name)
client.head_bucket(Bucket=name)
assert name in [v["Name"] for v in client.list_buckets()["Buckets"]]
assert client.get_bucket_location(Bucket=name)["LocationConstraint"] == "us-east-1"
print("PASS create/head/list/location")
data = b"model-weights\x00" * 1000
keys = ["weights.bin", "models/模型//../a%2Fb", "models/nested/b", "models/nested/c"]
for key in keys:
result = client.put_object(
Bucket=name, Key=key, Body=data, ContentType="application/octet-stream", Metadata={"test": "e2e"}
)
assert result["ETag"]
response = client.get_object(Bucket=name, Key=key)
assert response["Body"].read() == data
assert client.head_object(Bucket=name, Key=key)["Metadata"]["test"] == "e2e"
assert client.get_object(Bucket=name, Key="weights.bin", Range="bytes=2-7")["Body"].read() == data[2:8]
print("PASS streaming put/get/head, opaque keys, checksums, range")
first = client.list_objects_v2(Bucket=name, MaxKeys=2)
second = client.list_objects_v2(Bucket=name, MaxKeys=2, ContinuationToken=first["NextContinuationToken"])
assert len(first["Contents"]) + len(second["Contents"]) == 4
grouped = client.list_objects_v2(Bucket=name, Prefix="models/", Delimiter="/")
assert grouped.get("CommonPrefixes")
print("PASS list pagination and delimiter")
url = client.generate_presigned_url("get_object", Params={"Bucket": name, "Key": "weights.bin"}, ExpiresIn=60)
assert httpx.get(url, verify=ca, trust_env=False).content == data
print("PASS presigned URL")
upload = client.create_multipart_upload(Bucket=name, Key="multipart.bin")["UploadId"]
parts = []
for number, body in [(1, b"A" * (5 * 1024 * 1024)), (2, b"B" * 100)]:
value = client.upload_part(Bucket=name, Key="multipart.bin", UploadId=upload, PartNumber=number, Body=body)
parts.append({"PartNumber": number, "ETag": value["ETag"]})
assert len(client.list_parts(Bucket=name, Key="multipart.bin", UploadId=upload)["Parts"]) == 2
client.complete_multipart_upload(
Bucket=name, Key="multipart.bin", UploadId=upload, MultipartUpload={"Parts": parts}
)
assert client.head_object(Bucket=name, Key="multipart.bin")["ContentLength"] == 5 * 1024 * 1024 + 100
aborted = client.create_multipart_upload(Bucket=name, Key="aborted")["UploadId"]
client.abort_multipart_upload(Bucket=name, Key="aborted", UploadId=aborted)
print("PASS durable multipart/list/complete/abort")
other = httpx.Client(
base_url=base, verify=ca, headers={"Authorization": "Bearer " + tokens["user2_personal"]}, trust_env=False
)
other_credential = other.post("/v1/buckets/s3-credentials", json={"permissions": ["storage.read"]}).json()
try:
sdk(other_credential).get_object(Bucket=name, Key="weights.bin")
raise AssertionError("Cross tenant read succeeded")
except ClientError as exc:
assert exc.response["ResponseMetadata"]["HTTPStatusCode"] == 404
native.delete("/v1/buckets/s3-credentials/" + credential["id"]).raise_for_status()
try:
client.list_buckets()
raise AssertionError("Revoked key succeeded")
except ClientError as exc:
assert exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403
print("PASS two-user isolation and credential revocation")
# Clean up via the still-authorized native identity.
bucket = next(v for v in api("GET", "/v1/buckets")["buckets"] if v["name"] == name)
from urllib.parse import quote
for key in keys + ["multipart.bin"]:
api("DELETE", "/s3/" + name + "/" + quote(key, safe="").replace(".", "%2E"))
api("DELETE", "/v1/buckets/" + bucket["id"])
print("ALL HTTP E2E CHECKS PASSED")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
5.7 kB
·
Xet hash:
622f9f8cd985b2796908085eba3a0e11c5b75b79b85356479eff51a62bfcb2d7

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.