| """Real rclone transfers against only the disposable loopback storage sandbox.""" | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import ssl | |
| import subprocess | |
| import tempfile | |
| from urllib.parse import urlsplit | |
| from uuid import uuid4 | |
| import httpx | |
| ROOT = Path(__file__).resolve().parents[1] | |
| WORK = ROOT / "work/buckets-sandbox" | |
| def main(): | |
| config = json.loads((WORK / "config.json").read_text()) | |
| database = urlsplit(config["BUCKETS_DATABASE_URL"]) | |
| endpoint = urlsplit(config["BUCKETS_PUBLIC_ENDPOINT"]) | |
| if database.hostname not in {"localhost", "127.0.0.1"} or database.port != 55432 or database.path != "/buckets": | |
| raise RuntimeError("rclone drill requires the isolated local database") | |
| if endpoint.hostname not in {"localhost", "127.0.0.1"} or endpoint.port != 58080 or endpoint.scheme != "https": | |
| raise RuntimeError("rclone drill requires the isolated local HTTPS gateway") | |
| tokens = json.loads((WORK / "credentials.json").read_text()) | |
| ca = WORK / "localhost.pem" | |
| name = "rclone-" + uuid4().hex[:16] | |
| credential = bucket = None | |
| with httpx.Client( | |
| base_url=f"https://{endpoint.netloc}", | |
| verify=ssl.create_default_context(cafile=str(ca)), | |
| trust_env=False, | |
| headers={"Authorization": "Bearer " + tokens["user1_org"]}, | |
| ) as native: | |
| try: | |
| response = native.post("/v1/buckets", json={"name": name, "quota_bytes": 64 * 1024**2}) | |
| response.raise_for_status() | |
| bucket = response.json() | |
| response = native.post( | |
| "/v1/buckets/s3-credentials", | |
| json={ | |
| "permissions": ["storage.read", "storage.write", "storage.admin"], | |
| "bucket_ids": [bucket["id"]], | |
| "rate_limit_rpm": 600, | |
| }, | |
| ) | |
| response.raise_for_status() | |
| credential = response.json() | |
| env = { | |
| **os.environ, | |
| "RCLONE_CONFIG_IP_TYPE": "s3", | |
| "RCLONE_CONFIG_IP_PROVIDER": "Other", | |
| "RCLONE_CONFIG_IP_ENDPOINT": credential["endpoint"], | |
| "RCLONE_CONFIG_IP_REGION": "us-east-1", | |
| "RCLONE_CONFIG_IP_ACCESS_KEY_ID": credential["access_key_id"], | |
| "RCLONE_CONFIG_IP_SECRET_ACCESS_KEY": credential["secret_access_key"], | |
| "RCLONE_CONFIG_IP_FORCE_PATH_STYLE": "true", | |
| "RCLONE_CA_CERT": str(ca), | |
| } | |
| def rclone(*args): | |
| result = subprocess.run( | |
| [os.getenv("RCLONE_BINARY", "rclone"), "--config", os.devnull, *args], | |
| env=env, | |
| capture_output=True, | |
| timeout=120, | |
| ) | |
| if result.returncode: | |
| raise RuntimeError("rclone failed: " + result.stderr.decode(errors="replace")[-2000:]) | |
| return result.stdout | |
| with tempfile.TemporaryDirectory(dir=WORK) as folder: | |
| source, downloaded = Path(folder) / "source", Path(folder) / "downloaded" | |
| (source / "nested").mkdir(parents=True) | |
| files = { | |
| "model weights.bin": b"multipart-rclone\0" * (768 * 1024), | |
| "nested/λ.txt": "opaque UTF-8 key + literal plus".encode(), | |
| "empty": b"", | |
| } | |
| for key, body in files.items(): | |
| (source / key).write_bytes(body) | |
| remote = f"ip:{name}" | |
| rclone( | |
| "sync", | |
| str(source), | |
| remote + "/data", | |
| "--checksum", | |
| "--transfers", | |
| "2", | |
| "--s3-upload-cutoff", | |
| "5Mi", | |
| "--s3-chunk-size", | |
| "5Mi", | |
| ) | |
| listed = json.loads(rclone("lsjson", remote + "/data", "--recursive")) | |
| assert {row["Path"] for row in listed if not row["IsDir"]} == set(files) | |
| rclone("check", str(source), remote + "/data", "--one-way") | |
| rclone("copy", remote + "/data", str(downloaded), "--checksum") | |
| for key, body in files.items(): | |
| assert hashlib.sha256((downloaded / key).read_bytes()).digest() == hashlib.sha256(body).digest() | |
| rclone("copyto", remote + "/data/model weights.bin", remote + "/copied.bin") | |
| rclone("moveto", remote + "/copied.bin", remote + "/moved.bin") | |
| assert rclone("cat", remote + "/moved.bin", "--head", "11") == files["model weights.bin"][:11] | |
| rclone("touch", remote + "/moved.bin", "--no-create", "--timestamp", "2026-01-01T12:00:00Z") | |
| info = json.loads(rclone("lsjson", remote + "/moved.bin", "--stat")) | |
| assert info["ModTime"].startswith("2026-01-01T12:00:00") | |
| (source / "empty").unlink() | |
| rclone("sync", str(source), remote + "/data", "--checksum") | |
| listed = json.loads(rclone("lsjson", remote + "/data", "--recursive")) | |
| assert "empty" not in {row["Path"] for row in listed} | |
| rclone("purge", remote) | |
| assert native.get("/v1/buckets/" + bucket["id"]).status_code == 404 | |
| print( | |
| "PASS rclone: org-scoped multipart, UTF-8 keys, empty files, checksum sync/check, download, server copy/move, modtime, ranges, purge" | |
| ) | |
| finally: | |
| if bucket: | |
| response = native.get("/v1/buckets/" + bucket["id"] + "/objects") | |
| if response.status_code == 200: | |
| for obj in response.json()["objects"]: | |
| native.delete("/v1/buckets/" + bucket["id"] + "/object", params={"key": obj["object_key"]}) | |
| native.delete("/v1/buckets/" + bucket["id"]) | |
| if credential: | |
| native.delete("/v1/buckets/s3-credentials/" + credential["id"]) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 6.1 kB
- Xet hash:
- 799f985e4b15b975dd02f3e30bfd39f8944de4047162e56351963a9e47e69b46
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.