sharktide/tools / buckets_rclone_e2e.py
sharktide's picture
download
raw
6.1 kB
"""Real rclone transfers against only the disposable loopback storage sandbox."""
import hashlib
import json
import os
from pathlib import Path
import ssl
import subprocess
import tempfile
from urllib.parse import urlsplit
from uuid import uuid4
import httpx
ROOT = Path(__file__).resolve().parents[1]
WORK = ROOT / "work/buckets-sandbox"
def main():
config = json.loads((WORK / "config.json").read_text())
database = urlsplit(config["BUCKETS_DATABASE_URL"])
endpoint = urlsplit(config["BUCKETS_PUBLIC_ENDPOINT"])
if database.hostname not in {"localhost", "127.0.0.1"} or database.port != 55432 or database.path != "/buckets":
raise RuntimeError("rclone drill requires the isolated local database")
if endpoint.hostname not in {"localhost", "127.0.0.1"} or endpoint.port != 58080 or endpoint.scheme != "https":
raise RuntimeError("rclone drill requires the isolated local HTTPS gateway")
tokens = json.loads((WORK / "credentials.json").read_text())
ca = WORK / "localhost.pem"
name = "rclone-" + uuid4().hex[:16]
credential = bucket = None
with httpx.Client(
base_url=f"https://{endpoint.netloc}",
verify=ssl.create_default_context(cafile=str(ca)),
trust_env=False,
headers={"Authorization": "Bearer " + tokens["user1_org"]},
) as native:
try:
response = native.post("/v1/buckets", json={"name": name, "quota_bytes": 64 * 1024**2})
response.raise_for_status()
bucket = response.json()
response = native.post(
"/v1/buckets/s3-credentials",
json={
"permissions": ["storage.read", "storage.write", "storage.admin"],
"bucket_ids": [bucket["id"]],
"rate_limit_rpm": 600,
},
)
response.raise_for_status()
credential = response.json()
env = {
**os.environ,
"RCLONE_CONFIG_IP_TYPE": "s3",
"RCLONE_CONFIG_IP_PROVIDER": "Other",
"RCLONE_CONFIG_IP_ENDPOINT": credential["endpoint"],
"RCLONE_CONFIG_IP_REGION": "us-east-1",
"RCLONE_CONFIG_IP_ACCESS_KEY_ID": credential["access_key_id"],
"RCLONE_CONFIG_IP_SECRET_ACCESS_KEY": credential["secret_access_key"],
"RCLONE_CONFIG_IP_FORCE_PATH_STYLE": "true",
"RCLONE_CA_CERT": str(ca),
}
def rclone(*args):
result = subprocess.run(
[os.getenv("RCLONE_BINARY", "rclone"), "--config", os.devnull, *args],
env=env,
capture_output=True,
timeout=120,
)
if result.returncode:
raise RuntimeError("rclone failed: " + result.stderr.decode(errors="replace")[-2000:])
return result.stdout
with tempfile.TemporaryDirectory(dir=WORK) as folder:
source, downloaded = Path(folder) / "source", Path(folder) / "downloaded"
(source / "nested").mkdir(parents=True)
files = {
"model weights.bin": b"multipart-rclone\0" * (768 * 1024),
"nested/λ.txt": "opaque UTF-8 key + literal plus".encode(),
"empty": b"",
}
for key, body in files.items():
(source / key).write_bytes(body)
remote = f"ip:{name}"
rclone(
"sync",
str(source),
remote + "/data",
"--checksum",
"--transfers",
"2",
"--s3-upload-cutoff",
"5Mi",
"--s3-chunk-size",
"5Mi",
)
listed = json.loads(rclone("lsjson", remote + "/data", "--recursive"))
assert {row["Path"] for row in listed if not row["IsDir"]} == set(files)
rclone("check", str(source), remote + "/data", "--one-way")
rclone("copy", remote + "/data", str(downloaded), "--checksum")
for key, body in files.items():
assert hashlib.sha256((downloaded / key).read_bytes()).digest() == hashlib.sha256(body).digest()
rclone("copyto", remote + "/data/model weights.bin", remote + "/copied.bin")
rclone("moveto", remote + "/copied.bin", remote + "/moved.bin")
assert rclone("cat", remote + "/moved.bin", "--head", "11") == files["model weights.bin"][:11]
rclone("touch", remote + "/moved.bin", "--no-create", "--timestamp", "2026-01-01T12:00:00Z")
info = json.loads(rclone("lsjson", remote + "/moved.bin", "--stat"))
assert info["ModTime"].startswith("2026-01-01T12:00:00")
(source / "empty").unlink()
rclone("sync", str(source), remote + "/data", "--checksum")
listed = json.loads(rclone("lsjson", remote + "/data", "--recursive"))
assert "empty" not in {row["Path"] for row in listed}
rclone("purge", remote)
assert native.get("/v1/buckets/" + bucket["id"]).status_code == 404
print(
"PASS rclone: org-scoped multipart, UTF-8 keys, empty files, checksum sync/check, download, server copy/move, modtime, ranges, purge"
)
finally:
if bucket:
response = native.get("/v1/buckets/" + bucket["id"] + "/objects")
if response.status_code == 200:
for obj in response.json()["objects"]:
native.delete("/v1/buckets/" + bucket["id"] + "/object", params={"key": obj["object_key"]})
native.delete("/v1/buckets/" + bucket["id"])
if credential:
native.delete("/v1/buckets/s3-credentials/" + credential["id"])
if __name__ == "__main__":
main()

Xet Storage Details

Size:
6.1 kB
·
Xet hash:
799f985e4b15b975dd02f3e30bfd39f8944de4047162e56351963a9e47e69b46

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.