| """Isolated acceptance drill: load, restart, backup/restore, cleanup and accounting. | |
| Requires a sandbox started with tools/run_buckets_sandbox.py. Never accepts a | |
| production database or gateway. Backup files and receipts stay mode 0600 under | |
| work/. Cloud-provider, production deployment and invoice checks are separate. | |
| """ | |
| import argparse | |
| import asyncio | |
| from concurrent.futures import ThreadPoolExecutor | |
| from datetime import timedelta | |
| from decimal import Decimal, localcontext | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import statistics | |
| import re | |
| import resource | |
| import threading | |
| import subprocess | |
| import sys | |
| import time | |
| from uuid import UUID, uuid4 | |
| import boto3 | |
| from botocore.config import Config | |
| from botocore.exceptions import ClientError | |
| import httpx | |
| import psycopg | |
| from psycopg import sql | |
| from psycopg.rows import dict_row | |
| from urllib.parse import urlsplit | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT / "src")) | |
| def private_write(path, data): | |
| descriptor = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) | |
| with os.fdopen(descriptor, "wb") as output: | |
| output.write(data) | |
| def main(directory, concurrency, iterations): | |
| directory = directory.resolve() | |
| config = json.loads((directory / "config.json").read_text()) | |
| parsed = urlsplit(config["BUCKETS_DATABASE_URL"]) | |
| if ( | |
| parsed.hostname not in ("127.0.0.1", "localhost") | |
| or parsed.port != 55432 | |
| or not parsed.path.startswith("/buckets_acceptance") | |
| ): | |
| raise RuntimeError("Acceptance drills require their own local buckets_acceptance database") | |
| if config.get("BUCKETS_AZURE_ACCOUNT_URL"): | |
| raise RuntimeError("This restore drill is local-only; do not delete cloud generations") | |
| fields = dict( | |
| part.split("=", 1) for part in config.get("BUCKETS_AZURITE_CONNECTION_STRING", "").split(";") if "=" in part | |
| ) | |
| backing = urlsplit(fields.get("BlobEndpoint", "")) | |
| if ( | |
| config.get("BUCKETS_SANDBOX") != "1" | |
| or config.get("BUCKETS_KEY_VAULT_KEY_ID") | |
| or backing.hostname not in ("127.0.0.1", "localhost") | |
| or backing.port != 51000 | |
| ): | |
| raise RuntimeError("Acceptance restore only supports the explicitly local emulator and test wrapper") | |
| endpoint = urlsplit(config["BUCKETS_PUBLIC_ENDPOINT"]) | |
| if endpoint.hostname not in ("localhost", "127.0.0.1") or endpoint.port != 28221: | |
| raise RuntimeError("Expected the isolated gateway on localhost:28221") | |
| config["BUCKETS_RUN_WORKER"] = "0" | |
| tokens = json.loads((directory / "credentials.json").read_text()) | |
| ca = str(directory / "localhost.pem") | |
| dsn = config["BUCKETS_DATABASE_URL"] | |
| report = {"backend": "isolated-emulator", "concurrency": concurrency, "iterations_per_client": iterations} | |
| servers = [] | |
| def query(statement, params=(), target=dsn, many=False): | |
| with psycopg.connect(target, row_factory=dict_row) as conn: | |
| cursor = conn.execute(statement, params) | |
| return cursor.fetchall() if many else cursor.fetchone() if cursor.description else None | |
| def native(path, method="GET", body=None, port=28221): | |
| response = httpx.request( | |
| method, | |
| f"https://localhost:{port}" + path, | |
| headers={"Authorization": "Bearer " + tokens["user1_personal"]}, | |
| json=body, | |
| verify=ca, | |
| trust_env=False, | |
| timeout=60, | |
| ) | |
| response.raise_for_status() | |
| return response.json() if response.content else None | |
| def server(port, settings): | |
| env = {**os.environ, **settings, "PYTHONPATH": str(ROOT / "src")} | |
| log = (directory / f"acceptance-{port}.log").open("a") | |
| process = subprocess.Popen( | |
| [ | |
| sys.executable, | |
| "-m", | |
| "uvicorn", | |
| "buckets.sandbox:create_app", | |
| "--factory", | |
| "--host", | |
| "127.0.0.1", | |
| "--port", | |
| str(port), | |
| "--ssl-keyfile", | |
| str(directory / "localhost-key.pem"), | |
| "--ssl-certfile", | |
| ca, | |
| "--no-access-log", | |
| ], | |
| env=env, | |
| stdout=log, | |
| stderr=log, | |
| ) | |
| log.close() | |
| servers.append(process) | |
| for _ in range(100): | |
| if process.poll() is not None: | |
| # This app is guarded to a disposable database/emulator; its | |
| # startup log contains no production configuration or headers. | |
| startup = (directory / f"acceptance-{port}.log").read_text() | |
| raise RuntimeError("Isolated replica startup failed:\n" + startup[-4000:]) | |
| try: | |
| r = httpx.get(f"https://localhost:{port}/v1/buckets/status", verify=ca, trust_env=False, timeout=1) | |
| if r.status_code == 200: | |
| return process | |
| except httpx.HTTPError: | |
| pass | |
| time.sleep(0.1) | |
| raise RuntimeError("Isolated replica readiness timed out") | |
| def sdk(credential, port=28221): | |
| path = urlsplit(credential["endpoint"]).path | |
| return boto3.client( | |
| "s3", | |
| endpoint_url=f"https://localhost:{port}" + path, | |
| region_name="us-east-1", | |
| aws_access_key_id=credential["access_key_id"], | |
| aws_secret_access_key=credential["secret_access_key"], | |
| verify=ca, | |
| config=Config( | |
| signature_version="s3v4", | |
| s3={"addressing_style": "path"}, | |
| retries={"max_attempts": 0}, | |
| max_pool_connections=concurrency + 2, | |
| read_timeout=90, | |
| ), | |
| ) | |
| def checkpoint(step): | |
| print("PASS " + step, flush=True) | |
| # Keep listeners below Linux's ephemeral client-port range: thousands of | |
| # test connections can otherwise temporarily occupy a fixed high port. | |
| process = server(28222, config) | |
| bucket = native("/v1/buckets", "POST", {"name": "acceptance-" + uuid4().hex[:12], "quota_bytes": 1024**3}) | |
| credential = native( | |
| "/v1/buckets/s3-credentials", | |
| "POST", | |
| {"permissions": ["storage.read", "storage.write", "storage.admin"], "rate_limit_rpm": 600}, | |
| ) | |
| first, second = sdk(credential), sdk(credential, 28222) | |
| name, identifier = bucket["name"], bucket["id"] | |
| started = time.monotonic() | |
| try: | |
| latencies = [] | |
| def client_job(index): | |
| client = sdk(credential, 28221 if index % 2 else 28222) | |
| try: | |
| samples = [] | |
| for attempt in range(iterations): | |
| key = f"load/{index}/{attempt}" | |
| payload = hashlib.sha256(key.encode()).digest() * 32 | |
| stamp = time.monotonic() | |
| client.put_object(Bucket=name, Key=key, Body=payload) | |
| assert client.get_object(Bucket=name, Key=key)["Body"].read() == payload | |
| client.delete_object(Bucket=name, Key=key) | |
| samples.append(time.monotonic() - stamp) | |
| return samples | |
| finally: | |
| client.close() | |
| stop = threading.Event() | |
| peaks = {"database_connections": 0, "server_rss_mib": 0.0} | |
| primary_ids = re.findall(r"Started server process \[(\d+)\]", (directory / "server-1.log").read_text()) | |
| primary_pid = int(primary_ids[-1]) | |
| def monitor(): | |
| while not stop.is_set(): | |
| row = query("select numbackends from pg_stat_database where datname=%s", (parsed.path[1:],)) | |
| peaks["database_connections"] = max(peaks["database_connections"], row["numbackends"] - 1) | |
| for pid in [primary_pid, process.pid]: | |
| try: | |
| text = Path(f"/proc/{pid}/status").read_text() | |
| rss = int(re.search(r"VmRSS:\s+(\d+)", text).group(1)) / 1024 | |
| peaks["server_rss_mib"] = max(peaks["server_rss_mib"], rss) | |
| except (FileNotFoundError, AttributeError): | |
| pass | |
| stop.wait(0.1) | |
| observer = threading.Thread(target=monitor, daemon=True) | |
| observer.start() | |
| try: | |
| with ThreadPoolExecutor(concurrency) as pool: | |
| for samples in pool.map(client_job, range(concurrency)): | |
| latencies.extend(samples) | |
| finally: | |
| stop.set() | |
| observer.join(timeout=5) | |
| for _ in range(30): | |
| idle = query( | |
| "select count(*) n from pg_stat_activity where datname=%s and pid<>pg_backend_pid()", (parsed.path[1:],) | |
| )["n"] | |
| if idle == 0: | |
| break | |
| time.sleep(0.1) | |
| assert idle == 0, "Gateway database connections did not drain after load" | |
| report["load"] = { | |
| "workflows": len(latencies), | |
| "requests": len(latencies) * 3, | |
| "errors": 0, | |
| "peak_database_connections": peaks["database_connections"], | |
| "peak_server_rss_mib": round(peaks["server_rss_mib"], 1), | |
| "connections_after_load": idle, | |
| "elapsed_seconds": round(time.monotonic() - started, 3), | |
| "workflow_p50_seconds": round(statistics.median(latencies), 3), | |
| "workflow_p95_seconds": round(sorted(latencies)[max(0, int(len(latencies) * 0.95) - 1)], 3), | |
| } | |
| row = query("select bytes_used,bytes_reserved from storage_buckets where id=%s", (identifier,)) | |
| assert row == {"bytes_used": 0, "bytes_reserved": 0} | |
| checkpoint("concurrent mixed transfers across two replicas; exact quota counters") | |
| upload = second.create_multipart_upload(Bucket=name, Key="restart.bin", Tagging="recovery=verified")["UploadId"] | |
| body = b"A" * (8 * 1024**2) | |
| a = second.upload_part(Bucket=name, Key="restart.bin", UploadId=upload, PartNumber=1, Body=body) | |
| process.kill() | |
| process.wait(timeout=15) | |
| b = first.upload_part(Bucket=name, Key="restart.bin", UploadId=upload, PartNumber=2, Body=b"finish") | |
| first.complete_multipart_upload( | |
| Bucket=name, | |
| Key="restart.bin", | |
| UploadId=upload, | |
| MultipartUpload={"Parts": [{"PartNumber": 1, "ETag": a["ETag"]}, {"PartNumber": 2, "ETag": b["ETag"]}]}, | |
| ) | |
| process = server(28222, config) | |
| digest = hashlib.sha256(body + b"finish").hexdigest() | |
| assert hashlib.sha256(second.get_object(Bucket=name, Key="restart.bin")["Body"].read()).hexdigest() == digest | |
| checkpoint("replica stop/restart during multipart preserves data and result") | |
| first.put_bucket_tagging(Bucket=name, Tagging={"TagSet": [{"Key": "restore", "Value": "required"}]}) | |
| snapshot = directory / "recovery.dump" | |
| dumped = subprocess.run( | |
| [ | |
| "docker", | |
| "--host=unix:///var/run/docker.sock", | |
| "exec", | |
| "ip-buckets-postgres", | |
| "pg_dump", | |
| "-U", | |
| "postgres", | |
| "-d", | |
| parsed.path[1:], | |
| "-Fc", | |
| ], | |
| capture_output=True, | |
| check=True, | |
| ) | |
| private_write(snapshot, dumped.stdout) | |
| obj = query("select * from storage_objects where bucket_id=%s and object_key=%s", (identifier, "restart.bin")) | |
| async def save_generation(): | |
| os.environ.update(config) | |
| from buckets.sandbox import create_app | |
| runtime = create_app().state.bucket_runtime | |
| try: | |
| chunks = [] | |
| async for chunk in runtime.service.blobs.download(obj["locator"]): | |
| chunks.append(chunk) | |
| manifest = ( | |
| b"".join([v async for v in runtime.service.blobs.download(obj["locator"] + ".parts")]) | |
| if obj.get("s3_options", {}).get("manifest_sha256") | |
| else None | |
| ) | |
| return b"".join(chunks), manifest | |
| finally: | |
| await runtime.close() | |
| saved, saved_manifest = asyncio.run(save_generation()) | |
| # Backup is ciphertext; the restored API must recover the original plaintext hash. | |
| assert len(saved) >= obj["size_bytes"] | |
| backup = directory / "generation.bin" | |
| private_write(backup, saved) | |
| if saved_manifest is not None: | |
| private_write(directory / "generation.parts", saved_manifest) | |
| first.delete_object(Bucket=name, Key="restart.bin") | |
| native("/v1/buckets/s3-credentials/" + credential["id"], "DELETE") | |
| async def destroy_generation(): | |
| from buckets.sandbox import create_app | |
| runtime = create_app().state.bucket_runtime | |
| try: | |
| await runtime.service.blobs.delete(obj["locator"]) | |
| finally: | |
| await runtime.close() | |
| asyncio.run(destroy_generation()) | |
| restored_db = parsed.path[1:] + "_r_" + uuid4().hex[:6] | |
| admin_dsn = "postgresql://postgres:buckets-local-e2e@127.0.0.1:55432/postgres" | |
| with psycopg.connect(admin_dsn, autocommit=True) as admin: | |
| if admin.execute("select 1 from pg_database where datname=%s", (restored_db,)).fetchone(): | |
| raise RuntimeError("Refusing to overwrite an existing restored database") | |
| admin.execute(sql.SQL("create database {}").format(sql.Identifier(restored_db))) | |
| subprocess.run( | |
| [ | |
| "docker", | |
| "--host=unix:///var/run/docker.sock", | |
| "exec", | |
| "-i", | |
| "ip-buckets-postgres", | |
| "pg_restore", | |
| "-U", | |
| "postgres", | |
| "-d", | |
| restored_db, | |
| "--no-owner", | |
| ], | |
| input=dumped.stdout, | |
| capture_output=True, | |
| check=True, | |
| ) | |
| restored = {**config, "BUCKETS_DATABASE_URL": dsn.rsplit("/", 1)[0] + "/" + restored_db} | |
| async def restore_generation(): | |
| os.environ.update(restored) | |
| from buckets.sandbox import create_app | |
| runtime = create_app().state.bucket_runtime | |
| try: | |
| from buckets.keys import Owner | |
| locator = runtime.service.blobs.locator( | |
| Owner("user", UUID(bucket["owner_id"])), identifier, uuid4(), "restart.bin" | |
| ) | |
| result = await runtime.service.blobs.container.get_blob_client(locator).upload_blob( | |
| saved, overwrite=False | |
| ) | |
| if saved_manifest is not None: | |
| await runtime.service.blobs.container.get_blob_client(locator + ".parts").upload_blob( | |
| saved_manifest, overwrite=False | |
| ) | |
| return locator, result | |
| finally: | |
| await runtime.close() | |
| restored_locator, metadata = asyncio.run(restore_generation()) | |
| query( | |
| "update storage_objects set locator=%s,azure_etag=%s where bucket_id=%s and object_key=%s", | |
| (restored_locator, metadata["etag"], identifier, "restart.bin"), | |
| target=restored["BUCKETS_DATABASE_URL"], | |
| ) | |
| server(28223, restored) | |
| recovered = sdk(credential, 28223) | |
| assert hashlib.sha256(recovered.get_object(Bucket=name, Key="restart.bin")["Body"].read()).hexdigest() == digest | |
| assert recovered.get_object_tagging(Bucket=name, Key="restart.bin")["TagSet"] == [ | |
| {"Key": "recovery", "Value": "verified"} | |
| ] | |
| assert recovered.get_bucket_tagging(Bucket=name)["TagSet"] == [{"Key": "restore", "Value": "required"}] | |
| checkpoint("SQL + object generation + wrapping-key restore recovers exact hash, tags and encrypted credential") | |
| # A snapshot can predate revocation. Quarantine restored credentials before exposure. | |
| query( | |
| "update storage_s3_credentials set revoked_at=coalesce(revoked_at,now())", | |
| target=restored["BUCKETS_DATABASE_URL"], | |
| ) | |
| try: | |
| recovered.get_object(Bucket=name, Key="restart.bin") | |
| raise AssertionError("Restored revoked credential was usable after quarantine") | |
| except ClientError as exc: | |
| assert exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403 | |
| recovered.close() | |
| checkpoint("restored credential quarantine prevents post-backup revocations from being undone") | |
| fresh = native( | |
| "/v1/buckets/s3-credentials", | |
| "POST", | |
| {"permissions": ["storage.read", "storage.write"], "rate_limit_rpm": 600}, | |
| ) | |
| active = sdk(fresh) | |
| capacity_bytes = 1024**2 | |
| active.put_object(Bucket=name, Key="capacity.bin", Body=b"C" * capacity_bytes) | |
| orphan = active.create_multipart_upload(Bucket=name, Key="orphan")["UploadId"] | |
| active.upload_part(Bucket=name, Key="orphan", UploadId=orphan, PartNumber=1, Body=b"pending") | |
| active.close() | |
| # Controlled timestamps are confined to this disposable database. | |
| async def accounting(): | |
| os.environ.update(config) | |
| from buckets.sandbox import create_app | |
| from buckets import workers | |
| from buckets.repository import now | |
| runtime = create_app().state.bucket_runtime | |
| repo = runtime.service.repo | |
| try: | |
| await runtime.service.blobs.delete(restored_locator) | |
| async with repo.transaction() as conn: | |
| await conn.execute( | |
| "update storage_multipart_uploads set expires_at=now()-interval '1 minute' where id=%s", | |
| (orphan,), | |
| ) | |
| await conn.execute( | |
| "update storage_garbage set delete_after=now()-interval '1 minute' where bucket_id=%s", | |
| (identifier,), | |
| ) | |
| await workers.garbage(runtime) | |
| orphan_state = await repo.one( | |
| "select state,cleaned_at from storage_multipart_uploads where id=%s", (orphan,) | |
| ) | |
| assert orphan_state["state"] == "aborted" and orphan_state["cleaned_at"] is not None | |
| await workers.reconcile(runtime) | |
| day = now().date() - timedelta(days=1) | |
| clock = now().replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(days=1) | |
| # The recovery drill moves the sampler clock into the past. | |
| # Publish the matching historical customer rate in this isolated | |
| # database, rather than retroactively pricing it with a future | |
| # price check created when the sandbox started. | |
| async with repo.transaction() as conn: | |
| published = await ( | |
| await conn.execute( | |
| "select rate_version from storage_rate_checks order by checked_at desc limit 1" | |
| ) | |
| ).fetchone() | |
| assert published is not None | |
| await conn.execute( | |
| "insert into storage_rate_checks(rate_version,checked_at) values(%s,%s)", | |
| (published["rate_version"], clock - timedelta(hours=1)), | |
| ) | |
| original_now = workers.now | |
| try: | |
| workers.now = lambda: clock | |
| await workers.sample_capacity(runtime) | |
| sample = await repo.one( | |
| "select size_bytes from storage_capacity_samples where bucket_id=%s order by sampled_at desc limit 1", | |
| (identifier,), | |
| ) | |
| assert ( | |
| sample["size_bytes"] == capacity_bytes + 28 | |
| ) # One AES-GCM nonce/tag, charged as real physical storage. | |
| workers.now = lambda: clock + timedelta(days=1) | |
| await workers.sample_capacity(runtime) | |
| finally: | |
| workers.now = original_now | |
| async with repo.transaction() as conn: | |
| await conn.execute("update storage_usage_events set created_at=%s::date", (day,)) | |
| await conn.execute("update storage_billing_reservations set created_at=%s::date", (day,)) | |
| measured = await repo.all( | |
| "select meter,sum(quantity) quantity from storage_usage_events where bucket_id=%s group by meter", | |
| (identifier,), | |
| ) | |
| meters = {row["meter"]: row["quantity"] for row in measured} | |
| workflows = concurrency * iterations | |
| assert meters["write_operations"] == workflows * 2 + 8 # Includes durable multipart manifest. | |
| assert ( | |
| meters["read_operations"] == workflows * 2 + 6 | |
| ) # Manifest plus ciphertext split at the provider chunk boundary. | |
| assert meters["egress_bytes"] == workflows * 1024 + len(body) + 6 | |
| await workers.settle_day(runtime, day) | |
| balance = await repo.one( | |
| "select balance_credits from user_wallets where user_id=%s", (bucket["owner_id"],) | |
| ) | |
| await workers.settle_day(runtime, day) | |
| assert ( | |
| await repo.one("select balance_credits from user_wallets where user_id=%s", (bucket["owner_id"],)) | |
| == balance | |
| ) | |
| holds = await repo.one("select count(*) n from storage_billing_reservations where state='reserved'") | |
| assert holds["n"] == 0 | |
| exact = await repo.one("select coalesce(sum(amount_credits),0) n from storage_usage_daily") | |
| carry = await repo.one("select coalesce(sum(amount),0) n from storage_billing_residuals") | |
| debited = await repo.one( | |
| "select coalesce(-sum(delta_credits),0) n from credit_ledger where usage_kind='storage' and entry_type='usage'" | |
| ) | |
| with localcontext() as arithmetic: | |
| arithmetic.prec = 80 | |
| assert exact["n"] == debited["n"] + carry["n"] | |
| assert balance["balance_credits"] == Decimal(100) - debited["n"] | |
| return { | |
| "exact_credits": str(exact["n"]), | |
| "debited_credits": str(debited["n"]), | |
| "fractional_carry": str(carry["n"]), | |
| "outstanding_holds": 0, | |
| "inventory_bytes": capacity_bytes + 28, | |
| "logical_bytes": capacity_bytes, | |
| "expired_upload_cleaned": True, | |
| "read_operations": int(meters["read_operations"]), | |
| "write_operations": int(meters["write_operations"]), | |
| "egress_bytes": int(meters["egress_bytes"]), | |
| } | |
| finally: | |
| await runtime.close() | |
| report["accounting"] = asyncio.run(accounting()) | |
| report["restore"] = { | |
| "sha256_match": True, | |
| "encrypted_credential_recovered": True, | |
| "bucket_and_object_tags_recovered": True, | |
| "restored_credentials_quarantined": True, | |
| "backup_bytes": snapshot.stat().st_size, | |
| } | |
| report["restart"] = {"multipart_bytes": obj["size_bytes"], "sha256_match": True} | |
| checkpoint("closed-day settlement balances exact charge, carry and refunds; retry is idempotent") | |
| report["elapsed_seconds"] = round(time.monotonic() - started, 3) | |
| report["client_peak_rss_mib"] = round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 1) | |
| report_path = directory / "acceptance-results.json" | |
| private_write(report_path, json.dumps(report, indent=2).encode()) | |
| print(json.dumps(report, indent=2)) | |
| finally: | |
| first.close() | |
| second.close() | |
| for proc in servers: | |
| if proc.poll() is None: | |
| proc.terminate() | |
| try: | |
| proc.wait(timeout=15) | |
| except subprocess.TimeoutExpired: | |
| proc.kill() | |
| proc.wait() | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--directory", type=Path, default=ROOT / "work/buckets-acceptance") | |
| parser.add_argument("--concurrency", type=int, default=8) | |
| parser.add_argument("--iterations", type=int, default=4) | |
| args = parser.parse_args() | |
| if not 1 <= args.concurrency <= 32 or not 1 <= args.iterations <= 10: | |
| parser.error("Use 1..32 clients and 1..10 workflows per client") | |
| main(args.directory, args.concurrency, args.iterations) | |
Xet Storage Details
- Size:
- 25 kB
- Xet hash:
- 952cb299c31105e831c4e9eb4ebe987dc9f6f6b603f426d96c6c9c9206991d87
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.