sharktide/tools / buckets_acceptance.py
sharktide's picture
download
raw
25 kB
"""Isolated acceptance drill: load, restart, backup/restore, cleanup and accounting.
Requires a sandbox started with tools/run_buckets_sandbox.py. Never accepts a
production database or gateway. Backup files and receipts stay mode 0600 under
work/. Cloud-provider, production deployment and invoice checks are separate.
"""
import argparse
import asyncio
from concurrent.futures import ThreadPoolExecutor
from datetime import timedelta
from decimal import Decimal, localcontext
import hashlib
import json
import os
from pathlib import Path
import statistics
import re
import resource
import threading
import subprocess
import sys
import time
from uuid import UUID, uuid4
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
import httpx
import psycopg
from psycopg import sql
from psycopg.rows import dict_row
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
def private_write(path, data):
descriptor = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
with os.fdopen(descriptor, "wb") as output:
output.write(data)
def main(directory, concurrency, iterations):
directory = directory.resolve()
config = json.loads((directory / "config.json").read_text())
parsed = urlsplit(config["BUCKETS_DATABASE_URL"])
if (
parsed.hostname not in ("127.0.0.1", "localhost")
or parsed.port != 55432
or not parsed.path.startswith("/buckets_acceptance")
):
raise RuntimeError("Acceptance drills require their own local buckets_acceptance database")
if config.get("BUCKETS_AZURE_ACCOUNT_URL"):
raise RuntimeError("This restore drill is local-only; do not delete cloud generations")
fields = dict(
part.split("=", 1) for part in config.get("BUCKETS_AZURITE_CONNECTION_STRING", "").split(";") if "=" in part
)
backing = urlsplit(fields.get("BlobEndpoint", ""))
if (
config.get("BUCKETS_SANDBOX") != "1"
or config.get("BUCKETS_KEY_VAULT_KEY_ID")
or backing.hostname not in ("127.0.0.1", "localhost")
or backing.port != 51000
):
raise RuntimeError("Acceptance restore only supports the explicitly local emulator and test wrapper")
endpoint = urlsplit(config["BUCKETS_PUBLIC_ENDPOINT"])
if endpoint.hostname not in ("localhost", "127.0.0.1") or endpoint.port != 28221:
raise RuntimeError("Expected the isolated gateway on localhost:28221")
config["BUCKETS_RUN_WORKER"] = "0"
tokens = json.loads((directory / "credentials.json").read_text())
ca = str(directory / "localhost.pem")
dsn = config["BUCKETS_DATABASE_URL"]
report = {"backend": "isolated-emulator", "concurrency": concurrency, "iterations_per_client": iterations}
servers = []
def query(statement, params=(), target=dsn, many=False):
with psycopg.connect(target, row_factory=dict_row) as conn:
cursor = conn.execute(statement, params)
return cursor.fetchall() if many else cursor.fetchone() if cursor.description else None
def native(path, method="GET", body=None, port=28221):
response = httpx.request(
method,
f"https://localhost:{port}" + path,
headers={"Authorization": "Bearer " + tokens["user1_personal"]},
json=body,
verify=ca,
trust_env=False,
timeout=60,
)
response.raise_for_status()
return response.json() if response.content else None
def server(port, settings):
env = {**os.environ, **settings, "PYTHONPATH": str(ROOT / "src")}
log = (directory / f"acceptance-{port}.log").open("a")
process = subprocess.Popen(
[
sys.executable,
"-m",
"uvicorn",
"buckets.sandbox:create_app",
"--factory",
"--host",
"127.0.0.1",
"--port",
str(port),
"--ssl-keyfile",
str(directory / "localhost-key.pem"),
"--ssl-certfile",
ca,
"--no-access-log",
],
env=env,
stdout=log,
stderr=log,
)
log.close()
servers.append(process)
for _ in range(100):
if process.poll() is not None:
# This app is guarded to a disposable database/emulator; its
# startup log contains no production configuration or headers.
startup = (directory / f"acceptance-{port}.log").read_text()
raise RuntimeError("Isolated replica startup failed:\n" + startup[-4000:])
try:
r = httpx.get(f"https://localhost:{port}/v1/buckets/status", verify=ca, trust_env=False, timeout=1)
if r.status_code == 200:
return process
except httpx.HTTPError:
pass
time.sleep(0.1)
raise RuntimeError("Isolated replica readiness timed out")
def sdk(credential, port=28221):
path = urlsplit(credential["endpoint"]).path
return boto3.client(
"s3",
endpoint_url=f"https://localhost:{port}" + path,
region_name="us-east-1",
aws_access_key_id=credential["access_key_id"],
aws_secret_access_key=credential["secret_access_key"],
verify=ca,
config=Config(
signature_version="s3v4",
s3={"addressing_style": "path"},
retries={"max_attempts": 0},
max_pool_connections=concurrency + 2,
read_timeout=90,
),
)
def checkpoint(step):
print("PASS " + step, flush=True)
# Keep listeners below Linux's ephemeral client-port range: thousands of
# test connections can otherwise temporarily occupy a fixed high port.
process = server(28222, config)
bucket = native("/v1/buckets", "POST", {"name": "acceptance-" + uuid4().hex[:12], "quota_bytes": 1024**3})
credential = native(
"/v1/buckets/s3-credentials",
"POST",
{"permissions": ["storage.read", "storage.write", "storage.admin"], "rate_limit_rpm": 600},
)
first, second = sdk(credential), sdk(credential, 28222)
name, identifier = bucket["name"], bucket["id"]
started = time.monotonic()
try:
latencies = []
def client_job(index):
client = sdk(credential, 28221 if index % 2 else 28222)
try:
samples = []
for attempt in range(iterations):
key = f"load/{index}/{attempt}"
payload = hashlib.sha256(key.encode()).digest() * 32
stamp = time.monotonic()
client.put_object(Bucket=name, Key=key, Body=payload)
assert client.get_object(Bucket=name, Key=key)["Body"].read() == payload
client.delete_object(Bucket=name, Key=key)
samples.append(time.monotonic() - stamp)
return samples
finally:
client.close()
stop = threading.Event()
peaks = {"database_connections": 0, "server_rss_mib": 0.0}
primary_ids = re.findall(r"Started server process \[(\d+)\]", (directory / "server-1.log").read_text())
primary_pid = int(primary_ids[-1])
def monitor():
while not stop.is_set():
row = query("select numbackends from pg_stat_database where datname=%s", (parsed.path[1:],))
peaks["database_connections"] = max(peaks["database_connections"], row["numbackends"] - 1)
for pid in [primary_pid, process.pid]:
try:
text = Path(f"/proc/{pid}/status").read_text()
rss = int(re.search(r"VmRSS:\s+(\d+)", text).group(1)) / 1024
peaks["server_rss_mib"] = max(peaks["server_rss_mib"], rss)
except (FileNotFoundError, AttributeError):
pass
stop.wait(0.1)
observer = threading.Thread(target=monitor, daemon=True)
observer.start()
try:
with ThreadPoolExecutor(concurrency) as pool:
for samples in pool.map(client_job, range(concurrency)):
latencies.extend(samples)
finally:
stop.set()
observer.join(timeout=5)
for _ in range(30):
idle = query(
"select count(*) n from pg_stat_activity where datname=%s and pid<>pg_backend_pid()", (parsed.path[1:],)
)["n"]
if idle == 0:
break
time.sleep(0.1)
assert idle == 0, "Gateway database connections did not drain after load"
report["load"] = {
"workflows": len(latencies),
"requests": len(latencies) * 3,
"errors": 0,
"peak_database_connections": peaks["database_connections"],
"peak_server_rss_mib": round(peaks["server_rss_mib"], 1),
"connections_after_load": idle,
"elapsed_seconds": round(time.monotonic() - started, 3),
"workflow_p50_seconds": round(statistics.median(latencies), 3),
"workflow_p95_seconds": round(sorted(latencies)[max(0, int(len(latencies) * 0.95) - 1)], 3),
}
row = query("select bytes_used,bytes_reserved from storage_buckets where id=%s", (identifier,))
assert row == {"bytes_used": 0, "bytes_reserved": 0}
checkpoint("concurrent mixed transfers across two replicas; exact quota counters")
upload = second.create_multipart_upload(Bucket=name, Key="restart.bin", Tagging="recovery=verified")["UploadId"]
body = b"A" * (8 * 1024**2)
a = second.upload_part(Bucket=name, Key="restart.bin", UploadId=upload, PartNumber=1, Body=body)
process.kill()
process.wait(timeout=15)
b = first.upload_part(Bucket=name, Key="restart.bin", UploadId=upload, PartNumber=2, Body=b"finish")
first.complete_multipart_upload(
Bucket=name,
Key="restart.bin",
UploadId=upload,
MultipartUpload={"Parts": [{"PartNumber": 1, "ETag": a["ETag"]}, {"PartNumber": 2, "ETag": b["ETag"]}]},
)
process = server(28222, config)
digest = hashlib.sha256(body + b"finish").hexdigest()
assert hashlib.sha256(second.get_object(Bucket=name, Key="restart.bin")["Body"].read()).hexdigest() == digest
checkpoint("replica stop/restart during multipart preserves data and result")
first.put_bucket_tagging(Bucket=name, Tagging={"TagSet": [{"Key": "restore", "Value": "required"}]})
snapshot = directory / "recovery.dump"
dumped = subprocess.run(
[
"docker",
"--host=unix:///var/run/docker.sock",
"exec",
"ip-buckets-postgres",
"pg_dump",
"-U",
"postgres",
"-d",
parsed.path[1:],
"-Fc",
],
capture_output=True,
check=True,
)
private_write(snapshot, dumped.stdout)
obj = query("select * from storage_objects where bucket_id=%s and object_key=%s", (identifier, "restart.bin"))
async def save_generation():
os.environ.update(config)
from buckets.sandbox import create_app
runtime = create_app().state.bucket_runtime
try:
chunks = []
async for chunk in runtime.service.blobs.download(obj["locator"]):
chunks.append(chunk)
manifest = (
b"".join([v async for v in runtime.service.blobs.download(obj["locator"] + ".parts")])
if obj.get("s3_options", {}).get("manifest_sha256")
else None
)
return b"".join(chunks), manifest
finally:
await runtime.close()
saved, saved_manifest = asyncio.run(save_generation())
# Backup is ciphertext; the restored API must recover the original plaintext hash.
assert len(saved) >= obj["size_bytes"]
backup = directory / "generation.bin"
private_write(backup, saved)
if saved_manifest is not None:
private_write(directory / "generation.parts", saved_manifest)
first.delete_object(Bucket=name, Key="restart.bin")
native("/v1/buckets/s3-credentials/" + credential["id"], "DELETE")
async def destroy_generation():
from buckets.sandbox import create_app
runtime = create_app().state.bucket_runtime
try:
await runtime.service.blobs.delete(obj["locator"])
finally:
await runtime.close()
asyncio.run(destroy_generation())
restored_db = parsed.path[1:] + "_r_" + uuid4().hex[:6]
admin_dsn = "postgresql://postgres:buckets-local-e2e@127.0.0.1:55432/postgres"
with psycopg.connect(admin_dsn, autocommit=True) as admin:
if admin.execute("select 1 from pg_database where datname=%s", (restored_db,)).fetchone():
raise RuntimeError("Refusing to overwrite an existing restored database")
admin.execute(sql.SQL("create database {}").format(sql.Identifier(restored_db)))
subprocess.run(
[
"docker",
"--host=unix:///var/run/docker.sock",
"exec",
"-i",
"ip-buckets-postgres",
"pg_restore",
"-U",
"postgres",
"-d",
restored_db,
"--no-owner",
],
input=dumped.stdout,
capture_output=True,
check=True,
)
restored = {**config, "BUCKETS_DATABASE_URL": dsn.rsplit("/", 1)[0] + "/" + restored_db}
async def restore_generation():
os.environ.update(restored)
from buckets.sandbox import create_app
runtime = create_app().state.bucket_runtime
try:
from buckets.keys import Owner
locator = runtime.service.blobs.locator(
Owner("user", UUID(bucket["owner_id"])), identifier, uuid4(), "restart.bin"
)
result = await runtime.service.blobs.container.get_blob_client(locator).upload_blob(
saved, overwrite=False
)
if saved_manifest is not None:
await runtime.service.blobs.container.get_blob_client(locator + ".parts").upload_blob(
saved_manifest, overwrite=False
)
return locator, result
finally:
await runtime.close()
restored_locator, metadata = asyncio.run(restore_generation())
query(
"update storage_objects set locator=%s,azure_etag=%s where bucket_id=%s and object_key=%s",
(restored_locator, metadata["etag"], identifier, "restart.bin"),
target=restored["BUCKETS_DATABASE_URL"],
)
server(28223, restored)
recovered = sdk(credential, 28223)
assert hashlib.sha256(recovered.get_object(Bucket=name, Key="restart.bin")["Body"].read()).hexdigest() == digest
assert recovered.get_object_tagging(Bucket=name, Key="restart.bin")["TagSet"] == [
{"Key": "recovery", "Value": "verified"}
]
assert recovered.get_bucket_tagging(Bucket=name)["TagSet"] == [{"Key": "restore", "Value": "required"}]
checkpoint("SQL + object generation + wrapping-key restore recovers exact hash, tags and encrypted credential")
# A snapshot can predate revocation. Quarantine restored credentials before exposure.
query(
"update storage_s3_credentials set revoked_at=coalesce(revoked_at,now())",
target=restored["BUCKETS_DATABASE_URL"],
)
try:
recovered.get_object(Bucket=name, Key="restart.bin")
raise AssertionError("Restored revoked credential was usable after quarantine")
except ClientError as exc:
assert exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403
recovered.close()
checkpoint("restored credential quarantine prevents post-backup revocations from being undone")
fresh = native(
"/v1/buckets/s3-credentials",
"POST",
{"permissions": ["storage.read", "storage.write"], "rate_limit_rpm": 600},
)
active = sdk(fresh)
capacity_bytes = 1024**2
active.put_object(Bucket=name, Key="capacity.bin", Body=b"C" * capacity_bytes)
orphan = active.create_multipart_upload(Bucket=name, Key="orphan")["UploadId"]
active.upload_part(Bucket=name, Key="orphan", UploadId=orphan, PartNumber=1, Body=b"pending")
active.close()
# Controlled timestamps are confined to this disposable database.
async def accounting():
os.environ.update(config)
from buckets.sandbox import create_app
from buckets import workers
from buckets.repository import now
runtime = create_app().state.bucket_runtime
repo = runtime.service.repo
try:
await runtime.service.blobs.delete(restored_locator)
async with repo.transaction() as conn:
await conn.execute(
"update storage_multipart_uploads set expires_at=now()-interval '1 minute' where id=%s",
(orphan,),
)
await conn.execute(
"update storage_garbage set delete_after=now()-interval '1 minute' where bucket_id=%s",
(identifier,),
)
await workers.garbage(runtime)
orphan_state = await repo.one(
"select state,cleaned_at from storage_multipart_uploads where id=%s", (orphan,)
)
assert orphan_state["state"] == "aborted" and orphan_state["cleaned_at"] is not None
await workers.reconcile(runtime)
day = now().date() - timedelta(days=1)
clock = now().replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(days=1)
# The recovery drill moves the sampler clock into the past.
# Publish the matching historical customer rate in this isolated
# database, rather than retroactively pricing it with a future
# price check created when the sandbox started.
async with repo.transaction() as conn:
published = await (
await conn.execute(
"select rate_version from storage_rate_checks order by checked_at desc limit 1"
)
).fetchone()
assert published is not None
await conn.execute(
"insert into storage_rate_checks(rate_version,checked_at) values(%s,%s)",
(published["rate_version"], clock - timedelta(hours=1)),
)
original_now = workers.now
try:
workers.now = lambda: clock
await workers.sample_capacity(runtime)
sample = await repo.one(
"select size_bytes from storage_capacity_samples where bucket_id=%s order by sampled_at desc limit 1",
(identifier,),
)
assert (
sample["size_bytes"] == capacity_bytes + 28
) # One AES-GCM nonce/tag, charged as real physical storage.
workers.now = lambda: clock + timedelta(days=1)
await workers.sample_capacity(runtime)
finally:
workers.now = original_now
async with repo.transaction() as conn:
await conn.execute("update storage_usage_events set created_at=%s::date", (day,))
await conn.execute("update storage_billing_reservations set created_at=%s::date", (day,))
measured = await repo.all(
"select meter,sum(quantity) quantity from storage_usage_events where bucket_id=%s group by meter",
(identifier,),
)
meters = {row["meter"]: row["quantity"] for row in measured}
workflows = concurrency * iterations
assert meters["write_operations"] == workflows * 2 + 8 # Includes durable multipart manifest.
assert (
meters["read_operations"] == workflows * 2 + 6
) # Manifest plus ciphertext split at the provider chunk boundary.
assert meters["egress_bytes"] == workflows * 1024 + len(body) + 6
await workers.settle_day(runtime, day)
balance = await repo.one(
"select balance_credits from user_wallets where user_id=%s", (bucket["owner_id"],)
)
await workers.settle_day(runtime, day)
assert (
await repo.one("select balance_credits from user_wallets where user_id=%s", (bucket["owner_id"],))
== balance
)
holds = await repo.one("select count(*) n from storage_billing_reservations where state='reserved'")
assert holds["n"] == 0
exact = await repo.one("select coalesce(sum(amount_credits),0) n from storage_usage_daily")
carry = await repo.one("select coalesce(sum(amount),0) n from storage_billing_residuals")
debited = await repo.one(
"select coalesce(-sum(delta_credits),0) n from credit_ledger where usage_kind='storage' and entry_type='usage'"
)
with localcontext() as arithmetic:
arithmetic.prec = 80
assert exact["n"] == debited["n"] + carry["n"]
assert balance["balance_credits"] == Decimal(100) - debited["n"]
return {
"exact_credits": str(exact["n"]),
"debited_credits": str(debited["n"]),
"fractional_carry": str(carry["n"]),
"outstanding_holds": 0,
"inventory_bytes": capacity_bytes + 28,
"logical_bytes": capacity_bytes,
"expired_upload_cleaned": True,
"read_operations": int(meters["read_operations"]),
"write_operations": int(meters["write_operations"]),
"egress_bytes": int(meters["egress_bytes"]),
}
finally:
await runtime.close()
report["accounting"] = asyncio.run(accounting())
report["restore"] = {
"sha256_match": True,
"encrypted_credential_recovered": True,
"bucket_and_object_tags_recovered": True,
"restored_credentials_quarantined": True,
"backup_bytes": snapshot.stat().st_size,
}
report["restart"] = {"multipart_bytes": obj["size_bytes"], "sha256_match": True}
checkpoint("closed-day settlement balances exact charge, carry and refunds; retry is idempotent")
report["elapsed_seconds"] = round(time.monotonic() - started, 3)
report["client_peak_rss_mib"] = round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 1)
report_path = directory / "acceptance-results.json"
private_write(report_path, json.dumps(report, indent=2).encode())
print(json.dumps(report, indent=2))
finally:
first.close()
second.close()
for proc in servers:
if proc.poll() is None:
proc.terminate()
try:
proc.wait(timeout=15)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--directory", type=Path, default=ROOT / "work/buckets-acceptance")
parser.add_argument("--concurrency", type=int, default=8)
parser.add_argument("--iterations", type=int, default=4)
args = parser.parse_args()
if not 1 <= args.concurrency <= 32 or not 1 <= args.iterations <= 10:
parser.error("Use 1..32 clients and 1..10 workflows per client")
main(args.directory, args.concurrency, args.iterations)

Xet Storage Details

Size:
25 kB
·
Xet hash:
952cb299c31105e831c4e9eb4ebe987dc9f6f6b603f426d96c6c9c9206991d87

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.