sharktide/tools / run_buckets_sandbox.py
sharktide's picture
download
raw
11.9 kB
"""Create a persistent local E2E sandbox; never touches production identity/data.
Usage: .venv/bin/python tools/run_buckets_sandbox.py [--init-only] [--port 58080]
Test credentials and a local TLS CA are saved under work/buckets-sandbox (0600).
"""
import argparse
import asyncio
import base64
from datetime import datetime, timedelta, timezone
import hashlib
import ipaddress
import json
import os
from pathlib import Path
import secrets
import subprocess
import sys
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
WORK = ROOT / "work" / "buckets-sandbox"
def save(path, value):
path.write_text(value)
path.chmod(0o600)
async def initialize(port, database="buckets", azure_resources=None, catalog_source=None):
from azure.core.exceptions import ResourceExistsError
from azure.storage.blob.aio import BlobServiceClient
from buckets.catalog import active
from buckets.repository import Repository
WORK.mkdir(parents=True, exist_ok=True)
config_path = WORK / "config.json"
if config_path.exists():
config = json.loads(config_path.read_text())
else:
config = {
"BUCKETS_SANDBOX": "1",
"BUCKETS_DATABASE_URL": "postgresql://postgres:buckets-local-e2e@127.0.0.1:55432/" + database,
"BUCKETS_SANDBOX_WRAP_KEY": base64.b64encode(os.urandom(32)).decode(),
"BUCKETS_CURSOR_KEY": base64.b64encode(os.urandom(32)).decode(),
"BUCKETS_PUBLIC_ENDPOINT": f"https://localhost:{port}/s3/",
}
# The Azure-documented emulator key is public test material, not an Azure account secret.
# Read Azurite's bundled emulator default rather than inventing credentials.
result = subprocess.run(
[
"docker",
"--host=unix:///var/run/docker.sock",
"exec",
"ip-buckets-azurite",
"node",
"-e",
'const c=require("/opt/azurite/dist/src/common/utils/constants.js"); process.stdout.write(c.EMULATOR_ACCOUNT_KEY.toString("base64"))',
],
capture_output=True,
text=True,
check=True,
)
key = result.stdout.strip()
config["BUCKETS_AZURITE_CONNECTION_STRING"] = (
f"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey={key};BlobEndpoint=http://127.0.0.1:51000/devstoreaccount1;"
)
save(config_path, json.dumps(config, indent=2))
if azure_resources:
resources = json.loads(azure_resources.read_text())
config.update({k: resources[k] for k in ("BUCKETS_AZURE_ACCOUNT_URL", "BUCKETS_KEY_VAULT_KEY_ID")})
save(config_path, json.dumps(config, indent=2))
from urllib.parse import urlsplit
parsed = urlsplit(config["BUCKETS_DATABASE_URL"])
if (
parsed.hostname not in ("127.0.0.1", "localhost")
or parsed.port != 55432
or not parsed.path.startswith("/buckets")
):
raise RuntimeError("Refusing to initialize a non-sandbox database")
import psycopg
from psycopg import sql
async with await psycopg.AsyncConnection.connect(
"postgresql://postgres:buckets-local-e2e@127.0.0.1:55432/postgres", autocommit=True
) as admin:
name = parsed.path.lstrip("/")
if not await (await admin.execute("select 1 from pg_database where datname=%s", (name,))).fetchone():
await admin.execute(sql.SQL("create database {}").format(sql.Identifier(name)))
os.environ.update(config)
repo = Repository(config["BUCKETS_DATABASE_URL"])
existing = await repo.one("select to_regclass('public.storage_buckets') as name")
if not existing["name"]:
async with repo.transaction() as conn:
await conn.execute((ROOT / "tools/buckets-sandbox-base.sql").read_text())
await conn.execute((ROOT / "supabase/migrations/20261101151604_inferenceport_buckets.sql").read_text())
await conn.execute((ROOT / "supabase/migrations/20261101171055_buckets_retention_indexes.sql").read_text())
await conn.execute((ROOT / "supabase/migrations/20261106011903_buckets_public_access.sql").read_text())
await conn.execute((ROOT / "supabase/migrations/20261106120423_buckets_s3_tags.sql").read_text())
await conn.execute((ROOT / "supabase/migrations/20261106123450_buckets_versioning_and_metadata.sql").read_text())
await conn.execute((ROOT / "supabase/migrations/20261108153111_buckets_fk_indexes_rls.sql").read_text())
users = [uuid4(), uuid4()]
orgs = [uuid4(), uuid4()]
tokens = {}
async with repo.transaction() as conn:
for index, uid in enumerate(users):
await conn.execute(
"insert into auth.users(id,email) values(%s,%s)", (uid, f"user{index + 1}@sandbox.invalid")
)
await conn.execute("insert into public.user_wallets(user_id,balance_credits) values(%s,100)", (uid,))
await conn.execute(
"insert into public.organizations(id,slug,name) values(%s,%s,%s)",
(orgs[index], f"org-{index + 1}", f"Sandbox org {index + 1}"),
)
await conn.execute(
"insert into public.org_members(org_id,user_id,role) values(%s,%s,'owner')", (orgs[index], uid)
)
await conn.execute(
"insert into public.org_wallets(org_id,balance_credits) values(%s,100)", (orgs[index],)
)
for scope, org in [("personal", None), ("org", orgs[index])]:
token = "ipa_live_" + secrets.token_urlsafe(24)
salt = os.urandom(16)
derived = hashlib.scrypt(token.encode(), salt=salt, n=16384, r=8, p=1, dklen=32)
hashed = (
"scrypt$"
+ base64.urlsafe_b64encode(salt).decode().rstrip("=")
+ "$"
+ base64.urlsafe_b64encode(derived).decode().rstrip("=")
)
await conn.execute(
"insert into public.lightning_api_keys(id,user_id,org_id,key_prefix,key_hash) values(%s,%s,%s,%s,%s)",
(uuid4(), uid, org, token[:16], hashed),
)
tokens[f"user{index + 1}_{scope}"] = token
save(WORK / "credentials.json", json.dumps(tokens, indent=2))
if config.get("BUCKETS_AZURE_ACCOUNT_URL"):
from azure.identity.aio import DefaultAzureCredential
async with DefaultAzureCredential() as credential:
async with BlobServiceClient(
config["BUCKETS_AZURE_ACCOUNT_URL"], credential, connection_timeout=10, read_timeout=20, retry_total=1
) as client:
props = await client.get_container_client("inferenceport-buckets").get_container_properties()
if props.get("public_access"):
raise RuntimeError("Azure E2E container must be private")
else:
async with BlobServiceClient.from_connection_string(config["BUCKETS_AZURITE_CONNECTION_STRING"]) as client:
try:
await client.create_container("inferenceport-buckets")
except ResourceExistsError:
pass
if catalog_source:
source_config = json.loads(catalog_source.read_text())
source_dsn = source_config["BUCKETS_DATABASE_URL"]
source_url = urlsplit(source_dsn)
if (
source_url.hostname not in ("localhost", "127.0.0.1")
or source_url.port != 55432
or not source_url.path.startswith("/buckets")
):
raise RuntimeError("Catalog sources must also be isolated local sandbox databases")
source = Repository(source_dsn)
await active(source) # Enforce the real freshness and customer-pricing policy.
row = await source.one("""select v.*,c.checked_at from public.storage_rate_versions v
join public.storage_rate_checks c on c.rate_version=v.id order by c.checked_at desc limit 1""")
from psycopg.types.json import Jsonb
async with repo.transaction() as conn:
await conn.execute(
"""insert into public.storage_rate_versions(id,effective_at,catalog,source_payload)
values(%s,%s,%s,%s) on conflict(id) do nothing""",
(row["id"], row["effective_at"], Jsonb(row["catalog"]), Jsonb(row["source_payload"])),
)
await conn.execute(
"insert into public.storage_rate_checks(rate_version,checked_at) values(%s,%s)",
(row["id"], row["checked_at"]),
)
await active(repo)
if not (WORK / "localhost.pem").exists():
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import NameOID
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "InferencePort sandbox localhost")])
cert = (
x509.CertificateBuilder()
.subject_name(name)
.issuer_name(name)
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(datetime.now(timezone.utc) - timedelta(days=1))
.not_valid_after(datetime.now(timezone.utc) + timedelta(days=365))
.add_extension(
x509.SubjectAlternativeName(
[x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]
),
False,
)
.add_extension(x509.BasicConstraints(ca=True, path_length=None), True)
.sign(key, hashes.SHA256())
)
save(
WORK / "localhost-key.pem",
key.private_bytes(
serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()
).decode(),
)
save(WORK / "localhost.pem", cert.public_bytes(serialization.Encoding.PEM).decode())
print(f"Sandbox endpoint: {config['BUCKETS_PUBLIC_ENDPOINT']}")
print(f"Local test credentials: {WORK / 'credentials.json'}")
print(f"Local TLS CA: {WORK / 'localhost.pem'}")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--init-only", action="store_true")
parser.add_argument("--reload", action="store_true")
parser.add_argument("--worker", action="store_true")
parser.add_argument("--directory", type=Path, default=WORK)
parser.add_argument("--database", default="buckets")
parser.add_argument("--azure-resources", type=Path)
parser.add_argument(
"--catalog-source",
type=Path,
help="Reuse a still-verified local sandbox price snapshot; preserves its verification timestamp",
)
parser.add_argument("--port", type=int, default=58080)
args = parser.parse_args()
WORK = args.directory.resolve()
asyncio.run(initialize(args.port, args.database, args.azure_resources, args.catalog_source))
if args.worker:
from buckets.workers import main
asyncio.run(main(False))
sys.exit(0)
if not args.init_only:
import uvicorn
uvicorn.run(
"buckets.sandbox:create_app",
factory=True,
host="127.0.0.1",
port=args.port,
ssl_keyfile=str(WORK / "localhost-key.pem"),
ssl_certfile=str(WORK / "localhost.pem"),
access_log=False,
reload=args.reload,
reload_dirs=[str(ROOT / "src" / "buckets")],
)

Xet Storage Details

Size:
11.9 kB
·
Xet hash:
869c7c9e576d117a109be4211ddb0098708fd1be5c5813fc832e741dd012cb05

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.