| """Create a persistent local E2E sandbox; never touches production identity/data. | |
| Usage: .venv/bin/python tools/run_buckets_sandbox.py [--init-only] [--port 58080] | |
| Test credentials and a local TLS CA are saved under work/buckets-sandbox (0600). | |
| """ | |
| import argparse | |
| import asyncio | |
| import base64 | |
| from datetime import datetime, timedelta, timezone | |
| import hashlib | |
| import ipaddress | |
| import json | |
| import os | |
| from pathlib import Path | |
| import secrets | |
| import subprocess | |
| import sys | |
| from uuid import uuid4 | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT / "src")) | |
| WORK = ROOT / "work" / "buckets-sandbox" | |
| def save(path, value): | |
| path.write_text(value) | |
| path.chmod(0o600) | |
| async def initialize(port, database="buckets", azure_resources=None, catalog_source=None): | |
| from azure.core.exceptions import ResourceExistsError | |
| from azure.storage.blob.aio import BlobServiceClient | |
| from buckets.catalog import active | |
| from buckets.repository import Repository | |
| WORK.mkdir(parents=True, exist_ok=True) | |
| config_path = WORK / "config.json" | |
| if config_path.exists(): | |
| config = json.loads(config_path.read_text()) | |
| else: | |
| config = { | |
| "BUCKETS_SANDBOX": "1", | |
| "BUCKETS_DATABASE_URL": "postgresql://postgres:buckets-local-e2e@127.0.0.1:55432/" + database, | |
| "BUCKETS_SANDBOX_WRAP_KEY": base64.b64encode(os.urandom(32)).decode(), | |
| "BUCKETS_CURSOR_KEY": base64.b64encode(os.urandom(32)).decode(), | |
| "BUCKETS_PUBLIC_ENDPOINT": f"https://localhost:{port}/s3/", | |
| } | |
| # The Azure-documented emulator key is public test material, not an Azure account secret. | |
| # Read Azurite's bundled emulator default rather than inventing credentials. | |
| result = subprocess.run( | |
| [ | |
| "docker", | |
| "--host=unix:///var/run/docker.sock", | |
| "exec", | |
| "ip-buckets-azurite", | |
| "node", | |
| "-e", | |
| 'const c=require("/opt/azurite/dist/src/common/utils/constants.js"); process.stdout.write(c.EMULATOR_ACCOUNT_KEY.toString("base64"))', | |
| ], | |
| capture_output=True, | |
| text=True, | |
| check=True, | |
| ) | |
| key = result.stdout.strip() | |
| config["BUCKETS_AZURITE_CONNECTION_STRING"] = ( | |
| f"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey={key};BlobEndpoint=http://127.0.0.1:51000/devstoreaccount1;" | |
| ) | |
| save(config_path, json.dumps(config, indent=2)) | |
| if azure_resources: | |
| resources = json.loads(azure_resources.read_text()) | |
| config.update({k: resources[k] for k in ("BUCKETS_AZURE_ACCOUNT_URL", "BUCKETS_KEY_VAULT_KEY_ID")}) | |
| save(config_path, json.dumps(config, indent=2)) | |
| from urllib.parse import urlsplit | |
| parsed = urlsplit(config["BUCKETS_DATABASE_URL"]) | |
| if ( | |
| parsed.hostname not in ("127.0.0.1", "localhost") | |
| or parsed.port != 55432 | |
| or not parsed.path.startswith("/buckets") | |
| ): | |
| raise RuntimeError("Refusing to initialize a non-sandbox database") | |
| import psycopg | |
| from psycopg import sql | |
| async with await psycopg.AsyncConnection.connect( | |
| "postgresql://postgres:buckets-local-e2e@127.0.0.1:55432/postgres", autocommit=True | |
| ) as admin: | |
| name = parsed.path.lstrip("/") | |
| if not await (await admin.execute("select 1 from pg_database where datname=%s", (name,))).fetchone(): | |
| await admin.execute(sql.SQL("create database {}").format(sql.Identifier(name))) | |
| os.environ.update(config) | |
| repo = Repository(config["BUCKETS_DATABASE_URL"]) | |
| existing = await repo.one("select to_regclass('public.storage_buckets') as name") | |
| if not existing["name"]: | |
| async with repo.transaction() as conn: | |
| await conn.execute((ROOT / "tools/buckets-sandbox-base.sql").read_text()) | |
| await conn.execute((ROOT / "supabase/migrations/20261101151604_inferenceport_buckets.sql").read_text()) | |
| await conn.execute((ROOT / "supabase/migrations/20261101171055_buckets_retention_indexes.sql").read_text()) | |
| await conn.execute((ROOT / "supabase/migrations/20261106011903_buckets_public_access.sql").read_text()) | |
| await conn.execute((ROOT / "supabase/migrations/20261106120423_buckets_s3_tags.sql").read_text()) | |
| await conn.execute((ROOT / "supabase/migrations/20261106123450_buckets_versioning_and_metadata.sql").read_text()) | |
| await conn.execute((ROOT / "supabase/migrations/20261108153111_buckets_fk_indexes_rls.sql").read_text()) | |
| users = [uuid4(), uuid4()] | |
| orgs = [uuid4(), uuid4()] | |
| tokens = {} | |
| async with repo.transaction() as conn: | |
| for index, uid in enumerate(users): | |
| await conn.execute( | |
| "insert into auth.users(id,email) values(%s,%s)", (uid, f"user{index + 1}@sandbox.invalid") | |
| ) | |
| await conn.execute("insert into public.user_wallets(user_id,balance_credits) values(%s,100)", (uid,)) | |
| await conn.execute( | |
| "insert into public.organizations(id,slug,name) values(%s,%s,%s)", | |
| (orgs[index], f"org-{index + 1}", f"Sandbox org {index + 1}"), | |
| ) | |
| await conn.execute( | |
| "insert into public.org_members(org_id,user_id,role) values(%s,%s,'owner')", (orgs[index], uid) | |
| ) | |
| await conn.execute( | |
| "insert into public.org_wallets(org_id,balance_credits) values(%s,100)", (orgs[index],) | |
| ) | |
| for scope, org in [("personal", None), ("org", orgs[index])]: | |
| token = "ipa_live_" + secrets.token_urlsafe(24) | |
| salt = os.urandom(16) | |
| derived = hashlib.scrypt(token.encode(), salt=salt, n=16384, r=8, p=1, dklen=32) | |
| hashed = ( | |
| "scrypt$" | |
| + base64.urlsafe_b64encode(salt).decode().rstrip("=") | |
| + "$" | |
| + base64.urlsafe_b64encode(derived).decode().rstrip("=") | |
| ) | |
| await conn.execute( | |
| "insert into public.lightning_api_keys(id,user_id,org_id,key_prefix,key_hash) values(%s,%s,%s,%s,%s)", | |
| (uuid4(), uid, org, token[:16], hashed), | |
| ) | |
| tokens[f"user{index + 1}_{scope}"] = token | |
| save(WORK / "credentials.json", json.dumps(tokens, indent=2)) | |
| if config.get("BUCKETS_AZURE_ACCOUNT_URL"): | |
| from azure.identity.aio import DefaultAzureCredential | |
| async with DefaultAzureCredential() as credential: | |
| async with BlobServiceClient( | |
| config["BUCKETS_AZURE_ACCOUNT_URL"], credential, connection_timeout=10, read_timeout=20, retry_total=1 | |
| ) as client: | |
| props = await client.get_container_client("inferenceport-buckets").get_container_properties() | |
| if props.get("public_access"): | |
| raise RuntimeError("Azure E2E container must be private") | |
| else: | |
| async with BlobServiceClient.from_connection_string(config["BUCKETS_AZURITE_CONNECTION_STRING"]) as client: | |
| try: | |
| await client.create_container("inferenceport-buckets") | |
| except ResourceExistsError: | |
| pass | |
| if catalog_source: | |
| source_config = json.loads(catalog_source.read_text()) | |
| source_dsn = source_config["BUCKETS_DATABASE_URL"] | |
| source_url = urlsplit(source_dsn) | |
| if ( | |
| source_url.hostname not in ("localhost", "127.0.0.1") | |
| or source_url.port != 55432 | |
| or not source_url.path.startswith("/buckets") | |
| ): | |
| raise RuntimeError("Catalog sources must also be isolated local sandbox databases") | |
| source = Repository(source_dsn) | |
| await active(source) # Enforce the real freshness and customer-pricing policy. | |
| row = await source.one("""select v.*,c.checked_at from public.storage_rate_versions v | |
| join public.storage_rate_checks c on c.rate_version=v.id order by c.checked_at desc limit 1""") | |
| from psycopg.types.json import Jsonb | |
| async with repo.transaction() as conn: | |
| await conn.execute( | |
| """insert into public.storage_rate_versions(id,effective_at,catalog,source_payload) | |
| values(%s,%s,%s,%s) on conflict(id) do nothing""", | |
| (row["id"], row["effective_at"], Jsonb(row["catalog"]), Jsonb(row["source_payload"])), | |
| ) | |
| await conn.execute( | |
| "insert into public.storage_rate_checks(rate_version,checked_at) values(%s,%s)", | |
| (row["id"], row["checked_at"]), | |
| ) | |
| await active(repo) | |
| if not (WORK / "localhost.pem").exists(): | |
| from cryptography import x509 | |
| from cryptography.hazmat.primitives import hashes, serialization | |
| from cryptography.hazmat.primitives.asymmetric import rsa | |
| from cryptography.x509.oid import NameOID | |
| key = rsa.generate_private_key(public_exponent=65537, key_size=2048) | |
| name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "InferencePort sandbox localhost")]) | |
| cert = ( | |
| x509.CertificateBuilder() | |
| .subject_name(name) | |
| .issuer_name(name) | |
| .public_key(key.public_key()) | |
| .serial_number(x509.random_serial_number()) | |
| .not_valid_before(datetime.now(timezone.utc) - timedelta(days=1)) | |
| .not_valid_after(datetime.now(timezone.utc) + timedelta(days=365)) | |
| .add_extension( | |
| x509.SubjectAlternativeName( | |
| [x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))] | |
| ), | |
| False, | |
| ) | |
| .add_extension(x509.BasicConstraints(ca=True, path_length=None), True) | |
| .sign(key, hashes.SHA256()) | |
| ) | |
| save( | |
| WORK / "localhost-key.pem", | |
| key.private_bytes( | |
| serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption() | |
| ).decode(), | |
| ) | |
| save(WORK / "localhost.pem", cert.public_bytes(serialization.Encoding.PEM).decode()) | |
| print(f"Sandbox endpoint: {config['BUCKETS_PUBLIC_ENDPOINT']}") | |
| print(f"Local test credentials: {WORK / 'credentials.json'}") | |
| print(f"Local TLS CA: {WORK / 'localhost.pem'}") | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--init-only", action="store_true") | |
| parser.add_argument("--reload", action="store_true") | |
| parser.add_argument("--worker", action="store_true") | |
| parser.add_argument("--directory", type=Path, default=WORK) | |
| parser.add_argument("--database", default="buckets") | |
| parser.add_argument("--azure-resources", type=Path) | |
| parser.add_argument( | |
| "--catalog-source", | |
| type=Path, | |
| help="Reuse a still-verified local sandbox price snapshot; preserves its verification timestamp", | |
| ) | |
| parser.add_argument("--port", type=int, default=58080) | |
| args = parser.parse_args() | |
| WORK = args.directory.resolve() | |
| asyncio.run(initialize(args.port, args.database, args.azure_resources, args.catalog_source)) | |
| if args.worker: | |
| from buckets.workers import main | |
| asyncio.run(main(False)) | |
| sys.exit(0) | |
| if not args.init_only: | |
| import uvicorn | |
| uvicorn.run( | |
| "buckets.sandbox:create_app", | |
| factory=True, | |
| host="127.0.0.1", | |
| port=args.port, | |
| ssl_keyfile=str(WORK / "localhost-key.pem"), | |
| ssl_certfile=str(WORK / "localhost.pem"), | |
| access_log=False, | |
| reload=args.reload, | |
| reload_dirs=[str(ROOT / "src" / "buckets")], | |
| ) | |
Xet Storage Details
- Size:
- 11.9 kB
- Xet hash:
- 869c7c9e576d117a109be4211ddb0098708fd1be5c5813fc832e741dd012cb05
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.