sharktide/tools / provision_buckets_azure_test.py
sharktide's picture
download
raw
8.19 kB
"""Provision an isolated tagged Azure E2E account/container and Key Vault.
Requires AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID and --subscription.
Creates only its own new/tagged resource group; never touches the agents account.
Runtime configuration contains resource IDs, never the supplied client secret.
"""
import argparse
import base64
import json
import os
from pathlib import Path
import time
import urllib.error
import urllib.parse
import urllib.request
from uuid import uuid4
ARM = "https://management.azure.com"
def token(scope):
data = urllib.parse.urlencode(
{
"client_id": os.environ["AZURE_CLIENT_ID"],
"client_secret": os.environ["AZURE_CLIENT_SECRET"],
"grant_type": "client_credentials",
"scope": scope,
}
).encode()
request = urllib.request.Request(
"https://login.microsoftonline.com/" + os.environ["AZURE_TENANT_ID"] + "/oauth2/v2.0/token", data=data
)
with urllib.request.urlopen(request, timeout=60) as response:
return json.load(response)["access_token"]
def main(subscription, directory):
directory.mkdir(parents=True, exist_ok=True)
config_path = directory / "azure-resources.json"
config = (
json.loads(config_path.read_text())
if config_path.exists()
else {
"resource_group": "ip-buckets-e2e-" + uuid4().hex[:8],
"account": "ipbe2e" + uuid4().hex[:14],
"vault": "ipbe2e" + uuid4().hex[:14],
}
)
bearer = token(ARM + "/.default")
claims = json.loads(base64.urlsafe_b64decode(bearer.split(".")[1] + "=="))
principal = claims["oid"]
tenant = os.environ["AZURE_TENANT_ID"]
def call(method, path, body=None, bearer_token=bearer):
url = path if path.startswith("https://") else ARM + path
request = urllib.request.Request(
url,
data=json.dumps(body).encode() if body is not None else None,
method=method,
headers={"Authorization": "Bearer " + bearer_token, "Content-Type": "application/json"},
)
try:
with urllib.request.urlopen(request, timeout=90) as response:
raw = response.read()
return json.loads(raw) if raw else {}
except urllib.error.HTTPError as exc:
raw = exc.read()
try:
code = json.loads(raw).get("error", {}).get("code", "Unknown")
except ValueError:
code = "Unknown"
raise RuntimeError(f"Azure {method} failed HTTP {exc.code} code={code}") from None
group = f"/subscriptions/{subscription}/resourceGroups/{config['resource_group']}"
tags = {
"purpose": "InferencePort Buckets E2E",
"environment": "test",
"owner": "inferenceport",
"expires": "2026-11-07",
}
if config_path.exists():
existing = call("GET", group + "?api-version=2022-09-01")
if existing.get("tags", {}).get("purpose") != tags["purpose"]:
raise RuntimeError("Refusing to modify an unrecognized resource group")
else:
call("PUT", group + "?api-version=2022-09-01", {"location": "eastus", "tags": tags})
config_path.write_text(json.dumps(config, indent=2))
config_path.chmod(0o600)
storage = group + "/providers/Microsoft.Storage/storageAccounts/" + config["account"]
print("Provisioning isolated eastus/LRS storage account", flush=True)
call(
"PUT",
storage + "?api-version=2023-05-01",
{
"location": "eastus",
"tags": tags,
"kind": "StorageV2",
"sku": {"name": "Standard_LRS"},
"properties": {
"accessTier": "Hot",
"allowBlobPublicAccess": False,
"allowSharedKeyAccess": False,
"minimumTlsVersion": "TLS1_2",
"supportsHttpsTrafficOnly": True,
},
},
)
for _ in range(90):
account = call("GET", storage + "?api-version=2023-05-01")
if account["properties"].get("provisioningState") == "Succeeded":
break
time.sleep(2)
else:
raise RuntimeError("Storage provisioning timed out")
call(
"PUT",
storage + "/blobServices/default?api-version=2023-05-01",
{
"properties": {
"isVersioningEnabled": True,
"deleteRetentionPolicy": {"enabled": True, "days": 7},
"containerDeleteRetentionPolicy": {"enabled": True, "days": 7},
}
},
)
call(
"PUT",
storage + "/blobServices/default/containers/inferenceport-buckets?api-version=2023-05-01",
{"properties": {"publicAccess": "None"}},
)
role = storage + "/providers/Microsoft.Authorization/roleAssignments/" + str(uuid4()) + "?api-version=2022-04-01"
try:
call(
"PUT",
role,
{
"properties": {
"roleDefinitionId": f"/subscriptions/{subscription}/providers/Microsoft.Authorization/roleDefinitions/ba92f5b4-2d11-453d-a403-e96b0029c9fe",
"principalId": principal,
"principalType": "ServicePrincipal",
}
},
)
except RuntimeError as exc:
if "AuthorizationFailed" in str(exc):
config["blob_role_assignment"] = "manual_required"
print("Blob role assignment requires an Azure administrator; continuing Key Vault setup", flush=True)
elif "RoleAssignmentExists" not in str(exc):
raise
print("Provisioning isolated Key Vault wrapping key", flush=True)
vault = group + "/providers/Microsoft.KeyVault/vaults/" + config["vault"]
try:
call(
"PUT",
vault + "?api-version=2023-07-01",
{
"location": "eastus",
"tags": tags,
"properties": {
"tenantId": tenant,
"sku": {"family": "A", "name": "standard"},
"enableRbacAuthorization": False,
"enableSoftDelete": True,
"softDeleteRetentionInDays": 7,
"accessPolicies": [
{
"tenantId": tenant,
"objectId": principal,
"permissions": {"keys": ["get", "list", "create", "wrapKey", "unwrapKey"]},
}
],
},
},
)
except RuntimeError as exc:
if "MissingSubscriptionRegistration" not in str(exc):
raise
call("POST", f"/subscriptions/{subscription}/providers/Microsoft.KeyVault/register?api-version=2021-04-01")
raise RuntimeError("Key Vault provider registration requested; rerun after registration completes")
for _ in range(90):
current = call("GET", vault + "?api-version=2023-07-01")
if current["properties"].get("provisioningState") == "Succeeded":
break
time.sleep(2)
key_url = current["properties"]["vaultUri"] + "keys/buckets-e2e/create?api-version=7.4"
key = call(
"POST",
key_url,
{"kty": "RSA", "key_size": 2048, "key_ops": ["wrapKey", "unwrapKey"]},
token("https://vault.azure.net/.default"),
)
config.update(
{
"BUCKETS_AZURE_ACCOUNT_URL": account["properties"]["primaryEndpoints"]["blob"],
"BUCKETS_KEY_VAULT_KEY_ID": key["key"]["kid"],
"resource_group_id": group,
}
)
config_path.write_text(json.dumps(config, indent=2))
config_path.chmod(0o600)
print("Isolated Azure E2E infrastructure provisioned; verify Blob RBAC/network before use. Configuration:", config_path, flush=True)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--subscription", required=True)
parser.add_argument("--directory", type=Path, default=Path("work/buckets-sandbox"))
args = parser.parse_args()
main(args.subscription, args.directory)

Xet Storage Details

Size:
8.19 kB
·
Xet hash:
e938730df5bfe22f6456b0906bb1eb4fd18e6812e9a2b39e34d45fd689c00c1e

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.