runtime.py

import hashlib
import importlib.metadata
import json
import os
import platform
import sys
import time
import uuid
from pathlib import Path

ROOT = Path("/data")
UPSTREAM = Path("/opt/space/upstream")
MODEL_ROOT = ROOT / "models/make-it-animatable"
WEIGHTS = {"bw.pth":424693698, "bw_normal.pth":429008352,
           "joints.pth":525833771, "joints_coarse.pth":424706373,
           "pose.pth":525848433}

def configure_storage():
    if not ROOT.is_dir():
        raise RuntimeError("/data is not mounted. Mount the existing persistent bucket at /data.")
    for name in ("cache/huggingface","cache/torch","cache/xdg","cache/pip",
                 "models","inputs","outputs","temp","logs"):
        (ROOT/name).mkdir(parents=True,exist_ok=True)
    os.environ.update(HF_HOME=str(ROOT/"cache/huggingface"),
                      HF_HUB_CACHE=str(ROOT/"cache/huggingface/hub"),
                      TORCH_HOME=str(ROOT/"cache/torch"),
                      XDG_CACHE_HOME=str(ROOT/"cache/xdg"),
                      PIP_CACHE_DIR=str(ROOT/"cache/pip"),
                      TMPDIR=str(ROOT/"temp"),
                      GRADIO_TEMP_DIR=str(ROOT/"temp/gradio"))
    probe = ROOT/"temp"/("write-check-"+uuid.uuid4().hex)
    with probe.open("x") as stream:
        stream.write("ok")
    if probe.read_text() != "ok":
        raise RuntimeError("/data read/write verification failed")
    probe.unlink()
    import tempfile
    tempfile.tempdir = str(ROOT/"temp")
    missing = [str(MODEL_ROOT/"output/best/new"/name) for name,size in WEIGHTS.items()
               if not (MODEL_ROOT/"output/best/new"/name).is_file()
               or (MODEL_ROOT/"output/best/new"/name).stat().st_size != size]
    bones = MODEL_ROOT/"data/Mixamo/bones.fbx"
    if not bones.is_file() or bones.stat().st_size == 0:
        missing.append(str(bones))
    if missing:
        raise RuntimeError("Required cached assets missing or incomplete; startup stopped without "
                           "overwriting or redownloading assets: "+", ".join(missing))
    for name in ("data","output"):
        target = UPSTREAM/name
        expected = MODEL_ROOT/name
        if target.is_symlink():
            if target.resolve() != expected.resolve():
                raise RuntimeError("Unexpected asset link: "+str(target))
        elif target.exists():
            raise RuntimeError("Asset path must be an image-managed symlink: "+str(target))
        else:
            target.symlink_to(expected, target_is_directory=True)
    print("STORAGE_READY /data writable; cached weights reused; downloads=0; installs=0",flush=True)

def startup_checks():
    import torch
    import torch_cluster
    import pytorch3d
    from pytorch3d.ops import sample_farthest_points
    versions = {}
    for name in ("torch","torchvision","torch-cluster","torch-geometric",
                 "torch-scatter","torch-sparse","torch-spline-conv","pytorch3d","numpy","gradio"):
        try:
            versions[name] = importlib.metadata.version(name)
        except importlib.metadata.PackageNotFoundError:
            versions[name] = "not required / not installed"
    report = dict(python=platform.python_version(),packages=versions,cuda=torch.version.cuda,
                  cuda_available=torch.cuda.is_available(),
                  gpu=torch.cuda.get_device_name(0) if torch.cuda.is_available() else None)
    print("RUNTIME_VERSIONS "+json.dumps(report),flush=True)
    if not torch.cuda.is_available():
        raise RuntimeError("CUDA GPU unavailable. Dedicated GPU required; CPU inference is disabled.")
    if torch.__version__ != "2.1.2+cu121" or torch.version.cuda != "12.1":
        raise RuntimeError("Runtime Torch/CUDA differs from the pinned build")
    x = torch.rand((128,3),device="cuda")
    out = torch_cluster.fps(x,ratio=0.25)
    _, indices = sample_farthest_points(x[None],K=32)
    y = torch.ones((32,32),device="cuda") @ torch.ones((32,32),device="cuda")
    torch.cuda.synchronize()
    if len(out) != 32 or indices.shape != (1,32) or y[0,0].item() != 32:
        raise RuntimeError("CUDA binary smoke test failed")
    report["cuda_execution"] = "PASS: matmul, torch_cluster.fps, pytorch3d FPS"
    persistence = {}
    for name in ("cache/huggingface","cache/torch","cache/xdg","cache/pip","inputs","outputs"):
        marker = ROOT/name/".persistence-check"
        existed = marker.is_file()
        if not existed:
            with marker.open("x") as stream:
                stream.write(uuid.uuid4().hex)
        persistence[name] = {"marker":marker.read_text(),"survived_previous_start":existed}
    report["persistence"] = persistence
    report["cached_weights"] = WEIGHTS
    report["startup_time"] = time.time()
    report["startup_id"] = uuid.uuid4().hex
    report["outputs_present"] = sorted(str(p.relative_to(ROOT)) for p in (ROOT/"outputs").glob("*"))
    previous = ROOT/"logs/last-startup.json"
    if previous.is_file():
        report["previous_startup"] = json.loads(previous.read_text()).get("startup_id")
    (ROOT/"logs"/(report["startup_id"]+".json")).write_text(json.dumps(report,indent=2))
    previous.write_text(json.dumps(report,indent=2))
    print("CUDA_EXECUTION_PASS "+json.dumps(report),flush=True)
    return report