scripts/release_check.py

from __future__ import annotations

import importlib
import re
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
README = (ROOT / "README.md").read_text()
SRC = ROOT / "src"
sys.path.insert(0, str(SRC))

inferscale = importlib.import_module("inferscale")
internal_version = inferscale.__version__
design_space_search = inferscale.design_space_search
compare_agent_policies = inferscale.compare_agent_policies
adaptive_tiering_study = inferscale.adaptive_tiering_study
adaptive_alpha_sweep = inferscale.adaptive_alpha_sweep
compare_agent_memory_policies = inferscale.compare_agent_memory_policies
agent_memory_budget_sweep = inferscale.agent_memory_budget_sweep
agent_affinity_sweep = inferscale.agent_affinity_sweep
execution_prefetch_study = inferscale.execution_prefetch_study
execution_threshold_sweep = inferscale.execution_threshold_sweep
execution_decay_sweep = inferscale.execution_decay_sweep
execution_planning_study = inferscale.execution_planning_study
execution_horizon_sweep = inferscale.execution_horizon_sweep
execution_budget_sweep = inferscale.execution_budget_sweep
run_execution_learning = inferscale.run_execution_learning
paired_study = inferscale.paired_study
robustness_study = inferscale.robustness_study
run_agent_session_simulation = inferscale.run_agent_session_simulation
ttl_retention_sweep = inferscale.ttl_retention_sweep
run_simulation = inferscale.run_simulation
validate_cases = inferscale.validate_cases
repeated_seed_policy_study = inferscale.repeated_seed_policy_study
import_measurements = inferscale.import_measurements
calibrate_measurements = inferscale.calibrate_measurements
generate_research_report = inferscale.generate_research_report

errors: list[str] = []

match = re.search(r"^short_description:\s*(.+)$", README, re.MULTILINE)
if not match:
    errors.append("README metadata is missing short_description")
    short = ""
else:
    short = match.group(1).strip().strip('"\'')
    if len(short) > 60:
        errors.append(f"short_description is {len(short)} chars; HF limit is 60")

if "sdk: static" not in README:
    errors.append("README metadata must use sdk: static")
if internal_version != "1.0.0":
    errors.append(f"internal package version is {internal_version}; expected 1.0.0")

# Public-facing release/version branding is intentionally absent. Model names
# such as Mistral-7B-v0.3 are allowed; project headings/badges are not.
public_texts = [("README.md", README), ("index.html", (ROOT / "index.html").read_text())]
for name, text in public_texts:
    if re.search(r"InferScale(?:-Sim)?\s*/?\s*v\d", text, re.IGNORECASE):
        errors.append(f"{name} contains public project version branding")
if (ROOT / "CHANGELOG.md").exists():
    errors.append("CHANGELOG.md should be omitted from the public portfolio release")

src_files = sorted((ROOT / "src" / "inferscale").glob("*.py"))
web_files = sorted((ROOT / "py" / "inferscale").glob("*.py"))
if [path.name for path in src_files] != [path.name for path in web_files]:
    errors.append("browser Python mirror is stale; run python scripts/sync_web_python.py")
else:
    for src, web in zip(src_files, web_files, strict=True):
        if src.read_bytes() != web.read_bytes():
            errors.append(f"browser mirror differs for {src.name}; run sync_web_python.py")

worker_text = (ROOT / "worker.mjs").read_text()
for src in src_files:
    if f'"{src.name}"' not in worker_text:
        errors.append(f"worker module list is missing {src.name}")

for ui_file in (ROOT / "index.html", ROOT / "app.js"):
    try:
        ui_file.read_text().encode("ascii")
    except UnicodeEncodeError:
        errors.append(f"{ui_file.name} contains non-ASCII UI glyphs; use text labels for reliable rendering")

index = (ROOT / "index.html").read_text()
app = (ROOT / "app.js").read_text()
if "<footer" in index.lower():
    errors.append("UI should not include a product-style footer")
if "Download PNG" not in index or ".chart-download" not in app:
    errors.append("chart PNG export controls are missing")
if "Worst repetition" not in index or "Target" not in index:
    errors.append("capacity evidence columns are missing")
required_ui_ids = [
    "traceFile", "pairedStudyBtn", "robustStudyBtn", "agentRunBtn", "agentCompareBtn", "agentTtlBtn",
    "agentMemoryCompareBtn", "agentBudgetBtn", "agentAffinityBtn", "predictiveCompareBtn", "predictiveAlphaBtn",
    "execRunBtn", "execCompareBtn", "execThresholdBtn", "execDecayBtn", "execPlanningBtn", "execHorizonBtn",
    "execBudgetBtn", "consRunBtn", "measurementCalibrateBtn", "consReportBtn", "exportBtn", "agentRunJson",
]
for expected_id in required_ui_ids:
    if f'id="{expected_id}"' not in index:
        errors.append(f"UI is missing required control: {expected_id}")

# Design guardrails for the public research-workbench UI.
css = (ROOT / "styles.css").read_text()
if "gradient(" in css.lower():
    errors.append("UI should not use decorative CSS gradients")
if re.search(r"border-radius:\s*(?:999|9999|50)%?px?", css, re.IGNORECASE):
    errors.append("UI contains pill-style over-rounding")
if "feature freeze" in index.lower() or "feature freeze" in README.lower():
    errors.append("public UI/docs should not mention the release process")
if 'class="subtitle"' in index:
    errors.append("topbar should remain a compact tool header, not a product masthead")
if re.search(r"\.section-kicker\s*\{[^}]*text-transform:\s*uppercase", css, re.DOTALL | re.IGNORECASE):
    errors.append("section labels should not use repetitive AI-style uppercase kickers")
if 'class="metric emphasis"' in index or ".metric.emphasis" in css:
    errors.append("summary metric cards should remain visually uniform; avoid one-off accent-card emphasis")

# Every $("id") lookup in app.js should resolve to a static DOM id.
app_ids = set(re.findall(r'\$\("([A-Za-z0-9_-]+)"\)', app))
html_ids = set(re.findall(r'id="([A-Za-z0-9_-]+)"', index))
missing_ids = sorted(app_ids - html_ids)
if missing_ids:
    errors.append(f"app.js references missing DOM ids: {', '.join(missing_ids[:12])}")

smoke_cfg = {
    "model": "Qwen2.5-3B",
    "accelerator": "L4",
    "quantization": "int8",
    "duration_s": 4,
    "request_rate_rps": 1,
    "prompt_tokens_mean": 128,
    "output_tokens_mean": 8,
}
try:
    smoke = run_simulation(smoke_cfg)
    if smoke["summary"]["requests_completed"] <= 0:
        errors.append("simulation smoke test completed zero requests")
    if smoke["provenance"]["latency_profile_type"] != "analytical-reference":
        errors.append("profile provenance guard is missing")
    if smoke["diagnostics"].get("provenance") != "heuristic-simulator-diagnosis":
        errors.append("diagnosis provenance missing")
except Exception as exc:  # pragma: no cover
    errors.append(f"colocated smoke test raised: {exc}")

try:
    trace = run_simulation(smoke_cfg | {
        "arrival_process": "trace",
        "trace_requests": [
            {"arrival_time": 0.0, "prompt_tokens": 64, "output_tokens": 4},
            {"arrival_time": 0.2, "prompt_tokens": 96, "output_tokens": 6},
        ],
    })
    if trace["summary"]["requests_generated"] != 2:
        errors.append("trace replay smoke test did not preserve request count")
except Exception as exc:  # pragma: no cover
    errors.append(f"trace replay smoke test raised: {exc}")

try:
    pd = run_simulation(smoke_cfg | {
        "topology": "disaggregated_pd",
        "scheduler": "continuous_slo",
        "prefill_accelerator": "L4",
        "decode_accelerator": "L4",
        "interconnect_gbps": 50,
    })
    if pd["resource"].get("p95_transfer_ms", 0) <= 0:
        errors.append("P/D transfer telemetry missing")
except Exception as exc:  # pragma: no cover
    errors.append(f"P/D smoke test raised: {exc}")

try:
    design = design_space_search(smoke_cfg | {
        "shared_prefix_tokens": 64,
        "prefix_reuse_fraction": 0.5,
        "prefill_accelerator": "L4",
        "decode_accelerator": "L4",
    }, include_disaggregated=False)
    if design["candidate_count"] != 10 or design["pareto_count"] < 1 or design["efficiency_pareto_count"] < 1:
        errors.append("design-space smoke test did not return both expected Pareto frontiers")
except Exception as exc:  # pragma: no cover
    errors.append(f"design-space smoke test raised: {exc}")

try:
    paired = paired_study(
        smoke_cfg | {"shared_prefix_tokens": 64, "prefix_reuse_fraction": 0.75},
        study="prefix_cache",
        repetitions=4,
        bootstrap_samples=100,
    )
    if paired["protocol"] != "paired-common-random-numbers" or len(paired["metrics"]) != 4:
        errors.append("paired research study smoke test is incomplete")
except Exception as exc:  # pragma: no cover
    errors.append(f"paired study smoke test raised: {exc}")

try:
    robust = robustness_study(
        smoke_cfg | {"prefill_accelerator": "L4", "decode_accelerator": "L4"},
        study="pd_vs_colocated",
        samples=4,
        uncertainty=0.10,
    )
    if robust["method"] != "shared-multiplicative-latency-perturbation" or len(robust["rows"]) != 4:
        errors.append("robustness study smoke test is incomplete")
except Exception as exc:  # pragma: no cover
    errors.append(f"robustness study smoke test raised: {exc}")


try:
    agent_cfg = {
        "model": "Qwen2.5-3B", "accelerator": "L4", "quantization": "int8",
        "duration_s": 12, "session_rate_rps": 0.25, "replicas": 2, "seed": 7,
        "retention_policy": "ttl", "routing_policy": "session_affinity", "kv_ttl_s": 3,
    }
    agent = run_agent_session_simulation(agent_cfg)
    if agent["provenance"].get("mode") != "stateful-agent-session-simulation":
        errors.append("agent-session provenance guard is missing")
    if agent["summary"].get("turns_completed", 0) <= 0:
        errors.append("agent-session smoke test completed zero turns")
    agent_compare = compare_agent_policies(agent_cfg)
    if agent_compare.get("candidate_count") != 4:
        errors.append("agent policy arena did not return four candidates")
    agent_ttl = ttl_retention_sweep(agent_cfg, [0, 1, 3])
    if len(agent_ttl.get("rows", [])) != 3 or agent_ttl.get("pareto_count", 0) < 1:
        errors.append("agent TTL frontier smoke test is incomplete")
    agent_memory = compare_agent_memory_policies(agent_cfg | {"host_memory_gb": 4})
    if agent_memory.get("candidate_count") != 5 or not any(row.get("host_hit_rate", 0) > 0 for row in agent_memory.get("rows", [])):
        errors.append("agent tiered-memory comparison is incomplete")
    agent_budget = agent_memory_budget_sweep(agent_cfg | {"host_memory_gb": 4}, [0.5, 1.0])
    if len(agent_budget.get("rows", [])) != 6:
        errors.append("finite HBM budget study is incomplete")
    agent_affinity = agent_affinity_sweep(agent_cfg, [0, 150, 600])
    if len(agent_affinity.get("rows", [])) != 3:
        errors.append("bounded-affinity sweep is incomplete")
    adaptive = run_agent_session_simulation(agent_cfg | {
        "retention_policy": "adaptive",
        "routing_policy": "bounded_affinity",
        "adaptive_predictor_scope": "per_tool_ema",
    })
    if adaptive["provenance"].get("adaptive_policy") != "online-tool-gap-ewma-no-lookahead":
        errors.append("adaptive policy provenance guard is missing")
    if adaptive["resource"].get("adaptive_prediction_count", 0) <= 0:
        errors.append("adaptive policy produced no tool-gap predictions")
    predictive = adaptive_tiering_study(
        agent_cfg | {"host_memory_gb": 4, "routing_policy": "bounded_affinity"},
        horizon_s=40, shift_fraction=0.5, shift_multiplier=2.0, alpha=0.3,
    )
    if len(predictive.get("rows", [])) != 5 or predictive.get("shift_observation", 0) <= 0:
        errors.append("predictive tiering study is incomplete")
    alpha_sweep = adaptive_alpha_sweep(
        agent_cfg | {"host_memory_gb": 4, "routing_policy": "bounded_affinity"},
        [0.1, 0.3, 0.8], horizon_s=40, shift_fraction=0.5, shift_multiplier=2.0,
    )
    if len(alpha_sweep.get("rows", [])) != 3:
        errors.append("adaptation-rate sweep is incomplete")
except Exception as exc:  # pragma: no cover
    errors.append(f"agent-session smoke test raised: {exc}")


try:
    execution_cfg = {
        "model": "Qwen2.5-3B", "accelerator": "L4", "quantization": "int8",
        "duration_s": 50, "workflow_rate_rps": 0.2, "seed": 7,
        "shift_fraction": 0.5, "confidence_threshold": 0.5,
    }
    execution_run = run_execution_learning(execution_cfg | {"prefetch_policy": "decayed"})
    if execution_run["provenance"].get("mode") != "online-agent-execution-learning":
        errors.append("execution-learning provenance guard is missing")
    if execution_run["prediction"].get("count", 0) <= 0:
        errors.append("execution-learning run produced no transition observations")
    execution_compare = execution_prefetch_study(execution_cfg)
    if len(execution_compare.get("rows", [])) != 4:
        errors.append("execution prefetch policy study is incomplete")
    execution_threshold = execution_threshold_sweep(execution_cfg, [0.0, 0.5, 0.9])
    if len(execution_threshold.get("rows", [])) != 3:
        errors.append("execution confidence-threshold sweep is incomplete")
    execution_decay = execution_decay_sweep(execution_cfg, [0.5, 0.85, 1.0])
    if len(execution_decay.get("rows", [])) != 3:
        errors.append("execution transition-decay sweep is incomplete")
    execution_planning = execution_planning_study(execution_cfg | {"forecast_horizon": 3, "prefetch_top_k": 2})
    if len(execution_planning.get("rows", [])) != 4:
        errors.append("execution multi-step planning study is incomplete")
    execution_horizon = execution_horizon_sweep(execution_cfg, [1, 2, 3])
    if len(execution_horizon.get("rows", [])) != 3:
        errors.append("execution forecast-horizon sweep is incomplete")
    execution_budget = execution_budget_sweep(execution_cfg, [0.3, 0.6])
    if len(execution_budget.get("rows", [])) != 6:
        errors.append("execution cache-budget sweep is incomplete")
except Exception as exc:  # pragma: no cover
    errors.append(f"execution-learning smoke test raised: {exc}")

try:
    validation = validate_cases([{
        "name": "release-fixture",
        "config": smoke_cfg,
        "measured": {"p95_ttft_ms": 100.0, "goodput_rps": 0.8},
    }])
    if validation["observation_count"] != 2:
        errors.append("external-measurement validation hook failed")
except Exception as exc:  # pragma: no cover
    errors.append(f"validation hook smoke test raised: {exc}")


try:
    consolidation = repeated_seed_policy_study(
        {
            "model": "Qwen2.5-3B", "accelerator": "L4", "quantization": "int8",
            "duration_s": 32, "workflow_rate_rps": 0.12, "max_steps": 5, "seed": 7,
        },
        repetitions=4, bootstrap_samples=100,
    )
    if len(consolidation.get("policies", [])) != 4 or not consolidation.get("robust_winner"):
        errors.append("repeated-seed consolidation study is incomplete")
    if consolidation.get("oracle", {}).get("candidate_count_per_seed", 0) < 15:
        errors.append("bounded offline oracle search is incomplete")
    report = generate_research_report(consolidation)
    if "Robust policy ranking" not in report or "globally optimal" not in report:
        errors.append("research report guardrails are missing")
except Exception as exc:  # pragma: no cover
    errors.append(f"research consolidation smoke test raised: {exc}")

try:
    measurement_fixture = '{"backend":"sglang","request_rate":1.0,"random_input_len":128,"random_output_len":16,"p95_ttft_ms":120.0,"p95_e2e_latency_ms":900.0}'
    imported = import_measurements(measurement_fixture, "auto", smoke_cfg)
    if imported.get("case_count") != 1:
        errors.append("measurement importer failed")
    calibration = calibrate_measurements(imported["cases"], holdout_fraction=0.33, seed=7)
    if calibration.get("validation_mode") != "resubstitution-insufficient-cases-for-holdout":
        errors.append("small-sample calibration guard is missing")
except Exception as exc:  # pragma: no cover
    errors.append(f"measurement calibration smoke test raised: {exc}")

if errors:
    print("InferScale release check: FAIL")
    for error in errors:
        print(f"- {error}")
    raise SystemExit(1)

print("InferScale release check: PASS")
print(f"HF short_description: {len(short)}/60 characters")
print(f"Python modules mirrored: {len(src_files)}")
print(f"Colocated smoke requests: {smoke['summary']['requests_completed']}")
print(f"Trace replay requests: {trace['summary']['requests_generated']}")
print(f"P/D transfer p95: {pd['resource']['p95_transfer_ms']:.3f} ms")
print(f"Design candidates: {design['candidate_count']}")
print(f"Paired-study metrics: {len(paired['metrics'])}")
print(f"Robustness perturbations: {len(robust['rows'])}")
print(f"Agent turns: {agent['summary']['turns_completed']}")
print(f"Agent policy candidates: {agent_compare['candidate_count']}")
print(f"Agent TTL candidates: {len(agent_ttl['rows'])}")
print(f"Agent memory policies: {agent_memory['candidate_count']}")
print(f"HBM budget study rows: {len(agent_budget['rows'])}")
print(f"Affinity sweep points: {len(agent_affinity['rows'])}")
print(f"Adaptive predictions: {adaptive['resource']['adaptive_prediction_count']}")
print(f"Predictive-tiering candidates: {len(predictive['rows'])}")
print(f"Adaptation-rate points: {len(alpha_sweep['rows'])}")
print(f"Execution transition observations: {execution_run['prediction']['count']}")
print(f"Execution policy candidates: {len(execution_compare['rows'])}")
print(f"Execution threshold points: {len(execution_threshold['rows'])}")
print(f"Execution decay points: {len(execution_decay['rows'])}")
print(f"Execution planning candidates: {len(execution_planning['rows'])}")
print(f"Execution horizon points: {len(execution_horizon['rows'])}")
print(f"Execution cache-budget rows: {len(execution_budget['rows'])}")
print(f"Validation observations: {validation['observation_count']}")
print(f"Robust policy candidates: {len(consolidation['policies'])}")
print(f"Oracle candidates / seed: {consolidation['oracle']['candidate_count_per_seed']}")
print(f"Imported measurement cases: {imported['case_count']}")
print(f"Profile provenance: {smoke['provenance']['latency_profile_type']}")