"""Transparent estimates for an LLM inference capacity planning demo.""" from dataclasses import dataclass @dataclass(frozen=True) class Workload: requests_per_minute: float input_tokens: int output_tokens: int batch_size: int replicas: int tokens_per_second_per_replica: float target_p95_seconds: float = 8.0 concurrency_per_replica: int = 8 @dataclass(frozen=True) class Estimate: tokens_per_second_required: float tokens_per_second_capacity: float utilization: float estimated_queue_wait_seconds: float estimated_p95_seconds: float estimated_cost_per_hour: float meets_slo: bool notes: tuple[str, ...] def estimate(w: Workload, hourly_cost_per_replica: float = 1.20) -> Estimate: """Estimate capacity using explicit steady-state assumptions; not a benchmark.""" vals=(w.requests_per_minute,w.input_tokens,w.output_tokens,w.batch_size,w.replicas,w.tokens_per_second_per_replica,w.target_p95_seconds,w.concurrency_per_replica,hourly_cost_per_replica) if any(v < 0 for v in vals) or w.batch_size < 1 or w.replicas < 1 or w.concurrency_per_replica < 1: raise ValueError("rates and costs must be nonnegative; batch size, replicas and concurrency must be positive") tokens=w.input_tokens+w.output_tokens; required=w.requests_per_minute*tokens/60; capacity=w.replicas*w.tokens_per_second_per_replica utilization=required/capacity if capacity else (float("inf") if required else 0.0) effective=min(.999,utilization/max(1.0,min(w.batch_size,8)**.12)); service=tokens/max(w.tokens_per_second_per_replica,1e-9) wait=service*effective/max(1e-6,1-effective)/w.replicas; p95=service+wait notes=["Synthetic steady-state estimate; real serving latency depends on model, hardware, scheduler and traffic shape."] if utilization>=1: notes.append("Offered token load exceeds nominal aggregate capacity; add replicas or reduce load.") if w.requests_per_minute>w.replicas*w.concurrency_per_replica*60/max(service,1e-9): notes.append("Concurrency pressure may make the queue estimate optimistic.") return Estimate(required,capacity,utilization,wait,p95,w.replicas*hourly_cost_per_replica,p95<=w.target_p95_seconds,tuple(notes))