File size: 2,224 Bytes
c75a170 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | """Transparent estimates for an LLM inference capacity planning demo."""
from dataclasses import dataclass
@dataclass(frozen=True)
class Workload:
requests_per_minute: float
input_tokens: int
output_tokens: int
batch_size: int
replicas: int
tokens_per_second_per_replica: float
target_p95_seconds: float = 8.0
concurrency_per_replica: int = 8
@dataclass(frozen=True)
class Estimate:
tokens_per_second_required: float
tokens_per_second_capacity: float
utilization: float
estimated_queue_wait_seconds: float
estimated_p95_seconds: float
estimated_cost_per_hour: float
meets_slo: bool
notes: tuple[str, ...]
def estimate(w: Workload, hourly_cost_per_replica: float = 1.20) -> Estimate:
"""Estimate capacity using explicit steady-state assumptions; not a benchmark."""
vals=(w.requests_per_minute,w.input_tokens,w.output_tokens,w.batch_size,w.replicas,w.tokens_per_second_per_replica,w.target_p95_seconds,w.concurrency_per_replica,hourly_cost_per_replica)
if any(v < 0 for v in vals) or w.batch_size < 1 or w.replicas < 1 or w.concurrency_per_replica < 1:
raise ValueError("rates and costs must be nonnegative; batch size, replicas and concurrency must be positive")
tokens=w.input_tokens+w.output_tokens; required=w.requests_per_minute*tokens/60; capacity=w.replicas*w.tokens_per_second_per_replica
utilization=required/capacity if capacity else (float("inf") if required else 0.0)
effective=min(.999,utilization/max(1.0,min(w.batch_size,8)**.12)); service=tokens/max(w.tokens_per_second_per_replica,1e-9)
wait=service*effective/max(1e-6,1-effective)/w.replicas; p95=service+wait
notes=["Synthetic steady-state estimate; real serving latency depends on model, hardware, scheduler and traffic shape."]
if utilization>=1: notes.append("Offered token load exceeds nominal aggregate capacity; add replicas or reduce load.")
if w.requests_per_minute>w.replicas*w.concurrency_per_replica*60/max(service,1e-9): notes.append("Concurrency pressure may make the queue estimate optimistic.")
return Estimate(required,capacity,utilization,wait,p95,w.replicas*hourly_cost_per_replica,p95<=w.target_p95_seconds,tuple(notes))
|