File size: 2,224 Bytes
c75a170
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
"""Transparent estimates for an LLM inference capacity planning demo."""
from dataclasses import dataclass
@dataclass(frozen=True)
class Workload:
    requests_per_minute: float
    input_tokens: int
    output_tokens: int
    batch_size: int
    replicas: int
    tokens_per_second_per_replica: float
    target_p95_seconds: float = 8.0
    concurrency_per_replica: int = 8
@dataclass(frozen=True)
class Estimate:
    tokens_per_second_required: float
    tokens_per_second_capacity: float
    utilization: float
    estimated_queue_wait_seconds: float
    estimated_p95_seconds: float
    estimated_cost_per_hour: float
    meets_slo: bool
    notes: tuple[str, ...]
def estimate(w: Workload, hourly_cost_per_replica: float = 1.20) -> Estimate:
    """Estimate capacity using explicit steady-state assumptions; not a benchmark."""
    vals=(w.requests_per_minute,w.input_tokens,w.output_tokens,w.batch_size,w.replicas,w.tokens_per_second_per_replica,w.target_p95_seconds,w.concurrency_per_replica,hourly_cost_per_replica)
    if any(v < 0 for v in vals) or w.batch_size < 1 or w.replicas < 1 or w.concurrency_per_replica < 1:
        raise ValueError("rates and costs must be nonnegative; batch size, replicas and concurrency must be positive")
    tokens=w.input_tokens+w.output_tokens; required=w.requests_per_minute*tokens/60; capacity=w.replicas*w.tokens_per_second_per_replica
    utilization=required/capacity if capacity else (float("inf") if required else 0.0)
    effective=min(.999,utilization/max(1.0,min(w.batch_size,8)**.12)); service=tokens/max(w.tokens_per_second_per_replica,1e-9)
    wait=service*effective/max(1e-6,1-effective)/w.replicas; p95=service+wait
    notes=["Synthetic steady-state estimate; real serving latency depends on model, hardware, scheduler and traffic shape."]
    if utilization>=1: notes.append("Offered token load exceeds nominal aggregate capacity; add replicas or reduce load.")
    if w.requests_per_minute>w.replicas*w.concurrency_per_replica*60/max(service,1e-9): notes.append("Concurrency pressure may make the queue estimate optimistic.")
    return Estimate(required,capacity,utilization,wait,p95,w.replicas*hourly_cost_per_replica,p95<=w.target_p95_seconds,tuple(notes))