Download engine.py from DataDrivenAlex/inference-capacity-planner: direct link, hf CLI and curl.
- Browser
- Download file 2.22 kB
-
https://huggingface.co/spaces/DataDrivenAlex/inference-capacity-planner/resolve/main/engine.py
- Command line
-
hf download hf://spaces/DataDrivenAlex/inference-capacity-planner/engine.py
-
curl -L -o engine.py https://huggingface.co/spaces/DataDrivenAlex/inference-capacity-planner/resolve/main/engine.py
2.22 kB
| """Transparent estimates for an LLM inference capacity planning demo.""" | |
| from dataclasses import dataclass | |
| class Workload: | |
| requests_per_minute: float | |
| input_tokens: int | |
| output_tokens: int | |
| batch_size: int | |
| replicas: int | |
| tokens_per_second_per_replica: float | |
| target_p95_seconds: float = 8.0 | |
| concurrency_per_replica: int = 8 | |
| class Estimate: | |
| tokens_per_second_required: float | |
| tokens_per_second_capacity: float | |
| utilization: float | |
| estimated_queue_wait_seconds: float | |
| estimated_p95_seconds: float | |
| estimated_cost_per_hour: float | |
| meets_slo: bool | |
| notes: tuple[str, ...] | |
| def estimate(w: Workload, hourly_cost_per_replica: float = 1.20) -> Estimate: | |
| """Estimate capacity using explicit steady-state assumptions; not a benchmark.""" | |
| vals=(w.requests_per_minute,w.input_tokens,w.output_tokens,w.batch_size,w.replicas,w.tokens_per_second_per_replica,w.target_p95_seconds,w.concurrency_per_replica,hourly_cost_per_replica) | |
| if any(v < 0 for v in vals) or w.batch_size < 1 or w.replicas < 1 or w.concurrency_per_replica < 1: | |
| raise ValueError("rates and costs must be nonnegative; batch size, replicas and concurrency must be positive") | |
| tokens=w.input_tokens+w.output_tokens; required=w.requests_per_minute*tokens/60; capacity=w.replicas*w.tokens_per_second_per_replica | |
| utilization=required/capacity if capacity else (float("inf") if required else 0.0) | |
| effective=min(.999,utilization/max(1.0,min(w.batch_size,8)**.12)); service=tokens/max(w.tokens_per_second_per_replica,1e-9) | |
| wait=service*effective/max(1e-6,1-effective)/w.replicas; p95=service+wait | |
| notes=["Synthetic steady-state estimate; real serving latency depends on model, hardware, scheduler and traffic shape."] | |
| if utilization>=1: notes.append("Offered token load exceeds nominal aggregate capacity; add replicas or reduce load.") | |
| if w.requests_per_minute>w.replicas*w.concurrency_per_replica*60/max(service,1e-9): notes.append("Concurrency pressure may make the queue estimate optimistic.") | |
| return Estimate(required,capacity,utilization,wait,p95,w.replicas*hourly_cost_per_replica,p95<=w.target_p95_seconds,tuple(notes)) | |