Spaces:
Running
Running
File size: 2,085 Bytes
0c6c82c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | from __future__ import annotations
from .models import AcceleratorProfile, ModelProfile
MODELS: dict[str, ModelProfile] = {
"Llama-3.1-8B": ModelProfile(
name="Llama-3.1-8B",
params_b=8.03,
layers=32,
hidden_size=4096,
attention_heads=32,
kv_heads=8,
),
"Mistral-7B-v0.3": ModelProfile(
name="Mistral-7B-v0.3",
params_b=7.25,
layers=32,
hidden_size=4096,
attention_heads=32,
kv_heads=8,
),
"Qwen2.5-3B": ModelProfile(
name="Qwen2.5-3B",
params_b=3.09,
layers=36,
hidden_size=2048,
attention_heads=16,
kv_heads=2,
),
}
ACCELERATORS: dict[str, AcceleratorProfile] = {
"L4": AcceleratorProfile(
name="NVIDIA L4",
vram_gb=24.0,
peak_tflops_fp16=121.0,
bandwidth_gbps=300.0,
compute_efficiency=0.40,
bandwidth_efficiency=0.72,
),
"A10G": AcceleratorProfile(
name="NVIDIA A10G",
vram_gb=24.0,
peak_tflops_fp16=125.0,
bandwidth_gbps=600.0,
compute_efficiency=0.40,
bandwidth_efficiency=0.70,
),
"A100-40GB": AcceleratorProfile(
name="NVIDIA A100 40GB",
vram_gb=40.0,
peak_tflops_fp16=312.0,
bandwidth_gbps=1555.0,
compute_efficiency=0.47,
bandwidth_efficiency=0.76,
),
}
QUANTIZATION_BYTES = {
"fp16": 2.0,
"int8": 1.0,
"int4": 0.5,
}
# Compute dequantization / packing overheads are deliberately conservative analytical
# modifiers, not empirical benchmark claims.
QUANTIZATION_COMPUTE_MULTIPLIER = {
"fp16": 1.00,
"int8": 1.07,
"int4": 1.16,
}
def get_model(name: str) -> ModelProfile:
try:
return MODELS[name]
except KeyError as exc:
raise ValueError(f"Unknown model profile: {name}") from exc
def get_accelerator(name: str) -> AcceleratorProfile:
try:
return ACCELERATORS[name]
except KeyError as exc:
raise ValueError(f"Unknown accelerator profile: {name}") from exc
|