Download dflash/benchmark.py from yaro1214/daim: direct link, hf CLI and curl.
- Browser
- Download file 26.6 kB
-
https://huggingface.co/yaro1214/daim/resolve/main/dflash/benchmark.py
- Command line
-
hf download hf://yaro1214/daim/dflash/benchmark.py
-
curl -L -o benchmark.py https://huggingface.co/yaro1214/daim/resolve/main/dflash/benchmark.py
26.6 kB
| from __future__ import annotations | |
| import argparse | |
| import os | |
| import random | |
| import re | |
| import statistics | |
| import time | |
| import warnings | |
| from concurrent.futures import ThreadPoolExecutor, as_completed | |
| from itertools import chain | |
| from types import SimpleNamespace | |
| from typing import Any, List, Optional | |
| import numpy as np | |
| import requests | |
| import torch | |
| from torch import distributed as torch_dist | |
| from loguru import logger | |
| from rich import print | |
| from tqdm import tqdm | |
| from transformers import AutoModelForCausalLM, AutoTokenizer, DynamicCache | |
| from .model import DFlashDraftModel, sample, load_and_process_dataset, extract_context_feature | |
| def _dist_init() -> None: | |
| if "RANK" not in os.environ: | |
| warnings.warn("RANK not set. Skipping distributed initialization.") | |
| return | |
| torch_dist.init_process_group(backend="nccl", init_method="env://") | |
| def _dist_size() -> int: | |
| return int(os.environ.get("WORLD_SIZE", 1)) | |
| def _dist_rank() -> int: | |
| return int(os.environ.get("RANK", 0)) | |
| def _dist_local_rank() -> int: | |
| return int(os.environ.get("LOCAL_RANK", 0)) | |
| def _dist_is_main() -> bool: | |
| return _dist_rank() == 0 | |
| def _dist_gather(obj: Any, dst: int = 0) -> Optional[List[Any]]: | |
| if not torch_dist.is_initialized(): | |
| return [obj] | |
| if _dist_is_main(): | |
| objs: List[Any] = [None for _ in range(_dist_size())] | |
| torch_dist.gather_object(obj, objs, dst=dst) | |
| return objs | |
| else: | |
| torch_dist.gather_object(obj, dst=dst) | |
| return None | |
| def _cuda_time() -> float: | |
| if torch.cuda.is_available(): | |
| torch.cuda.synchronize() | |
| return time.perf_counter() | |
| def _target_hidden_suffix_embedding( | |
| *, | |
| verified_target_hidden: torch.Tensor, | |
| acceptance_length: int, | |
| embedding_weight: torch.Tensor, | |
| mask_token_id: int, | |
| model: DFlashDraftModel, | |
| bonus_token_id: int | None = None, | |
| first_reject_token_id: int | None = None, | |
| relay_overlap_mode: str = "strict", | |
| ) -> tuple[torch.Tensor, int]: | |
| """Shift verifier hidden by one token and pack the strict reject suffix.""" | |
| batch_size, num_verify_tokens, _ = verified_target_hidden.shape | |
| num_draft_tokens = num_verify_tokens - 1#由于包含anchor,所以需要-1进行调整 | |
| mask_embedding = embedding_weight[mask_token_id] | |
| suffix_embedding = mask_embedding.view(1, 1, -1).expand( | |
| batch_size, num_draft_tokens, -1 | |
| ).clone() | |
| # If d1..da are accepted, d(a+1) is rejected. Its verifier hidden predicts | |
| # d(a+2), the first reusable token. The rejected token itself is never reused. | |
| if relay_overlap_mode not in {"strict", "lookahead"}: | |
| raise ValueError( | |
| "relay_overlap_mode must be 'strict' or 'lookahead'" | |
| ) | |
| tail_offset = 0 if relay_overlap_mode == "lookahead" else 1 | |
| num_reuse = num_draft_tokens - acceptance_length - tail_offset | |
| #num draft为真实draft长度-exact match长度-tail | |
| #如果严格匹配,限制在前一轮draft的position内,strict需要多减1控制预测目标在前一轮draft范围内,主要在于match num draft-1 | |
| #最后一个draft的预测目标是在前一轮draft后一位,所以需要-1丢掉该项 | |
| if num_reuse <= 0: | |
| return suffix_embedding, 0 | |
| source_start = acceptance_length + 1 | |
| raw_hidden = verified_target_hidden[ | |
| :, source_start : source_start + num_reuse, : | |
| ] | |
| #假设接受长度为5,source_start=6,取d6对应的hidden state,实则对d7预测结果,所以从d7开始去取 | |
| adapter_type = getattr(model, "hidden_reuse_adapter_type", "relay_layer_norm") | |
| if adapter_type in {"relay_beta_gamma_mlp", "relay_dynamic_fusion"}: | |
| if bonus_token_id is None or first_reject_token_id is None: | |
| raise ValueError( | |
| "v5/v6 relay requires bonus and first-reject token ids" | |
| ) | |
| bonus_embedding = embedding_weight[bonus_token_id].view(1, 1, -1) | |
| first_reject_embedding = embedding_weight[first_reject_token_id].view( | |
| 1, 1, -1 | |
| ) | |
| bonus_embedding = bonus_embedding.expand( | |
| batch_size, num_reuse, -1 | |
| ) | |
| first_reject_embedding = first_reject_embedding.expand( | |
| batch_size, num_reuse, -1 | |
| ) | |
| if adapter_type == "relay_dynamic_fusion": | |
| projected_hidden, _ = model.project_target_hidden_dynamic( | |
| raw_hidden, | |
| bonus_embedding=bonus_embedding, | |
| first_reject_embedding=first_reject_embedding, | |
| ) | |
| else: | |
| projected_hidden = model.project_target_hidden(raw_hidden)#norm(fc(h)) | |
| delta = model.project_hidden_reuse( | |
| projected_hidden, | |
| bonus_embedding=bonus_embedding, | |
| first_reject_embedding=first_reject_embedding, | |
| ) | |
| else: | |
| projected_hidden = model.project_target_hidden(raw_hidden) | |
| delta = model.project_hidden_reuse(projected_hidden) | |
| suffix_embedding[:, :num_reuse] = mask_embedding + delta | |
| return suffix_embedding, num_reuse | |
| def _dflash_generate( | |
| model: DFlashDraftModel, | |
| target: AutoModelForCausalLM, | |
| input_ids: torch.Tensor, | |
| mask_token_id: int, | |
| max_new_tokens: int, | |
| block_size: int, | |
| stop_token_ids: list[int], | |
| temperature: float = 0.0, | |
| target_hidden_relay: bool = False, | |
| relay_overlap_mode: str = "strict", | |
| ) -> SimpleNamespace: | |
| if target_hidden_relay and not getattr(model, "hidden_reuse_enabled", False): | |
| raise ValueError("target-hidden relay requires a relay checkpoint") | |
| num_input_tokens = input_ids.shape[1] | |
| max_length = num_input_tokens + max_new_tokens | |
| output_ids = torch.full( | |
| (1, max_length + block_size), mask_token_id, dtype=torch.long, device=model.device, | |
| ) | |
| position_ids = torch.arange(output_ids.shape[1], device=model.device).unsqueeze(0) | |
| past_key_values_target = DynamicCache() | |
| past_key_values_draft = DynamicCache() | |
| prefill_start = _cuda_time() | |
| output = target( | |
| input_ids, | |
| position_ids=position_ids[:, :num_input_tokens], | |
| past_key_values=past_key_values_target, | |
| use_cache=True, | |
| logits_to_keep=1,#针对prefill阶段,只返回最后一个位置的logits,前面都是无效的不用返回 | |
| output_hidden_states=True if block_size > 1 else False, | |
| ) | |
| output_ids[:, :num_input_tokens] = input_ids | |
| output_ids[:, num_input_tokens:num_input_tokens+1] = sample(output.logits, temperature) | |
| if block_size > 1: | |
| target_hidden = extract_context_feature(output.hidden_states, model.target_layer_ids) | |
| time_to_first_token = _cuda_time() - prefill_start | |
| decode_start = _cuda_time() | |
| start = input_ids.shape[1] | |
| acceptance_lengths = [] | |
| draft_prefill = True | |
| hidden_suffix_embedding = None #传给下一轮的soft embedding | |
| hidden_reuse_lengths = [] | |
| next_hidden_reuse_tokens = 0 #可复用token数量,num_draft_tokens - acceptance_length -1 | |
| while start < max_length: | |
| block_output_ids = output_ids[:, start : start + block_size].clone() | |
| block_position_ids = position_ids[:, start : start + block_size] | |
| if block_size > 1: | |
| if target_hidden_relay and hidden_suffix_embedding is not None: | |
| #对于relay,anchor embedding + suffix embedding | |
| anchor_embedding = target.model.embed_tokens(block_output_ids[:, :1]) | |
| noise_embedding = torch.cat( | |
| [anchor_embedding, hidden_suffix_embedding], dim=1 | |
| ) | |
| else: | |
| #对于非relay,anchor embedding + mask embedding | |
| noise_embedding = target.model.embed_tokens(block_output_ids) | |
| hidden_reuse_lengths.append(next_hidden_reuse_tokens)#hidden_reuse_lengths用于记录每一轮复用的target hidden长度 | |
| draft_hidden = model( | |
| target_hidden=target_hidden, | |
| noise_embedding=noise_embedding, | |
| position_ids=position_ids[:, past_key_values_draft.get_seq_length(): start + block_size], | |
| past_key_values=past_key_values_draft, | |
| use_cache=True, | |
| is_causal=False, | |
| )[:, -block_size+1:, :] | |
| draft_logits = target.lm_head(draft_hidden) | |
| past_key_values_draft.crop(start) | |
| if getattr(model, "domino_enabled", False): | |
| if getattr(model, "shift_label", False): | |
| raise ValueError( | |
| "Domino + target-hidden relay requires shift_label=false" | |
| ) | |
| prefix_len = min( | |
| int(model.pure_draft_prefix_len), draft_logits.size(1) | |
| ) | |
| if prefix_len: | |
| block_output_ids[:, 1 : 1 + prefix_len] = sample( | |
| draft_logits[:, :prefix_len], temperature | |
| ) | |
| prefix_embeddings = target.model.embed_tokens( | |
| block_output_ids[:, : 1 + prefix_len] | |
| ) | |
| _, gru_state = model.prefix_gru(prefix_embeddings) | |
| for proposal_index in range(prefix_len, draft_logits.size(1)): | |
| correction_input = torch.cat( | |
| [draft_hidden[:, proposal_index], gru_state[-1]], dim=-1 | |
| ) | |
| corrected_logits = ( | |
| draft_logits[:, proposal_index] | |
| + model.embed_proj(correction_input) | |
| ).unsqueeze(1) | |
| next_token = sample(corrected_logits, temperature) | |
| block_output_ids[:, proposal_index + 1] = next_token[:, 0] | |
| next_embedding = target.model.embed_tokens(next_token) | |
| _, gru_state = model.prefix_gru(next_embedding, gru_state) | |
| else: | |
| block_output_ids[:, 1:] = sample(draft_logits, temperature) | |
| if draft_prefill: | |
| draft_prefill = False | |
| decode_start = _cuda_time() | |
| output = target( | |
| block_output_ids, | |
| position_ids=block_position_ids, | |
| past_key_values=past_key_values_target, | |
| use_cache=True, | |
| output_hidden_states=True if block_size > 1 else False, | |
| ) | |
| posterior = sample(output.logits, temperature)#target的verify结果 | |
| acceptance_length = (block_output_ids[:, 1:] == posterior[:, :-1]).cumprod(dim=1).sum(dim=1)[0].item() | |
| #verify对应的position通常会比draft多一位bonus位,由于bonus在确定接受长度后写入,所以posterior丢掉最后一位先逐位确定接受长度 | |
| output_ids[:, start : start + acceptance_length + 1] = block_output_ids[:, : acceptance_length + 1]#verify通过位置进行写入,接受长度为2时,anchor、d1、d2进行写入,所以[:3(acc+1)] | |
| output_ids[:, start + acceptance_length + 1] = posterior[:, acceptance_length]#bonus进行写入,d1、d2、d3,接受长度为2,则取d3对应位置的bonus | |
| acceptance_lengths.append(acceptance_length + 1)#统计指标时+1,算入bonus | |
| start += acceptance_length + 1 | |
| past_key_values_target.crop(start) | |
| if block_size > 1: | |
| #由于target的input是block_output_ids,所以verified_target_hidden涵盖anchor+d1、d2... | |
| verified_target_hidden = extract_context_feature( | |
| output.hidden_states, model.target_layer_ids | |
| ) | |
| target_hidden = verified_target_hidden[:, :acceptance_length + 1, :]#准备prefix的target hidden用于注入,接受长度为2取anchor、d1、d2的hidden(d2ntp得到bonus) | |
| if target_hidden_relay: | |
| reject_index = min(acceptance_length + 1, block_size - 1) | |
| #接受长度为2,d1、d2后的d3(即acc len+1)为reject位置,但是需要clip到bs-1,也就是draft长度,主要是如果全match本质上就没有reject | |
| bonus_token_id = posterior[:, acceptance_length].item()#bonus一定是存在的 | |
| first_reject_token_id = block_output_ids[:, reject_index].item() | |
| hidden_suffix_embedding, next_hidden_reuse_tokens = ( | |
| _target_hidden_suffix_embedding( | |
| verified_target_hidden=verified_target_hidden, | |
| acceptance_length=acceptance_length, | |
| embedding_weight=target.model.embed_tokens.weight, | |
| mask_token_id=mask_token_id, | |
| model=model, | |
| bonus_token_id=bonus_token_id, | |
| first_reject_token_id=first_reject_token_id, | |
| relay_overlap_mode=relay_overlap_mode, | |
| ) | |
| ) | |
| if stop_token_ids is not None and any( | |
| stop_token_id in output_ids[:, num_input_tokens:] for stop_token_id in stop_token_ids | |
| ): | |
| break | |
| output_ids = output_ids[:, :max_length] | |
| output_ids = output_ids[:, output_ids[0] != mask_token_id] | |
| if stop_token_ids is not None: | |
| stop_token_ids = torch.tensor(stop_token_ids, device=output_ids.device) | |
| stop_token_indices = torch.isin(output_ids[0][num_input_tokens:], stop_token_ids).nonzero(as_tuple=True)[0] | |
| if stop_token_indices.numel() > 0: | |
| output_ids = output_ids[:, : num_input_tokens + stop_token_indices[0] + 1] | |
| num_output_tokens = output_ids.shape[1] - num_input_tokens | |
| total_decode_time = _cuda_time() - decode_start | |
| time_per_output_token = total_decode_time / num_output_tokens | |
| return SimpleNamespace( | |
| output_ids=output_ids, | |
| num_input_tokens=num_input_tokens, | |
| num_output_tokens=num_output_tokens, | |
| time_to_first_token=time_to_first_token, | |
| time_per_output_token=time_per_output_token, | |
| acceptance_lengths=acceptance_lengths, | |
| hidden_reuse_lengths=hidden_reuse_lengths, | |
| ) | |
| _TRANSFORMERS_SUPPORTED_PATTERN = re.compile(r"qwen3(?!\.5)[\w-]*|llama.*3\.1.*8b.*instruct", re.IGNORECASE) | |
| def _check_transformers_model(model_name: str) -> None: | |
| if not _TRANSFORMERS_SUPPORTED_PATTERN.search(model_name): | |
| raise ValueError( | |
| f"Transformers backend does not support '{model_name}'. " | |
| f"Only Qwen3 series and LLaMA-3.1-8B-Instruct are supported. " | |
| f"Use --backend sglang or --backend vllm for other models." | |
| ) | |
| def _run_transformers(args: argparse.Namespace) -> None: | |
| _check_transformers_model(args.model) | |
| random.seed(0) | |
| np.random.seed(0) | |
| torch.manual_seed(0) | |
| torch.cuda.manual_seed_all(0) | |
| torch.backends.cudnn.deterministic = True | |
| torch.backends.cudnn.benchmark = False | |
| _dist_init() | |
| torch.cuda.set_device(_dist_local_rank()) | |
| device = torch.device(f"cuda:{_dist_local_rank()}") | |
| def has_flash_attn(): | |
| try: | |
| import flash_attn # noqa: F401 | |
| return True | |
| except ImportError: | |
| logger.warning( | |
| "flash_attn not installed. Falling back to torch.sdpa. Speedup will be lower. " | |
| "For optimal speedup in Transformers backend, please install: " | |
| "pip install flash-attn --no-build-isolation" | |
| ) | |
| return False | |
| installed_flash_attn = has_flash_attn() | |
| attn_impl = "flash_attention_2" if installed_flash_attn else "sdpa" | |
| target = AutoModelForCausalLM.from_pretrained( | |
| args.model, attn_implementation=attn_impl, dtype=torch.bfloat16, | |
| ).to(device).eval() | |
| draft_model = DFlashDraftModel.from_pretrained( | |
| args.draft_model, attn_implementation=attn_impl, dtype=torch.bfloat16, | |
| ).to(device).eval() | |
| checkpoint_overlap_mode = getattr( | |
| draft_model, "hidden_reuse_overlap_mode", "strict" | |
| ) | |
| relay_overlap_mode = ( | |
| checkpoint_overlap_mode | |
| if args.relay_overlap_mode == "checkpoint" | |
| else args.relay_overlap_mode | |
| ) | |
| if args.target_hidden_relay: | |
| if not getattr(draft_model, "hidden_reuse_enabled", False): | |
| raise ValueError("--target-hidden-relay requires a relay checkpoint") | |
| if draft_model.hidden_reuse_adapter_type not in { | |
| "relay_layer_norm", | |
| "relay_beta_gamma_mlp", | |
| "relay_dynamic_fusion", | |
| }: | |
| raise ValueError( | |
| "checkpoint is not a supported target-hidden relay checkpoint" | |
| ) | |
| block_size = args.block_size if args.block_size is not None else draft_model.block_size | |
| tokenizer = AutoTokenizer.from_pretrained(args.model) | |
| if torch_dist.is_initialized(): | |
| if _dist_is_main(): | |
| dataset = load_and_process_dataset( | |
| args.dataset, max_samples=args.max_samples | |
| ) | |
| torch_dist.barrier() | |
| if not _dist_is_main(): | |
| dataset = load_and_process_dataset( | |
| args.dataset, max_samples=args.max_samples | |
| ) | |
| else: | |
| dataset = load_and_process_dataset( | |
| args.dataset, max_samples=args.max_samples | |
| ) | |
| if args.max_samples is not None and len(dataset) > args.max_samples: | |
| random.shuffle(dataset) | |
| dataset = dataset[:args.max_samples] | |
| responses = [] | |
| indices = range(_dist_rank(), len(dataset), _dist_size()) | |
| for idx in tqdm(indices, disable=not _dist_is_main()): | |
| instance = dataset[idx] | |
| messages = [] | |
| for user_content in instance["turns"]: | |
| messages.append({"role": "user", "content": user_content}) | |
| input_text = tokenizer.apply_chat_template( | |
| messages, tokenize=False, add_generation_prompt=True, enable_thinking=False, | |
| ) | |
| input_ids = tokenizer.encode(input_text, return_tensors="pt").to(target.device) | |
| response = {} | |
| for bs in [1, block_size]: | |
| response[bs] = _dflash_generate( | |
| model=draft_model, target=target, input_ids=input_ids, | |
| mask_token_id=draft_model.mask_token_id, | |
| max_new_tokens=args.max_new_tokens, block_size=bs, | |
| stop_token_ids=[tokenizer.eos_token_id], temperature=args.temperature, | |
| target_hidden_relay=(args.target_hidden_relay and bs == block_size), | |
| relay_overlap_mode=relay_overlap_mode, | |
| ) | |
| spec_response = response[block_size] | |
| generated_ids = spec_response.output_ids[0, spec_response.num_input_tokens:] | |
| output_text = tokenizer.decode(generated_ids, skip_special_tokens=True) | |
| messages.append({"role": "assistant", "content": output_text}) | |
| responses.append(response) | |
| if _dist_size() > 1: | |
| responses = _dist_gather(responses, dst=0) | |
| if not _dist_is_main(): | |
| return | |
| responses = list(chain(*responses)) | |
| t1 = np.mean([r[1].time_per_output_token for r in responses]) | |
| tb = np.mean([r[block_size].time_per_output_token for r in responses]) | |
| print(f"Decoding speedup: {t1 / tb:.2f}") | |
| tau = np.mean([np.mean(r[block_size].acceptance_lengths) for r in responses]) | |
| print(f"Average Acceptance length: {tau:.2f}") | |
| acceptance_lengths = list(chain(*[r[block_size].acceptance_lengths for r in responses])) | |
| histogram = [acceptance_lengths.count(b) / len(acceptance_lengths) for b in range(block_size + 1)] | |
| print(f"Acceptance length histogram: {[f'{x * 100:.1f}%' for x in histogram]}") | |
| if args.target_hidden_relay: | |
| #response为dict,{1:基线结果,16:投机结果} | |
| relay_iterations = [ | |
| length | |
| for response in responses | |
| for length in response[block_size].hidden_reuse_lengths[1:]#prefill时hiddense_lengths为0,切片去掉 | |
| ] | |
| relay_rate = ( | |
| np.mean([length > 0 for length in relay_iterations]) | |
| if relay_iterations | |
| else 0.0 | |
| ) | |
| print(f"Target-hidden relay non-mask iteration rate: {relay_rate:.4f}") | |
| def _send_sglang( | |
| base_url: str, text: str, *, max_new_tokens: int, | |
| temperature: float, top_p: float, top_k: int, timeout_s: int, | |
| ) -> dict: | |
| resp = requests.post( | |
| base_url + "/generate", | |
| json={ | |
| "text": text, | |
| "sampling_params": { | |
| "temperature": temperature, "top_p": top_p, | |
| "top_k": top_k, "max_new_tokens": max_new_tokens, | |
| }, | |
| }, | |
| timeout=timeout_s, | |
| ) | |
| resp.raise_for_status() | |
| out = resp.json() | |
| return out if isinstance(out, dict) else out[0] | |
| def _send_vllm( | |
| base_url: str, text: str, *, model: str, max_new_tokens: int, | |
| temperature: float, top_p: float, timeout_s: int, | |
| enable_thinking: bool = False, | |
| ) -> dict: | |
| body: dict = { | |
| "model": model, | |
| "messages": [{"role": "user", "content": text}], | |
| "max_tokens": max_new_tokens, | |
| "temperature": temperature, | |
| "top_p": top_p, | |
| "chat_template_kwargs": {"enable_thinking": enable_thinking}, | |
| } | |
| resp = requests.post( | |
| base_url + "/v1/chat/completions", | |
| json=body, | |
| timeout=timeout_s, | |
| ) | |
| resp.raise_for_status() | |
| return resp.json() | |
| def _run_server(args: argparse.Namespace) -> None: | |
| is_vllm = args.backend == "vllm" | |
| dataset = load_and_process_dataset(args.dataset) | |
| tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True) | |
| num_prompts = args.num_prompts + args.concurrency | |
| prompts: list[str] = [] | |
| for i in range(num_prompts): | |
| item = dataset[i % len(dataset)] | |
| user_content = item["turns"][0] | |
| if is_vllm: | |
| prompts.append(user_content) | |
| else: | |
| prompts.append(tokenizer.apply_chat_template( | |
| [{"role": "user", "content": user_content}], | |
| tokenize=False, add_generation_prompt=True, | |
| enable_thinking=args.enable_thinking, | |
| )) | |
| def send_one(prompt: str) -> dict: | |
| if is_vllm: | |
| return _send_vllm( | |
| args.base_url, prompt, model=args.model, | |
| max_new_tokens=args.max_new_tokens, | |
| temperature=args.temperature, top_p=args.top_p, | |
| timeout_s=args.timeout_s, | |
| enable_thinking=args.enable_thinking, | |
| ) | |
| else: | |
| return _send_sglang( | |
| args.base_url, prompt, | |
| max_new_tokens=args.max_new_tokens, | |
| temperature=args.temperature, top_p=args.top_p, | |
| top_k=args.top_k, timeout_s=args.timeout_s, | |
| ) | |
| if not is_vllm: | |
| try: | |
| requests.get(args.base_url + "/flush_cache", timeout=60).raise_for_status() | |
| except Exception: | |
| print("Warning: /flush_cache failed. Continuing.") | |
| bs = max(args.concurrency, 1) | |
| if len(prompts) > bs: | |
| print(f"[warmup] {bs} requests ...") | |
| with ThreadPoolExecutor(max_workers=bs) as pool: | |
| list(pool.map(send_one, prompts[:bs])) | |
| prompts = prompts[bs:] | |
| print(f"Running benchmark: {args.num_prompts} prompts, concurrency={args.concurrency} ...") | |
| start = time.perf_counter() | |
| total_tokens = 0 | |
| spec_verify_ct_sum = 0 | |
| spec_accept_lengths: list[float] = [] | |
| with ThreadPoolExecutor(max_workers=args.concurrency) as pool: | |
| futures = {pool.submit(send_one, p): i for i, p in enumerate(prompts)} | |
| for fut in tqdm(as_completed(futures), total=len(prompts), desc="Benchmarking"): | |
| out = fut.result() | |
| if is_vllm: | |
| usage = out.get("usage", {}) | |
| total_tokens += int(usage.get("completion_tokens", 0)) | |
| else: | |
| meta = out.get("meta_info", {}) or {} | |
| total_tokens += int(meta.get("completion_tokens", 0)) | |
| spec_verify_ct_sum += int(meta.get("spec_verify_ct", 0)) | |
| if "spec_accept_length" in meta: | |
| try: | |
| spec_accept_lengths.append(float(meta["spec_accept_length"])) | |
| except (TypeError, ValueError): | |
| pass | |
| latency = time.perf_counter() - start | |
| toks_per_s = total_tokens / max(latency, 1e-6) | |
| print(f"\n{'='*50}") | |
| print(f"Backend: {args.backend}") | |
| print(f"Dataset: {args.dataset}") | |
| print(f"Num prompts: {args.num_prompts}") | |
| print(f"Concurrency: {args.concurrency}") | |
| print(f"Latency: {latency:.1f}s") | |
| print(f"Output tokens: {total_tokens}") | |
| print(f"Throughput: {toks_per_s:,.2f} tok/s") | |
| if spec_accept_lengths: | |
| print(f"Accept length: {statistics.mean(spec_accept_lengths):.3f}") | |
| if spec_verify_ct_sum > 0: | |
| print(f"Spec verify ct: {spec_verify_ct_sum}") | |
| print(f"{'='*50}") | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description="DFlash benchmark") | |
| parser.add_argument("--backend", choices=["transformers", "sglang", "vllm"], required=True) | |
| parser.add_argument("--model", type=str, required=True) | |
| parser.add_argument("--dataset", type=str, required=True) | |
| parser.add_argument("--max-new-tokens", type=int, default=2048) | |
| parser.add_argument("--temperature", type=float, default=0.0) | |
| parser.add_argument("--draft-model", type=str, default=None) | |
| parser.add_argument("--target-hidden-relay", action="store_true") | |
| parser.add_argument( | |
| "--relay-overlap-mode", | |
| choices=("checkpoint", "strict", "lookahead"), | |
| default="checkpoint", | |
| help=( | |
| "Relay boundary behavior. checkpoint reads the persisted mode and " | |
| "defaults legacy checkpoints to strict." | |
| ), | |
| ) | |
| parser.add_argument("--block-size", type=int, default=None) | |
| parser.add_argument("--max-samples", type=int, default=None) | |
| parser.add_argument("--base-url", type=str, default="http://127.0.0.1:30000") | |
| parser.add_argument("--num-prompts", type=int, default=1024) | |
| parser.add_argument("--concurrency", type=int, default=1) | |
| parser.add_argument("--top-p", type=float, default=1.0) | |
| parser.add_argument("--top-k", type=int, default=1) | |
| parser.add_argument("--enable-thinking", action="store_true") | |
| parser.add_argument("--timeout-s", type=int, default=3600) | |
| args = parser.parse_args() | |
| if args.backend == "transformers": | |
| if args.draft_model is None: | |
| parser.error("--draft-model is required for transformers backend") | |
| _run_transformers(args) | |
| else: | |
| _run_server(args) | |
| if __name__ == "__main__": | |
| main() | |