from __future__ import annotations import argparse import os import random import re import statistics import time import warnings from concurrent.futures import ThreadPoolExecutor, as_completed from itertools import chain from types import SimpleNamespace from typing import Any, List, Optional import numpy as np import requests import torch from torch import distributed as torch_dist from loguru import logger from rich import print from tqdm import tqdm from transformers import AutoModelForCausalLM, AutoTokenizer, DynamicCache from .model import DFlashDraftModel, sample, load_and_process_dataset, extract_context_feature def _dist_init() -> None: if "RANK" not in os.environ: warnings.warn("RANK not set. Skipping distributed initialization.") return torch_dist.init_process_group(backend="nccl", init_method="env://") def _dist_size() -> int: return int(os.environ.get("WORLD_SIZE", 1)) def _dist_rank() -> int: return int(os.environ.get("RANK", 0)) def _dist_local_rank() -> int: return int(os.environ.get("LOCAL_RANK", 0)) def _dist_is_main() -> bool: return _dist_rank() == 0 def _dist_gather(obj: Any, dst: int = 0) -> Optional[List[Any]]: if not torch_dist.is_initialized(): return [obj] if _dist_is_main(): objs: List[Any] = [None for _ in range(_dist_size())] torch_dist.gather_object(obj, objs, dst=dst) return objs else: torch_dist.gather_object(obj, dst=dst) return None def _cuda_time() -> float: if torch.cuda.is_available(): torch.cuda.synchronize() return time.perf_counter() def _target_hidden_suffix_embedding( *, verified_target_hidden: torch.Tensor, acceptance_length: int, embedding_weight: torch.Tensor, mask_token_id: int, model: DFlashDraftModel, bonus_token_id: int | None = None, first_reject_token_id: int | None = None, relay_overlap_mode: str = "strict", ) -> tuple[torch.Tensor, int]: """Shift verifier hidden by one token and pack the strict reject suffix.""" batch_size, num_verify_tokens, _ = verified_target_hidden.shape num_draft_tokens = num_verify_tokens - 1#由于包含anchor,所以需要-1进行调整 mask_embedding = embedding_weight[mask_token_id] suffix_embedding = mask_embedding.view(1, 1, -1).expand( batch_size, num_draft_tokens, -1 ).clone() # If d1..da are accepted, d(a+1) is rejected. Its verifier hidden predicts # d(a+2), the first reusable token. The rejected token itself is never reused. if relay_overlap_mode not in {"strict", "lookahead"}: raise ValueError( "relay_overlap_mode must be 'strict' or 'lookahead'" ) tail_offset = 0 if relay_overlap_mode == "lookahead" else 1 num_reuse = num_draft_tokens - acceptance_length - tail_offset #num draft为真实draft长度-exact match长度-tail #如果严格匹配,限制在前一轮draft的position内,strict需要多减1控制预测目标在前一轮draft范围内,主要在于match num draft-1 #最后一个draft的预测目标是在前一轮draft后一位,所以需要-1丢掉该项 if num_reuse <= 0: return suffix_embedding, 0 source_start = acceptance_length + 1 raw_hidden = verified_target_hidden[ :, source_start : source_start + num_reuse, : ] #假设接受长度为5,source_start=6,取d6对应的hidden state,实则对d7预测结果,所以从d7开始去取 adapter_type = getattr(model, "hidden_reuse_adapter_type", "relay_layer_norm") if adapter_type in {"relay_beta_gamma_mlp", "relay_dynamic_fusion"}: if bonus_token_id is None or first_reject_token_id is None: raise ValueError( "v5/v6 relay requires bonus and first-reject token ids" ) bonus_embedding = embedding_weight[bonus_token_id].view(1, 1, -1) first_reject_embedding = embedding_weight[first_reject_token_id].view( 1, 1, -1 ) bonus_embedding = bonus_embedding.expand( batch_size, num_reuse, -1 ) first_reject_embedding = first_reject_embedding.expand( batch_size, num_reuse, -1 ) if adapter_type == "relay_dynamic_fusion": projected_hidden, _ = model.project_target_hidden_dynamic( raw_hidden, bonus_embedding=bonus_embedding, first_reject_embedding=first_reject_embedding, ) else: projected_hidden = model.project_target_hidden(raw_hidden)#norm(fc(h)) delta = model.project_hidden_reuse( projected_hidden, bonus_embedding=bonus_embedding, first_reject_embedding=first_reject_embedding, ) else: projected_hidden = model.project_target_hidden(raw_hidden) delta = model.project_hidden_reuse(projected_hidden) suffix_embedding[:, :num_reuse] = mask_embedding + delta return suffix_embedding, num_reuse @torch.inference_mode() def _dflash_generate( model: DFlashDraftModel, target: AutoModelForCausalLM, input_ids: torch.Tensor, mask_token_id: int, max_new_tokens: int, block_size: int, stop_token_ids: list[int], temperature: float = 0.0, target_hidden_relay: bool = False, relay_overlap_mode: str = "strict", ) -> SimpleNamespace: if target_hidden_relay and not getattr(model, "hidden_reuse_enabled", False): raise ValueError("target-hidden relay requires a relay checkpoint") num_input_tokens = input_ids.shape[1] max_length = num_input_tokens + max_new_tokens output_ids = torch.full( (1, max_length + block_size), mask_token_id, dtype=torch.long, device=model.device, ) position_ids = torch.arange(output_ids.shape[1], device=model.device).unsqueeze(0) past_key_values_target = DynamicCache() past_key_values_draft = DynamicCache() prefill_start = _cuda_time() output = target( input_ids, position_ids=position_ids[:, :num_input_tokens], past_key_values=past_key_values_target, use_cache=True, logits_to_keep=1,#针对prefill阶段,只返回最后一个位置的logits,前面都是无效的不用返回 output_hidden_states=True if block_size > 1 else False, ) output_ids[:, :num_input_tokens] = input_ids output_ids[:, num_input_tokens:num_input_tokens+1] = sample(output.logits, temperature) if block_size > 1: target_hidden = extract_context_feature(output.hidden_states, model.target_layer_ids) time_to_first_token = _cuda_time() - prefill_start decode_start = _cuda_time() start = input_ids.shape[1] acceptance_lengths = [] draft_prefill = True hidden_suffix_embedding = None #传给下一轮的soft embedding hidden_reuse_lengths = [] next_hidden_reuse_tokens = 0 #可复用token数量,num_draft_tokens - acceptance_length -1 while start < max_length: block_output_ids = output_ids[:, start : start + block_size].clone() block_position_ids = position_ids[:, start : start + block_size] if block_size > 1: if target_hidden_relay and hidden_suffix_embedding is not None: #对于relay,anchor embedding + suffix embedding anchor_embedding = target.model.embed_tokens(block_output_ids[:, :1]) noise_embedding = torch.cat( [anchor_embedding, hidden_suffix_embedding], dim=1 ) else: #对于非relay,anchor embedding + mask embedding noise_embedding = target.model.embed_tokens(block_output_ids) hidden_reuse_lengths.append(next_hidden_reuse_tokens)#hidden_reuse_lengths用于记录每一轮复用的target hidden长度 draft_hidden = model( target_hidden=target_hidden, noise_embedding=noise_embedding, position_ids=position_ids[:, past_key_values_draft.get_seq_length(): start + block_size], past_key_values=past_key_values_draft, use_cache=True, is_causal=False, )[:, -block_size+1:, :] draft_logits = target.lm_head(draft_hidden) past_key_values_draft.crop(start) if getattr(model, "domino_enabled", False): if getattr(model, "shift_label", False): raise ValueError( "Domino + target-hidden relay requires shift_label=false" ) prefix_len = min( int(model.pure_draft_prefix_len), draft_logits.size(1) ) if prefix_len: block_output_ids[:, 1 : 1 + prefix_len] = sample( draft_logits[:, :prefix_len], temperature ) prefix_embeddings = target.model.embed_tokens( block_output_ids[:, : 1 + prefix_len] ) _, gru_state = model.prefix_gru(prefix_embeddings) for proposal_index in range(prefix_len, draft_logits.size(1)): correction_input = torch.cat( [draft_hidden[:, proposal_index], gru_state[-1]], dim=-1 ) corrected_logits = ( draft_logits[:, proposal_index] + model.embed_proj(correction_input) ).unsqueeze(1) next_token = sample(corrected_logits, temperature) block_output_ids[:, proposal_index + 1] = next_token[:, 0] next_embedding = target.model.embed_tokens(next_token) _, gru_state = model.prefix_gru(next_embedding, gru_state) else: block_output_ids[:, 1:] = sample(draft_logits, temperature) if draft_prefill: draft_prefill = False decode_start = _cuda_time() output = target( block_output_ids, position_ids=block_position_ids, past_key_values=past_key_values_target, use_cache=True, output_hidden_states=True if block_size > 1 else False, ) posterior = sample(output.logits, temperature)#target的verify结果 acceptance_length = (block_output_ids[:, 1:] == posterior[:, :-1]).cumprod(dim=1).sum(dim=1)[0].item() #verify对应的position通常会比draft多一位bonus位,由于bonus在确定接受长度后写入,所以posterior丢掉最后一位先逐位确定接受长度 output_ids[:, start : start + acceptance_length + 1] = block_output_ids[:, : acceptance_length + 1]#verify通过位置进行写入,接受长度为2时,anchor、d1、d2进行写入,所以[:3(acc+1)] output_ids[:, start + acceptance_length + 1] = posterior[:, acceptance_length]#bonus进行写入,d1、d2、d3,接受长度为2,则取d3对应位置的bonus acceptance_lengths.append(acceptance_length + 1)#统计指标时+1,算入bonus start += acceptance_length + 1 past_key_values_target.crop(start) if block_size > 1: #由于target的input是block_output_ids,所以verified_target_hidden涵盖anchor+d1、d2... verified_target_hidden = extract_context_feature( output.hidden_states, model.target_layer_ids ) target_hidden = verified_target_hidden[:, :acceptance_length + 1, :]#准备prefix的target hidden用于注入,接受长度为2取anchor、d1、d2的hidden(d2ntp得到bonus) if target_hidden_relay: reject_index = min(acceptance_length + 1, block_size - 1) #接受长度为2,d1、d2后的d3(即acc len+1)为reject位置,但是需要clip到bs-1,也就是draft长度,主要是如果全match本质上就没有reject bonus_token_id = posterior[:, acceptance_length].item()#bonus一定是存在的 first_reject_token_id = block_output_ids[:, reject_index].item() hidden_suffix_embedding, next_hidden_reuse_tokens = ( _target_hidden_suffix_embedding( verified_target_hidden=verified_target_hidden, acceptance_length=acceptance_length, embedding_weight=target.model.embed_tokens.weight, mask_token_id=mask_token_id, model=model, bonus_token_id=bonus_token_id, first_reject_token_id=first_reject_token_id, relay_overlap_mode=relay_overlap_mode, ) ) if stop_token_ids is not None and any( stop_token_id in output_ids[:, num_input_tokens:] for stop_token_id in stop_token_ids ): break output_ids = output_ids[:, :max_length] output_ids = output_ids[:, output_ids[0] != mask_token_id] if stop_token_ids is not None: stop_token_ids = torch.tensor(stop_token_ids, device=output_ids.device) stop_token_indices = torch.isin(output_ids[0][num_input_tokens:], stop_token_ids).nonzero(as_tuple=True)[0] if stop_token_indices.numel() > 0: output_ids = output_ids[:, : num_input_tokens + stop_token_indices[0] + 1] num_output_tokens = output_ids.shape[1] - num_input_tokens total_decode_time = _cuda_time() - decode_start time_per_output_token = total_decode_time / num_output_tokens return SimpleNamespace( output_ids=output_ids, num_input_tokens=num_input_tokens, num_output_tokens=num_output_tokens, time_to_first_token=time_to_first_token, time_per_output_token=time_per_output_token, acceptance_lengths=acceptance_lengths, hidden_reuse_lengths=hidden_reuse_lengths, ) _TRANSFORMERS_SUPPORTED_PATTERN = re.compile(r"qwen3(?!\.5)[\w-]*|llama.*3\.1.*8b.*instruct", re.IGNORECASE) def _check_transformers_model(model_name: str) -> None: if not _TRANSFORMERS_SUPPORTED_PATTERN.search(model_name): raise ValueError( f"Transformers backend does not support '{model_name}'. " f"Only Qwen3 series and LLaMA-3.1-8B-Instruct are supported. " f"Use --backend sglang or --backend vllm for other models." ) def _run_transformers(args: argparse.Namespace) -> None: _check_transformers_model(args.model) random.seed(0) np.random.seed(0) torch.manual_seed(0) torch.cuda.manual_seed_all(0) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False _dist_init() torch.cuda.set_device(_dist_local_rank()) device = torch.device(f"cuda:{_dist_local_rank()}") def has_flash_attn(): try: import flash_attn # noqa: F401 return True except ImportError: logger.warning( "flash_attn not installed. Falling back to torch.sdpa. Speedup will be lower. " "For optimal speedup in Transformers backend, please install: " "pip install flash-attn --no-build-isolation" ) return False installed_flash_attn = has_flash_attn() attn_impl = "flash_attention_2" if installed_flash_attn else "sdpa" target = AutoModelForCausalLM.from_pretrained( args.model, attn_implementation=attn_impl, dtype=torch.bfloat16, ).to(device).eval() draft_model = DFlashDraftModel.from_pretrained( args.draft_model, attn_implementation=attn_impl, dtype=torch.bfloat16, ).to(device).eval() checkpoint_overlap_mode = getattr( draft_model, "hidden_reuse_overlap_mode", "strict" ) relay_overlap_mode = ( checkpoint_overlap_mode if args.relay_overlap_mode == "checkpoint" else args.relay_overlap_mode ) if args.target_hidden_relay: if not getattr(draft_model, "hidden_reuse_enabled", False): raise ValueError("--target-hidden-relay requires a relay checkpoint") if draft_model.hidden_reuse_adapter_type not in { "relay_layer_norm", "relay_beta_gamma_mlp", "relay_dynamic_fusion", }: raise ValueError( "checkpoint is not a supported target-hidden relay checkpoint" ) block_size = args.block_size if args.block_size is not None else draft_model.block_size tokenizer = AutoTokenizer.from_pretrained(args.model) if torch_dist.is_initialized(): if _dist_is_main(): dataset = load_and_process_dataset( args.dataset, max_samples=args.max_samples ) torch_dist.barrier() if not _dist_is_main(): dataset = load_and_process_dataset( args.dataset, max_samples=args.max_samples ) else: dataset = load_and_process_dataset( args.dataset, max_samples=args.max_samples ) if args.max_samples is not None and len(dataset) > args.max_samples: random.shuffle(dataset) dataset = dataset[:args.max_samples] responses = [] indices = range(_dist_rank(), len(dataset), _dist_size()) for idx in tqdm(indices, disable=not _dist_is_main()): instance = dataset[idx] messages = [] for user_content in instance["turns"]: messages.append({"role": "user", "content": user_content}) input_text = tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True, enable_thinking=False, ) input_ids = tokenizer.encode(input_text, return_tensors="pt").to(target.device) response = {} for bs in [1, block_size]: response[bs] = _dflash_generate( model=draft_model, target=target, input_ids=input_ids, mask_token_id=draft_model.mask_token_id, max_new_tokens=args.max_new_tokens, block_size=bs, stop_token_ids=[tokenizer.eos_token_id], temperature=args.temperature, target_hidden_relay=(args.target_hidden_relay and bs == block_size), relay_overlap_mode=relay_overlap_mode, ) spec_response = response[block_size] generated_ids = spec_response.output_ids[0, spec_response.num_input_tokens:] output_text = tokenizer.decode(generated_ids, skip_special_tokens=True) messages.append({"role": "assistant", "content": output_text}) responses.append(response) if _dist_size() > 1: responses = _dist_gather(responses, dst=0) if not _dist_is_main(): return responses = list(chain(*responses)) t1 = np.mean([r[1].time_per_output_token for r in responses]) tb = np.mean([r[block_size].time_per_output_token for r in responses]) print(f"Decoding speedup: {t1 / tb:.2f}") tau = np.mean([np.mean(r[block_size].acceptance_lengths) for r in responses]) print(f"Average Acceptance length: {tau:.2f}") acceptance_lengths = list(chain(*[r[block_size].acceptance_lengths for r in responses])) histogram = [acceptance_lengths.count(b) / len(acceptance_lengths) for b in range(block_size + 1)] print(f"Acceptance length histogram: {[f'{x * 100:.1f}%' for x in histogram]}") if args.target_hidden_relay: #response为dict,{1:基线结果,16:投机结果} relay_iterations = [ length for response in responses for length in response[block_size].hidden_reuse_lengths[1:]#prefill时hiddense_lengths为0,切片去掉 ] relay_rate = ( np.mean([length > 0 for length in relay_iterations]) if relay_iterations else 0.0 ) print(f"Target-hidden relay non-mask iteration rate: {relay_rate:.4f}") def _send_sglang( base_url: str, text: str, *, max_new_tokens: int, temperature: float, top_p: float, top_k: int, timeout_s: int, ) -> dict: resp = requests.post( base_url + "/generate", json={ "text": text, "sampling_params": { "temperature": temperature, "top_p": top_p, "top_k": top_k, "max_new_tokens": max_new_tokens, }, }, timeout=timeout_s, ) resp.raise_for_status() out = resp.json() return out if isinstance(out, dict) else out[0] def _send_vllm( base_url: str, text: str, *, model: str, max_new_tokens: int, temperature: float, top_p: float, timeout_s: int, enable_thinking: bool = False, ) -> dict: body: dict = { "model": model, "messages": [{"role": "user", "content": text}], "max_tokens": max_new_tokens, "temperature": temperature, "top_p": top_p, "chat_template_kwargs": {"enable_thinking": enable_thinking}, } resp = requests.post( base_url + "/v1/chat/completions", json=body, timeout=timeout_s, ) resp.raise_for_status() return resp.json() def _run_server(args: argparse.Namespace) -> None: is_vllm = args.backend == "vllm" dataset = load_and_process_dataset(args.dataset) tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True) num_prompts = args.num_prompts + args.concurrency prompts: list[str] = [] for i in range(num_prompts): item = dataset[i % len(dataset)] user_content = item["turns"][0] if is_vllm: prompts.append(user_content) else: prompts.append(tokenizer.apply_chat_template( [{"role": "user", "content": user_content}], tokenize=False, add_generation_prompt=True, enable_thinking=args.enable_thinking, )) def send_one(prompt: str) -> dict: if is_vllm: return _send_vllm( args.base_url, prompt, model=args.model, max_new_tokens=args.max_new_tokens, temperature=args.temperature, top_p=args.top_p, timeout_s=args.timeout_s, enable_thinking=args.enable_thinking, ) else: return _send_sglang( args.base_url, prompt, max_new_tokens=args.max_new_tokens, temperature=args.temperature, top_p=args.top_p, top_k=args.top_k, timeout_s=args.timeout_s, ) if not is_vllm: try: requests.get(args.base_url + "/flush_cache", timeout=60).raise_for_status() except Exception: print("Warning: /flush_cache failed. Continuing.") bs = max(args.concurrency, 1) if len(prompts) > bs: print(f"[warmup] {bs} requests ...") with ThreadPoolExecutor(max_workers=bs) as pool: list(pool.map(send_one, prompts[:bs])) prompts = prompts[bs:] print(f"Running benchmark: {args.num_prompts} prompts, concurrency={args.concurrency} ...") start = time.perf_counter() total_tokens = 0 spec_verify_ct_sum = 0 spec_accept_lengths: list[float] = [] with ThreadPoolExecutor(max_workers=args.concurrency) as pool: futures = {pool.submit(send_one, p): i for i, p in enumerate(prompts)} for fut in tqdm(as_completed(futures), total=len(prompts), desc="Benchmarking"): out = fut.result() if is_vllm: usage = out.get("usage", {}) total_tokens += int(usage.get("completion_tokens", 0)) else: meta = out.get("meta_info", {}) or {} total_tokens += int(meta.get("completion_tokens", 0)) spec_verify_ct_sum += int(meta.get("spec_verify_ct", 0)) if "spec_accept_length" in meta: try: spec_accept_lengths.append(float(meta["spec_accept_length"])) except (TypeError, ValueError): pass latency = time.perf_counter() - start toks_per_s = total_tokens / max(latency, 1e-6) print(f"\n{'='*50}") print(f"Backend: {args.backend}") print(f"Dataset: {args.dataset}") print(f"Num prompts: {args.num_prompts}") print(f"Concurrency: {args.concurrency}") print(f"Latency: {latency:.1f}s") print(f"Output tokens: {total_tokens}") print(f"Throughput: {toks_per_s:,.2f} tok/s") if spec_accept_lengths: print(f"Accept length: {statistics.mean(spec_accept_lengths):.3f}") if spec_verify_ct_sum > 0: print(f"Spec verify ct: {spec_verify_ct_sum}") print(f"{'='*50}") def main() -> None: parser = argparse.ArgumentParser(description="DFlash benchmark") parser.add_argument("--backend", choices=["transformers", "sglang", "vllm"], required=True) parser.add_argument("--model", type=str, required=True) parser.add_argument("--dataset", type=str, required=True) parser.add_argument("--max-new-tokens", type=int, default=2048) parser.add_argument("--temperature", type=float, default=0.0) parser.add_argument("--draft-model", type=str, default=None) parser.add_argument("--target-hidden-relay", action="store_true") parser.add_argument( "--relay-overlap-mode", choices=("checkpoint", "strict", "lookahead"), default="checkpoint", help=( "Relay boundary behavior. checkpoint reads the persisted mode and " "defaults legacy checkpoints to strict." ), ) parser.add_argument("--block-size", type=int, default=None) parser.add_argument("--max-samples", type=int, default=None) parser.add_argument("--base-url", type=str, default="http://127.0.0.1:30000") parser.add_argument("--num-prompts", type=int, default=1024) parser.add_argument("--concurrency", type=int, default=1) parser.add_argument("--top-p", type=float, default=1.0) parser.add_argument("--top-k", type=int, default=1) parser.add_argument("--enable-thinking", action="store_true") parser.add_argument("--timeout-s", type=int, default=3600) args = parser.parse_args() if args.backend == "transformers": if args.draft_model is None: parser.error("--draft-model is required for transformers backend") _run_transformers(args) else: _run_server(args) if __name__ == "__main__": main()