import json import os import torch from itertools import islice from pathlib import Path from typing import Optional, Callable from typing_extensions import Unpack, Tuple from torch import nn from transformers.models.qwen3.modeling_qwen3 import ( Qwen3RMSNorm, Qwen3RotaryEmbedding, Qwen3Config, Qwen3PreTrainedModel, Qwen3MLP, GradientCheckpointingLayer, FlashAttentionKwargs, rotate_half, eager_attention_forward, ALL_ATTENTION_FUNCTIONS, ) from transformers import DynamicCache from transformers.modeling_outputs import CausalLMOutputWithPast from transformers.cache_utils import Cache # --------------------------------------------------------------------------- # Dataset loading (auto-downloads from HuggingFace on first use) # --------------------------------------------------------------------------- CACHE_DIR = Path(__file__).parent.parent / "cache" def _format_livecodebench(row: dict) -> str: prompt = ( "You are an expert Python programmer. You will be given a question " "(problem specification) and will generate a correct Python program " "that matches the specification and passes all tests. You will NOT " "return anything except for the program\n\n" f"### Question:\n{row['question_content']}\n\n" ) if row.get("starter_code"): prompt += ( "### Format: Use the following code structure:\n" f"```python\n{row['starter_code']}\n```\n\n" ) else: prompt += ( "### Format: Write your code in the following format:\n" "```python\n# YOUR CODE HERE\n```\n\n" ) return prompt + "### Answer: (use the provided format with backticks)" _LCB_BASE = ( "https://huggingface.co/datasets/livecodebench/" "code_generation_lite/resolve/main/" ) DATASETS = { "gsm8k": { "load_args": ("openai/gsm8k", "main"), "load_kwargs": {"split": "test"}, "format": lambda x: "{question}\nPlease reason step by step, and put your final answer within \\boxed{{}}.".format(**x), }, "math500": { "load_args": ("HuggingFaceH4/MATH-500",), "load_kwargs": {"split": "test"}, "format": lambda x: "{problem}\nPlease reason step by step, and put your final answer within \\boxed{{}}.".format(**x), }, "aime24": { "load_args": ("HuggingFaceH4/aime_2024",), "load_kwargs": {"split": "train"}, "format": lambda x: "{problem}\nPlease reason step by step, and put your final answer within \\boxed{{}}.".format(**x), }, "aime25": { "load_args": ("math-ai/aime25",), "load_kwargs": {"split": "test"}, "format": lambda x: "{problem}\nPlease reason step by step, and put your final answer within \\boxed{{}}.".format(**x), }, "humaneval": { "load_args": ("openai/openai_humaneval",), "load_kwargs": {"split": "test"}, "format": lambda x: "Write a solution to the following problem and make sure that it passes the tests:\n```python\n{prompt}\n```".format(**x), }, "mbpp": { "load_args": ("google-research-datasets/mbpp", "sanitized"), "load_kwargs": {"split": "test"}, "format": lambda x: x["prompt"], }, "livecodebench": { "load_args": ("json",), "load_kwargs": { "data_files": { "test": [ _LCB_BASE + f"test{suffix}.jsonl" for suffix in ("", "2", "3", "4", "5", "6") ] }, "split": "test", }, "format": _format_livecodebench, }, "swe-bench": { "load_args": ("princeton-nlp/SWE-bench_Lite",), "load_kwargs": {"split": "test"}, "format": lambda x: ( f"Problem Statement:\n{x['problem_statement']}\n" "Please fix the issue described above." ), }, "mt-bench": { "load_args": ("HuggingFaceH4/mt_bench_prompts",), "load_kwargs": {"split": "train"}, "format": lambda x: x["prompt"], # list of turns "multi_turn": True, }, "alpaca": { "load_args": ("tatsu-lab/alpaca",), "load_kwargs": {"split": "train"}, "format": lambda x: ( f"{x['instruction']}\n\nInput:\n{x['input']}" if x["input"] else x["instruction"] ), }, } def _dataset_cache_path(name: str, max_samples: Optional[int] = None) -> Path: if name == "livecodebench" and max_samples is not None: return CACHE_DIR / f"{name}.{max_samples}.jsonl" return CACHE_DIR / f"{name}.jsonl" def _prepare_dataset( name: str, out_path: Path, max_samples: Optional[int] = None, ) -> Path: from datasets import load_dataset cfg = DATASETS[name] CACHE_DIR.mkdir(exist_ok=True) print(f"[download] {name} ...") load_kwargs = dict(cfg["load_kwargs"]) if name == "livecodebench" and max_samples is not None: # The raw release is several GB because it includes private tests, while # generation only needs the prompt fields. Stream just the requested # prompt count instead of downloading every raw JSONL on every rank. load_kwargs["streaming"] = True dataset = load_dataset(*cfg["load_args"], **load_kwargs) rows = ( islice(dataset, max_samples) if name == "livecodebench" and max_samples is not None else dataset ) tmp_path = out_path.with_name(f".{out_path.name}.tmp.{os.getpid()}") count = 0 try: with open(tmp_path, "w") as f: for row in rows: if cfg.get("multi_turn"): turns = cfg["format"](row) else: turns = [cfg["format"](row)] f.write(json.dumps({"turns": turns}) + "\n") count += 1 os.replace(tmp_path, out_path) finally: tmp_path.unlink(missing_ok=True) print(f"[cached] {out_path} ({count} samples)") return out_path def load_and_process_dataset( data_name: str, max_samples: Optional[int] = None, ) -> list[dict]: if data_name not in DATASETS: raise ValueError(f"Unknown dataset '{data_name}'. Available: {list(DATASETS.keys())}") path = _dataset_cache_path(data_name, max_samples) if not path.exists(): CACHE_DIR.mkdir(exist_ok=True) _prepare_dataset(data_name, path, max_samples) with open(path) as f: return [json.loads(line) for line in f] # --------------------------------------------------------------------------- # Model utilities # --------------------------------------------------------------------------- def build_target_layer_ids(num_target_layers: int, num_draft_layers: int): if num_draft_layers == 1: return [(num_target_layers // 2)] start = 1 end = num_target_layers - 3 span = end - start return [ int(round(start + (i * span) / (num_draft_layers - 1))) for i in range(num_draft_layers) ] def extract_context_feature( hidden_states: list[torch.Tensor], layer_ids: Optional[list[int]], ) -> torch.Tensor: offset = 1 selected_states = [hidden_states[layer_id + offset] for layer_id in layer_ids] return torch.cat(selected_states, dim=-1) def sample(logits: torch.Tensor, temperature: float = 0.0) -> torch.Tensor: if temperature < 1e-5: return torch.argmax(logits, dim=-1) bsz, seq_len, vocab_size = logits.shape logits = logits.view(-1, vocab_size) / temperature probs = torch.softmax(logits, dim=-1) return torch.multinomial(probs, num_samples=1).view(bsz, seq_len) # --------------------------------------------------------------------------- # DFlash model # --------------------------------------------------------------------------- def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): cos = cos.unsqueeze(unsqueeze_dim) sin = sin.unsqueeze(unsqueeze_dim) q_len = q.size(-2) q_embed = (q * cos[..., -q_len:, :]) + (rotate_half(q) * sin[..., -q_len:, :]) k_embed = (k * cos) + (rotate_half(k) * sin) return q_embed, k_embed class Qwen3DFlashAttention(nn.Module): def __init__(self, config: Qwen3Config, layer_idx: int): super().__init__() self.config = config self.layer_idx = layer_idx self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads self.scaling = self.head_dim**-0.5 self.attention_dropout = config.attention_dropout self.is_causal = False self.q_proj = nn.Linear( config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias ) self.k_proj = nn.Linear( config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias ) self.v_proj = nn.Linear( config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias ) self.o_proj = nn.Linear( config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias ) self.q_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps) self.k_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps) self.sliding_window = config.sliding_window if config.layer_types[layer_idx] == "sliding_attention" else None def forward( self, hidden_states: torch.Tensor, target_hidden: torch.Tensor, position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: Optional[torch.Tensor], past_key_values: Optional[Cache] = None, cache_position: Optional[torch.LongTensor] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: bsz, q_len = hidden_states.shape[:-1] ctx_len = target_hidden.shape[1] q = self.q_proj(hidden_states) q = q.view(bsz, q_len, -1, self.head_dim) q = self.q_norm(q).transpose(1, 2) k_ctx = self.k_proj(target_hidden) k_noise = self.k_proj(hidden_states) v_ctx = self.v_proj(target_hidden) v_noise = self.v_proj(hidden_states) k = torch.cat([k_ctx, k_noise], dim=1).view(bsz, ctx_len + q_len, -1, self.head_dim) v = torch.cat([v_ctx, v_noise], dim=1).view(bsz, ctx_len + q_len, -1, self.head_dim) k = self.k_norm(k).transpose(1, 2) v = v.transpose(1, 2) cos, sin = position_embeddings q, k = apply_rotary_pos_emb(q, k, cos, sin) if past_key_values is not None: cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} k, v = past_key_values.update(k, v, self.layer_idx, cache_kwargs) attn_fn: Callable = eager_attention_forward if self.config._attn_implementation != "eager": attn_fn = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] attn_output, attn_weights = attn_fn( self, q, k, v, attention_mask, dropout=0.0 if not self.training else self.attention_dropout, scaling=self.scaling, sliding_window=self.sliding_window, **kwargs, ) attn_output = attn_output.reshape(bsz, q_len, -1) attn_output = self.o_proj(attn_output) return attn_output, attn_weights class Qwen3DFlashDecoderLayer(GradientCheckpointingLayer): def __init__(self, config: Qwen3Config, layer_idx: int): super().__init__() self.hidden_size = config.hidden_size self.self_attn = Qwen3DFlashAttention(config=config, layer_idx=layer_idx) self.mlp = Qwen3MLP(config) self.input_layernorm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) def forward( self, target_hidden: Optional[torch.Tensor] = None, hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.LongTensor] = None, past_key_value: Optional[Cache] = None, output_attentions: Optional[bool] = False, use_cache: Optional[bool] = False, cache_position: Optional[torch.LongTensor] = None, position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: residual = hidden_states hidden_states = self.input_layernorm(hidden_states) hidden_states = self.self_attn( hidden_states=hidden_states, target_hidden=target_hidden, attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_value, output_attentions=output_attentions, use_cache=use_cache, cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, )[0] hidden_states = residual + hidden_states residual = hidden_states hidden_states = self.post_attention_layernorm(hidden_states) hidden_states = self.mlp(hidden_states) hidden_states = residual + hidden_states return hidden_states class DFlashDraftModel(Qwen3PreTrainedModel): config_class = Qwen3Config _no_split_modules = ["Qwen3DFlashDecoderLayer"] def __init__(self, config) -> None: super().__init__(config) self.config = config self.layers = nn.ModuleList( [Qwen3DFlashDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] ) self.target_layer_ids = self.config.dflash_config.get( "target_layer_ids", build_target_layer_ids(config.num_target_layers, config.num_hidden_layers) ) self.norm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.rotary_emb = Qwen3RotaryEmbedding(config) self.fc = nn.Linear(len(self.target_layer_ids) * config.hidden_size, config.hidden_size, bias=False) self.hidden_norm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.block_size = config.block_size self.mask_token_id = self.config.dflash_config.get("mask_token_id", None) self.projector_type = self.config.dflash_config.get("projector_type", "dflash") if self.projector_type == "domino": self.emb_dim = int(self.config.dflash_config["emb_dim"]) self.gru_hidden_dim = int(self.config.dflash_config["gru_hidden_dim"]) self.pure_draft_prefix_len = int( self.config.dflash_config.get("pure_draft_prefix_len", 0) ) self.shift_label = bool(self.config.dflash_config.get("shift_label", False)) if self.shift_label: raise ValueError( "Domino + target-hidden relay inference requires shift_label=false" ) self.prefix_gru = nn.GRU( input_size=config.hidden_size, hidden_size=self.gru_hidden_dim, num_layers=1, batch_first=True, bias=False, ) self.embed_proj = nn.Sequential( nn.Linear(config.hidden_size + self.gru_hidden_dim, self.emb_dim, bias=False), nn.SiLU(), nn.Linear(self.emb_dim, config.vocab_size, bias=False), ) self.hidden_reuse_enabled = bool( self.config.dflash_config.get("hidden_reuse_enabled", False) ) self.hidden_reuse_adapter_type = self.config.dflash_config.get( "hidden_reuse_adapter_type", "relay_layer_norm" ) self.hidden_reuse_overlap_mode = self.config.dflash_config.get( "hidden_reuse_overlap_mode", "strict" ) if self.hidden_reuse_overlap_mode not in {"strict", "lookahead"}: raise ValueError( "unsupported hidden_reuse_overlap_mode: " f"{self.hidden_reuse_overlap_mode!r}" ) if self.hidden_reuse_enabled: if self.hidden_reuse_adapter_type not in { "relay_layer_norm", "relay_beta_gamma_mlp", "relay_dynamic_fusion", }: raise ValueError( "unsupported hidden_reuse_adapter_type: " f"{self.hidden_reuse_adapter_type!r}" ) if self.hidden_reuse_adapter_type == "relay_layer_norm": self.hidden_reuse_relay_norm = nn.LayerNorm( config.hidden_size, eps=1e-5, elementwise_affine=True, ) else: self.hidden_reuse_relay_norm = nn.LayerNorm( config.hidden_size, eps=1e-5, elementwise_affine=False, ) self.hidden_reuse_conditioner = nn.Sequential( nn.Linear(3 * config.hidden_size, config.hidden_size), nn.SiLU(), nn.Linear(config.hidden_size, 2 * config.hidden_size), ) if self.hidden_reuse_adapter_type == "relay_dynamic_fusion": query_dim = int( self.config.dflash_config.get( "hidden_reuse_query_dim", 128 ) ) if query_dim <= 0: raise ValueError("hidden_reuse_query_dim must be positive") self.hidden_reuse_query_dim = query_dim self.hidden_reuse_layer_norm = Qwen3RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) self.hidden_reuse_query_norm = nn.LayerNorm( 2 * config.hidden_size, eps=1e-5, elementwise_affine=False, ) self.hidden_reuse_query_proj = nn.Linear( 2 * config.hidden_size, query_dim, bias=False ) self.hidden_reuse_key_proj = nn.ModuleList( [ nn.Linear(config.hidden_size, query_dim, bias=False) for _ in self.target_layer_ids ] ) self.post_init() if self.hidden_reuse_enabled: if self.hidden_reuse_relay_norm.elementwise_affine: nn.init.zeros_(self.hidden_reuse_relay_norm.weight) nn.init.zeros_(self.hidden_reuse_relay_norm.bias) else: nn.init.zeros_(self.hidden_reuse_conditioner[-1].weight) nn.init.zeros_(self.hidden_reuse_conditioner[-1].bias) @property def domino_enabled(self) -> bool: return self.projector_type == "domino" def _init_weights(self, module: nn.Module) -> None: super()._init_weights(module) if module is getattr(self, "hidden_reuse_relay_norm", None): if module.elementwise_affine: nn.init.zeros_(module.weight) nn.init.zeros_(module.bias) if ( module is getattr(self, "hidden_reuse_conditioner", None) and self.hidden_reuse_adapter_type in {"relay_beta_gamma_mlp", "relay_dynamic_fusion"} ): nn.init.zeros_(module[-1].weight) nn.init.zeros_(module[-1].bias) def project_target_hidden(self, target_hidden: torch.Tensor) -> torch.Tensor: """Apply the ordinary DFLASH five-layer target projector.""" return self.hidden_norm(self.fc(target_hidden)) def project_target_hidden_dynamic( self, target_hidden: torch.Tensor, *, bonus_embedding: torch.Tensor, first_reject_embedding: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: """Fuse target layers with one boundary-conditioned depth-attention head.""" if self.hidden_reuse_adapter_type != "relay_dynamic_fusion": raise RuntimeError("dynamic target projection is only available for v6") hidden_size = self.config.hidden_size num_layers = len(self.target_layer_ids) expected_size = num_layers * hidden_size if target_hidden.size(-1) != expected_size: raise ValueError( f"expected {expected_size} concatenated target features, " f"got {target_hidden.size(-1)}" ) layer_hidden = target_hidden.unflatten(-1, (num_layers, hidden_size)) layer_hidden = self.hidden_reuse_layer_norm(layer_hidden) query_input = torch.cat( [bonus_embedding, first_reject_embedding], dim=-1 ) query = self.hidden_reuse_query_proj( self.hidden_reuse_query_norm(query_input) ) keys = torch.stack( [ key_proj(layer_hidden[..., layer_index, :]) for layer_index, key_proj in enumerate( self.hidden_reuse_key_proj ) ], dim=-2, ) scores = (keys * query.unsqueeze(-2)).sum(dim=-1) scores = scores * (self.hidden_reuse_query_dim**-0.5) layer_weights = torch.softmax(scores.float(), dim=-1).to(keys.dtype) gated_hidden = layer_hidden * layer_weights.unsqueeze(-1) fused_hidden = self.hidden_norm(self.fc(gated_hidden.flatten(-2))) return fused_hidden, layer_weights def project_hidden_reuse( self, source_hidden: torch.Tensor, *, bonus_embedding: Optional[torch.Tensor] = None, first_reject_embedding: Optional[torch.Tensor] = None, ) -> torch.Tensor: """Map projected verifier hidden states to a residual on mask embedding.""" if not self.hidden_reuse_enabled: raise RuntimeError("checkpoint does not enable target-hidden relay") if self.hidden_reuse_adapter_type == "relay_layer_norm": return self.hidden_reuse_relay_norm(source_hidden) if bonus_embedding is None or first_reject_embedding is None: raise ValueError( "v5/v6 target-hidden relay requires bonus and first-reject embeddings" ) condition = torch.cat( [bonus_embedding, first_reject_embedding, source_hidden], dim=-1 ) beta_gamma = self.hidden_reuse_conditioner(condition) beta, gamma = beta_gamma.chunk(2, dim=-1) return beta * self.hidden_reuse_relay_norm(source_hidden) + gamma def forward( self, position_ids: torch.LongTensor, attention_mask: Optional[torch.Tensor] = None, noise_embedding: Optional[torch.Tensor] = None, target_hidden: Optional[torch.Tensor] = None, past_key_values: Optional[Cache] = None, use_cache: bool = False, **kwargs, ) -> CausalLMOutputWithPast: hidden_states = noise_embedding target_hidden = self.project_target_hidden(target_hidden) position_embeddings = self.rotary_emb(hidden_states, position_ids) for layer in self.layers: hidden_states = layer( hidden_states=hidden_states, target_hidden=target_hidden, attention_mask=attention_mask, position_ids=position_ids, past_key_value=past_key_values, use_cache=use_cache, position_embeddings=position_embeddings, **kwargs, ) return self.norm(hidden_states) @torch.inference_mode() def spec_generate( self, target: nn.Module, input_ids: torch.LongTensor, max_new_tokens: int, stop_token_ids: list[int], temperature: float, ): self.eval() num_input_tokens = input_ids.shape[1] max_length = num_input_tokens + max_new_tokens block_size = self.block_size output_ids = torch.full( (1, max_length + block_size), self.mask_token_id, dtype=torch.long, device=target.device, ) position_ids = torch.arange(output_ids.shape[1], device=target.device).unsqueeze(0) past_key_values_target = DynamicCache() past_key_values_draft = DynamicCache() # Prefill stage output = target( input_ids, position_ids=position_ids[:, :num_input_tokens], past_key_values=past_key_values_target, use_cache=True, logits_to_keep=1, output_hidden_states=True, ) output_ids[:, :num_input_tokens] = input_ids output_ids[:, num_input_tokens:num_input_tokens+1] = sample(output.logits, temperature) target_hidden = extract_context_feature(output.hidden_states, self.target_layer_ids) # Decode stage acceptance_lengths = [] start = input_ids.shape[1] while start < max_length: block_output_ids = output_ids[:, start : start + block_size].clone() block_position_ids = position_ids[:, start : start + block_size] noise_embedding = target.model.embed_tokens(block_output_ids) draft_logits = target.lm_head(self( target_hidden=target_hidden, noise_embedding=noise_embedding, position_ids=position_ids[:, past_key_values_draft.get_seq_length(): start + block_size], past_key_values=past_key_values_draft, use_cache=True, is_causal=False, )[:, -block_size+1:, :]) past_key_values_draft.crop(start) block_output_ids[:, 1:] = sample(draft_logits) output = target( block_output_ids, position_ids=block_position_ids, past_key_values=past_key_values_target, use_cache=True, output_hidden_states=True, ) posterior = sample(output.logits, temperature) acceptance_length = (block_output_ids[:, 1:] == posterior[:, :-1]).cumprod(dim=1).sum(dim=1)[0].item() output_ids[:, start : start + acceptance_length + 1] = block_output_ids[:, : acceptance_length + 1] output_ids[:, start + acceptance_length + 1] = posterior[:, acceptance_length] start += acceptance_length + 1 past_key_values_target.crop(start) target_hidden = extract_context_feature(output.hidden_states, self.target_layer_ids)[:, :acceptance_length + 1, :] acceptance_lengths.append(acceptance_length+1) if stop_token_ids is not None and any( stop_token_id in output_ids[:, num_input_tokens:] for stop_token_id in stop_token_ids ): break output_ids = output_ids[:, :max_length] output_ids = output_ids[:, output_ids[0] != self.mask_token_id] if stop_token_ids is not None: stop_token_ids = torch.tensor(stop_token_ids, device=output_ids.device) stop_token_indices = torch.isin(output_ids[0][num_input_tokens:], stop_token_ids).nonzero(as_tuple=True)[0] if stop_token_indices.numel() > 0: output_ids = output_ids[:, : num_input_tokens + stop_token_indices[0] + 1] return output_ids