Download dflash/model.py from yaro1214/daim: direct link, hf CLI and curl.
- Browser
- Download file 27.9 kB
-
https://huggingface.co/yaro1214/daim/resolve/main/dflash/model.py
- Command line
-
hf download hf://yaro1214/daim/dflash/model.py
-
curl -L -o model.py https://huggingface.co/yaro1214/daim/resolve/main/dflash/model.py
27.9 kB
| import json | |
| import os | |
| import torch | |
| from itertools import islice | |
| from pathlib import Path | |
| from typing import Optional, Callable | |
| from typing_extensions import Unpack, Tuple | |
| from torch import nn | |
| from transformers.models.qwen3.modeling_qwen3 import ( | |
| Qwen3RMSNorm, | |
| Qwen3RotaryEmbedding, | |
| Qwen3Config, | |
| Qwen3PreTrainedModel, | |
| Qwen3MLP, | |
| GradientCheckpointingLayer, | |
| FlashAttentionKwargs, | |
| rotate_half, | |
| eager_attention_forward, | |
| ALL_ATTENTION_FUNCTIONS, | |
| ) | |
| from transformers import DynamicCache | |
| from transformers.modeling_outputs import CausalLMOutputWithPast | |
| from transformers.cache_utils import Cache | |
| # --------------------------------------------------------------------------- | |
| # Dataset loading (auto-downloads from HuggingFace on first use) | |
| # --------------------------------------------------------------------------- | |
| CACHE_DIR = Path(__file__).parent.parent / "cache" | |
| def _format_livecodebench(row: dict) -> str: | |
| prompt = ( | |
| "You are an expert Python programmer. You will be given a question " | |
| "(problem specification) and will generate a correct Python program " | |
| "that matches the specification and passes all tests. You will NOT " | |
| "return anything except for the program\n\n" | |
| f"### Question:\n{row['question_content']}\n\n" | |
| ) | |
| if row.get("starter_code"): | |
| prompt += ( | |
| "### Format: Use the following code structure:\n" | |
| f"```python\n{row['starter_code']}\n```\n\n" | |
| ) | |
| else: | |
| prompt += ( | |
| "### Format: Write your code in the following format:\n" | |
| "```python\n# YOUR CODE HERE\n```\n\n" | |
| ) | |
| return prompt + "### Answer: (use the provided format with backticks)" | |
| _LCB_BASE = ( | |
| "https://huggingface.co/datasets/livecodebench/" | |
| "code_generation_lite/resolve/main/" | |
| ) | |
| DATASETS = { | |
| "gsm8k": { | |
| "load_args": ("openai/gsm8k", "main"), | |
| "load_kwargs": {"split": "test"}, | |
| "format": lambda x: "{question}\nPlease reason step by step, and put your final answer within \\boxed{{}}.".format(**x), | |
| }, | |
| "math500": { | |
| "load_args": ("HuggingFaceH4/MATH-500",), | |
| "load_kwargs": {"split": "test"}, | |
| "format": lambda x: "{problem}\nPlease reason step by step, and put your final answer within \\boxed{{}}.".format(**x), | |
| }, | |
| "aime24": { | |
| "load_args": ("HuggingFaceH4/aime_2024",), | |
| "load_kwargs": {"split": "train"}, | |
| "format": lambda x: "{problem}\nPlease reason step by step, and put your final answer within \\boxed{{}}.".format(**x), | |
| }, | |
| "aime25": { | |
| "load_args": ("math-ai/aime25",), | |
| "load_kwargs": {"split": "test"}, | |
| "format": lambda x: "{problem}\nPlease reason step by step, and put your final answer within \\boxed{{}}.".format(**x), | |
| }, | |
| "humaneval": { | |
| "load_args": ("openai/openai_humaneval",), | |
| "load_kwargs": {"split": "test"}, | |
| "format": lambda x: "Write a solution to the following problem and make sure that it passes the tests:\n```python\n{prompt}\n```".format(**x), | |
| }, | |
| "mbpp": { | |
| "load_args": ("google-research-datasets/mbpp", "sanitized"), | |
| "load_kwargs": {"split": "test"}, | |
| "format": lambda x: x["prompt"], | |
| }, | |
| "livecodebench": { | |
| "load_args": ("json",), | |
| "load_kwargs": { | |
| "data_files": { | |
| "test": [ | |
| _LCB_BASE + f"test{suffix}.jsonl" | |
| for suffix in ("", "2", "3", "4", "5", "6") | |
| ] | |
| }, | |
| "split": "test", | |
| }, | |
| "format": _format_livecodebench, | |
| }, | |
| "swe-bench": { | |
| "load_args": ("princeton-nlp/SWE-bench_Lite",), | |
| "load_kwargs": {"split": "test"}, | |
| "format": lambda x: ( | |
| f"Problem Statement:\n{x['problem_statement']}\n" | |
| "Please fix the issue described above." | |
| ), | |
| }, | |
| "mt-bench": { | |
| "load_args": ("HuggingFaceH4/mt_bench_prompts",), | |
| "load_kwargs": {"split": "train"}, | |
| "format": lambda x: x["prompt"], # list of turns | |
| "multi_turn": True, | |
| }, | |
| "alpaca": { | |
| "load_args": ("tatsu-lab/alpaca",), | |
| "load_kwargs": {"split": "train"}, | |
| "format": lambda x: ( | |
| f"{x['instruction']}\n\nInput:\n{x['input']}" | |
| if x["input"] | |
| else x["instruction"] | |
| ), | |
| }, | |
| } | |
| def _dataset_cache_path(name: str, max_samples: Optional[int] = None) -> Path: | |
| if name == "livecodebench" and max_samples is not None: | |
| return CACHE_DIR / f"{name}.{max_samples}.jsonl" | |
| return CACHE_DIR / f"{name}.jsonl" | |
| def _prepare_dataset( | |
| name: str, | |
| out_path: Path, | |
| max_samples: Optional[int] = None, | |
| ) -> Path: | |
| from datasets import load_dataset | |
| cfg = DATASETS[name] | |
| CACHE_DIR.mkdir(exist_ok=True) | |
| print(f"[download] {name} ...") | |
| load_kwargs = dict(cfg["load_kwargs"]) | |
| if name == "livecodebench" and max_samples is not None: | |
| # The raw release is several GB because it includes private tests, while | |
| # generation only needs the prompt fields. Stream just the requested | |
| # prompt count instead of downloading every raw JSONL on every rank. | |
| load_kwargs["streaming"] = True | |
| dataset = load_dataset(*cfg["load_args"], **load_kwargs) | |
| rows = ( | |
| islice(dataset, max_samples) | |
| if name == "livecodebench" and max_samples is not None | |
| else dataset | |
| ) | |
| tmp_path = out_path.with_name(f".{out_path.name}.tmp.{os.getpid()}") | |
| count = 0 | |
| try: | |
| with open(tmp_path, "w") as f: | |
| for row in rows: | |
| if cfg.get("multi_turn"): | |
| turns = cfg["format"](row) | |
| else: | |
| turns = [cfg["format"](row)] | |
| f.write(json.dumps({"turns": turns}) + "\n") | |
| count += 1 | |
| os.replace(tmp_path, out_path) | |
| finally: | |
| tmp_path.unlink(missing_ok=True) | |
| print(f"[cached] {out_path} ({count} samples)") | |
| return out_path | |
| def load_and_process_dataset( | |
| data_name: str, | |
| max_samples: Optional[int] = None, | |
| ) -> list[dict]: | |
| if data_name not in DATASETS: | |
| raise ValueError(f"Unknown dataset '{data_name}'. Available: {list(DATASETS.keys())}") | |
| path = _dataset_cache_path(data_name, max_samples) | |
| if not path.exists(): | |
| CACHE_DIR.mkdir(exist_ok=True) | |
| _prepare_dataset(data_name, path, max_samples) | |
| with open(path) as f: | |
| return [json.loads(line) for line in f] | |
| # --------------------------------------------------------------------------- | |
| # Model utilities | |
| # --------------------------------------------------------------------------- | |
| def build_target_layer_ids(num_target_layers: int, num_draft_layers: int): | |
| if num_draft_layers == 1: | |
| return [(num_target_layers // 2)] | |
| start = 1 | |
| end = num_target_layers - 3 | |
| span = end - start | |
| return [ | |
| int(round(start + (i * span) / (num_draft_layers - 1))) | |
| for i in range(num_draft_layers) | |
| ] | |
| def extract_context_feature( | |
| hidden_states: list[torch.Tensor], | |
| layer_ids: Optional[list[int]], | |
| ) -> torch.Tensor: | |
| offset = 1 | |
| selected_states = [hidden_states[layer_id + offset] for layer_id in layer_ids] | |
| return torch.cat(selected_states, dim=-1) | |
| def sample(logits: torch.Tensor, temperature: float = 0.0) -> torch.Tensor: | |
| if temperature < 1e-5: | |
| return torch.argmax(logits, dim=-1) | |
| bsz, seq_len, vocab_size = logits.shape | |
| logits = logits.view(-1, vocab_size) / temperature | |
| probs = torch.softmax(logits, dim=-1) | |
| return torch.multinomial(probs, num_samples=1).view(bsz, seq_len) | |
| # --------------------------------------------------------------------------- | |
| # DFlash model | |
| # --------------------------------------------------------------------------- | |
| def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): | |
| cos = cos.unsqueeze(unsqueeze_dim) | |
| sin = sin.unsqueeze(unsqueeze_dim) | |
| q_len = q.size(-2) | |
| q_embed = (q * cos[..., -q_len:, :]) + (rotate_half(q) * sin[..., -q_len:, :]) | |
| k_embed = (k * cos) + (rotate_half(k) * sin) | |
| return q_embed, k_embed | |
| class Qwen3DFlashAttention(nn.Module): | |
| def __init__(self, config: Qwen3Config, layer_idx: int): | |
| super().__init__() | |
| self.config = config | |
| self.layer_idx = layer_idx | |
| self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) | |
| self.num_key_value_groups = config.num_attention_heads // config.num_key_value_heads | |
| self.scaling = self.head_dim**-0.5 | |
| self.attention_dropout = config.attention_dropout | |
| self.is_causal = False | |
| self.q_proj = nn.Linear( | |
| config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.attention_bias | |
| ) | |
| self.k_proj = nn.Linear( | |
| config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias | |
| ) | |
| self.v_proj = nn.Linear( | |
| config.hidden_size, config.num_key_value_heads * self.head_dim, bias=config.attention_bias | |
| ) | |
| self.o_proj = nn.Linear( | |
| config.num_attention_heads * self.head_dim, config.hidden_size, bias=config.attention_bias | |
| ) | |
| self.q_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps) | |
| self.k_norm = Qwen3RMSNorm(self.head_dim, eps=config.rms_norm_eps) | |
| self.sliding_window = config.sliding_window if config.layer_types[layer_idx] == "sliding_attention" else None | |
| def forward( | |
| self, | |
| hidden_states: torch.Tensor, | |
| target_hidden: torch.Tensor, | |
| position_embeddings: tuple[torch.Tensor, torch.Tensor], | |
| attention_mask: Optional[torch.Tensor], | |
| past_key_values: Optional[Cache] = None, | |
| cache_position: Optional[torch.LongTensor] = None, | |
| **kwargs: Unpack[FlashAttentionKwargs], | |
| ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: | |
| bsz, q_len = hidden_states.shape[:-1] | |
| ctx_len = target_hidden.shape[1] | |
| q = self.q_proj(hidden_states) | |
| q = q.view(bsz, q_len, -1, self.head_dim) | |
| q = self.q_norm(q).transpose(1, 2) | |
| k_ctx = self.k_proj(target_hidden) | |
| k_noise = self.k_proj(hidden_states) | |
| v_ctx = self.v_proj(target_hidden) | |
| v_noise = self.v_proj(hidden_states) | |
| k = torch.cat([k_ctx, k_noise], dim=1).view(bsz, ctx_len + q_len, -1, self.head_dim) | |
| v = torch.cat([v_ctx, v_noise], dim=1).view(bsz, ctx_len + q_len, -1, self.head_dim) | |
| k = self.k_norm(k).transpose(1, 2) | |
| v = v.transpose(1, 2) | |
| cos, sin = position_embeddings | |
| q, k = apply_rotary_pos_emb(q, k, cos, sin) | |
| if past_key_values is not None: | |
| cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} | |
| k, v = past_key_values.update(k, v, self.layer_idx, cache_kwargs) | |
| attn_fn: Callable = eager_attention_forward | |
| if self.config._attn_implementation != "eager": | |
| attn_fn = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] | |
| attn_output, attn_weights = attn_fn( | |
| self, | |
| q, | |
| k, | |
| v, | |
| attention_mask, | |
| dropout=0.0 if not self.training else self.attention_dropout, | |
| scaling=self.scaling, | |
| sliding_window=self.sliding_window, | |
| **kwargs, | |
| ) | |
| attn_output = attn_output.reshape(bsz, q_len, -1) | |
| attn_output = self.o_proj(attn_output) | |
| return attn_output, attn_weights | |
| class Qwen3DFlashDecoderLayer(GradientCheckpointingLayer): | |
| def __init__(self, config: Qwen3Config, layer_idx: int): | |
| super().__init__() | |
| self.hidden_size = config.hidden_size | |
| self.self_attn = Qwen3DFlashAttention(config=config, layer_idx=layer_idx) | |
| self.mlp = Qwen3MLP(config) | |
| self.input_layernorm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) | |
| self.post_attention_layernorm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) | |
| def forward( | |
| self, | |
| target_hidden: Optional[torch.Tensor] = None, | |
| hidden_states: Optional[torch.Tensor] = None, | |
| attention_mask: Optional[torch.Tensor] = None, | |
| position_ids: Optional[torch.LongTensor] = None, | |
| past_key_value: Optional[Cache] = None, | |
| output_attentions: Optional[bool] = False, | |
| use_cache: Optional[bool] = False, | |
| cache_position: Optional[torch.LongTensor] = None, | |
| position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, | |
| **kwargs: Unpack[FlashAttentionKwargs], | |
| ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: | |
| residual = hidden_states | |
| hidden_states = self.input_layernorm(hidden_states) | |
| hidden_states = self.self_attn( | |
| hidden_states=hidden_states, | |
| target_hidden=target_hidden, | |
| attention_mask=attention_mask, | |
| position_ids=position_ids, | |
| past_key_values=past_key_value, | |
| output_attentions=output_attentions, | |
| use_cache=use_cache, | |
| cache_position=cache_position, | |
| position_embeddings=position_embeddings, | |
| **kwargs, | |
| )[0] | |
| hidden_states = residual + hidden_states | |
| residual = hidden_states | |
| hidden_states = self.post_attention_layernorm(hidden_states) | |
| hidden_states = self.mlp(hidden_states) | |
| hidden_states = residual + hidden_states | |
| return hidden_states | |
| class DFlashDraftModel(Qwen3PreTrainedModel): | |
| config_class = Qwen3Config | |
| _no_split_modules = ["Qwen3DFlashDecoderLayer"] | |
| def __init__(self, config) -> None: | |
| super().__init__(config) | |
| self.config = config | |
| self.layers = nn.ModuleList( | |
| [Qwen3DFlashDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] | |
| ) | |
| self.target_layer_ids = self.config.dflash_config.get( | |
| "target_layer_ids", build_target_layer_ids(config.num_target_layers, config.num_hidden_layers) | |
| ) | |
| self.norm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) | |
| self.rotary_emb = Qwen3RotaryEmbedding(config) | |
| self.fc = nn.Linear(len(self.target_layer_ids) * config.hidden_size, config.hidden_size, bias=False) | |
| self.hidden_norm = Qwen3RMSNorm(config.hidden_size, eps=config.rms_norm_eps) | |
| self.block_size = config.block_size | |
| self.mask_token_id = self.config.dflash_config.get("mask_token_id", None) | |
| self.projector_type = self.config.dflash_config.get("projector_type", "dflash") | |
| if self.projector_type == "domino": | |
| self.emb_dim = int(self.config.dflash_config["emb_dim"]) | |
| self.gru_hidden_dim = int(self.config.dflash_config["gru_hidden_dim"]) | |
| self.pure_draft_prefix_len = int( | |
| self.config.dflash_config.get("pure_draft_prefix_len", 0) | |
| ) | |
| self.shift_label = bool(self.config.dflash_config.get("shift_label", False)) | |
| if self.shift_label: | |
| raise ValueError( | |
| "Domino + target-hidden relay inference requires shift_label=false" | |
| ) | |
| self.prefix_gru = nn.GRU( | |
| input_size=config.hidden_size, | |
| hidden_size=self.gru_hidden_dim, | |
| num_layers=1, | |
| batch_first=True, | |
| bias=False, | |
| ) | |
| self.embed_proj = nn.Sequential( | |
| nn.Linear(config.hidden_size + self.gru_hidden_dim, self.emb_dim, bias=False), | |
| nn.SiLU(), | |
| nn.Linear(self.emb_dim, config.vocab_size, bias=False), | |
| ) | |
| self.hidden_reuse_enabled = bool( | |
| self.config.dflash_config.get("hidden_reuse_enabled", False) | |
| ) | |
| self.hidden_reuse_adapter_type = self.config.dflash_config.get( | |
| "hidden_reuse_adapter_type", "relay_layer_norm" | |
| ) | |
| self.hidden_reuse_overlap_mode = self.config.dflash_config.get( | |
| "hidden_reuse_overlap_mode", "strict" | |
| ) | |
| if self.hidden_reuse_overlap_mode not in {"strict", "lookahead"}: | |
| raise ValueError( | |
| "unsupported hidden_reuse_overlap_mode: " | |
| f"{self.hidden_reuse_overlap_mode!r}" | |
| ) | |
| if self.hidden_reuse_enabled: | |
| if self.hidden_reuse_adapter_type not in { | |
| "relay_layer_norm", | |
| "relay_beta_gamma_mlp", | |
| "relay_dynamic_fusion", | |
| }: | |
| raise ValueError( | |
| "unsupported hidden_reuse_adapter_type: " | |
| f"{self.hidden_reuse_adapter_type!r}" | |
| ) | |
| if self.hidden_reuse_adapter_type == "relay_layer_norm": | |
| self.hidden_reuse_relay_norm = nn.LayerNorm( | |
| config.hidden_size, | |
| eps=1e-5, | |
| elementwise_affine=True, | |
| ) | |
| else: | |
| self.hidden_reuse_relay_norm = nn.LayerNorm( | |
| config.hidden_size, | |
| eps=1e-5, | |
| elementwise_affine=False, | |
| ) | |
| self.hidden_reuse_conditioner = nn.Sequential( | |
| nn.Linear(3 * config.hidden_size, config.hidden_size), | |
| nn.SiLU(), | |
| nn.Linear(config.hidden_size, 2 * config.hidden_size), | |
| ) | |
| if self.hidden_reuse_adapter_type == "relay_dynamic_fusion": | |
| query_dim = int( | |
| self.config.dflash_config.get( | |
| "hidden_reuse_query_dim", 128 | |
| ) | |
| ) | |
| if query_dim <= 0: | |
| raise ValueError("hidden_reuse_query_dim must be positive") | |
| self.hidden_reuse_query_dim = query_dim | |
| self.hidden_reuse_layer_norm = Qwen3RMSNorm( | |
| config.hidden_size, eps=config.rms_norm_eps | |
| ) | |
| self.hidden_reuse_query_norm = nn.LayerNorm( | |
| 2 * config.hidden_size, | |
| eps=1e-5, | |
| elementwise_affine=False, | |
| ) | |
| self.hidden_reuse_query_proj = nn.Linear( | |
| 2 * config.hidden_size, query_dim, bias=False | |
| ) | |
| self.hidden_reuse_key_proj = nn.ModuleList( | |
| [ | |
| nn.Linear(config.hidden_size, query_dim, bias=False) | |
| for _ in self.target_layer_ids | |
| ] | |
| ) | |
| self.post_init() | |
| if self.hidden_reuse_enabled: | |
| if self.hidden_reuse_relay_norm.elementwise_affine: | |
| nn.init.zeros_(self.hidden_reuse_relay_norm.weight) | |
| nn.init.zeros_(self.hidden_reuse_relay_norm.bias) | |
| else: | |
| nn.init.zeros_(self.hidden_reuse_conditioner[-1].weight) | |
| nn.init.zeros_(self.hidden_reuse_conditioner[-1].bias) | |
| def domino_enabled(self) -> bool: | |
| return self.projector_type == "domino" | |
| def _init_weights(self, module: nn.Module) -> None: | |
| super()._init_weights(module) | |
| if module is getattr(self, "hidden_reuse_relay_norm", None): | |
| if module.elementwise_affine: | |
| nn.init.zeros_(module.weight) | |
| nn.init.zeros_(module.bias) | |
| if ( | |
| module is getattr(self, "hidden_reuse_conditioner", None) | |
| and self.hidden_reuse_adapter_type | |
| in {"relay_beta_gamma_mlp", "relay_dynamic_fusion"} | |
| ): | |
| nn.init.zeros_(module[-1].weight) | |
| nn.init.zeros_(module[-1].bias) | |
| def project_target_hidden(self, target_hidden: torch.Tensor) -> torch.Tensor: | |
| """Apply the ordinary DFLASH five-layer target projector.""" | |
| return self.hidden_norm(self.fc(target_hidden)) | |
| def project_target_hidden_dynamic( | |
| self, | |
| target_hidden: torch.Tensor, | |
| *, | |
| bonus_embedding: torch.Tensor, | |
| first_reject_embedding: torch.Tensor, | |
| ) -> tuple[torch.Tensor, torch.Tensor]: | |
| """Fuse target layers with one boundary-conditioned depth-attention head.""" | |
| if self.hidden_reuse_adapter_type != "relay_dynamic_fusion": | |
| raise RuntimeError("dynamic target projection is only available for v6") | |
| hidden_size = self.config.hidden_size | |
| num_layers = len(self.target_layer_ids) | |
| expected_size = num_layers * hidden_size | |
| if target_hidden.size(-1) != expected_size: | |
| raise ValueError( | |
| f"expected {expected_size} concatenated target features, " | |
| f"got {target_hidden.size(-1)}" | |
| ) | |
| layer_hidden = target_hidden.unflatten(-1, (num_layers, hidden_size)) | |
| layer_hidden = self.hidden_reuse_layer_norm(layer_hidden) | |
| query_input = torch.cat( | |
| [bonus_embedding, first_reject_embedding], dim=-1 | |
| ) | |
| query = self.hidden_reuse_query_proj( | |
| self.hidden_reuse_query_norm(query_input) | |
| ) | |
| keys = torch.stack( | |
| [ | |
| key_proj(layer_hidden[..., layer_index, :]) | |
| for layer_index, key_proj in enumerate( | |
| self.hidden_reuse_key_proj | |
| ) | |
| ], | |
| dim=-2, | |
| ) | |
| scores = (keys * query.unsqueeze(-2)).sum(dim=-1) | |
| scores = scores * (self.hidden_reuse_query_dim**-0.5) | |
| layer_weights = torch.softmax(scores.float(), dim=-1).to(keys.dtype) | |
| gated_hidden = layer_hidden * layer_weights.unsqueeze(-1) | |
| fused_hidden = self.hidden_norm(self.fc(gated_hidden.flatten(-2))) | |
| return fused_hidden, layer_weights | |
| def project_hidden_reuse( | |
| self, | |
| source_hidden: torch.Tensor, | |
| *, | |
| bonus_embedding: Optional[torch.Tensor] = None, | |
| first_reject_embedding: Optional[torch.Tensor] = None, | |
| ) -> torch.Tensor: | |
| """Map projected verifier hidden states to a residual on mask embedding.""" | |
| if not self.hidden_reuse_enabled: | |
| raise RuntimeError("checkpoint does not enable target-hidden relay") | |
| if self.hidden_reuse_adapter_type == "relay_layer_norm": | |
| return self.hidden_reuse_relay_norm(source_hidden) | |
| if bonus_embedding is None or first_reject_embedding is None: | |
| raise ValueError( | |
| "v5/v6 target-hidden relay requires bonus and first-reject embeddings" | |
| ) | |
| condition = torch.cat( | |
| [bonus_embedding, first_reject_embedding, source_hidden], dim=-1 | |
| ) | |
| beta_gamma = self.hidden_reuse_conditioner(condition) | |
| beta, gamma = beta_gamma.chunk(2, dim=-1) | |
| return beta * self.hidden_reuse_relay_norm(source_hidden) + gamma | |
| def forward( | |
| self, | |
| position_ids: torch.LongTensor, | |
| attention_mask: Optional[torch.Tensor] = None, | |
| noise_embedding: Optional[torch.Tensor] = None, | |
| target_hidden: Optional[torch.Tensor] = None, | |
| past_key_values: Optional[Cache] = None, | |
| use_cache: bool = False, | |
| **kwargs, | |
| ) -> CausalLMOutputWithPast: | |
| hidden_states = noise_embedding | |
| target_hidden = self.project_target_hidden(target_hidden) | |
| position_embeddings = self.rotary_emb(hidden_states, position_ids) | |
| for layer in self.layers: | |
| hidden_states = layer( | |
| hidden_states=hidden_states, | |
| target_hidden=target_hidden, | |
| attention_mask=attention_mask, | |
| position_ids=position_ids, | |
| past_key_value=past_key_values, | |
| use_cache=use_cache, | |
| position_embeddings=position_embeddings, | |
| **kwargs, | |
| ) | |
| return self.norm(hidden_states) | |
| def spec_generate( | |
| self, | |
| target: nn.Module, | |
| input_ids: torch.LongTensor, | |
| max_new_tokens: int, | |
| stop_token_ids: list[int], | |
| temperature: float, | |
| ): | |
| self.eval() | |
| num_input_tokens = input_ids.shape[1] | |
| max_length = num_input_tokens + max_new_tokens | |
| block_size = self.block_size | |
| output_ids = torch.full( | |
| (1, max_length + block_size), | |
| self.mask_token_id, | |
| dtype=torch.long, | |
| device=target.device, | |
| ) | |
| position_ids = torch.arange(output_ids.shape[1], device=target.device).unsqueeze(0) | |
| past_key_values_target = DynamicCache() | |
| past_key_values_draft = DynamicCache() | |
| # Prefill stage | |
| output = target( | |
| input_ids, | |
| position_ids=position_ids[:, :num_input_tokens], | |
| past_key_values=past_key_values_target, | |
| use_cache=True, | |
| logits_to_keep=1, | |
| output_hidden_states=True, | |
| ) | |
| output_ids[:, :num_input_tokens] = input_ids | |
| output_ids[:, num_input_tokens:num_input_tokens+1] = sample(output.logits, temperature) | |
| target_hidden = extract_context_feature(output.hidden_states, self.target_layer_ids) | |
| # Decode stage | |
| acceptance_lengths = [] | |
| start = input_ids.shape[1] | |
| while start < max_length: | |
| block_output_ids = output_ids[:, start : start + block_size].clone() | |
| block_position_ids = position_ids[:, start : start + block_size] | |
| noise_embedding = target.model.embed_tokens(block_output_ids) | |
| draft_logits = target.lm_head(self( | |
| target_hidden=target_hidden, | |
| noise_embedding=noise_embedding, | |
| position_ids=position_ids[:, past_key_values_draft.get_seq_length(): start + block_size], | |
| past_key_values=past_key_values_draft, | |
| use_cache=True, | |
| is_causal=False, | |
| )[:, -block_size+1:, :]) | |
| past_key_values_draft.crop(start) | |
| block_output_ids[:, 1:] = sample(draft_logits) | |
| output = target( | |
| block_output_ids, | |
| position_ids=block_position_ids, | |
| past_key_values=past_key_values_target, | |
| use_cache=True, | |
| output_hidden_states=True, | |
| ) | |
| posterior = sample(output.logits, temperature) | |
| acceptance_length = (block_output_ids[:, 1:] == posterior[:, :-1]).cumprod(dim=1).sum(dim=1)[0].item() | |
| output_ids[:, start : start + acceptance_length + 1] = block_output_ids[:, : acceptance_length + 1] | |
| output_ids[:, start + acceptance_length + 1] = posterior[:, acceptance_length] | |
| start += acceptance_length + 1 | |
| past_key_values_target.crop(start) | |
| target_hidden = extract_context_feature(output.hidden_states, self.target_layer_ids)[:, :acceptance_length + 1, :] | |
| acceptance_lengths.append(acceptance_length+1) | |
| if stop_token_ids is not None and any( | |
| stop_token_id in output_ids[:, num_input_tokens:] for stop_token_id in stop_token_ids | |
| ): | |
| break | |
| output_ids = output_ids[:, :max_length] | |
| output_ids = output_ids[:, output_ids[0] != self.mask_token_id] | |
| if stop_token_ids is not None: | |
| stop_token_ids = torch.tensor(stop_token_ids, device=output_ids.device) | |
| stop_token_indices = torch.isin(output_ids[0][num_input_tokens:], stop_token_ids).nonzero(as_tuple=True)[0] | |
| if stop_token_indices.numel() > 0: | |
| output_ids = output_ids[:, : num_input_tokens + stop_token_indices[0] + 1] | |
| return output_ids | |