from dataclasses import dataclass, field from typing import Optional, Callable, Any @dataclass class SamplingParams: temperature: float = 1.0 max_tokens: int = 64 ignore_eos: bool = False cfg_scale: float = 1.0 # CFG guidance scale. When > 1.0, applies classifier-free guidance top_k: Optional[int] = None # Top-k sampling: consider only top k tokens top_p: Optional[float] = None # Top-p (nucleus) sampling: consider tokens with cumulative probability <= top_p min_p: Optional[float] = None # Min-p sampling: keep tokens with probability >= min_p * max_prob repetition_penalty: float = 1.0 # Repetition penalty: >1.0 reduces repetition, <1.0 increases it # Optional logits processor for constrained decoding # Should be a callable with signature: (input_ids: torch.Tensor, logits: torch.Tensor) -> torch.Tensor logits_processor: Optional[Any] = field(default=None, repr=False) # Optional callback to update processor state after each token # Should be a callable with signature: (token_id: int) -> None logits_processor_update_state: Optional[Callable[[int], None]] = field(default=None, repr=False) # Optional additive logits bias (shape [vocab_size]) applied each decode step. logits_bias: Optional[Any] = field(default=None, repr=False) # Optional RNG seed for deterministic sampling. seed: Optional[int] = None def __post_init__(self): assert self.temperature > 1e-10, "greedy sampling is not permitted" assert self.cfg_scale >= 1.0, "cfg_scale must be >= 1.0" if self.top_k is not None: assert self.top_k > 0, "top_k must be > 0" if self.top_p is not None: assert 0.0 < self.top_p <= 1.0, "top_p must be in (0.0, 1.0]" if self.min_p is not None: assert 0.0 < self.min_p <= 1.0, "min_p must be in (0.0, 1.0]" assert self.repetition_penalty > 0.0, "repetition_penalty must be > 0.0"