File size: 5,921 Bytes
15a7f95 8319e09 15a7f95 8319e09 15a7f95 7992fd3 15a7f95 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | """
handler.py — HuggingFace Inference Endpoint handler for SriRamanaAtmic/AtmicIntelv2
Compatible with transformers==4.51.3 (matches model's transformers_version in config.json).
Generation parameters (from Section A4 of technical review — do not change):
do_sample = False (greedy decoding — matches SFT + DPO training exactly)
max_new_tokens = 400
repetition_penalty = 1.01 (sole repetition control)
no_repeat_ngram_size = 0 (PERMANENTLY DISABLED — hard-coded, not overridable via API)
temperature / top_p (REMOVED — inactive under greedy decoding)
Token IDs (from added_tokens.json — verified):
<|endoftext|> = 32000 (pad_token_id)
<|assistant|> = 32001 (appears in INPUT prompt — must NEVER be eos_token_id)
<|end|> = 32007 (turn terminator — correct eos for generation)
Critical: generation_config.json in the repo contains eos_token_id=[32000, 32001, 32007].
Token 32001 (<|assistant|>) is present in every input prompt, causing generation to stop
at token 0. This handler explicitly overrides generation_config.json by setting
self.model.generation_config before any generate() call.
Input contract:
The caller (pipeline.py via prompt_builder.py) sends a fully-formatted Phi-3 prompt string.
This handler does NOT apply any chat template — prompt arrives ready to tokenize.
{"inputs": "<|system|>...<|end|>\n<|user|>...<|end|>\n<|assistant|>\n"}
"""
# ── DynamicCache compatibility shim (transformers >= 4.38) ──────────────────
# Must be first — before any other transformers import.
import transformers.cache_utils as _cu
if not hasattr(_cu.DynamicCache, "get_max_length"):
_cu.DynamicCache.get_max_length = lambda self: None
from transformers import DynamicCache
if not hasattr(DynamicCache, "get_max_length"):
DynamicCache.get_max_length = lambda self: None
# ────────────────────────────────────────────────────────────────────────────
from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
import torch
class EndpointHandler:
def __init__(self, path=""):
# ── Tokenizer ────────────────────────────────────────────────────
self.tokenizer = AutoTokenizer.from_pretrained(
path,
trust_remote_code=True,
)
# ── Model ────────────────────────────────────────────────────────
self.model = AutoModelForCausalLM.from_pretrained(
path,
torch_dtype=torch.bfloat16, # matches config.json torch_dtype
device_map="auto",
trust_remote_code=True,
attn_implementation="eager", # avoids flash-attn dependency
)
self.model.eval()
# ── Override generation_config.json ──────────────────────────────
# generation_config.json in the repo has eos_token_id=[32000, 32001, 32007].
# Token 32001 is <|assistant|>, which appears in every input prompt.
# This causes generate() to stop at token 0 — empty output.
# We override it here so model.generate() never reads the repo file.
self.model.generation_config = GenerationConfig(
do_sample=False, # greedy — matches SFT+DPO training
repetition_penalty=1.01,
no_repeat_ngram_size=0, # permanently disabled
eos_token_id=32007, # <|end|> only — turn terminator
pad_token_id=32000, # <|endoftext|>
bos_token_id=1,
)
def __call__(self, data: dict) -> list:
# ── Input: fully-formatted prompt string from prompt_builder.py ──
inputs = data.get("inputs", "")
parameters = data.get("parameters", {})
max_new_tokens = int(parameters.get("max_new_tokens", 400))
repetition_penalty = float(parameters.get("repetition_penalty", 1.02))
# ── Tokenize — prompt already contains all special tokens ─────────
tokenized = self.tokenizer(
inputs,
return_tensors="pt",
truncation=True,
max_length=3500, # leaves 596-token headroom within 4096
add_special_tokens=False, # prompt_builder adds
).to(self.model.device)
input_length = tokenized["input_ids"].shape[1]
# ── Generate ──────────────────────────────────────────────────────
# generation_config on the model is already overridden in __init__.
# kwargs here take final precedence for per-request overrides.
with torch.inference_mode():
output = self.model.generate(
**tokenized,
max_new_tokens=max_new_tokens,
repetition_penalty=repetition_penalty,
do_sample=False,
no_repeat_ngram_size=0,
eos_token_id=32007, # <|end|> — confirmed turn terminator
pad_token_id=32000,
)
# ── Decode new tokens only ────────────────────────────────────────
new_tokens = output[0][input_length:]
generated_text = self.tokenizer.decode(new_tokens, skip_special_tokens=True)
return [{"generated_text": generated_text}] |