sec10k-analyst-2b / serving /chat_runtime.py
usmane3's picture
v1.2: retrained planner (margins, growth, cash paid), clearer declines, several copies per GPU, CPU mode, setup fixes
5352590 verified
Raw History Blame Contribute Delete
16.2 kB
"""Five chat-role LoRAs borrowing the existing single Q4 model allocation.
Import is stdlib-only. Two causal contexts are pooled; all nine role adapters
stay resident. This changes quantization composition from old merged Q4 models.
It is source/mock validated until explicitly constructed by the owned service.
"""
from __future__ import annotations
from contextlib import ExitStack, contextmanager
import hashlib
import json
import os
from pathlib import Path
import queue
import threading
CHAT_ROLES = ("router", "router_legacy_v8", "prose_v6", "sections_v9", "writer_v10")
PRECISION = "one shared Q4_K_M base plus resident F16 runtime LoRA adapters"
_BORROW_INIT_LOCK = threading.Lock()
def _sha(path):
h = hashlib.sha256()
with Path(path).open("rb") as stream:
for block in iter(lambda: stream.read(8 * 1024 * 1024), b""):
h.update(block)
return h.hexdigest()
def validate_chat_manifest(shared, manifest):
base = manifest["base"]
if base["original_hf_identity"] != shared.manifest["base"]["original_hf_identity"]:
raise ValueError("Chat and decision adapters require the exact same common base")
if set(manifest["roles"]) != set(CHAT_ROLES):
raise ValueError("Exactly the five separately named chat roles are required")
if manifest.get("source_common_base_verified") is not True:
raise ValueError("Chat source common-base provenance is required")
for name, digest in base["tokenizer_sha256"].items():
if Path(name).name != name or _sha(Path(base["tokenizer_dir"]) / name) != digest:
raise ValueError("Chat tokenizer/template hash mismatch")
if not {"tokenizer.json", "tokenizer_config.json", "chat_template.jinja"} <= set(base["tokenizer_sha256"]):
raise ValueError("Exact tokenizer and chat template hashes required")
for role, row in manifest["roles"].items():
if row["common_base_identity"] != base["original_hf_identity"] or not row.get("original_adapter_provenance"):
raise ValueError("Chat adapter provenance mismatch: " + role)
if _sha(row["adapter_gguf_path"]) != row["adapter_gguf_sha256"]:
raise ValueError("Chat adapter GGUF hash mismatch: " + role)
return manifest
class SharedChatRuntime:
"""Server-agnostic chat endpoints using a pool of borrowed-model contexts.
`shared` owns the one base and four decision adapters. This owner keeps five
more adapters resident, borrows the base for two causal chat contexts, and
registers teardown before shared frees its base. There is no cross-request
KV reuse, and each request explicitly supplies all nine adapter scales.
"""
def __init__(self, shared, manifest, backend=None, *, validate=True, pool_size=2,
context_limit=8192, output_limit=1536):
if type(pool_size) is not int or not 1 <= pool_size <= 2:
raise ValueError("Use one or two shared chat contexts")
if (type(context_limit) is not int or type(output_limit) is not int
or context_limit <= output_limit or output_limit <= 0):
raise ValueError("Invalid context/output limits")
self.shared = shared
self.manifest = validate_chat_manifest(shared, manifest) if validate else manifest
self.backend = backend or BorrowedLlamaChatBackend(shared, manifest)
self.context_limit, self.output_limit = context_limit, output_limit
self._condition = threading.Condition()
self._closing = self._closed = False
self._active = 0
self._pool = queue.Queue()
self.contexts = []
self.adapters = {}
self._poisoned = set()
try:
for role in CHAT_ROLES:
self.adapters[role] = self.backend.load_adapter(shared.model, manifest["roles"][role]["adapter_gguf_path"])
for i in range(pool_size):
context = self.backend.create_context(shared.model, context_limit=context_limit, slot=i)
self.contexts.append(context)
self._pool.put(context)
shared.register_dependent(self.close)
except BaseException:
self.close()
raise
@contextmanager
def _slot(self):
with self._condition:
if self._closing or self._closed:
raise RuntimeError("Chat runtime is closing")
self._active += 1
context = None
try:
context = self._pool.get()
if id(context) in self._poisoned:
raise RuntimeError("Chat context cleanup failed; restart required")
yield context
finally:
if context is not None:
self._pool.put(context)
with self._condition:
self._active -= 1
self._condition.notify_all()
def _role(self, role):
if role not in CHAT_ROLES:
raise ValueError("Unknown chat role")
def _render(self, body):
messages = body.get("messages")
if not isinstance(messages, list) or not messages:
raise ValueError("Nonempty messages required")
for m in messages:
if (not isinstance(m, dict) or m.get("role") not in {"system", "user", "assistant"}
or not isinstance(m.get("content"), str)):
raise ValueError("Text system/user/assistant messages required")
kwargs = body.get("chat_template_kwargs", {})
if kwargs not in ({}, {"enable_thinking": False}):
raise ValueError("Only thinking-disabled rendering is supported")
if body.get("add_generation_prompt", True) is not True:
raise ValueError("Generation prompt is required")
return self.backend.render(messages)
def apply_template(self, role, body):
self._role(role)
return {"prompt": self._render(body)}
def tokenize(self, role, body):
self._role(role)
if (not isinstance(body.get("content"), str) or body.get("add_special", False) is not False
or body.get("parse_special", True) is not True):
raise ValueError("Exact tokenization requires text, add_special=false, parse_special=true")
with self._slot() as context:
tokens = self.backend.tokenize(context, body["content"])
return {"tokens": tokens}
def complete(self, role, body):
self._role(role)
if body.get("stream", False) is not False or body.get("temperature", 0) != 0:
raise ValueError("Only nonstreaming deterministic greedy chat is supported")
maximum = body.get("max_tokens", self.output_limit)
if type(maximum) is not int or not 0 < maximum <= self.output_limit:
raise ValueError("Output budget out of bounds")
if "tools" in body or "tool_choice" in body:
raise ValueError("Tool calling is not supported")
if body.get("grammar") is not None and body.get("response_format") is not None:
raise ValueError("Choose explicit grammar or response_format")
prompt = self._render(body)
with self._slot() as context:
tokens = self.backend.tokenize(context, prompt)
if (not tokens or any(type(t) is not int or t < 0 for t in tokens)
or len(tokens) + maximum > self.context_limit):
raise ValueError("Prompt/output exceeds exact context budget; no truncation")
all_adapters = {**self.shared.adapters, **self.adapters}
if len(all_adapters) != 9:
raise ValueError("Expected four decision plus five chat adapter identities")
scales = {name: float(name == role) for name in all_adapters}
try:
self.backend.clear(context)
self.backend.set_adapters(context, all_adapters, scales)
self.backend.assert_causal(context)
response = self.backend.generate(context, tokens, maximum,
grammar=body.get("grammar"),
response_format=body.get("response_format"))
finally:
try:
self.backend.clear(context)
except BaseException:
self._poisoned.add(id(context))
raise
model_id = self.manifest["roles"][role].get("model_id", "shared-base-" + role)
usage = response.get("usage")
if not isinstance(usage, dict) or usage.get("prompt_tokens") != len(tokens):
raise RuntimeError("Native completion input token accounting mismatch")
return {"id": response.get("id"), "object": "chat.completion", "created": response.get("created"),
"model": model_id, "precision": PRECISION,
"choices": [{"index": 0, "message": {"role": "assistant", "content": response["choices"][0]["text"]},
"finish_reason": response["choices"][0]["finish_reason"]}],
"usage": usage}
def dispatch(self, role, path, body):
if path == "/apply-template":
return self.apply_template(role, body)
if path == "/tokenize":
return self.tokenize(role, body)
if path == "/v1/chat/completions":
return self.complete(role, body)
raise ValueError("Unknown chat endpoint")
def close(self):
with self._condition:
if self._closed:
return
self._closing = True
while self._active:
self._condition.wait()
self._closed = True
# All borrowers must disappear before their adapters and the common base.
errors = []
for context in reversed(self.contexts):
try:
self.backend.free_context(context)
except BaseException as e:
errors.append(e)
self.contexts.clear()
for adapter in reversed(list(self.adapters.values())):
try:
self.backend.free_adapter(adapter)
except BaseException as e:
errors.append(e)
self.adapters.clear()
if errors:
raise RuntimeError("Chat teardown failed") from errors[0]
class BorrowedLlamaChatBackend:
"""Reuse installed high-level sampling while preventing any second base load.
The only constructor interception substitutes a borrowed LlamaModel wrapper
whose close stack never owns the base handle. Its regular context/batch and
sampler remain independently owned. Initialization is locked and restored
immediately. The default HF tokenizer renders the known exact base template;
native vocab token IDs must match it before generation.
"""
def __init__(self, shared, manifest):
self.shared = shared
self.L = shared.backend.L
from transformers import AutoTokenizer
self.tokenizer = AutoTokenizer.from_pretrained(manifest["base"]["tokenizer_dir"],
local_files_only=True, trust_remote_code=False)
def load_adapter(self, model, path):
return self.shared.backend.load_adapter(model, path)
def create_context(self, model, *, context_limit, slot):
import llama_cpp
import llama_cpp._internals as internals
L = self.L
# Fetch the original class inside the lock as well: another concurrent
# constructor must not accidentally subclass a temporary borrower.
with _BORROW_INIT_LOCK:
original = internals.LlamaModel
class BorrowedModel(original):
def __init__(self, *, path_model, params, verbose=True):
self.path_model, self.params, self.verbose = path_model, params, verbose
self._exit_stack = ExitStack()
self.sampler, self.custom_samplers = None, []
self.model = model
self.vocab = L.llama_model_get_vocab(model)
if not self.vocab:
raise RuntimeError("Shared model vocabulary is unavailable")
internals.LlamaModel = BorrowedModel
try:
chat = llama_cpp.Llama(model_path=self.shared.manifest["base"]["gguf_path"],
n_ctx=context_limit, n_batch=512, n_ubatch=512,
n_threads=int(os.environ.get("SEC10K_THREADS", "8") or 8), n_threads_batch=int(os.environ.get("SEC10K_THREADS", "8") or 8), logits_all=False,
attention_type=L.LLAMA_ATTENTION_TYPE_CAUSAL,
seed=0, verbose=False)
finally:
internals.LlamaModel = original
if chat._model.model != model:
chat.close()
raise RuntimeError("Chat context does not borrow the one common model")
if chat.n_ctx() != context_limit:
chat.close()
raise RuntimeError("Native chat context capacity differs from requested limit")
L.llama_set_causal_attn(chat._ctx.ctx, True)
return chat
def render(self, messages):
return self.tokenizer.apply_chat_template(messages, tokenize=False,
add_generation_prompt=True, enable_thinking=False)
def tokenize(self, context, prompt):
native = context.tokenize(prompt.encode("utf-8"), add_bos=False, special=True)
hf = self.tokenizer(prompt, add_special_tokens=False, truncation=False)["input_ids"]
if native != hf:
raise RuntimeError("Shared native vocab and exact HF tokenizer IDs differ")
return native
def clear(self, context):
if self.L.llama_set_adapters_lora(context._ctx.ctx, None, 0, None) != 0:
raise RuntimeError("Failed to clear chat adapters")
self.L.llama_memory_clear(self.L.llama_get_memory(context._ctx.ctx), True)
context.reset()
def set_adapters(self, context, adapters, scales):
import ctypes
names = list(adapters)
handles = (self.L.llama_adapter_lora_p_ctypes * len(names))(*(adapters[n] for n in names))
weights = (ctypes.c_float * len(names))(*(scales[n] for n in names))
if self.L.llama_set_adapters_lora(context._ctx.ctx, handles, len(names), weights) != 0:
raise RuntimeError("Failed to bind chat role adapter")
def assert_causal(self, context):
if not self.L.llama_get_causal_attn(context._ctx.ctx):
raise RuntimeError("Chat context is not causal")
def generate(self, context, tokens, maximum, *, grammar=None, response_format=None):
from llama_cpp import LlamaGrammar
if grammar is not None and response_format is not None:
raise ValueError("Choose explicit grammar or response_format")
constraint = None
if grammar is not None:
if not isinstance(grammar, str) or not grammar.strip():
raise ValueError("Grammar must be nonempty GBNF")
constraint = LlamaGrammar.from_string(grammar, verbose=False)
elif response_format is not None:
if not isinstance(response_format, dict) or response_format.get("type") != "json_object":
raise ValueError("Only json_object response_format is supported")
constraint = LlamaGrammar.from_json_schema(json.dumps(response_format.get("schema", {"type": "object"})), verbose=False)
return context.create_completion(prompt=tokens, max_tokens=maximum, temperature=0,
top_k=1, top_p=1, repeat_penalty=1, seed=0,
stream=False, grammar=constraint)
def free_context(self, context):
# Cleanup failure must not leave a borrowed context alive when its
# adapter handles and the owning shared model are subsequently freed.
try:
self.clear(context)
finally:
context.close()
def free_adapter(self, adapter):
self.shared.backend.free_adapter(adapter)