"""Five chat-role LoRAs borrowing the existing single Q4 model allocation. Import is stdlib-only. Two causal contexts are pooled; all nine role adapters stay resident. This changes quantization composition from old merged Q4 models. It is source/mock validated until explicitly constructed by the owned service. """ from __future__ import annotations from contextlib import ExitStack, contextmanager import hashlib import json import os from pathlib import Path import queue import threading CHAT_ROLES = ("router", "router_legacy_v8", "prose_v6", "sections_v9", "writer_v10") PRECISION = "one shared Q4_K_M base plus resident F16 runtime LoRA adapters" _BORROW_INIT_LOCK = threading.Lock() def _sha(path): h = hashlib.sha256() with Path(path).open("rb") as stream: for block in iter(lambda: stream.read(8 * 1024 * 1024), b""): h.update(block) return h.hexdigest() def validate_chat_manifest(shared, manifest): base = manifest["base"] if base["original_hf_identity"] != shared.manifest["base"]["original_hf_identity"]: raise ValueError("Chat and decision adapters require the exact same common base") if set(manifest["roles"]) != set(CHAT_ROLES): raise ValueError("Exactly the five separately named chat roles are required") if manifest.get("source_common_base_verified") is not True: raise ValueError("Chat source common-base provenance is required") for name, digest in base["tokenizer_sha256"].items(): if Path(name).name != name or _sha(Path(base["tokenizer_dir"]) / name) != digest: raise ValueError("Chat tokenizer/template hash mismatch") if not {"tokenizer.json", "tokenizer_config.json", "chat_template.jinja"} <= set(base["tokenizer_sha256"]): raise ValueError("Exact tokenizer and chat template hashes required") for role, row in manifest["roles"].items(): if row["common_base_identity"] != base["original_hf_identity"] or not row.get("original_adapter_provenance"): raise ValueError("Chat adapter provenance mismatch: " + role) if _sha(row["adapter_gguf_path"]) != row["adapter_gguf_sha256"]: raise ValueError("Chat adapter GGUF hash mismatch: " + role) return manifest class SharedChatRuntime: """Server-agnostic chat endpoints using a pool of borrowed-model contexts. `shared` owns the one base and four decision adapters. This owner keeps five more adapters resident, borrows the base for two causal chat contexts, and registers teardown before shared frees its base. There is no cross-request KV reuse, and each request explicitly supplies all nine adapter scales. """ def __init__(self, shared, manifest, backend=None, *, validate=True, pool_size=2, context_limit=8192, output_limit=1536): if type(pool_size) is not int or not 1 <= pool_size <= 2: raise ValueError("Use one or two shared chat contexts") if (type(context_limit) is not int or type(output_limit) is not int or context_limit <= output_limit or output_limit <= 0): raise ValueError("Invalid context/output limits") self.shared = shared self.manifest = validate_chat_manifest(shared, manifest) if validate else manifest self.backend = backend or BorrowedLlamaChatBackend(shared, manifest) self.context_limit, self.output_limit = context_limit, output_limit self._condition = threading.Condition() self._closing = self._closed = False self._active = 0 self._pool = queue.Queue() self.contexts = [] self.adapters = {} self._poisoned = set() try: for role in CHAT_ROLES: self.adapters[role] = self.backend.load_adapter(shared.model, manifest["roles"][role]["adapter_gguf_path"]) for i in range(pool_size): context = self.backend.create_context(shared.model, context_limit=context_limit, slot=i) self.contexts.append(context) self._pool.put(context) shared.register_dependent(self.close) except BaseException: self.close() raise @contextmanager def _slot(self): with self._condition: if self._closing or self._closed: raise RuntimeError("Chat runtime is closing") self._active += 1 context = None try: context = self._pool.get() if id(context) in self._poisoned: raise RuntimeError("Chat context cleanup failed; restart required") yield context finally: if context is not None: self._pool.put(context) with self._condition: self._active -= 1 self._condition.notify_all() def _role(self, role): if role not in CHAT_ROLES: raise ValueError("Unknown chat role") def _render(self, body): messages = body.get("messages") if not isinstance(messages, list) or not messages: raise ValueError("Nonempty messages required") for m in messages: if (not isinstance(m, dict) or m.get("role") not in {"system", "user", "assistant"} or not isinstance(m.get("content"), str)): raise ValueError("Text system/user/assistant messages required") kwargs = body.get("chat_template_kwargs", {}) if kwargs not in ({}, {"enable_thinking": False}): raise ValueError("Only thinking-disabled rendering is supported") if body.get("add_generation_prompt", True) is not True: raise ValueError("Generation prompt is required") return self.backend.render(messages) def apply_template(self, role, body): self._role(role) return {"prompt": self._render(body)} def tokenize(self, role, body): self._role(role) if (not isinstance(body.get("content"), str) or body.get("add_special", False) is not False or body.get("parse_special", True) is not True): raise ValueError("Exact tokenization requires text, add_special=false, parse_special=true") with self._slot() as context: tokens = self.backend.tokenize(context, body["content"]) return {"tokens": tokens} def complete(self, role, body): self._role(role) if body.get("stream", False) is not False or body.get("temperature", 0) != 0: raise ValueError("Only nonstreaming deterministic greedy chat is supported") maximum = body.get("max_tokens", self.output_limit) if type(maximum) is not int or not 0 < maximum <= self.output_limit: raise ValueError("Output budget out of bounds") if "tools" in body or "tool_choice" in body: raise ValueError("Tool calling is not supported") if body.get("grammar") is not None and body.get("response_format") is not None: raise ValueError("Choose explicit grammar or response_format") prompt = self._render(body) with self._slot() as context: tokens = self.backend.tokenize(context, prompt) if (not tokens or any(type(t) is not int or t < 0 for t in tokens) or len(tokens) + maximum > self.context_limit): raise ValueError("Prompt/output exceeds exact context budget; no truncation") all_adapters = {**self.shared.adapters, **self.adapters} if len(all_adapters) != 9: raise ValueError("Expected four decision plus five chat adapter identities") scales = {name: float(name == role) for name in all_adapters} try: self.backend.clear(context) self.backend.set_adapters(context, all_adapters, scales) self.backend.assert_causal(context) response = self.backend.generate(context, tokens, maximum, grammar=body.get("grammar"), response_format=body.get("response_format")) finally: try: self.backend.clear(context) except BaseException: self._poisoned.add(id(context)) raise model_id = self.manifest["roles"][role].get("model_id", "shared-base-" + role) usage = response.get("usage") if not isinstance(usage, dict) or usage.get("prompt_tokens") != len(tokens): raise RuntimeError("Native completion input token accounting mismatch") return {"id": response.get("id"), "object": "chat.completion", "created": response.get("created"), "model": model_id, "precision": PRECISION, "choices": [{"index": 0, "message": {"role": "assistant", "content": response["choices"][0]["text"]}, "finish_reason": response["choices"][0]["finish_reason"]}], "usage": usage} def dispatch(self, role, path, body): if path == "/apply-template": return self.apply_template(role, body) if path == "/tokenize": return self.tokenize(role, body) if path == "/v1/chat/completions": return self.complete(role, body) raise ValueError("Unknown chat endpoint") def close(self): with self._condition: if self._closed: return self._closing = True while self._active: self._condition.wait() self._closed = True # All borrowers must disappear before their adapters and the common base. errors = [] for context in reversed(self.contexts): try: self.backend.free_context(context) except BaseException as e: errors.append(e) self.contexts.clear() for adapter in reversed(list(self.adapters.values())): try: self.backend.free_adapter(adapter) except BaseException as e: errors.append(e) self.adapters.clear() if errors: raise RuntimeError("Chat teardown failed") from errors[0] class BorrowedLlamaChatBackend: """Reuse installed high-level sampling while preventing any second base load. The only constructor interception substitutes a borrowed LlamaModel wrapper whose close stack never owns the base handle. Its regular context/batch and sampler remain independently owned. Initialization is locked and restored immediately. The default HF tokenizer renders the known exact base template; native vocab token IDs must match it before generation. """ def __init__(self, shared, manifest): self.shared = shared self.L = shared.backend.L from transformers import AutoTokenizer self.tokenizer = AutoTokenizer.from_pretrained(manifest["base"]["tokenizer_dir"], local_files_only=True, trust_remote_code=False) def load_adapter(self, model, path): return self.shared.backend.load_adapter(model, path) def create_context(self, model, *, context_limit, slot): import llama_cpp import llama_cpp._internals as internals L = self.L # Fetch the original class inside the lock as well: another concurrent # constructor must not accidentally subclass a temporary borrower. with _BORROW_INIT_LOCK: original = internals.LlamaModel class BorrowedModel(original): def __init__(self, *, path_model, params, verbose=True): self.path_model, self.params, self.verbose = path_model, params, verbose self._exit_stack = ExitStack() self.sampler, self.custom_samplers = None, [] self.model = model self.vocab = L.llama_model_get_vocab(model) if not self.vocab: raise RuntimeError("Shared model vocabulary is unavailable") internals.LlamaModel = BorrowedModel try: chat = llama_cpp.Llama(model_path=self.shared.manifest["base"]["gguf_path"], n_ctx=context_limit, n_batch=512, n_ubatch=512, n_threads=int(os.environ.get("SEC10K_THREADS", "8") or 8), n_threads_batch=int(os.environ.get("SEC10K_THREADS", "8") or 8), logits_all=False, attention_type=L.LLAMA_ATTENTION_TYPE_CAUSAL, seed=0, verbose=False) finally: internals.LlamaModel = original if chat._model.model != model: chat.close() raise RuntimeError("Chat context does not borrow the one common model") if chat.n_ctx() != context_limit: chat.close() raise RuntimeError("Native chat context capacity differs from requested limit") L.llama_set_causal_attn(chat._ctx.ctx, True) return chat def render(self, messages): return self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, enable_thinking=False) def tokenize(self, context, prompt): native = context.tokenize(prompt.encode("utf-8"), add_bos=False, special=True) hf = self.tokenizer(prompt, add_special_tokens=False, truncation=False)["input_ids"] if native != hf: raise RuntimeError("Shared native vocab and exact HF tokenizer IDs differ") return native def clear(self, context): if self.L.llama_set_adapters_lora(context._ctx.ctx, None, 0, None) != 0: raise RuntimeError("Failed to clear chat adapters") self.L.llama_memory_clear(self.L.llama_get_memory(context._ctx.ctx), True) context.reset() def set_adapters(self, context, adapters, scales): import ctypes names = list(adapters) handles = (self.L.llama_adapter_lora_p_ctypes * len(names))(*(adapters[n] for n in names)) weights = (ctypes.c_float * len(names))(*(scales[n] for n in names)) if self.L.llama_set_adapters_lora(context._ctx.ctx, handles, len(names), weights) != 0: raise RuntimeError("Failed to bind chat role adapter") def assert_causal(self, context): if not self.L.llama_get_causal_attn(context._ctx.ctx): raise RuntimeError("Chat context is not causal") def generate(self, context, tokens, maximum, *, grammar=None, response_format=None): from llama_cpp import LlamaGrammar if grammar is not None and response_format is not None: raise ValueError("Choose explicit grammar or response_format") constraint = None if grammar is not None: if not isinstance(grammar, str) or not grammar.strip(): raise ValueError("Grammar must be nonempty GBNF") constraint = LlamaGrammar.from_string(grammar, verbose=False) elif response_format is not None: if not isinstance(response_format, dict) or response_format.get("type") != "json_object": raise ValueError("Only json_object response_format is supported") constraint = LlamaGrammar.from_json_schema(json.dumps(response_format.get("schema", {"type": "object"})), verbose=False) return context.create_completion(prompt=tokens, max_tokens=maximum, temperature=0, top_k=1, top_p=1, repeat_penalty=1, seed=0, stream=False, grammar=constraint) def free_context(self, context): # Cleanup failure must not leave a borrowed context alive when its # adapter handles and the owning shared model are subsequently freed. try: self.clear(context) finally: context.close() def free_adapter(self, adapter): self.shared.backend.free_adapter(adapter)