sec10k-analyst-2b / serving /runtime.py
usmane3's picture
v1.2: retrained planner (margins, growth, cash paid), clearer declines, several copies per GPU, CPU mode, setup fixes
5352590 verified
Raw History Blame Contribute Delete
16.3 kB
"""Isolated one-base/four-LoRA prototype. Import never loads models or CUDA.
This is a different numerical architecture from quantized merged checkpoints.
It has no accuracy claim and is not wired into the running app or benchmark.
"""
from __future__ import annotations
from contextlib import contextmanager
import hashlib
import os
from pathlib import Path
import threading
ROLES = ("first", "tournament", "judge_bidir", "judge_causal")
BIDIRECTIONAL = frozenset(("tournament", "judge_bidir"))
INT32_MAX = 0x7FFFFFFF
def file_sha256(path):
digest = hashlib.sha256()
with Path(path).open("rb") as stream:
for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def validate_manifest(manifest):
"""Require explicit provenance and identities; never guess a common base."""
if manifest.get("common_base_verified") is not True:
raise ValueError("Exact original common base has not been verified")
base = manifest["base"]
if file_sha256(base["gguf_path"]) != base["gguf_sha256"]:
raise ValueError("Shared base GGUF hash mismatch")
roles = manifest["roles"]
if set(roles) != set(ROLES):
raise ValueError("Exactly the four frozen role mappings are required")
for role in ROLES:
row = roles[role]
if row["common_base_identity"] != base["original_hf_identity"]:
raise ValueError("Adapter common-base lineage mismatch: " + role)
if file_sha256(row["adapter_gguf_path"]) != row["adapter_gguf_sha256"]:
raise ValueError("Adapter GGUF hash mismatch: " + role)
if not row.get("original_adapter_provenance"):
raise ValueError("Missing original adapter provenance: " + role)
# HF tokenizer/config files determine the frozen prompt/temperature recipe.
for name, expected in row["metadata_sha256"].items():
if Path(name).name != name or not (name.endswith(".json") or name == "chat_template.jinja"):
raise ValueError("Metadata must name local JSON or chat template files")
if file_sha256(Path(row["metadata_dir"]) / name) != expected:
raise ValueError("Frozen role metadata hash mismatch: " + role)
if not {"tokenizer.json", "tokenizer_config.json", "decider_config.json"}.issubset(row["metadata_sha256"]):
raise ValueError("Missing tokenizer or decider configuration identity")
return manifest
class SharedBaseRuntime:
"""Four persistent adapters, two attention-mode contexts, one base handle.
Different attention modes can execute concurrently. Each physical context is locked.
There is no cross-request KV cache. Every request clears memory and specifies
all four scales; cleanup removes every adapter even on inference failure.
The backend owns model construction and must implement these small methods.
"""
def __init__(self, manifest, backend, *, validate=True, context_pool="by_attention"):
if context_pool not in ("per_role", "by_attention"):
raise ValueError("Unknown context pool policy")
self.manifest = validate_manifest(manifest) if validate else manifest
self.backend = backend
self._condition = threading.Condition()
self._closing = False
self._active = 0
self._closed = False
self.context_pool = context_pool
self._dependents = []
self._locks = {role: threading.Lock() for role in ROLES}
self._poisoned = set()
self.model = None
self.adapters = {}
self.contexts = {}
self._physical_contexts = []
try:
self.model = backend.load_model(manifest["base"]["gguf_path"], n_gpu_layers=0 if os.environ.get("SEC10K_DEVICE", "").lower() == "cpu" else INT32_MAX)
for role in ROLES:
row = manifest["roles"][role]
self.adapters[role] = backend.load_adapter(self.model, row["adapter_gguf_path"])
# Initialize sequentially. There is no process-wide attention monkeypatch.
for role in ROLES:
representative = ("tournament" if role in BIDIRECTIONAL else "first") if context_pool == "by_attention" else role
if representative in self.contexts:
self.contexts[role] = backend.create_role_view(
self.contexts[representative], role=role,
metadata_dir=manifest["roles"][role]["metadata_dir"])
self._locks[role] = self._locks[representative]
continue
self.contexts[role] = backend.create_context(
self.model, role=role, metadata_dir=manifest["roles"][role]["metadata_dir"],
n_ctx=8192 if role in BIDIRECTIONAL else 16384,
n_seq_max=1, n_threads=int(os.environ.get("SEC10K_THREADS", "8") or 8), causal=role not in BIDIRECTIONAL,
n_ubatch=8192 if role in BIDIRECTIONAL else 2048)
self._physical_contexts.append(self.contexts[role])
except BaseException:
self.close()
raise
@contextmanager
def _request_slot(self, role):
if role not in ROLES:
raise ValueError("Unknown frozen role")
with self._condition:
if self._closing or self._closed:
raise RuntimeError("Runtime is closing or closed")
self._active += 1
try:
with self._locks[role]:
if role in self._poisoned:
raise RuntimeError("Context failed cleanup and cannot be reused")
yield self.contexts[role]
finally:
with self._condition:
self._active -= 1
self._condition.notify_all()
def request(self, role, path, body=None):
if path != "/v1/systemone":
raise ValueError("Only frozen systemone calls are supported")
if not isinstance(body, dict) or set(body) != {"state", "questions"}:
raise ValueError("Expected exact state/questions payload")
scales = {name: float(name == role) for name in ROLES}
with self._request_slot(role) as context:
try:
self.backend.clear_adapters(context)
self.backend.clear_memory(context)
self.backend.set_adapters(context, self.adapters, scales)
self.backend.assert_attention(context, causal=role not in BIDIRECTIONAL)
return self.backend.system_one(context, body["state"], body["questions"])
finally:
try:
self.backend.clear_adapters(context)
self.backend.clear_memory(context)
except BaseException:
# Cleanup failure invalidates every view of this physical context.
self._poisoned.update(name for name in ROLES if self._locks[name] is self._locks[role])
raise
def close(self):
with self._condition:
if self._closed:
return
self._closing = True
while self._active:
self._condition.wait()
self._closed = True
for close_dependent in reversed(self._dependents):
close_dependent()
self._dependents.clear()
# Contexts refer to adapter/model handles: free them before their owners.
for context in reversed(self._physical_contexts):
self.backend.free_context(context)
self._physical_contexts.clear()
self.contexts.clear()
for adapter in reversed(list(self.adapters.values())):
self.backend.free_adapter(adapter)
self.adapters.clear()
if self.model is not None:
self.backend.free_model(self.model)
self.model = None
def register_dependent(self, close_callback):
"""Register an owner of borrowed contexts/adapters before base teardown.
A dependent must stop accepting requests and drain its active calls in
its close callback. Registration happens before serving requests.
"""
with self._condition:
if self._closing or self._closed:
raise RuntimeError("Cannot register after shutdown starts")
if close_callback in self._dependents:
raise ValueError("Dependent already registered")
self._dependents.append(close_callback)
def __enter__(self):
return self
def __exit__(self, *exc):
self.close()
class NativeBackend:
"""llama-cpp-python C API + unchanged Decider scoring/prompt methods.
Construction/import is explicit. This code has only source/mock validation
until an exact common base and all four original adapters are available.
"""
def __init__(self):
import llama_cpp
self.L = llama_cpp
required = ("llama_adapter_lora_init", "llama_set_adapters_lora",
"llama_memory_clear", "llama_get_memory", "llama_set_causal_attn",
"llama_get_causal_attn")
if any(not hasattr(self.L, name) for name in required):
raise RuntimeError("Installed llama.cpp lacks required context-local APIs")
if not self.L.llama_supports_gpu_offload() and not os.environ.get("SEC10K_DEVICE", "").lower() == "cpu":
raise RuntimeError("CUDA offload support is required for this prototype")
self.L.llama_backend_init()
def load_model(self, path, *, n_gpu_layers):
import os
params = self.L.llama_model_default_params()
params.n_gpu_layers = n_gpu_layers
params.main_gpu = 0
model = self.L.llama_model_load_from_file(os.fsencode(path), params)
if not model:
raise RuntimeError("Shared base model load failed")
return model
def load_adapter(self, model, path):
import os
adapter = self.L.llama_adapter_lora_init(model, os.fsencode(path))
if not adapter:
raise RuntimeError("LoRA load failed: " + str(path))
return adapter
def create_context(self, model, *, role, metadata_dir, n_ctx, n_seq_max, n_threads, causal, n_ubatch):
import numpy as np
from types import SimpleNamespace
from transformers import AutoTokenizer
from decider.prompt import letter_ids
import decider.engine_gguf as module
from decider.infer import Decider
L = self.L
class BorrowedEngine(module.GGUFEngine):
def __del__(self):
# Model and adapter ownership belongs exclusively to runtime.
pass
engine = BorrowedEngine.__new__(BorrowedEngine)
engine.L, engine.model = L, model
engine.ctx = engine.batch = None
try:
engine.tok = AutoTokenizer.from_pretrained(metadata_dir, local_files_only=True)
engine.letters = np.asarray(letter_ids(engine.tok))
engine.m = SimpleNamespace(tok=engine.tok)
params = L.llama_context_default_params()
params.n_ctx = params.n_batch = n_ctx
params.n_ubatch, params.n_seq_max = n_ubatch, n_seq_max
params.kv_unified = True
params.n_threads = params.n_threads_batch = n_threads
if not causal:
if not hasattr(params, "attention_type") or not hasattr(L, "LLAMA_ATTENTION_TYPE_NON_CAUSAL"):
raise RuntimeError("Noncausal context construction is unavailable")
params.attention_type = L.LLAMA_ATTENTION_TYPE_NON_CAUSAL
engine.ctx = L.llama_init_from_model(model, params)
if not engine.ctx:
raise RuntimeError("Context allocation failed: " + role)
L.llama_set_causal_attn(engine.ctx, causal)
engine.n_ctx, engine.n_batch, engine.n_seq_max = n_ctx, min(8192, n_ctx), n_seq_max
engine.n_vocab = L.llama_vocab_n_tokens(L.llama_model_get_vocab(model))
engine.batch = L.llama_batch_init(n_ctx, 0, n_seq_max)
engine.dev = "gguf"
engine.cfg = dict(backend="llama.cpp", n_ctx=n_ctx, n_seq_max=n_seq_max)
engine.stats = dict(forwards=0)
# Use unchanged Decider config/temperature/layout initialization, while
# supplying our already constructed engine instead of loading a model.
# Startup is sequential in this isolated process; restore immediately.
original = module.GGUFEngine
try:
module.GGUFEngine = lambda *args, **kwargs: engine
decider = Decider(metadata_dir, gguf_file="shared-model-owned-by-runtime.gguf",
gguf_options={"n_ctx": n_ctx, "n_seq_max": n_seq_max})
finally:
module.GGUFEngine = original
context = SimpleNamespace(engine=engine, decider=decider, role=role)
self.assert_attention(context, causal=causal)
return context
except BaseException:
if engine.batch is not None:
L.llama_batch_free(engine.batch)
if engine.ctx:
L.llama_free(engine.ctx)
raise
def create_role_view(self, context, *, role, metadata_dir):
"""Separate frozen config/tokenizer/temperatures, one attention-mode engine."""
import numpy as np
from types import SimpleNamespace
from transformers import AutoTokenizer
from decider.prompt import letter_ids
import decider.engine_gguf as module
from decider.infer import Decider
tokenizer = AutoTokenizer.from_pretrained(metadata_dir, local_files_only=True)
# A shared scoring engine has one letter-logit projection and vocabulary.
# Fail closed if role-specific metadata changes that projection/template.
if not np.array_equal(np.asarray(letter_ids(tokenizer)), context.engine.letters):
raise ValueError("Pooled role has different answer-letter token IDs")
if tokenizer.get_vocab() != context.engine.tok.get_vocab() or tokenizer.chat_template != context.engine.tok.chat_template:
raise ValueError("Pooled role tokenizer/template differs")
original = module.GGUFEngine
try:
module.GGUFEngine = lambda *args, **kwargs: context.engine
decider = Decider(metadata_dir, gguf_file="shared-model-owned-by-runtime.gguf")
finally:
module.GGUFEngine = original
return SimpleNamespace(engine=context.engine, decider=decider, role=role)
def set_adapters(self, context, adapters, scales):
import ctypes
pointer_type = self.L.llama_adapter_lora_p_ctypes
handles = (pointer_type * len(ROLES))(*(adapters[r] for r in ROLES))
values = (ctypes.c_float * len(ROLES))(*(scales[r] for r in ROLES))
if self.L.llama_set_adapters_lora(context.engine.ctx, handles, len(ROLES), values) != 0:
raise RuntimeError("Context adapter assignment failed")
def clear_adapters(self, context):
if self.L.llama_set_adapters_lora(context.engine.ctx, None, 0, None) != 0:
raise RuntimeError("Context adapter cleanup failed")
def clear_memory(self, context):
self.L.llama_memory_clear(self.L.llama_get_memory(context.engine.ctx), True)
def assert_attention(self, context, *, causal):
if bool(self.L.llama_get_causal_attn(context.engine.ctx)) != causal:
raise RuntimeError("Context attention mode mismatch")
def system_one(self, context, state, questions):
return context.decider.system_one(state, questions)
def free_context(self, context):
try:
self.clear_adapters(context)
finally:
self.L.llama_batch_free(context.engine.batch)
self.L.llama_free(context.engine.ctx)
context.engine.batch = context.engine.ctx = None
context.engine.model = None
def free_adapter(self, adapter):
self.L.llama_adapter_lora_free(adapter)
def free_model(self, model):
self.L.llama_model_free(model)