"""Isolated one-base/four-LoRA prototype. Import never loads models or CUDA. This is a different numerical architecture from quantized merged checkpoints. It has no accuracy claim and is not wired into the running app or benchmark. """ from __future__ import annotations from contextlib import contextmanager import hashlib import os from pathlib import Path import threading ROLES = ("first", "tournament", "judge_bidir", "judge_causal") BIDIRECTIONAL = frozenset(("tournament", "judge_bidir")) INT32_MAX = 0x7FFFFFFF def file_sha256(path): digest = hashlib.sha256() with Path(path).open("rb") as stream: for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def validate_manifest(manifest): """Require explicit provenance and identities; never guess a common base.""" if manifest.get("common_base_verified") is not True: raise ValueError("Exact original common base has not been verified") base = manifest["base"] if file_sha256(base["gguf_path"]) != base["gguf_sha256"]: raise ValueError("Shared base GGUF hash mismatch") roles = manifest["roles"] if set(roles) != set(ROLES): raise ValueError("Exactly the four frozen role mappings are required") for role in ROLES: row = roles[role] if row["common_base_identity"] != base["original_hf_identity"]: raise ValueError("Adapter common-base lineage mismatch: " + role) if file_sha256(row["adapter_gguf_path"]) != row["adapter_gguf_sha256"]: raise ValueError("Adapter GGUF hash mismatch: " + role) if not row.get("original_adapter_provenance"): raise ValueError("Missing original adapter provenance: " + role) # HF tokenizer/config files determine the frozen prompt/temperature recipe. for name, expected in row["metadata_sha256"].items(): if Path(name).name != name or not (name.endswith(".json") or name == "chat_template.jinja"): raise ValueError("Metadata must name local JSON or chat template files") if file_sha256(Path(row["metadata_dir"]) / name) != expected: raise ValueError("Frozen role metadata hash mismatch: " + role) if not {"tokenizer.json", "tokenizer_config.json", "decider_config.json"}.issubset(row["metadata_sha256"]): raise ValueError("Missing tokenizer or decider configuration identity") return manifest class SharedBaseRuntime: """Four persistent adapters, two attention-mode contexts, one base handle. Different attention modes can execute concurrently. Each physical context is locked. There is no cross-request KV cache. Every request clears memory and specifies all four scales; cleanup removes every adapter even on inference failure. The backend owns model construction and must implement these small methods. """ def __init__(self, manifest, backend, *, validate=True, context_pool="by_attention"): if context_pool not in ("per_role", "by_attention"): raise ValueError("Unknown context pool policy") self.manifest = validate_manifest(manifest) if validate else manifest self.backend = backend self._condition = threading.Condition() self._closing = False self._active = 0 self._closed = False self.context_pool = context_pool self._dependents = [] self._locks = {role: threading.Lock() for role in ROLES} self._poisoned = set() self.model = None self.adapters = {} self.contexts = {} self._physical_contexts = [] try: self.model = backend.load_model(manifest["base"]["gguf_path"], n_gpu_layers=0 if os.environ.get("SEC10K_DEVICE", "").lower() == "cpu" else INT32_MAX) for role in ROLES: row = manifest["roles"][role] self.adapters[role] = backend.load_adapter(self.model, row["adapter_gguf_path"]) # Initialize sequentially. There is no process-wide attention monkeypatch. for role in ROLES: representative = ("tournament" if role in BIDIRECTIONAL else "first") if context_pool == "by_attention" else role if representative in self.contexts: self.contexts[role] = backend.create_role_view( self.contexts[representative], role=role, metadata_dir=manifest["roles"][role]["metadata_dir"]) self._locks[role] = self._locks[representative] continue self.contexts[role] = backend.create_context( self.model, role=role, metadata_dir=manifest["roles"][role]["metadata_dir"], n_ctx=8192 if role in BIDIRECTIONAL else 16384, n_seq_max=1, n_threads=int(os.environ.get("SEC10K_THREADS", "8") or 8), causal=role not in BIDIRECTIONAL, n_ubatch=8192 if role in BIDIRECTIONAL else 2048) self._physical_contexts.append(self.contexts[role]) except BaseException: self.close() raise @contextmanager def _request_slot(self, role): if role not in ROLES: raise ValueError("Unknown frozen role") with self._condition: if self._closing or self._closed: raise RuntimeError("Runtime is closing or closed") self._active += 1 try: with self._locks[role]: if role in self._poisoned: raise RuntimeError("Context failed cleanup and cannot be reused") yield self.contexts[role] finally: with self._condition: self._active -= 1 self._condition.notify_all() def request(self, role, path, body=None): if path != "/v1/systemone": raise ValueError("Only frozen systemone calls are supported") if not isinstance(body, dict) or set(body) != {"state", "questions"}: raise ValueError("Expected exact state/questions payload") scales = {name: float(name == role) for name in ROLES} with self._request_slot(role) as context: try: self.backend.clear_adapters(context) self.backend.clear_memory(context) self.backend.set_adapters(context, self.adapters, scales) self.backend.assert_attention(context, causal=role not in BIDIRECTIONAL) return self.backend.system_one(context, body["state"], body["questions"]) finally: try: self.backend.clear_adapters(context) self.backend.clear_memory(context) except BaseException: # Cleanup failure invalidates every view of this physical context. self._poisoned.update(name for name in ROLES if self._locks[name] is self._locks[role]) raise def close(self): with self._condition: if self._closed: return self._closing = True while self._active: self._condition.wait() self._closed = True for close_dependent in reversed(self._dependents): close_dependent() self._dependents.clear() # Contexts refer to adapter/model handles: free them before their owners. for context in reversed(self._physical_contexts): self.backend.free_context(context) self._physical_contexts.clear() self.contexts.clear() for adapter in reversed(list(self.adapters.values())): self.backend.free_adapter(adapter) self.adapters.clear() if self.model is not None: self.backend.free_model(self.model) self.model = None def register_dependent(self, close_callback): """Register an owner of borrowed contexts/adapters before base teardown. A dependent must stop accepting requests and drain its active calls in its close callback. Registration happens before serving requests. """ with self._condition: if self._closing or self._closed: raise RuntimeError("Cannot register after shutdown starts") if close_callback in self._dependents: raise ValueError("Dependent already registered") self._dependents.append(close_callback) def __enter__(self): return self def __exit__(self, *exc): self.close() class NativeBackend: """llama-cpp-python C API + unchanged Decider scoring/prompt methods. Construction/import is explicit. This code has only source/mock validation until an exact common base and all four original adapters are available. """ def __init__(self): import llama_cpp self.L = llama_cpp required = ("llama_adapter_lora_init", "llama_set_adapters_lora", "llama_memory_clear", "llama_get_memory", "llama_set_causal_attn", "llama_get_causal_attn") if any(not hasattr(self.L, name) for name in required): raise RuntimeError("Installed llama.cpp lacks required context-local APIs") if not self.L.llama_supports_gpu_offload() and not os.environ.get("SEC10K_DEVICE", "").lower() == "cpu": raise RuntimeError("CUDA offload support is required for this prototype") self.L.llama_backend_init() def load_model(self, path, *, n_gpu_layers): import os params = self.L.llama_model_default_params() params.n_gpu_layers = n_gpu_layers params.main_gpu = 0 model = self.L.llama_model_load_from_file(os.fsencode(path), params) if not model: raise RuntimeError("Shared base model load failed") return model def load_adapter(self, model, path): import os adapter = self.L.llama_adapter_lora_init(model, os.fsencode(path)) if not adapter: raise RuntimeError("LoRA load failed: " + str(path)) return adapter def create_context(self, model, *, role, metadata_dir, n_ctx, n_seq_max, n_threads, causal, n_ubatch): import numpy as np from types import SimpleNamespace from transformers import AutoTokenizer from decider.prompt import letter_ids import decider.engine_gguf as module from decider.infer import Decider L = self.L class BorrowedEngine(module.GGUFEngine): def __del__(self): # Model and adapter ownership belongs exclusively to runtime. pass engine = BorrowedEngine.__new__(BorrowedEngine) engine.L, engine.model = L, model engine.ctx = engine.batch = None try: engine.tok = AutoTokenizer.from_pretrained(metadata_dir, local_files_only=True) engine.letters = np.asarray(letter_ids(engine.tok)) engine.m = SimpleNamespace(tok=engine.tok) params = L.llama_context_default_params() params.n_ctx = params.n_batch = n_ctx params.n_ubatch, params.n_seq_max = n_ubatch, n_seq_max params.kv_unified = True params.n_threads = params.n_threads_batch = n_threads if not causal: if not hasattr(params, "attention_type") or not hasattr(L, "LLAMA_ATTENTION_TYPE_NON_CAUSAL"): raise RuntimeError("Noncausal context construction is unavailable") params.attention_type = L.LLAMA_ATTENTION_TYPE_NON_CAUSAL engine.ctx = L.llama_init_from_model(model, params) if not engine.ctx: raise RuntimeError("Context allocation failed: " + role) L.llama_set_causal_attn(engine.ctx, causal) engine.n_ctx, engine.n_batch, engine.n_seq_max = n_ctx, min(8192, n_ctx), n_seq_max engine.n_vocab = L.llama_vocab_n_tokens(L.llama_model_get_vocab(model)) engine.batch = L.llama_batch_init(n_ctx, 0, n_seq_max) engine.dev = "gguf" engine.cfg = dict(backend="llama.cpp", n_ctx=n_ctx, n_seq_max=n_seq_max) engine.stats = dict(forwards=0) # Use unchanged Decider config/temperature/layout initialization, while # supplying our already constructed engine instead of loading a model. # Startup is sequential in this isolated process; restore immediately. original = module.GGUFEngine try: module.GGUFEngine = lambda *args, **kwargs: engine decider = Decider(metadata_dir, gguf_file="shared-model-owned-by-runtime.gguf", gguf_options={"n_ctx": n_ctx, "n_seq_max": n_seq_max}) finally: module.GGUFEngine = original context = SimpleNamespace(engine=engine, decider=decider, role=role) self.assert_attention(context, causal=causal) return context except BaseException: if engine.batch is not None: L.llama_batch_free(engine.batch) if engine.ctx: L.llama_free(engine.ctx) raise def create_role_view(self, context, *, role, metadata_dir): """Separate frozen config/tokenizer/temperatures, one attention-mode engine.""" import numpy as np from types import SimpleNamespace from transformers import AutoTokenizer from decider.prompt import letter_ids import decider.engine_gguf as module from decider.infer import Decider tokenizer = AutoTokenizer.from_pretrained(metadata_dir, local_files_only=True) # A shared scoring engine has one letter-logit projection and vocabulary. # Fail closed if role-specific metadata changes that projection/template. if not np.array_equal(np.asarray(letter_ids(tokenizer)), context.engine.letters): raise ValueError("Pooled role has different answer-letter token IDs") if tokenizer.get_vocab() != context.engine.tok.get_vocab() or tokenizer.chat_template != context.engine.tok.chat_template: raise ValueError("Pooled role tokenizer/template differs") original = module.GGUFEngine try: module.GGUFEngine = lambda *args, **kwargs: context.engine decider = Decider(metadata_dir, gguf_file="shared-model-owned-by-runtime.gguf") finally: module.GGUFEngine = original return SimpleNamespace(engine=context.engine, decider=decider, role=role) def set_adapters(self, context, adapters, scales): import ctypes pointer_type = self.L.llama_adapter_lora_p_ctypes handles = (pointer_type * len(ROLES))(*(adapters[r] for r in ROLES)) values = (ctypes.c_float * len(ROLES))(*(scales[r] for r in ROLES)) if self.L.llama_set_adapters_lora(context.engine.ctx, handles, len(ROLES), values) != 0: raise RuntimeError("Context adapter assignment failed") def clear_adapters(self, context): if self.L.llama_set_adapters_lora(context.engine.ctx, None, 0, None) != 0: raise RuntimeError("Context adapter cleanup failed") def clear_memory(self, context): self.L.llama_memory_clear(self.L.llama_get_memory(context.engine.ctx), True) def assert_attention(self, context, *, causal): if bool(self.L.llama_get_causal_attn(context.engine.ctx)) != causal: raise RuntimeError("Context attention mode mismatch") def system_one(self, context, state, questions): return context.decider.system_one(state, questions) def free_context(self, context): try: self.clear_adapters(context) finally: self.L.llama_batch_free(context.engine.batch) self.L.llama_free(context.engine.ctx) context.engine.batch = context.engine.ctx = None context.engine.model = None def free_adapter(self, adapter): self.L.llama_adapter_lora_free(adapter) def free_model(self, model): self.L.llama_model_free(model)