File size: 9,096 Bytes
387a4cf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 | """Local streaming inference with replayable caching and per-run compute accounting.
Ollama returns prompt/evaluation token counts and nanosecond durations on the
last streaming record. https://docs.ollama.com/api/chat
"""
from __future__ import annotations
import json
import os
import sqlite3
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urlparse
import httpx
from .data import canonical, digest
MODEL = "qwen3.5:4b"
MODEL_DIGEST = "d8b0f5e9760cd1682034f292d7ef72ec46f432149be0df7574bf2d6e92e38c04"
def now() -> str:
"""Timestamp event boundaries in UTC; reports display their explicit time zone."""
return datetime.now(timezone.utc).isoformat(timespec="milliseconds")
class Broker:
"""Record full prompts and streamed outputs once for each logical call and attempt."""
def __init__(self, project: Path, run: str, ctx: int):
self.project, self.run, self.ctx = project, run, ctx
host = os.getenv("OLLAMA_HOST", "http://127.0.0.1:11434")
if urlparse(host).hostname not in ("127.0.0.1", "localhost", "::1"):
raise ValueError("This registration requires a loopback Ollama endpoint")
self.client = httpx.Client(
base_url=host, timeout=httpx.Timeout(1200, connect=10), trust_env=False
)
self.client.get("/api/version").raise_for_status()
self.version = self.client.get("/api/version").json()["version"]
models = self.client.get("/api/tags").json()["models"]
if not any(
row["name"] == MODEL and row["digest"] == MODEL_DIGEST for row in models
):
raise RuntimeError("The registered Ollama digest is unavailable")
project.joinpath(".private").mkdir(exist_ok=True)
self.db = sqlite3.connect(project / ".private" / "inference.sqlite", timeout=60)
self.db.execute("PRAGMA journal_mode=WAL")
self.db.execute(
"CREATE TABLE IF NOT EXISTS memo (key TEXT PRIMARY KEY, response TEXT NOT NULL)"
)
self.db.execute(
"CREATE TABLE IF NOT EXISTS calls (run TEXT, event TEXT, key TEXT, phase TEXT, role TEXT, input INTEGER, output INTEGER, seconds REAL, cached INTEGER, PRIMARY KEY(run,event))"
)
self.db.execute(
"CREATE TABLE IF NOT EXISTS attempts (run TEXT,event TEXT,started TEXT,seconds REAL,error TEXT)"
)
stamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f")
project.joinpath("logs").mkdir(exist_ok=True)
self.log = (project / "logs" / f"{run}-{stamp}.log").open(
"a", encoding="utf-8", buffering=1
)
def write(self, text: str):
"""Flush verbatim model traffic to disk and the worker's terminal in real time."""
self.log.write(text)
self.log.flush()
sys.stdout.write(text)
sys.stdout.flush()
def ask(
self, event, role, system, user, cap, *, schema=None, seed=13, temperature=0.0
):
payload = dict(
model=MODEL,
messages=[
dict(role="system", content=system),
dict(role="user", content=user),
],
stream=True,
think=False,
keep_alive="30m",
options=dict(
num_ctx=self.ctx, num_predict=cap, seed=seed, temperature=temperature
),
)
if schema:
payload["format"] = schema
if role == "reader":
compartment = "fixed-reader"
elif event.startswith("search/"):
compartment = "training" if event.startswith("search/train/") else "search"
elif event.startswith("endpoint/"):
compartment = "test" if self.run.startswith("eval-test-") else "pilot"
else:
compartment = self.run
key = digest(canonical([compartment, MODEL_DIGEST, self.version, payload]))
old = self.db.execute(
"SELECT key FROM calls WHERE run=? AND event=?", (self.run, event)
).fetchone()
if old and old[0] != key:
raise RuntimeError(f"Resume produced a different request for {event}")
saved = self.db.execute(
"SELECT response FROM memo WHERE key=?", (key,)
).fetchone()
self.write(
f"\n[{now()}] {self.run} {event} {role} request {key}\n{canonical(payload)}\n"
)
if saved:
response = json.loads(saved[0])
self.write(f"[{now()}] cache replay\n{response['content']}\n")
else:
start = time.monotonic()
parts, terminal = [], None
try:
# Count the complete two messages before requesting inference. Leave space
# for the chat template, decoder and a fixed margin on every backend.
from .reading import tokenizer
codec = tokenizer(self.project)
submitted = sum(
len(codec.encode(m["content"], add_special_tokens=False).ids)
for m in payload["messages"]
)
if submitted + cap + 128 > self.ctx:
raise ValueError(
"Complete input and reserved output exceed the frozen context"
)
with self.client.stream("POST", "/api/chat", json=payload) as stream:
stream.raise_for_status()
for line in stream.iter_lines():
if not line:
continue
chunk = json.loads(line)
if chunk.get("error"):
raise RuntimeError(chunk["error"])
message = chunk.get("message", {})
if message.get("thinking"):
self.write(message["thinking"])
piece = message.get("content", "")
parts.append(piece)
self.write(piece)
if chunk.get("done"):
terminal = chunk
if terminal is None:
raise RuntimeError("Ollama stream ended before final accounting")
response = dict(
content="".join(parts),
input=terminal["prompt_eval_count"],
output=terminal.get("eval_count", 0),
seconds=terminal.get("total_duration", 0) / 1e9,
reason=terminal.get("done_reason", ""),
)
if response["input"] >= self.ctx - 1:
raise ValueError(
"Prompt reached the context boundary; check coverage"
)
except Exception as exc:
with self.db:
self.db.execute(
"INSERT INTO attempts VALUES(?,?,?,?,?)",
(self.run, event, now(), time.monotonic() - start, str(exc)),
)
self.write(f"\n[{now()}] failed {exc}\n")
raise
self.write(
f"\n[{now()}] complete {canonical({k: v for k, v in response.items() if k != 'content'})}\n"
)
with self.db:
self.db.execute(
"INSERT OR IGNORE INTO memo VALUES(?,?)", (key, canonical(response))
)
self.db.execute(
"INSERT OR IGNORE INTO calls VALUES(?,?,?,?,?,?,?,?,?)",
(
self.run,
event,
key,
event.split("/")[0],
role,
response["input"],
response["output"],
response["seconds"],
int(bool(saved)),
),
)
return response["content"]
def metrics(self, phase=None):
"""Count unique logical events and actual GPU-resident service time separately."""
params = [self.run]
where = "run=?"
if phase:
where += " AND phase=?"
params.append(phase)
row = self.db.execute(
"SELECT count(*),sum(input),sum(output),sum(1-cached),sum(input*(1-cached)),sum(output*(1-cached)),sum(seconds*(1-cached)) FROM calls WHERE "
+ where,
params,
).fetchone()
result = dict(
zip(
(
"calls",
"input",
"output",
"actual_calls",
"actual_input",
"actual_output",
"service_seconds",
),
(v or 0 for v in row),
)
)
result["failed_attempts"] = self.db.execute(
"SELECT count(*) FROM attempts WHERE run=?", (self.run,)
).fetchone()[0]
return result
def close(self):
self.db.close()
self.client.close()
self.log.close()
|