Download src/rqstudy/inference.py from BurnyCoder/rqgm-editable-parent-selection: direct link, hf CLI and curl.
- Browser
- Download file 9.1 kB
-
https://huggingface.co/BurnyCoder/rqgm-editable-parent-selection/resolve/main/src/rqstudy/inference.py
- Command line
-
hf download hf://BurnyCoder/rqgm-editable-parent-selection/src/rqstudy/inference.py
-
curl -L -o inference.py https://huggingface.co/BurnyCoder/rqgm-editable-parent-selection/resolve/main/src/rqstudy/inference.py
9.1 kB
| """Local streaming inference with replayable caching and per-run compute accounting. | |
| Ollama returns prompt/evaluation token counts and nanosecond durations on the | |
| last streaming record. https://docs.ollama.com/api/chat | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import sqlite3 | |
| import sys | |
| import time | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from urllib.parse import urlparse | |
| import httpx | |
| from .data import canonical, digest | |
| MODEL = "qwen3.5:4b" | |
| MODEL_DIGEST = "d8b0f5e9760cd1682034f292d7ef72ec46f432149be0df7574bf2d6e92e38c04" | |
| def now() -> str: | |
| """Timestamp event boundaries in UTC; reports display their explicit time zone.""" | |
| return datetime.now(timezone.utc).isoformat(timespec="milliseconds") | |
| class Broker: | |
| """Record full prompts and streamed outputs once for each logical call and attempt.""" | |
| def __init__(self, project: Path, run: str, ctx: int): | |
| self.project, self.run, self.ctx = project, run, ctx | |
| host = os.getenv("OLLAMA_HOST", "http://127.0.0.1:11434") | |
| if urlparse(host).hostname not in ("127.0.0.1", "localhost", "::1"): | |
| raise ValueError("This registration requires a loopback Ollama endpoint") | |
| self.client = httpx.Client( | |
| base_url=host, timeout=httpx.Timeout(1200, connect=10), trust_env=False | |
| ) | |
| self.client.get("/api/version").raise_for_status() | |
| self.version = self.client.get("/api/version").json()["version"] | |
| models = self.client.get("/api/tags").json()["models"] | |
| if not any( | |
| row["name"] == MODEL and row["digest"] == MODEL_DIGEST for row in models | |
| ): | |
| raise RuntimeError("The registered Ollama digest is unavailable") | |
| project.joinpath(".private").mkdir(exist_ok=True) | |
| self.db = sqlite3.connect(project / ".private" / "inference.sqlite", timeout=60) | |
| self.db.execute("PRAGMA journal_mode=WAL") | |
| self.db.execute( | |
| "CREATE TABLE IF NOT EXISTS memo (key TEXT PRIMARY KEY, response TEXT NOT NULL)" | |
| ) | |
| self.db.execute( | |
| "CREATE TABLE IF NOT EXISTS calls (run TEXT, event TEXT, key TEXT, phase TEXT, role TEXT, input INTEGER, output INTEGER, seconds REAL, cached INTEGER, PRIMARY KEY(run,event))" | |
| ) | |
| self.db.execute( | |
| "CREATE TABLE IF NOT EXISTS attempts (run TEXT,event TEXT,started TEXT,seconds REAL,error TEXT)" | |
| ) | |
| stamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") | |
| project.joinpath("logs").mkdir(exist_ok=True) | |
| self.log = (project / "logs" / f"{run}-{stamp}.log").open( | |
| "a", encoding="utf-8", buffering=1 | |
| ) | |
| def write(self, text: str): | |
| """Flush verbatim model traffic to disk and the worker's terminal in real time.""" | |
| self.log.write(text) | |
| self.log.flush() | |
| sys.stdout.write(text) | |
| sys.stdout.flush() | |
| def ask( | |
| self, event, role, system, user, cap, *, schema=None, seed=13, temperature=0.0 | |
| ): | |
| payload = dict( | |
| model=MODEL, | |
| messages=[ | |
| dict(role="system", content=system), | |
| dict(role="user", content=user), | |
| ], | |
| stream=True, | |
| think=False, | |
| keep_alive="30m", | |
| options=dict( | |
| num_ctx=self.ctx, num_predict=cap, seed=seed, temperature=temperature | |
| ), | |
| ) | |
| if schema: | |
| payload["format"] = schema | |
| if role == "reader": | |
| compartment = "fixed-reader" | |
| elif event.startswith("search/"): | |
| compartment = "training" if event.startswith("search/train/") else "search" | |
| elif event.startswith("endpoint/"): | |
| compartment = "test" if self.run.startswith("eval-test-") else "pilot" | |
| else: | |
| compartment = self.run | |
| key = digest(canonical([compartment, MODEL_DIGEST, self.version, payload])) | |
| old = self.db.execute( | |
| "SELECT key FROM calls WHERE run=? AND event=?", (self.run, event) | |
| ).fetchone() | |
| if old and old[0] != key: | |
| raise RuntimeError(f"Resume produced a different request for {event}") | |
| saved = self.db.execute( | |
| "SELECT response FROM memo WHERE key=?", (key,) | |
| ).fetchone() | |
| self.write( | |
| f"\n[{now()}] {self.run} {event} {role} request {key}\n{canonical(payload)}\n" | |
| ) | |
| if saved: | |
| response = json.loads(saved[0]) | |
| self.write(f"[{now()}] cache replay\n{response['content']}\n") | |
| else: | |
| start = time.monotonic() | |
| parts, terminal = [], None | |
| try: | |
| # Count the complete two messages before requesting inference. Leave space | |
| # for the chat template, decoder and a fixed margin on every backend. | |
| from .reading import tokenizer | |
| codec = tokenizer(self.project) | |
| submitted = sum( | |
| len(codec.encode(m["content"], add_special_tokens=False).ids) | |
| for m in payload["messages"] | |
| ) | |
| if submitted + cap + 128 > self.ctx: | |
| raise ValueError( | |
| "Complete input and reserved output exceed the frozen context" | |
| ) | |
| with self.client.stream("POST", "/api/chat", json=payload) as stream: | |
| stream.raise_for_status() | |
| for line in stream.iter_lines(): | |
| if not line: | |
| continue | |
| chunk = json.loads(line) | |
| if chunk.get("error"): | |
| raise RuntimeError(chunk["error"]) | |
| message = chunk.get("message", {}) | |
| if message.get("thinking"): | |
| self.write(message["thinking"]) | |
| piece = message.get("content", "") | |
| parts.append(piece) | |
| self.write(piece) | |
| if chunk.get("done"): | |
| terminal = chunk | |
| if terminal is None: | |
| raise RuntimeError("Ollama stream ended before final accounting") | |
| response = dict( | |
| content="".join(parts), | |
| input=terminal["prompt_eval_count"], | |
| output=terminal.get("eval_count", 0), | |
| seconds=terminal.get("total_duration", 0) / 1e9, | |
| reason=terminal.get("done_reason", ""), | |
| ) | |
| if response["input"] >= self.ctx - 1: | |
| raise ValueError( | |
| "Prompt reached the context boundary; check coverage" | |
| ) | |
| except Exception as exc: | |
| with self.db: | |
| self.db.execute( | |
| "INSERT INTO attempts VALUES(?,?,?,?,?)", | |
| (self.run, event, now(), time.monotonic() - start, str(exc)), | |
| ) | |
| self.write(f"\n[{now()}] failed {exc}\n") | |
| raise | |
| self.write( | |
| f"\n[{now()}] complete {canonical({k: v for k, v in response.items() if k != 'content'})}\n" | |
| ) | |
| with self.db: | |
| self.db.execute( | |
| "INSERT OR IGNORE INTO memo VALUES(?,?)", (key, canonical(response)) | |
| ) | |
| self.db.execute( | |
| "INSERT OR IGNORE INTO calls VALUES(?,?,?,?,?,?,?,?,?)", | |
| ( | |
| self.run, | |
| event, | |
| key, | |
| event.split("/")[0], | |
| role, | |
| response["input"], | |
| response["output"], | |
| response["seconds"], | |
| int(bool(saved)), | |
| ), | |
| ) | |
| return response["content"] | |
| def metrics(self, phase=None): | |
| """Count unique logical events and actual GPU-resident service time separately.""" | |
| params = [self.run] | |
| where = "run=?" | |
| if phase: | |
| where += " AND phase=?" | |
| params.append(phase) | |
| row = self.db.execute( | |
| "SELECT count(*),sum(input),sum(output),sum(1-cached),sum(input*(1-cached)),sum(output*(1-cached)),sum(seconds*(1-cached)) FROM calls WHERE " | |
| + where, | |
| params, | |
| ).fetchone() | |
| result = dict( | |
| zip( | |
| ( | |
| "calls", | |
| "input", | |
| "output", | |
| "actual_calls", | |
| "actual_input", | |
| "actual_output", | |
| "service_seconds", | |
| ), | |
| (v or 0 for v in row), | |
| ) | |
| ) | |
| result["failed_attempts"] = self.db.execute( | |
| "SELECT count(*) FROM attempts WHERE run=?", (self.run,) | |
| ).fetchone()[0] | |
| return result | |
| def close(self): | |
| self.db.close() | |
| self.client.close() | |
| self.log.close() | |