File size: 9,096 Bytes
387a4cf
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
"""Local streaming inference with replayable caching and per-run compute accounting.

Ollama returns prompt/evaluation token counts and nanosecond durations on the
last streaming record. https://docs.ollama.com/api/chat
"""

from __future__ import annotations

import json
import os
import sqlite3
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urlparse

import httpx
from .data import canonical, digest

MODEL = "qwen3.5:4b"
MODEL_DIGEST = "d8b0f5e9760cd1682034f292d7ef72ec46f432149be0df7574bf2d6e92e38c04"


def now() -> str:
    """Timestamp event boundaries in UTC; reports display their explicit time zone."""
    return datetime.now(timezone.utc).isoformat(timespec="milliseconds")


class Broker:
    """Record full prompts and streamed outputs once for each logical call and attempt."""

    def __init__(self, project: Path, run: str, ctx: int):
        self.project, self.run, self.ctx = project, run, ctx
        host = os.getenv("OLLAMA_HOST", "http://127.0.0.1:11434")
        if urlparse(host).hostname not in ("127.0.0.1", "localhost", "::1"):
            raise ValueError("This registration requires a loopback Ollama endpoint")
        self.client = httpx.Client(
            base_url=host, timeout=httpx.Timeout(1200, connect=10), trust_env=False
        )
        self.client.get("/api/version").raise_for_status()
        self.version = self.client.get("/api/version").json()["version"]
        models = self.client.get("/api/tags").json()["models"]
        if not any(
            row["name"] == MODEL and row["digest"] == MODEL_DIGEST for row in models
        ):
            raise RuntimeError("The registered Ollama digest is unavailable")
        project.joinpath(".private").mkdir(exist_ok=True)
        self.db = sqlite3.connect(project / ".private" / "inference.sqlite", timeout=60)
        self.db.execute("PRAGMA journal_mode=WAL")
        self.db.execute(
            "CREATE TABLE IF NOT EXISTS memo (key TEXT PRIMARY KEY, response TEXT NOT NULL)"
        )
        self.db.execute(
            "CREATE TABLE IF NOT EXISTS calls (run TEXT, event TEXT, key TEXT, phase TEXT, role TEXT, input INTEGER, output INTEGER, seconds REAL, cached INTEGER, PRIMARY KEY(run,event))"
        )
        self.db.execute(
            "CREATE TABLE IF NOT EXISTS attempts (run TEXT,event TEXT,started TEXT,seconds REAL,error TEXT)"
        )
        stamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f")
        project.joinpath("logs").mkdir(exist_ok=True)
        self.log = (project / "logs" / f"{run}-{stamp}.log").open(
            "a", encoding="utf-8", buffering=1
        )

    def write(self, text: str):
        """Flush verbatim model traffic to disk and the worker's terminal in real time."""
        self.log.write(text)
        self.log.flush()
        sys.stdout.write(text)
        sys.stdout.flush()

    def ask(
        self, event, role, system, user, cap, *, schema=None, seed=13, temperature=0.0
    ):
        payload = dict(
            model=MODEL,
            messages=[
                dict(role="system", content=system),
                dict(role="user", content=user),
            ],
            stream=True,
            think=False,
            keep_alive="30m",
            options=dict(
                num_ctx=self.ctx, num_predict=cap, seed=seed, temperature=temperature
            ),
        )
        if schema:
            payload["format"] = schema
        if role == "reader":
            compartment = "fixed-reader"
        elif event.startswith("search/"):
            compartment = "training" if event.startswith("search/train/") else "search"
        elif event.startswith("endpoint/"):
            compartment = "test" if self.run.startswith("eval-test-") else "pilot"
        else:
            compartment = self.run
        key = digest(canonical([compartment, MODEL_DIGEST, self.version, payload]))
        old = self.db.execute(
            "SELECT key FROM calls WHERE run=? AND event=?", (self.run, event)
        ).fetchone()
        if old and old[0] != key:
            raise RuntimeError(f"Resume produced a different request for {event}")
        saved = self.db.execute(
            "SELECT response FROM memo WHERE key=?", (key,)
        ).fetchone()
        self.write(
            f"\n[{now()}] {self.run} {event} {role} request {key}\n{canonical(payload)}\n"
        )
        if saved:
            response = json.loads(saved[0])
            self.write(f"[{now()}] cache replay\n{response['content']}\n")
        else:
            start = time.monotonic()
            parts, terminal = [], None
            try:
                # Count the complete two messages before requesting inference. Leave space
                # for the chat template, decoder and a fixed margin on every backend.
                from .reading import tokenizer

                codec = tokenizer(self.project)
                submitted = sum(
                    len(codec.encode(m["content"], add_special_tokens=False).ids)
                    for m in payload["messages"]
                )
                if submitted + cap + 128 > self.ctx:
                    raise ValueError(
                        "Complete input and reserved output exceed the frozen context"
                    )
                with self.client.stream("POST", "/api/chat", json=payload) as stream:
                    stream.raise_for_status()
                    for line in stream.iter_lines():
                        if not line:
                            continue
                        chunk = json.loads(line)
                        if chunk.get("error"):
                            raise RuntimeError(chunk["error"])
                        message = chunk.get("message", {})
                        if message.get("thinking"):
                            self.write(message["thinking"])
                        piece = message.get("content", "")
                        parts.append(piece)
                        self.write(piece)
                        if chunk.get("done"):
                            terminal = chunk
                if terminal is None:
                    raise RuntimeError("Ollama stream ended before final accounting")
                response = dict(
                    content="".join(parts),
                    input=terminal["prompt_eval_count"],
                    output=terminal.get("eval_count", 0),
                    seconds=terminal.get("total_duration", 0) / 1e9,
                    reason=terminal.get("done_reason", ""),
                )
                if response["input"] >= self.ctx - 1:
                    raise ValueError(
                        "Prompt reached the context boundary; check coverage"
                    )
            except Exception as exc:
                with self.db:
                    self.db.execute(
                        "INSERT INTO attempts VALUES(?,?,?,?,?)",
                        (self.run, event, now(), time.monotonic() - start, str(exc)),
                    )
                self.write(f"\n[{now()}] failed {exc}\n")
                raise
            self.write(
                f"\n[{now()}] complete {canonical({k: v for k, v in response.items() if k != 'content'})}\n"
            )
        with self.db:
            self.db.execute(
                "INSERT OR IGNORE INTO memo VALUES(?,?)", (key, canonical(response))
            )
            self.db.execute(
                "INSERT OR IGNORE INTO calls VALUES(?,?,?,?,?,?,?,?,?)",
                (
                    self.run,
                    event,
                    key,
                    event.split("/")[0],
                    role,
                    response["input"],
                    response["output"],
                    response["seconds"],
                    int(bool(saved)),
                ),
            )
        return response["content"]

    def metrics(self, phase=None):
        """Count unique logical events and actual GPU-resident service time separately."""
        params = [self.run]
        where = "run=?"
        if phase:
            where += " AND phase=?"
            params.append(phase)
        row = self.db.execute(
            "SELECT count(*),sum(input),sum(output),sum(1-cached),sum(input*(1-cached)),sum(output*(1-cached)),sum(seconds*(1-cached)) FROM calls WHERE "
            + where,
            params,
        ).fetchone()
        result = dict(
            zip(
                (
                    "calls",
                    "input",
                    "output",
                    "actual_calls",
                    "actual_input",
                    "actual_output",
                    "service_seconds",
                ),
                (v or 0 for v in row),
            )
        )
        result["failed_attempts"] = self.db.execute(
            "SELECT count(*) FROM attempts WHERE run=?", (self.run,)
        ).fetchone()[0]
        return result

    def close(self):
        self.db.close()
        self.client.close()
        self.log.close()