Safe runtime fixes: ONNX session tuning + webapp per-token panel rebuild

rerank.make_ort_session (used by both the reranker and, via import, the
embedder): disables ONNX Runtime's spin-waiting between requests -- by
default worker threads busy-spin after an inference for a microsecond
latency win, which burns CPU quota between queries on the Space's shared
2 vCPUs -- and adds an env-configurable intra-op thread cap
(CANLEX_ORT_THREADS; unset keeps ORT's default, so dev behaviour and
numerics are unchanged). 159-Q eval re-run as a gate: identical
(Hit@1 .81 / Hit@3 .94 / Hit@5 .97 / MRR .88).

webapp/app.py: the sources panel was re-rendered from the FULL tool log
on every streamed token -- O(tokens x log-size) string churn plus a
re-sent multi-KB Gradio payload per token late in a session. The
rendered markdown is now cached and refreshed only when a tool call
completes. Display-identical.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

canlex/embed.py

@@ -71,8 +71,8 @@ class Embedder:
         if model_path is None:
             raise RuntimeError(f"Could not download an ONNX model from {EMB_REPO}.")
         tok_path = hf_hub_download(EMB_REPO, "tokenizer.json")
-        self.session = ort.InferenceSession(model_path,
-                                            providers=["CPUExecutionProvider"])
+        from .rerank import make_ort_session
+        self.session = make_ort_session(model_path)
         self.input_names = {i.name for i in self.session.get_inputs()}
         self.tokenizer = Tokenizer.from_file(tok_path)
         self.tokenizer.enable_truncation(max_length=_MAX_TOKENS)

canlex/rerank.py

@@ -20,6 +20,26 @@ _MAX_DOC_CHARS = int(os.environ.get("CANLEX_RERANK_DOC_CHARS", "1000"))
                        # 3000 -> 1000 (with pool 50 -> 16) held the eval exactly.
 
 
+def make_ort_session(model_path):
+    """A CPU InferenceSession tuned for shared, small-vCPU hosts.
+
+    Two deviations from onnxruntime defaults: (1) spin-waiting between requests
+    is disabled -- by default ORT worker threads busy-spin after an inference
+    for a microsecond-scale latency win, which on the Space's shared 2 vCPUs
+    burns quota between queries; (2) the intra-op thread count is
+    env-configurable (CANLEX_ORT_THREADS) so concurrent sessions can be capped
+    below core count if contention is ever observed. Unset, ORT's default
+    (all cores) is kept, so single-query numerics and speed are unchanged."""
+    opts = ort.SessionOptions()
+    threads = int(os.environ.get("CANLEX_ORT_THREADS", "0"))
+    if threads:
+        opts.intra_op_num_threads = threads
+    opts.add_session_config_entry("session.intra_op.allow_spinning", "0")
+    opts.add_session_config_entry("session.inter_op.allow_spinning", "0")
+    return ort.InferenceSession(model_path, sess_options=opts,
+                                providers=["CPUExecutionProvider"])
+
+
 class Reranker:
     """Cross-encoder that scores (query, section) pairs for relevance.
 
@@ -36,7 +56,7 @@ class Reranker:
         with open(cfg_path, encoding="utf-8") as fh:
             self.pad_id = json.load(fh).get("pad_token_id", 0)
 
-        self.session = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"])
+        self.session = make_ort_session(model_path)
         self.input_names = {i.name for i in self.session.get_inputs()}
 
         self.tokenizer = Tokenizer.from_file(tok_path)

webapp/app.py

@@ -427,6 +427,11 @@ async def _agentic_answer(question: str):
             trace: list[str] = []
 
             answer_buf = ""
+            # The rendered sources panel is cached and refreshed only when a
+            # tool call completes: rebuilding it from the full tool_log on
+            # every streamed token was O(tokens x log-size) string churn (tens
+            # of KB re-rendered and re-sent per token late in a session).
+            sources_md = ""
 
             def status_md(thinking: bool = True) -> str:
                 lines = [f"- {line}" for line in trace]
@@ -435,7 +440,7 @@ async def _agentic_answer(question: str):
                 return "\n".join(lines) if lines else ""
 
             for step in range(MAX_TOOL_ITERATIONS):
-                yield status_md(), answer_buf, _format_sources(tool_log)
+                yield status_md(), answer_buf, sources_md
 
                 # Stream Gemini's next turn. Stream text deltas to the answer
                 # panel optimistically; revert to the pre-turn answer if it
@@ -451,7 +456,7 @@ async def _agentic_answer(question: str):
                         turn_text += chunk["text"]
                         yield (status_md(),
                                answer_buf + turn_text,
-                               _format_sources(tool_log))
+                               sources_md)
                     elif chunk["type"] == "function_call":
                         turn_calls.append(chunk["call"])
                         if optimistic and turn_text:
@@ -460,7 +465,7 @@ async def _agentic_answer(question: str):
                             optimistic = False
                             yield (status_md(),
                                    answer_buf,
-                                   _format_sources(tool_log))
+                                   sources_md)
                     elif chunk["type"] == "finish":
                         turn_parts = chunk["parts"] or []
                         # Capture any text-only finish reason so the caller can
@@ -479,7 +484,7 @@ async def _agentic_answer(question: str):
                     answer_buf += turn_text
                     yield (status_md(thinking=False),
                            _match_badge(tool_log) + answer_buf,
-                           _format_sources(tool_log))
+                           sources_md)
                     return
 
                 # Tool turn. If the model emitted a commentary fragment before
@@ -499,10 +504,11 @@ async def _agentic_answer(question: str):
                     args = call.get("args") or {}
                     label = _summarize_call(name, args)
                     trace.append(label)
-                    yield status_md(), answer_buf, _format_sources(tool_log)
+                    yield status_md(), answer_buf, sources_md
 
                     output = await _run_tool(session, name, args)
                     tool_log.append((name, args, output))
+                    sources_md = _format_sources(tool_log)
                     function_responses.append({
                         "functionResponse": {
                             "name": name,
@@ -525,12 +531,12 @@ async def _agentic_answer(question: str):
                     turn_text += chunk["text"]
                     yield (status_md(thinking=False),
                            answer_buf + turn_text,
-                           _format_sources(tool_log))
+                           sources_md)
             answer_buf += turn_text or \
                 "_(no answer produced after the tool-call budget was exhausted)_"
             yield (status_md(thinking=False),
                    _match_badge(tool_log) + answer_buf,
-                   _format_sources(tool_log))
+                   sources_md)
 
 
 # --- Gradio handler -----------------------------------------------------------