Spaces:
Running
Running
github-actions[bot] commited on
Commit ·
0fea6ef
1
Parent(s): ed7a9b3
Deploy e90fe57
Browse filesTwo ways use compounds, neither of which is a model
Source: https://github.com/WINTER4000/turingDNA/commit/e90fe57533d072ec6e2e46c133a79b8d4793ed84
- dee/core/orchestrator.py +39 -0
- dee/core/resolution_cache.py +162 -0
- dee/core/resolve.py +23 -1
- dee/core/worked_examples.py +146 -0
- dee/server.py +39 -0
- tests/conftest.py +19 -0
- tests/test_compounding.py +249 -0
dee/core/orchestrator.py
CHANGED
|
@@ -2501,7 +2501,46 @@ def build_system_prompt(anonymous: bool, workspace: Optional[Dict[str, Any]] = N
|
|
| 2501 |
# reason: a durable preference must never read as a competing target.
|
| 2502 |
return (SYSTEM_PROMPT
|
| 2503 |
+ (_ANON_NOTE if anonymous else "")
|
|
|
|
| 2504 |
+ (continuity or "")
|
| 2505 |
+ _memory_note(memories or [])
|
| 2506 |
+ _workspace_note(workspace or {})
|
| 2507 |
+ _target_note(target or {}))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2501 |
# reason: a durable preference must never read as a competing target.
|
| 2502 |
return (SYSTEM_PROMPT
|
| 2503 |
+ (_ANON_NOTE if anonymous else "")
|
| 2504 |
+
+ _worked_examples_note()
|
| 2505 |
+ (continuity or "")
|
| 2506 |
+ _memory_note(memories or [])
|
| 2507 |
+ _workspace_note(workspace or {})
|
| 2508 |
+ _target_note(target or {}))
|
| 2509 |
+
|
| 2510 |
+
|
| 2511 |
+
# Tool chains that real users have completed cleanly, refreshed periodically
|
| 2512 |
+
# from the transcript corpus. Cached in-process because building it reads the
|
| 2513 |
+
# whole corpus and the answer changes on the scale of days, not requests.
|
| 2514 |
+
_WORKED_CACHE: Dict[str, Any] = {"text": "", "at": 0.0}
|
| 2515 |
+
_WORKED_TTL = 6 * 3600
|
| 2516 |
+
|
| 2517 |
+
|
| 2518 |
+
def _worked_examples_note() -> str:
|
| 2519 |
+
"""Observed successful tool chains, as a prompt section.
|
| 2520 |
+
|
| 2521 |
+
This is the point where the corpus feeds back into behaviour: the agent is
|
| 2522 |
+
told how this engine is actually driven by people who finished, rather
|
| 2523 |
+
than only what I guessed when writing the tool descriptions.
|
| 2524 |
+
|
| 2525 |
+
Empty string when nothing clears the k-anonymity floor, which is the
|
| 2526 |
+
normal state early on — an empty "PATHS THAT WORK" heading would read as
|
| 2527 |
+
the engine having no idea what works.
|
| 2528 |
+
"""
|
| 2529 |
+
now = time.time()
|
| 2530 |
+
if _WORKED_CACHE["text"] and now - _WORKED_CACHE["at"] < _WORKED_TTL:
|
| 2531 |
+
return _WORKED_CACHE["text"]
|
| 2532 |
+
text = ""
|
| 2533 |
+
try:
|
| 2534 |
+
from dee import auth as _auth
|
| 2535 |
+
from dee.core import worked_examples as _we
|
| 2536 |
+
runs = _auth.scan_agent_runs(limit=400)
|
| 2537 |
+
if runs:
|
| 2538 |
+
section = _we.as_prompt_section(_we.promote(runs))
|
| 2539 |
+
text = ("\n\n" + section) if section else ""
|
| 2540 |
+
except Exception: # noqa: BLE001
|
| 2541 |
+
# Never let this break a run. A missing prompt section costs a little
|
| 2542 |
+
# guidance; an exception here costs the whole conversation.
|
| 2543 |
+
logger.warning("worked-examples section unavailable", exc_info=True)
|
| 2544 |
+
text = ""
|
| 2545 |
+
_WORKED_CACHE.update(text=text, at=now)
|
| 2546 |
+
return text
|
dee/core/resolution_cache.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Resolve TP53 once. Never pay for it again.
|
| 2 |
+
|
| 3 |
+
THE PROBLEM
|
| 4 |
+
-----------
|
| 5 |
+
Every gene-symbol and accession lookup goes out to Ensembl, UniProt or NCBI,
|
| 6 |
+
every time, for every user. `exon.py` has a process-local dict, but a Hugging
|
| 7 |
+
Face Space sleeps on idle and restarts cold, so in practice the same handful of
|
| 8 |
+
genes — TP53, GFP, Cas9, pUC19 — are re-fetched from scratch forever.
|
| 9 |
+
|
| 10 |
+
That costs three things: latency on the request a user is actually waiting on,
|
| 11 |
+
a shared outbound IP burning someone else's rate limit, and correctness — when
|
| 12 |
+
Ensembl is slow or throttling, a lookup that has succeeded a thousand times
|
| 13 |
+
fails, and the user is told their gene might not exist.
|
| 14 |
+
|
| 15 |
+
WHY THIS IS THE HONEST KIND OF "GETS SMARTER"
|
| 16 |
+
---------------------------------------------
|
| 17 |
+
It compounds with use and involves no model, no training and no inference. The
|
| 18 |
+
thousandth user asking for TP53 gets an instant answer *because* nine hundred
|
| 19 |
+
and ninety-nine people asked first. That is a real flywheel, and unlike a
|
| 20 |
+
learned one it cannot be wrong in a way nobody notices — the cached value is
|
| 21 |
+
byte-identical to what the database returned.
|
| 22 |
+
|
| 23 |
+
WHAT MAY AND MAY NOT BE CACHED
|
| 24 |
+
------------------------------
|
| 25 |
+
Only resolutions keyed by a PUBLIC identifier: (gene symbol, organism) or an
|
| 26 |
+
accession. Those keys are not personal, and the values are public database
|
| 27 |
+
records, which is why one shared cache across all users is correct rather than
|
| 28 |
+
a leak.
|
| 29 |
+
|
| 30 |
+
A pasted sequence is never cached. Not by content, not by hash, not as a key.
|
| 31 |
+
`kind == "sequence"` is refused at the door — see :func:`cacheable`. The
|
| 32 |
+
standing rule is that a user's own sequence never leaves the Space, and a
|
| 33 |
+
cross-user cache is very much leaving.
|
| 34 |
+
|
| 35 |
+
STALENESS
|
| 36 |
+
---------
|
| 37 |
+
Database records change: RefSeq versions increment, Ensembl re-annotates. So
|
| 38 |
+
entries carry a TTL and are re-fetched after it. The TTL is long because these
|
| 39 |
+
records are stable on the timescale of a design project, and a stale-by-a-week
|
| 40 |
+
CDS is a far smaller problem than the lookup failing outright.
|
| 41 |
+
"""
|
| 42 |
+
from __future__ import annotations
|
| 43 |
+
|
| 44 |
+
import re
|
| 45 |
+
import threading
|
| 46 |
+
import time
|
| 47 |
+
from collections import OrderedDict
|
| 48 |
+
from typing import Any, Dict, Optional, Tuple
|
| 49 |
+
|
| 50 |
+
# Entries older than this are re-fetched. Sequence records are stable over
|
| 51 |
+
# weeks; this is about eventually noticing a re-annotation, not about
|
| 52 |
+
# freshness in any real-time sense.
|
| 53 |
+
TTL_SECONDS = 14 * 24 * 3600 # 14 days
|
| 54 |
+
MAX_ENTRIES = 2_000 # in-process ceiling; ~50 MB worst case
|
| 55 |
+
|
| 56 |
+
# Never cached. A pasted sequence is the user's own data and a shared cache is
|
| 57 |
+
# by definition cross-user.
|
| 58 |
+
UNCACHEABLE_KINDS = {"sequence", "empty", "unknown"}
|
| 59 |
+
|
| 60 |
+
_LOCK = threading.Lock()
|
| 61 |
+
_MEM: "OrderedDict[str, Dict[str, Any]]" = OrderedDict()
|
| 62 |
+
_STATS = {"hits": 0, "misses": 0, "stores": 0, "evictions": 0, "refused": 0}
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def make_key(kind: str, identifier: str, organism: str = "") -> str:
|
| 66 |
+
"""Stable cache key. Case- and whitespace-insensitive.
|
| 67 |
+
|
| 68 |
+
Organism is part of the key because TP53 exists in dozens of species and
|
| 69 |
+
they are different sequences — a key that dropped it would serve human
|
| 70 |
+
TP53 to someone who asked for zebrafish, which is worse than a miss.
|
| 71 |
+
"""
|
| 72 |
+
ident = re.sub(r"\s+", "", (identifier or "")).upper()
|
| 73 |
+
org = re.sub(r"\s+", " ", (organism or "")).strip().lower()
|
| 74 |
+
return f"{(kind or '').lower()}|{ident}|{org}"
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def cacheable(kind: str, result: Optional[Dict[str, Any]] = None) -> bool:
|
| 78 |
+
"""May a resolution of this kind be stored in a SHARED cache?
|
| 79 |
+
|
| 80 |
+
The gate that keeps user sequences out. Deliberately a whitelist-free
|
| 81 |
+
check on the one thing that disqualifies — a pasted sequence — rather than
|
| 82 |
+
a list of approved kinds, so a new public identifier type is cacheable the
|
| 83 |
+
day it is added instead of silently bypassing the cache forever.
|
| 84 |
+
"""
|
| 85 |
+
if (kind or "").lower() in UNCACHEABLE_KINDS:
|
| 86 |
+
return False
|
| 87 |
+
if result is not None and not result.get("ok"):
|
| 88 |
+
# A failure is not cached. Ensembl being down for ten seconds must not
|
| 89 |
+
# become "this gene does not exist" for the next fortnight.
|
| 90 |
+
return False
|
| 91 |
+
return True
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def get(kind: str, identifier: str, organism: str = "") -> Optional[Dict[str, Any]]:
|
| 95 |
+
"""A previously resolved record, or None. Never raises."""
|
| 96 |
+
if not cacheable(kind):
|
| 97 |
+
return None
|
| 98 |
+
key = make_key(kind, identifier, organism)
|
| 99 |
+
now = time.time()
|
| 100 |
+
with _LOCK:
|
| 101 |
+
entry = _MEM.get(key)
|
| 102 |
+
if entry is None:
|
| 103 |
+
_STATS["misses"] += 1
|
| 104 |
+
return None
|
| 105 |
+
if now - entry["at"] > TTL_SECONDS:
|
| 106 |
+
_MEM.pop(key, None)
|
| 107 |
+
_STATS["misses"] += 1
|
| 108 |
+
return None
|
| 109 |
+
_MEM.move_to_end(key) # LRU: a hit is a recency signal
|
| 110 |
+
_STATS["hits"] += 1
|
| 111 |
+
# A copy, so a caller mutating the result cannot poison the cache for
|
| 112 |
+
# everyone else — the failure mode of a shared cache that is very hard
|
| 113 |
+
# to trace back later.
|
| 114 |
+
out = dict(entry["value"])
|
| 115 |
+
out["cached"] = True
|
| 116 |
+
out["cached_age_seconds"] = int(now - entry["at"])
|
| 117 |
+
return out
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def put(kind: str, identifier: str, result: Dict[str, Any],
|
| 121 |
+
organism: str = "") -> bool:
|
| 122 |
+
"""Store a successful resolution. Returns whether it was stored."""
|
| 123 |
+
if not cacheable(kind, result):
|
| 124 |
+
with _LOCK:
|
| 125 |
+
_STATS["refused"] += 1
|
| 126 |
+
return False
|
| 127 |
+
if not (result or {}).get("sequence"):
|
| 128 |
+
return False
|
| 129 |
+
key = make_key(kind, identifier, organism)
|
| 130 |
+
# `cached` is a property of the read, not of the record. Storing it would
|
| 131 |
+
# make the first served copy claim it came from cache.
|
| 132 |
+
value = {k: v for k, v in result.items()
|
| 133 |
+
if k not in ("cached", "cached_age_seconds")}
|
| 134 |
+
with _LOCK:
|
| 135 |
+
_MEM[key] = {"value": value, "at": time.time()}
|
| 136 |
+
_MEM.move_to_end(key)
|
| 137 |
+
_STATS["stores"] += 1
|
| 138 |
+
while len(_MEM) > MAX_ENTRIES:
|
| 139 |
+
_MEM.popitem(last=False)
|
| 140 |
+
_STATS["evictions"] += 1
|
| 141 |
+
return True
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def stats() -> Dict[str, Any]:
|
| 145 |
+
"""Hit rate and size. The number worth watching is `hit_rate`: it is the
|
| 146 |
+
whole claim that this compounds."""
|
| 147 |
+
with _LOCK:
|
| 148 |
+
total = _STATS["hits"] + _STATS["misses"]
|
| 149 |
+
return {
|
| 150 |
+
**_STATS,
|
| 151 |
+
"entries": len(_MEM),
|
| 152 |
+
"hit_rate": round(_STATS["hits"] / total, 3) if total else 0.0,
|
| 153 |
+
"ttl_days": TTL_SECONDS // 86400,
|
| 154 |
+
}
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def clear() -> None:
|
| 158 |
+
"""Tests, and the admin path when a record is known to have changed."""
|
| 159 |
+
with _LOCK:
|
| 160 |
+
_MEM.clear()
|
| 161 |
+
for k in _STATS:
|
| 162 |
+
_STATS[k] = 0
|
dee/core/resolve.py
CHANGED
|
@@ -27,6 +27,7 @@ from typing import Dict, Tuple
|
|
| 27 |
|
| 28 |
from dee.core import accession as _accession
|
| 29 |
from dee.core import exon as _exon
|
|
|
|
| 30 |
|
| 31 |
# ─── Identifier patterns ─────────────────────────────────────────────
|
| 32 |
_ENSEMBL_TX = re.compile(r"^ENS[A-Z]*T\d{6,}(?:\.\d+)?$", re.I) # ENST…, ENSMUST…
|
|
@@ -362,13 +363,34 @@ def _resolve_via_uniprot(symbol: str, organism: str) -> Dict:
|
|
| 362 |
|
| 363 |
|
| 364 |
def resolve_target(text: str, organism: str = "") -> Dict:
|
| 365 |
-
"""Resolve pasted text to an editable sequence
|
|
|
|
| 366 |
|
| 367 |
Returns a dict: {ok, kind, sequence, gene_symbol, label, source, error?}.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 368 |
"""
|
| 369 |
organism = (organism or "").lower().strip()
|
| 370 |
kind, val = classify(text)
|
| 371 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 372 |
if kind == "empty":
|
| 373 |
return _err("Paste a DNA sequence, a gene symbol, or an accession.")
|
| 374 |
|
|
|
|
| 27 |
|
| 28 |
from dee.core import accession as _accession
|
| 29 |
from dee.core import exon as _exon
|
| 30 |
+
from dee.core import resolution_cache as _rcache
|
| 31 |
|
| 32 |
# ─── Identifier patterns ─────────────────────────────────────────────
|
| 33 |
_ENSEMBL_TX = re.compile(r"^ENS[A-Z]*T\d{6,}(?:\.\d+)?$", re.I) # ENST…, ENSMUST…
|
|
|
|
| 363 |
|
| 364 |
|
| 365 |
def resolve_target(text: str, organism: str = "") -> Dict:
|
| 366 |
+
"""Resolve pasted text to an editable sequence, serving a cached record
|
| 367 |
+
when this identifier has been resolved before.
|
| 368 |
|
| 369 |
Returns a dict: {ok, kind, sequence, gene_symbol, label, source, error?}.
|
| 370 |
+
|
| 371 |
+
A thin wrapper rather than a check inside the resolver, because the
|
| 372 |
+
resolver has eight success paths and a cache written at each of them is a
|
| 373 |
+
cache that will eventually miss one.
|
| 374 |
+
|
| 375 |
+
A pasted sequence is never cached — `resolution_cache.cacheable` refuses
|
| 376 |
+
`kind == "sequence"` outright, so only public identifiers reach the store.
|
| 377 |
"""
|
| 378 |
organism = (organism or "").lower().strip()
|
| 379 |
kind, val = classify(text)
|
| 380 |
|
| 381 |
+
hit = _rcache.get(kind, val, organism)
|
| 382 |
+
if hit is not None:
|
| 383 |
+
return hit
|
| 384 |
+
|
| 385 |
+
result = _resolve_uncached(text, organism, kind, val)
|
| 386 |
+
# Failures are deliberately not stored: Ensembl being down for ten seconds
|
| 387 |
+
# must not become "this gene does not exist" for the next fortnight.
|
| 388 |
+
_rcache.put(kind, val, result, organism)
|
| 389 |
+
return result
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
def _resolve_uncached(text: str, organism: str, kind: str, val: str) -> Dict:
|
| 393 |
+
"""The actual resolution. Every network call lives below here."""
|
| 394 |
if kind == "empty":
|
| 395 |
return _err("Paste a DNA sequence, a gene symbol, or an accession.")
|
| 396 |
|
dee/core/worked_examples.py
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Paths that already worked, promoted from the corpus so the agent reuses them.
|
| 2 |
+
|
| 3 |
+
THE IDEA
|
| 4 |
+
--------
|
| 5 |
+
`learn_signal.py` reads the transcripts for what went WRONG. The same corpus
|
| 6 |
+
holds what went right, and that half has a use the failure half does not: a
|
| 7 |
+
tool chain that N different people completed successfully is evidence about
|
| 8 |
+
how this engine is actually driven, and it can be handed back to the agent.
|
| 9 |
+
|
| 10 |
+
"To build a construct, people call lookup_vector → simulate_assembly →
|
| 11 |
+
check_synthesis" is worth more in a system prompt than any sentence I could
|
| 12 |
+
write about it, because it is observed rather than imagined, and it updates
|
| 13 |
+
itself as the tools change.
|
| 14 |
+
|
| 15 |
+
This is the honest kind of learning, same as the resolution cache: no model,
|
| 16 |
+
no training, nothing that can be quietly wrong. It is counting.
|
| 17 |
+
|
| 18 |
+
WHAT IS PROMOTED, AND WHAT IS NOT
|
| 19 |
+
---------------------------------
|
| 20 |
+
Only the SHAPE — the ordered list of tool names. Never the user's prompt,
|
| 21 |
+
never arguments, never a sequence. A worked example that carried the question
|
| 22 |
+
that produced it would be a transcript excerpt, and transcript excerpts across
|
| 23 |
+
users are exactly what `learn_signal.field_report` refuses to emit.
|
| 24 |
+
|
| 25 |
+
A run qualifies only if it:
|
| 26 |
+
* finished cleanly (`status == "done"`),
|
| 27 |
+
* called at least two tools — one call is not a path,
|
| 28 |
+
* had no failed tool call, and
|
| 29 |
+
* was never corrected by the user mid-run.
|
| 30 |
+
|
| 31 |
+
That last one matters most. A run the user had to steer is not a worked
|
| 32 |
+
example; it is a near-miss, and promoting it would teach the agent the route
|
| 33 |
+
that needed fixing.
|
| 34 |
+
|
| 35 |
+
And the same k-anonymity floor as everything else cross-user: a chain must
|
| 36 |
+
have been completed by at least `min_users` DISTINCT users. One lab's unusual
|
| 37 |
+
workflow is that lab's business, and a chain seen once is an anecdote.
|
| 38 |
+
"""
|
| 39 |
+
from __future__ import annotations
|
| 40 |
+
|
| 41 |
+
import datetime as _dt
|
| 42 |
+
from collections import Counter, defaultdict
|
| 43 |
+
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
|
| 44 |
+
|
| 45 |
+
from dee.core.aggregate import EFFECTIVE_DATE, MIN_USERS, AggregationGateError
|
| 46 |
+
from dee.core.learn_signal import _events, _uid, is_correction
|
| 47 |
+
|
| 48 |
+
# A chain longer than this is usually one exploratory session rather than a
|
| 49 |
+
# repeatable path, and it will not generalise.
|
| 50 |
+
MAX_CHAIN = 8
|
| 51 |
+
MIN_CHAIN = 2
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def chain_of(run: Dict[str, Any]) -> Optional[Tuple[str, ...]]:
|
| 55 |
+
"""The ordered tool names of a clean run, or None if it doesn't qualify.
|
| 56 |
+
|
| 57 |
+
Consecutive repeats collapse: calling fetch_sequence three times for three
|
| 58 |
+
genes is the same PATH as calling it once, and keeping the repetition
|
| 59 |
+
would fragment the counts across chains that mean the same thing.
|
| 60 |
+
"""
|
| 61 |
+
if str(run.get("status") or "") != "done":
|
| 62 |
+
return None
|
| 63 |
+
|
| 64 |
+
names: List[str] = []
|
| 65 |
+
for ev in _events(run):
|
| 66 |
+
kind = ev.get("kind")
|
| 67 |
+
if kind == "steer" and is_correction(str(ev.get("text") or "")):
|
| 68 |
+
return None # a corrected run is a near-miss
|
| 69 |
+
if kind == "tool_result" and not ev.get("ok"):
|
| 70 |
+
return None # something failed; not a clean path
|
| 71 |
+
if kind == "tool_call":
|
| 72 |
+
name = str(ev.get("name") or "")
|
| 73 |
+
if name and (not names or names[-1] != name):
|
| 74 |
+
names.append(name)
|
| 75 |
+
|
| 76 |
+
if not (MIN_CHAIN <= len(names) <= MAX_CHAIN):
|
| 77 |
+
return None
|
| 78 |
+
return tuple(names)
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def promote(runs: Iterable[Dict[str, Any]], *,
|
| 82 |
+
min_users: int = MIN_USERS,
|
| 83 |
+
limit: int = 12,
|
| 84 |
+
today: Optional[_dt.date] = None,
|
| 85 |
+
enforce_gate: bool = True) -> Dict[str, Any]:
|
| 86 |
+
"""Tool chains proven by real completed runs, ranked by how many distinct
|
| 87 |
+
users completed them.
|
| 88 |
+
|
| 89 |
+
Cross-user, so it inherits the same date gate as every other aggregation
|
| 90 |
+
in this engine.
|
| 91 |
+
"""
|
| 92 |
+
day = today or _dt.date.today()
|
| 93 |
+
if enforce_gate and day < EFFECTIVE_DATE:
|
| 94 |
+
raise AggregationGateError(
|
| 95 |
+
f"Cross-user aggregation is gated until {EFFECTIVE_DATE.isoformat()} "
|
| 96 |
+
f"(Privacy/Terms v2.0 effective date); today is {day.isoformat()}."
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
runs = list(runs)
|
| 100 |
+
counts: Counter = Counter()
|
| 101 |
+
users: Dict[Tuple[str, ...], Set[str]] = defaultdict(set)
|
| 102 |
+
for run in runs:
|
| 103 |
+
chain = chain_of(run)
|
| 104 |
+
if chain is None:
|
| 105 |
+
continue
|
| 106 |
+
counts[chain] += 1
|
| 107 |
+
if _uid(run):
|
| 108 |
+
users[chain].add(_uid(run))
|
| 109 |
+
|
| 110 |
+
examples = [
|
| 111 |
+
{"chain": list(chain), "runs": n, "users": len(users[chain]),
|
| 112 |
+
"steps": len(chain)}
|
| 113 |
+
for chain, n in counts.most_common()
|
| 114 |
+
if len(users[chain]) >= min_users
|
| 115 |
+
]
|
| 116 |
+
return {
|
| 117 |
+
"ok": True,
|
| 118 |
+
"examples": examples[:limit],
|
| 119 |
+
"runs_considered": len(runs),
|
| 120 |
+
"chains_seen": len(counts),
|
| 121 |
+
"min_users": min_users,
|
| 122 |
+
"effective_date": EFFECTIVE_DATE.isoformat(),
|
| 123 |
+
"note": ("Tool names only. No prompts, no arguments, no sequences — a "
|
| 124 |
+
"worked example carrying the question that produced it would "
|
| 125 |
+
"be a cross-user transcript excerpt."),
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def as_prompt_section(report: Dict[str, Any], limit: int = 6) -> str:
|
| 130 |
+
"""The promoted chains, formatted for the agent's system prompt.
|
| 131 |
+
|
| 132 |
+
Returns "" when nothing clears the floor, and the caller must treat that
|
| 133 |
+
as "add no section" rather than "add an empty heading" — an empty
|
| 134 |
+
"PATHS THAT WORK" header reads as the engine having no idea what works.
|
| 135 |
+
"""
|
| 136 |
+
rows = (report or {}).get("examples") or []
|
| 137 |
+
if not rows:
|
| 138 |
+
return ""
|
| 139 |
+
lines = ["PATHS THAT HAVE WORKED",
|
| 140 |
+
"Observed from completed runs — not rules, and not a list of the "
|
| 141 |
+
"only valid routes. Prefer them when they fit; deviate when the "
|
| 142 |
+
"task differs."]
|
| 143 |
+
for r in rows[:limit]:
|
| 144 |
+
lines.append(f" {' -> '.join(r['chain'])} "
|
| 145 |
+
f"({r['users']} users, {r['runs']} runs)")
|
| 146 |
+
return "\n".join(lines)
|
dee/server.py
CHANGED
|
@@ -3258,6 +3258,45 @@ def create_app() -> Flask:
|
|
| 3258 |
return Response(_ls.as_text(rep), mimetype="text/plain")
|
| 3259 |
return jsonify(rep)
|
| 3260 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3261 |
@app.post("/api/admin/run-benchmarks")
|
| 3262 |
def admin_run_benchmarks() -> Response:
|
| 3263 |
"""Produce REAL validation numbers — runs the live ESM-2 model already
|
|
|
|
| 3258 |
return Response(_ls.as_text(rep), mimetype="text/plain")
|
| 3259 |
return jsonify(rep)
|
| 3260 |
|
| 3261 |
+
@app.get("/api/admin/worked-examples")
|
| 3262 |
+
def admin_worked_examples() -> Response:
|
| 3263 |
+
"""Tool chains real users have completed cleanly.
|
| 3264 |
+
|
| 3265 |
+
The other half of /api/admin/learn-signal: that one reads the corpus
|
| 3266 |
+
for what went wrong, this one for what went right. Same privacy
|
| 3267 |
+
regime — tool names only, k-anonymous, date-gated. These are fed back
|
| 3268 |
+
into the agent's system prompt, so this endpoint is also how you see
|
| 3269 |
+
what it is currently being told.
|
| 3270 |
+
"""
|
| 3271 |
+
from dee.core import worked_examples as _we
|
| 3272 |
+
from dee.core.aggregate import AggregationGateError as _Gate
|
| 3273 |
+
|
| 3274 |
+
if not _admin_ok():
|
| 3275 |
+
return jsonify({"ok": False, "error": "forbidden"}), 403
|
| 3276 |
+
try:
|
| 3277 |
+
limit = max(1, min(int(request.args.get("limit", 400)), 5000))
|
| 3278 |
+
except ValueError:
|
| 3279 |
+
limit = 400
|
| 3280 |
+
runs = _auth.scan_agent_runs(limit=limit)
|
| 3281 |
+
try:
|
| 3282 |
+
rep = _we.promote(runs)
|
| 3283 |
+
except _Gate as exc:
|
| 3284 |
+
return jsonify({"ok": False, "error": str(exc)}), 403
|
| 3285 |
+
if request.args.get("format") == "prompt":
|
| 3286 |
+
return Response(_we.as_prompt_section(rep) or
|
| 3287 |
+
"(nothing has cleared the k-anonymity floor yet)",
|
| 3288 |
+
mimetype="text/plain")
|
| 3289 |
+
return jsonify(rep)
|
| 3290 |
+
|
| 3291 |
+
@app.get("/api/admin/cache-stats")
|
| 3292 |
+
def admin_cache_stats() -> Response:
|
| 3293 |
+
"""Resolution-cache hit rate — the number that says whether repeat
|
| 3294 |
+
lookups are actually compounding."""
|
| 3295 |
+
from dee.core import resolution_cache as _rc
|
| 3296 |
+
if not _admin_ok():
|
| 3297 |
+
return jsonify({"ok": False, "error": "forbidden"}), 403
|
| 3298 |
+
return jsonify({"ok": True, "resolution_cache": _rc.stats()})
|
| 3299 |
+
|
| 3300 |
@app.post("/api/admin/run-benchmarks")
|
| 3301 |
def admin_run_benchmarks() -> Response:
|
| 3302 |
"""Produce REAL validation numbers — runs the live ESM-2 model already
|
tests/conftest.py
CHANGED
|
@@ -30,3 +30,22 @@ def _no_leaked_handover():
|
|
| 30 |
_orch._HANDOVERS.clear()
|
| 31 |
yield
|
| 32 |
_orch._HANDOVERS.clear()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
_orch._HANDOVERS.clear()
|
| 31 |
yield
|
| 32 |
_orch._HANDOVERS.clear()
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@pytest.fixture(autouse=True)
|
| 36 |
+
def _no_leaked_resolutions():
|
| 37 |
+
"""The resolution cache is process-global by design — it is shared across
|
| 38 |
+
users on purpose, because a gene symbol is public and re-fetching TP53 for
|
| 39 |
+
every visitor is the waste it exists to remove.
|
| 40 |
+
|
| 41 |
+
That makes it the same hazard as _HANDOVERS above. Caught immediately:
|
| 42 |
+
test_resolve_ensembl_tx resolves ENST00000269305 successfully, and
|
| 43 |
+
test_resolve_ensembl_fetch_failure then asserts the SAME id fails when
|
| 44 |
+
Ensembl returns nothing — which it does not, because the first test's
|
| 45 |
+
result is still cached. The product behaviour is correct (surviving a
|
| 46 |
+
network blip is the whole feature); the test needs a cold cache.
|
| 47 |
+
"""
|
| 48 |
+
from dee.core import resolution_cache as _rc
|
| 49 |
+
_rc.clear()
|
| 50 |
+
yield
|
| 51 |
+
_rc.clear()
|
tests/test_compounding.py
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Two ways the engine gets better with use, neither of which is a model.
|
| 2 |
+
|
| 3 |
+
The founder's ask was that "both engine and turing chat panel get smarter each
|
| 4 |
+
day". The tempting answer is to train something. The honest answer, given that
|
| 5 |
+
platform labs = 0 and there is no measured data to train on, is to make use
|
| 6 |
+
itself compound:
|
| 7 |
+
|
| 8 |
+
resolution cache the thousandth person to ask for TP53 gets an instant
|
| 9 |
+
answer BECAUSE nine hundred and ninety-nine asked first.
|
| 10 |
+
worked examples a tool chain N different users completed is evidence
|
| 11 |
+
about how this engine is actually driven, and it can go
|
| 12 |
+
straight back into the agent's prompt.
|
| 13 |
+
|
| 14 |
+
Neither can be quietly wrong. The cached value is byte-identical to what the
|
| 15 |
+
database returned, and the promoted chain is a count. That is the whole appeal
|
| 16 |
+
— a learned component that degrades silently is the failure this product is
|
| 17 |
+
positioned against.
|
| 18 |
+
"""
|
| 19 |
+
import datetime as _dt
|
| 20 |
+
import json
|
| 21 |
+
|
| 22 |
+
import pytest
|
| 23 |
+
|
| 24 |
+
from dee.core import resolution_cache as rc
|
| 25 |
+
from dee.core import worked_examples as we
|
| 26 |
+
from dee.core.aggregate import EFFECTIVE_DATE, AggregationGateError
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
@pytest.fixture(autouse=True)
|
| 30 |
+
def _clean():
|
| 31 |
+
rc.clear()
|
| 32 |
+
yield
|
| 33 |
+
rc.clear()
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def ev(kind, seq, **kw):
|
| 37 |
+
return dict(kind=kind, seq=seq, at=1754000000.0 + seq, **kw)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def clean_run(chain, user="u1", status="done", extra=()):
|
| 41 |
+
"""A run that called `chain` in order and finished."""
|
| 42 |
+
events = []
|
| 43 |
+
for i, name in enumerate(chain):
|
| 44 |
+
events.append(ev("tool_call", i * 2 + 1, id=f"c{i}", name=name))
|
| 45 |
+
events.append(ev("tool_result", i * 2 + 2, id=f"c{i}", name=name, ok=True))
|
| 46 |
+
events.extend(extra)
|
| 47 |
+
return {"run_id": f"r{user}{'-'.join(chain)}", "user_id": user,
|
| 48 |
+
"status": status, "events": events}
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
# --------------------------------------------------------------------------- #
|
| 52 |
+
# resolution cache
|
| 53 |
+
# --------------------------------------------------------------------------- #
|
| 54 |
+
RECORD = {"ok": True, "kind": "refseq", "sequence": "ATGC" * 50,
|
| 55 |
+
"label": "NM_000546 · 200 nt", "source": "ncbi", "gene_symbol": ""}
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def test_a_second_lookup_is_served_from_the_first():
|
| 59 |
+
assert rc.get("refseq", "NM_000546") is None
|
| 60 |
+
rc.put("refseq", "NM_000546", RECORD)
|
| 61 |
+
hit = rc.get("refseq", "NM_000546")
|
| 62 |
+
assert hit is not None
|
| 63 |
+
assert hit["sequence"] == RECORD["sequence"], "cached value must be identical"
|
| 64 |
+
assert hit["cached"] is True
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def test_a_pasted_sequence_is_never_stored_in_a_shared_cache():
|
| 68 |
+
"""The standing rule is that a user's own sequence never leaves the Space.
|
| 69 |
+
A cross-user cache is very much leaving."""
|
| 70 |
+
pasted = {"ok": True, "kind": "sequence", "sequence": "ACGT" * 40}
|
| 71 |
+
assert rc.cacheable("sequence") is False
|
| 72 |
+
assert rc.put("sequence", "ACGT" * 40, pasted) is False
|
| 73 |
+
assert rc.get("sequence", "ACGT" * 40) is None
|
| 74 |
+
assert rc.stats()["stores"] == 0
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def test_the_refusal_is_on_kind_not_on_a_list_of_approved_types():
|
| 78 |
+
"""A whitelist would mean each new public identifier type silently
|
| 79 |
+
bypasses the cache until someone remembers to add it."""
|
| 80 |
+
assert rc.cacheable("refseq") and rc.cacheable("uniprot")
|
| 81 |
+
assert rc.cacheable("some_new_database_added_next_year") is True
|
| 82 |
+
assert rc.cacheable("sequence") is False
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def test_a_failure_is_not_cached():
|
| 86 |
+
"""Ensembl down for ten seconds must not become 'this gene does not
|
| 87 |
+
exist' for the next fortnight."""
|
| 88 |
+
assert rc.put("symbol", "TP53", {"ok": False, "error": "timeout"}, "human") is False
|
| 89 |
+
assert rc.get("symbol", "TP53", "human") is None
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def test_organism_is_part_of_the_key():
|
| 93 |
+
"""TP53 exists in dozens of species and they are different sequences.
|
| 94 |
+
Serving human TP53 to someone who asked for zebrafish is worse than a
|
| 95 |
+
miss."""
|
| 96 |
+
rc.put("symbol", "TP53", RECORD, "human")
|
| 97 |
+
assert rc.get("symbol", "TP53", "human") is not None
|
| 98 |
+
assert rc.get("symbol", "TP53", "zebrafish") is None
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def test_keys_are_case_and_whitespace_insensitive():
|
| 102 |
+
rc.put("symbol", "TP53", RECORD, "human")
|
| 103 |
+
assert rc.get("symbol", " tp53 ", "Human") is not None
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def test_a_caller_mutating_the_result_cannot_poison_the_cache():
|
| 107 |
+
"""The hard-to-trace failure mode of any shared cache."""
|
| 108 |
+
rc.put("refseq", "NM_1", RECORD)
|
| 109 |
+
first = rc.get("refseq", "NM_1")
|
| 110 |
+
first["sequence"] = "TAMPERED"
|
| 111 |
+
assert rc.get("refseq", "NM_1")["sequence"] == RECORD["sequence"]
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def test_entries_expire_so_a_reannotated_record_is_refetched(monkeypatch):
|
| 115 |
+
rc.put("refseq", "NM_1", RECORD)
|
| 116 |
+
assert rc.get("refseq", "NM_1") is not None
|
| 117 |
+
monkeypatch.setattr(rc, "TTL_SECONDS", -1)
|
| 118 |
+
assert rc.get("refseq", "NM_1") is None
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def test_the_cache_is_bounded():
|
| 122 |
+
monkey = rc.MAX_ENTRIES
|
| 123 |
+
try:
|
| 124 |
+
rc.MAX_ENTRIES = 5
|
| 125 |
+
for i in range(20):
|
| 126 |
+
rc.put("refseq", f"NM_{i}", RECORD)
|
| 127 |
+
assert rc.stats()["entries"] <= 5
|
| 128 |
+
assert rc.stats()["evictions"] >= 15
|
| 129 |
+
finally:
|
| 130 |
+
rc.MAX_ENTRIES = monkey
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def test_the_resolver_serves_repeats_without_going_back_out(monkeypatch):
|
| 134 |
+
"""End to end through resolve_target, which is where it has to work."""
|
| 135 |
+
from dee.core import resolve
|
| 136 |
+
calls = []
|
| 137 |
+
|
| 138 |
+
def fake(text, organism, kind, val):
|
| 139 |
+
calls.append(val)
|
| 140 |
+
return {"ok": True, "kind": "refseq", "sequence": "ATGC" * 30,
|
| 141 |
+
"label": "x", "source": "ncbi", "gene_symbol": ""}
|
| 142 |
+
monkeypatch.setattr(resolve, "_resolve_uncached", fake)
|
| 143 |
+
|
| 144 |
+
a = resolve.resolve_target("NM_000546")
|
| 145 |
+
b = resolve.resolve_target("NM_000546")
|
| 146 |
+
assert calls == ["NM_000546"], "the second lookup went back to the network"
|
| 147 |
+
assert a["sequence"] == b["sequence"]
|
| 148 |
+
assert b.get("cached") is True
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def test_hit_rate_is_reported_because_it_is_the_whole_claim():
|
| 152 |
+
rc.put("refseq", "NM_1", RECORD)
|
| 153 |
+
rc.get("refseq", "NM_1")
|
| 154 |
+
rc.get("refseq", "NM_2")
|
| 155 |
+
assert rc.stats()["hit_rate"] == 0.5
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
# --------------------------------------------------------------------------- #
|
| 159 |
+
# worked examples
|
| 160 |
+
# --------------------------------------------------------------------------- #
|
| 161 |
+
BUILD = ("lookup_vector", "simulate_assembly", "check_synthesis")
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def test_a_chain_completed_by_enough_users_is_promoted():
|
| 165 |
+
runs = [clean_run(BUILD, user=f"u{i}") for i in range(4)]
|
| 166 |
+
out = we.promote(runs, min_users=3)
|
| 167 |
+
assert out["examples"][0]["chain"] == list(BUILD)
|
| 168 |
+
assert out["examples"][0]["users"] == 4
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def test_a_corrected_run_is_not_a_worked_example():
|
| 172 |
+
"""A run the user had to steer is a near-miss. Promoting it teaches the
|
| 173 |
+
agent the route that needed fixing."""
|
| 174 |
+
runs = [clean_run(BUILD, user=f"u{i}") for i in range(3)]
|
| 175 |
+
runs.append(clean_run(BUILD, user="u9", extra=[
|
| 176 |
+
ev("steer", 99, text="no, that's the wrong backbone")]))
|
| 177 |
+
out = we.promote(runs, min_users=1)
|
| 178 |
+
assert out["examples"][0]["users"] == 3, "the corrected run was counted"
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
def test_a_run_with_a_failed_tool_is_not_promoted():
|
| 182 |
+
bad = {"run_id": "b", "user_id": "u1", "status": "done", "events": [
|
| 183 |
+
ev("tool_call", 1, id="c1", name="fetch_sequence"),
|
| 184 |
+
ev("tool_result", 2, id="c1", name="fetch_sequence", ok=False, error="x"),
|
| 185 |
+
ev("tool_call", 3, id="c2", name="lookup_vector"),
|
| 186 |
+
ev("tool_result", 4, id="c2", name="lookup_vector", ok=True)]}
|
| 187 |
+
assert we.chain_of(bad) is None
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def test_an_unfinished_run_is_not_promoted():
|
| 191 |
+
assert we.chain_of(clean_run(BUILD, status="awaiting_input")) is None
|
| 192 |
+
assert we.chain_of(clean_run(BUILD, status="error")) is None
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def test_a_single_tool_call_is_not_a_path():
|
| 196 |
+
assert we.chain_of(clean_run(("fetch_sequence",))) is None
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def test_consecutive_repeats_collapse():
|
| 200 |
+
"""Fetching three genes is the same PATH as fetching one. Keeping the
|
| 201 |
+
repetition fragments the counts across chains that mean the same thing."""
|
| 202 |
+
chain = we.chain_of(clean_run(
|
| 203 |
+
("fetch_sequence", "fetch_sequence", "fetch_sequence", "fold_structure")))
|
| 204 |
+
assert chain == ("fetch_sequence", "fold_structure")
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def test_one_lab_s_unusual_workflow_is_not_published_to_everyone():
|
| 208 |
+
runs = [clean_run(BUILD, user=f"u{i}") for i in range(3)]
|
| 209 |
+
runs.append(clean_run(("design_crispr_guides", "check_prior_art"), user="solo"))
|
| 210 |
+
out = we.promote(runs, min_users=3)
|
| 211 |
+
chains = [tuple(e["chain"]) for e in out["examples"]]
|
| 212 |
+
assert BUILD in chains
|
| 213 |
+
assert ("design_crispr_guides", "check_prior_art") not in chains
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def test_no_user_content_survives_promotion():
|
| 217 |
+
"""Same rule as the field report: a worked example carrying the question
|
| 218 |
+
that produced it is a cross-user transcript excerpt."""
|
| 219 |
+
secret = "ZZQXSECRETZZ"
|
| 220 |
+
runs = [clean_run(BUILD, user=f"u{i}", extra=[
|
| 221 |
+
ev("user", 90, text=f"engineer {secret}"),
|
| 222 |
+
ev("text", 91, text=f"done with {secret}")]) for i in range(4)]
|
| 223 |
+
out = we.promote(runs, min_users=1)
|
| 224 |
+
assert secret not in json.dumps(out)
|
| 225 |
+
assert secret not in we.as_prompt_section(out)
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
def test_promotion_obeys_the_same_date_gate():
|
| 229 |
+
runs = [clean_run(BUILD, user=f"u{i}") for i in range(4)]
|
| 230 |
+
with pytest.raises(AggregationGateError):
|
| 231 |
+
we.promote(runs, today=EFFECTIVE_DATE - _dt.timedelta(days=1))
|
| 232 |
+
assert we.promote(runs, today=EFFECTIVE_DATE)["ok"] is True
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
def test_the_prompt_section_is_empty_when_nothing_qualifies():
|
| 236 |
+
"""An empty 'PATHS THAT WORK' heading reads as the engine having no idea
|
| 237 |
+
what works — worse than saying nothing."""
|
| 238 |
+
assert we.as_prompt_section(we.promote([], min_users=3)) == ""
|
| 239 |
+
assert we.as_prompt_section({}) == ""
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
def test_the_prompt_section_frames_them_as_evidence_not_rules():
|
| 243 |
+
"""Presented as law, an observed chain becomes a cage: the agent stops
|
| 244 |
+
solving tasks that need a different route."""
|
| 245 |
+
runs = [clean_run(BUILD, user=f"u{i}") for i in range(4)]
|
| 246 |
+
text = we.as_prompt_section(we.promote(runs, min_users=3))
|
| 247 |
+
assert "lookup_vector -> simulate_assembly" in text
|
| 248 |
+
assert "not rules" in text and "deviate" in text
|
| 249 |
+
assert "4 users" in text
|