fieldnotes-rag / worker.mjs
iankamar's picture
Fix answer relevance, ownership context and unsupported-budget refusal; pin models and record 10 passing live checks
f33fa8f verified
Raw History Blame Contribute Delete
9.46 kB
import { retrievalQuestion, supportsQuestion } from "./relevance.mjs?v=3";
import { ANSWER_CASES, checkAnswer } from "./audit.mjs?v=3";
import {
pipeline,
env,
LogitsProcessor,
LogitsProcessorList,
} from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.7.2";
import { retrieve, selectionMessages, selectionAnswer } from "./rag.mjs";
import {
hybridSearch,
retrievalMetrics,
rankSemantic,
verifySelection,
} from "./search.mjs?v=3";
import { CORPUS, BENCHMARK } from "./corpus.mjs";
env.allowLocalModels = false;
env.backends.onnx.wasm.numThreads = 1;
let generator,
embedder,
cacheKey = "",
vectors = [];
let requestId;
function send(type, data = {}) {
self.postMessage({ type, requestId, ...data });
}
function status(text) {
send("status", { text });
}
async function embed(texts) {
if (!embedder) {
status("Loading the small semantic search model…");
embedder = await pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2", {
device: "wasm",
revision: "751bff37182d3f1213fa05d7196b954e230abad9",
dtype: "q8",
});
}
const result = [];
for (let i = 0; i < texts.length; i += 12) {
const batch = await embedder(texts.slice(i, i + 12), {
pooling: "mean",
normalize: true,
});
result.push(...batch.tolist());
}
return result;
}
async function index(chunks) {
const key = JSON.stringify(chunks.map((c) => [c.source, c.location, c.text]));
if (key !== cacheKey) {
status(`Embedding ${chunks.length} document passages locally…`);
const next = await embed(chunks.map((c) => c.text));
vectors = next;
cacheKey = key;
}
return vectors;
}
class EvidenceOnly extends LogitsProcessor {
constructor(ids) {
super();
this.ids = ids;
}
_call(_, logits) {
const kept = this.ids.map((i) => logits.data[i]);
logits.data.fill(-Infinity);
this.ids.forEach((id, i) => {
logits.data[id] = kept[i];
});
return logits;
}
}
async function select(question, hits) {
if (!generator) {
status("Loading the browser LLM (~500 MB on first use)…");
generator = await pipeline(
"text-generation",
"onnx-community/Qwen2.5-0.5B-Instruct",
{
device: "wasm",
revision: "22942cb7d7ba4cc81bb4673549ca4d4614469b5e",
dtype: "q8",
progress_callback: (p) => {
if (p.status === "progress")
status(`LLM download ${Math.round(p.progress || 0)}% · ${p.file}`);
},
},
);
}
const sentences = hits.flatMap((h, sourceIndex) =>
h.text
.split(/\bwho\b/i.test(question) ? /(?!) / : /(?<=[.!?])\s+/)
.filter((s) => s.trim())
.map((text) => ({
text,
source: h.source,
location: h.location,
sourceIndex,
})),
);
const sentenceVectors = await embed(sentences.map((s) => s.text));
const [query] = await embed([retrievalQuestion(question)]);
const candidates = rankSemantic(sentences, sentenceVectors, query).filter(c => supportsQuestion(question, c.text)).slice(
0,
8,
);
if (!candidates.length) return { text: "I cannot determine the answer from the retrieved evidence.", method: "withheld" };
const ids = candidates.map((_, i) =>
generator.tokenizer.encode(String(i + 1), { add_special_tokens: false }),
);
if (ids.some((x) => x.length !== 1))
throw Error("Sentence IDs must be single tokens.");
const processors = new LogitsProcessorList();
processors.push(new EvidenceOnly(ids.map((x) => Number(x[0]))));
status("Selecting a source quotation with constrained LLM decoding…");
const output = await generator(selectionMessages(question, candidates), {
max_new_tokens: 1,
logits_processor: processors,
do_sample: false,
return_full_text: false,
});
const generated = output[0].generated_text;
const raw =
typeof generated === "string" ? generated : generated.at(-1).content;
const checked = verifySelection(
candidates,
/^[1-8]$/.test(raw.trim()) ? Number(raw.trim()) - 1 : -1,
);
if (!checked.candidate)
return {
text: "No answer passed the relevance check. Inspect the retrieved passages.",
method: "withheld",
};
if (checked.method === "semantic-fallback")
return {
text: `Closest evidence (model selection disagreed):\n${checked.candidate.text} [${checked.candidate.sourceIndex + 1}]`,
method: checked.method,
guard: "question constraints and sentence semantic margin 0.03",
};
return {
text: selectionAnswer(raw, candidates),
method: checked.method,
guard: "question constraints and sentence semantic margin 0.03",
};
}
async function evaluate() {
const start = performance.now(),
v = await index(CORPUS),
queries = await embed(BENCHMARK.map((q) => q.question));
const rows = BENCHMARK.map((q, i) => {
const baseline = retrieve(q.question, CORPUS, 3),
hybrid = hybridSearch(q.question, CORPUS, v, queries[i]);
return {
...q,
baseline: baseline.map((c) => c.id),
hybrid: hybrid.map((c) => c.id),
similarity: rankSemantic(CORPUS, v, queries[i])[0]?.semantic,
};
});
const metrics = (key) =>
retrievalMetrics(rows.map((r) => ({ ...r, ids: r[key] })));
return {
version: "fieldnotes-v2.1-synthetic-30",
embeddingModel: "Xenova/all-MiniLM-L6-v2",
threshold: 0.3,
fusion: "RRF semantic 2 : BM25 1, constant 60",
currentSourceRule:
"Explicit current/latest/active/effective queries exclude sources whose provenance is marked superseded",
scope:
"Retrieval only. Not LLM answer accuracy. Agent-authored synthetic queries and labels; not an independent benchmark.",
baseline: metrics("baseline"),
hybrid: metrics("hybrid"),
elapsedMs: Math.round(performance.now() - start),
rows,
};
}
async function answerAudit() {
const started = performance.now(),
v = await index(CORPUS),
rows = [];
for (const [i, test] of ANSWER_CASES.entries()) {
status(`Answer check ${i + 1}/${ANSWER_CASES.length}: ${test.question}`);
const tick = performance.now(),
[query] = await embed([retrievalQuestion(test.question)]);
const hits = hybridSearch(retrievalQuestion(test.question), CORPUS, v, query, { k: 8 });
const result = hits.length
? await select(test.question, hits)
: {
text: "No evidence met the retrieval threshold.",
method: "withheld",
};
rows.push({
question: test.question,
answer: result.text,
method: result.method,
check: checkAnswer(test, result.text, hits),
sources: hits.map((h) => ({
id: h.id,
source: h.source,
location: h.location,
})),
elapsedMs: Math.round(performance.now() - tick),
});
send("audit-progress", {
complete: rows.length,
total: ANSWER_CASES.length,
});
}
return {
version: "fieldnotes-answer-audit-2",
scope:
"Ten synthetic development checks, including two audit regression questions. Required facts, citation IDs and two refusals; not independent answer accuracy.",
passed: rows.filter((r) => r.check.pass).length,
total: rows.length,
fallbacks: rows.filter((r) => r.method === "semantic-fallback").length,
elapsedMs: Math.round(performance.now() - started),
rows,
};
}
// The UI permits only one job. Clear terminates this worker; IDs reject stale replies.
self.onmessage = async ({ data }) => {
requestId = data.requestId;
try {
if (data.action === "answer-audit") {
send("answer-audit", { result: await answerAudit() });
return;
}
if (data.action === "evaluate") {
send("evaluation", { result: await evaluate() });
return;
}
const start = performance.now();
if (
!data.question?.trim() ||
data.question.length > 1500 ||
!data.chunks?.length ||
data.chunks.length > 600
)
throw Error("Invalid question or document count.");
let hits;
if (data.mode === "lexical") hits = retrieve(data.question, data.chunks, 3);
else {
const v = await index(data.chunks),
[q] = await embed([retrievalQuestion(data.question)]);
hits = hybridSearch(retrievalQuestion(data.question), data.chunks, v, q, { k: 8 });
}
send("evidence", { hits });
const retrievalMs = Math.round(performance.now() - start);
const selected =
data.generate && hits.length ? await select(data.question, hits) : null;
const answer = !hits.length
? "No evidence met the retrieval threshold. Try another question or inspect the source documents."
: (selected?.text ?? "Evidence-only mode. No LLM answer was generated.");
send("answer", {
text: answer,
hits,
metadata: {
retrievalMs,
totalMs: Math.round(performance.now() - start),
answerMethod: selected?.method ?? "retrieval-only",
relevanceGuard: selected?.guard ?? null,
embeddingModel:
data.mode === "lexical" ? null : "Xenova/all-MiniLM-L6-v2",
llm: data.generate && hits.length ? "Qwen2.5-0.5B-Instruct q8" : null,
threshold: data.mode === "lexical" ? null : 0.3,
returnedPassages: hits.length,
},
});
} catch (e) {
send("error", {
text:
"Local processing failed. You can cancel, retry, or use keyword search with Evidence only. " +
String(e?.message || e).slice(0, 180),
});
}
};