File size: 3,906 Bytes
496163b 8551660 9af7af9 8551660 9af7af9 8551660 9af7af9 8551660 9af7af9 8551660 9af7af9 8551660 5e7271c 9af7af9 5e7271c 35dc819 5e7271c 35dc819 5e7271c 35dc819 5e7271c 35dc819 5e7271c 35dc819 5e7271c b14f0ba 5e7271c 9af7af9 8551660 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 | // src/verifier/verifier_core.mjs
import fs from "fs/promises";
import path from "path";
async function loadTemplate() {
const filePath = path.resolve(
path.dirname(new URL(import.meta.url).pathname),
"..",
"..",
"prompts",
"verifier_prompt.txt"
);
return await fs.readFile(filePath, "utf8");
}
export async function runVerifier({ question, context, gen }, provider) {
const tmpl = await loadTemplate();
const ctxText = context
.map((c) => c.content || c.text || "")
.join("\n\n---\n\n");
const prompt = tmpl
.replace(/{question}/g, question)
.replace(/{answer}/g, gen.answer || '')
.replace(/{context}/g, ctxText);
const raw = await provider.generate(prompt);
// Parse strict JSON format:
// {"REASONING": <bullet points>, "SCORE": <final score>}
let ok = false;
let score = null;
let reasoning = null;
let error = null;
const rawLower = typeof raw === 'string' ? raw.toLowerCase() : '';
if (/score/i.test(raw) && /pass/i.test(raw)) {
score = 'PASS';
ok = true;
} else if (/score/i.test(raw) && /fail/i.test(raw)) {
score = 'FAIL';
ok = false;
}
const safeParse = (txt) => {
try {
return JSON.parse(txt);
} catch {
return null;
}
};
const parseJsonLoose = (text) => {
if (!text || typeof text !== 'string') return null;
// Trim and try direct parse first
const direct = text.trim();
const parsedDirect = safeParse(direct);
if (parsedDirect) return parsedDirect;
// Heuristic: grab the substring between first { and last }
const start = direct.indexOf('{');
const end = direct.lastIndexOf('}');
let candidate =
start !== -1 && end !== -1 && end > start
? direct.slice(start, end + 1)
: direct;
// Fix stray PROMPT = ... prefix into JSON key
candidate = candidate.replace(/^\s*PROMPT\s*=/i, '"PROMPT":');
// Fix unquoted PASS/FAIL tokens after "SCORE":
candidate = candidate
.replace(/"SCORE"\s*:\s*PASS/gi, '"SCORE":"PASS"')
.replace(/"SCORE"\s*:\s*FAIL/gi, '"SCORE":"FAIL"')
// also tolerate SCORE: PASS without quotes or braces nearby
.replace(/\bSCORE\s*:\s*PASS\b/gi, '"SCORE":"PASS"')
.replace(/\bSCORE\s*:\s*FAIL\b/gi, '"SCORE":"FAIL"');
// If still missing braces, wrap
const trimmed = candidate.trim();
const hasLeadingBrace = trimmed.startsWith('{');
const hasTrailingBrace = trimmed.endsWith('}');
if (!hasLeadingBrace) candidate = `{${candidate}`;
if (!hasTrailingBrace) candidate = `${candidate}}`;
return safeParse(candidate);
};
const parsed = parseJsonLoose(raw);
if (!parsed) {
error = 'invalid_json';
} else {
reasoning = parsed?.REASONING ?? null;
if (Object.prototype.hasOwnProperty.call(parsed, 'SCORE')) {
const s = parsed.SCORE;
// Accept PASS/FAIL strings from finetuned verifier
if (typeof s === 'string') {
const trimmed = s.trim().toLowerCase();
if (trimmed === 'pass') {
score = 'PASS';
ok = true;
} else if (trimmed === 'fail') {
score = 'FAIL';
ok = false;
} else {
const num = Number(s);
if (Number.isFinite(num)) {
score = num;
ok = num >= 0.5;
}
}
} else if (typeof s === 'number') {
score = s;
ok = s >= 0.5;
}
} else {
error = 'missing_score';
}
}
// Fallback: raw PASS/FAIL tokens even if parsing failed
if (!ok && typeof raw === 'string') {
if (/pass/i.test(raw) && !/fail/i.test(raw)) {
score = score ?? 'PASS';
ok = true;
error = null;
} else if (/fail/i.test(raw) && !/pass/i.test(raw)) {
score = score ?? 'FAIL';
ok = false;
error = null;
}
}
return { raw, ok, score, reasoning, error };
}
export default { runVerifier };
|