#!/usr/bin/env node // scripts/gold_preview.mjs // Quick preview of gold JSONL entries (questions and answers). import fs from 'fs/promises'; import path from 'path'; import { fileURLToPath } from 'url'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const PROJECT_ROOT = path.join(__dirname, '..'); function parseArgs() { const args = process.argv.slice(2); let limit = 5; let fileArg; let full = false; let maxQuestion = 500; let maxAnswer = 800; let maxContext = 300; for (let i = 0; i < args.length; i++) { const a = args[i]; if (a === '--limit' || a === '-n') { const v = Number(args[i + 1]); if (Number.isFinite(v)) limit = v; i++; } else if (a === '--file' || a === '-f') { fileArg = args[i + 1]; i++; } else if (a === '--full') { full = true; } else if (a === '--max-question') { const v = Number(args[i + 1]); if (Number.isFinite(v)) maxQuestion = v; i++; } else if (a === '--max-answer') { const v = Number(args[i + 1]); if (Number.isFinite(v)) maxAnswer = v; i++; } else if (a === '--max-context') { const v = Number(args[i + 1]); if (Number.isFinite(v)) maxContext = v; i++; } } const goldPath = fileArg || process.env.GOLD_PATH || path.join(PROJECT_ROOT, 'gold', 'pipeline_gold.jsonl'); if (full) { maxQuestion = Infinity; maxAnswer = Infinity; maxContext = Infinity; } return { limit, goldPath, full, maxQuestion, maxAnswer, maxContext }; } function preview(text, max = 200, full = false) { if (full) return Array.isArray(text) ? text.join(' ') : String(text ?? ''); if (text == null) return ''; const str = Array.isArray(text) ? text.join(' ') : String(text); if (str.length <= max) return str; return str.slice(0, max) + `… [+${str.length - max} chars]`; } async function main() { const { limit, goldPath, full, maxQuestion, maxAnswer, maxContext, } = parseArgs(); let raw; try { raw = await fs.readFile(goldPath, 'utf8'); } catch (e) { if (e.code === 'ENOENT') { console.error(`Gold file not found: ${goldPath}`); process.exit(1); } throw e; } const lines = raw .split('\n') .map((l) => l.trim()) .filter(Boolean) .slice(0, limit); console.log(`Gold preview (${lines.length} of max ${limit}) from ${goldPath}\n`); lines.forEach((line, idx) => { let obj; try { obj = JSON.parse(line); } catch { console.log(`#${idx + 1}: [invalid JSON] ${preview(line, 120)}`); return; } const q = obj.question || '[no question]'; const ans = obj.sample?.answer || obj.sample?.raw || '[no answer]'; const rawGen = obj.sample?.raw; const thought = obj.sample?.thought; const thinking = obj.sample?.thinking; const confidence = obj.sample?.confidence ?? obj.sample?.confidence_level; const evidence = obj.sample?.evidence; const limitations = obj.sample?.limitations; const chunkId = obj.sourceChunkId || obj.context?.[0]?.id || '[unknown chunk]'; const ctxSnippet = obj.context?.[0]?.content || obj.sourceChunk || ''; const rew = obj.reward?.score ?? obj.reward?.ok; const verOk = obj.verifier?.ok ?? obj.ver?.ok; const verScore = obj.verifier?.score ?? obj.ver?.score; console.log(`#${idx + 1}`); console.log(`Chunk: ${chunkId}`); console.log(`Q: ${preview(q, maxQuestion, full)}`); console.log(`A: ${preview(ans, maxAnswer, full)}`); if (thought !== undefined) { const tVal = typeof thought === 'string' ? thought : JSON.stringify(thought, null, 2); console.log(`Thought: ${preview(tVal, maxAnswer, full)}`); } if (rawGen !== undefined) { console.log(`Raw: ${preview(rawGen, maxAnswer, full)}`); } if (confidence !== undefined) console.log(`Gen confidence: ${confidence}`); if (evidence) console.log(`Evidence: ${preview(Array.isArray(evidence) ? evidence.join(' | ') : evidence, 400, full)}`); if (limitations) console.log(`Limitations: ${preview(limitations, 200, full)}`); if (thinking !== undefined) { const tVal = typeof thinking === 'string' ? thinking : JSON.stringify(thinking, null, 2); console.log(`Thinking: ${preview(tVal, maxAnswer, full)}`); } if (ctxSnippet) console.log(`Ctx: ${preview(ctxSnippet, maxContext, full)}`); if (verOk !== undefined) console.log(`Verifier ok: ${verOk}${verScore !== undefined ? ` (score: ${verScore})` : ''}`); if (rew !== undefined) console.log(`Reward: ${rew}`); console.log(''); }); } // Exported for tests export async function capturePreview() { const { limit, goldPath, full, maxQuestion, maxAnswer, maxContext, } = parseArgs(); let raw; try { raw = await fs.readFile(goldPath, 'utf8'); } catch (e) { if (e.code === 'ENOENT') { throw new Error(`Gold file not found: ${goldPath}`); } throw e; } const lines = raw .split('\n') .map((l) => l.trim()) .filter(Boolean) .slice(0, limit); const chunks = []; chunks.push(`Gold preview (${lines.length} of max ${limit}) from ${goldPath}\n`); lines.forEach((line, idx) => { let obj; try { obj = JSON.parse(line); } catch { chunks.push(`#${idx + 1}: [invalid JSON] ${preview(line, 120)}`); return; } const q = obj.question || '[no question]'; const ans = obj.sample?.answer || obj.sample?.raw || '[no answer]'; const chunkId = obj.sourceChunkId || obj.context?.[0]?.id || '[unknown chunk]'; const ctxSnippet = obj.context?.[0]?.content || obj.sourceChunk || ''; const rew = obj.reward?.score ?? obj.reward?.ok; const verOk = obj.verifier?.ok ?? obj.ver?.ok; const verScore = obj.verifier?.score ?? obj.ver?.score; chunks.push(`#${idx + 1}`); chunks.push(`Chunk: ${chunkId}`); chunks.push(`Q: ${preview(q, maxQuestion, full)}`); chunks.push(`A: ${preview(ans, maxAnswer, full)}`); if (ctxSnippet) chunks.push(`Ctx: ${preview(ctxSnippet, maxContext, full)}`); if (verOk !== undefined) chunks.push(`Verifier ok: ${verOk}${verScore !== undefined ? ` (score: ${verScore})` : ''}`); if (rew !== undefined) chunks.push(`Reward: ${rew}`); chunks.push(''); }); return chunks.join('\n'); } if (process.argv[1] && process.argv[1].endsWith('gold_preview.mjs')) { main().catch((err) => { console.error('Gold preview error:', err); process.exit(1); }); }