scripts/gold_preview.mjs

#!/usr/bin/env node
// scripts/gold_preview.mjs
// Quick preview of gold JSONL entries (questions and answers).

import fs from 'fs/promises';
import path from 'path';
import { fileURLToPath } from 'url';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const PROJECT_ROOT = path.join(__dirname, '..');

function parseArgs() {
  const args = process.argv.slice(2);
  let limit = 5;
  let fileArg;
  let full = false;
  let maxQuestion = 500;
  let maxAnswer = 800;
  let maxContext = 300;

  for (let i = 0; i < args.length; i++) {
    const a = args[i];
    if (a === '--limit' || a === '-n') {
      const v = Number(args[i + 1]);
      if (Number.isFinite(v)) limit = v;
      i++;
    } else if (a === '--file' || a === '-f') {
      fileArg = args[i + 1];
      i++;
    } else if (a === '--full') {
      full = true;
    } else if (a === '--max-question') {
      const v = Number(args[i + 1]);
      if (Number.isFinite(v)) maxQuestion = v;
      i++;
    } else if (a === '--max-answer') {
      const v = Number(args[i + 1]);
      if (Number.isFinite(v)) maxAnswer = v;
      i++;
    } else if (a === '--max-context') {
      const v = Number(args[i + 1]);
      if (Number.isFinite(v)) maxContext = v;
      i++;
    }
  }

  const goldPath =
    fileArg ||
    process.env.GOLD_PATH ||
    path.join(PROJECT_ROOT, 'gold', 'pipeline_gold.jsonl');

  if (full) {
    maxQuestion = Infinity;
    maxAnswer = Infinity;
    maxContext = Infinity;
  }

  return { limit, goldPath, full, maxQuestion, maxAnswer, maxContext };
}

function preview(text, max = 200, full = false) {
  if (full) return Array.isArray(text) ? text.join(' ') : String(text ?? '');
  if (text == null) return '';
  const str = Array.isArray(text) ? text.join(' ') : String(text);
  if (str.length <= max) return str;
  return str.slice(0, max) + `… [+${str.length - max} chars]`;
}

async function main() {
  const {
    limit,
    goldPath,
    full,
    maxQuestion,
    maxAnswer,
    maxContext,
  } = parseArgs();

  let raw;
  try {
    raw = await fs.readFile(goldPath, 'utf8');
  } catch (e) {
    if (e.code === 'ENOENT') {
      console.error(`Gold file not found: ${goldPath}`);
      process.exit(1);
    }
    throw e;
  }

  const lines = raw
    .split('\n')
    .map((l) => l.trim())
    .filter(Boolean)
    .slice(0, limit);

  console.log(`Gold preview (${lines.length} of max ${limit}) from ${goldPath}\n`);

  lines.forEach((line, idx) => {
    let obj;
    try {
      obj = JSON.parse(line);
    } catch {
      console.log(`#${idx + 1}: [invalid JSON] ${preview(line, 120)}`);
      return;
    }

    const q = obj.question || '[no question]';
    const ans = obj.sample?.answer || obj.sample?.raw || '[no answer]';
    const rawGen = obj.sample?.raw;
    const thought = obj.sample?.thought;
    const thinking = obj.sample?.thinking;
    const confidence = obj.sample?.confidence ?? obj.sample?.confidence_level;
    const evidence = obj.sample?.evidence;
    const limitations = obj.sample?.limitations;
    const chunkId = obj.sourceChunkId || obj.context?.[0]?.id || '[unknown chunk]';
    const ctxSnippet = obj.context?.[0]?.content || obj.sourceChunk || '';
    const rew = obj.reward?.score ?? obj.reward?.ok;
    const verOk = obj.verifier?.ok ?? obj.ver?.ok;
    const verScore = obj.verifier?.score ?? obj.ver?.score;

    console.log(`#${idx + 1}`);
    console.log(`Chunk: ${chunkId}`);
    console.log(`Q: ${preview(q, maxQuestion, full)}`);
    console.log(`A: ${preview(ans, maxAnswer, full)}`);
    if (thought !== undefined) {
      const tVal =
        typeof thought === 'string'
          ? thought
          : JSON.stringify(thought, null, 2);
      console.log(`Thought: ${preview(tVal, maxAnswer, full)}`);
    }
    if (rawGen !== undefined) {
      console.log(`Raw: ${preview(rawGen, maxAnswer, full)}`);
    }
    if (confidence !== undefined) console.log(`Gen confidence: ${confidence}`);
    if (evidence) console.log(`Evidence: ${preview(Array.isArray(evidence) ? evidence.join(' | ') : evidence, 400, full)}`);
    if (limitations) console.log(`Limitations: ${preview(limitations, 200, full)}`);
    if (thinking !== undefined) {
      const tVal =
        typeof thinking === 'string'
          ? thinking
          : JSON.stringify(thinking, null, 2);
      console.log(`Thinking: ${preview(tVal, maxAnswer, full)}`);
    }
    if (ctxSnippet) console.log(`Ctx: ${preview(ctxSnippet, maxContext, full)}`);
    if (verOk !== undefined) console.log(`Verifier ok: ${verOk}${verScore !== undefined ? ` (score: ${verScore})` : ''}`);
    if (rew !== undefined) console.log(`Reward: ${rew}`);
    console.log('');
  });
}

// Exported for tests
export async function capturePreview() {
  const {
    limit,
    goldPath,
    full,
    maxQuestion,
    maxAnswer,
    maxContext,
  } = parseArgs();

  let raw;
  try {
    raw = await fs.readFile(goldPath, 'utf8');
  } catch (e) {
    if (e.code === 'ENOENT') {
      throw new Error(`Gold file not found: ${goldPath}`);
    }
    throw e;
  }

  const lines = raw
    .split('\n')
    .map((l) => l.trim())
    .filter(Boolean)
    .slice(0, limit);

  const chunks = [];

  chunks.push(`Gold preview (${lines.length} of max ${limit}) from ${goldPath}\n`);

  lines.forEach((line, idx) => {
    let obj;
    try {
      obj = JSON.parse(line);
    } catch {
      chunks.push(`#${idx + 1}: [invalid JSON] ${preview(line, 120)}`);
      return;
    }

    const q = obj.question || '[no question]';
    const ans = obj.sample?.answer || obj.sample?.raw || '[no answer]';
    const chunkId = obj.sourceChunkId || obj.context?.[0]?.id || '[unknown chunk]';
    const ctxSnippet = obj.context?.[0]?.content || obj.sourceChunk || '';
    const rew = obj.reward?.score ?? obj.reward?.ok;
    const verOk = obj.verifier?.ok ?? obj.ver?.ok;
    const verScore = obj.verifier?.score ?? obj.ver?.score;

    chunks.push(`#${idx + 1}`);
    chunks.push(`Chunk: ${chunkId}`);
    chunks.push(`Q: ${preview(q, maxQuestion, full)}`);
    chunks.push(`A: ${preview(ans, maxAnswer, full)}`);
    if (ctxSnippet) chunks.push(`Ctx: ${preview(ctxSnippet, maxContext, full)}`);
    if (verOk !== undefined) chunks.push(`Verifier ok: ${verOk}${verScore !== undefined ? ` (score: ${verScore})` : ''}`);
    if (rew !== undefined) chunks.push(`Reward: ${rew}`);
    chunks.push('');
  });

  return chunks.join('\n');
}

if (process.argv[1] && process.argv[1].endsWith('gold_preview.mjs')) {
  main().catch((err) => {
    console.error('Gold preview error:', err);
    process.exit(1);
  });
}