frontend/viz2d-demo/scripts/build-qa-summary.mjs

#!/usr/bin/env node

import { mkdir, readFile, writeFile } from "node:fs/promises";
import { existsSync } from "node:fs";
import path from "node:path";

const cwd = process.cwd();
const reportPath = path.resolve(cwd, process.argv[2] ?? "test-results/real-qa.json");
const outputPath = path.resolve(cwd, process.argv[3] ?? "test-results/qa-summary.md");
const outputDir = path.dirname(outputPath);

try {
  await main();
} catch (error) {
  await writeFailureSummary(error);
  console.error(error);
}

async function main() {
  await mkdir(outputDir, { recursive: true });

  if (!existsSync(reportPath)) {
    await writeFile(
      outputPath,
      [
        "# Real Browser QA Summary",
        "",
        `Playwright JSON report was not found at \`${relativeToCwd(reportPath)}\`.`,
        "",
        "Run the Playwright suite first so this summary can include screenshots and Gemini judgments.",
        "",
      ].join("\n"),
    );
    console.warn(`QA summary skipped source report: ${relativeToCwd(reportPath)}`);
    return;
  }

  const report = JSON.parse(await readFile(reportPath, "utf8"));
  const entries = collectEntries(report);
  const hydratedEntries = [];

  for (const entry of entries) {
    const result = finalResult(entry.test);
    const groupedEntries = await groupedCaseEntries(entry, result);
    if (groupedEntries.length) {
      hydratedEntries.push(...groupedEntries);
      continue;
    }

    const beforeImage = await imageAttachmentDataUri(result, "before-canvas");
    const afterImage = await imageAttachmentDataUri(result, "after-texture-canvas");
    const geminiAttachment = findAttachment(result, "gemini-judge");
    const gemini = geminiAttachment ? parseJsonLoose(await readTextAttachment(geminiAttachment)) : null;

    hydratedEntries.push({
      ...entry,
      afterImage,
      beforeImage,
      gemini,
      result,
      status: displayStatus(entry.test, result),
    });
  }

  await writeFile(outputPath, buildMarkdown(hydratedEntries));
  console.log(`QA markdown summary written to ${relativeToCwd(outputPath)}`);
}

function collectEntries(report) {
  const entries = [];

  for (const suite of asArray(report.suites)) {
    collectSuite(suite, entries);
  }

  return entries;
}

async function writeFailureSummary(error) {
  await mkdir(outputDir, { recursive: true });
  const message = error instanceof Error ? error.stack ?? error.message : String(error);

  await writeFile(
    outputPath,
    [
      "# Real Browser QA Summary",
      "",
      "The compact Markdown summary could not be generated.",
      "",
      `Source report: \`${relativeToCwd(reportPath)}\``,
      "",
      "```text",
      message,
      "```",
      "",
      "The full Playwright artifacts are still available in this artifact bundle.",
      "",
    ].join("\n"),
  );
}

function collectSuite(suite, entries) {
  for (const spec of asArray(suite.specs)) {
    const title = typeof spec.title === "string" && spec.title.trim() ? spec.title.trim() : "Untitled QA case";
    for (const test of asArray(spec.tests)) {
      entries.push({ spec, test, title });
    }
  }

  for (const child of asArray(suite.suites)) {
    collectSuite(child, entries);
  }
}

function finalResult(test) {
  const results = asArray(test.results);
  return results[results.length - 1] ?? {};
}

function displayStatus(test, result) {
  const resultStatus = typeof result.status === "string" ? result.status : "";
  const testStatus = typeof test.status === "string" ? test.status : "";

  if (resultStatus) return resultStatus;
  if (testStatus === "expected") return "passed";
  if (testStatus === "unexpected") return "failed";
  return testStatus || "unknown";
}

async function groupedCaseEntries(entry, result) {
  const metas = await groupedCaseMetas(result);
  const hydratedEntries = [];

  for (const { meta, prefix } of metas) {
    const beforeImage = await imageAttachmentDataUri(result, `${prefix}-before-canvas`);
    const afterImage = await imageAttachmentDataUri(result, `${prefix}-after-texture-canvas`);
    const geminiAttachment = findAttachment(result, `${prefix}-gemini-judge`);
    const gemini = geminiAttachment ? parseJsonLoose(await readTextAttachment(geminiAttachment)) : null;
    const imageTitle = meta.imageName || meta.roomName || imageName(entry.title);
    const tileName = meta.tileName || "Unknown tile";

    hydratedEntries.push({
      ...entry,
      afterImage,
      beforeImage,
      gemini,
      result,
      status: typeof meta.status === "string" ? meta.status : displayStatus(entry.test, result),
      title: `${imageTitle}: apply ${tileName}`,
    });
  }

  return hydratedEntries;
}

async function groupedCaseMetas(result) {
  const metas = [];
  for (const attachment of asArray(result.attachments)) {
    const name = typeof attachment?.name === "string" ? attachment.name : "";
    if (!name.startsWith("qa-case-") || !name.endsWith("-meta")) continue;
    const meta = parseJsonLoose(await readTextAttachment(attachment));
    if (!meta || typeof meta !== "object" || Array.isArray(meta)) continue;
    metas.push({
      meta,
      prefix: typeof meta.attachmentPrefix === "string" && meta.attachmentPrefix
        ? meta.attachmentPrefix
        : name.slice(0, -"-meta".length),
    });
  }

  return metas.sort((left, right) => Number(left.meta.caseIndex ?? 0) - Number(right.meta.caseIndex ?? 0));
}

async function imageAttachmentDataUri(result, attachmentName) {
  const attachment = findAttachment(result, attachmentName);
  if (!attachment) return null;

  const sourcePath = resolveAttachmentPath(attachment.path);
  const contentType = imageContentType(attachment);

  if (sourcePath) {
    const buffer = await readFile(sourcePath);
    return `data:${contentType};base64,${buffer.toString("base64")}`;
  }

  if (typeof attachment.body === "string") {
    return `data:${contentType};base64,${attachment.body}`;
  }

  return null;
}

function findAttachment(result, name) {
  return asArray(result.attachments).find((attachment) => attachment?.name === name) ?? null;
}

async function readTextAttachment(attachment) {
  const sourcePath = resolveAttachmentPath(attachment.path);
  if (sourcePath) return readFile(sourcePath, "utf8");

  if (typeof attachment.body !== "string") return "";

  const direct = attachment.body.trim();
  if (direct.startsWith("{") || direct.startsWith("[")) return attachment.body;

  return Buffer.from(attachment.body, "base64").toString("utf8");
}

function resolveAttachmentPath(value) {
  if (typeof value !== "string" || !value.trim()) return null;

  const candidates = [
    path.isAbsolute(value) ? value : path.resolve(cwd, value),
    path.resolve(path.dirname(reportPath), value),
  ];

  return candidates.find((candidate) => existsSync(candidate)) ?? null;
}

function parseJsonLoose(text) {
  if (!text.trim()) return null;

  try {
    return JSON.parse(text);
  } catch {
    const match = text.match(/\{[\s\S]*\}/);
    if (!match) return null;
    try {
      return JSON.parse(match[0]);
    } catch {
      return null;
    }
  }
}

function buildMarkdown(entries) {
  const passed = entries.filter((entry) => entry.status === "passed").length;
  const skipped = entries.filter((entry) => entry.status === "skipped").length;
  const needsReview = entries.length - passed - skipped;
  const lines = [
    "# Real Browser QA Summary",
    "",
    `Generated from \`${relativeToCwd(reportPath)}\`.`,
    "",
    `Total images: ${entries.length}`,
    `Passed: ${passed}`,
    `Needs review: ${needsReview}`,
    `Skipped: ${skipped}`,
    "",
  ];

  for (const [index, entry] of entries.entries()) {
    lines.push("", `## ${index + 1}. ${escapeHeading(imageName(entry.title))}`, "");
    lines.push(`Status: **${entry.status}**`, "");
    lines.push("### Before", "");
    lines.push(imageTag(entry.beforeImage), "");
    lines.push("### After Tile Applied", "");
    lines.push(imageTag(entry.afterImage), "");
    lines.push("", "### Gemini Judgment", "");
    lines.push(...geminiSummary(entry.gemini));
  }

  return `${lines.join("\n")}\n`;
}

function geminiSummary(gemini) {
  if (!gemini) {
    return ["No Gemini judgment attachment was found for this image.", ""];
  }

  const lines = [
    `- Overall pass: ${yesNo(gemini.overallPass)}`,
    `- Tile applied: ${yesNo(gemini.tileApplied)}`,
    `- Coverage gap detected: ${yesNo(gemini.coverageGapDetected)}`,
    `- Missed floor areas: ${listOrNone(gemini.missedFloorAreas)}`,
    `- Direction mismatch detected: ${yesNo(gemini.directionMismatchDetected)}`,
    `- Direction issues: ${listOrNone(gemini.directionIssues)}`,
    `- Bleed detected: ${yesNo(gemini.bleedDetected)}`,
    `- Scores: surface ${score(gemini.surfaceCorrectness)}, edge ${score(gemini.edgeCompleteness)}, occlusion ${score(gemini.occlusionBoundaryQuality)}, perspective ${score(gemini.perspectiveQuality)}, direction ${score(gemini.directionQuality)}, lighting ${score(gemini.lightingPreserved)}, realism ${score(gemini.realism)}`,
    `- Notes: ${gemini.notes ?? "none"}`,
    "",
  ];

  return lines;
}

function imageTag(relativePath) {
  if (!relativePath) return "Missing screenshot";
  return `<img src="${relativePath}" width="720" />`;
}

function yesNo(value) {
  return typeof value === "boolean" ? (value ? "yes" : "no") : "n/a";
}

function score(value) {
  return typeof value === "number" && Number.isFinite(value) ? value.toFixed(2) : "n/a";
}

function listOrNone(value) {
  return Array.isArray(value) && value.length > 0 ? value.join("; ") : "none";
}

function imageContentType(attachment) {
  if (typeof attachment.contentType === "string" && attachment.contentType.startsWith("image/")) {
    return attachment.contentType;
  }

  const extension = typeof attachment.path === "string" ? path.extname(attachment.path).toLowerCase() : "";
  if (extension === ".jpg" || extension === ".jpeg") return "image/jpeg";
  if (extension === ".webp") return "image/webp";
  return "image/png";
}

function imageName(title) {
  return title.split(": apply ")[0] || title;
}

function escapeHeading(value) {
  return String(value).replace(/\n/g, " ").trim();
}

function asArray(value) {
  return Array.isArray(value) ? value : [];
}

function relativeToCwd(value) {
  return path.relative(cwd, value).replaceAll(path.sep, "/") || ".";
}