#!/usr/bin/env node import { mkdir, readFile, writeFile } from "node:fs/promises"; import { existsSync } from "node:fs"; import path from "node:path"; const cwd = process.cwd(); const reportPath = path.resolve(cwd, process.argv[2] ?? "test-results/real-qa.json"); const outputPath = path.resolve(cwd, process.argv[3] ?? "test-results/qa-summary.md"); const outputDir = path.dirname(outputPath); try { await main(); } catch (error) { await writeFailureSummary(error); console.error(error); } async function main() { await mkdir(outputDir, { recursive: true }); if (!existsSync(reportPath)) { await writeFile( outputPath, [ "# Real Browser QA Summary", "", `Playwright JSON report was not found at \`${relativeToCwd(reportPath)}\`.`, "", "Run the Playwright suite first so this summary can include screenshots and Gemini judgments.", "", ].join("\n"), ); console.warn(`QA summary skipped source report: ${relativeToCwd(reportPath)}`); return; } const report = JSON.parse(await readFile(reportPath, "utf8")); const entries = collectEntries(report); const hydratedEntries = []; for (const entry of entries) { const result = finalResult(entry.test); const groupedEntries = await groupedCaseEntries(entry, result); if (groupedEntries.length) { hydratedEntries.push(...groupedEntries); continue; } const beforeImage = await imageAttachmentDataUri(result, "before-canvas"); const afterImage = await imageAttachmentDataUri(result, "after-texture-canvas"); const geminiAttachment = findAttachment(result, "gemini-judge"); const gemini = geminiAttachment ? parseJsonLoose(await readTextAttachment(geminiAttachment)) : null; hydratedEntries.push({ ...entry, afterImage, beforeImage, gemini, result, status: displayStatus(entry.test, result), }); } await writeFile(outputPath, buildMarkdown(hydratedEntries)); console.log(`QA markdown summary written to ${relativeToCwd(outputPath)}`); } function collectEntries(report) { const entries = []; for (const suite of asArray(report.suites)) { collectSuite(suite, entries); } return entries; } async function writeFailureSummary(error) { await mkdir(outputDir, { recursive: true }); const message = error instanceof Error ? error.stack ?? error.message : String(error); await writeFile( outputPath, [ "# Real Browser QA Summary", "", "The compact Markdown summary could not be generated.", "", `Source report: \`${relativeToCwd(reportPath)}\``, "", "```text", message, "```", "", "The full Playwright artifacts are still available in this artifact bundle.", "", ].join("\n"), ); } function collectSuite(suite, entries) { for (const spec of asArray(suite.specs)) { const title = typeof spec.title === "string" && spec.title.trim() ? spec.title.trim() : "Untitled QA case"; for (const test of asArray(spec.tests)) { entries.push({ spec, test, title }); } } for (const child of asArray(suite.suites)) { collectSuite(child, entries); } } function finalResult(test) { const results = asArray(test.results); return results[results.length - 1] ?? {}; } function displayStatus(test, result) { const resultStatus = typeof result.status === "string" ? result.status : ""; const testStatus = typeof test.status === "string" ? test.status : ""; if (resultStatus) return resultStatus; if (testStatus === "expected") return "passed"; if (testStatus === "unexpected") return "failed"; return testStatus || "unknown"; } async function groupedCaseEntries(entry, result) { const metas = await groupedCaseMetas(result); const hydratedEntries = []; for (const { meta, prefix } of metas) { const beforeImage = await imageAttachmentDataUri(result, `${prefix}-before-canvas`); const afterImage = await imageAttachmentDataUri(result, `${prefix}-after-texture-canvas`); const geminiAttachment = findAttachment(result, `${prefix}-gemini-judge`); const gemini = geminiAttachment ? parseJsonLoose(await readTextAttachment(geminiAttachment)) : null; const imageTitle = meta.imageName || meta.roomName || imageName(entry.title); const tileName = meta.tileName || "Unknown tile"; hydratedEntries.push({ ...entry, afterImage, beforeImage, gemini, result, status: typeof meta.status === "string" ? meta.status : displayStatus(entry.test, result), title: `${imageTitle}: apply ${tileName}`, }); } return hydratedEntries; } async function groupedCaseMetas(result) { const metas = []; for (const attachment of asArray(result.attachments)) { const name = typeof attachment?.name === "string" ? attachment.name : ""; if (!name.startsWith("qa-case-") || !name.endsWith("-meta")) continue; const meta = parseJsonLoose(await readTextAttachment(attachment)); if (!meta || typeof meta !== "object" || Array.isArray(meta)) continue; metas.push({ meta, prefix: typeof meta.attachmentPrefix === "string" && meta.attachmentPrefix ? meta.attachmentPrefix : name.slice(0, -"-meta".length), }); } return metas.sort((left, right) => Number(left.meta.caseIndex ?? 0) - Number(right.meta.caseIndex ?? 0)); } async function imageAttachmentDataUri(result, attachmentName) { const attachment = findAttachment(result, attachmentName); if (!attachment) return null; const sourcePath = resolveAttachmentPath(attachment.path); const contentType = imageContentType(attachment); if (sourcePath) { const buffer = await readFile(sourcePath); return `data:${contentType};base64,${buffer.toString("base64")}`; } if (typeof attachment.body === "string") { return `data:${contentType};base64,${attachment.body}`; } return null; } function findAttachment(result, name) { return asArray(result.attachments).find((attachment) => attachment?.name === name) ?? null; } async function readTextAttachment(attachment) { const sourcePath = resolveAttachmentPath(attachment.path); if (sourcePath) return readFile(sourcePath, "utf8"); if (typeof attachment.body !== "string") return ""; const direct = attachment.body.trim(); if (direct.startsWith("{") || direct.startsWith("[")) return attachment.body; return Buffer.from(attachment.body, "base64").toString("utf8"); } function resolveAttachmentPath(value) { if (typeof value !== "string" || !value.trim()) return null; const candidates = [ path.isAbsolute(value) ? value : path.resolve(cwd, value), path.resolve(path.dirname(reportPath), value), ]; return candidates.find((candidate) => existsSync(candidate)) ?? null; } function parseJsonLoose(text) { if (!text.trim()) return null; try { return JSON.parse(text); } catch { const match = text.match(/\{[\s\S]*\}/); if (!match) return null; try { return JSON.parse(match[0]); } catch { return null; } } } function buildMarkdown(entries) { const passed = entries.filter((entry) => entry.status === "passed").length; const skipped = entries.filter((entry) => entry.status === "skipped").length; const needsReview = entries.length - passed - skipped; const lines = [ "# Real Browser QA Summary", "", `Generated from \`${relativeToCwd(reportPath)}\`.`, "", `Total images: ${entries.length}`, `Passed: ${passed}`, `Needs review: ${needsReview}`, `Skipped: ${skipped}`, "", ]; for (const [index, entry] of entries.entries()) { lines.push("", `## ${index + 1}. ${escapeHeading(imageName(entry.title))}`, ""); lines.push(`Status: **${entry.status}**`, ""); lines.push("### Before", ""); lines.push(imageTag(entry.beforeImage), ""); lines.push("### After Tile Applied", ""); lines.push(imageTag(entry.afterImage), ""); lines.push("", "### Gemini Judgment", ""); lines.push(...geminiSummary(entry.gemini)); } return `${lines.join("\n")}\n`; } function geminiSummary(gemini) { if (!gemini) { return ["No Gemini judgment attachment was found for this image.", ""]; } const lines = [ `- Overall pass: ${yesNo(gemini.overallPass)}`, `- Tile applied: ${yesNo(gemini.tileApplied)}`, `- Coverage gap detected: ${yesNo(gemini.coverageGapDetected)}`, `- Missed floor areas: ${listOrNone(gemini.missedFloorAreas)}`, `- Direction mismatch detected: ${yesNo(gemini.directionMismatchDetected)}`, `- Direction issues: ${listOrNone(gemini.directionIssues)}`, `- Bleed detected: ${yesNo(gemini.bleedDetected)}`, `- Scores: surface ${score(gemini.surfaceCorrectness)}, edge ${score(gemini.edgeCompleteness)}, occlusion ${score(gemini.occlusionBoundaryQuality)}, perspective ${score(gemini.perspectiveQuality)}, direction ${score(gemini.directionQuality)}, lighting ${score(gemini.lightingPreserved)}, realism ${score(gemini.realism)}`, `- Notes: ${gemini.notes ?? "none"}`, "", ]; return lines; } function imageTag(relativePath) { if (!relativePath) return "Missing screenshot"; return ``; } function yesNo(value) { return typeof value === "boolean" ? (value ? "yes" : "no") : "n/a"; } function score(value) { return typeof value === "number" && Number.isFinite(value) ? value.toFixed(2) : "n/a"; } function listOrNone(value) { return Array.isArray(value) && value.length > 0 ? value.join("; ") : "none"; } function imageContentType(attachment) { if (typeof attachment.contentType === "string" && attachment.contentType.startsWith("image/")) { return attachment.contentType; } const extension = typeof attachment.path === "string" ? path.extname(attachment.path).toLowerCase() : ""; if (extension === ".jpg" || extension === ".jpeg") return "image/jpeg"; if (extension === ".webp") return "image/webp"; return "image/png"; } function imageName(title) { return title.split(": apply ")[0] || title; } function escapeHeading(value) { return String(value).replace(/\n/g, " ").trim(); } function asArray(value) { return Array.isArray(value) ? value : []; } function relativeToCwd(value) { return path.relative(cwd, value).replaceAll(path.sep, "/") || "."; }