backend/src/publisher/index.ts

/**
 * Publish pipeline entry point.
 *
 * Orchestrates: Y.Doc → JSON → HTML → PDF/Thumbnail → Upload to HF
 */

import { readFileSync, existsSync } from "fs";
import { join, dirname } from "path";
import { fileURLToPath } from "url";
import * as Y from "yjs";
import { getDataDir, docPath, sanitizeName } from "../utils.js";
import { TiptapTransformer } from "@hocuspocus/transformer";
import { renderArticleHTML, type PublishMeta, type CitationData } from "./html-renderer.js";
import { renderArticleMarkdown } from "./markdown-renderer.js";
import { formatBibliographyServer } from "./format-bibliography.js";
import { getServerExtensions } from "./extensions.js";
import { isPdfEnabled, generatePdfAndThumbnail } from "./pdf-generator.js";
import {
  isHfStorageEnabled,
  uploadPublishedAssets,
  getPublishedAssetUrl,
} from "../hf-storage.js";

const __dirname = dirname(fileURLToPath(import.meta.url));
// Resolved at call-time for testability
function dataDir() { return getDataDir(); }

export interface PublishCSS {
  variables: string;
  reset: string;
  base: string;
  layout: string;
  print: string;
  editorTokens: string;
  article: string;
  components: string;
  publisher: string;
}

/**
 * Resolve @custom-media at-rules so the CSS works in browsers.
 * PostCSS handles this in the Vite build, but the publisher reads raw source files.
 */
/** @internal - exported for testing */
export function resolveCustomMedia(allCss: string): string {
  const mediaMap = new Map<string, string>();

  // Extract @custom-media declarations
  const declRegex = /@custom-media\s+(--[\w-]+)\s+\(([^)]+)\)\s*;/g;
  let match: RegExpExecArray | null;
  while ((match = declRegex.exec(allCss)) !== null) {
    mediaMap.set(match[1], `(${match[2]})`);
  }

  // Remove @custom-media declarations
  let resolved = allCss.replace(/@custom-media\s+--[\w-]+\s+\([^)]+\)\s*;/g, "");

  // Replace @media (--custom-name) with the resolved value
  for (const [name, value] of mediaMap) {
    resolved = resolved.replaceAll(`(${name})`, value);
  }

  return resolved;
}

function loadCSS(): PublishCSS {
  const candidates = [
    join(__dirname, "..", "..", "..", "frontend", "src", "styles"),
    join(__dirname, "..", "..", "frontend-styles"),
    join(__dirname, "..", "frontend-styles"),
  ];

  const readSafe = (p: string) => (existsSync(p) ? readFileSync(p, "utf-8") : "");

  for (const dir of candidates) {
    const varsPath = join(dir, "_variables.css");
    const basePath = join(dir, "_base.css");
    if (existsSync(varsPath) && existsSync(basePath)) {
      console.log("[publish] CSS found at:", dir);

      const componentFiles = [
        "_code.css", "_button.css", "_table.css", "_tag.css",
        "_card.css", "_form.css", "_mermaid.css", "_hero.css", "_toc.css",
        "_footer.css", "_embed.css", "_hf-user.css",
      ];
      const componentsCss = componentFiles
        .map((f) => readSafe(join(dir, "components", f)))
        .join("\n");

      const variablesRaw = readFileSync(varsPath, "utf-8");
      const resetRaw = readSafe(join(dir, "_reset.css"));
      const baseRaw = readFileSync(basePath, "utf-8");
      const layoutRaw = readSafe(join(dir, "_layout.css"));
      const printRaw = readSafe(join(dir, "_print.css"));
      const editorTokensRaw = readSafe(join(dir, "tokens.css"));
      const articleRaw = readSafe(join(dir, "article.css"));
      const publisherRaw = readSafe(join(dir, "_publisher.css"));

      // @custom-media declarations live in _variables.css - resolve across all files
      const allRaw = [variablesRaw, resetRaw, baseRaw, layoutRaw, printRaw, editorTokensRaw, articleRaw, componentsCss, publisherRaw].join("\n/* __CSS_BOUNDARY__ */\n");
      const resolved = resolveCustomMedia(allRaw);
      const parts = resolved.split("/* __CSS_BOUNDARY__ */");

      return {
        variables: parts[0]?.trim() ?? "",
        reset: parts[1]?.trim() ?? "",
        base: parts[2]?.trim() ?? "",
        layout: parts[3]?.trim() ?? "",
        print: parts[4]?.trim() ?? "",
        editorTokens: parts[5]?.trim() ?? "",
        article: parts[6]?.trim() ?? "",
        components: parts[7]?.trim() ?? "",
        publisher: parts[8]?.trim() ?? "",
      };
    }
  }

  console.warn("[publish] CSS files not found, tried:", candidates);
  return { variables: "", reset: "", base: "", layout: "", print: "", editorTokens: "", article: "", components: "", publisher: "" };
}

interface AuthorObj {
  name: string;
  url?: string;
  affiliations: number[];
}

interface AffiliationObj {
  name: string;
  url?: string;
}

function buildPublishMeta(
  frontmatter: Record<string, unknown>,
  authors: AuthorObj[],
  affiliations: AffiliationObj[],
  fallbackTitle: string,
  settings?: Record<string, unknown>,
): PublishMeta {
  const rawHue = settings?.primaryHue;
  const primaryHue = typeof rawHue === "number" && !Number.isNaN(rawHue) ? rawHue : undefined;
  return {
    title: (frontmatter.title as string) || fallbackTitle,
    subtitle: (frontmatter.subtitle as string) || undefined,
    description: (frontmatter.description as string) || "",
    authors: authors.map((a) => ({
      name: a.name,
      url: a.url,
      affiliationIndices: a.affiliations || [],
      affiliationNames: (a.affiliations || [])
        .map((idx) => affiliations[idx - 1]?.name)
        .filter(Boolean) as string[],
    })),
    affiliations: affiliations.map((a) => ({ name: a.name, url: a.url })),
    date: (frontmatter.published as string) || (frontmatter.date as string) || new Date().toISOString(),
    doi: (frontmatter.doi as string) || undefined,
    licence: (frontmatter.licence as string) || (frontmatter.license as string) || undefined,
    banner: (frontmatter.banner as string) || "banner.html",
    tableOfContentsAutoCollapse: frontmatter.tableOfContentsAutoCollapse === true,
    primaryHue,
  };
}

/**
 * Extract TipTap JSON and frontmatter from a Yjs document.
 * Uses @hocuspocus/transformer to convert the XmlFragment to ProseMirror JSON.
 *
 * Scalar fields live in Y.Map("frontmatter"), while authors/affiliations/links
 * each have their own Y.Array for concurrent-safe editing.
 */
/** @internal - exported for testing */
export function extractFromYDoc(ydoc: Y.Doc): {
  json: Record<string, unknown>;
  frontmatter: Record<string, unknown>;
  authors: AuthorObj[];
  affiliations: AffiliationObj[];
  embeds: Record<string, string>;
  settings: Record<string, unknown>;
} {
  const metaMap = ydoc.getMap("frontmatter");
  const frontmatter: Record<string, unknown> = {};
  metaMap.forEach((value, key) => {
    frontmatter[key] = value;
  });

  const settingsMap = ydoc.getMap("settings");
  const settings: Record<string, unknown> = {};
  settingsMap.forEach((value, key) => {
    settings[key] = value;
  });

  const yAuthors = ydoc.getArray<AuthorObj>("frontmatter.authors");
  const yAffiliations = ydoc.getArray<AffiliationObj>("frontmatter.affiliations");
  const authors = yAuthors.toArray();
  const affiliations = yAffiliations.toArray();

  const sharedTypes: string[] = [];
  ydoc.share.forEach((_value, key) => sharedTypes.push(key));
  console.log("[publish] Y.Doc shared types:", sharedTypes);
  console.log("[publish] frontmatter scalars:", JSON.stringify(frontmatter));
  console.log("[publish] authors:", JSON.stringify(authors));
  console.log("[publish] affiliations:", JSON.stringify(affiliations));

  const fragment = ydoc.getXmlFragment("default");
  console.log("[publish] XmlFragment 'default' length:", fragment.length);
  if (fragment.length > 0) {
    console.log("[publish] XmlFragment first child tag:", fragment.toArray()[0]?.constructor?.name);
  }

  const extensions = getServerExtensions();
  const json = TiptapTransformer
    .extensions(extensions)
    .fromYdoc(ydoc, "default");

  console.log("[publish] TiptapTransformer result type:", json?.type);
  const contentArr = (json as any)?.content;
  console.log("[publish] TiptapTransformer content count:", Array.isArray(contentArr) ? contentArr.length : "N/A");
  if (Array.isArray(contentArr) && contentArr.length > 0) {
    const types = contentArr.slice(0, 10).map((n: any) => n.type);
    console.log("[publish] First 10 node types:", types);
  }
  console.log("[publish] Full JSON (truncated):", JSON.stringify(json).slice(0, 2000));

  // Extract embed HTML from Y.Map("embeds")
  const embedsMap = ydoc.getMap<string>("embeds");
  const embeds: Record<string, string> = {};
  embedsMap.forEach((value, key) => {
    embeds[key] = value;
  });
  console.log("[publish] embeds:", Object.keys(embeds).length, "entries:", Object.keys(embeds));

  if (json && typeof json === "object" && json.type === "doc") {
    return { json, frontmatter, authors, affiliations, embeds, settings };
  }

  console.warn("[publish] TiptapTransformer returned unexpected format, using fallback");
  return {
    json: { type: "doc", content: [{ type: "paragraph" }] },
    frontmatter,
    authors,
    affiliations,
    embeds,
    settings,
  };
}

/**
 * Extract citation entries and document order from Y.Doc.
 */
/** @internal - exported for testing */
export function extractCitationsFromYDoc(ydoc: Y.Doc, json: Record<string, unknown>): CitationData {
  const citationsMap = ydoc.getMap("citations");
  const settingsMap = ydoc.getMap("settings");

  const style = (settingsMap.get("citationStyle") as string) || "apa";

  // Collect all citation keys in document order
  const orderedKeys: string[] = [];
  function walkCitations(nodes: any[]) {
    for (const node of nodes) {
      if (node.type === "citation" && node.attrs?.key) {
        const k = node.attrs.key as string;
        if (!orderedKeys.includes(k)) orderedKeys.push(k);
      }
      if (Array.isArray(node.content)) walkCitations(node.content);
    }
  }
  const content = (json as any)?.content;
  if (Array.isArray(content)) walkCitations(content);

  // Build entries list in document order
  const entries: any[] = [];
  for (const key of orderedKeys) {
    const entry = citationsMap.get(key);
    if (entry) {
      entries.push({ ...(entry as any), id: key });
    }
  }

  console.log("[publish] Citations from Y.Doc:", orderedKeys.length, "keys, style:", style);
  console.log("[publish] Citation map size:", citationsMap.size);

  return { entries, orderedKeys, style };
}

export interface PublishResult {
  htmlUrl: string | null;
  pdfUrl: string | null;
  thumbUrl: string | null;
  success: boolean;
  error?: string;
}

/**
 * High-level publish stages emitted to the SSE stream so the UI can show
 * honest progress labels instead of a silent spinner. The PDF generator
 * emits additional sub-stages (load, thumbnail, pdf-setup, pdf-toc, rasterize,
 * pdf-render) on the same callback.
 */
export type PublishStage =
  | "extract"
  | "bibliography"
  | "render"
  | "load"
  | "thumbnail"
  | "pdf-setup"
  | "pdf-toc"
  | "rasterize"
  | "pdf-render"
  | "write"
  | "upload";

export type PublishStageEmitter = (stage: PublishStage | string, detail?: Record<string, unknown>) => void;

export interface PublishOptions {
  onStage?: PublishStageEmitter;
}

/**
 * Render a preview HTML of the document without saving or uploading anything.
 * Useful for testing the publisher pipeline.
 */
export async function previewDocument(docName: string): Promise<{ html: string } | { error: string }> {
  const p = docPath(docName);
  if (!existsSync(p)) return { error: "Document not found" };

  const ydoc = new Y.Doc();
  const data = readFileSync(p);
  Y.applyUpdate(ydoc, new Uint8Array(data));

  const { json, frontmatter, authors, affiliations, embeds, settings } = extractFromYDoc(ydoc);
  const citationData = extractCitationsFromYDoc(ydoc, json);
  const meta = buildPublishMeta(frontmatter, authors, affiliations, docName, settings);

  const css = loadCSS();

  let biblioHtml = "";
  if (citationData.entries.length > 0) {
    try {
      biblioHtml = await formatBibliographyServer(citationData.entries, citationData.style);
    } catch (err) {
      console.error("[preview] bibliography formatting failed:", err);
    }
  }

  const html = await renderArticleHTML(json, meta, css, citationData, biblioHtml, embeds);
  return { html };
}

/**
 * Compute the base URL Playwright uses when navigating to the published page.
 * Defaults to the backend's own HTTP listener so the PDF is rendered from
 * the exact same HTML users see.
 */
function getLocalBaseUrl(): string {
  const port = process.env.PORT || "8080";
  return process.env.PUBLISH_BASE_URL || `http://127.0.0.1:${port}`;
}

/**
 * Run the full publish pipeline for a document.
 * Emits stage events via `opts.onStage` so the route can stream SSE updates.
 *
 * Pipeline (in order):
 *   1. extract TipTap JSON + metadata from the Y.Doc
 *   2. render the HTML (with local /published/* URLs baked in)
 *   3. write index.html + meta.json to disk *first* so the URL is live
 *   4. Playwright goto() that URL -> thumbnail + PDF
 *   5. on PDF failure, rewrite index.html/meta.json without the broken PDF link
 *   6. on HF upload, re-render with public URLs and push everything
 */
export async function publishDocument(
  docName: string,
  token?: string,
  opts: PublishOptions = {}
): Promise<PublishResult> {
  const onStage = opts.onStage ?? (() => {});
  const p = docPath(docName);
  const tPublishStart = Date.now();

  if (!existsSync(p)) {
    return { htmlUrl: null, pdfUrl: null, thumbUrl: null, success: false, error: "Document not found" };
  }

  onStage("extract");

  const ydoc = new Y.Doc();
  const data = readFileSync(p);
  Y.applyUpdate(ydoc, new Uint8Array(data));

  const { json, frontmatter, authors, affiliations, embeds, settings } = extractFromYDoc(ydoc);
  const citationData = extractCitationsFromYDoc(ydoc, json);
  const baseMeta = buildPublishMeta(frontmatter, authors, affiliations, docName, settings);

  const wantPdf = frontmatter.showPdf !== false;
  const useHf = isHfStorageEnabled();
  const pdfEnabled = isPdfEnabled();
  const safeDirName = sanitizeName(docName);

  const css = loadCSS();
  console.log("[publish] CSS variables length:", css.variables.length, "article length:", css.article.length);

  let biblioHtml = "";
  if (citationData.entries.length > 0) {
    onStage("bibliography", { entries: citationData.entries.length });
    try {
      biblioHtml = await formatBibliographyServer(citationData.entries, citationData.style);
      console.log("[publish] Bibliography formatted server-side:", citationData.entries.length, "entries");
    } catch (err) {
      console.error("[publish] Server-side bibliography formatting failed:", err);
    }
  }

  // Meta used for the *local* published copy. `article.pdf`/`thumb.jpg`
  // don't exist on disk yet, but referencing them here means Playwright
  // sees the final hrefs when the PDF is rendered (print CSS hides the
  // Download button anyway, so the broken-href window is harmless).
  const localMeta: PublishMeta = { ...baseMeta };
  if (wantPdf && pdfEnabled) {
    localMeta.ogImage = `/published/${safeDirName}/thumb.jpg`;
    localMeta.pdfUrl = `/published/${safeDirName}/article.pdf`;
  }
  console.log("[publish] Meta:", JSON.stringify(localMeta));

  onStage("render");
  const tRenderStart = Date.now();
  let localHtml = await renderArticleHTML(json, localMeta, css, citationData, biblioHtml, embeds);
  console.log(
    `[publish] Generated HTML length: ${localHtml.length} (${Date.now() - tRenderStart}ms)`
  );

  // --- Step 1: write HTML + meta to disk BEFORE Playwright runs ------------
  onStage("write");
  const { writeFileSync: fsWrite, mkdirSync: fsMkdir } = await import("fs");
  const publishDir = join(dataDir(), "published", safeDirName);
  fsMkdir(publishDir, { recursive: true });
  fsWrite(join(publishDir, "index.html"), localHtml);
  fsWrite(join(publishDir, "meta.json"), JSON.stringify(localMeta, null, 2));

  // Co-located `llms.txt` (https://llmstxt.org/) for LLM agents and
  // crawlers that struggle with the heavy published HTML. Cheap to
  // generate (one walk over the same TipTap-JSON, no Playwright) so we
  // always write it; serving is gated separately by `create-app.ts`.
  let llmsTxt = "";
  try {
    llmsTxt = renderArticleMarkdown(json, localMeta, citationData, biblioHtml);
    fsWrite(join(publishDir, "llms.txt"), llmsTxt);
  } catch (err) {
    console.warn("[publish] llms.txt generation failed:", (err as Error).message);
  }
  console.log("[publish] index.html written to", publishDir);

  // --- Step 2: generate PDF + thumbnail from the served URL ---------------
  let pdf: Buffer | null = null;
  let thumbnail: Buffer | null = null;

  if (wantPdf && pdfEnabled) {
    const url = `${getLocalBaseUrl()}/published/${safeDirName}/index.html`;
    const tPdfStart = Date.now();
    try {
      console.log("[publish] Launching Playwright against", url);
      const assets = await generatePdfAndThumbnail("", { url, onStage });
      pdf = assets.pdf;
      thumbnail = assets.thumbnail;
      console.log(
        `[publish] Playwright done in ${Date.now() - tPdfStart}ms - pdf=${pdf ? pdf.byteLength : 0}B thumb=${thumbnail ? thumbnail.byteLength : 0}B`
      );
    } catch (err) {
      console.error("[publish] PDF generation failed:", err);
    }
  } else if (wantPdf && !pdfEnabled) {
    console.log("[publish] PDF generation skipped (Playwright unavailable or ENABLE_PDF=false)");
  }

  // --- Step 3: if PDF/thumbnail succeeded, persist them. If either failed,
  // strip the references from the meta & rewrite the HTML to avoid broken
  // Download/og:image links.
  if (pdf) fsWrite(join(publishDir, "article.pdf"), pdf);
  if (thumbnail) fsWrite(join(publishDir, "thumb.jpg"), thumbnail);

  const finalLocalMeta = { ...localMeta };
  if (!thumbnail) delete finalLocalMeta.ogImage;
  if (!pdf) delete finalLocalMeta.pdfUrl;

  if (wantPdf && pdfEnabled && (!pdf || !thumbnail)) {
    console.log("[publish] rewriting index.html without missing asset references");
    localHtml = await renderArticleHTML(json, finalLocalMeta, css, citationData, biblioHtml, embeds);
    fsWrite(join(publishDir, "index.html"), localHtml);
  }
  fsWrite(join(publishDir, "meta.json"), JSON.stringify(finalLocalMeta, null, 2));
  console.log(
    `[publish] local copy ready at ${publishDir} (total ${Date.now() - tPublishStart}ms)`
  );

  // --- Step 4: HF upload (if enabled): rebuild HTML once more with public URLs
  if (useHf) {
    onStage("upload");
    const hfMeta: PublishMeta = { ...baseMeta };
    if (thumbnail) hfMeta.ogImage = getPublishedAssetUrl(docName, "thumb.jpg");
    if (pdf) hfMeta.pdfUrl = getPublishedAssetUrl(docName, "article.pdf");
    const hfHtml = await renderArticleHTML(json, hfMeta, css, citationData, biblioHtml, embeds);
    try {
      // Re-render llms.txt with the HF meta so internal links (DOI, etc.)
      // match what's in the uploaded HTML. Cheap, deterministic, and we
      // already have everything in memory.
      let hfLlmsTxt: string | undefined = undefined;
      try {
        hfLlmsTxt = renderArticleMarkdown(json, hfMeta, citationData, biblioHtml);
      } catch (err) {
        console.warn("[publish] HF llms.txt render failed:", (err as Error).message);
      }
      const urls = await uploadPublishedAssets(docName, {
        html: hfHtml,
        pdf,
        thumbnail,
        meta: hfMeta as any,
        llmsTxt: hfLlmsTxt,
      }, token);
      return { ...urls, success: true };
    } catch (err) {
      console.warn("[publish] HF upload failed, local copy is still available:", (err as Error).message);
    }
  }

  return {
    htmlUrl: `/published/${safeDirName}/index.html`,
    pdfUrl: pdf ? `/published/${safeDirName}/article.pdf` : null,
    thumbUrl: thumbnail ? `/published/${safeDirName}/thumb.jpg` : null,
    success: true,
  };
}