/** * lib/yjs-client.ts * * Robust Yjs-level peer for demo personas (bob, carol). * * Why this exists * --------------- * The original demo drove Bob and Carol through Playwright + Chromium: * - each persona ran an actual editor in a real browser, * - Playwright "typed" characters into the DOM, * - positions were tracked through MappablePosition / slot claims. * * That works but is fragile: ProseMirror input rules, concurrent * AgentRewrite animations, and DOM-level selection changes all * introduce races that occasionally drop Bob's text into the wrong * slot or delete half a paragraph. * * Yjs (the CRDT Tiptap uses for collaboration) is built exactly for * this kind of headless peer. A Node process can open a * `HocuspocusProvider` against the same server the browsers connect * to, sync the shared `Y.Doc`, and mutate it directly through * `Y.XmlElement` / `Y.XmlText` APIs. Every mutation is atomic at * the CRDT level and shows up in Alice's editor in real time, with * zero input-rule surprises and zero PM position drift. * * This module exposes: * - `connectPersona()` - open a Hocuspocus connection as a named * peer, wait for initial sync, return a handle with the shared * fragment, citations map, and frontmatter helpers. * - a small DOM-like helper set to build paragraphs, headings, * inline math, citations, and to type text character by * character (so Alice visually sees the peer typing). */ import { WebSocket } from "ws"; import { HocuspocusProvider } from "@hocuspocus/provider"; import * as Y from "yjs"; // The Hocuspocus provider expects a DOM `WebSocket` global; Node // doesn't ship one so we plumb `ws` in before any provider is // constructed. Idempotent. if (!(globalThis as unknown as { WebSocket?: unknown }).WebSocket) { (globalThis as unknown as { WebSocket: unknown }).WebSocket = WebSocket; } // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- export interface PersonaUser { name: string; color: string; avatarUrl?: string; } export interface PersonaHandle { ydoc: Y.Doc; provider: HocuspocusProvider; /** * Shared editor content (Tiptap `field: "default"`). */ fragment: Y.XmlFragment; /** * Citations keyed by citation-key (Vaswani2017 -> {title, authors, ...}). */ citationsMap: Y.Map; /** * Whole-article settings (primaryHue, etc.). */ settingsMap: Y.Map; /** * Frontmatter arrays (authors, affiliations, links). */ frontmatter: { scalars: Y.Map; authors: Y.Array; affiliations: Y.Array; links: Y.Array; }; /** * Cancel the awareness ticker / close the connection. */ destroy: () => void; } export interface Author { name: string; url?: string; affiliations: number[]; } export interface Affiliation { name: string; url?: string; } // --------------------------------------------------------------------------- // Connection // --------------------------------------------------------------------------- /** * Open a Hocuspocus connection as a named peer and resolve once the * server has finished the initial sync (so the Y.Doc mirrors what * Alice / the disk state contain). */ export async function connectPersona(args: { url: string; docName: string; user: PersonaUser; token?: string; syncTimeoutMs?: number; }): Promise { const ydoc = new Y.Doc(); const provider = new HocuspocusProvider({ url: args.url, name: args.docName, document: ydoc, token: args.token ?? "", }); // Broadcast ourselves as a collaborator so the other peers see a // name + color in their byline / awareness list immediately. The // Tiptap CollaborationCursor extension reads awareness.user on // every update, so this is all that's needed for the avatar bubble // to appear. provider.awareness?.setLocalStateField("user", args.user); await new Promise((resolve, reject) => { const timer = setTimeout( () => reject(new Error(`persona "${args.user.name}" sync timeout`)), args.syncTimeoutMs ?? 15_000, ); provider.on("synced", () => { clearTimeout(timer); resolve(); }); }); const fragment = ydoc.getXmlFragment("default"); const citationsMap = ydoc.getMap("citations"); const settingsMap = ydoc.getMap("settings"); const frontmatter = { scalars: ydoc.getMap("frontmatter"), authors: ydoc.getArray("frontmatter.authors"), affiliations: ydoc.getArray("frontmatter.affiliations"), links: ydoc.getArray("frontmatter.links"), }; const destroy = () => { try { provider.destroy(); } catch { /* provider may already be detached */ } try { ydoc.destroy(); } catch { /* noop */ } }; return { ydoc, provider, fragment, citationsMap, settingsMap, frontmatter, destroy, }; } // --------------------------------------------------------------------------- // Inline content builders // --------------------------------------------------------------------------- /** * Describes a chunk of inline content. `text` entries can carry * marks like `bold` or `link`; `inlineMath` entries render as a * KaTeX node. */ export type InlineItem = | { type: "text"; text: string; marks?: InlineMarks } | { type: "inlineMath"; latex: string } | { type: "citation"; key: string; label: string }; export interface InlineMarks { bold?: boolean; italic?: boolean; underline?: boolean; code?: boolean; link?: { href: string; target?: string; rel?: string; class?: string; title?: string | null; }; } /** * Convert our plain JS `InlineMarks` into the Yjs attribute object * expected by `Y.XmlText.insert(index, text, attrs)`. Mirrors the * shape Tiptap's y-prosemirror binding serializes (empty objects * for booleans, full attr maps for parametrized marks). */ function marksToAttrs( marks: InlineMarks | undefined, ): Record | undefined { if (!marks) return undefined; const out: Record = {}; if (marks.bold) out.bold = {}; if (marks.italic) out.italic = {}; if (marks.underline) out.underline = {}; if (marks.code) out.code = {}; if (marks.link) { out.link = { href: marks.link.href, target: marks.link.target ?? "_blank", rel: marks.link.rel ?? "noopener noreferrer nofollow", class: marks.link.class ?? "editor-link", title: marks.link.title ?? null, }; } return Object.keys(out).length ? out : undefined; } /** * Append inline content to the END of a paragraph Y.XmlElement. * Text chunks merge into adjacent Y.XmlText when possible (so marks * stay contiguous); non-text items are inserted as standalone * children. */ export function appendInline( paragraph: Y.XmlElement, items: InlineItem[], ): void { for (const item of items) { if (item.type === "text") { const attrs = marksToAttrs(item.marks); // Try to reuse the trailing Y.XmlText node - merging marks // with a previous text run gives Tiptap a cleaner delta and // avoids spurious splits that the ProseMirror renderer has // to collapse afterwards. const last = paragraph.toArray().at(-1); if (last instanceof Y.XmlText) { last.insert(last.length, item.text, attrs); } else { // Attach first, then insert - Y.XmlText emits observer // events on `.insert`, and doing that while detached // triggers "Invalid access: Add Yjs type to a document // before reading data". const t = new Y.XmlText(); paragraph.push([t]); t.insert(0, item.text, attrs); } } else if (item.type === "inlineMath") { const el = new Y.XmlElement("inlineMath"); paragraph.push([el]); el.setAttribute("latex", item.latex); } else if (item.type === "citation") { const el = new Y.XmlElement("citation"); paragraph.push([el]); el.setAttribute("key", item.key); el.setAttribute("label", item.label); } } } // --------------------------------------------------------------------------- // Block builders // --------------------------------------------------------------------------- /** * Create an empty, detached `` Y.XmlElement. Caller * must attach it (via `frag.push([p])` / `frag.insert(n, [p])`) * before any content insertion. `appendInline` and * `typeIntoParagraph` take care of creating the first Y.XmlText * child lazily after attachment, so an empty paragraph is a valid * seed for all writer helpers below. */ export function buildEmptyParagraph(): Y.XmlElement { return new Y.XmlElement("paragraph"); } /** * Build a paragraph pre-populated with inline content. Requires a * parent to insert into so the Y.XmlText children can be attached * immediately (a detached Y.XmlText insert would throw the * "Invalid access" warning). The new paragraph is pushed onto the * end of `parent`. */ export function pushParagraphInto( parent: Y.XmlFragment | Y.XmlElement, items: InlineItem[] = [], ): Y.XmlElement { const el = new Y.XmlElement("paragraph"); parent.push([el]); if (items.length) appendInline(el, items); return el; } // --------------------------------------------------------------------------- // Top-level navigation // --------------------------------------------------------------------------- /** * Find the index of the nth occurrence of an H2 whose textContent * matches `text` (case-insensitive). Returns -1 if not found. */ export function findHeadingIndex( frag: Y.XmlFragment, text: string, level: number = 2, ): number { const wanted = text.trim().toLowerCase(); const children = frag.toArray(); for (let i = 0; i < children.length; i++) { const c = children[i]; if (!(c instanceof Y.XmlElement)) continue; if (c.nodeName !== "heading") continue; if (Number(c.getAttribute("level")) !== level) continue; const textContent = c .toArray() .filter((n): n is Y.XmlText => n instanceof Y.XmlText) .map((n) => n.toString()) .join(""); if (textContent.trim().toLowerCase() === wanted) return i; } return -1; } /** * Return the Nth non-heading child that sits right after * `headingText`, scanning top-level fragment children. Creates * empty paragraphs on demand to satisfy `slotIndex` if the section * exists but doesn't have enough slots yet (useful when a peer * lands ahead of the scaffold writer). */ export function claimSlotAfterHeading( frag: Y.XmlFragment, headingText: string, slotIndex: number, ): Y.XmlElement | null { const headingIdx = findHeadingIndex(frag, headingText); if (headingIdx === -1) return null; const children = frag.toArray(); // Collect section slots up to the next H-level boundary. const slots: { el: Y.XmlElement; fragIndex: number }[] = []; for (let i = headingIdx + 1; i < children.length; i++) { const c = children[i]; if (!(c instanceof Y.XmlElement)) continue; if (c.nodeName === "heading") break; slots.push({ el: c, fragIndex: i }); } if (slots[slotIndex]) return slots[slotIndex].el; // Section is present but too few slots. Grow it by inserting // empty paragraphs right after the last slot we found (or right // after the heading if the section was bare). Elements are // minted detached, then pushed as a single batch via Y.insert // so attribute writes don't emit detached-node warnings. const insertAt = slots.length ? slots[slots.length - 1].fragIndex + 1 : headingIdx + 1; const needed = slotIndex + 1 - slots.length; const fresh: Y.XmlElement[] = []; for (let i = 0; i < needed; i++) { fresh.push(new Y.XmlElement("paragraph")); } frag.insert(insertAt, fresh); return fresh[needed - 1]; } /** * Replace (or create) the paragraph at fragment index `slotIndex` * within a section with a `blockMath` atom carrying the given * LaTeX. `blockMath` is a Tiptap atom: nodeName="blockMath", no * children, single `latex` attribute (matches * @tiptap/extension-mathematics/BlockMath.ts). The frontend * NodeView picks it up and renders KaTeX the moment the CRDT * sync lands in Alice's editor. * * Using an atomic replace (delete slot + insert blockMath) keeps * the insertion visually "crisp" on camera: no half-parsed LaTeX * intermediate state, no input-rule roundtrip. */ export function writeBlockMathAtSlot( frag: Y.XmlFragment, headingText: string, slotIndex: number, latex: string, ): Y.XmlElement | null { const headingIdx = findHeadingIndex(frag, headingText); if (headingIdx === -1) return null; const children = frag.toArray(); // Collect slot fragment indices up to the next heading boundary. const slots: number[] = []; for (let i = headingIdx + 1; i < children.length; i++) { const c = children[i]; if (!(c instanceof Y.XmlElement)) continue; if (c.nodeName === "heading") break; slots.push(i); } const block = new Y.XmlElement("blockMath"); frag.doc?.transact(() => { if (slots[slotIndex] !== undefined) { frag.delete(slots[slotIndex], 1); frag.insert(slots[slotIndex], [block]); } else { // Section exists but doesn't have a slot at slotIndex yet. // Pad with empty paragraphs then append the blockMath so // downstream slot indices stay aligned with the scaffold. const insertAt = slots.length ? slots[slots.length - 1] + 1 : headingIdx + 1; const pad = slotIndex - slots.length; const batch: Y.XmlElement[] = []; for (let k = 0; k < pad; k++) batch.push(new Y.XmlElement("paragraph")); batch.push(block); frag.insert(insertAt, batch); } block.setAttribute("latex", latex); }); return block; } /** * Wait until `predicate(fragment)` returns true, polling on every * fragment update. Used to gate a peer on "Alice has written the * scaffold" (`findHeadingIndex(frag, "The recipe") !== -1`). */ export async function waitForCondition( frag: Y.XmlFragment, predicate: () => boolean, timeoutMs: number = 30_000, ): Promise { if (predicate()) return true; return new Promise((resolve) => { let done = false; const finish = (v: boolean) => { if (done) return; done = true; frag.unobserveDeep(onChange); clearTimeout(timer); resolve(v); }; const onChange = () => { try { if (predicate()) finish(true); } catch { /* predicate may temporarily throw while the doc is mid- * mutation; just retry on the next event */ } }; frag.observeDeep(onChange); const timer = setTimeout(() => finish(false), timeoutMs); }); } // --------------------------------------------------------------------------- // Text "typing" (human-paced) // --------------------------------------------------------------------------- export interface TypingOptions { /** * Characters per second (roughly). Default 24 cps ≈ a fast touch * typist. */ cps?: number; /** * Jitter on the inter-keystroke delay (0..1). Default 0.35. */ jitter?: number; /** * If set, group keystrokes of that many chars into one Y * transaction to keep Alice's render smooth without killing the * "typing" feel. Default 1 (every char is its own tick). */ chunkSize?: number; } export function sleep(ms: number): Promise { return new Promise((r) => setTimeout(r, ms)); } /** * Broadcast a Tiptap-compatible cursor position through awareness. * * `yCursorPlugin` (from `@tiptap/y-tiptap`) reads `awareness.cursor` * as `{anchor, head}`, each being a JSON-serialized Yjs relative * position. Any peer that sets those two fields will have its caret * rendered inside every other peer's editor - that's exactly the * "floating Bob Mercier / Carol Dubois cursors" Alice sees during * normal human collaboration. * * We build the relative position from the target Y.XmlText node + * offset directly (no ProseMirror state involved), which is safe for * headless Node peers that don't have an editor. */ export function setAwarenessCursor( handle: PersonaHandle, node: Y.XmlText, offset: number, ): void { try { const rel = Y.createRelativePositionFromTypeIndex(node, offset, -1); const json = Y.relativePositionToJSON(rel); handle.provider.awareness?.setLocalStateField("cursor", { anchor: json, head: json, }); } catch { /* type not yet attached to the doc; skip this tick, caller will * retry on the next chunk */ } } /** * Clear the broadcast cursor (peer has nothing selected / is idle). * Call this between scenes so Alice doesn't see a ghost caret * lingering after a persona finishes typing. */ export function clearAwarenessCursor(handle: PersonaHandle): void { handle.provider.awareness?.setLocalStateField("cursor", null); } /** * Insert `chunk` at the end of `paragraph`, reusing the trailing * Y.XmlText run when possible. Returns the Y.XmlText the chunk * landed in plus the offset AFTER the insert, so callers can * broadcast an up-to-date cursor position right after the write. */ function pushChunkIntoParagraph( paragraph: Y.XmlElement, chunk: string, attrs: Record | undefined, ): { node: Y.XmlText; offset: number } { const last = paragraph.toArray().at(-1); if (last instanceof Y.XmlText) { const base = last.length; last.insert(base, chunk, attrs); return { node: last, offset: base + chunk.length }; } // Attach the XmlText BEFORE inserting: Y.XmlText.insert emits // observer events, and firing those while detached throws the // "Invalid access" warning even though the end result is // correct. const t = new Y.XmlText(); paragraph.push([t]); t.insert(0, chunk, attrs); return { node: t, offset: chunk.length }; } /** * Append text, chunk by chunk, to a paragraph's trailing Y.XmlText * node with human-like cadence. Each chunk is a real Yjs insert * that propagates to Alice, so she sees the text stream in at a * natural speed. * * Side effect: after every chunk we also broadcast the peer's * cursor through awareness (if `handle` is provided). That's what * makes Bob's / Carol's caret visibly travel along their paragraph * in Alice's editor while they type, same as a human collaborator. */ export async function typeIntoParagraph( paragraph: Y.XmlElement, text: string, options: TypingOptions = {}, marks?: InlineMarks, handle?: PersonaHandle, ): Promise { const cps = options.cps ?? 24; const jitter = options.jitter ?? 0.35; const chunkSize = Math.max(1, options.chunkSize ?? 1); const baseDelay = 1000 / cps; const attrs = marksToAttrs(marks); let i = 0; while (i < text.length) { const take = Math.min(chunkSize, text.length - i); const chunk = text.slice(i, i + take); const landed = pushChunkIntoParagraph(paragraph, chunk, attrs); if (handle) setAwarenessCursor(handle, landed.node, landed.offset); i += take; const jitterMul = 1 + (Math.random() * 2 - 1) * jitter; await sleep(Math.max(5, baseDelay * jitterMul)); } } /** * Like `typeIntoParagraph` but inserts an inline node (math, * citation) as a single Yjs op after an optional small pause - so * the viewer experiences the formula "clicking in" rather than * appearing with no warning. When `handle` is provided the peer's * awareness cursor is nudged to the end of the paragraph after the * insert. */ export async function insertInlineNode( paragraph: Y.XmlElement, item: | { type: "inlineMath"; latex: string } | { type: "citation"; key: string; label: string }, preDelayMs: number = 0, handle?: PersonaHandle, ): Promise { if (preDelayMs > 0) await sleep(preDelayMs); appendInline(paragraph, [item]); if (handle) broadcastCursorAtEnd(handle, paragraph); } /** * Broadcast the peer's awareness cursor at the end of `paragraph`. * Useful after bulk inline inserts (citation chip, inline math) that * don't go through `typeIntoParagraph`'s per-chunk broadcasting. */ export function broadcastCursorAtEnd( handle: PersonaHandle, paragraph: Y.XmlElement, ): void { const last = paragraph.toArray().at(-1); if (last instanceof Y.XmlText) { setAwarenessCursor(handle, last, last.length); } } // --------------------------------------------------------------------------- // Frontmatter helpers // --------------------------------------------------------------------------- /** * Add an author + (optionally) a new affiliation atomically. * Mirrors `frontmatterStore.addAuthor` in the frontend but runs * directly against the shared Y.Arrays so there's no race between * "affiliation index allocation" and "author push". */ export function addAuthorRecord( handle: PersonaHandle, args: { name: string; url?: string; newAffiliation?: { name: string; url?: string }; existingAffiliationIndices?: number[]; }, ): void { const { ydoc, frontmatter } = handle; ydoc.transact(() => { let affIndices = args.existingAffiliationIndices ?? []; if (args.newAffiliation) { frontmatter.affiliations.push([args.newAffiliation]); // addAffiliation returns 1-based index in the frontend store, // so we match that convention: after push, length equals the // new entry's 1-based index. affIndices = [...affIndices, frontmatter.affiliations.length]; } frontmatter.authors.push([ { name: args.name, url: args.url, affiliations: affIndices, }, ]); }); } /** * Patch the URL of an existing affiliation by 1-based index, matching * the convention used everywhere else in the frontmatter. Used by the * "other" persona to backfill a `url` on the affiliation seeded by * the main agent (e.g. Alice adds "Hugging Face" with no URL via the * Playwright modal, then Bob attaches `https://huggingface.co` to it * so the byline renders the affiliation as a real link). * * Done inside a single Yjs transaction so the change shows up atomically * on every peer (no flicker between "name only" and "name + url"). */ export function setAffiliationUrl( handle: PersonaHandle, oneBasedIndex: number, url: string, ): void { const { ydoc, frontmatter } = handle; const idx = oneBasedIndex - 1; const current = frontmatter.affiliations.get(idx); if (!current) return; ydoc.transact(() => { frontmatter.affiliations.delete(idx, 1); frontmatter.affiliations.insert(idx, [{ ...current, url }]); }); } /** * Wait for at least `n` affiliations to be present in the shared * frontmatter. Used by the "other" persona to gate on the main agent * having seeded the Hugging Face affiliation before reusing its index. */ export async function waitForAffiliationCount( handle: PersonaHandle, minCount: number, timeoutMs: number = 30_000, ): Promise { if (handle.frontmatter.affiliations.length >= minCount) return true; return new Promise((resolve) => { let done = false; const onChange = () => { if (done) return; if (handle.frontmatter.affiliations.length >= minCount) { done = true; handle.frontmatter.affiliations.unobserve(onChange); clearTimeout(timer); resolve(true); } }; handle.frontmatter.affiliations.observe(onChange); const timer = setTimeout(() => { if (done) return; done = true; handle.frontmatter.affiliations.unobserve(onChange); resolve(false); }, timeoutMs); }); } /** * Wait for at least `n` authors to be present in the shared * frontmatter. Useful for persona gating ("Bob waits for Alice to * seed the first author before appending his own"). */ export async function waitForAuthorCount( handle: PersonaHandle, minCount: number, timeoutMs: number = 30_000, ): Promise { if (handle.frontmatter.authors.length >= minCount) return true; return new Promise((resolve) => { let done = false; const onChange = () => { if (done) return; if (handle.frontmatter.authors.length >= minCount) { done = true; handle.frontmatter.authors.unobserve(onChange); clearTimeout(timer); resolve(true); } }; handle.frontmatter.authors.observe(onChange); const timer = setTimeout(() => { if (done) return; done = true; handle.frontmatter.authors.unobserve(onChange); resolve(false); }, timeoutMs); }); } // --------------------------------------------------------------------------- // Citations // --------------------------------------------------------------------------- export interface CitationEntry { id: string; "citation-key": string; type: string; title?: string; author?: Array<{ family: string; given: string }>; issued?: { "date-parts": number[][] }; "container-title"?: string; volume?: string; URL?: string; DOI?: string; [key: string]: unknown; } /** * Register a citation entry in the shared `citationsMap`. Tiptap's * frontend listens on this map; a citation chip with matching * `key` attr will resolve to the entry automatically. */ export function registerCitation( handle: PersonaHandle, key: string, entry: CitationEntry, ): void { handle.citationsMap.set(key, entry); }