server/collab.js

import { Hocuspocus } from '@hocuspocus/server'
import * as Y from 'yjs'
import { yDocToProsemirrorJSON } from 'y-prosemirror'
import { newId, b64encode, b64decode } from './util.js'
import * as store from './store.js'
import { pmToMarkdown, blockToMarkdown, markdownToBlocks } from './md.js'
import { sessionFromCookieHeader, userFromToken, BUILD_ID } from './auth.js'
import { DOC_NAME_RE, projectIdOf, pageSlugOf, docNameFor, slugify, PAGE_SLUG_RE } from './util.js'
import { defaultStructureYaml, clearStructureCache, primeStructureCache } from './pages.js'

const FIELD = 'default'
const CLAIM_TIMEOUT_MS = 5 * 60 * 1000
const MAX_ATTEMPTS = 3

// --- mention queue state (persisted via store) ---
const mentionState = store.getMentionState()
const waiters = [] // { handles:Set, resolve, timer, done }
const agentLastPoll = new Map() // handle -> ts
const docWatchers = new Map() // docName -> { document, un } for the LIVE instance

function persistMentions() {
  store.saveMentionState(mentionState)
}

// --- hocuspocus server ---

async function authenticate({ requestHeaders, token, documentName }) {
  if (documentName && !DOC_NAME_RE.test(documentName)) throw new Error('bad document name')
  let user = null
  if (token && token.startsWith('cookie')) {
    // browser clients carry their bundle's build id ("cookie:<id>") — a stale
    // tab holding a diverged replica must NOT be allowed to sync: it re-inserts
    // long-deleted content into everyone's doc
    const clientBuild = token.split(':')[1] || null
    if (BUILD_ID !== 'dev' && clientBuild !== BUILD_ID) {
      throw new Error('outdated client — reload the page to get the current version')
    }
  } else if (token) {
    if (token.startsWith('ak_')) {
      const agent = store.agentByKey(token)
      if (agent) user = { username: agent.owner, name: agent.owner, avatar: null }
    } else {
      user = await userFromToken(token)
    }
  }
  if (!user) user = sessionFromCookieHeader(requestHeaders?.cookie || requestHeaders?.get?.('cookie'))
  if (!user) throw new Error('unauthorized')
  if (documentName && !store.canAccessDoc(documentName, user.username)) throw new Error('no access to this document')
  return { user }
}

export const hocuspocus = new Hocuspocus({
  debounce: 2000,
  maxDebounce: 10000,

  async onAuthenticate(data) {
    return authenticate(data)
  },

  async onLoadDocument({ documentName, document }) {
    const state = store.loadDocState(documentName)
    if (state) Y.applyUpdate(document, state)
    return document
  },

  async afterLoadDocument({ documentName, document }) {
    attachWatcher(documentName, document)
    // catch up on anything added while the server was down
    setTimeout(() => scanDocForMentions(documentName, document), 0)
    // one-time upgrade of pre-v2 suggestions to item-based anchors so their
    // ranges stop drifting when neighbouring blocks are added. Must operate on
    // the ALREADY-LOADED document — opening a direct connection here would
    // re-load the doc after unload and loop forever.
    if (!migratedAnchorDocs.has(documentName)) {
      migratedAnchorDocs.add(documentName)
      setTimeout(() => {
        try {
          migrateLegacySuggestionAnchors(document)
        } catch {}
        try {
          migrateCheckboxLists(document)
        } catch {}
        // structure pages are just the yaml field now — drop old heading/intro
        try {
          if (String(documentName).endsWith('::_structure') && !document.isDestroyed) {
            const fragment = document.getXmlFragment(FIELD)
            let hasCode = false
            for (let i = 0; i < fragment.length; i++) {
              if (fragment.get(i) instanceof Y.XmlElement && fragment.get(i).nodeName === 'codeBlock') hasCode = true
            }
            if (hasCode) {
              document.transact(() => {
                for (let i = fragment.length - 1; i >= 0; i--) {
                  const el2 = fragment.get(i)
                  if (!(el2 instanceof Y.XmlElement) || el2.nodeName !== 'codeBlock') fragment.delete(i, 1)
                }
              })
            }
          }
        } catch {}
      }, 50)
    }
  },

  async afterUnloadDocument({ documentName }) {
    releaseWatcher(documentName, { onlyIfDestroyed: true })
  },

  async onStoreDocument({ documentName, document }) {
    let markdown = null
    let title = null
    try {
      const pm = yDocToProsemirrorJSON(document, FIELD)
      markdown = pmToMarkdown(pm)
      title = titleFromPm(pm)
    } catch {}
    store.saveDocState(documentName, Y.encodeStateAsUpdate(document), markdown)
    store.touchProjectOnStore(projectIdOf(documentName), pageSlugOf(documentName), title)
  },
})

function titleFromPm(pm) {
  for (const node of pm.content || []) {
    const text = blockToMarkdown(node)
      ?.replace(/^#+\s*/, '')
      .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') // links -> their text
      .replace(/[*_`]/g, '')
      .trim()
    if (text) return text.slice(0, 80)
  }
  return null
}

// --- document watcher: detect new @mentions in comment threads ---

// Watch the LIVE document instance, not just the name. A document can be
// unloaded and re-loaded (every direct connection that disconnects can trigger
// it), and afterUnloadDocument for the old instance may fire *after*
// afterLoadDocument for the new one. Keying only by name meant the new instance
// could end up with no observer at all — and then @mentions in that document
// were silently never detected for the rest of the process's life.
function attachWatcher(docName, document) {
  const existing = docWatchers.get(docName)
  if (existing?.document === document) return
  if (existing) {
    try {
      existing.un()
    } catch {}
  }
  const threads = document.getMap('threads')
  const observer = () => setTimeout(() => scanDocForMentions(docName, document), 0)
  threads.observeDeep(observer)
  docWatchers.set(docName, { document, un: () => threads.unobserveDeep(observer) })
}

// Drop a watcher only when its own instance is gone: afterUnloadDocument only
// tells us the name, and the unloaded instance is always destroy()ed first, so
// a live replacement keeps its observer.
function releaseWatcher(docName, { onlyIfDestroyed = false } = {}) {
  const entry = docWatchers.get(docName)
  if (!entry) return
  if (onlyIfDestroyed && !entry.document.isDestroyed) return
  try {
    entry.un()
  } catch {}
  docWatchers.delete(docName)
}

function scanDocForMentions(docName, document) {
  if (document.isDestroyed) return
  const threads = document.getMap('threads')
  const agents = store.getAgents()
  const created = []
  const explicitHandles = text =>
    [...new Set([...String(text || '').matchAll(/@([a-z0-9][a-z0-9_.-]{1,38})/g)].map(m => m[1]))].filter(h => agents[h])

  threads.forEach((ythread, threadId) => {
    const messages = ythread.get('messages')
    if (!messages) return
    const all = messages.toArray()
    all.forEach((msg, msgIdx) => {
      if (!msg?.id || msg.authorType === 'agent') return
      const key = `${docName}:${threadId}:${msg.id}`
      if (mentionState.processed[key]) return
      mentionState.processed[key] = true
      // an agent only receives work for docs its owner can access, and a
      // PRIVATE agent only answers mentions written by its owner
      const usable = h =>
        store.canAccessDoc(docName, agents[h]?.owner) && (agents[h]?.shared !== false || agents[h]?.owner === msg.author)
      const explicit = explicitHandles(msg.text)
      let handles = explicit.filter(usable)
      let implicit = false
      // implicit routing only when the author tagged nobody — an explicit
      // mention of an unreachable agent must not be redirected to another one
      if (!explicit.length) {
        // No explicit tag: deliver to the ONE agent clearly involved in this
        // thread (it authored the suggestion under discussion, replied earlier,
        // or was mentioned earlier). Ambiguous (0 or 2+ candidates) => nobody.
        const candidates = new Set()
        const suggestionId = ythread.get('suggestionId')
        if (suggestionId) {
          const sugg = document.getMap('suggestions').get(suggestionId)
          if (sugg?.authorType === 'agent' && agents[sugg.author]) candidates.add(sugg.author)
        }
        for (const prev of all.slice(0, msgIdx)) {
          if (prev.authorType === 'agent' && agents[prev.author]) candidates.add(prev.author)
          for (const h of explicitHandles(prev.text)) candidates.add(h)
        }
        if (candidates.size === 1) {
          handles = [...candidates].filter(usable)
          implicit = true
        }
      }
      const chips = []
      for (const handle of handles) {
        const task = {
          id: newId(12),
          docId: docName,
          threadId,
          messageId: msg.id,
          handle,
          requestedBy: msg.author,
          text: msg.text,
          implicit,
          status: 'pending',
          attempts: 0,
          ts: Date.now(),
        }
        mentionState.tasks[task.id] = task
        created.push(task)
        chips.push({ handle, mentionId: task.id, status: 'pending' })
      }
      if (chips.length) {
        setTimeout(() => writeChips(docName, threadId, msg.id, chips), 0)
      }
    })
  })
  if (created.length) {
    persistMentions()
    wakeWaiters()
  }
}

// The chip is the author's only sign that the mention was picked up, and the
// message is marked processed before it is written — so a write that loses the
// race with the client's own thread transaction must be retried, not dropped.
async function writeChips(docName, threadId, messageId, chips, attempt = 0) {
  const ok = await updateMessage(docName, threadId, messageId, { mentions: chips }).catch(() => false)
  if (ok || attempt >= 5) return
  setTimeout(() => writeChips(docName, threadId, messageId, chips, attempt + 1), 150 * (attempt + 1))
}

// --- privileged Y.Doc writes ---

// Server-side writes/reads open a direct connection. Disconnecting one defaults
// to unloading the document *immediately* — and since every API call does this,
// it kept racing with browsers connecting to the same document: hocuspocus could
// drop the instance a client had just been attached to, leaving that tab talking
// to a Y.Doc the server no longer knows about. From then on server writes (agent
// replies, suggestions, mention chips) landed in a fresh instance and never
// reached the tab. `unloadImmediately: false` defers the unload to the store
// debounce, which re-checks the connection count, so an arriving client wins.
const KEEP_LOADED = { unloadImmediately: false }

async function withDoc(docName, fn) {
  const conn = await hocuspocus.openDirectConnection(docName, { user: { username: '__server__' } })
  try {
    let result
    await conn.transact(document => {
      result = fn(document)
    })
    return result
  } finally {
    await conn.disconnect(KEEP_LOADED)
  }
}

async function readDoc(docName, fn) {
  const conn = await hocuspocus.openDirectConnection(docName, { user: { username: '__server__' } })
  try {
    return fn(conn.document)
  } finally {
    await conn.disconnect(KEEP_LOADED)
  }
}

export async function updateMessage(docName, threadId, messageId, patch) {
  return withDoc(docName, document => {
    const ythread = document.getMap('threads').get(threadId)
    const messages = ythread?.get('messages')
    if (!messages) return false
    const idx = messages.toArray().findIndex(m => m?.id === messageId)
    if (idx === -1) return false
    const updated = { ...messages.get(idx), ...patch }
    messages.delete(idx, 1)
    messages.insert(idx, [updated])
    return true
  })
}

export async function addMessage(docName, threadId, { author, authorType, text }) {
  const msg = { id: newId(10), author, authorType, text, ts: Date.now() }
  const ok = await withDoc(docName, document => {
    const ythread = document.getMap('threads').get(threadId)
    const messages = ythread?.get('messages')
    if (!messages) return false
    messages.push([msg])
    return true
  })
  return ok ? msg : null
}

export async function setMentionChip(task, status) {
  try {
    const ythreadOk = await readDoc(task.docId, document => {
      const ythread = document.getMap('threads').get(task.threadId)
      const messages = ythread?.get('messages')
      if (!messages) return false
      return messages.toArray().some(m => m?.id === task.messageId)
    })
    if (!ythreadOk) return
    await withDoc(task.docId, document => {
      const messages = document.getMap('threads').get(task.threadId).get('messages')
      const idx = messages.toArray().findIndex(m => m?.id === task.messageId)
      if (idx === -1) return
      const msg = { ...messages.get(idx) }
      msg.mentions = (msg.mentions || []).map(c => (c.mentionId === task.id ? { ...c, status } : c))
      messages.delete(idx, 1)
      messages.insert(idx, [msg])
    })
  } catch {}
}

function setTaskStatus(task, status, extra = {}) {
  Object.assign(task, { status, ...extra })
  persistMentions()
  setTimeout(() => setMentionChip(task, status), 0)
}

// --- mention polling (agents) ---

export function handlesOwnedBy(username) {
  const agents = store.getAgents()
  return Object.keys(agents).filter(h => agents[h].owner === username)
}

function claimable(handles) {
  return Object.values(mentionState.tasks).filter(t => t.status === 'pending' && handles.includes(t.handle))
}

async function claimAndEnrich(tasks) {
  const out = []
  for (const task of tasks) {
    setTaskStatus(task, 'claimed', { claimedAt: Date.now(), attempts: (task.attempts || 0) + 1 })
    let context = null
    try {
      context = await readDoc(task.docId, document => {
        const ythread = document.getMap('threads').get(task.threadId)
        const suggestionId = ythread?.get('suggestionId') || null
        const sugg = suggestionId ? document.getMap('suggestions').get(suggestionId) : null
        return {
          excerpt: ythread?.get('excerpt') || null,
          thread_messages: ythread?.get('messages')?.toArray() || [],
          suggestion: sugg
            ? {
                suggestion_id: sugg.id,
                author: sugg.author,
                status: sugg.status,
                original_markdown: sugg.originalMarkdown,
                replacement_markdown: sugg.replacementMarkdown,
                rationale: sugg.rationale,
              }
            : null,
        }
      })
    } catch {}
    const registry = store.getRegistry()
    const projectId = projectIdOf(task.docId)
    const page = pageSlugOf(task.docId)
    out.push({
      mention_id: task.id,
      doc_id: projectId,
      page,
      doc_title: registry[projectId]?.title || projectId,
      page_title: page === 'home' ? null : store.pagesOf(projectId)?.[page]?.title || page,
      thread_id: task.threadId,
      handle: task.handle,
      requested_by: task.requestedBy,
      instruction: task.text,
      anchored_text: context?.excerpt || null,
      thread_messages: (context?.thread_messages || []).map(m => ({ author: m.author, authorType: m.authorType, text: m.text })),
      about_suggestion: context?.suggestion || null,
      implicit_follow_up: !!task.implicit,
      created_at: task.ts,
    })
  }
  return out
}

function pollableHandles(username, onlyHandle) {
  let handles = handlesOwnedBy(username)
  if (onlyHandle) {
    // agent-key poll: exactly this handle
    handles = handles.filter(h => h === onlyHandle)
  } else {
    // HF-token poll: only legacy handles without a dedicated key — once a key
    // exists, possession of the key IS the agent, and owner-level polls must
    // not steal its work
    const agents = store.getAgents()
    handles = handles.filter(h => !agents[h]?.keyHash)
  }
  return handles
}

const NO_HANDLES_ERROR = 'no agent handles registered for your account (handles with a key must poll with that key) — POST /api/agents first'

// Instant, non-claiming view of outstanding work — for debugging. Receiving
// (and claiming) work happens through the streaming poll.
export function snapshotMentions(username, onlyHandle = null) {
  const handles = pollableHandles(username, onlyHandle)
  if (!handles.length) return { error: NO_HANDLES_ERROR }
  const mentions = Object.values(mentionState.tasks)
    .filter(t => handles.includes(t.handle) && (t.status === 'pending' || t.status === 'claimed'))
    .map(t => ({ mention_id: t.id, doc_id: t.docId, thread_id: t.threadId, handle: t.handle, status: t.status, instruction: t.text, created_at: t.ts }))
  return { snapshot: true, mentions }
}

export function pollMentions(username, waitSeconds, onlyHandle = null, opts = {}) {
  const handles = pollableHandles(username, onlyHandle)
  if (!handles.length) return Promise.resolve({ error: NO_HANDLES_ERROR })
  for (const h of handles) agentLastPoll.set(h, Date.now())

  const now = claimable(handles)
  if (now.length || !waitSeconds) return claimAndEnrich(now).then(mentions => ({ mentions }))

  const { maxWait = 55, registerCancel = null } = opts
  return new Promise(resolve => {
    const waiter = {
      handles: new Set(handles),
      done: false,
      fire: async tasks => {
        if (waiter.fired) return
        waiter.fired = true
        clearTimeout(waiter.timer)
        const idx = waiters.indexOf(waiter)
        if (idx !== -1) waiters.splice(idx, 1)
        for (const h of handles) agentLastPoll.set(h, Date.now())
        resolve({ mentions: await claimAndEnrich(tasks) })
      },
    }
    waiter.timer = setTimeout(() => waiter.fire([]), Math.min(waitSeconds, maxWait) * 1000)
    waiters.push(waiter)
    // let the caller cancel when the client disconnects, so a parked waiter
    // can't claim mentions into a dead socket
    if (registerCancel) {
      registerCancel(() => {
        if (waiter.fired) return
        waiter.fired = true
        clearTimeout(waiter.timer)
        const idx = waiters.indexOf(waiter)
        if (idx !== -1) waiters.splice(idx, 1)
        resolve({ mentions: [], cancelled: true })
      })
    }
  })
}

function wakeWaiters() {
  for (const waiter of [...waiters]) {
    if (waiter.fired) continue
    const tasks = claimable([...waiter.handles])
    if (tasks.length) waiter.fire(tasks)
  }
}

export function agentPresence() {
  const agents = store.getAgents()
  return Object.entries(agents).map(([handle, info]) => {
    const hasWaiter = waiters.some(w => !w.fired && w.handles.has(handle))
    const last = agentLastPoll.get(handle) || 0
    return {
      handle,
      owner: info.owner,
      shared: info.shared !== false,
      online: hasWaiter || Date.now() - last < 70 * 1000,
      last_seen: last || null,
    }
  })
}

export function dismissMention(mentionId, username) {
  const task = mentionState.tasks[mentionId]
  if (!task) return { error: 'mention not found' }
  const agents = store.getAgents()
  if (agents[task.handle]?.owner !== username) return { error: 'not your mention' }
  if (task.status !== 'pending' && task.status !== 'claimed') return { error: `mention already ${task.status}` }
  setTaskStatus(task, 'done', { doneAt: Date.now(), dismissed: true })
  return { ok: true }
}

// When an agent handle is removed, fail its outstanding work so chips don't hang
export function cancelAgentTasks(handle) {
  for (const task of Object.values(mentionState.tasks)) {
    if (task.handle === handle && (task.status === 'pending' || task.status === 'claimed')) {
      setTaskStatus(task, 'failed')
    }
  }
}

export function completeMention(mentionId, { docId, threadId, handle } = {}) {
  let completed = 0
  for (const task of Object.values(mentionState.tasks)) {
    const explicit = mentionId && task.id === mentionId
    const implicit = !mentionId && docId && task.docId === docId && task.threadId === threadId && task.handle === handle
    if ((explicit || implicit) && (task.status === 'pending' || task.status === 'claimed')) {
      setTaskStatus(task, 'done', { doneAt: Date.now() })
      completed++
    }
  }
  return completed
}

// requeue stale claims, fail after MAX_ATTEMPTS
setInterval(() => {
  let changed = false
  for (const task of Object.values(mentionState.tasks)) {
    if (task.status !== 'claimed' || Date.now() - (task.claimedAt || 0) < CLAIM_TIMEOUT_MS) continue
    changed = true
    if ((task.attempts || 0) >= MAX_ATTEMPTS) setTaskStatus(task, 'failed')
    else setTaskStatus(task, 'pending')
  }
  if (changed) wakeWaiters()
}, 30 * 1000).unref()

// --- doc reading for agents ---

export async function getDocSnapshot(docName) {
  return readDoc(docName, document => {
    const pm = yDocToProsemirrorJSON(document, FIELD)
    const fragment = document.getXmlFragment(FIELD)
    const blocks = (pm.content || []).map((node, i) => ({
      index: i,
      anchor_start: b64encode(Y.encodeRelativePosition(Y.createRelativePositionFromTypeIndex(fragment, i))),
      anchor_end: b64encode(Y.encodeRelativePosition(Y.createRelativePositionFromTypeIndex(fragment, i + 1))),
      markdown: blockToMarkdown(node) ?? '',
    }))
    const threads = []
    document.getMap('threads').forEach((ythread, id) => {
      threads.push({
        thread_id: id,
        excerpt: ythread.get('excerpt') || null,
        resolved: !!ythread.get('resolved'),
        messages: (ythread.get('messages')?.toArray() || []).map(m => ({ author: m.author, authorType: m.authorType, text: m.text })),
      })
    })
    const suggestions = []
    document.getMap('suggestions').forEach(s => suggestions.push(s))
    return {
      markdown: pmToMarkdown(pm),
      blocks,
      threads,
      suggestions: suggestions.map(s => ({ id: s.id, author: s.author, status: s.status, rationale: s.rationale })),
    }
  })
}

// --- suggestions ---

export async function createSuggestion(docName, { anchorStart, anchorEnd, blockIndex, replacementMarkdown, rationale, author, authorType, threadId, supersedes }) {
  return withDoc(docName, document => {
    // supersedes only LINKS a revision to the original (and reuses its anchors);
    // both stay open side by side — a human decides which, if any, to accept
    let revises = null
    if (supersedes) {
      const old = document.getMap('suggestions').get(supersedes)
      if (old && old.author === author) {
        revises = supersedes
        if (anchorStart == null && blockIndex == null) {
          anchorStart = old.anchorStart
          anchorEnd = old.anchorEnd
        }
      }
    }
    const fragment = document.getXmlFragment(FIELD)
    if (blockIndex != null && (anchorStart == null || anchorEnd == null)) {
      const i = Math.max(0, Math.min(Number(blockIndex), fragment.length - 1))
      anchorStart = b64encode(Y.encodeRelativePosition(Y.createRelativePositionFromTypeIndex(fragment, i)))
      anchorEnd = b64encode(Y.encodeRelativePosition(Y.createRelativePositionFromTypeIndex(fragment, i + 1)))
    }
    const range = resolveBlockRange(document, fragment, anchorStart, anchorEnd)
    if (!range) return { error: 'anchor could not be resolved against the current document' }
    const pm = yDocToProsemirrorJSON(document, FIELD)
    const original = (pm.content || []).slice(range.from, range.to).map(blockToMarkdown).join('\n\n')
    const suggestion = {
      id: newId(10),
      anchorStart,
      anchorEnd,
      author,
      authorType,
      replacementMarkdown: String(replacementMarkdown ?? ''),
      rationale: rationale ? String(rationale) : null,
      originalMarkdown: original,
      status: 'open',
      revises,
      threadId: threadId || null,
      ts: Date.now(),
    }
    // re-anchor to the target block items themselves (see resolveBlockRange)
    suggestion.anchorStart = b64encode(Y.encodeRelativePosition(Y.createRelativePositionFromTypeIndex(fragment, range.from)))
    suggestion.anchorEndIncl = b64encode(Y.encodeRelativePosition(Y.createRelativePositionFromTypeIndex(fragment, Math.max(range.from, range.to - 1))))
    // forward anchor: tracks the first stable block AFTER the range (or the
    // end-of-doc sentinel), so phantom insertion panels get pushed down by
    // content typed at the boundary
    const afterIdx = stableAfterIndex(pm, range.to)
    suggestion.anchorAfter = b64encode(Y.encodeRelativePosition(Y.createRelativePositionFromTypeIndex(fragment, afterIdx ?? fragment.length)))
    document.getMap('suggestions').set(suggestion.id, suggestion)
    return { suggestion }
  })
}

// First content-bearing block at or after `from` — empty paragraphs are skipped
// because y-prosemirror reuses them as shells when users type at boundaries,
// which would drag an anchor onto the newly typed text.
function stableAfterIndex(pm, from) {
  const content = pm.content || []
  for (let i = from; i < content.length; i++) {
    const n = content[i]
    const empty = n.type === 'paragraph' && !(n.content || []).some(c => (c.text || '').trim())
    if (!empty) return i
  }
  return null
}

// anchorEndIncl (when present) references the LAST target block itself, so
// blocks inserted right after the suggestion stay outside the range. The
// legacy exclusive anchorEnd referenced the NEXT block (or the end-of-doc
// sentinel), which swallowed newly typed neighbours into the suggestion.
function resolveBlockRange(document, fragment, anchorStart, anchorEnd, anchorEndIncl = null) {
  try {
    const absStart = Y.createAbsolutePositionFromRelativePosition(Y.decodeRelativePosition(b64decode(anchorStart)), document)
    const absEndRel = anchorEndIncl || anchorEnd
    const absEnd = Y.createAbsolutePositionFromRelativePosition(Y.decodeRelativePosition(b64decode(absEndRel)), document)
    if (!absStart || !absEnd || absStart.type !== fragment || absEnd.type !== fragment) return null
    const from = Math.max(0, Math.min(absStart.index, fragment.length))
    const endIdx = anchorEndIncl ? absEnd.index + 1 : absEnd.index
    const to = Math.max(from, Math.min(Math.max(endIdx, from + 1), fragment.length))
    return { from, to }
  } catch {
    return null
  }
}

const migratedAnchorDocs = new Set()

function migrateLegacySuggestionAnchors(document) {
  if (document.isDestroyed) return
  const suggMap = document.getMap('suggestions')
  const fragment = document.getXmlFragment(FIELD)
  let pm = { content: [] }
  try {
    pm = yDocToProsemirrorJSON(document, FIELD)
  } catch {}
  const updates = []
  suggMap.forEach((s, id) => {
    if (!s || (s.anchorEndIncl && s.anchorAfter) || (s.status !== 'open' && s.status !== 'superseded')) return
    const range = resolveBlockRange(document, fragment, s.anchorStart, s.anchorEnd, s.anchorEndIncl || null)
    if (!range) return
    updates.push([
      id,
      {
        ...s,
        anchorStart: b64encode(Y.encodeRelativePosition(Y.createRelativePositionFromTypeIndex(fragment, range.from))),
        anchorEndIncl: b64encode(Y.encodeRelativePosition(Y.createRelativePositionFromTypeIndex(fragment, Math.max(range.from, range.to - 1)))),
        anchorAfter: b64encode(
          Y.encodeRelativePosition(Y.createRelativePositionFromTypeIndex(fragment, stableAfterIndex(pm, range.to) ?? fragment.length))
        ),
      },
    ])
  })
  if (updates.length) {
    document.transact(() => {
      for (const [id, value] of updates) suggMap.set(id, value)
    })
  }
}

// Did the accepting client's local apply really reach this document? Two
// independent checks, because getting this wrong in either direction is bad:
// trusting a dead client loses the content, doubting a live one duplicates it.
//   1. presence — a synced editor publishes awareness (CollaborationCaret);
//      a tab with a rejected/closed socket has none.
//   2. content — the anchored blocks already read as the replacement.
// Either one is enough to conclude the change is in the document.
function clientApplied(document, suggestion, username) {
  try {
    for (const st of document.awareness.getStates().values()) {
      if (st?.user?.name === username) return true
    }
  } catch {}
  try {
    // Anchor-independent on purpose: a client that applied the replacement has
    // already moved the anchor, so compare against the whole page instead. Each
    // replacement block must appear in full — matching a prefix would mistake a
    // shrinking edit (same opening, sentence removed) for one already applied.
    const pm = yDocToProsemirrorJSON(document, FIELD)
    const flat = node =>
      (node.text || '') + (node.content || []).map(flat).join(node.type === 'paragraph' ? '' : ' ')
    const docText = norm((pm.content || []).map(flat).join(' '))
    const blocks = markdownToBlocks(suggestion.replacementMarkdown)
    if (!blocks.length) return false
    return blocks.every(b => {
      const want = norm(blockPlainText(b))
      return !want || docText.includes(want)
    })
  } catch {
    return false
  }
}

const norm = s => String(s || '').replace(/\s+/g, ' ').trim()

// reader-visible text of a parsed block (marks and structure dropped)
function blockPlainText(b) {
  const seg = ss => (ss || []).map(x => x.text || '').join('')
  if (b.type === 'bulletList' || b.type === 'orderedList') return (b.items || []).map(seg).join(' ')
  if (b.type === 'taskList') return (b.items || []).map(it => seg(it.inline)).join(' ')
  if (b.type === 'table') return (b.rows || []).map(r => r.map(seg).join(' ')).join(' ')
  if (b.type === 'codeBlock' || b.type === 'htmlBlock') return b.text || ''
  if (b.type === 'image' || b.type === 'horizontalRule') return ''
  return seg(b.inline)
}

export async function resolveSuggestionAction(docName, suggestionId, action, username, { markOnly = false } = {}) {
  return withDoc(docName, document => {
    const suggestions = document.getMap('suggestions')
    const suggestion = suggestions.get(suggestionId)
    if (!suggestion) return { error: 'suggestion not found' }

    if (action === 'reopen') {
      // undo of an accept: the text change was reverted client-side, bring the suggestion back
      if (suggestion.status !== 'accepted') return { error: `cannot reopen a ${suggestion.status} suggestion` }
      suggestions.set(suggestionId, { ...suggestion, status: 'open', resolvedBy: null, resolvedAt: null })
      return { ok: true }
    }

    if (suggestion.status !== 'open' && suggestion.status !== 'superseded') return { error: `suggestion already ${suggestion.status}` }

    if (action === 'reject') {
      suggestions.set(suggestionId, { ...suggestion, status: 'rejected', resolvedBy: username, resolvedAt: Date.now() })
      return { ok: true }
    }

    // mark_only means "I already applied the replacement in my editor" (so it
    // lands in that user's undo history). It is only true if that client's
    // websocket actually delivered the change — a tab whose socket is dead
    // (rejected as outdated, offline) still reaches this endpoint over HTTP, and
    // trusting it there recorded accepts whose content was silently dropped.
    if (markOnly && clientApplied(document, suggestion, username)) {
      suggestions.set(suggestionId, { ...suggestion, status: 'accepted', resolvedBy: username, resolvedAt: Date.now() })
      return { ok: true }
    }

    const fragment = document.getXmlFragment(FIELD)
    const range = resolveBlockRange(document, fragment, suggestion.anchorStart, suggestion.anchorEnd, suggestion.anchorEndIncl || null)
    if (!range) {
      suggestions.set(suggestionId, { ...suggestion, status: 'orphaned' })
      return { error: 'the anchored text no longer exists; suggestion marked orphaned' }
    }
    const nodes = markdownToBlocks(suggestion.replacementMarkdown)
    fragment.delete(range.from, range.to - range.from)
    fragment.insert(range.from, nodes.map(buildYBlock))
    suggestions.set(suggestionId, { ...suggestion, status: 'accepted', resolvedBy: username, resolvedAt: Date.now() })
    if (String(docName).endsWith('::_structure')) clearStructureCache(projectIdOf(docName))
    return { ok: true }
  })
}

// --- building Y nodes from markdown block descriptors ---

function yInlineText(segments) {
  // Insert the full plain string first, then format() the marked ranges.
  // Inserting formatted segments sequentially would make later unformatted
  // segments inherit the previous segment's formatting (Yjs semantics).
  const t = new Y.XmlText()
  t.insert(0, (segments || []).map(s => s.text).join(''))
  let pos = 0
  for (const seg of segments || []) {
    if (Object.keys(seg.attrs || {}).length) t.format(pos, seg.text.length, seg.attrs)
    pos += seg.text.length
  }
  return t
}

// Inline content is usually one XmlText run, but a formula is an element, so a
// paragraph containing math becomes an alternating list of text runs and math
// elements. Each text run is built on its own — formatting must not leak across
// a formula, and the offsets above are per-run.
function yInlineChildren(segments) {
  const out = []
  let run = []
  const flush = () => {
    if (run.length) out.push(yInlineText(run))
    run = []
  }
  for (const seg of segments || []) {
    if (seg.math != null) {
      flush()
      const m = new Y.XmlElement('mathInline')
      m.setAttribute('latex', seg.math)
      out.push(m)
    } else if (seg.text) {
      run.push(seg)
    }
  }
  flush()
  // an empty paragraph still needs its text node, or the editor sees no content
  return out.length ? out : [yInlineText([])]
}

function paragraphOf(segments) {
  const p = new Y.XmlElement('paragraph')
  p.insert(0, yInlineChildren(segments))
  return p
}

export function buildYBlock(block) {
  switch (block.type) {
    case 'heading': {
      const el = new Y.XmlElement('heading')
      el.setAttribute('level', block.attrs?.level || 1)
      el.insert(0, yInlineChildren(block.inline))
      return el
    }
    case 'mathBlock': {
      const el = new Y.XmlElement('mathBlock')
      el.setAttribute('latex', block.attrs?.latex || '')
      return el
    }
    case 'codeBlock': {
      const el = new Y.XmlElement('codeBlock')
      if (block.attrs?.language) el.setAttribute('language', block.attrs.language)
      const t = new Y.XmlText()
      t.insert(0, block.text || '')
      el.insert(0, [t])
      return el
    }
    case 'bulletList':
    case 'orderedList': {
      const el = new Y.XmlElement(block.type)
      el.insert(0, (block.items || []).map(segments => {
        const li = new Y.XmlElement('listItem')
        li.insert(0, [paragraphOf(segments)])
        return li
      }))
      return el
    }
    case 'taskList': {
      const el = new Y.XmlElement('taskList')
      el.insert(0, (block.items || []).map(it => {
        const li = new Y.XmlElement('taskItem')
        li.setAttribute('checked', !!it.checked)
        li.insert(0, [paragraphOf(it.inline)])
        return li
      }))
      return el
    }
    case 'table': {
      const el = new Y.XmlElement('table')
      el.insert(0, (block.rows || []).map((cells, r) => {
        const row = new Y.XmlElement('tableRow')
        row.insert(0, cells.map(segs => {
          const cell = new Y.XmlElement(r === 0 ? 'tableHeader' : 'tableCell')
          cell.setAttribute('colspan', 1)
          cell.setAttribute('rowspan', 1)
          cell.insert(0, [paragraphOf(segs)])
          return cell
        }))
        return row
      }))
      return el
    }
    case 'blockquote': {
      const el = new Y.XmlElement('blockquote')
      el.insert(0, [paragraphOf(block.inline)])
      return el
    }
    case 'horizontalRule':
      return new Y.XmlElement('horizontalRule')
    case 'htmlBlock': {
      const el = new Y.XmlElement('htmlBlock')
      el.setAttribute('html', block.text || '')
      return el
    }
    case 'image': {
      const el = new Y.XmlElement('image')
      el.setAttribute('src', block.attrs?.src || '')
      if (block.attrs?.alt) el.setAttribute('alt', block.attrs.alt)
      return el
    }
    default:
      return paragraphOf(block.inline)
  }
}

export async function deleteDoc(id) {
  const pages = Object.keys(store.pagesOf(id) || { home: {} })
  for (const slug of [...new Set(['home', '_structure', ...pages])]) {
    const name = docNameFor(id, slug)
    try { hocuspocus.closeConnections(name) } catch {}
    releaseWatcher(name)
  }
  for (const [taskId, task] of Object.entries(mentionState.tasks)) {
    if (projectIdOf(task.docId) === id) delete mentionState.tasks[taskId]
  }
  persistMentions()
  store.deleteDocFiles(id)
}

// --- doc creation ---

export async function createDoc(title, username) {
  const id = newId(8)
  store.upsertRegistry(id, { title: title || 'Untitled', createdAt: Date.now(), updatedAt: Date.now(), createdBy: username, pages: {} })
  await withDoc(id, document => {
    const fragment = document.getXmlFragment(FIELD)
    if (fragment.length === 0) {
      const h = new Y.XmlElement('heading')
      h.setAttribute('level', 1)
      const t = new Y.XmlText()
      t.insert(0, title || 'Untitled')
      h.insert(0, [t])
      const p = new Y.XmlElement('paragraph')
      fragment.insert(0, [h, p])
    }
  })
  await ensureStructurePage(id)
  return id
}

// --- pages ---

function structureCodeBlock(document) {
  const fragment = document.getXmlFragment(FIELD)
  for (let i = 0; i < fragment.length; i++) {
    const el = fragment.get(i)
    if (el instanceof Y.XmlElement && el.nodeName === 'codeBlock') return el
  }
  return null
}

const ensuredStructure = new Set()

export async function ensureStructurePage(projectId) {
  if (ensuredStructure.has(projectId)) return
  const name = docNameFor(projectId, '_structure')
  if (store.loadDocState(name)) {
    ensuredStructure.add(projectId)
    return
  }
  const slugs = Object.keys(store.pagesOf(projectId) || { home: {} })
  await withDoc(name, document => {
    const fragment = document.getXmlFragment(FIELD)
    if (fragment.length > 0) return
    // just the yaml field — the explanation lives in the page chrome
    const code = new Y.XmlElement('codeBlock')
    code.setAttribute('language', 'yaml')
    const ct = new Y.XmlText()
    ct.insert(0, defaultStructureYaml(slugs))
    code.insert(0, [ct])
    fragment.insert(0, [code])
  })
  store.upsertPageMeta(projectId, '_structure', { title: 'Structure', createdAt: Date.now() })
  ensuredStructure.add(projectId)
  clearStructureCache(projectId)
}

export async function createPage(projectId, title, username, { slug: explicitSlug = null, contentMarkdown = null } = {}) {
  const pages = store.pagesOf(projectId) || {}
  let slug
  if (explicitSlug) {
    // creating a page that the structure yaml references but doesn't exist yet
    slug = String(explicitSlug)
    if (!PAGE_SLUG_RE.test(slug) || slug === '_structure' || slug === 'home') return { error: 'bad page slug' }
    if (pages[slug]) return { error: 'page already exists' }
  } else {
    const base = slugify(title)
    slug = base === '_structure' ? 'page' : base
    let n = 2
    while (pages[slug] || slug === 'home') slug = `${base}-${n++}`.slice(0, 40)
    if (!PAGE_SLUG_RE.test(slug) || slug === '_structure') return { error: 'bad page title' }
  }
  const name = docNameFor(projectId, slug)
  await withDoc(name, document => {
    const fragment = document.getXmlFragment(FIELD)
    if (fragment.length > 0) return
    if (contentMarkdown != null && String(contentMarkdown).trim()) {
      // seed with the proposed content (accepting a new-page suggestion)
      fragment.insert(0, markdownToBlocks(contentMarkdown).map(buildYBlock))
    } else {
      const h = new Y.XmlElement('heading')
      h.setAttribute('level', 1)
      const t = new Y.XmlText()
      t.insert(0, title || slug)
      h.insert(0, [t])
      const p = new Y.XmlElement('paragraph')
      fragment.insert(0, [h, p])
    }
  })
  store.upsertPageMeta(projectId, slug, { title: title || slug, createdAt: Date.now(), createdBy: username })
  await ensureStructurePage(projectId)
  // append to the structure yaml so the new page shows in the tree right away
  let newRaw = null
  await withDoc(docNameFor(projectId, '_structure'), document => {
    const code = structureCodeBlock(document)
    if (!code) return
    const text = code.get(0)
    if (!(text instanceof Y.XmlText)) return
    const current = text.toString()
    if (new RegExp(`^\\s*-\\s*${slug}\\s*$`, 'm').test(current)) return
    text.insert(current.length, `${current.endsWith('\n') || !current ? '' : '\n'}- ${slug}`)
    newRaw = text.toString()
  })
  if (newRaw != null) primeStructureCache(projectId, newRaw)
  else clearStructureCache(projectId)
  return { slug }
}

export async function deletePage(projectId, slug) {
  if (slug === 'home' || slug === '_structure') return { error: 'this page cannot be deleted' }
  const name = docNameFor(projectId, slug)
  try { hocuspocus.closeConnections(name) } catch {}
  releaseWatcher(name)
  for (const [taskId, task] of Object.entries(mentionState.tasks)) {
    if (task.docId === name) delete mentionState.tasks[taskId]
  }
  persistMentions()
  store.deletePageFiles(name)
  store.removePageMeta(projectId, slug)
  // drop its line from the structure yaml
  let newRaw = null
  try {
    await withDoc(docNameFor(projectId, '_structure'), document => {
      const code = structureCodeBlock(document)
      const text = code?.get(0)
      if (!(text instanceof Y.XmlText)) return
      const lines = text.toString().split('\n')
      const keep = lines.filter(l => !new RegExp(`^\\s*-\\s*${slug}\\s*:?\\s*$`).test(l))
      if (keep.length !== lines.length) {
        text.delete(0, text.length)
        text.insert(0, keep.join('\n'))
      }
      newRaw = text.toString()
    })
  } catch {}
  if (newRaw != null) primeStructureCache(projectId, newRaw)
  else clearStructureCache(projectId)
  return { ok: true }
}

// One-time upgrade of legacy bullet lists whose every item is a checkbox
// ("[ ] ..." / "[x] ...") — written before task-list support — into real task
// lists. Only converts a list when ALL items match, so mixed lists are left
// alone. Reuses the markdown round-trip (preserves inline formatting).
function migrateCheckboxLists(document) {
  if (document.isDestroyed) return
  const fragment = document.getXmlFragment(FIELD)
  let pm
  try {
    pm = yDocToProsemirrorJSON(document, FIELD)
  } catch {
    return
  }
  const content = pm.content || []
  const CHECKBOX = /^\[[ xX]?\]\s+\S/
  const itemText = li => (li.content || []).map(b => (b.content || []).map(n => n.text || '').join('')).join(' ')
  const updates = []
  content.forEach((node, i) => {
    if (node.type !== 'bulletList') return
    const items = node.content || []
    if (items.length && items.every(li => CHECKBOX.test(itemText(li)))) {
      const blocks = markdownToBlocks(blockToMarkdown(node))
      if (blocks.length === 1 && blocks[0].type === 'taskList') updates.push([i, blocks[0], itemText(items[0])])
    }
  })
  if (!updates.length) return
  document.transact(() => {
    // apply high index first so earlier indices stay valid; re-verify each
    // target is still the expected bullet list (guards against a concurrent
    // edit having shifted the doc between the snapshot and this transaction)
    for (const [i, block, firstText] of updates.reverse()) {
      const el = i < fragment.length ? fragment.get(i) : null
      if (!(el instanceof Y.XmlElement) || el.nodeName !== 'bulletList') continue
      if (!el.toString().replace(/<[^>]+>/g, '').trim().startsWith(firstText.slice(0, 12))) continue
      fragment.delete(i, 1)
      fragment.insert(i, [buildYBlock(block)])
    }
  })
}

// --- new-page proposals -------------------------------------------------------
// A proposal for a whole new page (title + content) lives in the project's
// _structure doc, so it is project-level and visible before acceptance. On
// accept it becomes a real page (with content) added to the structure.

export async function createPageProposal(projectId, { title, contentMarkdown, rationale, author, authorType }) {
  const clean = String(title || '').trim().slice(0, 120)
  if (!clean) return { error: 'title required' }
  const pages = store.pagesOf(projectId) || {}
  await ensureStructurePage(projectId)
  const id = newId(10)
  let existingSlugs = new Set(Object.keys(pages))
  const result = await withDoc(docNameFor(projectId, '_structure'), document => {
    const map = document.getMap('pageProposals')
    map.forEach(p => {
      if (p.status === 'open') existingSlugs.add(p.slug)
    })
    const base = slugify(clean)
    let slug = base === '_structure' ? 'page' : base
    let n = 2
    while (existingSlugs.has(slug) || slug === 'home') slug = `${base}-${n++}`.slice(0, 40)
    map.set(id, {
      id,
      kind: 'newPage',
      slug,
      title: clean,
      contentMarkdown: String(contentMarkdown || ''),
      rationale: rationale ? String(rationale) : null,
      author,
      authorType,
      status: 'open',
      ts: Date.now(),
    })
    return { slug }
  })
  clearStructureCache(projectId)
  return { id, slug: result.slug }
}

export async function getPageProposal(projectId, pid) {
  return readDoc(docNameFor(projectId, '_structure'), document => {
    const p = document.getMap('pageProposals').get(pid)
    return p || null
  })
}

export async function resolvePageProposal(projectId, pid, action, username) {
  const proposal = await getPageProposal(projectId, pid)
  if (!proposal) return { error: 'proposal not found' }
  if (proposal.status !== 'open') return { error: `proposal already ${proposal.status}` }
  if (action === 'reject') {
    await withDoc(docNameFor(projectId, '_structure'), document => {
      const map = document.getMap('pageProposals')
      const p = map.get(pid)
      if (p) map.set(pid, { ...p, status: 'rejected', resolvedBy: username, resolvedAt: Date.now() })
    })
    clearStructureCache(projectId)
    return { ok: true }
  }
  // accept: create the page with its content, add to structure, mark accepted
  const created = await createPage(projectId, proposal.title, username, {
    slug: proposal.slug,
    contentMarkdown: proposal.contentMarkdown,
  })
  if (created.error) return created
  await withDoc(docNameFor(projectId, '_structure'), document => {
    const map = document.getMap('pageProposals')
    const p = map.get(pid)
    if (p) map.set(pid, { ...p, status: 'accepted', resolvedBy: username, resolvedAt: Date.now(), slug: created.slug })
  })
  clearStructureCache(projectId)
  return { ok: true, slug: created.slug }
}