import type { TLShapeId } from '@tldraw/tlschema' import type { IncomingMessage, ServerResponse } from 'node:http' import { z } from 'zod' import { nodeLabel, normaliseType } from '../shared/display.ts' import { paperShapeId } from '../shared/ids.ts' import type { PaperCardProps } from '../shared/paperCard.ts' import { defaultPaperCardProps, NODE_SIZE, type PaperCardShape } from '../shared/shapes.ts' import { isAgentActive } from './acpPipe.ts' import { addRemovedLink, boardsOf, createBoard, ensureBoard, findCard, findMember, findPaper, joinMember, markSeen, membersOf, papersOf, seedCount, syncPapers, unvotedCount, updateBoard, upsertPaper, type BoardPaper, type BoardRow, } from './boardDb.ts' import { addEvent, lastSeq, latestEvents, newSince, streamEvents } from './events.ts' import { placeNear } from './layout.ts' import { papersClient, PapersError, type PaperRecord, type PapersClient } from './papersClient.ts' import { PdfError, servePdf } from './pdfProxy.ts' import { MAX_SEED_PDF_BYTES, pdfSeed, savePdf } from './pdfSeed.ts' import { HttpError, readRoom, writeRoom, writer } from './rooms.ts' import { castVote, triage } from './triage.ts' // The board service's REST API at /api/v1 (T08): boards, members, seeds, papers, votes, removed // links, the activity feed and its SSE stream. JSON in and out, ISO 8601 UTC times, and errors as // {"error": {"code", "message"}}. Writes need `Authorization: Bearer `, trusted like the // sync socket (decision D5); reads don't, so an EventSource (which can't send headers) can listen. const BEARER = /^Bearer\s+(m_[A-Za-z0-9_-]{1,64})$/ /** A board ID is its room ID; the same characters the ACP pipe accepts. */ const BOARD_ID = /^[A-Za-z0-9_-]{1,128}$/ const PAPER_ID = /^[^\s\u0000-\u001f]{1,200}$/u const MAX_BODY = 64 * 1024 export class ApiError extends Error { status: number code: string constructor(status: number, code: string, message: string) { super(message) this.status = status this.code = code } } interface Call { req: IncomingMessage res: ServerResponse url: URL /** Checked against BOARD_ID; empty for /boards. */ boardId: string paperId: string member: string | null papers: PapersClient } /** A JSON reply (no body: 204), or null when the handler wrote the response itself. */ type Reply = { status: number; body?: unknown } | null type Handler = (c: Call) => Reply | Promise const ok = (body: unknown): Reply => ({ status: 200, body }) const NO_CONTENT: Reply = { status: 204 } function send(res: ServerResponse, status: number, body?: unknown, headers: Record = {}) { if (res.headersSent) return res.end() if (body === undefined) { res.writeHead(status, headers) return res.end() } res.writeHead(status, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store', ...headers }) res.end(JSON.stringify(body)) } const CODES: Record = { 400: 'bad_request', 404: 'not_found', 409: 'conflict', 503: 'unavailable' } function fail(res: ServerResponse, e: unknown) { const error = (status: number, code: string, message: string, headers?: Record) => send(res, status, { error: { code, message } }, headers) if (e instanceof ApiError) { const headers: Record = e.status === 401 ? { 'www-authenticate': 'Bearer' } : {} return error(e.status, e.code, e.message, headers) } if (e instanceof HttpError) return error(e.status, CODES[e.status] ?? 'error', e.message) if (e instanceof PapersError) { return e.code === 'not_found' ? error(404, 'paper_not_found', e.message) : error(502, 'papers_unavailable', e.message) } if (e instanceof PdfError) return error(502, e.refused ? 'pdf_refused' : 'pdf_unavailable', e.message) if (e instanceof Error && e.name === 'ValidationError') return error(400, 'invalid', String(e.message)) console.error('REST API:', e) error(500, 'internal', 'Internal error') } function needMember(c: Call): string { if (!c.member) throw new ApiError(401, 'unauthorized', 'Send Authorization: Bearer ') return c.member } function param(raw: string, pattern: RegExp, what: string): string { let value: string try { value = decodeURIComponent(raw) } catch { throw new ApiError(400, 'bad_request', `Bad ${what} in the path`) } if (!pattern.test(value)) throw new ApiError(400, 'bad_request', `Bad ${what}: ${value.slice(0, 200)}`) return value } const issues = (error: z.ZodError) => error.issues.map((i) => (i.path.length ? `${i.path.join('.')}: ${i.message}` : i.message)).join('; ') async function readBody(req: IncomingMessage, schema: S): Promise> { const chunks: Buffer[] = [] let size = 0 for await (const chunk of req as AsyncIterable) { size += chunk.length if (size > MAX_BODY) throw new ApiError(413, 'too_large', `The body is over ${MAX_BODY} bytes`) chunks.push(chunk) } let value: unknown try { value = JSON.parse(Buffer.concat(chunks).toString() || 'null') } catch { throw new ApiError(400, 'bad_json', 'The body must be JSON') } const parsed = schema.safeParse(value) if (!parsed.success) throw new ApiError(400, 'invalid', issues(parsed.error)) return parsed.data } function readQuery(url: URL, schema: S): z.infer { const parsed = schema.safeParse(Object.fromEntries(url.searchParams)) if (!parsed.success) throw new ApiError(400, 'invalid', issues(parsed.error)) return parsed.data } // ---- input const text = (max: number) => z.string().trim().max(max) const goalId = z.string().regex(/^[a-z0-9][a-z0-9-]{0,39}$/, 'goal IDs look like new-findings') const goals = z .array(z.object({ id: goalId, label: text(60).min(1) }).strict()) .min(1) .max(8) .refine((g) => new Set(g.map((x) => x.id)).size === g.length, 'goal IDs must differ') const boardInput = z.object({ title: text(200).optional(), question: text(2000).optional(), context: text(10_000).optional() }).strict() const boardPatch = boardInput .extend({ goals: goals.optional() }) .refine((b) => Object.values(b).some((v) => v !== undefined), 'Give at least one of title, question, context, goals') // without a name, a new member joins with an empty one and a known one keeps theirs const memberInput = z.object({ displayName: text(40).optional() }).strict() const seenInput = z.object({ seq: z.number().int().min(0) }).strict() const seedInput = z.union([ z.object({ doi: z.string().trim().min(3).max(300) }).strict(), z.object({ pmid: z.string().trim().regex(/^\d{1,9}$/, 'a PMID is digits') }).strict(), ]) const voteInput = z.object({ decision: z.enum(['keep', 'discard']).nullable() }).strict() const paperIdInput = z.string().regex(PAPER_ID, 'not a paper ID') const removedLinkInput = z.object({ from: paperIdInput, to: paperIdInput, kind: z.enum(['cites', 'similar', 'user']) }).strict() const STATUSES = ['new', 'kept', 'contested', 'discarded'] as const const KINDS = ['seed', 'kept', 'suggestion'] as const const list = (values: readonly [T, ...T[]]) => z .string() .transform((s) => s.split(',').filter(Boolean)) .pipe(z.array(z.enum(values))) const papersQuery = z.object({ sort: z.enum(['rank', 'added']).default('rank'), status: list(STATUSES).optional(), kind: list(KINDS).optional(), goal: goalId.optional(), }) const seq = z.coerce.number().int().min(0) const activityQuery = z.object({ after: seq.default(0), limit: z.coerce.number().int().min(1).max(500).default(100) }) // ---- views // the room's cards are read first, so the seed and paper counts are up to date function boardView(board: BoardRow, caller: string | null) { const seen = caller ? (findMember(board.id, caller)?.lastSeenSeq ?? 0) : 0 const paperCount = syncPapers(board.id).length const fresh = caller ? newSince(board.id, seen, caller) : 0 return { ...board, members: membersOf(board.id), seedCount: seedCount(board.id), paperCount, lastSeq: lastSeq(board.id), agentActive: isAgentActive(board.id), // for the caller: papers they haven't voted on, and the "Catch me up" badge (T13; Home calls it newCount) ...(caller ? { unvoted: unvotedCount(board.id, caller), newSinceLastSeen: fresh, newCount: fresh } : {}), } } function paperRow(card: PaperCardShape, row: BoardPaper | null) { const p = card.props return { paperId: p.paperId, label: nodeLabel(p.authors, p.year), title: p.title, authors: p.authors, year: p.year, venue: p.venue, type: p.type, kind: p.kind, status: p.status, scores: p.scores, keepBy: p.keepBy, discardBy: p.discardBy, working: p.working, addedBy: row?.addedBy ?? null, onBehalfOf: row?.onBehalfOf ?? null, runId: row?.runId ?? null, addedAt: row?.addedAt ?? null, } } // drops empty strings, lists and objects, so missing parts are left out rather than blank function compact(o: Record): Record | undefined { const kept = Object.entries(o).filter( ([, v]) => v !== undefined && v !== null && v !== '' && !(Array.isArray(v) && !v.length) && !(typeof v === 'object' && !Array.isArray(v) && !Object.keys(v).length) ) return kept.length ? Object.fromEntries(kept) : undefined } /** Concept's paper record: the papers MCP's fields, what the card shows, and the board's part. */ function paperView(boardId: string, card: PaperCardShape, row: BoardPaper | null, record: PaperRecord | null, skipped: string[]) { const p = card.props const pdf = record?.openAccessPdf ? `/api/v1/boards/${boardId}/papers/${encodeURIComponent(p.paperId)}/pdf` : undefined return { ...compact({ paperId: p.paperId, ids: record?.ids, title: record?.title || p.title, authors: record?.authors?.length ? record.authors : p.authors, year: record?.year ?? p.year, venue: record?.venue || p.venue, url: record?.url || p.url, abstract: record?.abstract, openAccessPdf: record?.openAccessPdf, pdf, type: p.type, typeRaw: p.typeRaw, peerReviewed: p.peerReviewed, citedBy: p.citedBy, relevance: compact({ scores: p.scores, claims: p.claims, quotes: p.quotes, glance: p.glance, figure: p.figure, gaps: p.gaps }), reading: compact({ whyItMatters: p.oneLine, summary: p.summary, provenance: p.provenance }), foundBy: p.foundBy, }), board: { kind: p.kind, status: p.status, votes: { keep: p.keepBy, discard: p.discardBy }, working: p.working, addedBy: row?.addedBy ?? null, onBehalfOf: row?.onBehalfOf ?? null, runId: row?.runId ?? null, addedAt: row?.addedAt ?? null, }, ...(skipped.length ? { skipped } : {}), } } function needCard(boardId: string, paperId: string): PaperCardShape { const card = readRoom(boardId, (txn) => findCard(txn, paperId)) if (!card) throw new ApiError(404, 'not_found', `No card for ${paperId} on this board`) return card } /** Adds the member to the board if they're new, with a member.joined event. */ function join(boardId: string, memberId: string, displayName?: string) { const { member, joined } = joinMember(boardId, memberId, displayName) if (joined) addEvent({ boardId, actor: memberId, type: 'member.joined', payload: { memberId, displayName: member.displayName } }) return member } // ---- routes async function postBoard(c: Call): Promise { const member = needMember(c) const board = createBoard(await readBody(c.req, boardInput), member) addEvent({ boardId: board.id, actor: member, type: 'board.created', payload: { title: board.title } }) join(board.id, member) return { status: 201, body: boardView(board, member) } } function listBoards(c: Call): Reply { const member = needMember(c) return ok(boardsOf(member).map((b) => boardView(b, member))) } function getBoard(c: Call): Reply { // opening an old room makes its row return ok(boardView(ensureBoard(c.boardId), c.member)) } async function patchBoard(c: Call): Promise { const member = needMember(c) const board = updateBoard(c.boardId, await readBody(c.req, boardPatch)) join(board.id, member) return ok(boardView(board, member)) } async function putMember(c: Call): Promise { const member = needMember(c) const { displayName } = await readBody(c.req, memberInput) return ok(join(c.boardId, member, displayName)) } async function postSeen(c: Call): Promise { const member = needMember(c) const { seq } = await readBody(c.req, seenInput) join(c.boardId, member) markSeen(c.boardId, member, Math.min(seq, lastSeq(c.boardId))) return NO_CONTENT } function seedFields(r: PaperRecord): Partial { return { title: r.title || r.paperId, authors: Array.isArray(r.authors) ? r.authors.filter((a) => typeof a === 'string') : [], year: Number.isInteger(r.year) ? (r.year as number) : null, venue: r.venue ?? '', url: r.url ?? '', type: normaliseType(r.type), typeRaw: r.type ?? '', } } // the form around the file: field headers and boundaries const MAX_FORM = MAX_SEED_PDF_BYTES + 64 * 1024 /** The `file` of a multipart/form-data body: a PDF of at most 20 MB. */ async function readPdfForm(req: IncomingMessage): Promise<{ name: string; data: Uint8Array }> { const tooLarge = () => new ApiError(413, 'too_large', `A PDF can be at most ${MAX_SEED_PDF_BYTES / 1024 / 1024} MB`) if (Number(req.headers['content-length'] ?? 0) > MAX_FORM) throw tooLarge() const chunks: Buffer[] = [] let size = 0 for await (const chunk of req as AsyncIterable) { size += chunk.length if (size > MAX_FORM) throw tooLarge() chunks.push(chunk) } let form: FormData try { form = await new Response(Buffer.concat(chunks), { headers: { 'content-type': req.headers['content-type'] ?? '' } }).formData() } catch { throw new ApiError(400, 'bad_request', 'The body must be multipart/form-data') } const file = form.get('file') if (!(file instanceof File)) throw new ApiError(400, 'invalid', 'Send the PDF as the form field "file"') const name = file.name || 'the file' if (file.type !== 'application/pdf') throw new ApiError(415, 'unsupported_type', `${name} isn't a PDF`) if (file.size > MAX_SEED_PDF_BYTES) throw tooLarge() return { name, data: new Uint8Array(await file.arrayBuffer()) } } const notFound = (e: unknown): e is PapersError => e instanceof PapersError && e.code === 'not_found' // A dropped PDF (T16) is looked up by the first DOI in it, else by its title; it's kept with the // uploads once it has become a seed. async function resolvePdf(c: Call, resolveSeed: NonNullable) { const { name, data } = await readPdfForm(c.req) let found: Awaited> try { found = await pdfSeed(data) } catch { throw new ApiError(422, 'bad_pdf', `Couldn't read ${name} as a PDF. Paste its DOI instead.`) } let doiMiss: PapersError | null = null for (const seed of [found.doi && { doi: found.doi }, found.title && { title: found.title }]) { if (!seed) continue try { return { record: await resolveSeed(seed), upload: savePdf(data) } } catch (e) { if (!notFound(e)) throw e if ('doi' in seed) doiMiss = e } } throw doiMiss ?? new ApiError(422, 'no_doi', `Couldn't find a DOI in ${name}. Paste its DOI instead.`) } // A new seed card is placed right of the board's content, authored by the member; a card already // there (an agent's suggestion, say) becomes a seed. The seed is a DOI, a PMID, or a PDF. async function postSeed(c: Call): Promise { const member = needMember(c) const pdf = /^multipart\/form-data\b/i.test(c.req.headers['content-type'] ?? '') const seed = pdf ? null : await readBody(c.req, seedInput) if (!c.papers.resolveSeed) throw new ApiError(501, 'not_implemented', 'This papers client has no resolve_seed') const { record, upload } = seed ? { record: await c.papers.resolveSeed(seed), upload: null } : await resolvePdf(c, c.papers.resolveSeed) join(c.boardId, member) const card = writeRoom(c.boardId, (txn) => { const w = writer(txn) const existing = findCard(txn, record.paperId) if (existing?.props.kind === 'seed') return existing const size = NODE_SIZE.square.min if (existing) { const { status, kind } = triage('seed', existing.props) const grown = Math.max(existing.props.w, size) return w.patchShape({ id: existing.id, props: { kind, status, w: grown, h: grown } }) as PaperCardShape } const at = placeNear([...txn.values()], { w: size, h: size }, null) return w.createShape({ id: paperShapeId(record.paperId) as TLShapeId, type: 'paper-card', ...at, props: defaultPaperCardProps(record.paperId, { ...seedFields(record), kind: 'seed', status: 'kept', w: size, h: size }), meta: { author: member, paperId: record.paperId }, }) as PaperCardShape }) upsertPaper(c.boardId, card) addEvent({ boardId: c.boardId, actor: member, type: 'seed.added', payload: { paperId: record.paperId, title: card.props.title } }) const body = paperView(c.boardId, card, findPaper(c.boardId, record.paperId), record, []) return { status: 201, body: upload ? { ...body, upload: `/api/uploads/${upload}` } : body } } function listPapers(c: Call): Reply { const { sort, status, kind, goal } = readQuery(c.url, papersQuery) const cards = syncPapers(c.boardId).filter( (card) => (!status || status.includes(card.props.status)) && (!kind || kind.includes(card.props.kind)) ) const rows = new Map(papersOf(c.boardId).map((r) => [r.paperId, r])) const papers = cards.map((card) => paperRow(card, rows.get(card.props.paperId) ?? null)) if (sort === 'added') papers.sort((a, b) => (b.addedAt ?? '').localeCompare(a.addedAt ?? '')) else { // best fit for the goal first (the board's first goal unless one is named); unscored last const g = goal ?? ensureBoard(c.boardId).goals[0]?.id ?? '' papers.sort((a, b) => (b.scores[g]?.score ?? -1) - (a.scores[g]?.score ?? -1)) } return ok(papers) } async function getPaper(c: Call): Promise { const card = needCard(c.boardId, c.paperId) let record: PaperRecord | null = null const skipped: string[] = [] try { record = await c.papers.getPaper(c.paperId) } catch (e) { if (!(e instanceof PapersError)) throw e skipped.push(e.message) } return ok(paperView(c.boardId, card, findPaper(c.boardId, c.paperId), record, skipped)) } async function putVote(c: Call): Promise { const member = needMember(c) const { decision } = await readBody(c.req, voteInput) join(c.boardId, member) return ok(castVote(c.boardId, c.paperId, member, decision)) } async function getPdf(c: Call): Promise { needCard(c.boardId, c.paperId) await servePdf(c.res, c.paperId, async () => { const link = (await c.papers.getPaper(c.paperId)).openAccessPdf if (!link) throw new ApiError(404, 'no_pdf', `No open-access PDF is known for ${c.paperId}`) return link }) return null } async function putRemovedLink(c: Call): Promise { const member = needMember(c) const link = await readBody(c.req, removedLinkInput) join(c.boardId, member) if (addRemovedLink(c.boardId, link, member)) addEvent({ boardId: c.boardId, actor: member, type: 'link.removed', payload: link }) return NO_CONTENT } function getActivity(c: Call): Reply { const { after, limit } = readQuery(c.url, activityQuery) ensureBoard(c.boardId) return ok(latestEvents(c.boardId, after, limit)) } function getEvents(c: Call): Reply { const raw = c.req.headers['last-event-id'] ?? c.url.searchParams.get('after') let after: number | null = null if (raw !== undefined && raw !== null && raw !== '') { const parsed = seq.safeParse(raw) if (!parsed.success) throw new ApiError(400, 'invalid', 'Last-Event-ID and ?after= take an event seq') after = parsed.data } streamEvents(c.res, c.boardId, after) return null } const B = '([^/]+)' const route = (method: string, path: string, handler: Handler) => ({ method, pattern: new RegExp(`^/api/v1${path}$`), handler }) const routes = [ route('POST', '/boards', postBoard), route('GET', '/boards', listBoards), route('GET', `/boards/${B}`, getBoard), route('PATCH', `/boards/${B}`, patchBoard), route('PUT', `/boards/${B}/members/me`, putMember), route('POST', `/boards/${B}/seen`, postSeen), route('POST', `/boards/${B}/seeds`, postSeed), route('GET', `/boards/${B}/papers`, listPapers), route('GET', `/boards/${B}/papers/${B}`, getPaper), route('PUT', `/boards/${B}/papers/${B}/vote`, putVote), route('GET', `/boards/${B}/papers/${B}/pdf`, getPdf), route('PUT', `/boards/${B}/removed-links`, putRemovedLink), route('GET', `/boards/${B}/activity`, getActivity), route('GET', `/boards/${B}/events`, getEvents), ] /** Everything under /api/v1. */ export async function handleApi(req: IncomingMessage, res: ServerResponse, url: URL, papers: PapersClient = papersClient()) { try { const matching = routes.filter((r) => r.pattern.test(url.pathname)) if (!matching.length) throw new ApiError(404, 'not_found', `No route ${url.pathname}`) const r = matching.find((r) => r.method === req.method) if (!r) { res.setHeader('allow', matching.map((m) => m.method).join(', ')) throw new ApiError(405, 'method_not_allowed', `${req.method} isn't allowed here`) } const [, board, paper] = url.pathname.match(r.pattern)! const header = req.headers.authorization const reply = await r.handler({ req, res, url, boardId: board ? param(board, BOARD_ID, 'board ID') : '', paperId: paper ? param(paper, PAPER_ID, 'paper ID') : '', member: (Array.isArray(header) ? header[0] : header)?.trim().match(BEARER)?.[1] ?? null, papers, }) if (reply) send(res, reply.status, reply.body) } catch (e) { fail(res, e) } }