Markdown rendering update (#1989)

* Enable markdown rendering via web worker and improve code highlighting

* LanguageFn typing

src/lib/components/chat/MarkdownRenderer.svelte

@@ -1,11 +1,10 @@
 <script lang="ts">
 	import { processBlocks, processBlocksSync, type BlockToken } from "$lib/utils/marked";
-	// import MarkdownWorker from "$lib/workers/markdownWorker?worker";
+	import MarkdownWorker from "$lib/workers/markdownWorker?worker";
 	import MarkdownBlock from "./MarkdownBlock.svelte";
 	import { browser } from "$app/environment";
 
-	import DOMPurify from "isomorphic-dompurify";
-	import { onMount } from "svelte";
+	import { onMount, onDestroy } from "svelte";
 	import { updateDebouncer } from "$lib/utils/updates";
 
 	interface Props {
@@ -17,53 +16,51 @@
 	let { content, sources = [], loading = false }: Props = $props();
 
 	let blocks: BlockToken[] = $state(processBlocksSync(content, sources));
+	let worker: Worker | null = null;
+	let latestRequestId = 0;
 
-	async function processContent(
-		content: string,
-		sources: { title?: string; link: string }[]
-	): Promise<BlockToken[]> {
-		// Note: Worker support for blocks can be added later if needed
-		// For now, use direct processing which is still efficient due to block memoization
-		return processBlocks(content, sources);
+	function handleBlocks(result: BlockToken[], requestId: number) {
+		if (requestId !== latestRequestId) return;
+		blocks = result;
+		updateDebouncer.endRender();
 	}
 
 	$effect(() => {
 		if (!browser) {
 			blocks = processBlocksSync(content, sources);
-		} else {
-			(async () => {
-				updateDebouncer.startRender();
-				blocks = await processContent(content, sources).then(async (processedBlocks) =>
-					Promise.all(
-						processedBlocks.map(async (block) => ({
-							...block,
-							tokens: await Promise.all(
-								block.tokens.map(async (token) => {
-									if (token.type === "text") {
-										token.html = DOMPurify.sanitize(await token.html);
-									}
-									return token;
-								})
-							),
-						}))
-					)
-				);
+			return;
+		}
+
+		const requestId = ++latestRequestId;
 
-				updateDebouncer.endRender();
-			})();
+		if (worker) {
+			updateDebouncer.startRender();
+			worker.postMessage({ type: "process", content, sources, requestId });
+			return;
 		}
+
+		(async () => {
+			updateDebouncer.startRender();
+			const processed = await processBlocks(content, sources);
+			// Only apply if this is still the latest request
+			handleBlocks(processed, requestId);
+		})();
 	});
 
 	onMount(() => {
-		// todo: fix worker, seems to be transmitting a lot of data
-		// worker = browser && window.Worker ? new MarkdownWorker() : null;
+		if (typeof Worker !== "undefined") {
+			worker = new MarkdownWorker();
+			worker.onmessage = (event: MessageEvent) => {
+				const data = event.data as { type?: string; blocks?: BlockToken[]; requestId?: number };
+				if (data?.type !== "processed" || !data.blocks || data.requestId === undefined) return;
+				handleBlocks(data.blocks, data.requestId);
+			};
+		}
+	});
 
-		DOMPurify.addHook("afterSanitizeAttributes", (node) => {
-			if (node.tagName === "A") {
-				node.setAttribute("target", "_blank");
-				node.setAttribute("rel", "noreferrer");
-			}
-		});
+	onDestroy(() => {
+		worker?.terminate();
+		worker = null;
 	});
 </script>
 

src/lib/utils/marked.ts

@@ -7,10 +7,55 @@ type SimpleSource = {
 	title?: string;
 	link: string;
 };
-import hljs from "highlight.js";
+import hljs from "highlight.js/lib/core";
+import type { LanguageFn } from "highlight.js";
+import javascript from "highlight.js/lib/languages/javascript";
+import typescript from "highlight.js/lib/languages/typescript";
+import json from "highlight.js/lib/languages/json";
+import bash from "highlight.js/lib/languages/bash";
+import shell from "highlight.js/lib/languages/shell";
+import python from "highlight.js/lib/languages/python";
+import go from "highlight.js/lib/languages/go";
+import rust from "highlight.js/lib/languages/rust";
+import java from "highlight.js/lib/languages/java";
+import csharp from "highlight.js/lib/languages/csharp";
+import cpp from "highlight.js/lib/languages/cpp";
+import cLang from "highlight.js/lib/languages/c";
+import xml from "highlight.js/lib/languages/xml";
+import css from "highlight.js/lib/languages/css";
+import scss from "highlight.js/lib/languages/scss";
+import markdownLang from "highlight.js/lib/languages/markdown";
+import yaml from "highlight.js/lib/languages/yaml";
+import sql from "highlight.js/lib/languages/sql";
+import plaintext from "highlight.js/lib/languages/plaintext";
 import { parseIncompleteMarkdown } from "./parseIncompleteMarkdown";
 import { parseMarkdownIntoBlocks } from "./parseBlocks";
 
+const bundledLanguages: [string, LanguageFn][] = [
+	["javascript", javascript],
+	["typescript", typescript],
+	["json", json],
+	["bash", bash],
+	["shell", shell],
+	["python", python],
+	["go", go],
+	["rust", rust],
+	["java", java],
+	["csharp", csharp],
+	["cpp", cpp],
+	["c", cLang],
+	["xml", xml],
+	["html", xml],
+	["css", css],
+	["scss", scss],
+	["markdown", markdownLang],
+	["yaml", yaml],
+	["sql", sql],
+	["plaintext", plaintext],
+];
+
+bundledLanguages.forEach(([name, language]) => hljs.registerLanguage(name, language));
+
 interface katexBlockToken extends Tokens.Generic {
 	type: "katexBlock";
 	raw: string;
@@ -159,6 +204,27 @@ function addInlineCitations(md: string, webSearchSources: SimpleSource[] = []):
 	});
 }
 
+function sanitizeHref(href?: string | null): string | undefined {
+	if (!href) return undefined;
+	const trimmed = href.trim();
+	const lower = trimmed.toLowerCase();
+	if (lower.startsWith("javascript:") || lower.startsWith("data:text/html")) {
+		return undefined;
+	}
+	return trimmed.replace(/>$/, "");
+}
+
+function highlightCode(text: string, lang?: string): string {
+	if (lang && hljs.getLanguage(lang)) {
+		try {
+			return hljs.highlight(text, { language: lang, ignoreIllegals: true }).value;
+		} catch {
+			// fall through to auto-detect
+		}
+	}
+	return hljs.highlightAuto(text).value;
+}
+
 function createMarkedInstance(sources: SimpleSource[]): Marked {
 	return new Marked({
 		hooks: {
@@ -166,8 +232,12 @@ function createMarkedInstance(sources: SimpleSource[]): Marked {
 		},
 		extensions: [katexBlockExtension, katexInlineExtension],
 		renderer: {
-			link: (href, title, text) =>
-				`<a href="${href?.replace(/>$/, "")}" target="_blank" rel="noreferrer">${text}</a>`,
+			link: (href, title, text) => {
+				const safeHref = sanitizeHref(href);
+				return safeHref
+					? `<a href="${safeHref}" target="_blank" rel="noreferrer">${text}</a>`
+					: `<span>${escapeHTML(text ?? "")}</span>`;
+			},
 			html: (html) => escapeHTML(html),
 		},
 		gfm: true,
@@ -200,6 +270,13 @@ type TextToken = {
 	html: string | Promise<string>;
 };
 
+const blockCache = new Map<string, BlockToken>();
+
+function cacheKey(index: number, blockContent: string, sources: SimpleSource[]) {
+	const sourceKey = sources.map((s) => s.link).join("|");
+	return `${index}-${hashString(blockContent)}|${sourceKey}`;
+}
+
 export async function processTokens(content: string, sources: SimpleSource[]): Promise<Token[]> {
 	// Apply incomplete markdown preprocessing for smooth streaming
 	const processedContent = parseIncompleteMarkdown(content);
@@ -213,7 +290,7 @@ export async function processTokens(content: string, sources: SimpleSource[]): P
 				return {
 					type: "code" as const,
 					lang: token.lang,
-					code: hljs.highlightAuto(token.text, hljs.getLanguage(token.lang)?.aliases).value,
+					code: highlightCode(token.text, token.lang),
 					rawCode: token.text,
 					isClosed: isFencedBlockClosed(token.raw ?? ""),
 				};
@@ -240,7 +317,7 @@ export function processTokensSync(content: string, sources: SimpleSource[]): Tok
 			return {
 				type: "code" as const,
 				lang: token.lang,
-				code: hljs.highlightAuto(token.text, hljs.getLanguage(token.lang)?.aliases).value,
+				code: highlightCode(token.text, token.lang),
 				rawCode: token.text,
 				isClosed: isFencedBlockClosed(token.raw ?? ""),
 			};
@@ -282,12 +359,18 @@ export async function processBlocks(
 
 	return await Promise.all(
 		blocks.map(async (blockContent, index) => {
+			const key = cacheKey(index, blockContent, sources);
+			const cached = blockCache.get(key);
+			if (cached) return cached;
+
 			const tokens = await processTokens(blockContent, sources);
-			return {
+			const block: BlockToken = {
 				id: `${index}-${hashString(blockContent)}`,
 				content: blockContent,
 				tokens,
 			};
+			blockCache.set(key, block);
+			return block;
 		})
 	);
 }
@@ -299,11 +382,17 @@ export function processBlocksSync(content: string, sources: SimpleSource[] = [])
 	const blocks = parseMarkdownIntoBlocks(content);
 
 	return blocks.map((blockContent, index) => {
+		const key = cacheKey(index, blockContent, sources);
+		const cached = blockCache.get(key);
+		if (cached) return cached;
+
 		const tokens = processTokensSync(blockContent, sources);
-		return {
+		const block: BlockToken = {
 			id: `${index}-${hashString(blockContent)}`,
 			content: blockContent,
 			tokens,
 		};
+		blockCache.set(key, block);
+		return block;
 	});
 }

src/lib/workers/markdownWorker.ts

@@ -3,17 +3,19 @@ type SimpleSource = {
 	title?: string;
 	link: string;
 };
-import { processTokens, type Token } from "$lib/utils/marked";
+import { processBlocks, type BlockToken } from "$lib/utils/marked";
 
 export type IncomingMessage = {
 	type: "process";
 	content: string;
 	sources: SimpleSource[];
+	requestId: number;
 };
 
 export type OutgoingMessage = {
 	type: "processed";
-	tokens: Token[];
+	blocks: BlockToken[];
+	requestId: number;
 };
 
 // Flag to track if the worker is currently processing a message
@@ -31,9 +33,11 @@ async function processMessage() {
 		isProcessing = true;
 
 		try {
-			const { content, sources } = nextMessage;
-			const processedTokens = await processTokens(content, sources);
-			postMessage(JSON.parse(JSON.stringify({ type: "processed", tokens: processedTokens })));
+			const { content, sources, requestId } = nextMessage;
+			const processedBlocks = await processBlocks(content, sources);
+			postMessage(
+				JSON.parse(JSON.stringify({ type: "processed", blocks: processedBlocks, requestId }))
+			);
 		} finally {
 			isProcessing = false;