Spaces:
Sleeping
Sleeping
Enhance code token processing to include fenced block closure detection
Browse files
src/lib/components/chat/MarkdownRenderer.svelte
CHANGED
|
@@ -86,6 +86,10 @@
|
|
| 86 |
<!-- eslint-disable-next-line svelte/no-at-html-tags -->
|
| 87 |
{@html token.html}
|
| 88 |
{:else if token.type === "code"}
|
| 89 |
-
<CodeBlock
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
{/if}
|
| 91 |
{/each}
|
|
|
|
| 86 |
<!-- eslint-disable-next-line svelte/no-at-html-tags -->
|
| 87 |
{@html token.html}
|
| 88 |
{:else if token.type === "code"}
|
| 89 |
+
<CodeBlock
|
| 90 |
+
code={token.code}
|
| 91 |
+
rawCode={token.rawCode}
|
| 92 |
+
loading={loading && !token.isClosed}
|
| 93 |
+
/>
|
| 94 |
{/if}
|
| 95 |
{/each}
|
src/lib/utils/marked.ts
CHANGED
|
@@ -172,11 +172,24 @@ function createMarkedInstance(sources: SimpleSource[]): Marked {
|
|
| 172 |
breaks: true,
|
| 173 |
});
|
| 174 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
type CodeToken = {
|
| 176 |
type: "code";
|
| 177 |
lang: string;
|
| 178 |
code: string;
|
| 179 |
rawCode: string;
|
|
|
|
| 180 |
};
|
| 181 |
|
| 182 |
type TextToken = {
|
|
@@ -190,13 +203,14 @@ export async function processTokens(content: string, sources: SimpleSource[]): P
|
|
| 190 |
|
| 191 |
const processedTokens = await Promise.all(
|
| 192 |
tokens.map(async (token) => {
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
|
|
|
| 200 |
} else {
|
| 201 |
return {
|
| 202 |
type: "text" as const,
|
|
@@ -213,13 +227,14 @@ export function processTokensSync(content: string, sources: SimpleSource[]): Tok
|
|
| 213 |
const marked = createMarkedInstance(sources);
|
| 214 |
const tokens = marked.lexer(content);
|
| 215 |
return tokens.map((token) => {
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
|
|
|
| 223 |
}
|
| 224 |
return { type: "text" as const, html: marked.parse(token.raw) };
|
| 225 |
});
|
|
|
|
| 172 |
breaks: true,
|
| 173 |
});
|
| 174 |
}
|
| 175 |
+
function isFencedBlockClosed(raw?: string): boolean {
|
| 176 |
+
if (!raw) return true;
|
| 177 |
+
const trimmed = raw.replace(/[\s\u0000]+$/, "");
|
| 178 |
+
const openingFenceMatch = trimmed.match(/^([`~]{3,})/);
|
| 179 |
+
if (!openingFenceMatch) {
|
| 180 |
+
return true;
|
| 181 |
+
}
|
| 182 |
+
const fence = openingFenceMatch[1];
|
| 183 |
+
const closingFencePattern = new RegExp(`(?:\n|\r\n)${fence}(?:[\t ]+)?$`);
|
| 184 |
+
return closingFencePattern.test(trimmed);
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
type CodeToken = {
|
| 188 |
type: "code";
|
| 189 |
lang: string;
|
| 190 |
code: string;
|
| 191 |
rawCode: string;
|
| 192 |
+
isClosed: boolean;
|
| 193 |
};
|
| 194 |
|
| 195 |
type TextToken = {
|
|
|
|
| 203 |
|
| 204 |
const processedTokens = await Promise.all(
|
| 205 |
tokens.map(async (token) => {
|
| 206 |
+
if (token.type === "code") {
|
| 207 |
+
return {
|
| 208 |
+
type: "code" as const,
|
| 209 |
+
lang: token.lang,
|
| 210 |
+
code: hljs.highlightAuto(token.text, hljs.getLanguage(token.lang)?.aliases).value,
|
| 211 |
+
rawCode: token.text,
|
| 212 |
+
isClosed: isFencedBlockClosed(token.raw ?? ""),
|
| 213 |
+
};
|
| 214 |
} else {
|
| 215 |
return {
|
| 216 |
type: "text" as const,
|
|
|
|
| 227 |
const marked = createMarkedInstance(sources);
|
| 228 |
const tokens = marked.lexer(content);
|
| 229 |
return tokens.map((token) => {
|
| 230 |
+
if (token.type === "code") {
|
| 231 |
+
return {
|
| 232 |
+
type: "code" as const,
|
| 233 |
+
lang: token.lang,
|
| 234 |
+
code: hljs.highlightAuto(token.text, hljs.getLanguage(token.lang)?.aliases).value,
|
| 235 |
+
rawCode: token.text,
|
| 236 |
+
isClosed: isFencedBlockClosed(token.raw ?? ""),
|
| 237 |
+
};
|
| 238 |
}
|
| 239 |
return { type: "text" as const, html: marked.parse(token.raw) };
|
| 240 |
});
|