victor HF Staff commited on
Commit
aa9a52b
·
1 Parent(s): dd2acb7

Enhance code token processing to include fenced block closure detection

Browse files
src/lib/components/chat/MarkdownRenderer.svelte CHANGED
@@ -86,6 +86,10 @@
86
  <!-- eslint-disable-next-line svelte/no-at-html-tags -->
87
  {@html token.html}
88
  {:else if token.type === "code"}
89
- <CodeBlock code={token.code} rawCode={token.rawCode} {loading} />
 
 
 
 
90
  {/if}
91
  {/each}
 
86
  <!-- eslint-disable-next-line svelte/no-at-html-tags -->
87
  {@html token.html}
88
  {:else if token.type === "code"}
89
+ <CodeBlock
90
+ code={token.code}
91
+ rawCode={token.rawCode}
92
+ loading={loading && !token.isClosed}
93
+ />
94
  {/if}
95
  {/each}
src/lib/utils/marked.ts CHANGED
@@ -172,11 +172,24 @@ function createMarkedInstance(sources: SimpleSource[]): Marked {
172
  breaks: true,
173
  });
174
  }
 
 
 
 
 
 
 
 
 
 
 
 
175
  type CodeToken = {
176
  type: "code";
177
  lang: string;
178
  code: string;
179
  rawCode: string;
 
180
  };
181
 
182
  type TextToken = {
@@ -190,13 +203,14 @@ export async function processTokens(content: string, sources: SimpleSource[]): P
190
 
191
  const processedTokens = await Promise.all(
192
  tokens.map(async (token) => {
193
- if (token.type === "code") {
194
- return {
195
- type: "code" as const,
196
- lang: token.lang,
197
- code: hljs.highlightAuto(token.text, hljs.getLanguage(token.lang)?.aliases).value,
198
- rawCode: token.text,
199
- };
 
200
  } else {
201
  return {
202
  type: "text" as const,
@@ -213,13 +227,14 @@ export function processTokensSync(content: string, sources: SimpleSource[]): Tok
213
  const marked = createMarkedInstance(sources);
214
  const tokens = marked.lexer(content);
215
  return tokens.map((token) => {
216
- if (token.type === "code") {
217
- return {
218
- type: "code" as const,
219
- lang: token.lang,
220
- code: hljs.highlightAuto(token.text, hljs.getLanguage(token.lang)?.aliases).value,
221
- rawCode: token.text,
222
- };
 
223
  }
224
  return { type: "text" as const, html: marked.parse(token.raw) };
225
  });
 
172
  breaks: true,
173
  });
174
  }
175
+ function isFencedBlockClosed(raw?: string): boolean {
176
+ if (!raw) return true;
177
+ const trimmed = raw.replace(/[\s\u0000]+$/, "");
178
+ const openingFenceMatch = trimmed.match(/^([`~]{3,})/);
179
+ if (!openingFenceMatch) {
180
+ return true;
181
+ }
182
+ const fence = openingFenceMatch[1];
183
+ const closingFencePattern = new RegExp(`(?:\n|\r\n)${fence}(?:[\t ]+)?$`);
184
+ return closingFencePattern.test(trimmed);
185
+ }
186
+
187
  type CodeToken = {
188
  type: "code";
189
  lang: string;
190
  code: string;
191
  rawCode: string;
192
+ isClosed: boolean;
193
  };
194
 
195
  type TextToken = {
 
203
 
204
  const processedTokens = await Promise.all(
205
  tokens.map(async (token) => {
206
+ if (token.type === "code") {
207
+ return {
208
+ type: "code" as const,
209
+ lang: token.lang,
210
+ code: hljs.highlightAuto(token.text, hljs.getLanguage(token.lang)?.aliases).value,
211
+ rawCode: token.text,
212
+ isClosed: isFencedBlockClosed(token.raw ?? ""),
213
+ };
214
  } else {
215
  return {
216
  type: "text" as const,
 
227
  const marked = createMarkedInstance(sources);
228
  const tokens = marked.lexer(content);
229
  return tokens.map((token) => {
230
+ if (token.type === "code") {
231
+ return {
232
+ type: "code" as const,
233
+ lang: token.lang,
234
+ code: hljs.highlightAuto(token.text, hljs.getLanguage(token.lang)?.aliases).value,
235
+ rawCode: token.text,
236
+ isClosed: isFencedBlockClosed(token.raw ?? ""),
237
+ };
238
  }
239
  return { type: "text" as const, html: marked.parse(token.raw) };
240
  });