carbon-tokenization / backend /tests /highlight-autodetect.test.ts
tfrere's picture
tfrere HF Staff
fix(highlight): auto-detect language for code blocks without explicit lang
bf2abd0
Raw
History Blame Contribute Delete
2 kB
/**
* Auto-detection of syntax highlighting for language-less code blocks.
*
* Code blocks authored without an explicit language must still be highlighted
* (in both the editor and the published output). These tests cover the
* publisher path end-to-end plus the shared detector in isolation.
*/
import { describe, it, expect } from "vitest";
import { renderArticleHTML, type PublishMeta } from "../src/publisher/html-renderer.js";
import type { PublishCSS } from "../src/publisher/index.js";
import { detectShikiLang } from "../src/shared/detect-lang.js";
const EMPTY_CSS: PublishCSS = {
variables: "",
reset: "",
base: "",
layout: "",
print: "",
editorTokens: "",
article: "",
components: "",
publisher: "",
};
const META: PublishMeta = {
title: "Code Article",
description: "code highlighting",
authors: [{ name: "Alice", affiliationIndices: [1], affiliationNames: ["MIT"] }],
affiliations: [{ name: "MIT" }],
date: "2025-01-01",
};
const PY = `import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
def load(name):
return AutoModelForCausalLM.from_pretrained(name)`;
describe("detectShikiLang", () => {
it("detects python from a typical snippet", () => {
expect(detectShikiLang(PY)).toBe("python");
});
it("returns empty for trivially short input", () => {
expect(detectShikiLang("x")).toBe("");
});
});
describe("publisher highlights language-less code blocks", () => {
it("colors a code block that has no language attribute", async () => {
const json = {
type: "doc",
content: [
// Note: no `attrs.language` - mirrors how these blocks are authored.
{ type: "codeBlock", content: [{ type: "text", text: PY }] },
],
};
const html = await renderArticleHTML(json as any, META, EMPTY_CSS);
// Detected language label and per-token color vars must be present.
expect(html).toContain('data-lang="python"');
expect(html).toContain("--shiki-light:");
});
});