carbon-tokenization / backend /src /shared /shiki-config.ts
tfrere's picture
tfrere HF Staff
refactor(backend/publisher): transformer pipeline + shared registry + shiki highlighting
7843436
Raw
History Blame Contribute Delete
3.4 kB
/**
* Shared Shiki configuration.
*
* Single source of truth for syntax highlighting across the editor (live
* decorations) and the publisher (static SSR). Using the same config in both
* environments guarantees:
* - identical supported languages
* - identical themes / token colors
* - identical output structure (<span class="line"><span style=...>)
*
* Shiki is async-initialized (it loads an Oniguruma WASM engine + grammars).
* A singleton highlighter is created on first use and reused forever. The
* editor kicks off creation at mount; the publisher awaits it synchronously
* during render (publish is a one-shot pipeline).
*/
import type { BundledLanguage, BundledTheme, HighlighterGeneric } from "shiki";
import { createHighlighter } from "shiki";
/**
* Languages bundled into the highlighter. Chosen for research/tech articles.
* Keep this list in sync with what the editor's language selector advertises.
* Adding a language = one line here + redeploy; the cost is ~50KB gzipped per
* grammar loaded lazily.
*/
export const SHIKI_LANGS: BundledLanguage[] = [
"javascript",
"typescript",
"tsx",
"jsx",
"python",
"bash",
"shell",
"json",
"yaml",
"toml",
"css",
"html",
"xml",
"markdown",
"sql",
"rust",
"go",
"c",
"cpp",
"csharp",
"java",
"r",
"julia",
"ruby",
"php",
"swift",
"kotlin",
"scala",
"haskell",
"lua",
"dockerfile",
"diff",
"latex",
"ini",
"makefile",
"graphql",
];
/**
* Dual theme: Shiki emits each token with inline `color` (light) plus a
* `--shiki-dark` CSS variable (dark). A small CSS rule toggles which wins
* based on the `data-theme` attribute on <html>. See `_code-blocks.css`.
*
* Pick well-balanced, muted themes that read well in an academic context.
*/
export const SHIKI_THEMES = {
light: "github-light" as BundledTheme,
dark: "github-dark" as BundledTheme,
};
export type ShikiHighlighter = HighlighterGeneric<BundledLanguage, BundledTheme>;
let highlighterPromise: Promise<ShikiHighlighter> | null = null;
/**
* Returns the shared highlighter, creating it on first call. All subsequent
* callers wait on the same promise (deduped).
*/
export function getSharedHighlighter(): Promise<ShikiHighlighter> {
if (!highlighterPromise) {
highlighterPromise = createHighlighter({
themes: [SHIKI_THEMES.light, SHIKI_THEMES.dark],
langs: SHIKI_LANGS,
});
}
return highlighterPromise;
}
/**
* Checks whether a language string is supported. Falls back to plain-text
* rendering if not, which is what both editor and publisher should do.
*/
export function isSupportedLang(lang: string | null | undefined): lang is BundledLanguage {
if (!lang) return false;
return (SHIKI_LANGS as readonly string[]).includes(lang);
}
/**
* Normalize a language alias to the canonical name Shiki expects. Handles
* common aliases that users type (`js` → `javascript`, `py` → `python`, etc.).
*/
export function normalizeLang(lang: string | null | undefined): string {
if (!lang) return "";
const l = lang.toLowerCase().trim();
const aliases: Record<string, string> = {
js: "javascript",
ts: "typescript",
py: "python",
rb: "ruby",
sh: "bash",
zsh: "bash",
yml: "yaml",
md: "markdown",
"c++": "cpp",
"c#": "csharp",
docker: "dockerfile",
tex: "latex",
};
return aliases[l] ?? l;
}