841 lines
28 KiB
JavaScript
841 lines
28 KiB
JavaScript
import { ShikiError } from "@shikijs/types";
|
|
import { EncodedTokenMetadata, INITIAL, Registry as Registry$1, Theme } from "@shikijs/vscode-textmate";
|
|
|
|
export * from "@shikijs/types"
|
|
|
|
//#region src/utils/colors.ts
|
|
function resolveColorReplacements(theme, options) {
|
|
const replacements = typeof theme === "string" ? {} : { ...theme.colorReplacements };
|
|
const themeName = typeof theme === "string" ? theme : theme.name;
|
|
for (const [key, value] of Object.entries(options?.colorReplacements || {})) if (typeof value === "string") replacements[key] = value;
|
|
else if (key === themeName) Object.assign(replacements, value);
|
|
return replacements;
|
|
}
|
|
function applyColorReplacements(color, replacements) {
|
|
if (!color) return color;
|
|
return replacements?.[color?.toLowerCase()] || color;
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/utils/general.ts
|
|
function toArray(x) {
|
|
return Array.isArray(x) ? x : [x];
|
|
}
|
|
/**
|
|
* Normalize a getter to a promise.
|
|
*/
|
|
async function normalizeGetter(p) {
|
|
return Promise.resolve(typeof p === "function" ? p() : p).then((r) => r.default || r);
|
|
}
|
|
/**
|
|
* Check if the language is plaintext that is ignored by Shiki.
|
|
*
|
|
* Hard-coded plain text languages: `plaintext`, `txt`, `text`, `plain`.
|
|
*/
|
|
function isPlainLang(lang) {
|
|
return !lang || [
|
|
"plaintext",
|
|
"txt",
|
|
"text",
|
|
"plain"
|
|
].includes(lang);
|
|
}
|
|
/**
|
|
* Check if the language is specially handled or bypassed by Shiki.
|
|
*
|
|
* Hard-coded languages: `ansi` and plaintexts like `plaintext`, `txt`, `text`, `plain`.
|
|
*/
|
|
function isSpecialLang(lang) {
|
|
return lang === "ansi" || isPlainLang(lang);
|
|
}
|
|
/**
|
|
* Check if the theme is specially handled or bypassed by Shiki.
|
|
*
|
|
* Hard-coded themes: `none`.
|
|
*/
|
|
function isNoneTheme(theme) {
|
|
return theme === "none";
|
|
}
|
|
/**
|
|
* Check if the theme is specially handled or bypassed by Shiki.
|
|
*
|
|
* Hard-coded themes: `none`.
|
|
*/
|
|
function isSpecialTheme(theme) {
|
|
return isNoneTheme(theme);
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/utils/strings.ts
|
|
/**
|
|
* Split a string into lines, each line preserves the line ending.
|
|
*
|
|
* @param code - The code string to split into lines
|
|
* @param preserveEnding - Whether to preserve line endings in the result
|
|
* @returns Array of tuples containing [line content, offset index]
|
|
*
|
|
* @example
|
|
* ```ts
|
|
* splitLines('hello\nworld', false)
|
|
* // => [['hello', 0], ['world', 6]]
|
|
*
|
|
* splitLines('hello\nworld', true)
|
|
* // => [['hello\n', 0], ['world', 6]]
|
|
* ```
|
|
*/
|
|
function splitLines(code, preserveEnding = false) {
|
|
if (code.length === 0) return [["", 0]];
|
|
const parts = code.split(/(\r?\n)/g);
|
|
let index = 0;
|
|
const lines = [];
|
|
for (let i = 0; i < parts.length; i += 2) {
|
|
const line = preserveEnding ? parts[i] + (parts[i + 1] || "") : parts[i];
|
|
lines.push([line, index]);
|
|
index += parts[i].length;
|
|
index += parts[i + 1]?.length || 0;
|
|
}
|
|
return lines;
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/textmate/normalize-theme.ts
|
|
/**
|
|
* https://github.com/microsoft/vscode/blob/f7f05dee53fb33fe023db2e06e30a89d3094488f/src/vs/platform/theme/common/colorRegistry.ts#L258-L268
|
|
*/
|
|
const VSCODE_FALLBACK_EDITOR_FG = {
|
|
light: "#333333",
|
|
dark: "#bbbbbb"
|
|
};
|
|
const VSCODE_FALLBACK_EDITOR_BG = {
|
|
light: "#fffffe",
|
|
dark: "#1e1e1e"
|
|
};
|
|
const RESOLVED_KEY = "__shiki_resolved";
|
|
/**
|
|
* Normalize a textmate theme to shiki theme
|
|
*/
|
|
function normalizeTheme(rawTheme) {
|
|
if (rawTheme?.[RESOLVED_KEY]) return rawTheme;
|
|
const theme = { ...rawTheme };
|
|
if (theme.tokenColors && !theme.settings) {
|
|
theme.settings = theme.tokenColors;
|
|
delete theme.tokenColors;
|
|
}
|
|
theme.type ||= "dark";
|
|
theme.colorReplacements = { ...theme.colorReplacements };
|
|
theme.settings ||= [];
|
|
let { bg, fg } = theme;
|
|
if (!bg || !fg) {
|
|
/**
|
|
* First try:
|
|
* Theme might contain a global `tokenColor` without `name` or `scope`
|
|
* Used as default value for foreground/background
|
|
*/
|
|
const globalSetting = theme.settings ? theme.settings.find((s) => !s.name && !s.scope) : void 0;
|
|
if (globalSetting?.settings?.foreground) fg = globalSetting.settings.foreground;
|
|
if (globalSetting?.settings?.background) bg = globalSetting.settings.background;
|
|
/**
|
|
* Second try:
|
|
* If there's no global `tokenColor` without `name` or `scope`
|
|
* Use `editor.foreground` and `editor.background`
|
|
*/
|
|
if (!fg && theme?.colors?.["editor.foreground"]) fg = theme.colors["editor.foreground"];
|
|
if (!bg && theme?.colors?.["editor.background"]) bg = theme.colors["editor.background"];
|
|
/**
|
|
* Last try:
|
|
* If there's no fg/bg color specified in theme, use default
|
|
*/
|
|
if (!fg) fg = theme.type === "light" ? VSCODE_FALLBACK_EDITOR_FG.light : VSCODE_FALLBACK_EDITOR_FG.dark;
|
|
if (!bg) bg = theme.type === "light" ? VSCODE_FALLBACK_EDITOR_BG.light : VSCODE_FALLBACK_EDITOR_BG.dark;
|
|
theme.fg = fg;
|
|
theme.bg = bg;
|
|
}
|
|
if (!(theme.settings[0] && theme.settings[0].settings && !theme.settings[0].scope)) theme.settings.unshift({ settings: {
|
|
foreground: theme.fg,
|
|
background: theme.bg
|
|
} });
|
|
let replacementCount = 0;
|
|
const replacementMap = /* @__PURE__ */ new Map();
|
|
function getReplacementColor(value) {
|
|
if (replacementMap.has(value)) return replacementMap.get(value);
|
|
replacementCount += 1;
|
|
const hex = `#${replacementCount.toString(16).padStart(8, "0").toLowerCase()}`;
|
|
if (theme.colorReplacements?.[`#${hex}`]) return getReplacementColor(value);
|
|
replacementMap.set(value, hex);
|
|
return hex;
|
|
}
|
|
theme.settings = theme.settings.map((setting) => {
|
|
const replaceFg = setting.settings?.foreground && !setting.settings.foreground.startsWith("#");
|
|
const replaceBg = setting.settings?.background && !setting.settings.background.startsWith("#");
|
|
if (!replaceFg && !replaceBg) return setting;
|
|
const clone = {
|
|
...setting,
|
|
settings: { ...setting.settings }
|
|
};
|
|
if (replaceFg) {
|
|
const replacement = getReplacementColor(setting.settings.foreground);
|
|
theme.colorReplacements[replacement] = setting.settings.foreground;
|
|
clone.settings.foreground = replacement;
|
|
}
|
|
if (replaceBg) {
|
|
const replacement = getReplacementColor(setting.settings.background);
|
|
theme.colorReplacements[replacement] = setting.settings.background;
|
|
clone.settings.background = replacement;
|
|
}
|
|
return clone;
|
|
});
|
|
for (const key of Object.keys(theme.colors || {})) if (key === "editor.foreground" || key === "editor.background" || key.startsWith("terminal.ansi")) {
|
|
if (!theme.colors[key]?.startsWith("#")) {
|
|
const replacement = getReplacementColor(theme.colors[key]);
|
|
theme.colorReplacements[replacement] = theme.colors[key];
|
|
theme.colors[key] = replacement;
|
|
}
|
|
}
|
|
Object.defineProperty(theme, RESOLVED_KEY, {
|
|
enumerable: false,
|
|
writable: false,
|
|
value: true
|
|
});
|
|
return theme;
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/textmate/getters-resolve.ts
|
|
/**
|
|
* Resolve
|
|
*/
|
|
async function resolveLangs(langs) {
|
|
return Array.from(new Set((await Promise.all(langs.filter((l) => !isSpecialLang(l)).map(async (lang) => await normalizeGetter(lang).then((r) => Array.isArray(r) ? r : [r])))).flat()));
|
|
}
|
|
async function resolveThemes(themes) {
|
|
return (await Promise.all(themes.map(async (theme) => isSpecialTheme(theme) ? null : normalizeTheme(await normalizeGetter(theme))))).filter((i) => !!i);
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/utils/alias.ts
|
|
function resolveLangAlias(name, alias) {
|
|
if (!alias) return name;
|
|
if (alias[name]) {
|
|
const resolved = new Set([name]);
|
|
while (alias[name]) {
|
|
name = alias[name];
|
|
if (resolved.has(name)) throw new ShikiError(`Circular alias \`${Array.from(resolved).join(" -> ")} -> ${name}\``);
|
|
resolved.add(name);
|
|
}
|
|
}
|
|
return name;
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/textmate/registry.ts
|
|
var Registry = class extends Registry$1 {
|
|
_resolvedThemes = /* @__PURE__ */ new Map();
|
|
_resolvedGrammars = /* @__PURE__ */ new Map();
|
|
_langMap = /* @__PURE__ */ new Map();
|
|
_langGraph = /* @__PURE__ */ new Map();
|
|
_textmateThemeCache = /* @__PURE__ */ new WeakMap();
|
|
_loadedThemesCache = null;
|
|
_loadedLanguagesCache = null;
|
|
constructor(_resolver, _themes, _langs, _alias = {}) {
|
|
super(_resolver);
|
|
this._resolver = _resolver;
|
|
this._themes = _themes;
|
|
this._langs = _langs;
|
|
this._alias = _alias;
|
|
this._themes.map((t) => this.loadTheme(t));
|
|
this.loadLanguages(this._langs);
|
|
}
|
|
getTheme(theme) {
|
|
if (typeof theme === "string") return this._resolvedThemes.get(theme);
|
|
else return this.loadTheme(theme);
|
|
}
|
|
loadTheme(theme) {
|
|
const _theme = normalizeTheme(theme);
|
|
if (_theme.name) {
|
|
this._resolvedThemes.set(_theme.name, _theme);
|
|
this._loadedThemesCache = null;
|
|
}
|
|
return _theme;
|
|
}
|
|
getLoadedThemes() {
|
|
if (!this._loadedThemesCache) this._loadedThemesCache = [...this._resolvedThemes.keys()];
|
|
return this._loadedThemesCache;
|
|
}
|
|
setTheme(theme) {
|
|
let textmateTheme = this._textmateThemeCache.get(theme);
|
|
if (!textmateTheme) {
|
|
textmateTheme = Theme.createFromRawTheme(theme);
|
|
this._textmateThemeCache.set(theme, textmateTheme);
|
|
}
|
|
this._syncRegistry.setTheme(textmateTheme);
|
|
}
|
|
getGrammar(name) {
|
|
name = resolveLangAlias(name, this._alias);
|
|
return this._resolvedGrammars.get(name);
|
|
}
|
|
loadLanguage(lang) {
|
|
if (this.getGrammar(lang.name)) return;
|
|
const embeddedLazilyBy = new Set([...this._langMap.values()].filter((i) => i.embeddedLangsLazy?.includes(lang.name)));
|
|
this._resolver.addLanguage(lang);
|
|
const grammarConfig = {
|
|
balancedBracketSelectors: lang.balancedBracketSelectors || ["*"],
|
|
unbalancedBracketSelectors: lang.unbalancedBracketSelectors || []
|
|
};
|
|
this._syncRegistry._rawGrammars.set(lang.scopeName, lang);
|
|
const g = this.loadGrammarWithConfiguration(lang.scopeName, 1, grammarConfig);
|
|
g.name = lang.name;
|
|
this._resolvedGrammars.set(lang.name, g);
|
|
if (lang.aliases) lang.aliases.forEach((alias) => {
|
|
this._alias[alias] = lang.name;
|
|
});
|
|
this._loadedLanguagesCache = null;
|
|
if (embeddedLazilyBy.size) for (const e of embeddedLazilyBy) {
|
|
this._resolvedGrammars.delete(e.name);
|
|
this._loadedLanguagesCache = null;
|
|
this._syncRegistry?._injectionGrammars?.delete(e.scopeName);
|
|
this._syncRegistry?._grammars?.delete(e.scopeName);
|
|
this.loadLanguage(this._langMap.get(e.name));
|
|
}
|
|
}
|
|
dispose() {
|
|
super.dispose();
|
|
this._resolvedThemes.clear();
|
|
this._resolvedGrammars.clear();
|
|
this._langMap.clear();
|
|
this._langGraph.clear();
|
|
this._loadedThemesCache = null;
|
|
}
|
|
loadLanguages(langs) {
|
|
for (const lang of langs) this.resolveEmbeddedLanguages(lang);
|
|
const langsGraphArray = Array.from(this._langGraph.entries());
|
|
const missingLangs = langsGraphArray.filter(([_, lang]) => !lang);
|
|
if (missingLangs.length) {
|
|
const dependents = langsGraphArray.filter(([_, lang]) => {
|
|
if (!lang) return false;
|
|
return (lang.embeddedLanguages || lang.embeddedLangs)?.some((l) => missingLangs.map(([name]) => name).includes(l));
|
|
}).filter((lang) => !missingLangs.includes(lang));
|
|
throw new ShikiError(`Missing languages ${missingLangs.map(([name]) => `\`${name}\``).join(", ")}, required by ${dependents.map(([name]) => `\`${name}\``).join(", ")}`);
|
|
}
|
|
for (const [_, lang] of langsGraphArray) this._resolver.addLanguage(lang);
|
|
for (const [_, lang] of langsGraphArray) this.loadLanguage(lang);
|
|
}
|
|
getLoadedLanguages() {
|
|
if (!this._loadedLanguagesCache) this._loadedLanguagesCache = [...new Set([...this._resolvedGrammars.keys(), ...Object.keys(this._alias)])];
|
|
return this._loadedLanguagesCache;
|
|
}
|
|
resolveEmbeddedLanguages(lang) {
|
|
this._langMap.set(lang.name, lang);
|
|
this._langGraph.set(lang.name, lang);
|
|
const embedded = lang.embeddedLanguages ?? lang.embeddedLangs;
|
|
if (embedded) for (const embeddedLang of embedded) this._langGraph.set(embeddedLang, this._langMap.get(embeddedLang));
|
|
}
|
|
};
|
|
|
|
//#endregion
|
|
//#region src/textmate/resolver.ts
|
|
var Resolver = class {
|
|
_langs = /* @__PURE__ */ new Map();
|
|
_scopeToLang = /* @__PURE__ */ new Map();
|
|
_injections = /* @__PURE__ */ new Map();
|
|
_onigLib;
|
|
constructor(engine, langs) {
|
|
this._onigLib = {
|
|
createOnigScanner: (patterns) => engine.createScanner(patterns),
|
|
createOnigString: (s) => engine.createString(s)
|
|
};
|
|
langs.forEach((i) => this.addLanguage(i));
|
|
}
|
|
get onigLib() {
|
|
return this._onigLib;
|
|
}
|
|
getLangRegistration(langIdOrAlias) {
|
|
return this._langs.get(langIdOrAlias);
|
|
}
|
|
loadGrammar(scopeName) {
|
|
return this._scopeToLang.get(scopeName);
|
|
}
|
|
addLanguage(l) {
|
|
this._langs.set(l.name, l);
|
|
if (l.aliases) l.aliases.forEach((a) => {
|
|
this._langs.set(a, l);
|
|
});
|
|
this._scopeToLang.set(l.scopeName, l);
|
|
if (l.injectTo) l.injectTo.forEach((i) => {
|
|
if (!this._injections.get(i)) this._injections.set(i, []);
|
|
this._injections.get(i).push(l.scopeName);
|
|
});
|
|
}
|
|
getInjections(scopeName) {
|
|
const scopeParts = scopeName.split(".");
|
|
let injections = [];
|
|
for (let i = 1; i <= scopeParts.length; i++) {
|
|
const subScopeName = scopeParts.slice(0, i).join(".");
|
|
injections = [...injections, ...this._injections.get(subScopeName) || []];
|
|
}
|
|
return injections;
|
|
}
|
|
};
|
|
|
|
//#endregion
|
|
//#region src/constructors/primitive.ts
|
|
let instancesCount = 0;
|
|
/**
|
|
* Get the minimal shiki primitive instance.
|
|
*
|
|
* Requires to provide the engine and all themes and languages upfront.
|
|
*/
|
|
function createShikiPrimitive(options) {
|
|
instancesCount += 1;
|
|
if (options.warnings !== false && instancesCount >= 10 && instancesCount % 10 === 0) console.warn(`[Shiki] ${instancesCount} instances have been created. Shiki is supposed to be used as a singleton, consider refactoring your code to cache your highlighter instance; Or call \`highlighter.dispose()\` to release unused instances.`);
|
|
let isDisposed = false;
|
|
if (!options.engine) throw new ShikiError("`engine` option is required for synchronous mode");
|
|
const langs = (options.langs || []).flat(1);
|
|
const themes = (options.themes || []).flat(1).map(normalizeTheme);
|
|
const _registry = new Registry(new Resolver(options.engine, langs), themes, langs, options.langAlias);
|
|
let _lastTheme;
|
|
function resolveLangAlias$1(name) {
|
|
return resolveLangAlias(name, options.langAlias);
|
|
}
|
|
function getLanguage(name) {
|
|
ensureNotDisposed();
|
|
const _lang = _registry.getGrammar(typeof name === "string" ? name : name.name);
|
|
if (!_lang) throw new ShikiError(`Language \`${name}\` not found, you may need to load it first`);
|
|
return _lang;
|
|
}
|
|
function getTheme(name) {
|
|
if (name === "none") return {
|
|
bg: "",
|
|
fg: "",
|
|
name: "none",
|
|
settings: [],
|
|
type: "dark"
|
|
};
|
|
ensureNotDisposed();
|
|
const _theme = _registry.getTheme(name);
|
|
if (!_theme) throw new ShikiError(`Theme \`${name}\` not found, you may need to load it first`);
|
|
return _theme;
|
|
}
|
|
function setTheme(name) {
|
|
ensureNotDisposed();
|
|
const theme = getTheme(name);
|
|
if (_lastTheme !== name) {
|
|
_registry.setTheme(theme);
|
|
_lastTheme = name;
|
|
}
|
|
return {
|
|
theme,
|
|
colorMap: _registry.getColorMap()
|
|
};
|
|
}
|
|
function getLoadedThemes() {
|
|
ensureNotDisposed();
|
|
return _registry.getLoadedThemes();
|
|
}
|
|
function getLoadedLanguages() {
|
|
ensureNotDisposed();
|
|
return _registry.getLoadedLanguages();
|
|
}
|
|
function loadLanguageSync(...langs) {
|
|
ensureNotDisposed();
|
|
_registry.loadLanguages(langs.flat(1));
|
|
}
|
|
async function loadLanguage(...langs) {
|
|
return loadLanguageSync(await resolveLangs(langs));
|
|
}
|
|
function loadThemeSync(...themes) {
|
|
ensureNotDisposed();
|
|
for (const theme of themes.flat(1)) _registry.loadTheme(theme);
|
|
}
|
|
async function loadTheme(...themes) {
|
|
ensureNotDisposed();
|
|
return loadThemeSync(await resolveThemes(themes));
|
|
}
|
|
function ensureNotDisposed() {
|
|
if (isDisposed) throw new ShikiError("Shiki instance has been disposed");
|
|
}
|
|
function dispose() {
|
|
if (isDisposed) return;
|
|
isDisposed = true;
|
|
_registry.dispose();
|
|
instancesCount -= 1;
|
|
}
|
|
return {
|
|
setTheme,
|
|
getTheme,
|
|
getLanguage,
|
|
getLoadedThemes,
|
|
getLoadedLanguages,
|
|
resolveLangAlias: resolveLangAlias$1,
|
|
loadLanguage,
|
|
loadLanguageSync,
|
|
loadTheme,
|
|
loadThemeSync,
|
|
dispose,
|
|
[Symbol.dispose]: dispose
|
|
};
|
|
}
|
|
/**
|
|
* @deprecated Use `createShikiPrimitive` instead.
|
|
*/
|
|
const createShikiInternalSync = createShikiPrimitive;
|
|
|
|
//#endregion
|
|
//#region src/constructors/async.ts
|
|
/**
|
|
* Get the minimal shiki primitive instance.
|
|
*/
|
|
async function createShikiPrimitiveAsync(options) {
|
|
if (!options.engine) console.warn("`engine` option is required. Use `createOnigurumaEngine` or `createJavaScriptRegexEngine` to create an engine.");
|
|
const [themes, langs, engine] = await Promise.all([
|
|
resolveThemes(options.themes || []),
|
|
resolveLangs(options.langs || []),
|
|
options.engine
|
|
]);
|
|
return createShikiPrimitive({
|
|
...options,
|
|
themes,
|
|
langs,
|
|
engine
|
|
});
|
|
}
|
|
/**
|
|
* @deprecated Use `createShikiPrimitiveAsync` instead.
|
|
*/
|
|
const createShikiInternal = createShikiPrimitiveAsync;
|
|
|
|
//#endregion
|
|
//#region src/textmate/grammar-state.ts
|
|
const _grammarStateMap = /* @__PURE__ */ new WeakMap();
|
|
function setLastGrammarStateToMap(keys, state) {
|
|
_grammarStateMap.set(keys, state);
|
|
}
|
|
function getLastGrammarStateFromMap(keys) {
|
|
return _grammarStateMap.get(keys);
|
|
}
|
|
/**
|
|
* GrammarState is a special reference object that holds the state of a grammar.
|
|
*
|
|
* It's used to highlight code snippets that are part of the target language.
|
|
*/
|
|
var GrammarState = class GrammarState {
|
|
/**
|
|
* Theme to Stack mapping
|
|
*/
|
|
_stacks = {};
|
|
lang;
|
|
get themes() {
|
|
return Object.keys(this._stacks);
|
|
}
|
|
get theme() {
|
|
return this.themes[0];
|
|
}
|
|
get _stack() {
|
|
return this._stacks[this.theme];
|
|
}
|
|
/**
|
|
* Static method to create a initial grammar state.
|
|
*/
|
|
static initial(lang, themes) {
|
|
return new GrammarState(Object.fromEntries(toArray(themes).map((theme) => [theme, INITIAL])), lang);
|
|
}
|
|
constructor(...args) {
|
|
if (args.length === 2) {
|
|
const [stacksMap, lang] = args;
|
|
this.lang = lang;
|
|
this._stacks = stacksMap;
|
|
} else {
|
|
const [stack, lang, theme] = args;
|
|
this.lang = lang;
|
|
this._stacks = { [theme]: stack };
|
|
}
|
|
}
|
|
/**
|
|
* Get the internal stack object.
|
|
* @internal
|
|
*/
|
|
getInternalStack(theme = this.theme) {
|
|
return this._stacks[theme];
|
|
}
|
|
getScopes(theme = this.theme) {
|
|
return getScopes(this._stacks[theme]);
|
|
}
|
|
toJSON() {
|
|
return {
|
|
lang: this.lang,
|
|
theme: this.theme,
|
|
themes: this.themes,
|
|
scopes: this.getScopes()
|
|
};
|
|
}
|
|
};
|
|
function getScopes(stack) {
|
|
const scopes = [];
|
|
const visited = /* @__PURE__ */ new Set();
|
|
function pushScope(stack) {
|
|
if (visited.has(stack)) return;
|
|
visited.add(stack);
|
|
const name = stack?.nameScopesList?.scopeName;
|
|
if (name) scopes.push(name);
|
|
if (stack.parent) pushScope(stack.parent);
|
|
}
|
|
pushScope(stack);
|
|
return scopes;
|
|
}
|
|
function getGrammarStack(state, theme) {
|
|
if (!(state instanceof GrammarState)) throw new ShikiError("Invalid grammar state");
|
|
return state.getInternalStack(theme);
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/highlight/code-to-tokens-base.ts
|
|
/**
|
|
* Code to tokens, with a simple theme.
|
|
*/
|
|
function codeToTokensBase(primitive, code, options = {}) {
|
|
const { theme: themeName = primitive.getLoadedThemes()[0] } = options;
|
|
if (isPlainLang(primitive.resolveLangAlias(options.lang || "text")) || isNoneTheme(themeName)) return splitLines(code).map((line) => [{
|
|
content: line[0],
|
|
offset: line[1]
|
|
}]);
|
|
const { theme, colorMap } = primitive.setTheme(themeName);
|
|
const _grammar = primitive.getLanguage(options.lang || "text");
|
|
if (options.grammarState) {
|
|
if (options.grammarState.lang !== _grammar.name) throw new ShikiError(`Grammar state language "${options.grammarState.lang}" does not match highlight language "${_grammar.name}"`);
|
|
if (!options.grammarState.themes.includes(theme.name)) throw new ShikiError(`Grammar state themes "${options.grammarState.themes}" do not contain highlight theme "${theme.name}"`);
|
|
}
|
|
return tokenizeWithTheme(code, _grammar, theme, colorMap, options);
|
|
}
|
|
function getLastGrammarState(...args) {
|
|
if (args.length === 2) return getLastGrammarStateFromMap(args[1]);
|
|
const [primitive, code, options = {}] = args;
|
|
const { lang = "text", theme: themeName = primitive.getLoadedThemes()[0] } = options;
|
|
if (isPlainLang(lang) || isNoneTheme(themeName)) throw new ShikiError("Plain language does not have grammar state");
|
|
if (lang === "ansi") throw new ShikiError("ANSI language does not have grammar state");
|
|
const { theme, colorMap } = primitive.setTheme(themeName);
|
|
const _grammar = primitive.getLanguage(lang);
|
|
return new GrammarState(_tokenizeWithTheme(code, _grammar, theme, colorMap, options).stateStack, _grammar.name, theme.name);
|
|
}
|
|
function tokenizeWithTheme(code, grammar, theme, colorMap, options) {
|
|
const result = _tokenizeWithTheme(code, grammar, theme, colorMap, options);
|
|
const grammarState = new GrammarState(result.stateStack, grammar.name, theme.name);
|
|
setLastGrammarStateToMap(result.tokens, grammarState);
|
|
return result.tokens;
|
|
}
|
|
function _tokenizeWithTheme(code, grammar, theme, colorMap, options) {
|
|
const colorReplacements = resolveColorReplacements(theme, options);
|
|
const { tokenizeMaxLineLength = 0, tokenizeTimeLimit = 500 } = options;
|
|
const lines = splitLines(code);
|
|
let stateStack = options.grammarState ? getGrammarStack(options.grammarState, theme.name) ?? INITIAL : options.grammarContextCode != null ? _tokenizeWithTheme(options.grammarContextCode, grammar, theme, colorMap, {
|
|
...options,
|
|
grammarState: void 0,
|
|
grammarContextCode: void 0
|
|
}).stateStack : INITIAL;
|
|
let actual = [];
|
|
const final = [];
|
|
for (let i = 0, len = lines.length; i < len; i++) {
|
|
const [line, lineOffset] = lines[i];
|
|
if (line === "") {
|
|
actual = [];
|
|
final.push([]);
|
|
continue;
|
|
}
|
|
if (tokenizeMaxLineLength > 0 && line.length >= tokenizeMaxLineLength) {
|
|
actual = [];
|
|
final.push([{
|
|
content: line,
|
|
offset: lineOffset,
|
|
color: "",
|
|
fontStyle: 0
|
|
}]);
|
|
continue;
|
|
}
|
|
let resultWithScopes;
|
|
let tokensWithScopes;
|
|
let tokensWithScopesIndex;
|
|
if (options.includeExplanation) {
|
|
resultWithScopes = grammar.tokenizeLine(line, stateStack, tokenizeTimeLimit);
|
|
tokensWithScopes = resultWithScopes.tokens;
|
|
tokensWithScopesIndex = 0;
|
|
}
|
|
const result = grammar.tokenizeLine2(line, stateStack, tokenizeTimeLimit);
|
|
const tokensLength = result.tokens.length / 2;
|
|
for (let j = 0; j < tokensLength; j++) {
|
|
const startIndex = result.tokens[2 * j];
|
|
const nextStartIndex = j + 1 < tokensLength ? result.tokens[2 * j + 2] : line.length;
|
|
if (startIndex === nextStartIndex) continue;
|
|
const metadata = result.tokens[2 * j + 1];
|
|
const color = applyColorReplacements(colorMap[EncodedTokenMetadata.getForeground(metadata)], colorReplacements);
|
|
const fontStyle = EncodedTokenMetadata.getFontStyle(metadata);
|
|
const token = {
|
|
content: line.substring(startIndex, nextStartIndex),
|
|
offset: lineOffset + startIndex,
|
|
color,
|
|
fontStyle
|
|
};
|
|
if (options.includeExplanation) {
|
|
const themeSettingsSelectors = [];
|
|
if (options.includeExplanation !== "scopeName") for (const setting of theme.settings) {
|
|
let selectors;
|
|
switch (typeof setting.scope) {
|
|
case "string":
|
|
selectors = setting.scope.split(/,/).map((scope) => scope.trim());
|
|
break;
|
|
case "object":
|
|
selectors = setting.scope;
|
|
break;
|
|
default: continue;
|
|
}
|
|
themeSettingsSelectors.push({
|
|
settings: setting,
|
|
selectors: selectors.map((selector) => selector.split(/ /))
|
|
});
|
|
}
|
|
token.explanation = [];
|
|
let offset = 0;
|
|
while (startIndex + offset < nextStartIndex) {
|
|
const tokenWithScopes = tokensWithScopes[tokensWithScopesIndex];
|
|
const tokenWithScopesText = line.substring(tokenWithScopes.startIndex, tokenWithScopes.endIndex);
|
|
offset += tokenWithScopesText.length;
|
|
token.explanation.push({
|
|
content: tokenWithScopesText,
|
|
scopes: options.includeExplanation === "scopeName" ? explainThemeScopesNameOnly(tokenWithScopes.scopes) : explainThemeScopesFull(themeSettingsSelectors, tokenWithScopes.scopes)
|
|
});
|
|
tokensWithScopesIndex += 1;
|
|
}
|
|
}
|
|
actual.push(token);
|
|
}
|
|
final.push(actual);
|
|
actual = [];
|
|
stateStack = result.ruleStack;
|
|
}
|
|
return {
|
|
tokens: final,
|
|
stateStack
|
|
};
|
|
}
|
|
function explainThemeScopesNameOnly(scopes) {
|
|
return scopes.map((scope) => ({ scopeName: scope }));
|
|
}
|
|
function explainThemeScopesFull(themeSelectors, scopes) {
|
|
const result = [];
|
|
for (let i = 0, len = scopes.length; i < len; i++) {
|
|
const scope = scopes[i];
|
|
result[i] = {
|
|
scopeName: scope,
|
|
themeMatches: explainThemeScope(themeSelectors, scope, scopes.slice(0, i))
|
|
};
|
|
}
|
|
return result;
|
|
}
|
|
function matchesOne(selector, scope) {
|
|
return selector === scope || scope.substring(0, selector.length) === selector && scope[selector.length] === ".";
|
|
}
|
|
function matches(selectors, scope, parentScopes) {
|
|
if (!matchesOne(selectors[selectors.length - 1], scope)) return false;
|
|
let selectorParentIndex = selectors.length - 2;
|
|
let parentIndex = parentScopes.length - 1;
|
|
while (selectorParentIndex >= 0 && parentIndex >= 0) {
|
|
if (matchesOne(selectors[selectorParentIndex], parentScopes[parentIndex])) selectorParentIndex -= 1;
|
|
parentIndex -= 1;
|
|
}
|
|
if (selectorParentIndex === -1) return true;
|
|
return false;
|
|
}
|
|
function explainThemeScope(themeSettingsSelectors, scope, parentScopes) {
|
|
const result = [];
|
|
for (const { selectors, settings } of themeSettingsSelectors) for (const selectorPieces of selectors) if (matches(selectorPieces, scope, parentScopes)) {
|
|
result.push(settings);
|
|
break;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
//#endregion
|
|
//#region src/highlight/code-to-tokens-themes.ts
|
|
/**
|
|
* Get tokens with multiple themes
|
|
*/
|
|
function codeToTokensWithThemes(primitive, code, options, codeToTokensBaseFn = codeToTokensBase) {
|
|
const themes = Object.entries(options.themes).filter((i) => i[1]).map((i) => ({
|
|
color: i[0],
|
|
theme: i[1]
|
|
}));
|
|
const themedTokens = themes.map((t) => {
|
|
const tokens = codeToTokensBaseFn(primitive, code, {
|
|
...options,
|
|
theme: t.theme
|
|
});
|
|
return {
|
|
tokens,
|
|
state: getLastGrammarStateFromMap(tokens),
|
|
theme: typeof t.theme === "string" ? t.theme : t.theme.name
|
|
};
|
|
});
|
|
const tokens = alignThemesTokenization(...themedTokens.map((i) => i.tokens));
|
|
const mergedTokens = tokens[0].map((line, lineIdx) => line.map((_token, tokenIdx) => {
|
|
const mergedToken = {
|
|
content: _token.content,
|
|
variants: {},
|
|
offset: _token.offset
|
|
};
|
|
if ("includeExplanation" in options && options.includeExplanation) mergedToken.explanation = _token.explanation;
|
|
tokens.forEach((t, themeIdx) => {
|
|
const { content: _, explanation: __, offset: ___, ...styles } = t[lineIdx][tokenIdx];
|
|
mergedToken.variants[themes[themeIdx].color] = styles;
|
|
});
|
|
return mergedToken;
|
|
}));
|
|
const mergedGrammarState = themedTokens[0].state ? new GrammarState(Object.fromEntries(themedTokens.map((s) => [s.theme, s.state?.getInternalStack(s.theme)])), themedTokens[0].state.lang) : void 0;
|
|
if (mergedGrammarState) setLastGrammarStateToMap(mergedTokens, mergedGrammarState);
|
|
return mergedTokens;
|
|
}
|
|
/**
|
|
* Break tokens from multiple themes into same tokenization.
|
|
*
|
|
* For example, given two themes that tokenize `console.log("hello")` as:
|
|
*
|
|
* - `console . log (" hello ")` (6 tokens)
|
|
* - `console .log ( "hello" )` (5 tokens)
|
|
*
|
|
* This function will return:
|
|
*
|
|
* - `console . log ( " hello " )` (8 tokens)
|
|
* - `console . log ( " hello " )` (8 tokens)
|
|
*/
|
|
function alignThemesTokenization(...themes) {
|
|
const outThemes = themes.map(() => []);
|
|
const count = themes.length;
|
|
for (let i = 0; i < themes[0].length; i++) {
|
|
const lines = themes.map((t) => t[i]);
|
|
const outLines = outThemes.map(() => []);
|
|
outThemes.forEach((t, i) => t.push(outLines[i]));
|
|
const indexes = lines.map(() => 0);
|
|
const current = lines.map((l) => l[0]);
|
|
while (current.every((t) => t)) {
|
|
const minLength = Math.min(...current.map((t) => t.content.length));
|
|
for (let n = 0; n < count; n++) {
|
|
const token = current[n];
|
|
if (token.content.length === minLength) {
|
|
outLines[n].push(token);
|
|
indexes[n] += 1;
|
|
current[n] = lines[n][indexes[n]];
|
|
} else {
|
|
outLines[n].push({
|
|
...token,
|
|
content: token.content.slice(0, minLength)
|
|
});
|
|
current[n] = {
|
|
...token,
|
|
content: token.content.slice(minLength),
|
|
offset: token.offset + minLength
|
|
};
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return outThemes;
|
|
}
|
|
|
|
//#endregion
|
|
export { GrammarState, Registry, Resolver, alignThemesTokenization, applyColorReplacements, codeToTokensBase, codeToTokensWithThemes, createShikiInternal, createShikiInternalSync, createShikiPrimitive, createShikiPrimitiveAsync, getGrammarStack, getLastGrammarState, getLastGrammarStateFromMap, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, normalizeGetter, normalizeTheme, resolveColorReplacements, resolveLangAlias, resolveLangs, resolveThemes, setLastGrammarStateToMap, splitLines, toArray, tokenizeWithTheme }; |