feat: add shiki targeting

dist/index.d.ts

@@ -1,6 +1,7 @@
 import { Theme as Theme$1 } from 'rehype-pretty-code';
 import { QuartzTransformerPlugin } from '@quartz-community/types';
 export { QuartzTransformerPlugin } from '@quartz-community/types';
+import { ShikiTransformer } from 'shiki';
 
 interface Theme extends Record<string, Theme$1> {
     light: Theme$1;
@@ -10,7 +11,21 @@ interface SyntaxHighlightingOptions {
     theme?: Theme;
     keepBackground?: boolean;
     clipboard?: boolean;
+    tokenClassification?: boolean;
 }
 declare const SyntaxHighlighting: QuartzTransformerPlugin<Partial<SyntaxHighlightingOptions>>;
 
-export { SyntaxHighlighting, type SyntaxHighlightingOptions };
+/**
+ * Shiki transformer that adds `data-token-type` attributes to token spans
+ * based on TextMate scope classification.
+ *
+ * Requires `includeExplanation: "scopeName"` on the Shiki options to have
+ * access to scope data. When scopes are unavailable, tokens are left untouched.
+ *
+ * This is purely additive — inline styles are preserved as-is. Downstream
+ * consumers (e.g. Quartz Themes) can target `span[data-token-type="keyword"]`
+ * to override colors with theme-specific values.
+ */
+declare function tokenClassifierTransformer(): ShikiTransformer;
+
+export { SyntaxHighlighting, type SyntaxHighlightingOptions, tokenClassifierTransformer };

dist/index.js

@@ -28712,6 +28712,142 @@ function rehypePrettyCode(options = {}) {
   };
 }
 
+// src/token-classifier.ts
+var SCOPE_TO_TOKEN = [
+  // Invalid / error scopes
+  ["invalid.broken", "error"],
+  ["invalid.deprecated", "error"],
+  ["invalid.illegal", "error"],
+  ["invalid.unimplemented", "error"],
+  ["invalid", "error"],
+  ["message.error", "error"],
+  // Comments
+  ["comment", "comment"],
+  ["punctuation.definition.comment", "comment"],
+  ["string.comment", "comment"],
+  // Strings & regexp
+  ["string.regexp", "regexp"],
+  ["constant.other.reference.link", "regexp"],
+  ["string.other.link", "string"],
+  ["string", "string"],
+  ["punctuation.definition.string", "string"],
+  // Keywords & control flow
+  ["keyword.control", "keyword"],
+  ["keyword.operator.new", "keyword"],
+  ["keyword.operator.expression", "keyword"],
+  ["keyword.operator", "operator"],
+  ["keyword", "keyword"],
+  // Storage (function, class, const, let, var, async, static)
+  ["storage.modifier.import", "normal"],
+  ["storage.modifier.package", "normal"],
+  ["storage.modifier", "keyword"],
+  ["storage.type", "keyword"],
+  ["storage", "keyword"],
+  // Constants & literal values
+  ["constant.numeric", "value"],
+  ["constant.language", "value"],
+  ["constant.character.escape", "string"],
+  ["constant", "value"],
+  // Variables
+  ["variable.parameter", "important"],
+  ["variable.language", "value"],
+  ["variable.other.constant", "value"],
+  ["variable.other.enummember", "value"],
+  ["variable.other.readwrite", "normal"],
+  ["variable.other.property", "property"],
+  ["variable.other.object", "normal"],
+  ["variable.function", "function"],
+  ["variable", "normal"],
+  // Entity names
+  ["entity.name.function", "function"],
+  ["entity.name.constant", "value"],
+  ["entity.name.type.class", "property"],
+  ["entity.name.type.module", "property"],
+  ["entity.name.type", "property"],
+  ["entity.name.tag", "tag"],
+  ["entity.name.section", "tag"],
+  ["entity.name", "function"],
+  ["entity.other.inherited-class", "property"],
+  ["entity.other.attribute-name", "important"],
+  // Support (library/framework builtins)
+  ["support.function", "function"],
+  ["support.class", "property"],
+  ["support.type", "property"],
+  ["support.constant", "value"],
+  ["support.variable", "normal"],
+  ["support", "function"],
+  // Markup (markdown, diffs, etc.)
+  ["markup.inserted", "string"],
+  ["markup.deleted", "error"],
+  ["markup.changed", "important"],
+  ["markup.heading", "tag"],
+  ["markup.bold", "normal"],
+  ["markup.italic", "normal"],
+  ["markup.inline.raw", "value"],
+  ["markup.quote", "comment"],
+  ["markup.ignored", "comment"],
+  ["markup.untracked", "comment"],
+  ["markup", "normal"],
+  // Punctuation
+  ["punctuation.definition.template-expression", "keyword"],
+  ["punctuation.definition.inserted", "string"],
+  ["punctuation.definition.deleted", "error"],
+  ["punctuation.definition.changed", "important"],
+  ["punctuation.definition.list.begin.markdown", "important"],
+  ["punctuation.section", "punctuation"],
+  ["punctuation.separator", "punctuation"],
+  ["punctuation.accessor", "punctuation"],
+  ["punctuation.terminator", "punctuation"],
+  ["punctuation", "punctuation"],
+  // Bracket highlighter (used by some themes)
+  ["brackethighlighter.unmatched", "error"],
+  ["brackethighlighter", "punctuation"],
+  // Meta
+  ["meta.property-name", "property"],
+  ["meta.module-reference", "value"],
+  ["meta.diff.header", "tag"],
+  ["meta.diff.range", "function"],
+  ["meta.separator", "comment"],
+  ["meta.output", "value"],
+  ["meta.function-call", "function"],
+  ["meta.import", "keyword"],
+  ["meta.export", "keyword"],
+  // Source-level fallback for embedded code
+  ["source.regexp", "regexp"]
+];
+function classifyScopes(scopes) {
+  for (let i2 = scopes.length - 1; i2 >= 0; i2--) {
+    const scope = scopes[i2];
+    for (const [prefix, tokenType] of SCOPE_TO_TOKEN) {
+      if (scope.startsWith(prefix)) {
+        return tokenType;
+      }
+    }
+  }
+  return void 0;
+}
+function classifyToken(token2) {
+  if (!token2.explanation) return void 0;
+  for (const explanation of token2.explanation) {
+    const scopeNames = explanation.scopes.map((s3) => s3.scopeName);
+    const result = classifyScopes(scopeNames);
+    if (result) return result;
+  }
+  return void 0;
+}
+function tokenClassifierTransformer() {
+  return {
+    name: "token-classifier",
+    span(_hast, _line, _col, _lineElement, token2) {
+      const tokenType = classifyToken(token2);
+      if (tokenType) {
+        _hast.properties = _hast.properties || {};
+        _hast.properties["data-token-type"] = tokenType;
+      }
+    }
+  };
+}
+
 // src/scripts/clipboard.inline.ts
 var clipboard_inline_default = `var i='<svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true"><path fill-rule="evenodd" d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 010 1.5h-1.5a.25.25 0 00-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 00.25-.25v-1.5a.75.75 0 011.5 0v1.5A1.75 1.75 0 019.25 16h-7.5A1.75 1.75 0 010 14.25v-7.5z"></path><path fill-rule="evenodd" d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0114.25 11h-7.5A1.75 1.75 0 015 9.25v-7.5zm1.75-.25a.25.25 0 00-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 00.25-.25v-7.5a.25.25 0 00-.25-.25h-7.5z"></path></svg>',s='<svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true"><path fill-rule="evenodd" fill="rgb(63, 185, 80)" d="M13.78 4.22a.75.75 0 010 1.06l-7.25 7.25a.75.75 0 01-1.06 0L2.22 9.28a.75.75 0 011.06-1.06L6 10.94l6.72-6.72a.75.75 0 011.06 0z"></path></svg>',d=()=>{let o=document.getElementsByTagName("pre");for(let n=0;n<o.length;n++){let a=o[n];if(!a)continue;let t=a.getElementsByTagName("code")[0];if(t){let c=(t.dataset.clipboard?JSON.parse(t.dataset.clipboard):t.innerText).replace(/\\n\\n/g,\`
 \`),e=document.createElement("button");e.className="clipboard-button",e.type="button",e.innerHTML=i,e.ariaLabel="Copy source";let r=()=>{navigator.clipboard.writeText(c).then(()=>{e.blur(),e.innerHTML=s,setTimeout(()=>{e.innerHTML=i,e.style.borderColor=""},2e3)},l=>console.error(l))};e.addEventListener("click",r),window.addCleanup(()=>e.removeEventListener("click",r)),a.prepend(e)}}};document.addEventListener("nav",d);document.addEventListener("render",d);
@@ -28727,15 +28863,21 @@ var defaultOptions = {
     dark: "github-dark"
   },
   keepBackground: false,
-  clipboard: true
+  clipboard: true,
+  tokenClassification: true
 };
 var SyntaxHighlighting = (userOpts) => {
   const opts = { ...defaultOptions, ...userOpts };
-  const { clipboard, ...codeOpts } = opts;
+  const { clipboard, tokenClassification, ...codeOpts } = opts;
+  const rehypeOpts = { ...codeOpts };
+  if (tokenClassification) {
+    rehypeOpts.includeExplanation = "scopeName";
+    rehypeOpts.transformers = [...rehypeOpts.transformers ?? [], tokenClassifierTransformer()];
+  }
   return {
     name: "SyntaxHighlighting",
     htmlPlugins() {
-      return [[src_default, codeOpts]];
+      return [[src_default, rehypeOpts]];
     },
     externalResources() {
       const js = [];
@@ -28756,6 +28898,6 @@ var SyntaxHighlighting = (userOpts) => {
   };
 };
 
-export { SyntaxHighlighting };
+export { SyntaxHighlighting, tokenClassifierTransformer };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map

package.json

@@ -91,7 +91,8 @@
         "light": "github-light",
         "dark": "github-dark"
       },
-      "keepBackground": false
+      "keepBackground": false,
+      "tokenClassification": true
     }
   }
 }

src/index.ts

@@ -1,3 +1,4 @@
 export { SyntaxHighlighting } from "./transformer";
 export type { SyntaxHighlightingOptions } from "./transformer";
+export { tokenClassifierTransformer } from "./token-classifier";
 export type { QuartzTransformerPlugin } from "@quartz-community/types";

src/token-classifier.ts

@@ -0,0 +1,163 @@
+import type { ShikiTransformer, ThemedToken } from "shiki";
+
+const SCOPE_TO_TOKEN: [prefix: string, tokenType: string][] = [
+  // Invalid / error scopes
+  ["invalid.broken", "error"],
+  ["invalid.deprecated", "error"],
+  ["invalid.illegal", "error"],
+  ["invalid.unimplemented", "error"],
+  ["invalid", "error"],
+  ["message.error", "error"],
+
+  // Comments
+  ["comment", "comment"],
+  ["punctuation.definition.comment", "comment"],
+  ["string.comment", "comment"],
+
+  // Strings & regexp
+  ["string.regexp", "regexp"],
+  ["constant.other.reference.link", "regexp"],
+  ["string.other.link", "string"],
+  ["string", "string"],
+  ["punctuation.definition.string", "string"],
+
+  // Keywords & control flow
+  ["keyword.control", "keyword"],
+  ["keyword.operator.new", "keyword"],
+  ["keyword.operator.expression", "keyword"],
+  ["keyword.operator", "operator"],
+  ["keyword", "keyword"],
+
+  // Storage (function, class, const, let, var, async, static)
+  ["storage.modifier.import", "normal"],
+  ["storage.modifier.package", "normal"],
+  ["storage.modifier", "keyword"],
+  ["storage.type", "keyword"],
+  ["storage", "keyword"],
+
+  // Constants & literal values
+  ["constant.numeric", "value"],
+  ["constant.language", "value"],
+  ["constant.character.escape", "string"],
+  ["constant", "value"],
+
+  // Variables
+  ["variable.parameter", "important"],
+  ["variable.language", "value"],
+  ["variable.other.constant", "value"],
+  ["variable.other.enummember", "value"],
+  ["variable.other.readwrite", "normal"],
+  ["variable.other.property", "property"],
+  ["variable.other.object", "normal"],
+  ["variable.function", "function"],
+  ["variable", "normal"],
+
+  // Entity names
+  ["entity.name.function", "function"],
+  ["entity.name.constant", "value"],
+  ["entity.name.type.class", "property"],
+  ["entity.name.type.module", "property"],
+  ["entity.name.type", "property"],
+  ["entity.name.tag", "tag"],
+  ["entity.name.section", "tag"],
+  ["entity.name", "function"],
+  ["entity.other.inherited-class", "property"],
+  ["entity.other.attribute-name", "important"],
+
+  // Support (library/framework builtins)
+  ["support.function", "function"],
+  ["support.class", "property"],
+  ["support.type", "property"],
+  ["support.constant", "value"],
+  ["support.variable", "normal"],
+  ["support", "function"],
+
+  // Markup (markdown, diffs, etc.)
+  ["markup.inserted", "string"],
+  ["markup.deleted", "error"],
+  ["markup.changed", "important"],
+  ["markup.heading", "tag"],
+  ["markup.bold", "normal"],
+  ["markup.italic", "normal"],
+  ["markup.inline.raw", "value"],
+  ["markup.quote", "comment"],
+  ["markup.ignored", "comment"],
+  ["markup.untracked", "comment"],
+  ["markup", "normal"],
+
+  // Punctuation
+  ["punctuation.definition.template-expression", "keyword"],
+  ["punctuation.definition.inserted", "string"],
+  ["punctuation.definition.deleted", "error"],
+  ["punctuation.definition.changed", "important"],
+  ["punctuation.definition.list.begin.markdown", "important"],
+  ["punctuation.section", "punctuation"],
+  ["punctuation.separator", "punctuation"],
+  ["punctuation.accessor", "punctuation"],
+  ["punctuation.terminator", "punctuation"],
+  ["punctuation", "punctuation"],
+
+  // Bracket highlighter (used by some themes)
+  ["brackethighlighter.unmatched", "error"],
+  ["brackethighlighter", "punctuation"],
+
+  // Meta
+  ["meta.property-name", "property"],
+  ["meta.module-reference", "value"],
+  ["meta.diff.header", "tag"],
+  ["meta.diff.range", "function"],
+  ["meta.separator", "comment"],
+  ["meta.output", "value"],
+  ["meta.function-call", "function"],
+  ["meta.import", "keyword"],
+  ["meta.export", "keyword"],
+
+  // Source-level fallback for embedded code
+  ["source.regexp", "regexp"],
+];
+
+function classifyScopes(scopes: string[]): string | undefined {
+  for (let i = scopes.length - 1; i >= 0; i--) {
+    const scope = scopes[i]!;
+    for (const [prefix, tokenType] of SCOPE_TO_TOKEN) {
+      if (scope.startsWith(prefix)) {
+        return tokenType;
+      }
+    }
+  }
+  return undefined;
+}
+
+function classifyToken(token: ThemedToken): string | undefined {
+  if (!token.explanation) return undefined;
+  for (const explanation of token.explanation) {
+    const scopeNames = explanation.scopes.map((s) => s.scopeName);
+    const result = classifyScopes(scopeNames);
+    if (result) return result;
+  }
+  return undefined;
+}
+
+/**
+ * Shiki transformer that adds `data-token-type` attributes to token spans
+ * based on TextMate scope classification.
+ *
+ * Requires `includeExplanation: "scopeName"` on the Shiki options to have
+ * access to scope data. When scopes are unavailable, tokens are left untouched.
+ *
+ * This is purely additive — inline styles are preserved as-is. Downstream
+ * consumers (e.g. Quartz Themes) can target `span[data-token-type="keyword"]`
+ * to override colors with theme-specific values.
+ */
+export function tokenClassifierTransformer(): ShikiTransformer {
+  return {
+    name: "token-classifier",
+    span(_hast, _line, _col, _lineElement, token) {
+      const tokenType = classifyToken(token);
+      if (tokenType) {
+        _hast.properties = _hast.properties || {};
+        _hast.properties["data-token-type"] = tokenType;
+      }
+    },
+  };
+}

src/transformer.ts

@@ -1,6 +1,7 @@
 import rehypePrettyCode from "rehype-pretty-code";
 import type { Options as CodeOptions, Theme as CodeTheme } from "rehype-pretty-code";
 import type { QuartzTransformerPlugin, JSResource, CSSResource } from "@quartz-community/types";
+import { tokenClassifierTransformer } from "./token-classifier";
 // @ts-expect-error - inline script import handled by Quartz bundler
 import clipboardScript from "./scripts/clipboard.inline";
 import clipboardStyle from "./styles/clipboard.scss";
@@ -14,6 +15,7 @@ export interface SyntaxHighlightingOptions {
   theme?: Theme;
   keepBackground?: boolean;
   clipboard?: boolean;
+  tokenClassification?: boolean;
 }
 
 const defaultOptions: SyntaxHighlightingOptions = {
@@ -23,17 +25,26 @@ const defaultOptions: SyntaxHighlightingOptions = {
   },
   keepBackground: false,
   clipboard: true,
+  tokenClassification: true,
 };
 
 export const SyntaxHighlighting: QuartzTransformerPlugin<Partial<SyntaxHighlightingOptions>> = (
   userOpts,
 ) => {
   const opts = { ...defaultOptions, ...userOpts };
-  const { clipboard, ...codeOpts } = opts;
+  const { clipboard, tokenClassification, ...codeOpts } = opts;
+
+  const rehypeOpts: CodeOptions = { ...(codeOpts as CodeOptions) };
+
+  if (tokenClassification) {
+    (rehypeOpts as Record<string, unknown>).includeExplanation = "scopeName";
+    rehypeOpts.transformers = [...(rehypeOpts.transformers ?? []), tokenClassifierTransformer()];
+  }
+
   return {
     name: "SyntaxHighlighting",
     htmlPlugins() {
-      return [[rehypePrettyCode, codeOpts as CodeOptions]];
+      return [[rehypePrettyCode, rehypeOpts]];
     },
     externalResources() {
       const js: JSResource[] = [];

tsup.config.ts

@@ -43,6 +43,18 @@ const inlineScriptPlugin: Plugin = {
   },
 };
 
+const SINGLETON_EXTERNALS = [
+  "preact",
+  "preact/hooks",
+  "preact/jsx-runtime",
+  "preact/compat",
+  "@jackyzha0/quartz",
+  "@jackyzha0/quartz/*",
+  "vfile",
+  "vfile/*",
+  "unified",
+];
+
 export default defineConfig({
   entry: {
     index: "src/index.ts",
@@ -55,6 +67,8 @@ export default defineConfig({
   treeshake: true,
   target: "es2022",
   splitting: false,
+  noExternal: [/.*/],
+  external: SINGLETON_EXTERNALS,
   outDir: "dist",
   platform: "node",
   esbuildPlugins: [inlineScriptPlugin],