Add syntax highlighting with Shikiji

.changeset/unlucky-jokes-battle.md

@@ -0,0 +1,5 @@
+---
+"neogrok": minor
+---
+
+Add syntax highlighting with shikiji

.eslintignore

@@ -2,5 +2,4 @@
 node_modules
 /build
 /.svelte-kit
-package-lock.json
 svelte.config.js

.gitattributes

@@ -0,0 +1 @@
+.yarn/releases/* -diff

.github/workflows/ci.yml

@@ -12,10 +12,10 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: 20.x
-      - run: npm install
-      - run: npm run lint
-      - run: npm run test
-      - run: npm run build
+      - run: yarn install
+      - run: yarn run lint
+      - run: yarn run test
+      - run: yarn run build
   deploy-demo:
     # Only run on master commits; this is a further restriction of the top-level `on`.
     if: github.event_name == 'push'
@@ -47,7 +47,7 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: 20.x
-      - run: npm install
+      - run: yarn install
       - uses: changesets/action@v1
         with:
           commit: Publish release

.gitignore

@@ -2,8 +2,8 @@
 *.log
 node_modules/
 *.tsbuildinfo
-.npm
 .svelte-kit
 build/
 .env
 .envrc
+.yarn/install-state.gz

.ignore

@@ -0,0 +1 @@
+.yarn/releases

.nvmrc

@@ -0,0 +1 @@
+v20.10.0

.prettierignore

@@ -2,4 +2,4 @@
 node_modules
 /build
 /.svelte-kit
-package-lock.json
+/.yarn

.yarn/patches/shikiji-core-npm-0.9.18-826293a3df.patch

@@ -0,0 +1,49 @@
+diff --git a/dist/chunk-types.d.mts b/dist/chunk-types.d.mts
+index 2992357a1d0e670afce9ec11133a58a4e17593da..788cab9adb4134796ab6cd91fa9fd25b3a971dc5 100644
+--- a/dist/chunk-types.d.mts
++++ b/dist/chunk-types.d.mts
+@@ -1124,10 +1124,8 @@ interface ThemedTokenExplanation {
+ interface ThemedToken extends TokenStyles, TokenBase {
+ }
+ interface TokenBase {
+-    /**
+-     * The content of the token
+-     */
+-    content: string;
++    start: number;
++    end: number;
+     /**
+      * Explanation of
+      *
+diff --git a/dist/index.mjs b/dist/index.mjs
+index 6f891f5e37470cb673f8bcb8914c44634ca47091..1b52e8c87fa19d739e66f824715a18f37ae05fc6 100644
+--- a/dist/index.mjs
++++ b/dist/index.mjs
+@@ -3709,7 +3709,7 @@ function dimColor(color) {
+ function codeToThemedTokens(internal, code, options = {}) {
+     const { lang = 'text', theme: themeName = internal.getLoadedThemes()[0], } = options;
+     if (isPlaintext(lang)) {
+-        const lines = code.split(/\r\n|\r|\n/);
++        const lines = code;
+         return [...lines.map(line => [{ content: line }])];
+     }
+     const { theme, colorMap } = internal.setTheme(themeName);
+@@ -3723,7 +3723,7 @@ function tokenizeWithTheme(code, grammar, theme, colorMap, options) {
+         ...theme.colorReplacements,
+         ...options?.colorReplacements,
+     };
+-    const lines = code.split(/\r\n|\r|\n/);
++    const lines = code;
+     let ruleStack = INITIAL;
+     let actual = [];
+     const final = [];
+@@ -3754,7 +3754,8 @@ function tokenizeWithTheme(code, grammar, theme, colorMap, options) {
+             const foregroundColor = applyColorReplacements(colorMap[foreground], colorReplacements);
+             const fontStyle = StackElementMetadata.getFontStyle(metadata);
+             const token = {
+-                content: line.substring(startIndex, nextStartIndex),
++                start: startIndex,
++                end: nextStartIndex,
+                 color: foregroundColor,
+                 fontStyle,
+             };

.yarn/patches/shikiji-npm-0.9.18-945ea5efcb.patch

@@ -0,0 +1,13 @@
+diff --git a/dist/bundle-full.d.mts b/dist/bundle-full.d.mts
+index b8e655052e4a1b6e10e617eb06008a0782ff3a70..db1121f519c64bcada055f10f07500c27afca436 100644
+--- a/dist/bundle-full.d.mts
++++ b/dist/bundle-full.d.mts
+@@ -12,7 +12,7 @@ type Highlighter = HighlighterGeneric<BundledLanguage, BundledTheme>;
+ declare const getHighlighter: shikiji_core.GetHighlighterFactory<BundledLanguage, BundledTheme>;
+ declare const codeToHtml: (code: string, options: shikiji_core.CodeToHastOptions<BundledLanguage, BundledTheme>) => Promise<string>;
+ declare const codeToHast: (code: string, options: shikiji_core.CodeToHastOptions<BundledLanguage, BundledTheme>) => Promise<Root>;
+-declare const codeToThemedTokens: (code: string, options: shikiji_core.RequireKeys<shikiji_core.CodeToThemedTokensOptions<BundledLanguage, BundledTheme>, "lang" | "theme">) => Promise<shikiji_core.ThemedToken[][]>;
++declare const codeToThemedTokens: (code: string[], options: shikiji_core.RequireKeys<shikiji_core.CodeToThemedTokensOptions<BundledLanguage, BundledTheme>, "lang" | "theme">) => Promise<shikiji_core.ThemedToken[][]>;
+ declare const codeToTokensWithThemes: (code: string, options: shikiji_core.RequireKeys<shikiji_core.CodeToTokensWithThemesOptions<BundledLanguage, BundledTheme>, "lang" | "themes">) => Promise<shikiji_core.ThemedTokenWithVariants[][]>;
+ declare const getSingletonHighlighter: () => Promise<HighlighterGeneric<BundledLanguage, BundledTheme>>;
+

.yarnrc.yml

@@ -0,0 +1,4 @@
+yarnPath: .yarn/releases/yarn-4.0.2.cjs
+nodeLinker: node-modules
+defaultSemverRangePrefix: ""
+enableTelemetry: false

README.md

@@ -39,9 +39,9 @@ Neogrok is packaged for installation on
 install an executable.
 
 Alternatively, building from source is easy. Clone the repository,
-`npm install && npm run build && npm run start`. You can of course run the server
-without intermediation by `npm`, by doing whatever `npm run start` does directly;
-but the relevant commands may change in the future, whereas `npm run start` will
+`yarn install && yarn run build && yarn run start`. You can of course run the server
+without intermediation by `yarn`, by doing whatever `yarn run start` does directly;
+but the relevant commands may change in the future, whereas `yarn run start` will
 not.
 
 ## Deploying

demo/Dockerfile.neogrok

@@ -1,9 +1,9 @@
 FROM node:20-slim as app-builder
 WORKDIR /app
 COPY . .
-RUN npm install \
-    && npm run build \
-    && npm install --omit=dev
+RUN yarn install \
+    && yarn run build \
+    && yarn workspaces focus --production
 
 FROM gcr.io/distroless/nodejs20-debian12
 WORKDIR /app

demo/Dockerfile.neogrok.dockerignore

@@ -1,7 +1,7 @@
 *
-!.npmrc
+!.yarn*
+!yarn.lock
 !.postcssrc
-!package-lock.json
 !package.json
 !src/
 !static/