feat: replace regex HTML-to-markdown with turndown

The hand-rolled regex converter silently dropped tables, nested lists, and other non-trivial HTML structures returned by `fetch_and_index`. Turndown (with the GFM plugin) handles these correctly and is already battle-tested. Turndown, turndown-plugin-gfm, and domino are declared as external dependencies in the esbuild bundle and installed on-demand by both `start.mjs` and `start.sh` so the bundle stays small and the server still starts without a prior `npm install`.
2026-10-02 04:14:38 +08:00 · 2026-03-01 13:19:27 -05:00
parent 8c36b4c647
commit 5894a68001
6 changed files with 246 additions and 87 deletions
@@ -1,18 +1,21 @@
 {
  "name": "context-mode",
-  "version": "0.9.6",
+  "version": "0.9.9",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "context-mode",
-      "version": "0.9.6",
+      "version": "0.9.9",
      "license": "MIT",
      "dependencies": {
        "@clack/prompts": "^1.0.1",
+        "@mixmark-io/domino": "^2.2.0",
        "@modelcontextprotocol/sdk": "^1.26.0",
        "better-sqlite3": "^12.6.2",
        "picocolors": "^1.1.1",
+        "turndown": "^7.2.0",
+        "turndown-plugin-gfm": "^1.0.2",
        "zod": "^3.25.0"
      },
      "bin": {
@@ -21,6 +24,7 @@
      "devDependencies": {
        "@types/better-sqlite3": "^7.6.13",
        "@types/node": "^22.19.11",
+        "@types/turndown": "^5.0.5",
        "esbuild": "^0.27.3",
        "tsx": "^4.21.0",
        "typescript": "^5.7.0"
@@ -499,6 +503,12 @@
        "hono": "^4"
      }
    },
+    "node_modules/@mixmark-io/domino": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz",
+      "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==",
+      "license": "BSD-2-Clause"
+    },
    "node_modules/@modelcontextprotocol/sdk": {
      "version": "1.26.0",
      "license": "MIT",
@@ -557,6 +567,13 @@
        "undici-types": "~6.21.0"
      }
    },
+    "node_modules/@types/turndown": {
+      "version": "5.0.6",
+      "resolved": "https://registry.npmjs.org/@types/turndown/-/turndown-5.0.6.tgz",
+      "integrity": "sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==",
+      "dev": true,
+      "license": "MIT"
+    },
    "node_modules/accepts": {
      "version": "2.0.0",
      "license": "MIT",
@@ -1930,6 +1947,21 @@
        "node": "*"
      }
    },
+    "node_modules/turndown": {
+      "version": "7.2.2",
+      "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.2.tgz",
+      "integrity": "sha512-1F7db8BiExOKxjSMU2b7if62D/XOyQyZbPKq/nUwopfgnHlqXHqQ0lvfUTeUIr1lZJzOPFn43dODyMSIfvWRKQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@mixmark-io/domino": "^2.2.0"
+      }
+    },
+    "node_modules/turndown-plugin-gfm": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.2.tgz",
+      "integrity": "sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg==",
+      "license": "MIT"
+    },
    "node_modules/type-is": {
      "version": "2.0.1",
      "license": "MIT",
@@ -38,7 +38,7 @@
  ],
  "scripts": {
    "build": "tsc",
-    "bundle": "esbuild src/server.ts --bundle --platform=node --target=node18 --format=esm --outfile=server.bundle.mjs --external:better-sqlite3 --minify",
+    "bundle": "esbuild src/server.ts --bundle --platform=node --target=node18 --format=esm --outfile=server.bundle.mjs --external:better-sqlite3 --external:turndown --external:turndown-plugin-gfm --external:@mixmark-io/domino --minify",
    "prepublishOnly": "npm run build",
    "dev": "npx tsx src/server.ts",
    "setup": "npx tsx src/cli.ts setup",
@@ -56,18 +56,23 @@
    "test:stream-cap": "npx tsx tests/stream-cap.test.ts",
    "test:search-wiring": "npx tsx tests/search-wiring.test.ts",
    "test:search-fallback": "npx tsx tests/search-fallback-integration.test.ts",
+    "test:turndown": "npx tsx tests/turndown.test.ts",
    "test:all": "for f in tests/*.test.ts; do npx tsx \"$f\" || exit 1; done"
  },
  "dependencies": {
    "@clack/prompts": "^1.0.1",
+    "@mixmark-io/domino": "^2.2.0",
    "@modelcontextprotocol/sdk": "^1.26.0",
    "better-sqlite3": "^12.6.2",
    "picocolors": "^1.1.1",
+    "turndown": "^7.2.0",
+    "turndown-plugin-gfm": "^1.0.2",
    "zod": "^3.25.0"
  },
  "devDependencies": {
    "@types/better-sqlite3": "^7.6.13",
    "@types/node": "^22.19.11",
+    "@types/turndown": "^5.0.5",
    "esbuild": "^0.27.3",
    "tsx": "^4.21.0",
    "typescript": "^5.7.0"
@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { createRequire } from "node:module";
 import { z } from "zod";
 import { PolyglotExecutor } from "./executor.js";
 import { ContentStore, cleanupStaleDBs, type SearchResult } from "./store.js";
@@ -752,77 +753,54 @@ server.registerTool(
  },
 );

+// ─────────────────────────────────────────────────────────
+// Turndown path resolution (external dep, like better-sqlite3)
+// ─────────────────────────────────────────────────────────
+
+let _turndownPath: string | null = null;
+let _gfmPluginPath: string | null = null;
+
+function resolveTurndownPath(): string {
+  if (!_turndownPath) {
+    const require = createRequire(import.meta.url);
+    _turndownPath = require.resolve("turndown");
+  }
+  return _turndownPath;
+}
+
+function resolveGfmPluginPath(): string {
+  if (!_gfmPluginPath) {
+    const require = createRequire(import.meta.url);
+    _gfmPluginPath = require.resolve("turndown-plugin-gfm");
+  }
+  return _gfmPluginPath;
+}
+
 // ─────────────────────────────────────────────────────────
 // Tool: fetch_and_index
 // ─────────────────────────────────────────────────────────

-const HTML_TO_MARKDOWN_CODE = `
-const url = process.argv[1];
-if (!url) { console.error("No URL provided"); process.exit(1); }
+function buildFetchCode(url: string): string {
+  const turndownPath = JSON.stringify(resolveTurndownPath());
+  const gfmPath = JSON.stringify(resolveGfmPluginPath());
+  return `
+const TurndownService = require(${turndownPath});
+const { gfm } = require(${gfmPath});
+const url = ${JSON.stringify(url)};

 async function main() {
  const resp = await fetch(url);
  if (!resp.ok) { console.error("HTTP " + resp.status); process.exit(1); }
+  const html = await resp.text();

-  let html = await resp.text();
-
-  // Strip script, style, nav, header, footer tags with content
-  html = html.replace(/<script[^>]*>[\\s\\S]*?<\\/script>/gi, "");
-  html = html.replace(/<style[^>]*>[\\s\\S]*?<\\/style>/gi, "");
-  html = html.replace(/<nav[^>]*>[\\s\\S]*?<\\/nav>/gi, "");
-  html = html.replace(/<header[^>]*>[\\s\\S]*?<\\/header>/gi, "");
-  html = html.replace(/<footer[^>]*>[\\s\\S]*?<\\/footer>/gi, "");
-
-  // Convert headings to markdown
-  html = html.replace(/<h1[^>]*>(.*?)<\\/h1>/gi, "\\n# $1\\n");
-  html = html.replace(/<h2[^>]*>(.*?)<\\/h2>/gi, "\\n## $1\\n");
-  html = html.replace(/<h3[^>]*>(.*?)<\\/h3>/gi, "\\n### $1\\n");
-  html = html.replace(/<h4[^>]*>(.*?)<\\/h4>/gi, "\\n#### $1\\n");
-
-  // Convert code blocks
-  html = html.replace(/<pre[^>]*><code[^>]*class="[^"]*language-(\\w+)"[^>]*>([\\s\\S]*?)<\\/code><\\/pre>/gi,
-    (_, lang, code) => "\\n\\\`\\\`\\\`" + lang + "\\n" + decodeEntities(code) + "\\n\\\`\\\`\\\`\\n");
-  html = html.replace(/<pre[^>]*><code[^>]*>([\\s\\S]*?)<\\/code><\\/pre>/gi,
-    (_, code) => "\\n\\\`\\\`\\\`\\n" + decodeEntities(code) + "\\n\\\`\\\`\\\`\\n");
-  html = html.replace(/<code[^>]*>([^<]*)<\\/code>/gi, "\\\`$1\\\`");
-
-  // Convert links
-  html = html.replace(/<a[^>]*href="([^"]*)"[^>]*>(.*?)<\\/a>/gi, "[$2]($1)");
-
-  // Convert lists
-  html = html.replace(/<li[^>]*>(.*?)<\\/li>/gi, "- $1\\n");
-
-  // Convert paragraphs and line breaks
-  html = html.replace(/<p[^>]*>(.*?)<\\/p>/gi, "\\n$1\\n");
-  html = html.replace(/<br\\s*\\/?>/gi, "\\n");
-  html = html.replace(/<hr\\s*\\/?>/gi, "\\n---\\n");
-
-  // Strip remaining HTML tags
-  html = html.replace(/<[^>]+>/g, "");
-
-  // Decode HTML entities
-  html = decodeEntities(html);
-
-  // Clean up whitespace
-  html = html.replace(/\\n{3,}/g, "\\n\\n").trim();
-
-  console.log(html);
+  const td = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+  td.use(gfm);
+  td.remove(['script', 'style', 'nav', 'header', 'footer', 'noscript']);
+  console.log(td.turndown(html));
 }
-
-function decodeEntities(s) {
-  return s
-    .replace(/&amp;/g, "&")
-    .replace(/&lt;/g, "<")
-    .replace(/&gt;/g, ">")
-    .replace(/&quot;/g, '"')
-    .replace(/&#39;/g, "'")
-    .replace(/&#x27;/g, "'")
-    .replace(/&#x2F;/g, "/")
-    .replace(/&nbsp;/g, " ");
-}
-
 main();
 `;
+}

 server.registerTool(
  "fetch_and_index",
@@ -845,7 +823,7 @@ server.registerTool(
  async ({ url, source }) => {
    try {
      // Execute fetch inside subprocess — raw HTML never enters context
-      const fetchCode = `process.argv[1] = ${JSON.stringify(url)};\n${HTML_TO_MARKDOWN_CODE}`;
+      const fetchCode = buildFetchCode(url);
      const result = await executor.execute({
        language: "javascript",
        code: fetchCode,
@@ -62,15 +62,17 @@ if (cacheMatch) {
  }
 }

-// Ensure native module is available
-if (!existsSync(resolve(__dirname, "node_modules", "better-sqlite3"))) {
-  try {
-    execSync("npm install better-sqlite3 --no-package-lock --no-save --silent", {
-      cwd: __dirname,
-      stdio: "pipe",
-      timeout: 60000,
-    });
-  } catch { /* best effort */ }
+// Ensure external dependencies are available
+for (const pkg of ["better-sqlite3", "turndown", "turndown-plugin-gfm", "@mixmark-io/domino"]) {
+  if (!existsSync(resolve(__dirname, "node_modules", pkg))) {
+    try {
+      execSync(`npm install ${pkg} --no-package-lock --no-save --silent`, {
+        cwd: __dirname,
+        stdio: "pipe",
+        timeout: 60000,
+      });
+    } catch { /* best effort */ }
+  }
 }

 // Bundle exists (CI-built) — start instantly
@@ -1,15 +0,0 @@
-#!/bin/sh
-CLAUDE_PROJECT_DIR="${CLAUDE_PROJECT_DIR:-$(pwd)}"
-DIR="$(cd "$(dirname "$0")" && pwd)"
-cd "$DIR"
-
-# Bundle exists (CI-built) — start instantly, install native module in background
-if [ -f server.bundle.mjs ]; then
-  [ -d node_modules/better-sqlite3 ] || npm install better-sqlite3 --no-package-lock --no-save --silent 2>/dev/null &
-  CLAUDE_PROJECT_DIR="$CLAUDE_PROJECT_DIR" exec node server.bundle.mjs
-fi
-
-# Fallback: no bundle (dev or npm install) — full build
-[ -d node_modules ] || npm install --silent 2>/dev/null
-[ -f build/server.js ] || npx tsc --silent 2>/dev/null
-CLAUDE_PROJECT_DIR="$CLAUDE_PROJECT_DIR" exec node build/server.js
@@ -0,0 +1,157 @@
+import { strict as assert } from "node:assert";
+import { PolyglotExecutor } from "../src/executor.js";
+import { createRequire } from "node:module";
+
+let passed = 0;
+let failed = 0;
+const results: { name: string; status: "PASS" | "FAIL"; time: number; error?: string }[] = [];
+
+async function test(name: string, fn: () => Promise<void>) {
+  const start = performance.now();
+  try {
+    await fn();
+    const time = performance.now() - start;
+    passed++;
+    results.push({ name, status: "PASS", time });
+    console.log(`  ✓ ${name} (${time.toFixed(0)}ms)`);
+  } catch (err: unknown) {
+    const time = performance.now() - start;
+    failed++;
+    const message = err instanceof Error ? err.message : String(err);
+    results.push({ name, status: "FAIL", time, error: message });
+    console.log(`  ✗ ${name} (${time.toFixed(0)}ms)`);
+    console.log(`    ${message}`);
+  }
+}
+
+// Resolve turndown path the same way server.ts will
+const require = createRequire(import.meta.url);
+const turndownPath = require.resolve("turndown");
+const gfmPath = require.resolve("turndown-plugin-gfm");
+
+const executor = new PolyglotExecutor();
+
+function buildConversionCode(html: string): string {
+  return `
+const TurndownService = require(${JSON.stringify(turndownPath)});
+const { gfm } = require(${JSON.stringify(gfmPath)});
+const td = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+td.use(gfm);
+td.remove(['script', 'style', 'nav', 'header', 'footer', 'noscript']);
+console.log(td.turndown(${JSON.stringify(html)}));
+`;
+}
+
+console.log("\nturndown HTML-to-markdown conversion tests\n");
+
+await test("converts headings", async () => {
+  const result = await executor.execute({
+    language: "javascript",
+    code: buildConversionCode("<h1>Title</h1><h2>Subtitle</h2><h3>Section</h3>"),
+  });
+  assert.equal(result.exitCode, 0, `stderr: ${result.stderr}`);
+  assert(result.stdout.includes("# Title"), `expected '# Title', got: ${result.stdout}`);
+  assert(result.stdout.includes("## Subtitle"));
+  assert(result.stdout.includes("### Section"));
+});
+
+await test("converts links", async () => {
+  const result = await executor.execute({
+    language: "javascript",
+    code: buildConversionCode('<p>Visit <a href="https://example.com">Example</a></p>'),
+  });
+  assert.equal(result.exitCode, 0, `stderr: ${result.stderr}`);
+  assert(result.stdout.includes("[Example](https://example.com)"));
+});
+
+await test("converts fenced code blocks", async () => {
+  const result = await executor.execute({
+    language: "javascript",
+    code: buildConversionCode('<pre><code class="language-js">const x = 1;</code></pre>'),
+  });
+  assert.equal(result.exitCode, 0, `stderr: ${result.stderr}`);
+  assert(result.stdout.includes("```"), `expected fenced code block, got: ${result.stdout}`);
+  assert(result.stdout.includes("const x = 1;"));
+});
+
+await test("strips script tags", async () => {
+  const result = await executor.execute({
+    language: "javascript",
+    code: buildConversionCode("<p>Hello</p><script>alert('xss')</script><p>World</p>"),
+  });
+  assert.equal(result.exitCode, 0, `stderr: ${result.stderr}`);
+  assert(!result.stdout.includes("alert"), `script content leaked: ${result.stdout}`);
+  assert(result.stdout.includes("Hello"));
+  assert(result.stdout.includes("World"));
+});
+
+await test("strips style, nav, header, footer, noscript tags", async () => {
+  const html = [
+    "<style>body { color: red; }</style>",
+    "<header><nav>Menu</nav></header>",
+    "<main><p>Content</p></main>",
+    "<footer>Footer</footer>",
+    "<noscript>Enable JS</noscript>",
+  ].join("");
+  const result = await executor.execute({
+    language: "javascript",
+    code: buildConversionCode(html),
+  });
+  assert.equal(result.exitCode, 0, `stderr: ${result.stderr}`);
+  assert(result.stdout.includes("Content"), `lost main content: ${result.stdout}`);
+  assert(!result.stdout.includes("Menu"), `nav leaked: ${result.stdout}`);
+  assert(!result.stdout.includes("Footer"), `footer leaked: ${result.stdout}`);
+  assert(!result.stdout.includes("Enable JS"), `noscript leaked: ${result.stdout}`);
+  assert(!result.stdout.includes("color: red"), `style leaked: ${result.stdout}`);
+});
+
+await test("converts tables", async () => {
+  const html = `
+    <table>
+      <thead><tr><th>Name</th><th>Age</th></tr></thead>
+      <tbody><tr><td>Alice</td><td>30</td></tr></tbody>
+    </table>`;
+  const result = await executor.execute({
+    language: "javascript",
+    code: buildConversionCode(html),
+  });
+  assert.equal(result.exitCode, 0, `stderr: ${result.stderr}`);
+  assert(result.stdout.includes("| Name"), `expected pipe table, got: ${result.stdout}`);
+  assert(result.stdout.includes("| Alice"));
+  assert(result.stdout.includes("| ---"), `expected table separator, got: ${result.stdout}`);
+});
+
+await test("handles nested tags correctly", async () => {
+  const html = '<div><p>Outer <strong>bold <em>and italic</em></strong> text</p></div>';
+  const result = await executor.execute({
+    language: "javascript",
+    code: buildConversionCode(html),
+  });
+  assert.equal(result.exitCode, 0, `stderr: ${result.stderr}`);
+  assert(result.stdout.includes("**bold"), `missing bold: ${result.stdout}`);
+  assert(result.stdout.includes("italic"), `missing italic: ${result.stdout}`);
+});
+
+await test("handles malformed HTML gracefully", async () => {
+  const html = "<p>Unclosed paragraph<p>Another<div>Nested badly</p></div>";
+  const result = await executor.execute({
+    language: "javascript",
+    code: buildConversionCode(html),
+  });
+  assert.equal(result.exitCode, 0, `stderr: ${result.stderr}`);
+  assert(result.stdout.includes("Unclosed paragraph"), `lost content: ${result.stdout}`);
+  assert(result.stdout.includes("Nested badly"), `lost nested content: ${result.stdout}`);
+});
+
+await test("decodes HTML entities", async () => {
+  const result = await executor.execute({
+    language: "javascript",
+    code: buildConversionCode("<p>Tom &amp; Jerry &lt;3 &quot;cheese&quot;</p>"),
+  });
+  assert.equal(result.exitCode, 0, `stderr: ${result.stderr}`);
+  assert(result.stdout.includes('Tom & Jerry <3 "cheese"'), `entities not decoded: ${result.stdout}`);
+});
+
+// Summary
+console.log(`\n  ${passed} passed, ${failed} failed\n`);
+if (failed > 0) process.exit(1);