ci: add OS matrix, tool-parity guard, and coverage job

Expand CI to ubuntu + windows + macos. Add a tool-parity check that
verifies all four surfaces (mcp-ts index.ts, vscode-ext mcp.ts,
vscode-ext package.json, mcpb manifest.json) declare the same tools.
Add a coverage job that runs vitest with @vitest/coverage-v8 and
enforces 80/65/90/80 thresholds for statements/branches/functions/lines.
This commit is contained in:
Erik van der Boom 2026-04-19 22:33:16 +02:00
parent 60eefee806
commit a88b47bf19
3 changed files with 149 additions and 3 deletions

View file

@ -10,7 +10,11 @@ permissions:
jobs:
test:
name: Test & template-sync check
runs-on: ubuntu-latest
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
@ -39,7 +43,7 @@ jobs:
if: always()
uses: actions/upload-artifact@v7
with:
name: mcp-test-results
name: mcp-test-results-${{ matrix.os }}
path: mcp-ts/test-results.json
# ── Template sync ─────────────────────────────────────────────────────────
@ -65,5 +69,43 @@ jobs:
if: always()
uses: actions/upload-artifact@v7
with:
name: vscode-ext-test-results
name: vscode-ext-test-results-${{ matrix.os }}
path: vscode-ext/test-results.json
# ── Job: Coverage gates ──────────────────────────────────────────────────────
# Runs coverage-with-thresholds on ubuntu only (one platform is enough — coverage
# numbers don't depend on OS, and per-OS coverage runs would triple CI time).
coverage:
name: Coverage (v8)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/setup-node@v6
with:
node-version: '22'
cache: 'npm'
cache-dependency-path: |
mcp-ts/package-lock.json
vscode-ext/package-lock.json
- name: Install mcp-ts
run: cd mcp-ts && npm ci
- name: mcp-ts coverage
run: cd mcp-ts && npm run test:coverage
- name: Sync templates to vscode-ext
run: cp mcp-ts/src/templates.ts vscode-ext/src/ai-setup-templates.ts
- name: Install vscode-ext
run: cd vscode-ext && npm ci
- name: vscode-ext coverage
run: cd vscode-ext && npm run test:coverage
- name: Upload coverage reports
if: always()
uses: actions/upload-artifact@v7
with:
name: coverage
path: |
mcp-ts/coverage/
vscode-ext/coverage/

1
.gitignore vendored
View file

@ -12,6 +12,7 @@ mcp-rust/target/
# Test artifacts
test-results.json
coverage/
# Environment (contains personal paths)
.env

View file

@ -0,0 +1,103 @@
#!/usr/bin/env node
/**
* MCP tool parity guard.
*
* Verifies every tool name appears in all 5 required surfaces:
* 1. mcp-ts/src/tools.ts implementation function `toolXxx`
* 2. mcp-ts/src/index.ts server.registerTool('bindery_xxx', ...)
* 3. vscode-ext/src/mcp.ts vscode.lm.registerTool('bindery_xxx', ...)
* 4. vscode-ext/package.json languageModelTools[].name = 'bindery_xxx'
* 5. mcpb/manifest.json tools[].name = 'xxx' (no prefix)
*
* Source of truth: the set of tool names in mcp-ts/src/index.ts.
* Exits with code 1 if any surface diverges.
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import { fileURLToPath } from 'node:url';
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
function read(p) {
return fs.readFileSync(path.join(repoRoot, p), 'utf-8');
}
function extractMatches(source, regex) {
const names = new Set();
for (const m of source.matchAll(regex)) {
names.add(m[1]);
}
return names;
}
// ── Surface 1: mcp-ts/src/index.ts — registerTool('xxx', ...) (no prefix in server)
const indexTs = read('mcp-ts/src/index.ts');
const sourceOfTruth = extractMatches(indexTs, /registerTool\s*\(\s*['"]([a-z0-9_]+)['"]/gi);
/**
* Tools that only make sense when the MCP server runs standalone against a
* multi-book registry. The VS Code extension is always single-book (the
* workspace), so these aren't exposed via vscode.lm.registerTool or listed
* in languageModelTools[]. They must still appear in mcpb/manifest.json.
*/
const serverOnlyTools = new Set(['list_books', 'identify_book']);
const vscodeExpected = new Set([...sourceOfTruth].filter(t => !serverOnlyTools.has(t)));
// ── Surface 2: vscode-ext/src/mcp.ts — vscode.lm.registerTool('bindery_xxx', ...)
const mcpTs = read('vscode-ext/src/mcp.ts');
const vscodeLmTools = extractMatches(mcpTs, /registerTool\s*(?:<[^>]+>)?\s*\(\s*['"]bindery_([a-z0-9_]+)['"]/gi);
// ── Surface 3: vscode-ext/package.json — languageModelTools[].name
const vsPkg = JSON.parse(read('vscode-ext/package.json'));
const lmManifest = new Set(
(vsPkg.contributes?.languageModelTools ?? [])
.map((t) => (t.name || '').replace(/^bindery_/, ''))
.filter(Boolean)
);
// ── Surface 4: mcpb/manifest.json — tools[].name (no prefix)
const mcpbManifest = JSON.parse(read('mcpb/manifest.json'));
const mcpbTools = new Set((mcpbManifest.tools ?? []).map((t) => t.name).filter(Boolean));
// ── Surface 5: mcp-ts/src/tools.ts — implementation must exist for each
// We don't enforce naming convention strictly (tool names don't always map to camelCase),
// so we just confirm the file is non-empty — the TypeScript compiler already validates imports.
const toolsTs = read('mcp-ts/src/tools.ts');
if (toolsTs.length < 100) {
console.error('mcp-ts/src/tools.ts is suspiciously short');
process.exit(1);
}
// ── Compare ──
const surfaces = {
'mcp-ts/src/index.ts (registerTool)': { set: sourceOfTruth, expected: sourceOfTruth },
'vscode-ext/src/mcp.ts (vscode.lm.registerTool)': { set: vscodeLmTools, expected: vscodeExpected },
'vscode-ext/package.json (languageModelTools)': { set: lmManifest, expected: vscodeExpected },
'mcpb/manifest.json (tools)': { set: mcpbTools, expected: sourceOfTruth },
};
let failed = false;
console.log(`Source of truth: ${sourceOfTruth.size} tools in mcp-ts/src/index.ts`);
console.log(`Server-only (excluded from VS Code surfaces): ${[...serverOnlyTools].join(', ')}\n`);
for (const [label, { set, expected }] of Object.entries(surfaces)) {
const missing = [...expected].filter((t) => !set.has(t)).sort();
const extra = [...set].filter((t) => !expected.has(t)).sort();
if (missing.length === 0 && extra.length === 0) {
console.log(`${label}: ${set.size} tools`);
continue;
}
failed = true;
console.error(`${label}: ${set.size} tools (expected ${expected.size})`);
if (missing.length > 0) { console.error(` missing: ${missing.join(', ')}`); }
if (extra.length > 0) { console.error(` extra: ${extra.join(', ')}`); }
}
if (failed) {
console.error('\nTool parity check FAILED. Every MCP tool must be registered in all required surfaces.');
process.exit(1);
}
console.log('\nAll surfaces agree.');