mirror of
https://github.com/evdboom/Bindery.git
synced 2026-07-22 06:49:36 +00:00
ci: add OS matrix, tool-parity guard, and coverage job
Expand CI to ubuntu + windows + macos. Add a tool-parity check that verifies all four surfaces (mcp-ts index.ts, vscode-ext mcp.ts, vscode-ext package.json, mcpb manifest.json) declare the same tools. Add a coverage job that runs vitest with @vitest/coverage-v8 and enforces 80/65/90/80 thresholds for statements/branches/functions/lines.
This commit is contained in:
parent
60eefee806
commit
a88b47bf19
3 changed files with 149 additions and 3 deletions
48
.github/workflows/ci.yml
vendored
48
.github/workflows/ci.yml
vendored
|
|
@ -10,7 +10,11 @@ permissions:
|
|||
jobs:
|
||||
test:
|
||||
name: Test & template-sync check
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-latest]
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
||||
|
||||
|
|
@ -39,7 +43,7 @@ jobs:
|
|||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: mcp-test-results
|
||||
name: mcp-test-results-${{ matrix.os }}
|
||||
path: mcp-ts/test-results.json
|
||||
|
||||
# ── Template sync ─────────────────────────────────────────────────────────
|
||||
|
|
@ -65,5 +69,43 @@ jobs:
|
|||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: vscode-ext-test-results
|
||||
name: vscode-ext-test-results-${{ matrix.os }}
|
||||
path: vscode-ext/test-results.json
|
||||
|
||||
# ── Job: Coverage gates ──────────────────────────────────────────────────────
|
||||
# Runs coverage-with-thresholds on ubuntu only (one platform is enough — coverage
|
||||
# numbers don't depend on OS, and per-OS coverage runs would triple CI time).
|
||||
coverage:
|
||||
name: Coverage (v8)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: |
|
||||
mcp-ts/package-lock.json
|
||||
vscode-ext/package-lock.json
|
||||
|
||||
- name: Install mcp-ts
|
||||
run: cd mcp-ts && npm ci
|
||||
- name: mcp-ts coverage
|
||||
run: cd mcp-ts && npm run test:coverage
|
||||
|
||||
- name: Sync templates to vscode-ext
|
||||
run: cp mcp-ts/src/templates.ts vscode-ext/src/ai-setup-templates.ts
|
||||
|
||||
- name: Install vscode-ext
|
||||
run: cd vscode-ext && npm ci
|
||||
- name: vscode-ext coverage
|
||||
run: cd vscode-ext && npm run test:coverage
|
||||
|
||||
- name: Upload coverage reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: coverage
|
||||
path: |
|
||||
mcp-ts/coverage/
|
||||
vscode-ext/coverage/
|
||||
|
|
|
|||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -12,6 +12,7 @@ mcp-rust/target/
|
|||
|
||||
# Test artifacts
|
||||
test-results.json
|
||||
coverage/
|
||||
|
||||
# Environment (contains personal paths)
|
||||
.env
|
||||
|
|
|
|||
103
scripts/check-tool-parity.mjs
Normal file
103
scripts/check-tool-parity.mjs
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* MCP tool parity guard.
|
||||
*
|
||||
* Verifies every tool name appears in all 5 required surfaces:
|
||||
* 1. mcp-ts/src/tools.ts — implementation function `toolXxx`
|
||||
* 2. mcp-ts/src/index.ts — server.registerTool('bindery_xxx', ...)
|
||||
* 3. vscode-ext/src/mcp.ts — vscode.lm.registerTool('bindery_xxx', ...)
|
||||
* 4. vscode-ext/package.json — languageModelTools[].name = 'bindery_xxx'
|
||||
* 5. mcpb/manifest.json — tools[].name = 'xxx' (no prefix)
|
||||
*
|
||||
* Source of truth: the set of tool names in mcp-ts/src/index.ts.
|
||||
* Exits with code 1 if any surface diverges.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
|
||||
function read(p) {
|
||||
return fs.readFileSync(path.join(repoRoot, p), 'utf-8');
|
||||
}
|
||||
|
||||
function extractMatches(source, regex) {
|
||||
const names = new Set();
|
||||
for (const m of source.matchAll(regex)) {
|
||||
names.add(m[1]);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
// ── Surface 1: mcp-ts/src/index.ts — registerTool('xxx', ...) (no prefix in server)
|
||||
const indexTs = read('mcp-ts/src/index.ts');
|
||||
const sourceOfTruth = extractMatches(indexTs, /registerTool\s*\(\s*['"]([a-z0-9_]+)['"]/gi);
|
||||
|
||||
/**
|
||||
* Tools that only make sense when the MCP server runs standalone against a
|
||||
* multi-book registry. The VS Code extension is always single-book (the
|
||||
* workspace), so these aren't exposed via vscode.lm.registerTool or listed
|
||||
* in languageModelTools[]. They must still appear in mcpb/manifest.json.
|
||||
*/
|
||||
const serverOnlyTools = new Set(['list_books', 'identify_book']);
|
||||
const vscodeExpected = new Set([...sourceOfTruth].filter(t => !serverOnlyTools.has(t)));
|
||||
|
||||
// ── Surface 2: vscode-ext/src/mcp.ts — vscode.lm.registerTool('bindery_xxx', ...)
|
||||
const mcpTs = read('vscode-ext/src/mcp.ts');
|
||||
const vscodeLmTools = extractMatches(mcpTs, /registerTool\s*(?:<[^>]+>)?\s*\(\s*['"]bindery_([a-z0-9_]+)['"]/gi);
|
||||
|
||||
// ── Surface 3: vscode-ext/package.json — languageModelTools[].name
|
||||
const vsPkg = JSON.parse(read('vscode-ext/package.json'));
|
||||
const lmManifest = new Set(
|
||||
(vsPkg.contributes?.languageModelTools ?? [])
|
||||
.map((t) => (t.name || '').replace(/^bindery_/, ''))
|
||||
.filter(Boolean)
|
||||
);
|
||||
|
||||
// ── Surface 4: mcpb/manifest.json — tools[].name (no prefix)
|
||||
const mcpbManifest = JSON.parse(read('mcpb/manifest.json'));
|
||||
const mcpbTools = new Set((mcpbManifest.tools ?? []).map((t) => t.name).filter(Boolean));
|
||||
|
||||
// ── Surface 5: mcp-ts/src/tools.ts — implementation must exist for each
|
||||
// We don't enforce naming convention strictly (tool names don't always map to camelCase),
|
||||
// so we just confirm the file is non-empty — the TypeScript compiler already validates imports.
|
||||
const toolsTs = read('mcp-ts/src/tools.ts');
|
||||
if (toolsTs.length < 100) {
|
||||
console.error('mcp-ts/src/tools.ts is suspiciously short');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// ── Compare ──
|
||||
const surfaces = {
|
||||
'mcp-ts/src/index.ts (registerTool)': { set: sourceOfTruth, expected: sourceOfTruth },
|
||||
'vscode-ext/src/mcp.ts (vscode.lm.registerTool)': { set: vscodeLmTools, expected: vscodeExpected },
|
||||
'vscode-ext/package.json (languageModelTools)': { set: lmManifest, expected: vscodeExpected },
|
||||
'mcpb/manifest.json (tools)': { set: mcpbTools, expected: sourceOfTruth },
|
||||
};
|
||||
|
||||
let failed = false;
|
||||
console.log(`Source of truth: ${sourceOfTruth.size} tools in mcp-ts/src/index.ts`);
|
||||
console.log(`Server-only (excluded from VS Code surfaces): ${[...serverOnlyTools].join(', ')}\n`);
|
||||
|
||||
for (const [label, { set, expected }] of Object.entries(surfaces)) {
|
||||
const missing = [...expected].filter((t) => !set.has(t)).sort();
|
||||
const extra = [...set].filter((t) => !expected.has(t)).sort();
|
||||
|
||||
if (missing.length === 0 && extra.length === 0) {
|
||||
console.log(`✓ ${label}: ${set.size} tools`);
|
||||
continue;
|
||||
}
|
||||
failed = true;
|
||||
console.error(`✗ ${label}: ${set.size} tools (expected ${expected.size})`);
|
||||
if (missing.length > 0) { console.error(` missing: ${missing.join(', ')}`); }
|
||||
if (extra.length > 0) { console.error(` extra: ${extra.join(', ')}`); }
|
||||
}
|
||||
|
||||
if (failed) {
|
||||
console.error('\nTool parity check FAILED. Every MCP tool must be registered in all required surfaces.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log('\nAll surfaces agree.');
|
||||
Loading…
Reference in a new issue