diff --git a/.github/workflows/scorecard-watch.yml b/.github/workflows/scorecard-watch.yml new file mode 100644 index 0000000..183b2e7 --- /dev/null +++ b/.github/workflows/scorecard-watch.yml @@ -0,0 +1,54 @@ +name: Scorecard watch + +# Standalone scheduled regression watch over the Obsidian community portal +# scorecard (#165). It gates NOTHING in the release path — it does not run +# on PRs or releases. Its only job is to turn a silent portal regression +# (Health/Review downgrade, more automated issues, a new permission/ +# behaviour finding vs the committed baseline) into a visible red check. +# +# The gate never inspects advisory wording — only structured deltas vs +# scripts/scorecard-baseline.json. Scraper drift (exit 3) is reported +# distinctly from a real regression (exit 1); a transient unreachable +# portal is inconclusive and never fails (exit 0). + +on: + schedule: + - cron: '17 14 * * 1' # Mondays 14:17 UTC — offset off the hour + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: scorecard-watch + cancel-in-progress: false + +jobs: + scorecard: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v6 + + - name: Set up Node + uses: actions/setup-node@v6 + with: + node-version: '20' + + # No dependency install: scorecard.mjs + scorecard-gate.mjs use only + # Node builtins (fetch, fs, child_process). Keeps this fast and off + # the dependency surface. + - name: Scorecard regression gate + id: gate + run: | + set +e + node scripts/scorecard-gate.mjs + code=$? + echo "exit=$code" >> "$GITHUB_OUTPUT" + case $code in + 0) echo "::notice::Scorecard gate: no regression vs baseline." ;; + 1) echo "::error::Scorecard gate: REGRESSION vs baseline — see log. Re-baseline only if the change is accepted (make scorecard-baseline)." ;; + 3) echo "::error::Scorecard gate: SCRAPER DRIFT — fix scripts/scorecard.mjs. This is NOT a portal/plugin regression (#183-class)." ;; + *) echo "::error::Scorecard gate: unexpected exit $code." ;; + esac + exit $code diff --git a/Makefile b/Makefile index 8b0a172..7457367 100644 --- a/Makefile +++ b/Makefile @@ -93,6 +93,12 @@ mcpb: ## Build MCPB bundle (obsidian-mcp-.mcpb) for Claude Desktop scorecard: ## Pull the Obsidian community portal scorecard + freshness delta (free post-release signal) @node scripts/scorecard.mjs +scorecard-gate: ## Diff the live scorecard vs the committed baseline (exit 1 = regression, 3 = drift) + @node scripts/scorecard-gate.mjs + +scorecard-baseline: ## DELIBERATELY re-snapshot scorecard-baseline.json (only after an ACCEPTED change; commit it) + @node scripts/scorecard-baseline.mjs + set-description: ## Set plugin description (SoT: package.json) + sync. Usage: make set-description DESC='...' @node scripts/set-description.mjs "$(DESC)" @node sync-version.mjs diff --git a/scripts/scorecard-baseline.json b/scripts/scorecard-baseline.json new file mode 100644 index 0000000..fd0c843 --- /dev/null +++ b/scripts/scorecard-baseline.json @@ -0,0 +1,24 @@ +{ + "note": "Regenerate intentionally via `make scorecard-baseline` after an ACCEPTED scorecard change. scripts/scorecard-gate.mjs fails on regressions vs this snapshot.", + "capturedFor": "0.11.25", + "capturedAt": "2026-05-17", + "health": "Excellent", + "healthScore": "4/4", + "review": "Satisfactory", + "reviewScore": "3/4", + "issuesFound": 5, + "findings": [ + "**Clipboard Access**: Reads or writes the system clipboard. May expose content copied from outside Obsidian.", + "**Direct Filesystem Access**: Uses the Node.js `fs` module to access the filesystem outside of the Obsidian vault API. Can read and write any file on the system.", + "**Dynamic Code Execution**: Executes dynamically generated code via `eval()` or `new Function()`. Prevents full static analysis of plugin behavior.", + "**Vault Enumeration**: Enumerates all files in the vault (`vault.getFiles`, `getMarkdownFiles`, etc.). Gives the plugin access to every file path in the vault.", + "**Vault Read**: Reads individual vault files via the Obsidian API (`vault.read`, `vault.cachedRead`)", + "**Vault Write**: Creates or modifies vault files via the Obsidian API (`vault.modify`, `vault.create`, etc.)", + "Malware scan not available.", + "Network requests scan not available.", + "Obfuscation scan not available.", + "The `main.js` release asset has a verified GitHub artifact attestation.", + "The `styles.css` release asset has a verified GitHub artifact attestation.", + "The release contains additional files: `obsidian-mcp-0.11.25.mcpb`, `obsidian-mcp.mcpb`. Only `main.js`, `manifest.json`, and `styles.css` are supported." + ] +} diff --git a/scripts/scorecard-baseline.mjs b/scripts/scorecard-baseline.mjs new file mode 100644 index 0000000..d029b34 --- /dev/null +++ b/scripts/scorecard-baseline.mjs @@ -0,0 +1,53 @@ +#!/usr/bin/env node +// Regenerate scripts/scorecard-baseline.json from the live portal (#165). +// +// This is a DELIBERATE act: run it only when the scorecard genuinely +// changed in an ACCEPTED way (e.g. a finding cleared by a shipped fix, or +// a new finding analysed and accepted), then commit the new baseline. The +// scorecard-watch workflow fails against whatever this file records, so an +// accidental re-baseline silently disarms the gate — hence a separate, +// named command, not an automatic refresh. +// +// Refuses to write on scraper drift (the live read is untrustworthy then). + +import { writeFileSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; + +const run = spawnSync(process.execPath, ['scripts/scorecard.mjs', '--json'], { + encoding: 'utf8', + maxBuffer: 8 * 1024 * 1024, +}); +const jsonLine = (run.stdout || '') + .split('\n') + .reverse() + .find((l) => l.trim().startsWith('{')); +if (!jsonLine) { + console.error('scorecard-baseline: portal unreachable / no JSON — not writing.'); + process.exit(1); +} +const j = JSON.parse(jsonLine); +if (j.integrity === 'DRIFT' || run.status === 2) { + console.error( + 'scorecard-baseline: SCRAPER DRIFT — refusing to baseline an untrustworthy read. Fix scripts/scorecard.mjs first.', + ); + process.exit(3); +} + +const baseline = { + note: 'Regenerate intentionally via `make scorecard-baseline` after an ACCEPTED scorecard change. scripts/scorecard-gate.mjs fails on regressions vs this snapshot.', + capturedFor: j.portal.currentVersion, + capturedAt: new Date().toISOString().slice(0, 10), + health: j.portal.health, + healthScore: j.portal.healthScore, + review: j.portal.review, + reviewScore: j.portal.reviewScore, + issuesFound: Number(j.portal.issuesFound), + findings: j.findings.slice().sort(), +}; +writeFileSync( + 'scripts/scorecard-baseline.json', + JSON.stringify(baseline, null, 2) + '\n', +); +console.log( + `scorecard-baseline: wrote baseline for ${baseline.capturedFor} — Health ${baseline.health} ${baseline.healthScore} | Review ${baseline.review} ${baseline.reviewScore} | ${baseline.issuesFound} issues | ${baseline.findings.length} findings.\nReview the diff and commit scripts/scorecard-baseline.json.`, +); diff --git a/scripts/scorecard-gate.mjs b/scripts/scorecard-gate.mjs new file mode 100644 index 0000000..100c084 --- /dev/null +++ b/scripts/scorecard-gate.mjs @@ -0,0 +1,142 @@ +#!/usr/bin/env node +// Post-release regression gate over the Obsidian portal scorecard (#165). +// +// Runs `scorecard.mjs --json`, diffs the structured signals against the +// committed baseline (scripts/scorecard-baseline.json), and FAILS the +// workflow only on a genuine regression: +// +// - Health or Review score ratio dropped vs baseline (a downgrade) +// - automated issue count increased vs baseline +// - a new permission/behaviour finding appeared (a "**Title**:" +// entry absent from the baseline) +// +// It never inspects advisory *wording* — only structured deltas — so a +// reworded-but-equivalent finding does not fail the gate (that is the +// scraper drift guard's job, handled distinctly below). +// +// Exit codes: +// 0 no regression (or inconclusive: portal unreachable — never a false fail) +// 1 regression vs baseline → fail the workflow (intended signal) +// 3 scraper drift → distinct: fix scorecard.mjs, NOT a portal change +// +// This is a standalone scheduled job. It does not run on PRs or releases +// and gates nothing in the release path — it only turns a silent portal +// regression into a visible red check. + +import { readFileSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; + +const BASELINE = 'scripts/scorecard-baseline.json'; + +// "filled/total" → ratio in [0,1], or null if unparseable / total 0. +function ratio(score) { + if (typeof score !== 'string') return null; + const m = score.match(/^(\d+)\s*\/\s*(\d+)$/); + if (!m) return null; + const total = Number(m[2]); + return total > 0 ? Number(m[1]) / total : null; +} + +// The behaviour/permission findings are the bold-titled ones +// ("**Direct Filesystem Access**: …"). Neutral sentences ("… scan not +// available.", "… attestation.") are not regressions when they appear. +const titled = (findings) => + new Set( + (findings || []) + .map((f) => (f.match(/^\*\*([^*]+)\*\*:/) || [])[1]) + .filter(Boolean), + ); + +function main() { + const run = spawnSync(process.execPath, ['scripts/scorecard.mjs', '--json'], { + encoding: 'utf8', + maxBuffer: 8 * 1024 * 1024, + }); + + const jsonLine = (run.stdout || '') + .split('\n') + .reverse() + .find((l) => l.trim().startsWith('{')); + + if (!jsonLine) { + // scorecard.mjs exits 0 and prints to stderr when the portal is + // unreachable. A transient network failure must never fail the gate. + console.log( + 'scorecard-gate: portal unreachable / no JSON — inconclusive, not a regression.', + ); + if (run.stderr) console.log(run.stderr.trim()); + process.exit(0); + } + + const live = JSON.parse(jsonLine); + + if (live.integrity === 'DRIFT' || run.status === 2) { + const bang = '!'.repeat(64); + console.error(bang); + console.error('SCRAPER DRIFT — the portal parser no longer matches the'); + console.error('page. This is NOT a plugin/portal regression: fix'); + console.error('scripts/scorecard.mjs. Gate is inconclusive.'); + if (live.missingAnchors?.length) + console.error(` missing anchors : ${live.missingAnchors.join(', ')}`); + console.error(bang); + process.exit(3); + } + + const base = JSON.parse(readFileSync(BASELINE, 'utf8')); + const p = live.portal || {}; + const regressions = []; + + for (const [label, liveScore, baseScore] of [ + ['Health', p.healthScore, base.healthScore], + ['Review', p.reviewScore, base.reviewScore], + ]) { + const lr = ratio(liveScore); + const br = ratio(baseScore); + if (lr != null && br != null && lr < br) { + regressions.push( + `${label} score downgraded: ${baseScore} → ${liveScore}`, + ); + } + } + + const liveIssues = Number(p.issuesFound); + if (Number.isFinite(liveIssues) && liveIssues > Number(base.issuesFound)) { + regressions.push( + `automated issue count rose: ${base.issuesFound} → ${liveIssues}`, + ); + } + + const baseTitled = titled(base.findings); + const newTitled = [...titled(live.findings)].filter((t) => !baseTitled.has(t)); + if (newTitled.length) { + regressions.push(`new behaviour/permission finding(s): ${newTitled.join('; ')}`); + } + + const line = '─'.repeat(64); + console.log(line); + console.log(`Scorecard gate — baseline captured for ${base.capturedFor} (${base.capturedAt})`); + console.log( + `Live: Health ${p.health} ${p.healthScore} | Review ${p.review} ${p.reviewScore} | ${p.issuesFound} issues | freshness: ${live.freshness}`, + ); + console.log(line); + + if (regressions.length) { + console.error('REGRESSION vs baseline:'); + for (const r of regressions) console.error(` ✗ ${r}`); + console.error(line); + console.error( + 'If this reflects an ACCEPTED change, re-baseline deliberately:\n make scorecard-baseline (then commit scripts/scorecard-baseline.json)', + ); + process.exit(1); + } + + // Surface improvements for the log (informational, never gating). + const baseTitledArr = [...baseTitled]; + const liveTitledSet = titled(live.findings); + const cleared = baseTitledArr.filter((t) => !liveTitledSet.has(t)); + if (cleared.length) console.log(`Improved — finding(s) cleared: ${cleared.join('; ')}`); + console.log('No regression vs baseline. ✓'); + process.exit(0); +} + +main();