diff --git a/.e2e/README.md b/.e2e/README.md index f8e8ad2..4cd6dd3 100644 --- a/.e2e/README.md +++ b/.e2e/README.md @@ -1,18 +1,65 @@ # E2E Contract This repository uses `.e2e/run.sh` as the project-local runtime entry point. -The runner clears stale artifacts, executes the real `npm test` gate, records -`.e2e/results/npm-test.log`, and writes `.e2e/artifact.json` in CTRF format. +The runner clears stale artifacts, executes the release-grade test pipeline +(build → typecheck → unit/component/contract category suites → headless +product-shell smoke → optional live smoke), records logs under +`.e2e/results/`, and writes `.e2e/artifact.json` in CTRF format with one entry +per real evidence source. Run the host-neutral gate with: ```bash -bash .e2e/gate.sh --json +npm run e2e ``` -If `e2e_contract_validator` is not installed, set -`E2E_CONTRACT_VALIDATOR_PYTHONPATH` to the `claude-code-addons/scripts` -directory before running the gate. +The gate is self-contained in this repository. It does not depend on local +agent skills or an external validator checkout. Generated artifacts under `.e2e/artifact.json` and `.e2e/results/` are runtime evidence and are intentionally ignored by git. + +## Test Categories + +The source of truth for ordinary test classification is `tests/catalog.json`. + +- `unit`: pure logic tests with no Obsidian runtime, filesystem, process, or + provider boundary. +- `component`: plugin components exercised with controlled adapters, job + managers, or Obsidian shims. +- `contract`: provider protocols, CLI command shapes, exported test surfaces, + and architecture invariants. +- `e2e`: default project-local product-shell smoke. It installs the built plugin + package into a disposable Vault filesystem and boots the packaged `main.js` + against a recording Obsidian API shim. This checks package/install/lifecycle + boundaries, but it is not a full Obsidian GUI run. +- `live`: opt-in check against a real local Vault install. It verifies plugin + files under `.obsidian/plugins/parallel-reader/` and does not launch Obsidian, + call a provider, or run GUI interactions. Run with `TEST_LIVE=1` and set + `OBSIDIAN_VAULT_PATH` when the default iCloud Vault path is not the target. + +Useful commands: + +```bash +npm run e2e +npm run test:unit +npm run test:component +npm run test:contract +npm run test:e2e +TEST_LIVE=1 npm run test:live +``` + +The `.e2e` gate emits one CTRF entry per real evidence source so risk-tag +coverage maps to actual runs: + +| Entry | Risk tags | +|-------|-----------| +| `build and typecheck` | `boundary_io, wiring, regression` | +| `unit category` | `regression` | +| `component category` | `data_integrity, wiring, resource_lifecycle, regression` | +| `contract category` | `contract, wiring, regression` | +| `headless product-shell e2e smoke` | `boundary_io, wiring, resource_lifecycle, regression` | +| `live Vault install smoke` (skipped unless `TEST_LIVE=1`) | `boundary_io, contract, regression` | + +`npm test` remains a developer-facing convenience that runs the same build, +typecheck, and category suites without the e2e/live legs. diff --git a/.e2e/cases/live/run.mjs b/.e2e/cases/live/run.mjs new file mode 100644 index 0000000..68a48a6 --- /dev/null +++ b/.e2e/cases/live/run.mjs @@ -0,0 +1,77 @@ +import { createHash } from 'crypto'; +import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs'; +import os from 'os'; +import path from 'path'; + +const repoRoot = path.resolve(import.meta.dirname, '../../..'); +const resultPath = path.resolve(repoRoot, process.env.LIVE_E2E_RESULT || '.e2e/results/live.json'); +const startedAt = Date.now(); + +function writeResult(result) { + mkdirSync(path.dirname(resultPath), { recursive: true }); + writeFileSync(resultPath, JSON.stringify(result, null, 2) + '\n'); +} + +function sha256(filePath) { + return createHash('sha256').update(readFileSync(filePath)).digest('hex'); +} + +function defaultVaultPath() { + return path.join(os.homedir(), 'Library/Mobile Documents/iCloud~md~obsidian/Documents/Vault'); +} + +function main() { + if (process.env.TEST_LIVE !== '1') { + writeResult({ + name: 'live Vault install smoke', + category: 'live', + status: 'skipped', + durationMs: Date.now() - startedAt, + reason: 'Set TEST_LIVE=1 to check a real local Vault installation.', + }); + return; + } + + const vaultRoot = process.env.OBSIDIAN_VAULT_PATH || defaultVaultPath(); + const pluginDir = path.join(vaultRoot, '.obsidian/plugins/parallel-reader'); + const files = ['main.js', 'manifest.json', 'styles.css']; + const missing = files.filter((file) => !existsSync(path.join(pluginDir, file))); + if (missing.length) { + writeResult({ + name: 'live Vault install smoke', + category: 'live', + status: 'failed', + durationMs: Date.now() - startedAt, + vaultRoot, + missing, + }); + process.exit(1); + } + + const manifest = JSON.parse(readFileSync(path.join(pluginDir, 'manifest.json'), 'utf8')); + const sourceManifest = JSON.parse(readFileSync(path.join(repoRoot, 'manifest.json'), 'utf8')); + const hashChecks = files.map((file) => ({ + file, + sourceSha256: sha256(path.join(repoRoot, file)), + installedSha256: sha256(path.join(pluginDir, file)), + })); + const mismatched = hashChecks.filter((check) => check.sourceSha256 !== check.installedSha256); + + writeResult({ + name: 'live Vault install smoke', + category: 'live', + status: mismatched.length ? 'failed' : 'passed', + durationMs: Date.now() - startedAt, + vaultRoot, + manifest: { + installedId: manifest.id, + installedVersion: manifest.version, + sourceVersion: sourceManifest.version, + }, + hashChecks, + }); + + if (mismatched.length) process.exit(1); +} + +main(); diff --git a/.e2e/cases/npm-test-risk-map.json b/.e2e/cases/npm-test-risk-map.json deleted file mode 100644 index a56cf62..0000000 --- a/.e2e/cases/npm-test-risk-map.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "tests": [ - { - "name": "build and typecheck gate", - "risk_tags": ["boundary_io", "wiring", "regression"], - "source_files": ["package.json", "esbuild.config.mjs", "tsconfig.json"] - }, - { - "name": "provider protocol and schema contracts", - "risk_tags": ["contract", "failure_path", "regression"], - "source_files": ["tests/providers.test.js", "tests/schema.test.js", "tests/direct-providers.test.js"] - }, - { - "name": "cache, markdown, and vault data integrity", - "risk_tags": ["data_integrity", "boundary_io", "regression"], - "source_files": ["tests/cache.test.js", "tests/direct-cache.test.js", "tests/vault-batch.test.js"] - }, - { - "name": "batch generation failure and cancellation paths", - "risk_tags": ["failure_path", "data_integrity", "regression"], - "source_files": ["tests/plugin-batch.test.js", "tests/direct-batch.test.js"] - }, - { - "name": "plugin module wiring and export surface", - "risk_tags": ["wiring", "contract", "regression"], - "source_files": ["tests/architecture.test.js", "tests/test-exports.test.js"] - } - ] -} diff --git a/.e2e/cases/product-shell/run.mjs b/.e2e/cases/product-shell/run.mjs new file mode 100644 index 0000000..f26138b --- /dev/null +++ b/.e2e/cases/product-shell/run.mjs @@ -0,0 +1,481 @@ +import { createHash } from 'crypto'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync, copyFileSync } from 'fs'; +import Module, { createRequire } from 'module'; +import { tmpdir } from 'os'; +import path from 'path'; + +const require = createRequire(import.meta.url); +const repoRoot = path.resolve(import.meta.dirname, '../../..'); +const resultPath = path.resolve(repoRoot, process.env.PRODUCT_SHELL_RESULT || '.e2e/results/product-shell.json'); +const startedAt = Date.now(); + +function sha256(filePath) { + return createHash('sha256').update(readFileSync(filePath)).digest('hex'); +} + +function writeResult(result) { + mkdirSync(path.dirname(resultPath), { recursive: true }); + writeFileSync(resultPath, JSON.stringify(result, null, 2) + '\n'); +} + +function assert(condition, message) { + if (!condition) throw new Error(message); +} + +class FakeElement { + constructor(tag = 'div') { + this.tagName = tag.toUpperCase(); + this.classList = new Set(); + this.attrs = new Map(); + this.children = []; + this.parent = null; + this._listeners = new Map(); + this._textContent = ''; + this._title = ''; + } + + get textContent() { + if (this._textContent && this.children.length === 0) return this._textContent; + return this.children.map((c) => c.textContent).join(''); + } + set textContent(value) { + this._textContent = String(value); + this.children = []; + } + + setText(text) { + this.textContent = text; + } + + empty() { + for (const child of this.children) child.parent = null; + this.children = []; + this._textContent = ''; + } + + addClass(cls) { + if (!cls) return; + for (const part of String(cls).split(/\s+/)) if (part) this.classList.add(part); + } + removeClass(cls) { + if (!cls) return; + for (const part of String(cls).split(/\s+/)) this.classList.delete(part); + } + toggleClass(cls, force) { + if (force === true) this.addClass(cls); + else if (force === false) this.removeClass(cls); + else if (this.classList.has(cls)) this.removeClass(cls); + else this.addClass(cls); + } + hasClass(cls) { + return this.classList.has(cls); + } + + setAttr(name, value) { + this.attrs.set(name, String(value)); + } + getAttr(name) { + return this.attrs.has(name) ? this.attrs.get(name) : null; + } + + appendChild(child) { + if (child.parent) child.parent.children = child.parent.children.filter((c) => c !== child); + child.parent = this; + this.children.push(child); + return child; + } + + createEl(tag, options = {}) { + const child = new FakeElement(tag); + if (options.cls) child.addClass(options.cls); + if (options.text != null) child.setText(options.text); + if (options.title != null) child._title = String(options.title); + if (options.href != null) child.setAttr('href', options.href); + return this.appendChild(child); + } + + createDiv(options = {}) { + return this.createEl('div', options); + } + + addEventListener(type, handler) { + if (typeof handler !== 'function') return; + if (!this._listeners.has(type)) this._listeners.set(type, []); + this._listeners.get(type).push(handler); + } + + removeEventListener(type, handler) { + const list = this._listeners.get(type); + if (!list) return; + this._listeners.set( + type, + list.filter((h) => h !== handler), + ); + } + + focus() { + /* recorded by view.ts focusSummaryPane which checks typeof container.focus === 'function' */ + } + + querySelector(selector) { + const match = this._matcher(selector); + return this._findFirst(match); + } + + querySelectorAll(selector) { + const match = this._matcher(selector); + const out = []; + this._collect(match, out); + return out; + } + + _matcher(selector) { + const trimmed = String(selector).trim(); + if (trimmed.startsWith('.')) { + const cls = trimmed.slice(1); + return (el) => el.classList.has(cls); + } + const tag = trimmed.toUpperCase(); + return (el) => el.tagName === tag; + } + + _findFirst(match) { + for (const child of this.children) { + if (match(child)) return child; + const nested = child._findFirst(match); + if (nested) return nested; + } + return null; + } + + _collect(match, out) { + for (const child of this.children) { + if (match(child)) out.push(child); + child._collect(match, out); + } + } + + snapshot() { + return { + tag: this.tagName.toLowerCase(), + classes: [...this.classList].sort(), + attrs: Object.fromEntries(this.attrs), + text: this._textContent || undefined, + children: this.children.map((c) => c.snapshot()), + }; + } +} + +class DataAdapter { + constructor(vaultRoot) { + this.vaultRoot = vaultRoot; + } + + fullPath(filePath) { + return path.join(this.vaultRoot, filePath); + } + + async exists(filePath) { + return existsSync(this.fullPath(filePath)); + } + + async mkdir(filePath) { + mkdirSync(this.fullPath(filePath), { recursive: true }); + } + + async read(filePath) { + return readFileSync(this.fullPath(filePath), 'utf8'); + } + + async write(filePath, content) { + mkdirSync(path.dirname(this.fullPath(filePath)), { recursive: true }); + writeFileSync(this.fullPath(filePath), content); + } + + async remove(filePath) { + rmSync(this.fullPath(filePath), { force: true }); + } +} + +function makeObsidianStub(record) { + class Plugin { + constructor(app, manifest) { + this.app = app; + this.manifest = manifest; + } + + async loadData() { + return {}; + } + + async saveData(data) { + record.savedData = data; + } + + addRibbonIcon(icon, title, callback) { + record.ribbonIcons.push({ icon, title, callback }); + } + + registerView(type, factory) { + record.viewFactories.set(type, factory); + record.views.push({ type }); + } + + addCommand(command) { + record.commands.push(command); + } + + addSettingTab(tab) { + record.settingTabs.push(tab); + } + + registerEvent(eventRef) { + record.events.push(eventRef); + } + } + + class ItemView { + constructor(leaf) { + this.leaf = leaf; + this.containerEl = new FakeElement('div'); + this.containerEl.appendChild(new FakeElement('div')); + this.containerEl.appendChild(new FakeElement('div')); + } + } + + class PluginSettingTab { + constructor(app, plugin) { + this.app = app; + this.plugin = plugin; + this.containerEl = new FakeElement('div'); + } + } + + class Setting {} + class Notice { + constructor(message) { + record.notices.push(typeof message === 'string' ? message : ''); + } + } + class MarkdownView {} + class TFile {} + class Menu {} + class Modal {} + + return { + Plugin, + ItemView, + PluginSettingTab, + Setting, + Notice, + MarkdownView, + TFile, + Menu, + Modal, + MarkdownRenderer: { render: async () => {} }, + requestUrl: async () => { + throw new Error('requestUrl is not available in product-shell smoke'); + }, + setIcon: () => {}, + }; +} + +function makeApp(vaultRoot) { + const adapter = new DataAdapter(vaultRoot); + return { + vault: { + adapter, + configDir: '.obsidian', + on: (name) => ({ scope: 'vault', name }), + read: async (file) => adapter.read(file.path), + getMarkdownFiles: () => [], + getAbstractFileByPath: () => null, + }, + workspace: { + on: (name) => ({ scope: 'workspace', name }), + getActiveViewOfType: () => null, + getLeavesOfType: () => [], + getRightLeaf: () => null, + revealLeaf: async () => {}, + setActiveLeaf: () => {}, + getLeaf: () => null, + }, + }; +} + +function installPackage(vaultRoot) { + const pluginDir = path.join(vaultRoot, '.obsidian/plugins/parallel-reader'); + mkdirSync(pluginDir, { recursive: true }); + mkdirSync(path.join(vaultRoot, 'Reading'), { recursive: true }); + writeFileSync(path.join(vaultRoot, '.obsidian/community-plugins.json'), JSON.stringify(['parallel-reader'], null, 2)); + writeFileSync(path.join(vaultRoot, 'Reading/source.md'), '# Source\n\nThis note exercises plugin package installation.\n'); + + const files = ['main.js', 'main.js.map', 'manifest.json', 'styles.css']; + for (const file of files) { + const source = path.join(repoRoot, file); + assert(existsSync(source), `missing build artifact: ${file}`); + assert(statSync(source).size > 0, `empty build artifact: ${file}`); + copyFileSync(source, path.join(pluginDir, file)); + } + return { pluginDir, files }; +} + +const SAFE_COMMANDS = ['card-prev', 'card-next', 'card-jump', 'clear-all']; +const UNSAFE_COMMANDS = ['run', 'regen', 'open-view', 'export-current', 'copy-current-markdown', 'batch-generate']; + +async function exerciseView(record, plugin) { + const factory = record.viewFactories.get('parallel-reader-view'); + assert(typeof factory === 'function', 'view factory is not a function'); + + const leaf = { containerEl: new FakeElement('div') }; + const view = factory(leaf); + assert(view, 'view factory returned no view'); + assert(typeof view.onOpen === 'function', 'view has no onOpen'); + + await view.onOpen(); + + const container = view.containerEl.children[1]; + assert(container.hasClass('parallel-reader-container'), 'container missing parallel-reader-container class'); + assert(container.getAttr('tabindex') === '0', 'container missing tabindex'); + + const empty = container.querySelector('.parallel-reader-empty'); + assert(empty, 'empty state element not rendered'); + const heading = empty.querySelector('h3'); + assert(heading && heading.textContent, 'empty state heading missing'); + const paragraph = empty.querySelector('p'); + assert(paragraph && paragraph.textContent, 'empty state paragraph missing'); + const code = empty.querySelector('code'); + assert(code && code.textContent, 'empty state code hint missing'); + + await view.onClose(); + + return { + container: { + classes: [...container.classList].sort(), + tabindex: container.getAttr('tabindex'), + childCount: container.children.length, + }, + empty: empty.snapshot(), + }; +} + +async function invokeSafeCommands(record, plugin) { + const byId = new Map(record.commands.map((cmd) => [cmd.id, cmd])); + const invoked = []; + for (const id of SAFE_COMMANDS) { + const cmd = byId.get(id); + assert(cmd && typeof cmd.callback === 'function', `safe command ${id} missing callback`); + const startedAt = Date.now(); + let error = null; + try { + await cmd.callback(); + } catch (err) { + error = err instanceof Error ? err.message : String(err); + } + invoked.push({ id, durationMs: Date.now() - startedAt, error }); + } + return invoked; +} + +function recordUnsafeCommands(record) { + const byId = new Map(record.commands.map((cmd) => [cmd.id, cmd])); + return UNSAFE_COMMANDS.map((id) => ({ + id, + registered: byId.has(id), + attempted: false, + reason: 'requires workspace/network mocks beyond product-shell shim', + })); +} + +async function main() { + const vaultRoot = mkdtempSync(path.join(tmpdir(), 'parallel-reader-e2e-vault-')); + const record = { + commands: [], + events: [], + notices: [], + ribbonIcons: [], + settingTabs: [], + viewFactories: new Map(), + views: [], + }; + + try { + const { pluginDir, files } = installPackage(vaultRoot); + const manifestPath = path.join(pluginDir, 'manifest.json'); + const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); + assert(manifest.id === 'parallel-reader', `unexpected manifest id: ${manifest.id}`); + assert(manifest.isDesktopOnly === true, 'manifest must remain desktop-only'); + + const originalLoad = Module._load; + const obsidianStub = makeObsidianStub(record); + Module._load = function load(request, parent, isMain) { + if (request === 'obsidian') return obsidianStub; + return originalLoad.call(this, request, parent, isMain); + }; + + let viewRender = null; + let commandsInvoked = null; + let commandsAttempted = null; + + try { + const pluginModule = require(path.join(pluginDir, 'main.js')); + const PluginClass = pluginModule.default || pluginModule; + const plugin = new PluginClass(makeApp(vaultRoot), manifest); + await plugin.onload(); + assert(record.views.some((view) => view.type === 'parallel-reader-view'), 'parallel-reader view was not registered'); + const commandIds = new Set(record.commands.map((command) => command.id)); + for (const id of ['run', 'regen', 'open-view', 'export-current', 'copy-current-markdown', 'batch-generate']) { + assert(commandIds.has(id), `missing command registration: ${id}`); + } + assert(record.settingTabs.length === 1, 'settings tab was not registered'); + + viewRender = await exerciseView(record, plugin); + commandsInvoked = await invokeSafeCommands(record, plugin); + commandsAttempted = recordUnsafeCommands(record); + + const failedSafe = commandsInvoked.filter((c) => c.error); + assert(failedSafe.length === 0, `safe commands raised: ${failedSafe.map((c) => `${c.id}=${c.error}`).join('; ')}`); + + await plugin.onunload(); + } finally { + Module._load = originalLoad; + } + + writeResult({ + name: 'packaged plugin boot and disposable vault install smoke', + category: 'e2e', + status: 'passed', + durationMs: Date.now() - startedAt, + checks: { + manifest: { id: manifest.id, version: manifest.version, desktopOnly: manifest.isDesktopOnly }, + installedFiles: files.map((file) => ({ + file, + sha256: sha256(path.join(pluginDir, file)), + })), + registeredCommands: record.commands.map((command) => command.id).sort(), + registeredViews: record.views.map((view) => view.type).sort(), + registeredEventCount: record.events.length, + }, + viewRender, + commandsInvoked, + commandsAttempted, + noticeCount: record.notices.length, + }); + } catch (error) { + writeResult({ + name: 'packaged plugin boot and disposable vault install smoke', + category: 'e2e', + status: 'failed', + durationMs: Date.now() - startedAt, + error: error instanceof Error ? error.stack || error.message : String(error), + }); + throw error; + } finally { + rmSync(vaultRoot, { recursive: true, force: true }); + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); diff --git a/.e2e/config.yaml b/.e2e/config.yaml index 3e14e1e..6cf010f 100644 --- a/.e2e/config.yaml +++ b/.e2e/config.yaml @@ -3,10 +3,10 @@ schema_version: "2.0" required_risk_tags: - boundary_io - wiring - - failure_path - data_integrity - contract - regression + - resource_lifecycle preconditions: [] diff --git a/.e2e/gate.sh b/.e2e/gate.sh index 64ecc48..9c2887c 100755 --- a/.e2e/gate.sh +++ b/.e2e/gate.sh @@ -3,13 +3,4 @@ set -euo pipefail cd "$(dirname "$0")/.." -if ! python -c 'import e2e_contract_validator' >/dev/null 2>&1; then - if [ -n "${E2E_CONTRACT_VALIDATOR_PYTHONPATH:-}" ]; then - export PYTHONPATH="${E2E_CONTRACT_VALIDATOR_PYTHONPATH}${PYTHONPATH:+:$PYTHONPATH}" - else - echo "e2e_contract_validator is not importable; set E2E_CONTRACT_VALIDATOR_PYTHONPATH or install it" >&2 - exit 3 - fi -fi - -python -m e2e_contract_validator gate --project . --json "$@" +node .e2e/scripts/gate.mjs --project . "$@" diff --git a/.e2e/run.sh b/.e2e/run.sh index 7c20136..3b8f821 100755 --- a/.e2e/run.sh +++ b/.e2e/run.sh @@ -5,20 +5,64 @@ cd "$(dirname "$0")/.." ARTIFACT=".e2e/artifact.json" RESULTS_DIR=".e2e/results" -LOG="$RESULTS_DIR/npm-test.log" -START_MS="$(node -e 'process.stdout.write(String(Date.now()))')" +BUILD_LOG="$RESULTS_DIR/build.log" +TESTS_LOG="$RESULTS_DIR/tests-default.log" +TESTS_RESULT="$RESULTS_DIR/tests-default.json" +PRODUCT_SHELL_LOG="$RESULTS_DIR/product-shell.log" +PRODUCT_SHELL_RESULT="$RESULTS_DIR/product-shell.json" +LIVE_LOG="$RESULTS_DIR/live.log" +LIVE_RESULT="$RESULTS_DIR/live.json" rm -f "$ARTIFACT" +rm -rf "$RESULTS_DIR" mkdir -p "$RESULTS_DIR" +now_ms() { + node -e 'process.stdout.write(String(Date.now()))' +} + +START_MS="$(now_ms)" + set +e -npm test >"$LOG" 2>&1 -RUN_EXIT=$? +BUILD_START_MS="$(now_ms)" +{ npm run build && npm run typecheck; } >"$BUILD_LOG" 2>&1 +BUILD_EXIT=$? +BUILD_END_MS="$(now_ms)" +BUILD_DURATION_MS=$((BUILD_END_MS - BUILD_START_MS)) + +if [ "$BUILD_EXIT" -eq 0 ]; then + TEST_RESULTS_JSON="$TESTS_RESULT" node scripts/run-tests.mjs >"$TESTS_LOG" 2>&1 + TESTS_EXIT=$? +else + TESTS_EXIT=1 + echo "skipped: build/typecheck failed" >"$TESTS_LOG" +fi + +if [ "$BUILD_EXIT" -eq 0 ]; then + PRODUCT_SHELL_RESULT="$PRODUCT_SHELL_RESULT" node .e2e/cases/product-shell/run.mjs >"$PRODUCT_SHELL_LOG" 2>&1 + PRODUCT_SHELL_EXIT=$? +else + PRODUCT_SHELL_EXIT=1 + echo "skipped: build/typecheck failed" >"$PRODUCT_SHELL_LOG" +fi + +if [ "${TEST_LIVE:-0}" = "1" ]; then + LIVE_E2E_RESULT="$LIVE_RESULT" node .e2e/cases/live/run.mjs >"$LIVE_LOG" 2>&1 + LIVE_EXIT=$? +else + LIVE_EXIT=0 +fi set -e -END_MS="$(node -e 'process.stdout.write(String(Date.now()))')" -LOG_SHA="$(shasum -a 256 "$LOG" | awk '{print $1}')" -export RUN_EXIT START_MS END_MS LOG LOG_SHA +END_MS="$(now_ms)" +export START_MS END_MS +export BUILD_LOG BUILD_EXIT BUILD_DURATION_MS +export TESTS_LOG TESTS_RESULT +export PRODUCT_SHELL_LOG PRODUCT_SHELL_EXIT +export LIVE_LOG LIVE_EXIT node .e2e/scripts/write-artifact.mjs -exit "$RUN_EXIT" + +if [ "$BUILD_EXIT" -ne 0 ] || [ "$TESTS_EXIT" -ne 0 ] || [ "$PRODUCT_SHELL_EXIT" -ne 0 ] || [ "$LIVE_EXIT" -ne 0 ]; then + exit 1 +fi diff --git a/.e2e/scripts/gate.mjs b/.e2e/scripts/gate.mjs new file mode 100644 index 0000000..436c869 --- /dev/null +++ b/.e2e/scripts/gate.mjs @@ -0,0 +1,395 @@ +import { spawnSync } from 'child_process'; +import { createHash } from 'crypto'; +import { existsSync, readFileSync } from 'fs'; +import path from 'path'; + +const VALID_RISK_TAGS = new Set([ + 'boundary_io', + 'failure_path', + 'concurrency', + 'wiring', + 'regression', + 'security', + 'data_integrity', + 'resource_lifecycle', + 'contract', +]); + +function parseArgs(argv) { + const args = { + json: false, + project: '.', + required: false, + skipRun: false, + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--json') args.json = true; + else if (arg === '--required') args.required = true; + else if (arg === '--skip-run') args.skipRun = true; + else if (arg === '--project') { + const value = argv[++i]; + if (!value) throw new Error('--project requires a value'); + args.project = value; + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + + return args; +} + +function readJson(filePath) { + return JSON.parse(readFileSync(filePath, 'utf8')); +} + +function parseScalar(value) { + const trimmed = value.trim(); + if (trimmed === '[]') return []; + if (trimmed === 'true') return true; + if (trimmed === 'false') return false; + if ((trimmed.startsWith('"') && trimmed.endsWith('"')) || (trimmed.startsWith("'") && trimmed.endsWith("'"))) { + return trimmed.slice(1, -1); + } + return trimmed; +} + +function loadConfig(configPath) { + const root = {}; + let currentList = null; + let currentObject = null; + + for (const rawLine of readFileSync(configPath, 'utf8').split(/\r?\n/)) { + const withoutComment = rawLine.replace(/\s+#.*$/, ''); + if (!withoutComment.trim()) continue; + + const indent = withoutComment.match(/^\s*/)[0].length; + const line = withoutComment.trim(); + if (indent === 0) { + currentList = null; + currentObject = null; + const match = line.match(/^([^:]+):(.*)$/); + if (!match) throw new Error(`unsupported config line: ${rawLine}`); + const key = match[1].trim(); + const value = match[2].trim(); + if (!value) { + if (key === 'required_risk_tags' || key === 'preconditions') { + root[key] = []; + currentList = root[key]; + } else { + root[key] = {}; + currentObject = root[key]; + } + } else { + root[key] = parseScalar(value); + } + } else if (currentList && line.startsWith('- ')) { + currentList.push(parseScalar(line.slice(2))); + } else if (currentObject) { + const match = line.match(/^([^:]+):(.*)$/); + if (!match) throw new Error(`unsupported nested config line: ${rawLine}`); + currentObject[match[1].trim()] = parseScalar(match[2].trim()); + } else { + throw new Error(`unsupported config indentation: ${rawLine}`); + } + } + + const schemaVersion = String(root.schema_version || ''); + if (!schemaVersion) throw new Error('config.schema_version is required'); + if (!schemaVersion.startsWith('2.')) throw new Error(`unsupported config schema_version ${schemaVersion}`); + + const required = root.required_risk_tags; + if (!Array.isArray(required) || required.length === 0) { + throw new Error('config.required_risk_tags must be a non-empty list'); + } + const invalid = required.filter( + (tag) => typeof tag !== 'string' || (!VALID_RISK_TAGS.has(tag) && !tag.startsWith('x-')), + ); + if (invalid.length) throw new Error(`unknown required risk tag(s): ${invalid.join(', ')}`); + + if (!root.provenance || typeof root.provenance !== 'object') root.provenance = {}; + if (!Array.isArray(root.preconditions)) root.preconditions = []; + + return root; +} + +function loadCtrf(artifactPath) { + const ctrf = readJson(artifactPath); + if (!ctrf || typeof ctrf !== 'object' || Array.isArray(ctrf)) throw new Error('artifact must be a JSON object'); + const results = ctrf.results; + if (!results || typeof results !== 'object') throw new Error('artifact.results is required'); + if (!results.tool || typeof results.tool !== 'object') throw new Error('artifact.results.tool is required'); + if (!results.summary || typeof results.summary !== 'object') throw new Error('artifact.results.summary is required'); + if (!Array.isArray(results.tests)) throw new Error('artifact.results.tests must be a list'); + return ctrf; +} + +function sha256(filePath) { + return createHash('sha256').update(readFileSync(filePath)).digest('hex'); +} + +function* walkDicts(value) { + if (value && typeof value === 'object' && !Array.isArray(value)) { + yield value; + for (const child of Object.values(value)) yield* walkDicts(child); + } else if (Array.isArray(value)) { + for (const child of value) yield* walkDicts(child); + } +} + +function* iterContracts(ctrf) { + for (const test of ctrf.results.tests) { + const contract = test?.extra?.e2e_contract; + if (contract && typeof contract === 'object' && !Array.isArray(contract)) { + yield [test, contract]; + } + } +} + +function verifyEvidenceSha256(ctrf, evidenceDir) { + const base = path.resolve(evidenceDir); + const errors = []; + for (const [test, contract] of iterContracts(ctrf)) { + const testName = String(test.name || ''); + for (const mapping of walkDicts(contract)) { + for (const [key, expected] of Object.entries(mapping)) { + if (!key.endsWith('_sha256') || typeof expected !== 'string') continue; + const pathKey = `${key.slice(0, -7)}_path`; + const relPath = mapping[pathKey]; + if (typeof relPath !== 'string' || !relPath) continue; + const filePath = path.resolve(base, relPath); + if (!filePath.startsWith(`${base}${path.sep}`) && filePath !== base) { + errors.push(`${testName}: evidence path escapes base directory ${relPath}`); + continue; + } + if (!existsSync(filePath)) { + errors.push(`${testName}: missing evidence file ${relPath}`); + continue; + } + if (sha256(filePath) !== expected) { + errors.push(`${testName}: ${relPath} sha256 mismatch`); + } + } + } + } + return errors; +} + +function checkInternalConsistency(ctrf) { + const errors = []; + const summary = ctrf.results.summary; + const tests = ctrf.results.tests; + const statuses = ['passed', 'failed', 'skipped', 'pending', 'other']; + + for (const field of ['tests', ...statuses]) { + if (!(field in summary)) errors.push(`summary.${field} is required`); + } + + const countTotal = statuses.reduce((sum, status) => sum + Number(summary[status] || 0), 0); + const declaredTotal = Number(summary.tests || tests.length); + if (declaredTotal !== tests.length) { + errors.push(`summary.tests=${declaredTotal} but artifact has ${tests.length} test entries`); + } + if (countTotal !== tests.length) { + errors.push(`summary status counts total ${countTotal} but artifact has ${tests.length} test entries`); + } + if (Number(summary.failed || 0) > 0) { + errors.push(`summary.failed=${summary.failed}; e2e gate requires all tests to pass`); + } + + for (const [test, contract] of iterContracts(ctrf)) { + if (test.status !== 'passed') continue; + const command = contract.command_recorded; + if (!command || typeof command !== 'object') continue; + if (!Object.hasOwn(command, 'exit_code')) { + errors.push(`${test.name || ''}: command_recorded.exit_code missing`); + continue; + } + const expected = Object.hasOwn(contract, 'expected_exit_code') ? contract.expected_exit_code : 0; + if (command.exit_code !== expected) { + errors.push(`${test.name || ''}: exit_code ${command.exit_code} != expected ${expected}`); + } + } + + return errors; +} + +function checkRiskCoverage(config, ctrf) { + const required = new Set(config.required_risk_tags || []); + const covered = new Set(); + for (const test of ctrf.results.tests) { + if (test.status !== 'passed') continue; + for (const tag of test.tags || []) { + if (typeof tag === 'string' && tag.startsWith('risk:')) covered.add(tag.slice('risk:'.length)); + } + } + return [...required].filter((tag) => !covered.has(tag)).sort(); +} + +function runCheck(configPath, artifactPath, evidenceDir) { + const result = { + artifact_path: artifactPath, + config_path: configPath, + failures: [], + missing: [], + mode: 'ctrf', + schema_version: '2.0', + status: 'failed', + steps: {}, + }; + + let config; + let ctrf; + try { + config = loadConfig(configPath); + ctrf = loadCtrf(artifactPath); + result.steps.parse = 'pass'; + } catch (error) { + result.status = 'invalid'; + result.failures.push(error instanceof Error ? error.message : String(error)); + result.steps.parse = 'fail'; + return [result, 3]; + } + + if (config.provenance?.cosign_required) { + result.failures.push('cosign provenance verification is not available in this repo-local gate'); + result.steps.provenance = 'fail'; + return [result, 1]; + } + result.steps.provenance = 'skipped'; + + const evidenceErrors = verifyEvidenceSha256(ctrf, evidenceDir); + if (evidenceErrors.length) { + result.status = 'invalid'; + result.failures.push(...evidenceErrors); + result.steps.evidence_sha256 = 'fail'; + return [result, 4]; + } + result.steps.evidence_sha256 = 'pass'; + + const consistencyErrors = checkInternalConsistency(ctrf); + if (consistencyErrors.length) { + result.failures.push(...consistencyErrors); + result.steps.internal_consistency = 'fail'; + return [result, 1]; + } + result.steps.internal_consistency = 'pass'; + + const missing = checkRiskCoverage(config, ctrf); + result.missing = missing; + if (missing.length) { + result.failures.push(`missing required risk tags: ${missing.join(', ')}`); + result.steps.risk_coverage = 'fail'; + return [result, 1]; + } + result.steps.risk_coverage = 'pass'; + result.status = 'pass'; + return [result, 0]; +} + +function runPreconditions(config, projectRoot) { + const records = []; + for (const item of config.preconditions || []) { + if (!item || typeof item !== 'object' || !item.check) continue; + const completed = spawnSync(String(item.check), { + cwd: projectRoot, + encoding: 'utf8', + shell: true, + stdio: ['ignore', 'pipe', 'pipe'], + }); + records.push({ + check: item.check, + exit_code: completed.status ?? 1, + name: item.name || 'precondition', + remediation: item.remediation || '', + status: completed.status === 0 ? 'pass' : 'fail', + stderr: (completed.stderr || '').trim(), + stdout: (completed.stdout || '').trim(), + }); + } + return records; +} + +function runGate(args) { + const projectRoot = path.resolve(args.project); + const e2eDir = path.join(projectRoot, '.e2e'); + const configPath = path.join(e2eDir, 'config.yaml'); + const artifactPath = path.join(e2eDir, 'artifact.json'); + const result = { + artifact_path: artifactPath, + check_exit_code: null, + config_path: configPath, + exit_code: 1, + missing: [], + mode: 'ctrf', + preconditions: [], + run_exit_code: null, + schema_version: '2.0', + status: 'failed', + }; + + if (!existsSync(configPath)) { + result.status = args.required ? 'missing_contract' : 'no_contract'; + result.exit_code = args.required ? 3 : 0; + return [result, result.exit_code]; + } + + let config; + try { + config = loadConfig(configPath); + } catch (error) { + result.status = 'invalid'; + result.failures = [error instanceof Error ? error.message : String(error)]; + result.exit_code = 3; + return [result, 3]; + } + + const preconditions = runPreconditions(config, projectRoot); + result.preconditions = preconditions; + if (preconditions.some((item) => item.status !== 'pass')) { + result.status = 'precondition_failed'; + result.exit_code = 2; + return [result, 2]; + } + + let runExit = 0; + if (!args.skipRun) { + const runScript = path.join(e2eDir, 'run.sh'); + if (!existsSync(runScript)) { + result.status = 'invalid'; + result.failures = ['.e2e/run.sh is missing']; + result.exit_code = 3; + return [result, 3]; + } + const completed = spawnSync('bash', [runScript], { + cwd: projectRoot, + encoding: 'utf8', + env: { ...process.env, E2E_CONTRACT_ACTIVE: 'true' }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + runExit = completed.status ?? 1; + result.run_stdout = (completed.stdout || '').trim(); + result.run_stderr = (completed.stderr || '').trim(); + } + result.run_exit_code = runExit; + + const [check, checkExit] = runCheck(configPath, artifactPath, e2eDir); + result.check = check; + result.check_exit_code = checkExit; + result.missing = check.missing || []; + result.exit_code = checkExit || (runExit !== 0 ? 1 : 0); + result.status = result.exit_code === 0 ? 'pass' : 'fail'; + return [result, result.exit_code]; +} + +try { + const args = parseArgs(process.argv.slice(2)); + const [result, exitCode] = runGate(args); + process.stdout.write(JSON.stringify(result, null, args.json ? 2 : 0) + '\n'); + process.exit(exitCode); +} catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${message}\n`); + process.exit(3); +} diff --git a/.e2e/scripts/write-artifact.mjs b/.e2e/scripts/write-artifact.mjs index 646ee7a..5250dfb 100644 --- a/.e2e/scripts/write-artifact.mjs +++ b/.e2e/scripts/write-artifact.mjs @@ -1,63 +1,193 @@ -import fs from 'fs'; +import { createHash } from 'crypto'; +import { existsSync, readFileSync, writeFileSync } from 'fs'; -const riskMap = JSON.parse(fs.readFileSync('.e2e/cases/npm-test-risk-map.json', 'utf8')); -const runExit = Number(process.env.RUN_EXIT || 1); -const startMs = Number(process.env.START_MS || Date.now()); -const endMs = Number(process.env.END_MS || startMs); -const durationMs = Math.max(0, endMs - startMs); -const logPath = String(process.env.LOG || '.e2e/results/npm-test.log').replace(/^\.e2e\//, ''); -const logSha = String(process.env.LOG_SHA || ''); +const resultsDir = '.e2e/results'; +const catalogPath = 'tests/catalog.json'; -const command = { - command: 'npm test', - exit_code: runExit, -}; +const RISK_TAGS_BY_CATEGORY = Object.freeze({ + unit: ['regression'], + component: ['data_integrity', 'wiring', 'resource_lifecycle', 'regression'], + contract: ['contract', 'wiring', 'regression'], +}); -const tests = - runExit === 0 - ? riskMap.tests.map((test) => ({ - name: test.name, - status: 'passed', - duration: durationMs / riskMap.tests.length, - tags: test.risk_tags.map((tag) => `risk:${tag}`), - extra: { - e2e_contract: { - command_recorded: command, - expected_exit_code: 0, - evidence: { - log_path: logPath, - log_sha256: logSha, - }, - source_files: test.source_files, - }, +const PRODUCT_SHELL_RISK_TAGS = ['boundary_io', 'wiring', 'resource_lifecycle', 'regression']; +const LIVE_RISK_TAGS = ['boundary_io', 'contract', 'regression']; +const BUILD_RISK_TAGS = ['boundary_io', 'wiring', 'regression']; + +function readJson(path, fallback = null) { + try { + return JSON.parse(readFileSync(path, 'utf8')); + } catch { + return fallback; + } +} + +function sha256(path) { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +function rel(path) { + return path.replace(/^\.e2e\//, ''); +} + +function evidence(logPath, resultPath = '') { + const value = {}; + if (logPath && existsSync(logPath)) { + value.log_path = rel(logPath); + value.log_sha256 = sha256(logPath); + } + if (resultPath && existsSync(resultPath)) { + value.result_path = rel(resultPath); + value.result_sha256 = sha256(resultPath); + } + return value; +} + +function makeTest({ name, status, durationMs, tags, command, exitCode, logPath, resultPath, sourceFiles, extra = {} }) { + return { + name, + status, + duration: Math.max(0, durationMs || 0), + tags: status === 'passed' ? tags.map((tag) => `risk:${tag}`) : [], + extra: { + e2e_contract: { + command_recorded: { + command, + exit_code: Number(exitCode), }, - })) - : [ - { - name: 'npm test', - status: 'failed', - duration: durationMs, - tags: [], - extra: { - e2e_contract: { - command_recorded: command, - expected_exit_code: 0, - evidence: { - log_path: logPath, - log_sha256: logSha, - }, - }, - }, - }, - ]; + expected_exit_code: 0, + evidence: evidence(logPath, resultPath), + source_files: sourceFiles, + ...extra, + }, + }, + }; +} + +function makeSkippedTest(name, reason) { + return { + name, + status: 'skipped', + duration: 0, + tags: [], + extra: { + e2e_contract: { + skip_reason: reason, + }, + }, + }; +} + +function buildAndTypecheckTest() { + const logPath = process.env.BUILD_LOG || `${resultsDir}/build.log`; + const exitCode = Number(process.env.BUILD_EXIT || 1); + const durationMs = Number(process.env.BUILD_DURATION_MS || 0); + return makeTest({ + name: 'build and typecheck', + status: exitCode === 0 ? 'passed' : 'failed', + durationMs, + tags: BUILD_RISK_TAGS, + command: 'npm run build && npm run typecheck', + exitCode, + logPath, + sourceFiles: ['package.json', 'esbuild.config.mjs', 'tsconfig.json'], + extra: { category: 'build' }, + }); +} + +function categoryTest(category, summary, defaultCategories) { + const tags = RISK_TAGS_BY_CATEGORY[category]; + if (!tags) throw new Error(`Unmapped test category: ${category}`); + const stats = summary?.byCategory?.[category] || { tests: 0, passed: 0, failed: 0, files: [] }; + const exitCode = stats.failed === 0 && stats.tests > 0 ? 0 : 1; + const durationMs = (summary?.results || []) + .filter((r) => r.category === category) + .reduce((sum, r) => sum + (r.durationMs || 0), 0); + const recordedCommand = `TEST_RESULTS_JSON=${process.env.TESTS_RESULT || `${resultsDir}/tests-default.json`} node scripts/run-tests.mjs (categories: ${defaultCategories.join(',')})`; + return makeTest({ + name: `${category} category`, + status: exitCode === 0 ? 'passed' : 'failed', + durationMs, + tags, + command: recordedCommand, + exitCode, + logPath: process.env.TESTS_LOG || `${resultsDir}/tests-default.log`, + resultPath: process.env.TESTS_RESULT || `${resultsDir}/tests-default.json`, + sourceFiles: stats.files.map((f) => `tests/${f}`), + extra: { category, stats: { tests: stats.tests, passed: stats.passed, failed: stats.failed } }, + }); +} + +function productShellTest() { + const logPath = process.env.PRODUCT_SHELL_LOG || `${resultsDir}/product-shell.log`; + const resultPath = `${resultsDir}/product-shell.json`; + const exitCode = Number(process.env.PRODUCT_SHELL_EXIT || 1); + const result = readJson(resultPath, {}); + return makeTest({ + name: 'headless product-shell e2e smoke', + status: result.status === 'passed' && exitCode === 0 ? 'passed' : 'failed', + durationMs: result.durationMs || 0, + tags: PRODUCT_SHELL_RISK_TAGS, + command: 'node .e2e/cases/product-shell/run.mjs', + exitCode, + logPath, + resultPath, + sourceFiles: ['main.js', 'manifest.json', 'styles.css', '.e2e/cases/product-shell/run.mjs'], + extra: { + category: 'e2e', + checks: result.checks || {}, + viewRender: result.viewRender || null, + commandsInvoked: result.commandsInvoked || null, + commandsAttempted: result.commandsAttempted || null, + }, + }); +} + +function liveTest() { + if (process.env.TEST_LIVE !== '1') { + return makeSkippedTest('live Vault install smoke', 'Set TEST_LIVE=1 to check a real local Vault installation.'); + } + const logPath = process.env.LIVE_LOG || `${resultsDir}/live.log`; + const resultPath = `${resultsDir}/live.json`; + const exitCode = Number(process.env.LIVE_EXIT || 1); + const result = readJson(resultPath, {}); + return makeTest({ + name: 'live Vault install smoke', + status: result.status === 'passed' && exitCode === 0 ? 'passed' : 'failed', + durationMs: result.durationMs || 0, + tags: LIVE_RISK_TAGS, + command: 'TEST_LIVE=1 node .e2e/cases/live/run.mjs', + exitCode, + logPath, + resultPath, + sourceFiles: ['.e2e/cases/live/run.mjs'], + extra: { + category: 'live', + checks: result, + }, + }); +} + +const catalog = readJson(catalogPath); +if (!catalog) throw new Error(`failed to read ${catalogPath}`); +const defaultCategories = catalog.defaultCategories || ['unit', 'component', 'contract']; + +const testsSummary = readJson(`${resultsDir}/tests-default.json`, null); + +const tests = [ + buildAndTypecheckTest(), + ...defaultCategories.map((category) => categoryTest(category, testsSummary, defaultCategories)), + productShellTest(), + liveTest(), +]; const summary = { tests: tests.length, passed: tests.filter((test) => test.status === 'passed').length, failed: tests.filter((test) => test.status === 'failed').length, pending: 0, - skipped: 0, - other: 0, + skipped: tests.filter((test) => test.status === 'skipped').length, + other: tests.filter((test) => !['passed', 'failed', 'skipped'].includes(test.status)).length, }; const artifact = { @@ -65,12 +195,12 @@ const artifact = { specVersion: '0.0.0', results: { tool: { - name: 'npm', - version: 'test', + name: 'obsidian-parallel-reader-e2e', + version: '3', }, summary, tests, }, }; -fs.writeFileSync('.e2e/artifact.json', JSON.stringify(artifact, null, 2) + '\n'); +writeFileSync('.e2e/artifact.json', JSON.stringify(artifact, null, 2) + '\n'); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc546f7..4e6554c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,8 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: @@ -20,14 +21,18 @@ jobs: - run: npm ci - - name: Build - run: npm run build - - - name: Typecheck - run: npm run typecheck - - - name: Test - run: npm run test:only - - name: Lint run: npm run lint + + - name: E2E gate + run: timeout 600 npm run e2e + + - name: Upload e2e artifacts on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: e2e-${{ github.run_id }} + if-no-files-found: ignore + path: | + .e2e/artifact.json + .e2e/results/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 09f20bb..cc37e23 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,8 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: @@ -22,14 +23,21 @@ jobs: - run: npm ci - - name: Build - run: npm run build + - name: Lint + run: npm run lint - - name: Typecheck - run: npm run typecheck + - name: E2E gate + run: timeout 600 npm run e2e - - name: Test - run: npm run test:only + - name: Upload e2e artifacts on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: e2e-${{ github.run_id }} + if-no-files-found: ignore + path: | + .e2e/artifact.json + .e2e/results/ - name: Create Release env: diff --git a/.gitignore b/.gitignore index 9408df7..c954fa7 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,8 @@ main.js.map # Agent state .agent/ +.claude/ +.orchestration/ # E2E contract runtime artifacts .e2e/artifact.json diff --git a/biome.json b/biome.json index 345d1f0..0792ef9 100644 --- a/biome.json +++ b/biome.json @@ -1,7 +1,14 @@ { "$schema": "https://biomejs.dev/schemas/2.0.0/schema.json", "files": { - "includes": ["main.ts", "src/**/*.ts", "tests/**/*.js", "scripts/**/*.mjs", "esbuild.config.mjs"] + "includes": [ + "main.ts", + "src/**/*.ts", + "tests/**/*.js", + "scripts/**/*.mjs", + ".e2e/scripts/**/*.mjs", + "esbuild.config.mjs" + ] }, "linter": { "enabled": true, diff --git a/package.json b/package.json index 767924b..d2f98bc 100644 --- a/package.json +++ b/package.json @@ -7,11 +7,17 @@ "build": "node esbuild.config.mjs production", "dev": "node esbuild.config.mjs watch", "typecheck": "tsc --noEmit", - "lint": "biome check main.ts src/ tests/ scripts/", - "lint:fix": "biome check --write main.ts src/ tests/ scripts/", + "lint": "biome check main.ts src/ tests/ scripts/ .e2e/scripts/", + "lint:fix": "biome check --write main.ts src/ tests/ scripts/ .e2e/scripts/", "version": "node scripts/bump-version.mjs", "test": "npm run build && npm run typecheck && node scripts/run-tests.mjs", - "test:only": "node scripts/run-tests.mjs" + "test:only": "node scripts/run-tests.mjs", + "test:unit": "node scripts/run-tests.mjs --category unit", + "test:component": "node scripts/run-tests.mjs --category component", + "test:contract": "node scripts/run-tests.mjs --category contract", + "e2e": "bash .e2e/gate.sh --json", + "test:e2e": "npm run build && node .e2e/cases/product-shell/run.mjs", + "test:live": "npm run build && node .e2e/cases/live/run.mjs" }, "devDependencies": { "@biomejs/biome": "^2.4.13", diff --git a/scripts/run-tests.mjs b/scripts/run-tests.mjs index 524ca04..1a2cd73 100644 --- a/scripts/run-tests.mjs +++ b/scripts/run-tests.mjs @@ -1,24 +1,164 @@ import { execFileSync } from 'child_process'; -import { readdirSync } from 'fs'; +import { existsSync, readdirSync, readFileSync, writeFileSync } from 'fs'; import { join } from 'path'; -const testsDir = join(import.meta.dirname, '..', 'tests'); -const files = readdirSync(testsDir) +const repoRoot = join(import.meta.dirname, '..'); +const testsDir = join(repoRoot, 'tests'); +const catalogPath = join(testsDir, 'catalog.json'); +const catalog = JSON.parse(readFileSync(catalogPath, 'utf8')); +const allTestFiles = readdirSync(testsDir) .filter((f) => f.endsWith('.test.js')) .sort(); -let failed = 0; -for (const file of files) { - const filePath = join(testsDir, file); - try { - execFileSync('node', [filePath], { stdio: 'inherit' }); - } catch { - failed++; +function usage() { + console.error(`Usage: node scripts/run-tests.mjs [--category ...] [--all] [--list] [--json-output ] + +Categories: ${Object.keys(catalog.categories).join(', ')} +Default: ${catalog.defaultCategories.join(', ')}`); +} + +function parseArgs(argv) { + const categories = []; + let all = false; + let list = false; + let jsonOutput = process.env.TEST_RESULTS_JSON || ''; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--all') { + all = true; + } else if (arg === '--list') { + list = true; + } else if (arg === '--category') { + const value = argv[++i]; + if (!value) throw new Error('--category requires a value'); + categories.push(value); + } else if (arg === '--json-output') { + const value = argv[++i]; + if (!value) throw new Error('--json-output requires a path'); + jsonOutput = value; + } else if (catalog.categories[arg]) { + categories.push(arg); + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + + return { + categories: all ? Object.keys(catalog.categories) : categories.length ? categories : catalog.defaultCategories, + list, + jsonOutput, + }; +} + +function validateCatalog() { + const classified = new Map(); + for (const [category, config] of Object.entries(catalog.categories)) { + if (!Array.isArray(config.files)) throw new Error(`catalog category ${category} must define files[]`); + for (const file of config.files) { + if (!file.endsWith('.test.js')) throw new Error(`catalog entry is not a test file: ${category}/${file}`); + if (classified.has(file)) throw new Error(`test file classified twice: ${file}`); + classified.set(file, category); + } + } + + const missing = allTestFiles.filter((file) => !classified.has(file)); + const extra = [...classified.keys()].filter((file) => !allTestFiles.includes(file)); + if (missing.length || extra.length) { + throw new Error( + [ + missing.length ? `unclassified test files: ${missing.join(', ')}` : '', + extra.length ? `catalog references missing files: ${extra.join(', ')}` : '', + ] + .filter(Boolean) + .join('; '), + ); } } -if (failed > 0) { - console.error(`\n${failed} test file(s) failed.`); +function filesForCategories(categories) { + const files = []; + for (const category of categories) { + const config = catalog.categories[category]; + if (!config) throw new Error(`Unknown test category: ${category}`); + for (const file of config.files) files.push({ category, file }); + } + return files; +} + +function runTest(category, file) { + const filePath = join(testsDir, file); + const startedAt = Date.now(); + let exitCode = 0; + try { + execFileSync('node', [filePath], { stdio: 'inherit' }); + } catch (error) { + exitCode = typeof error.status === 'number' ? error.status : 1; + } + const durationMs = Date.now() - startedAt; + return { + category, + file, + status: exitCode === 0 ? 'passed' : 'failed', + exitCode, + durationMs, + }; +} + +function summarize(results, categories) { + const byCategory = {}; + for (const category of categories) { + const categoryResults = results.filter((result) => result.category === category); + byCategory[category] = { + tests: categoryResults.length, + passed: categoryResults.filter((result) => result.status === 'passed').length, + failed: categoryResults.filter((result) => result.status === 'failed').length, + files: categoryResults.map((result) => result.file), + }; + } + return { + catalogVersion: catalog.version, + categories, + total: results.length, + passed: results.filter((result) => result.status === 'passed').length, + failed: results.filter((result) => result.status === 'failed').length, + byCategory, + results, + }; +} + +function main() { + const options = parseArgs(process.argv.slice(2)); + validateCatalog(); + + const selected = filesForCategories(options.categories); + if (options.list) { + for (const { category, file } of selected) console.log(`${category}\t${file}`); + return; + } + + const missingFiles = selected.filter(({ file }) => !existsSync(join(testsDir, file))).map(({ file }) => file); + if (missingFiles.length) throw new Error(`Selected test files do not exist: ${missingFiles.join(', ')}`); + + const results = selected.map(({ category, file }) => runTest(category, file)); + const summary = summarize(results, options.categories); + + if (options.jsonOutput) { + writeFileSync(join(repoRoot, options.jsonOutput), JSON.stringify(summary, null, 2) + '\n'); + } + + if (summary.failed > 0) { + console.error(`\n${summary.failed} test file(s) failed.`); + process.exit(1); + } + + console.log(`\nAll ${summary.total} test files passed across ${options.categories.join(', ')}.`); +} + +try { + main(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + usage(); process.exit(1); } -console.log(`\nAll ${files.length} test files passed.`); diff --git a/tests/catalog.json b/tests/catalog.json new file mode 100644 index 0000000..e588a94 --- /dev/null +++ b/tests/catalog.json @@ -0,0 +1,57 @@ +{ + "version": 1, + "defaultCategories": ["unit", "component", "contract"], + "categories": { + "unit": { + "description": "Pure logic tests with no Obsidian runtime, filesystem, process, or provider boundary.", + "files": [ + "anchor.test.js", + "cards-nav.test.js", + "direct-batch.test.js", + "direct-generation.test.js", + "direct-i18n.test.js", + "direct-markdown.test.js", + "direct-prompt.test.js", + "direct-settings-extra.test.js", + "direct-settings.test.js", + "direct-streaming.test.js", + "i18n.test.js", + "markdown.test.js", + "schema.test.js", + "scroll.test.js", + "settings.test.js", + "streaming.test.js", + "vault-batch.test.js" + ] + }, + "component": { + "description": "Tests that exercise plugin components with controlled adapters, job managers, or Obsidian shims.", + "files": [ + "cache.test.js", + "direct-cache.test.js", + "generation-job-manager.test.js", + "plugin-batch.test.js" + ] + }, + "contract": { + "description": "Tests for provider protocols, CLI command contracts, exported test surfaces, and architecture invariants.", + "files": [ + "architecture.test.js", + "cli.test.js", + "direct-providers.test.js", + "providers.test.js", + "test-exports.test.js" + ] + } + }, + "externalCategories": { + "e2e": { + "description": "Headless product-shell smoke using the built plugin package and a disposable Vault filesystem.", + "command": "npm run test:e2e" + }, + "live": { + "description": "Opt-in check against a real local Vault plugin install. Does not launch Obsidian or call a provider.", + "command": "TEST_LIVE=1 npm run test:live" + } + } +}