From 2e97525d4f9ba4cac00235b26b8e99b8a44ca52f Mon Sep 17 00:00:00 2001 From: murashit Date: Tue, 14 Jul 2026 21:15:21 +0900 Subject: [PATCH] build: verify app-server binding provenance --- .github/workflows/check.yml | 10 +- .github/workflows/release.yml | 9 ++ README.md | 2 +- package.json | 1 + scripts/api-baseline.mjs | 89 ++++++----- scripts/app-server-compatibility.mjs | 49 ++++++ scripts/generate-app-server-types.mjs | 76 ++++++++- scripts/release/preflight.mjs | 1 + src/app-server/connection/client-profile.ts | 3 +- src/app-server/connection/compatibility.json | 3 +- tests/scripts/development-scripts.test.ts | 159 +++++++++++++++++-- 11 files changed, 339 insertions(+), 63 deletions(-) create mode 100644 scripts/app-server-compatibility.mjs diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2693162f..8a3bd9ed 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -30,8 +30,14 @@ jobs: - name: Install dependencies run: npm ci --ignore-scripts - - name: Check recorded API baselines - run: npm run api:baseline -- --recorded-only + - name: Install recorded Codex CLI + run: npm install --global "@openai/codex@$(node scripts/app-server-compatibility.mjs --tested-cli-version)" + + - name: Check API baselines + run: npm run api:baseline + + - name: Check generated app-server bindings + run: npm run generate:app-server-types:check - name: Check pull request commit messages if: github.event_name == 'pull_request' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 93d41761..103214a7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,6 +31,15 @@ jobs: - name: Install dependencies run: npm ci --ignore-scripts + - name: Install recorded Codex CLI + run: npm install --global "@openai/codex@$(node scripts/app-server-compatibility.mjs --tested-cli-version)" + + - name: Check API baselines + run: npm run api:baseline + + - name: Check generated app-server bindings + run: npm run generate:app-server-types:check + - name: Check release version run: npm run release:check env: diff --git a/README.md b/README.md index e935e5a6..112ab9d2 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Threads can be archived as Markdown notes with a configurable folder, filename t | ------------------------ | --------- | --------------------------------------------------------------------------------------------------- | | `manifest.minAppVersion` | `1.12.0` | Minimum Obsidian desktop version declared for plugin loading. | | `obsidian` API types | `1.12.3` | TypeScript API package used for compile-time checks; kept in the same minor as `manifest` baseline. | -| `codex.testedCliVersion` | `0.144.0` | Track app-server compatibility by Codex CLI minor version. | +| `codex.testedCliVersion` | `0.144.4` | Exact CLI patch used to generate and verify bindings; compatibility is tracked by minor version. | Codex Panel depends on the experimental `codex app-server` API. diff --git a/package.json b/package.json index c1ea4fc3..f2af30ba 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "check:unused": "knip --no-progress", "fix": "biome check --fix --skip=plugin --diagnostic-level=warn --error-on-warnings && knip --fix --no-progress", "generate:app-server-types": "node scripts/generate-app-server-types.mjs", + "generate:app-server-types:check": "node scripts/generate-app-server-types.mjs --check", "release:check": "node scripts/release/check.mjs", "release:notes": "node scripts/release/notes.mjs", "release:preflight": "node scripts/release/preflight.mjs", diff --git a/scripts/api-baseline.mjs b/scripts/api-baseline.mjs index 5f9ee338..d83b62ce 100644 --- a/scripts/api-baseline.mjs +++ b/scripts/api-baseline.mjs @@ -3,20 +3,19 @@ import { readFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; -const validArgs = new Set(["--json", "--recorded-only"]); +const validArgs = new Set(["--json"]); if (isMain()) { const args = new Set(process.argv.slice(2)); const asJson = args.has("--json"); - const recordedOnly = args.has("--recorded-only"); for (const arg of args) { if (!validArgs.has(arg)) { - console.error("Usage: node scripts/api-baseline.mjs [--json] [--recorded-only]"); + console.error("Usage: node scripts/api-baseline.mjs [--json]"); process.exit(1); } } - const report = await createApiBaselineReport({ skipLocalCodex: recordedOnly }); + const report = await createApiBaselineReport(); if (asJson) { console.log(JSON.stringify(report, null, 2)); } else { @@ -60,12 +59,10 @@ export async function createApiBaselineReport(options = {}) { const readmeBaselines = readCompatibilityBaselines(inputs.readme); const codexReadmeVersion = readmeBaselines.codexTestedCliVersion; - const codexLocalVersion = options.skipLocalCodex - ? null - : options.readCodexVersion - ? await options.readCodexVersion() - : readCodexVersion(); + const codexRecordedVersion = inputs.appServerCompatibilityJson.codexAppServer?.testedCliVersion ?? null; + const codexLocalVersion = options.readCodexVersion ? await options.readCodexVersion() : readCodexVersion(); const codexReadmeSemver = parseSemver(codexReadmeVersion); + const codexRecordedSemver = parseSemver(codexRecordedVersion); const codexLocalSemver = parseSemver(codexLocalVersion); const obsidianMinVersion = inputs.manifestJson.minAppVersion; @@ -78,11 +75,12 @@ export async function createApiBaselineReport(options = {}) { const obsidianLockSemver = parseSemver(obsidianLockVersion); const obsidianMinSemver = parseSemver(obsidianMinVersion); - const appServerGenerationExperimentalDeclared = inputs.appServerCompatibilityJson.codexAppServer?.typeGeneration?.experimental === true; - const appServerGenerationExperimental = - inputs.appServerGenerateSource.includes("app-server") && - inputs.appServerGenerateSource.includes("generate-ts") && - inputs.appServerGenerateSource.includes("--experimental"); + const generationArguments = inputs.appServerCompatibilityJson.codexAppServer?.typeGeneration?.arguments ?? null; + const expectedGenerationArguments = ["app-server", "generate-ts", "--experimental"]; + const appServerGenerationArgumentsDeclared = + Array.isArray(generationArguments) && generationArguments.every((argument) => typeof argument === "string"); + const appServerGenerationArgumentsSupported = + appServerGenerationArgumentsDeclared && arraysEqual(generationArguments, expectedGenerationArguments); const initializeCapabilities = inputs.appServerCompatibilityJson.codexAppServer?.initialize?.capabilities ?? {}; const initializeExperimentalApi = initializeCapabilities.experimentalApi === true; const initializeRequestAttestationDisabled = initializeCapabilities.requestAttestation === false; @@ -90,18 +88,22 @@ export async function createApiBaselineReport(options = {}) { if (!codexReadmeSemver) { fail("README.md Compatibility table must define `codex.testedCliVersion` as X.Y.Z."); } - if (!options.skipLocalCodex && !codexLocalSemver) { + if (!codexRecordedSemver || codexRecordedSemver.version !== codexRecordedVersion) { + fail("src/app-server/connection/compatibility.json must declare codexAppServer.testedCliVersion as X.Y.Z."); + } + if (!codexLocalSemver) { fail("local codex --version could not be read."); } - if (codexReadmeSemver && codexLocalSemver && minorKey(codexReadmeSemver) !== minorKey(codexLocalSemver)) { - fail(`local Codex CLI minor ${minorKey(codexLocalSemver)} does not match compatibility table minor ${minorKey(codexReadmeSemver)}.`); + if (codexReadmeVersion && codexRecordedVersion && codexReadmeVersion !== codexRecordedVersion) { + fail(`README Codex CLI ${codexReadmeVersion} does not match recorded tested CLI ${codexRecordedVersion}.`); } - if (!appServerGenerationExperimentalDeclared) { - fail("src/app-server/connection/compatibility.json must declare codexAppServer.typeGeneration.experimental: true."); + if (codexLocalSemver && codexRecordedSemver && codexLocalSemver.version !== codexRecordedSemver.version) { + fail(`local Codex CLI ${codexLocalSemver.version} does not match recorded tested CLI ${codexRecordedSemver.version}.`); } - if (!appServerGenerationExperimental) fail("generate:app-server-types must use codex app-server generate-ts --experimental."); - if (appServerGenerationExperimentalDeclared !== appServerGenerationExperimental) { - fail("app-server type generation declaration must match scripts/generate-app-server-types.mjs."); + if (!appServerGenerationArgumentsDeclared) { + fail("src/app-server/connection/compatibility.json must declare codexAppServer.typeGeneration.arguments as a string array."); + } else if (!appServerGenerationArgumentsSupported) { + fail(`app-server type generation arguments must be ${expectedGenerationArguments.join(" ")}.`); } if (!initializeExperimentalApi) { fail("src/app-server/connection/compatibility.json must declare codexAppServer.initialize.capabilities.experimentalApi: true."); @@ -143,15 +145,17 @@ export async function createApiBaselineReport(options = {}) { return { codex: { - policy: "managed by minor version", + policy: "compatibility is managed by minor; generated bindings are proven against an exact CLI patch", + recordedTestedCliVersion: codexRecordedVersion, readmeTestedCliVersion: codexReadmeVersion, readmeTestedMinor: minorKey(codexReadmeSemver), localCliVersion: codexLocalVersion, localCliMinor: minorKey(codexLocalSemver), - localCliCheckSkipped: options.skipLocalCodex === true, - localCliMatchesTestedMinor: codexReadmeSemver && codexLocalSemver ? minorKey(codexReadmeSemver) === minorKey(codexLocalSemver) : null, - appServerGenerationExperimentalDeclared, - appServerGenerationExperimental, + readmeMatchesRecordedVersion: codexReadmeVersion && codexRecordedVersion ? codexReadmeVersion === codexRecordedVersion : false, + localCliMatchesRecordedVersion: + codexLocalSemver && codexRecordedSemver ? codexLocalSemver.version === codexRecordedSemver.version : null, + generationArguments, + appServerGenerationArgumentsSupported, initializeExperimentalApi, initializeRequestAttestationDisabled, }, @@ -243,21 +247,22 @@ function displayValue(value) { return value ?? "(missing)"; } +function arraysEqual(left, right) { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + async function readBaselineInputs(cwd) { - const [packageJson, packageLockJson, manifestJson, versionsJson, readme, appServerCompatibilityJson, appServerGenerateSource] = - await Promise.all([ - readJson(cwd, "package.json"), - readJson(cwd, "package-lock.json"), - readJson(cwd, "manifest.json"), - readJson(cwd, "versions.json"), - readFile(path.join(cwd, "README.md"), "utf8"), - readJson(cwd, "src/app-server/connection/compatibility.json"), - readFile(path.join(cwd, "scripts/generate-app-server-types.mjs"), "utf8"), - ]); + const [packageJson, packageLockJson, manifestJson, versionsJson, readme, appServerCompatibilityJson] = await Promise.all([ + readJson(cwd, "package.json"), + readJson(cwd, "package-lock.json"), + readJson(cwd, "manifest.json"), + readJson(cwd, "versions.json"), + readFile(path.join(cwd, "README.md"), "utf8"), + readJson(cwd, "src/app-server/connection/compatibility.json"), + ]); return { appServerCompatibilityJson, - appServerGenerateSource, manifestJson, packageJson, packageLockJson, @@ -280,11 +285,11 @@ function printReport(report) { console.log("Codex app-server"); console.log(` policy: ${report.codex.policy}`); console.log(` compatibility table CLI: ${displayValue(report.codex.readmeTestedCliVersion)}`); + console.log(` recorded tested CLI: ${displayValue(report.codex.recordedTestedCliVersion)}`); console.log(` compatibility table minor: ${displayValue(report.codex.readmeTestedMinor)}`); - console.log(` local codex CLI: ${report.codex.localCliCheckSkipped ? "(skipped)" : displayValue(report.codex.localCliVersion)}`); - console.log(` local codex minor: ${report.codex.localCliCheckSkipped ? "(skipped)" : displayValue(report.codex.localCliMinor)}`); - console.log(` declared generate-ts --experimental: ${report.codex.appServerGenerationExperimentalDeclared ? "yes" : "no"}`); - console.log(` generate-ts --experimental: ${report.codex.appServerGenerationExperimental ? "yes" : "no"}`); + console.log(` local codex CLI: ${displayValue(report.codex.localCliVersion)}`); + console.log(` local codex minor: ${displayValue(report.codex.localCliMinor)}`); + console.log(` generation arguments: ${report.codex.generationArguments?.join(" ") ?? "(missing)"}`); console.log(` initialize experimentalApi: ${report.codex.initializeExperimentalApi ? "yes" : "no"}`); console.log(` initialize requestAttestation disabled: ${report.codex.initializeRequestAttestationDisabled ? "yes" : "no"}`); console.log(""); diff --git a/scripts/app-server-compatibility.mjs b/scripts/app-server-compatibility.mjs new file mode 100644 index 00000000..509bdf08 --- /dev/null +++ b/scripts/app-server-compatibility.mjs @@ -0,0 +1,49 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const compatibilityRelativePath = "src/app-server/connection/compatibility.json"; +const exactSemverPattern = /^\d+\.\d+\.\d+$/; + +if (isMain()) { + const args = process.argv.slice(2); + if (args.length !== 1 || args[0] !== "--tested-cli-version") { + console.error("Usage: node scripts/app-server-compatibility.mjs --tested-cli-version"); + process.exit(1); + } + + try { + const policy = await readAppServerGenerationPolicy(process.cwd()); + console.log(policy.testedCliVersion); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} + +export async function readAppServerGenerationPolicy(cwd) { + const compatibilityPath = path.join(cwd, compatibilityRelativePath); + const compatibility = JSON.parse(await readFile(compatibilityPath, "utf8")); + const testedCliVersion = compatibility.codexAppServer?.testedCliVersion; + const generationArguments = compatibility.codexAppServer?.typeGeneration?.arguments; + + if (typeof testedCliVersion !== "string" || !exactSemverPattern.test(testedCliVersion)) { + throw new Error(`${compatibilityRelativePath} must declare codexAppServer.testedCliVersion as X.Y.Z.`); + } + if (!isStringArray(generationArguments) || generationArguments.length === 0) { + throw new Error(`${compatibilityRelativePath} must declare codexAppServer.typeGeneration.arguments as a non-empty string array.`); + } + if (generationArguments.includes("--out")) { + throw new Error(`${compatibilityRelativePath} must not include --out in codexAppServer.typeGeneration.arguments.`); + } + + return { generationArguments, testedCliVersion }; +} + +function isStringArray(value) { + return Array.isArray(value) && value.every((entry) => typeof entry === "string" && entry.length > 0); +} + +function isMain() { + return process.argv[1] ? path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) : false; +} diff --git a/scripts/generate-app-server-types.mjs b/scripts/generate-app-server-types.mjs index 16fb405c..179ae98b 100644 --- a/scripts/generate-app-server-types.mjs +++ b/scripts/generate-app-server-types.mjs @@ -2,6 +2,7 @@ import { spawnSync } from "node:child_process"; import { mkdir, mkdtemp, readdir, readFile, rename, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { readAppServerGenerationPolicy } from "./app-server-compatibility.mjs"; const generatedRelativeDir = "src/generated/app-server"; const generatedHeader = "// GENERATED CODE! DO NOT MODIFY BY HAND!"; @@ -9,7 +10,11 @@ const normalizationNotice = "// This file was mechanically normalized after gene if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { try { - await generateAppServerTypes(); + const args = process.argv.slice(2); + if (args.some((arg) => arg !== "--check") || args.length > 1) { + throw new Error("Usage: node scripts/generate-app-server-types.mjs [--check]"); + } + await generateAppServerTypes({ check: args.includes("--check") }); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; @@ -20,19 +25,55 @@ export async function generateAppServerTypes(options = {}) { const cwd = options.cwd ?? process.cwd(); const generatedDir = path.resolve(cwd, generatedRelativeDir); const runCommand = options.runCommand ?? run; + const readCodexVersion = options.readCodexVersion ?? readInstalledCodexVersion; + const policy = await readAppServerGenerationPolicy(cwd); + const installedCliVersion = await readCodexVersion(cwd); + if (installedCliVersion !== policy.testedCliVersion) { + throw new Error( + `Codex CLI ${policy.testedCliVersion} is required to generate app-server bindings; found ${installedCliVersion ?? "an unreadable version"}.`, + ); + } const generatedParent = path.dirname(generatedDir); await mkdir(generatedParent, { recursive: true }); const stagedDir = await mkdtemp(path.join(generatedParent, ".app-server-")); try { const stagedRelativeDir = path.relative(cwd, stagedDir); - await runCommand("codex", ["app-server", "generate-ts", "--experimental", "--out", stagedRelativeDir], { cwd }); + await runCommand("codex", [...policy.generationArguments, "--out", stagedRelativeDir], { cwd }); await normalizeGeneratedTypes(stagedDir); - await replaceGeneratedTypes(generatedDir, stagedDir); + if (options.check === true) { + const differences = await compareGeneratedTrees(generatedDir, stagedDir); + if (differences.length > 0) { + throw new Error(`generated app-server bindings are out of date:\n${differences.map((difference) => ` ${difference}`).join("\n")}`); + } + } else { + await replaceGeneratedTypes(generatedDir, stagedDir); + } } finally { await rm(stagedDir, { recursive: true, force: true }); } } +async function compareGeneratedTrees(trackedDir, stagedDir) { + const [trackedFiles, stagedFiles] = await Promise.all([listFiles(trackedDir), listFiles(stagedDir)]); + const trackedSet = new Set(trackedFiles); + const stagedSet = new Set(stagedFiles); + const differences = []; + + for (const file of stagedFiles) { + if (!trackedSet.has(file)) { + differences.push(`added: ${file}`); + continue; + } + const [trackedSource, stagedSource] = await Promise.all([readFile(path.join(trackedDir, file)), readFile(path.join(stagedDir, file))]); + if (!trackedSource.equals(stagedSource)) differences.push(`changed: ${file}`); + } + for (const file of trackedFiles) { + if (!stagedSet.has(file)) differences.push(`removed: ${file}`); + } + + return differences; +} + async function replaceGeneratedTypes(generatedDir, stagedDir) { const backupDir = `${generatedDir}.backup-${process.pid.toString()}-${Date.now().toString()}`; let hasBackup = false; @@ -83,6 +124,24 @@ async function listTypeScriptFiles(dir) { return files.flat(); } +async function listFiles(dir, relativeDir = "") { + let entries; + try { + entries = await readdir(path.join(dir, relativeDir), { withFileTypes: true }); + } catch (error) { + if (isMissingPathError(error)) return []; + throw error; + } + const files = await Promise.all( + entries.map((entry) => { + const relativePath = path.join(relativeDir, entry.name); + if (entry.isDirectory()) return listFiles(dir, relativePath); + return entry.isFile() ? [relativePath.replaceAll(path.sep, "/")] : []; + }), + ); + return files.flat().sort(); +} + function normalizeSource(source) { let normalized = source; do { @@ -109,3 +168,14 @@ function run(command, args, options = {}) { } if (result.status !== 0) throw new Error(`${command} ${args.join(" ")} exited with status ${(result.status ?? 1).toString()}.`); } + +function readInstalledCodexVersion(cwd) { + const result = spawnSync("codex", ["--version"], { + cwd, + encoding: "utf8", + stdio: "pipe", + shell: false, + }); + if (result.error || result.status !== 0) return null; + return `${result.stdout}\n${result.stderr}`.match(/\b\d+\.\d+\.\d+\b/)?.[0] ?? null; +} diff --git a/scripts/release/preflight.mjs b/scripts/release/preflight.mjs index 60668091..4e7e3b1d 100644 --- a/scripts/release/preflight.mjs +++ b/scripts/release/preflight.mjs @@ -89,6 +89,7 @@ run("git", ["rev-parse", "--verify", `refs/tags/${previousTag}`], { capture: tru run("npm", ["run", "commitlint", "--", "--from", previousTag, "--to", "main", "--verbose"]); run("npm", ["run", "release:check"]); run("npm", ["run", "api:baseline"]); +run("npm", ["run", "generate:app-server-types:check"]); run("npm", ["ci", "--dry-run"]); run("npm", ["run", "check"]); diff --git a/src/app-server/connection/client-profile.ts b/src/app-server/connection/client-profile.ts index c8d3ce84..69ac0ba9 100644 --- a/src/app-server/connection/client-profile.ts +++ b/src/app-server/connection/client-profile.ts @@ -5,8 +5,9 @@ import compatibility from "./compatibility.json"; interface AppServerCompatibility { codexAppServer: { + testedCliVersion: string; typeGeneration: { - experimental: boolean; + arguments: string[]; }; initialize: { capabilities: Pick; diff --git a/src/app-server/connection/compatibility.json b/src/app-server/connection/compatibility.json index 51fe03b7..a8c20e68 100644 --- a/src/app-server/connection/compatibility.json +++ b/src/app-server/connection/compatibility.json @@ -1,7 +1,8 @@ { "codexAppServer": { + "testedCliVersion": "0.144.4", "typeGeneration": { - "experimental": true + "arguments": ["app-server", "generate-ts", "--experimental"] }, "initialize": { "capabilities": { diff --git a/tests/scripts/development-scripts.test.ts b/tests/scripts/development-scripts.test.ts index 1c56d357..9e73b680 100644 --- a/tests/scripts/development-scripts.test.ts +++ b/tests/scripts/development-scripts.test.ts @@ -39,8 +39,11 @@ describe("development scripts", () => { const calls: { command: string; args: string[]; cwd: string }[] = []; const { generateAppServerTypes } = await import(pathToFileURL(path.join(repoRoot, "scripts", "generate-app-server-types.mjs")).href); + await writeAppServerCompatibility(cwd, "0.144.4", ["app-server", "generate-ts", "--experimental", "--test-setting"]); + await generateAppServerTypes({ cwd, + readCodexVersion: () => "0.144.4", async runCommand(command: string, args: string[], options: { cwd: string }) { calls.push({ command, args, cwd: options.cwd }); const outIndex = args.indexOf("--out"); @@ -56,8 +59,8 @@ describe("development scripts", () => { expect(calls).toHaveLength(1); expect(calls[0]).toMatchObject({ command: "codex", cwd }); - expect(calls[0]?.args.slice(0, 4)).toEqual(["app-server", "generate-ts", "--experimental", "--out"]); - expect(calls[0]?.args[4]?.replaceAll("\\", "/")).toMatch(/^src\/generated\/\.app-server-/); + expect(calls[0]?.args.slice(0, 5)).toEqual(["app-server", "generate-ts", "--experimental", "--test-setting", "--out"]); + expect(calls[0]?.args[5]?.replaceAll("\\", "/")).toMatch(/^src\/generated\/\.app-server-/); await expect(readFile(path.join(cwd, "src", "generated", "app-server", "v2", "Example.ts"), "utf8")).resolves.toContain( "export type Example = string | null;", ); @@ -68,11 +71,13 @@ describe("development scripts", () => { const generatedDir = path.join(cwd, "src", "generated", "app-server"); await mkdir(generatedDir, { recursive: true }); await writeFile(path.join(generatedDir, "Existing.ts"), "export type Existing = true;\n"); + await writeAppServerCompatibility(cwd, "0.144.4"); const { generateAppServerTypes } = await import(pathToFileURL(path.join(repoRoot, "scripts", "generate-app-server-types.mjs")).href); await expect( generateAppServerTypes({ cwd, + readCodexVersion: () => "0.144.4", runCommand: async () => { throw new Error("generation failed"); }, @@ -83,9 +88,61 @@ describe("development scripts", () => { await expect(readdir(path.join(cwd, "src", "generated"))).resolves.toEqual(["app-server"]); }); + it("checks normalized generated bindings without replacing the tracked tree", async () => { + const cwd = await tempWorkspace(); + const generatedDir = path.join(cwd, "src", "generated", "app-server"); + await mkdir(path.join(generatedDir, "v2"), { recursive: true }); + await writeFile( + path.join(generatedDir, "v2", "Example.ts"), + "// GENERATED CODE! DO NOT MODIFY BY HAND!\n// This file was mechanically normalized after generation by scripts/generate-app-server-types.mjs.\nexport type Example = string | null;\n", + ); + await writeAppServerCompatibility(cwd, "0.144.4"); + const { generateAppServerTypes } = await import(pathToFileURL(path.join(repoRoot, "scripts", "generate-app-server-types.mjs")).href); + + const generate = async (_command: string, args: string[], options: { cwd: string }) => { + const outputDir = args[args.indexOf("--out") + 1]; + if (!outputDir) throw new Error("Missing generated output directory"); + await mkdir(path.join(options.cwd, outputDir, "v2"), { recursive: true }); + await writeFile( + path.join(options.cwd, outputDir, "v2", "Example.ts"), + "// GENERATED CODE! DO NOT MODIFY BY HAND!\nexport type Example = string | null | null;\n", + ); + }; + + await expect( + generateAppServerTypes({ cwd, check: true, readCodexVersion: () => "0.144.4", runCommand: generate }), + ).resolves.toBeUndefined(); + + const trackedSource = await readFile(path.join(generatedDir, "v2", "Example.ts"), "utf8"); + await expect( + generateAppServerTypes({ + cwd, + check: true, + readCodexVersion: () => "0.144.4", + async runCommand(command: string, args: string[], options: { cwd: string }) { + await generate(command, args, options); + const outputDir = args[args.indexOf("--out") + 1]; + if (!outputDir) throw new Error("Missing generated output directory"); + await writeFile(path.join(options.cwd, outputDir, "v2", "Extra.ts"), "export type Extra = true;\n"); + }, + }), + ).rejects.toThrow("generated app-server bindings are out of date:\n added: v2/Extra.ts"); + await expect(readFile(path.join(generatedDir, "v2", "Example.ts"), "utf8")).resolves.toBe(trackedSource); + await expect(readdir(path.join(cwd, "src", "generated"))).resolves.toEqual(["app-server"]); + }); + + it("refuses to generate bindings with a different Codex CLI patch", async () => { + const cwd = await tempWorkspace(); + await writeAppServerCompatibility(cwd, "0.144.4"); + const { generateAppServerTypes } = await import(pathToFileURL(path.join(repoRoot, "scripts", "generate-app-server-types.mjs")).href); + + await expect(generateAppServerTypes({ cwd, readCodexVersion: () => "0.144.5", runCommand: async () => undefined })).rejects.toThrow( + "Codex CLI 0.144.4 is required to generate app-server bindings; found 0.144.5.", + ); + }); + it("reads app-server compatibility policy from the declared baseline in API baseline checks", async () => { const cwd = await tempWorkspace(); - await mkdir(path.join(cwd, "scripts"), { recursive: true }); await mkdir(path.join(cwd, "src", "app-server"), { recursive: true }); const { createApiBaselineReport } = await import(pathToFileURL(path.join(repoRoot, "scripts", "api-baseline.mjs")).href); @@ -120,15 +177,12 @@ describe("development scripts", () => { "| `obsidian` API types | `1.12.3` | Compile-time API package. |", ].join("\n"), ); - await writeFile( - path.join(cwd, "scripts", "generate-app-server-types.mjs"), - 'run("codex", ["app-server", "generate-ts", "--experimental"]);\n', - ); await mkdir(path.join(cwd, "src", "app-server", "connection"), { recursive: true }); await writeJson(path.join(cwd, "src", "app-server", "connection", "compatibility.json"), { codexAppServer: { + testedCliVersion: "0.139.0", typeGeneration: { - experimental: true, + arguments: ["app-server", "generate-ts", "--experimental"], }, initialize: { capabilities: { @@ -144,18 +198,51 @@ describe("development scripts", () => { readCodexVersion: () => "0.139.0", }); - expect(report.codex.appServerGenerationExperimentalDeclared).toBe(true); + expect(report.codex.recordedTestedCliVersion).toBe("0.139.0"); + expect(report.codex.generationArguments).toEqual(["app-server", "generate-ts", "--experimental"]); + expect(report.codex.readmeMatchesRecordedVersion).toBe(true); + expect(report.codex.localCliMatchesRecordedVersion).toBe(true); expect(report.codex.initializeExperimentalApi).toBe(true); expect(report.codex.initializeRequestAttestationDisabled).toBe(true); expect(report.failures).toEqual([]); - const recordedOnlyReport = await createApiBaselineReport({ + const missingLocalCliReport = await createApiBaselineReport({ cwd, readCodexVersion: () => null, - skipLocalCodex: true, }); - expect(recordedOnlyReport.codex.localCliCheckSkipped).toBe(true); - expect(recordedOnlyReport.failures).toEqual([]); + expect(missingLocalCliReport.failures).toContain("local codex --version could not be read."); + }); + + it("rejects the obsolete recorded-only API baseline mode", () => { + const result = runNodeScript("scripts/api-baseline.mjs", ["--recorded-only"], repoRoot); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("Usage: node scripts/api-baseline.mjs [--json]"); + }); + + it("reports app-server provenance drift between the recorded, README, and local CLI versions", async () => { + const cwd = await apiBaselineFixture({ recordedCodexVersion: "0.144.4", readmeCodexVersion: "0.144.0" }); + const { createApiBaselineReport } = await import(pathToFileURL(path.join(repoRoot, "scripts", "api-baseline.mjs")).href); + + const report = await createApiBaselineReport({ cwd, readCodexVersion: () => "0.144.5" }); + + expect(report.failures).toContain("README Codex CLI 0.144.0 does not match recorded tested CLI 0.144.4."); + expect(report.failures).toContain("local Codex CLI 0.144.5 does not match recorded tested CLI 0.144.4."); + }); + + it("checks generated app-server bindings in CI and release paths", async () => { + const [checkWorkflow, releaseWorkflow, releasePreflight] = await Promise.all([ + readFile(path.join(repoRoot, ".github", "workflows", "check.yml"), "utf8"), + readFile(path.join(repoRoot, ".github", "workflows", "release.yml"), "utf8"), + readFile(path.join(repoRoot, "scripts", "release", "preflight.mjs"), "utf8"), + ]); + + for (const workflow of [checkWorkflow, releaseWorkflow]) { + expect(workflow).toContain("npm install --global"); + expect(workflow).toContain("node scripts/app-server-compatibility.mjs --tested-cli-version"); + expect(workflow).toContain("npm run generate:app-server-types:check"); + } + expect(releasePreflight).toContain('run("npm", ["run", "generate:app-server-types:check"]);'); }); it("reports representative CSS usage policy failures", async () => { @@ -252,6 +339,52 @@ async function styleOrderFixture(): Promise { return cwd; } +async function apiBaselineFixture(options: { recordedCodexVersion: string; readmeCodexVersion: string }): Promise { + const cwd = await tempWorkspace(); + await writeJson(path.join(cwd, "package.json"), { version: "1.0.0", devDependencies: { obsidian: "~1.12.3" } }); + await writeJson(path.join(cwd, "package-lock.json"), { + packages: { "node_modules/obsidian": { version: "1.12.3" } }, + }); + await writeJson(path.join(cwd, "manifest.json"), { minAppVersion: "1.12.0" }); + await writeJson(path.join(cwd, "versions.json"), { "1.0.0": "1.12.0" }); + await writeFile( + path.join(cwd, "README.md"), + [ + "## Compatibility", + "", + "| Key | Version | Notes |", + "| --- | --- | --- |", + `| \`codex.testedCliVersion\` | \`${options.readmeCodexVersion}\` | Tested CLI. |`, + "| `manifest.minAppVersion` | `1.12.0` | Minimum app version. |", + "| `obsidian` API types | `1.12.3` | Compile-time API package. |", + ].join("\n"), + ); + await writeAppServerCompatibility(cwd, options.recordedCodexVersion); + return cwd; +} + +async function writeAppServerCompatibility( + cwd: string, + testedCliVersion: string, + generationArguments = ["app-server", "generate-ts", "--experimental"], +): Promise { + await mkdir(path.join(cwd, "src", "app-server", "connection"), { recursive: true }); + await writeJson(path.join(cwd, "src", "app-server", "connection", "compatibility.json"), { + codexAppServer: { + testedCliVersion, + typeGeneration: { + arguments: generationArguments, + }, + initialize: { + capabilities: { + experimentalApi: true, + requestAttestation: false, + }, + }, + }, + }); +} + async function cssUsageFixture(files: Record): Promise { const cwd = await tempWorkspace(); await mkdir(path.join(cwd, "src", "styles"), { recursive: true });