From d170b5f5b3e50d2952b823a72fbaf531da6d05a1 Mon Sep 17 00:00:00 2001 From: alexcd Date: Thu, 18 Jun 2026 15:21:16 +0200 Subject: [PATCH] fix: harden write tools release readiness --- README.md | 8 ++ packages/mcp-server/src/config.test.ts | 79 ++++++++++----- packages/mcp-server/src/config.ts | 12 ++- packages/mcp-server/src/database.test.ts | 35 ++++++- packages/mcp-server/src/database.ts | 26 ++++- packages/mcp-server/src/index.ts | 2 +- packages/mcp-server/src/indexer.test.ts | 34 ++++++- packages/mcp-server/src/indexer.ts | 26 ++++- .../mcp-server/src/mcp-write-tools.test.ts | 46 ++++++++- packages/mcp-server/src/mcp.test.ts | 31 +++++- packages/mcp-server/src/mcp.ts | 66 ++++++++++++- packages/plugin/src/server.test.ts | 98 +++++++++++++++++++ packages/plugin/src/server.ts | 9 +- packages/plugin/src/settings.ts | 93 ++++++++++++++++++ packages/plugin/src/test-obsidian.ts | 25 +++++ packages/shared/src/index.ts | 1 + packages/shared/src/prune.ts | 74 ++++++++++++++ packages/shared/src/types.ts | 10 ++ vitest.config.ts | 1 + 19 files changed, 631 insertions(+), 45 deletions(-) create mode 100644 packages/plugin/src/server.test.ts create mode 100644 packages/plugin/src/test-obsidian.ts create mode 100644 packages/shared/src/prune.ts diff --git a/README.md b/README.md index 74b831e..c017884 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,10 @@ Add the same config to LM Studio's `mcp.json`. Without embeddings, vault search Embeddings are optional. Without them, search uses SQLite full-text search: it is local, fast, and good for exact words, titles, tags, and phrases. With embeddings enabled, the MCP server can also do semantic/vector search, so queries like “notes about long-term planning” can find notes that use different wording. +Embeddings are cached in the local SQLite index by note chunk content hash. When notes are edited, rewritten, deleted, or later excluded from the vault scope, old chunk vectors can become stale cache data. Pruning removes vectors that are no longer referenced by current indexed chunks; it does not change your Obsidian notes. + +Most users do not need to prune manually because **Auto-prune embeddings** is enabled by default in the plugin settings and runs after note writes and `refresh_index`. Manual pruning is useful if you disable auto-prune, change exclusions, troubleshoot a large `index.sqlite`, rebuild/delete index data, or see orphaned embeddings reported by `index_status`. You can trigger it from Obsidian with **Advanced settings → Prune now**, or from an MCP client by asking it to run `prune_embeddings`. + The plugin/MCP server does not automatically know which embedding model LM Studio loaded. The model value must match the identifier exposed by LM Studio's embeddings endpoint. If no embedding model is loaded, search still works through SQLite full-text search, but semantic search and `related_notes` are unavailable. For LM Studio, load an embedding model such as `nomic-embed-text-v1.5`, start the local server, then add these env vars to the same MCP config: @@ -168,6 +172,8 @@ When you add or edit notes later, ask your MCP client to refresh the index: This calls `refresh_index`, updates SQLite, and refreshes embeddings too if they are enabled. +The SQLite file is a rebuildable cache, not the source of truth. If `index.sqlite` is deleted, run `refresh_index` to recreate it; embeddings may need to be recomputed. + ## MCP Tools | Tool | Description | @@ -175,6 +181,7 @@ This calls `refresh_index`, updates SQLite, and refreshes embeddings too if they | `vault_status` | Plugin version, vault name, included note count, scope summary | | `refresh_index` | Rebuild the SQLite index (and embeddings if enabled) | | `index_status` | Index freshness, row counts, embedding state | +| `prune_embeddings` | Remove stale cached embedding vectors no longer used by indexed note chunks | | `ask_vault` | Default tool for natural vault questions; uses embeddings automatically when available | | `analyze_vault` | Find candidate notes/snippets for themes, patterns, and vault-wide synthesis | | `list_notes` | Paginated list of included notes with metadata | @@ -199,6 +206,7 @@ All tools return JSON text. Note content is marked untrusted so hosts do not tre | `OBSIDIAN_MCP_TOKEN` | — | Required. Bearer token from the plugin settings | | `OBSIDIAN_MCP_DB` | plugin folder `index.sqlite` | Override the SQLite cache path | | `OBSIDIAN_MCP_MAX_RESULTS` | `20` | Default result cap for list/search tools | +| `OBSIDIAN_MCP_AUTO_PRUNE_EMBEDDINGS` | plugin setting, usually `on` | Override automatic stale embedding pruning | | `OBSIDIAN_MCP_EMBEDDINGS` | `off` | Set to `on` to enable semantic search | | `OBSIDIAN_MCP_EMBEDDING_BASE_URL` | — | OpenAI-compatible endpoint, e.g. `http://127.0.0.1:1234/v1` | | `OBSIDIAN_MCP_EMBEDDING_API_KEY` | — | Optional API key for the embedding endpoint | diff --git a/packages/mcp-server/src/config.test.ts b/packages/mcp-server/src/config.test.ts index 9ffcadb..c1cdaae 100644 --- a/packages/mcp-server/src/config.test.ts +++ b/packages/mcp-server/src/config.test.ts @@ -9,6 +9,8 @@ describe("loadConfig", () => { expect(config.dbPath).toBeNull(); expect(config.dbPathSource).toBe("bridge"); expect(config.autoIndex).toBe(true); + expect(config.autoPruneEmbeddings).toBe(true); + expect(config.autoPruneEmbeddingsSource).toBe("bridge"); }); it("requires explicit embedding opt-in", () => { @@ -26,46 +28,73 @@ describe("loadConfig", () => { expect(config.autoIndex).toBe(false); }); + it("allows automatic embedding pruning to be disabled by env", () => { + const config = loadConfig({ OBSIDIAN_MCP_AUTO_PRUNE_EMBEDDINGS: "off" }); + expect(config.autoPruneEmbeddings).toBe(false); + expect(config.autoPruneEmbeddingsSource).toBe("env"); + }); + it("resolves the default database path from the Obsidian plugin folder", async () => { const config = loadConfig({ OBSIDIAN_MCP_TOKEN: "token" }); const resolved = await resolveRuntimeConfig(config, { - status: () => Promise.resolve({ - ok: true, - vaultName: "Test", - pluginVersion: "0.4.2", - bridgeVersion: "0.4.2", - readOnly: true, - writeToolsEnabled: false, - pluginDirectory: { - vaultPath: "custom-config/plugins/mcp-vault-bridge", - filesystemPath: "/vault/custom-config/plugins/mcp-vault-bridge", - defaultDatabasePath: "/vault/custom-config/plugins/mcp-vault-bridge/index.sqlite" - }, - scope: { excludedFolders: [], excludedFiles: [], excludedTags: [] }, - vaultPreview: { - detectedFolders: [], - detectedFiles: [], - detectedTags: [], - includedNoteCount: 0, - excludedNoteCount: 0 - }, - includedNoteCount: 0, - maxNoteBytes: 120000, - auditEnabled: true - }) + status: () => Promise.resolve(createBridgeStatus({ autoPruneEmbeddings: false })) } as never); expect(resolved.dbPath).toBe("/vault/custom-config/plugins/mcp-vault-bridge/index.sqlite"); expect(resolved.dbPathSource).toBe("bridge"); + expect(resolved.autoPruneEmbeddings).toBe(false); }); it("keeps explicit database paths", async () => { const config = loadConfig({ OBSIDIAN_MCP_DB: "/tmp/custom.sqlite" }); const resolved = await resolveRuntimeConfig(config, { - status: () => Promise.reject(new Error("should not be called")) + status: () => Promise.resolve(createBridgeStatus({ autoPruneEmbeddings: false })) } as never); expect(resolved.dbPath).toBe("/tmp/custom.sqlite"); expect(resolved.dbPathSource).toBe("env"); + expect(resolved.autoPruneEmbeddings).toBe(false); + }); + + it("lets the env pruning override win over plugin status", async () => { + const config = loadConfig({ + OBSIDIAN_MCP_DB: "/tmp/custom.sqlite", + OBSIDIAN_MCP_AUTO_PRUNE_EMBEDDINGS: "off" + }); + const resolved = await resolveRuntimeConfig(config, { + status: () => Promise.resolve(createBridgeStatus({ autoPruneEmbeddings: true })) + } as never); + + expect(resolved.dbPath).toBe("/tmp/custom.sqlite"); + expect(resolved.autoPruneEmbeddings).toBe(false); + expect(resolved.autoPruneEmbeddingsSource).toBe("env"); }); }); + +function createBridgeStatus(options: { autoPruneEmbeddings: boolean }) { + return { + ok: true, + vaultName: "Test", + pluginVersion: "0.4.2", + bridgeVersion: "0.4.2", + readOnly: true, + writeToolsEnabled: false, + autoPruneEmbeddings: options.autoPruneEmbeddings, + pluginDirectory: { + vaultPath: "custom-config/plugins/mcp-vault-bridge", + filesystemPath: "/vault/custom-config/plugins/mcp-vault-bridge", + defaultDatabasePath: "/vault/custom-config/plugins/mcp-vault-bridge/index.sqlite" + }, + scope: { excludedFolders: [], excludedFiles: [], excludedTags: [] }, + vaultPreview: { + detectedFolders: [], + detectedFiles: [], + detectedTags: [], + includedNoteCount: 0, + excludedNoteCount: 0 + }, + includedNoteCount: 0, + maxNoteBytes: 120000, + auditEnabled: true + }; +} diff --git a/packages/mcp-server/src/config.ts b/packages/mcp-server/src/config.ts index 0a9ddc2..2020901 100644 --- a/packages/mcp-server/src/config.ts +++ b/packages/mcp-server/src/config.ts @@ -17,6 +17,8 @@ export interface ServerConfig { dbPathSource: "env" | "bridge"; maxResults: number; autoIndex: boolean; + autoPruneEmbeddings: boolean; + autoPruneEmbeddingsSource: "env" | "bridge"; embeddings: EmbeddingConfig; } @@ -28,6 +30,8 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): ServerConfig { dbPathSource: env.OBSIDIAN_MCP_DB ? "env" : "bridge", maxResults: clampNumber(env.OBSIDIAN_MCP_MAX_RESULTS, DEFAULT_PAGE_LIMIT, 1, 100), autoIndex: parseEnabled(env.OBSIDIAN_MCP_AUTO_INDEX, true), + autoPruneEmbeddings: parseEnabled(env.OBSIDIAN_MCP_AUTO_PRUNE_EMBEDDINGS, true), + autoPruneEmbeddingsSource: env.OBSIDIAN_MCP_AUTO_PRUNE_EMBEDDINGS === undefined ? "bridge" : "env", embeddings: { enabled: parseEnabled(env.OBSIDIAN_MCP_EMBEDDINGS, false), baseUrl: env.OBSIDIAN_MCP_EMBEDDING_BASE_URL ?? null, @@ -39,11 +43,13 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): ServerConfig { } export async function resolveRuntimeConfig(config: ServerConfig, bridge: BridgeClient): Promise { + const status = await bridge.status(); + const autoPruneEmbeddings = + config.autoPruneEmbeddingsSource === "env" ? config.autoPruneEmbeddings : status.autoPruneEmbeddings; if (config.dbPath) { - return { ...config, dbPath: config.dbPath }; + return { ...config, dbPath: config.dbPath, autoPruneEmbeddings }; } - const status = await bridge.status(); const dbPath = status.pluginDirectory.defaultDatabasePath; if (!dbPath) { throw new Error( @@ -51,7 +57,7 @@ export async function resolveRuntimeConfig(config: ServerConfig, bridge: BridgeC ); } - return { ...config, dbPath, dbPathSource: "bridge" }; + return { ...config, dbPath, dbPathSource: "bridge", autoPruneEmbeddings }; } function expandHome(path: string): string { diff --git a/packages/mcp-server/src/database.test.ts b/packages/mcp-server/src/database.test.ts index e0ae4c3..6c8bf0c 100644 --- a/packages/mcp-server/src/database.test.ts +++ b/packages/mcp-server/src/database.test.ts @@ -27,6 +27,40 @@ describe("VaultDatabase", () => { expect(db.getNote("B.md")?.content).toBe("two"); db.close(); }); + + it("prunes orphaned embeddings while preserving current embeddings", () => { + const db = new VaultDatabase(join(mkdtempSync(join(tmpdir(), "obsidian-mcp-")), "index.sqlite")); + db.replaceNotes([makeNote("A.md", "# A\nold content", [])]); + const [oldChunk] = db.chunksMissingEmbeddings("test", "model", 10); + expect(oldChunk).toBeDefined(); + db.upsertEmbedding(oldChunk!.contentHash, "test", "model", [1, 0, 0]); + + db.replaceNotes([makeNote("A.md", "# A\nnew content", [])]); + const [newChunk] = db.chunksMissingEmbeddings("test", "model", 10); + expect(newChunk).toBeDefined(); + db.upsertEmbedding(newChunk!.contentHash, "test", "model", [0, 1, 0]); + + expect(db.stats()).toMatchObject({ + embeddingCount: 2, + orphanedEmbeddingCount: 1 + }); + + const result = db.pruneOrphanedEmbeddings(); + + expect(result).toMatchObject({ + beforeCount: 2, + afterCount: 1, + orphanedBeforeCount: 1, + orphanedAfterCount: 0, + deletedEmbeddings: 1 + }); + expect(result.estimatedBytesFreed).toBeGreaterThan(0); + expect(db.stats()).toMatchObject({ + embeddingCount: 1, + orphanedEmbeddingCount: 0 + }); + db.close(); + }); }); function makeNote(path: string, content: string, tags: string[]): VaultNote { @@ -55,4 +89,3 @@ function makeNote(path: string, content: string, tags: string[]): VaultNote { } }; } - diff --git a/packages/mcp-server/src/database.ts b/packages/mcp-server/src/database.ts index 8897c49..fdbb63f 100644 --- a/packages/mcp-server/src/database.ts +++ b/packages/mcp-server/src/database.ts @@ -2,13 +2,23 @@ import { mkdirSync } from "node:fs"; import { createRequire } from "node:module"; import { dirname } from "node:path"; import type Database from "better-sqlite3"; -import { chunkMarkdown, titleFromPath, type MarkdownChunk, type SearchResult, type VaultNote } from "@obsidian-mcp/shared"; +import { + chunkMarkdown, + countOrphanedEmbeddingsInDatabase, + pruneOrphanedEmbeddingsInDatabase, + titleFromPath, + type MarkdownChunk, + type PruneEmbeddingsResult, + type SearchResult, + type VaultNote +} from "@obsidian-mcp/shared"; import { sha256 } from "./hash.js"; export interface IndexStats { noteCount: number; chunkCount: number; embeddingCount: number; + orphanedEmbeddingCount: number; lastIndexedAt: string | null; } @@ -52,11 +62,13 @@ export class VaultDatabase { const noteCount = this.db.prepare("SELECT COUNT(*) AS count FROM notes").get() as { count: number }; const chunkCount = this.db.prepare("SELECT COUNT(*) AS count FROM chunks").get() as { count: number }; const embeddingCount = this.db.prepare("SELECT COUNT(*) AS count FROM embeddings").get() as { count: number }; + const orphanedEmbeddingCount = this.countOrphanedEmbeddings(); const last = this.db.prepare("SELECT MAX(indexed_at) AS lastIndexedAt FROM notes").get() as { lastIndexedAt: string | null }; return { noteCount: noteCount.count, chunkCount: chunkCount.count, embeddingCount: embeddingCount.count, + orphanedEmbeddingCount, lastIndexedAt: last.lastIndexedAt }; } @@ -269,6 +281,10 @@ export class VaultDatabase { .run(contentHash, provider, model, JSON.stringify(vector), vector.length, new Date().toISOString()); } + pruneOrphanedEmbeddings(): PruneEmbeddingsResult { + return pruneOrphanedEmbeddingsInDatabase(this.db); + } + semanticSearch(queryVector: number[], provider: string, model: string, limit: number): SearchResult[] { const rows = this.db .prepare( @@ -349,6 +365,14 @@ export class VaultDatabase { ); `); } + + private embeddingCount(): number { + return (this.db.prepare("SELECT COUNT(*) AS count FROM embeddings").get() as { count: number }).count; + } + + private countOrphanedEmbeddings(): number { + return countOrphanedEmbeddingsInDatabase(this.db); + } } function loadBetterSqlite3(): typeof Database { diff --git a/packages/mcp-server/src/index.ts b/packages/mcp-server/src/index.ts index c3fcdb2..b8eecf6 100644 --- a/packages/mcp-server/src/index.ts +++ b/packages/mcp-server/src/index.ts @@ -12,7 +12,7 @@ async function main(): Promise { const runtimeConfig = await resolveRuntimeConfig(config, bridge); const db = new VaultDatabase(runtimeConfig.dbPath); const embeddings = new EmbeddingClient(config.embeddings); - const indexer = new VaultIndexer(bridge, db, embeddings); + const indexer = new VaultIndexer(bridge, db, embeddings, runtimeConfig.autoPruneEmbeddings); await startMcpServer({ config: runtimeConfig, bridge, db, embeddings, indexer }); } diff --git a/packages/mcp-server/src/indexer.test.ts b/packages/mcp-server/src/indexer.test.ts index 7098445..3584eac 100644 --- a/packages/mcp-server/src/indexer.test.ts +++ b/packages/mcp-server/src/indexer.test.ts @@ -45,8 +45,18 @@ describe("VaultIndexer", () => { const result = await indexer.refreshIfEmpty(); - expect(result).toEqual({ indexedNotes: 1, embeddingChunks: 0 }); + expect(result).toEqual({ + indexedNotes: 1, + embeddingChunks: 0, + maintenance: { + prunedEmbeddings: 0, + orphanedEmbeddingsRemaining: 0, + estimatedBytesFreed: 0, + summary: "No orphaned embedding vectors to prune." + } + }); expect(exportNotes).toHaveBeenCalledOnce(); + expect(db.pruneOrphanedEmbeddings).toHaveBeenCalledOnce(); expect(indexer.status()).toMatchObject({ indexing: false, lastError: null }); }); @@ -69,6 +79,17 @@ describe("VaultIndexer", () => { expect(left).toEqual(right); expect(exportNotes).toHaveBeenCalledOnce(); }); + + it("can skip pruning after refresh when auto-prune is disabled", async () => { + const { bridge } = createBridge(); + const db = createDb(() => 0); + const indexer = new VaultIndexer(bridge, db, createEmbeddings(), false); + + const result = await indexer.refresh(); + + expect(result.maintenance).toBeUndefined(); + expect(db.pruneOrphanedEmbeddings).not.toHaveBeenCalled(); + }); }); function createBridge(): { bridge: BridgeClient; exportNotes: ReturnType } { @@ -85,9 +106,18 @@ function createDb(getNoteCount: () => number, onReplace: (notes: VaultNote[]) => noteCount: getNoteCount(), chunkCount: getNoteCount(), embeddingCount: 0, + orphanedEmbeddingCount: 0, lastIndexedAt: getNoteCount() > 0 ? "2026-01-01T00:00:00.000Z" : null }), - replaceNotes: vi.fn(onReplace) + replaceNotes: vi.fn(onReplace), + pruneOrphanedEmbeddings: vi.fn(() => ({ + beforeCount: 0, + afterCount: 0, + orphanedBeforeCount: 0, + orphanedAfterCount: 0, + deletedEmbeddings: 0, + estimatedBytesFreed: 0 + })) } as unknown as VaultDatabase; } diff --git a/packages/mcp-server/src/indexer.ts b/packages/mcp-server/src/indexer.ts index a2fae60..dab0daf 100644 --- a/packages/mcp-server/src/indexer.ts +++ b/packages/mcp-server/src/indexer.ts @@ -1,3 +1,4 @@ +import type { PruneEmbeddingsResult } from "@obsidian-mcp/shared"; import type { BridgeClient } from "./bridge-client.js"; import type { VaultDatabase } from "./database.js"; import { backfillEmbeddings, type EmbeddingClient } from "./embeddings.js"; @@ -5,6 +6,12 @@ import { backfillEmbeddings, type EmbeddingClient } from "./embeddings.js"; export interface RefreshResult { indexedNotes: number; embeddingChunks: number; + maintenance?: { + prunedEmbeddings: number; + orphanedEmbeddingsRemaining: number; + estimatedBytesFreed: number; + summary: string; + }; } export interface IndexerStatus { @@ -21,7 +28,8 @@ export class VaultIndexer { constructor( private readonly bridge: BridgeClient, private readonly db: VaultDatabase, - private readonly embeddings: EmbeddingClient + private readonly embeddings: EmbeddingClient, + private readonly autoPruneEmbeddings = true ) {} async refresh(): Promise { @@ -87,9 +95,23 @@ export class VaultIndexer { } } + const maintenance = this.autoPruneEmbeddings ? formatMaintenance(this.db.pruneOrphanedEmbeddings()) : undefined; return { indexedNotes: notes.length, - embeddingChunks + embeddingChunks, + maintenance }; } } + +function formatMaintenance(result: PruneEmbeddingsResult): RefreshResult["maintenance"] { + return { + prunedEmbeddings: result.deletedEmbeddings, + orphanedEmbeddingsRemaining: result.orphanedAfterCount, + estimatedBytesFreed: result.estimatedBytesFreed, + summary: + result.deletedEmbeddings > 0 + ? `Pruned ${result.deletedEmbeddings} orphaned embedding vector(s).` + : "No orphaned embedding vectors to prune." + }; +} diff --git a/packages/mcp-server/src/mcp-write-tools.test.ts b/packages/mcp-server/src/mcp-write-tools.test.ts index fa3b22a..e96c046 100644 --- a/packages/mcp-server/src/mcp-write-tools.test.ts +++ b/packages/mcp-server/src/mcp-write-tools.test.ts @@ -63,12 +63,17 @@ describe("MCP write tools", () => { throw new Error("create_note was not registered"); } const result = await tool.handler({ path: "Notes/New.md", content: "# New", overwrite: false }); - const parsed = JSON.parse(result.content[0]!.text) as WriteNoteResponse & { index: { noteCount: number } }; + const parsed = JSON.parse(result.content[0]!.text) as WriteNoteResponse & { + index: { noteCount: number }; + maintenance?: { summary: string }; + }; expect(runtime.bridge.createNote).toHaveBeenCalledWith("Notes/New.md", "# New", false); expect(runtime.db.upsertNote).toHaveBeenCalledWith(expect.objectContaining({ path: "Notes/New.md", content: "# New" })); + expect(runtime.db.pruneOrphanedEmbeddings).toHaveBeenCalledOnce(); expect(parsed.operation).toBe("create"); expect(parsed.index.noteCount).toBe(1); + expect(parsed.maintenance?.summary).toContain("No orphaned"); }); it("replace_note_text preserves occurrenceIndex and returns an embedding refresh hint", async () => { @@ -86,10 +91,27 @@ describe("MCP write tools", () => { expect(runtime.db.upsertNote).toHaveBeenCalled(); expect(parsed.hint).toContain("refresh_index"); }); + + it("registers prune_embeddings and returns cleanup counts", async () => { + const runtime = createRuntime({ orphanedEmbeddings: 2 }); + await startMcpServer(runtime); + + const tool = sdkMock.registeredTools.get("prune_embeddings"); + if (!tool) { + throw new Error("prune_embeddings was not registered"); + } + const result = await tool.handler({}); + const parsed = JSON.parse(result.content[0]!.text) as { maintenance: { prunedEmbeddings: number }; index: { orphanedEmbeddingCount: number } }; + + expect(runtime.db.pruneOrphanedEmbeddings).toHaveBeenCalledOnce(); + expect(parsed.maintenance.prunedEmbeddings).toBe(2); + expect(parsed.index.orphanedEmbeddingCount).toBe(0); + }); }); -function createRuntime(options: { embeddingsEnabled?: boolean } = {}): McpRuntime { +function createRuntime(options: { embeddingsEnabled?: boolean; orphanedEmbeddings?: number } = {}): McpRuntime { const embeddingsEnabled = options.embeddingsEnabled ?? false; + let orphanedEmbeddings = options.orphanedEmbeddings ?? 0; const note = createWrittenNote("Notes/New.md", "# New"); return { config: { @@ -99,6 +121,8 @@ function createRuntime(options: { embeddingsEnabled?: boolean } = {}): McpRuntim dbPathSource: "env", maxResults: 20, autoIndex: false, + autoPruneEmbeddings: true, + autoPruneEmbeddingsSource: "bridge", embeddings: { enabled: embeddingsEnabled, baseUrl: embeddingsEnabled ? "http://127.0.0.1:1234/v1" : null, @@ -116,6 +140,7 @@ function createRuntime(options: { embeddingsEnabled?: boolean } = {}): McpRuntim bridgeVersion: "0.4.3", readOnly: false, writeToolsEnabled: true, + autoPruneEmbeddings: true, pluginDirectory: { vaultPath: ".obsidian/plugins/mcp-vault-bridge", filesystemPath: "/vault/.obsidian/plugins/mcp-vault-bridge", @@ -144,10 +169,23 @@ function createRuntime(options: { embeddingsEnabled?: boolean } = {}): McpRuntim stats: vi.fn(() => ({ noteCount: 1, chunkCount: 1, - embeddingCount: embeddingsEnabled ? 0 : 0, + embeddingCount: orphanedEmbeddings, + orphanedEmbeddingCount: orphanedEmbeddings, lastIndexedAt: "2026-01-01T00:00:00.000Z" })), - upsertNote: vi.fn() + upsertNote: vi.fn(), + pruneOrphanedEmbeddings: vi.fn(() => { + const deletedEmbeddings = orphanedEmbeddings; + orphanedEmbeddings = 0; + return { + beforeCount: deletedEmbeddings, + afterCount: 0, + orphanedBeforeCount: deletedEmbeddings, + orphanedAfterCount: 0, + deletedEmbeddings, + estimatedBytesFreed: deletedEmbeddings * 128 + }; + }) } as unknown as McpRuntime["db"], embeddings: { enabled: embeddingsEnabled, diff --git a/packages/mcp-server/src/mcp.test.ts b/packages/mcp-server/src/mcp.test.ts index 7459cfc..c63ff09 100644 --- a/packages/mcp-server/src/mcp.test.ts +++ b/packages/mcp-server/src/mcp.test.ts @@ -8,6 +8,7 @@ const indexStats: IndexStats = { noteCount: 2, chunkCount: 4, embeddingCount: 4, + orphanedEmbeddingCount: 0, lastIndexedAt: "2026-01-01T00:00:00.000Z" }; @@ -114,7 +115,9 @@ describe("indexWrittenNote", () => { const result = indexWrittenNote(runtime, response); expect(upsertNote).toHaveBeenCalledWith(response.note); + expect(runtime.db.pruneOrphanedEmbeddings).toHaveBeenCalledOnce(); expect(result.operation).toBe("create"); + expect(result.maintenance?.prunedEmbeddings).toBe(0); expect(result.hint).toBeUndefined(); }); @@ -129,6 +132,21 @@ describe("indexWrittenNote", () => { expect(result.hint).toContain("refresh_index"); }); + + it("skips automatic pruning when disabled", () => { + const runtime = createRuntime({ + embeddingsEnabled: true, + autoPruneEmbeddings: false, + stats: indexStats, + upsertNote: vi.fn() + }); + + const result = indexWrittenNote(runtime, createWriteResponse("Notes/New.md")); + + expect(runtime.db.pruneOrphanedEmbeddings).not.toHaveBeenCalled(); + expect(result.maintenance).toBeUndefined(); + expect(result.hint).toContain("prune_embeddings"); + }); }); function createRuntime(options: { @@ -139,6 +157,7 @@ function createRuntime(options: { semanticSearch?: () => SearchResult[]; listNotes?: () => IndexedNote[]; upsertNote?: (note: WriteNoteResponse["note"]) => void; + autoPruneEmbeddings?: boolean; }): McpRuntime { return { config: { @@ -148,6 +167,8 @@ function createRuntime(options: { dbPathSource: "env", maxResults: 20, autoIndex: true, + autoPruneEmbeddings: options.autoPruneEmbeddings ?? true, + autoPruneEmbeddingsSource: "bridge", embeddings: { enabled: options.embeddingsEnabled, baseUrl: options.embeddingsEnabled ? "http://127.0.0.1:1234/v1" : null, @@ -162,7 +183,15 @@ function createRuntime(options: { searchFts: options.searchFts ?? (() => []), semanticSearch: options.semanticSearch ?? (() => []), listNotes: options.listNotes ?? (() => []), - upsertNote: options.upsertNote ?? (() => undefined) + upsertNote: options.upsertNote ?? (() => undefined), + pruneOrphanedEmbeddings: vi.fn(() => ({ + beforeCount: 0, + afterCount: 0, + orphanedBeforeCount: 0, + orphanedAfterCount: 0, + deletedEmbeddings: 0, + estimatedBytesFreed: 0 + })) } as unknown as McpRuntime["db"], embeddings: { enabled: options.embeddingsEnabled, diff --git a/packages/mcp-server/src/mcp.ts b/packages/mcp-server/src/mcp.ts index 7e0e814..56b99b2 100644 --- a/packages/mcp-server/src/mcp.ts +++ b/packages/mcp-server/src/mcp.ts @@ -1,7 +1,13 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; import { z } from "zod"; -import { DEFAULT_MAX_TOOL_TEXT_BYTES, truncateText, type SearchResult, type WriteNoteResponse } from "@obsidian-mcp/shared"; +import { + DEFAULT_MAX_TOOL_TEXT_BYTES, + truncateText, + type PruneEmbeddingsResult, + type SearchResult, + type WriteNoteResponse +} from "@obsidian-mcp/shared"; import type { BridgeClient } from "./bridge-client.js"; import type { ServerConfig } from "./config.js"; import type { VaultDatabase } from "./database.js"; @@ -92,6 +98,17 @@ export async function startMcpServer(runtime: McpRuntime): Promise { () => jsonResponse(getIndexStatus(runtime)) ); + server.registerTool( + "prune_embeddings", + { + title: "Prune Embeddings", + description: + "Remove stale cached embedding vectors no longer used by indexed note chunks. Usually automatic after writes and refresh_index; run manually after maintenance or when index_status reports orphaned embeddings.", + inputSchema: {} + }, + () => jsonResponse({ maintenance: formatMaintenance(runtime.db.pruneOrphanedEmbeddings()), index: getIndexStatus(runtime) }) + ); + server.registerTool( "ask_vault", { @@ -354,16 +371,50 @@ function jsonResponse(value: unknown): { content: Array<{ type: "text"; text: st export function indexWrittenNote(runtime: McpRuntime, response: WriteNoteResponse): WriteNoteResponse & { index: ReturnType; + maintenance?: ReturnType; hint?: string; } { runtime.db.upsertNote(response.note); + const maintenance = runtime.config.autoPruneEmbeddings ? formatMaintenance(runtime.db.pruneOrphanedEmbeddings()) : undefined; return { ...response, + maintenance, index: getIndexStatus(runtime), - hint: runtime.embeddings.enabled ? "Note content was updated in the local index. Run refresh_index to refresh embeddings." : undefined + hint: writeMaintenanceHint(runtime, maintenance) }; } +function formatMaintenance(result: PruneEmbeddingsResult): { + prunedEmbeddings: number; + orphanedEmbeddingsRemaining: number; + estimatedBytesFreed: number; + summary: string; +} { + return { + prunedEmbeddings: result.deletedEmbeddings, + orphanedEmbeddingsRemaining: result.orphanedAfterCount, + estimatedBytesFreed: result.estimatedBytesFreed, + summary: + result.deletedEmbeddings > 0 + ? `Pruned ${result.deletedEmbeddings} orphaned embedding vector(s). Estimated ${result.estimatedBytesFreed} byte(s) of stale vector data removed.` + : "No orphaned embedding vectors to prune." + }; +} + +function writeMaintenanceHint(runtime: McpRuntime, maintenance: ReturnType | undefined): string | undefined { + if (!runtime.embeddings.enabled) { + return undefined; + } + const refresh = "Run refresh_index to refresh embeddings for changed note chunks."; + if (!runtime.config.autoPruneEmbeddings) { + return `${refresh} Auto-prune is disabled; run prune_embeddings to clean stale vectors.`; + } + if (maintenance && maintenance.prunedEmbeddings > 0) { + return `${refresh} ${maintenance.summary}`; + } + return refresh; +} + function mergeResults(a: T[], b: T[], limit: number): T[] { const map = new Map(); for (const item of [...a, ...b]) { @@ -460,17 +511,24 @@ function getIndexStatus(runtime: McpRuntime): ReturnType databasePath: string | null; databasePathSource: ServerConfig["dbPathSource"]; autoIndexEnabled: boolean; + autoPruneEmbeddingsEnabled: boolean; + autoPruneEmbeddingsSource: ServerConfig["autoPruneEmbeddingsSource"]; indexing: boolean; lastError: string | null; + hint?: string; } { const indexer = runtime.indexer.status(); + const stats = runtime.db.stats(); return { - ...runtime.db.stats(), + ...stats, databasePath: runtime.config.dbPath, databasePathSource: runtime.config.dbPathSource, autoIndexEnabled: runtime.config.autoIndex, + autoPruneEmbeddingsEnabled: runtime.config.autoPruneEmbeddings, + autoPruneEmbeddingsSource: runtime.config.autoPruneEmbeddingsSource, indexing: indexer.indexing, - lastError: indexer.lastError + lastError: indexer.lastError, + hint: stats.orphanedEmbeddingCount > 0 ? "Run prune_embeddings to clean stale cached embedding vectors." : undefined }; } diff --git a/packages/plugin/src/server.test.ts b/packages/plugin/src/server.test.ts new file mode 100644 index 0000000..4cca4f9 --- /dev/null +++ b/packages/plugin/src/server.test.ts @@ -0,0 +1,98 @@ +import { createServer } from "node:net"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type ObsidianMcpPlugin from "./main.js"; +import { createBridgeServer, type BridgeServerHandle } from "./server.js"; + +describe("plugin bridge write routes", () => { + let handles: BridgeServerHandle[] = []; + + beforeEach(() => { + (globalThis as unknown as { window: { require: NodeJS.Require } }).window = { require }; + }); + + afterEach(async () => { + await Promise.all(handles.map((handle) => handle.close())); + handles = []; + vi.restoreAllMocks(); + }); + + it("checks write authorization before append content validation", async () => { + const port = await getFreePort(); + const plugin = createPlugin({ port, writeToolsEnabled: false }); + const handle = await createBridgeServer(plugin, "token"); + handles.push(handle); + + const response = await fetch(`http://127.0.0.1:${port}/notes/append`, { + method: "POST", + headers: { + Authorization: "Bearer token", + "Content-Type": "application/json" + }, + body: JSON.stringify({ path: "Notes/Test.md", content: "" }) + }); + + expect(response.status).toBe(403); + expect(await response.json()).toEqual({ error: "Write tools are disabled in the Obsidian plugin settings." }); + expect(plugin.audit).toHaveBeenCalledWith({ + route: "/notes/append", + path: "Notes/Test.md", + allowed: false, + reason: "writes_disabled" + }); + }); +}); + +function createPlugin(options: { port: number; writeToolsEnabled: boolean }): ObsidianMcpPlugin { + return { + manifest: { + id: "mcp-vault-bridge", + version: "0.4.3" + }, + settings: { + bridgeEnabled: true, + port: options.port, + excludedFolders: [], + excludedFiles: [], + excludedTags: [], + maxNoteBytes: 120000, + writeToolsEnabled: options.writeToolsEnabled, + autoPruneEmbeddings: true, + auditEnabled: true, + tokenSecretName: "obsidian-mcp-bridge-token", + nodeCommandOverride: "", + npmCommandOverride: "" + }, + app: { + vault: { + configDir: ".obsidian", + getName: () => "Test Vault", + getMarkdownFiles: () => [], + getAbstractFileByPath: () => null, + cachedRead: vi.fn(), + create: vi.fn(), + modify: vi.fn() + }, + metadataCache: { + getFileCache: () => null, + resolvedLinks: {} + } + }, + audit: vi.fn(() => Promise.resolve()) + } as unknown as ObsidianMcpPlugin; +} + +async function getFreePort(): Promise { + return new Promise((resolve, reject) => { + const server = createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + server.close(() => reject(new Error("Could not allocate a local port."))); + return; + } + const { port } = address; + server.close(() => resolve(port)); + }); + }); +} diff --git a/packages/plugin/src/server.ts b/packages/plugin/src/server.ts index 26ca70e..c373819 100644 --- a/packages/plugin/src/server.ts +++ b/packages/plugin/src/server.ts @@ -149,6 +149,7 @@ function buildStatus(plugin: ObsidianMcpPlugin): BridgeStatus { bridgeVersion: plugin.manifest.version, readOnly: !plugin.settings.writeToolsEnabled, writeToolsEnabled: plugin.settings.writeToolsEnabled, + autoPruneEmbeddings: plugin.settings.autoPruneEmbeddings, pluginDirectory, scope: normalizeVaultScope(plugin.settings), vaultPreview: buildVaultScopePreview(plugin), @@ -338,6 +339,10 @@ async function routeAppendNote( route: string, body: JsonRecord ): Promise { + if (!(await ensureWritesEnabled(plugin, response, route, stringField(body.path)))) { + return; + } + const content = stringField(body.content); if (!content) { await plugin.audit({ route, path: stringField(body.path), allowed: false, reason: "empty_append" }); @@ -389,7 +394,9 @@ async function routeMutateExistingNote( edit: (existing: string) => string ): Promise { const rawPath = stringField(body.path); - if (!(await ensureWritesEnabled(plugin, response, route, rawPath))) { + if (!plugin.settings.writeToolsEnabled) { + await plugin.audit({ route, path: rawPath, allowed: false, reason: "writes_disabled" }); + sendJson(response, 403, { error: "Write tools are disabled in the Obsidian plugin settings." }); return; } diff --git a/packages/plugin/src/settings.ts b/packages/plugin/src/settings.ts index 4e59cbf..e9c3c27 100644 --- a/packages/plugin/src/settings.ts +++ b/packages/plugin/src/settings.ts @@ -16,6 +16,9 @@ import { normalizeTag, normalizeVaultPath, parseDelimitedList, + pruneOrphanedEmbeddingsInDatabase, + type PrunableEmbeddingDatabase, + type PruneEmbeddingsResult, type VaultScopePreview } from "@obsidian-mcp/shared"; import type ObsidianMcpPlugin from "./main.js"; @@ -29,6 +32,7 @@ export interface ObsidianMcpSettings { excludedTags: string[]; maxNoteBytes: number; writeToolsEnabled: boolean; + autoPruneEmbeddings: boolean; auditEnabled: boolean; tokenSecretName: string; nodeCommandOverride: string; @@ -43,6 +47,7 @@ export const DEFAULT_SETTINGS: ObsidianMcpSettings = { excludedTags: [], maxNoteBytes: DEFAULT_MAX_NOTE_BYTES, writeToolsEnabled: false, + autoPruneEmbeddings: true, auditEnabled: true, tokenSecretName: "obsidian-mcp-bridge-token", nodeCommandOverride: "", @@ -87,6 +92,14 @@ interface NodePath { join(...paths: string[]): string; } +interface NodeModule { + createRequire(filename: string): (moduleName: string) => unknown; +} + +interface BetterSqlite3Constructor { + new (path: string): PrunableEmbeddingDatabase & { close(): void }; +} + interface ExecFileOptions { cwd?: string; timeout?: number; @@ -542,6 +555,34 @@ export class ObsidianMcpSettingTab extends PluginSettingTab { }) ); + new Setting(section) + .setName("Auto-prune embeddings") + .setDesc("Automatically removes stale cached embedding vectors after writes and index refreshes. This keeps the local SQLite cache tidy.") + .addToggle((toggle) => + toggle.setValue(this.plugin.settings.autoPruneEmbeddings).onChange(async (value) => { + this.plugin.settings.autoPruneEmbeddings = value; + await this.plugin.saveSettings(); + this.display(); + }) + ) + .addButton((button) => { + const dbPath = getIndexDatabasePath(this.plugin); + button + .setButtonText("Prune now") + .setTooltip(dbPath ? "Remove stale cached embedding vectors now." : "Plugin folder unavailable.") + .setDisabled(!dbPath) + .onClick(async () => { + try { + await withBusyButton(button, "Pruning", "Prune now", async () => { + const result = await pruneEmbeddingsFromSettings(this.plugin); + new Notice(formatPruneNotice(result)); + }); + } catch (error) { + new Notice(error instanceof Error ? error.message : String(error)); + } + }); + }); + new Setting(section) .setName("Loopback port") .setDesc("The mcp adapter connects to this local port.") @@ -1373,6 +1414,58 @@ function getMcpServerPath(plugin: ObsidianMcpPlugin): string | null { return pluginDirectory ? joinFilesystemPath(pluginDirectory, "mcp-server.cjs") : null; } +function getIndexDatabasePath(plugin: ObsidianMcpPlugin): string | null { + const pluginDirectory = getPluginFilesystemDirectory(plugin); + return pluginDirectory ? joinFilesystemPath(pluginDirectory, "index.sqlite") : null; +} + +async function pruneEmbeddingsFromSettings(plugin: ObsidianMcpPlugin): Promise { + const dbPath = getIndexDatabasePath(plugin); + if (!dbPath) { + throw new Error("Plugin folder unavailable. Use Obsidian desktop with a filesystem vault."); + } + if (!(await fileExists(dbPath))) { + throw new Error("Index database is not ready yet. Run refresh_index first."); + } + + const pluginDirectory = getPluginFilesystemDirectory(plugin); + if (!pluginDirectory) { + throw new Error("Plugin folder unavailable. Use Obsidian desktop with a filesystem vault."); + } + const Database = loadRuntimeSqlite(pluginDirectory); + const db = new Database(dbPath); + try { + return pruneOrphanedEmbeddingsInDatabase(db); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (message.includes("no such table")) { + throw new Error("Index database is not ready yet. Run refresh_index first."); + } + throw error; + } finally { + db.close(); + } +} + +function loadRuntimeSqlite(pluginDirectory: string): BetterSqlite3Constructor { + const nodeModule = loadNodeModule("module"); + if (!nodeModule) { + throw new Error("Node module loader is not available inside Obsidian."); + } + const requireFromRuntime = nodeModule.createRequire(joinFilesystemPath(pluginDirectory, "package.json")); + try { + return requireFromRuntime("better-sqlite3") as BetterSqlite3Constructor; + } catch { + throw new Error("SQLite runtime is missing. Click Install SQLite runtime."); + } +} + +function formatPruneNotice(result: PruneEmbeddingsResult): string { + return result.deletedEmbeddings > 0 + ? `Pruned ${result.deletedEmbeddings} stale embedding vector(s).` + : "No stale embedding vectors found."; +} + function joinFilesystemPath(...parts: string[]): string { const path = loadNodeModule("path"); if (path) { diff --git a/packages/plugin/src/test-obsidian.ts b/packages/plugin/src/test-obsidian.ts new file mode 100644 index 0000000..139dc3f --- /dev/null +++ b/packages/plugin/src/test-obsidian.ts @@ -0,0 +1,25 @@ +export class FileSystemAdapter { + getBasePath(): string { + return ""; + } +} + +export class Modal {} + +export class Notice { + constructor(readonly message: string) {} +} + +export class PluginSettingTab {} + +export class Setting {} + +export class TFile { + path = ""; + basename = ""; + extension = "md"; + stat = { ctime: 0, mtime: 0, size: 0 }; +} + +export type App = unknown; +export type ButtonComponent = unknown; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 40cbf69..dee6839 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1,4 +1,5 @@ export * from "./markdown.js"; +export * from "./prune.js"; export * from "./security.js"; export * from "./types.js"; export * from "./write.js"; diff --git a/packages/shared/src/prune.ts b/packages/shared/src/prune.ts new file mode 100644 index 0000000..0d67d6c --- /dev/null +++ b/packages/shared/src/prune.ts @@ -0,0 +1,74 @@ +import type { PruneEmbeddingsResult } from "./types.js"; + +export interface SqliteStatement { + get(): Result | undefined; + run(): unknown; +} + +export interface PrunableEmbeddingDatabase { + prepare(sql: string): SqliteStatement; +} + +export function pruneOrphanedEmbeddingsInDatabase(db: PrunableEmbeddingDatabase): PruneEmbeddingsResult { + const beforeCount = countEmbeddings(db); + const orphanedBeforeCount = countOrphanedEmbeddings(db); + const estimatedBytesFreed = estimateOrphanedEmbeddingBytes(db); + if (orphanedBeforeCount > 0) { + db.prepare( + `DELETE FROM embeddings + WHERE NOT EXISTS ( + SELECT 1 FROM chunks + WHERE chunks.content_hash = embeddings.content_hash + )` + ).run(); + } + const afterCount = countEmbeddings(db); + const orphanedAfterCount = countOrphanedEmbeddings(db); + return { + beforeCount, + afterCount, + orphanedBeforeCount, + orphanedAfterCount, + deletedEmbeddings: beforeCount - afterCount, + estimatedBytesFreed + }; +} + +export function countOrphanedEmbeddingsInDatabase(db: PrunableEmbeddingDatabase): number { + return countOrphanedEmbeddings(db); +} + +function countEmbeddings(db: PrunableEmbeddingDatabase): number { + const row = db.prepare<{ count: number }>("SELECT COUNT(*) AS count FROM embeddings").get(); + return row?.count ?? 0; +} + +function countOrphanedEmbeddings(db: PrunableEmbeddingDatabase): number { + const row = db + .prepare<{ count: number }>( + `SELECT COUNT(*) AS count + FROM embeddings e + WHERE NOT EXISTS ( + SELECT 1 FROM chunks c + WHERE c.content_hash = e.content_hash + )` + ) + .get(); + return row?.count ?? 0; +} + +function estimateOrphanedEmbeddingBytes(db: PrunableEmbeddingDatabase): number { + const row = db + .prepare<{ bytes: number }>( + `SELECT COALESCE(SUM( + length(content_hash) + length(provider) + length(model) + length(vector_json) + 32 + ), 0) AS bytes + FROM embeddings e + WHERE NOT EXISTS ( + SELECT 1 FROM chunks c + WHERE c.content_hash = e.content_hash + )` + ) + .get(); + return row?.bytes ?? 0; +} diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 21f09aa..33351ec 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -22,6 +22,7 @@ export interface BridgeStatus { bridgeVersion: string; readOnly: boolean; writeToolsEnabled: boolean; + autoPruneEmbeddings: boolean; pluginDirectory: { vaultPath: string; filesystemPath: string | null; @@ -106,3 +107,12 @@ export interface WriteNoteResponse { operation: "create" | "append" | "replace" | "delete_text" | "rewrite"; note: VaultNote; } + +export interface PruneEmbeddingsResult { + beforeCount: number; + afterCount: number; + orphanedBeforeCount: number; + orphanedAfterCount: number; + deletedEmbeddings: number; + estimatedBytesFreed: number; +} diff --git a/vitest.config.ts b/vitest.config.ts index 78cc394..d6acfa2 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -5,6 +5,7 @@ export default defineConfig({ resolve: { alias: { "@obsidian-mcp/shared": resolve("packages/shared/src/index.ts"), + "obsidian": resolve("packages/plugin/src/test-obsidian.ts"), "virtual:mcp-server-payload": resolve("packages/plugin/src/test-mcp-server-payload.ts") } },