diff --git a/docs/MIYO_SEARCH_PLAN.md b/docs/MIYO_SEARCH_PLAN.md new file mode 100644 index 00000000..d52a4644 --- /dev/null +++ b/docs/MIYO_SEARCH_PLAN.md @@ -0,0 +1,132 @@ +# Miyo Search Integration Plan (Revised) + +## Summary + +Integrate Miyo as the semantic index backend while **preserving Copilot’s existing chunking, indexing, and Search v3 UX**. Copilot continues to generate chunks and embeddings locally, but stores vectors and metadata in Miyo via new index-management endpoints. Search v3 keeps its lexical + semantic merge and query expansion, with the semantic leg served by Miyo. Implementation will proceed in two phases: Phase 1 refactors Copilot to abstract Orama; Phase 2 adds Miyo as an additional backend. + +## Inputs Read + +- `discover_miyo.md`: Service discovery via `~/Library/Application Support/Miyo/service.json` with `host`, `port`, and `pid`. +- `miyo_openapi.json`: Existing `/v0/search` and `/v0/health` endpoints. + +## Goals + +- Preserve current Copilot chunking and indexing flow (Search v3 chunk IDs, metadata injection). +- Preserve Search v3 experience (lexical + semantic merge, query expansion, dedupe behavior). +- Store vectors and semantic index data in Miyo (not in Orama) when enabled. +- Reuse existing indexing hooks (refresh, force reindex, clear, delete, list) without UI regressions. +- Allow easy switching between Orama and Miyo via a backend abstraction. +- Keep Miyo usage behind the self-host toggle and Miyo toggle. + +## Non-Goals + +- Changing AI prompts or chat behavior. +- Rewriting lexical search or QueryExpander. +- Forcing a single embedding model for all users (the system remains configurable). + +## Proposed Design + +### 1. Embedding Provider: Miyo (Toggle-Controlled) + +- Add `EmbeddingModelProviders.MIYO` and an internal Miyo embedding model entry. +- **Users do not select Miyo embeddings directly**. The Miyo toggle under self-host mode controls this automatically. +- Implement using `CustomOpenAIEmbeddings` so Miyo’s embeddings are **OpenAI-compatible** (same request/response as Brevilabs embeddings). +- Base URL should resolve to Miyo (self-host URL or discovery result). The embedding endpoint should be `POST /v1/embeddings` on that base URL. +- When Miyo indexing is enabled, automatically switch the embedding model to the Miyo provider and restore the previous embedding model when disabled. + +### 2. Index Backend Abstraction (Phase 1) + +Introduce a backend interface (example name `SemanticIndexBackend`) to decouple indexing from Orama: + +- `initialize(embeddingInstance)` +- `clearIndex()` +- `upsert(doc | docs[])` +- `removeByPath(path)` +- `getIndexedFiles()` +- `getLatestFileMtime()` +- `isIndexEmpty()` / `hasIndex(path)` +- `checkAndHandleEmbeddingModelChange(embeddingInstance)` +- `save()` (no-op for Miyo) +- `checkIndexIntegrity()` (optional for Miyo) + +Implement two backends: + +- **OramaIndexBackend**: wraps existing `DBOperations` unchanged. +- **MiyoIndexBackend**: calls new Miyo index-management endpoints (Phase 2). + +VectorStoreManager should select the backend based on settings: + +- Orama when `enableSemanticSearchV3` and Miyo disabled. +- Miyo when self-host + Miyo indexing enabled. + +### 3. Reuse Existing Chunking + Indexing Flow + +- Keep `IndexOperations.prepareAllChunks()` and its metadata injection. +- Keep chunk IDs (`note_path#chunk_index`) and metadata fields (heading, frontmatter, created/modified). These must be stored in Miyo. +- Replace direct `DBOperations` calls with backend interface calls. + +### 4. Search v3: Merge + Expansion Preserved + +- Keep `TieredLexicalRetriever` for lexical + query expansion. +- Modify `MergedSemanticRetriever` to accept an injected semantic retriever. +- Implement `MiyoSemanticRetriever` that queries Miyo for semantic results (hybrid search), mapping results into LangChain `Document` objects. +- When Miyo indexing is enabled, semantic leg uses Miyo; otherwise fallback to `HybridRetriever` (Orama). + +### 5. Indexing Hooks (No UX Change) + +Existing commands and UI should continue to work: + +- Refresh Vault Index +- Force Reindex Vault +- Clear Index +- List Indexed Files +- Remove From Index + +These hooks should call VectorStoreManager which delegates to the chosen backend. + +### 6. Service Discovery + +- Keep `MiyoServiceDiscovery` to resolve base URL from `service.json`. +- Use discovery for **search**, **embeddings**, and **index management** endpoints. + +### 7. Migration + Switching + +- Switching from Orama to Miyo should trigger a full reindex (force rebuild) because vectors move stores. +- Switching back from Miyo to Orama should also trigger a full reindex to rebuild local Orama state. +- Miyo can keep data in a single collection for now; Copilot will still pass `source_id` for future isolation. + +## Two-Phase Delivery Plan + +### Phase 1: Orama Abstraction (No Behavior Change) + +- Introduce the index backend interface and refactor Orama usage behind it. +- Keep all existing commands, UI, and indexing flows unchanged. +- Validate that Search v3 behavior is unchanged. + +### Phase 2: Add Miyo Backend + +- Implement Miyo index backend and retriever. +- Enable Miyo embedding provider toggled by self-host + Miyo switch. +- Wire collection naming per vault and hybrid search endpoint usage. + +## Integration Points + +- `src/LLMProviders/embeddingManager.ts` (new provider `MIYO`) +- `src/constants.ts` (provider enum + built-in model entry) +- `src/search/indexBackend/*` (new backend abstraction) +- `src/search/vectorStoreManager.ts` (backend selection) +- `src/search/indexOperations.ts` (backend interface usage) +- `src/search/v3/MergedSemanticRetriever.ts` (semantic retriever injection) +- `src/search/miyo/*` (discovery + Miyo client) + +## Testing Plan + +- Unit tests for backend abstraction (Orama adapter, Miyo adapter) +- Integration test for “force reindex” using Miyo backend (verify calls) +- Manual test: enable Miyo + reindex; confirm search results still include query expansion + lexical merge + +## Open Questions (Resolved) + +- Miyo embeddings are **not user-selectable**; the Miyo toggle controls them. +- Miyo search should be **hybrid** (BM25 + vector). +- Each vault is a **separate Miyo collection**. diff --git a/src/search/findRelevantNotes.ts b/src/search/findRelevantNotes.ts index 304d4430..72430eae 100644 --- a/src/search/findRelevantNotes.ts +++ b/src/search/findRelevantNotes.ts @@ -1,5 +1,6 @@ import { getBacklinkedNotes, getLinkedNotes } from "@/noteUtils"; import { DBOperations } from "@/search/dbOperations"; +import VectorStoreManager from "@/search/vectorStoreManager"; import { getSettings } from "@/settings/model"; import { InternalTypedDocument, Orama, Result } from "@orama/orama"; import { TFile } from "obsidian"; @@ -11,13 +12,12 @@ const LINKS_WEIGHT = 0.3; /** * Gets the embeddings for the given note path. * @param notePath - The note path to get embeddings for. - * @param db - The Orama database. * @returns The embeddings for the given note path. */ -async function getNoteEmbeddings(notePath: string, db: Orama): Promise { +async function getNoteEmbeddings(notePath: string): Promise { const debug = getSettings().debug; - const hits = await DBOperations.getDocsByPath(db, notePath); - if (!hits) { + const docs = await VectorStoreManager.getInstance().getDocumentsByPath(notePath); + if (docs.length === 0) { if (debug) { console.log("No hits found for note:", notePath); } @@ -25,14 +25,14 @@ async function getNoteEmbeddings(notePath: string, db: Orama): Promise> { const debug = getSettings().debug; - const currentNoteEmbeddings = await getNoteEmbeddings(filePath, db); + const currentNoteEmbeddings = await getNoteEmbeddings(filePath); if (currentNoteEmbeddings.length === 0) { if (debug) { console.log("No embeddings found for note:", filePath); diff --git a/src/search/hybridRetriever.ts b/src/search/hybridRetriever.ts index e957a46a..d5b646a2 100644 --- a/src/search/hybridRetriever.ts +++ b/src/search/hybridRetriever.ts @@ -19,7 +19,6 @@ import { ChatPromptTemplate } from "@langchain/core/prompts"; import { BaseRetriever } from "@langchain/core/retrievers"; import { search } from "@orama/orama"; import { TFile } from "obsidian"; -import { DBOperations } from "./dbOperations"; export class HybridRetriever extends BaseRetriever { public lc_namespace = ["hybrid_retriever"]; @@ -170,33 +169,33 @@ export class HybridRetriever extends BaseRetriever { private async getExplicitChunks(noteFiles: TFile[]): Promise { const explicitChunks: Document[] = []; for (const noteFile of noteFiles) { - const db = await VectorStoreManager.getInstance().getDb(); - const hits = await DBOperations.getDocsByPath(db, noteFile.path); - if (hits) { - const matchingChunks = hits.map( - (hit: any) => - new Document({ - pageContent: hit.document.content, - metadata: { - ...hit.document.metadata, - score: hit.score, - path: hit.document.path, - mtime: hit.document.mtime, - ctime: hit.document.ctime, - title: hit.document.title, - id: hit.document.id, - embeddingModel: hit.document.embeddingModel, - tags: hit.document.tags, - extension: hit.document.extension, - created_at: hit.document.created_at, - nchars: hit.document.nchars, - // Expose chunkId explicitly for cross-engine deduplication - chunkId: hit.document.metadata?.chunkId, - }, - }) - ); - explicitChunks.push(...matchingChunks); + const docs = await VectorStoreManager.getInstance().getDocumentsByPath(noteFile.path); + if (docs.length === 0) { + continue; } + const matchingChunks = docs.map( + (doc) => + new Document({ + pageContent: doc.content, + metadata: { + ...doc.metadata, + score: 1, + path: doc.path, + mtime: doc.mtime, + ctime: doc.ctime, + title: doc.title, + id: doc.id, + embeddingModel: doc.embeddingModel, + tags: doc.tags, + extension: doc.extension, + created_at: doc.created_at, + nchars: doc.nchars, + // Expose chunkId explicitly for cross-engine deduplication + chunkId: doc.metadata?.chunkId, + }, + }) + ); + explicitChunks.push(...matchingChunks); } return explicitChunks; } diff --git a/src/search/indexBackend/OramaIndexBackend.ts b/src/search/indexBackend/OramaIndexBackend.ts new file mode 100644 index 00000000..60a12b9d --- /dev/null +++ b/src/search/indexBackend/OramaIndexBackend.ts @@ -0,0 +1,184 @@ +import { Embeddings } from "@langchain/core/embeddings"; +import { Orama } from "@orama/orama"; +import { App } from "obsidian"; +import { DBOperations } from "@/search/dbOperations"; +import type { + SemanticIndexBackend, + SemanticIndexDocument, +} from "@/search/indexBackend/SemanticIndexBackend"; + +/** + * Orama-backed implementation of the semantic index backend. + */ +export class OramaIndexBackend implements SemanticIndexBackend { + private dbOps: DBOperations; + + /** + * Create a new Orama backend tied to the current Obsidian app instance. + */ + constructor(app: App) { + this.dbOps = new DBOperations(app); + } + + /** + * Initialize the underlying Orama database. + */ + public async initialize(embeddingInstance: Embeddings | undefined): Promise { + await this.dbOps.initializeDB(embeddingInstance); + } + + /** + * Clear the Orama index and reset storage. + */ + public async clearIndex(embeddingInstance: Embeddings | undefined): Promise { + await this.dbOps.clearIndex(embeddingInstance); + } + + /** + * Insert or update a document in Orama. + */ + public async upsert(doc: SemanticIndexDocument): Promise { + return this.dbOps.upsert(doc); + } + + /** + * Remove all documents associated with a file path. + */ + public async removeByPath(path: string): Promise { + await this.dbOps.removeDocs(path); + } + + /** + * Return all indexed file paths. + */ + public async getIndexedFiles(): Promise { + return this.dbOps.getIndexedFiles(); + } + + /** + * Return the latest modified time across indexed files. + */ + public async getLatestFileMtime(): Promise { + return this.dbOps.getLatestFileMtime(); + } + + /** + * Return true when the index has no documents. + */ + public async isIndexEmpty(): Promise { + return this.dbOps.isIndexEmpty(); + } + + /** + * Return true when the specified file path has indexed documents. + */ + public async hasIndex(path: string): Promise { + return this.dbOps.hasIndex(path); + } + + /** + * Return all indexed documents for a given file path. + */ + public async getDocumentsByPath(path: string): Promise { + const db = this.dbOps.getDb(); + if (!db) { + throw new Error("Database is not loaded. Please restart the plugin."); + } + const hits = await DBOperations.getDocsByPath(db, path); + if (!hits) { + return []; + } + return hits.map((hit) => hit.document as SemanticIndexDocument); + } + + /** + * Detect embedding model changes and rebuild as needed. + */ + public async checkAndHandleEmbeddingModelChange(embeddingInstance: Embeddings): Promise { + return this.dbOps.checkAndHandleEmbeddingModelChange(embeddingInstance); + } + + /** + * Persist the Orama index to disk. + */ + public async save(): Promise { + await this.dbOps.saveDB(); + } + + /** + * Validate index integrity and track missing embeddings. + */ + public async checkIndexIntegrity(): Promise { + await this.dbOps.checkIndexIntegrity(); + } + + /** + * Remove stale documents that no longer belong in the index. + */ + public async garbageCollect(): Promise { + return this.dbOps.garbageCollect(); + } + + /** + * Mark a file as missing embeddings. + */ + public markFileMissingEmbeddings(path: string): void { + this.dbOps.markFileMissingEmbeddings(path); + } + + /** + * Clear tracked files missing embeddings. + */ + public clearFilesMissingEmbeddings(): void { + this.dbOps.clearFilesMissingEmbeddings(); + } + + /** + * Return tracked files missing embeddings. + */ + public getFilesMissingEmbeddings(): string[] { + return this.dbOps.getFilesMissingEmbeddings(); + } + + /** + * Mark index state as dirty without forcing a save. + */ + public markUnsavedChanges(): void { + this.dbOps.markUnsavedChanges(); + } + + /** + * Ensure pending changes are flushed on unload. + */ + public onunload(): void { + this.dbOps.onunload(); + } + + /** + * Return the underlying Orama database instance when available. + */ + public getDb(): Orama | undefined { + return this.dbOps.getDb(); + } + + /** + * Reinitialize the Orama DB when index sync settings change the storage path. + */ + public async reinitializeForIndexSyncChange( + embeddingInstance: Embeddings | undefined + ): Promise { + const newPath = await this.dbOps.getDbPath(); + const oldPath = this.dbOps.getCurrentDbPath(); + + if (oldPath !== newPath) { + await this.dbOps.initializeDB(embeddingInstance); + } + } + + /** + * Return the underlying DBOperations instance (legacy access). + */ + public getDbOperations(): DBOperations { + return this.dbOps; + } +} diff --git a/src/search/indexBackend/SemanticIndexBackend.ts b/src/search/indexBackend/SemanticIndexBackend.ts new file mode 100644 index 00000000..a4a2eacc --- /dev/null +++ b/src/search/indexBackend/SemanticIndexBackend.ts @@ -0,0 +1,115 @@ +import { Embeddings } from "@langchain/core/embeddings"; + +/** + * Represents a single semantic index document stored by a backend. + */ +export interface SemanticIndexDocument { + id: string; + title: string; + content: string; + embedding: number[]; + path: string; + embeddingModel: string; + created_at: number; + ctime: number; + mtime: number; + tags: string[]; + extension: string; + nchars: number; + metadata: Record; +} + +/** + * Defines the minimal contract for a semantic index backend. + */ +export interface SemanticIndexBackend { + /** + * Initialize the backend with the provided embeddings instance. + */ + initialize(embeddingInstance: Embeddings | undefined): Promise; + + /** + * Clear all indexed data and reset the backend state. + */ + clearIndex(embeddingInstance: Embeddings | undefined): Promise; + + /** + * Insert or update a semantic document in the backend. + */ + upsert(doc: SemanticIndexDocument): Promise; + + /** + * Remove all indexed documents associated with a file path. + */ + removeByPath(path: string): Promise; + + /** + * Return the list of indexed file paths. + */ + getIndexedFiles(): Promise; + + /** + * Return the latest modified time across indexed files. + */ + getLatestFileMtime(): Promise; + + /** + * Return true when the backend has no indexed content. + */ + isIndexEmpty(): Promise; + + /** + * Return true when a given file path has indexed content. + */ + hasIndex(path: string): Promise; + + /** + * Return all indexed documents for a given file path. + */ + getDocumentsByPath(path: string): Promise; + + /** + * Check for embedding model changes and trigger required rebuilds. + */ + checkAndHandleEmbeddingModelChange(embeddingInstance: Embeddings): Promise; + + /** + * Persist backend changes, if applicable. + */ + save(): Promise; + + /** + * Validate index integrity and mark files needing reindexing. + */ + checkIndexIntegrity(): Promise; + + /** + * Remove stale documents that no longer match vault state. + */ + garbageCollect(): Promise; + + /** + * Track a file path that failed embedding generation. + */ + markFileMissingEmbeddings(path: string): void; + + /** + * Clear tracked files missing embeddings. + */ + clearFilesMissingEmbeddings(): void; + + /** + * Return tracked files missing embeddings. + */ + getFilesMissingEmbeddings(): string[]; + + /** + * Mark backend data as dirty without saving immediately. + */ + markUnsavedChanges(): void; + + /** + * Flush or persist any pending backend work before unload. + */ + onunload(): void; +} diff --git a/src/search/indexEventHandler.ts b/src/search/indexEventHandler.ts index d3e8f544..03ce94f3 100644 --- a/src/search/indexEventHandler.ts +++ b/src/search/indexEventHandler.ts @@ -3,7 +3,7 @@ import { ChainType } from "@/chainFactory"; import { logInfo } from "@/logger"; import { getSettings, subscribeToSettingsChange } from "@/settings/model"; import { App, MarkdownView, Platform, TAbstractFile, TFile } from "obsidian"; -import { DBOperations } from "./dbOperations"; +import type { SemanticIndexBackend } from "./indexBackend/SemanticIndexBackend"; import { IndexOperations } from "./indexOperations"; import { getMatchingPatterns, shouldIndexFile } from "./searchUtils"; @@ -18,7 +18,7 @@ export class IndexEventHandler { constructor( private app: App, private indexOps: IndexOperations, - private dbOps: DBOperations + private indexBackend: SemanticIndexBackend ) { this.syncEventListeners(); subscribeToSettingsChange(() => { @@ -136,7 +136,7 @@ export class IndexEventHandler { return; } if (file instanceof TFile) { - await this.dbOps.removeDocs(file.path); + await this.indexBackend.removeByPath(file.path); } }; diff --git a/src/search/indexOperations.ts b/src/search/indexOperations.ts index d56be623..75f8ae63 100644 --- a/src/search/indexOperations.ts +++ b/src/search/indexOperations.ts @@ -6,7 +6,7 @@ import { getSettings, subscribeToSettingsChange } from "@/settings/model"; import { formatDateTime } from "@/utils"; import { MD5 } from "crypto-js"; import { App, Notice, TFile } from "obsidian"; -import { DBOperations } from "./dbOperations"; +import type { SemanticIndexBackend } from "./indexBackend/SemanticIndexBackend"; import { extractAppIgnoreSettings, getDecodedPatterns, @@ -41,7 +41,7 @@ export class IndexOperations { constructor( private app: App, - private dbOps: DBOperations, + private indexBackend: SemanticIndexBackend, private embeddingsManager: EmbeddingsManager ) { const settings = getSettings(); @@ -70,7 +70,8 @@ export class IndexOperations { } // Check for model change first - const modelChanged = await this.dbOps.checkAndHandleEmbeddingModelChange(embeddingInstance); + const modelChanged = + await this.indexBackend.checkAndHandleEmbeddingModelChange(embeddingInstance); if (modelChanged) { // If model changed, force a full reindex by setting overwrite to true overwrite = true; @@ -78,11 +79,11 @@ export class IndexOperations { // Clear index and tracking if overwrite is true if (overwrite) { - await this.dbOps.clearIndex(embeddingInstance); - this.dbOps.clearFilesMissingEmbeddings(); + await this.indexBackend.clearIndex(embeddingInstance); + this.indexBackend.clearFilesMissingEmbeddings(); } else { // Run garbage collection first to clean up stale documents - await this.dbOps.garbageCollect(); + await this.indexBackend.garbageCollect(); } const files = await this.getFilesToIndex(overwrite); @@ -95,7 +96,7 @@ export class IndexOperations { this.createIndexingNotice(); // Clear the missing embeddings list before starting new indexing - this.dbOps.clearFilesMissingEmbeddings(); + this.indexBackend.clearFilesMissingEmbeddings(); // New: Prepare all chunks first const allChunks = await this.prepareAllChunks(files); @@ -131,12 +132,12 @@ export class IndexOperations { // Skip documents with invalid embeddings if (!embedding || !Array.isArray(embedding) || embedding.length === 0) { logError(`Invalid embedding for document ${chunk.fileInfo.path}: ${embedding}`); - this.dbOps.markFileMissingEmbeddings(chunk.fileInfo.path); + this.indexBackend.markFileMissingEmbeddings(chunk.fileInfo.path); continue; } try { - await this.dbOps.upsert({ + await this.indexBackend.upsert({ ...chunk.fileInfo, id: this.getDocHash(chunk.content), content: chunk.content, @@ -152,7 +153,7 @@ export class IndexOperations { filePath: chunk.fileInfo.path, errors, }); - this.dbOps.markFileMissingEmbeddings(chunk.fileInfo.path); + this.indexBackend.markFileMissingEmbeddings(chunk.fileInfo.path); continue; } } @@ -168,7 +169,7 @@ export class IndexOperations { const currentCheckpoint = Math.floor(this.state.indexedCount / this.checkpointInterval); if (currentCheckpoint > previousCheckpoint) { - await this.dbOps.saveDB(); + await this.indexBackend.save(); console.log("Copilot index checkpoint save completed."); } } catch (err) { @@ -188,11 +189,11 @@ export class IndexOperations { // Run save and integrity check with setTimeout to ensure it's non-blocking setTimeout(() => { - this.dbOps - .saveDB() + this.indexBackend + .save() .then(() => { logInfo("Copilot index final save completed."); - this.dbOps.checkIndexIntegrity().catch((err) => { + this.indexBackend.checkIndexIntegrity().catch((err) => { logError("Background integrity check failed:", err); }); }) @@ -298,9 +299,9 @@ export class IndexOperations { } // Get currently indexed files and latest mtime - const indexedFilePaths = new Set(await this.dbOps.getIndexedFiles()); - const latestMtime = await this.dbOps.getLatestFileMtime(); - const filesMissingEmbeddings = new Set(this.dbOps.getFilesMissingEmbeddings()); + const indexedFilePaths = new Set(await this.indexBackend.getIndexedFiles()); + const latestMtime = await this.indexBackend.getLatestFileMtime(); + const filesMissingEmbeddings = new Set(this.indexBackend.getFilesMissingEmbeddings()); // Get all markdown files that should be indexed under current rules const filesToIndex = new Set(); @@ -551,10 +552,11 @@ export class IndexOperations { return; } - await this.dbOps.removeDocs(file.path); + await this.indexBackend.removeByPath(file.path); // Check for model change - const modelChanged = await this.dbOps.checkAndHandleEmbeddingModelChange(embeddingInstance); + const modelChanged = + await this.indexBackend.checkAndHandleEmbeddingModelChange(embeddingInstance); if (modelChanged) { await this.indexVaultToVectorStore(true); return; @@ -572,7 +574,7 @@ export class IndexOperations { // Save to database for (let i = 0; i < chunks.length; i++) { const chunk = chunks[i]; - await this.dbOps.upsert({ + await this.indexBackend.upsert({ ...chunk.fileInfo, id: this.getDocHash(chunk.content), content: chunk.content, @@ -583,7 +585,7 @@ export class IndexOperations { } // Mark that we have unsaved changes instead of saving immediately - this.dbOps.markUnsavedChanges(); + this.indexBackend.markUnsavedChanges(); if (getSettings().debug) { console.log(`Reindexed file: ${file.path}`); diff --git a/src/search/vectorStoreManager.ts b/src/search/vectorStoreManager.ts index 1fe1fa35..7c6b0c5f 100644 --- a/src/search/vectorStoreManager.ts +++ b/src/search/vectorStoreManager.ts @@ -5,7 +5,12 @@ import EmbeddingsManager from "@/LLMProviders/embeddingManager"; import { CopilotSettings, getSettings, subscribeToSettingsChange } from "@/settings/model"; import { Orama } from "@orama/orama"; import { Notice, Platform, TFile } from "obsidian"; -import { DBOperations } from "./dbOperations"; +import type { DBOperations } from "./dbOperations"; +import { OramaIndexBackend } from "./indexBackend/OramaIndexBackend"; +import type { + SemanticIndexBackend, + SemanticIndexDocument, +} from "./indexBackend/SemanticIndexBackend"; import { IndexEventHandler } from "./indexEventHandler"; import { IndexOperations } from "./indexOperations"; @@ -16,13 +21,15 @@ export default class VectorStoreManager { private initializationPromise: Promise; private lastKnownSettings: CopilotSettings | undefined; private embeddingsManager: EmbeddingsManager; - private dbOps: DBOperations; + private indexBackend: SemanticIndexBackend; + private oramaBackend: OramaIndexBackend; private constructor() { this.embeddingsManager = EmbeddingsManager.getInstance(); - this.dbOps = new DBOperations(app); - this.indexOps = new IndexOperations(app, this.dbOps, this.embeddingsManager); - this.eventHandler = new IndexEventHandler(app, this.indexOps, this.dbOps); + this.oramaBackend = new OramaIndexBackend(app); + this.indexBackend = this.oramaBackend; + this.indexOps = new IndexOperations(app, this.indexBackend, this.embeddingsManager); + this.eventHandler = new IndexEventHandler(app, this.indexOps, this.indexBackend); this.initializationPromise = this.initialize(); this.setupSettingsSubscription(); @@ -46,12 +53,8 @@ export default class VectorStoreManager { // Handle path changes (enableIndexSync) if (settings.enableIndexSync !== prevSettings?.enableIndexSync) { - const newPath = await this.dbOps.getDbPath(); - const oldPath = this.dbOps.getCurrentDbPath(); - - if (oldPath !== newPath) { - await this.dbOps.initializeDB(await this.embeddingsManager.getEmbeddingsAPI()); - } + const embeddingInstance = await this.embeddingsManager.getEmbeddingsAPI(); + await this.oramaBackend.reinitializeForIndexSyncChange(embeddingInstance); } }; @@ -71,7 +74,7 @@ export default class VectorStoreManager { while (retries > 0) { try { const embeddingAPI = await this.embeddingsManager.getEmbeddingsAPI(); - await this.dbOps.initializeDB(embeddingAPI); + await this.indexBackend.initialize(embeddingAPI); break; } catch (error) { if ( @@ -112,42 +115,53 @@ export default class VectorStoreManager { public async clearIndex(): Promise { await this.waitForInitialization(); - await this.dbOps.clearIndex(await this.embeddingsManager.getEmbeddingsAPI()); + await this.indexBackend.clearIndex(await this.embeddingsManager.getEmbeddingsAPI()); } public async garbageCollectVectorStore(): Promise { await this.waitForInitialization(); - return this.dbOps.garbageCollect(); + return this.indexBackend.garbageCollect(); } public async getIndexedFiles(): Promise { await this.waitForInitialization(); - return this.dbOps.getIndexedFiles(); + return this.indexBackend.getIndexedFiles(); } public async isIndexEmpty(): Promise { await this.waitForInitialization(); - return await this.dbOps.isIndexEmpty(); + return await this.indexBackend.isIndexEmpty(); } public async hasIndex(notePath: string): Promise { await this.waitForInitialization(); - return this.dbOps.hasIndex(notePath); + return this.indexBackend.hasIndex(notePath); + } + + /** + * Retrieve all indexed documents for a file path. + * + * @param notePath - The vault-relative file path to look up. + * @returns The list of indexed documents for the file. + */ + public async getDocumentsByPath(notePath: string): Promise { + await this.waitForInitialization(); + return this.indexBackend.getDocumentsByPath(notePath); } public onunload(): void { this.eventHandler.cleanup(); - this.dbOps.onunload(); + this.indexBackend.onunload(); } public async getDbOps(): Promise { await this.waitForInitialization(); - return this.dbOps; + return this.oramaBackend.getDbOperations(); } public async getDb(): Promise> { await this.waitForInitialization(); - const db = this.dbOps.getDb(); + const db = this.oramaBackend.getDb(); if (!db) { throw new Error("Database is not loaded. Please restart the plugin."); }