From e26561ba40dc0b0f7dcffef86c96a30347968434 Mon Sep 17 00:00:00 2001 From: Logan Yang Date: Sun, 17 Aug 2025 16:32:21 -0700 Subject: [PATCH] feat: introduce unified search system with legacy support (#1728) * feat: introduce unified search system with legacy support - Added a new `SearchSystem` interface and `SearchSystemFactory` to manage both legacy and v3 search implementations. - Updated various components to utilize the new factory for creating retrievers and managing indexing operations. - Introduced `useLegacySearch` setting to toggle between legacy and v3 search systems, enhancing user control over search functionality. - Refactored existing search-related code to streamline indexing and retrieval processes, improving overall performance and maintainability. - Updated documentation to reflect changes in search architecture and settings management. * feat: enhance V3 indexing responses with document count - Updated the V3SearchIndexer to return the document count after incremental and full indexing operations. - Modified success messages to include the number of documents indexed, improving user feedback and clarity on indexing results. * feat: improve error handling and user feedback for indexing operations - Enhanced the indexing process by adding error handling for both incremental and full indexing operations, providing users with clear notifications in case of failures. - Updated the `refreshVaultIndex` and `forceReindexVault` functions to log errors and display messages when indexing fails, improving overall user experience and transparency. - Refactored the main indexing logic in `CopilotPlugin` to ensure proper loading of indexes based on the selected strategy, enhancing functionality and reliability. * fix: update indexing logic for search system consistency - Changed comments to clarify that the index loading occurs even when search is disabled. - Refactored the index loading process to consistently use `SearchSystemFactory`, enhancing code clarity and maintainability. - Updated the condition for enabling semantic search to account for the new `useLegacySearch` setting, improving search functionality control. --- src/LLMProviders/chainManager.ts | 12 +- .../chainRunner/VaultQAChainRunner.ts | 6 +- src/LLMProviders/projectManager.ts | 2 +- src/commands/index.ts | 36 ++- .../chat-components/ChatControls.tsx | 26 +- src/constants.ts | 1 + src/main.ts | 36 +-- src/search/SearchSystem.ts | 260 ++++++++++++++++++ src/search/v3/README.md | 82 ++++++ src/settings/model.ts | 2 + src/settings/v2/components/QASettings.tsx | 91 ++++-- src/tools/SearchTools.ts | 49 +++- 12 files changed, 523 insertions(+), 80 deletions(-) create mode 100644 src/search/SearchSystem.ts diff --git a/src/LLMProviders/chainManager.ts b/src/LLMProviders/chainManager.ts index 383d4690..5e32b09d 100644 --- a/src/LLMProviders/chainManager.ts +++ b/src/LLMProviders/chainManager.ts @@ -16,7 +16,7 @@ import { VaultQAChainRunner, } from "@/LLMProviders/chainRunner/index"; import { logError, logInfo } from "@/logger"; -import { TieredLexicalRetriever } from "@/search/v3/TieredLexicalRetriever"; +import { SearchSystemFactory } from "@/search/SearchSystem"; import { getSettings, getSystemPrompt, subscribeToSettingsChange } from "@/settings/model"; import { ChatMessage } from "@/types/message"; import { findCustomModel, isOSeriesModel, isSupportedChain } from "@/utils"; @@ -206,7 +206,7 @@ export default class ChainManager { // TODO: VaultQAChainRunner now handles this directly without chains await this.initializeQAChain(options); - const retriever = new TieredLexicalRetriever(app, { + const retriever = SearchSystemFactory.createRetriever(app, { minSimilarityScore: 0.01, maxK: getSettings().maxSourceChunks, salientTerms: [], @@ -289,10 +289,10 @@ export default class ChainManager { private async initializeQAChain(options: SetChainOptions) { // Handle index refresh if needed if (options.refreshIndex) { - // New semantic index auto-refresh path - const { MemoryIndexManager } = await import("@/search/v3/MemoryIndexManager"); - await MemoryIndexManager.getInstance(this.app).indexVaultIncremental(); - await MemoryIndexManager.getInstance(this.app).ensureLoaded(); + // Auto-refresh index based on active search system + const { SearchSystemFactory } = await import("@/search/SearchSystem"); + await SearchSystemFactory.getIndexer().indexVaultIncremental(this.app); + await SearchSystemFactory.getIndexer().ensureLoaded(this.app); } } diff --git a/src/LLMProviders/chainRunner/VaultQAChainRunner.ts b/src/LLMProviders/chainRunner/VaultQAChainRunner.ts index 37595fbb..ef61bdba 100644 --- a/src/LLMProviders/chainRunner/VaultQAChainRunner.ts +++ b/src/LLMProviders/chainRunner/VaultQAChainRunner.ts @@ -1,6 +1,6 @@ import { ABORT_REASON, RETRIEVED_DOCUMENT_TAG } from "@/constants"; import { logInfo } from "@/logger"; -import { TieredLexicalRetriever } from "@/search/v3/TieredLexicalRetriever"; +import { SearchSystemFactory } from "@/search/SearchSystem"; import { getSettings, getSystemPrompt } from "@/settings/model"; import { ChatMessage } from "@/types/message"; import { @@ -43,8 +43,8 @@ export class VaultQAChainRunner extends BaseChainRunner { standaloneQuestion = userMessage.message; } - // Create retriever using tiered lexical approach - const retriever = new TieredLexicalRetriever(app, { + // Create retriever using factory (selects between legacy and v3 based on settings) + const retriever = SearchSystemFactory.createRetriever(app, { minSimilarityScore: 0.01, maxK: getSettings().maxSourceChunks, salientTerms: [], diff --git a/src/LLMProviders/projectManager.ts b/src/LLMProviders/projectManager.ts index a68fa0bc..f63e12ed 100644 --- a/src/LLMProviders/projectManager.ts +++ b/src/LLMProviders/projectManager.ts @@ -61,7 +61,7 @@ export default class ProjectManager { } const settings = getSettings(); const shouldAutoIndex = - settings.enableSemanticSearchV3 && + (settings.useLegacySearch || settings.enableSemanticSearchV3) && settings.indexVaultToVectorStore === VAULT_VECTOR_STORE_STRATEGY.ON_MODE_SWITCH && (getChainType() === ChainType.VAULT_QA_CHAIN || getChainType() === ChainType.COPILOT_PLUS_CHAIN); diff --git a/src/commands/index.ts b/src/commands/index.ts index 01658a5e..335a7c39 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -176,23 +176,39 @@ export function registerCommands( addCommand(plugin, COMMAND_IDS.INDEX_VAULT_TO_COPILOT_INDEX, async () => { try { - const { MemoryIndexManager } = await import("@/search/v3/MemoryIndexManager"); - await MemoryIndexManager.getInstance(plugin.app).indexVaultIncremental(); - await MemoryIndexManager.getInstance(plugin.app).ensureLoaded(); + const { SearchSystemFactory } = await import("@/search/SearchSystem"); + const result = await SearchSystemFactory.getIndexer().indexVaultIncremental(plugin.app); + + if (!result.success) { + logError("Index refresh failed:", result.message); + new Notice(`Failed to refresh index: ${result.message || "Unknown error"}`); + return; + } + + const count = result.documentCount ?? 0; + new Notice(`Index refreshed with ${count} documents.`); } catch (err) { - logError("Error building semantic memory index:", err); - new Notice("An error occurred while building the semantic memory index."); + logError("Error building index:", err); + new Notice("An error occurred while building the index."); } }); addCommand(plugin, COMMAND_IDS.FORCE_REINDEX_VAULT_TO_COPILOT_INDEX, async () => { try { - const { MemoryIndexManager } = await import("@/search/v3/MemoryIndexManager"); - await MemoryIndexManager.getInstance(plugin.app).indexVault(); - await MemoryIndexManager.getInstance(plugin.app).ensureLoaded(); + const { SearchSystemFactory } = await import("@/search/SearchSystem"); + const result = await SearchSystemFactory.getIndexer().indexVaultFull(plugin.app); + + if (!result.success) { + logError("Index rebuild failed:", result.message); + new Notice(`Failed to rebuild index: ${result.message || "Unknown error"}`); + return; + } + + const count = result.documentCount ?? 0; + new Notice(`Index rebuilt with ${count} documents.`); } catch (err) { - logError("Error rebuilding semantic memory index:", err); - new Notice("An error occurred while rebuilding the semantic memory index."); + logError("Error rebuilding index:", err); + new Notice("An error occurred while rebuilding the index."); } }); diff --git a/src/components/chat-components/ChatControls.tsx b/src/components/chat-components/ChatControls.tsx index 8ddf4159..06515670 100644 --- a/src/components/chat-components/ChatControls.tsx +++ b/src/components/chat-components/ChatControls.tsx @@ -9,7 +9,6 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip import { PLUS_UTM_MEDIUMS } from "@/constants"; import { logError } from "@/logger"; import { navigateToPlusPage, useIsPlusUser } from "@/plusUtils"; -import { MemoryIndexManager } from "@/search/v3/MemoryIndexManager"; import { updateSetting, useSettingsValue } from "@/settings/model"; import { Docs4LLMParser } from "@/tools/FileParserManager"; import { isRateLimitError } from "@/utils/rateLimitUtils"; @@ -32,9 +31,16 @@ import React from "react"; export async function refreshVaultIndex() { try { - // v3 semantic index: show progress and perform incremental update - await MemoryIndexManager.getInstance(app).indexVaultIncremental(); - await MemoryIndexManager.getInstance(app).ensureLoaded(); + const { SearchSystemFactory } = await import("@/search/SearchSystem"); + const result = await SearchSystemFactory.getIndexer().indexVaultIncremental(app); + + if (!result.success) { + new Notice(`Failed to refresh index: ${result.message || "Unknown error"}`); + return; + } + + const count = result.documentCount ?? 0; + new Notice(`Index refreshed with ${count} documents.`); } catch (error) { console.error("Error refreshing vault index:", error); new Notice("Failed to refresh vault index. Check console for details."); @@ -43,8 +49,16 @@ export async function refreshVaultIndex() { export async function forceReindexVault() { try { - await MemoryIndexManager.getInstance(app).indexVault(); - await MemoryIndexManager.getInstance(app).ensureLoaded(); + const { SearchSystemFactory } = await import("@/search/SearchSystem"); + const result = await SearchSystemFactory.getIndexer().indexVaultFull(app); + + if (!result.success) { + new Notice(`Failed to rebuild index: ${result.message || "Unknown error"}`); + return; + } + + const count = result.documentCount ?? 0; + new Notice(`Index rebuilt with ${count} documents.`); } catch (error) { console.error("Error force reindexing vault:", error); new Notice("Failed to force reindex vault. Check console for details."); diff --git a/src/constants.ts b/src/constants.ts index 90990a14..e26a91fe 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -718,6 +718,7 @@ export const DEFAULT_SETTINGS: CopilotSettings = { enableAutonomousAgent: false, enableCustomPromptTemplating: true, enableSemanticSearchV3: false, + useLegacySearch: false, suggestedDefaultCommands: false, autonomousAgentMaxIterations: 4, autonomousAgentEnabledToolIds: [ diff --git a/src/main.ts b/src/main.ts index e888c6b5..040ea76e 100644 --- a/src/main.ts +++ b/src/main.ts @@ -25,8 +25,8 @@ import { MessageRepository } from "@/core/MessageRepository"; import { encryptAllKeys } from "@/encryptionService"; import { logInfo, logWarn } from "@/logger"; import { checkIsPlusUser } from "@/plusUtils"; +import { SearchSystemFactory } from "@/search/SearchSystem"; import { MemoryIndexManager } from "@/search/v3/MemoryIndexManager"; -import { TieredLexicalRetriever } from "@/search/v3/TieredLexicalRetriever"; import { CopilotSettingTab } from "@/settings/SettingsPage"; import { getModelKeyFromModel, @@ -126,29 +126,31 @@ export default class CopilotPlugin extends Plugin { IntentAnalyzer.initTools(this.app.vault); - // Auto-index per strategy when semantic toggle is enabled + // Auto-index per strategy when search is enabled try { const settings = getSettings(); - const semanticOn = settings.enableSemanticSearchV3; - if (semanticOn) { + const searchEnabled = settings.useLegacySearch || settings.enableSemanticSearchV3; + if (searchEnabled) { const strategy = settings.indexVaultToVectorStore; const isMobileDisabled = settings.disableIndexOnMobile && (this.app as any).isMobile; if (!isMobileDisabled && strategy === VAULT_VECTOR_STORE_STRATEGY.ON_STARTUP) { - await MemoryIndexManager.getInstance(this.app).indexVaultIncremental(); - await MemoryIndexManager.getInstance(this.app).ensureLoaded(); - } else { - const loaded = isMobileDisabled - ? false - : await MemoryIndexManager.getInstance(this.app).loadIfExists(); - if (!loaded) { - logWarn("MemoryIndex: embedding index not found; falling back to full-text only"); - new Notice("embedding index doesn't exist, fall back to full-text search"); + // Use SearchSystemFactory for both legacy and v3 + const { SearchSystemFactory } = await import("@/search/SearchSystem"); + const result = await SearchSystemFactory.getIndexer().indexVaultIncremental(this.app); + if (!result.success) { + logWarn(`Index loading failed: ${result.message}`); } + } else if (!isMobileDisabled) { + // For non-ON_STARTUP strategies, ensure index is loaded for the active system + const { SearchSystemFactory } = await import("@/search/SearchSystem"); + await SearchSystemFactory.getIndexer().ensureLoaded(this.app); } } else { - // If semantic is off, we still try to load index for features that depend on it + // If search is off, we still try to load index for features that depend on it if (!(settings.disableIndexOnMobile && (this.app as any).isMobile)) { - await MemoryIndexManager.getInstance(this.app).loadIfExists(); + // Use SearchSystemFactory for consistency, even when search is disabled + const { SearchSystemFactory } = await import("@/search/SearchSystem"); + await SearchSystemFactory.getIndexer().ensureLoaded(this.app); } } } catch { @@ -170,7 +172,7 @@ export default class CopilotPlugin extends Plugin { // if semantic search v3 is enabled and file was modified while active try { const settings = getSettings(); - if (settings.enableSemanticSearchV3) { + if (settings.enableSemanticSearchV3 && !settings.useLegacySearch) { const { lastActiveFile, lastActiveMtime } = this.fileTracker; if ( lastActiveFile && @@ -476,7 +478,7 @@ export default class CopilotPlugin extends Plugin { } async customSearchDB(query: string, salientTerms: string[], textWeight: number): Promise { - const retriever = new TieredLexicalRetriever(app, { + const retriever = SearchSystemFactory.createRetriever(app, { minSimilarityScore: 0.3, maxK: 20, salientTerms: salientTerms, diff --git a/src/search/SearchSystem.ts b/src/search/SearchSystem.ts new file mode 100644 index 00000000..0ca927b5 --- /dev/null +++ b/src/search/SearchSystem.ts @@ -0,0 +1,260 @@ +import { logInfo, logWarn } from "@/logger"; +import { getSettings } from "@/settings/model"; +import { BaseRetriever } from "@langchain/core/retrievers"; +import { App, Notice } from "obsidian"; +import { HybridRetriever } from "./hybridRetriever"; +import { TieredLexicalRetriever } from "./v3/TieredLexicalRetriever"; + +/** + * Unified interface for search operations (retrieval and indexing) + */ +export interface IndexingResult { + success: boolean; + documentCount?: number; + message?: string; +} + +/** + * Interface for search indexing operations + */ +export interface SearchIndexer { + indexVaultIncremental(app: App): Promise; + indexVaultFull(app: App): Promise; + ensureLoaded(app: App): Promise; + clearIndex(app: App): Promise; +} + +/** + * Retriever options interface + */ +export interface RetrieverOptions { + minSimilarityScore?: number; + maxK: number; + salientTerms: string[]; + timeRange?: { startTime: number; endTime: number }; + textWeight?: number; + returnAll?: boolean; + useRerankerThreshold?: number; +} + +/** + * Abstract interface for search systems + */ +export interface SearchSystem { + readonly name: string; + readonly version: string; + readonly isLegacy: boolean; + + createRetriever(app: App, options: RetrieverOptions): BaseRetriever; + getIndexer(): SearchIndexer; + isSemanticSearchEnabled(): boolean; +} + +/** + * Legacy Orama-based search indexer implementation + */ +class LegacySearchIndexer implements SearchIndexer { + async indexVaultIncremental(app: App): Promise { + logInfo("LegacySearchIndexer: Incremental indexing"); + try { + const VectorStoreManager = (await import("@/search/vectorStoreManager")).default; + const count = await VectorStoreManager.getInstance().indexVaultToVectorStore(false); + return { success: true, documentCount: count }; + } catch (error) { + return { success: false, message: error.message }; + } + } + + async indexVaultFull(app: App): Promise { + logInfo("LegacySearchIndexer: Full indexing"); + try { + const VectorStoreManager = (await import("@/search/vectorStoreManager")).default; + const count = await VectorStoreManager.getInstance().indexVaultToVectorStore(true); + return { success: true, documentCount: count }; + } catch (error) { + return { success: false, message: error.message }; + } + } + + async ensureLoaded(app: App): Promise { + logInfo("LegacySearchIndexer: Ensuring index is loaded"); + const VectorStoreManager = (await import("@/search/vectorStoreManager")).default; + const isEmpty = await VectorStoreManager.getInstance().isIndexEmpty(); + if (isEmpty) { + new Notice("Legacy Orama index is empty. Please rebuild the index."); + } + } + + async clearIndex(app: App): Promise { + logInfo("LegacySearchIndexer: Clearing index"); + const VectorStoreManager = (await import("@/search/vectorStoreManager")).default; + await VectorStoreManager.getInstance().clearIndex(); + } +} + +/** + * V3 MemoryIndexManager-based search indexer implementation + */ +class V3SearchIndexer implements SearchIndexer { + async indexVaultIncremental(app: App): Promise { + logInfo("V3SearchIndexer: Incremental indexing"); + try { + const { MemoryIndexManager } = await import("@/search/v3/MemoryIndexManager"); + const documentCount = await MemoryIndexManager.getInstance(app).indexVaultIncremental(); + await MemoryIndexManager.getInstance(app).ensureLoaded(); + return { + success: true, + documentCount, + message: `V3 index updated with ${documentCount} documents`, + }; + } catch (error) { + return { success: false, message: error.message }; + } + } + + async indexVaultFull(app: App): Promise { + logInfo("V3SearchIndexer: Full indexing"); + try { + const { MemoryIndexManager } = await import("@/search/v3/MemoryIndexManager"); + const documentCount = await MemoryIndexManager.getInstance(app).indexVault(); + await MemoryIndexManager.getInstance(app).ensureLoaded(); + return { + success: true, + documentCount, + message: `V3 index rebuilt with ${documentCount} documents`, + }; + } catch (error) { + return { success: false, message: error.message }; + } + } + + async ensureLoaded(app: App): Promise { + logInfo("V3SearchIndexer: Ensuring index is loaded"); + const { MemoryIndexManager } = await import("@/search/v3/MemoryIndexManager"); + await MemoryIndexManager.getInstance(app).ensureLoaded(); + } + + async clearIndex(app: App): Promise { + logInfo("V3SearchIndexer: Clear not supported - index rebuilds on demand"); + new Notice("V3 search rebuilds index automatically on demand."); + } +} + +/** + * Legacy Orama-based search system implementation + * @deprecated Will be removed in v3.0.0 + */ +class LegacySearchSystem implements SearchSystem { + readonly name = "Orama"; + readonly version = "legacy"; + readonly isLegacy = true; + private indexer = new LegacySearchIndexer(); + + createRetriever(app: App, options: RetrieverOptions): BaseRetriever { + return new HybridRetriever({ + ...options, + minSimilarityScore: options.minSimilarityScore ?? 0.1, + }); + } + + getIndexer(): SearchIndexer { + return this.indexer; + } + + isSemanticSearchEnabled(): boolean { + // Legacy always has semantic capability via embeddings + return true; + } +} + +/** + * V3 TieredLexical search system implementation + */ +class V3SearchSystem implements SearchSystem { + readonly name = "TieredLexical"; + readonly version = "v3"; + readonly isLegacy = false; + private indexer = new V3SearchIndexer(); + + createRetriever(app: App, options: RetrieverOptions): BaseRetriever { + return new TieredLexicalRetriever(app, options); + } + + getIndexer(): SearchIndexer { + return this.indexer; + } + + isSemanticSearchEnabled(): boolean { + return getSettings().enableSemanticSearchV3; + } +} + +/** + * Factory for creating and managing search systems + */ +export class SearchSystemFactory { + private static searchSystem: SearchSystem | null = null; + + /** + * Get the current search system based on settings + */ + static getSearchSystem(): SearchSystem { + const settings = getSettings(); + + // Check if we need to recreate due to settings change + if (this.searchSystem) { + const isCurrentlyLegacy = this.searchSystem.isLegacy; + const shouldBeLegacy = settings.useLegacySearch; + + if (isCurrentlyLegacy !== shouldBeLegacy) { + this.searchSystem = null; + } + } + + if (!this.searchSystem) { + if (settings.useLegacySearch) { + logWarn( + "DEPRECATED: Legacy Orama search is deprecated and will be removed in a future version. Please switch to v3 search in QA Settings." + ); + this.searchSystem = new LegacySearchSystem(); + } else { + this.searchSystem = new V3SearchSystem(); + } + } + + return this.searchSystem; + } + + /** + * Reset the cached search system (use when settings change) + */ + static reset(): void { + this.searchSystem = null; + } + + /** + * Create a retriever using the current search system + */ + static createRetriever(app: App, options: RetrieverOptions): BaseRetriever { + const searchSystem = this.getSearchSystem(); + logInfo(`Creating retriever using ${searchSystem.name} (${searchSystem.version})`); + + try { + return searchSystem.createRetriever(app, options); + } catch (error) { + // Fallback to v3 if legacy fails + if (searchSystem.isLegacy) { + logWarn("Failed to create legacy retriever, falling back to v3"); + return new V3SearchSystem().createRetriever(app, options); + } + throw error; + } + } + + /** + * Get the indexer for the current search system + */ + static getIndexer(): SearchIndexer { + return this.getSearchSystem().getIndexer(); + } +} diff --git a/src/search/v3/README.md b/src/search/v3/README.md index f4d2f497..c91622c9 100644 --- a/src/search/v3/README.md +++ b/src/search/v3/README.md @@ -265,3 +265,85 @@ interface NoteIdRank { 7. **Explainable Rankings**: Track contributing factors for transparency 8. **Intelligent Graph Boost**: Only analyzes highly relevant results (≥75% similarity) to balance quality and performance 9. **Token-Aware Indexing**: Automatic chunk truncation when embedding token limits approached + +## Legacy Search Migration + +### Current State (v2.x) + +The codebase currently supports both legacy Orama-based search and v3 tiered lexical search via a toggle in settings. The implementation uses a factory pattern (`SearchSystemFactory`) to route between systems. + +### Architecture for Dual Support + +```typescript +// SearchSystem.ts provides unified interface +interface SearchSystem { + createRetriever(app: App, options: RetrieverOptions): BaseRetriever; + getIndexer(): SearchIndexer; + isSemanticSearchEnabled(): boolean; +} + +// Factory routes based on settings +SearchSystemFactory.getSearchSystem(); // Returns appropriate implementation +``` + +### Legacy Removal Steps (v3.0.0) + +When ready to remove legacy search support: + +#### 1. Remove Legacy Classes from `SearchSystem.ts` + +- Delete `LegacySearchIndexer` class +- Delete `LegacySearchSystem` class +- Update `SearchSystemFactory.getSearchSystem()` to always return `V3SearchSystem` + +#### 2. Remove Settings Properties + +- Delete `useLegacySearch` from: + - `src/settings/model.ts` + - `src/constants.ts` (DEFAULT_SETTINGS) +- Delete `numPartitions` setting (legacy partitioning) + +#### 3. Remove UI Components + +- Remove legacy search toggle from `src/settings/v2/components/QASettings.tsx` +- Remove partition dropdown UI +- Clean up conditional rendering logic + +#### 4. Delete Legacy Files + +```bash +# Core legacy files to remove +src/search/hybridRetriever.ts +src/search/vectorStoreManager.ts +src/search/chunkedStorage.ts +src/search/dbOperations.ts +src/search/indexOperations.ts +src/search/indexEventHandler.ts +``` + +#### 5. Clean Package Dependencies + +Remove from `package.json`: + +- `@orama/orama` +- Any Orama-related plugins + +#### 6. Update Documentation + +- Remove legacy search references from user documentation +- Update API documentation +- Remove deprecation warnings + +### Benefits After Removal + +- **Reduced Bundle Size**: ~200KB reduction from removing Orama +- **Simpler Codebase**: Single search implementation path +- **Better Performance**: No conditional checks or dynamic imports +- **Cleaner Architecture**: Direct use of v3 components without abstraction layer +- **Easier Maintenance**: One search system to maintain and optimize + +### Migration Timeline + +1. **v2.x** (Current): Dual support with deprecation warnings +2. **v2.x+1**: Legacy search disabled by default, requires opt-in +3. **v3.0.0**: Complete removal of legacy search code diff --git a/src/settings/model.ts b/src/settings/model.ts index 34f5c0bf..f3db88f3 100644 --- a/src/settings/model.ts +++ b/src/settings/model.ts @@ -112,6 +112,8 @@ export interface CopilotSettings { enableCustomPromptTemplating: boolean; /** Enable semantic stage in v3 search (requires Memory Index JSONL) */ enableSemanticSearchV3: boolean; + /** Use legacy HybridRetriever with Orama instead of v3 search */ + useLegacySearch: boolean; /** * Weight for semantic search in hybrid retrieval (0-1 range) * - 0 = fully lexical (keyword-based) search diff --git a/src/settings/v2/components/QASettings.tsx b/src/settings/v2/components/QASettings.tsx index 3f286508..5589c786 100644 --- a/src/settings/v2/components/QASettings.tsx +++ b/src/settings/v2/components/QASettings.tsx @@ -19,14 +19,16 @@ export const QASettings: React.FC = () => {
- {/* Enable Semantic Search (v3) */} - updateSetting("enableSemanticSearchV3", checked)} - /> + {/* Enable Semantic Search (v3) - Only show when not using legacy search */} + {!settings.useLegacySearch && ( + updateSetting("enableSemanticSearchV3", checked)} + /> + )} {/* Auto-Index Strategy */} { onChange={(value) => updateSetting("embeddingBatchSize", value)} /> - {/* Semantic vs Lexical Weight */} - {settings.enableSemanticSearchV3 && ( + {/* Semantic vs Lexical Weight - Only show when using v3 search with semantic enabled */} + {!settings.useLegacySearch && settings.enableSemanticSearchV3 && ( { /> )} - {/* Lexical Search RAM Limit */} - updateSetting("lexicalSearchRamLimit", value)} - suffix=" MB" - /> - - {/* Number of Partitions removed (auto-managed in v3) */} + {/* Lexical Search RAM Limit - Only show when using v3 search */} + {!settings.useLegacySearch && ( + updateSetting("lexicalSearchRamLimit", value)} + suffix=" MB" + /> + )} {/* Exclusions */} { checked={settings.disableIndexOnMobile} onCheckedChange={(checked) => updateSetting("disableIndexOnMobile", checked)} /> + + {/* Legacy Search Section - At the bottom */} +
+

Legacy Search Options

+ + {/* Use Legacy Search */} + { + updateSetting("useLegacySearch", checked); + // Disable v3 semantic search when enabling legacy search + if (checked) { + updateSetting("enableSemanticSearchV3", false); + } + }} + /> + + {/* Number of Partitions - Only show when using legacy search */} + {settings.useLegacySearch && ( + updateSetting("numPartitions", Number(value))} + options={[ + { label: "1", value: "1" }, + { label: "2", value: "2" }, + { label: "4", value: "4" }, + { label: "8", value: "8" }, + { label: "16", value: "16" }, + { label: "32", value: "32" }, + { label: "40", value: "40" }, + ]} + placeholder="Select partitions" + /> + )} +
diff --git a/src/tools/SearchTools.ts b/src/tools/SearchTools.ts index 9a68fdce..6adf065f 100644 --- a/src/tools/SearchTools.ts +++ b/src/tools/SearchTools.ts @@ -2,7 +2,7 @@ import { getStandaloneQuestion } from "@/chainUtils"; import { TEXT_WEIGHT } from "@/constants"; import { BrevilabsClient } from "@/LLMProviders/brevilabsClient"; import { logInfo } from "@/logger"; -import { TieredLexicalRetriever } from "@/search/v3/TieredLexicalRetriever"; +import { SearchSystemFactory } from "@/search/SearchSystem"; import { getSettings } from "@/settings/model"; import { z } from "zod"; import { createTool, SimpleTool } from "./SimpleTool"; @@ -35,8 +35,8 @@ const localSearchTool = createTool({ logInfo(`returnAll: ${returnAll}`); - // Use tiered lexical retriever for multi-stage search - const retriever = new TieredLexicalRetriever(app, { + // Use retriever factory to get appropriate retriever based on settings + const retriever = SearchSystemFactory.createRetriever(app, { minSimilarityScore: returnAll ? 0.0 : 0.1, maxK: effectiveMaxK, salientTerms, @@ -85,21 +85,44 @@ const localSearchTool = createTool({ }, }); -// Note: indexTool is kept for backward compatibility but is no longer used with v3 search +// Note: indexTool behavior depends on which retriever is active const indexTool = createTool({ name: "indexVault", description: "Index the vault to the Copilot index", schema: z.void(), // No parameters handler: async () => { - // Tiered lexical retriever doesn't require manual indexing - it builds ephemeral indexes on demand - const indexResultPrompt = `The tiered lexical retriever builds indexes on demand and doesn't require manual indexing.\n`; - return ( - indexResultPrompt + - JSON.stringify({ - success: true, - message: "Tiered lexical retriever uses on-demand indexing. No manual indexing required.", - }) - ); + const settings = getSettings(); + if (settings.useLegacySearch) { + // Legacy search uses persistent Orama index - trigger actual indexing + try { + const VectorStoreManager = (await import("@/search/vectorStoreManager")).default; + const count = await VectorStoreManager.getInstance().indexVaultToVectorStore(); + const indexResultPrompt = `Legacy search (Orama) index refreshed with ${count} documents.\n`; + return ( + indexResultPrompt + + JSON.stringify({ + success: true, + message: `Legacy Orama index has been refreshed with ${count} documents.`, + documentCount: count, + }) + ); + } catch (error) { + return JSON.stringify({ + success: false, + message: `Failed to index with legacy search: ${error.message}`, + }); + } + } else { + // V3 search builds indexes on demand + const indexResultPrompt = `The tiered lexical retriever builds indexes on demand and doesn't require manual indexing.\n`; + return ( + indexResultPrompt + + JSON.stringify({ + success: true, + message: "Tiered lexical retriever uses on-demand indexing. No manual indexing required.", + }) + ); + } }, isBackground: true, });