mirror of
https://github.com/logancyang/obsidian-copilot.git
synced 2026-07-22 07:50:24 +00:00
feat: introduce unified search system with legacy support (#1728)
* feat: introduce unified search system with legacy support - Added a new `SearchSystem` interface and `SearchSystemFactory` to manage both legacy and v3 search implementations. - Updated various components to utilize the new factory for creating retrievers and managing indexing operations. - Introduced `useLegacySearch` setting to toggle between legacy and v3 search systems, enhancing user control over search functionality. - Refactored existing search-related code to streamline indexing and retrieval processes, improving overall performance and maintainability. - Updated documentation to reflect changes in search architecture and settings management. * feat: enhance V3 indexing responses with document count - Updated the V3SearchIndexer to return the document count after incremental and full indexing operations. - Modified success messages to include the number of documents indexed, improving user feedback and clarity on indexing results. * feat: improve error handling and user feedback for indexing operations - Enhanced the indexing process by adding error handling for both incremental and full indexing operations, providing users with clear notifications in case of failures. - Updated the `refreshVaultIndex` and `forceReindexVault` functions to log errors and display messages when indexing fails, improving overall user experience and transparency. - Refactored the main indexing logic in `CopilotPlugin` to ensure proper loading of indexes based on the selected strategy, enhancing functionality and reliability. * fix: update indexing logic for search system consistency - Changed comments to clarify that the index loading occurs even when search is disabled. - Refactored the index loading process to consistently use `SearchSystemFactory`, enhancing code clarity and maintainability. - Updated the condition for enabling semantic search to account for the new `useLegacySearch` setting, improving search functionality control.
This commit is contained in:
Logan Yang
GitHub
parent
214fd8d44d
commit
e26561ba40
12 changed files with 523 additions and 80 deletions
|
|
@ -16,7 +16,7 @@ import {
|
|||
VaultQAChainRunner,
|
||||
} from "@/LLMProviders/chainRunner/index";
|
||||
import { logError, logInfo } from "@/logger";
|
||||
import { TieredLexicalRetriever } from "@/search/v3/TieredLexicalRetriever";
|
||||
import { SearchSystemFactory } from "@/search/SearchSystem";
|
||||
import { getSettings, getSystemPrompt, subscribeToSettingsChange } from "@/settings/model";
|
||||
import { ChatMessage } from "@/types/message";
|
||||
import { findCustomModel, isOSeriesModel, isSupportedChain } from "@/utils";
|
||||
|
|
@ -206,7 +206,7 @@ export default class ChainManager {
|
|||
// TODO: VaultQAChainRunner now handles this directly without chains
|
||||
await this.initializeQAChain(options);
|
||||
|
||||
const retriever = new TieredLexicalRetriever(app, {
|
||||
const retriever = SearchSystemFactory.createRetriever(app, {
|
||||
minSimilarityScore: 0.01,
|
||||
maxK: getSettings().maxSourceChunks,
|
||||
salientTerms: [],
|
||||
|
|
@ -289,10 +289,10 @@ export default class ChainManager {
|
|||
private async initializeQAChain(options: SetChainOptions) {
|
||||
// Handle index refresh if needed
|
||||
if (options.refreshIndex) {
|
||||
// New semantic index auto-refresh path
|
||||
const { MemoryIndexManager } = await import("@/search/v3/MemoryIndexManager");
|
||||
await MemoryIndexManager.getInstance(this.app).indexVaultIncremental();
|
||||
await MemoryIndexManager.getInstance(this.app).ensureLoaded();
|
||||
// Auto-refresh index based on active search system
|
||||
const { SearchSystemFactory } = await import("@/search/SearchSystem");
|
||||
await SearchSystemFactory.getIndexer().indexVaultIncremental(this.app);
|
||||
await SearchSystemFactory.getIndexer().ensureLoaded(this.app);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { ABORT_REASON, RETRIEVED_DOCUMENT_TAG } from "@/constants";
|
||||
import { logInfo } from "@/logger";
|
||||
import { TieredLexicalRetriever } from "@/search/v3/TieredLexicalRetriever";
|
||||
import { SearchSystemFactory } from "@/search/SearchSystem";
|
||||
import { getSettings, getSystemPrompt } from "@/settings/model";
|
||||
import { ChatMessage } from "@/types/message";
|
||||
import {
|
||||
|
|
@ -43,8 +43,8 @@ export class VaultQAChainRunner extends BaseChainRunner {
|
|||
standaloneQuestion = userMessage.message;
|
||||
}
|
||||
|
||||
// Create retriever using tiered lexical approach
|
||||
const retriever = new TieredLexicalRetriever(app, {
|
||||
// Create retriever using factory (selects between legacy and v3 based on settings)
|
||||
const retriever = SearchSystemFactory.createRetriever(app, {
|
||||
minSimilarityScore: 0.01,
|
||||
maxK: getSettings().maxSourceChunks,
|
||||
salientTerms: [],
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ export default class ProjectManager {
|
|||
}
|
||||
const settings = getSettings();
|
||||
const shouldAutoIndex =
|
||||
settings.enableSemanticSearchV3 &&
|
||||
(settings.useLegacySearch || settings.enableSemanticSearchV3) &&
|
||||
settings.indexVaultToVectorStore === VAULT_VECTOR_STORE_STRATEGY.ON_MODE_SWITCH &&
|
||||
(getChainType() === ChainType.VAULT_QA_CHAIN ||
|
||||
getChainType() === ChainType.COPILOT_PLUS_CHAIN);
|
||||
|
|
|
|||
|
|
@ -176,23 +176,39 @@ export function registerCommands(
|
|||
|
||||
addCommand(plugin, COMMAND_IDS.INDEX_VAULT_TO_COPILOT_INDEX, async () => {
|
||||
try {
|
||||
const { MemoryIndexManager } = await import("@/search/v3/MemoryIndexManager");
|
||||
await MemoryIndexManager.getInstance(plugin.app).indexVaultIncremental();
|
||||
await MemoryIndexManager.getInstance(plugin.app).ensureLoaded();
|
||||
const { SearchSystemFactory } = await import("@/search/SearchSystem");
|
||||
const result = await SearchSystemFactory.getIndexer().indexVaultIncremental(plugin.app);
|
||||
|
||||
if (!result.success) {
|
||||
logError("Index refresh failed:", result.message);
|
||||
new Notice(`Failed to refresh index: ${result.message || "Unknown error"}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const count = result.documentCount ?? 0;
|
||||
new Notice(`Index refreshed with ${count} documents.`);
|
||||
} catch (err) {
|
||||
logError("Error building semantic memory index:", err);
|
||||
new Notice("An error occurred while building the semantic memory index.");
|
||||
logError("Error building index:", err);
|
||||
new Notice("An error occurred while building the index.");
|
||||
}
|
||||
});
|
||||
|
||||
addCommand(plugin, COMMAND_IDS.FORCE_REINDEX_VAULT_TO_COPILOT_INDEX, async () => {
|
||||
try {
|
||||
const { MemoryIndexManager } = await import("@/search/v3/MemoryIndexManager");
|
||||
await MemoryIndexManager.getInstance(plugin.app).indexVault();
|
||||
await MemoryIndexManager.getInstance(plugin.app).ensureLoaded();
|
||||
const { SearchSystemFactory } = await import("@/search/SearchSystem");
|
||||
const result = await SearchSystemFactory.getIndexer().indexVaultFull(plugin.app);
|
||||
|
||||
if (!result.success) {
|
||||
logError("Index rebuild failed:", result.message);
|
||||
new Notice(`Failed to rebuild index: ${result.message || "Unknown error"}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const count = result.documentCount ?? 0;
|
||||
new Notice(`Index rebuilt with ${count} documents.`);
|
||||
} catch (err) {
|
||||
logError("Error rebuilding semantic memory index:", err);
|
||||
new Notice("An error occurred while rebuilding the semantic memory index.");
|
||||
logError("Error rebuilding index:", err);
|
||||
new Notice("An error occurred while rebuilding the index.");
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip
|
|||
import { PLUS_UTM_MEDIUMS } from "@/constants";
|
||||
import { logError } from "@/logger";
|
||||
import { navigateToPlusPage, useIsPlusUser } from "@/plusUtils";
|
||||
import { MemoryIndexManager } from "@/search/v3/MemoryIndexManager";
|
||||
import { updateSetting, useSettingsValue } from "@/settings/model";
|
||||
import { Docs4LLMParser } from "@/tools/FileParserManager";
|
||||
import { isRateLimitError } from "@/utils/rateLimitUtils";
|
||||
|
|
@ -32,9 +31,16 @@ import React from "react";
|
|||
|
||||
export async function refreshVaultIndex() {
|
||||
try {
|
||||
// v3 semantic index: show progress and perform incremental update
|
||||
await MemoryIndexManager.getInstance(app).indexVaultIncremental();
|
||||
await MemoryIndexManager.getInstance(app).ensureLoaded();
|
||||
const { SearchSystemFactory } = await import("@/search/SearchSystem");
|
||||
const result = await SearchSystemFactory.getIndexer().indexVaultIncremental(app);
|
||||
|
||||
if (!result.success) {
|
||||
new Notice(`Failed to refresh index: ${result.message || "Unknown error"}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const count = result.documentCount ?? 0;
|
||||
new Notice(`Index refreshed with ${count} documents.`);
|
||||
} catch (error) {
|
||||
console.error("Error refreshing vault index:", error);
|
||||
new Notice("Failed to refresh vault index. Check console for details.");
|
||||
|
|
@ -43,8 +49,16 @@ export async function refreshVaultIndex() {
|
|||
|
||||
export async function forceReindexVault() {
|
||||
try {
|
||||
await MemoryIndexManager.getInstance(app).indexVault();
|
||||
await MemoryIndexManager.getInstance(app).ensureLoaded();
|
||||
const { SearchSystemFactory } = await import("@/search/SearchSystem");
|
||||
const result = await SearchSystemFactory.getIndexer().indexVaultFull(app);
|
||||
|
||||
if (!result.success) {
|
||||
new Notice(`Failed to rebuild index: ${result.message || "Unknown error"}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const count = result.documentCount ?? 0;
|
||||
new Notice(`Index rebuilt with ${count} documents.`);
|
||||
} catch (error) {
|
||||
console.error("Error force reindexing vault:", error);
|
||||
new Notice("Failed to force reindex vault. Check console for details.");
|
||||
|
|
|
|||
|
|
@ -718,6 +718,7 @@ export const DEFAULT_SETTINGS: CopilotSettings = {
|
|||
enableAutonomousAgent: false,
|
||||
enableCustomPromptTemplating: true,
|
||||
enableSemanticSearchV3: false,
|
||||
useLegacySearch: false,
|
||||
suggestedDefaultCommands: false,
|
||||
autonomousAgentMaxIterations: 4,
|
||||
autonomousAgentEnabledToolIds: [
|
||||
|
|
|
|||
36
src/main.ts
36
src/main.ts
|
|
@ -25,8 +25,8 @@ import { MessageRepository } from "@/core/MessageRepository";
|
|||
import { encryptAllKeys } from "@/encryptionService";
|
||||
import { logInfo, logWarn } from "@/logger";
|
||||
import { checkIsPlusUser } from "@/plusUtils";
|
||||
import { SearchSystemFactory } from "@/search/SearchSystem";
|
||||
import { MemoryIndexManager } from "@/search/v3/MemoryIndexManager";
|
||||
import { TieredLexicalRetriever } from "@/search/v3/TieredLexicalRetriever";
|
||||
import { CopilotSettingTab } from "@/settings/SettingsPage";
|
||||
import {
|
||||
getModelKeyFromModel,
|
||||
|
|
@ -126,29 +126,31 @@ export default class CopilotPlugin extends Plugin {
|
|||
|
||||
IntentAnalyzer.initTools(this.app.vault);
|
||||
|
||||
// Auto-index per strategy when semantic toggle is enabled
|
||||
// Auto-index per strategy when search is enabled
|
||||
try {
|
||||
const settings = getSettings();
|
||||
const semanticOn = settings.enableSemanticSearchV3;
|
||||
if (semanticOn) {
|
||||
const searchEnabled = settings.useLegacySearch || settings.enableSemanticSearchV3;
|
||||
if (searchEnabled) {
|
||||
const strategy = settings.indexVaultToVectorStore;
|
||||
const isMobileDisabled = settings.disableIndexOnMobile && (this.app as any).isMobile;
|
||||
if (!isMobileDisabled && strategy === VAULT_VECTOR_STORE_STRATEGY.ON_STARTUP) {
|
||||
await MemoryIndexManager.getInstance(this.app).indexVaultIncremental();
|
||||
await MemoryIndexManager.getInstance(this.app).ensureLoaded();
|
||||
} else {
|
||||
const loaded = isMobileDisabled
|
||||
? false
|
||||
: await MemoryIndexManager.getInstance(this.app).loadIfExists();
|
||||
if (!loaded) {
|
||||
logWarn("MemoryIndex: embedding index not found; falling back to full-text only");
|
||||
new Notice("embedding index doesn't exist, fall back to full-text search");
|
||||
// Use SearchSystemFactory for both legacy and v3
|
||||
const { SearchSystemFactory } = await import("@/search/SearchSystem");
|
||||
const result = await SearchSystemFactory.getIndexer().indexVaultIncremental(this.app);
|
||||
if (!result.success) {
|
||||
logWarn(`Index loading failed: ${result.message}`);
|
||||
}
|
||||
} else if (!isMobileDisabled) {
|
||||
// For non-ON_STARTUP strategies, ensure index is loaded for the active system
|
||||
const { SearchSystemFactory } = await import("@/search/SearchSystem");
|
||||
await SearchSystemFactory.getIndexer().ensureLoaded(this.app);
|
||||
}
|
||||
} else {
|
||||
// If semantic is off, we still try to load index for features that depend on it
|
||||
// If search is off, we still try to load index for features that depend on it
|
||||
if (!(settings.disableIndexOnMobile && (this.app as any).isMobile)) {
|
||||
await MemoryIndexManager.getInstance(this.app).loadIfExists();
|
||||
// Use SearchSystemFactory for consistency, even when search is disabled
|
||||
const { SearchSystemFactory } = await import("@/search/SearchSystem");
|
||||
await SearchSystemFactory.getIndexer().ensureLoaded(this.app);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
|
|
@ -170,7 +172,7 @@ export default class CopilotPlugin extends Plugin {
|
|||
// if semantic search v3 is enabled and file was modified while active
|
||||
try {
|
||||
const settings = getSettings();
|
||||
if (settings.enableSemanticSearchV3) {
|
||||
if (settings.enableSemanticSearchV3 && !settings.useLegacySearch) {
|
||||
const { lastActiveFile, lastActiveMtime } = this.fileTracker;
|
||||
if (
|
||||
lastActiveFile &&
|
||||
|
|
@ -476,7 +478,7 @@ export default class CopilotPlugin extends Plugin {
|
|||
}
|
||||
|
||||
async customSearchDB(query: string, salientTerms: string[], textWeight: number): Promise<any[]> {
|
||||
const retriever = new TieredLexicalRetriever(app, {
|
||||
const retriever = SearchSystemFactory.createRetriever(app, {
|
||||
minSimilarityScore: 0.3,
|
||||
maxK: 20,
|
||||
salientTerms: salientTerms,
|
||||
|
|
|
|||
260
src/search/SearchSystem.ts
Normal file
260
src/search/SearchSystem.ts
Normal file
|
|
@ -0,0 +1,260 @@
|
|||
import { logInfo, logWarn } from "@/logger";
|
||||
import { getSettings } from "@/settings/model";
|
||||
import { BaseRetriever } from "@langchain/core/retrievers";
|
||||
import { App, Notice } from "obsidian";
|
||||
import { HybridRetriever } from "./hybridRetriever";
|
||||
import { TieredLexicalRetriever } from "./v3/TieredLexicalRetriever";
|
||||
|
||||
/**
|
||||
* Unified interface for search operations (retrieval and indexing)
|
||||
*/
|
||||
export interface IndexingResult {
|
||||
success: boolean;
|
||||
documentCount?: number;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Interface for search indexing operations
|
||||
*/
|
||||
export interface SearchIndexer {
|
||||
indexVaultIncremental(app: App): Promise<IndexingResult>;
|
||||
indexVaultFull(app: App): Promise<IndexingResult>;
|
||||
ensureLoaded(app: App): Promise<void>;
|
||||
clearIndex(app: App): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retriever options interface
|
||||
*/
|
||||
export interface RetrieverOptions {
|
||||
minSimilarityScore?: number;
|
||||
maxK: number;
|
||||
salientTerms: string[];
|
||||
timeRange?: { startTime: number; endTime: number };
|
||||
textWeight?: number;
|
||||
returnAll?: boolean;
|
||||
useRerankerThreshold?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract interface for search systems
|
||||
*/
|
||||
export interface SearchSystem {
|
||||
readonly name: string;
|
||||
readonly version: string;
|
||||
readonly isLegacy: boolean;
|
||||
|
||||
createRetriever(app: App, options: RetrieverOptions): BaseRetriever;
|
||||
getIndexer(): SearchIndexer;
|
||||
isSemanticSearchEnabled(): boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy Orama-based search indexer implementation
|
||||
*/
|
||||
class LegacySearchIndexer implements SearchIndexer {
|
||||
async indexVaultIncremental(app: App): Promise<IndexingResult> {
|
||||
logInfo("LegacySearchIndexer: Incremental indexing");
|
||||
try {
|
||||
const VectorStoreManager = (await import("@/search/vectorStoreManager")).default;
|
||||
const count = await VectorStoreManager.getInstance().indexVaultToVectorStore(false);
|
||||
return { success: true, documentCount: count };
|
||||
} catch (error) {
|
||||
return { success: false, message: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
async indexVaultFull(app: App): Promise<IndexingResult> {
|
||||
logInfo("LegacySearchIndexer: Full indexing");
|
||||
try {
|
||||
const VectorStoreManager = (await import("@/search/vectorStoreManager")).default;
|
||||
const count = await VectorStoreManager.getInstance().indexVaultToVectorStore(true);
|
||||
return { success: true, documentCount: count };
|
||||
} catch (error) {
|
||||
return { success: false, message: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
async ensureLoaded(app: App): Promise<void> {
|
||||
logInfo("LegacySearchIndexer: Ensuring index is loaded");
|
||||
const VectorStoreManager = (await import("@/search/vectorStoreManager")).default;
|
||||
const isEmpty = await VectorStoreManager.getInstance().isIndexEmpty();
|
||||
if (isEmpty) {
|
||||
new Notice("Legacy Orama index is empty. Please rebuild the index.");
|
||||
}
|
||||
}
|
||||
|
||||
async clearIndex(app: App): Promise<void> {
|
||||
logInfo("LegacySearchIndexer: Clearing index");
|
||||
const VectorStoreManager = (await import("@/search/vectorStoreManager")).default;
|
||||
await VectorStoreManager.getInstance().clearIndex();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* V3 MemoryIndexManager-based search indexer implementation
|
||||
*/
|
||||
class V3SearchIndexer implements SearchIndexer {
|
||||
async indexVaultIncremental(app: App): Promise<IndexingResult> {
|
||||
logInfo("V3SearchIndexer: Incremental indexing");
|
||||
try {
|
||||
const { MemoryIndexManager } = await import("@/search/v3/MemoryIndexManager");
|
||||
const documentCount = await MemoryIndexManager.getInstance(app).indexVaultIncremental();
|
||||
await MemoryIndexManager.getInstance(app).ensureLoaded();
|
||||
return {
|
||||
success: true,
|
||||
documentCount,
|
||||
message: `V3 index updated with ${documentCount} documents`,
|
||||
};
|
||||
} catch (error) {
|
||||
return { success: false, message: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
async indexVaultFull(app: App): Promise<IndexingResult> {
|
||||
logInfo("V3SearchIndexer: Full indexing");
|
||||
try {
|
||||
const { MemoryIndexManager } = await import("@/search/v3/MemoryIndexManager");
|
||||
const documentCount = await MemoryIndexManager.getInstance(app).indexVault();
|
||||
await MemoryIndexManager.getInstance(app).ensureLoaded();
|
||||
return {
|
||||
success: true,
|
||||
documentCount,
|
||||
message: `V3 index rebuilt with ${documentCount} documents`,
|
||||
};
|
||||
} catch (error) {
|
||||
return { success: false, message: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
async ensureLoaded(app: App): Promise<void> {
|
||||
logInfo("V3SearchIndexer: Ensuring index is loaded");
|
||||
const { MemoryIndexManager } = await import("@/search/v3/MemoryIndexManager");
|
||||
await MemoryIndexManager.getInstance(app).ensureLoaded();
|
||||
}
|
||||
|
||||
async clearIndex(app: App): Promise<void> {
|
||||
logInfo("V3SearchIndexer: Clear not supported - index rebuilds on demand");
|
||||
new Notice("V3 search rebuilds index automatically on demand.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy Orama-based search system implementation
|
||||
* @deprecated Will be removed in v3.0.0
|
||||
*/
|
||||
class LegacySearchSystem implements SearchSystem {
|
||||
readonly name = "Orama";
|
||||
readonly version = "legacy";
|
||||
readonly isLegacy = true;
|
||||
private indexer = new LegacySearchIndexer();
|
||||
|
||||
createRetriever(app: App, options: RetrieverOptions): BaseRetriever {
|
||||
return new HybridRetriever({
|
||||
...options,
|
||||
minSimilarityScore: options.minSimilarityScore ?? 0.1,
|
||||
});
|
||||
}
|
||||
|
||||
getIndexer(): SearchIndexer {
|
||||
return this.indexer;
|
||||
}
|
||||
|
||||
isSemanticSearchEnabled(): boolean {
|
||||
// Legacy always has semantic capability via embeddings
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* V3 TieredLexical search system implementation
|
||||
*/
|
||||
class V3SearchSystem implements SearchSystem {
|
||||
readonly name = "TieredLexical";
|
||||
readonly version = "v3";
|
||||
readonly isLegacy = false;
|
||||
private indexer = new V3SearchIndexer();
|
||||
|
||||
createRetriever(app: App, options: RetrieverOptions): BaseRetriever {
|
||||
return new TieredLexicalRetriever(app, options);
|
||||
}
|
||||
|
||||
getIndexer(): SearchIndexer {
|
||||
return this.indexer;
|
||||
}
|
||||
|
||||
isSemanticSearchEnabled(): boolean {
|
||||
return getSettings().enableSemanticSearchV3;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Factory for creating and managing search systems
|
||||
*/
|
||||
export class SearchSystemFactory {
|
||||
private static searchSystem: SearchSystem | null = null;
|
||||
|
||||
/**
|
||||
* Get the current search system based on settings
|
||||
*/
|
||||
static getSearchSystem(): SearchSystem {
|
||||
const settings = getSettings();
|
||||
|
||||
// Check if we need to recreate due to settings change
|
||||
if (this.searchSystem) {
|
||||
const isCurrentlyLegacy = this.searchSystem.isLegacy;
|
||||
const shouldBeLegacy = settings.useLegacySearch;
|
||||
|
||||
if (isCurrentlyLegacy !== shouldBeLegacy) {
|
||||
this.searchSystem = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.searchSystem) {
|
||||
if (settings.useLegacySearch) {
|
||||
logWarn(
|
||||
"DEPRECATED: Legacy Orama search is deprecated and will be removed in a future version. Please switch to v3 search in QA Settings."
|
||||
);
|
||||
this.searchSystem = new LegacySearchSystem();
|
||||
} else {
|
||||
this.searchSystem = new V3SearchSystem();
|
||||
}
|
||||
}
|
||||
|
||||
return this.searchSystem;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the cached search system (use when settings change)
|
||||
*/
|
||||
static reset(): void {
|
||||
this.searchSystem = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a retriever using the current search system
|
||||
*/
|
||||
static createRetriever(app: App, options: RetrieverOptions): BaseRetriever {
|
||||
const searchSystem = this.getSearchSystem();
|
||||
logInfo(`Creating retriever using ${searchSystem.name} (${searchSystem.version})`);
|
||||
|
||||
try {
|
||||
return searchSystem.createRetriever(app, options);
|
||||
} catch (error) {
|
||||
// Fallback to v3 if legacy fails
|
||||
if (searchSystem.isLegacy) {
|
||||
logWarn("Failed to create legacy retriever, falling back to v3");
|
||||
return new V3SearchSystem().createRetriever(app, options);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the indexer for the current search system
|
||||
*/
|
||||
static getIndexer(): SearchIndexer {
|
||||
return this.getSearchSystem().getIndexer();
|
||||
}
|
||||
}
|
||||
|
|
@ -265,3 +265,85 @@ interface NoteIdRank {
|
|||
7. **Explainable Rankings**: Track contributing factors for transparency
|
||||
8. **Intelligent Graph Boost**: Only analyzes highly relevant results (≥75% similarity) to balance quality and performance
|
||||
9. **Token-Aware Indexing**: Automatic chunk truncation when embedding token limits approached
|
||||
|
||||
## Legacy Search Migration
|
||||
|
||||
### Current State (v2.x)
|
||||
|
||||
The codebase currently supports both legacy Orama-based search and v3 tiered lexical search via a toggle in settings. The implementation uses a factory pattern (`SearchSystemFactory`) to route between systems.
|
||||
|
||||
### Architecture for Dual Support
|
||||
|
||||
```typescript
|
||||
// SearchSystem.ts provides unified interface
|
||||
interface SearchSystem {
|
||||
createRetriever(app: App, options: RetrieverOptions): BaseRetriever;
|
||||
getIndexer(): SearchIndexer;
|
||||
isSemanticSearchEnabled(): boolean;
|
||||
}
|
||||
|
||||
// Factory routes based on settings
|
||||
SearchSystemFactory.getSearchSystem(); // Returns appropriate implementation
|
||||
```
|
||||
|
||||
### Legacy Removal Steps (v3.0.0)
|
||||
|
||||
When ready to remove legacy search support:
|
||||
|
||||
#### 1. Remove Legacy Classes from `SearchSystem.ts`
|
||||
|
||||
- Delete `LegacySearchIndexer` class
|
||||
- Delete `LegacySearchSystem` class
|
||||
- Update `SearchSystemFactory.getSearchSystem()` to always return `V3SearchSystem`
|
||||
|
||||
#### 2. Remove Settings Properties
|
||||
|
||||
- Delete `useLegacySearch` from:
|
||||
- `src/settings/model.ts`
|
||||
- `src/constants.ts` (DEFAULT_SETTINGS)
|
||||
- Delete `numPartitions` setting (legacy partitioning)
|
||||
|
||||
#### 3. Remove UI Components
|
||||
|
||||
- Remove legacy search toggle from `src/settings/v2/components/QASettings.tsx`
|
||||
- Remove partition dropdown UI
|
||||
- Clean up conditional rendering logic
|
||||
|
||||
#### 4. Delete Legacy Files
|
||||
|
||||
```bash
|
||||
# Core legacy files to remove
|
||||
src/search/hybridRetriever.ts
|
||||
src/search/vectorStoreManager.ts
|
||||
src/search/chunkedStorage.ts
|
||||
src/search/dbOperations.ts
|
||||
src/search/indexOperations.ts
|
||||
src/search/indexEventHandler.ts
|
||||
```
|
||||
|
||||
#### 5. Clean Package Dependencies
|
||||
|
||||
Remove from `package.json`:
|
||||
|
||||
- `@orama/orama`
|
||||
- Any Orama-related plugins
|
||||
|
||||
#### 6. Update Documentation
|
||||
|
||||
- Remove legacy search references from user documentation
|
||||
- Update API documentation
|
||||
- Remove deprecation warnings
|
||||
|
||||
### Benefits After Removal
|
||||
|
||||
- **Reduced Bundle Size**: ~200KB reduction from removing Orama
|
||||
- **Simpler Codebase**: Single search implementation path
|
||||
- **Better Performance**: No conditional checks or dynamic imports
|
||||
- **Cleaner Architecture**: Direct use of v3 components without abstraction layer
|
||||
- **Easier Maintenance**: One search system to maintain and optimize
|
||||
|
||||
### Migration Timeline
|
||||
|
||||
1. **v2.x** (Current): Dual support with deprecation warnings
|
||||
2. **v2.x+1**: Legacy search disabled by default, requires opt-in
|
||||
3. **v3.0.0**: Complete removal of legacy search code
|
||||
|
|
|
|||
|
|
@ -112,6 +112,8 @@ export interface CopilotSettings {
|
|||
enableCustomPromptTemplating: boolean;
|
||||
/** Enable semantic stage in v3 search (requires Memory Index JSONL) */
|
||||
enableSemanticSearchV3: boolean;
|
||||
/** Use legacy HybridRetriever with Orama instead of v3 search */
|
||||
useLegacySearch: boolean;
|
||||
/**
|
||||
* Weight for semantic search in hybrid retrieval (0-1 range)
|
||||
* - 0 = fully lexical (keyword-based) search
|
||||
|
|
|
|||
|
|
@ -19,14 +19,16 @@ export const QASettings: React.FC = () => {
|
|||
<div className="tw-space-y-4">
|
||||
<section>
|
||||
<div className="tw-space-y-4">
|
||||
{/* Enable Semantic Search (v3) */}
|
||||
<SettingItem
|
||||
type="switch"
|
||||
title="Enable Semantic Search (v3)"
|
||||
description="Optional semantic search component to boost the default search performance. Use 'Refresh Vault Index' or 'Force Reindex Vault' to build the embedding index."
|
||||
checked={settings.enableSemanticSearchV3}
|
||||
onCheckedChange={(checked) => updateSetting("enableSemanticSearchV3", checked)}
|
||||
/>
|
||||
{/* Enable Semantic Search (v3) - Only show when not using legacy search */}
|
||||
{!settings.useLegacySearch && (
|
||||
<SettingItem
|
||||
type="switch"
|
||||
title="Enable Semantic Search (v3)"
|
||||
description="Optional semantic search component to boost the default search performance. Use 'Refresh Vault Index' or 'Force Reindex Vault' to build the embedding index."
|
||||
checked={settings.enableSemanticSearchV3}
|
||||
onCheckedChange={(checked) => updateSetting("enableSemanticSearchV3", checked)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Auto-Index Strategy */}
|
||||
<SettingItem
|
||||
|
|
@ -131,8 +133,8 @@ export const QASettings: React.FC = () => {
|
|||
onChange={(value) => updateSetting("embeddingBatchSize", value)}
|
||||
/>
|
||||
|
||||
{/* Semantic vs Lexical Weight */}
|
||||
{settings.enableSemanticSearchV3 && (
|
||||
{/* Semantic vs Lexical Weight - Only show when using v3 search with semantic enabled */}
|
||||
{!settings.useLegacySearch && settings.enableSemanticSearchV3 && (
|
||||
<SettingItem
|
||||
type="slider"
|
||||
title="Semantic Search Weight"
|
||||
|
|
@ -146,20 +148,20 @@ export const QASettings: React.FC = () => {
|
|||
/>
|
||||
)}
|
||||
|
||||
{/* Lexical Search RAM Limit */}
|
||||
<SettingItem
|
||||
type="slider"
|
||||
title="Lexical Search RAM Limit"
|
||||
description="Maximum RAM usage for full-text search index. Lower values use less memory but may limit search performance on large vaults. Default is 100 MB."
|
||||
min={20}
|
||||
max={1000}
|
||||
step={20}
|
||||
value={settings.lexicalSearchRamLimit || 100}
|
||||
onChange={(value) => updateSetting("lexicalSearchRamLimit", value)}
|
||||
suffix=" MB"
|
||||
/>
|
||||
|
||||
{/* Number of Partitions removed (auto-managed in v3) */}
|
||||
{/* Lexical Search RAM Limit - Only show when using v3 search */}
|
||||
{!settings.useLegacySearch && (
|
||||
<SettingItem
|
||||
type="slider"
|
||||
title="Lexical Search RAM Limit"
|
||||
description="Maximum RAM usage for full-text search index. Lower values use less memory but may limit search performance on large vaults. Default is 100 MB."
|
||||
min={20}
|
||||
max={1000}
|
||||
step={20}
|
||||
value={settings.lexicalSearchRamLimit || 100}
|
||||
onChange={(value) => updateSetting("lexicalSearchRamLimit", value)}
|
||||
suffix=" MB"
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Exclusions */}
|
||||
<SettingItem
|
||||
|
|
@ -233,6 +235,47 @@ export const QASettings: React.FC = () => {
|
|||
checked={settings.disableIndexOnMobile}
|
||||
onCheckedChange={(checked) => updateSetting("disableIndexOnMobile", checked)}
|
||||
/>
|
||||
|
||||
{/* Legacy Search Section - At the bottom */}
|
||||
<div className="tw-mt-6 tw-border-t tw-pt-4">
|
||||
<h3 className="tw-mb-4 tw-text-lg tw-font-semibold">Legacy Search Options</h3>
|
||||
|
||||
{/* Use Legacy Search */}
|
||||
<SettingItem
|
||||
type="switch"
|
||||
title="Use Legacy Search (Orama)"
|
||||
description="Fallback to the legacy HybridRetriever with Orama instead of the new v3 search system. Enable this if you experience issues with the new search."
|
||||
checked={settings.useLegacySearch}
|
||||
onCheckedChange={(checked) => {
|
||||
updateSetting("useLegacySearch", checked);
|
||||
// Disable v3 semantic search when enabling legacy search
|
||||
if (checked) {
|
||||
updateSetting("enableSemanticSearchV3", false);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
|
||||
{/* Number of Partitions - Only show when using legacy search */}
|
||||
{settings.useLegacySearch && (
|
||||
<SettingItem
|
||||
type="select"
|
||||
title="Number of Partitions"
|
||||
description="Split the Orama index into multiple partitions to handle large vaults. Increase if you get 'string length' errors. Default is 1."
|
||||
value={String(settings.numPartitions || 1)}
|
||||
onChange={(value) => updateSetting("numPartitions", Number(value))}
|
||||
options={[
|
||||
{ label: "1", value: "1" },
|
||||
{ label: "2", value: "2" },
|
||||
{ label: "4", value: "4" },
|
||||
{ label: "8", value: "8" },
|
||||
{ label: "16", value: "16" },
|
||||
{ label: "32", value: "32" },
|
||||
{ label: "40", value: "40" },
|
||||
]}
|
||||
placeholder="Select partitions"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { getStandaloneQuestion } from "@/chainUtils";
|
|||
import { TEXT_WEIGHT } from "@/constants";
|
||||
import { BrevilabsClient } from "@/LLMProviders/brevilabsClient";
|
||||
import { logInfo } from "@/logger";
|
||||
import { TieredLexicalRetriever } from "@/search/v3/TieredLexicalRetriever";
|
||||
import { SearchSystemFactory } from "@/search/SearchSystem";
|
||||
import { getSettings } from "@/settings/model";
|
||||
import { z } from "zod";
|
||||
import { createTool, SimpleTool } from "./SimpleTool";
|
||||
|
|
@ -35,8 +35,8 @@ const localSearchTool = createTool({
|
|||
|
||||
logInfo(`returnAll: ${returnAll}`);
|
||||
|
||||
// Use tiered lexical retriever for multi-stage search
|
||||
const retriever = new TieredLexicalRetriever(app, {
|
||||
// Use retriever factory to get appropriate retriever based on settings
|
||||
const retriever = SearchSystemFactory.createRetriever(app, {
|
||||
minSimilarityScore: returnAll ? 0.0 : 0.1,
|
||||
maxK: effectiveMaxK,
|
||||
salientTerms,
|
||||
|
|
@ -85,21 +85,44 @@ const localSearchTool = createTool({
|
|||
},
|
||||
});
|
||||
|
||||
// Note: indexTool is kept for backward compatibility but is no longer used with v3 search
|
||||
// Note: indexTool behavior depends on which retriever is active
|
||||
const indexTool = createTool({
|
||||
name: "indexVault",
|
||||
description: "Index the vault to the Copilot index",
|
||||
schema: z.void(), // No parameters
|
||||
handler: async () => {
|
||||
// Tiered lexical retriever doesn't require manual indexing - it builds ephemeral indexes on demand
|
||||
const indexResultPrompt = `The tiered lexical retriever builds indexes on demand and doesn't require manual indexing.\n`;
|
||||
return (
|
||||
indexResultPrompt +
|
||||
JSON.stringify({
|
||||
success: true,
|
||||
message: "Tiered lexical retriever uses on-demand indexing. No manual indexing required.",
|
||||
})
|
||||
);
|
||||
const settings = getSettings();
|
||||
if (settings.useLegacySearch) {
|
||||
// Legacy search uses persistent Orama index - trigger actual indexing
|
||||
try {
|
||||
const VectorStoreManager = (await import("@/search/vectorStoreManager")).default;
|
||||
const count = await VectorStoreManager.getInstance().indexVaultToVectorStore();
|
||||
const indexResultPrompt = `Legacy search (Orama) index refreshed with ${count} documents.\n`;
|
||||
return (
|
||||
indexResultPrompt +
|
||||
JSON.stringify({
|
||||
success: true,
|
||||
message: `Legacy Orama index has been refreshed with ${count} documents.`,
|
||||
documentCount: count,
|
||||
})
|
||||
);
|
||||
} catch (error) {
|
||||
return JSON.stringify({
|
||||
success: false,
|
||||
message: `Failed to index with legacy search: ${error.message}`,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// V3 search builds indexes on demand
|
||||
const indexResultPrompt = `The tiered lexical retriever builds indexes on demand and doesn't require manual indexing.\n`;
|
||||
return (
|
||||
indexResultPrompt +
|
||||
JSON.stringify({
|
||||
success: true,
|
||||
message: "Tiered lexical retriever uses on-demand indexing. No manual indexing required.",
|
||||
})
|
||||
);
|
||||
}
|
||||
},
|
||||
isBackground: true,
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue