From da59d0a7ac29d1fcb508117ac1bb957a0bf205ad Mon Sep 17 00:00:00 2001 From: Logan Yang Date: Sat, 9 Mar 2024 21:29:35 -0800 Subject: [PATCH] Implement index exclusion list and tests (#352) --- src/components/QAExclusionModal.tsx | 50 +++++++++ src/constants.ts | 158 +++++++++++++--------------- src/main.ts | 43 ++++++-- src/settings/SettingsPage.tsx | 1 + src/utils.ts | 33 ++++++ tests/utils.test.ts | 57 ++++++++++ 6 files changed, 250 insertions(+), 92 deletions(-) create mode 100644 src/components/QAExclusionModal.tsx diff --git a/src/components/QAExclusionModal.tsx b/src/components/QAExclusionModal.tsx new file mode 100644 index 00000000..66aa7b38 --- /dev/null +++ b/src/components/QAExclusionModal.tsx @@ -0,0 +1,50 @@ +import { CopilotSettings } from "@/settings/SettingsPage"; +import { App, Modal } from "obsidian"; + +export class QAExclusionModal extends Modal { + private settings: CopilotSettings; + private onSubmit: (paths: string) => void; + + constructor(app: App, settings: CopilotSettings, onSubmit: (paths: string) => void) { + super(app); + this.settings = settings; + this.onSubmit = onSubmit; + } + + onOpen() { + const formContainer = this.contentEl.createEl('div', { cls: 'copilot-command-modal' }); + const pathContainer = formContainer.createEl('div', { cls: 'copilot-command-input-container' }); + + pathContainer.createEl('h3', { text: 'Exclude by Folder Path or Note Title', cls: 'copilot-command-header' }); + const descFragment = createFragment((frag) => { + frag.appendText('All notes under the paths will be excluded from indexing'); + }); + pathContainer.appendChild(descFragment); + + const pathField = pathContainer.createEl( + 'input', + { + type: 'text', + cls: 'copilot-command-input', + value: this.settings.qaExclusionPaths, + placeholder: 'Enter /folderPath, [[note title]] separated by commas', + } + ); + pathField.setAttribute('name', 'folderPath'); + + const submitButtonContainer = formContainer.createEl('div', { cls: 'copilot-command-save-btn-container' }); + const submitButton = submitButtonContainer.createEl('button', { text: 'Submit', cls: 'copilot-command-save-btn' }); + + submitButton.addEventListener('click', () => { + // Parse the input list + const pathsValue = pathField.value + .split(',') + .map(pathValue => pathValue.trim()) + .filter(pathValue => pathValue !== '') + .join(','); + + this.onSubmit(pathsValue); + this.close(); + }); + } +} \ No newline at end of file diff --git a/src/constants.ts b/src/constants.ts index 2bee99c1..3f45d9e7 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,32 +1,33 @@ -import { CopilotSettings } from '@/settings/SettingsPage'; +import { CopilotSettings } from "@/settings/SettingsPage"; -export const CHAT_VIEWTYPE = 'copilot-chat-view'; -export const USER_SENDER = 'user'; -export const AI_SENDER = 'ai'; -export const DEFAULT_SYSTEM_PROMPT = 'You are Obsidian Copilot, a helpful assistant that integrates AI to Obsidian note-taking.'; +export const CHAT_VIEWTYPE = "copilot-chat-view"; +export const USER_SENDER = "user"; +export const AI_SENDER = "ai"; +export const DEFAULT_SYSTEM_PROMPT = + "You are Obsidian Copilot, a helpful assistant that integrates AI to Obsidian note-taking."; export enum ChatModels { - GPT_35_TURBO = 'gpt-3.5-turbo', - GPT_35_TURBO_16K = 'gpt-3.5-turbo-16k', - GPT_4 = 'gpt-4', - GPT_4_TURBO = 'gpt-4-turbo-preview', - GPT_4_32K = 'gpt-4-32k', - GEMINI_PRO = 'gemini-pro', - OLLAMA = 'ollama', + GPT_35_TURBO = "gpt-3.5-turbo", + GPT_35_TURBO_16K = "gpt-3.5-turbo-16k", + GPT_4 = "gpt-4", + GPT_4_TURBO = "gpt-4-turbo-preview", + GPT_4_32K = "gpt-4-32k", + GEMINI_PRO = "gemini-pro", + OLLAMA = "ollama", } export enum ChatModelDisplayNames { - GPT_35_TURBO = 'GPT-3.5', - GPT_35_TURBO_16K = 'GPT-3.5 16K', - GPT_4 = 'GPT-4', - GPT_4_TURBO = 'GPT-4 TURBO', - GPT_4_32K = 'GPT-4 32K', - AZURE_OPENAI = 'AZURE OPENAI', - CLAUDE = 'CLAUDE 3', - GEMINI_PRO = 'GEMINI PRO', - OPENROUTERAI = 'OPENROUTER.AI', - OLLAMA = 'OLLAMA (LOCAL)', - LM_STUDIO = 'LM STUDIO (LOCAL)', + GPT_35_TURBO = "GPT-3.5", + GPT_35_TURBO_16K = "GPT-3.5 16K", + GPT_4 = "GPT-4", + GPT_4_TURBO = "GPT-4 TURBO", + GPT_4_32K = "GPT-4 32K", + AZURE_OPENAI = "AZURE OPENAI", + CLAUDE = "CLAUDE 3", + GEMINI_PRO = "GEMINI PRO", + OPENROUTERAI = "OPENROUTER.AI", + OLLAMA = "OLLAMA (LOCAL)", + LM_STUDIO = "LM STUDIO (LOCAL)", } export const OPENAI_MODELS = new Set([ @@ -38,29 +39,19 @@ export const OPENAI_MODELS = new Set([ ChatModelDisplayNames.LM_STUDIO, ]); -export const AZURE_MODELS = new Set([ - ChatModelDisplayNames.AZURE_OPENAI, -]); +export const AZURE_MODELS = new Set([ChatModelDisplayNames.AZURE_OPENAI]); -export const GOOGLE_MODELS = new Set([ - ChatModelDisplayNames.GEMINI_PRO, -]); +export const GOOGLE_MODELS = new Set([ChatModelDisplayNames.GEMINI_PRO]); -export const ANTHROPIC_MODELS = new Set([ - ChatModelDisplayNames.CLAUDE, -]); +export const ANTHROPIC_MODELS = new Set([ChatModelDisplayNames.CLAUDE]); export const OPENROUTERAI_MODELS = new Set([ ChatModelDisplayNames.OPENROUTERAI, -]) - -export const OLLAMA_MODELS = new Set([ - ChatModelDisplayNames.OLLAMA, ]); -export const LM_STUDIO_MODELS = new Set([ - ChatModelDisplayNames.LM_STUDIO, -]); +export const OLLAMA_MODELS = new Set([ChatModelDisplayNames.OLLAMA]); + +export const LM_STUDIO_MODELS = new Set([ChatModelDisplayNames.LM_STUDIO]); export const DISPLAY_NAME_TO_MODEL: Record = { [ChatModelDisplayNames.GPT_35_TURBO]: ChatModels.GPT_35_TURBO, @@ -68,21 +59,21 @@ export const DISPLAY_NAME_TO_MODEL: Record = { [ChatModelDisplayNames.GPT_4]: ChatModels.GPT_4, [ChatModelDisplayNames.GPT_4_TURBO]: ChatModels.GPT_4_TURBO, [ChatModelDisplayNames.GPT_4_32K]: ChatModels.GPT_4_32K, - [ChatModelDisplayNames.AZURE_OPENAI]: 'azure_openai', + [ChatModelDisplayNames.AZURE_OPENAI]: "azure_openai", [ChatModelDisplayNames.GEMINI_PRO]: ChatModels.GEMINI_PRO, }; // Model Providers export enum ModelProviders { - OPENAI = 'openai', - HUGGINGFACE = 'huggingface', - COHEREAI = 'cohereai', - AZURE_OPENAI = 'azure_openai', - ANTHROPIC = 'anthropic', - GOOGLE = 'google', - OPENROUTERAI = 'openrouterai', - LM_STUDIO = 'lm_studio', - OLLAMA = 'ollama', + OPENAI = "openai", + HUGGINGFACE = "huggingface", + COHEREAI = "cohereai", + AZURE_OPENAI = "azure_openai", + ANTHROPIC = "anthropic", + GOOGLE = "google", + OPENROUTERAI = "openrouterai", + LM_STUDIO = "lm_studio", + OLLAMA = "ollama", } export const VENDOR_MODELS: Record> = { @@ -104,12 +95,12 @@ export const EMBEDDING_PROVIDERS = [ ]; export enum EmbeddingModels { - OPENAI_EMBEDDING_ADA_V2 = 'text-embedding-ada-002', - OPENAI_EMBEDDING_SMALL = 'text-embedding-3-small', - OPENAI_EMBEDDING_LARGE = 'text-embedding-3-large', - AZURE_OPENAI = 'azure-openai', - COHEREAI = 'cohereai', - OLLAMA_NOMIC = 'ollama-nomic-embed-text', + OPENAI_EMBEDDING_ADA_V2 = "text-embedding-ada-002", + OPENAI_EMBEDDING_SMALL = "text-embedding-3-small", + OPENAI_EMBEDDING_LARGE = "text-embedding-3-large", + AZURE_OPENAI = "azure-openai", + COHEREAI = "cohereai", + OLLAMA_NOMIC = "ollama-nomic-embed-text", } export const EMBEDDING_MODEL_TO_PROVIDERS: Record = { @@ -119,18 +110,18 @@ export const EMBEDDING_MODEL_TO_PROVIDERS: Record = { [EmbeddingModels.AZURE_OPENAI]: ModelProviders.AZURE_OPENAI, [EmbeddingModels.COHEREAI]: ModelProviders.COHEREAI, [EmbeddingModels.OLLAMA_NOMIC]: ModelProviders.OLLAMA, -} +}; // Embedding Models -export const NOMIC_EMBED_TEXT = 'nomic-embed-text'; +export const NOMIC_EMBED_TEXT = "nomic-embed-text"; // export const DISTILBERT_NLI = 'sentence-transformers/distilbert-base-nli-mean-tokens'; // export const INSTRUCTOR_XL = 'hkunlp/instructor-xl'; // Inference API is off for this // export const MPNET_V2 = 'sentence-transformers/all-mpnet-base-v2'; // Inference API returns 400 export enum VAULT_VECTOR_STORE_STRATEGY { - NEVER = 'NEVER', - ON_STARTUP = 'ON STARTUP', - ON_MODE_SWITCH = 'ON MODE SWITCH', + NEVER = "NEVER", + ON_STARTUP = "ON STARTUP", + ON_MODE_SWITCH = "ON MODE SWITCH", } export const VAULT_VECTOR_STORE_STRATEGIES = [ @@ -142,37 +133,38 @@ export const VAULT_VECTOR_STORE_STRATEGIES = [ export const PROXY_SERVER_PORT = 53001; export const DEFAULT_SETTINGS: CopilotSettings = { - openAIApiKey: '', - huggingfaceApiKey: '', - cohereApiKey: '', - anthropicApiKey: '', - anthropicModel: 'claude-3-sonnet-20240229', - azureOpenAIApiKey: '', - azureOpenAIApiInstanceName: '', - azureOpenAIApiDeploymentName: '', - azureOpenAIApiVersion: '', - azureOpenAIApiEmbeddingDeploymentName: '', - googleApiKey: '', - openRouterAiApiKey: '', - openRouterModel: 'cognitivecomputations/dolphin-mixtral-8x7b', + openAIApiKey: "", + huggingfaceApiKey: "", + cohereApiKey: "", + anthropicApiKey: "", + anthropicModel: "claude-3-sonnet-20240229", + azureOpenAIApiKey: "", + azureOpenAIApiInstanceName: "", + azureOpenAIApiDeploymentName: "", + azureOpenAIApiVersion: "", + azureOpenAIApiEmbeddingDeploymentName: "", + googleApiKey: "", + openRouterAiApiKey: "", + openRouterModel: "cognitivecomputations/dolphin-mixtral-8x7b", defaultModel: ChatModels.GPT_4_TURBO, defaultModelDisplayName: ChatModelDisplayNames.GPT_4_TURBO, embeddingModel: EmbeddingModels.OPENAI_EMBEDDING_SMALL, temperature: 0.1, maxTokens: 1000, contextTurns: 15, - userSystemPrompt: '', - openAIProxyBaseUrl: '', - openAIProxyModelName: '', - openAIEmbeddingProxyBaseUrl: '', - openAIEmbeddingProxyModelName: '', - ollamaModel: 'llama2', - ollamaBaseUrl: '', - lmStudioBaseUrl: 'http://localhost:1234/v1', + userSystemPrompt: "", + openAIProxyBaseUrl: "", + openAIProxyModelName: "", + openAIEmbeddingProxyBaseUrl: "", + openAIEmbeddingProxyModelName: "", + ollamaModel: "llama2", + ollamaBaseUrl: "", + lmStudioBaseUrl: "http://localhost:1234/v1", stream: true, - defaultSaveFolder: 'copilot-conversations', + defaultSaveFolder: "copilot-conversations", indexVaultToVectorStore: VAULT_VECTOR_STORE_STRATEGY.NEVER, - chatNoteContextPath: '', + qaExclusionPaths: "", + chatNoteContextPath: "", chatNoteContextTags: [], debug: false, enableEncryption: false, diff --git a/src/main.ts b/src/main.ts index cd4d6001..e0cba5d9 100644 --- a/src/main.ts +++ b/src/main.ts @@ -8,6 +8,7 @@ import { AdhocPromptModal } from "@/components/AdhocPromptModal"; import { ChatNoteContextModal } from "@/components/ChatNoteContextModal"; import CopilotView from "@/components/CopilotView"; import { ListPromptModal } from "@/components/ListPromptModal"; +import { QAExclusionModal } from "@/components/QAExclusionModal"; import { CHAT_VIEWTYPE, DEFAULT_SETTINGS, @@ -23,6 +24,7 @@ import SharedState from "@/sharedState"; import { areEmbeddingModelsSame, getAllNotesContent, + isPathInList, sanitizeSettings, } from "@/utils"; import VectorDBManager, { VectorStoreDocument } from "@/vectorDBManager"; @@ -412,6 +414,18 @@ export default class CopilotPlugin extends Plugin { }, }); + this.addCommand({ + id: "set-vault-qa-exclusion", + name: "Set exclusion for Vault QA mode", + callback: async () => { + new QAExclusionModal(this.app, this.settings, async (paths: string) => { + // Store the path in the plugin's settings, default to empty string + this.settings.qaExclusionPaths = paths; + await this.saveSettings(); + }).open(); + }, + }); + this.registerEvent( this.app.vault.on("delete", (file) => { const docHash = VectorDBManager.getDocumentHash(file.path); @@ -471,11 +485,12 @@ export default class CopilotPlugin extends Plugin { const currEmbeddingModel = EmbeddingsManager.getModelName(embeddingInstance); - console.log( - "Prev vs Current embedding models:", - prevEmbeddingModel, - currEmbeddingModel, - ); + if (this.settings.debug) + console.log( + "Prev vs Current embedding models:", + prevEmbeddingModel, + currEmbeddingModel, + ); if (!areEmbeddingModelsSame(prevEmbeddingModel, currEmbeddingModel)) { // Model has changed, clear DB and reindex from scratch @@ -504,10 +519,18 @@ export default class CopilotPlugin extends Plugin { this.dbVectorStores, ); - const files = this.app.vault.getMarkdownFiles().filter((file) => { - if (!latestMtime || overwrite) return true; - return file.stat.mtime > latestMtime; - }); + const files = this.app.vault + .getMarkdownFiles() + .filter((file) => { + if (!latestMtime || overwrite) return true; + return file.stat.mtime > latestMtime; + }) + // file not in qaExclusionPaths + .filter((file) => { + if (!this.settings.qaExclusionPaths) return true; + return !isPathInList(file.path, this.settings.qaExclusionPaths); + }); + const fileContents: string[] = await Promise.all( files.map((file) => this.app.vault.cachedRead(file)), ); @@ -540,6 +563,8 @@ export default class CopilotPlugin extends Plugin { embeddingInstance, noteFile, ); + if (this.settings.debug) console.log(`Indexed: [[${file.basename}]].`); + indexedCount++; indexNotice.setMessage( `Copilot is indexing your vault...\n${indexedCount}/${totalFiles} files processed.`, diff --git a/src/settings/SettingsPage.tsx b/src/settings/SettingsPage.tsx index 72524d18..1828597b 100644 --- a/src/settings/SettingsPage.tsx +++ b/src/settings/SettingsPage.tsx @@ -40,6 +40,7 @@ export interface CopilotSettings { debug: boolean; enableEncryption: boolean; maxSourceChunks: number; + qaExclusionPaths: string; } export class CopilotSettingTab extends PluginSettingTab { diff --git a/src/utils.ts b/src/utils.ts index c121eb73..3f8a1917 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -101,6 +101,39 @@ export async function getNotesFromTags( return filesWithTag; } +export function isPathInList(filePath: string, pathList: string): boolean { + if (!pathList) return false; + + // Extract the file name from the filePath + const fileName = filePath.split("/").pop()?.toLowerCase(); + + // Normalize the file path for case-insensitive comparison + const normalizedFilePath = filePath.toLowerCase(); + + return pathList + .split(",") + .map( + (path) => + path + .trim() // Trim whitespace + .replace(/^\[\[|\]\]$/g, "") // Remove surrounding [[ and ]] + .replace(/^\//, "") // Remove leading slash + .toLowerCase(), // Convert to lowercase for case-insensitive comparison + ) + .some((normalizedPath) => { + // Check for exact match or proper segmentation + const isExactMatch = + normalizedFilePath === normalizedPath || + normalizedFilePath.startsWith(normalizedPath + "/") || + normalizedFilePath.endsWith("/" + normalizedPath) || + normalizedFilePath.includes("/" + normalizedPath + "/"); + // Check for file name match (for cases like [[note1]]) + const isFileNameMatch = fileName === normalizedPath + ".md"; + + return isExactMatch || isFileNameMatch; + }); +} + export const stringToChainType = (chain: string): ChainType => { switch (chain) { case "llm_chain": diff --git a/tests/utils.test.ts b/tests/utils.test.ts index 88c740aa..c79efe48 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -4,6 +4,7 @@ import { getNotesFromPath, getNotesFromTags, isFolderMatch, + isPathInList, processVariableNameForNotePath, } from "../src/utils"; @@ -213,3 +214,59 @@ describe("getNotesFromTags", () => { expect(resultPaths.length).toEqual(expectedPaths.length); }); }); + +describe("isPathInList", () => { + it("should exclude a file path that exactly matches an excluded path", () => { + const result = isPathInList("test/folder/note.md", "test/folder"); + expect(result).toBe(true); + }); + + it("should not exclude a file path if there is no match", () => { + const result = isPathInList("test/folder/note.md", "another/folder"); + expect(result).toBe(false); + }); + + it("should exclude a file path that matches an excluded path with leading slash", () => { + const result = isPathInList("test/folder/note.md", "/test/folder"); + expect(result).toBe(true); + }); + + it("should exclude a note title that matches an excluded path with surrounding [[ and ]]", () => { + const result = isPathInList("test/folder/note1.md", "[[note1]]"); + expect(result).toBe(true); + }); + + it("should be case insensitive when excluding a file path", () => { + const result = isPathInList("Test/Folder/Note.md", "test/folder"); + expect(result).toBe(true); + }); + + it("should handle multiple excluded paths separated by commas", () => { + const result = isPathInList( + "test/folder/note.md", + "another/folder,test/folder", + ); + expect(result).toBe(true); + }); + + it("should not exclude a file path if it partially matches an excluded path without proper segmentation", () => { + const result = isPathInList("test/folder123/note.md", "test/folder"); + expect(result).toBe(false); + }); + + it("should exclude a file path that matches any one of multiple excluded paths", () => { + const result = isPathInList( + "test/folder/note.md", + "another/folder, test/folder, yet/another/folder", + ); + expect(result).toBe(true); + }); + + it("should trim spaces around excluded paths", () => { + const result = isPathInList( + "test/folder/note.md", + " another/folder , test/folder ", + ); + expect(result).toBe(true); + }); +});