diff --git a/package-lock.json b/package-lock.json index eab950cb..5f124896 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3803,15 +3803,15 @@ } }, "node_modules/@next/env": { - "version": "14.2.5", - "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.5.tgz", - "integrity": "sha512-/zZGkrTOsraVfYjGP8uM0p6r0BDT6xWpkjdVbcz66PJVSpwXX3yNiRycxAuDfBKGWBrZBXRuK/YVlkNgxHGwmA==", + "version": "14.2.13", + "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.13.tgz", + "integrity": "sha512-s3lh6K8cbW1h5Nga7NNeXrbe0+2jIIYK9YaA9T7IufDWnZpozdFUp6Hf0d5rNWUKu4fEuSX2rCKlGjCrtylfDw==", "peer": true }, "node_modules/@next/swc-darwin-arm64": { - "version": "14.2.5", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.5.tgz", - "integrity": "sha512-/9zVxJ+K9lrzSGli1///ujyRfon/ZneeZ+v4ptpiPoOU+GKZnm8Wj8ELWU1Pm7GHltYRBklmXMTUqM/DqQ99FQ==", + "version": "14.2.13", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.13.tgz", + "integrity": "sha512-IkAmQEa2Htq+wHACBxOsslt+jMoV3msvxCn0WFSfJSkv/scy+i/EukBKNad36grRxywaXUYJc9mxEGkeIs8Bzg==", "cpu": [ "arm64" ], @@ -3825,9 +3825,9 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "14.2.5", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.5.tgz", - "integrity": "sha512-vXHOPCwfDe9qLDuq7U1OYM2wUY+KQ4Ex6ozwsKxp26BlJ6XXbHleOUldenM67JRyBfVjv371oneEvYd3H2gNSA==", + "version": "14.2.13", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.13.tgz", + "integrity": "sha512-Dv1RBGs2TTjkwEnFMVL5XIfJEavnLqqwYSD6LXgTPdEy/u6FlSrLBSSfe1pcfqhFEXRAgVL3Wpjibe5wXJzWog==", "cpu": [ "x64" ], @@ -3841,9 +3841,9 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "14.2.5", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.5.tgz", - "integrity": "sha512-vlhB8wI+lj8q1ExFW8lbWutA4M2ZazQNvMWuEDqZcuJJc78iUnLdPPunBPX8rC4IgT6lIx/adB+Cwrl99MzNaA==", + "version": "14.2.13", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.13.tgz", + "integrity": "sha512-yB1tYEFFqo4ZNWkwrJultbsw7NPAAxlPXURXioRl9SdW6aIefOLS+0TEsKrWBtbJ9moTDgU3HRILL6QBQnMevg==", "cpu": [ "arm64" ], @@ -3857,9 +3857,9 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "14.2.5", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.5.tgz", - "integrity": "sha512-NpDB9NUR2t0hXzJJwQSGu1IAOYybsfeB+LxpGsXrRIb7QOrYmidJz3shzY8cM6+rO4Aojuef0N/PEaX18pi9OA==", + "version": "14.2.13", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.13.tgz", + "integrity": "sha512-v5jZ/FV/eHGoWhMKYrsAweQ7CWb8xsWGM/8m1mwwZQ/sutJjoFaXchwK4pX8NqwImILEvQmZWyb8pPTcP7htWg==", "cpu": [ "arm64" ], @@ -3873,9 +3873,9 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "14.2.5", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.5.tgz", - "integrity": "sha512-8XFikMSxWleYNryWIjiCX+gU201YS+erTUidKdyOVYi5qUQo/gRxv/3N1oZFCgqpesN6FPeqGM72Zve+nReVXQ==", + "version": "14.2.13", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.13.tgz", + "integrity": "sha512-aVc7m4YL7ViiRv7SOXK3RplXzOEe/qQzRA5R2vpXboHABs3w8vtFslGTz+5tKiQzWUmTmBNVW0UQdhkKRORmGA==", "cpu": [ "x64" ], @@ -3889,9 +3889,9 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "14.2.5", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.5.tgz", - "integrity": "sha512-6QLwi7RaYiQDcRDSU/os40r5o06b5ue7Jsk5JgdRBGGp8l37RZEh9JsLSM8QF0YDsgcosSeHjglgqi25+m04IQ==", + "version": "14.2.13", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.13.tgz", + "integrity": "sha512-4wWY7/OsSaJOOKvMsu1Teylku7vKyTuocvDLTZQq0TYv9OjiYYWt63PiE1nTuZnqQ4RPvME7Xai+9enoiN0Wrg==", "cpu": [ "x64" ], @@ -3905,9 +3905,9 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "14.2.5", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.5.tgz", - "integrity": "sha512-1GpG2VhbspO+aYoMOQPQiqc/tG3LzmsdBH0LhnDS3JrtDx2QmzXe0B6mSZZiN3Bq7IOMXxv1nlsjzoS1+9mzZw==", + "version": "14.2.13", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.13.tgz", + "integrity": "sha512-uP1XkqCqV2NVH9+g2sC7qIw+w2tRbcMiXFEbMihkQ8B1+V6m28sshBwAB0SDmOe0u44ne1vFU66+gx/28RsBVQ==", "cpu": [ "arm64" ], @@ -3921,9 +3921,9 @@ } }, "node_modules/@next/swc-win32-ia32-msvc": { - "version": "14.2.5", - "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.5.tgz", - "integrity": "sha512-Igh9ZlxwvCDsu6438FXlQTHlRno4gFpJzqPjSIBZooD22tKeI4fE/YMRoHVJHmrQ2P5YL1DoZ0qaOKkbeFWeMg==", + "version": "14.2.13", + "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.13.tgz", + "integrity": "sha512-V26ezyjPqQpDBV4lcWIh8B/QICQ4v+M5Bo9ykLN+sqeKKBxJVDpEc6biDVyluTXTC40f5IqCU0ttth7Es2ZuMw==", "cpu": [ "ia32" ], @@ -3937,9 +3937,9 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "14.2.5", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.5.tgz", - "integrity": "sha512-tEQ7oinq1/CjSG9uSTerca3v4AZ+dFa+4Yu6ihaG8Ud8ddqLQgFGcnwYls13H5X5CPDPZJdYxyeMui6muOLd4g==", + "version": "14.2.13", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.13.tgz", + "integrity": "sha512-WwzOEAFBGhlDHE5Z73mNU8CO8mqMNLqaG+AO9ETmzdCQlJhVtWZnOl2+rqgVQS+YHunjOWptdFmNfbpwcUuEsw==", "cpu": [ "x64" ], @@ -12549,9 +12549,9 @@ ] }, "node_modules/micromatch": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", - "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "dev": true, "dependencies": { "braces": "^3.0.3", @@ -12697,12 +12697,12 @@ } }, "node_modules/next": { - "version": "14.2.5", - "resolved": "https://registry.npmjs.org/next/-/next-14.2.5.tgz", - "integrity": "sha512-0f8aRfBVL+mpzfBjYfQuLWh2WyAwtJXCRfkPF4UJ5qd2YwrHczsrSzXU4tRMV0OAxR8ZJZWPFn6uhSC56UTsLA==", + "version": "14.2.13", + "resolved": "https://registry.npmjs.org/next/-/next-14.2.13.tgz", + "integrity": "sha512-BseY9YNw8QJSwLYD7hlZzl6QVDoSFHL/URN5K64kVEVpCsSOWeyjbIGK+dZUaRViHTaMQX8aqmnn0PHBbGZezg==", "peer": true, "dependencies": { - "@next/env": "14.2.5", + "@next/env": "14.2.13", "@swc/helpers": "0.5.5", "busboy": "1.6.0", "caniuse-lite": "^1.0.30001579", @@ -12717,15 +12717,15 @@ "node": ">=18.17.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "14.2.5", - "@next/swc-darwin-x64": "14.2.5", - "@next/swc-linux-arm64-gnu": "14.2.5", - "@next/swc-linux-arm64-musl": "14.2.5", - "@next/swc-linux-x64-gnu": "14.2.5", - "@next/swc-linux-x64-musl": "14.2.5", - "@next/swc-win32-arm64-msvc": "14.2.5", - "@next/swc-win32-ia32-msvc": "14.2.5", - "@next/swc-win32-x64-msvc": "14.2.5" + "@next/swc-darwin-arm64": "14.2.13", + "@next/swc-darwin-x64": "14.2.13", + "@next/swc-linux-arm64-gnu": "14.2.13", + "@next/swc-linux-arm64-musl": "14.2.13", + "@next/swc-linux-x64-gnu": "14.2.13", + "@next/swc-linux-x64-musl": "14.2.13", + "@next/swc-win32-arm64-msvc": "14.2.13", + "@next/swc-win32-ia32-msvc": "14.2.13", + "@next/swc-win32-x64-msvc": "14.2.13" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", @@ -13297,9 +13297,9 @@ "dev": true }, "node_modules/path-to-regexp": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz", - "integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.9.0.tgz", + "integrity": "sha512-xIp7/apCFJuUHdDLWe8O1HIkb0kQrOMb/0u6FXQjemHn/ii5LrIzU6bdECnsiTF/GjZkMEKg1xdiZwNqDYlZ6g==", "dependencies": { "isarray": "0.0.1" } diff --git a/src/LLMProviders/chainManager.ts b/src/LLMProviders/chainManager.ts index dbe93153..20386fcf 100644 --- a/src/LLMProviders/chainManager.ts +++ b/src/LLMProviders/chainManager.ts @@ -12,13 +12,13 @@ import { isSupportedChain, } from "@/utils"; import VectorDBManager, { MemoryVector, NoteFile, VectorStoreDocument } from "@/vectorDBManager"; -import { RunnableSequence } from "@langchain/core/runnables"; -import { BaseChatMemory } from "langchain/memory"; import { ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, } from "@langchain/core/prompts"; +import { RunnableSequence } from "@langchain/core/runnables"; +import { BaseChatMemory } from "langchain/memory"; import { MultiQueryRetriever } from "langchain/retrievers/multi_query"; import { MemoryVectorStore } from "langchain/vectorstores/memory"; import { App, Notice } from "obsidian"; @@ -479,12 +479,12 @@ export default class ChainManager { const markdownLinks = docTitles .map( (title) => - `[${title}](obsidian://open?vault=${encodeURIComponent(this.app.vault.getName())}&file=${encodeURIComponent( + `- [${title}](obsidian://open?vault=${encodeURIComponent(this.app.vault.getName())}&file=${encodeURIComponent( title )})` ) .join("\n"); - fullAIResponse += "\n\n**Sources**:\n" + markdownLinks; + fullAIResponse += "\n\n#### Sources:\n" + markdownLinks; } return fullAIResponse; diff --git a/src/LLMProviders/chatModelManager.ts b/src/LLMProviders/chatModelManager.ts index f0f3dc37..4544476a 100644 --- a/src/LLMProviders/chatModelManager.ts +++ b/src/LLMProviders/chatModelManager.ts @@ -3,10 +3,10 @@ import { BUILTIN_CHAT_MODELS, ChatModelProviders } from "@/constants"; import EncryptionService from "@/encryptionService"; import { ChatAnthropicWrapped, ProxyChatOpenAI } from "@/langchainWrappers"; import { ChatCohere } from "@langchain/cohere"; +import { BaseChatModel } from "@langchain/core/language_models/chat_models"; import { ChatGoogleGenerativeAI } from "@langchain/google-genai"; import { ChatGroq } from "@langchain/groq"; import { ChatOllama } from "@langchain/ollama"; -import { BaseChatModel } from "@langchain/core/language_models/chat_models"; import { ChatOpenAI } from "@langchain/openai"; import { Notice } from "obsidian"; diff --git a/src/LLMProviders/embeddingManager.ts b/src/LLMProviders/embeddingManager.ts index 7117ba7f..9ad46f49 100644 --- a/src/LLMProviders/embeddingManager.ts +++ b/src/LLMProviders/embeddingManager.ts @@ -4,9 +4,9 @@ import { EmbeddingModelProviders } from "@/constants"; import EncryptionService from "@/encryptionService"; import { ProxyOpenAIEmbeddings } from "@/langchainWrappers"; import { CohereEmbeddings } from "@langchain/cohere"; -import { GoogleGenerativeAIEmbeddings } from "@langchain/google-genai"; import { Embeddings } from "@langchain/core/embeddings"; -import { OllamaEmbeddings } from "@langchain/community/embeddings/ollama"; +import { GoogleGenerativeAIEmbeddings } from "@langchain/google-genai"; +import { OllamaEmbeddings } from "@langchain/ollama"; import { OpenAIEmbeddings } from "@langchain/openai"; export default class EmbeddingManager { private encryptionService: EncryptionService; diff --git a/src/main.ts b/src/main.ts index 011b7cd7..7ac075ab 100644 --- a/src/main.ts +++ b/src/main.ts @@ -271,19 +271,7 @@ export default class CopilotPlugin extends Plugin { id: "clear-local-vector-store", name: "Clear local vector store", callback: async () => { - try { - // Clear the vectorstore db - await this.dbVectorStores.destroy(); - // Reinitialize the database - this.dbVectorStores = new PouchDB( - `copilot_vector_stores_${this.getVaultIdentifier()}` - ); - new Notice("Local vector store cleared successfully."); - console.log("Local vector store cleared successfully, new instance created."); - } catch (err) { - console.error("Error clearing the local vector store:", err); - new Notice("An error occurred while clearing the local vector store."); - } + await this.clearVectorStore(); }, }); @@ -339,10 +327,11 @@ export default class CopilotPlugin extends Plugin { name: "Force re-index vault for QA", callback: async () => { try { + await this.clearVectorStore(); const indexedFileCount = await this.indexVaultToVectorStore(true); - new Notice(`${indexedFileCount} vault files indexed to vector store.`); - console.log(`${indexedFileCount} vault files indexed to vector store.`); + new Notice(`${indexedFileCount} vault files re-indexed to vector store.`); + console.log(`${indexedFileCount} vault files re-indexed to vector store.`); } catch (err) { console.error("Error re-indexing vault to vector store:", err); new Notice("An error occurred while re-indexing vault to vector store."); @@ -481,8 +470,12 @@ export default class CopilotPlugin extends Plugin { // TODO: Remove this when Ollama model is dynamically set const currEmbeddingModel = EmbeddingsManager.getModelName(embeddingInstance); - if (this.settings.debug) + if (this.settings.debug) { + console.log( + `\nVault QA exclusion paths: ${this.settings.qaExclusionPaths ? this.settings.qaExclusionPaths : "None"}` + ); console.log("Prev vs Current embedding models:", prevEmbeddingModel, currEmbeddingModel); + } if (!areEmbeddingModelsSame(prevEmbeddingModel, currEmbeddingModel)) { // Model has changed, clear DB and reindex from scratch @@ -870,4 +863,21 @@ export default class CopilotPlugin extends Plugin { copilotView.updateView(); } } + + async clearVectorStore(): Promise { + try { + // Clear the vectorstore db + await this.dbVectorStores.destroy(); + // Reinitialize the database + this.dbVectorStores = new PouchDB( + `copilot_vector_stores_${this.getVaultIdentifier()}` + ); + new Notice("Local vector store cleared successfully."); + console.log("Local vector store cleared successfully, new instance created."); + } catch (err) { + console.error("Error clearing the local vector store:", err); + new Notice("An error occurred while clearing the local vector store."); + throw err; + } + } } diff --git a/src/search/hybridRetriever.ts b/src/search/hybridRetriever.ts index 1f345c23..406896ac 100644 --- a/src/search/hybridRetriever.ts +++ b/src/search/hybridRetriever.ts @@ -1,10 +1,10 @@ import { extractNoteTitles, getNoteFileFromTitle } from "@/utils"; import VectorDBManager from "@/vectorDBManager"; +import { Document } from "@langchain/core/documents"; +import { BaseLanguageModel } from "@langchain/core/language_models/base"; +import { ChatPromptTemplate } from "@langchain/core/prompts"; import { BaseRetriever } from "@langchain/core/retrievers"; import { VectorStore } from "@langchain/core/vectorstores"; -import { BaseLanguageModel } from "langchain/base_language"; -import { Document } from "langchain/document"; -import { ChatPromptTemplate } from "langchain/prompts"; import { ScoreThresholdRetriever, ScoreThresholdRetrieverInput,