diff --git a/package-lock.json b/package-lock.json index 0fd29bab..445f7b5b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,12 +18,13 @@ "@langchain/anthropic": "^0.3.3", "@langchain/cohere": "^0.3.0", "@langchain/community": "^0.3.22", - "@langchain/core": "^0.3.27", + "@langchain/core": "^0.3.45", "@langchain/deepseek": "^0.0.1", "@langchain/google-genai": "^0.1.6", "@langchain/groq": "^0.1.2", "@langchain/mistralai": "^0.2.0", - "@langchain/openai": "^0.4.2", + "@langchain/openai": "^0.5.6", + "@langchain/xai": "^0.0.2", "@orama/orama": "^3.0.0-rc-2", "@radix-ui/react-checkbox": "^1.1.3", "@radix-ui/react-collapsible": "^1.1.2", @@ -69,7 +70,7 @@ "tailwindcss-animate": "^1.0.7" }, "devDependencies": { - "@langchain/ollama": "^0.1.1", + "@langchain/ollama": "^0.2.0", "@tailwindcss/container-queries": "^0.1.1", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^14.0.0", @@ -4332,17 +4333,16 @@ } }, "node_modules/@langchain/core": { - "version": "0.3.39", - "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.3.39.tgz", - "integrity": "sha512-muXs4asy1A7qDtcdznxqyBfxf4N6qxofY/S0c95vbsWa0r9YAE2PttHIjcuxSy1q2jUiTkpCcgFEjNJRQRVhEw==", - "license": "MIT", + "version": "0.3.45", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.3.45.tgz", + "integrity": "sha512-4icCMCeuauVvwH43zgCE6Hk2Zbke5OccfVhFpTiIJPo8WoU/ALdaZlS22S+6+qZ9lFzFHsgZ6K33Av9aWhxkQA==", "dependencies": { "@cfworker/json-schema": "^4.0.2", "ansi-styles": "^5.0.0", "camelcase": "6", "decamelize": "1.2.0", "js-tiktoken": "^1.0.12", - "langsmith": ">=0.2.8 <0.4.0", + "langsmith": "^0.3.16", "mustache": "^4.2.0", "p-queue": "^6.6.2", "p-retry": "4", @@ -4376,6 +4376,57 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@langchain/core/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/@langchain/core/node_modules/chalk/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@langchain/core/node_modules/langsmith": { + "version": "0.3.20", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.3.20.tgz", + "integrity": "sha512-zwVQos6tjcksCTfdM67QKq7yyED4GmQiZw/sJ6UCMYZxlvTMMg3PeQ9tOePXAWNWoJygOnH+EwGXr7gYOOETDg==", + "dependencies": { + "@types/uuid": "^10.0.0", + "chalk": "^4.1.2", + "console-table-printer": "^2.12.1", + "p-queue": "^6.6.2", + "p-retry": "4", + "semver": "^7.6.3", + "uuid": "^10.0.0" + }, + "peerDependencies": { + "openai": "*" + }, + "peerDependenciesMeta": { + "openai": { + "optional": true + } + } + }, "node_modules/@langchain/core/node_modules/uuid": { "version": "10.0.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", @@ -4403,6 +4454,23 @@ "@langchain/core": ">=0.3.0 <0.4.0" } }, + "node_modules/@langchain/deepseek/node_modules/@langchain/openai": { + "version": "0.4.9", + "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.4.9.tgz", + "integrity": "sha512-NAsaionRHNdqaMjVLPkFCyjUDze+OqRHghA1Cn4fPoAafz+FXcl9c7LlEl9Xo0FH6/8yiCl7Rw2t780C/SBVxQ==", + "dependencies": { + "js-tiktoken": "^1.0.12", + "openai": "^4.87.3", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": ">=0.3.39 <0.4.0" + } + }, "node_modules/@langchain/google-genai": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/@langchain/google-genai/-/google-genai-0.1.6.tgz", @@ -4483,13 +4551,16 @@ } }, "node_modules/@langchain/ollama": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@langchain/ollama/-/ollama-0.1.1.tgz", - "integrity": "sha512-IQEdzGkfKzdoyys3GW5hCXc64d/u1xkrYXved73BLO+bnyQfzrM224jdsiYGUpjW3cUaO1ebD6PUiMYcANPPFQ==", + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@langchain/ollama/-/ollama-0.2.0.tgz", + "integrity": "sha512-jLlYFqt+nbhaJKLakk7lRTWHZJ7wHeJLM6yuv4jToQ8zPzpL//372+MjggDoW0mnw8ofysg1T2C6mEJspKJtiA==", "devOptional": true, + "license": "MIT", "dependencies": { - "ollama": "^0.5.9", - "uuid": "^10.0.0" + "ollama": "^0.5.12", + "uuid": "^10.0.0", + "zod": "^3.24.1", + "zod-to-json-schema": "^3.24.1" }, "engines": { "node": ">=18" @@ -4512,13 +4583,12 @@ } }, "node_modules/@langchain/openai": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.4.2.tgz", - "integrity": "sha512-Cuj7qbVcycALTP0aqZuPpEc7As8cwiGaU21MhXRyZFs+dnWxKYxZ1Q1z4kcx6cYkq/I+CNwwmk+sP+YruU73Aw==", - "license": "MIT", + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.5.6.tgz", + "integrity": "sha512-zN0iyJthPNmcefIBVybZwcTBgcqu/ElJFov42ZntxEncK4heOMAE9lkq9LQ5CaPU/SgrduibrM1oL57+tLUtaA==", "dependencies": { "js-tiktoken": "^1.0.12", - "openai": "^4.77.0", + "openai": "^4.93.0", "zod": "^3.22.4", "zod-to-json-schema": "^3.22.3" }, @@ -4526,7 +4596,7 @@ "node": ">=18" }, "peerDependencies": { - "@langchain/core": ">=0.3.29 <0.4.0" + "@langchain/core": ">=0.3.39 <0.4.0" } }, "node_modules/@langchain/textsplitters": { @@ -4543,6 +4613,38 @@ "@langchain/core": ">=0.2.21 <0.4.0" } }, + "node_modules/@langchain/xai": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/@langchain/xai/-/xai-0.0.2.tgz", + "integrity": "sha512-wVOs7SfJs4VWk/oiHJomaoaZ+r9nQhPqbEXlQ2D8L0d54PxYhb1ILR9rub9LT1RpqazSX8HG4A8+hX4R01qkSg==", + "dependencies": { + "@langchain/openai": "~0.4.4", + "zod": "^3.24.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": ">=0.2.21 <0.4.0" + } + }, + "node_modules/@langchain/xai/node_modules/@langchain/openai": { + "version": "0.4.9", + "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.4.9.tgz", + "integrity": "sha512-NAsaionRHNdqaMjVLPkFCyjUDze+OqRHghA1Cn4fPoAafz+FXcl9c7LlEl9Xo0FH6/8yiCl7Rw2t780C/SBVxQ==", + "dependencies": { + "js-tiktoken": "^1.0.12", + "openai": "^4.87.3", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": ">=0.3.39 <0.4.0" + } + }, "node_modules/@lezer/common": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.2.3.tgz", @@ -10218,6 +10320,14 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "dev": true }, + "node_modules/console-table-printer": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/console-table-printer/-/console-table-printer-2.12.1.tgz", + "integrity": "sha512-wKGOQRRvdnd89pCeH96e2Fn4wkbenSP6LMHfjfyNLMbGuHEFbMqQNuxXqd0oXG9caIOQ1FTvc5Uijp9/4jujnQ==", + "dependencies": { + "simple-wcswidth": "^1.0.1" + } + }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -12475,7 +12585,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, "engines": { "node": ">=8" } @@ -17400,10 +17509,11 @@ } }, "node_modules/ollama": { - "version": "0.5.9", - "resolved": "https://registry.npmjs.org/ollama/-/ollama-0.5.9.tgz", - "integrity": "sha512-F/KZuDRC+ZsVCuMvcOYuQ6zj42/idzCkkuknGyyGVmNStMZ/sU3jQpvhnl4SyC0+zBzLiKNZJnJeuPFuieWZvQ==", + "version": "0.5.14", + "resolved": "https://registry.npmjs.org/ollama/-/ollama-0.5.14.tgz", + "integrity": "sha512-pvOuEYa2WkkAumxzJP0RdEYHkbZ64AYyyUszXVX7ruLvk5L+EiO2G71da2GqEQ4IAk4j6eLoUbGk5arzFT1wJA==", "devOptional": true, + "license": "MIT", "dependencies": { "whatwg-fetch": "^3.6.20" } @@ -17449,10 +17559,9 @@ "integrity": "sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ==" }, "node_modules/openai": { - "version": "4.83.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.83.0.tgz", - "integrity": "sha512-fmTsqud0uTtRKsPC7L8Lu55dkaTwYucqncDHzVvO64DKOpNTuiYwjbR/nVgpapXuYy8xSnhQQPUm+3jQaxICgw==", - "license": "Apache-2.0", + "version": "4.95.1", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.95.1.tgz", + "integrity": "sha512-IqJy+ymeW+k/Wq+2YVN3693OQMMcODRtHEYOlz263MdUwnN/Dwdl9c2EXSxLLtGEHkSHAfvzpDMHI5MaWJKXjQ==", "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", @@ -19301,6 +19410,11 @@ "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", "peer": true }, + "node_modules/simple-wcswidth": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-wcswidth/-/simple-wcswidth-1.0.1.tgz", + "integrity": "sha512-xMO/8eNREtaROt7tJvWJqHBDTMFN4eiQ5I4JRMuilwfnFcV5W9u7RUkueNkdw0jPqGMX36iCywelS5yilTuOxg==" + }, "node_modules/sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -19895,7 +20009,6 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, "dependencies": { "has-flag": "^4.0.0" }, @@ -20766,7 +20879,8 @@ "version": "3.6.20", "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz", "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==", - "devOptional": true + "devOptional": true, + "license": "MIT" }, "node_modules/whatwg-mimetype": { "version": "3.0.0", @@ -21080,9 +21194,9 @@ } }, "node_modules/zod": { - "version": "3.24.1", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.1.tgz", - "integrity": "sha512-muH7gBL9sI1nciMZV67X5fTKKBLtwpZ5VBp1vsOQzj1MhrBZ4wlVCm3gedKZWLp0Oyel8sIGfeiz54Su+OVT+A==", + "version": "3.24.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.3.tgz", + "integrity": "sha512-HhY1oqzWCQWuUqvBFnsyrtZRhyPeR7SUGv+C4+MsisMuVfSPx8HpwWqH8tRahSlt6M3PiFAcoeFhZAqIXTxoSg==", "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index 8d70a1bb..959ba022 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ }, "license": "AGPL-3.0", "devDependencies": { - "@langchain/ollama": "^0.1.1", + "@langchain/ollama": "^0.2.0", "@tailwindcss/container-queries": "^0.1.1", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^14.0.0", @@ -80,12 +80,13 @@ "@langchain/anthropic": "^0.3.3", "@langchain/cohere": "^0.3.0", "@langchain/community": "^0.3.22", - "@langchain/core": "^0.3.27", + "@langchain/core": "^0.3.45", "@langchain/deepseek": "^0.0.1", "@langchain/google-genai": "^0.1.6", "@langchain/groq": "^0.1.2", "@langchain/mistralai": "^0.2.0", - "@langchain/openai": "^0.4.2", + "@langchain/openai": "^0.5.6", + "@langchain/xai": "^0.0.2", "@orama/orama": "^3.0.0-rc-2", "@radix-ui/react-checkbox": "^1.1.3", "@radix-ui/react-collapsible": "^1.1.2", diff --git a/src/LLMProviders/brevilabsClient.ts b/src/LLMProviders/brevilabsClient.ts index b075281e..6b851212 100644 --- a/src/LLMProviders/brevilabsClient.ts +++ b/src/LLMProviders/brevilabsClient.ts @@ -111,9 +111,14 @@ export class BrevilabsClient { endpoint: string, body: any, method = "POST", - excludeAuthHeader = false + excludeAuthHeader = false, + skipLicenseCheck = false ): Promise<{ data: T | null; error?: Error }> { - this.checkLicenseKey(); + if (!skipLicenseCheck) { + this.checkLicenseKey(); + } + + body.user_id = getSettings().userId; const url = new URL(`${BREVILABS_API_BASE_URL}${endpoint}`); if (method === "GET") { @@ -162,6 +167,7 @@ export class BrevilabsClient { license_key: await getDecryptedKey(getSettings().plusLicenseKey), }, "POST", + true, true ); if (error) { diff --git a/src/LLMProviders/chainRunner.ts b/src/LLMProviders/chainRunner.ts index 9aff8f08..ff1e4c86 100644 --- a/src/LLMProviders/chainRunner.ts +++ b/src/LLMProviders/chainRunner.ts @@ -1,7 +1,5 @@ import { getCurrentProject } from "@/aiParams"; import { getStandaloneQuestion } from "@/chainUtils"; -import { Composer } from "./composer"; -import { getSystemPrompt } from "@/settings/model"; import { ABORT_REASON, AI_SENDER, @@ -10,8 +8,15 @@ import { MAX_CHARS_FOR_LOCAL_SEARCH_CONTEXT, ModelCapability, } from "@/constants"; +import { + ImageBatchProcessor, + ImageContent, + ImageProcessingResult, + MessageContent, +} from "@/imageProcessing/imageProcessor"; import { BrevilabsClient } from "@/LLMProviders/brevilabsClient"; -import { logError } from "@/logger"; +import { logInfo } from "@/logger"; +import { getSettings, getSystemPrompt } from "@/settings/model"; import { ChatMessage } from "@/sharedState"; import { ToolManager } from "@/tools/toolManager"; import { @@ -22,13 +27,11 @@ import { formatDateTime, getApiErrorMessage, getMessageRole, - ImageContent, - ImageProcessor, - MessageContent, } from "@/utils"; import { BaseChatModel } from "@langchain/core/language_models/chat_models"; import { Notice } from "obsidian"; import ChainManager from "./chainManager"; +import { Composer } from "./composer"; import { COPILOT_TOOL_NAMES, IntentAnalyzer } from "./intentAnalyzer"; import ProjectManager from "./projectManager"; @@ -314,58 +317,26 @@ class CopilotPlusChainRunner extends BaseChainRunner { return hasYoutubeCommand && youtubeUrl !== null && words.length === 1; } - private async processImageUrls(urls: string[]): Promise { - try { - const imageUrls = await Promise.all( - urls.map(async (url) => { - if (await ImageProcessor.isImageUrl(url, this.chainManager.app.vault)) { - const imageContent = await ImageProcessor.convertToBase64( - url, - this.chainManager.app.vault - ); - if (!imageContent) { - logError(`Failed to process image: ${url}`); - return null; - } - return imageContent; - } - return null; - }) - ); - - // Filter out null values and return valid image URLs - const validImages = imageUrls.filter((item): item is ImageContent => item !== null); - return validImages; - } catch (error) { - logError("Error processing image URLs:", error); - return []; - } + private async processImageUrls(urls: string[]): Promise { + const failedImages: string[] = []; + const processedImages = await ImageBatchProcessor.processUrlBatch( + urls, + failedImages, + this.chainManager.app.vault + ); + ImageBatchProcessor.showFailedImagesNotice(failedImages); + return processedImages; } - private async processExistingImages(content: MessageContent[]): Promise { - try { - const imageContent = await Promise.all( - content - .filter( - (item): item is ImageContent => item.type === "image_url" && !!item.image_url?.url - ) - .map(async (item) => { - const processedContent = await ImageProcessor.convertToBase64( - item.image_url.url, - this.chainManager.app.vault - ); - if (!processedContent) { - logError(`Failed to process existing image: ${item.image_url.url}`); - return null; - } - return processedContent; - }) - ); - return imageContent.filter((item): item is ImageContent => item !== null); - } catch (error) { - logError("Error processing images:", error); - return []; - } + private async processChatInputImages(content: MessageContent[]): Promise { + const failedImages: string[] = []; + const processedImages = await ImageBatchProcessor.processChatImageBatch( + content, + failedImages, + this.chainManager.app.vault + ); + ImageBatchProcessor.showFailedImagesNotice(failedImages); + return processedImages; } private async extractEmbeddedImages(content: string): Promise { @@ -379,33 +350,61 @@ class CopilotPlusChainRunner extends BaseChainRunner { textContent: string, userMessage: ChatMessage ): Promise { - const content: MessageContent[] = [ + const failureMessages: string[] = []; + const successfulImages: ImageContent[] = []; + const settings = getSettings(); + + // Collect all image sources + const imageSources: { urls: string[]; type: string }[] = []; + + // Safely check and add context URLs + const contextUrls = userMessage.context?.urls; + if (contextUrls && contextUrls.length > 0) { + imageSources.push({ urls: contextUrls, type: "context" }); + } + + // Process embedded images only if setting is enabled + if (settings.passMarkdownImages) { + const embeddedImages = await this.extractEmbeddedImages(textContent); + if (embeddedImages.length > 0) { + imageSources.push({ urls: embeddedImages, type: "embedded" }); + } + } + + // Process all image sources + for (const source of imageSources) { + const result = await this.processImageUrls(source.urls); + successfulImages.push(...result.successfulImages); + failureMessages.push(...result.failureDescriptions); + } + + // Process existing chat content images if present + const existingContent = userMessage.content; + if (existingContent && existingContent.length > 0) { + const result = await this.processChatInputImages(existingContent); + successfulImages.push(...result.successfulImages); + failureMessages.push(...result.failureDescriptions); + } + + // Let the LLM know about the image processing failures + let finalText = textContent; + if (failureMessages.length > 0) { + finalText = `${textContent}\n\nNote: \n${failureMessages.join("\n")}\n`; + } + + const messageContent: MessageContent[] = [ { type: "text", - text: textContent, + text: finalText, }, ]; - // Process URLs in the message to identify images - if (userMessage.context?.urls && userMessage.context.urls.length > 0) { - const imageContents = await this.processImageUrls(userMessage.context.urls); - content.push(...imageContents); + // Add successful images after the text content + if (successfulImages.length > 0) { + messageContent.push(...successfulImages); } - // Process embedded images from the text content - const embeddedImages = await this.extractEmbeddedImages(textContent); - if (embeddedImages.length > 0) { - const imageContents = await this.processImageUrls(embeddedImages); - content.push(...imageContents); - } - - // Add existing image content if present - if (userMessage.content && userMessage.content.length > 0) { - const imageContents = await this.processExistingImages(userMessage.content); - content.push(...imageContents); - } - - return content; + return messageContent; } private hasCapability(model: BaseChatModel, capability: ModelCapability): boolean { @@ -473,11 +472,9 @@ class CopilotPlusChainRunner extends BaseChainRunner { content, }); - // Add debug logging for final request - if (debug) { - console.log("==== Final Request to AI ====\n", messages); - } - + const enhancedUserMessage = content instanceof Array ? (content[0] as any).text : content; + logInfo("Enhanced user message: ", enhancedUserMessage); + logInfo("==== Final Request to AI ====\n", messages); const streamer = new ThinkBlockStreamer(updateCurrentAiMessage); const chatStream = await this.chainManager.chatModelManager.getChatModel().stream(messages); @@ -620,10 +617,7 @@ class CopilotPlusChainRunner extends BaseChainRunner { // Enhance with tool outputs. let enhancedUserMessage = this.prepareEnhancedUserMessage(cleanedUserMessage, toolOutputs); // If no results, default to LLM Chain - if (debug) { - console.log("No local search results. Using standard LLM Chain."); - console.log("Enhanced user message:", enhancedUserMessage); - } + logInfo("No local search results. Using standard LLM Chain."); // Enhance with composer output. if (messageForAnalysis.includes("@composer")) { @@ -733,7 +727,7 @@ class CopilotPlusChainRunner extends BaseChainRunner { .join("\n\n"); } } - return `User message: ${userMessage}${context}`; + return `${userMessage}${context}`; } private getTimeExpression(toolCalls: any[]): string { diff --git a/src/LLMProviders/chatModelManager.ts b/src/LLMProviders/chatModelManager.ts index bee13e2a..fcd0bf29 100644 --- a/src/LLMProviders/chatModelManager.ts +++ b/src/LLMProviders/chatModelManager.ts @@ -19,6 +19,7 @@ import { ChatGroq } from "@langchain/groq"; import { ChatMistralAI } from "@langchain/mistralai"; import { ChatOllama } from "@langchain/ollama"; import { ChatOpenAI } from "@langchain/openai"; +import { ChatXAI } from "@langchain/xai"; import { Notice } from "obsidian"; type ChatConstructorType = { @@ -31,6 +32,7 @@ const CHAT_PROVIDER_CONSTRUCTORS = { [ChatModelProviders.ANTHROPIC]: ChatAnthropic, [ChatModelProviders.COHEREAI]: ChatCohere, [ChatModelProviders.GOOGLE]: ChatGoogleGenerativeAI, + [ChatModelProviders.XAI]: ChatXAI, [ChatModelProviders.OPENROUTERAI]: ChatOpenAI, [ChatModelProviders.OLLAMA]: ChatOllama, [ChatModelProviders.LM_STUDIO]: ChatOpenAI, @@ -63,6 +65,7 @@ export default class ChatModelManager { [ChatModelProviders.COHEREAI]: () => getSettings().cohereApiKey, [ChatModelProviders.OPENROUTERAI]: () => getSettings().openRouterAiApiKey, [ChatModelProviders.GROQ]: () => getSettings().groqApiKey, + [ChatModelProviders.XAI]: () => getSettings().xaiApiKey, [ChatModelProviders.OLLAMA]: () => "default-key", [ChatModelProviders.LM_STUDIO]: () => "default-key", [ChatModelProviders.OPENAI_FORMAT]: () => "default-key", @@ -167,6 +170,11 @@ export default class ChatModelManager { ], baseUrl: customModel.baseUrl, }, + [ChatModelProviders.XAI]: { + apiKey: await getDecryptedKey(customModel.apiKey || settings.xaiApiKey), + model: modelName, + // This langchainjs XAI client does not support baseURL override + }, [ChatModelProviders.OPENROUTERAI]: { modelName: modelName, openAIApiKey: await getDecryptedKey(customModel.apiKey || settings.openRouterAiApiKey), @@ -182,10 +190,11 @@ export default class ChatModelManager { [ChatModelProviders.OLLAMA]: { // ChatOllama has `model` instead of `modelName`!! model: modelName, - // @ts-ignore - apiKey: customModel.apiKey || "default-key", // MUST NOT use /v1 in the baseUrl for ollama baseUrl: customModel.baseUrl || "http://localhost:11434", + headers: new Headers({ + Authorization: `Bearer ${await getDecryptedKey(customModel.apiKey || "default-key")}`, + }), }, [ChatModelProviders.LM_STUDIO]: { modelName: modelName, diff --git a/src/LLMProviders/embeddingManager.ts b/src/LLMProviders/embeddingManager.ts index e907b592..91772d8f 100644 --- a/src/LLMProviders/embeddingManager.ts +++ b/src/LLMProviders/embeddingManager.ts @@ -256,6 +256,9 @@ export default class EmbeddingManager { baseUrl: customModel.baseUrl || "http://localhost:11434", model: modelName, truncate: true, + headers: { + Authorization: `Bearer ${await getDecryptedKey(customModel.apiKey || "default-key")}`, + }, }, [EmbeddingModelProviders.LM_STUDIO]: { modelName, diff --git a/src/commands/constants.ts b/src/commands/constants.ts index 385f2060..66be1cb6 100644 --- a/src/commands/constants.ts +++ b/src/commands/constants.ts @@ -6,132 +6,107 @@ export const COMMAND_NAME_MAX_LENGTH = 50; export const DEFAULT_INLINE_EDIT_COMMANDS: InlineEditCommandSettings[] = [ { name: "Fix grammar and spelling", - prompt: - `Fix the grammar and spelling of the text below. Preserve all formatting, line breaks, and special characters. Do not add or remove any content. Return only the corrected text.\n\n` + - `${SELECTED_TEXT_PLACEHOLDER}`, + prompt: `Fix the grammar and spelling of {}. Preserve all formatting, line breaks, and special characters. Do not add or remove any content. Return only the corrected text.`, showInContextMenu: true, }, { name: "Translate to Chinese", - prompt: - `Translate the text below into Chinese: + prompt: `Translate {} into Chinese: 1. Preserve the meaning and tone 2. Maintain appropriate cultural context 3. Keep formatting and structure - Return only the translated text.\n\n` + - `${SELECTED_TEXT_PLACEHOLDER}`, + Return only the translated text.`, showInContextMenu: true, }, { name: "Summarize", - prompt: - `Create a bullet-point summary of the text below. Each bullet point should capture a key point. Return only the bullet-point summary.\n\n` + - `${SELECTED_TEXT_PLACEHOLDER}`, + prompt: `Create a bullet-point summary of {}. Each bullet point should capture a key point. Return only the bullet-point summary.`, showInContextMenu: true, }, { name: "Simplify", - prompt: - `Simplify the text below to a 6th-grade reading level (ages 11-12). Use simple sentences, common words, and clear explanations. Maintain the original key concepts. Return only the simplified text.\n\n` + - `{copilot-selection}`, + prompt: `Simplify {} to a 6th-grade reading level (ages 11-12). Use simple sentences, common words, and clear explanations. Maintain the original key concepts. Return only the simplified text.`, showInContextMenu: true, }, { name: "Emojify", - prompt: - `Add relevant emojis to enhance the text below. Follow these rules: + prompt: `Add relevant emojis to enhance {}. Follow these rules: 1. Insert emojis at natural breaks in the text 2. Never place two emojis next to each other 3. Keep all original text unchanged 4. Choose emojis that match the context and tone - Return only the emojified text.\n\n` + `{copilot-selection}`, + Return only the emojified text.`, showInContextMenu: true, }, { name: "Make shorter", - prompt: - `Reduce the text below to half its length while preserving these elements: + prompt: `Reduce {} to half its length while preserving these elements: 1. Main ideas and key points 2. Essential details 3. Original tone and style - Return only the shortened text.\n\n` + - `${SELECTED_TEXT_PLACEHOLDER}`, + Return only the shortened text.`, showInContextMenu: true, }, { name: "Make longer", - prompt: - `Expand the text below to twice its length by: + prompt: `Expand {} to twice its length by: 1. Adding relevant details and examples 2. Elaborating on key points 3. Maintaining the original tone and style - Return only the expanded text.\n\n` + `${SELECTED_TEXT_PLACEHOLDER}`, + Return only the expanded text.`, showInContextMenu: true, }, { name: "Generate table of contents", - prompt: - `Generate a hierarchical table of contents for the text below. Use appropriate heading levels (H1, H2, H3, etc.). Include page numbers if present. Return only the table of contents.\n\n` + - `${SELECTED_TEXT_PLACEHOLDER}`, + prompt: `Generate a hierarchical table of contents for {}. Use appropriate heading levels (H1, H2, H3, etc.). Include page numbers if present. Return only the table of contents.`, showInContextMenu: false, }, { name: "Generate glossary", - prompt: - `Create a glossary of important terms, concepts, and phrases from the text below. Format each entry as "Term: Definition". Sort entries alphabetically. Return only the glossary.\n\n` + - `{copilot-selection}`, + prompt: `Create a glossary of important terms, concepts, and phrases from {}. Format each entry as "Term: Definition". Sort entries alphabetically. Return only the glossary.`, showInContextMenu: false, }, { name: "Remove URLs", - prompt: - `Remove all URLs from the text below. Preserve all other content and formatting. URLs may be in various formats (http, https, www). Return only the text with URLs removed.\n\n` + - `${SELECTED_TEXT_PLACEHOLDER}`, + prompt: `Remove all URLs from {}. Preserve all other content and formatting. URLs may be in various formats (http, https, www). Return only the text with URLs removed.`, showInContextMenu: false, }, { name: "Rewrite as tweet", - prompt: - `Rewrite the text below as a single tweet with these requirements: + prompt: `Rewrite {} as a single tweet with these requirements: 1. Maximum 280 characters 2. Use concise, impactful language 3. Maintain the core message - Return only the tweet text.\n\n` + `${SELECTED_TEXT_PLACEHOLDER}`, + Return only the tweet text.`, showInContextMenu: false, }, { name: "Rewrite as tweet thread", - prompt: - `Convert the text below into a Twitter thread following these rules: + prompt: `Convert {} into a Twitter thread following these rules: 1. Each tweet must be under 240 characters 2. Start with "THREAD START" on its own line 3. Separate tweets with "\n\n---\n\n" 4. End with "THREAD END" on its own line 5. Make content engaging and clear - Return only the formatted thread.\n\n` + - `${SELECTED_TEXT_PLACEHOLDER}`, + Return only the formatted thread.`, showInContextMenu: false, }, { name: "Explain like I am 5", - prompt: - `Explain the text below in simple terms that a 5-year-old would understand: + prompt: `Explain {} in simple terms that a 5-year-old would understand: 1. Use basic vocabulary 2. Include simple analogies 3. Break down complex concepts - Return only the simplified explanation.\n\n` + - `${SELECTED_TEXT_PLACEHOLDER}`, + Return only the simplified explanation.`, showInContextMenu: false, }, { name: "Rewrite as press release", - prompt: - `Transform the text below into a professional press release: + prompt: `Transform {} into a professional press release: 1. Use formal, journalistic style 2. Include headline and dateline 3. Follow inverted pyramid structure - Return only the press release format.\n\n` + - `${SELECTED_TEXT_PLACEHOLDER}`, + Return only the press release format.`, showInContextMenu: false, }, ]; diff --git a/src/commands/inlineEditCommandUtils.ts b/src/commands/inlineEditCommandUtils.ts index 4ce50d4e..6d9eca05 100644 --- a/src/commands/inlineEditCommandUtils.ts +++ b/src/commands/inlineEditCommandUtils.ts @@ -1,8 +1,9 @@ import { COMMAND_NAME_MAX_LENGTH, DEFAULT_INLINE_EDIT_COMMANDS, - SELECTED_TEXT_PLACEHOLDER, + SELECTED_TEXT_PLACEHOLDER as LEGACY_SELECTED_TEXT_PLACEHOLDER, } from "@/commands/constants"; +import { processPrompt } from "@/customPromptProcessor"; import { InlineEditCommandSettings, getSettings, useSettingsValue } from "@/settings/model"; export function getCommandId(commandName: string) { @@ -54,16 +55,44 @@ export function useInlineEditCommands(): InlineEditCommandSettings[] { } /** - * Replace the {copilot-selection} placeholder with the selected text. - * If the placeholder is not found, append the selected text to the prompt. + * Process the command prompt. */ -export function processCommandPrompt(prompt: string, selectedText: string) { - const index = prompt.indexOf(SELECTED_TEXT_PLACEHOLDER); +export async function processCommandPrompt( + prompt: string, + selectedText: string, + skipAppendingSelectedText = false +) { + const result = await processPrompt( + prompt, + selectedText, + app.vault, + app.workspace.getActiveFile() + ); + + const processedPrompt = result.processedPrompt; + + if (processedPrompt.includes("{selectedText}") || skipAppendingSelectedText) { + // Containing {selectedText} means the prompt was using the custom prompt + // processor way of handling the selected text. No need to go through the + // legacy placeholder. + return processedPrompt; + } + + // This is the legacy custom command selected text placeholder. It replaced + // {copilot-selection} in the prompt with the selected text. This is different + // from the custom prompt processor which uses {} in the prompt and appends + // the selected text to the prompt. We cannot change user's custom commands + // that have the old placeholder, so we need to support both. + // Also, selected text is required for custom commands. If neither `{}` nor + // `{copilot-selection}` is found, append the selected text to the prompt. + const index = processedPrompt.indexOf(LEGACY_SELECTED_TEXT_PLACEHOLDER); if (index === -1) { - return prompt + "\n\n" + selectedText; + return processedPrompt + "\n\n" + selectedText; } return ( - prompt.slice(0, index) + selectedText + prompt.slice(index + SELECTED_TEXT_PLACEHOLDER.length) + processedPrompt.slice(0, index) + + selectedText + + processedPrompt.slice(index + LEGACY_SELECTED_TEXT_PLACEHOLDER.length) ); } diff --git a/src/components/Chat.tsx b/src/components/Chat.tsx index c3e1eca8..38bb5758 100644 --- a/src/components/Chat.tsx +++ b/src/components/Chat.tsx @@ -10,6 +10,7 @@ import { updateChatMemory } from "@/chatUtils"; import { ChatControls } from "@/components/chat-components/ChatControls"; import ChatInput from "@/components/chat-components/ChatInput"; import ChatMessages from "@/components/chat-components/ChatMessages"; +import { NewVersionBanner } from "@/components/chat-components/NewVersionBanner"; import { ProjectList } from "@/components/chat-components/ProjectList"; import { ABORT_REASON, COMMAND_IDS, EVENT_NAMES, LOADING_MESSAGES, USER_SENDER } from "@/constants"; import { AppContext, EventTargetContext } from "@/context"; @@ -88,26 +89,10 @@ const Chat: React.FC = ({ const appContext = useContext(AppContext); const app = plugin.app || appContext; - const processContextNotes = async ( - customPromptProcessor: CustomPromptProcessor, - fileParserManager: FileParserManager - ) => { - const activeNote = app.workspace.getActiveFile(); - return await contextProcessor.processContextNotes( - customPromptProcessor, - fileParserManager, - app.vault, - contextNotes, - includeActiveNote, - activeNote, - currentChain - ); - }; - const handleSendMessage = async ({ toolCalls, urls, - contextNotes, + contextNotes: passedContextNotes, // Rename to avoid shadowing }: { toolCalls?: string[]; urls?: string[]; @@ -140,7 +125,7 @@ const Chat: React.FC = ({ }); } - const notes = [...(contextNotes || [])]; + const notes = [...(passedContextNotes || [])]; const activeNote = app.workspace.getActiveFile(); if (includeActiveNote && activeNote && !notes.some((note) => note.path === activeNote.path)) { notes.push(activeNote); @@ -170,11 +155,12 @@ const Chat: React.FC = ({ // First, process the original user message for custom prompts const customPromptProcessor = CustomPromptProcessor.getInstance(app.vault); - let processedUserMessage = await customPromptProcessor.processCustomPrompt( - inputMessage || "", - "", - app.workspace.getActiveFile() as TFile | undefined - ); + const { processedPrompt: processedUserMessage, includedFiles } = + await customPromptProcessor.processCustomPrompt( + inputMessage || "", + "", + app.workspace.getActiveFile() as TFile | undefined + ); // Extract Mentions (such as URLs) from original input message only if using Copilot Plus chain const urlContextAddition = @@ -182,11 +168,22 @@ const Chat: React.FC = ({ ? await mention.processUrls(inputMessage || "") : { urlContext: "", imageUrls: [] }; - // Add context notes - const noteContextAddition = await processContextNotes(customPromptProcessor, fileParserManager); + // Create set of file paths that were included in the custom prompt + const excludedNotePaths = new Set(includedFiles.map((file) => file.path)); + + // Add context notes, excluding those already processed by custom prompt + const noteContextAddition = await contextProcessor.processContextNotes( + excludedNotePaths, + fileParserManager, + app.vault, + notes, + includeActiveNote, + activeNote, + currentChain + ); // Combine everything - processedUserMessage = + const finalProcessedMessage = processedUserMessage + urlContextAddition.urlContext + noteContextAddition; let messageWithToolCalls = inputMessage; @@ -196,7 +193,7 @@ const Chat: React.FC = ({ } const promptMessageHidden: ChatMessage = { - message: processedUserMessage, + message: finalProcessedMessage, originalMessage: messageWithToolCalls, sender: USER_SENDER, isVisible: false, @@ -504,11 +501,12 @@ ${chatContent}`; if (!customPrompt) { return selectedText; } - return await customPromptProcessor.processCustomPrompt( + const result = await customPromptProcessor.processCustomPrompt( customPrompt, selectedText, app.workspace.getActiveFile() ?? undefined ); + return result.processedPrompt; // Extract just the processed prompt string }), [] ); @@ -518,11 +516,12 @@ ${chatContent}`; if (!customPrompt) { return selectedText; } - return await customPromptProcessor.processCustomPrompt( + const result = await customPromptProcessor.processCustomPrompt( customPrompt, selectedText, app.workspace.getActiveFile() as TFile | undefined ); + return result.processedPrompt; // Extract just the processed prompt string }), [] ); @@ -613,7 +612,8 @@ ${chatContent}`; const renderChatComponents = () => ( <> -
+
+ -
-
handleSaveAsNote(true)} diff --git a/src/components/CustomPromptSyntaxInstruction.tsx b/src/components/CustomPromptSyntaxInstruction.tsx new file mode 100644 index 00000000..86afb397 --- /dev/null +++ b/src/components/CustomPromptSyntaxInstruction.tsx @@ -0,0 +1,22 @@ +import React from "react"; + +export function CustomPromptSyntaxInstruction() { + return ( +
    +
  • + {"{}"} represents the selected text. +
    • +
  • + {`{[[Note Title]]}`} represents a note. +
    • +
  • + {`{activeNote}`} represents the active + note. +
    • +
  • + {`{#tag1, #tag2}`} represents ALL notes + with ANY of the specified tags in their property (an OR operation). +
    • +
+ ); +} diff --git a/src/components/chat-components/ChatButtons.tsx b/src/components/chat-components/ChatButtons.tsx index c5cf66f2..fdab30cf 100644 --- a/src/components/chat-components/ChatButtons.tsx +++ b/src/components/chat-components/ChatButtons.tsx @@ -40,18 +40,10 @@ export const ChatButtons: React.FC = ({ }) => { return (
- - - - - Copy - {message.sender === USER_SENDER ? ( <> @@ -96,6 +88,14 @@ export const ChatButtons: React.FC = ({ Insert / Replace at cursor + + + + + Copy + + +
+
+
+ ); +} diff --git a/src/components/modals/AddPromptModal.tsx b/src/components/modals/AddPromptModal.tsx index 4f87a771..d158d340 100644 --- a/src/components/modals/AddPromptModal.tsx +++ b/src/components/modals/AddPromptModal.tsx @@ -5,6 +5,7 @@ import { err2String } from "@/utils"; import { Input } from "@/components/ui/input"; import { Textarea } from "@/components/ui/textarea"; import { Button } from "@/components/ui/button"; +import { CustomPromptSyntaxInstruction } from "@/components/CustomPromptSyntaxInstruction"; interface AddPromptModalContentProps { initialTitle?: string; @@ -97,21 +98,7 @@ function AddPromptModalContent({
Use the following syntax in your prompt:
-
- - {"{}"} represents the selected text (not required). - - {`{[[Note Title]]}`} represents a note. - - {`{activeNote}`} represents the active note. - - {`{FolderPath}`} represents a folder of notes. - - - {`{#tag1, #tag2}`} represents ALL notes with ANY of the specified tags in their - property (an OR operation).{" "} - -
- - Tip: turn on debug mode to show the processed prompt in the chat window. - -
-
+