Squashed new commits from master to preview

commit 730d0c82b1
Author: Logan Yang <logancyang@gmail.com>
Date:   Mon Apr 21 20:13:21 2025 -0700

    Switch insert and copy button positions (#1463)

commit 4efdc941f2
Author: Logan Yang <logancyang@gmail.com>
Date:   Mon Apr 21 16:15:50 2025 -0700

    Fix canvas read (#1462)

commit 93e4a04ef1
Author: Emt-lin <41323133+Emt-lin@users.noreply.github.com>
Date:   Tue Apr 22 07:13:47 2025 +0800

    fix: Add a new line when press the Enter key on mobile. (#1450)

commit 3c65d79a74
Author: Logan Yang <logancyang@gmail.com>
Date:   Mon Apr 21 15:38:36 2025 -0700

    Implement canvas adaptor (#1461)

    * Implement canvas parser
    * Refactor context processing and custom prompt handling

    - Update `processContextNotes` to exclude notes already processed by custom prompts using a set of excluded note paths.
    - Modify `extractVariablesFromPrompt` to return both a variables map and a set of included files.
    - Adjust `processPrompt` to track included files during prompt processing.
    - Update `Chat` component to handle context notes more efficiently by avoiding duplication of processed notes.
    - Ensure that the `FileParserManager` is initialized with the vault in `main.ts`.

    * Include canvas files in note context modal
    * Fix tests

commit 6a9085a71c
Author: Logan Yang <logancyang@gmail.com>
Date:   Sun Apr 20 16:14:54 2025 -0700

    Add a toggle to turn custom prompt templating off (#1460)

commit 448a64c383
Author: Logan Yang <logancyang@gmail.com>
Date:   Sun Apr 20 15:52:42 2025 -0700

    Update dependencies and support gpt 4.1 series, o4-mini and grok 3 (#1459)

commit 1d46ab90b0
Author: Logan Yang <logancyang@gmail.com>
Date:   Sun Apr 20 12:14:56 2025 -0700

    Fix image in note logic (#1457)

    * Refactor image processing in note context
    * Add passMarkdownImages setting

commit f4bf334c27
Author: Felix Haase <felix.haase@feki.de>
Date:   Fri Apr 18 08:19:35 2025 +0200

    Ollama ApiKey support (#1421)

    * Update @langchain/ollama and ollama
    * Ollama: support api keys by passing headers

commit 19d8b50a74
Author: Emt-lin <41323133+Emt-lin@users.noreply.github.com>
Date:   Fri Apr 18 14:13:25 2025 +0800

    refactor: Optimize some user experiences. (#1441)

commit fe9b9311ba
Author: Zero Liu <zerolxy@gmail.com>
Date:   Sun Apr 13 17:40:42 2025 -0700

    Improve custom command (v3) (#1446)

    * Trim responses and auto focus replace on finishing
    * Make textarea auto scroll to bottom when generating
    * Support custom prompt syntax in custom command
    * Support follow up instruction
    * Fix unit test
    * Allow followup instruction to use custom prompt syntax

commit 648412a914
Author: Zero Liu <zerolxy@gmail.com>
Date:   Wed Apr 2 21:44:37 2025 -0700

    Add update notification (#1415)

commit cb4510e920
Author: Zero Liu <zerolxy@gmail.com>
Date:   Wed Apr 2 21:43:26 2025 -0700

    Add user_id to broca requests (#1414)
This commit is contained in:
Logan Yang 2025-04-22 00:03:19 -07:00
parent 4b4d09924f
commit 684a3fc2aa
37 changed files with 2325 additions and 937 deletions

178
package-lock.json generated
View file

@ -18,12 +18,13 @@
"@langchain/anthropic": "^0.3.3",
"@langchain/cohere": "^0.3.0",
"@langchain/community": "^0.3.22",
"@langchain/core": "^0.3.27",
"@langchain/core": "^0.3.45",
"@langchain/deepseek": "^0.0.1",
"@langchain/google-genai": "^0.1.6",
"@langchain/groq": "^0.1.2",
"@langchain/mistralai": "^0.2.0",
"@langchain/openai": "^0.4.2",
"@langchain/openai": "^0.5.6",
"@langchain/xai": "^0.0.2",
"@orama/orama": "^3.0.0-rc-2",
"@radix-ui/react-checkbox": "^1.1.3",
"@radix-ui/react-collapsible": "^1.1.2",
@ -69,7 +70,7 @@
"tailwindcss-animate": "^1.0.7"
},
"devDependencies": {
"@langchain/ollama": "^0.1.1",
"@langchain/ollama": "^0.2.0",
"@tailwindcss/container-queries": "^0.1.1",
"@testing-library/jest-dom": "^5.16.5",
"@testing-library/react": "^14.0.0",
@ -4332,17 +4333,16 @@
}
},
"node_modules/@langchain/core": {
"version": "0.3.39",
"resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.3.39.tgz",
"integrity": "sha512-muXs4asy1A7qDtcdznxqyBfxf4N6qxofY/S0c95vbsWa0r9YAE2PttHIjcuxSy1q2jUiTkpCcgFEjNJRQRVhEw==",
"license": "MIT",
"version": "0.3.45",
"resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.3.45.tgz",
"integrity": "sha512-4icCMCeuauVvwH43zgCE6Hk2Zbke5OccfVhFpTiIJPo8WoU/ALdaZlS22S+6+qZ9lFzFHsgZ6K33Av9aWhxkQA==",
"dependencies": {
"@cfworker/json-schema": "^4.0.2",
"ansi-styles": "^5.0.0",
"camelcase": "6",
"decamelize": "1.2.0",
"js-tiktoken": "^1.0.12",
"langsmith": ">=0.2.8 <0.4.0",
"langsmith": "^0.3.16",
"mustache": "^4.2.0",
"p-queue": "^6.6.2",
"p-retry": "4",
@ -4376,6 +4376,57 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/@langchain/core/node_modules/chalk": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
"dependencies": {
"ansi-styles": "^4.1.0",
"supports-color": "^7.1.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/chalk/chalk?sponsor=1"
}
},
"node_modules/@langchain/core/node_modules/chalk/node_modules/ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"dependencies": {
"color-convert": "^2.0.1"
},
"engines": {
"node": ">=8"
},
"funding": {
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/@langchain/core/node_modules/langsmith": {
"version": "0.3.20",
"resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.3.20.tgz",
"integrity": "sha512-zwVQos6tjcksCTfdM67QKq7yyED4GmQiZw/sJ6UCMYZxlvTMMg3PeQ9tOePXAWNWoJygOnH+EwGXr7gYOOETDg==",
"dependencies": {
"@types/uuid": "^10.0.0",
"chalk": "^4.1.2",
"console-table-printer": "^2.12.1",
"p-queue": "^6.6.2",
"p-retry": "4",
"semver": "^7.6.3",
"uuid": "^10.0.0"
},
"peerDependencies": {
"openai": "*"
},
"peerDependenciesMeta": {
"openai": {
"optional": true
}
}
},
"node_modules/@langchain/core/node_modules/uuid": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",
@ -4403,6 +4454,23 @@
"@langchain/core": ">=0.3.0 <0.4.0"
}
},
"node_modules/@langchain/deepseek/node_modules/@langchain/openai": {
"version": "0.4.9",
"resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.4.9.tgz",
"integrity": "sha512-NAsaionRHNdqaMjVLPkFCyjUDze+OqRHghA1Cn4fPoAafz+FXcl9c7LlEl9Xo0FH6/8yiCl7Rw2t780C/SBVxQ==",
"dependencies": {
"js-tiktoken": "^1.0.12",
"openai": "^4.87.3",
"zod": "^3.22.4",
"zod-to-json-schema": "^3.22.3"
},
"engines": {
"node": ">=18"
},
"peerDependencies": {
"@langchain/core": ">=0.3.39 <0.4.0"
}
},
"node_modules/@langchain/google-genai": {
"version": "0.1.6",
"resolved": "https://registry.npmjs.org/@langchain/google-genai/-/google-genai-0.1.6.tgz",
@ -4483,13 +4551,16 @@
}
},
"node_modules/@langchain/ollama": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/@langchain/ollama/-/ollama-0.1.1.tgz",
"integrity": "sha512-IQEdzGkfKzdoyys3GW5hCXc64d/u1xkrYXved73BLO+bnyQfzrM224jdsiYGUpjW3cUaO1ebD6PUiMYcANPPFQ==",
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/@langchain/ollama/-/ollama-0.2.0.tgz",
"integrity": "sha512-jLlYFqt+nbhaJKLakk7lRTWHZJ7wHeJLM6yuv4jToQ8zPzpL//372+MjggDoW0mnw8ofysg1T2C6mEJspKJtiA==",
"devOptional": true,
"license": "MIT",
"dependencies": {
"ollama": "^0.5.9",
"uuid": "^10.0.0"
"ollama": "^0.5.12",
"uuid": "^10.0.0",
"zod": "^3.24.1",
"zod-to-json-schema": "^3.24.1"
},
"engines": {
"node": ">=18"
@ -4512,13 +4583,12 @@
}
},
"node_modules/@langchain/openai": {
"version": "0.4.2",
"resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.4.2.tgz",
"integrity": "sha512-Cuj7qbVcycALTP0aqZuPpEc7As8cwiGaU21MhXRyZFs+dnWxKYxZ1Q1z4kcx6cYkq/I+CNwwmk+sP+YruU73Aw==",
"license": "MIT",
"version": "0.5.6",
"resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.5.6.tgz",
"integrity": "sha512-zN0iyJthPNmcefIBVybZwcTBgcqu/ElJFov42ZntxEncK4heOMAE9lkq9LQ5CaPU/SgrduibrM1oL57+tLUtaA==",
"dependencies": {
"js-tiktoken": "^1.0.12",
"openai": "^4.77.0",
"openai": "^4.93.0",
"zod": "^3.22.4",
"zod-to-json-schema": "^3.22.3"
},
@ -4526,7 +4596,7 @@
"node": ">=18"
},
"peerDependencies": {
"@langchain/core": ">=0.3.29 <0.4.0"
"@langchain/core": ">=0.3.39 <0.4.0"
}
},
"node_modules/@langchain/textsplitters": {
@ -4543,6 +4613,38 @@
"@langchain/core": ">=0.2.21 <0.4.0"
}
},
"node_modules/@langchain/xai": {
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/@langchain/xai/-/xai-0.0.2.tgz",
"integrity": "sha512-wVOs7SfJs4VWk/oiHJomaoaZ+r9nQhPqbEXlQ2D8L0d54PxYhb1ILR9rub9LT1RpqazSX8HG4A8+hX4R01qkSg==",
"dependencies": {
"@langchain/openai": "~0.4.4",
"zod": "^3.24.2"
},
"engines": {
"node": ">=18"
},
"peerDependencies": {
"@langchain/core": ">=0.2.21 <0.4.0"
}
},
"node_modules/@langchain/xai/node_modules/@langchain/openai": {
"version": "0.4.9",
"resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.4.9.tgz",
"integrity": "sha512-NAsaionRHNdqaMjVLPkFCyjUDze+OqRHghA1Cn4fPoAafz+FXcl9c7LlEl9Xo0FH6/8yiCl7Rw2t780C/SBVxQ==",
"dependencies": {
"js-tiktoken": "^1.0.12",
"openai": "^4.87.3",
"zod": "^3.22.4",
"zod-to-json-schema": "^3.22.3"
},
"engines": {
"node": ">=18"
},
"peerDependencies": {
"@langchain/core": ">=0.3.39 <0.4.0"
}
},
"node_modules/@lezer/common": {
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.2.3.tgz",
@ -10218,6 +10320,14 @@
"integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
"dev": true
},
"node_modules/console-table-printer": {
"version": "2.12.1",
"resolved": "https://registry.npmjs.org/console-table-printer/-/console-table-printer-2.12.1.tgz",
"integrity": "sha512-wKGOQRRvdnd89pCeH96e2Fn4wkbenSP6LMHfjfyNLMbGuHEFbMqQNuxXqd0oXG9caIOQ1FTvc5Uijp9/4jujnQ==",
"dependencies": {
"simple-wcswidth": "^1.0.1"
}
},
"node_modules/content-disposition": {
"version": "0.5.4",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
@ -12475,7 +12585,6 @@
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
"dev": true,
"engines": {
"node": ">=8"
}
@ -17400,10 +17509,11 @@
}
},
"node_modules/ollama": {
"version": "0.5.9",
"resolved": "https://registry.npmjs.org/ollama/-/ollama-0.5.9.tgz",
"integrity": "sha512-F/KZuDRC+ZsVCuMvcOYuQ6zj42/idzCkkuknGyyGVmNStMZ/sU3jQpvhnl4SyC0+zBzLiKNZJnJeuPFuieWZvQ==",
"version": "0.5.14",
"resolved": "https://registry.npmjs.org/ollama/-/ollama-0.5.14.tgz",
"integrity": "sha512-pvOuEYa2WkkAumxzJP0RdEYHkbZ64AYyyUszXVX7ruLvk5L+EiO2G71da2GqEQ4IAk4j6eLoUbGk5arzFT1wJA==",
"devOptional": true,
"license": "MIT",
"dependencies": {
"whatwg-fetch": "^3.6.20"
}
@ -17449,10 +17559,9 @@
"integrity": "sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ=="
},
"node_modules/openai": {
"version": "4.83.0",
"resolved": "https://registry.npmjs.org/openai/-/openai-4.83.0.tgz",
"integrity": "sha512-fmTsqud0uTtRKsPC7L8Lu55dkaTwYucqncDHzVvO64DKOpNTuiYwjbR/nVgpapXuYy8xSnhQQPUm+3jQaxICgw==",
"license": "Apache-2.0",
"version": "4.95.1",
"resolved": "https://registry.npmjs.org/openai/-/openai-4.95.1.tgz",
"integrity": "sha512-IqJy+ymeW+k/Wq+2YVN3693OQMMcODRtHEYOlz263MdUwnN/Dwdl9c2EXSxLLtGEHkSHAfvzpDMHI5MaWJKXjQ==",
"dependencies": {
"@types/node": "^18.11.18",
"@types/node-fetch": "^2.6.4",
@ -19301,6 +19410,11 @@
"integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==",
"peer": true
},
"node_modules/simple-wcswidth": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/simple-wcswidth/-/simple-wcswidth-1.0.1.tgz",
"integrity": "sha512-xMO/8eNREtaROt7tJvWJqHBDTMFN4eiQ5I4JRMuilwfnFcV5W9u7RUkueNkdw0jPqGMX36iCywelS5yilTuOxg=="
},
"node_modules/sisteransi": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz",
@ -19895,7 +20009,6 @@
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
"dev": true,
"dependencies": {
"has-flag": "^4.0.0"
},
@ -20766,7 +20879,8 @@
"version": "3.6.20",
"resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz",
"integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==",
"devOptional": true
"devOptional": true,
"license": "MIT"
},
"node_modules/whatwg-mimetype": {
"version": "3.0.0",
@ -21080,9 +21194,9 @@
}
},
"node_modules/zod": {
"version": "3.24.1",
"resolved": "https://registry.npmjs.org/zod/-/zod-3.24.1.tgz",
"integrity": "sha512-muH7gBL9sI1nciMZV67X5fTKKBLtwpZ5VBp1vsOQzj1MhrBZ4wlVCm3gedKZWLp0Oyel8sIGfeiz54Su+OVT+A==",
"version": "3.24.3",
"resolved": "https://registry.npmjs.org/zod/-/zod-3.24.3.tgz",
"integrity": "sha512-HhY1oqzWCQWuUqvBFnsyrtZRhyPeR7SUGv+C4+MsisMuVfSPx8HpwWqH8tRahSlt6M3PiFAcoeFhZAqIXTxoSg==",
"funding": {
"url": "https://github.com/sponsors/colinhacks"
}

View file

@ -33,7 +33,7 @@
},
"license": "AGPL-3.0",
"devDependencies": {
"@langchain/ollama": "^0.1.1",
"@langchain/ollama": "^0.2.0",
"@tailwindcss/container-queries": "^0.1.1",
"@testing-library/jest-dom": "^5.16.5",
"@testing-library/react": "^14.0.0",
@ -80,12 +80,13 @@
"@langchain/anthropic": "^0.3.3",
"@langchain/cohere": "^0.3.0",
"@langchain/community": "^0.3.22",
"@langchain/core": "^0.3.27",
"@langchain/core": "^0.3.45",
"@langchain/deepseek": "^0.0.1",
"@langchain/google-genai": "^0.1.6",
"@langchain/groq": "^0.1.2",
"@langchain/mistralai": "^0.2.0",
"@langchain/openai": "^0.4.2",
"@langchain/openai": "^0.5.6",
"@langchain/xai": "^0.0.2",
"@orama/orama": "^3.0.0-rc-2",
"@radix-ui/react-checkbox": "^1.1.3",
"@radix-ui/react-collapsible": "^1.1.2",

View file

@ -111,9 +111,14 @@ export class BrevilabsClient {
endpoint: string,
body: any,
method = "POST",
excludeAuthHeader = false
excludeAuthHeader = false,
skipLicenseCheck = false
): Promise<{ data: T | null; error?: Error }> {
this.checkLicenseKey();
if (!skipLicenseCheck) {
this.checkLicenseKey();
}
body.user_id = getSettings().userId;
const url = new URL(`${BREVILABS_API_BASE_URL}${endpoint}`);
if (method === "GET") {
@ -162,6 +167,7 @@ export class BrevilabsClient {
license_key: await getDecryptedKey(getSettings().plusLicenseKey),
},
"POST",
true,
true
);
if (error) {

View file

@ -1,7 +1,5 @@
import { getCurrentProject } from "@/aiParams";
import { getStandaloneQuestion } from "@/chainUtils";
import { Composer } from "./composer";
import { getSystemPrompt } from "@/settings/model";
import {
ABORT_REASON,
AI_SENDER,
@ -10,8 +8,15 @@ import {
MAX_CHARS_FOR_LOCAL_SEARCH_CONTEXT,
ModelCapability,
} from "@/constants";
import {
ImageBatchProcessor,
ImageContent,
ImageProcessingResult,
MessageContent,
} from "@/imageProcessing/imageProcessor";
import { BrevilabsClient } from "@/LLMProviders/brevilabsClient";
import { logError } from "@/logger";
import { logInfo } from "@/logger";
import { getSettings, getSystemPrompt } from "@/settings/model";
import { ChatMessage } from "@/sharedState";
import { ToolManager } from "@/tools/toolManager";
import {
@ -22,13 +27,11 @@ import {
formatDateTime,
getApiErrorMessage,
getMessageRole,
ImageContent,
ImageProcessor,
MessageContent,
} from "@/utils";
import { BaseChatModel } from "@langchain/core/language_models/chat_models";
import { Notice } from "obsidian";
import ChainManager from "./chainManager";
import { Composer } from "./composer";
import { COPILOT_TOOL_NAMES, IntentAnalyzer } from "./intentAnalyzer";
import ProjectManager from "./projectManager";
@ -314,58 +317,26 @@ class CopilotPlusChainRunner extends BaseChainRunner {
return hasYoutubeCommand && youtubeUrl !== null && words.length === 1;
}
private async processImageUrls(urls: string[]): Promise<ImageContent[]> {
try {
const imageUrls = await Promise.all(
urls.map(async (url) => {
if (await ImageProcessor.isImageUrl(url, this.chainManager.app.vault)) {
const imageContent = await ImageProcessor.convertToBase64(
url,
this.chainManager.app.vault
);
if (!imageContent) {
logError(`Failed to process image: ${url}`);
return null;
}
return imageContent;
}
return null;
})
);
// Filter out null values and return valid image URLs
const validImages = imageUrls.filter((item): item is ImageContent => item !== null);
return validImages;
} catch (error) {
logError("Error processing image URLs:", error);
return [];
}
private async processImageUrls(urls: string[]): Promise<ImageProcessingResult> {
const failedImages: string[] = [];
const processedImages = await ImageBatchProcessor.processUrlBatch(
urls,
failedImages,
this.chainManager.app.vault
);
ImageBatchProcessor.showFailedImagesNotice(failedImages);
return processedImages;
}
private async processExistingImages(content: MessageContent[]): Promise<ImageContent[]> {
try {
const imageContent = await Promise.all(
content
.filter(
(item): item is ImageContent => item.type === "image_url" && !!item.image_url?.url
)
.map(async (item) => {
const processedContent = await ImageProcessor.convertToBase64(
item.image_url.url,
this.chainManager.app.vault
);
if (!processedContent) {
logError(`Failed to process existing image: ${item.image_url.url}`);
return null;
}
return processedContent;
})
);
return imageContent.filter((item): item is ImageContent => item !== null);
} catch (error) {
logError("Error processing images:", error);
return [];
}
private async processChatInputImages(content: MessageContent[]): Promise<ImageProcessingResult> {
const failedImages: string[] = [];
const processedImages = await ImageBatchProcessor.processChatImageBatch(
content,
failedImages,
this.chainManager.app.vault
);
ImageBatchProcessor.showFailedImagesNotice(failedImages);
return processedImages;
}
private async extractEmbeddedImages(content: string): Promise<string[]> {
@ -379,33 +350,61 @@ class CopilotPlusChainRunner extends BaseChainRunner {
textContent: string,
userMessage: ChatMessage
): Promise<MessageContent[]> {
const content: MessageContent[] = [
const failureMessages: string[] = [];
const successfulImages: ImageContent[] = [];
const settings = getSettings();
// Collect all image sources
const imageSources: { urls: string[]; type: string }[] = [];
// Safely check and add context URLs
const contextUrls = userMessage.context?.urls;
if (contextUrls && contextUrls.length > 0) {
imageSources.push({ urls: contextUrls, type: "context" });
}
// Process embedded images only if setting is enabled
if (settings.passMarkdownImages) {
const embeddedImages = await this.extractEmbeddedImages(textContent);
if (embeddedImages.length > 0) {
imageSources.push({ urls: embeddedImages, type: "embedded" });
}
}
// Process all image sources
for (const source of imageSources) {
const result = await this.processImageUrls(source.urls);
successfulImages.push(...result.successfulImages);
failureMessages.push(...result.failureDescriptions);
}
// Process existing chat content images if present
const existingContent = userMessage.content;
if (existingContent && existingContent.length > 0) {
const result = await this.processChatInputImages(existingContent);
successfulImages.push(...result.successfulImages);
failureMessages.push(...result.failureDescriptions);
}
// Let the LLM know about the image processing failures
let finalText = textContent;
if (failureMessages.length > 0) {
finalText = `${textContent}\n\nNote: \n${failureMessages.join("\n")}\n`;
}
const messageContent: MessageContent[] = [
{
type: "text",
text: textContent,
text: finalText,
},
];
// Process URLs in the message to identify images
if (userMessage.context?.urls && userMessage.context.urls.length > 0) {
const imageContents = await this.processImageUrls(userMessage.context.urls);
content.push(...imageContents);
// Add successful images after the text content
if (successfulImages.length > 0) {
messageContent.push(...successfulImages);
}
// Process embedded images from the text content
const embeddedImages = await this.extractEmbeddedImages(textContent);
if (embeddedImages.length > 0) {
const imageContents = await this.processImageUrls(embeddedImages);
content.push(...imageContents);
}
// Add existing image content if present
if (userMessage.content && userMessage.content.length > 0) {
const imageContents = await this.processExistingImages(userMessage.content);
content.push(...imageContents);
}
return content;
return messageContent;
}
private hasCapability(model: BaseChatModel, capability: ModelCapability): boolean {
@ -473,11 +472,9 @@ class CopilotPlusChainRunner extends BaseChainRunner {
content,
});
// Add debug logging for final request
if (debug) {
console.log("==== Final Request to AI ====\n", messages);
}
const enhancedUserMessage = content instanceof Array ? (content[0] as any).text : content;
logInfo("Enhanced user message: ", enhancedUserMessage);
logInfo("==== Final Request to AI ====\n", messages);
const streamer = new ThinkBlockStreamer(updateCurrentAiMessage);
const chatStream = await this.chainManager.chatModelManager.getChatModel().stream(messages);
@ -620,10 +617,7 @@ class CopilotPlusChainRunner extends BaseChainRunner {
// Enhance with tool outputs.
let enhancedUserMessage = this.prepareEnhancedUserMessage(cleanedUserMessage, toolOutputs);
// If no results, default to LLM Chain
if (debug) {
console.log("No local search results. Using standard LLM Chain.");
console.log("Enhanced user message:", enhancedUserMessage);
}
logInfo("No local search results. Using standard LLM Chain.");
// Enhance with composer output.
if (messageForAnalysis.includes("@composer")) {
@ -733,7 +727,7 @@ class CopilotPlusChainRunner extends BaseChainRunner {
.join("\n\n");
}
}
return `User message: ${userMessage}${context}`;
return `${userMessage}${context}`;
}
private getTimeExpression(toolCalls: any[]): string {

View file

@ -19,6 +19,7 @@ import { ChatGroq } from "@langchain/groq";
import { ChatMistralAI } from "@langchain/mistralai";
import { ChatOllama } from "@langchain/ollama";
import { ChatOpenAI } from "@langchain/openai";
import { ChatXAI } from "@langchain/xai";
import { Notice } from "obsidian";
type ChatConstructorType = {
@ -31,6 +32,7 @@ const CHAT_PROVIDER_CONSTRUCTORS = {
[ChatModelProviders.ANTHROPIC]: ChatAnthropic,
[ChatModelProviders.COHEREAI]: ChatCohere,
[ChatModelProviders.GOOGLE]: ChatGoogleGenerativeAI,
[ChatModelProviders.XAI]: ChatXAI,
[ChatModelProviders.OPENROUTERAI]: ChatOpenAI,
[ChatModelProviders.OLLAMA]: ChatOllama,
[ChatModelProviders.LM_STUDIO]: ChatOpenAI,
@ -63,6 +65,7 @@ export default class ChatModelManager {
[ChatModelProviders.COHEREAI]: () => getSettings().cohereApiKey,
[ChatModelProviders.OPENROUTERAI]: () => getSettings().openRouterAiApiKey,
[ChatModelProviders.GROQ]: () => getSettings().groqApiKey,
[ChatModelProviders.XAI]: () => getSettings().xaiApiKey,
[ChatModelProviders.OLLAMA]: () => "default-key",
[ChatModelProviders.LM_STUDIO]: () => "default-key",
[ChatModelProviders.OPENAI_FORMAT]: () => "default-key",
@ -167,6 +170,11 @@ export default class ChatModelManager {
],
baseUrl: customModel.baseUrl,
},
[ChatModelProviders.XAI]: {
apiKey: await getDecryptedKey(customModel.apiKey || settings.xaiApiKey),
model: modelName,
// This langchainjs XAI client does not support baseURL override
},
[ChatModelProviders.OPENROUTERAI]: {
modelName: modelName,
openAIApiKey: await getDecryptedKey(customModel.apiKey || settings.openRouterAiApiKey),
@ -182,10 +190,11 @@ export default class ChatModelManager {
[ChatModelProviders.OLLAMA]: {
// ChatOllama has `model` instead of `modelName`!!
model: modelName,
// @ts-ignore
apiKey: customModel.apiKey || "default-key",
// MUST NOT use /v1 in the baseUrl for ollama
baseUrl: customModel.baseUrl || "http://localhost:11434",
headers: new Headers({
Authorization: `Bearer ${await getDecryptedKey(customModel.apiKey || "default-key")}`,
}),
},
[ChatModelProviders.LM_STUDIO]: {
modelName: modelName,

View file

@ -256,6 +256,9 @@ export default class EmbeddingManager {
baseUrl: customModel.baseUrl || "http://localhost:11434",
model: modelName,
truncate: true,
headers: {
Authorization: `Bearer ${await getDecryptedKey(customModel.apiKey || "default-key")}`,
},
},
[EmbeddingModelProviders.LM_STUDIO]: {
modelName,

View file

@ -6,132 +6,107 @@ export const COMMAND_NAME_MAX_LENGTH = 50;
export const DEFAULT_INLINE_EDIT_COMMANDS: InlineEditCommandSettings[] = [
{
name: "Fix grammar and spelling",
prompt:
`<instruction>Fix the grammar and spelling of the text below. Preserve all formatting, line breaks, and special characters. Do not add or remove any content. Return only the corrected text.</instruction>\n\n` +
`<text>${SELECTED_TEXT_PLACEHOLDER}</text>`,
prompt: `Fix the grammar and spelling of {}. Preserve all formatting, line breaks, and special characters. Do not add or remove any content. Return only the corrected text.`,
showInContextMenu: true,
},
{
name: "Translate to Chinese",
prompt:
`<instruction>Translate the text below into Chinese:
prompt: `Translate {} into Chinese:
1. Preserve the meaning and tone
2. Maintain appropriate cultural context
3. Keep formatting and structure
Return only the translated text.</instruction>\n\n` +
`<text>${SELECTED_TEXT_PLACEHOLDER}</text>`,
Return only the translated text.`,
showInContextMenu: true,
},
{
name: "Summarize",
prompt:
`<instruction>Create a bullet-point summary of the text below. Each bullet point should capture a key point. Return only the bullet-point summary.</instruction>\n\n` +
`<text>${SELECTED_TEXT_PLACEHOLDER}</text>`,
prompt: `Create a bullet-point summary of {}. Each bullet point should capture a key point. Return only the bullet-point summary.`,
showInContextMenu: true,
},
{
name: "Simplify",
prompt:
`<instruction>Simplify the text below to a 6th-grade reading level (ages 11-12). Use simple sentences, common words, and clear explanations. Maintain the original key concepts. Return only the simplified text.</instruction>\n\n` +
`<text>{copilot-selection}</text>`,
prompt: `Simplify {} to a 6th-grade reading level (ages 11-12). Use simple sentences, common words, and clear explanations. Maintain the original key concepts. Return only the simplified text.`,
showInContextMenu: true,
},
{
name: "Emojify",
prompt:
`<instruction>Add relevant emojis to enhance the text below. Follow these rules:
prompt: `Add relevant emojis to enhance {}. Follow these rules:
1. Insert emojis at natural breaks in the text
2. Never place two emojis next to each other
3. Keep all original text unchanged
4. Choose emojis that match the context and tone
Return only the emojified text.</instruction>\n\n` + `<text>{copilot-selection}</text>`,
Return only the emojified text.`,
showInContextMenu: true,
},
{
name: "Make shorter",
prompt:
`<instruction>Reduce the text below to half its length while preserving these elements:
prompt: `Reduce {} to half its length while preserving these elements:
1. Main ideas and key points
2. Essential details
3. Original tone and style
Return only the shortened text.</instruction>\n\n` +
`<text>${SELECTED_TEXT_PLACEHOLDER}</text>`,
Return only the shortened text.`,
showInContextMenu: true,
},
{
name: "Make longer",
prompt:
`<instruction>Expand the text below to twice its length by:
prompt: `Expand {} to twice its length by:
1. Adding relevant details and examples
2. Elaborating on key points
3. Maintaining the original tone and style
Return only the expanded text.</instruction>\n\n` + `<text>${SELECTED_TEXT_PLACEHOLDER}</text>`,
Return only the expanded text.`,
showInContextMenu: true,
},
{
name: "Generate table of contents",
prompt:
`<instruction>Generate a hierarchical table of contents for the text below. Use appropriate heading levels (H1, H2, H3, etc.). Include page numbers if present. Return only the table of contents.</instruction>\n\n` +
`<text>${SELECTED_TEXT_PLACEHOLDER}</text>`,
prompt: `Generate a hierarchical table of contents for {}. Use appropriate heading levels (H1, H2, H3, etc.). Include page numbers if present. Return only the table of contents.`,
showInContextMenu: false,
},
{
name: "Generate glossary",
prompt:
`<instruction>Create a glossary of important terms, concepts, and phrases from the text below. Format each entry as "Term: Definition". Sort entries alphabetically. Return only the glossary.</instruction>\n\n` +
`<text>{copilot-selection}</text>`,
prompt: `Create a glossary of important terms, concepts, and phrases from {}. Format each entry as "Term: Definition". Sort entries alphabetically. Return only the glossary.`,
showInContextMenu: false,
},
{
name: "Remove URLs",
prompt:
`<instruction>Remove all URLs from the text below. Preserve all other content and formatting. URLs may be in various formats (http, https, www). Return only the text with URLs removed.</instruction>\n\n` +
`<text>${SELECTED_TEXT_PLACEHOLDER}</text>`,
prompt: `Remove all URLs from {}. Preserve all other content and formatting. URLs may be in various formats (http, https, www). Return only the text with URLs removed.`,
showInContextMenu: false,
},
{
name: "Rewrite as tweet",
prompt:
`<instruction>Rewrite the text below as a single tweet with these requirements:
prompt: `Rewrite {} as a single tweet with these requirements:
1. Maximum 280 characters
2. Use concise, impactful language
3. Maintain the core message
Return only the tweet text.</instruction>\n\n` + `<text>${SELECTED_TEXT_PLACEHOLDER}</text>`,
Return only the tweet text.`,
showInContextMenu: false,
},
{
name: "Rewrite as tweet thread",
prompt:
`<instruction>Convert the text below into a Twitter thread following these rules:
prompt: `Convert {} into a Twitter thread following these rules:
1. Each tweet must be under 240 characters
2. Start with "THREAD START" on its own line
3. Separate tweets with "\n\n---\n\n"
4. End with "THREAD END" on its own line
5. Make content engaging and clear
Return only the formatted thread.</instruction>\n\n` +
`<text>${SELECTED_TEXT_PLACEHOLDER}</text>`,
Return only the formatted thread.`,
showInContextMenu: false,
},
{
name: "Explain like I am 5",
prompt:
`<instruction>Explain the text below in simple terms that a 5-year-old would understand:
prompt: `Explain {} in simple terms that a 5-year-old would understand:
1. Use basic vocabulary
2. Include simple analogies
3. Break down complex concepts
Return only the simplified explanation.</instruction>\n\n` +
`<text>${SELECTED_TEXT_PLACEHOLDER}</text>`,
Return only the simplified explanation.`,
showInContextMenu: false,
},
{
name: "Rewrite as press release",
prompt:
`<instruction>Transform the text below into a professional press release:
prompt: `Transform {} into a professional press release:
1. Use formal, journalistic style
2. Include headline and dateline
3. Follow inverted pyramid structure
Return only the press release format.</instruction>\n\n` +
`<text>${SELECTED_TEXT_PLACEHOLDER}</text>`,
Return only the press release format.`,
showInContextMenu: false,
},
];

View file

@ -1,8 +1,9 @@
import {
COMMAND_NAME_MAX_LENGTH,
DEFAULT_INLINE_EDIT_COMMANDS,
SELECTED_TEXT_PLACEHOLDER,
SELECTED_TEXT_PLACEHOLDER as LEGACY_SELECTED_TEXT_PLACEHOLDER,
} from "@/commands/constants";
import { processPrompt } from "@/customPromptProcessor";
import { InlineEditCommandSettings, getSettings, useSettingsValue } from "@/settings/model";
export function getCommandId(commandName: string) {
@ -54,16 +55,44 @@ export function useInlineEditCommands(): InlineEditCommandSettings[] {
}
/**
* Replace the {copilot-selection} placeholder with the selected text.
* If the placeholder is not found, append the selected text to the prompt.
* Process the command prompt.
*/
export function processCommandPrompt(prompt: string, selectedText: string) {
const index = prompt.indexOf(SELECTED_TEXT_PLACEHOLDER);
export async function processCommandPrompt(
prompt: string,
selectedText: string,
skipAppendingSelectedText = false
) {
const result = await processPrompt(
prompt,
selectedText,
app.vault,
app.workspace.getActiveFile()
);
const processedPrompt = result.processedPrompt;
if (processedPrompt.includes("{selectedText}") || skipAppendingSelectedText) {
// Containing {selectedText} means the prompt was using the custom prompt
// processor way of handling the selected text. No need to go through the
// legacy placeholder.
return processedPrompt;
}
// This is the legacy custom command selected text placeholder. It replaced
// {copilot-selection} in the prompt with the selected text. This is different
// from the custom prompt processor which uses {} in the prompt and appends
// the selected text to the prompt. We cannot change user's custom commands
// that have the old placeholder, so we need to support both.
// Also, selected text is required for custom commands. If neither `{}` nor
// `{copilot-selection}` is found, append the selected text to the prompt.
const index = processedPrompt.indexOf(LEGACY_SELECTED_TEXT_PLACEHOLDER);
if (index === -1) {
return prompt + "\n\n" + selectedText;
return processedPrompt + "\n\n" + selectedText;
}
return (
prompt.slice(0, index) + selectedText + prompt.slice(index + SELECTED_TEXT_PLACEHOLDER.length)
processedPrompt.slice(0, index) +
selectedText +
processedPrompt.slice(index + LEGACY_SELECTED_TEXT_PLACEHOLDER.length)
);
}

View file

@ -10,6 +10,7 @@ import { updateChatMemory } from "@/chatUtils";
import { ChatControls } from "@/components/chat-components/ChatControls";
import ChatInput from "@/components/chat-components/ChatInput";
import ChatMessages from "@/components/chat-components/ChatMessages";
import { NewVersionBanner } from "@/components/chat-components/NewVersionBanner";
import { ProjectList } from "@/components/chat-components/ProjectList";
import { ABORT_REASON, COMMAND_IDS, EVENT_NAMES, LOADING_MESSAGES, USER_SENDER } from "@/constants";
import { AppContext, EventTargetContext } from "@/context";
@ -88,26 +89,10 @@ const Chat: React.FC<ChatProps> = ({
const appContext = useContext(AppContext);
const app = plugin.app || appContext;
const processContextNotes = async (
customPromptProcessor: CustomPromptProcessor,
fileParserManager: FileParserManager
) => {
const activeNote = app.workspace.getActiveFile();
return await contextProcessor.processContextNotes(
customPromptProcessor,
fileParserManager,
app.vault,
contextNotes,
includeActiveNote,
activeNote,
currentChain
);
};
const handleSendMessage = async ({
toolCalls,
urls,
contextNotes,
contextNotes: passedContextNotes, // Rename to avoid shadowing
}: {
toolCalls?: string[];
urls?: string[];
@ -140,7 +125,7 @@ const Chat: React.FC<ChatProps> = ({
});
}
const notes = [...(contextNotes || [])];
const notes = [...(passedContextNotes || [])];
const activeNote = app.workspace.getActiveFile();
if (includeActiveNote && activeNote && !notes.some((note) => note.path === activeNote.path)) {
notes.push(activeNote);
@ -170,11 +155,12 @@ const Chat: React.FC<ChatProps> = ({
// First, process the original user message for custom prompts
const customPromptProcessor = CustomPromptProcessor.getInstance(app.vault);
let processedUserMessage = await customPromptProcessor.processCustomPrompt(
inputMessage || "",
"",
app.workspace.getActiveFile() as TFile | undefined
);
const { processedPrompt: processedUserMessage, includedFiles } =
await customPromptProcessor.processCustomPrompt(
inputMessage || "",
"",
app.workspace.getActiveFile() as TFile | undefined
);
// Extract Mentions (such as URLs) from original input message only if using Copilot Plus chain
const urlContextAddition =
@ -182,11 +168,22 @@ const Chat: React.FC<ChatProps> = ({
? await mention.processUrls(inputMessage || "")
: { urlContext: "", imageUrls: [] };
// Add context notes
const noteContextAddition = await processContextNotes(customPromptProcessor, fileParserManager);
// Create set of file paths that were included in the custom prompt
const excludedNotePaths = new Set(includedFiles.map((file) => file.path));
// Add context notes, excluding those already processed by custom prompt
const noteContextAddition = await contextProcessor.processContextNotes(
excludedNotePaths,
fileParserManager,
app.vault,
notes,
includeActiveNote,
activeNote,
currentChain
);
// Combine everything
processedUserMessage =
const finalProcessedMessage =
processedUserMessage + urlContextAddition.urlContext + noteContextAddition;
let messageWithToolCalls = inputMessage;
@ -196,7 +193,7 @@ const Chat: React.FC<ChatProps> = ({
}
const promptMessageHidden: ChatMessage = {
message: processedUserMessage,
message: finalProcessedMessage,
originalMessage: messageWithToolCalls,
sender: USER_SENDER,
isVisible: false,
@ -504,11 +501,12 @@ ${chatContent}`;
if (!customPrompt) {
return selectedText;
}
return await customPromptProcessor.processCustomPrompt(
const result = await customPromptProcessor.processCustomPrompt(
customPrompt,
selectedText,
app.workspace.getActiveFile() ?? undefined
);
return result.processedPrompt; // Extract just the processed prompt string
}),
[]
);
@ -518,11 +516,12 @@ ${chatContent}`;
if (!customPrompt) {
return selectedText;
}
return await customPromptProcessor.processCustomPrompt(
const result = await customPromptProcessor.processCustomPrompt(
customPrompt,
selectedText,
app.workspace.getActiveFile() as TFile | undefined
);
return result.processedPrompt; // Extract just the processed prompt string
}),
[]
);
@ -613,7 +612,8 @@ ${chatContent}`;
const renderChatComponents = () => (
<>
<div className="flex-1 overflow-y-auto">
<div className="flex flex-col size-full overflow-hidden">
<NewVersionBanner currentVersion={plugin.manifest.version} />
<ChatMessages
chatHistory={chatHistory}
currentAiMessage={currentAiMessage}
@ -627,8 +627,6 @@ ${chatContent}`;
onReplaceChat={setInputMessage}
showHelperComponents={mode !== "project"}
/>
</div>
<div className="flex flex-col items-center justify-end w-full flex-none box-border sticky bottom-0 border-t border-border">
<ChatControls
onNewChat={handleNewChat}
onSaveAsNote={() => handleSaveAsNote(true)}

View file

@ -0,0 +1,22 @@
import React from "react";
export function CustomPromptSyntaxInstruction() {
return (
<ul className="text-sm px-4 m-0">
<li>
<span className="font-medium text-accent">{"{}"}</span> represents the selected text.
</li>
<li>
<span className="font-medium text-accent">{`{[[Note Title]]}`}</span> represents a note.
</li>
<li>
<span className="font-medium text-accent">{`{activeNote}`}</span> represents the active
note.
</li>
<li>
<span className="font-medium text-accent">{`{#tag1, #tag2}`}</span> represents ALL notes
with ANY of the specified tags in their property (an OR operation).
</li>
</ul>
);
}

View file

@ -40,18 +40,10 @@ export const ChatButtons: React.FC<ChatButtonsProps> = ({
}) => {
return (
<div
className={cn("flex", {
className={cn("flex gap-1", {
"group-hover:opacity-100 opacity-0": !Platform.isMobile,
})}
>
<Tooltip>
<TooltipTrigger asChild>
<Button variant="ghost2" size="fit" onClick={onCopy} title="Copy">
{isCopied ? <Check className="size-4" /> : <Copy className="size-4" />}
</Button>
</TooltipTrigger>
<TooltipContent>Copy</TooltipContent>
</Tooltip>
{message.sender === USER_SENDER ? (
<>
<Tooltip>
@ -96,6 +88,14 @@ export const ChatButtons: React.FC<ChatButtonsProps> = ({
</TooltipTrigger>
<TooltipContent>Insert / Replace at cursor</TooltipContent>
</Tooltip>
<Tooltip>
<TooltipTrigger asChild>
<Button variant="ghost2" size="fit" onClick={onCopy} title="Copy">
{isCopied ? <Check className="size-4" /> : <Copy className="size-4" />}
</Button>
</TooltipTrigger>
<TooltipContent>Copy</TooltipContent>
</Tooltip>
<Tooltip>
<TooltipTrigger asChild>
<Button onClick={onRegenerate} variant="ghost2" size="fit" title="Regenerate">

View file

@ -292,10 +292,14 @@ const ChatInput = forwardRef<{ focus: () => void }, ChatInputProps>(
}
if (e.key === "Enter" && !e.shiftKey) {
e.preventDefault();
onSendMessage(false);
setHistoryIndex(-1);
setTempInput("");
// Only prevent default and send message on non-mobile platforms
if (!Platform.isMobile) {
e.preventDefault();
onSendMessage(false);
setHistoryIndex(-1);
setTempInput("");
}
// On mobile, do nothing here, allowing the default newline behavior
} else if (e.key === "ArrowUp") {
if (currentLineIndex > 0 || selectionStart > 0) {
// Allow normal cursor movement within multi-line input

View file

@ -0,0 +1,78 @@
import { Button } from "@/components/ui/button";
import { useLatestVersion } from "@/hooks/useLatestVersion";
import { cn } from "@/lib/utils";
import { updateSetting, useSettingsValue } from "@/settings/model";
import { isNewerVersion } from "@/utils";
import { XIcon } from "lucide-react";
import React, { useState } from "react";
interface NewVersionBannerProps {
currentVersion: string;
}
export function NewVersionBanner({ currentVersion }: NewVersionBannerProps) {
const { latestVersion, hasUpdate } = useLatestVersion(currentVersion);
const lastDismissedVersion = useSettingsValue().lastDismissedVersion;
const [isVisible, setIsVisible] = useState(true);
const showBanner =
hasUpdate &&
latestVersion &&
isNewerVersion(latestVersion, currentVersion) &&
lastDismissedVersion !== latestVersion;
const handleDismiss = () => {
if (latestVersion) {
setIsVisible(false);
// Wait for animation to complete before updating setting
setTimeout(() => {
updateSetting("lastDismissedVersion", latestVersion);
}, 300);
}
};
if (!showBanner) {
return null;
}
return (
<div
className={cn(
"min-h-14 overflow-hidden",
isVisible
? "animate-in slide-in-from-top duration-300"
: "animate-out slide-out-to-top duration-300"
)}
>
<div className="flex items-center justify-between gap-2 p-2 pl-3 mb-1 text-xs border border-border border-solid rounded-md">
<div className="flex items-center gap-2">
<span className="font-medium">Update available:</span>(
<a
href={`https://github.com/logancyang/obsidian-copilot/releases/latest`}
target="_blank"
rel="noopener noreferrer"
className="text-normal"
>
v{latestVersion}
</a>
</div>
<div className="flex items-center gap-2">
<Button
size="fit"
variant="ghost2"
className="text-accent hover:text-accent-hover"
onClick={() => {
window.open(`obsidian://show-plugin?id=copilot`, "_blank");
handleDismiss();
}}
>
Update
</Button>
<Button variant="ghost2" size="icon" onClick={handleDismiss}>
<XIcon className="size-4" />
</Button>
</div>
</div>
</div>
);
}

View file

@ -5,6 +5,7 @@ import { err2String } from "@/utils";
import { Input } from "@/components/ui/input";
import { Textarea } from "@/components/ui/textarea";
import { Button } from "@/components/ui/button";
import { CustomPromptSyntaxInstruction } from "@/components/CustomPromptSyntaxInstruction";
interface AddPromptModalContentProps {
initialTitle?: string;
@ -97,21 +98,7 @@ function AddPromptModalContent({
</div>
<div className="text-sm text-muted -mt-1">Use the following syntax in your prompt:</div>
</div>
<div className="text-sm flex flex-col gap-1 bg-secondary/30 rounded-md p-2">
<strong>- {"{}"} represents the selected text (not required). </strong>
<strong>- {`{[[Note Title]]}`} represents a note. </strong>
<strong>- {`{activeNote}`} represents the active note. </strong>
<strong>- {`{FolderPath}`} represents a folder of notes. </strong>
<strong>
- {`{#tag1, #tag2}`} represents ALL notes with ANY of the specified tags in their
property (an OR operation).{" "}
</strong>
<div className="mt-1">
<span className="text-muted">
Tip: turn on debug mode to show the processed prompt in the chat window.
</span>
</div>
</div>
<CustomPromptSyntaxInstruction />
<Textarea
value={prompt}

View file

@ -17,7 +17,7 @@ export abstract class BaseNoteModal<T> extends FuzzySuggestModal<T> {
.filter(
(file): file is TFile =>
file instanceof TFile &&
(file.extension === "md" || file.extension === "pdf") &&
(file.extension === "md" || file.extension === "pdf" || file.extension === "canvas") &&
!excludeNotePaths.includes(file.path) &&
file.path !== this.activeNote?.path
);
@ -25,7 +25,9 @@ export abstract class BaseNoteModal<T> extends FuzzySuggestModal<T> {
// Get all other files that weren't recently opened
const allFiles = this.app.vault
.getFiles()
.filter((file) => file.extension === "md" || file.extension === "pdf");
.filter(
(file) => file.extension === "md" || file.extension === "pdf" || file.extension === "canvas"
);
const otherFiles = allFiles.filter(
(file) =>
@ -45,6 +47,8 @@ export abstract class BaseNoteModal<T> extends FuzzySuggestModal<T> {
}
if (extension === "pdf") {
title += " (PDF)";
} else if (extension === "canvas") {
title += " (Canvas)";
}
return title;
}

View file

@ -16,6 +16,7 @@ import { getModelDisplayText } from "@/components/ui/model-display";
import { cn } from "@/lib/utils";
import { ChevronDown } from "lucide-react";
import { logError } from "@/logger";
import { CustomPromptSyntaxInstruction } from "@/components/CustomPromptSyntaxInstruction";
type FormErrors = {
name?: string;
@ -90,10 +91,7 @@ function InlineEditCommandSettingsModalContent({
<div className="flex flex-col gap-2">
<Label htmlFor="prompt">Prompt</Label>
<div className="text-sm text-muted mb-2">
Use <code>{"{copilot-selection}"}</code> as a placeholder for the selected text. If not
included, the selected text will be appended to the prompt.
</div>
<CustomPromptSyntaxInstruction />
<Textarea
id="prompt"
value={command.prompt}

View file

@ -1,21 +1,71 @@
import { useModelKey } from "@/aiParams";
import { CustomModel, useModelKey } from "@/aiParams";
import { processCommandPrompt } from "@/commands/inlineEditCommandUtils";
import { Button } from "@/components/ui/button";
import { getModelDisplayText } from "@/components/ui/model-display";
import ChatModelManager from "@/LLMProviders/chatModelManager";
import { logError } from "@/logger";
import { InlineEditCommandSettings, useSettingsValue } from "@/settings/model";
import { ChatMessage } from "@/sharedState";
import { findCustomModel, insertIntoEditor } from "@/utils";
import { Bot, Copy, PenLine } from "lucide-react";
import { App, Modal, Notice } from "obsidian";
import React, { useCallback, useEffect, useMemo, useState } from "react";
import {
ChatPromptTemplate,
HumanMessagePromptTemplate,
MessagesPlaceholder,
SystemMessagePromptTemplate,
} from "@langchain/core/prompts";
import { RunnableSequence } from "@langchain/core/runnables";
import { BaseChatMemory, BufferMemory } from "langchain/memory";
import { ArrowBigUp, Bot, Command, Copy, CornerDownLeft, PenLine } from "lucide-react";
import { App, Modal, Notice, Platform } from "obsidian";
import React, { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { createRoot, Root } from "react-dom/client";
import ProjectManager from "@/LLMProviders/projectManager";
// Custom hook for managing chat chain
function useChatChain(selectedModel: CustomModel) {
const [chatMemory] = useState<BaseChatMemory>(
new BufferMemory({ returnMessages: true, memoryKey: "history" })
);
const [chatChain, setChatChain] = useState<RunnableSequence | null>(null);
// Initialize chat chain
useEffect(() => {
async function initChatChain() {
const chatModel = await ChatModelManager.getInstance().createModelInstance(selectedModel);
const chatPrompt = ChatPromptTemplate.fromMessages([
SystemMessagePromptTemplate.fromTemplate(
"You are a helpful assistant. You'll help the user with their content editing needs."
),
new MessagesPlaceholder("history"),
HumanMessagePromptTemplate.fromTemplate("{input}"),
]);
const newChatChain = RunnableSequence.from([
{
input: (initialInput) => initialInput.input,
memory: () => chatMemory.loadMemoryVariables({}),
},
{
input: (previousOutput) => previousOutput.input,
history: (previousOutput) => previousOutput.memory.history,
},
chatPrompt,
chatModel,
]);
setChatChain(newChatChain);
}
initChatChain();
}, [selectedModel, chatMemory]);
return { chatChain, chatMemory };
}
interface InlineEditModalContentProps {
originalText: string;
command: InlineEditCommandSettings;
onInsert: (message: string) => void;
onReplace: (message: string) => void;
onClose: () => void;
}
function InlineEditModalContent({
@ -23,10 +73,13 @@ function InlineEditModalContent({
command,
onInsert,
onReplace,
onClose,
}: InlineEditModalContentProps) {
const [aiCurrentMessage, setAiCurrentMessage] = useState<string | null>(null);
const [processedMessage, setProcessedMessage] = useState<string | null>(null);
const [followupInstruction, setFollowupInstruction] = useState<string>("");
const textareaRef = useRef<HTMLTextAreaElement>(null);
const followupRef = useRef<HTMLTextAreaElement>(null);
const [generating, setGenerating] = useState(true);
const [modelKey] = useModelKey();
const settings = useSettingsValue();
const selectedModel = useMemo(
@ -34,37 +87,163 @@ function InlineEditModalContent({
[command.modelKey, modelKey, settings.activeModels]
);
const commandName = command.name;
const handleAddMessage = useCallback((message: ChatMessage) => {
setProcessedMessage(message.message);
}, []);
const { chatChain, chatMemory } = useChatChain(selectedModel);
const commandName = command.name;
// Reusable function to handle streaming responses wrapped in useCallback
const streamResponse = useCallback(
async (input: string, abortController: AbortController) => {
if (!chatChain) {
console.error("Chat chain not initialized");
new Notice("Chat engine not ready. Please try again.");
setGenerating(false);
return null;
}
try {
setAiCurrentMessage(null);
setProcessedMessage(null);
setGenerating(true);
let fullResponse = "";
const chainWithSignal = chatChain.bind({ signal: abortController.signal });
const stream = await chainWithSignal.stream({ input });
for await (const chunk of stream) {
if (abortController.signal.aborted) break;
const chunkContent = typeof chunk.content === "string" ? chunk.content : "";
fullResponse += chunkContent;
setAiCurrentMessage(fullResponse);
}
if (!abortController.signal.aborted) {
const trimmedResponse = fullResponse.trim();
setProcessedMessage(trimmedResponse);
setGenerating(false);
await chatMemory.saveContext({ input }, { output: trimmedResponse });
return trimmedResponse;
}
return null;
} catch (error) {
logError("Error generating response:", error);
setGenerating(false);
return null;
}
},
[chatChain, chatMemory]
);
// Generate initial response
useEffect(() => {
const abortController = new AbortController();
async function stream() {
const prompt = processCommandPrompt(command.prompt, originalText);
let fullAIResponse = "";
const chatModel = await ProjectManager.instance
.getCurrentChainManager()
.chatModelManager.createModelInstance(selectedModel);
const chatStream = await chatModel.stream(prompt);
for await (const chunk of chatStream) {
if (abortController?.signal.aborted) break;
fullAIResponse += chunk.content;
setAiCurrentMessage(fullAIResponse);
abortControllerRef.current = abortController;
async function generateInitialResponse() {
if (!chatChain) {
// We'll wait for the chain to be ready
return;
}
if (!abortController?.signal.aborted) {
setProcessedMessage(fullAIResponse);
try {
const prompt = await processCommandPrompt(command.prompt, originalText);
await streamResponse(prompt, abortController);
} catch (error) {
logError("Error in initial response:", error);
setGenerating(false);
}
}
stream();
generateInitialResponse();
return () => {
abortController.abort();
};
}, [command.prompt, originalText, handleAddMessage, selectedModel]);
}, [command.prompt, originalText, chatChain, streamResponse]);
const abortControllerRef = useRef<AbortController | null>(null);
const handleFollowupSubmit = async () => {
if (!followupInstruction.trim() || !chatChain) {
if (!chatChain) {
new Notice("Chat engine not ready. Please try again.");
}
return;
}
const abortController = new AbortController();
abortControllerRef.current = abortController;
// Skip appending the selected text to the prompt because it's already
// included in the original prompt.
const prompt = await processCommandPrompt(followupInstruction, originalText, true);
try {
const result = await streamResponse(prompt, abortController);
if (result) {
// Reset follow-up instruction on success
setFollowupInstruction("");
}
} finally {
if (abortController.signal.aborted) {
setGenerating(false);
setProcessedMessage(aiCurrentMessage ?? "");
}
abortControllerRef.current = null;
}
};
// Handle stopping generation
const handleStopGeneration = () => {
if (abortControllerRef.current) {
abortControllerRef.current.abort();
abortControllerRef.current = null;
setGenerating(false);
}
};
// Handle keyboard shortcuts
const handleKeyDown = (e: React.KeyboardEvent) => {
if (e.nativeEvent.isComposing) return;
// For insert/replace buttons
if (!generating && processedMessage && !showFollowupSubmit) {
// Command+Enter (Mac) or Ctrl+Enter (Windows) for Replace
if (e.key === "Enter" && (Platform.isMacOS ? e.metaKey : e.ctrlKey) && !e.shiftKey) {
e.preventDefault();
onReplace(processedMessage);
}
// Command+Shift+Enter (Mac) or Ctrl+Shift+Enter (Windows) for Insert
if (e.key === "Enter" && (Platform.isMacOS ? e.metaKey : e.ctrlKey) && e.shiftKey) {
e.preventDefault();
onInsert(processedMessage);
}
}
// For submit button (follow-up instruction)
if (showFollowupSubmit && e.key === "Enter" && !e.shiftKey && !e.metaKey && !e.ctrlKey) {
e.preventDefault();
handleFollowupSubmit();
}
};
// Scroll textarea to bottom when generating content
useEffect(() => {
if (textareaRef.current && aiCurrentMessage && generating) {
const textarea = textareaRef.current;
textarea.scrollTop = textarea.scrollHeight;
}
}, [aiCurrentMessage, generating]);
// Determine if follow-up submit button should be shown
const showFollowupSubmit = !generating && followupInstruction.trim().length > 0;
return (
<div className="flex flex-col gap-4">
<div className="flex flex-col gap-4" onKeyDown={handleKeyDown}>
<div className="max-h-60 overflow-y-auto text-muted whitespace-pre-wrap">{originalText}</div>
<div className="flex flex-col gap-2">
{commandName && (
@ -73,13 +252,10 @@ function InlineEditModalContent({
{commandName}
</div>
)}
<div className="text-muted flex items-center gap-2 font-bold">
<Bot className="w-4 h-4" />
{getModelDisplayText(selectedModel)}
</div>
</div>
<div className="relative group">
<textarea
ref={textareaRef}
className="w-full h-60 text-text peer"
value={processedMessage ?? aiCurrentMessage ?? "loading..."}
disabled={processedMessage == null}
@ -97,22 +273,83 @@ function InlineEditModalContent({
</button>
)}
</div>
<div className="flex justify-end gap-2">
<button onClick={onClose}>Close</button>
<button
disabled={processedMessage == null}
className="!bg-interactive-accent !text-on-accent cursor-pointer"
onClick={() => onInsert(processedMessage ?? "")}
>
Insert
</button>
<button
disabled={processedMessage == null}
className="!bg-interactive-accent !text-on-accent cursor-pointer"
onClick={() => onReplace(processedMessage ?? "")}
>
Replace
</button>
{!generating && processedMessage && (
<div className="flex flex-col gap-2">
<textarea
autoFocus
ref={followupRef}
className="w-full h-20 text-text"
placeholder="Enter follow-up instructions..."
value={followupInstruction}
onChange={(e) => setFollowupInstruction(e.target.value)}
/>
</div>
)}
<div className="flex justify-between gap-2">
<div className="text-faint text-xs flex items-center gap-2 font-bold">
<Bot className="w-4 h-4" />
{getModelDisplayText(selectedModel)}
</div>
<div className="flex gap-2">
{generating ? (
// When generating, show Stop button
<Button variant="secondary" onClick={handleStopGeneration}>
Stop
</Button>
) : showFollowupSubmit ? (
// When follow-up instruction has content, show Submit button with Enter shortcut
<Button onClick={handleFollowupSubmit} className="flex items-center gap-1">
<span>Submit</span>
<CornerDownLeft className="size-3" />
</Button>
) : (
// Otherwise, show Insert and Replace buttons with shortcut indicators
<>
<Button
onClick={() => onInsert(processedMessage ?? "")}
className="flex items-center gap-1"
>
<span>Insert</span>
<div className="flex items-center text-xs text-muted">
{Platform.isMacOS ? (
<>
<Command className="size-3" />
<ArrowBigUp className="size-3" />
<CornerDownLeft className="size-3" />
</>
) : (
<>
<span className="text-xs">Ctrl</span>
<ArrowBigUp className="size-3" />
<CornerDownLeft className="size-3" />
</>
)}
</div>
</Button>
<Button
onClick={() => onReplace(processedMessage ?? "")}
className="flex items-center gap-1"
>
<span>Replace</span>
<div className="flex items-center text-xs text-muted">
{Platform.isMacOS ? (
<>
<Command className="size-3" />
<CornerDownLeft className="size-3" />
</>
) : (
<>
<span className="text-xs">Ctrl</span>
<CornerDownLeft className="size-3" />
</>
)}
</div>
</Button>
</>
)}
</div>
</div>
</div>
);
@ -146,17 +383,12 @@ export class InlineEditModal extends Modal {
this.close();
};
const handleClose = () => {
this.close();
};
this.root.render(
<InlineEditModalContent
originalText={selectedText}
command={command}
onInsert={handleInsert}
onReplace={handleReplace}
onClose={handleClose}
/>
);
}

View file

@ -10,7 +10,7 @@ const Input = React.forwardRef<HTMLInputElement, React.ComponentProps<"input">>(
className={cn(
"!h-9 !min-w-[50px] !border border-border border-solid !rounded-md !bg-transparent !px-3 !py-1 md:!text-base !text-sm !transition-colors",
"focus-visible:!shadow-sm focus-visible:!outline-none focus-visible:!ring-1 focus-visible:!ring-ring",
"placeholder:text-xs", // custom styles
"placeholder:text-sm",
"flex w-full shadow-sm placeholder:text-muted disabled:cursor-not-allowed disabled:opacity-50",
className
)}

View file

@ -1,6 +1,7 @@
import { CustomModel } from "@/aiParams";
import { DEFAULT_INLINE_EDIT_COMMANDS } from "@/commands/constants";
import { type CopilotSettings } from "@/settings/model";
import { v4 as uuidv4 } from "uuid";
import { ChainType } from "./chainFactory";
export const BREVILABS_API_BASE_URL = "https://api.brevilabs.com/v1";
@ -43,10 +44,10 @@ export type PlusUtmMedium = (typeof PLUS_UTM_MEDIUMS)[keyof typeof PLUS_UTM_MEDI
export enum ChatModels {
COPILOT_PLUS_FLASH = "copilot-plus-flash",
GPT_4o = "gpt-4o",
GPT_4o_mini = "gpt-4o-mini",
O1_mini = "o1-mini",
O3_mini = "o3-mini",
GPT_41 = "gpt-4.1",
GPT_41_mini = "gpt-4.1-mini",
GPT_41_nano = "gpt-4.1-nano",
O4_mini = "o4-mini",
AZURE_OPENAI = "azure-openai",
GEMINI_PRO = "gemini-2.0-pro-exp",
GEMINI_FLASH = "gemini-2.0-flash",
@ -54,6 +55,8 @@ export enum ChatModels {
GEMINI_15_FLASH = "gemini-1.5-flash-001", // TODO(logan): Project should use 2.0 flash once it supports context caching
CLAUDE_3_5_SONNET = "claude-3-5-sonnet-latest",
CLAUDE_3_5_HAIKU = "claude-3-5-haiku-latest",
GROK3 = "grok-3-beta",
GROK3_MINI = "grok-3-mini-beta",
COMMAND_R = "command-r",
COMMAND_R_PLUS = "command-r-plus",
OPENROUTER_GPT_4o = "openai/chatgpt-4o-latest",
@ -71,6 +74,7 @@ export enum ChatModelProviders {
ANTHROPIC = "anthropic",
COHEREAI = "cohereai",
GOOGLE = "google",
XAI = "xai",
OPENROUTERAI = "openrouterai",
GROQ = "groq",
OLLAMA = "ollama",
@ -103,7 +107,7 @@ export const BUILTIN_CHAT_MODELS: CustomModel[] = [
capabilities: [ModelCapability.VISION],
},
{
name: ChatModels.GPT_4o,
name: ChatModels.GPT_41,
provider: ChatModelProviders.OPENAI,
enabled: true,
isBuiltIn: true,
@ -111,7 +115,7 @@ export const BUILTIN_CHAT_MODELS: CustomModel[] = [
capabilities: [ModelCapability.VISION],
},
{
name: ChatModels.GPT_4o_mini,
name: ChatModels.GPT_41_mini,
provider: ChatModelProviders.OPENAI,
enabled: true,
isBuiltIn: true,
@ -120,17 +124,19 @@ export const BUILTIN_CHAT_MODELS: CustomModel[] = [
capabilities: [ModelCapability.VISION],
},
{
name: ChatModels.O1_mini,
name: ChatModels.GPT_41_nano,
provider: ChatModelProviders.OPENAI,
enabled: true,
isBuiltIn: true,
capabilities: [ModelCapability.REASONING],
core: true,
capabilities: [ModelCapability.VISION],
},
{
name: ChatModels.O3_mini,
name: ChatModels.O4_mini,
provider: ChatModelProviders.OPENAI,
enabled: true,
isBuiltIn: true,
core: true,
capabilities: [ModelCapability.REASONING],
},
{
@ -147,6 +153,18 @@ export const BUILTIN_CHAT_MODELS: CustomModel[] = [
enabled: true,
isBuiltIn: true,
},
{
name: ChatModels.GROK3,
provider: ChatModelProviders.XAI,
enabled: true,
isBuiltIn: true,
},
{
name: ChatModels.GROK3_MINI,
provider: ChatModelProviders.XAI,
enabled: true,
isBuiltIn: true,
},
{
name: ChatModels.COMMAND_R,
provider: ChatModelProviders.COHEREAI,
@ -332,7 +350,7 @@ export const ProviderInfo: Record<Provider, ProviderMetadata> = {
label: "OpenAI",
host: "https://api.openai.com",
keyManagementURL: "https://platform.openai.com/api-keys",
testModel: ChatModels.GPT_4o,
testModel: ChatModels.GPT_41,
},
[ChatModelProviders.AZURE_OPENAI]: {
label: "Azure OpenAI",
@ -358,6 +376,12 @@ export const ProviderInfo: Record<Provider, ProviderMetadata> = {
keyManagementURL: "https://makersuite.google.com/app/apikey",
testModel: ChatModels.GEMINI_FLASH,
},
[ChatModelProviders.XAI]: {
label: "XAI",
host: "https://api.x.ai/v1",
keyManagementURL: "https://console.x.ai",
testModel: ChatModels.GROK3,
},
[ChatModelProviders.OPENROUTERAI]: {
label: "OpenRouter",
host: "https://openrouter.ai/api/v1/",
@ -418,6 +442,7 @@ export const ProviderSettingsKeyMap: Record<SettingKeyProviders, keyof CopilotSe
groq: "groqApiKey",
openrouterai: "openRouterAiApiKey",
cohereai: "cohereApiKey",
xai: "xaiApiKey",
"copilot-plus": "plusLicenseKey",
mistralai: "mistralApiKey",
deepseek: "deepseekApiKey",
@ -502,6 +527,7 @@ export const AUTOCOMPLETE_CONFIG = {
} as const;
export const DEFAULT_SETTINGS: CopilotSettings = {
userId: uuidv4(),
isPlusUser: false,
plusLicenseKey: "",
openAIApiKey: "",
@ -516,10 +542,11 @@ export const DEFAULT_SETTINGS: CopilotSettings = {
azureOpenAIApiEmbeddingDeploymentName: "",
googleApiKey: "",
openRouterAiApiKey: "",
xaiApiKey: "",
mistralApiKey: "",
deepseekApiKey: "",
defaultChainType: ChainType.LLM_CHAIN,
defaultModelKey: ChatModels.GPT_4o + "|" + ChatModelProviders.OPENAI,
defaultModelKey: ChatModels.GPT_41 + "|" + ChatModelProviders.OPENAI,
embeddingModelKey: EmbeddingModels.OPENAI_EMBEDDING_SMALL + "|" + EmbeddingModelProviders.OPENAI,
temperature: 0.1,
maxTokens: 1000,
@ -556,6 +583,9 @@ export const DEFAULT_SETTINGS: CopilotSettings = {
inlineEditCommands: DEFAULT_INLINE_EDIT_COMMANDS,
projectList: [],
enableAutocomplete: true,
lastDismissedVersion: null,
passMarkdownImages: true,
enableCustomPromptTemplating: true,
};
export enum ABORT_REASON {

View file

@ -1,5 +1,4 @@
import { ChainType } from "@/chainFactory";
import { CustomPromptProcessor } from "@/customPromptProcessor";
import { FileParserManager } from "@/tools/FileParserManager";
import { TFile, Vault } from "obsidian";
@ -43,8 +42,20 @@ export class ContextProcessor {
return content;
}
/**
* Processes context notes, excluding any already handled by custom prompts.
*
* @param excludedNotePaths A set of file paths that should be skipped.
* @param fileParserManager
* @param vault
* @param contextNotes
* @param includeActiveNote
* @param activeNote
* @param currentChain
* @returns The combined content string of the processed context notes.
*/
async processContextNotes(
customPromptProcessor: CustomPromptProcessor,
excludedNotePaths: Set<string>,
fileParserManager: FileParserManager,
vault: Vault,
contextNotes: TFile[],
@ -52,33 +63,43 @@ export class ContextProcessor {
activeNote: TFile | null,
currentChain: ChainType
): Promise<string> {
const processedVars = await customPromptProcessor.getProcessedVariables();
let additionalContext = "";
const processNote = async (note: TFile) => {
try {
// Skip if this note was already processed by processCustomPrompt
const noteRef = `[[${note.basename}]]`;
if (processedVars.has(noteRef)) {
// Check if this note was already processed (via custom prompt)
if (excludedNotePaths.has(note.path)) {
console.log(`Skipping note ${note.path} as it was included via custom prompt.`);
return;
}
if (currentChain !== ChainType.COPILOT_PLUS_CHAIN && note.extension !== "md") {
if (!fileParserManager.supportsExtension(note.extension)) {
console.warn(`Unsupported file type: ${note.extension}`);
} else {
console.warn(`File type ${note.extension} only supported in Copilot Plus mode`);
}
return;
}
console.log(
`Processing note: ${note.path}, extension: ${note.extension}, chain: ${currentChain}`
);
// 1. Check if the file extension is supported by any parser
if (!fileParserManager.supportsExtension(note.extension)) {
console.warn(`Unsupported file type: ${note.extension}`);
return;
}
// 2. Apply chain restrictions only to supported files that are NOT md or canvas
if (
currentChain !== ChainType.COPILOT_PLUS_CHAIN &&
note.extension !== "md" &&
note.extension !== "canvas"
) {
// This file type is supported, but requires Plus mode (e.g., PDF)
console.warn(
`File type ${note.extension} requires Copilot Plus mode for context processing.`
);
return;
}
// 3. If we reach here, parse the file (md, canvas, or other supported type in Plus mode)
let content = await fileParserManager.parseFile(note, vault);
// Special handling for embedded PDFs within markdown (only in Plus mode)
if (note.extension === "md" && currentChain === ChainType.COPILOT_PLUS_CHAIN) {
content = await this.processEmbeddedPDFs(content, vault, fileParserManager);
}
@ -92,11 +113,7 @@ export class ContextProcessor {
// Process active note if included
if (includeActiveNote && activeNote) {
const activeNoteVar = `activeNote`;
const activeNotePath = `[[${activeNote.basename}]]`;
if (!processedVars.has(activeNoteVar) && !processedVars.has(activeNotePath)) {
await processNote(activeNote);
}
await processNote(activeNote);
}
// Process context notes

View file

@ -1,5 +1,11 @@
import { CustomPrompt, CustomPromptProcessor } from "@/customPromptProcessor";
import { extractNoteFiles, getFileContent, getNotesFromPath } from "@/utils";
import {
extractNoteFiles,
getFileContent,
getFileName,
getNotesFromPath,
getNotesFromTags,
} from "@/utils";
import { Notice, TFile, Vault } from "obsidian";
// Mock Obsidian
@ -19,14 +25,34 @@ jest.mock("@/utils", () => ({
processVariableNameForNotePath: jest.fn(),
}));
// Mock the specific function from the same module
jest.mock("@/customPromptProcessor", () => {
// Keep original implementations for things we don't want to mock
const originalModule = jest.requireActual("@/customPromptProcessor");
return {
...originalModule,
extractVariablesFromPrompt: jest.fn(), // Mock the specific function
};
});
describe("CustomPromptProcessor", () => {
let processor: CustomPromptProcessor;
let mockVault: Vault;
let mockActiveNote: TFile;
let originalConsoleWarn: typeof console.warn;
beforeEach(() => {
// Save original console.warn
originalConsoleWarn = console.warn;
// Mock console.warn
console.warn = jest.fn();
// Reset mocks before each test
jest.clearAllMocks();
jest.resetAllMocks();
// Set default implementations for critical mocks
(extractNoteFiles as jest.Mock).mockReturnValue([]);
// Create mock objects
mockVault = {} as Vault;
@ -39,6 +65,11 @@ describe("CustomPromptProcessor", () => {
processor = CustomPromptProcessor.getInstance(mockVault);
});
afterEach(() => {
// Restore original console.warn
console.warn = originalConsoleWarn;
});
it("should add 1 context and selectedText", async () => {
const doc: CustomPrompt = {
title: "test-prompt",
@ -48,13 +79,25 @@ describe("CustomPromptProcessor", () => {
// Mock getFileContent to return content for {variable}
(getFileContent as jest.Mock).mockResolvedValueOnce("here is the note content for note0");
(getFileName as jest.Mock).mockReturnValueOnce("Variable Note");
(getNotesFromPath as jest.Mock).mockResolvedValueOnce([mockActiveNote]);
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
mockExtractVariables.mockResolvedValue({
variablesMap: new Map([
["variable", "## Variable Note\n\nhere is the note content for note0"],
]),
includedFiles: new Set([mockActiveNote]),
});
const result = await processor.processCustomPrompt(doc.content, selectedText, mockActiveNote);
expect(result).toContain("This is a {variable} and {selectedText}.");
expect(result).toContain("here is some selected text 12345");
expect(result).toContain("here is the note content for note0");
expect(result.processedPrompt).toBe(
"This is a {variable} and {selectedText}.\n\nselectedText:\n\nhere is some selected text 12345\n\nvariable:\n\n## Variable Note\n\nhere is the note content for note0"
);
expect(result.includedFiles).toContain(mockActiveNote);
});
it("should add 2 context and no selectedText", async () => {
@ -69,14 +112,35 @@ describe("CustomPromptProcessor", () => {
.mockResolvedValueOnce("here is the note content for note0")
.mockResolvedValueOnce("note content for note1");
// Mock getNotesFromPath to return an array with a single mock file
(getNotesFromPath as jest.Mock).mockResolvedValue([mockActiveNote]);
// Mock getFileName to return appropriate names
const { getFileName } = jest.requireMock("@/utils") as any;
getFileName.mockReturnValueOnce("Variable1 Note").mockReturnValueOnce("Variable2 Note");
// Mock getNotesFromPath to return two mock files, one for each variable
const mockNote1 = { path: "path/to/note1.md", basename: "Variable1 Note" } as TFile;
const mockNote2 = { path: "path/to/note2.md", basename: "Variable2 Note" } as TFile;
(getNotesFromPath as jest.Mock)
.mockResolvedValueOnce([mockNote1])
.mockResolvedValueOnce([mockNote2]);
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
mockExtractVariables.mockResolvedValue({
variablesMap: new Map([
["variable1", "## Variable1 Note\n\nhere is the note content for note0"],
["variable2", "## Variable2 Note\n\nnote content for note1"],
]),
includedFiles: new Set([mockNote1, mockNote2]),
});
const result = await processor.processCustomPrompt(doc.content, selectedText, mockActiveNote);
expect(result).toContain("This is a {variable1} and {variable2}.");
expect(result).toContain("here is the note content for note0");
expect(result).toContain("note content for note1");
expect(result.processedPrompt).toBe(
"This is a {variable1} and {variable2}.\n\nvariable1:\n\n## Variable1 Note\n\nhere is the note content for note0\n\nvariable2:\n\n## Variable2 Note\n\nnote content for note1"
);
expect(result.includedFiles).toContain(mockNote1);
expect(result.includedFiles).toContain(mockNote2);
});
it("should add 1 selectedText and no context", async () => {
@ -86,10 +150,20 @@ describe("CustomPromptProcessor", () => {
};
const selectedText = "here is some selected text 12345";
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
mockExtractVariables.mockResolvedValue({
variablesMap: new Map(),
includedFiles: new Set(),
});
const result = await processor.processCustomPrompt(doc.content, selectedText, mockActiveNote);
expect(result).toContain("Rewrite the following text {selectedText}");
expect(result).toContain("here is some selected text 12345");
expect(result.processedPrompt).toBe(
"Rewrite the following text {selectedText}\n\nselectedText:\n\nhere is some selected text 12345"
);
expect(result.includedFiles).toEqual([]);
});
it("should process {activeNote} correctly", async () => {
@ -101,11 +175,23 @@ describe("CustomPromptProcessor", () => {
// Mock the getFileContent function to return a predefined content for the active note
(getFileContent as jest.Mock).mockResolvedValue("Content of the active note");
const { getFileName } = jest.requireMock("@/utils") as any;
getFileName.mockReturnValue("Active Note");
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
mockExtractVariables.mockResolvedValue({
variablesMap: new Map([["activenote", "## Active Note\n\nContent of the active note"]]),
includedFiles: new Set([mockActiveNote]),
});
const result = await processor.processCustomPrompt(doc.content, selectedText, mockActiveNote);
expect(result).toContain("This is the active note: {activenote}");
expect(result).toContain("Content of the active note");
expect(result.processedPrompt).toBe(
"This is the active note: {activenote}\n\nactivenote:\n\n## Active Note\n\nContent of the active note"
);
expect(result.includedFiles).toContain(mockActiveNote);
expect(getFileContent).toHaveBeenCalledWith(mockActiveNote, mockVault);
});
@ -116,11 +202,18 @@ describe("CustomPromptProcessor", () => {
};
const selectedText = "";
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
mockExtractVariables.mockResolvedValue({
variablesMap: new Map(), // ActiveNote variable won't be found
includedFiles: new Set(),
});
const result = await processor.processCustomPrompt(doc.content, selectedText, undefined);
expect(result).toContain("This is the active note: {activeNote}");
expect(result).not.toContain("Content of the active note");
expect(getFileContent).not.toHaveBeenCalled();
expect(result.processedPrompt).toBe("This is the active note: {activeNote}\n\n");
expect(result.includedFiles).toEqual([]);
expect(Notice).toHaveBeenCalledWith("No active note found.");
});
@ -131,80 +224,170 @@ describe("CustomPromptProcessor", () => {
};
const selectedText = "selected text";
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
mockExtractVariables.mockResolvedValue({
variablesMap: new Map(),
includedFiles: new Set(),
});
const result = await processor.processCustomPrompt(doc.content, selectedText, mockActiveNote);
expect(result).toBe("This is a test prompt with no variables.\n\n");
expect(result.processedPrompt).toBe("This is a test prompt with no variables.\n\n");
expect(result.includedFiles).toEqual([]);
});
it("should process a single tag variable correctly", async () => {
const customPrompt = "Notes related to {#tag} are:";
const selectedText = "";
// Mock the extractVariablesFromPrompt method to simulate tag processing
jest
.spyOn(processor, "extractVariablesFromPrompt")
.mockResolvedValue(['[{"name":"note","content":"Note content for #tag"}]']);
// Mock note file for the tag
const mockNoteForTag = {
path: "path/to/tagged/note.md",
basename: "Tagged Note",
} as TFile;
// Mock getNotesFromTags to return our mock note
(getNotesFromTags as jest.Mock).mockResolvedValue([mockNoteForTag]);
// Mock getFileName to return the basename
const { getFileName } = jest.requireMock("@/utils") as any;
getFileName.mockReturnValue("Tagged Note");
// Mock getFileContent to return content for the note
(getFileContent as jest.Mock).mockResolvedValue("Note content for #tag");
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
mockExtractVariables.mockResolvedValue({
variablesMap: new Map([["#tag", "## Tagged Note\n\nNote content for #tag"]]),
includedFiles: new Set([mockNoteForTag]),
});
const result = await processor.processCustomPrompt(customPrompt, selectedText, mockActiveNote);
expect(result).toContain("Notes related to {#tag} are:");
expect(result).toContain('[{"name":"note","content":"Note content for #tag"}]');
expect(result.processedPrompt).toBe(
"Notes related to {#tag} are:\n\n#tag:\n\n## Tagged Note\n\nNote content for #tag"
);
expect(result.includedFiles).toContain(mockNoteForTag);
});
it("should process multiple tag variables correctly", async () => {
const customPrompt = "Notes related to {#tag1,#tag2,#tag3} are:";
const selectedText = "";
// Mock the extractVariablesFromPrompt method to simulate processing of multiple tags
jest
.spyOn(processor, "extractVariablesFromPrompt")
.mockResolvedValue([
'[{"name":"note1","content":"Note content for #tag1"},{"name":"note2","content":"Note content for #tag2"}]',
]);
// Mock note files for the tags
const mockNoteForTag1 = {
basename: "Tagged Note 1",
path: "path/to/tagged/note1.md",
} as TFile;
const mockNoteForTag2 = {
basename: "Tagged Note 2",
path: "path/to/tagged/note2.md",
} as TFile;
// Mock getNotesFromTags to return our mock notes
(getNotesFromTags as jest.Mock).mockResolvedValue([mockNoteForTag1, mockNoteForTag2]);
// Mock getFileName to return the basename
const { getFileName } = jest.requireMock("@/utils") as any;
getFileName.mockImplementation((file: TFile) => file.basename);
// Mock getFileContent to return content for each note
(getFileContent as jest.Mock).mockImplementation((file: TFile) => {
if (file.basename === "Tagged Note 1") {
return "Note content for #tag1";
} else if (file.basename === "Tagged Note 2") {
return "Note content for #tag2";
}
return "";
});
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
mockExtractVariables.mockResolvedValue({
variablesMap: new Map([
[
"#tag1,#tag2,#tag3",
"## Tagged Note 1\n\nNote content for #tag1\n\n## Tagged Note 2\n\nNote content for #tag2",
],
]),
includedFiles: new Set([mockNoteForTag1, mockNoteForTag2]),
});
const result = await processor.processCustomPrompt(customPrompt, selectedText, mockActiveNote);
expect(result).toContain("Notes related to {#tag1,#tag2,#tag3} are:");
expect(result).toContain(
'[{"name":"note1","content":"Note content for #tag1"},{"name":"note2","content":"Note content for #tag2"}]'
expect(result.processedPrompt).toBe(
"Notes related to {#tag1,#tag2,#tag3} are:\n\n#tag1,#tag2,#tag3:\n\n## Tagged Note 1\n\nNote content for #tag1\n\n## Tagged Note 2\n\nNote content for #tag2"
);
expect(result.includedFiles).toContain(mockNoteForTag1);
expect(result.includedFiles).toContain(mockNoteForTag2);
});
it("should process [[note title]] syntax correctly", async () => {
const customPrompt = "Content of [[Test Note]] is important.";
const selectedText = "";
const mockTestNote = { basename: "Test Note", path: "Test Note.md" } as TFile;
// Mock the necessary functions
(extractNoteFiles as jest.Mock).mockReturnValue([
{ basename: "Test Note", path: "Test Note.md" },
]);
(extractNoteFiles as jest.Mock).mockReturnValue([mockTestNote]);
(getFileContent as jest.Mock).mockResolvedValue("Test note content");
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
mockExtractVariables.mockResolvedValue({
variablesMap: new Map(),
includedFiles: new Set(),
});
const result = await processor.processCustomPrompt(customPrompt, selectedText, mockActiveNote);
expect(result).toContain("Content of [[Test Note]] is important.");
expect(result).toContain("Title: [[Test Note]]\nPath: Test Note.md\n\nTest note content");
expect(result.processedPrompt).toBe(
"Content of [[Test Note]] is important.\n\nTitle: [[Test Note]]\nPath: Test Note.md\n\nTest note content"
);
expect(result.includedFiles).toContain(mockTestNote);
});
it("should process {[[note title]]} syntax correctly without duplication", async () => {
const customPrompt = "Content of {[[Test Note]]} is important.";
const customPrompt = "Content of {[[Test Note]]} is important. Look at [[Test Note]].";
const selectedText = "";
// Mock the necessary functions
jest
.spyOn(processor, "extractVariablesFromPrompt")
.mockResolvedValue([JSON.stringify([{ name: "Test Note", content: "Test note content" }])]);
(extractNoteFiles as jest.Mock).mockReturnValue([
{ basename: "Test Note", path: "Test Note.md" },
]);
const mockNoteFile = {
basename: "Test Note",
path: "Test Note.md",
} as TFile;
(extractNoteFiles as jest.Mock).mockReturnValue([mockNoteFile]);
const { getFileName } = jest.requireMock("@/utils") as any;
getFileName.mockReturnValue("Test Note");
(getFileContent as jest.Mock).mockResolvedValue("Test note content");
// Mock getNotesFromPath to return our mock note
(getNotesFromPath as jest.Mock).mockResolvedValue([mockNoteFile]);
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
mockExtractVariables.mockResolvedValue({
variablesMap: new Map([["[[Test Note]]", "## Test Note\n\nTest note content"]]),
includedFiles: new Set([mockNoteFile]),
});
const result = await processor.processCustomPrompt(customPrompt, selectedText, mockActiveNote);
expect(result).toContain("Content of {[[Test Note]]} is important.");
expect(result).toContain(
'[[Test Note]]:\n\n[{"name":"Test Note","content":"Test note content"}]'
expect(result.processedPrompt).toBe(
"Content of {[[Test Note]]} is important. Look at [[Test Note]].\n\n[[Test Note]]:\n\n## Test Note\n\nTest note content"
);
expect((result.match(/Test note content/g) || []).length).toBe(1);
// Note: extractNoteFiles will still find [[Test Note]], but processPrompt should skip adding it again because it's already in includedFiles from the variable processing
expect(result.includedFiles).toEqual([mockNoteFile]);
expect(extractNoteFiles).toHaveBeenCalledWith(customPrompt, mockVault);
});
it("should process both {[[note title]]} and [[note title]] syntax correctly", async () => {
@ -212,43 +395,86 @@ describe("CustomPromptProcessor", () => {
const selectedText = "";
// Mock the necessary functions
jest
.spyOn(processor, "extractVariablesFromPrompt")
.mockResolvedValue([JSON.stringify([{ name: "Note1", content: "Note1 content" }])]);
(extractNoteFiles as jest.Mock).mockReturnValue([
{ basename: "Note1", path: "Note1.md" },
{ basename: "Note2", path: "Note2.md" },
]);
(getFileContent as jest.Mock).mockResolvedValue("Note2 content");
const mockNote1 = {
basename: "Note1",
path: "Note1.md",
} as TFile;
const mockNote2 = {
basename: "Note2",
path: "Note2.md",
} as TFile;
(extractNoteFiles as jest.Mock).mockReturnValue([mockNote1, mockNote2]);
const { getFileName } = jest.requireMock("@/utils") as any;
getFileName.mockImplementation((file: TFile) => file.basename);
(getFileContent as jest.Mock).mockImplementation((file: TFile) => {
if (file.basename === "Note1") {
return "Note1 content";
} else if (file.basename === "Note2") {
return "Note2 content";
}
return "";
});
// Mock getNotesFromPath to return our mock note
(getNotesFromPath as jest.Mock).mockResolvedValue([mockNote1]);
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
mockExtractVariables.mockResolvedValue({
variablesMap: new Map([["[[Note1]]", "## Note1\n\nNote1 content"]]),
includedFiles: new Set([mockNote1]), // Only Note1 comes from variables
});
const result = await processor.processCustomPrompt(customPrompt, selectedText, mockActiveNote);
expect(result).toContain("{[[Note1]]} content and [[Note2]] are both important.");
expect(result).toContain('[[Note1]]:\n\n[{"name":"Note1","content":"Note1 content"}]');
expect(result).toContain("Title: [[Note2]]\nPath: Note2.md\n\nNote2 content");
expect(result.processedPrompt).toBe(
"{[[Note1]]} content and [[Note2]] are both important.\n\n[[Note1]]:\n\n## Note1\n\nNote1 content\n\nTitle: [[Note2]]\nPath: Note2.md\n\nNote2 content"
);
// Note2 is added via [[Note2]] processing
expect(result.includedFiles).toEqual(expect.arrayContaining([mockNote1, mockNote2]));
expect(result.includedFiles.length).toBe(2);
});
it("should handle multiple occurrences of [[note title]] syntax", async () => {
const customPrompt = "[[Note1]] is related to [[Note2]] and [[Note3]].";
const selectedText = "";
const mockNote1 = { basename: "Note1", path: "Note1.md" } as TFile;
const mockNote2 = { basename: "Note2", path: "Note2.md" } as TFile;
const mockNote3 = { basename: "Note3", path: "Note3.md" } as TFile;
// Mock the necessary functions
(extractNoteFiles as jest.Mock).mockReturnValue([
{ basename: "Note1", path: "Note1.md" },
{ basename: "Note2", path: "Note2.md" },
{ basename: "Note3", path: "Note3.md" },
]);
(getFileContent as jest.Mock)
.mockResolvedValueOnce("Note1 content")
.mockResolvedValueOnce("Note2 content")
.mockResolvedValueOnce("Note3 content");
(extractNoteFiles as jest.Mock).mockReturnValue([mockNote1, mockNote2, mockNote3]);
(getFileContent as jest.Mock).mockImplementation((file: TFile) => {
if (file.basename === "Note1") {
return "Note1 content";
} else if (file.basename === "Note2") {
return "Note2 content";
} else if (file.basename === "Note3") {
return "Note3 content";
}
return "";
});
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
mockExtractVariables.mockResolvedValue({
variablesMap: new Map(),
includedFiles: new Set(),
});
const result = await processor.processCustomPrompt(customPrompt, selectedText, mockActiveNote);
expect(result).toContain("[[Note1]] is related to [[Note2]] and [[Note3]].");
expect(result).toContain("Title: [[Note1]]\nPath: Note1.md\n\nNote1 content");
expect(result).toContain("Title: [[Note2]]\nPath: Note2.md\n\nNote2 content");
expect(result).toContain("Title: [[Note3]]\nPath: Note3.md\n\nNote3 content");
expect(result.processedPrompt).toBe(
"[[Note1]] is related to [[Note2]] and [[Note3]].\n\nTitle: [[Note1]]\nPath: Note1.md\n\nNote1 content\n\nTitle: [[Note2]]\nPath: Note2.md\n\nNote2 content\n\nTitle: [[Note3]]\nPath: Note3.md\n\nNote3 content"
);
expect(result.includedFiles).toEqual(expect.arrayContaining([mockNote1, mockNote2, mockNote3]));
expect(result.includedFiles.length).toBe(3);
});
it("should handle non-existent note titles gracefully", async () => {
@ -256,12 +482,20 @@ describe("CustomPromptProcessor", () => {
const selectedText = "";
// Mock the necessary functions
(extractNoteFiles as jest.Mock).mockReturnValue(["Non-existent Note"]);
(extractNoteFiles as jest.Mock).mockReturnValue([]); // Assume it returns empty if note doesn't exist
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
mockExtractVariables.mockResolvedValue({
variablesMap: new Map(),
includedFiles: new Set(),
});
const result = await processor.processCustomPrompt(customPrompt, selectedText, mockActiveNote);
expect(result).toContain("[[Non-existent Note]] should not cause errors.");
expect(result).not.toContain("[[Non-existent Note]]:");
expect(result.processedPrompt).toBe("[[Non-existent Note]] should not cause errors.\n\n");
expect(result.includedFiles).toEqual([]);
});
it("should process {activenote} only once when it appears multiple times", async () => {
@ -271,19 +505,29 @@ describe("CustomPromptProcessor", () => {
};
const selectedText = "";
// Mock the extractVariablesFromPrompt method to simulate processing of {activeNote}
jest
.spyOn(processor, "extractVariablesFromPrompt")
.mockResolvedValue([
JSON.stringify([{ name: "Active Note", content: "Content of the active note" }]),
]);
// Mock getFileName and getFileContent
const { getFileName } = jest.requireMock("@/utils") as any;
getFileName.mockReturnValue("Active Note");
(getFileContent as jest.Mock).mockResolvedValue("Content of the active note");
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
// Even if the regex matches twice, extractVariablesFromPrompt should handle it
mockExtractVariables.mockResolvedValue({
variablesMap: new Map([["activeNote", "## Active Note\n\nContent of the active note"]]),
includedFiles: new Set([mockActiveNote]),
});
const result = await processor.processCustomPrompt(doc.content, selectedText, mockActiveNote);
expect(result).toContain("This is the active note: {activeNote}. And again: {activeNote}");
expect(result).toContain("Content of the active note");
expect((result.match(/activeNote:/g) || []).length).toBe(1);
expect(processor.extractVariablesFromPrompt).toHaveBeenCalledTimes(1);
// Check that getFileContent was called with the active note at least once
expect(getFileContent).toHaveBeenCalledWith(mockActiveNote, mockVault);
expect(result.processedPrompt).toBe(
"This is the active note: {activeNote}. And again: {activeNote}\n\nactiveNote:\n\n## Active Note\n\nContent of the active note"
);
expect(result.includedFiles).toContain(mockActiveNote);
});
it("should use active note content when {} is present and no selected text", async () => {
@ -295,11 +539,21 @@ describe("CustomPromptProcessor", () => {
(getFileContent as jest.Mock).mockResolvedValue("Content of the active note");
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
mockExtractVariables.mockResolvedValue({
variablesMap: new Map(),
includedFiles: new Set(),
});
const result = await processor.processCustomPrompt(doc.content, selectedText, mockActiveNote);
expect(result).toContain("Summarize this: {selectedText}");
expect(result).toContain("selectedText (entire active note):\n\n Content of the active note");
expect(getFileContent).toHaveBeenCalledWith(mockActiveNote, mockVault);
expect(result.processedPrompt).toBe(
"Summarize this: {selectedText}\n\nselectedText (entire active note):\n\nContent of the active note"
);
// Active note should be included because of {}
expect(result.includedFiles).toContain(mockActiveNote);
});
it("should not duplicate active note content when both {} and {activeNote} are present", async () => {
@ -309,18 +563,30 @@ describe("CustomPromptProcessor", () => {
};
const selectedText = "";
// Mock getFileContent for the active note when processed via {}
(getFileContent as jest.Mock).mockResolvedValue("Content of the active note");
jest
.spyOn(processor, "extractVariablesFromPrompt")
.mockResolvedValue([
JSON.stringify([{ name: "Active Note", content: "Content of the active note" }]),
]);
const { getFileName } = jest.requireMock("@/utils") as any;
getFileName.mockReturnValue("Active Note");
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
mockExtractVariables.mockResolvedValue({
variablesMap: new Map([["activeNote", "## Active Note\n\nContent of the active note"]]),
includedFiles: new Set([mockActiveNote]),
});
const result = await processor.processCustomPrompt(doc.content, selectedText, mockActiveNote);
expect(result).toContain("Summarize this: {selectedText}. Additional info: {activeNote}");
expect(result).toContain("selectedText (entire active note):\n\n Content of the active note");
expect(result).not.toContain("activeNote:");
expect(result.processedPrompt).toBe(
"Summarize this: {selectedText}. Additional info: {activeNote}\n\nselectedText (entire active note):\n\nContent of the active note"
);
// Ensure extractVariablesFromPrompt was NOT necessarily called if active note was handled by {}
// expect(mockExtractVariables).toHaveBeenCalled(); // Removed this check
// Ensure getFileContent was called for the {} replacement
expect(getFileContent).toHaveBeenCalledWith(mockActiveNote, mockVault);
// Active note should be included only once
expect(result.includedFiles).toEqual([mockActiveNote]);
});
it("should prioritize selected text over active note when both are available", async () => {
@ -332,10 +598,66 @@ describe("CustomPromptProcessor", () => {
(getFileContent as jest.Mock).mockResolvedValue("Content of the active note");
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
mockExtractVariables.mockResolvedValue({
variablesMap: new Map(),
includedFiles: new Set(),
});
const result = await processor.processCustomPrompt(doc.content, selectedText, mockActiveNote);
expect(result).toContain("Analyze this: {selectedText}");
expect(result).toContain("selectedText:\n\n This is the selected text");
expect(result).not.toContain("selectedText (entire active note):");
expect(result.processedPrompt).toBe(
"Analyze this: {selectedText}\n\nselectedText:\n\nThis is the selected text"
);
// Active note should not be included when selected text is present for {}
expect(result.includedFiles).toEqual([]);
});
it("should handle invalid variable names correctly", async () => {
const doc: CustomPrompt = {
title: "test-prompt",
content: "This is a test prompt with {invalidVariable} name and {activeNote}",
};
const selectedText = "";
(getNotesFromPath as jest.Mock).mockImplementation((_vault: Vault, variableName: string) => {
if (variableName === "Active Note") {
// Assuming processVariableNameForNotePath is mocked to return this
return [mockActiveNote];
}
return [];
});
(getFileName as jest.Mock).mockReturnValue(mockActiveNote.basename);
(getFileContent as jest.Mock).mockImplementation((file: TFile) => {
if (file.basename === "Active Note") {
return "Active Note Content";
}
return "";
});
// Mock the module-level extractVariablesFromPrompt function
const mockExtractVariables = jest.requireMock("@/customPromptProcessor")
.extractVariablesFromPrompt as jest.Mock;
// Simulate that only activeNote is found
mockExtractVariables.mockResolvedValue({
variablesMap: new Map([["activeNote", "## Active Note\n\nActive Note Content"]]),
includedFiles: new Set([mockActiveNote]),
});
// We also need to mock processVariableNameForNotePath for the "activeNote" variable
const { processVariableNameForNotePath } = jest.requireMock("@/utils") as any;
processVariableNameForNotePath.mockImplementation((name: string) => name); // Simple passthrough for test
const result = await processor.processCustomPrompt(doc.content, selectedText, mockActiveNote);
expect(result.processedPrompt).toBe(
"This is a test prompt with {invalidVariable} name and {activeNote}\n\nactiveNote:\n\n## Active Note\n\nActive Note Content"
);
expect(result.includedFiles).toContain(mockActiveNote);
// Expect the warning for the invalid variable
expect(console.warn).toHaveBeenCalledWith("No notes found for variable: invalidVariable");
});
});

View file

@ -16,6 +16,186 @@ export interface CustomPrompt {
content: string;
}
/**
* {copilot-selection} is the legacy custom command special placeholder. It must
* be skipped when processing custom prompts because it's handled differently
* by the custom command prompt processor.
*/
const VARIABLE_REGEX = /\{(?!copilot-selection\})([^}]+)\}/g;
/**
* Represents the result of processing a custom prompt variable.
*/
interface VariableProcessingResult {
content: string;
files: TFile[];
}
/**
* Extract variables from a custom prompt and get their content and associated files.
*/
async function extractVariablesFromPrompt(
customPrompt: string,
vault: Vault,
activeNote?: TFile | null
): Promise<{ variablesMap: Map<string, string>; includedFiles: Set<TFile> }> {
const variablesMap = new Map<string, string>();
const includedFiles = new Set<TFile>();
let match;
while ((match = VARIABLE_REGEX.exec(customPrompt)) !== null) {
const variableName = match[1].trim();
const variableResult: VariableProcessingResult = { content: "", files: [] };
if (variableName.toLowerCase() === "activenote") {
if (activeNote) {
const content = await getFileContent(activeNote, vault);
if (content) {
variableResult.content = `## ${getFileName(activeNote)}\n\n${content}`;
variableResult.files.push(activeNote);
}
} else {
new Notice("No active note found.");
}
} else if (variableName.startsWith("#")) {
// Handle tag-based variable for multiple tags
const tagNames = variableName
.slice(1)
.split(",")
.map((tag) => tag.trim());
const noteFiles = await getNotesFromTags(vault, tagNames);
const notesContent: string[] = [];
for (const file of noteFiles) {
const content = await getFileContent(file, vault);
if (content) {
notesContent.push(`## ${getFileName(file)}\n\n${content}`);
variableResult.files.push(file);
}
}
variableResult.content = notesContent.join("\n\n");
} else {
const processedVariableName = processVariableNameForNotePath(variableName);
const noteFiles = await getNotesFromPath(vault, processedVariableName);
const notesContent: string[] = [];
for (const file of noteFiles) {
const content = await getFileContent(file, vault);
if (content) {
notesContent.push(`## ${getFileName(file)}\n\n${content}`);
variableResult.files.push(file);
}
}
variableResult.content = notesContent.join("\n\n");
}
if (variableResult.content) {
variablesMap.set(variableName, variableResult.content);
variableResult.files.forEach((file) => includedFiles.add(file));
} else if (variableName.toLowerCase() !== "activenote") {
console.warn(`No notes found for variable: ${variableName}`);
}
}
return { variablesMap, includedFiles };
}
/**
* Represents the result of processing a custom prompt.
*/
export interface ProcessedPromptResult {
processedPrompt: string;
includedFiles: TFile[];
}
/**
* Process a custom prompt by replacing variables and adding note contents.
* Returns the processed prompt string and a list of files included in the processing.
*/
export async function processPrompt(
customPrompt: string,
selectedText: string,
vault: Vault,
activeNote?: TFile | null
): Promise<ProcessedPromptResult> {
const settings = getSettings();
const includedFiles = new Set<TFile>();
if (!settings.enableCustomPromptTemplating) {
// If templating is disabled, check if activeNote should be included for {}
if (customPrompt.includes("{}") && !selectedText && activeNote) {
includedFiles.add(activeNote);
}
return {
processedPrompt: customPrompt + "\n\n",
includedFiles: Array.from(includedFiles),
};
}
// Extract variables and track files included through them
const { variablesMap, includedFiles: variableFiles } = await extractVariablesFromPrompt(
customPrompt,
vault,
activeNote
);
variableFiles.forEach((file) => includedFiles.add(file));
let processedPrompt = customPrompt;
let additionalInfo = "";
let activeNoteContent: string | null = null;
if (processedPrompt.includes("{}")) {
processedPrompt = processedPrompt.replace(/\{\}/g, "{selectedText}");
if (selectedText) {
additionalInfo += `selectedText:\n\n${selectedText}`;
// Note: selectedText doesn't directly correspond to a file inclusion here
} else if (activeNote) {
activeNoteContent = await getFileContent(activeNote, vault);
additionalInfo += `selectedText (entire active note):\n\n${activeNoteContent}`;
includedFiles.add(activeNote); // Ensure active note is tracked if used for {}
} else {
additionalInfo += `selectedText:\n\n(No selected text or active note available)`;
}
}
// Add variable contents to the additional info
// The files are already tracked via includedFiles set
for (const [varName, content] of variablesMap.entries()) {
if (varName.toLowerCase() === "activenote" && activeNoteContent !== null) {
// Content already added via {} handling, but file tracking is done.
continue;
}
if (additionalInfo) {
additionalInfo += `\n\n${varName}:\n\n${content}`;
} else {
additionalInfo += `${varName}:\n\n${content}`;
}
}
// Process [[note title]] syntax
const noteLinkFiles = extractNoteFiles(processedPrompt, vault);
for (const noteFile of noteLinkFiles) {
// Check if this note wasn't already included via a variable
// We use the Set's reference equality which works for TFile objects
if (!includedFiles.has(noteFile)) {
const noteContent = await getFileContent(noteFile, vault);
if (noteContent) {
if (additionalInfo) {
additionalInfo += `\n\nTitle: [[${noteFile.basename}]]\nPath: ${noteFile.path}\n\n${noteContent}`;
} else {
additionalInfo += `Title: [[${noteFile.basename}]]\nPath: ${noteFile.path}\n\n${noteContent}`;
}
includedFiles.add(noteFile); // Track files included via [[links]]
}
}
}
return {
processedPrompt: additionalInfo
? `${processedPrompt}\n\n${additionalInfo}`
: `${processedPrompt}\n\n`,
includedFiles: Array.from(includedFiles),
};
}
export class CustomPromptProcessor {
private static instance: CustomPromptProcessor;
private usageStrategy: TimestampUsageStrategy;
@ -116,140 +296,21 @@ export class CustomPromptProcessor {
}
/**
* Extract variables and get their content.
* Process a custom prompt by replacing variables, adding note contents,
* and tracking which files were included.
*
* @param {CustomPrompt} doc - the custom prompt to process
* @return {Promise<string[]>} the processed custom prompt
* @param {string} customPrompt - the custom prompt template
* @param {string} selectedText - the text selected by the user
* @param {TFile} [activeNote] - the currently active note (optional)
* @return {Promise<ProcessedPromptResult>} An object containing the processed prompt string
* and an array of TFile objects included.
*/
public async extractVariablesFromPrompt(
customPrompt: string,
activeNote?: TFile
): Promise<string[]> {
const variablesWithContent: string[] = [];
const variableRegex = /\{([^}]+)\}/g;
let match;
while ((match = variableRegex.exec(customPrompt)) !== null) {
const variableName = match[1].trim();
const notes = [];
if (variableName.toLowerCase() === "activenote") {
if (activeNote) {
const content = await getFileContent(activeNote, this.vault);
if (content) {
notes.push({ name: getFileName(activeNote), content });
}
} else {
new Notice("No active note found.");
}
} else if (variableName.startsWith("#")) {
// Handle tag-based variable for multiple tags
const tagNames = variableName
.slice(1)
.split(",")
.map((tag) => tag.trim());
const noteFiles = await getNotesFromTags(this.vault, tagNames);
for (const file of noteFiles) {
const content = await getFileContent(file, this.vault);
if (content) {
notes.push({ name: getFileName(file), content });
}
}
} else {
const processedVariableName = processVariableNameForNotePath(variableName);
const noteFiles = await getNotesFromPath(this.vault, processedVariableName);
for (const file of noteFiles) {
const content = await getFileContent(file, this.vault);
if (content) {
notes.push({ name: getFileName(file), content });
}
}
}
if (notes.length > 0) {
const markdownContent = notes
.map((note) => `## ${note.name}\n\n${note.content}`)
.join("\n\n");
variablesWithContent.push(markdownContent);
} else {
console.warn(`No notes found for variable: ${variableName}`);
}
}
return variablesWithContent;
}
// TODO: return the processed variables along with the processed prompt and
// remove getProcessedVariables
async processCustomPrompt(
customPrompt: string,
selectedText: string,
activeNote?: TFile
): Promise<string> {
this.lastProcessedPrompt = customPrompt;
const variablesWithContent = await this.extractVariablesFromPrompt(customPrompt, activeNote);
let processedPrompt = customPrompt;
const matches = [...processedPrompt.matchAll(/\{([^}]+)\}/g)];
let additionalInfo = "";
let activeNoteContent: string | null = null;
if (processedPrompt.includes("{}")) {
processedPrompt = processedPrompt.replace(/\{\}/g, "{selectedText}");
if (selectedText) {
additionalInfo += `selectedText:\n\n ${selectedText}`;
} else if (activeNote) {
activeNoteContent = await getFileContent(activeNote, this.vault);
additionalInfo += `selectedText (entire active note):\n\n ${activeNoteContent}`;
} else {
additionalInfo += `selectedText:\n\n (No selected text or active note available)`;
}
}
for (let i = 0; i < variablesWithContent.length; i++) {
if (matches[i]) {
const varname = matches[i][1];
if (varname.toLowerCase() === "activenote" && activeNoteContent) {
// Skip adding activeNote content if it's already added as selectedText
continue;
}
additionalInfo += `\n\n${varname}:\n\n${variablesWithContent[i]}`;
}
}
// Process [[note title]] syntax with new reference system
const noteFiles = extractNoteFiles(processedPrompt, this.vault);
for (const noteFile of noteFiles) {
// Check if this note wasn't already processed in extractVariablesFromPrompt
if (!matches.some((match) => match[1].includes(`[[${noteFile.basename}]]`))) {
const noteContent = await getFileContent(noteFile, this.vault);
if (noteContent) {
additionalInfo += `\n\nTitle: [[${noteFile.basename}]]\nPath: ${noteFile.path}\n\n${noteContent}`;
}
}
}
return processedPrompt + "\n\n" + additionalInfo;
): Promise<ProcessedPromptResult> {
// Remove dependency on lastProcessedPrompt state
return processPrompt(customPrompt, selectedText, this.vault, activeNote);
}
// TODO: remove this
async getProcessedVariables(): Promise<Set<string>> {
const processedVars = new Set<string>();
// Add variables from the last processed prompt
const matches = this.lastProcessedPrompt?.matchAll(/\{([^}]+)\}/g) || [];
for (const match of matches) {
processedVars.add(match[1]);
}
// Add explicitly referenced note titles
const noteFiles = extractNoteFiles(this.lastProcessedPrompt || "", this.vault);
for (const file of noteFiles) {
processedVars.add(`[[${file.basename}]]`);
}
return processedVars;
}
private lastProcessedPrompt: string | null = null;
}

View file

@ -0,0 +1,28 @@
import { checkLatestVersion, isNewerVersion } from "@/utils";
import { useEffect, useState } from "react";
interface UseLatestVersionResult {
latestVersion: string | null;
hasUpdate: boolean;
}
export function useLatestVersion(currentVersion: string): UseLatestVersionResult {
const [latestVersion, setLatestVersion] = useState<string | null>(null);
useEffect(() => {
const checkVersion = async () => {
const result = await checkLatestVersion();
if (result.version) {
setLatestVersion(result.version);
}
};
checkVersion();
}, []);
const hasUpdate = latestVersion !== null && isNewerVersion(latestVersion, currentVersion);
return {
latestVersion,
hasUpdate,
};
}

View file

@ -0,0 +1,400 @@
import { logError } from "@/logger";
import { safeFetch } from "@/utils";
import { Notice, TFile, Vault } from "obsidian";
export interface ImageContent {
type: "image_url";
image_url: {
url: string;
};
}
export interface TextContent {
type: "text";
text: string;
}
export interface ImageProcessingResult {
successfulImages: ImageContent[];
failureDescriptions: string[];
}
export type MessageContent = ImageContent | TextContent;
export class ImageProcessor {
private static readonly IMAGE_EXTENSIONS = [".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp"];
private static readonly MAX_IMAGE_SIZE = 3 * 1024 * 1024; // 3MB
private static readonly MIME_TYPES = {
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
".png": "image/png",
".gif": "image/gif",
".webp": "image/webp",
".bmp": "image/bmp",
};
static async isImageUrl(url: string, vault: Vault): Promise<boolean> {
try {
// Extract extension from the URL or file path
const extension = url.split(".").pop()?.toLowerCase();
if (extension) {
// Check if the extension is supported
const isSupported = this.IMAGE_EXTENSIONS.some(
(ext) => ext.toLowerCase() === `.${extension}`
);
if (!isSupported) {
const msg = `Unsupported image format: .${extension}. Supported formats: ${this.IMAGE_EXTENSIONS.join(", ")}`;
logError(msg);
new Notice(msg);
return false;
}
}
// First check if it's an Obsidian vault image path
if (this.IMAGE_EXTENSIONS.some((ext) => url.toLowerCase().endsWith(ext))) {
// Verify the file exists and is accessible
const file = vault.getAbstractFileByPath(url);
if (!file || !(file instanceof TFile)) {
logError(`File not found in vault: ${url}`);
return false;
}
// Check file size
if (file.stat.size > this.MAX_IMAGE_SIZE) {
logError(`File too large: ${file.stat.size} bytes`);
return false;
}
return true;
}
// Then check if it's a valid URL
const urlObj = new URL(url);
// First check: URL path ends with image extension
if (this.IMAGE_EXTENSIONS.some((ext) => urlObj.pathname.toLowerCase().endsWith(ext))) {
return true;
}
// Second check: Try HEAD request to check content-type
try {
const response = await safeFetch(url, {
method: "HEAD",
headers: {}, // Explicitly set empty headers
});
const contentType = response.headers.get("content-type");
if (contentType?.startsWith("image/")) {
return true;
}
} catch (error) {
logError(`Error checking content-type for URL: ${url}`, error);
}
// Final check: Analyze URL patterns that commonly indicate image content
const searchParams = urlObj.searchParams;
const imageIndicators = [
// Image dimensions
searchParams.has("w") || searchParams.has("width"),
searchParams.has("h") || searchParams.has("height"),
// Image processing
searchParams.has("format"),
searchParams.has("fit"),
// Image quality
searchParams.has("q") || searchParams.has("quality"),
// Common CDN image path patterns
urlObj.pathname.includes("/image/"),
urlObj.pathname.includes("/images/"),
urlObj.pathname.includes("/img/"),
// Common image processing parameters
searchParams.has("auto"),
searchParams.has("crop"),
];
// If multiple image-related indicators are present, likely an image URL
const imageIndicatorCount = imageIndicators.filter(Boolean).length;
return imageIndicatorCount >= 2; // Require at least 2 indicators to consider it an image URL
} catch {
// If URL construction fails, it might still be a valid Obsidian vault image path
return this.IMAGE_EXTENSIONS.some((ext) => url.toLowerCase().endsWith(ext));
}
}
private static async handleVaultImage(file: TFile, vault: Vault): Promise<string | null> {
try {
// Check file size first
if (file.stat.size > this.MAX_IMAGE_SIZE) {
logError(`Image too large: ${file.stat.size} bytes, skipping: ${file.path}`);
return null;
}
// Read the file as array buffer
const arrayBuffer = await vault.readBinary(file);
// Validate MIME type
const mimeType = await this.getMimeType(arrayBuffer, file.extension);
if (!mimeType.startsWith("image/")) {
logError(`Invalid MIME type: ${mimeType}, skipping: ${file.path}`);
return null;
}
const buffer = Buffer.from(arrayBuffer);
const base64 = buffer.toString("base64");
const result = `data:${mimeType};base64,${base64}`;
return result;
} catch (error) {
logError("Error processing vault image:", error);
return null;
}
}
private static async handleWebImage(imageUrl: string): Promise<string | null> {
try {
const response = await safeFetch(imageUrl, {
method: "GET",
headers: {},
});
if (!response.ok) {
logError(`Failed to fetch image: ${response.statusText}, URL: ${imageUrl}`);
return null;
}
// Try to get content type from response headers
const contentType = response.headers.get("content-type");
if (!contentType?.startsWith("image/")) {
logError(`Invalid content type: ${contentType}, URL: ${imageUrl}`);
return null;
}
const arrayBuffer = await response.arrayBuffer();
// Check file size
if (arrayBuffer.byteLength > this.MAX_IMAGE_SIZE) {
logError(`Image too large: ${arrayBuffer.byteLength} bytes, URL: ${imageUrl}`);
return null;
}
const buffer = Buffer.from(arrayBuffer);
const base64 = buffer.toString("base64");
return `data:${contentType};base64,${base64}`;
} catch (error) {
logError("Error converting web image to base64:", error);
return null;
}
}
private static async handleLocalImage(imageUrl: string, vault: Vault): Promise<string | null> {
try {
const localPath = decodeURIComponent(imageUrl.replace("app://", ""));
const file = vault.getAbstractFileByPath(localPath);
if (!file || !(file instanceof TFile)) {
logError(`Local image not found: ${localPath}`);
return null;
}
// Check file size
if (file.stat.size > this.MAX_IMAGE_SIZE) {
logError(`Image too large: ${file.stat.size} bytes, path: ${localPath}`);
return null;
}
// Read the file as array buffer
const arrayBuffer = await vault.readBinary(file);
// Validate MIME type
const mimeType = await this.getMimeType(arrayBuffer, file.extension);
if (!mimeType.startsWith("image/")) {
logError(`Invalid MIME type: ${mimeType}, path: ${localPath}`);
return null;
}
const buffer = Buffer.from(arrayBuffer);
const base64 = buffer.toString("base64");
const result = `data:${mimeType};base64,${base64}`;
return result;
} catch (error) {
logError("Error processing local image:", error);
return null;
}
}
private static async imageToBase64(imageUrl: string, vault: Vault): Promise<string | null> {
// If it's already a data URL, return it as is
if (imageUrl.startsWith("data:")) {
return imageUrl;
}
// Check if it's a local vault image
if (imageUrl.startsWith("app://")) {
return await this.handleLocalImage(imageUrl, vault);
}
// Check if it's an Obsidian vault image (direct file path)
const file = vault.getAbstractFileByPath(imageUrl);
if (file instanceof TFile) {
return await this.handleVaultImage(file, vault);
}
// Handle web images
return await this.handleWebImage(imageUrl);
}
static async convertToBase64(imageUrl: string, vault: Vault): Promise<ImageContent | null> {
const base64Url = await this.imageToBase64(imageUrl, vault);
if (!base64Url) {
logError(`Failed to convert image to base64: ${imageUrl}`);
return null;
}
return {
type: "image_url",
image_url: {
url: base64Url,
},
};
}
private static async getMimeType(arrayBuffer: ArrayBuffer, extension: string): Promise<string> {
// Get the first few bytes to check for magic numbers
const bytes = new Uint8Array(arrayBuffer.slice(0, 4));
// Check for common image magic numbers
if (bytes[0] === 0xff && bytes[1] === 0xd8) return "image/jpeg";
if (bytes[0] === 0x89 && bytes[1] === 0x50) return "image/png";
if (bytes[0] === 0x47 && bytes[1] === 0x49) return "image/gif";
if (bytes[0] === 0x52 && bytes[1] === 0x49) return "image/webp";
if (bytes[0] === 0x42 && bytes[1] === 0x4d) return "image/bmp";
if (bytes[0] === 0x3c && bytes[1] === 0x73) {
throw new Error("SVG files are not supported");
}
// Fall back to extension-based detection
const mimeType = this.MIME_TYPES[extension.toLowerCase() as keyof typeof this.MIME_TYPES];
if (!mimeType) {
const error = `Unsupported image extension: ${extension}`;
logError(error);
throw new Error(error);
}
return mimeType;
}
}
export class ImageBatchProcessor {
static async processUrlBatch(
urls: string[],
failedImages: string[],
vault: Vault
): Promise<ImageProcessingResult> {
try {
const results = await Promise.all(
urls.map((url) => ImageBatchProcessor.processSingleUrl(url, failedImages, vault))
);
const successfulImages = results.filter((item): item is ImageContent => item !== null);
const failureDescriptions = failedImages.map((url) => `Image read failed for: ${url}`);
return {
successfulImages,
failureDescriptions,
};
} catch (error) {
logError("Error processing URL batch:", error);
return {
successfulImages: [],
failureDescriptions: urls.map((url) => `Image read failed for: ${url}`),
};
}
}
static async processSingleUrl(
url: string,
failedImages: string[],
vault: Vault
): Promise<ImageContent | null> {
try {
if (!(await ImageProcessor.isImageUrl(url, vault))) {
failedImages.push(url);
return null;
}
const imageContent = await ImageProcessor.convertToBase64(url, vault);
if (!imageContent) {
failedImages.push(url);
return null;
}
return imageContent;
} catch (error) {
logError(`Failed to process image: ${url}`, error);
failedImages.push(url);
return null;
}
}
static async processChatImageBatch(
content: MessageContent[],
failedImages: string[],
vault: Vault
): Promise<ImageProcessingResult> {
try {
const imageItems = content.filter(
(item): item is ImageContent => item.type === "image_url" && !!item.image_url?.url
);
const results = await Promise.all(
imageItems.map((item) =>
ImageBatchProcessor.processChatSingleImage(item, failedImages, vault)
)
);
const successfulImages = results.filter((item): item is ImageContent => item !== null);
const failureDescriptions = failedImages.map((url) => `Image read failed for: ${url}`);
return {
successfulImages,
failureDescriptions,
};
} catch (error) {
logError("Error processing chat image batch:", error);
const imageUrls = content
.filter((item): item is ImageContent => item.type === "image_url" && !!item.image_url?.url)
.map((item) => item.image_url.url);
return {
successfulImages: [],
failureDescriptions: imageUrls.map((url) => `Image read failed for: ${url}`),
};
}
}
static async processChatSingleImage(
item: ImageContent,
failedImages: string[],
vault: Vault
): Promise<ImageContent | null> {
try {
const processedContent = await ImageProcessor.convertToBase64(item.image_url.url, vault);
if (!processedContent) {
failedImages.push(item.image_url.url);
return null;
}
return processedContent;
} catch (error) {
logError(`Failed to process chat image: ${item.image_url.url}`, error);
failedImages.push(item.image_url.url);
return null;
}
}
static showFailedImagesNotice(failedImages: string[]): void {
if (failedImages.length > 0) {
new Notice(`Failed to process images:\n${failedImages.join("\n")}`);
}
}
}

View file

@ -74,7 +74,7 @@ export default class CopilotPlugin extends Plugin {
this.projectManager = ProjectManager.getInstance(this.app, this.vectorStoreManager, this);
// Initialize FileParserManager early with other core services
this.fileParserManager = new FileParserManager(this.brevilabsClient);
this.fileParserManager = new FileParserManager(this.brevilabsClient, this.app.vault);
this.registerView(CHAT_VIEWTYPE, (leaf: WorkspaceLeaf) => new CopilotView(leaf, this));
this.registerView(APPLY_VIEW_TYPE, (leaf: WorkspaceLeaf) => new ApplyView(leaf));

View file

@ -1,5 +1,6 @@
import { ImageProcessor } from "@/imageProcessing/imageProcessor";
import { BrevilabsClient, Url4llmResponse } from "@/LLMProviders/brevilabsClient";
import { ImageProcessor, isYoutubeUrl } from "@/utils";
import { isYoutubeUrl } from "@/utils";
export interface MentionData {
type: string;

View file

@ -32,10 +32,6 @@ export function useIsPlusUser(): boolean | undefined {
/** Check if the user is a Plus user. */
export async function checkIsPlusUser(): Promise<boolean | undefined> {
if (!getSettings().plusLicenseKey) {
turnOffPlus();
return false;
}
const brevilabsClient = BrevilabsClient.getInstance();
const result = await brevilabsClient.validateLicenseKey();
return result.isValid;

View file

@ -1,5 +1,6 @@
import { CustomModel, ProjectConfig } from "@/aiParams";
import { atom, createStore, useAtomValue } from "jotai";
import { v4 as uuidv4 } from "uuid";
import { type ChainType } from "@/chainFactory";
import {
@ -36,6 +37,7 @@ export interface InlineEditCommandSettings {
}
export interface CopilotSettings {
userId: string;
plusLicenseKey: string;
openAIApiKey: string;
openAIOrgId: string;
@ -49,6 +51,7 @@ export interface CopilotSettings {
azureOpenAIApiEmbeddingDeploymentName: string;
googleApiKey: string;
openRouterAiApiKey: string;
xaiApiKey: string;
mistralApiKey: string;
deepseekApiKey: string;
defaultChainType: ChainType;
@ -57,6 +60,7 @@ export interface CopilotSettings {
temperature: number;
maxTokens: number;
contextTurns: number;
lastDismissedVersion: string | null;
// Do not use this directly, use getSystemPrompt() instead
userSystemPrompt: string;
openAIProxyBaseUrl: string;
@ -90,9 +94,10 @@ export interface CopilotSettings {
// undefined means never checked
isPlusUser: boolean | undefined;
inlineEditCommands: InlineEditCommandSettings[] | undefined;
// Autocomplete settings
enableAutocomplete: boolean;
projectList: Array<ProjectConfig>;
passMarkdownImages: boolean;
enableCustomPromptTemplating: boolean;
}
export const settingsStore = createStore();
@ -166,6 +171,10 @@ export function sanitizeSettings(settings: CopilotSettings): CopilotSettings {
// If settings is null/undefined, use DEFAULT_SETTINGS
const settingsToSanitize = settings || DEFAULT_SETTINGS;
if (!settingsToSanitize.userId) {
settingsToSanitize.userId = uuidv4();
}
// fix: Maintain consistency between EmbeddingModelProviders.AZURE_OPENAI and ChatModelProviders.AZURE_OPENAI,
// where it was 'azure_openai' before EmbeddingModelProviders.AZURE_OPENAI.
if (!settingsToSanitize.activeEmbeddingModels) {
@ -206,6 +215,16 @@ export function sanitizeSettings(settings: CopilotSettings): CopilotSettings {
? DEFAULT_SETTINGS.embeddingBatchSize
: embeddingBatchSize;
// Ensure passMarkdownImages has a default value
if (typeof sanitizedSettings.passMarkdownImages !== "boolean") {
sanitizedSettings.passMarkdownImages = DEFAULT_SETTINGS.passMarkdownImages;
}
// Ensure enableCustomPromptTemplating has a default value
if (typeof sanitizedSettings.enableCustomPromptTemplating !== "boolean") {
sanitizedSettings.enableCustomPromptTemplating = DEFAULT_SETTINGS.enableCustomPromptTemplating;
}
return sanitizedSettings;
}

View file

@ -5,9 +5,9 @@ import { TabProvider, useTab } from "@/contexts/TabContext";
import CopilotPlugin from "@/main";
import { resetSettings } from "@/settings/model";
import { CommandSettings } from "@/settings/v2/components/CommandSettings";
import { checkLatestVersion, isNewerVersion } from "@/utils";
import { useLatestVersion } from "@/hooks/useLatestVersion";
import { Cog, Command, Cpu, Database, Wrench } from "lucide-react";
import React, { useEffect, useState } from "react";
import React from "react";
import { AdvancedSettings } from "./components/AdvancedSettings";
import { BasicSettings } from "./components/BasicSettings";
import { ModelSettings } from "./components/ModelSettings";
@ -81,24 +81,7 @@ interface SettingsMainV2Props {
const SettingsMainV2: React.FC<SettingsMainV2Props> = ({ plugin }) => {
// Add a key state that we'll change when resetting
const [resetKey, setResetKey] = React.useState(0);
const [latestVersion, setLatestVersion] = useState<string | null>(null);
const [updateError, setUpdateError] = useState<string | null>(null);
useEffect(() => {
// Check version when settings tab is opened
const checkForUpdates = async () => {
const { version, error } = await checkLatestVersion();
if (error) {
console.error("Version check failed:", error);
setUpdateError(error);
} else if (version) {
setLatestVersion(version);
setUpdateError(null);
}
};
checkForUpdates();
}, [plugin.manifest.version]); // Only re-run if plugin version changes
const { latestVersion, hasUpdate } = useLatestVersion(plugin.manifest.version);
const handleReset = async () => {
const modal = new ResetSettingsConfirmModal(app, async () => {
@ -109,9 +92,6 @@ const SettingsMainV2: React.FC<SettingsMainV2Props> = ({ plugin }) => {
modal.open();
};
const isNewerVersionAvailable =
latestVersion && isNewerVersion(latestVersion, plugin.manifest.version);
return (
<TabProvider>
<div>
@ -119,35 +99,25 @@ const SettingsMainV2: React.FC<SettingsMainV2Props> = ({ plugin }) => {
<h1 className="flex flex-col sm:flex-row sm:items-center sm:justify-between gap-2">
<div className="flex items-center gap-2">
<span>Copilot Settings</span>
<span className="text-xs text-muted">
<a
href="https://github.com/logancyang/obsidian-copilot/releases/"
target="_blank"
rel="noopener noreferrer"
className="text-accent hover:underline"
>
v{plugin.manifest.version}
</a>
{updateError ? (
<span className="text-error" title={updateError}>
{" "}
(update check failed)
</span>
) : (
latestVersion && (
<>
{isNewerVersionAvailable ? (
<span className="text-accent" title="A new version is available">
{" "}
(latest: v{latestVersion})
</span>
) : (
<span className="text-accent"> (up to date)</span>
)}
</>
)
<div className="flex items-center gap-1">
<span className="text-xs text-muted">v{plugin.manifest.version}</span>
{latestVersion && (
<>
{hasUpdate ? (
<a
href="obsidian://show-plugin?id=copilot"
target="_blank"
rel="noopener noreferrer"
className="text-xs text-accent hover:underline"
>
(Update to v{latestVersion})
</a>
) : (
<span className="text-xs text-normal"> (up to date)</span>
)}
</>
)}
</span>
</div>
</div>
<div className="self-end sm:self-auto">
<Button variant="secondary" size="sm" onClick={handleReset}>

View file

@ -1,6 +1,6 @@
import React from "react";
import { SettingItem } from "@/components/ui/setting-item";
import { updateSetting, useSettingsValue } from "@/settings/model";
import React from "react";
export const AdvancedSettings: React.FC = () => {
const settings = useSettingsValue();
@ -19,6 +19,26 @@ export const AdvancedSettings: React.FC = () => {
/>
<div className="space-y-4">
<SettingItem
type="switch"
title="Custom Prompt Templating"
description="Enable templating to process variables like {activenote}, {foldername} or {#tag} in prompts. Disable to use raw prompts without any processing."
checked={settings.enableCustomPromptTemplating}
onCheckedChange={(checked) => {
updateSetting("enableCustomPromptTemplating", checked);
}}
/>
<SettingItem
type="switch"
title="Images in Markdown (Plus)"
description="Pass embedded images in markdown to the AI along with the text. Only works with multimodal models (plus only)."
checked={settings.passMarkdownImages}
onCheckedChange={(checked) => {
updateSetting("passMarkdownImages", checked);
}}
/>
<SettingItem
type="switch"
title="Enable Encryption"

View file

@ -3,13 +3,13 @@ import { useTab } from "@/contexts/TabContext";
import { getSettings } from "@/settings/model";
import {
ChatModelProviders,
SettingKeyProviders,
EmbeddingModelProviders,
MODEL_CAPABILITIES,
ModelCapability,
Provider,
ProviderMetadata,
ProviderSettingsKeyMap,
SettingKeyProviders,
} from "@/constants";
import { CustomModel } from "@/aiParams";
import { err2String, getProviderInfo, getProviderLabel, omit } from "@/utils";
@ -18,7 +18,7 @@ import { Label } from "@/components/ui/label";
import { Input } from "@/components/ui/input";
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
import { Button } from "@/components/ui/button";
import { ChevronDown, Loader2, HelpCircle } from "lucide-react";
import { ChevronDown, HelpCircle, Loader2 } from "lucide-react";
import {
Dialog,
DialogContent,
@ -496,7 +496,25 @@ export const ModelAddDialog: React.FC<ModelAddDialogProps> = ({
)}
</FormField>
<FormField label="Model Capabilities">
<FormField
label={
<div className="flex items-center gap-1.5">
<span className="leading-none">Model Capabilities</span>
<TooltipProvider delayDuration={0}>
<Tooltip>
<TooltipTrigger asChild>
<HelpCircle className="size-4" />
</TooltipTrigger>
<TooltipContent align="start" className="max-w-96" side="bottom">
<div className="text-sm text-muted">
Only used to display model capabilities, does not affect model functionality
</div>
</TooltipContent>
</Tooltip>
</TooltipProvider>
</div>
}
>
<div className="flex gap-4 items-center">
{capabilityOptions.map(({ id, label, description }) => (
<div key={id} className="flex items-center gap-2">
@ -539,7 +557,21 @@ export const ModelAddDialog: React.FC<ModelAddDialogProps> = ({
onCheckedChange={(checked: boolean) => setModel({ ...model, enableCors: checked })}
/>
<Label htmlFor="enable-cors" className="text-sm">
Enable CORS
<div className="flex items-center gap-1.5">
<span>Enable CORS</span>
<TooltipProvider delayDuration={0}>
<Tooltip>
<TooltipTrigger asChild>
<HelpCircle className="size-4" />
</TooltipTrigger>
<TooltipContent align="start" className="max-w-96" side="bottom">
<div className="text-sm text-muted">
Only check this option when prompted that CORS is needed
</div>
</TooltipContent>
</Tooltip>
</TooltipProvider>
</div>
</Label>
</div>
<div className="flex gap-2">

View file

@ -1,4 +1,4 @@
import React, { useEffect, useState } from "react";
import React, { useEffect, useState, useMemo, useCallback } from "react";
import { useTab } from "@/contexts/TabContext";
import { CustomModel } from "@/aiParams";
import {
@ -14,13 +14,24 @@ import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/comp
import { FormField } from "@/components/ui/form-field";
import { Checkbox } from "@/components/ui/checkbox";
import { Label } from "@/components/ui/label";
import { MODEL_CAPABILITIES, ModelCapability } from "@/constants";
import {
MODEL_CAPABILITIES,
ModelCapability,
Provider,
ProviderMetadata,
ProviderSettingsKeyMap,
SettingKeyProviders,
} from "@/constants";
import { getProviderInfo, getProviderLabel } from "@/utils";
import { PasswordInput } from "@/components/ui/password-input";
import { getSettings } from "@/settings/model";
import { debounce } from "@/utils";
interface ModelEditDialogProps {
open: boolean;
onOpenChange: (open: boolean) => void;
model: CustomModel | null;
onUpdate: (model: CustomModel) => void;
onUpdate: (originalModel: CustomModel, updatedModel: CustomModel) => void;
}
export const ModelEditDialog: React.FC<ModelEditDialogProps> = ({
@ -31,20 +42,65 @@ export const ModelEditDialog: React.FC<ModelEditDialogProps> = ({
}) => {
const { modalContainer } = useTab();
const [localModel, setLocalModel] = useState<CustomModel | null>(model);
const [originalModel, setOriginalModel] = useState<CustomModel | null>(model);
const [providerInfo, setProviderInfo] = useState<ProviderMetadata>({} as ProviderMetadata);
const settings = getSettings();
const getDefaultApiKey = (provider: Provider): string => {
return (settings[ProviderSettingsKeyMap[provider as SettingKeyProviders]] as string) || "";
};
useEffect(() => {
setLocalModel(model);
setOriginalModel(model);
if (model?.provider) {
setProviderInfo(getProviderInfo(model.provider));
}
}, [model]);
// Debounce the onUpdate callback
const debouncedOnUpdate = useMemo(
() =>
debounce((currentOriginalModel: CustomModel | null, updatedModel: CustomModel) => {
if (currentOriginalModel) {
onUpdate(currentOriginalModel, updatedModel);
}
}, 500),
[onUpdate]
);
// Function to update local state immediately
const handleLocalUpdate = useCallback(
(field: keyof CustomModel, value: any) => {
setLocalModel((prevModel) => {
if (!prevModel) return null;
const updatedModel = {
...prevModel,
[field]: value,
};
// Call the debounced update function, passing the stable originalModel and the new updatedModel
debouncedOnUpdate(originalModel, updatedModel);
return updatedModel; // Return the updated model for immediate state update
});
},
[originalModel, debouncedOnUpdate]
);
if (!localModel) return null;
const handleUpdate = (field: keyof CustomModel, value: any) => {
const updatedModel = {
...localModel,
[field]: value,
};
setLocalModel(updatedModel);
onUpdate(updatedModel);
const getPlaceholderUrl = () => {
if (!localModel || !localModel.provider || localModel.provider !== "azure-openai") {
return providerInfo.host || "https://api.example.com/v1";
}
const instanceName = localModel.azureOpenAIApiInstanceName || "[instance]";
const deploymentName = localModel.isEmbeddingModel
? localModel.azureOpenAIApiEmbeddingDeploymentName || "[deployment]"
: localModel.azureOpenAIApiDeploymentName || "[deployment]";
const apiVersion = localModel.azureOpenAIApiVersion || "[api-version]";
const endpoint = localModel.isEmbeddingModel ? "embeddings" : "chat/completions";
return `https://${instanceName}.openai.azure.com/openai/deployments/${deploymentName}/${endpoint}?api-version=${apiVersion}`;
};
const capabilityOptions = Object.entries(MODEL_CAPABILITIES).map(([id, description]) => ({
@ -53,6 +109,8 @@ export const ModelEditDialog: React.FC<ModelEditDialogProps> = ({
description,
})) as Array<{ id: ModelCapability; label: string; description: string }>;
const displayApiKey = localModel.apiKey || getDefaultApiKey(localModel.provider as Provider);
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className="sm:max-w-[425px]" container={modalContainer}>
@ -61,13 +119,13 @@ export const ModelEditDialog: React.FC<ModelEditDialogProps> = ({
<DialogDescription>Customize model parameters.</DialogDescription>
</DialogHeader>
<div className="space-y-6">
<div className="space-y-3">
<FormField label="Model Name" required>
<Input
type="text"
disabled={localModel.core}
value={localModel.name}
onChange={(e) => handleUpdate("name", e.target.value)}
onChange={(e) => handleLocalUpdate("name", e.target.value)}
placeholder="Enter model name"
/>
</FormField>
@ -102,11 +160,62 @@ export const ModelEditDialog: React.FC<ModelEditDialogProps> = ({
type="text"
placeholder="Custom display name (optional)"
value={localModel.displayName || ""}
onChange={(e) => handleUpdate("displayName", e.target.value)}
onChange={(e) => handleLocalUpdate("displayName", e.target.value)}
/>
</FormField>
<FormField label="Model Capabilities">
<FormField label="Provider">
<Input
type="text"
value={getProviderLabel(localModel.provider)}
disabled
className="bg-muted"
/>
</FormField>
<FormField label="Base URL" description="Leave it blank, unless you are using a proxy.">
<Input
type="text"
placeholder={getPlaceholderUrl()}
value={localModel.baseUrl || ""}
onChange={(e) => handleLocalUpdate("baseUrl", e.target.value)}
/>
</FormField>
<FormField label="API Key">
<PasswordInput
placeholder={`Enter ${providerInfo.label || "Provider"} API Key`}
value={displayApiKey}
onChange={(value) => handleLocalUpdate("apiKey", value)}
/>
{providerInfo.keyManagementURL && (
<p className="text-xs text-muted">
<a href={providerInfo.keyManagementURL} target="_blank" rel="noopener noreferrer">
Get {providerInfo.label} API Key
</a>
</p>
)}
</FormField>
<FormField
label={
<div className="flex items-center gap-1.5">
<span className="leading-none">Model Capabilities</span>
<TooltipProvider delayDuration={0}>
<Tooltip>
<TooltipTrigger asChild>
<HelpCircle className="size-4" />
</TooltipTrigger>
<TooltipContent align="start" className="max-w-96" side="bottom">
<div className="text-sm text-muted">
Only used to display model capabilities, does not affect model functionality
</div>
</TooltipContent>
</Tooltip>
</TooltipProvider>
</div>
}
>
<div className="flex gap-4 items-center">
{capabilityOptions.map(({ id, label, description }) => (
<div key={id} className="flex items-center gap-2">
@ -115,12 +224,10 @@ export const ModelEditDialog: React.FC<ModelEditDialogProps> = ({
checked={localModel.capabilities?.includes(id)}
onCheckedChange={(checked) => {
const newCapabilities = localModel.capabilities || [];
handleUpdate(
"capabilities",
checked
? [...newCapabilities, id]
: newCapabilities.filter((cap) => cap !== id)
);
const value = checked
? [...newCapabilities, id]
: newCapabilities.filter((cap) => cap !== id);
handleLocalUpdate("capabilities", value);
}}
/>
<Label htmlFor={id} className="text-sm">
@ -157,7 +264,7 @@ export const ModelEditDialog: React.FC<ModelEditDialogProps> = ({
>
<SettingSlider
value={localModel.temperature ?? 0.1}
onChange={(value) => handleUpdate("temperature", value)}
onChange={(value) => handleLocalUpdate("temperature", value)}
max={2}
min={0}
step={0.1}
@ -183,7 +290,7 @@ export const ModelEditDialog: React.FC<ModelEditDialogProps> = ({
>
<SettingSlider
value={localModel.context ?? 1000}
onChange={(value) => handleUpdate("context", value)}
onChange={(value) => handleLocalUpdate("context", value)}
max={16000}
min={0}
step={100}
@ -206,7 +313,7 @@ export const ModelEditDialog: React.FC<ModelEditDialogProps> = ({
</div>
<SettingSwitch
checked={localModel.stream ?? true}
onCheckedChange={(checked) => handleUpdate("stream", checked)}
onCheckedChange={(checked) => handleLocalUpdate("stream", checked)}
/>
</div>*/}
</div>

View file

@ -1,12 +1,14 @@
import React, { useState } from "react";
import { SettingItem } from "@/components/ui/setting-item";
import { setSettings, updateSetting, useSettingsValue } from "@/settings/model";
import { CustomModel } from "@/aiParams";
import { SettingItem } from "@/components/ui/setting-item";
import EmbeddingManager from "@/LLMProviders/embeddingManager";
import { ModelAddDialog } from "@/settings/v2/components/ModelAddDialog";
import { ModelTable } from "@/settings/v2/components/ModelTable";
import { ModelEditDialog } from "@/settings/v2/components/ModelEditDialog";
import ProjectManager from "@/LLMProviders/projectManager";
import { logError } from "@/logger";
import { setSettings, updateSetting, useSettingsValue } from "@/settings/model";
import { ModelAddDialog } from "@/settings/v2/components/ModelAddDialog";
import { ModelEditDialog } from "@/settings/v2/components/ModelEditDialog";
import { ModelTable } from "@/settings/v2/components/ModelTable";
import { Notice } from "obsidian";
import React, { useState } from "react";
export const ModelSettings: React.FC = () => {
const settings = useSettingsValue();
@ -34,7 +36,22 @@ export const ModelSettings: React.FC = () => {
});
};
const handleModelUpdate = (updatedModel: CustomModel) => {
const handleModelUpdate = (originalModel: CustomModel, updatedModel: CustomModel) => {
const modelIndex = settings.activeModels.findIndex(
(m) => m.name === originalModel.name && m.provider === originalModel.provider
);
if (modelIndex !== -1) {
const updatedModels = [...settings.activeModels];
updatedModels[modelIndex] = updatedModel;
updateSetting("activeModels", updatedModels);
} else {
new Notice("Could not find model to update");
logError("Could not find model to update:", originalModel);
}
};
// Handler for updates originating from the ModelTable itself (e.g., checkbox toggles)
const handleTableUpdate = (updatedModel: CustomModel) => {
const updatedModels = settings.activeModels.map((m) =>
m.name === updatedModel.name && m.provider === updatedModel.provider ? updatedModel : m
);
@ -73,7 +90,7 @@ export const ModelSettings: React.FC = () => {
onEdit={setEditingModel}
onDelete={onDeleteModel}
onAdd={() => setShowAddDialog(true)}
onUpdateModel={handleModelUpdate}
onUpdateModel={handleTableUpdate}
onReorderModels={handleModelReorder}
title="Chat Model"
/>

170
src/tools/CanvasLoader.ts Normal file
View file

@ -0,0 +1,170 @@
import { TFile, Vault } from "obsidian";
/* ---------- Core data types ---------- */
interface CanvasNodeBase {
id: string;
x: number;
y: number;
width: number;
height: number;
type: "file" | "text" | "link" | "group";
label?: string; // groups
color?: string; // files / links
url?: string; // links
file?: string; // files
text?: string; // text cards
}
export interface RichNode extends CanvasNodeBase {
/** Inlined markdown or plaintext content (empty for groups/links). */
content: string;
}
export interface CanvasEdge {
id: string;
fromNode: string;
toNode: string;
fromSide?: string;
toSide?: string;
/** Synthetic labels such as "contains". */
label?: string;
}
export interface CanvasData {
nodes: RichNode[];
edges: CanvasEdge[];
byId: Record<string, RichNode>;
}
/* ---------- Loader class ---------- */
export class CanvasLoader {
constructor(private vault: Vault) {}
/** Load & enrich a `.canvas` file. */
async load(file: TFile): Promise<CanvasData> {
const raw = await this.vault.read(file);
const { nodes = [], edges = [] } = JSON.parse(raw) as {
nodes: CanvasNodeBase[];
edges: CanvasEdge[];
};
const richNodes: RichNode[] = await Promise.all(
nodes.map(async (n) => {
if (n.type === "file" && n.file) {
const file = this.vault.getAbstractFileByPath(n.file);
const md = file instanceof TFile ? await this.vault.cachedRead(file) : "";
return { ...n, content: md };
}
if (n.type === "text") return { ...n, content: n.text ?? "" };
return { ...n, content: "" }; // link / group
})
);
const allEdges = [...edges];
this.#deriveGroupEdges(richNodes, allEdges);
const byId = Object.fromEntries(richNodes.map((n) => [n.id, n]));
return { nodes: richNodes, edges: allEdges, byId };
}
/** Build a concise prompt for an LLM. */
buildPrompt(canvas: CanvasData): string {
// First, build a map of group contents
const groupContents = new Map<string, RichNode[]>();
const groups = canvas.nodes.filter((n) => n.type === "group");
for (const group of groups) {
const containedNodes = canvas.nodes.filter((n) => {
if (n.id === group.id) return false;
const nodeX = n.x + n.width / 2;
const nodeY = n.y + n.height / 2;
return (
nodeX >= group.x &&
nodeY >= group.y &&
nodeX <= group.x + group.width &&
nodeY <= group.y + group.height
);
});
groupContents.set(group.label || group.id, containedNodes);
}
// Build a clear, structured description
let description = `This canvas contains the following elements:\n\n`;
// Helper function to format node content
const formatNodeContent = (node: RichNode): string => {
switch (node.type) {
case "file":
return `- File: ${node.file}\nContent:\n${node.content}\n`;
case "text":
return `- Text: "${node.text}"\n`;
case "link":
return `- Link: ${node.url}\n`;
default:
return "";
}
};
// Describe groups and their contents
groups.forEach((group) => {
const groupName = group.label || group.id;
const contents = groupContents.get(groupName) || [];
description += `Group "${groupName}" contains:\n`;
contents.forEach((node) => {
description += formatNodeContent(node);
});
description += "\n";
});
// Describe non-grouped elements
const ungroupedNodes = canvas.nodes.filter((n) => {
if (n.type === "group") return false;
return !Array.from(groupContents.values())
.flat()
.some((gn) => gn.id === n.id);
});
if (ungroupedNodes.length > 0) {
description += "Elements outside of groups:\n";
ungroupedNodes.forEach((node) => {
description += formatNodeContent(node);
});
}
description += "\nWhen describing this canvas, please:\n";
description += "- Use the actual titles/names of elements instead of their IDs\n";
description += "- Pay attention to the content and relationships between elements\n";
description +=
"- Describe files by their names, links by their URLs, and text nodes by their content\n";
return description;
}
/* ---------- private helpers ---------- */
/** Add synthetic 'contains' edges for group membership. */
#deriveGroupEdges(nodes: RichNode[], edges: CanvasEdge[]) {
const groups = nodes.filter((n) => n.type === "group");
for (const g of groups) {
for (const n of nodes) {
if (n.id === g.id) continue;
// Check if node's center point is within the group's bounds
const nodeX = n.x + n.width / 2;
const nodeY = n.y + n.height / 2;
const inside =
nodeX >= g.x && nodeY >= g.y && nodeX <= g.x + g.width && nodeY <= g.y + g.height;
if (inside) {
edges.push({
id: crypto.randomUUID(),
fromNode: g.id,
toNode: n.id,
label: "contains",
});
}
}
}
}
}

View file

@ -2,6 +2,7 @@ import { BrevilabsClient } from "@/LLMProviders/brevilabsClient";
import { PDFCache } from "@/cache/pdfCache";
import { logError, logInfo } from "@/logger";
import { TFile, Vault } from "obsidian";
import { CanvasLoader } from "./CanvasLoader";
interface FileParser {
supportedExtensions: string[];
@ -55,6 +56,24 @@ export class PDFParser implements FileParser {
}
}
export class CanvasParser implements FileParser {
supportedExtensions = ["canvas"];
async parseFile(file: TFile, vault: Vault): Promise<string> {
try {
logInfo("Parsing Canvas file:", file.path);
const canvasLoader = new CanvasLoader(vault);
const canvasData = await canvasLoader.load(file);
// Use the specialized buildPrompt method to create LLM-friendly format
return canvasLoader.buildPrompt(canvasData);
} catch (error) {
logError(`Error parsing Canvas file ${file.path}:`, error);
return `[Error: Could not parse Canvas file ${file.basename}]`;
}
}
}
// Future parsers can be added like this:
/*
class DocxParser implements FileParser {
@ -69,10 +88,11 @@ class DocxParser implements FileParser {
export class FileParserManager {
private parsers: Map<string, FileParser> = new Map();
constructor(brevilabsClient: BrevilabsClient) {
// Register more parsers here
constructor(brevilabsClient: BrevilabsClient, vault: Vault) {
// Register parsers
this.registerParser(new MarkdownParser());
this.registerParser(new PDFParser(brevilabsClient));
this.registerParser(new CanvasParser());
}
registerParser(parser: FileParser) {

View file

@ -1,6 +1,7 @@
import { TFolder } from "obsidian";
import { createGetFileTreeTool, buildFileTree } from "./FileTreeTools";
import * as searchUtils from "@/search/searchUtils";
import { ToolManager } from "@/tools/toolManager";
import { TFolder } from "obsidian";
import { buildFileTree, createGetFileTreeTool } from "./FileTreeTools";
// Mock the searchUtils functions
jest.mock("@/search/searchUtils", () => ({
@ -129,7 +130,7 @@ describe("FileTreeTools", () => {
// Also test the tool to ensure it uses buildFileTree correctly
const tool = createGetFileTreeTool(root);
const result = await tool.invoke({});
const result = await ToolManager.callTool(tool, {});
// Extract JSON part after the prompt
const jsonPart = result.substring(result.indexOf("{"));

View file

@ -10,13 +10,12 @@ import {
SettingKeyProviders,
USER_SENDER,
} from "@/constants";
import { logError, logInfo } from "@/logger";
import { logInfo } from "@/logger";
import { CopilotSettings } from "@/settings/model";
import { ChatMessage } from "@/sharedState";
import { BaseChatModel } from "@langchain/core/language_models/chat_models";
import { MemoryVariables } from "@langchain/core/memory";
import { RunnableSequence } from "@langchain/core/runnables";
import { Buffer } from "buffer";
import { BaseChain, RetrievalQAChain } from "langchain/chains";
import moment from "moment";
import { MarkdownView, Notice, TFile, Vault, requestUrl } from "obsidian";
@ -269,7 +268,7 @@ export function stringToFormattedDateTime(timestamp: string): FormattedDateTime
}
export async function getFileContent(file: TFile, vault: Vault): Promise<string | null> {
if (file.extension != "md") return null;
if (file.extension != "md" && file.extension != "canvas") return null;
return await vault.cachedRead(file);
}
@ -474,282 +473,6 @@ export function extractYoutubeUrl(text: string): string | null {
return match ? match[0] : null;
}
export interface ImageContent {
type: "image_url";
image_url: {
url: string;
};
}
export interface TextContent {
type: "text";
text: string;
}
export type MessageContent = ImageContent | TextContent;
export class ImageProcessor {
private static readonly IMAGE_EXTENSIONS = [
".jpg",
".jpeg",
".png",
".gif",
".webp",
".bmp",
".svg",
];
private static readonly MAX_IMAGE_SIZE = 3 * 1024 * 1024; // 3MB
private static readonly MIME_TYPES = {
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
".png": "image/png",
".gif": "image/gif",
".webp": "image/webp",
".bmp": "image/bmp",
".svg": "image/svg+xml",
};
static async isImageUrl(url: string, vault: Vault): Promise<boolean> {
try {
// First check if it's an Obsidian vault image path
if (this.IMAGE_EXTENSIONS.some((ext) => url.toLowerCase().endsWith(ext))) {
// Verify the file exists and is accessible
const file = vault.getAbstractFileByPath(url);
if (!file || !(file instanceof TFile)) {
logError("File not found in vault");
return false;
}
// Check file size
if (file.stat.size > this.MAX_IMAGE_SIZE) {
logError("File too large:", file.stat.size, "bytes");
return false;
}
return true;
}
// Then check if it's a valid URL
const urlObj = new URL(url);
// First check: URL path ends with image extension
if (this.IMAGE_EXTENSIONS.some((ext) => urlObj.pathname.toLowerCase().endsWith(ext))) {
return true;
}
// Second check: Try HEAD request to check content-type
try {
const response = await safeFetch(url, {
method: "HEAD",
headers: {}, // Explicitly set empty headers
});
const contentType = response.headers.get("content-type");
if (contentType?.startsWith("image/")) {
return true;
}
} catch (error) {
logError("Error checking content-type:", error);
}
// Final check: Analyze URL patterns that commonly indicate image content
const searchParams = urlObj.searchParams;
const imageIndicators = [
// Image dimensions
searchParams.has("w") || searchParams.has("width"),
searchParams.has("h") || searchParams.has("height"),
// Image processing
searchParams.has("format"),
searchParams.has("fit"),
// Image quality
searchParams.has("q") || searchParams.has("quality"),
// Common CDN image path patterns
urlObj.pathname.includes("/image/"),
urlObj.pathname.includes("/images/"),
urlObj.pathname.includes("/img/"),
// Common image processing parameters
searchParams.has("auto"),
searchParams.has("crop"),
];
// If multiple image-related indicators are present, likely an image URL
const imageIndicatorCount = imageIndicators.filter(Boolean).length;
return imageIndicatorCount >= 2; // Require at least 2 indicators to consider it an image URL
} catch {
// If URL construction fails, it might still be a valid Obsidian vault image path
return this.IMAGE_EXTENSIONS.some((ext) => url.toLowerCase().endsWith(ext));
}
}
private static async handleVaultImage(file: TFile, vault: Vault): Promise<string | null> {
try {
// Check file size first
if (file.stat.size > this.MAX_IMAGE_SIZE) {
logError(`Image too large: ${file.stat.size} bytes, skipping: ${file.path}`);
return null;
}
// Read the file as array buffer
const arrayBuffer = await vault.readBinary(file);
// Validate MIME type
const mimeType = await this.getMimeType(arrayBuffer, file.extension);
if (!mimeType.startsWith("image/")) {
logError(`Invalid MIME type: ${mimeType}, skipping: ${file.path}`);
return null;
}
const buffer = Buffer.from(arrayBuffer);
const base64 = buffer.toString("base64");
const result = `data:${mimeType};base64,${base64}`;
return result;
} catch (error) {
logError("Error in handleVaultImage:", error);
return null;
}
}
private static async handleWebImage(imageUrl: string): Promise<string | null> {
try {
const response = await safeFetch(imageUrl, {
method: "GET",
headers: {},
});
if (!response.ok) {
logError(`Failed to fetch image: ${response.statusText}, skipping: ${imageUrl}`);
return null;
}
// Try to get content type from response headers
const contentType = response.headers.get("content-type");
if (!contentType?.startsWith("image/")) {
logError(`Invalid content type: ${contentType}, skipping: ${imageUrl}`);
return null;
}
const arrayBuffer = await response.arrayBuffer();
// Check file size
if (arrayBuffer.byteLength > this.MAX_IMAGE_SIZE) {
logError(`Image too large: ${arrayBuffer.byteLength} bytes, skipping: ${imageUrl}`);
return null;
}
const buffer = Buffer.from(arrayBuffer);
const base64 = buffer.toString("base64");
return `data:${contentType};base64,${base64}`;
} catch (error) {
logError("Error converting image to base64:", error);
return null;
}
}
private static async handleLocalImage(imageUrl: string, vault: Vault): Promise<string | null> {
try {
const localPath = decodeURIComponent(imageUrl.replace("app://", ""));
const file = vault.getAbstractFileByPath(localPath);
if (!file || !(file instanceof TFile)) {
logError(`Local image not found: ${localPath}`);
return null;
}
// Check file size
if (file.stat.size > this.MAX_IMAGE_SIZE) {
logError(`Image too large: ${file.stat.size} bytes, skipping: ${localPath}`);
return null;
}
// Read the file as array buffer
const arrayBuffer = await vault.readBinary(file);
// Validate MIME type
const mimeType = await this.getMimeType(arrayBuffer, file.extension);
if (!mimeType.startsWith("image/")) {
logError(`Invalid MIME type: ${mimeType}, skipping: ${localPath}`);
return null;
}
const buffer = Buffer.from(arrayBuffer);
const base64 = buffer.toString("base64");
const result = `data:${mimeType};base64,${base64}`;
return result;
} catch (error) {
logError("Error in handleLocalImage:", error);
return null;
}
}
private static async imageToBase64(imageUrl: string, vault: Vault): Promise<string | null> {
// If it's already a data URL, return it as is
if (imageUrl.startsWith("data:")) {
return imageUrl;
}
// Check if it's a local vault image
if (imageUrl.startsWith("app://")) {
return await this.handleLocalImage(imageUrl, vault);
}
// Check if it's an Obsidian vault image (direct file path)
const file = vault.getAbstractFileByPath(imageUrl);
if (file instanceof TFile) {
return await this.handleVaultImage(file, vault);
}
// Handle web images
return await this.handleWebImage(imageUrl);
}
static async convertToBase64(imageUrl: string, vault: Vault): Promise<ImageContent | null> {
const base64Url = await this.imageToBase64(imageUrl, vault);
if (!base64Url) {
return null;
}
return {
type: "image_url",
image_url: {
url: base64Url,
},
};
}
private static async getMimeType(arrayBuffer: ArrayBuffer, extension: string): Promise<string> {
// Get the first few bytes to check for magic numbers
const bytes = new Uint8Array(arrayBuffer.slice(0, 4));
// Check for common image magic numbers
if (bytes[0] === 0xff && bytes[1] === 0xd8) {
return "image/jpeg";
}
if (bytes[0] === 0x89 && bytes[1] === 0x50) {
return "image/png";
}
if (bytes[0] === 0x47 && bytes[1] === 0x49) {
return "image/gif";
}
if (bytes[0] === 0x52 && bytes[1] === 0x49) {
return "image/webp";
}
if (bytes[0] === 0x42 && bytes[1] === 0x4d) {
return "image/bmp";
}
if (bytes[0] === 0x3c && bytes[1] === 0x73) {
return "image/svg+xml";
}
// Fall back to extension-based detection
const mimeType = this.MIME_TYPES[extension.toLowerCase() as keyof typeof this.MIME_TYPES];
if (!mimeType) {
throw new Error(`Unsupported image extension: ${extension}`);
}
return mimeType;
}
}
/** Proxy function to use in place of fetch() to bypass CORS restrictions.
* It currently doesn't support streaming until this is implemented
* https://forum.obsidian.md/t/support-streaming-the-request-and-requesturl-response-body/87381 */
@ -996,12 +719,12 @@ export async function checkLatestVersion(): Promise<{
export function isOSeriesModel(model: BaseChatModel | string): boolean {
if (typeof model === "string") {
return model.startsWith("o1") || model.startsWith("o3");
return model.startsWith("o1") || model.startsWith("o3") || model.startsWith("o4");
}
// For BaseChatModel instances
const modelName = (model as any).modelName || (model as any).model || "";
return modelName.startsWith("o1") || modelName.startsWith("o3");
return modelName.startsWith("o1") || modelName.startsWith("o3") || modelName.startsWith("o4");
}
export function getMessageRole(