From ff3636637f81ce9096ae0c3ae7e6160877485437 Mon Sep 17 00:00:00 2001 From: Emt-lin <41323133+Emt-lin@users.noreply.github.com> Date: Tue, 13 Jan 2026 09:54:54 +0800 Subject: [PATCH] feat: Add Web Viewer bridge for referencing open web tabs in chat (#2096) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add Web Viewer bridge for referencing open web tabs in chat. * refactor: Simplify markdown image extraction parser。 * fix: Prevent web selection from auto-reappearing after removal or new chat. * fix: Suppress active web tab when web selection exists. * feat: Add YouTube transcript extraction for Web Viewer tabs. * feat: Built-in slash prompts for web clippers. * feat: Improve web selection tracking and simplify context settings - Merge context settings: combine note/web active content and selection settings - Add auto-clear for web selection badge when deselected on page - Make note and web selections mutually exclusive - Show website favicon in web selection badge - Add migration logic and tests for settings --- package-lock.json | 26 +- package.json | 4 +- .../chainRunner/CopilotPlusChainRunner.ts | 95 ++- .../chainRunner/utils/imageExtraction.test.ts | 390 ++++++++++- .../chainRunner/utils/imageExtraction.ts | 318 +++++++++ .../chainRunner/utils/xmlParsing.ts | 20 + src/commands/constants.ts | 142 ++++ src/commands/customCommandUtils.test.ts | 21 +- src/commands/customCommandUtils.ts | 12 +- src/commands/index.ts | 59 +- src/components/Chat.tsx | 58 +- .../AtMentionTypeahead.test.tsx | 226 ++++++ .../chat-components/AtMentionTypeahead.tsx | 32 +- .../chat-components/ChatContextMenu.tsx | 75 +- .../chat-components/ChatInput.test.ts | 39 ++ src/components/chat-components/ChatInput.tsx | 162 ++++- .../chat-components/ChatSingleMessage.tsx | 54 +- .../chat-components/ContextBadges.tsx | 189 ++++- .../chat-components/ContextControl.tsx | 11 +- .../chat-components/InlineMessageEditor.tsx | 6 + .../chat-components/LexicalEditor.tsx | 23 +- .../chat-components/TypeaheadMenuContent.tsx | 17 +- .../hooks/useActiveWebTabState.ts | 55 ++ .../hooks/useAtMentionCategories.tsx | 23 +- .../hooks/useAtMentionSearch.ts | 95 ++- .../chat-components/hooks/useOpenWebTabs.ts | 182 +++++ .../hooks/useTypeaheadPlugin.ts | 25 +- .../pills/ActiveWebTabPillNode.tsx | 225 ++++++ .../chat-components/pills/WebTabPillNode.tsx | 244 +++++++ .../plugins/GenericPillSyncPlugin.tsx | 43 +- .../plugins/WebTabPillSyncPlugin.tsx | 106 +++ .../chat-components/utils/lexicalTextUtils.ts | 16 +- src/constants.ts | 12 +- src/contextProcessor.selectedText.test.ts | 174 +++++ src/contextProcessor.ts | 423 +++++++++++- src/core/ChatManager.test.ts | 541 ++++++++++++++- src/core/ChatManager.ts | 135 +++- src/core/ChatPersistenceManager.ts | 19 +- src/core/ContextManager.ts | 14 +- src/core/MessageLifecycle.xmltags.test.ts | 136 +++- src/hooks/useChatManager.ts | 11 +- src/main.ts | 110 ++- .../webViewerService/webViewerService.ts | 383 ++++++++++ .../webViewerServiceActions.ts | 510 ++++++++++++++ .../webViewerServiceHelpers.ts | 307 +++++++++ .../webViewerServiceSelection.ts | 403 +++++++++++ .../webViewerServiceSingleton.ts | 109 +++ .../webViewerService/webViewerServiceState.ts | 652 ++++++++++++++++++ .../webViewerService/webViewerServiceTypes.ts | 242 +++++++ src/settings/model.test.ts | 74 ++ src/settings/model.ts | 28 +- src/settings/v2/components/BasicSettings.tsx | 16 +- src/state/ChatUIState.ts | 2 + src/types/message.ts | 60 +- src/utils.ts | 15 + src/utils/urlNormalization.test.ts | 307 +++++++++ src/utils/urlNormalization.ts | 172 +++++ 57 files changed, 7650 insertions(+), 198 deletions(-) create mode 100644 src/LLMProviders/chainRunner/utils/imageExtraction.ts create mode 100644 src/components/chat-components/AtMentionTypeahead.test.tsx create mode 100644 src/components/chat-components/hooks/useActiveWebTabState.ts create mode 100644 src/components/chat-components/hooks/useOpenWebTabs.ts create mode 100644 src/components/chat-components/pills/ActiveWebTabPillNode.tsx create mode 100644 src/components/chat-components/pills/WebTabPillNode.tsx create mode 100644 src/components/chat-components/plugins/WebTabPillSyncPlugin.tsx create mode 100644 src/contextProcessor.selectedText.test.ts create mode 100644 src/services/webViewerService/webViewerService.ts create mode 100644 src/services/webViewerService/webViewerServiceActions.ts create mode 100644 src/services/webViewerService/webViewerServiceHelpers.ts create mode 100644 src/services/webViewerService/webViewerServiceSelection.ts create mode 100644 src/services/webViewerService/webViewerServiceSingleton.ts create mode 100644 src/services/webViewerService/webViewerServiceState.ts create mode 100644 src/services/webViewerService/webViewerServiceTypes.ts create mode 100644 src/utils/urlNormalization.test.ts create mode 100644 src/utils/urlNormalization.ts diff --git a/package-lock.json b/package-lock.json index 949a3ed0..e2c5f5f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -81,7 +81,8 @@ "sse": "github:mpetazzoni/sse.js", "tailwind-merge": "^2.5.5", "tailwindcss-animate": "^1.0.7", - "trie-search": "^2.2.0" + "trie-search": "^2.2.0", + "turndown": "^7.2.2" }, "devDependencies": { "@langchain/ollama": "^1.0.0", @@ -101,6 +102,7 @@ "@types/react": "^18.0.33", "@types/react-dom": "^18.0.11", "@types/react-syntax-highlighter": "^15.5.6", + "@types/turndown": "^5.0.6", "@typescript-eslint/eslint-plugin": "^8.19.1", "@typescript-eslint/parser": "^8.19.1", "builtin-modules": "3.3.0", @@ -5037,6 +5039,12 @@ "zod-to-json-schema": "^3.24.1" } }, + "node_modules/@mixmark-io/domino": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz", + "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==", + "license": "BSD-2-Clause" + }, "node_modules/@next/env": { "version": "15.5.4", "resolved": "https://registry.npmjs.org/@next/env/-/env-15.5.4.tgz", @@ -10190,6 +10198,13 @@ "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.2.tgz", "integrity": "sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw==" }, + "node_modules/@types/turndown": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@types/turndown/-/turndown-5.0.6.tgz", + "integrity": "sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", @@ -22504,6 +22519,15 @@ "node": ">=0.6.x" } }, + "node_modules/turndown": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.2.tgz", + "integrity": "sha512-1F7db8BiExOKxjSMU2b7if62D/XOyQyZbPKq/nUwopfgnHlqXHqQ0lvfUTeUIr1lZJzOPFn43dODyMSIfvWRKQ==", + "license": "MIT", + "dependencies": { + "@mixmark-io/domino": "^2.2.0" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index 28dc61b0..dcdf79a3 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "@types/react": "^18.0.33", "@types/react-dom": "^18.0.11", "@types/react-syntax-highlighter": "^15.5.6", + "@types/turndown": "^5.0.6", "@typescript-eslint/eslint-plugin": "^8.19.1", "@typescript-eslint/parser": "^8.19.1", "builtin-modules": "3.3.0", @@ -149,6 +150,7 @@ "sse": "github:mpetazzoni/sse.js", "tailwind-merge": "^2.5.5", "tailwindcss-animate": "^1.0.7", - "trie-search": "^2.2.0" + "trie-search": "^2.2.0", + "turndown": "^7.2.2" } } diff --git a/src/LLMProviders/chainRunner/CopilotPlusChainRunner.ts b/src/LLMProviders/chainRunner/CopilotPlusChainRunner.ts index 23578bbc..3a49adc8 100644 --- a/src/LLMProviders/chainRunner/CopilotPlusChainRunner.ts +++ b/src/LLMProviders/chainRunner/CopilotPlusChainRunner.ts @@ -48,11 +48,12 @@ import { renderCiCMessage, wrapLocalSearchPayload, } from "./utils/cicPromptUtils"; +import { extractMarkdownImagePaths } from "./utils/imageExtraction"; import { ThinkBlockStreamer } from "./utils/ThinkBlockStreamer"; import { deduplicateSources } from "./utils/toolExecution"; import { recordPromptPayload } from "./utils/promptPayloadRecorder"; import { ModelAdapterFactory, joinPromptSections } from "./utils/modelAdapter"; -import { parseXMLToolCalls } from "./utils/xmlParsing"; +import { parseXMLToolCalls, unescapeXml } from "./utils/xmlParsing"; import { extractParametersFromZod, SimpleTool } from "@/tools/SimpleTool"; import ProjectManager from "@/LLMProviders/projectManager"; import { isProjectMode } from "@/aiParams"; @@ -328,11 +329,47 @@ OUTPUT ONLY XML - NO OTHER TEXT.`; return processedImages; } + /** + * Extracts images from a context block (active_note or active_web_tab) in L3 content. + * @param l3Text - The full L3_TURN layer text + * @param source - Configuration for the context source type + * @returns Array of image URLs/paths found in the block + */ + private async extractImagesFromContextBlock( + l3Text: string, + source: { tagName: string; identifierTag: string; displayName: string; useForResolution: boolean } + ): Promise { + // Match the context block + const blockRegex = new RegExp(`<${source.tagName}>([\\s\\S]*?)<\\/${source.tagName}>`); + const blockMatch = blockRegex.exec(l3Text); + if (!blockMatch) return []; + + const block = blockMatch[1]; + + // Extract content - unescape XML entities for correct URL parsing + const contentRegex = /([\s\S]*?)<\/content>/; + const contentMatch = contentRegex.exec(block); + const content = contentMatch ? unescapeXml(contentMatch[1]) : ""; + if (!content) return []; + + // Extract identifier (path or url) for logging and optional resolution + const identifierRegex = new RegExp(`<${source.identifierTag}>(.*?)<\\/${source.identifierTag}>`); + const identifierMatch = identifierRegex.exec(block); + const identifier = identifierMatch ? identifierMatch[1] : undefined; + + logInfo( + `[CopilotPlus] Extracting images from ${source.displayName}:`, + identifier || `no ${source.identifierTag}` + ); + + // Use identifier for vault path resolution only if configured + const sourcePath = source.useForResolution ? identifier : undefined; + return this.extractEmbeddedImages(content, sourcePath); + } + private async extractEmbeddedImages(content: string, sourcePath?: string): Promise { - // Match both wiki-style ![[image.ext]] and standard markdown ![alt](image.ext) + // Match wiki-style ![[image.ext]] const wikiImageRegex = /!\[\[(.*?\.(png|jpg|jpeg|gif|webp|bmp|svg))\]\]/g; - // Updated regex to handle URLs with or without file extensions - const markdownImageRegex = /!\[.*?\]\(([^)]+)\)/g; const resolvedImages: string[] = []; @@ -359,11 +396,9 @@ OUTPUT ONLY XML - NO OTHER TEXT.`; } } - // Process standard markdown images - const mdMatches = [...content.matchAll(markdownImageRegex)]; - for (const match of mdMatches) { - const imagePath = match[1].trim(); - + // Process standard markdown images using robust character-scanning parser + const mdImagePaths = extractMarkdownImagePaths(content); + for (const imagePath of mdImagePaths) { // Skip empty paths if (!imagePath) continue; @@ -417,7 +452,7 @@ OUTPUT ONLY XML - NO OTHER TEXT.`; // Process embedded images only if setting is enabled if (settings.passMarkdownImages) { - // IMPORTANT: Only extract images from the active note + // IMPORTANT: Only extract images from the active context (note or web tab) // Never from L2 (promoted notes) or attached context notes const envelope = userMessage.contextEnvelope; @@ -427,35 +462,23 @@ OUTPUT ONLY XML - NO OTHER TEXT.`; ); } - // Extract ONLY from block in L3 + // Extract from active context blocks in L3 (active_note or active_web_tab) const l3Turn = envelope.layers.find((l) => l.id === "L3_TURN"); if (l3Turn) { - // Find block - const activeNoteRegex = /([\s\S]*?)<\/active_note>/; - const activeNoteMatch = activeNoteRegex.exec(l3Turn.text); + // Define context sources to extract images from + // - tagName: XML tag name in L3 content + // - identifierTag: tag containing source identifier (path/url) for logging + // - displayName: human-readable name for logs + // - useForResolution: whether to use identifier for vault path resolution + const contextSources = [ + { tagName: "active_note", identifierTag: "path", displayName: "active note", useForResolution: true }, + { tagName: "active_web_tab", identifierTag: "url", displayName: "active web tab", useForResolution: false }, + ]; - if (activeNoteMatch) { - const activeNoteBlock = activeNoteMatch[1]; - - // Extract path from tag - const pathRegex = /(.*?)<\/path>/; - const pathMatch = pathRegex.exec(activeNoteBlock); - const sourcePath = pathMatch ? pathMatch[1] : undefined; - - // Extract content from tag - const contentRegex = /([\s\S]*?)<\/content>/; - const contentMatch = contentRegex.exec(activeNoteBlock); - const activeNoteContent = contentMatch ? contentMatch[1] : ""; - - if (activeNoteContent) { - logInfo( - "[CopilotPlus] Extracting images from active note only:", - sourcePath || "no source path" - ); - const embeddedImages = await this.extractEmbeddedImages(activeNoteContent, sourcePath); - if (embeddedImages.length > 0) { - imageSources.push({ urls: embeddedImages, type: "embedded" }); - } + for (const source of contextSources) { + const images = await this.extractImagesFromContextBlock(l3Turn.text, source); + if (images.length > 0) { + imageSources.push({ urls: images, type: "embedded" }); } } } diff --git a/src/LLMProviders/chainRunner/utils/imageExtraction.test.ts b/src/LLMProviders/chainRunner/utils/imageExtraction.test.ts index 3fb25ab1..acf39ff8 100644 --- a/src/LLMProviders/chainRunner/utils/imageExtraction.test.ts +++ b/src/LLMProviders/chainRunner/utils/imageExtraction.test.ts @@ -1,3 +1,5 @@ +import { extractMarkdownImagePaths } from "./imageExtraction"; + // Test for the image extraction logic describe("Image extraction from content", () => { // Mock the global app object @@ -9,18 +11,14 @@ describe("Image extraction from content", () => { (global as any).app = mockApp; - // Mock logger (removed since we're no longer logging warnings) - beforeEach(() => { jest.clearAllMocks(); }); - // Helper function that replicates the extractEmbeddedImages logic + // Helper function that replicates the extractEmbeddedImages logic from CopilotPlusChainRunner async function extractEmbeddedImages(content: string, sourcePath?: string): Promise { - // Match both wiki-style ![[image.ext]] and standard markdown ![alt](image.ext) + // Match wiki-style ![[image.ext]] const wikiImageRegex = /!\[\[(.*?\.(png|jpg|jpeg|gif|webp|bmp|svg))\]\]/g; - // Updated regex to handle URLs with or without file extensions - const markdownImageRegex = /!\[.*?\]\(([^)]+)\)/g; const resolvedImages: string[] = []; @@ -46,11 +44,9 @@ describe("Image extraction from content", () => { } } - // Process standard markdown images - const mdMatches = [...content.matchAll(markdownImageRegex)]; - for (const match of mdMatches) { - const imagePath = match[1].trim(); - + // Process standard markdown images using robust character-scanning parser + const mdImagePaths = extractMarkdownImagePaths(content); + for (const imagePath of mdImagePaths) { // Skip empty paths if (!imagePath) continue; @@ -339,4 +335,376 @@ describe("Image extraction from content", () => { expect(result).toEqual(["attachments/image (1).png", "attachments/file-name_2.jpg"]); }); }); + + describe("Angle bracket syntax ![alt]()", () => { + it("should extract URL with parentheses using angle brackets", async () => { + const content = "Wikipedia link ![Mars]()"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["https://en.wikipedia.org/wiki/Mars_(planet).jpg"]); + }); + + it("should extract URL with spaces using angle brackets", async () => { + const content = "Spaced path ![alt]()"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["path/my image file.png"]); + }); + + it("should extract multiple angle bracket URLs", async () => { + const content = ` + First ![img1]() + Second ![img2]() + `; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual([ + "https://example.com/image(1).png", + "https://example.com/image (2).jpg", + ]); + }); + }); + + describe("Image with title syntax ![alt](url \"title\")", () => { + it("should extract URL with double-quoted title", async () => { + const content = 'Image with title ![alt](https://example.com/img.png "This is the title")'; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["https://example.com/img.png"]); + }); + + it("should extract URL with single-quoted title", async () => { + const content = "Image with title ![alt](https://example.com/img.png 'Single quoted title')"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["https://example.com/img.png"]); + }); + + it("should extract URL with parentheses title", async () => { + const content = "Image with title ![alt](https://example.com/img.png (Parentheses title))"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["https://example.com/img.png"]); + }); + + it("should extract angle bracket URL with title", async () => { + const content = + 'Angle bracket with title ![alt]( "Title here")'; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["https://example.com/image(1).png"]); + }); + + it("should handle local path with title", async () => { + const content = 'Local image ![alt](images/photo.jpg "My photo")'; + const sourcePath = "notes/test.md"; + + mockApp.metadataCache.getFirstLinkpathDest.mockReturnValueOnce({ + path: "images/photo.jpg", + }); + + const result = await extractEmbeddedImages(content, sourcePath); + + expect(result).toEqual(["images/photo.jpg"]); + }); + }); + + describe("Mixed enhanced syntaxes", () => { + it("should handle all syntax variations together", async () => { + const content = ` + Standard: ![](https://example.com/standard.png) + With title: ![alt](https://example.com/titled.png "Title") + Angle bracket: ![alt]() + Angle with title: ![alt]( "Both features") + Wiki style: ![[local.png]] + `; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual([ + "local.png", + "https://example.com/standard.png", + "https://example.com/titled.png", + "https://example.com/path (special).png", + "https://example.com/both(1).png", + ]); + }); + + it("should not be confused by malformed syntax", async () => { + const content = ` + Valid: ![](https://example.com/valid.png) + Missing close bracket: ![alt](https://example.com/missing.png + Empty angle brackets: ![alt](<>) + `; + + const result = await extractEmbeddedImages(content); + + // Only the valid one should be extracted + expect(result).toEqual(["https://example.com/valid.png"]); + }); + }); + + // NEW TESTS: Paths with parentheses (without angle brackets) + describe("Paths with parentheses (balanced parentheses support)", () => { + it("should extract path with single pair of parentheses", async () => { + const content = "Image ![](foo(bar).png)"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["foo(bar).png"]); + }); + + it("should extract path with multiple pairs of parentheses", async () => { + const content = "Image ![](image(1)(2).png)"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["image(1)(2).png"]); + }); + + it("should extract Wikipedia-style URL with parentheses", async () => { + const content = "![Mars](https://en.wikipedia.org/wiki/Mars_(planet).jpg)"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["https://en.wikipedia.org/wiki/Mars_(planet).jpg"]); + }); + + it("should extract local path with parentheses", async () => { + const content = "![](images/screenshot (1).png)"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["images/screenshot (1).png"]); + }); + }); + + // NEW TESTS: Paths with spaces (without angle brackets) + describe("Paths with spaces (space support)", () => { + it("should extract path with spaces", async () => { + const content = "Image ![](foo bar.png)"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["foo bar.png"]); + }); + + it("should extract path with multiple spaces", async () => { + const content = "Image ![](path/my image file.png)"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["path/my image file.png"]); + }); + + it("should extract path with spaces and special chars", async () => { + const content = "Image ![](my folder/image - copy (1).png)"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["my folder/image - copy (1).png"]); + }); + }); + + // NEW TESTS: Multiple images on same line + describe("Multiple images on same line", () => { + it("should extract multiple images on same line", async () => { + const content = "Images: ![](a.png) ![](b.png) ![](c.png)"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["a.png", "b.png", "c.png"]); + }); + + it("should extract multiple images with text between", async () => { + const content = "First ![](a.png) then ![](b.png) and finally ![](c.png)"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["a.png", "b.png", "c.png"]); + }); + + it("should handle image followed by parenthetical text", async () => { + const content = "Image ![](a.png) (this is a note) more text"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["a.png"]); + }); + }); + + // NEW TESTS: Alt text with special characters + describe("Alt text with special characters", () => { + it("should handle alt text with parentheses", async () => { + const content = "![a (b)](img.png)"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["img.png"]); + }); + + it("should handle alt text with brackets", async () => { + const content = "![alt [text]](img.png)"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["img.png"]); + }); + + it("should handle complex alt text", async () => { + const content = "![Figure 1 (a): Test [ref]](diagram.png)"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["diagram.png"]); + }); + }); + + // NEW TESTS: Edge cases for the new parser + describe("Parser edge cases", () => { + it("should handle escaped characters in path", async () => { + const content = "![](path\\(1\\).png)"; + + const result = await extractEmbeddedImages(content); + + // The parser should handle escaped parens + expect(result.length).toBe(1); + }); + + it("should handle leading/trailing whitespace in path", async () => { + const content = "![]( image.png )"; + + const result = await extractEmbeddedImages(content); + + expect(result).toEqual(["image.png"]); + }); + + it("should handle newline between ] and (", async () => { + const content = "![alt]\n(image.png)"; + + const result = await extractEmbeddedImages(content); + + // CommonMark allows whitespace between ] and ( + expect(result).toEqual(["image.png"]); + }); + }); +}); + +// Direct tests for the extractMarkdownImagePaths function +describe("extractMarkdownImagePaths", () => { + it("should extract simple paths", () => { + const result = extractMarkdownImagePaths("![](simple.png)"); + expect(result).toEqual(["simple.png"]); + }); + + it("should extract paths with parentheses", () => { + const result = extractMarkdownImagePaths("![](foo(bar).png)"); + expect(result).toEqual(["foo(bar).png"]); + }); + + it("should extract paths with spaces", () => { + const result = extractMarkdownImagePaths("![](foo bar.png)"); + expect(result).toEqual(["foo bar.png"]); + }); + + it("should extract multiple images", () => { + const result = extractMarkdownImagePaths("![](a.png) ![](b.png)"); + expect(result).toEqual(["a.png", "b.png"]); + }); + + it("should handle angle bracket syntax", () => { + const result = extractMarkdownImagePaths("![alt]()"); + expect(result).toEqual(["path with spaces.png"]); + }); + + it("should handle title syntax", () => { + const result = extractMarkdownImagePaths('![alt](image.png "title")'); + expect(result).toEqual(["image.png"]); + }); + + it("should return empty array for no images", () => { + const result = extractMarkdownImagePaths("no images here"); + expect(result).toEqual([]); + }); + + it("should skip empty paths", () => { + const result = extractMarkdownImagePaths("![]()"); + expect(result).toEqual([]); + }); + + it("should trim inside angle destinations (bug fix)", () => { + const result = extractMarkdownImagePaths("![](< image.png >)"); + expect(result).toEqual(["image.png"]); + }); + + // Combination edge cases: parentheses destination + title + it("should handle parentheses in path with double-quoted title", () => { + const result = extractMarkdownImagePaths('![](foo(bar).png "title")'); + expect(result).toEqual(["foo(bar).png"]); + }); + + it("should handle parentheses in path with parentheses title", () => { + const result = extractMarkdownImagePaths("![](foo(bar).png (title))"); + expect(result).toEqual(["foo(bar).png"]); + }); + + // Combination edge cases: spaces destination + title + it("should handle spaces in path with double-quoted title", () => { + const result = extractMarkdownImagePaths('![](foo bar.png "title")'); + expect(result).toEqual(["foo bar.png"]); + }); + + it("should handle spaces in path with parentheses title", () => { + const result = extractMarkdownImagePaths("![](foo bar.png (title))"); + expect(result).toEqual(["foo bar.png"]); + }); + + // Ambiguous case: path ending with parentheses preceded by space + // Current behavior: treats (1) as title, returns "image" + // This matches CommonMark behavior where space + (...) is a title + it("should treat space + parentheses at end as title (ambiguous case)", () => { + const result = extractMarkdownImagePaths("![](image (1))"); + expect(result).toEqual(["image"]); + }); + + // To preserve parentheses in filename with spaces, use angle brackets + it("should preserve parentheses in filename when using angle brackets", () => { + const result = extractMarkdownImagePaths("![]()"); + expect(result).toEqual(["image (1).png"]); + }); + + // Edge cases: original regex would match but current implementation handles differently + // These tests lock down the improved behavior vs the old regex /!\[.*?\]\(([^)]+)\)/g + + // Unclosed parenthesis in destination - old regex would capture "foo(bar", current skips + it("should skip unclosed parenthesis in destination", () => { + const result = extractMarkdownImagePaths("![](foo(bar)"); + expect(result).toEqual([]); + }); + + // Missing closing angle bracket - old regex would capture " { + const result = extractMarkdownImagePaths("![]( { + const result = extractMarkdownImagePaths("![a [b](inner.png)](outer.png)"); + expect(result).toEqual(["outer.png"]); + }); + + // Balanced parentheses - old regex would truncate at first ), current handles correctly + it("should handle balanced parentheses without truncation", () => { + const result = extractMarkdownImagePaths("![](foo(bar))"); + expect(result).toEqual(["foo(bar)"]); + }); }); diff --git a/src/LLMProviders/chainRunner/utils/imageExtraction.ts b/src/LLMProviders/chainRunner/utils/imageExtraction.ts new file mode 100644 index 00000000..fe3d62f1 --- /dev/null +++ b/src/LLMProviders/chainRunner/utils/imageExtraction.ts @@ -0,0 +1,318 @@ +/** + * Extracts Markdown image destinations (paths/URLs) from inline image syntax: + * `![alt](destination "title")`. + * + * Supported: + * - Non-angle destinations can include spaces: `![](foo bar.png)` (loose compatibility) + * - Balanced parentheses in destinations: `![](foo(bar).png)` + * - Angle destinations: `![]()` (content inside `<...>` is trimmed) + * - Optional titles (ignored): `"..."`, `'...'`, `( ... )` + * + * Note: Obsidian wiki embeds (`![[image.png]]`) are handled separately by the caller. + */ +export function extractMarkdownImagePaths(markdown: string): string[] { + const results: string[] = []; + let searchIndex = 0; + + while (searchIndex < markdown.length) { + const startIndex = markdown.indexOf("![", searchIndex); + if (startIndex === -1) { + break; + } + + const closeBracketIndex = findClosingBracketIndex(markdown, startIndex + 2); + if (closeBracketIndex === null) { + searchIndex = startIndex + 2; + continue; + } + + // Skip whitespace between ] and ( + let i = closeBracketIndex + 1; + while (i < markdown.length && isWhitespaceChar(markdown[i])) { + i++; + } + + if (markdown[i] !== "(") { + searchIndex = i; + continue; + } + + const closeParenIndex = findClosingParenIndexForImage(markdown, i); + if (closeParenIndex === null) { + searchIndex = i + 1; + continue; + } + + const destination = parseImageDestination(markdown.slice(i + 1, closeParenIndex)); + if (destination) { + results.push(destination); + } + + searchIndex = closeParenIndex + 1; + } + + return results; +} + +/** + * Parses the inside of `(...)` and returns only the destination. + * + * Rules: + * - Angle destinations `<...>`: content is trimmed (fixes `![](< image.png >)` → `image.png`) + * - Non-angle destinations: supports spaces (loose compatibility), strips optional title from end + */ +function parseImageDestination(innerRaw: string): string | null { + const inner = innerRaw.trim(); + if (inner.length === 0) { + return null; + } + + // Handle angle bracket destinations: `<...>` + if (inner.startsWith("<")) { + const closeAngleIndex = findClosingAngleBracket(inner, 0); + if (closeAngleIndex === null) { + return null; + } + + // Fix: trim inside `< ... >`, so `![](< image.png >)` returns `image.png` + const destination = inner.slice(1, closeAngleIndex).trim(); + return destination.length > 0 ? destination : null; + } + + // Non-angle destination: strip optional title from end, keep spaces (loose compatibility) + const destination = stripOptionalTitleFromEnd(inner).trim(); + return destination.length > 0 ? destination : null; +} + +/** + * Strips an optional trailing title from a non-angle destination string. + * Title forms (ignored): `"..."`, `'...'`, `( ... )`, each preceded by whitespace. + */ +function stripOptionalTitleFromEnd(value: string): string { + const s = value.trimEnd(); + if (s.length === 0) { + return ""; + } + + const lastChar = s[s.length - 1]; + + // Check for quoted title: `"..."` or `'...'` + if (lastChar === '"' || lastChar === "'") { + const quote = lastChar; + const openIndex = findMatchingUnescapedQuoteFromEnd(s, quote, s.length - 1); + if (openIndex !== null) { + const before = s.slice(0, openIndex); + const hasSeparator = before.length > 0 && isWhitespaceChar(before[before.length - 1]); + if (hasSeparator) { + return before.trimEnd(); + } + } + } + + // Check for parentheses title: `( ... )` + if (lastChar === ")") { + const openIndex = findMatchingOpeningParenForEnd(s); + if (openIndex !== null) { + const before = s.slice(0, openIndex); + const hasSeparator = before.length > 0 && isWhitespaceChar(before[before.length - 1]); + if (hasSeparator) { + return before.trimEnd(); + } + } + } + + return s; +} + +/** + * Finds the closing `]` for image/link text, supporting nested brackets and backslash escapes. + */ +function findClosingBracketIndex(source: string, startIndex: number): number | null { + let i = startIndex; + let nestedDepth = 0; + + while (i < source.length) { + const ch = source[i]; + + if (ch === "\\") { + i += 2; + continue; + } + + if (ch === "[") { + nestedDepth++; + i++; + continue; + } + + if (ch === "]") { + if (nestedDepth === 0) { + return i; + } + nestedDepth--; + i++; + continue; + } + + i++; + } + + return null; +} + +/** + * Finds the correct closing `)` for the image parens starting at `openParenIndex`. + * Counts nested parentheses, but ignores any parentheses inside an initial `<...>` destination. + */ +function findClosingParenIndexForImage(source: string, openParenIndex: number): number | null { + let i = openParenIndex + 1; + let parenDepth = 1; + + let beforeDestination = true; + let inAngleDestination = false; + + while (i < source.length) { + const ch = source[i]; + + if (inAngleDestination) { + if (ch === "\\") { + i += 2; + continue; + } + if (ch === ">") { + inAngleDestination = false; + beforeDestination = false; + i++; + continue; + } + i++; + continue; + } + + if (beforeDestination) { + if (isWhitespaceChar(ch)) { + i++; + continue; + } + if (ch === "<") { + inAngleDestination = true; + i++; + continue; + } + beforeDestination = false; + } + + if (ch === "\\") { + i += 2; + continue; + } + + if (ch === "(") { + parenDepth++; + i++; + continue; + } + + if (ch === ")") { + parenDepth--; + if (parenDepth === 0) { + return i; + } + i++; + continue; + } + + i++; + } + + return null; +} + +/** + * Finds the closing `>` for an angle destination starting at `openIndex` (which points to `<`). + */ +function findClosingAngleBracket(source: string, openIndex: number): number | null { + for (let i = openIndex + 1; i < source.length; i++) { + const ch = source[i]; + if (ch === "\\") { + i++; + continue; + } + if (ch === ">") { + return i; + } + } + return null; +} + +/** + * Finds the opening quote matching a closing quote at `closeIndex`, scanning backward. + */ +function findMatchingUnescapedQuoteFromEnd( + source: string, + quote: '"' | "'", + closeIndex: number +): number | null { + for (let i = closeIndex - 1; i >= 0; i--) { + if (source[i] === quote && !isCharEscaped(source, i)) { + return i; + } + } + return null; +} + +/** + * Finds the matching opening `(` for a string ending in `)`, supporting nested parentheses and escapes. + */ +function findMatchingOpeningParenForEnd(source: string): number | null { + if (source.length === 0 || source[source.length - 1] !== ")") { + return null; + } + + let depth = 0; + + for (let i = source.length - 1; i >= 0; i--) { + const ch = source[i]; + + if ((ch === ")" || ch === "(") && isCharEscaped(source, i)) { + continue; + } + + if (ch === ")") { + depth++; + continue; + } + + if (ch === "(") { + depth--; + if (depth === 0) { + return i; + } + } + } + + return null; +} + +/** + * Checks whether `source[index]` is escaped by an odd number of consecutive backslashes. + */ +function isCharEscaped(source: string, index: number): boolean { + let backslashCount = 0; + + for (let i = index - 1; i >= 0; i--) { + if (source[i] !== "\\") { + break; + } + backslashCount++; + } + + return backslashCount % 2 === 1; +} + +/** + * Returns true if a character is treated as whitespace for parsing. + */ +function isWhitespaceChar(ch: string): boolean { + return ch === " " || ch === "\t" || ch === "\n" || ch === "\r"; +} diff --git a/src/LLMProviders/chainRunner/utils/xmlParsing.ts b/src/LLMProviders/chainRunner/utils/xmlParsing.ts index 75278b30..cabbe0d2 100644 --- a/src/LLMProviders/chainRunner/utils/xmlParsing.ts +++ b/src/LLMProviders/chainRunner/utils/xmlParsing.ts @@ -18,6 +18,26 @@ export function escapeXml(str: string): string { .replace(/'/g, "'"); } +/** + * Unescapes XML entities back to their original characters. + * Used when extracting content that was previously escaped (e.g., image URLs). + * @param str - The XML-escaped string to unescape + * @returns The unescaped string with original characters restored + */ +export function unescapeXml(str: string): string { + if (typeof str !== "string") { + return ""; + } + + // Reason: & must be unescaped last to prevent double-unescaping + return str + .replace(/'/g, "'") + .replace(/"/g, '"') + .replace(/>/g, ">") + .replace(/</g, "<") + .replace(/&/g, "&"); +} + /** * Escapes special XML characters for use in XML attributes * @param str - The string to escape for attribute use diff --git a/src/commands/constants.ts b/src/commands/constants.ts index 4fc882ac..756f82d6 100644 --- a/src/commands/constants.ts +++ b/src/commands/constants.ts @@ -168,4 +168,146 @@ export const DEFAULT_COMMANDS: CustomCommand[] = [ modelKey: "", lastUsedMs: 0, }, + { + title: "Clip YouTube Transcript", + content: ` +Based on the YouTube video information and transcript provided in the context, generate a complete Obsidian note in the following format. + +IMPORTANT: If no YouTube video context is found, remind the user to: +1. Open a YouTube video in Web Viewer (or use @ to select a YouTube web tab) +2. Then use this command again + +Generate the note with this exact structure: + +--- +title: "