From 1d46ab90b0648bb2a9ca38e1311ce9a0bf07bd07 Mon Sep 17 00:00:00 2001 From: Logan Yang Date: Sun, 20 Apr 2025 12:14:56 -0700 Subject: [PATCH] Fix image in note logic (#1457) * Refactor image processing in note context * Add passMarkdownImages setting --- src/LLMProviders/chainRunner.ts | 164 ++++--- src/constants.ts | 3 +- src/imageProcessing/imageProcessor.ts | 400 ++++++++++++++++++ src/mentions/Mention.ts | 3 +- src/settings/model.ts | 6 + .../v2/components/AdvancedSettings.tsx | 12 +- src/utils.ts | 279 +----------- 7 files changed, 501 insertions(+), 366 deletions(-) create mode 100644 src/imageProcessing/imageProcessor.ts diff --git a/src/LLMProviders/chainRunner.ts b/src/LLMProviders/chainRunner.ts index 330fa1dd..0cb1426c 100644 --- a/src/LLMProviders/chainRunner.ts +++ b/src/LLMProviders/chainRunner.ts @@ -7,9 +7,15 @@ import { MAX_CHARS_FOR_LOCAL_SEARCH_CONTEXT, ModelCapability, } from "@/constants"; +import { + ImageBatchProcessor, + ImageContent, + ImageProcessingResult, + MessageContent, +} from "@/imageProcessing/imageProcessor"; import { BrevilabsClient } from "@/LLMProviders/brevilabsClient"; -import { logError } from "@/logger"; -import { getSystemPrompt } from "@/settings/model"; +import { logInfo } from "@/logger"; +import { getSettings, getSystemPrompt } from "@/settings/model"; import { ChatMessage } from "@/sharedState"; import { ToolManager } from "@/tools/toolManager"; import { @@ -20,9 +26,6 @@ import { formatDateTime, getApiErrorMessage, getMessageRole, - ImageContent, - ImageProcessor, - MessageContent, } from "@/utils"; import { BaseChatModel } from "@langchain/core/language_models/chat_models"; import { Notice } from "obsidian"; @@ -307,58 +310,26 @@ class CopilotPlusChainRunner extends BaseChainRunner { return hasYoutubeCommand && youtubeUrl !== null && words.length === 1; } - private async processImageUrls(urls: string[]): Promise { - try { - const imageUrls = await Promise.all( - urls.map(async (url) => { - if (await ImageProcessor.isImageUrl(url, this.chainManager.app.vault)) { - const imageContent = await ImageProcessor.convertToBase64( - url, - this.chainManager.app.vault - ); - if (!imageContent) { - logError(`Failed to process image: ${url}`); - return null; - } - return imageContent; - } - return null; - }) - ); - - // Filter out null values and return valid image URLs - const validImages = imageUrls.filter((item): item is ImageContent => item !== null); - return validImages; - } catch (error) { - logError("Error processing image URLs:", error); - return []; - } + private async processImageUrls(urls: string[]): Promise { + const failedImages: string[] = []; + const processedImages = await ImageBatchProcessor.processUrlBatch( + urls, + failedImages, + this.chainManager.app.vault + ); + ImageBatchProcessor.showFailedImagesNotice(failedImages); + return processedImages; } - private async processExistingImages(content: MessageContent[]): Promise { - try { - const imageContent = await Promise.all( - content - .filter( - (item): item is ImageContent => item.type === "image_url" && !!item.image_url?.url - ) - .map(async (item) => { - const processedContent = await ImageProcessor.convertToBase64( - item.image_url.url, - this.chainManager.app.vault - ); - if (!processedContent) { - logError(`Failed to process existing image: ${item.image_url.url}`); - return null; - } - return processedContent; - }) - ); - return imageContent.filter((item): item is ImageContent => item !== null); - } catch (error) { - logError("Error processing images:", error); - return []; - } + private async processChatInputImages(content: MessageContent[]): Promise { + const failedImages: string[] = []; + const processedImages = await ImageBatchProcessor.processChatImageBatch( + content, + failedImages, + this.chainManager.app.vault + ); + ImageBatchProcessor.showFailedImagesNotice(failedImages); + return processedImages; } private async extractEmbeddedImages(content: string): Promise { @@ -372,33 +343,61 @@ class CopilotPlusChainRunner extends BaseChainRunner { textContent: string, userMessage: ChatMessage ): Promise { - const content: MessageContent[] = [ + const failureMessages: string[] = []; + const successfulImages: ImageContent[] = []; + const settings = getSettings(); + + // Collect all image sources + const imageSources: { urls: string[]; type: string }[] = []; + + // Safely check and add context URLs + const contextUrls = userMessage.context?.urls; + if (contextUrls && contextUrls.length > 0) { + imageSources.push({ urls: contextUrls, type: "context" }); + } + + // Process embedded images only if setting is enabled + if (settings.passMarkdownImages) { + const embeddedImages = await this.extractEmbeddedImages(textContent); + if (embeddedImages.length > 0) { + imageSources.push({ urls: embeddedImages, type: "embedded" }); + } + } + + // Process all image sources + for (const source of imageSources) { + const result = await this.processImageUrls(source.urls); + successfulImages.push(...result.successfulImages); + failureMessages.push(...result.failureDescriptions); + } + + // Process existing chat content images if present + const existingContent = userMessage.content; + if (existingContent && existingContent.length > 0) { + const result = await this.processChatInputImages(existingContent); + successfulImages.push(...result.successfulImages); + failureMessages.push(...result.failureDescriptions); + } + + // Let the LLM know about the image processing failures + let finalText = textContent; + if (failureMessages.length > 0) { + finalText = `${textContent}\n\nNote: \n${failureMessages.join("\n")}\n`; + } + + const messageContent: MessageContent[] = [ { type: "text", - text: textContent, + text: finalText, }, ]; - // Process URLs in the message to identify images - if (userMessage.context?.urls && userMessage.context.urls.length > 0) { - const imageContents = await this.processImageUrls(userMessage.context.urls); - content.push(...imageContents); + // Add successful images after the text content + if (successfulImages.length > 0) { + messageContent.push(...successfulImages); } - // Process embedded images from the text content - const embeddedImages = await this.extractEmbeddedImages(textContent); - if (embeddedImages.length > 0) { - const imageContents = await this.processImageUrls(embeddedImages); - content.push(...imageContents); - } - - // Add existing image content if present - if (userMessage.content && userMessage.content.length > 0) { - const imageContents = await this.processExistingImages(userMessage.content); - content.push(...imageContents); - } - - return content; + return messageContent; } private hasCapability(model: BaseChatModel, capability: ModelCapability): boolean { @@ -467,11 +466,9 @@ class CopilotPlusChainRunner extends BaseChainRunner { content, }); - // Add debug logging for final request - if (debug) { - console.log("==== Final Request to AI ====\n", messages); - } - + const enhancedUserMessage = content instanceof Array ? (content[0] as any).text : content; + logInfo("Enhanced user message: ", enhancedUserMessage); + logInfo("==== Final Request to AI ====\n", messages); const streamer = new ThinkBlockStreamer(updateCurrentAiMessage); const chatStream = await this.chainManager.chatModelManager.getChatModel().stream(messages); @@ -616,10 +613,7 @@ class CopilotPlusChainRunner extends BaseChainRunner { toolOutputs ); // If no results, default to LLM Chain - if (debug) { - console.log("No local search results. Using standard LLM Chain."); - console.log("Enhanced user message:", enhancedUserMessage); - } + logInfo("No local search results. Using standard LLM Chain."); fullAIResponse = await this.streamMultimodalResponse( enhancedUserMessage, @@ -720,7 +714,7 @@ class CopilotPlusChainRunner extends BaseChainRunner { .join("\n\n"); } } - return `User message: ${userMessage}${context}`; + return `${userMessage}${context}`; } private getTimeExpression(toolCalls: any[]): string { diff --git a/src/constants.ts b/src/constants.ts index 5471f3a5..4b1d64a4 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,8 +1,8 @@ import { CustomModel } from "@/aiParams"; import { DEFAULT_INLINE_EDIT_COMMANDS } from "@/commands/constants"; import { type CopilotSettings } from "@/settings/model"; -import { ChainType } from "./chainFactory"; import { v4 as uuidv4 } from "uuid"; +import { ChainType } from "./chainFactory"; export const BREVILABS_API_BASE_URL = "https://api.brevilabs.com/v1"; export const CHAT_VIEWTYPE = "copilot-chat-view"; @@ -524,6 +524,7 @@ export const DEFAULT_SETTINGS: CopilotSettings = { defaultConversationNoteName: "{$topic}@{$date}_{$time}", inlineEditCommands: DEFAULT_INLINE_EDIT_COMMANDS, lastDismissedVersion: null, + passMarkdownImages: true, }; export const EVENT_NAMES = { diff --git a/src/imageProcessing/imageProcessor.ts b/src/imageProcessing/imageProcessor.ts new file mode 100644 index 00000000..fb264b2a --- /dev/null +++ b/src/imageProcessing/imageProcessor.ts @@ -0,0 +1,400 @@ +import { logError } from "@/logger"; +import { safeFetch } from "@/utils"; +import { Notice, TFile, Vault } from "obsidian"; + +export interface ImageContent { + type: "image_url"; + image_url: { + url: string; + }; +} + +export interface TextContent { + type: "text"; + text: string; +} + +export interface ImageProcessingResult { + successfulImages: ImageContent[]; + failureDescriptions: string[]; +} + +export type MessageContent = ImageContent | TextContent; + +export class ImageProcessor { + private static readonly IMAGE_EXTENSIONS = [".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp"]; + + private static readonly MAX_IMAGE_SIZE = 3 * 1024 * 1024; // 3MB + private static readonly MIME_TYPES = { + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".png": "image/png", + ".gif": "image/gif", + ".webp": "image/webp", + ".bmp": "image/bmp", + }; + + static async isImageUrl(url: string, vault: Vault): Promise { + try { + // Extract extension from the URL or file path + const extension = url.split(".").pop()?.toLowerCase(); + if (extension) { + // Check if the extension is supported + const isSupported = this.IMAGE_EXTENSIONS.some( + (ext) => ext.toLowerCase() === `.${extension}` + ); + + if (!isSupported) { + const msg = `Unsupported image format: .${extension}. Supported formats: ${this.IMAGE_EXTENSIONS.join(", ")}`; + logError(msg); + new Notice(msg); + return false; + } + } + + // First check if it's an Obsidian vault image path + if (this.IMAGE_EXTENSIONS.some((ext) => url.toLowerCase().endsWith(ext))) { + // Verify the file exists and is accessible + const file = vault.getAbstractFileByPath(url); + if (!file || !(file instanceof TFile)) { + logError(`File not found in vault: ${url}`); + return false; + } + + // Check file size + if (file.stat.size > this.MAX_IMAGE_SIZE) { + logError(`File too large: ${file.stat.size} bytes`); + return false; + } + + return true; + } + + // Then check if it's a valid URL + const urlObj = new URL(url); + + // First check: URL path ends with image extension + if (this.IMAGE_EXTENSIONS.some((ext) => urlObj.pathname.toLowerCase().endsWith(ext))) { + return true; + } + + // Second check: Try HEAD request to check content-type + try { + const response = await safeFetch(url, { + method: "HEAD", + headers: {}, // Explicitly set empty headers + }); + + const contentType = response.headers.get("content-type"); + if (contentType?.startsWith("image/")) { + return true; + } + } catch (error) { + logError(`Error checking content-type for URL: ${url}`, error); + } + + // Final check: Analyze URL patterns that commonly indicate image content + const searchParams = urlObj.searchParams; + const imageIndicators = [ + // Image dimensions + searchParams.has("w") || searchParams.has("width"), + searchParams.has("h") || searchParams.has("height"), + // Image processing + searchParams.has("format"), + searchParams.has("fit"), + // Image quality + searchParams.has("q") || searchParams.has("quality"), + // Common CDN image path patterns + urlObj.pathname.includes("/image/"), + urlObj.pathname.includes("/images/"), + urlObj.pathname.includes("/img/"), + // Common image processing parameters + searchParams.has("auto"), + searchParams.has("crop"), + ]; + + // If multiple image-related indicators are present, likely an image URL + const imageIndicatorCount = imageIndicators.filter(Boolean).length; + return imageIndicatorCount >= 2; // Require at least 2 indicators to consider it an image URL + } catch { + // If URL construction fails, it might still be a valid Obsidian vault image path + return this.IMAGE_EXTENSIONS.some((ext) => url.toLowerCase().endsWith(ext)); + } + } + + private static async handleVaultImage(file: TFile, vault: Vault): Promise { + try { + // Check file size first + if (file.stat.size > this.MAX_IMAGE_SIZE) { + logError(`Image too large: ${file.stat.size} bytes, skipping: ${file.path}`); + return null; + } + + // Read the file as array buffer + const arrayBuffer = await vault.readBinary(file); + + // Validate MIME type + const mimeType = await this.getMimeType(arrayBuffer, file.extension); + if (!mimeType.startsWith("image/")) { + logError(`Invalid MIME type: ${mimeType}, skipping: ${file.path}`); + return null; + } + + const buffer = Buffer.from(arrayBuffer); + const base64 = buffer.toString("base64"); + + const result = `data:${mimeType};base64,${base64}`; + return result; + } catch (error) { + logError("Error processing vault image:", error); + return null; + } + } + + private static async handleWebImage(imageUrl: string): Promise { + try { + const response = await safeFetch(imageUrl, { + method: "GET", + headers: {}, + }); + + if (!response.ok) { + logError(`Failed to fetch image: ${response.statusText}, URL: ${imageUrl}`); + return null; + } + + // Try to get content type from response headers + const contentType = response.headers.get("content-type"); + if (!contentType?.startsWith("image/")) { + logError(`Invalid content type: ${contentType}, URL: ${imageUrl}`); + return null; + } + + const arrayBuffer = await response.arrayBuffer(); + + // Check file size + if (arrayBuffer.byteLength > this.MAX_IMAGE_SIZE) { + logError(`Image too large: ${arrayBuffer.byteLength} bytes, URL: ${imageUrl}`); + return null; + } + + const buffer = Buffer.from(arrayBuffer); + const base64 = buffer.toString("base64"); + return `data:${contentType};base64,${base64}`; + } catch (error) { + logError("Error converting web image to base64:", error); + return null; + } + } + + private static async handleLocalImage(imageUrl: string, vault: Vault): Promise { + try { + const localPath = decodeURIComponent(imageUrl.replace("app://", "")); + const file = vault.getAbstractFileByPath(localPath); + if (!file || !(file instanceof TFile)) { + logError(`Local image not found: ${localPath}`); + return null; + } + + // Check file size + if (file.stat.size > this.MAX_IMAGE_SIZE) { + logError(`Image too large: ${file.stat.size} bytes, path: ${localPath}`); + return null; + } + + // Read the file as array buffer + const arrayBuffer = await vault.readBinary(file); + + // Validate MIME type + const mimeType = await this.getMimeType(arrayBuffer, file.extension); + if (!mimeType.startsWith("image/")) { + logError(`Invalid MIME type: ${mimeType}, path: ${localPath}`); + return null; + } + + const buffer = Buffer.from(arrayBuffer); + const base64 = buffer.toString("base64"); + + const result = `data:${mimeType};base64,${base64}`; + return result; + } catch (error) { + logError("Error processing local image:", error); + return null; + } + } + + private static async imageToBase64(imageUrl: string, vault: Vault): Promise { + // If it's already a data URL, return it as is + if (imageUrl.startsWith("data:")) { + return imageUrl; + } + + // Check if it's a local vault image + if (imageUrl.startsWith("app://")) { + return await this.handleLocalImage(imageUrl, vault); + } + + // Check if it's an Obsidian vault image (direct file path) + const file = vault.getAbstractFileByPath(imageUrl); + if (file instanceof TFile) { + return await this.handleVaultImage(file, vault); + } + + // Handle web images + return await this.handleWebImage(imageUrl); + } + + static async convertToBase64(imageUrl: string, vault: Vault): Promise { + const base64Url = await this.imageToBase64(imageUrl, vault); + if (!base64Url) { + logError(`Failed to convert image to base64: ${imageUrl}`); + return null; + } + return { + type: "image_url", + image_url: { + url: base64Url, + }, + }; + } + + private static async getMimeType(arrayBuffer: ArrayBuffer, extension: string): Promise { + // Get the first few bytes to check for magic numbers + const bytes = new Uint8Array(arrayBuffer.slice(0, 4)); + + // Check for common image magic numbers + if (bytes[0] === 0xff && bytes[1] === 0xd8) return "image/jpeg"; + if (bytes[0] === 0x89 && bytes[1] === 0x50) return "image/png"; + if (bytes[0] === 0x47 && bytes[1] === 0x49) return "image/gif"; + if (bytes[0] === 0x52 && bytes[1] === 0x49) return "image/webp"; + if (bytes[0] === 0x42 && bytes[1] === 0x4d) return "image/bmp"; + if (bytes[0] === 0x3c && bytes[1] === 0x73) { + throw new Error("SVG files are not supported"); + } + + // Fall back to extension-based detection + const mimeType = this.MIME_TYPES[extension.toLowerCase() as keyof typeof this.MIME_TYPES]; + if (!mimeType) { + const error = `Unsupported image extension: ${extension}`; + logError(error); + throw new Error(error); + } + return mimeType; + } +} + +export class ImageBatchProcessor { + static async processUrlBatch( + urls: string[], + failedImages: string[], + vault: Vault + ): Promise { + try { + const results = await Promise.all( + urls.map((url) => ImageBatchProcessor.processSingleUrl(url, failedImages, vault)) + ); + + const successfulImages = results.filter((item): item is ImageContent => item !== null); + const failureDescriptions = failedImages.map((url) => `Image read failed for: ${url}`); + + return { + successfulImages, + failureDescriptions, + }; + } catch (error) { + logError("Error processing URL batch:", error); + return { + successfulImages: [], + failureDescriptions: urls.map((url) => `Image read failed for: ${url}`), + }; + } + } + + static async processSingleUrl( + url: string, + failedImages: string[], + vault: Vault + ): Promise { + try { + if (!(await ImageProcessor.isImageUrl(url, vault))) { + failedImages.push(url); + return null; + } + + const imageContent = await ImageProcessor.convertToBase64(url, vault); + + if (!imageContent) { + failedImages.push(url); + return null; + } + + return imageContent; + } catch (error) { + logError(`Failed to process image: ${url}`, error); + failedImages.push(url); + return null; + } + } + + static async processChatImageBatch( + content: MessageContent[], + failedImages: string[], + vault: Vault + ): Promise { + try { + const imageItems = content.filter( + (item): item is ImageContent => item.type === "image_url" && !!item.image_url?.url + ); + + const results = await Promise.all( + imageItems.map((item) => + ImageBatchProcessor.processChatSingleImage(item, failedImages, vault) + ) + ); + + const successfulImages = results.filter((item): item is ImageContent => item !== null); + const failureDescriptions = failedImages.map((url) => `Image read failed for: ${url}`); + + return { + successfulImages, + failureDescriptions, + }; + } catch (error) { + logError("Error processing chat image batch:", error); + const imageUrls = content + .filter((item): item is ImageContent => item.type === "image_url" && !!item.image_url?.url) + .map((item) => item.image_url.url); + return { + successfulImages: [], + failureDescriptions: imageUrls.map((url) => `Image read failed for: ${url}`), + }; + } + } + + static async processChatSingleImage( + item: ImageContent, + failedImages: string[], + vault: Vault + ): Promise { + try { + const processedContent = await ImageProcessor.convertToBase64(item.image_url.url, vault); + + if (!processedContent) { + failedImages.push(item.image_url.url); + return null; + } + + return processedContent; + } catch (error) { + logError(`Failed to process chat image: ${item.image_url.url}`, error); + failedImages.push(item.image_url.url); + return null; + } + } + + static showFailedImagesNotice(failedImages: string[]): void { + if (failedImages.length > 0) { + new Notice(`Failed to process images:\n${failedImages.join("\n")}`); + } + } +} diff --git a/src/mentions/Mention.ts b/src/mentions/Mention.ts index 037161ad..bbf12fdd 100644 --- a/src/mentions/Mention.ts +++ b/src/mentions/Mention.ts @@ -1,5 +1,6 @@ +import { ImageProcessor } from "@/imageProcessing/imageProcessor"; import { BrevilabsClient, Url4llmResponse } from "@/LLMProviders/brevilabsClient"; -import { ImageProcessor, isYoutubeUrl } from "@/utils"; +import { isYoutubeUrl } from "@/utils"; export interface MentionData { type: string; diff --git a/src/settings/model.ts b/src/settings/model.ts index fb7efa47..292d6bdf 100644 --- a/src/settings/model.ts +++ b/src/settings/model.ts @@ -93,6 +93,7 @@ export interface CopilotSettings { // undefined means never checked isPlusUser: boolean | undefined; inlineEditCommands: InlineEditCommandSettings[] | undefined; + passMarkdownImages: boolean; } export const settingsStore = createStore(); @@ -210,6 +211,11 @@ export function sanitizeSettings(settings: CopilotSettings): CopilotSettings { ? DEFAULT_SETTINGS.embeddingBatchSize : embeddingBatchSize; + // Ensure passMarkdownImages has a default value + if (typeof sanitizedSettings.passMarkdownImages !== "boolean") { + sanitizedSettings.passMarkdownImages = DEFAULT_SETTINGS.passMarkdownImages; + } + return sanitizedSettings; } diff --git a/src/settings/v2/components/AdvancedSettings.tsx b/src/settings/v2/components/AdvancedSettings.tsx index 670c02a7..bbb71b91 100644 --- a/src/settings/v2/components/AdvancedSettings.tsx +++ b/src/settings/v2/components/AdvancedSettings.tsx @@ -1,6 +1,6 @@ -import React from "react"; import { SettingItem } from "@/components/ui/setting-item"; import { updateSetting, useSettingsValue } from "@/settings/model"; +import React from "react"; export const AdvancedSettings: React.FC = () => { const settings = useSettingsValue(); @@ -19,6 +19,16 @@ export const AdvancedSettings: React.FC = () => { />
+ { + updateSetting("passMarkdownImages", checked); + }} + /> + { - try { - // First check if it's an Obsidian vault image path - if (this.IMAGE_EXTENSIONS.some((ext) => url.toLowerCase().endsWith(ext))) { - // Verify the file exists and is accessible - const file = vault.getAbstractFileByPath(url); - if (!file || !(file instanceof TFile)) { - logError("File not found in vault"); - return false; - } - - // Check file size - if (file.stat.size > this.MAX_IMAGE_SIZE) { - logError("File too large:", file.stat.size, "bytes"); - return false; - } - - return true; - } - - // Then check if it's a valid URL - const urlObj = new URL(url); - - // First check: URL path ends with image extension - if (this.IMAGE_EXTENSIONS.some((ext) => urlObj.pathname.toLowerCase().endsWith(ext))) { - return true; - } - - // Second check: Try HEAD request to check content-type - try { - const response = await safeFetch(url, { - method: "HEAD", - headers: {}, // Explicitly set empty headers - }); - - const contentType = response.headers.get("content-type"); - if (contentType?.startsWith("image/")) { - return true; - } - } catch (error) { - logError("Error checking content-type:", error); - } - - // Final check: Analyze URL patterns that commonly indicate image content - const searchParams = urlObj.searchParams; - const imageIndicators = [ - // Image dimensions - searchParams.has("w") || searchParams.has("width"), - searchParams.has("h") || searchParams.has("height"), - // Image processing - searchParams.has("format"), - searchParams.has("fit"), - // Image quality - searchParams.has("q") || searchParams.has("quality"), - // Common CDN image path patterns - urlObj.pathname.includes("/image/"), - urlObj.pathname.includes("/images/"), - urlObj.pathname.includes("/img/"), - // Common image processing parameters - searchParams.has("auto"), - searchParams.has("crop"), - ]; - - // If multiple image-related indicators are present, likely an image URL - const imageIndicatorCount = imageIndicators.filter(Boolean).length; - return imageIndicatorCount >= 2; // Require at least 2 indicators to consider it an image URL - } catch { - // If URL construction fails, it might still be a valid Obsidian vault image path - return this.IMAGE_EXTENSIONS.some((ext) => url.toLowerCase().endsWith(ext)); - } - } - - private static async handleVaultImage(file: TFile, vault: Vault): Promise { - try { - // Check file size first - if (file.stat.size > this.MAX_IMAGE_SIZE) { - logError(`Image too large: ${file.stat.size} bytes, skipping: ${file.path}`); - return null; - } - - // Read the file as array buffer - const arrayBuffer = await vault.readBinary(file); - - // Validate MIME type - const mimeType = await this.getMimeType(arrayBuffer, file.extension); - if (!mimeType.startsWith("image/")) { - logError(`Invalid MIME type: ${mimeType}, skipping: ${file.path}`); - return null; - } - - const buffer = Buffer.from(arrayBuffer); - const base64 = buffer.toString("base64"); - - const result = `data:${mimeType};base64,${base64}`; - return result; - } catch (error) { - logError("Error in handleVaultImage:", error); - return null; - } - } - - private static async handleWebImage(imageUrl: string): Promise { - try { - const response = await safeFetch(imageUrl, { - method: "GET", - headers: {}, - }); - - if (!response.ok) { - logError(`Failed to fetch image: ${response.statusText}, skipping: ${imageUrl}`); - return null; - } - - // Try to get content type from response headers - const contentType = response.headers.get("content-type"); - if (!contentType?.startsWith("image/")) { - logError(`Invalid content type: ${contentType}, skipping: ${imageUrl}`); - return null; - } - - const arrayBuffer = await response.arrayBuffer(); - - // Check file size - if (arrayBuffer.byteLength > this.MAX_IMAGE_SIZE) { - logError(`Image too large: ${arrayBuffer.byteLength} bytes, skipping: ${imageUrl}`); - return null; - } - - const buffer = Buffer.from(arrayBuffer); - const base64 = buffer.toString("base64"); - return `data:${contentType};base64,${base64}`; - } catch (error) { - logError("Error converting image to base64:", error); - return null; - } - } - - private static async handleLocalImage(imageUrl: string, vault: Vault): Promise { - try { - const localPath = decodeURIComponent(imageUrl.replace("app://", "")); - const file = vault.getAbstractFileByPath(localPath); - if (!file || !(file instanceof TFile)) { - logError(`Local image not found: ${localPath}`); - return null; - } - - // Check file size - if (file.stat.size > this.MAX_IMAGE_SIZE) { - logError(`Image too large: ${file.stat.size} bytes, skipping: ${localPath}`); - return null; - } - - // Read the file as array buffer - const arrayBuffer = await vault.readBinary(file); - - // Validate MIME type - const mimeType = await this.getMimeType(arrayBuffer, file.extension); - if (!mimeType.startsWith("image/")) { - logError(`Invalid MIME type: ${mimeType}, skipping: ${localPath}`); - return null; - } - - const buffer = Buffer.from(arrayBuffer); - const base64 = buffer.toString("base64"); - - const result = `data:${mimeType};base64,${base64}`; - return result; - } catch (error) { - logError("Error in handleLocalImage:", error); - return null; - } - } - - private static async imageToBase64(imageUrl: string, vault: Vault): Promise { - // If it's already a data URL, return it as is - if (imageUrl.startsWith("data:")) { - return imageUrl; - } - - // Check if it's a local vault image - if (imageUrl.startsWith("app://")) { - return await this.handleLocalImage(imageUrl, vault); - } - - // Check if it's an Obsidian vault image (direct file path) - const file = vault.getAbstractFileByPath(imageUrl); - if (file instanceof TFile) { - return await this.handleVaultImage(file, vault); - } - - // Handle web images - return await this.handleWebImage(imageUrl); - } - - static async convertToBase64(imageUrl: string, vault: Vault): Promise { - const base64Url = await this.imageToBase64(imageUrl, vault); - if (!base64Url) { - return null; - } - return { - type: "image_url", - image_url: { - url: base64Url, - }, - }; - } - - private static async getMimeType(arrayBuffer: ArrayBuffer, extension: string): Promise { - // Get the first few bytes to check for magic numbers - const bytes = new Uint8Array(arrayBuffer.slice(0, 4)); - - // Check for common image magic numbers - if (bytes[0] === 0xff && bytes[1] === 0xd8) { - return "image/jpeg"; - } - if (bytes[0] === 0x89 && bytes[1] === 0x50) { - return "image/png"; - } - if (bytes[0] === 0x47 && bytes[1] === 0x49) { - return "image/gif"; - } - if (bytes[0] === 0x52 && bytes[1] === 0x49) { - return "image/webp"; - } - if (bytes[0] === 0x42 && bytes[1] === 0x4d) { - return "image/bmp"; - } - if (bytes[0] === 0x3c && bytes[1] === 0x73) { - return "image/svg+xml"; - } - - // Fall back to extension-based detection - const mimeType = this.MIME_TYPES[extension.toLowerCase() as keyof typeof this.MIME_TYPES]; - if (!mimeType) { - throw new Error(`Unsupported image extension: ${extension}`); - } - return mimeType; - } -} - /** Proxy function to use in place of fetch() to bypass CORS restrictions. * It currently doesn't support streaming until this is implemented * https://forum.obsidian.md/t/support-streaming-the-request-and-requesturl-response-body/87381 */