mirror of
https://github.com/logancyang/obsidian-copilot.git
synced 2026-07-22 07:50:24 +00:00
Fix image in note logic (#1457)
* Refactor image processing in note context * Add passMarkdownImages setting
This commit is contained in:
Logan Yang
GitHub
parent
f4bf334c27
commit
1d46ab90b0
7 changed files with 501 additions and 366 deletions
|
|
@ -7,9 +7,15 @@ import {
|
|||
MAX_CHARS_FOR_LOCAL_SEARCH_CONTEXT,
|
||||
ModelCapability,
|
||||
} from "@/constants";
|
||||
import {
|
||||
ImageBatchProcessor,
|
||||
ImageContent,
|
||||
ImageProcessingResult,
|
||||
MessageContent,
|
||||
} from "@/imageProcessing/imageProcessor";
|
||||
import { BrevilabsClient } from "@/LLMProviders/brevilabsClient";
|
||||
import { logError } from "@/logger";
|
||||
import { getSystemPrompt } from "@/settings/model";
|
||||
import { logInfo } from "@/logger";
|
||||
import { getSettings, getSystemPrompt } from "@/settings/model";
|
||||
import { ChatMessage } from "@/sharedState";
|
||||
import { ToolManager } from "@/tools/toolManager";
|
||||
import {
|
||||
|
|
@ -20,9 +26,6 @@ import {
|
|||
formatDateTime,
|
||||
getApiErrorMessage,
|
||||
getMessageRole,
|
||||
ImageContent,
|
||||
ImageProcessor,
|
||||
MessageContent,
|
||||
} from "@/utils";
|
||||
import { BaseChatModel } from "@langchain/core/language_models/chat_models";
|
||||
import { Notice } from "obsidian";
|
||||
|
|
@ -307,58 +310,26 @@ class CopilotPlusChainRunner extends BaseChainRunner {
|
|||
return hasYoutubeCommand && youtubeUrl !== null && words.length === 1;
|
||||
}
|
||||
|
||||
private async processImageUrls(urls: string[]): Promise<ImageContent[]> {
|
||||
try {
|
||||
const imageUrls = await Promise.all(
|
||||
urls.map(async (url) => {
|
||||
if (await ImageProcessor.isImageUrl(url, this.chainManager.app.vault)) {
|
||||
const imageContent = await ImageProcessor.convertToBase64(
|
||||
url,
|
||||
this.chainManager.app.vault
|
||||
);
|
||||
if (!imageContent) {
|
||||
logError(`Failed to process image: ${url}`);
|
||||
return null;
|
||||
}
|
||||
return imageContent;
|
||||
}
|
||||
return null;
|
||||
})
|
||||
);
|
||||
|
||||
// Filter out null values and return valid image URLs
|
||||
const validImages = imageUrls.filter((item): item is ImageContent => item !== null);
|
||||
return validImages;
|
||||
} catch (error) {
|
||||
logError("Error processing image URLs:", error);
|
||||
return [];
|
||||
}
|
||||
private async processImageUrls(urls: string[]): Promise<ImageProcessingResult> {
|
||||
const failedImages: string[] = [];
|
||||
const processedImages = await ImageBatchProcessor.processUrlBatch(
|
||||
urls,
|
||||
failedImages,
|
||||
this.chainManager.app.vault
|
||||
);
|
||||
ImageBatchProcessor.showFailedImagesNotice(failedImages);
|
||||
return processedImages;
|
||||
}
|
||||
|
||||
private async processExistingImages(content: MessageContent[]): Promise<ImageContent[]> {
|
||||
try {
|
||||
const imageContent = await Promise.all(
|
||||
content
|
||||
.filter(
|
||||
(item): item is ImageContent => item.type === "image_url" && !!item.image_url?.url
|
||||
)
|
||||
.map(async (item) => {
|
||||
const processedContent = await ImageProcessor.convertToBase64(
|
||||
item.image_url.url,
|
||||
this.chainManager.app.vault
|
||||
);
|
||||
if (!processedContent) {
|
||||
logError(`Failed to process existing image: ${item.image_url.url}`);
|
||||
return null;
|
||||
}
|
||||
return processedContent;
|
||||
})
|
||||
);
|
||||
return imageContent.filter((item): item is ImageContent => item !== null);
|
||||
} catch (error) {
|
||||
logError("Error processing images:", error);
|
||||
return [];
|
||||
}
|
||||
private async processChatInputImages(content: MessageContent[]): Promise<ImageProcessingResult> {
|
||||
const failedImages: string[] = [];
|
||||
const processedImages = await ImageBatchProcessor.processChatImageBatch(
|
||||
content,
|
||||
failedImages,
|
||||
this.chainManager.app.vault
|
||||
);
|
||||
ImageBatchProcessor.showFailedImagesNotice(failedImages);
|
||||
return processedImages;
|
||||
}
|
||||
|
||||
private async extractEmbeddedImages(content: string): Promise<string[]> {
|
||||
|
|
@ -372,33 +343,61 @@ class CopilotPlusChainRunner extends BaseChainRunner {
|
|||
textContent: string,
|
||||
userMessage: ChatMessage
|
||||
): Promise<MessageContent[]> {
|
||||
const content: MessageContent[] = [
|
||||
const failureMessages: string[] = [];
|
||||
const successfulImages: ImageContent[] = [];
|
||||
const settings = getSettings();
|
||||
|
||||
// Collect all image sources
|
||||
const imageSources: { urls: string[]; type: string }[] = [];
|
||||
|
||||
// Safely check and add context URLs
|
||||
const contextUrls = userMessage.context?.urls;
|
||||
if (contextUrls && contextUrls.length > 0) {
|
||||
imageSources.push({ urls: contextUrls, type: "context" });
|
||||
}
|
||||
|
||||
// Process embedded images only if setting is enabled
|
||||
if (settings.passMarkdownImages) {
|
||||
const embeddedImages = await this.extractEmbeddedImages(textContent);
|
||||
if (embeddedImages.length > 0) {
|
||||
imageSources.push({ urls: embeddedImages, type: "embedded" });
|
||||
}
|
||||
}
|
||||
|
||||
// Process all image sources
|
||||
for (const source of imageSources) {
|
||||
const result = await this.processImageUrls(source.urls);
|
||||
successfulImages.push(...result.successfulImages);
|
||||
failureMessages.push(...result.failureDescriptions);
|
||||
}
|
||||
|
||||
// Process existing chat content images if present
|
||||
const existingContent = userMessage.content;
|
||||
if (existingContent && existingContent.length > 0) {
|
||||
const result = await this.processChatInputImages(existingContent);
|
||||
successfulImages.push(...result.successfulImages);
|
||||
failureMessages.push(...result.failureDescriptions);
|
||||
}
|
||||
|
||||
// Let the LLM know about the image processing failures
|
||||
let finalText = textContent;
|
||||
if (failureMessages.length > 0) {
|
||||
finalText = `${textContent}\n\nNote: \n${failureMessages.join("\n")}\n`;
|
||||
}
|
||||
|
||||
const messageContent: MessageContent[] = [
|
||||
{
|
||||
type: "text",
|
||||
text: textContent,
|
||||
text: finalText,
|
||||
},
|
||||
];
|
||||
|
||||
// Process URLs in the message to identify images
|
||||
if (userMessage.context?.urls && userMessage.context.urls.length > 0) {
|
||||
const imageContents = await this.processImageUrls(userMessage.context.urls);
|
||||
content.push(...imageContents);
|
||||
// Add successful images after the text content
|
||||
if (successfulImages.length > 0) {
|
||||
messageContent.push(...successfulImages);
|
||||
}
|
||||
|
||||
// Process embedded images from the text content
|
||||
const embeddedImages = await this.extractEmbeddedImages(textContent);
|
||||
if (embeddedImages.length > 0) {
|
||||
const imageContents = await this.processImageUrls(embeddedImages);
|
||||
content.push(...imageContents);
|
||||
}
|
||||
|
||||
// Add existing image content if present
|
||||
if (userMessage.content && userMessage.content.length > 0) {
|
||||
const imageContents = await this.processExistingImages(userMessage.content);
|
||||
content.push(...imageContents);
|
||||
}
|
||||
|
||||
return content;
|
||||
return messageContent;
|
||||
}
|
||||
|
||||
private hasCapability(model: BaseChatModel, capability: ModelCapability): boolean {
|
||||
|
|
@ -467,11 +466,9 @@ class CopilotPlusChainRunner extends BaseChainRunner {
|
|||
content,
|
||||
});
|
||||
|
||||
// Add debug logging for final request
|
||||
if (debug) {
|
||||
console.log("==== Final Request to AI ====\n", messages);
|
||||
}
|
||||
|
||||
const enhancedUserMessage = content instanceof Array ? (content[0] as any).text : content;
|
||||
logInfo("Enhanced user message: ", enhancedUserMessage);
|
||||
logInfo("==== Final Request to AI ====\n", messages);
|
||||
const streamer = new ThinkBlockStreamer(updateCurrentAiMessage);
|
||||
const chatStream = await this.chainManager.chatModelManager.getChatModel().stream(messages);
|
||||
|
||||
|
|
@ -616,10 +613,7 @@ class CopilotPlusChainRunner extends BaseChainRunner {
|
|||
toolOutputs
|
||||
);
|
||||
// If no results, default to LLM Chain
|
||||
if (debug) {
|
||||
console.log("No local search results. Using standard LLM Chain.");
|
||||
console.log("Enhanced user message:", enhancedUserMessage);
|
||||
}
|
||||
logInfo("No local search results. Using standard LLM Chain.");
|
||||
|
||||
fullAIResponse = await this.streamMultimodalResponse(
|
||||
enhancedUserMessage,
|
||||
|
|
@ -720,7 +714,7 @@ class CopilotPlusChainRunner extends BaseChainRunner {
|
|||
.join("\n\n");
|
||||
}
|
||||
}
|
||||
return `User message: ${userMessage}${context}`;
|
||||
return `${userMessage}${context}`;
|
||||
}
|
||||
|
||||
private getTimeExpression(toolCalls: any[]): string {
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { CustomModel } from "@/aiParams";
|
||||
import { DEFAULT_INLINE_EDIT_COMMANDS } from "@/commands/constants";
|
||||
import { type CopilotSettings } from "@/settings/model";
|
||||
import { ChainType } from "./chainFactory";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { ChainType } from "./chainFactory";
|
||||
|
||||
export const BREVILABS_API_BASE_URL = "https://api.brevilabs.com/v1";
|
||||
export const CHAT_VIEWTYPE = "copilot-chat-view";
|
||||
|
|
@ -524,6 +524,7 @@ export const DEFAULT_SETTINGS: CopilotSettings = {
|
|||
defaultConversationNoteName: "{$topic}@{$date}_{$time}",
|
||||
inlineEditCommands: DEFAULT_INLINE_EDIT_COMMANDS,
|
||||
lastDismissedVersion: null,
|
||||
passMarkdownImages: true,
|
||||
};
|
||||
|
||||
export const EVENT_NAMES = {
|
||||
|
|
|
|||
400
src/imageProcessing/imageProcessor.ts
Normal file
400
src/imageProcessing/imageProcessor.ts
Normal file
|
|
@ -0,0 +1,400 @@
|
|||
import { logError } from "@/logger";
|
||||
import { safeFetch } from "@/utils";
|
||||
import { Notice, TFile, Vault } from "obsidian";
|
||||
|
||||
export interface ImageContent {
|
||||
type: "image_url";
|
||||
image_url: {
|
||||
url: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface TextContent {
|
||||
type: "text";
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface ImageProcessingResult {
|
||||
successfulImages: ImageContent[];
|
||||
failureDescriptions: string[];
|
||||
}
|
||||
|
||||
export type MessageContent = ImageContent | TextContent;
|
||||
|
||||
export class ImageProcessor {
|
||||
private static readonly IMAGE_EXTENSIONS = [".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp"];
|
||||
|
||||
private static readonly MAX_IMAGE_SIZE = 3 * 1024 * 1024; // 3MB
|
||||
private static readonly MIME_TYPES = {
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".png": "image/png",
|
||||
".gif": "image/gif",
|
||||
".webp": "image/webp",
|
||||
".bmp": "image/bmp",
|
||||
};
|
||||
|
||||
static async isImageUrl(url: string, vault: Vault): Promise<boolean> {
|
||||
try {
|
||||
// Extract extension from the URL or file path
|
||||
const extension = url.split(".").pop()?.toLowerCase();
|
||||
if (extension) {
|
||||
// Check if the extension is supported
|
||||
const isSupported = this.IMAGE_EXTENSIONS.some(
|
||||
(ext) => ext.toLowerCase() === `.${extension}`
|
||||
);
|
||||
|
||||
if (!isSupported) {
|
||||
const msg = `Unsupported image format: .${extension}. Supported formats: ${this.IMAGE_EXTENSIONS.join(", ")}`;
|
||||
logError(msg);
|
||||
new Notice(msg);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// First check if it's an Obsidian vault image path
|
||||
if (this.IMAGE_EXTENSIONS.some((ext) => url.toLowerCase().endsWith(ext))) {
|
||||
// Verify the file exists and is accessible
|
||||
const file = vault.getAbstractFileByPath(url);
|
||||
if (!file || !(file instanceof TFile)) {
|
||||
logError(`File not found in vault: ${url}`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check file size
|
||||
if (file.stat.size > this.MAX_IMAGE_SIZE) {
|
||||
logError(`File too large: ${file.stat.size} bytes`);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// Then check if it's a valid URL
|
||||
const urlObj = new URL(url);
|
||||
|
||||
// First check: URL path ends with image extension
|
||||
if (this.IMAGE_EXTENSIONS.some((ext) => urlObj.pathname.toLowerCase().endsWith(ext))) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Second check: Try HEAD request to check content-type
|
||||
try {
|
||||
const response = await safeFetch(url, {
|
||||
method: "HEAD",
|
||||
headers: {}, // Explicitly set empty headers
|
||||
});
|
||||
|
||||
const contentType = response.headers.get("content-type");
|
||||
if (contentType?.startsWith("image/")) {
|
||||
return true;
|
||||
}
|
||||
} catch (error) {
|
||||
logError(`Error checking content-type for URL: ${url}`, error);
|
||||
}
|
||||
|
||||
// Final check: Analyze URL patterns that commonly indicate image content
|
||||
const searchParams = urlObj.searchParams;
|
||||
const imageIndicators = [
|
||||
// Image dimensions
|
||||
searchParams.has("w") || searchParams.has("width"),
|
||||
searchParams.has("h") || searchParams.has("height"),
|
||||
// Image processing
|
||||
searchParams.has("format"),
|
||||
searchParams.has("fit"),
|
||||
// Image quality
|
||||
searchParams.has("q") || searchParams.has("quality"),
|
||||
// Common CDN image path patterns
|
||||
urlObj.pathname.includes("/image/"),
|
||||
urlObj.pathname.includes("/images/"),
|
||||
urlObj.pathname.includes("/img/"),
|
||||
// Common image processing parameters
|
||||
searchParams.has("auto"),
|
||||
searchParams.has("crop"),
|
||||
];
|
||||
|
||||
// If multiple image-related indicators are present, likely an image URL
|
||||
const imageIndicatorCount = imageIndicators.filter(Boolean).length;
|
||||
return imageIndicatorCount >= 2; // Require at least 2 indicators to consider it an image URL
|
||||
} catch {
|
||||
// If URL construction fails, it might still be a valid Obsidian vault image path
|
||||
return this.IMAGE_EXTENSIONS.some((ext) => url.toLowerCase().endsWith(ext));
|
||||
}
|
||||
}
|
||||
|
||||
private static async handleVaultImage(file: TFile, vault: Vault): Promise<string | null> {
|
||||
try {
|
||||
// Check file size first
|
||||
if (file.stat.size > this.MAX_IMAGE_SIZE) {
|
||||
logError(`Image too large: ${file.stat.size} bytes, skipping: ${file.path}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Read the file as array buffer
|
||||
const arrayBuffer = await vault.readBinary(file);
|
||||
|
||||
// Validate MIME type
|
||||
const mimeType = await this.getMimeType(arrayBuffer, file.extension);
|
||||
if (!mimeType.startsWith("image/")) {
|
||||
logError(`Invalid MIME type: ${mimeType}, skipping: ${file.path}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(arrayBuffer);
|
||||
const base64 = buffer.toString("base64");
|
||||
|
||||
const result = `data:${mimeType};base64,${base64}`;
|
||||
return result;
|
||||
} catch (error) {
|
||||
logError("Error processing vault image:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static async handleWebImage(imageUrl: string): Promise<string | null> {
|
||||
try {
|
||||
const response = await safeFetch(imageUrl, {
|
||||
method: "GET",
|
||||
headers: {},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
logError(`Failed to fetch image: ${response.statusText}, URL: ${imageUrl}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Try to get content type from response headers
|
||||
const contentType = response.headers.get("content-type");
|
||||
if (!contentType?.startsWith("image/")) {
|
||||
logError(`Invalid content type: ${contentType}, URL: ${imageUrl}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
|
||||
// Check file size
|
||||
if (arrayBuffer.byteLength > this.MAX_IMAGE_SIZE) {
|
||||
logError(`Image too large: ${arrayBuffer.byteLength} bytes, URL: ${imageUrl}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(arrayBuffer);
|
||||
const base64 = buffer.toString("base64");
|
||||
return `data:${contentType};base64,${base64}`;
|
||||
} catch (error) {
|
||||
logError("Error converting web image to base64:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static async handleLocalImage(imageUrl: string, vault: Vault): Promise<string | null> {
|
||||
try {
|
||||
const localPath = decodeURIComponent(imageUrl.replace("app://", ""));
|
||||
const file = vault.getAbstractFileByPath(localPath);
|
||||
if (!file || !(file instanceof TFile)) {
|
||||
logError(`Local image not found: ${localPath}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check file size
|
||||
if (file.stat.size > this.MAX_IMAGE_SIZE) {
|
||||
logError(`Image too large: ${file.stat.size} bytes, path: ${localPath}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Read the file as array buffer
|
||||
const arrayBuffer = await vault.readBinary(file);
|
||||
|
||||
// Validate MIME type
|
||||
const mimeType = await this.getMimeType(arrayBuffer, file.extension);
|
||||
if (!mimeType.startsWith("image/")) {
|
||||
logError(`Invalid MIME type: ${mimeType}, path: ${localPath}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(arrayBuffer);
|
||||
const base64 = buffer.toString("base64");
|
||||
|
||||
const result = `data:${mimeType};base64,${base64}`;
|
||||
return result;
|
||||
} catch (error) {
|
||||
logError("Error processing local image:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static async imageToBase64(imageUrl: string, vault: Vault): Promise<string | null> {
|
||||
// If it's already a data URL, return it as is
|
||||
if (imageUrl.startsWith("data:")) {
|
||||
return imageUrl;
|
||||
}
|
||||
|
||||
// Check if it's a local vault image
|
||||
if (imageUrl.startsWith("app://")) {
|
||||
return await this.handleLocalImage(imageUrl, vault);
|
||||
}
|
||||
|
||||
// Check if it's an Obsidian vault image (direct file path)
|
||||
const file = vault.getAbstractFileByPath(imageUrl);
|
||||
if (file instanceof TFile) {
|
||||
return await this.handleVaultImage(file, vault);
|
||||
}
|
||||
|
||||
// Handle web images
|
||||
return await this.handleWebImage(imageUrl);
|
||||
}
|
||||
|
||||
static async convertToBase64(imageUrl: string, vault: Vault): Promise<ImageContent | null> {
|
||||
const base64Url = await this.imageToBase64(imageUrl, vault);
|
||||
if (!base64Url) {
|
||||
logError(`Failed to convert image to base64: ${imageUrl}`);
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: base64Url,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static async getMimeType(arrayBuffer: ArrayBuffer, extension: string): Promise<string> {
|
||||
// Get the first few bytes to check for magic numbers
|
||||
const bytes = new Uint8Array(arrayBuffer.slice(0, 4));
|
||||
|
||||
// Check for common image magic numbers
|
||||
if (bytes[0] === 0xff && bytes[1] === 0xd8) return "image/jpeg";
|
||||
if (bytes[0] === 0x89 && bytes[1] === 0x50) return "image/png";
|
||||
if (bytes[0] === 0x47 && bytes[1] === 0x49) return "image/gif";
|
||||
if (bytes[0] === 0x52 && bytes[1] === 0x49) return "image/webp";
|
||||
if (bytes[0] === 0x42 && bytes[1] === 0x4d) return "image/bmp";
|
||||
if (bytes[0] === 0x3c && bytes[1] === 0x73) {
|
||||
throw new Error("SVG files are not supported");
|
||||
}
|
||||
|
||||
// Fall back to extension-based detection
|
||||
const mimeType = this.MIME_TYPES[extension.toLowerCase() as keyof typeof this.MIME_TYPES];
|
||||
if (!mimeType) {
|
||||
const error = `Unsupported image extension: ${extension}`;
|
||||
logError(error);
|
||||
throw new Error(error);
|
||||
}
|
||||
return mimeType;
|
||||
}
|
||||
}
|
||||
|
||||
export class ImageBatchProcessor {
|
||||
static async processUrlBatch(
|
||||
urls: string[],
|
||||
failedImages: string[],
|
||||
vault: Vault
|
||||
): Promise<ImageProcessingResult> {
|
||||
try {
|
||||
const results = await Promise.all(
|
||||
urls.map((url) => ImageBatchProcessor.processSingleUrl(url, failedImages, vault))
|
||||
);
|
||||
|
||||
const successfulImages = results.filter((item): item is ImageContent => item !== null);
|
||||
const failureDescriptions = failedImages.map((url) => `Image read failed for: ${url}`);
|
||||
|
||||
return {
|
||||
successfulImages,
|
||||
failureDescriptions,
|
||||
};
|
||||
} catch (error) {
|
||||
logError("Error processing URL batch:", error);
|
||||
return {
|
||||
successfulImages: [],
|
||||
failureDescriptions: urls.map((url) => `Image read failed for: ${url}`),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
static async processSingleUrl(
|
||||
url: string,
|
||||
failedImages: string[],
|
||||
vault: Vault
|
||||
): Promise<ImageContent | null> {
|
||||
try {
|
||||
if (!(await ImageProcessor.isImageUrl(url, vault))) {
|
||||
failedImages.push(url);
|
||||
return null;
|
||||
}
|
||||
|
||||
const imageContent = await ImageProcessor.convertToBase64(url, vault);
|
||||
|
||||
if (!imageContent) {
|
||||
failedImages.push(url);
|
||||
return null;
|
||||
}
|
||||
|
||||
return imageContent;
|
||||
} catch (error) {
|
||||
logError(`Failed to process image: ${url}`, error);
|
||||
failedImages.push(url);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
static async processChatImageBatch(
|
||||
content: MessageContent[],
|
||||
failedImages: string[],
|
||||
vault: Vault
|
||||
): Promise<ImageProcessingResult> {
|
||||
try {
|
||||
const imageItems = content.filter(
|
||||
(item): item is ImageContent => item.type === "image_url" && !!item.image_url?.url
|
||||
);
|
||||
|
||||
const results = await Promise.all(
|
||||
imageItems.map((item) =>
|
||||
ImageBatchProcessor.processChatSingleImage(item, failedImages, vault)
|
||||
)
|
||||
);
|
||||
|
||||
const successfulImages = results.filter((item): item is ImageContent => item !== null);
|
||||
const failureDescriptions = failedImages.map((url) => `Image read failed for: ${url}`);
|
||||
|
||||
return {
|
||||
successfulImages,
|
||||
failureDescriptions,
|
||||
};
|
||||
} catch (error) {
|
||||
logError("Error processing chat image batch:", error);
|
||||
const imageUrls = content
|
||||
.filter((item): item is ImageContent => item.type === "image_url" && !!item.image_url?.url)
|
||||
.map((item) => item.image_url.url);
|
||||
return {
|
||||
successfulImages: [],
|
||||
failureDescriptions: imageUrls.map((url) => `Image read failed for: ${url}`),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
static async processChatSingleImage(
|
||||
item: ImageContent,
|
||||
failedImages: string[],
|
||||
vault: Vault
|
||||
): Promise<ImageContent | null> {
|
||||
try {
|
||||
const processedContent = await ImageProcessor.convertToBase64(item.image_url.url, vault);
|
||||
|
||||
if (!processedContent) {
|
||||
failedImages.push(item.image_url.url);
|
||||
return null;
|
||||
}
|
||||
|
||||
return processedContent;
|
||||
} catch (error) {
|
||||
logError(`Failed to process chat image: ${item.image_url.url}`, error);
|
||||
failedImages.push(item.image_url.url);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
static showFailedImagesNotice(failedImages: string[]): void {
|
||||
if (failedImages.length > 0) {
|
||||
new Notice(`Failed to process images:\n${failedImages.join("\n")}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
import { ImageProcessor } from "@/imageProcessing/imageProcessor";
|
||||
import { BrevilabsClient, Url4llmResponse } from "@/LLMProviders/brevilabsClient";
|
||||
import { ImageProcessor, isYoutubeUrl } from "@/utils";
|
||||
import { isYoutubeUrl } from "@/utils";
|
||||
|
||||
export interface MentionData {
|
||||
type: string;
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@ export interface CopilotSettings {
|
|||
// undefined means never checked
|
||||
isPlusUser: boolean | undefined;
|
||||
inlineEditCommands: InlineEditCommandSettings[] | undefined;
|
||||
passMarkdownImages: boolean;
|
||||
}
|
||||
|
||||
export const settingsStore = createStore();
|
||||
|
|
@ -210,6 +211,11 @@ export function sanitizeSettings(settings: CopilotSettings): CopilotSettings {
|
|||
? DEFAULT_SETTINGS.embeddingBatchSize
|
||||
: embeddingBatchSize;
|
||||
|
||||
// Ensure passMarkdownImages has a default value
|
||||
if (typeof sanitizedSettings.passMarkdownImages !== "boolean") {
|
||||
sanitizedSettings.passMarkdownImages = DEFAULT_SETTINGS.passMarkdownImages;
|
||||
}
|
||||
|
||||
return sanitizedSettings;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import React from "react";
|
||||
import { SettingItem } from "@/components/ui/setting-item";
|
||||
import { updateSetting, useSettingsValue } from "@/settings/model";
|
||||
import React from "react";
|
||||
|
||||
export const AdvancedSettings: React.FC = () => {
|
||||
const settings = useSettingsValue();
|
||||
|
|
@ -19,6 +19,16 @@ export const AdvancedSettings: React.FC = () => {
|
|||
/>
|
||||
|
||||
<div className="space-y-4">
|
||||
<SettingItem
|
||||
type="switch"
|
||||
title="Pass Images in Markdown (Plus)"
|
||||
description="Pass embedded images in markdown to the AI along with the text. Only works with multimodal models (plus only)."
|
||||
checked={settings.passMarkdownImages}
|
||||
onCheckedChange={(checked) => {
|
||||
updateSetting("passMarkdownImages", checked);
|
||||
}}
|
||||
/>
|
||||
|
||||
<SettingItem
|
||||
type="switch"
|
||||
title="Enable Encryption"
|
||||
|
|
|
|||
279
src/utils.ts
279
src/utils.ts
|
|
@ -10,13 +10,12 @@ import {
|
|||
SettingKeyProviders,
|
||||
USER_SENDER,
|
||||
} from "@/constants";
|
||||
import { logError, logInfo } from "@/logger";
|
||||
import { logInfo } from "@/logger";
|
||||
import { CopilotSettings } from "@/settings/model";
|
||||
import { ChatMessage } from "@/sharedState";
|
||||
import { BaseChatModel } from "@langchain/core/language_models/chat_models";
|
||||
import { MemoryVariables } from "@langchain/core/memory";
|
||||
import { RunnableSequence } from "@langchain/core/runnables";
|
||||
import { Buffer } from "buffer";
|
||||
import { BaseChain, RetrievalQAChain } from "langchain/chains";
|
||||
import moment from "moment";
|
||||
import { MarkdownView, Notice, TFile, Vault, requestUrl } from "obsidian";
|
||||
|
|
@ -465,282 +464,6 @@ export function extractYoutubeUrl(text: string): string | null {
|
|||
return match ? match[0] : null;
|
||||
}
|
||||
|
||||
export interface ImageContent {
|
||||
type: "image_url";
|
||||
image_url: {
|
||||
url: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface TextContent {
|
||||
type: "text";
|
||||
text: string;
|
||||
}
|
||||
|
||||
export type MessageContent = ImageContent | TextContent;
|
||||
|
||||
export class ImageProcessor {
|
||||
private static readonly IMAGE_EXTENSIONS = [
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".png",
|
||||
".gif",
|
||||
".webp",
|
||||
".bmp",
|
||||
".svg",
|
||||
];
|
||||
|
||||
private static readonly MAX_IMAGE_SIZE = 3 * 1024 * 1024; // 3MB
|
||||
private static readonly MIME_TYPES = {
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".png": "image/png",
|
||||
".gif": "image/gif",
|
||||
".webp": "image/webp",
|
||||
".bmp": "image/bmp",
|
||||
".svg": "image/svg+xml",
|
||||
};
|
||||
|
||||
static async isImageUrl(url: string, vault: Vault): Promise<boolean> {
|
||||
try {
|
||||
// First check if it's an Obsidian vault image path
|
||||
if (this.IMAGE_EXTENSIONS.some((ext) => url.toLowerCase().endsWith(ext))) {
|
||||
// Verify the file exists and is accessible
|
||||
const file = vault.getAbstractFileByPath(url);
|
||||
if (!file || !(file instanceof TFile)) {
|
||||
logError("File not found in vault");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check file size
|
||||
if (file.stat.size > this.MAX_IMAGE_SIZE) {
|
||||
logError("File too large:", file.stat.size, "bytes");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// Then check if it's a valid URL
|
||||
const urlObj = new URL(url);
|
||||
|
||||
// First check: URL path ends with image extension
|
||||
if (this.IMAGE_EXTENSIONS.some((ext) => urlObj.pathname.toLowerCase().endsWith(ext))) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Second check: Try HEAD request to check content-type
|
||||
try {
|
||||
const response = await safeFetch(url, {
|
||||
method: "HEAD",
|
||||
headers: {}, // Explicitly set empty headers
|
||||
});
|
||||
|
||||
const contentType = response.headers.get("content-type");
|
||||
if (contentType?.startsWith("image/")) {
|
||||
return true;
|
||||
}
|
||||
} catch (error) {
|
||||
logError("Error checking content-type:", error);
|
||||
}
|
||||
|
||||
// Final check: Analyze URL patterns that commonly indicate image content
|
||||
const searchParams = urlObj.searchParams;
|
||||
const imageIndicators = [
|
||||
// Image dimensions
|
||||
searchParams.has("w") || searchParams.has("width"),
|
||||
searchParams.has("h") || searchParams.has("height"),
|
||||
// Image processing
|
||||
searchParams.has("format"),
|
||||
searchParams.has("fit"),
|
||||
// Image quality
|
||||
searchParams.has("q") || searchParams.has("quality"),
|
||||
// Common CDN image path patterns
|
||||
urlObj.pathname.includes("/image/"),
|
||||
urlObj.pathname.includes("/images/"),
|
||||
urlObj.pathname.includes("/img/"),
|
||||
// Common image processing parameters
|
||||
searchParams.has("auto"),
|
||||
searchParams.has("crop"),
|
||||
];
|
||||
|
||||
// If multiple image-related indicators are present, likely an image URL
|
||||
const imageIndicatorCount = imageIndicators.filter(Boolean).length;
|
||||
return imageIndicatorCount >= 2; // Require at least 2 indicators to consider it an image URL
|
||||
} catch {
|
||||
// If URL construction fails, it might still be a valid Obsidian vault image path
|
||||
return this.IMAGE_EXTENSIONS.some((ext) => url.toLowerCase().endsWith(ext));
|
||||
}
|
||||
}
|
||||
|
||||
private static async handleVaultImage(file: TFile, vault: Vault): Promise<string | null> {
|
||||
try {
|
||||
// Check file size first
|
||||
if (file.stat.size > this.MAX_IMAGE_SIZE) {
|
||||
logError(`Image too large: ${file.stat.size} bytes, skipping: ${file.path}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Read the file as array buffer
|
||||
const arrayBuffer = await vault.readBinary(file);
|
||||
|
||||
// Validate MIME type
|
||||
const mimeType = await this.getMimeType(arrayBuffer, file.extension);
|
||||
if (!mimeType.startsWith("image/")) {
|
||||
logError(`Invalid MIME type: ${mimeType}, skipping: ${file.path}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(arrayBuffer);
|
||||
const base64 = buffer.toString("base64");
|
||||
|
||||
const result = `data:${mimeType};base64,${base64}`;
|
||||
return result;
|
||||
} catch (error) {
|
||||
logError("Error in handleVaultImage:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static async handleWebImage(imageUrl: string): Promise<string | null> {
|
||||
try {
|
||||
const response = await safeFetch(imageUrl, {
|
||||
method: "GET",
|
||||
headers: {},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
logError(`Failed to fetch image: ${response.statusText}, skipping: ${imageUrl}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Try to get content type from response headers
|
||||
const contentType = response.headers.get("content-type");
|
||||
if (!contentType?.startsWith("image/")) {
|
||||
logError(`Invalid content type: ${contentType}, skipping: ${imageUrl}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
|
||||
// Check file size
|
||||
if (arrayBuffer.byteLength > this.MAX_IMAGE_SIZE) {
|
||||
logError(`Image too large: ${arrayBuffer.byteLength} bytes, skipping: ${imageUrl}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(arrayBuffer);
|
||||
const base64 = buffer.toString("base64");
|
||||
return `data:${contentType};base64,${base64}`;
|
||||
} catch (error) {
|
||||
logError("Error converting image to base64:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static async handleLocalImage(imageUrl: string, vault: Vault): Promise<string | null> {
|
||||
try {
|
||||
const localPath = decodeURIComponent(imageUrl.replace("app://", ""));
|
||||
const file = vault.getAbstractFileByPath(localPath);
|
||||
if (!file || !(file instanceof TFile)) {
|
||||
logError(`Local image not found: ${localPath}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check file size
|
||||
if (file.stat.size > this.MAX_IMAGE_SIZE) {
|
||||
logError(`Image too large: ${file.stat.size} bytes, skipping: ${localPath}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Read the file as array buffer
|
||||
const arrayBuffer = await vault.readBinary(file);
|
||||
|
||||
// Validate MIME type
|
||||
const mimeType = await this.getMimeType(arrayBuffer, file.extension);
|
||||
if (!mimeType.startsWith("image/")) {
|
||||
logError(`Invalid MIME type: ${mimeType}, skipping: ${localPath}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(arrayBuffer);
|
||||
const base64 = buffer.toString("base64");
|
||||
|
||||
const result = `data:${mimeType};base64,${base64}`;
|
||||
return result;
|
||||
} catch (error) {
|
||||
logError("Error in handleLocalImage:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static async imageToBase64(imageUrl: string, vault: Vault): Promise<string | null> {
|
||||
// If it's already a data URL, return it as is
|
||||
if (imageUrl.startsWith("data:")) {
|
||||
return imageUrl;
|
||||
}
|
||||
|
||||
// Check if it's a local vault image
|
||||
if (imageUrl.startsWith("app://")) {
|
||||
return await this.handleLocalImage(imageUrl, vault);
|
||||
}
|
||||
|
||||
// Check if it's an Obsidian vault image (direct file path)
|
||||
const file = vault.getAbstractFileByPath(imageUrl);
|
||||
if (file instanceof TFile) {
|
||||
return await this.handleVaultImage(file, vault);
|
||||
}
|
||||
|
||||
// Handle web images
|
||||
return await this.handleWebImage(imageUrl);
|
||||
}
|
||||
|
||||
static async convertToBase64(imageUrl: string, vault: Vault): Promise<ImageContent | null> {
|
||||
const base64Url = await this.imageToBase64(imageUrl, vault);
|
||||
if (!base64Url) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: base64Url,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static async getMimeType(arrayBuffer: ArrayBuffer, extension: string): Promise<string> {
|
||||
// Get the first few bytes to check for magic numbers
|
||||
const bytes = new Uint8Array(arrayBuffer.slice(0, 4));
|
||||
|
||||
// Check for common image magic numbers
|
||||
if (bytes[0] === 0xff && bytes[1] === 0xd8) {
|
||||
return "image/jpeg";
|
||||
}
|
||||
if (bytes[0] === 0x89 && bytes[1] === 0x50) {
|
||||
return "image/png";
|
||||
}
|
||||
if (bytes[0] === 0x47 && bytes[1] === 0x49) {
|
||||
return "image/gif";
|
||||
}
|
||||
if (bytes[0] === 0x52 && bytes[1] === 0x49) {
|
||||
return "image/webp";
|
||||
}
|
||||
if (bytes[0] === 0x42 && bytes[1] === 0x4d) {
|
||||
return "image/bmp";
|
||||
}
|
||||
if (bytes[0] === 0x3c && bytes[1] === 0x73) {
|
||||
return "image/svg+xml";
|
||||
}
|
||||
|
||||
// Fall back to extension-based detection
|
||||
const mimeType = this.MIME_TYPES[extension.toLowerCase() as keyof typeof this.MIME_TYPES];
|
||||
if (!mimeType) {
|
||||
throw new Error(`Unsupported image extension: ${extension}`);
|
||||
}
|
||||
return mimeType;
|
||||
}
|
||||
}
|
||||
|
||||
/** Proxy function to use in place of fetch() to bypass CORS restrictions.
|
||||
* It currently doesn't support streaming until this is implemented
|
||||
* https://forum.obsidian.md/t/support-streaming-the-request-and-requesturl-response-body/87381 */
|
||||
|
|
|
|||
Loading…
Reference in a new issue