feat: add image and PDF support to ReadVaultFiles function

Add support for reading images (PNG, JPG, JPEG, WebP) and PDFs in ReadVaultFiles. Implement provider-specific formatting for binary content (Claude, OpenAI, Gemini). Update conversation structure to handle binary files separately from text responses. Add unpdf library for PDF text extraction in search.

AIClasses/BaseAIClass.ts

@@ -43,6 +43,8 @@ export abstract class BaseAIClass implements IAIClass {
         abortSignal?: AbortSignal
     ): AsyncGenerator<IStreamChunk, void, unknown>;
 
+    public abstract formatBinaryFilesForUser(files: Array<{type: string, path: string, contents: string}>): string;
+
     protected abstract parseStreamChunk(chunk: string): IStreamChunk;
     protected abstract extractContents(conversationContent: ConversationContent[]): unknown;
     protected abstract mapFunctionDefinitions(aiFunctionDefinitions: IAIFunctionDefinition[]): object;

AIClasses/Claude/Claude.ts

@@ -9,10 +9,13 @@ import type { ConversationContent } from "Conversations/ConversationContent";
 import { Role } from "Enums/Role";
 import type { RawMessageStreamEvent, ContentBlockParam, Tool } from '@anthropic-ai/sdk/resources/messages';
 import { Exception } from "Helpers/Exception";
+import * as path from "path-browserify";
+import { FileType, getImageMimeType, isFileType } from "Enums/FileType";
 
 export class Claude extends BaseAIClass {
 
     private readonly STOP_REASON_TOOL_USE: string = "tool_use";
+    private readonly SUPPORTED_IMAGE_TYPES: string[] = ["image/jpeg", "image/png", "image/gif", "image/webp"];
 
     private accumulatedFunctionName: string | null = null;
     private accumulatedFunctionArgs: string = "";
@@ -184,6 +187,12 @@ export class Claude extends BaseAIClass {
                     }
                 }
 
+                // Add provider-specific content if present (e.g., binary files)
+                if (content.isProviderSpecificContent && contentToExtract.trim() !== "") {
+                    const rawContent = JSON.parse(contentToExtract) as ContentBlockParam[];
+                    contentBlocks.push(...rawContent);
+                }
+
                 // Add function response if present
                 if (content.isFunctionCallResponse && contentToExtract.trim() !== "") {
                     const parsedContent = this.parseFunctionResponse(contentToExtract);
@@ -228,4 +237,46 @@ export class Claude extends BaseAIClass {
             }
         }));
     }
+
+    public formatBinaryFilesForUser(files: Array<{type: string, path: string, contents: string}>): string {
+        const contentBlocks = files.flatMap(file => {
+            const extension = path.extname(file.path).substring(1).toLowerCase();
+
+            let mimeType: string;
+            let blockType: string;
+
+            if (isFileType(file.type, FileType.PDF)) {
+                mimeType = "application/pdf";
+                blockType = "document";
+            } else {
+                try {
+                    mimeType = getImageMimeType(extension);
+                    blockType = "image";
+
+                    if (!this.SUPPORTED_IMAGE_TYPES.includes(mimeType)) {
+                        return [
+                            { type: "text", text: `Unsupported image format: ${path.basename(file.path)}` }
+                        ];
+                    }
+                } catch (error) {
+                    return [
+                        { type: "text", text: Exception.messageFrom(error) }
+                    ];
+                }
+            }
+
+            return [
+                {type: "text", text: path.basename(file.path)},
+                {
+                    type: blockType,
+                    source: {
+                        type: "base64",
+                        media_type: mimeType,
+                        data: file.contents
+                    }
+                }
+            ];
+        });
+        return JSON.stringify(contentBlocks);
+    }
 }

AIClasses/FunctionDefinitions/AIFunctionResponse.ts

@@ -1,7 +1,10 @@
 // Platform agnostic class for function responses
+
+import type { AIFunction } from "Enums/AIFunction";
+
 // Used by AI providers to format function execution results for API calls
 export class AIFunctionResponse {
-    public readonly name: string;
+    public readonly name: AIFunction;
     public readonly response: object;
     public readonly toolId?: string;
 
@@ -12,7 +15,7 @@ export class AIFunctionResponse {
                                                            
     public static readonly UserSuggestionMessage: string = "The user has rejected the change with the following suggestion: ";
 
-    constructor(name: string, response: object, toolId?: string) {
+    constructor(name: AIFunction, response: object, toolId?: string) {
         this.name = name;
         this.response = response;
         this.toolId = toolId;

AIClasses/FunctionDefinitions/Functions/ReadVaultFiles.ts

@@ -4,13 +4,15 @@ import type { IAIFunctionDefinition } from "../IAIFunctionDefinition";
 export const ReadVaultFiles: IAIFunctionDefinition = {
   name: AIFunction.ReadVaultFiles,
   description: `Reads and returns the complete content of one or more files from the vault.
-                Call this when you need to access existing note content to answer questions,
+                Call this when you need to access existing file content to answer questions,
                 provide summaries, verify information, or gather context before making updates.
                 Use proactively before updating files to understand current content and avoid
                 data loss. Essential for any operation that references or builds upon existing notes.
 
                 For multiple files: Use when comparing content, gathering related context, or
-                analyzing information across several documents.`,
+                analyzing information across several documents.
+
+                Supports text files (.md, .txt, etc.), images (.png, .jpg, .jpeg, etc), and PDFs (.pdf).`,
   parameters: {
     type: "object",
     properties: {
@@ -19,7 +21,7 @@ export const ReadVaultFiles: IAIFunctionDefinition = {
         items: {
           type: "string"
         },
-        description: "Array of full paths to files within the vault. Can contain a single file path ['folder/note.md'] or multiple paths ['folder/note1.md', 'folder/note2.md']. Each path must be exact and point to an existing file.",
+        description: "Array of full paths to files within the vault. Can contain a single file path ['folder/note.md'] or multiple paths ['folder/note1.md', 'folder/note2.pdf']. Each path must be exact and point to an existing file.",
       },
       user_message: {
         type: "string",

AIClasses/Gemini/Gemini.ts

@@ -9,10 +9,14 @@ import type { IAIFunctionDefinition } from "AIClasses/FunctionDefinitions/IAIFun
 import type { ConversationContent } from "Conversations/ConversationContent";
 import type { Candidate, Part, FunctionDeclaration } from "@google/genai";
 import { FinishReason } from "@google/genai";
+import * as path from "path-browserify";
+import { FileType, getImageMimeType, isFileType } from "Enums/FileType";
+import { Exception } from "Helpers/Exception";
 
 export class Gemini extends BaseAIClass {
 
   private readonly REQUEST_WEB_SEARCH: string = "request_web_search";
+  private readonly SUPPORTED_IMAGE_TYPES: string[] = ["image/jpeg", "image/png"];
 
   private accumulatedFunctionName: string | null = null;
   private accumulatedFunctionArgs: Record<string, unknown> = {};
@@ -190,6 +194,12 @@ export class Gemini extends BaseAIClass {
           }
         }
 
+        // Add provider-specific content if present (e.g., binary files)
+        if (content.isProviderSpecificContent && contentToExtract.trim() !== "") {
+          const rawContent = JSON.parse(contentToExtract) as Part[];
+          parts.push(...rawContent);
+        }
+
         // Add function response if present
         if (content.isFunctionCallResponse && contentToExtract.trim() !== "") {
           const parsedContent = this.parseFunctionResponse(contentToExtract);
@@ -232,4 +242,44 @@ export class Gemini extends BaseAIClass {
       parameters: functionDefinition.parameters as FunctionDeclaration['parameters']
     }));
   }
+
+  public formatBinaryFilesForUser(files: Array<{type: string, path: string, contents: string}>): string {
+    const parts: unknown[] = [];
+
+    for (const file of files) {
+      const extension = path.extname(file.path).substring(1).toLowerCase();
+
+      let mimeType: string;
+
+      if (isFileType(file.type, FileType.PDF)) {
+        mimeType = "application/pdf";
+      } else {
+        try {
+          mimeType = getImageMimeType(extension);
+
+          if (!this.SUPPORTED_IMAGE_TYPES.includes(mimeType)) {
+            parts.push({
+              text: `Unsupported image format: ${path.basename(file.path)}`
+            });
+            continue;
+          }
+        } catch (error) {
+          parts.push({
+            text: Exception.messageFrom(error)
+          });
+          continue;
+        }
+      }
+
+      parts.push({text: path.basename(file.path)});
+      parts.push({
+        inlineData: {
+          mimeType: mimeType,
+          data: file.contents
+        }
+      });
+    }
+
+    return JSON.stringify(parts);
+  }
 }

AIClasses/IAIClass.ts

@@ -3,4 +3,5 @@ import type { Conversation } from "Conversations/Conversation";
 
 export interface IAIClass {
     streamRequest(conversation: Conversation, allowDestructiveActions: boolean): AsyncGenerator<IStreamChunk, void, unknown>;
+    formatBinaryFilesForUser(files: Array<{type: string, path: string, contents: string}>): string;
 }

AIClasses/OpenAI/OpenAI.ts

@@ -9,9 +9,13 @@ import type { IAIFunctionDefinition } from "AIClasses/FunctionDefinitions/IAIFun
 import type { ResponseEvent, ResponseOutputTextDelta, ResponseOutputItemDone, ResponseErrorEvent, ResponseFailedEvent, OpenAIFunctionTool, ResponsesAPIInput } from "./OpenAITypes";
 import { Exception } from "Helpers/Exception";
 import { ApiErrorType } from "Types/ApiError";
+import * as path from "path-browserify";
+import { FileType, getImageMimeType, isFileType } from "Enums/FileType";
 
 export class OpenAI extends BaseAIClass {
 
+    private readonly SUPPORTED_IMAGE_TYPES: string[] = ["image/jpeg", "image/png", "image/webp"];
+
     public constructor() {
         super(AIProvider.OpenAI);
     }
@@ -228,7 +232,14 @@ export class OpenAI extends BaseAIClass {
                 continue;
             }
 
-            // Case 2: Function call response
+            // Case 2: Provider-specific content (e.g., binary files)
+            if (content.isProviderSpecificContent && contentToExtract.trim() !== "") {
+                const rawContent = JSON.parse(contentToExtract) as ResponsesAPIInput[];
+                results.push(...rawContent);
+                continue;
+            }
+
+            // Case 3: Function call response
             if (content.isFunctionCallResponse && contentToExtract.trim() !== "") {
                 const parsedContent = this.parseFunctionResponse(contentToExtract);
 
@@ -258,7 +269,7 @@ export class OpenAI extends BaseAIClass {
                 continue;
             }
 
-            // Case 3: Regular text message (user or assistant)
+            // Case 4: Regular text message (user or assistant)
             if (contentToExtract.trim() !== "") {
                 results.push({
                     role: content.role as "user" | "assistant",
@@ -278,4 +289,44 @@ export class OpenAI extends BaseAIClass {
             parameters: functionDefinition.parameters
         }));
     }
+
+    public formatBinaryFilesForUser(files: Array<{type: string, path: string, contents: string}>): string {
+        const contentBlocks: unknown[] = [];
+
+        for (const file of files) {
+            const extension = path.extname(file.path).substring(1).toLowerCase();
+
+            let mimeType: string;
+
+            if (isFileType(file.type, FileType.PDF)) {
+                mimeType = "application/pdf";
+            } else {
+                try {
+                    mimeType = getImageMimeType(extension);
+
+                    if (!this.SUPPORTED_IMAGE_TYPES.includes(mimeType)) {
+                        contentBlocks.push({
+                            type: "input_text",
+                            text: `Unsupported image format: ${path.basename(file.path)}`
+                        });
+                        continue;
+                    }
+                } catch (error) {
+                    contentBlocks.push({
+                        type: "input_text",
+                        text: Exception.messageFrom(error)
+                    });
+                    continue;
+                }
+            }
+
+            contentBlocks.push({
+                type: "input_file",
+                filename: path.basename(file.path),
+                file_data: `data:${mimeType};base64,${file.contents}`
+            });
+        }
+
+        return JSON.stringify(contentBlocks);
+    }
 }

AIClasses/SystemPrompt.ts

@@ -114,6 +114,13 @@ Regex is your most versatile search capability. Use it aggressively:
 
 **Only after exhausting all tiers**: Acknowledge search scope, explain strategies attempted, suggest alternatives.
 
+### Non-Markdown Content
+
+When searches return or reference images or PDFs:
+- **Read them** rather than just noting their existence
+- Extract relevant information to answer the user's query
+- Reference the source file with [[wiki-links]] as usual
+
 ## Multi-Tool Workflow Architecture
 
 ### Planning Phase (for Complex Queries)
@@ -143,6 +150,7 @@ Before executing complex queries:
 ## Core Capabilities
 
 **Knowledge Operations**
+- Reading and analyzing images and PDFs stored in the vault
 - Finding and synthesizing information across notes with bi-directional links
 - Understanding graph connections, tags, and metadata relationships
 - Creating atomic notes with proper [[wiki-link]] syntax
@@ -167,6 +175,7 @@ Before executing complex queries:
 ❌ Describing what you'd create instead of creating it
 ❌ Providing generic answers when vault contains specific information
 ❌ Mimicking historical tool call formats instead of using native functions
+❌ Noting that a PDF/image exists without reading its contents when relevant
 
 ## Decision Framework
 

Components/ChatArea.svelte

@@ -21,6 +21,7 @@
   export let editModeActive: boolean = false;
 
   export function resetChatArea() {
+    autoScroll = true;
     cancelling = false;
     messageElements = [];
     lastProcessedContent.clear();
@@ -68,7 +69,7 @@
       chatAreaPaddingElement.style.padding = `${Math.max(0, padding / 2)}px`;
 
       tick().then(() => {
-        if (autoScroll && behavior) {
+        if (behavior && (autoScroll || shouldSettle)) {
           chatContainer.scroll({ top: chatContainer.scrollHeight, behavior: behavior })
         }
       });
@@ -76,7 +77,9 @@
 
     if (behavior === "instant" || shouldSettle) {
       tick().then(() => {
-        performLayoutCalculations();
+        requestAnimationFrame(() => {
+          performLayoutCalculations();
+        });
       });
     } else {
       layoutUpdateTimeout = setTimeout(() => {
@@ -243,7 +246,7 @@
 
 <div class="chat-area" bind:this={chatContainer} on:scroll={handleScroll} use:observeResize>
   {#each messages as message, index}
-    {#if !message.isFunctionCallResponse && message.content.trim() !== ""}
+    {#if !message.isFunctionCallResponse && !message.isProviderSpecificContent && message.content.trim() !== ""}
       {#if message.role === Role.User}
         <div class="message-container {Role.User}" use:trackingAction={index}>
           <div class="message-bubble {Role.User}">

Conversations/Conversation.ts

@@ -1,5 +1,9 @@
 import { StringTools } from "Helpers/StringTools";
 import { ConversationContent } from "./ConversationContent";
+import type { AIFunctionResponse } from "AIClasses/FunctionDefinitions/AIFunctionResponse";
+import { Role } from "Enums/Role";
+import { AIFunction } from "Enums/AIFunction";
+import { isTextFile } from "Enums/FileType";
 
 export class Conversation {
 
@@ -15,6 +19,89 @@ export class Conversation {
         this.updated = new Date();
         this.title = `${StringTools.dateToString(this.created)}`;
     }
+
+    public addFunctionResponse(
+        functionResponse: AIFunctionResponse,
+        formatBinaryFiles?: (files: Array<{type: string, path: string, contents: string}>) => string
+    ): void {
+        if (functionResponse.name !== AIFunction.ReadVaultFiles) {
+            const functionResponseString = functionResponse.toConversationString();
+            this.contents.push(new ConversationContent(
+                Role.User,
+                functionResponseString,
+                functionResponseString,
+                "",
+                new Date(),
+                false,
+                true,
+                false,
+                functionResponse.toolId
+            ));
+            return;
+        }
+
+        const results = (functionResponse.response as
+            {results: Array<{type: string, path: string, contents: string}>}).results;
+
+        const textResults = results.filter(result => isTextFile(result.type));
+        const binaryResults = results.filter(result => !isTextFile(result.type));
+
+        // 1. Function response with text files (or success message if none)
+        let functionResponseData;
+        if (textResults.length > 0) {
+            // Include text file contents in function response
+            functionResponseData = {
+                id: functionResponse.toolId,
+                functionResponse: {
+                    name: functionResponse.name,
+                    response: {
+                        results: textResults,
+                        ...(binaryResults.length > 0 && {message: "Binary files follow in next message"})
+                    }
+                }
+            };
+        } else {
+            // No text files - just acknowledge success
+            functionResponseData = {
+                id: functionResponse.toolId,
+                functionResponse: {
+                    name: functionResponse.name,
+                    response: {
+                        message: "Files retrieved successfully. Binary content follows in next message.",
+                        count: binaryResults.length
+                    }
+                }
+            };
+        }
+
+        this.contents.push(new ConversationContent(
+            Role.User,
+            JSON.stringify(functionResponseData),
+            JSON.stringify(functionResponseData),
+            "",
+            new Date(),
+            false,
+            true,
+            false,
+            functionResponse.toolId
+        ));
+
+        // 2. If there are non-text files, add follow-up user message
+        if (binaryResults.length > 0 && formatBinaryFiles) {
+            const providerContent = formatBinaryFiles(binaryResults);
+
+            this.contents.push(new ConversationContent(
+                Role.User,
+                providerContent,
+                providerContent,
+                "",
+                new Date(),
+                false,
+                false,
+                true
+            ));
+        }
+    }
     
     public static isConversationData(data: unknown): data is { title: string; created: string; updated: string; contents: ConversationContent[] } {
         return (

Conversations/ConversationContent.ts

@@ -9,11 +9,39 @@ export class ConversationContent {
     timestamp: Date;
     isFunctionCall: boolean;
     isFunctionCallResponse: boolean;
+    isProviderSpecificContent: boolean;
     toolId?: string;
     thoughtSignature?: string;
     errorType?: ApiErrorType;
 
-    constructor(role: Role, content: string = "", promptContent: string = "", functionCall: string = "", timestamp: Date = new Date(), isFunctionCall = false, isFunctionCallResponse = false, toolId?: string, thoughtSignature?: string, errorType?: ApiErrorType) {
+    /**
+     * Creates a conversation content entry.
+     *
+     * @param role - The role of the message sender (User or Assistant)
+     * @param content - The display content shown in the UI (for User messages, this is the user's input; for Assistant, it's the AI response)
+     * @param promptContent - The content sent to the AI provider (often same as content, but may differ for system-generated messages)
+     * @param functionCall - JSON string of the function call data (only set when isFunctionCall=true)
+     * @param timestamp - When this content was created
+     * @param isFunctionCall - True if this is an Assistant message containing a function/tool call
+     * @param isFunctionCallResponse - True if this is a User message containing the response to a function call (hides from UI)
+     * @param isProviderSpecificContent - True if this contains provider-specific formatted content (e.g., binary files in Claude/OpenAI/Gemini format; hides from UI)
+     * @param toolId - Unique identifier for tool calls/responses (used to match calls with their responses)
+     * @param thoughtSignature - Gemini-specific thought signature for extended thinking
+     * @param errorType - If present, indicates this message contains an error
+     */
+    constructor(
+        role: Role,
+        content: string = "",
+        promptContent: string = "",
+        functionCall: string = "",
+        timestamp: Date = new Date(),
+        isFunctionCall = false,
+        isFunctionCallResponse = false,
+        isProviderSpecificContent = false,
+        toolId?: string,
+        thoughtSignature?: string,
+        errorType?: ApiErrorType
+    ) {
         this.role = role;
         this.content = content;
         this.promptContent = promptContent;
@@ -21,6 +49,7 @@ export class ConversationContent {
         this.timestamp = timestamp;
         this.isFunctionCall = isFunctionCall;
         this.isFunctionCallResponse = isFunctionCallResponse;
+        this.isProviderSpecificContent = isProviderSpecificContent;
         this.toolId = toolId;
         this.thoughtSignature = thoughtSignature;
         this.errorType = errorType;
@@ -28,7 +57,7 @@ export class ConversationContent {
 
     public static isConversationContentData(this: void, data: unknown): data is {
         role: string; content: string; promptContent: string; functionCall: string; timestamp: string, isFunctionCall: boolean,
-        isFunctionCallResponse: boolean, toolId?: string, thoughtSignature?: string, errorType?: string
+        isFunctionCallResponse: boolean, isProviderSpecificContent: boolean, toolId?: string, thoughtSignature?: string, errorType?: string
     } {
         return (
             data !== null &&
@@ -40,6 +69,7 @@ export class ConversationContent {
             "timestamp" in data &&
             "isFunctionCall" in data &&
             "isFunctionCallResponse" in data &&
+            "isProviderSpecificContent" in data &&
             typeof data.role === "string" &&
             typeof data.content === "string" &&
             typeof data.promptContent === "string" &&
@@ -47,6 +77,7 @@ export class ConversationContent {
             typeof data.timestamp === "string" &&
             typeof data.isFunctionCall === "boolean" &&
             typeof data.isFunctionCallResponse === "boolean" &&
+            typeof data.isProviderSpecificContent === "boolean" &&
 
             // optional conversation data fields
             (!("toolId" in data) || typeof data.toolId === "string") &&

Enums/FileType.ts

@@ -0,0 +1,126 @@
+import { Exception } from "Helpers/Exception";
+
+export enum FileType {
+
+    // ----- Types officially supported by Obsidian -----
+
+    MD = "md",
+    BASE = "base",
+    CANVAS = "canvas",
+
+    // images
+    AVIF = "avif",
+    BMP = "bmp",
+    GIF = "gif",
+    JPEG = "jpeg",
+    JPG = "jpg",
+    PNG = "png",
+    SVG = "svg",
+    WEBP = "webp",
+
+    WEBM = "webm",
+
+    // audio
+    FLAC = "flac",
+    M4A = "m4a",
+    MP3 = "mp3",
+    OGG = "ogg",
+    WAV = "wav",
+    WEBM_AUDIO = WEBM,
+    THREE_GP = "3gp",
+
+    // video
+    MKV = "mkv",
+    MOV = "mov",
+    MP4 = "mp4",
+    OGV = "ogv",
+    WEBM_VIDEO = WEBM,
+
+    // pdf
+    PDF = "pdf",
+
+    // ----- Types not officially supported by Obsidian -----
+
+    TXT = "txt",
+    JSON = "json",
+    XML = "xml",
+    HTML = "html",
+    CSV = "csv",
+    YAML = "yaml",
+    YML = "yml",
+    CSS = "css",
+    JS = "js",
+    TS = "ts",
+    JSX = "jsx",
+    TSX = "tsx",
+    SASS = "sass",
+    SCSS = "scss",
+    TEX = "tex",
+    INI = "ini",
+    CFG = "cfg",
+    CONF = "conf",
+    LOG = "log",
+    TOML = "toml",
+    RTF = "rtf",
+}
+
+export function getImageMimeType(extension: string): string {
+    switch (extension.toLowerCase()) {
+        case FileType.AVIF as string:
+            return "image/avif";
+        case FileType.BMP as string:
+            return "image/bmp";
+        case FileType.GIF as string:
+            return "image/gif";
+        case FileType.JPEG as string:
+        case FileType.JPG as string:
+            return "image/jpeg";
+        case FileType.PNG as string:
+            return "image/png";
+        case FileType.SVG as string:
+            return "image/svg+xml";
+        case FileType.WEBP as string:
+            return "image/webp";
+        default:
+            Exception.throw(`Image type not supported: ${extension}`);
+    }
+}
+
+export function isKnownFileType(value: string): value is FileType {
+    return Object.values(FileType).includes(value as FileType);
+}
+
+export function isFileType(value: string, fileType: FileType) {
+    return value === fileType.toString();
+}
+
+export function isBinaryFile(extension: string) {
+    return isKnownFileType(extension) && !isTextFile(extension);
+}
+
+export function isTextFile(extension: string) {
+    return isFileType(extension, FileType.MD)
+        || isFileType(extension, FileType.BASE)
+        || isFileType(extension, FileType.CANVAS)
+        || isFileType(extension, FileType.TXT)
+        || isFileType(extension, FileType.JSON)
+        || isFileType(extension, FileType.XML)
+        || isFileType(extension, FileType.HTML)
+        || isFileType(extension, FileType.CSV)
+        || isFileType(extension, FileType.YAML)
+        || isFileType(extension, FileType.YML)
+        || isFileType(extension, FileType.CSS)
+        || isFileType(extension, FileType.JS)
+        || isFileType(extension, FileType.TS)
+        || isFileType(extension, FileType.JSX)
+        || isFileType(extension, FileType.TSX)
+        || isFileType(extension, FileType.SASS)
+        || isFileType(extension, FileType.SCSS)
+        || isFileType(extension, FileType.TEX)
+        || isFileType(extension, FileType.INI)
+        || isFileType(extension, FileType.CFG)
+        || isFileType(extension, FileType.CONF)
+        || isFileType(extension, FileType.LOG)
+        || isFileType(extension, FileType.TOML)
+        || isFileType(extension, FileType.RTF);
+}

Services/AIFunctionService.ts

@@ -8,6 +8,7 @@ import type { ISearchMatch } from "../Helpers/SearchTypes";
 import { AbortService } from "./AbortService";
 import { normalizePath, TAbstractFile, TFile } from "obsidian";
 import { Exception } from "Helpers/Exception";
+import * as path from "path-browserify";
 import { 
     SearchVaultFilesArgsSchema,
     ReadVaultFilesArgsSchema,
@@ -159,7 +160,11 @@ export class AIFunctionService {
                 if (result instanceof Error) {
                     return { path: filePath, error: result.message }
                 }
-                return { path: filePath, contents: result }
+                return {
+                    type: path.extname(filePath).substring(1).toLocaleLowerCase(),
+                    path: filePath,
+                    contents: result
+                }
             })
         );
         return { results };

Services/ChatService.ts

@@ -60,7 +60,7 @@ export class ChatService {
 	}
 
 	public async submit(conversation: Conversation, allowDestructiveActions: boolean, userRequest: string, formattedRequest: string, callbacks: IChatServiceCallbacks) {
-		if (!await this.semaphore.wait()) {
+		if (this.ai === undefined || !await this.semaphore.wait()) {
 			return;
 		}
 
@@ -86,7 +86,7 @@ export class ChatService {
 				}
 
 				// Process AI responses and function calls
-				let response = await this.streamRequestResponse(conversation, allowDestructiveActions, callbacks);
+				let response = await this.streamRequestResponse(this.ensureCorrectConversationStructure(conversation), allowDestructiveActions, callbacks);
 				while (response.functionCall || response.shouldContinue) {
 					if (response.functionCall) {
 						const userMessage = response.functionCall.arguments.user_message;
@@ -95,17 +95,15 @@ export class ChatService {
 						}
 
 						const functionResponse = await this.aiFunctionService.performAIFunction(response.functionCall);
-
-						const functionResponseString = functionResponse.toConversationString();
-						conversation.contents.push(new ConversationContent(
-							Role.User, functionResponseString, functionResponseString, "", new Date(), false, true, functionResponse.toolId
-						));
+						conversation.addFunctionResponse(
+							functionResponse,
+							(files) => this.ai!.formatBinaryFilesForUser(files)
+						);
 					} else {
 						callbacks.onThoughtUpdate(Copy.AIThoughtMessage);
 					}
 
-					this.ensureCorrectConversationStructure(conversation);
-					response = await this.streamRequestResponse(conversation, allowDestructiveActions, callbacks);
+					response = await this.streamRequestResponse(this.ensureCorrectConversationStructure(conversation), allowDestructiveActions, callbacks);
 				}
 			});
 		} catch (error) {
@@ -168,7 +166,7 @@ export class ChatService {
 		}
 	}
 
-	private ensureCorrectConversationStructure(conversation: Conversation) {
+	private ensureCorrectConversationStructure(conversation: Conversation): Conversation {
 		// Check if the last message is from the assistant to prevent assistant-to-assistant structure
 		// This can happen when the assistant's last message had no function call and the user sends a new request
 		if (conversation.contents.length > 0) {
@@ -178,6 +176,7 @@ export class ChatService {
 				conversation.contents.push(ConversationContent.safeContinue());
 			}
 		}
+		return conversation;
 	}
 
 	private async streamRequestResponse(

Services/ConversationFileSystemService.ts

@@ -45,6 +45,7 @@ export class ConversationFileSystemService {
                     timestamp: content.timestamp.toISOString(),
                     isFunctionCall: content.isFunctionCall,
                     isFunctionCallResponse: content.isFunctionCallResponse,
+                    isProviderSpecificContent: content.isProviderSpecificContent,
                     toolId: content.toolId,
                     thoughtSignature: content.thoughtSignature,
                     errorType: content.errorType
@@ -110,6 +111,7 @@ export class ConversationFileSystemService {
                         new Date(content.timestamp),
                         content.isFunctionCall,
                         content.isFunctionCallResponse,
+                        content.isProviderSpecificContent,
                         content.toolId,
                         content.thoughtSignature,
                         content.errorType

Services/VaultService.ts

@@ -1,4 +1,4 @@
-import { FileManager, TAbstractFile, TFile, TFolder, type Vault } from "obsidian";
+import { arrayBufferToBase64, FileManager, TAbstractFile, TFile, TFolder, type Vault } from "obsidian";
 import { Resolve } from "./DependencyService";
 import { Services } from "./Services";
 import type VaultkeeperAIPlugin from "main";
@@ -16,6 +16,8 @@ import * as path from "path-browserify";
 import { Event } from "Enums/Event";
 import { AbortService } from "./AbortService";
 import { AIFunctionResponse } from "AIClasses/FunctionDefinitions/AIFunctionResponse";
+import { extractText } from 'unpdf';
+import { FileType, isBinaryFile, isFileType } from "Enums/FileType";
 
 interface IFileEventArgs {
     oldPath: string;
@@ -37,7 +39,7 @@ export class VaultService {
 
     public constructor() {
         this.plugin = Resolve<VaultkeeperAIPlugin>(Services.VaultkeeperAIPlugin);
-        
+
         this.vault = this.plugin.app.vault;
         this.fileManager = this.plugin.app.fileManager
 
@@ -77,12 +79,18 @@ export class VaultService {
         return await this.vault.adapter.exists(filePath);
     }
 
-    public async read(file: TFile, allowAccessToPluginRoot: boolean = false): Promise<string> {
+    public async read(file: TFile, allowAccessToPluginRoot: boolean = false): Promise<string | Error> {
         const filePath = this.sanitiserService.sanitize(file.path);
         if (this.isExclusion(filePath, allowAccessToPluginRoot)) {
             Exception.log(`Plugin attempted to read a file that is in the exclusions list: ${filePath}`);
-            return "";
+            return Exception.new(`File does not exist: ${filePath}`);
         }
+
+        if (isBinaryFile(file.extension.toLowerCase())) {
+            const arrayBuffer = await this.vault.readBinary(file);
+            return arrayBufferToBase64(arrayBuffer);
+        }
+
         return await this.vault.read(file);
     }
 
@@ -93,6 +101,10 @@ export class VaultService {
             return Exception.new(`Failed to create file, permission denied: ${filePath}`);
         }
 
+        if (path.extname(filePath) === "pdf") {
+            return Exception.new("Creating PDF files is not supported");
+        }
+
         const fileName = path.basename(filePath);
         return this.proposeChange(fileName, fileName, "", content, requiresConfirmation, async () => {
             await this.createDirectories(filePath, allowAccessToPluginRoot);
@@ -107,7 +119,17 @@ export class VaultService {
             return Exception.new(`File does not exist: ${filePath}`);
         }
 
-        return this.proposeChange(file.name, file.name, await this.read(file, allowAccessToPluginRoot), content, requiresConfirmation, async () => {
+        if (isFileType(file.extension.toLocaleLowerCase(), FileType.PDF)) {
+            return Exception.new("Modifying PDF files is not supported");
+        }
+
+        const currentContent = await this.read(file, allowAccessToPluginRoot);
+
+        if (currentContent instanceof Error) {
+            return currentContent;
+        }
+
+        return this.proposeChange(file.name, file.name, currentContent, content, requiresConfirmation, async () => {
             await this.vault.process(file, () => content);
             return file;
         });
@@ -120,8 +142,16 @@ export class VaultService {
             return Exception.new(`File does not exist: ${filePath}`);
         }
 
+        if (isFileType(path.extname(filePath), FileType.PDF)) {
+            return Exception.new("Creating PDF files is not supported");
+        }
+
         const currentContent = await this.read(file, allowAccessToPluginRoot);
 
+        if (currentContent instanceof Error) {
+            return currentContent;
+        }
+
         if (!currentContent.includes(oldContent)) {
             return Exception.new(`Content to replace was not found in the file. The old content must match exactly.`);
         }
@@ -143,8 +173,18 @@ export class VaultService {
 
         // handle file deletion
         if (file instanceof TFile) {
-            return this.proposeChange(file.name, file.name, await this.read(file, allowAccessToPluginRoot), "", requiresConfirmation, async () => {
-                await this.fileManager.trashFile(file);    
+            const currentContent = await this.read(file, allowAccessToPluginRoot)
+
+            if (currentContent instanceof Error) {
+                return currentContent;
+            }
+
+            if (isFileType(path.extname(filePath), FileType.PDF)) {
+                await this.fileManager.trashFile(file);
+            }
+
+            return this.proposeChange(file.name, file.name, currentContent, "", requiresConfirmation, async () => {
+                await this.fileManager.trashFile(file);
             });
         }
 
@@ -161,7 +201,7 @@ export class VaultService {
         sourcePath = this.sanitiserService.sanitize(sourcePath);
         destinationPath = this.sanitiserService.sanitize(destinationPath);
         const file = this.getAbstractFileByPath(sourcePath, allowAccessToPluginRoot);
-        
+
         if (file === null) {
             return Exception.new(`File does not exist: ${sourcePath}`);
         }
@@ -283,7 +323,14 @@ export class VaultService {
             const batch = shuffledFiles.slice(i, i + BATCH_SIZE);
             const batchPromises = batch.map(async (file) => {
                 try {
-                    const content = await this.vault.cachedRead(file);
+                    let content;
+                    if (isFileType(file.extension.toLocaleLowerCase(), FileType.PDF)) {
+                        const arrayBuffer = await this.vault.readBinary(file);
+                        content = (await extractText(new Uint8Array(arrayBuffer), { mergePages: true })).text;
+                    } else {
+                        content = await this.vault.cachedRead(file);
+                    }
+
                     const snippets = this.extractSnippets(content, regex);
 
                     // Check filename match

package-lock.json

@@ -34,6 +34,7 @@
 				"remark-rehype": "^11.1.2",
 				"remark-wiki-link": "^2.0.1",
 				"unified": "^11.0.5",
+				"unpdf": "^1.4.0",
 				"uuid": "^13.0.0",
 				"zod": "^4.1.13"
 			},
@@ -9448,6 +9449,20 @@
 				"url": "https://opencollective.com/unified"
 			}
 		},
+		"node_modules/unpdf": {
+			"version": "1.4.0",
+			"resolved": "https://registry.npmjs.org/unpdf/-/unpdf-1.4.0.tgz",
+			"integrity": "sha512-TahIk0xdH/4jh/MxfclzU79g40OyxtP00VnEUZdEkJoYtXAHWLiir6t3FC6z3vDqQTzc2ZHcla6uEiVTNjejuA==",
+			"license": "MIT",
+			"peerDependencies": {
+				"@napi-rs/canvas": "^0.1.69"
+			},
+			"peerDependenciesMeta": {
+				"@napi-rs/canvas": {
+					"optional": true
+				}
+			}
+		},
 		"node_modules/unpipe": {
 			"version": "1.0.0",
 			"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",

package.json

@@ -69,6 +69,7 @@
 		"remark-rehype": "^11.1.2",
 		"remark-wiki-link": "^2.0.1",
 		"unified": "^11.0.5",
+		"unpdf": "^1.4.0",
 		"uuid": "^13.0.0",
 		"zod": "^4.1.13"
 	}