diff --git a/AIClasses/Claude/Claude.ts b/AIClasses/Claude/Claude.ts index e797086..7666cfc 100644 --- a/AIClasses/Claude/Claude.ts +++ b/AIClasses/Claude/Claude.ts @@ -153,7 +153,7 @@ export class Claude extends BaseAIClass { const contentBlocks: ContentBlockParam[] = []; const contentToExtract = this.getContentToExtract(content); - if (contentToExtract.trim() !== "" && !content.isFunctionCallResponse) { + if (contentToExtract.trim() !== "" && !content.isFunctionCallResponse && !content.isProviderSpecificContent) { contentBlocks.push({ type: "text", text: contentToExtract diff --git a/AIClasses/FunctionDefinitions/Functions/ReadVaultFiles.ts b/AIClasses/FunctionDefinitions/Functions/ReadVaultFiles.ts index 20fedd8..cb08825 100644 --- a/AIClasses/FunctionDefinitions/Functions/ReadVaultFiles.ts +++ b/AIClasses/FunctionDefinitions/Functions/ReadVaultFiles.ts @@ -4,6 +4,11 @@ import type { IAIFunctionDefinition } from "../IAIFunctionDefinition"; export const ReadVaultFiles: IAIFunctionDefinition = { name: AIFunction.ReadVaultFiles, description: `Reads and returns the complete content of one or more files from the vault. + + **IMPORTANT: This function gives you the ability to SEE and ANALYZE images + and PDFs. When users ask about image or PDF files, USE THIS FUNCTION to + read them—do not claim you cannot see images.** + Call this when you need to access existing file content to answer questions, provide summaries, verify information, or gather context before making updates. Use proactively before updating files to understand current content and avoid diff --git a/AIClasses/SystemPrompt.ts b/AIClasses/SystemPrompt.ts index 422b9df..2cce678 100644 --- a/AIClasses/SystemPrompt.ts +++ b/AIClasses/SystemPrompt.ts @@ -24,6 +24,7 @@ When users issue directives, their instruction IS your authorization: - Task verbs (create, generate, update, delete) → Execute corresponding function - Implied actions ("I need X") → Call the function that produces X - Outcome requests ("Show me Y") → Use tools to retrieve/generate Y +- Image/PDF references ("What's in X.png", "Summarize Y.pdf") → Read the file first **Example:** User: "Create a note about today's meeting with Sarah" @@ -176,6 +177,8 @@ Before executing complex queries: ❌ Providing generic answers when vault contains specific information ❌ Mimicking historical tool call formats instead of using native functions ❌ Noting that a PDF/image exists without reading its contents when relevant +❌ Asking users to describe images instead of reading them yourself +❌ Saying "I cannot see/interpret images" ## Decision Framework diff --git a/__tests__/AIClasses/Claude.test.ts b/__tests__/AIClasses/Claude.test.ts index f7dbb2b..7711f89 100644 --- a/__tests__/AIClasses/Claude.test.ts +++ b/__tests__/AIClasses/Claude.test.ts @@ -699,6 +699,89 @@ describe('Claude', () => { expect(result[1].content[0].text).toBe('Second message'); expect(result[2].content[0].text).toBe('Third message'); }); + + it('should handle provider-specific content (images/PDFs) without stringifying', () => { + // Simulate what formatBinaryFilesForUser returns for an image + const imageContentBlocks = [ + { type: 'text', text: 'test-image.png' }, + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==' + } + } + ]; + + const providerSpecificContent = new ConversationContent( + Role.User, + JSON.stringify(imageContentBlocks), + JSON.stringify(imageContentBlocks), + '', + new Date(), + false, + false, + true // isProviderSpecificContent = true + ); + + const result = (claude as any).extractContents([providerSpecificContent]); + + expect(result).toHaveLength(1); + expect(result[0].content).toHaveLength(2); + + // First block should be the filename text + expect(result[0].content[0]).toEqual({ + type: 'text', + text: 'test-image.png' + }); + + // Second block should be the image with base64 data (NOT stringified) + expect(result[0].content[1]).toEqual({ + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==' + } + }); + }); + + it('should not add provider-specific content as text when isProviderSpecificContent is true', () => { + // This test ensures the fix for the token usage issue + const pdfContentBlocks = [ + { type: 'text', text: 'document.pdf' }, + { + type: 'document', + source: { + type: 'base64', + media_type: 'application/pdf', + data: 'JVBERi0xLjQKJeLjz9MK' + } + } + ]; + + const providerSpecificContent = new ConversationContent( + Role.User, + JSON.stringify(pdfContentBlocks), + JSON.stringify(pdfContentBlocks), + '', + new Date(), + false, + false, + true // isProviderSpecificContent = true + ); + + const result = (claude as any).extractContents([providerSpecificContent]); + + expect(result).toHaveLength(1); + expect(result[0].content).toHaveLength(2); + + // Verify no text block with stringified JSON was added + const textBlocks = result[0].content.filter((block: any) => block.type === 'text'); + expect(textBlocks).toHaveLength(1); + expect(textBlocks[0].text).toBe('document.pdf'); // Only the filename, not the stringified JSON + }); }); describe('mapFunctionDefinitions', () => {