fix: prevent double-stringification of provider-specific content

Excludes image/PDF content blocks from text extraction when
isProviderSpecificContent flag is set, avoiding unnecessary token
usage. Updates ReadVaultFiles documentation to clarify image/PDF
reading capabilities and adds system prompt guidance for handling
binary file references.
This commit is contained in:
Andrew Beal 2025-12-16 22:14:42 +00:00
parent fe086363a9
commit 30badb186a
4 changed files with 92 additions and 1 deletions

View file

@ -153,7 +153,7 @@ export class Claude extends BaseAIClass {
const contentBlocks: ContentBlockParam[] = [];
const contentToExtract = this.getContentToExtract(content);
if (contentToExtract.trim() !== "" && !content.isFunctionCallResponse) {
if (contentToExtract.trim() !== "" && !content.isFunctionCallResponse && !content.isProviderSpecificContent) {
contentBlocks.push({
type: "text",
text: contentToExtract

View file

@ -4,6 +4,11 @@ import type { IAIFunctionDefinition } from "../IAIFunctionDefinition";
export const ReadVaultFiles: IAIFunctionDefinition = {
name: AIFunction.ReadVaultFiles,
description: `Reads and returns the complete content of one or more files from the vault.
**IMPORTANT: This function gives you the ability to SEE and ANALYZE images
and PDFs. When users ask about image or PDF files, USE THIS FUNCTION to
read themdo not claim you cannot see images.**
Call this when you need to access existing file content to answer questions,
provide summaries, verify information, or gather context before making updates.
Use proactively before updating files to understand current content and avoid

View file

@ -24,6 +24,7 @@ When users issue directives, their instruction IS your authorization:
- Task verbs (create, generate, update, delete) Execute corresponding function
- Implied actions ("I need X") Call the function that produces X
- Outcome requests ("Show me Y") Use tools to retrieve/generate Y
- Image/PDF references ("What's in X.png", "Summarize Y.pdf") Read the file first
**Example:**
User: "Create a note about today's meeting with Sarah"
@ -176,6 +177,8 @@ Before executing complex queries:
Providing generic answers when vault contains specific information
Mimicking historical tool call formats instead of using native functions
Noting that a PDF/image exists without reading its contents when relevant
Asking users to describe images instead of reading them yourself
Saying "I cannot see/interpret images"
## Decision Framework

View file

@ -699,6 +699,89 @@ describe('Claude', () => {
expect(result[1].content[0].text).toBe('Second message');
expect(result[2].content[0].text).toBe('Third message');
});
it('should handle provider-specific content (images/PDFs) without stringifying', () => {
// Simulate what formatBinaryFilesForUser returns for an image
const imageContentBlocks = [
{ type: 'text', text: 'test-image.png' },
{
type: 'image',
source: {
type: 'base64',
media_type: 'image/png',
data: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='
}
}
];
const providerSpecificContent = new ConversationContent(
Role.User,
JSON.stringify(imageContentBlocks),
JSON.stringify(imageContentBlocks),
'',
new Date(),
false,
false,
true // isProviderSpecificContent = true
);
const result = (claude as any).extractContents([providerSpecificContent]);
expect(result).toHaveLength(1);
expect(result[0].content).toHaveLength(2);
// First block should be the filename text
expect(result[0].content[0]).toEqual({
type: 'text',
text: 'test-image.png'
});
// Second block should be the image with base64 data (NOT stringified)
expect(result[0].content[1]).toEqual({
type: 'image',
source: {
type: 'base64',
media_type: 'image/png',
data: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='
}
});
});
it('should not add provider-specific content as text when isProviderSpecificContent is true', () => {
// This test ensures the fix for the token usage issue
const pdfContentBlocks = [
{ type: 'text', text: 'document.pdf' },
{
type: 'document',
source: {
type: 'base64',
media_type: 'application/pdf',
data: 'JVBERi0xLjQKJeLjz9MK'
}
}
];
const providerSpecificContent = new ConversationContent(
Role.User,
JSON.stringify(pdfContentBlocks),
JSON.stringify(pdfContentBlocks),
'',
new Date(),
false,
false,
true // isProviderSpecificContent = true
);
const result = (claude as any).extractContents([providerSpecificContent]);
expect(result).toHaveLength(1);
expect(result[0].content).toHaveLength(2);
// Verify no text block with stringified JSON was added
const textBlocks = result[0].content.filter((block: any) => block.type === 'text');
expect(textBlocks).toHaveLength(1);
expect(textBlocks[0].text).toBe('document.pdf'); // Only the filename, not the stringified JSON
});
});
describe('mapFunctionDefinitions', () => {