mirror of
https://github.com/andy-stack/vaultkeeper-ai.git
synced 2026-07-22 06:42:03 +00:00
fix: prevent double-stringification of provider-specific content
Excludes image/PDF content blocks from text extraction when isProviderSpecificContent flag is set, avoiding unnecessary token usage. Updates ReadVaultFiles documentation to clarify image/PDF reading capabilities and adds system prompt guidance for handling binary file references.
This commit is contained in:
parent
fe086363a9
commit
30badb186a
4 changed files with 92 additions and 1 deletions
|
|
@ -153,7 +153,7 @@ export class Claude extends BaseAIClass {
|
|||
const contentBlocks: ContentBlockParam[] = [];
|
||||
const contentToExtract = this.getContentToExtract(content);
|
||||
|
||||
if (contentToExtract.trim() !== "" && !content.isFunctionCallResponse) {
|
||||
if (contentToExtract.trim() !== "" && !content.isFunctionCallResponse && !content.isProviderSpecificContent) {
|
||||
contentBlocks.push({
|
||||
type: "text",
|
||||
text: contentToExtract
|
||||
|
|
|
|||
|
|
@ -4,6 +4,11 @@ import type { IAIFunctionDefinition } from "../IAIFunctionDefinition";
|
|||
export const ReadVaultFiles: IAIFunctionDefinition = {
|
||||
name: AIFunction.ReadVaultFiles,
|
||||
description: `Reads and returns the complete content of one or more files from the vault.
|
||||
|
||||
**IMPORTANT: This function gives you the ability to SEE and ANALYZE images
|
||||
and PDFs. When users ask about image or PDF files, USE THIS FUNCTION to
|
||||
read them—do not claim you cannot see images.**
|
||||
|
||||
Call this when you need to access existing file content to answer questions,
|
||||
provide summaries, verify information, or gather context before making updates.
|
||||
Use proactively before updating files to understand current content and avoid
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ When users issue directives, their instruction IS your authorization:
|
|||
- Task verbs (create, generate, update, delete) → Execute corresponding function
|
||||
- Implied actions ("I need X") → Call the function that produces X
|
||||
- Outcome requests ("Show me Y") → Use tools to retrieve/generate Y
|
||||
- Image/PDF references ("What's in X.png", "Summarize Y.pdf") → Read the file first
|
||||
|
||||
**Example:**
|
||||
User: "Create a note about today's meeting with Sarah"
|
||||
|
|
@ -176,6 +177,8 @@ Before executing complex queries:
|
|||
❌ Providing generic answers when vault contains specific information
|
||||
❌ Mimicking historical tool call formats instead of using native functions
|
||||
❌ Noting that a PDF/image exists without reading its contents when relevant
|
||||
❌ Asking users to describe images instead of reading them yourself
|
||||
❌ Saying "I cannot see/interpret images"
|
||||
|
||||
## Decision Framework
|
||||
|
||||
|
|
|
|||
|
|
@ -699,6 +699,89 @@ describe('Claude', () => {
|
|||
expect(result[1].content[0].text).toBe('Second message');
|
||||
expect(result[2].content[0].text).toBe('Third message');
|
||||
});
|
||||
|
||||
it('should handle provider-specific content (images/PDFs) without stringifying', () => {
|
||||
// Simulate what formatBinaryFilesForUser returns for an image
|
||||
const imageContentBlocks = [
|
||||
{ type: 'text', text: 'test-image.png' },
|
||||
{
|
||||
type: 'image',
|
||||
source: {
|
||||
type: 'base64',
|
||||
media_type: 'image/png',
|
||||
data: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
const providerSpecificContent = new ConversationContent(
|
||||
Role.User,
|
||||
JSON.stringify(imageContentBlocks),
|
||||
JSON.stringify(imageContentBlocks),
|
||||
'',
|
||||
new Date(),
|
||||
false,
|
||||
false,
|
||||
true // isProviderSpecificContent = true
|
||||
);
|
||||
|
||||
const result = (claude as any).extractContents([providerSpecificContent]);
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].content).toHaveLength(2);
|
||||
|
||||
// First block should be the filename text
|
||||
expect(result[0].content[0]).toEqual({
|
||||
type: 'text',
|
||||
text: 'test-image.png'
|
||||
});
|
||||
|
||||
// Second block should be the image with base64 data (NOT stringified)
|
||||
expect(result[0].content[1]).toEqual({
|
||||
type: 'image',
|
||||
source: {
|
||||
type: 'base64',
|
||||
media_type: 'image/png',
|
||||
data: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it('should not add provider-specific content as text when isProviderSpecificContent is true', () => {
|
||||
// This test ensures the fix for the token usage issue
|
||||
const pdfContentBlocks = [
|
||||
{ type: 'text', text: 'document.pdf' },
|
||||
{
|
||||
type: 'document',
|
||||
source: {
|
||||
type: 'base64',
|
||||
media_type: 'application/pdf',
|
||||
data: 'JVBERi0xLjQKJeLjz9MK'
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
const providerSpecificContent = new ConversationContent(
|
||||
Role.User,
|
||||
JSON.stringify(pdfContentBlocks),
|
||||
JSON.stringify(pdfContentBlocks),
|
||||
'',
|
||||
new Date(),
|
||||
false,
|
||||
false,
|
||||
true // isProviderSpecificContent = true
|
||||
);
|
||||
|
||||
const result = (claude as any).extractContents([providerSpecificContent]);
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].content).toHaveLength(2);
|
||||
|
||||
// Verify no text block with stringified JSON was added
|
||||
const textBlocks = result[0].content.filter((block: any) => block.type === 'text');
|
||||
expect(textBlocks).toHaveLength(1);
|
||||
expect(textBlocks[0].text).toBe('document.pdf'); // Only the filename, not the stringified JSON
|
||||
});
|
||||
});
|
||||
|
||||
describe('mapFunctionDefinitions', () => {
|
||||
|
|
|
|||
Loading…
Reference in a new issue