From d96d4b4b29a31d319f0c58ebac0891fef86b2eab Mon Sep 17 00:00:00 2001 From: Andrew Beal Date: Tue, 16 Dec 2025 23:14:34 +0000 Subject: [PATCH] Correct OpenAI binary image data handling. Update model tests. --- AIClasses/BaseAIClass.ts | 2 +- AIClasses/Claude/Claude.ts | 2 +- AIClasses/Gemini/Gemini.ts | 2 +- AIClasses/IAIClass.ts | 2 +- AIClasses/OpenAI/OpenAI.ts | 31 +-- Services/ChatService.ts | 3 +- __tests__/AIClasses/Claude.test.ts | 309 ++++++++++++++++++++++++++++- __tests__/AIClasses/Gemini.test.ts | 261 ++++++++++++++++++++++++ __tests__/AIClasses/OpenAI.test.ts | 249 +++++++++++++++++++++++ 9 files changed, 842 insertions(+), 19 deletions(-) diff --git a/AIClasses/BaseAIClass.ts b/AIClasses/BaseAIClass.ts index 9d968b8..85d2dfe 100644 --- a/AIClasses/BaseAIClass.ts +++ b/AIClasses/BaseAIClass.ts @@ -43,7 +43,7 @@ export abstract class BaseAIClass implements IAIClass { abortSignal?: AbortSignal ): AsyncGenerator; - public abstract formatBinaryFilesForUser(files: Array<{type: string, path: string, contents: string}>): string; + public abstract formatBinaryFiles(files: Array<{type: string, path: string, contents: string}>): string; protected abstract parseStreamChunk(chunk: string): IStreamChunk; protected abstract extractContents(conversationContent: ConversationContent[]): unknown; diff --git a/AIClasses/Claude/Claude.ts b/AIClasses/Claude/Claude.ts index 7666cfc..bc1e1ea 100644 --- a/AIClasses/Claude/Claude.ts +++ b/AIClasses/Claude/Claude.ts @@ -238,7 +238,7 @@ export class Claude extends BaseAIClass { })); } - public formatBinaryFilesForUser(files: Array<{type: string, path: string, contents: string}>): string { + public formatBinaryFiles(files: Array<{type: string, path: string, contents: string}>): string { const contentBlocks = files.flatMap(file => { const extension = path.extname(file.path).substring(1).toLowerCase(); diff --git a/AIClasses/Gemini/Gemini.ts b/AIClasses/Gemini/Gemini.ts index 0eeddd4..45c97c9 100644 --- a/AIClasses/Gemini/Gemini.ts +++ b/AIClasses/Gemini/Gemini.ts @@ -243,7 +243,7 @@ export class Gemini extends BaseAIClass { })); } - public formatBinaryFilesForUser(files: Array<{type: string, path: string, contents: string}>): string { + public formatBinaryFiles(files: Array<{type: string, path: string, contents: string}>): string { const parts: unknown[] = []; for (const file of files) { diff --git a/AIClasses/IAIClass.ts b/AIClasses/IAIClass.ts index 50ca166..2d4e008 100644 --- a/AIClasses/IAIClass.ts +++ b/AIClasses/IAIClass.ts @@ -3,5 +3,5 @@ import type { Conversation } from "Conversations/Conversation"; export interface IAIClass { streamRequest(conversation: Conversation, allowDestructiveActions: boolean): AsyncGenerator; - formatBinaryFilesForUser(files: Array<{type: string, path: string, contents: string}>): string; + formatBinaryFiles(files: Array<{type: string, path: string, contents: string}>): string; } \ No newline at end of file diff --git a/AIClasses/OpenAI/OpenAI.ts b/AIClasses/OpenAI/OpenAI.ts index 8c645fd..39711b9 100644 --- a/AIClasses/OpenAI/OpenAI.ts +++ b/AIClasses/OpenAI/OpenAI.ts @@ -290,20 +290,23 @@ export class OpenAI extends BaseAIClass { })); } - public formatBinaryFilesForUser(files: Array<{type: string, path: string, contents: string}>): string { + public formatBinaryFiles(files: Array<{type: string, path: string, contents: string}>): string { const contentBlocks: unknown[] = []; - + for (const file of files) { const extension = path.extname(file.path).substring(1).toLowerCase(); - let mimeType: string; - + if (isFileType(file.type, FileType.PDF)) { mimeType = "application/pdf"; + contentBlocks.push({ + type: "input_file", + filename: path.basename(file.path), + file_data: `data:${mimeType};base64,${file.contents}` + }); } else { try { mimeType = getImageMimeType(extension); - if (!this.SUPPORTED_IMAGE_TYPES.includes(mimeType)) { contentBlocks.push({ type: "input_text", @@ -311,6 +314,11 @@ export class OpenAI extends BaseAIClass { }); continue; } + + contentBlocks.push({ + type: "input_image", + image_url: `data:${mimeType};base64,${file.contents}` + }); } catch (error) { contentBlocks.push({ type: "input_text", @@ -319,14 +327,11 @@ export class OpenAI extends BaseAIClass { continue; } } - - contentBlocks.push({ - type: "input_file", - filename: path.basename(file.path), - file_data: `data:${mimeType};base64,${file.contents}` - }); } - - return JSON.stringify(contentBlocks); + + return JSON.stringify([{ + role: "user", + content: contentBlocks + }]); } } \ No newline at end of file diff --git a/Services/ChatService.ts b/Services/ChatService.ts index f0d88f5..df6aca7 100644 --- a/Services/ChatService.ts +++ b/Services/ChatService.ts @@ -63,6 +63,7 @@ export class ChatService { if (this.ai === undefined || !await this.semaphore.wait()) { return; } + const ai = this.ai; this.semaphoreHeld = true; @@ -97,7 +98,7 @@ export class ChatService { const functionResponse = await this.aiFunctionService.performAIFunction(response.functionCall); conversation.addFunctionResponse( functionResponse, - (files) => this.ai!.formatBinaryFilesForUser(files) + (files) => ai.formatBinaryFiles(files) ); } else { callbacks.onThoughtUpdate(Copy.AIThoughtMessage); diff --git a/__tests__/AIClasses/Claude.test.ts b/__tests__/AIClasses/Claude.test.ts index 7711f89..96dcb95 100644 --- a/__tests__/AIClasses/Claude.test.ts +++ b/__tests__/AIClasses/Claude.test.ts @@ -701,7 +701,7 @@ describe('Claude', () => { }); it('should handle provider-specific content (images/PDFs) without stringifying', () => { - // Simulate what formatBinaryFilesForUser returns for an image + // Simulate what formatBinaryFiles returns for an image const imageContentBlocks = [ { type: 'text', text: 'test-image.png' }, { @@ -891,4 +891,311 @@ describe('Claude', () => { expect((claude as any).accumulatedFunctionId).toBeNull(); }); }); + + describe('formatBinaryFiles', () => { + it('should format PDF files with document type', () => { + const files = [{ + type: 'pdf', + path: '/vault/documents/report.pdf', + contents: 'base64encodedcontent' + }]; + + const result = claude.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(2); + expect(parsed[0]).toEqual({ + type: 'text', + text: 'report.pdf' + }); + expect(parsed[1]).toEqual({ + type: 'document', + source: { + type: 'base64', + media_type: 'application/pdf', + data: 'base64encodedcontent' + } + }); + }); + + it('should format JPEG images with image type', () => { + const files = [{ + type: 'image', + path: '/vault/images/photo.jpg', + contents: 'base64imagedata' + }]; + + const result = claude.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(2); + expect(parsed[0]).toEqual({ + type: 'text', + text: 'photo.jpg' + }); + expect(parsed[1]).toEqual({ + type: 'image', + source: { + type: 'base64', + media_type: 'image/jpeg', + data: 'base64imagedata' + } + }); + }); + + it('should format PNG images with image type', () => { + const files = [{ + type: 'image', + path: '/vault/images/diagram.png', + contents: 'base64pngdata' + }]; + + const result = claude.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(2); + expect(parsed[1]).toEqual({ + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'base64pngdata' + } + }); + }); + + it('should format GIF images with image type', () => { + const files = [{ + type: 'image', + path: '/vault/images/animation.gif', + contents: 'base64gifdata' + }]; + + const result = claude.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(2); + expect(parsed[1]).toEqual({ + type: 'image', + source: { + type: 'base64', + media_type: 'image/gif', + data: 'base64gifdata' + } + }); + }); + + it('should format WebP images with image type', () => { + const files = [{ + type: 'image', + path: '/vault/images/modern.webp', + contents: 'base64webpdata' + }]; + + const result = claude.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(2); + expect(parsed[1]).toEqual({ + type: 'image', + source: { + type: 'base64', + media_type: 'image/webp', + data: 'base64webpdata' + } + }); + }); + + it('should handle unsupported image formats with error message', () => { + const files = [{ + type: 'image', + path: '/vault/images/photo.bmp', + contents: 'base64bmpdata' + }]; + + const result = claude.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(1); + expect(parsed[0]).toEqual({ + type: 'text', + text: 'Unsupported image format: photo.bmp' + }); + }); + + it('should handle multiple files of different types', () => { + const files = [ + { + type: 'pdf', + path: '/vault/doc.pdf', + contents: 'pdfdata' + }, + { + type: 'image', + path: '/vault/image.jpg', + contents: 'jpegdata' + }, + { + type: 'image', + path: '/vault/screenshot.png', + contents: 'pngdata' + } + ]; + + const result = claude.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(6); + + // PDF file + expect(parsed[0]).toEqual({ type: 'text', text: 'doc.pdf' }); + expect(parsed[1]).toEqual({ + type: 'document', + source: { + type: 'base64', + media_type: 'application/pdf', + data: 'pdfdata' + } + }); + + // JPEG image + expect(parsed[2]).toEqual({ type: 'text', text: 'image.jpg' }); + expect(parsed[3]).toEqual({ + type: 'image', + source: { + type: 'base64', + media_type: 'image/jpeg', + data: 'jpegdata' + } + }); + + // PNG image + expect(parsed[4]).toEqual({ type: 'text', text: 'screenshot.png' }); + expect(parsed[5]).toEqual({ + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'pngdata' + } + }); + }); + + it('should handle mixed supported and unsupported files', () => { + const files = [ + { + type: 'image', + path: '/vault/good.jpg', + contents: 'jpegdata' + }, + { + type: 'image', + path: '/vault/bad.bmp', + contents: 'bmpdata' + }, + { + type: 'pdf', + path: '/vault/doc.pdf', + contents: 'pdfdata' + } + ]; + + const result = claude.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(5); + + expect(parsed[0].type).toBe('text'); + expect(parsed[0].text).toBe('good.jpg'); + expect(parsed[1].type).toBe('image'); + expect(parsed[2]).toEqual({ + type: 'text', + text: 'Unsupported image format: bad.bmp' + }); + expect(parsed[3].type).toBe('text'); + expect(parsed[3].text).toBe('doc.pdf'); + expect(parsed[4].type).toBe('document'); + }); + + it('should handle error when getting image mime type fails', () => { + const files = [{ + type: 'image', + path: '/vault/image.unknown', + contents: 'imagedata' + }]; + + const result = claude.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(1); + expect(parsed[0].type).toBe('text'); + expect(parsed[0].text).toContain('Image type not supported'); + }); + + it('should handle files with uppercase extensions', () => { + const files = [ + { + type: 'pdf', + path: '/vault/document.PDF', + contents: 'pdfdata' + }, + { + type: 'image', + path: '/vault/photo.JPG', + contents: 'jpegdata' + } + ]; + + const result = claude.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(4); + expect(parsed[0].text).toBe('document.PDF'); + expect(parsed[1].type).toBe('document'); + expect(parsed[2].text).toBe('photo.JPG'); + expect(parsed[3].type).toBe('image'); + }); + + it('should handle empty files array', () => { + const files: Array<{type: string, path: string, contents: string}> = []; + + const result = claude.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(0); + }); + + it('should properly encode filenames with special characters', () => { + const files = [{ + type: 'pdf', + path: '/vault/documents/report (final) v2.pdf', + contents: 'pdfdata' + }]; + + const result = claude.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed[0].text).toBe('report (final) v2.pdf'); + }); + + it('should handle JPEG files with .jpeg extension', () => { + const files = [{ + type: 'image', + path: '/vault/photo.jpeg', + contents: 'jpegdata' + }]; + + const result = claude.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed[1]).toEqual({ + type: 'image', + source: { + type: 'base64', + media_type: 'image/jpeg', + data: 'jpegdata' + } + }); + }); + }); }); diff --git a/__tests__/AIClasses/Gemini.test.ts b/__tests__/AIClasses/Gemini.test.ts index a330b8f..fa9642b 100644 --- a/__tests__/AIClasses/Gemini.test.ts +++ b/__tests__/AIClasses/Gemini.test.ts @@ -1076,4 +1076,265 @@ describe('Gemini', () => { expect((gemini as any).accumulatedThoughtSignature).toBeNull(); }); }); + + describe('formatBinaryFiles', () => { + it('should format PDF files with inlineData', () => { + const files = [{ + type: 'pdf', + path: '/vault/documents/report.pdf', + contents: 'base64encodedcontent' + }]; + + const result = gemini.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(2); + expect(parsed[0]).toEqual({ + text: 'report.pdf' + }); + expect(parsed[1]).toEqual({ + inlineData: { + mimeType: 'application/pdf', + data: 'base64encodedcontent' + } + }); + }); + + it('should format JPEG images with inlineData', () => { + const files = [{ + type: 'image', + path: '/vault/images/photo.jpg', + contents: 'base64imagedata' + }]; + + const result = gemini.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(2); + expect(parsed[0]).toEqual({ + text: 'photo.jpg' + }); + expect(parsed[1]).toEqual({ + inlineData: { + mimeType: 'image/jpeg', + data: 'base64imagedata' + } + }); + }); + + it('should format PNG images with inlineData', () => { + const files = [{ + type: 'image', + path: '/vault/images/diagram.png', + contents: 'base64pngdata' + }]; + + const result = gemini.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(2); + expect(parsed[1]).toEqual({ + inlineData: { + mimeType: 'image/png', + data: 'base64pngdata' + } + }); + }); + + it('should handle unsupported image formats (GIF) with error message', () => { + const files = [{ + type: 'image', + path: '/vault/images/animation.gif', + contents: 'base64gifdata' + }]; + + const result = gemini.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(1); + expect(parsed[0]).toEqual({ + text: 'Unsupported image format: animation.gif' + }); + }); + + it('should handle unsupported image formats (BMP) with error message', () => { + const files = [{ + type: 'image', + path: '/vault/images/photo.bmp', + contents: 'base64bmpdata' + }]; + + const result = gemini.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(1); + expect(parsed[0]).toEqual({ + text: 'Unsupported image format: photo.bmp' + }); + }); + + it('should handle multiple files of different types', () => { + const files = [ + { + type: 'pdf', + path: '/vault/doc.pdf', + contents: 'pdfdata' + }, + { + type: 'image', + path: '/vault/image.jpg', + contents: 'jpegdata' + }, + { + type: 'image', + path: '/vault/screenshot.png', + contents: 'pngdata' + } + ]; + + const result = gemini.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(6); + + // PDF file + expect(parsed[0]).toEqual({ text: 'doc.pdf' }); + expect(parsed[1]).toEqual({ + inlineData: { + mimeType: 'application/pdf', + data: 'pdfdata' + } + }); + + // JPEG image + expect(parsed[2]).toEqual({ text: 'image.jpg' }); + expect(parsed[3]).toEqual({ + inlineData: { + mimeType: 'image/jpeg', + data: 'jpegdata' + } + }); + + // PNG image + expect(parsed[4]).toEqual({ text: 'screenshot.png' }); + expect(parsed[5]).toEqual({ + inlineData: { + mimeType: 'image/png', + data: 'pngdata' + } + }); + }); + + it('should handle mixed supported and unsupported files', () => { + const files = [ + { + type: 'image', + path: '/vault/good.jpg', + contents: 'jpegdata' + }, + { + type: 'image', + path: '/vault/bad.bmp', + contents: 'bmpdata' + }, + { + type: 'pdf', + path: '/vault/doc.pdf', + contents: 'pdfdata' + } + ]; + + const result = gemini.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(5); + + expect(parsed[0]).toEqual({ text: 'good.jpg' }); + expect(parsed[1]).toHaveProperty('inlineData'); + expect(parsed[2]).toEqual({ + text: 'Unsupported image format: bad.bmp' + }); + expect(parsed[3]).toEqual({ text: 'doc.pdf' }); + expect(parsed[4]).toHaveProperty('inlineData'); + }); + + it('should handle error when getting image mime type fails', () => { + const files = [{ + type: 'image', + path: '/vault/image.unknown', + contents: 'imagedata' + }]; + + const result = gemini.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(1); + expect(parsed[0]).toHaveProperty('text'); + expect(parsed[0].text).toContain('Image type not supported'); + }); + + it('should handle files with uppercase extensions', () => { + const files = [ + { + type: 'pdf', + path: '/vault/document.PDF', + contents: 'pdfdata' + }, + { + type: 'image', + path: '/vault/photo.JPG', + contents: 'jpegdata' + } + ]; + + const result = gemini.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(4); + expect(parsed[0].text).toBe('document.PDF'); + expect(parsed[1]).toHaveProperty('inlineData'); + expect(parsed[2].text).toBe('photo.JPG'); + expect(parsed[3]).toHaveProperty('inlineData'); + }); + + it('should handle empty files array', () => { + const files: Array<{type: string, path: string, contents: string}> = []; + + const result = gemini.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(0); + }); + + it('should properly encode filenames with special characters', () => { + const files = [{ + type: 'pdf', + path: '/vault/documents/report (final) v2.pdf', + contents: 'pdfdata' + }]; + + const result = gemini.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed[0].text).toBe('report (final) v2.pdf'); + }); + + it('should handle JPEG files with .jpeg extension', () => { + const files = [{ + type: 'image', + path: '/vault/photo.jpeg', + contents: 'jpegdata' + }]; + + const result = gemini.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed[1]).toEqual({ + inlineData: { + mimeType: 'image/jpeg', + data: 'jpegdata' + } + }); + }); + }); }); diff --git a/__tests__/AIClasses/OpenAI.test.ts b/__tests__/AIClasses/OpenAI.test.ts index e528c0a..9a0cf4e 100644 --- a/__tests__/AIClasses/OpenAI.test.ts +++ b/__tests__/AIClasses/OpenAI.test.ts @@ -1004,4 +1004,253 @@ describe('OpenAI', () => { expect(webSearchTool.name).toBe(undefined); }); }); + + describe('formatBinaryFiles', () => { + it('should format PDF files with input_file type', () => { + const files = [{ + type: 'pdf', + path: '/vault/documents/report.pdf', + contents: 'base64encodedcontent' + }]; + + const result = openai.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(1); + expect(parsed[0].role).toBe('user'); + expect(parsed[0].content).toHaveLength(1); + expect(parsed[0].content[0]).toEqual({ + type: 'input_file', + filename: 'report.pdf', + file_data: 'data:application/pdf;base64,base64encodedcontent' + }); + }); + + it('should format JPEG images with input_image type', () => { + const files = [{ + type: 'image', + path: '/vault/images/photo.jpg', + contents: 'base64imagedata' + }]; + + const result = openai.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(1); + expect(parsed[0].role).toBe('user'); + expect(parsed[0].content).toHaveLength(1); + expect(parsed[0].content[0]).toEqual({ + type: 'input_image', + image_url: 'data:image/jpeg;base64,base64imagedata' + }); + }); + + it('should format PNG images with input_image type', () => { + const files = [{ + type: 'image', + path: '/vault/images/diagram.png', + contents: 'base64pngdata' + }]; + + const result = openai.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(1); + expect(parsed[0].content[0]).toEqual({ + type: 'input_image', + image_url: 'data:image/png;base64,base64pngdata' + }); + }); + + it('should format WebP images with input_image type', () => { + const files = [{ + type: 'image', + path: '/vault/images/modern.webp', + contents: 'base64webpdata' + }]; + + const result = openai.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(1); + expect(parsed[0].content[0]).toEqual({ + type: 'input_image', + image_url: 'data:image/webp;base64,base64webpdata' + }); + }); + + it('should handle unsupported image formats with error message', () => { + const files = [{ + type: 'image', + path: '/vault/images/photo.gif', + contents: 'base64gifdata' + }]; + + const result = openai.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(1); + expect(parsed[0].content).toHaveLength(1); + expect(parsed[0].content[0]).toEqual({ + type: 'input_text', + text: 'Unsupported image format: photo.gif' + }); + }); + + it('should handle multiple files of different types', () => { + const files = [ + { + type: 'pdf', + path: '/vault/doc.pdf', + contents: 'pdfdata' + }, + { + type: 'image', + path: '/vault/image.jpg', + contents: 'jpegdata' + }, + { + type: 'image', + path: '/vault/screenshot.png', + contents: 'pngdata' + } + ]; + + const result = openai.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(1); + expect(parsed[0].role).toBe('user'); + expect(parsed[0].content).toHaveLength(3); + + expect(parsed[0].content[0]).toEqual({ + type: 'input_file', + filename: 'doc.pdf', + file_data: 'data:application/pdf;base64,pdfdata' + }); + + expect(parsed[0].content[1]).toEqual({ + type: 'input_image', + image_url: 'data:image/jpeg;base64,jpegdata' + }); + + expect(parsed[0].content[2]).toEqual({ + type: 'input_image', + image_url: 'data:image/png;base64,pngdata' + }); + }); + + it('should handle mixed supported and unsupported files', () => { + const files = [ + { + type: 'image', + path: '/vault/good.jpg', + contents: 'jpegdata' + }, + { + type: 'image', + path: '/vault/bad.bmp', + contents: 'bmpdata' + }, + { + type: 'pdf', + path: '/vault/doc.pdf', + contents: 'pdfdata' + } + ]; + + const result = openai.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(1); + expect(parsed[0].content).toHaveLength(3); + + expect(parsed[0].content[0].type).toBe('input_image'); + expect(parsed[0].content[1]).toEqual({ + type: 'input_text', + text: 'Unsupported image format: bad.bmp' + }); + expect(parsed[0].content[2].type).toBe('input_file'); + }); + + it('should handle error when getting image mime type fails', () => { + const files = [{ + type: 'image', + path: '/vault/image.unknown', + contents: 'imagedata' + }]; + + const result = openai.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(1); + expect(parsed[0].content).toHaveLength(1); + expect(parsed[0].content[0].type).toBe('input_text'); + expect(parsed[0].content[0].text).toContain('Image type not supported'); + }); + + it('should handle files with uppercase extensions', () => { + const files = [ + { + type: 'pdf', + path: '/vault/document.PDF', + contents: 'pdfdata' + }, + { + type: 'image', + path: '/vault/photo.JPG', + contents: 'jpegdata' + } + ]; + + const result = openai.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(1); + expect(parsed[0].content).toHaveLength(2); + expect(parsed[0].content[0].type).toBe('input_file'); + expect(parsed[0].content[0].filename).toBe('document.PDF'); + expect(parsed[0].content[1].type).toBe('input_image'); + }); + + it('should handle empty files array', () => { + const files: Array<{type: string, path: string, contents: string}> = []; + + const result = openai.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed).toHaveLength(1); + expect(parsed[0].role).toBe('user'); + expect(parsed[0].content).toHaveLength(0); + }); + + it('should properly encode filenames with special characters', () => { + const files = [{ + type: 'pdf', + path: '/vault/documents/report (final) v2.pdf', + contents: 'pdfdata' + }]; + + const result = openai.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed[0].content[0].filename).toBe('report (final) v2.pdf'); + }); + + it('should handle JPEG files with .jpeg extension', () => { + const files = [{ + type: 'image', + path: '/vault/photo.jpeg', + contents: 'jpegdata' + }]; + + const result = openai.formatBinaryFiles(files); + const parsed = JSON.parse(result); + + expect(parsed[0].content[0]).toEqual({ + type: 'input_image', + image_url: 'data:image/jpeg;base64,jpegdata' + }); + }); + }); });