obsidian-plugin/andy-stack_vaultkeeper-ai
1
0
Fork 0
mirror of https://github.com/andy-stack/vaultkeeper-ai.git synced 2026-07-22 06:42:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Correct OpenAI binary image data handling. Update model tests.

Browse source
This commit is contained in:
Andrew Beal 2025-12-16 23:14:34 +00:00
parent 30badb186a
commit d96d4b4b29
9 changed files with 842 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
AIClasses/BaseAIClass.ts
View file

@ -43,7 +43,7 @@ export abstract class BaseAIClass implements IAIClass {
abortSignal?: AbortSignal
): AsyncGenerator<IStreamChunk, void, unknown>;
public abstract formatBinaryFilesForUser(files: Array<{type: string, path: string, contents: string}>): string;
public abstract formatBinaryFiles(files: Array<{type: string, path: string, contents: string}>): string;
protected abstract parseStreamChunk(chunk: string): IStreamChunk;
protected abstract extractContents(conversationContent: ConversationContent[]): unknown;

2
AIClasses/Claude/Claude.ts
View file

@ -238,7 +238,7 @@ export class Claude extends BaseAIClass {
}));
}
public formatBinaryFilesForUser(files: Array<{type: string, path: string, contents: string}>): string {
public formatBinaryFiles(files: Array<{type: string, path: string, contents: string}>): string {
const contentBlocks = files.flatMap(file => {
const extension = path.extname(file.path).substring(1).toLowerCase();

2
AIClasses/Gemini/Gemini.ts
View file

@ -243,7 +243,7 @@ export class Gemini extends BaseAIClass {
}));
}
public formatBinaryFilesForUser(files: Array<{type: string, path: string, contents: string}>): string {
public formatBinaryFiles(files: Array<{type: string, path: string, contents: string}>): string {
const parts: unknown[] = [];
for (const file of files) {

2
AIClasses/IAIClass.ts
View file

@ -3,5 +3,5 @@ import type { Conversation } from "Conversations/Conversation";
export interface IAIClass {
streamRequest(conversation: Conversation, allowDestructiveActions: boolean): AsyncGenerator<IStreamChunk, void, unknown>;
formatBinaryFilesForUser(files: Array<{type: string, path: string, contents: string}>): string;
formatBinaryFiles(files: Array<{type: string, path: string, contents: string}>): string;
}

31
AIClasses/OpenAI/OpenAI.ts
View file

@ -290,20 +290,23 @@ export class OpenAI extends BaseAIClass {
}));
}
public formatBinaryFilesForUser(files: Array<{type: string, path: string, contents: string}>): string {
public formatBinaryFiles(files: Array<{type: string, path: string, contents: string}>): string {
const contentBlocks: unknown[] = [];
for (const file of files) {
const extension = path.extname(file.path).substring(1).toLowerCase();
let mimeType: string;
if (isFileType(file.type, FileType.PDF)) {
mimeType = "application/pdf";
contentBlocks.push({
type: "input_file",
filename: path.basename(file.path),
file_data: `data:${mimeType};base64,${file.contents}`
});
} else {
try {
mimeType = getImageMimeType(extension);
if (!this.SUPPORTED_IMAGE_TYPES.includes(mimeType)) {
contentBlocks.push({
type: "input_text",
@ -311,6 +314,11 @@ export class OpenAI extends BaseAIClass {
});
continue;
}
contentBlocks.push({
type: "input_image",
image_url: `data:${mimeType};base64,${file.contents}`
});
} catch (error) {
contentBlocks.push({
type: "input_text",
@ -319,14 +327,11 @@ export class OpenAI extends BaseAIClass {
continue;
}
}
contentBlocks.push({
type: "input_file",
filename: path.basename(file.path),
file_data: `data:${mimeType};base64,${file.contents}`
});
}
return JSON.stringify(contentBlocks);
return JSON.stringify([{
role: "user",
content: contentBlocks
}]);
}
}

3
Services/ChatService.ts
View file

@ -63,6 +63,7 @@ export class ChatService {
if (this.ai === undefined || !await this.semaphore.wait()) {
return;
}
const ai = this.ai;
this.semaphoreHeld = true;
@ -97,7 +98,7 @@ export class ChatService {
const functionResponse = await this.aiFunctionService.performAIFunction(response.functionCall);
conversation.addFunctionResponse(
functionResponse,
(files) => this.ai!.formatBinaryFilesForUser(files)
(files) => ai.formatBinaryFiles(files)
);
} else {
callbacks.onThoughtUpdate(Copy.AIThoughtMessage);

309
__tests__/AIClasses/Claude.test.ts
View file

@ -701,7 +701,7 @@ describe('Claude', () => {
});
it('should handle provider-specific content (images/PDFs) without stringifying', () => {
// Simulate what formatBinaryFilesForUser returns for an image
// Simulate what formatBinaryFiles returns for an image
const imageContentBlocks = [
{ type: 'text', text: 'test-image.png' },
{
@ -891,4 +891,311 @@ describe('Claude', () => {
expect((claude as any).accumulatedFunctionId).toBeNull();
});
});
describe('formatBinaryFiles', () => {
it('should format PDF files with document type', () => {
const files = [{
type: 'pdf',
path: '/vault/documents/report.pdf',
contents: 'base64encodedcontent'
}];
const result = claude.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(2);
expect(parsed[0]).toEqual({
type: 'text',
text: 'report.pdf'
});
expect(parsed[1]).toEqual({
type: 'document',
source: {
type: 'base64',
media_type: 'application/pdf',
data: 'base64encodedcontent'
}
});
});
it('should format JPEG images with image type', () => {
const files = [{
type: 'image',
path: '/vault/images/photo.jpg',
contents: 'base64imagedata'
}];
const result = claude.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(2);
expect(parsed[0]).toEqual({
type: 'text',
text: 'photo.jpg'
});
expect(parsed[1]).toEqual({
type: 'image',
source: {
type: 'base64',
media_type: 'image/jpeg',
data: 'base64imagedata'
}
});
});
it('should format PNG images with image type', () => {
const files = [{
type: 'image',
path: '/vault/images/diagram.png',
contents: 'base64pngdata'
}];
const result = claude.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(2);
expect(parsed[1]).toEqual({
type: 'image',
source: {
type: 'base64',
media_type: 'image/png',
data: 'base64pngdata'
}
});
});
it('should format GIF images with image type', () => {
const files = [{
type: 'image',
path: '/vault/images/animation.gif',
contents: 'base64gifdata'
}];
const result = claude.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(2);
expect(parsed[1]).toEqual({
type: 'image',
source: {
type: 'base64',
media_type: 'image/gif',
data: 'base64gifdata'
}
});
});
it('should format WebP images with image type', () => {
const files = [{
type: 'image',
path: '/vault/images/modern.webp',
contents: 'base64webpdata'
}];
const result = claude.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(2);
expect(parsed[1]).toEqual({
type: 'image',
source: {
type: 'base64',
media_type: 'image/webp',
data: 'base64webpdata'
}
});
});
it('should handle unsupported image formats with error message', () => {
const files = [{
type: 'image',
path: '/vault/images/photo.bmp',
contents: 'base64bmpdata'
}];
const result = claude.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(1);
expect(parsed[0]).toEqual({
type: 'text',
text: 'Unsupported image format: photo.bmp'
});
});
it('should handle multiple files of different types', () => {
const files = [
{
type: 'pdf',
path: '/vault/doc.pdf',
contents: 'pdfdata'
},
{
type: 'image',
path: '/vault/image.jpg',
contents: 'jpegdata'
},
{
type: 'image',
path: '/vault/screenshot.png',
contents: 'pngdata'
}
];
const result = claude.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(6);
// PDF file
expect(parsed[0]).toEqual({ type: 'text', text: 'doc.pdf' });
expect(parsed[1]).toEqual({
type: 'document',
source: {
type: 'base64',
media_type: 'application/pdf',
data: 'pdfdata'
}
});
// JPEG image
expect(parsed[2]).toEqual({ type: 'text', text: 'image.jpg' });
expect(parsed[3]).toEqual({
type: 'image',
source: {
type: 'base64',
media_type: 'image/jpeg',
data: 'jpegdata'
}
});
// PNG image
expect(parsed[4]).toEqual({ type: 'text', text: 'screenshot.png' });
expect(parsed[5]).toEqual({
type: 'image',
source: {
type: 'base64',
media_type: 'image/png',
data: 'pngdata'
}
});
});
it('should handle mixed supported and unsupported files', () => {
const files = [
{
type: 'image',
path: '/vault/good.jpg',
contents: 'jpegdata'
},
{
type: 'image',
path: '/vault/bad.bmp',
contents: 'bmpdata'
},
{
type: 'pdf',
path: '/vault/doc.pdf',
contents: 'pdfdata'
}
];
const result = claude.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(5);
expect(parsed[0].type).toBe('text');
expect(parsed[0].text).toBe('good.jpg');
expect(parsed[1].type).toBe('image');
expect(parsed[2]).toEqual({
type: 'text',
text: 'Unsupported image format: bad.bmp'
});
expect(parsed[3].type).toBe('text');
expect(parsed[3].text).toBe('doc.pdf');
expect(parsed[4].type).toBe('document');
});
it('should handle error when getting image mime type fails', () => {
const files = [{
type: 'image',
path: '/vault/image.unknown',
contents: 'imagedata'
}];
const result = claude.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(1);
expect(parsed[0].type).toBe('text');
expect(parsed[0].text).toContain('Image type not supported');
});
it('should handle files with uppercase extensions', () => {
const files = [
{
type: 'pdf',
path: '/vault/document.PDF',
contents: 'pdfdata'
},
{
type: 'image',
path: '/vault/photo.JPG',
contents: 'jpegdata'
}
];
const result = claude.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(4);
expect(parsed[0].text).toBe('document.PDF');
expect(parsed[1].type).toBe('document');
expect(parsed[2].text).toBe('photo.JPG');
expect(parsed[3].type).toBe('image');
});
it('should handle empty files array', () => {
const files: Array<{type: string, path: string, contents: string}> = [];
const result = claude.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(0);
});
it('should properly encode filenames with special characters', () => {
const files = [{
type: 'pdf',
path: '/vault/documents/report (final) v2.pdf',
contents: 'pdfdata'
}];
const result = claude.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed[0].text).toBe('report (final) v2.pdf');
});
it('should handle JPEG files with .jpeg extension', () => {
const files = [{
type: 'image',
path: '/vault/photo.jpeg',
contents: 'jpegdata'
}];
const result = claude.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed[1]).toEqual({
type: 'image',
source: {
type: 'base64',
media_type: 'image/jpeg',
data: 'jpegdata'
}
});
});
});
});

261
__tests__/AIClasses/Gemini.test.ts
View file

@ -1076,4 +1076,265 @@ describe('Gemini', () => {
expect((gemini as any).accumulatedThoughtSignature).toBeNull();
});
});
describe('formatBinaryFiles', () => {
it('should format PDF files with inlineData', () => {
const files = [{
type: 'pdf',
path: '/vault/documents/report.pdf',
contents: 'base64encodedcontent'
}];
const result = gemini.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(2);
expect(parsed[0]).toEqual({
text: 'report.pdf'
});
expect(parsed[1]).toEqual({
inlineData: {
mimeType: 'application/pdf',
data: 'base64encodedcontent'
}
});
});
it('should format JPEG images with inlineData', () => {
const files = [{
type: 'image',
path: '/vault/images/photo.jpg',
contents: 'base64imagedata'
}];
const result = gemini.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(2);
expect(parsed[0]).toEqual({
text: 'photo.jpg'
});
expect(parsed[1]).toEqual({
inlineData: {
mimeType: 'image/jpeg',
data: 'base64imagedata'
}
});
});
it('should format PNG images with inlineData', () => {
const files = [{
type: 'image',
path: '/vault/images/diagram.png',
contents: 'base64pngdata'
}];
const result = gemini.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(2);
expect(parsed[1]).toEqual({
inlineData: {
mimeType: 'image/png',
data: 'base64pngdata'
}
});
});
it('should handle unsupported image formats (GIF) with error message', () => {
const files = [{
type: 'image',
path: '/vault/images/animation.gif',
contents: 'base64gifdata'
}];
const result = gemini.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(1);
expect(parsed[0]).toEqual({
text: 'Unsupported image format: animation.gif'
});
});
it('should handle unsupported image formats (BMP) with error message', () => {
const files = [{
type: 'image',
path: '/vault/images/photo.bmp',
contents: 'base64bmpdata'
}];
const result = gemini.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(1);
expect(parsed[0]).toEqual({
text: 'Unsupported image format: photo.bmp'
});
});
it('should handle multiple files of different types', () => {
const files = [
{
type: 'pdf',
path: '/vault/doc.pdf',
contents: 'pdfdata'
},
{
type: 'image',
path: '/vault/image.jpg',
contents: 'jpegdata'
},
{
type: 'image',
path: '/vault/screenshot.png',
contents: 'pngdata'
}
];
const result = gemini.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(6);
// PDF file
expect(parsed[0]).toEqual({ text: 'doc.pdf' });
expect(parsed[1]).toEqual({
inlineData: {
mimeType: 'application/pdf',
data: 'pdfdata'
}
});
// JPEG image
expect(parsed[2]).toEqual({ text: 'image.jpg' });
expect(parsed[3]).toEqual({
inlineData: {
mimeType: 'image/jpeg',
data: 'jpegdata'
}
});
// PNG image
expect(parsed[4]).toEqual({ text: 'screenshot.png' });
expect(parsed[5]).toEqual({
inlineData: {
mimeType: 'image/png',
data: 'pngdata'
}
});
});
it('should handle mixed supported and unsupported files', () => {
const files = [
{
type: 'image',
path: '/vault/good.jpg',
contents: 'jpegdata'
},
{
type: 'image',
path: '/vault/bad.bmp',
contents: 'bmpdata'
},
{
type: 'pdf',
path: '/vault/doc.pdf',
contents: 'pdfdata'
}
];
const result = gemini.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(5);
expect(parsed[0]).toEqual({ text: 'good.jpg' });
expect(parsed[1]).toHaveProperty('inlineData');
expect(parsed[2]).toEqual({
text: 'Unsupported image format: bad.bmp'
});
expect(parsed[3]).toEqual({ text: 'doc.pdf' });
expect(parsed[4]).toHaveProperty('inlineData');
});
it('should handle error when getting image mime type fails', () => {
const files = [{
type: 'image',
path: '/vault/image.unknown',
contents: 'imagedata'
}];
const result = gemini.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(1);
expect(parsed[0]).toHaveProperty('text');
expect(parsed[0].text).toContain('Image type not supported');
});
it('should handle files with uppercase extensions', () => {
const files = [
{
type: 'pdf',
path: '/vault/document.PDF',
contents: 'pdfdata'
},
{
type: 'image',
path: '/vault/photo.JPG',
contents: 'jpegdata'
}
];
const result = gemini.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(4);
expect(parsed[0].text).toBe('document.PDF');
expect(parsed[1]).toHaveProperty('inlineData');
expect(parsed[2].text).toBe('photo.JPG');
expect(parsed[3]).toHaveProperty('inlineData');
});
it('should handle empty files array', () => {
const files: Array<{type: string, path: string, contents: string}> = [];
const result = gemini.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(0);
});
it('should properly encode filenames with special characters', () => {
const files = [{
type: 'pdf',
path: '/vault/documents/report (final) v2.pdf',
contents: 'pdfdata'
}];
const result = gemini.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed[0].text).toBe('report (final) v2.pdf');
});
it('should handle JPEG files with .jpeg extension', () => {
const files = [{
type: 'image',
path: '/vault/photo.jpeg',
contents: 'jpegdata'
}];
const result = gemini.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed[1]).toEqual({
inlineData: {
mimeType: 'image/jpeg',
data: 'jpegdata'
}
});
});
});
});

249
__tests__/AIClasses/OpenAI.test.ts
View file

@ -1004,4 +1004,253 @@ describe('OpenAI', () => {
expect(webSearchTool.name).toBe(undefined);
});
});
describe('formatBinaryFiles', () => {
it('should format PDF files with input_file type', () => {
const files = [{
type: 'pdf',
path: '/vault/documents/report.pdf',
contents: 'base64encodedcontent'
}];
const result = openai.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(1);
expect(parsed[0].role).toBe('user');
expect(parsed[0].content).toHaveLength(1);
expect(parsed[0].content[0]).toEqual({
type: 'input_file',
filename: 'report.pdf',
file_data: 'data:application/pdf;base64,base64encodedcontent'
});
});
it('should format JPEG images with input_image type', () => {
const files = [{
type: 'image',
path: '/vault/images/photo.jpg',
contents: 'base64imagedata'
}];
const result = openai.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(1);
expect(parsed[0].role).toBe('user');
expect(parsed[0].content).toHaveLength(1);
expect(parsed[0].content[0]).toEqual({
type: 'input_image',
image_url: 'data:image/jpeg;base64,base64imagedata'
});
});
it('should format PNG images with input_image type', () => {
const files = [{
type: 'image',
path: '/vault/images/diagram.png',
contents: 'base64pngdata'
}];
const result = openai.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(1);
expect(parsed[0].content[0]).toEqual({
type: 'input_image',
image_url: 'data:image/png;base64,base64pngdata'
});
});
it('should format WebP images with input_image type', () => {
const files = [{
type: 'image',
path: '/vault/images/modern.webp',
contents: 'base64webpdata'
}];
const result = openai.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(1);
expect(parsed[0].content[0]).toEqual({
type: 'input_image',
image_url: 'data:image/webp;base64,base64webpdata'
});
});
it('should handle unsupported image formats with error message', () => {
const files = [{
type: 'image',
path: '/vault/images/photo.gif',
contents: 'base64gifdata'
}];
const result = openai.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(1);
expect(parsed[0].content).toHaveLength(1);
expect(parsed[0].content[0]).toEqual({
type: 'input_text',
text: 'Unsupported image format: photo.gif'
});
});
it('should handle multiple files of different types', () => {
const files = [
{
type: 'pdf',
path: '/vault/doc.pdf',
contents: 'pdfdata'
},
{
type: 'image',
path: '/vault/image.jpg',
contents: 'jpegdata'
},
{
type: 'image',
path: '/vault/screenshot.png',
contents: 'pngdata'
}
];
const result = openai.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(1);
expect(parsed[0].role).toBe('user');
expect(parsed[0].content).toHaveLength(3);
expect(parsed[0].content[0]).toEqual({
type: 'input_file',
filename: 'doc.pdf',
file_data: 'data:application/pdf;base64,pdfdata'
});
expect(parsed[0].content[1]).toEqual({
type: 'input_image',
image_url: 'data:image/jpeg;base64,jpegdata'
});
expect(parsed[0].content[2]).toEqual({
type: 'input_image',
image_url: 'data:image/png;base64,pngdata'
});
});
it('should handle mixed supported and unsupported files', () => {
const files = [
{
type: 'image',
path: '/vault/good.jpg',
contents: 'jpegdata'
},
{
type: 'image',
path: '/vault/bad.bmp',
contents: 'bmpdata'
},
{
type: 'pdf',
path: '/vault/doc.pdf',
contents: 'pdfdata'
}
];
const result = openai.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(1);
expect(parsed[0].content).toHaveLength(3);
expect(parsed[0].content[0].type).toBe('input_image');
expect(parsed[0].content[1]).toEqual({
type: 'input_text',
text: 'Unsupported image format: bad.bmp'
});
expect(parsed[0].content[2].type).toBe('input_file');
});
it('should handle error when getting image mime type fails', () => {
const files = [{
type: 'image',
path: '/vault/image.unknown',
contents: 'imagedata'
}];
const result = openai.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(1);
expect(parsed[0].content).toHaveLength(1);
expect(parsed[0].content[0].type).toBe('input_text');
expect(parsed[0].content[0].text).toContain('Image type not supported');
});
it('should handle files with uppercase extensions', () => {
const files = [
{
type: 'pdf',
path: '/vault/document.PDF',
contents: 'pdfdata'
},
{
type: 'image',
path: '/vault/photo.JPG',
contents: 'jpegdata'
}
];
const result = openai.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(1);
expect(parsed[0].content).toHaveLength(2);
expect(parsed[0].content[0].type).toBe('input_file');
expect(parsed[0].content[0].filename).toBe('document.PDF');
expect(parsed[0].content[1].type).toBe('input_image');
});
it('should handle empty files array', () => {
const files: Array<{type: string, path: string, contents: string}> = [];
const result = openai.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed).toHaveLength(1);
expect(parsed[0].role).toBe('user');
expect(parsed[0].content).toHaveLength(0);
});
it('should properly encode filenames with special characters', () => {
const files = [{
type: 'pdf',
path: '/vault/documents/report (final) v2.pdf',
contents: 'pdfdata'
}];
const result = openai.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed[0].content[0].filename).toBe('report (final) v2.pdf');
});
it('should handle JPEG files with .jpeg extension', () => {
const files = [{
type: 'image',
path: '/vault/photo.jpeg',
contents: 'jpegdata'
}];
const result = openai.formatBinaryFiles(files);
const parsed = JSON.parse(result);
expect(parsed[0].content[0]).toEqual({
type: 'input_image',
image_url: 'data:image/jpeg;base64,jpegdata'
});
});
});
});