From 8a16ee125b069d22d8409347a1e74053946daee7 Mon Sep 17 00:00:00 2001 From: Andrew Beal Date: Fri, 19 Dec 2025 20:05:46 +0000 Subject: [PATCH] refactor: migrate file upload from base64 to file ID API for all providers - Add file ID storage and upload tracking to Attachment class - Store provider field in BaseAIClass for file service integration - Update Claude, Gemini, and OpenAI to use file ID references - Replace fetch with requestUrl for native Obsidian HTTP handling - Implement graceful upload failure handling with user notifications - Add retry logic with default values to prevent conversation breaks - Optimize file cache refresh to run only when attachments present - Filter empty function responses in conversation content validation --- AIClasses/BaseAIClass.ts | 4 +- AIClasses/BaseAIFileService.ts | 114 ++++++++++++------ AIClasses/Claude/Claude.ts | 165 +++++++++++++++----------- AIClasses/Claude/ClaudeFileService.ts | 59 ++++----- AIClasses/Claude/ClaudeTypes.ts | 12 +- AIClasses/Gemini/Gemini.ts | 65 +++++++--- AIClasses/Gemini/GeminiFileService.ts | 59 ++++----- AIClasses/Gemini/GeminiTypes.ts | 6 +- AIClasses/IAIFileService.ts | 2 +- AIClasses/OpenAI/OpenAI.ts | 47 +++++++- AIClasses/OpenAI/OpenAIFileService.ts | 55 ++++----- Conversations/Conversation.ts | 11 +- 12 files changed, 368 insertions(+), 231 deletions(-) diff --git a/AIClasses/BaseAIClass.ts b/AIClasses/BaseAIClass.ts index 0576359..eaf883d 100644 --- a/AIClasses/BaseAIClass.ts +++ b/AIClasses/BaseAIClass.ts @@ -20,6 +20,7 @@ import type { IAIFileService } from "./IAIFileService"; export abstract class BaseAIClass implements IAIClass { + protected readonly provider: AIProvider; protected readonly apiKey: string; protected readonly aiPrompt: IPrompt; protected readonly abortService: AbortService; @@ -29,6 +30,7 @@ export abstract class BaseAIClass implements IAIClass { protected readonly aiFunctionDefinitions: AIFunctionDefinitions; protected constructor(provider: AIProvider) { + this.provider = provider; this.aiPrompt = Resolve(Services.IPrompt); this.abortService = Resolve(Services.AbortService); this.aiFileService = Resolve(Services.IAIFileService); @@ -54,7 +56,7 @@ export abstract class BaseAIClass implements IAIClass { protected filterConversationContents(conversationContent: ConversationContent[]): ConversationContent[] { return conversationContent.filter((content, index, array) => { - if (!content.content && !content.functionCall && (!content.attachments || content.attachments.length === 0)) { + if (!content.content && !content.functionCall && !content.functionResponse && (!content.attachments || content.attachments.length === 0)) { return false; // Filter out empty content } diff --git a/AIClasses/BaseAIFileService.ts b/AIClasses/BaseAIFileService.ts index ef2ec72..8332e86 100644 --- a/AIClasses/BaseAIFileService.ts +++ b/AIClasses/BaseAIFileService.ts @@ -39,21 +39,24 @@ export abstract class BaseAIFileService implements IAIFileService { return [...this.fileIDs]; } - public async uploadFile(attachment: Attachment): Promise { + public async uploadFile(attachment: Attachment): Promise { const existingFileID = attachment.getFileID(this.provider); if (existingFileID && this.fileIDs.contains(existingFileID)) { - return existingFileID; + return; } const fileID = await this.uploadFileToAPI(attachment.base64, attachment.mimeType, attachment.fileName); + + if (fileID.trim() === "") { + return; // We tried, the agent will be notified of the failure + } + attachment.setFileID(this.provider, fileID); if (!this.fileIDs.contains(fileID)) { this.fileIDs.push(fileID); } - - return fileID; } public async deleteFile(attachment: Attachment): Promise { @@ -77,48 +80,87 @@ export abstract class BaseAIFileService implements IAIFileService { } // Retries operation on retryable errors (500, 502, 503, 504) with exponential backoff - protected async withRetry(operationName: string, operation: () => Promise): Promise { - let lastError: unknown; + protected async withRetry(operationName: string, operation: () => Promise, defaultValue: T): Promise { + return await this.abortService.abortableOperation(async () => { + for (let attempt = 1; attempt <= BaseAIFileService.MAX_RETRIES; attempt++) { + try { + return await operation(); + } catch (error) { - for (let attempt = 1; attempt <= BaseAIFileService.MAX_RETRIES; attempt++) { - try { - return await operation(); - } catch (error) { - lastError = error; - - // Don't retry on abort errors - if (AbortService.isAbortError(error)) { - throw error; - } - - if (ApiError.isApiError(error) && error.info.isRetryable) { - if (attempt === BaseAIFileService.MAX_RETRIES) { - Exception.log(`${operationName}: Max retries (${BaseAIFileService.MAX_RETRIES}) exhausted`); + // Don't retry on abort errors - throw immediately + if (AbortService.isAbortError(error)) { throw error; } - const delay = BaseAIFileService.RETRY_DELAYS[attempt]; - Exception.warn(`${operationName}: Attempt ${attempt}/${BaseAIFileService.MAX_RETRIES} failed with ${error.info.type} (status ${error.info.statusCode}). Retrying in ${delay}ms...`); + if (ApiError.isApiError(error) && error.info.isRetryable) { + if (attempt === BaseAIFileService.MAX_RETRIES) { + Exception.log(`${operationName}: Max retries (${BaseAIFileService.MAX_RETRIES}) exhausted. Returning default value.`); + return defaultValue; + } - if (this.abortService.signal().aborted) { - this.abortService.throw(); + const delay = BaseAIFileService.RETRY_DELAYS[attempt]; + Exception.warn(`${operationName}: Attempt ${attempt}/${BaseAIFileService.MAX_RETRIES} failed with ${error.info.type} (status ${error.info.statusCode}). Retrying in ${delay}ms...`); + + if (this.abortService.signal().aborted) { + this.abortService.throw(); + } + + await sleep(delay); + + if (this.abortService.signal().aborted) { + this.abortService.throw(); + } + } else { + // Non-retryable error - return default value + Exception.log(`${operationName}: Non-retryable error. Returning default value.`); + return defaultValue; } - - await sleep(delay); - - if (this.abortService.signal().aborted) { - this.abortService.throw(); - } - } else { - throw error; } } - } - throw lastError; + return defaultValue; + }); } - protected createBlob(bytes: Uint8Array, mimeType: string): Blob { - return new Blob([bytes], { type: mimeType }); + protected createBoundary(): string { + return `----FormBoundary${Date.now()}${Math.random().toString(36).substring(2)}`; + } + + protected createFormData(displayName: string | undefined, mimeType: string, boundary: string, bytes: Uint8Array, additionalFields?: Record): Buffer { + const parts: Buffer[] = []; + + // Add the file field + parts.push( + Buffer.from( + `--${boundary}\r\n` + + `Content-Disposition: form-data; name="file"; filename="${displayName || 'file'}"\r\n` + + `Content-Type: ${mimeType}\r\n\r\n`, + 'utf8' + ) + ); + parts.push(Buffer.from(bytes)); + + // Add any additional fields + if (additionalFields) { + for (const [key, value] of Object.entries(additionalFields)) { + parts.push( + Buffer.from( + `\r\n--${boundary}\r\n` + + `Content-Disposition: form-data; name="${key}"\r\n\r\n` + + value, + 'utf8' + ) + ); + } + } + + // Add closing boundary + parts.push(Buffer.from(`\r\n--${boundary}--\r\n`, 'utf8')); + + return Buffer.concat(parts); + } + + protected bytesToBuffer(bytes: Uint8Array): ArrayBuffer { + return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); } } diff --git a/AIClasses/Claude/Claude.ts b/AIClasses/Claude/Claude.ts index 1c9fc1c..972b34d 100644 --- a/AIClasses/Claude/Claude.ts +++ b/AIClasses/Claude/Claude.ts @@ -31,9 +31,14 @@ export class Claude extends BaseAIClass { this.accumulatedFunctionArgs = ""; this.accumulatedFunctionId = null; + // Refresh file cache only if conversation has attachments + if (conversation.hasAttachments()) { + await this.aiFileService.refreshCache(); + } + const systemPrompt = await this.buildSystemPrompt(); - const messages = this.extractContents(conversation.contents); + const messages = await this.extractContents(conversation.contents); const tools = [{ type: "web_search_20250305", @@ -56,6 +61,7 @@ export class Claude extends BaseAIClass { const headers = { "x-api-key": this.apiKey, "anthropic-version": "2023-06-01", + "anthropic-beta": "files-api-2025-04-14", "anthropic-dangerous-direct-browser-access": "true" }; @@ -146,83 +152,106 @@ export class Claude extends BaseAIClass { } } - protected extractContents(conversationContent: ConversationContent[]): { role: Role; content: ContentBlockParam[]; }[] { - return this.filterConversationContents(conversationContent) - .map(content => { - const contentBlocks: ContentBlockParam[] = []; - const contentToExtract = content.content ?? ""; + protected async extractContents(conversationContent: ConversationContent[]): Promise<{ role: Role; content: ContentBlockParam[]; }[]> { + const results = []; - if (contentToExtract.trim() !== "" && !content.functionResponse && (!content.attachments || content.attachments.length === 0)) { + for (const content of this.filterConversationContents(conversationContent)) { + const contentBlocks: ContentBlockParam[] = []; + const contentToExtract = content.content ?? ""; + + if (contentToExtract.trim() !== "" && !content.functionResponse && (!content.attachments || content.attachments.length === 0)) { + contentBlocks.push({ + type: "text", + text: contentToExtract + }); + } + + // Add function call if present + if (content.functionCall) { + const parsedContent = this.parseFunctionCall(content.functionCall); + + if (parsedContent) { + if (parsedContent.functionCall.id && parsedContent.functionCall.id.trim() !== "") { + contentBlocks.push({ + type: "tool_use", + id: parsedContent.functionCall.id, + name: parsedContent.functionCall.name, + input: parsedContent.functionCall.args + }); + } else { + contentBlocks.push({ + type: "text", + text: this.convertFunctionCallToText(parsedContent) + }); + } + } else { contentBlocks.push({ type: "text", - text: contentToExtract + text: "Error parsing function call" }); } + } - // Add function call if present - if (content.functionCall) { - const parsedContent = this.parseFunctionCall(content.functionCall); + // Add binary file attachments if present + if (content.attachments && content.attachments.length > 0) { + // Upload all attachments and track failures + const failedUploads: string[] = []; - if (parsedContent) { - if (parsedContent.functionCall.id && parsedContent.functionCall.id.trim() !== "") { - contentBlocks.push({ - type: "tool_use", - id: parsedContent.functionCall.id, - name: parsedContent.functionCall.name, - input: parsedContent.functionCall.args - }); - } else { - contentBlocks.push({ - type: "text", - text: this.convertFunctionCallToText(parsedContent) - }); - } - } else { - contentBlocks.push({ - type: "text", - text: "Error parsing function call" - }); + for (const attachment of content.attachments) { + try { + await this.aiFileService.uploadFile(attachment); + } catch (error) { + Exception.log(`Failed to upload ${attachment.fileName}: ${Exception.messageFrom(error)}`); + failedUploads.push(attachment.fileName); } } - // Add binary file attachments if present - if (content.attachments && content.attachments.length > 0) { - const formattedContent = this.formatBinaryFiles(content.attachments); - const rawContent = JSON.parse(formattedContent) as ContentBlockParam[]; - contentBlocks.push(...rawContent); + // Format successfully uploaded files + const formattedContent = this.formatBinaryFiles(content.attachments); + const rawContent = JSON.parse(formattedContent) as ContentBlockParam[]; + contentBlocks.push(...rawContent); + + // Add error messages for failed uploads + if (failedUploads.length > 0) { + contentBlocks.push({ + type: "text", + text: `[Upload failed for: ${failedUploads.join(', ')}.]` + }); } + } - // Add function response if present - if (content.functionResponse) { - const parsedContent = this.parseFunctionResponse(content.functionResponse); + // Add function response if present + if (content.functionResponse) { + const parsedContent = this.parseFunctionResponse(content.functionResponse); - if (parsedContent) { - if (parsedContent.id && parsedContent.id.trim() !== "") { - contentBlocks.push({ - type: "tool_result", - tool_use_id: parsedContent.id, - content: JSON.stringify(parsedContent.functionResponse.response) - }); - } else { - contentBlocks.push({ - type: "text", - text: this.convertFunctionResponseToText(parsedContent) - }); - } + if (parsedContent) { + if (parsedContent.id && parsedContent.id.trim() !== "") { + contentBlocks.push({ + type: "tool_result", + tool_use_id: parsedContent.id, + content: JSON.stringify(parsedContent.functionResponse.response) + }); } else { contentBlocks.push({ type: "text", - text: content.functionResponse + text: this.convertFunctionResponseToText(parsedContent) }); } + } else { + contentBlocks.push({ + type: "text", + text: content.functionResponse + }); } + } - return { - role: content.role, - content: contentBlocks - }; - }) - .filter(message => message.content.length > 0); + results.push({ + role: content.role, + content: contentBlocks + }); + } + + return results.filter(message => message.content.length > 0); } protected mapFunctionDefinitions(aiFunctionDefinitions: IAIFunctionDefinition[]): Tool[] { @@ -239,19 +268,20 @@ export class Claude extends BaseAIClass { public formatBinaryFiles(attachments: Attachment[]): string { const contentBlocks = attachments.flatMap(attachment => { - let blockType: string; + // Check for uploaded file ID + const fileID = attachment.getFileID(this.provider); + if (!fileID) { + // Skip - upload failed, error message added in extractContents() + return []; + } + let blockType: string; if (attachment.mimeType === "application/pdf") { blockType = "document"; } else { - // Image handling blockType = "image"; - - // Validate supported image types if (!this.SUPPORTED_IMAGE_TYPES.includes(attachment.mimeType)) { - return [ - { type: "text", text: `Unsupported image format: ${attachment.fileName}` } - ]; + return [{ type: "text", text: `Unsupported image format: ${attachment.fileName}` }]; } } @@ -260,9 +290,8 @@ export class Claude extends BaseAIClass { { type: blockType, source: { - type: "base64", - media_type: attachment.mimeType, - data: attachment.base64 + type: "file", + file_id: fileID } } ]; diff --git a/AIClasses/Claude/ClaudeFileService.ts b/AIClasses/Claude/ClaudeFileService.ts index a2fd39a..74a2c59 100644 --- a/AIClasses/Claude/ClaudeFileService.ts +++ b/AIClasses/Claude/ClaudeFileService.ts @@ -1,90 +1,91 @@ import { BaseAIFileService } from "AIClasses/BaseAIFileService"; import { AIFileServiceURL, AIProvider } from "Enums/ApiProvider"; import { StringTools } from "Helpers/StringTools"; +import { requestUrl } from "obsidian"; import { ApiError } from "Types/ApiError"; +import type { ClaudeFile, ClaudeListFilesResponse } from "./ClaudeTypes"; export class ClaudeFileService extends BaseAIFileService { - private readonly betaHeader = "files-api-2025-04-14"; - public constructor() { super(AIProvider.Claude); } protected async listFilesFromAPI(): Promise { return this.withRetry("List files", async () => { - const response = await fetch(AIFileServiceURL.Claude, { + + const response = await requestUrl({ + url: AIFileServiceURL.Claude, method: "GET", headers: { "x-api-key": this.apiKey, "anthropic-version": "2023-06-01", - "anthropic-beta": this.betaHeader + "anthropic-beta": "files-api-2025-04-14" }, - signal: this.abortService.signal() + throw: false }); - if (!response.ok) { - const responseBody = await response.text(); - throw ApiError.fromResponse(response.status, response.statusText, responseBody); + if (response.status !== 200) { + throw ApiError.fromResponse(response.status, "List files failed", response.text); } - const data = await response.json() as ClaudeListFilesResponse; + const data = response.json as ClaudeListFilesResponse; if (!data.data || data.data.length === 0) { return []; } return data.data.map(file => file.id); - }); + }, []); } protected async uploadFileToAPI(data: string, mimeType: string, displayName?: string): Promise { return this.withRetry("Upload file", async () => { const bytes = StringTools.toBytes(data); - const blob = this.createBlob(bytes, mimeType); - const formData = new FormData(); - formData.append("file", blob, displayName || "file"); + const boundary = this.createBoundary(); + const formData = this.createFormData(displayName, mimeType, boundary, bytes); - const response = await fetch(AIFileServiceURL.Claude, { + const response = await requestUrl({ + url: AIFileServiceURL.Claude, method: "POST", headers: { "x-api-key": this.apiKey, "anthropic-version": "2023-06-01", - "anthropic-beta": this.betaHeader + "anthropic-beta": "files-api-2025-04-14", + "Content-Type": `multipart/form-data; boundary=${boundary}` }, - body: formData, - signal: this.abortService.signal() + body: formData.buffer, + throw: false }); - if (!response.ok) { - const responseBody = await response.text(); - throw ApiError.fromResponse(response.status, response.statusText, responseBody); + if (response.status !== 200 && response.status !== 201) { + throw ApiError.fromResponse(response.status, "Upload file failed", response.text); } - const responseData = await response.json() as ClaudeFile; + const responseData = response.json as ClaudeFile; return responseData.id; - }); + }, ""); } protected async deleteFileFromAPI(id: string): Promise { return this.withRetry("Delete file", async () => { - const response = await fetch(`${AIFileServiceURL.Claude}/${id}`, { + const response = await requestUrl({ + url: `${AIFileServiceURL.Claude}/${id}`, method: "DELETE", headers: { "x-api-key": this.apiKey, "anthropic-version": "2023-06-01", - "anthropic-beta": this.betaHeader + "anthropic-beta": "files-api-2025-04-14" }, - signal: this.abortService.signal() + throw: false }); - if (!response.ok && response.status !== 204 && response.status !== 404) { - const responseBody = await response.text(); - throw ApiError.fromResponse(response.status, response.statusText, responseBody); + if (response.status !== 200 && response.status !== 204 && response.status !== 404) { + throw ApiError.fromResponse(response.status, "Delete file failed", response.text); } - }); + }, undefined); } } \ No newline at end of file diff --git a/AIClasses/Claude/ClaudeTypes.ts b/AIClasses/Claude/ClaudeTypes.ts index 3817d37..39c8c8b 100644 --- a/AIClasses/Claude/ClaudeTypes.ts +++ b/AIClasses/Claude/ClaudeTypes.ts @@ -1,4 +1,4 @@ -interface ClaudeFile { +export interface ClaudeFile { id: string; type: "file"; filename: string; @@ -8,15 +8,9 @@ interface ClaudeFile { downloadable: boolean; } -interface ClaudeListFilesResponse { +export interface ClaudeListFilesResponse { data: ClaudeFile[]; has_more: boolean; first_id?: string; last_id?: string; -} - -interface ClaudeDeleteResponse { - id: string; - type: "file"; - deleted: boolean; -} +} \ No newline at end of file diff --git a/AIClasses/Gemini/Gemini.ts b/AIClasses/Gemini/Gemini.ts index eeb3eef..c574620 100644 --- a/AIClasses/Gemini/Gemini.ts +++ b/AIClasses/Gemini/Gemini.ts @@ -10,6 +10,7 @@ import type { IAIFunctionDefinition } from "AIClasses/FunctionDefinitions/IAIFun import type { ConversationContent } from "Conversations/ConversationContent"; import type { Candidate, Part, FunctionDeclaration } from "@google/genai"; import { FinishReason } from "@google/genai"; +import { Exception } from "Helpers/Exception"; export class Gemini extends BaseAIClass { @@ -34,7 +35,12 @@ export class Gemini extends BaseAIClass { this.accumulatedFunctionArgs = {}; this.accumulatedThoughtSignature = null; - const contents = this.extractContents(conversation.contents); + // Refresh file cache only if conversation has attachments + if (conversation.hasAttachments()) { + await this.aiFileService.refreshCache(); + } + + const contents = await this.extractContents(conversation.contents); const tools = requestWebSearch ? { google_search: {} } : { @@ -152,9 +158,10 @@ export class Gemini extends BaseAIClass { } } - protected extractContents(conversationContent: ConversationContent[]): { role: Role, parts: Part[] }[] { - return this.filterConversationContents(conversationContent) - .map(content => { + protected async extractContents(conversationContent: ConversationContent[]): Promise<{ role: Role, parts: Part[] }[]> { + const results = []; + + for (const content of this.filterConversationContents(conversationContent)) { const parts: Part[] = []; const contentToExtract = content.content ?? ""; @@ -193,9 +200,29 @@ export class Gemini extends BaseAIClass { // Add binary file attachments if present if (content.attachments && content.attachments.length > 0) { + // Upload all attachments and track failures + const failedUploads: string[] = []; + + for (const attachment of content.attachments) { + try { + await this.aiFileService.uploadFile(attachment); + } catch (error) { + Exception.log(`Failed to upload ${attachment.fileName}: ${Exception.messageFrom(error)}`); + failedUploads.push(attachment.fileName); + } + } + + // Format successfully uploaded files const formattedContent = this.formatBinaryFiles(content.attachments); const rawContent = JSON.parse(formattedContent) as Part[]; parts.push(...rawContent); + + // Add error messages for failed uploads + if (failedUploads.length > 0) { + parts.push({ + text: `[Upload failed for: ${failedUploads.join(', ')}.]` + }); + } } // Add function response if present @@ -225,12 +252,13 @@ export class Gemini extends BaseAIClass { } } - return { - role: content.role === Role.User ? Role.User : Role.Model, - parts: parts - }; - }) - .filter(message => message.parts.length > 0); + results.push({ + role: content.role === Role.User ? Role.User : Role.Model, + parts: parts + }); + } + + return results.filter(message => message.parts.length > 0); } protected mapFunctionDefinitions(aiFunctionDefinitions: IAIFunctionDefinition[]): FunctionDeclaration[] { @@ -245,22 +273,27 @@ export class Gemini extends BaseAIClass { const parts: unknown[] = []; for (const attachment of attachments) { + // Check for uploaded file ID + const fileID = attachment.getFileID(this.provider); + if (!fileID) { + // Skip - upload failed, error message added in extractContents() + continue; + } + // Validate image types (Gemini only supports JPEG and PNG) if (attachment.mimeType.startsWith('image/')) { if (!this.SUPPORTED_IMAGE_TYPES.includes(attachment.mimeType)) { - parts.push({ - text: `Unsupported image format: ${attachment.fileName}` - }); + parts.push({ text: `Unsupported image format: ${attachment.fileName}` }); continue; } } - // Add filename text block, then binary data + // Add filename and file data parts.push({text: attachment.fileName}); parts.push({ - inlineData: { + fileData: { mimeType: attachment.mimeType, - data: attachment.base64 + fileUri: fileID // Format: "files/abc123" } }); } diff --git a/AIClasses/Gemini/GeminiFileService.ts b/AIClasses/Gemini/GeminiFileService.ts index 3bf849a..6b99d9e 100644 --- a/AIClasses/Gemini/GeminiFileService.ts +++ b/AIClasses/Gemini/GeminiFileService.ts @@ -2,7 +2,9 @@ import { BaseAIFileService } from "AIClasses/BaseAIFileService"; import { AIFileServiceURL, AIProvider } from "Enums/ApiProvider"; import { Exception } from "Helpers/Exception"; import { StringTools } from "Helpers/StringTools"; +import { requestUrl } from "obsidian"; import { ApiError } from "Types/ApiError"; +import type { GeminiListFilesResponse, GeminiUploadResponse } from "./GeminiTypes"; export class GeminiFileService extends BaseAIFileService { @@ -12,27 +14,28 @@ export class GeminiFileService extends BaseAIFileService { protected async listFilesFromAPI(): Promise { return this.withRetry("List files", async () => { - const response = await fetch(`${AIFileServiceURL.Gemini}/files?key=${this.apiKey}`, { + + const response = await requestUrl({ + url: `${AIFileServiceURL.Gemini}/files?key=${this.apiKey}`, method: "GET", headers: { "Content-Type": "application/json" }, - signal: this.abortService.signal() + throw: false }); - if (!response.ok) { - const responseBody = await response.text(); - throw ApiError.fromResponse(response.status, response.statusText, responseBody); + if (response.status !== 200) { + throw ApiError.fromResponse(response.status, "List files failed", response.text); } - const data = await response.json() as GeminiListFilesResponse; + const data = response.json as GeminiListFilesResponse; if (!data.files || data.files.length === 0) { return []; } return data.files.map(file => file.name); - }); + }, []); } protected async uploadFileToAPI(data: string, mimeType: string, displayName?: string): Promise { @@ -43,7 +46,8 @@ export class GeminiFileService extends BaseAIFileService { const metadata = displayName ? { file: { displayName } } : {}; // Step 1: Initiate resumable upload - const initiateResponse = await fetch(`${AIFileServiceURL.GeminiUpload}/files?key=${this.apiKey}`, { + const initiateResponse = await requestUrl({ + url: `${AIFileServiceURL.GeminiUpload}/files?key=${this.apiKey}`, method: "POST", headers: { "X-Goog-Upload-Protocol": "resumable", @@ -53,55 +57,52 @@ export class GeminiFileService extends BaseAIFileService { "Content-Type": "application/json" }, body: JSON.stringify(metadata), - signal: this.abortService.signal() + throw: false }); - if (!initiateResponse.ok) { - const responseBody = await initiateResponse.text(); - throw ApiError.fromResponse(initiateResponse.status, initiateResponse.statusText, responseBody); + if (initiateResponse.status !== 200) { + throw ApiError.fromResponse(initiateResponse.status, "Upload file failed", initiateResponse.text); } - const uploadUrl = initiateResponse.headers.get("x-goog-upload-url"); + const uploadUrl = initiateResponse.headers["x-goog-upload-url"]; if (!uploadUrl) { Exception.throw("No upload URL received from initiate request"); } // Step 2: Upload file data - const blob = this.createBlob(bytes, mimeType); - const uploadResponse = await fetch(uploadUrl, { + const uploadResponse = await requestUrl({ + url: uploadUrl, method: "POST", headers: { - "Content-Length": numBytes.toString(), "X-Goog-Upload-Offset": "0", "X-Goog-Upload-Command": "upload, finalize" }, - body: blob, - signal: this.abortService.signal() + body: this.bytesToBuffer(bytes), + throw: false }); - if (!uploadResponse.ok) { - const responseBody = await uploadResponse.text(); - throw ApiError.fromResponse(uploadResponse.status, uploadResponse.statusText, responseBody); + if (uploadResponse.status !== 200 && uploadResponse.status !== 201) { + throw ApiError.fromResponse(uploadResponse.status, "Upload file failed", uploadResponse.text); } - const responseData = await uploadResponse.json() as GeminiUploadResponse; + const responseData = uploadResponse.json as GeminiUploadResponse; return responseData.file.uri; - }); + }, ""); } protected async deleteFileFromAPI(name: string): Promise { return this.withRetry("Delete file", async () => { - const response = await fetch(`${AIFileServiceURL.Gemini}/${name}?key=${this.apiKey}`, { + const response = await requestUrl({ + url: `${AIFileServiceURL.Gemini}/${name}?key=${this.apiKey}`, method: "DELETE", - signal: this.abortService.signal() + throw: false }); - if (!response.ok && response.status !== 204 && response.status !== 403) { - const responseBody = await response.text(); - throw ApiError.fromResponse(response.status, response.statusText, responseBody); + if (response.status !== 200 && response.status !== 204 && response.status !== 404) { + throw ApiError.fromResponse(response.status, "Delete file failed", response.text); } - }); + }, undefined); } } \ No newline at end of file diff --git a/AIClasses/Gemini/GeminiTypes.ts b/AIClasses/Gemini/GeminiTypes.ts index bb508ff..f9ad412 100644 --- a/AIClasses/Gemini/GeminiTypes.ts +++ b/AIClasses/Gemini/GeminiTypes.ts @@ -1,4 +1,4 @@ -interface GeminiFile { +export interface GeminiFile { name: string; displayName?: string; mimeType: string; @@ -14,11 +14,11 @@ interface GeminiFile { }; } -interface GeminiListFilesResponse { +export interface GeminiListFilesResponse { files?: GeminiFile[]; nextPageToken?: string; } -interface GeminiUploadResponse { +export interface GeminiUploadResponse { file: GeminiFile; } \ No newline at end of file diff --git a/AIClasses/IAIFileService.ts b/AIClasses/IAIFileService.ts index fc59c10..8c70d7a 100644 --- a/AIClasses/IAIFileService.ts +++ b/AIClasses/IAIFileService.ts @@ -3,6 +3,6 @@ import type { Attachment } from "Conversations/Attachment"; export interface IAIFileService { refreshCache(): Promise; listFiles(): string[]; - uploadFile(attachment: Attachment): Promise; + uploadFile(attachment: Attachment): Promise; deleteFile(attachment: Attachment): Promise; } \ No newline at end of file diff --git a/AIClasses/OpenAI/OpenAI.ts b/AIClasses/OpenAI/OpenAI.ts index 0927639..f9ac112 100644 --- a/AIClasses/OpenAI/OpenAI.ts +++ b/AIClasses/OpenAI/OpenAI.ts @@ -23,9 +23,14 @@ export class OpenAI extends BaseAIClass { conversation: Conversation, allowDestructiveActions: boolean ): AsyncGenerator { + // Refresh file cache only if conversation has attachments + if (conversation.hasAttachments()) { + await this.aiFileService.refreshCache(); + } + const systemPrompt = await this.buildSystemPrompt(); - const input = this.extractContents(conversation.contents); + const input = await this.extractContents(conversation.contents); const tools = [{ type: "web_search" @@ -179,7 +184,7 @@ export class OpenAI extends BaseAIClass { } } - protected extractContents(conversationContent: ConversationContent[]): ResponsesAPIInput[] { + protected async extractContents(conversationContent: ConversationContent[]): Promise { const results: ResponsesAPIInput[] = []; for (const content of this.filterConversationContents(conversationContent)) { @@ -232,9 +237,31 @@ export class OpenAI extends BaseAIClass { // Case 2: Binary file attachments if (content.attachments && content.attachments.length > 0) { + // Upload all attachments and track failures + const failedUploads: string[] = []; + + for (const attachment of content.attachments) { + try { + await this.aiFileService.uploadFile(attachment); + } catch (error) { + Exception.log(`Failed to upload ${attachment.fileName}: ${Exception.messageFrom(error)}`); + failedUploads.push(attachment.fileName); + } + } + + // Format successfully uploaded files const formattedContent = this.formatBinaryFiles(content.attachments); const rawContent = JSON.parse(formattedContent) as ResponsesAPIInput[]; results.push(...rawContent); + + // Add error messages for failed uploads + if (failedUploads.length > 0) { + // OpenAI formatBinaryFiles returns array with role wrapper, so add as separate message + results.push({ + role: "user", + content: `[Upload failed for: ${failedUploads.join(', ')}.]` + }); + } continue; } @@ -293,14 +320,21 @@ export class OpenAI extends BaseAIClass { const contentBlocks: unknown[] = []; for (const attachment of attachments) { + // Check for uploaded file ID + const fileID = attachment.getFileID(this.provider); + if (!fileID) { + // Skip - upload failed, error message added in extractContents() + continue; + } + if (attachment.mimeType === "application/pdf") { + // Use file ID format for PDFs contentBlocks.push({ type: "input_file", - filename: attachment.fileName, - file_data: `data:${attachment.mimeType};base64,${attachment.base64}` + file_id: fileID }); } else { - // Image handling - validate supported types + // Images if (!this.SUPPORTED_IMAGE_TYPES.includes(attachment.mimeType)) { contentBlocks.push({ type: "input_text", @@ -309,9 +343,10 @@ export class OpenAI extends BaseAIClass { continue; } + // Use file ID format for images contentBlocks.push({ type: "input_image", - image_url: `data:${attachment.mimeType};base64,${attachment.base64}` + file_id: fileID }); } } diff --git a/AIClasses/OpenAI/OpenAIFileService.ts b/AIClasses/OpenAI/OpenAIFileService.ts index e562110..c89af05 100644 --- a/AIClasses/OpenAI/OpenAIFileService.ts +++ b/AIClasses/OpenAI/OpenAIFileService.ts @@ -2,6 +2,7 @@ import { BaseAIFileService } from "AIClasses/BaseAIFileService"; import { AIFileServiceURL, AIProvider } from "Enums/ApiProvider"; import type { OpenAIFile, OpenAIListFilesResponse } from "./OpenAITypes"; import { StringTools } from "Helpers/StringTools"; +import { requestUrl } from "obsidian"; import { ApiError } from "Types/ApiError"; export class OpenAIFileService extends BaseAIFileService { @@ -12,76 +13,76 @@ export class OpenAIFileService extends BaseAIFileService { protected async listFilesFromAPI(): Promise { return this.withRetry("List files", async () => { - const response = await fetch(AIFileServiceURL.OpenAI, { + + const response = await requestUrl({ + url: AIFileServiceURL.OpenAI, method: "GET", headers: { "Authorization": `Bearer ${this.apiKey}` }, - signal: this.abortService.signal() + throw: false }); - if (!response.ok) { - const responseBody = await response.text(); - throw ApiError.fromResponse(response.status, response.statusText, responseBody); + if (response.status !== 200) { + throw ApiError.fromResponse(response.status, "List files failed", response.text); } - const data = await response.json() as OpenAIListFilesResponse; + const data = response.json as OpenAIListFilesResponse; if (!data.data || data.data.length === 0) { return []; } return data.data.map(file => file.id); - }); + }, []); } protected async uploadFileToAPI(data: string, mimeType: string, displayName?: string): Promise { return this.withRetry("Upload file", async () => { const bytes = StringTools.toBytes(data); - const blob = this.createBlob(bytes, mimeType); - - const formData = new FormData(); - formData.append("file", blob, displayName || "file"); // Use 'vision' for images, 'user_data' for other files const purpose = mimeType.startsWith('image/') ? 'vision' : 'user_data'; - formData.append("purpose", purpose); - const response = await fetch(AIFileServiceURL.OpenAI, { + const boundary = this.createBoundary(); + const formData = this.createFormData(displayName, mimeType, boundary, bytes, { purpose }); + + const response = await requestUrl({ + url: AIFileServiceURL.OpenAI, method: "POST", headers: { - "Authorization": `Bearer ${this.apiKey}` + "Authorization": `Bearer ${this.apiKey}`, + "Content-Type": `multipart/form-data; boundary=${boundary}` }, - body: formData, - signal: this.abortService.signal() + body: formData.buffer, + throw: false }); - if (!response.ok) { - const responseBody = await response.text(); - throw ApiError.fromResponse(response.status, response.statusText, responseBody); + if (response.status !== 200 && response.status !== 201) { + throw ApiError.fromResponse(response.status, "Upload file failed", response.text); } - const responseData = await response.json() as OpenAIFile; + const responseData = response.json as OpenAIFile; return responseData.id; - }); + }, ""); } protected async deleteFileFromAPI(id: string): Promise { return this.withRetry("Delete file", async () => { - const response = await fetch(`${AIFileServiceURL.OpenAI}/${id}`, { + const response = await requestUrl({ + url: `${AIFileServiceURL.OpenAI}/${id}`, method: "DELETE", headers: { "Authorization": `Bearer ${this.apiKey}` }, - signal: this.abortService.signal() + throw: false }); - if (!response.ok && response.status !== 204 && response.status !== 404) { - const responseBody = await response.text(); - throw ApiError.fromResponse(response.status, response.statusText, responseBody); + if (response.status !== 200 && response.status !== 204 && response.status !== 404) { + throw ApiError.fromResponse(response.status, "Delete file failed", response.text); } - }); + }, undefined); } } \ No newline at end of file diff --git a/Conversations/Conversation.ts b/Conversations/Conversation.ts index 8284d68..c706531 100644 --- a/Conversations/Conversation.ts +++ b/Conversations/Conversation.ts @@ -22,6 +22,10 @@ export class Conversation { this.title = `${StringTools.dateToString(this.created)}`; } + public hasAttachments(): boolean { + return this.contents.some(c => c.attachments.length > 0); + } + public addFunctionResponse(functionResponse: AIFunctionResponse): void { if (functionResponse.name !== AIFunction.ReadVaultFiles) { const functionResponseString = functionResponse.toConversationString(); @@ -96,12 +100,7 @@ export class Conversation { } } - return new Attachment( - fileName, - mimeType, - file.contents, // base64 string - {} // empty fileID map (phase 2 feature) - ); + return new Attachment(fileName, mimeType, file.contents); }); this.contents.push(new ConversationContent({