mirror of
https://github.com/andy-stack/vaultkeeper-ai.git
synced 2026-07-22 06:42:03 +00:00
refactor: migrate file upload from base64 to file ID API for all providers
- Add file ID storage and upload tracking to Attachment class - Store provider field in BaseAIClass for file service integration - Update Claude, Gemini, and OpenAI to use file ID references - Replace fetch with requestUrl for native Obsidian HTTP handling - Implement graceful upload failure handling with user notifications - Add retry logic with default values to prevent conversation breaks - Optimize file cache refresh to run only when attachments present - Filter empty function responses in conversation content validation
This commit is contained in:
Andrew Beal
parent
e4f56e3877
commit
8a16ee125b
12 changed files with 368 additions and 231 deletions
|
|
@ -20,6 +20,7 @@ import type { IAIFileService } from "./IAIFileService";
|
|||
|
||||
export abstract class BaseAIClass implements IAIClass {
|
||||
|
||||
protected readonly provider: AIProvider;
|
||||
protected readonly apiKey: string;
|
||||
protected readonly aiPrompt: IPrompt;
|
||||
protected readonly abortService: AbortService;
|
||||
|
|
@ -29,6 +30,7 @@ export abstract class BaseAIClass implements IAIClass {
|
|||
protected readonly aiFunctionDefinitions: AIFunctionDefinitions;
|
||||
|
||||
protected constructor(provider: AIProvider) {
|
||||
this.provider = provider;
|
||||
this.aiPrompt = Resolve<IPrompt>(Services.IPrompt);
|
||||
this.abortService = Resolve<AbortService>(Services.AbortService);
|
||||
this.aiFileService = Resolve<IAIFileService>(Services.IAIFileService);
|
||||
|
|
@ -54,7 +56,7 @@ export abstract class BaseAIClass implements IAIClass {
|
|||
|
||||
protected filterConversationContents(conversationContent: ConversationContent[]): ConversationContent[] {
|
||||
return conversationContent.filter((content, index, array) => {
|
||||
if (!content.content && !content.functionCall && (!content.attachments || content.attachments.length === 0)) {
|
||||
if (!content.content && !content.functionCall && !content.functionResponse && (!content.attachments || content.attachments.length === 0)) {
|
||||
return false; // Filter out empty content
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -39,21 +39,24 @@ export abstract class BaseAIFileService implements IAIFileService {
|
|||
return [...this.fileIDs];
|
||||
}
|
||||
|
||||
public async uploadFile(attachment: Attachment): Promise<string> {
|
||||
public async uploadFile(attachment: Attachment): Promise<void> {
|
||||
const existingFileID = attachment.getFileID(this.provider);
|
||||
|
||||
if (existingFileID && this.fileIDs.contains(existingFileID)) {
|
||||
return existingFileID;
|
||||
return;
|
||||
}
|
||||
|
||||
const fileID = await this.uploadFileToAPI(attachment.base64, attachment.mimeType, attachment.fileName);
|
||||
|
||||
if (fileID.trim() === "") {
|
||||
return; // We tried, the agent will be notified of the failure
|
||||
}
|
||||
|
||||
attachment.setFileID(this.provider, fileID);
|
||||
|
||||
if (!this.fileIDs.contains(fileID)) {
|
||||
this.fileIDs.push(fileID);
|
||||
}
|
||||
|
||||
return fileID;
|
||||
}
|
||||
|
||||
public async deleteFile(attachment: Attachment): Promise<void> {
|
||||
|
|
@ -77,48 +80,87 @@ export abstract class BaseAIFileService implements IAIFileService {
|
|||
}
|
||||
|
||||
// Retries operation on retryable errors (500, 502, 503, 504) with exponential backoff
|
||||
protected async withRetry<T>(operationName: string, operation: () => Promise<T>): Promise<T> {
|
||||
let lastError: unknown;
|
||||
protected async withRetry<T>(operationName: string, operation: () => Promise<T>, defaultValue: T): Promise<T> {
|
||||
return await this.abortService.abortableOperation(async () => {
|
||||
for (let attempt = 1; attempt <= BaseAIFileService.MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error) {
|
||||
|
||||
for (let attempt = 1; attempt <= BaseAIFileService.MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
|
||||
// Don't retry on abort errors
|
||||
if (AbortService.isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (ApiError.isApiError(error) && error.info.isRetryable) {
|
||||
if (attempt === BaseAIFileService.MAX_RETRIES) {
|
||||
Exception.log(`${operationName}: Max retries (${BaseAIFileService.MAX_RETRIES}) exhausted`);
|
||||
// Don't retry on abort errors - throw immediately
|
||||
if (AbortService.isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
const delay = BaseAIFileService.RETRY_DELAYS[attempt];
|
||||
Exception.warn(`${operationName}: Attempt ${attempt}/${BaseAIFileService.MAX_RETRIES} failed with ${error.info.type} (status ${error.info.statusCode}). Retrying in ${delay}ms...`);
|
||||
if (ApiError.isApiError(error) && error.info.isRetryable) {
|
||||
if (attempt === BaseAIFileService.MAX_RETRIES) {
|
||||
Exception.log(`${operationName}: Max retries (${BaseAIFileService.MAX_RETRIES}) exhausted. Returning default value.`);
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
if (this.abortService.signal().aborted) {
|
||||
this.abortService.throw();
|
||||
const delay = BaseAIFileService.RETRY_DELAYS[attempt];
|
||||
Exception.warn(`${operationName}: Attempt ${attempt}/${BaseAIFileService.MAX_RETRIES} failed with ${error.info.type} (status ${error.info.statusCode}). Retrying in ${delay}ms...`);
|
||||
|
||||
if (this.abortService.signal().aborted) {
|
||||
this.abortService.throw();
|
||||
}
|
||||
|
||||
await sleep(delay);
|
||||
|
||||
if (this.abortService.signal().aborted) {
|
||||
this.abortService.throw();
|
||||
}
|
||||
} else {
|
||||
// Non-retryable error - return default value
|
||||
Exception.log(`${operationName}: Non-retryable error. Returning default value.`);
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
await sleep(delay);
|
||||
|
||||
if (this.abortService.signal().aborted) {
|
||||
this.abortService.throw();
|
||||
}
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
return defaultValue;
|
||||
});
|
||||
}
|
||||
|
||||
protected createBlob(bytes: Uint8Array<ArrayBuffer>, mimeType: string): Blob {
|
||||
return new Blob([bytes], { type: mimeType });
|
||||
protected createBoundary(): string {
|
||||
return `----FormBoundary${Date.now()}${Math.random().toString(36).substring(2)}`;
|
||||
}
|
||||
|
||||
protected createFormData(displayName: string | undefined, mimeType: string, boundary: string, bytes: Uint8Array<ArrayBuffer>, additionalFields?: Record<string, string>): Buffer<ArrayBuffer> {
|
||||
const parts: Buffer[] = [];
|
||||
|
||||
// Add the file field
|
||||
parts.push(
|
||||
Buffer.from(
|
||||
`--${boundary}\r\n` +
|
||||
`Content-Disposition: form-data; name="file"; filename="${displayName || 'file'}"\r\n` +
|
||||
`Content-Type: ${mimeType}\r\n\r\n`,
|
||||
'utf8'
|
||||
)
|
||||
);
|
||||
parts.push(Buffer.from(bytes));
|
||||
|
||||
// Add any additional fields
|
||||
if (additionalFields) {
|
||||
for (const [key, value] of Object.entries(additionalFields)) {
|
||||
parts.push(
|
||||
Buffer.from(
|
||||
`\r\n--${boundary}\r\n` +
|
||||
`Content-Disposition: form-data; name="${key}"\r\n\r\n` +
|
||||
value,
|
||||
'utf8'
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Add closing boundary
|
||||
parts.push(Buffer.from(`\r\n--${boundary}--\r\n`, 'utf8'));
|
||||
|
||||
return Buffer.concat(parts);
|
||||
}
|
||||
|
||||
protected bytesToBuffer(bytes: Uint8Array<ArrayBuffer>): ArrayBuffer {
|
||||
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,9 +31,14 @@ export class Claude extends BaseAIClass {
|
|||
this.accumulatedFunctionArgs = "";
|
||||
this.accumulatedFunctionId = null;
|
||||
|
||||
// Refresh file cache only if conversation has attachments
|
||||
if (conversation.hasAttachments()) {
|
||||
await this.aiFileService.refreshCache();
|
||||
}
|
||||
|
||||
const systemPrompt = await this.buildSystemPrompt();
|
||||
|
||||
const messages = this.extractContents(conversation.contents);
|
||||
const messages = await this.extractContents(conversation.contents);
|
||||
|
||||
const tools = [{
|
||||
type: "web_search_20250305",
|
||||
|
|
@ -56,6 +61,7 @@ export class Claude extends BaseAIClass {
|
|||
const headers = {
|
||||
"x-api-key": this.apiKey,
|
||||
"anthropic-version": "2023-06-01",
|
||||
"anthropic-beta": "files-api-2025-04-14",
|
||||
"anthropic-dangerous-direct-browser-access": "true"
|
||||
};
|
||||
|
||||
|
|
@ -146,83 +152,106 @@ export class Claude extends BaseAIClass {
|
|||
}
|
||||
}
|
||||
|
||||
protected extractContents(conversationContent: ConversationContent[]): { role: Role; content: ContentBlockParam[]; }[] {
|
||||
return this.filterConversationContents(conversationContent)
|
||||
.map(content => {
|
||||
const contentBlocks: ContentBlockParam[] = [];
|
||||
const contentToExtract = content.content ?? "";
|
||||
protected async extractContents(conversationContent: ConversationContent[]): Promise<{ role: Role; content: ContentBlockParam[]; }[]> {
|
||||
const results = [];
|
||||
|
||||
if (contentToExtract.trim() !== "" && !content.functionResponse && (!content.attachments || content.attachments.length === 0)) {
|
||||
for (const content of this.filterConversationContents(conversationContent)) {
|
||||
const contentBlocks: ContentBlockParam[] = [];
|
||||
const contentToExtract = content.content ?? "";
|
||||
|
||||
if (contentToExtract.trim() !== "" && !content.functionResponse && (!content.attachments || content.attachments.length === 0)) {
|
||||
contentBlocks.push({
|
||||
type: "text",
|
||||
text: contentToExtract
|
||||
});
|
||||
}
|
||||
|
||||
// Add function call if present
|
||||
if (content.functionCall) {
|
||||
const parsedContent = this.parseFunctionCall(content.functionCall);
|
||||
|
||||
if (parsedContent) {
|
||||
if (parsedContent.functionCall.id && parsedContent.functionCall.id.trim() !== "") {
|
||||
contentBlocks.push({
|
||||
type: "tool_use",
|
||||
id: parsedContent.functionCall.id,
|
||||
name: parsedContent.functionCall.name,
|
||||
input: parsedContent.functionCall.args
|
||||
});
|
||||
} else {
|
||||
contentBlocks.push({
|
||||
type: "text",
|
||||
text: this.convertFunctionCallToText(parsedContent)
|
||||
});
|
||||
}
|
||||
} else {
|
||||
contentBlocks.push({
|
||||
type: "text",
|
||||
text: contentToExtract
|
||||
text: "Error parsing function call"
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Add function call if present
|
||||
if (content.functionCall) {
|
||||
const parsedContent = this.parseFunctionCall(content.functionCall);
|
||||
// Add binary file attachments if present
|
||||
if (content.attachments && content.attachments.length > 0) {
|
||||
// Upload all attachments and track failures
|
||||
const failedUploads: string[] = [];
|
||||
|
||||
if (parsedContent) {
|
||||
if (parsedContent.functionCall.id && parsedContent.functionCall.id.trim() !== "") {
|
||||
contentBlocks.push({
|
||||
type: "tool_use",
|
||||
id: parsedContent.functionCall.id,
|
||||
name: parsedContent.functionCall.name,
|
||||
input: parsedContent.functionCall.args
|
||||
});
|
||||
} else {
|
||||
contentBlocks.push({
|
||||
type: "text",
|
||||
text: this.convertFunctionCallToText(parsedContent)
|
||||
});
|
||||
}
|
||||
} else {
|
||||
contentBlocks.push({
|
||||
type: "text",
|
||||
text: "Error parsing function call"
|
||||
});
|
||||
for (const attachment of content.attachments) {
|
||||
try {
|
||||
await this.aiFileService.uploadFile(attachment);
|
||||
} catch (error) {
|
||||
Exception.log(`Failed to upload ${attachment.fileName}: ${Exception.messageFrom(error)}`);
|
||||
failedUploads.push(attachment.fileName);
|
||||
}
|
||||
}
|
||||
|
||||
// Add binary file attachments if present
|
||||
if (content.attachments && content.attachments.length > 0) {
|
||||
const formattedContent = this.formatBinaryFiles(content.attachments);
|
||||
const rawContent = JSON.parse(formattedContent) as ContentBlockParam[];
|
||||
contentBlocks.push(...rawContent);
|
||||
// Format successfully uploaded files
|
||||
const formattedContent = this.formatBinaryFiles(content.attachments);
|
||||
const rawContent = JSON.parse(formattedContent) as ContentBlockParam[];
|
||||
contentBlocks.push(...rawContent);
|
||||
|
||||
// Add error messages for failed uploads
|
||||
if (failedUploads.length > 0) {
|
||||
contentBlocks.push({
|
||||
type: "text",
|
||||
text: `[Upload failed for: ${failedUploads.join(', ')}.]`
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Add function response if present
|
||||
if (content.functionResponse) {
|
||||
const parsedContent = this.parseFunctionResponse(content.functionResponse);
|
||||
// Add function response if present
|
||||
if (content.functionResponse) {
|
||||
const parsedContent = this.parseFunctionResponse(content.functionResponse);
|
||||
|
||||
if (parsedContent) {
|
||||
if (parsedContent.id && parsedContent.id.trim() !== "") {
|
||||
contentBlocks.push({
|
||||
type: "tool_result",
|
||||
tool_use_id: parsedContent.id,
|
||||
content: JSON.stringify(parsedContent.functionResponse.response)
|
||||
});
|
||||
} else {
|
||||
contentBlocks.push({
|
||||
type: "text",
|
||||
text: this.convertFunctionResponseToText(parsedContent)
|
||||
});
|
||||
}
|
||||
if (parsedContent) {
|
||||
if (parsedContent.id && parsedContent.id.trim() !== "") {
|
||||
contentBlocks.push({
|
||||
type: "tool_result",
|
||||
tool_use_id: parsedContent.id,
|
||||
content: JSON.stringify(parsedContent.functionResponse.response)
|
||||
});
|
||||
} else {
|
||||
contentBlocks.push({
|
||||
type: "text",
|
||||
text: content.functionResponse
|
||||
text: this.convertFunctionResponseToText(parsedContent)
|
||||
});
|
||||
}
|
||||
} else {
|
||||
contentBlocks.push({
|
||||
type: "text",
|
||||
text: content.functionResponse
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
role: content.role,
|
||||
content: contentBlocks
|
||||
};
|
||||
})
|
||||
.filter(message => message.content.length > 0);
|
||||
results.push({
|
||||
role: content.role,
|
||||
content: contentBlocks
|
||||
});
|
||||
}
|
||||
|
||||
return results.filter(message => message.content.length > 0);
|
||||
}
|
||||
|
||||
protected mapFunctionDefinitions(aiFunctionDefinitions: IAIFunctionDefinition[]): Tool[] {
|
||||
|
|
@ -239,19 +268,20 @@ export class Claude extends BaseAIClass {
|
|||
|
||||
public formatBinaryFiles(attachments: Attachment[]): string {
|
||||
const contentBlocks = attachments.flatMap(attachment => {
|
||||
let blockType: string;
|
||||
// Check for uploaded file ID
|
||||
const fileID = attachment.getFileID(this.provider);
|
||||
if (!fileID) {
|
||||
// Skip - upload failed, error message added in extractContents()
|
||||
return [];
|
||||
}
|
||||
|
||||
let blockType: string;
|
||||
if (attachment.mimeType === "application/pdf") {
|
||||
blockType = "document";
|
||||
} else {
|
||||
// Image handling
|
||||
blockType = "image";
|
||||
|
||||
// Validate supported image types
|
||||
if (!this.SUPPORTED_IMAGE_TYPES.includes(attachment.mimeType)) {
|
||||
return [
|
||||
{ type: "text", text: `Unsupported image format: ${attachment.fileName}` }
|
||||
];
|
||||
return [{ type: "text", text: `Unsupported image format: ${attachment.fileName}` }];
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -260,9 +290,8 @@ export class Claude extends BaseAIClass {
|
|||
{
|
||||
type: blockType,
|
||||
source: {
|
||||
type: "base64",
|
||||
media_type: attachment.mimeType,
|
||||
data: attachment.base64
|
||||
type: "file",
|
||||
file_id: fileID
|
||||
}
|
||||
}
|
||||
];
|
||||
|
|
|
|||
|
|
@ -1,90 +1,91 @@
|
|||
import { BaseAIFileService } from "AIClasses/BaseAIFileService";
|
||||
import { AIFileServiceURL, AIProvider } from "Enums/ApiProvider";
|
||||
import { StringTools } from "Helpers/StringTools";
|
||||
import { requestUrl } from "obsidian";
|
||||
import { ApiError } from "Types/ApiError";
|
||||
import type { ClaudeFile, ClaudeListFilesResponse } from "./ClaudeTypes";
|
||||
|
||||
export class ClaudeFileService extends BaseAIFileService {
|
||||
|
||||
private readonly betaHeader = "files-api-2025-04-14";
|
||||
|
||||
public constructor() {
|
||||
super(AIProvider.Claude);
|
||||
}
|
||||
|
||||
protected async listFilesFromAPI(): Promise<string[]> {
|
||||
return this.withRetry("List files", async () => {
|
||||
const response = await fetch(AIFileServiceURL.Claude, {
|
||||
|
||||
const response = await requestUrl({
|
||||
url: AIFileServiceURL.Claude,
|
||||
method: "GET",
|
||||
headers: {
|
||||
"x-api-key": this.apiKey,
|
||||
"anthropic-version": "2023-06-01",
|
||||
"anthropic-beta": this.betaHeader
|
||||
"anthropic-beta": "files-api-2025-04-14"
|
||||
},
|
||||
signal: this.abortService.signal()
|
||||
throw: false
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const responseBody = await response.text();
|
||||
throw ApiError.fromResponse(response.status, response.statusText, responseBody);
|
||||
if (response.status !== 200) {
|
||||
throw ApiError.fromResponse(response.status, "List files failed", response.text);
|
||||
}
|
||||
|
||||
const data = await response.json() as ClaudeListFilesResponse;
|
||||
const data = response.json as ClaudeListFilesResponse;
|
||||
|
||||
if (!data.data || data.data.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return data.data.map(file => file.id);
|
||||
});
|
||||
}, []);
|
||||
}
|
||||
|
||||
protected async uploadFileToAPI(data: string, mimeType: string, displayName?: string): Promise<string> {
|
||||
return this.withRetry("Upload file", async () => {
|
||||
const bytes = StringTools.toBytes(data);
|
||||
const blob = this.createBlob(bytes, mimeType);
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("file", blob, displayName || "file");
|
||||
const boundary = this.createBoundary();
|
||||
const formData = this.createFormData(displayName, mimeType, boundary, bytes);
|
||||
|
||||
const response = await fetch(AIFileServiceURL.Claude, {
|
||||
const response = await requestUrl({
|
||||
url: AIFileServiceURL.Claude,
|
||||
method: "POST",
|
||||
headers: {
|
||||
"x-api-key": this.apiKey,
|
||||
"anthropic-version": "2023-06-01",
|
||||
"anthropic-beta": this.betaHeader
|
||||
"anthropic-beta": "files-api-2025-04-14",
|
||||
"Content-Type": `multipart/form-data; boundary=${boundary}`
|
||||
},
|
||||
body: formData,
|
||||
signal: this.abortService.signal()
|
||||
body: formData.buffer,
|
||||
throw: false
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const responseBody = await response.text();
|
||||
throw ApiError.fromResponse(response.status, response.statusText, responseBody);
|
||||
if (response.status !== 200 && response.status !== 201) {
|
||||
throw ApiError.fromResponse(response.status, "Upload file failed", response.text);
|
||||
}
|
||||
|
||||
const responseData = await response.json() as ClaudeFile;
|
||||
const responseData = response.json as ClaudeFile;
|
||||
|
||||
return responseData.id;
|
||||
});
|
||||
}, "");
|
||||
}
|
||||
|
||||
protected async deleteFileFromAPI(id: string): Promise<void> {
|
||||
return this.withRetry("Delete file", async () => {
|
||||
const response = await fetch(`${AIFileServiceURL.Claude}/${id}`, {
|
||||
const response = await requestUrl({
|
||||
url: `${AIFileServiceURL.Claude}/${id}`,
|
||||
method: "DELETE",
|
||||
headers: {
|
||||
"x-api-key": this.apiKey,
|
||||
"anthropic-version": "2023-06-01",
|
||||
"anthropic-beta": this.betaHeader
|
||||
"anthropic-beta": "files-api-2025-04-14"
|
||||
},
|
||||
signal: this.abortService.signal()
|
||||
throw: false
|
||||
});
|
||||
|
||||
if (!response.ok && response.status !== 204 && response.status !== 404) {
|
||||
const responseBody = await response.text();
|
||||
throw ApiError.fromResponse(response.status, response.statusText, responseBody);
|
||||
if (response.status !== 200 && response.status !== 204 && response.status !== 404) {
|
||||
throw ApiError.fromResponse(response.status, "Delete file failed", response.text);
|
||||
}
|
||||
});
|
||||
}, undefined);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
interface ClaudeFile {
|
||||
export interface ClaudeFile {
|
||||
id: string;
|
||||
type: "file";
|
||||
filename: string;
|
||||
|
|
@ -8,15 +8,9 @@ interface ClaudeFile {
|
|||
downloadable: boolean;
|
||||
}
|
||||
|
||||
interface ClaudeListFilesResponse {
|
||||
export interface ClaudeListFilesResponse {
|
||||
data: ClaudeFile[];
|
||||
has_more: boolean;
|
||||
first_id?: string;
|
||||
last_id?: string;
|
||||
}
|
||||
|
||||
interface ClaudeDeleteResponse {
|
||||
id: string;
|
||||
type: "file";
|
||||
deleted: boolean;
|
||||
}
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@ import type { IAIFunctionDefinition } from "AIClasses/FunctionDefinitions/IAIFun
|
|||
import type { ConversationContent } from "Conversations/ConversationContent";
|
||||
import type { Candidate, Part, FunctionDeclaration } from "@google/genai";
|
||||
import { FinishReason } from "@google/genai";
|
||||
import { Exception } from "Helpers/Exception";
|
||||
|
||||
export class Gemini extends BaseAIClass {
|
||||
|
||||
|
|
@ -34,7 +35,12 @@ export class Gemini extends BaseAIClass {
|
|||
this.accumulatedFunctionArgs = {};
|
||||
this.accumulatedThoughtSignature = null;
|
||||
|
||||
const contents = this.extractContents(conversation.contents);
|
||||
// Refresh file cache only if conversation has attachments
|
||||
if (conversation.hasAttachments()) {
|
||||
await this.aiFileService.refreshCache();
|
||||
}
|
||||
|
||||
const contents = await this.extractContents(conversation.contents);
|
||||
|
||||
const tools = requestWebSearch ? { google_search: {} } :
|
||||
{
|
||||
|
|
@ -152,9 +158,10 @@ export class Gemini extends BaseAIClass {
|
|||
}
|
||||
}
|
||||
|
||||
protected extractContents(conversationContent: ConversationContent[]): { role: Role, parts: Part[] }[] {
|
||||
return this.filterConversationContents(conversationContent)
|
||||
.map(content => {
|
||||
protected async extractContents(conversationContent: ConversationContent[]): Promise<{ role: Role, parts: Part[] }[]> {
|
||||
const results = [];
|
||||
|
||||
for (const content of this.filterConversationContents(conversationContent)) {
|
||||
const parts: Part[] = [];
|
||||
const contentToExtract = content.content ?? "";
|
||||
|
||||
|
|
@ -193,9 +200,29 @@ export class Gemini extends BaseAIClass {
|
|||
|
||||
// Add binary file attachments if present
|
||||
if (content.attachments && content.attachments.length > 0) {
|
||||
// Upload all attachments and track failures
|
||||
const failedUploads: string[] = [];
|
||||
|
||||
for (const attachment of content.attachments) {
|
||||
try {
|
||||
await this.aiFileService.uploadFile(attachment);
|
||||
} catch (error) {
|
||||
Exception.log(`Failed to upload ${attachment.fileName}: ${Exception.messageFrom(error)}`);
|
||||
failedUploads.push(attachment.fileName);
|
||||
}
|
||||
}
|
||||
|
||||
// Format successfully uploaded files
|
||||
const formattedContent = this.formatBinaryFiles(content.attachments);
|
||||
const rawContent = JSON.parse(formattedContent) as Part[];
|
||||
parts.push(...rawContent);
|
||||
|
||||
// Add error messages for failed uploads
|
||||
if (failedUploads.length > 0) {
|
||||
parts.push({
|
||||
text: `[Upload failed for: ${failedUploads.join(', ')}.]`
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Add function response if present
|
||||
|
|
@ -225,12 +252,13 @@ export class Gemini extends BaseAIClass {
|
|||
}
|
||||
}
|
||||
|
||||
return {
|
||||
role: content.role === Role.User ? Role.User : Role.Model,
|
||||
parts: parts
|
||||
};
|
||||
})
|
||||
.filter(message => message.parts.length > 0);
|
||||
results.push({
|
||||
role: content.role === Role.User ? Role.User : Role.Model,
|
||||
parts: parts
|
||||
});
|
||||
}
|
||||
|
||||
return results.filter(message => message.parts.length > 0);
|
||||
}
|
||||
|
||||
protected mapFunctionDefinitions(aiFunctionDefinitions: IAIFunctionDefinition[]): FunctionDeclaration[] {
|
||||
|
|
@ -245,22 +273,27 @@ export class Gemini extends BaseAIClass {
|
|||
const parts: unknown[] = [];
|
||||
|
||||
for (const attachment of attachments) {
|
||||
// Check for uploaded file ID
|
||||
const fileID = attachment.getFileID(this.provider);
|
||||
if (!fileID) {
|
||||
// Skip - upload failed, error message added in extractContents()
|
||||
continue;
|
||||
}
|
||||
|
||||
// Validate image types (Gemini only supports JPEG and PNG)
|
||||
if (attachment.mimeType.startsWith('image/')) {
|
||||
if (!this.SUPPORTED_IMAGE_TYPES.includes(attachment.mimeType)) {
|
||||
parts.push({
|
||||
text: `Unsupported image format: ${attachment.fileName}`
|
||||
});
|
||||
parts.push({ text: `Unsupported image format: ${attachment.fileName}` });
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Add filename text block, then binary data
|
||||
// Add filename and file data
|
||||
parts.push({text: attachment.fileName});
|
||||
parts.push({
|
||||
inlineData: {
|
||||
fileData: {
|
||||
mimeType: attachment.mimeType,
|
||||
data: attachment.base64
|
||||
fileUri: fileID // Format: "files/abc123"
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@ import { BaseAIFileService } from "AIClasses/BaseAIFileService";
|
|||
import { AIFileServiceURL, AIProvider } from "Enums/ApiProvider";
|
||||
import { Exception } from "Helpers/Exception";
|
||||
import { StringTools } from "Helpers/StringTools";
|
||||
import { requestUrl } from "obsidian";
|
||||
import { ApiError } from "Types/ApiError";
|
||||
import type { GeminiListFilesResponse, GeminiUploadResponse } from "./GeminiTypes";
|
||||
|
||||
export class GeminiFileService extends BaseAIFileService {
|
||||
|
||||
|
|
@ -12,27 +14,28 @@ export class GeminiFileService extends BaseAIFileService {
|
|||
|
||||
protected async listFilesFromAPI(): Promise<string[]> {
|
||||
return this.withRetry("List files", async () => {
|
||||
const response = await fetch(`${AIFileServiceURL.Gemini}/files?key=${this.apiKey}`, {
|
||||
|
||||
const response = await requestUrl({
|
||||
url: `${AIFileServiceURL.Gemini}/files?key=${this.apiKey}`,
|
||||
method: "GET",
|
||||
headers: {
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
signal: this.abortService.signal()
|
||||
throw: false
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const responseBody = await response.text();
|
||||
throw ApiError.fromResponse(response.status, response.statusText, responseBody);
|
||||
if (response.status !== 200) {
|
||||
throw ApiError.fromResponse(response.status, "List files failed", response.text);
|
||||
}
|
||||
|
||||
const data = await response.json() as GeminiListFilesResponse;
|
||||
const data = response.json as GeminiListFilesResponse;
|
||||
|
||||
if (!data.files || data.files.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return data.files.map(file => file.name);
|
||||
});
|
||||
}, []);
|
||||
}
|
||||
|
||||
protected async uploadFileToAPI(data: string, mimeType: string, displayName?: string): Promise<string> {
|
||||
|
|
@ -43,7 +46,8 @@ export class GeminiFileService extends BaseAIFileService {
|
|||
const metadata = displayName ? { file: { displayName } } : {};
|
||||
|
||||
// Step 1: Initiate resumable upload
|
||||
const initiateResponse = await fetch(`${AIFileServiceURL.GeminiUpload}/files?key=${this.apiKey}`, {
|
||||
const initiateResponse = await requestUrl({
|
||||
url: `${AIFileServiceURL.GeminiUpload}/files?key=${this.apiKey}`,
|
||||
method: "POST",
|
||||
headers: {
|
||||
"X-Goog-Upload-Protocol": "resumable",
|
||||
|
|
@ -53,55 +57,52 @@ export class GeminiFileService extends BaseAIFileService {
|
|||
"Content-Type": "application/json"
|
||||
},
|
||||
body: JSON.stringify(metadata),
|
||||
signal: this.abortService.signal()
|
||||
throw: false
|
||||
});
|
||||
|
||||
if (!initiateResponse.ok) {
|
||||
const responseBody = await initiateResponse.text();
|
||||
throw ApiError.fromResponse(initiateResponse.status, initiateResponse.statusText, responseBody);
|
||||
if (initiateResponse.status !== 200) {
|
||||
throw ApiError.fromResponse(initiateResponse.status, "Upload file failed", initiateResponse.text);
|
||||
}
|
||||
|
||||
const uploadUrl = initiateResponse.headers.get("x-goog-upload-url");
|
||||
const uploadUrl = initiateResponse.headers["x-goog-upload-url"];
|
||||
if (!uploadUrl) {
|
||||
Exception.throw("No upload URL received from initiate request");
|
||||
}
|
||||
|
||||
// Step 2: Upload file data
|
||||
const blob = this.createBlob(bytes, mimeType);
|
||||
const uploadResponse = await fetch(uploadUrl, {
|
||||
const uploadResponse = await requestUrl({
|
||||
url: uploadUrl,
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Length": numBytes.toString(),
|
||||
"X-Goog-Upload-Offset": "0",
|
||||
"X-Goog-Upload-Command": "upload, finalize"
|
||||
},
|
||||
body: blob,
|
||||
signal: this.abortService.signal()
|
||||
body: this.bytesToBuffer(bytes),
|
||||
throw: false
|
||||
});
|
||||
|
||||
if (!uploadResponse.ok) {
|
||||
const responseBody = await uploadResponse.text();
|
||||
throw ApiError.fromResponse(uploadResponse.status, uploadResponse.statusText, responseBody);
|
||||
if (uploadResponse.status !== 200 && uploadResponse.status !== 201) {
|
||||
throw ApiError.fromResponse(uploadResponse.status, "Upload file failed", uploadResponse.text);
|
||||
}
|
||||
|
||||
const responseData = await uploadResponse.json() as GeminiUploadResponse;
|
||||
const responseData = uploadResponse.json as GeminiUploadResponse;
|
||||
|
||||
return responseData.file.uri;
|
||||
});
|
||||
}, "");
|
||||
}
|
||||
|
||||
protected async deleteFileFromAPI(name: string): Promise<void> {
|
||||
return this.withRetry("Delete file", async () => {
|
||||
const response = await fetch(`${AIFileServiceURL.Gemini}/${name}?key=${this.apiKey}`, {
|
||||
const response = await requestUrl({
|
||||
url: `${AIFileServiceURL.Gemini}/${name}?key=${this.apiKey}`,
|
||||
method: "DELETE",
|
||||
signal: this.abortService.signal()
|
||||
throw: false
|
||||
});
|
||||
|
||||
if (!response.ok && response.status !== 204 && response.status !== 403) {
|
||||
const responseBody = await response.text();
|
||||
throw ApiError.fromResponse(response.status, response.statusText, responseBody);
|
||||
if (response.status !== 200 && response.status !== 204 && response.status !== 404) {
|
||||
throw ApiError.fromResponse(response.status, "Delete file failed", response.text);
|
||||
}
|
||||
});
|
||||
}, undefined);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
interface GeminiFile {
|
||||
export interface GeminiFile {
|
||||
name: string;
|
||||
displayName?: string;
|
||||
mimeType: string;
|
||||
|
|
@ -14,11 +14,11 @@ interface GeminiFile {
|
|||
};
|
||||
}
|
||||
|
||||
interface GeminiListFilesResponse {
|
||||
export interface GeminiListFilesResponse {
|
||||
files?: GeminiFile[];
|
||||
nextPageToken?: string;
|
||||
}
|
||||
|
||||
interface GeminiUploadResponse {
|
||||
export interface GeminiUploadResponse {
|
||||
file: GeminiFile;
|
||||
}
|
||||
|
|
@ -3,6 +3,6 @@ import type { Attachment } from "Conversations/Attachment";
|
|||
export interface IAIFileService {
|
||||
refreshCache(): Promise<void>;
|
||||
listFiles(): string[];
|
||||
uploadFile(attachment: Attachment): Promise<string>;
|
||||
uploadFile(attachment: Attachment): Promise<void>;
|
||||
deleteFile(attachment: Attachment): Promise<void>;
|
||||
}
|
||||
|
|
@ -23,9 +23,14 @@ export class OpenAI extends BaseAIClass {
|
|||
conversation: Conversation, allowDestructiveActions: boolean
|
||||
): AsyncGenerator<IStreamChunk, void, unknown> {
|
||||
|
||||
// Refresh file cache only if conversation has attachments
|
||||
if (conversation.hasAttachments()) {
|
||||
await this.aiFileService.refreshCache();
|
||||
}
|
||||
|
||||
const systemPrompt = await this.buildSystemPrompt();
|
||||
|
||||
const input = this.extractContents(conversation.contents);
|
||||
const input = await this.extractContents(conversation.contents);
|
||||
|
||||
const tools = [{
|
||||
type: "web_search"
|
||||
|
|
@ -179,7 +184,7 @@ export class OpenAI extends BaseAIClass {
|
|||
}
|
||||
}
|
||||
|
||||
protected extractContents(conversationContent: ConversationContent[]): ResponsesAPIInput[] {
|
||||
protected async extractContents(conversationContent: ConversationContent[]): Promise<ResponsesAPIInput[]> {
|
||||
const results: ResponsesAPIInput[] = [];
|
||||
|
||||
for (const content of this.filterConversationContents(conversationContent)) {
|
||||
|
|
@ -232,9 +237,31 @@ export class OpenAI extends BaseAIClass {
|
|||
|
||||
// Case 2: Binary file attachments
|
||||
if (content.attachments && content.attachments.length > 0) {
|
||||
// Upload all attachments and track failures
|
||||
const failedUploads: string[] = [];
|
||||
|
||||
for (const attachment of content.attachments) {
|
||||
try {
|
||||
await this.aiFileService.uploadFile(attachment);
|
||||
} catch (error) {
|
||||
Exception.log(`Failed to upload ${attachment.fileName}: ${Exception.messageFrom(error)}`);
|
||||
failedUploads.push(attachment.fileName);
|
||||
}
|
||||
}
|
||||
|
||||
// Format successfully uploaded files
|
||||
const formattedContent = this.formatBinaryFiles(content.attachments);
|
||||
const rawContent = JSON.parse(formattedContent) as ResponsesAPIInput[];
|
||||
results.push(...rawContent);
|
||||
|
||||
// Add error messages for failed uploads
|
||||
if (failedUploads.length > 0) {
|
||||
// OpenAI formatBinaryFiles returns array with role wrapper, so add as separate message
|
||||
results.push({
|
||||
role: "user",
|
||||
content: `[Upload failed for: ${failedUploads.join(', ')}.]`
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -293,14 +320,21 @@ export class OpenAI extends BaseAIClass {
|
|||
const contentBlocks: unknown[] = [];
|
||||
|
||||
for (const attachment of attachments) {
|
||||
// Check for uploaded file ID
|
||||
const fileID = attachment.getFileID(this.provider);
|
||||
if (!fileID) {
|
||||
// Skip - upload failed, error message added in extractContents()
|
||||
continue;
|
||||
}
|
||||
|
||||
if (attachment.mimeType === "application/pdf") {
|
||||
// Use file ID format for PDFs
|
||||
contentBlocks.push({
|
||||
type: "input_file",
|
||||
filename: attachment.fileName,
|
||||
file_data: `data:${attachment.mimeType};base64,${attachment.base64}`
|
||||
file_id: fileID
|
||||
});
|
||||
} else {
|
||||
// Image handling - validate supported types
|
||||
// Images
|
||||
if (!this.SUPPORTED_IMAGE_TYPES.includes(attachment.mimeType)) {
|
||||
contentBlocks.push({
|
||||
type: "input_text",
|
||||
|
|
@ -309,9 +343,10 @@ export class OpenAI extends BaseAIClass {
|
|||
continue;
|
||||
}
|
||||
|
||||
// Use file ID format for images
|
||||
contentBlocks.push({
|
||||
type: "input_image",
|
||||
image_url: `data:${attachment.mimeType};base64,${attachment.base64}`
|
||||
file_id: fileID
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { BaseAIFileService } from "AIClasses/BaseAIFileService";
|
|||
import { AIFileServiceURL, AIProvider } from "Enums/ApiProvider";
|
||||
import type { OpenAIFile, OpenAIListFilesResponse } from "./OpenAITypes";
|
||||
import { StringTools } from "Helpers/StringTools";
|
||||
import { requestUrl } from "obsidian";
|
||||
import { ApiError } from "Types/ApiError";
|
||||
|
||||
export class OpenAIFileService extends BaseAIFileService {
|
||||
|
|
@ -12,76 +13,76 @@ export class OpenAIFileService extends BaseAIFileService {
|
|||
|
||||
protected async listFilesFromAPI(): Promise<string[]> {
|
||||
return this.withRetry("List files", async () => {
|
||||
const response = await fetch(AIFileServiceURL.OpenAI, {
|
||||
|
||||
const response = await requestUrl({
|
||||
url: AIFileServiceURL.OpenAI,
|
||||
method: "GET",
|
||||
headers: {
|
||||
"Authorization": `Bearer ${this.apiKey}`
|
||||
},
|
||||
signal: this.abortService.signal()
|
||||
throw: false
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const responseBody = await response.text();
|
||||
throw ApiError.fromResponse(response.status, response.statusText, responseBody);
|
||||
if (response.status !== 200) {
|
||||
throw ApiError.fromResponse(response.status, "List files failed", response.text);
|
||||
}
|
||||
|
||||
const data = await response.json() as OpenAIListFilesResponse;
|
||||
const data = response.json as OpenAIListFilesResponse;
|
||||
|
||||
if (!data.data || data.data.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return data.data.map(file => file.id);
|
||||
});
|
||||
}, []);
|
||||
}
|
||||
|
||||
protected async uploadFileToAPI(data: string, mimeType: string, displayName?: string): Promise<string> {
|
||||
return this.withRetry("Upload file", async () => {
|
||||
const bytes = StringTools.toBytes(data);
|
||||
const blob = this.createBlob(bytes, mimeType);
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("file", blob, displayName || "file");
|
||||
|
||||
// Use 'vision' for images, 'user_data' for other files
|
||||
const purpose = mimeType.startsWith('image/') ? 'vision' : 'user_data';
|
||||
formData.append("purpose", purpose);
|
||||
|
||||
const response = await fetch(AIFileServiceURL.OpenAI, {
|
||||
const boundary = this.createBoundary();
|
||||
const formData = this.createFormData(displayName, mimeType, boundary, bytes, { purpose });
|
||||
|
||||
const response = await requestUrl({
|
||||
url: AIFileServiceURL.OpenAI,
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Authorization": `Bearer ${this.apiKey}`
|
||||
"Authorization": `Bearer ${this.apiKey}`,
|
||||
"Content-Type": `multipart/form-data; boundary=${boundary}`
|
||||
},
|
||||
body: formData,
|
||||
signal: this.abortService.signal()
|
||||
body: formData.buffer,
|
||||
throw: false
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const responseBody = await response.text();
|
||||
throw ApiError.fromResponse(response.status, response.statusText, responseBody);
|
||||
if (response.status !== 200 && response.status !== 201) {
|
||||
throw ApiError.fromResponse(response.status, "Upload file failed", response.text);
|
||||
}
|
||||
|
||||
const responseData = await response.json() as OpenAIFile;
|
||||
const responseData = response.json as OpenAIFile;
|
||||
|
||||
return responseData.id;
|
||||
});
|
||||
}, "");
|
||||
}
|
||||
|
||||
protected async deleteFileFromAPI(id: string): Promise<void> {
|
||||
return this.withRetry("Delete file", async () => {
|
||||
const response = await fetch(`${AIFileServiceURL.OpenAI}/${id}`, {
|
||||
const response = await requestUrl({
|
||||
url: `${AIFileServiceURL.OpenAI}/${id}`,
|
||||
method: "DELETE",
|
||||
headers: {
|
||||
"Authorization": `Bearer ${this.apiKey}`
|
||||
},
|
||||
signal: this.abortService.signal()
|
||||
throw: false
|
||||
});
|
||||
|
||||
if (!response.ok && response.status !== 204 && response.status !== 404) {
|
||||
const responseBody = await response.text();
|
||||
throw ApiError.fromResponse(response.status, response.statusText, responseBody);
|
||||
if (response.status !== 200 && response.status !== 204 && response.status !== 404) {
|
||||
throw ApiError.fromResponse(response.status, "Delete file failed", response.text);
|
||||
}
|
||||
});
|
||||
}, undefined);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -22,6 +22,10 @@ export class Conversation {
|
|||
this.title = `${StringTools.dateToString(this.created)}`;
|
||||
}
|
||||
|
||||
public hasAttachments(): boolean {
|
||||
return this.contents.some(c => c.attachments.length > 0);
|
||||
}
|
||||
|
||||
public addFunctionResponse(functionResponse: AIFunctionResponse): void {
|
||||
if (functionResponse.name !== AIFunction.ReadVaultFiles) {
|
||||
const functionResponseString = functionResponse.toConversationString();
|
||||
|
|
@ -96,12 +100,7 @@ export class Conversation {
|
|||
}
|
||||
}
|
||||
|
||||
return new Attachment(
|
||||
fileName,
|
||||
mimeType,
|
||||
file.contents, // base64 string
|
||||
{} // empty fileID map (phase 2 feature)
|
||||
);
|
||||
return new Attachment(fileName, mimeType, file.contents);
|
||||
});
|
||||
|
||||
this.contents.push(new ConversationContent({
|
||||
|
|
|
|||
Loading…
Reference in a new issue