diff --git a/AIClasses/BaseAIClass.ts b/AIClasses/BaseAIClass.ts index 5864fa5..292d4a0 100644 --- a/AIClasses/BaseAIClass.ts +++ b/AIClasses/BaseAIClass.ts @@ -42,7 +42,7 @@ export abstract class BaseAIClass implements IAIClass { this.apiKey = this.settingsService.getApiKeyForProvider(provider); } - get currentProvider(): AIProvider { + public get currentProvider(): AIProvider { return this.provider; } diff --git a/AIClasses/BaseAIFileService.ts b/AIClasses/BaseAIFileService.ts index 7b94872..7b261c6 100644 --- a/AIClasses/BaseAIFileService.ts +++ b/AIClasses/BaseAIFileService.ts @@ -48,7 +48,7 @@ export abstract class BaseAIFileService implements IAIFileService { } attachment.deleteFileID(this.provider); - const fileID = await this.uploadFileToAPI(attachment.base64, attachment.mimeType, attachment.fileName); + const fileID = await this.uploadFileToAPI(await attachment.getBase64(), attachment.getMimeType(), attachment.fileName); if (fileID.trim() === "") { return; // We tried, the agent will be notified of the failure diff --git a/AIClasses/Claude/Claude.ts b/AIClasses/Claude/Claude.ts index 3be3e9b..0b4b4e4 100644 --- a/AIClasses/Claude/Claude.ts +++ b/AIClasses/Claude/Claude.ts @@ -287,7 +287,7 @@ export class Claude extends BaseAIClass { return []; // Skip - upload failed, error message added in extractContents() } - const mimeType = toMimeType(attachment.mimeType); + const mimeType = toMimeType(attachment.getMimeType()); let isPlainText = false; diff --git a/AIClasses/Gemini/Gemini.ts b/AIClasses/Gemini/Gemini.ts index 0341fff..e810aa7 100644 --- a/AIClasses/Gemini/Gemini.ts +++ b/AIClasses/Gemini/Gemini.ts @@ -334,7 +334,7 @@ export class Gemini extends BaseAIClass { continue; // Skip - upload failed, error message added in extractContents() } - const mimeType = toMimeType(attachment.mimeType); + const mimeType = toMimeType(attachment.getMimeType()); let isPlainText = false; diff --git a/AIClasses/OpenAI/OpenAI.ts b/AIClasses/OpenAI/OpenAI.ts index 3425182..98d713f 100644 --- a/AIClasses/OpenAI/OpenAI.ts +++ b/AIClasses/OpenAI/OpenAI.ts @@ -323,7 +323,7 @@ export class OpenAI extends BaseAIClass { continue; // Skip - upload failed, error message added in extractContents() } - const mimeType = toMimeType(attachment.mimeType); + const mimeType = toMimeType(attachment.getMimeType()); let isPlainText = false; diff --git a/Components/ChatArea.svelte b/Components/ChatArea.svelte index 77869dd..b435405 100644 --- a/Components/ChatArea.svelte +++ b/Components/ChatArea.svelte @@ -81,7 +81,7 @@ chatAreaPaddingElement.style.paddingBottom = `${padding}px`; - if (behavior && (autoScroll || shouldSettle)) { + if (behavior && autoScroll) { chatContainer.scroll({ top: chatContainer.scrollHeight, behavior }); } } diff --git a/Conversations/Attachment.ts b/Conversations/Attachment.ts index 5875ce9..b4c6337 100644 --- a/Conversations/Attachment.ts +++ b/Conversations/Attachment.ts @@ -1,25 +1,44 @@ import type { AIProvider } from "Enums/ApiProvider"; import { isAudioFile, isImageFile, isKnownFileType, isTextFile, isVideoFile } from "Enums/FileType"; import { MimeTypeToFileTypes } from "Enums/FileTypeMimeTypeMapping"; -import { toMimeType } from "Enums/MimeType"; +import { isImageMimeType, isTextMimeType, MimeType, toMimeType } from "Enums/MimeType"; +import { StringTools } from "Helpers/StringTools"; export class Attachment { - + public fileName: string; public mimeType: string; - public base64: string; public fileID: Partial>; + public base64: string; + public filePath?: string; constructor( fileName: string, mimeType: string, base64: string, - fileID: Partial> = {} + fileID: Partial> = {}, + filePath?: string ) { this.fileName = fileName; this.mimeType = mimeType; - this.base64 = base64; this.fileID = fileID; + this.base64 = base64; + this.filePath = filePath; + } + + public getMimeType(): string { + const mimeTypeEnum = toMimeType(this.mimeType); + if (isTextMimeType(mimeTypeEnum)) { + return MimeType.TEXT_PLAIN; + } + return this.mimeType; + } + + public async getBase64(): Promise { + if (isImageMimeType(toMimeType(this.mimeType))) { + return await StringTools.resizeB64Image(this.base64, this.mimeType); + } + return this.base64; } public getFileID(provider: AIProvider): string | undefined { @@ -70,7 +89,7 @@ export class Attachment { public static isAttachmentData(this: void, data: unknown): data is { fileName: string; mimeType: string; - base64: string; + filePath: string; fileID?: Partial>; } { return ( @@ -78,10 +97,10 @@ export class Attachment { typeof data === "object" && "fileName" in data && "mimeType" in data && - "base64" in data && + "filePath" in data && typeof data.fileName === "string" && typeof data.mimeType === "string" && - typeof data.base64 === "string" && + typeof data.filePath === "string" && (!("fileID" in data) || typeof data.fileID === "object") ); } diff --git a/Enums/MimeType.ts b/Enums/MimeType.ts index 80d3847..d7908df 100644 --- a/Enums/MimeType.ts +++ b/Enums/MimeType.ts @@ -107,3 +107,42 @@ export function toMimeType(mimeType: string): MimeType { export function isKnownMimeType(value: string): value is MimeType { return Object.values(MimeType).includes(value as MimeType) && value !== MimeType.UNKNOWN.toString(); } + +export function isImageMimeType(mimeType: MimeType) { + return mimeType === MimeType.IMAGE_AVIF || + mimeType === MimeType.IMAGE_BMP || + mimeType === MimeType.IMAGE_GIF || + mimeType === MimeType.IMAGE_JPEG || + mimeType === MimeType.IMAGE_PNG || + mimeType === MimeType.IMAGE_SVG || + mimeType === MimeType.IMAGE_WEBP; +} + +export function isTextMimeType(mimeType: MimeType): boolean { + if (mimeType === MimeType.APPLICATION_PDF || mimeType === MimeType.UNKNOWN) { + return false; + } + + if (mimeType.startsWith("text/")) { + return true; + } + + const textApplicationTypes = [ + MimeType.APPLICATION_JSON, + MimeType.APPLICATION_XML, + MimeType.APPLICATION_RTF, + MimeType.APPLICATION_YAML, + MimeType.APPLICATION_TOML, + MimeType.APPLICATION_TEX, + MimeType.APPLICATION_LATEX, + MimeType.APPLICATION_MAKEFILE, + MimeType.APPLICATION_GRADLE, + MimeType.APPLICATION_DOCKERFILE, + MimeType.APPLICATION_PYTHON_CODE, + MimeType.APPLICATION_JAVASCRIPT, + MimeType.APPLICATION_TYPESCRIPT, + MimeType.APPLICATION_SH + ]; + + return textApplicationTypes.includes(mimeType); +} \ No newline at end of file diff --git a/Enums/Path.ts b/Enums/Path.ts index 000277b..66c8dce 100644 --- a/Enums/Path.ts +++ b/Enums/Path.ts @@ -2,6 +2,7 @@ export enum Path { Root = "/", VaultkeeperAIDir = "Vaultkeeper AI", Conversations = `${Path.VaultkeeperAIDir}/Conversations`, + Attachments = `${Path.Conversations}/Attachments`, UserInstructions = `${Path.VaultkeeperAIDir}/User Instructions`, ExampleUserInstructions = `${Path.UserInstructions}/EXAMPLE_INSTRUCTIONS.md` }; \ No newline at end of file diff --git a/Helpers/StringTools.ts b/Helpers/StringTools.ts index 1b7076f..613010d 100644 --- a/Helpers/StringTools.ts +++ b/Helpers/StringTools.ts @@ -1,3 +1,5 @@ +import { Exception } from "./Exception"; + export abstract class StringTools { // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-unsafe-assignment -- regex-parser is a CommonJS module without ESM support @@ -69,4 +71,56 @@ export abstract class StringTools { return bytes; } + public static async computeSHA256Hash(base64: string): Promise { + const bytes = this.toBytes(base64); + const hashBuffer = await crypto.subtle.digest('SHA-256', bytes); + const hashArray = Array.from(new Uint8Array(hashBuffer)); + return hashArray.map(b => b.toString(16).padStart(2, '0')).join(''); + } + + public static async resizeB64Image(base64: string, mimeType: string, maxWidth: number = 1000, maxHeight: number = 1000): Promise { + return new Promise((resolve, reject) => { + const img = new Image(); + + img.onload = () => { + let width = img.width; + let height = img.height; + + if (width > maxWidth || height > maxHeight) { + const aspectRatio = width / height; + if (width > height) { + width = maxWidth; + height = maxWidth / aspectRatio; + } else { + height = maxHeight; + width = maxHeight * aspectRatio; + } + } + + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext("2d"); + + if (!context) { + reject(Exception.new("Failed to get canvas context")); + return; + } + + context.drawImage(img, 0, 0, width, height); + + const dataURL = canvas.toDataURL(mimeType, 0.92); + const base64Only = dataURL.split(',')[1]; + resolve(base64Only); + }; + + img.onerror = () => reject(Exception.new("Failed to load image")); + + if (!base64.startsWith('data:')) { + base64 = `data:${mimeType};base64,${base64}`; + } + img.src = base64; + }); + } + } \ No newline at end of file diff --git a/Services/ConversationFileSystemService.ts b/Services/ConversationFileSystemService.ts index bba9c32..b7daeda 100644 --- a/Services/ConversationFileSystemService.ts +++ b/Services/ConversationFileSystemService.ts @@ -8,6 +8,8 @@ import { Attachment } from "Conversations/Attachment"; import { Exception } from "Helpers/Exception"; import type { IAIFileService } from "AIClasses/IAIFileService"; import { Reference } from "Conversations/Reference"; +import { arrayBufferToBase64 } from "obsidian"; +import { StringTools } from "Helpers/StringTools"; export class ConversationFileSystemService { @@ -46,7 +48,19 @@ export class ConversationFileSystemService { } conversation.updated = new Date(); - + + // Save attachment files and update filePaths + for (const content of conversation.contents) { + for (const attachment of content.attachments) { + if (!attachment.filePath && attachment.base64) { + const filePath = await this.saveAttachmentFile(attachment); + if (!(filePath instanceof Error)) { + attachment.filePath = filePath.replace(`${Path.Conversations}/`, ''); + } + } + } + } + const conversationData = { title: conversation.title, created: conversation.created.toISOString(), @@ -59,7 +73,12 @@ export class ConversationFileSystemService { displayContent: content.displayContent, functionCall: content.functionCall, functionResponse: content.functionResponse, - attachments: content.attachments, + attachments: content.attachments.map(att => ({ + fileName: att.fileName, + mimeType: att.mimeType, + filePath: att.filePath, + fileID: att.fileID + })), references: content.references, shouldDisplayContent: content.shouldDisplayContent, toolId: content.toolId, @@ -114,6 +133,11 @@ export class ConversationFileSystemService { // Mark as deleted to prevent subsequent saves during ongoing operations this.isDeleted = true; this.currentConversationPath = null; + + // Queue garbage collection after AI file deletion + this.deletionQueue = this.deletionQueue.then(async () => { + await this.garbageCollectAttachments(); + }); } public async getAllConversations(): Promise { @@ -130,6 +154,57 @@ export class ConversationFileSystemService { return conversations; } + public async garbageCollectAttachments(): Promise { + try { + // 1. Get all attachment files + const attachmentFiles = await this.fileSystemService.listFilesInDirectory( + Path.Attachments, + false, + true + ); + + if (attachmentFiles.length === 0) { + return; + } + + // 2. Build reference count map + const referenceCount = new Map(); + + const conversations = await this.getAllConversations(); + for (const conversation of conversations) { + for (const content of conversation.contents) { + for (const attachment of content.attachments) { + if (attachment.filePath) { + const count = referenceCount.get(attachment.filePath) || 0; + referenceCount.set(attachment.filePath, count + 1); + } + } + } + } + + // 3. Delete unreferenced files + for (const file of attachmentFiles) { + const relativePath = file.path.replace(`${Path.Conversations}/`, ''); + const refCount = referenceCount.get(relativePath) || 0; + + if (refCount === 0) { + const deleteResult = await this.fileSystemService.deleteFile( + file.path, + true, + false + ); + + if (deleteResult instanceof Error) { + Exception.log(deleteResult); + } + } + } + } catch (error) { + Exception.log(error); + return Exception.new(error); + } + } + public async updateConversationTitle(oldPath: string, newTitle: string): Promise { const newPath = `${Path.Conversations}/${newTitle}.json`; @@ -144,6 +219,41 @@ export class ConversationFileSystemService { } } + private async saveAttachmentFile(attachment: Attachment): Promise { + const hash = await StringTools.computeSHA256Hash(attachment.base64); + const fileName = `${hash}.bin`; + const filePath = `${Path.Attachments}/${fileName}`; + + const exists = await this.fileSystemService.exists(filePath, true); + if (exists) { + return filePath; + } + + const bytes = StringTools.toBytes(attachment.base64); + const arrayBuffer = bytes.buffer; + + const result = await this.fileSystemService.writeBinaryFile(filePath, arrayBuffer, true); + + if (result instanceof Error) { + Exception.log(result); + return filePath; + } + + return filePath; + } + + private async loadAttachmentFile(filePath: string): Promise { + const fullPath = `${Path.Conversations}/${filePath}`; + const arrayBuffer = await this.fileSystemService.readBinaryFile(fullPath, true); + + if (arrayBuffer instanceof Error) { + Exception.log(arrayBuffer); + return ""; + } + + return arrayBufferToBase64(arrayBuffer); + } + private async readConversation(path: string): Promise { const result = await this.fileSystemService.readObjectFromFile(path, true); @@ -158,9 +268,9 @@ export class ConversationFileSystemService { conversation.title = result.title; conversation.created = new Date(result.created); conversation.updated = new Date(result.updated); - conversation.contents = result.contents.map(content => { - // Reconstruct Attachment instances from plain objects - const attachments = this.deserializeAttachments(content.attachments); + + const contentPromises = result.contents.map(async content => { + const attachments = await this.deserializeAttachments(content.attachments); const references = this.deserializeReferences(content.references); return new ConversationContent({ @@ -178,24 +288,44 @@ export class ConversationFileSystemService { errorType: content.errorType }); }); + + conversation.contents = await Promise.all(contentPromises); } return conversation; } - private deserializeAttachments(attachmentsData: unknown): Attachment[] { + private async deserializeAttachments(attachmentsData: unknown): Promise { if (!Array.isArray(attachmentsData)) { return []; } - return attachmentsData - .filter(Attachment.isAttachmentData) - .map(attachmentData => new Attachment( + const attachments: Attachment[] = []; + + for (const attachmentData of attachmentsData) { + if (!Attachment.isAttachmentData(attachmentData)) { + continue; + } + + const base64 = await this.loadAttachmentFile(attachmentData.filePath); + + if (!base64) { + Exception.warn(`Skipping attachment with missing file: ${attachmentData.fileName} (${attachmentData.filePath})`); + continue; + } + + const attachment = new Attachment( attachmentData.fileName, attachmentData.mimeType, - attachmentData.base64, - attachmentData.fileID || {} - )); + base64, + attachmentData.fileID || {}, + attachmentData.filePath + ); + + attachments.push(attachment); + } + + return attachments; } private deserializeReferences(referencesData: unknown): Reference[] { diff --git a/Services/FileSystemService.ts b/Services/FileSystemService.ts index f23a950..af575a7 100644 --- a/Services/FileSystemService.ts +++ b/Services/FileSystemService.ts @@ -51,6 +51,14 @@ export class FileSystemService { return await this.vaultService.modify(file, content, allowAccessToPluginRoot, requiresConfirmation); } + public async writeBinaryFile(filePath: string, data: ArrayBuffer, allowAccessToPluginRoot: boolean = false): Promise { + const file: TAbstractFile | null = this.vaultService.getAbstractFileByPath(filePath, allowAccessToPluginRoot); + if (file == null || !(file instanceof TFile)) { + return await this.vaultService.createBinary(filePath, data, allowAccessToPluginRoot); + } + return await this.vaultService.modifyBinary(file, data, allowAccessToPluginRoot); + } + public async patchFile(filePath: string, oldContent: string, newContent: string, allowAccessToPluginRoot: boolean = false, requiresConfirmation: boolean = true): Promise { const file: TAbstractFile | null = this.vaultService.getAbstractFileByPath(filePath, allowAccessToPluginRoot); diff --git a/Services/HTMLService.ts b/Services/HTMLService.ts index d865c8a..658c7ff 100644 --- a/Services/HTMLService.ts +++ b/Services/HTMLService.ts @@ -23,7 +23,6 @@ export class HTMLService { return fragment; } - // Creates a temporary container, parses HTML, and returns the container. // Useful for parsing HTML when you need to traverse the resulting DOM structure. public parseHTMLToContainer(htmlString: string): HTMLDivElement { diff --git a/Services/VaultService.ts b/Services/VaultService.ts index f624f21..fcda6c4 100644 --- a/Services/VaultService.ts +++ b/Services/VaultService.ts @@ -230,6 +230,38 @@ export class VaultService { } } + public async createBinary(filePath: string, data: ArrayBuffer, allowAccessToPluginRoot: boolean = false): Promise { + filePath = this.sanitiserService.sanitize(filePath); + if (this.isExclusion(filePath, allowAccessToPluginRoot)) { + Exception.log(`Plugin attempted to create a binary file that is in the exclusion list: ${filePath}`); + return Exception.new(`Failed to create file, permission denied: ${filePath}`); + } + + try { + await this.createDirectories(filePath, allowAccessToPluginRoot); + return await this.vault.createBinary(filePath, data); + } catch (error) { + Exception.log(error); + return Exception.new(error); + } + } + + public async modifyBinary(file: TFile, data: ArrayBuffer, allowAccessToPluginRoot: boolean = false): Promise { + const filePath = this.sanitiserService.sanitize(file.path); + if (this.isExclusion(file.path, allowAccessToPluginRoot)) { + Exception.log(`Plugin attempted to modify a binary file that is in the exclusion list: ${filePath}`); + return Exception.new(`File does not exist: ${filePath}`); + } + + try { + await this.vault.modifyBinary(file, data); + return file; + } catch (error) { + Exception.log(error); + return Exception.new(error); + } + } + public async createFolder(path: string, allowAccessToPluginRoot: boolean = false): Promise { path = this.sanitiserService.sanitize(path); if (this.isExclusion(path, allowAccessToPluginRoot)) { diff --git a/manifest.json b/manifest.json index ba79b09..753771a 100644 --- a/manifest.json +++ b/manifest.json @@ -1,11 +1,11 @@ { - "id": "vaultkeeper-ai", - "name": "Vaultkeeper AI", - "version": "1.2.1", - "minAppVersion": "1.9.14", - "description": "Multi-AI assistant. Read, search, create, and edit notes with multiple AI providers.", - "author": "Andy-Stack", - "authorUrl": "https://github.com/Andy-Stack", - "fundingUrl": "https://buymeacoffee.com/andy.stack", - "isDesktopOnly": false + "id": "vaultkeeper-ai", + "name": "Vaultkeeper AI", + "version": "1.2.1", + "minAppVersion": "1.9.14", + "description": "Multi-AI assistant. Read, search, create, and edit notes with multiple AI providers.", + "author": "Andy-Stack", + "authorUrl": "https://github.com/Andy-Stack", + "fundingUrl": "https://buymeacoffee.com/andy.stack", + "isDesktopOnly": false } \ No newline at end of file