From 62104f0abd2177bea1e165105b0e1793ea6b2374 Mon Sep 17 00:00:00 2001 From: Andrew Beal Date: Mon, 6 Oct 2025 23:01:20 +0100 Subject: [PATCH] Refactor AI function handling by introducing AIFunctionCall and AIFunctionResponse classes, updating function definitions, and enhancing conversation content management. Remove unused code and improve service registration for AIFunctionService. --- AIClasses/AIFunctionCall.ts | 19 +++ .../AIFunctionDefinitions.ts | 8 + .../FunctionDefinitions/AIFunctionResponse.ts | 20 +++ ...Definition.ts => IAIFunctionDefinition.ts} | 3 +- .../FunctionDefinitions/ListVaultFiles.ts | 11 +- AIClasses/Gemini/Gemini.ts | 118 +++++++++++--- AIClasses/Response/AIResponse.ts | 6 - AIClasses/SystemPrompt.ts | 1 + Components/ChatArea.svelte | 42 ++--- Components/ChatWindow.svelte | 145 +++++++++++------- Conversations/ConversationContent.ts | 14 +- Modals/ConversationHistoryModal.ts | 4 +- Modals/ConversationHistoryModalSvelte.svelte | 10 +- Services/AIFunctionService.ts | 22 ++- Services/ConversationFileSystemService.ts | 14 +- Services/ServiceRegistration.ts | 2 + Services/Services.ts | 1 + Services/StreamingService.ts | 3 + 18 files changed, 321 insertions(+), 122 deletions(-) create mode 100644 AIClasses/AIFunctionCall.ts create mode 100644 AIClasses/FunctionDefinitions/AIFunctionDefinitions.ts create mode 100644 AIClasses/FunctionDefinitions/AIFunctionResponse.ts rename AIClasses/FunctionDefinitions/{AIFunctionDefinition.ts => IAIFunctionDefinition.ts} (55%) delete mode 100644 AIClasses/Response/AIResponse.ts diff --git a/AIClasses/AIFunctionCall.ts b/AIClasses/AIFunctionCall.ts new file mode 100644 index 0000000..e058a21 --- /dev/null +++ b/AIClasses/AIFunctionCall.ts @@ -0,0 +1,19 @@ +// platform agnostic function call class used to execute the requested function +export class AIFunctionCall { + public readonly name: string; + public readonly arguments: Record; + + constructor(name: string, args: Record) { + this.name = name; + this.arguments = args; + } + + public toConversationString(): string { + return JSON.stringify({ + functionCall: { + name: this.name, + args: this.arguments + } + }); + } +} \ No newline at end of file diff --git a/AIClasses/FunctionDefinitions/AIFunctionDefinitions.ts b/AIClasses/FunctionDefinitions/AIFunctionDefinitions.ts new file mode 100644 index 0000000..a0b5165 --- /dev/null +++ b/AIClasses/FunctionDefinitions/AIFunctionDefinitions.ts @@ -0,0 +1,8 @@ +import type { IAIFunctionDefinition } from "./IAIFunctionDefinition"; +import { ListVaultFiles } from "./ListVaultFiles"; + +export class AIFunctionDefinitions { + public getQueryActions(): IAIFunctionDefinition[] { + return [ListVaultFiles]; + } +} \ No newline at end of file diff --git a/AIClasses/FunctionDefinitions/AIFunctionResponse.ts b/AIClasses/FunctionDefinitions/AIFunctionResponse.ts new file mode 100644 index 0000000..6c5a6ee --- /dev/null +++ b/AIClasses/FunctionDefinitions/AIFunctionResponse.ts @@ -0,0 +1,20 @@ +// Platform agnostic class for function responses +// Used by AI providers to format function execution results for API calls +export class AIFunctionResponse { + public readonly name: string; + public readonly response: object; + + constructor(name: string, response: object) { + this.name = name; + this.response = response; + } + + public toConversationString(): string { + return JSON.stringify({ + functionResponse: { + name: this.name, + response: { result: this.response } + } + }); + } +} \ No newline at end of file diff --git a/AIClasses/FunctionDefinitions/AIFunctionDefinition.ts b/AIClasses/FunctionDefinitions/IAIFunctionDefinition.ts similarity index 55% rename from AIClasses/FunctionDefinitions/AIFunctionDefinition.ts rename to AIClasses/FunctionDefinitions/IAIFunctionDefinition.ts index 7d125a3..22a3590 100644 --- a/AIClasses/FunctionDefinitions/AIFunctionDefinition.ts +++ b/AIClasses/FunctionDefinitions/IAIFunctionDefinition.ts @@ -1,4 +1,5 @@ -export interface FunctionDefinition { +// platform agnostic function definition used to present function calls in an API call +export interface IAIFunctionDefinition { name: string; description: string; parameters: { diff --git a/AIClasses/FunctionDefinitions/ListVaultFiles.ts b/AIClasses/FunctionDefinitions/ListVaultFiles.ts index da1f9b7..bc2a0af 100644 --- a/AIClasses/FunctionDefinitions/ListVaultFiles.ts +++ b/AIClasses/FunctionDefinitions/ListVaultFiles.ts @@ -1,7 +1,7 @@ import { AIFunction } from "Enums/AIFunction"; -import type { FunctionDefinition } from "./AIFunctionDefinition"; +import type { IAIFunctionDefinition } from "./IAIFunctionDefinition"; -export const ListVaultFiles: FunctionDefinition = { +export const ListVaultFiles: IAIFunctionDefinition = { name: AIFunction.ListVaultFiles, description: `Returns complete list of vault files with metadata (names, paths, sizes). Call this whenever you need to know what files exist in the vault to answer questions, @@ -9,6 +9,11 @@ export const ListVaultFiles: FunctionDefinition = { when vault contents would inform your response.`, parameters: { type: "object", - properties: {} + properties: { + user_message: { + type: "string", + description: "A short message to be displayed to the user that explains the action being taken" + } + } } } \ No newline at end of file diff --git a/AIClasses/Gemini/Gemini.ts b/AIClasses/Gemini/Gemini.ts index 8c29d46..c2efc7d 100644 --- a/AIClasses/Gemini/Gemini.ts +++ b/AIClasses/Gemini/Gemini.ts @@ -6,76 +6,146 @@ import { StreamingService, type StreamChunk } from "Services/StreamingService"; import type { Conversation } from "Conversations/Conversation"; import { Role } from "Enums/Role"; import { AIProviderURL } from "Enums/ApiProvider"; +import { AIFunctionCall } from "AIClasses/AIFunctionCall"; +import type { AIFunctionDefinitions } from "AIClasses/FunctionDefinitions/AIFunctionDefinitions"; +import type { IAIFunctionDefinition } from "AIClasses/FunctionDefinitions/IAIFunctionDefinition"; export class Gemini implements IAIClass { + private readonly REQUEST_WEB_SEARCH: string = "request_web_search"; + private readonly apiKey: string; private readonly aiPrompt: IPrompt; private readonly streamingService: StreamingService; + private readonly aiFunctionDefinitions: AIFunctionDefinitions; + private accumulatedFunctionName: string | null = null; + private accumulatedFunctionArgs: Record = {}; public constructor(apiKey: string) { this.apiKey = apiKey; this.aiPrompt = Resolve(Services.IPrompt); this.streamingService = Resolve(Services.StreamingService); + this.aiFunctionDefinitions = Resolve(Services.AIFunctionDefinitions); } public async* streamRequest(conversation: Conversation): AsyncGenerator { - + // next request should use web search only (gemini api doesn't support custom tooling and grounding at the same time) + let requestWebSearch = this.accumulatedFunctionName == this.REQUEST_WEB_SEARCH; + + // Reset function call accumulation state for new request + this.accumulatedFunctionName = null; + this.accumulatedFunctionArgs = {}; + const contents = conversation.contents.map(content => ({ role: content.role === Role.User ? "user" : "model", - parts: [ - { - text: content.content - } - ] + parts: (content.isFunctionCall || content.isFunctionCallResponse) + ? [JSON.parse(content.content)] : [{ text: content.content }] })); + const tools = requestWebSearch ? { google_search: {} } : + { + functionDeclarations: [ + { + name: "request_web_search", + description: `Use this function when you need to search the web for current + information, recent events, news, or facts that may have changed. + After calling this, you will be able to perform web searches.`, + parameters: { + type: "object", + properties: { + reasoning: { + type: "string", + description: "Brief explanation of why web search is needed" + } + }, + required: ["reasoning"] + } + }, + ...this.mapFunctionDefinitions(this.aiFunctionDefinitions.getQueryActions()), + ] + } + const requestBody = { system_instruction: { parts: [ { text: this.aiPrompt.systemInstruction() }, + { + text: `IMPORTANT: When you need current information from the web (recent events, news, current prices, weather, etc.), you should: + 1. First call the 'request_web_search' function to indicate you need web access + 2. After that, you'll be given access to Google Search + 3. Once you have the information from the search, you can answer the user's question + 4. Subsequent communication will return to providing custom function calls` + }, { text: await this.aiPrompt.userInstruction() } ] }, contents: contents, - tools: [ - { - google_search: {}, - functionDeclarations: [], - }, - ] + tools: [tools] }; + console.log(requestBody); + yield* this.streamingService.streamRequest( AIProviderURL.Gemini.replace("API_KEY", this.apiKey), requestBody, - this.parseStreamChunk + this.parseStreamChunk.bind(this) ); } private parseStreamChunk(chunk: string): StreamChunk { try { const data = JSON.parse(chunk); - + let text = ""; + let functionCall: AIFunctionCall | undefined = undefined; const candidate = data.candidates?.[0]; if (candidate) { - if (candidate.content?.parts?.[0]?.text) { - text = candidate.content.parts[0].text; - } else if (candidate.text) { - text = candidate.text; + // Check for text content + if (candidate.content?.parts?.[0]?.text) { + text = candidate.content.parts[0].text; + } else if (candidate.text) { + text = candidate.text; + } + + // Check for function call and accumulate + const parts = candidate.content?.parts || []; + for (const part of parts) { + if (part.functionCall) { + // Accumulate function name + if (part.functionCall.name) { + this.accumulatedFunctionName = part.functionCall.name; + } + + // Accumulate function arguments (merge with existing) + if (part.functionCall.args) { + this.accumulatedFunctionArgs = { + ...this.accumulatedFunctionArgs, + ...part.functionCall.args + }; + } + break; // Only handle first function call per chunk } + } } - + const isComplete = !!candidate?.finishReason; - + + // If streaming is complete and we have accumulated a function call, return it + if (isComplete && this.accumulatedFunctionName) { + functionCall = new AIFunctionCall( + this.accumulatedFunctionName, + this.accumulatedFunctionArgs + ); + } + return { content: text, isComplete: isComplete, + functionCall: functionCall, }; } catch (error) { const message = error instanceof Error ? error.message : "Unknown parsing error"; @@ -83,4 +153,12 @@ export class Gemini implements IAIClass { return { content: "", isComplete: false, error: `Failed to parse chunk: ${message}` }; } } + + private mapFunctionDefinitions(aiFunctionDefinitions: IAIFunctionDefinition[]): object[] { + return aiFunctionDefinitions.map((functionDefinition) => ({ + name: functionDefinition.name, + description: functionDefinition.description, + parameters: functionDefinition.parameters + })); + } } \ No newline at end of file diff --git a/AIClasses/Response/AIResponse.ts b/AIClasses/Response/AIResponse.ts deleted file mode 100644 index 844fb4f..0000000 --- a/AIClasses/Response/AIResponse.ts +++ /dev/null @@ -1,6 +0,0 @@ - - -interface CreateFileRequest { - file_path: string; - file_content: string; -} \ No newline at end of file diff --git a/AIClasses/SystemPrompt.ts b/AIClasses/SystemPrompt.ts index 7099de9..123d647 100644 --- a/AIClasses/SystemPrompt.ts +++ b/AIClasses/SystemPrompt.ts @@ -54,6 +54,7 @@ You are also capable of helping with: - Simple conversational exchanges **Best practices:** +- Always provide a user friendly description when using a tool in addition to the function object - Use tools proactively when vault content would enhance your answer - Offer to search rather than assuming - e.g., "Would you like me to check your notes for related information?" - Don't over-use tools for simple questions that don't require vault access diff --git a/Components/ChatArea.svelte b/Components/ChatArea.svelte index 342dd82..bc52f46 100644 --- a/Components/ChatArea.svelte +++ b/Components/ChatArea.svelte @@ -163,28 +163,30 @@
{#each messages as message, messageIndex (`${message.role}-${messageIndex}`)} - {#if message.role === Role.User} -
-
-

{message.content}

-
-
- {:else} -
-
-
- - {#if isStreaming && messageIndex === messages.length - 1} -
- - - {:else} - - {@html getStaticHTML(message, messageIndex)} - {/if} + {#if !message.isFunctionCall && !message.isFunctionCallResponse} + {#if message.role === Role.User} +
+
+

{message.content}

-
+ {:else} +
+
+
+ + {#if isStreaming && messageIndex === messages.length - 1} +
+ + + {:else} + + {@html getStaticHTML(message, messageIndex)} + {/if} +
+
+
+ {/if} {/if} {/each} diff --git a/Components/ChatWindow.svelte b/Components/ChatWindow.svelte index a106afa..7d74693 100644 --- a/Components/ChatWindow.svelte +++ b/Components/ChatWindow.svelte @@ -11,9 +11,13 @@ import { Role } from "Enums/Role"; import { Conversation } from "Conversations/Conversation"; import { ConversationContent } from "Conversations/ConversationContent"; + import type { AIFunctionCall } from "AIClasses/AIFunctionCall"; + import type { AIFunctionService } from "Services/AIFunctionService"; + import type { AIFunctionResponse } from "AIClasses/FunctionDefinitions/AIFunctionResponse"; let ai: IAIClass = Resolve(Services.IAIClass); let conversationService: ConversationFileSystemService = Resolve(Services.ConversationFileSystemService); + let aiFunctionService: AIFunctionService = Resolve(Services.AIFunctionService); let semaphore: Semaphore = new Semaphore(1, false); let textareaElement: HTMLTextAreaElement; @@ -27,76 +31,103 @@ let conversation = new Conversation(); async function handleSubmit() { - if (userRequest.trim() === "" || isSubmitting) { - return; - } - isSubmitting = true; - - const requestToSend = userRequest; - userRequest = ""; - textareaElement.value = ""; - autoResize(); - if (!await semaphore.wait()) { return; } - // Add user message to chat - conversation.contents = [...conversation.contents, new ConversationContent(Role.User, requestToSend)]; - - await conversationService.saveConversation(conversation); - - scrollToBottom(); - try { - // Create AI message placeholder - const aiMessageIndex = conversation.contents.length; - conversation.contents = [...conversation.contents, new ConversationContent(Role.Assistant, "")]; - isStreaming = true; + if (userRequest.trim() === "" || isSubmitting) { + return; + } + isSubmitting = true; - // Stream the response - let accumulatedContent = ""; + conversation.contents = [...conversation.contents, new ConversationContent(Role.User, userRequest)]; + await conversationService.saveConversation(conversation); - for await (const chunk of ai.streamRequest(conversation)) { - if (chunk.error) { - console.error("Streaming error:", chunk.error); - // Update message with error - conversation.contents = conversation.contents.map((msg, messageIndex) => - messageIndex === aiMessageIndex - ? { ...msg, content: "Error: " + chunk.error } - : msg - ); - isStreaming = false; - break; - } + textareaElement.value = ""; + userRequest = ""; + autoResize(); - if (chunk.content) { - accumulatedContent += chunk.content; - // Update the message with accumulated content - conversation.contents = conversation.contents.map((msg, messageIndex) => - messageIndex === aiMessageIndex - ? { ...msg, content: accumulatedContent } - : msg - ); - } + scrollToBottom(); - if (chunk.isComplete) { - // Mark streaming as complete - isStreaming = false; - conversation.contents = conversation.contents.map((msg, messageIndex) => - messageIndex === aiMessageIndex - ? { ...msg, content: accumulatedContent } - : msg - ); + let functionCall: AIFunctionCall | null = await streamRequestResponse(); + while (functionCall) { + + if ('user_message' in functionCall.arguments) { + conversation.contents = [...conversation.contents, new ConversationContent( + Role.Assistant, functionCall.arguments.user_message)]; await conversationService.saveConversation(conversation); } + + conversation.contents = [...conversation.contents, new ConversationContent( + Role.Assistant, functionCall.toConversationString(), new Date(), true)]; + await conversationService.saveConversation(conversation); + + const functionResponse: AIFunctionResponse = await aiFunctionService.performAIFunction(functionCall); + conversation.contents = [...conversation.contents, new ConversationContent( + Role.User, functionResponse.toConversationString(), new Date(), false, true)]; + await conversationService.saveConversation(conversation); + + functionCall = await streamRequestResponse(); } } finally { semaphore.release(); isSubmitting = false; + tick().then(() => { + textareaElement?.focus(); + }); } } + async function streamRequestResponse(): Promise { + // Create AI message placeholder + const aiMessageIndex = conversation.contents.length; + conversation.contents = [...conversation.contents, new ConversationContent(Role.Assistant, "")]; + isStreaming = true; + + let accumulatedContent = ""; + let capturedFunctionCall: AIFunctionCall | null = null; + + for await (const chunk of ai.streamRequest(conversation)) { + if (chunk.error) { + console.error("Streaming error:", chunk.error); + conversation.contents = conversation.contents.map((msg, messageIndex) => + messageIndex === aiMessageIndex + ? { ...msg, content: "Error: " + chunk.error } + : msg + ); + isStreaming = false; + break; + } + + if (chunk.content) { + accumulatedContent += chunk.content; + conversation.contents = conversation.contents.map((msg, messageIndex) => + messageIndex === aiMessageIndex + ? { ...msg, content: accumulatedContent } + : msg + ); + } + + if (chunk.functionCall) { + capturedFunctionCall = chunk.functionCall; + } + + if (chunk.isComplete) { + isStreaming = false; + conversation.contents = conversation.contents.map((msg, messageIndex) => + messageIndex === aiMessageIndex + ? { ...msg, content: accumulatedContent } + : msg + ); + + await conversationService.saveConversation(conversation); + } + } + + return capturedFunctionCall; + } + function handleKeydown(e: KeyboardEvent) { if (e.key === 'Enter') { if (e.shiftKey) { @@ -161,10 +192,10 @@ rows="1"> - @@ -213,6 +244,12 @@ resize: none; overflow-y: auto; color: var(--font-interface-theme); + transition: border-color 0.3s ease-out; + } + + #input:focus { + border-color: var(--color-accent); + transition: border-color 0.3s ease-out; } #submit { diff --git a/Conversations/ConversationContent.ts b/Conversations/ConversationContent.ts index 5d97d5e..c7f17c9 100644 --- a/Conversations/ConversationContent.ts +++ b/Conversations/ConversationContent.ts @@ -2,6 +2,8 @@ export class ConversationContent { role: string; content: string timestamp: Date; + isFunctionCall: boolean; + isFunctionCallResponse: boolean; public static isConversationContentData(data: unknown): data is { role: string; content: string; timestamp: string } { return ( @@ -10,15 +12,21 @@ export class ConversationContent { 'role' in data && 'content' in data && 'timestamp' in data && + 'isFunctionCall' in data && + 'isFunctionCallResponse' in data && typeof data.role === 'string' && typeof data.content === 'string' && - typeof data.timestamp === 'string' + typeof data.timestamp === 'string' && + typeof data.isFunctionCall == 'boolean' && + typeof data.isFunctionCallResponse == 'boolean' ); } - constructor(role: string, content: string) { + constructor(role: string, content: string, timestamp: Date = new Date(), isFunctionCall = false, isFunctionCallResponse = false) { this.role = role; this.content = content; - this.timestamp = new Date(); + this.timestamp = timestamp; + this.isFunctionCall = isFunctionCall; + this.isFunctionCallResponse = isFunctionCallResponse; } } \ No newline at end of file diff --git a/Modals/ConversationHistoryModal.ts b/Modals/ConversationHistoryModal.ts index f17ae5c..90f3809 100644 --- a/Modals/ConversationHistoryModal.ts +++ b/Modals/ConversationHistoryModal.ts @@ -33,7 +33,9 @@ export class ConversationHistoryModal extends Modal { override async open() { this.conversations = await this.conversationFileSystemService.getAllConversations(); - this.items = this.conversations.map((conversation, index) => ({ + this.items = this.conversations + .sort((a, b) => b.created.getTime() - a.created.getTime()) + .map((conversation, index) => ({ id: index.toString(), date: dateToString(conversation.created, false), title: conversation.title, diff --git a/Modals/ConversationHistoryModalSvelte.svelte b/Modals/ConversationHistoryModalSvelte.svelte index 69f44c6..ff02327 100644 --- a/Modals/ConversationHistoryModalSvelte.svelte +++ b/Modals/ConversationHistoryModalSvelte.svelte @@ -1,6 +1,6 @@