From a32f15308784cc38f4f433cb0a80963b97d3ee55 Mon Sep 17 00:00:00 2001 From: Andrew Beal Date: Mon, 6 Apr 2026 11:28:45 +0100 Subject: [PATCH] Add provider-level tool call handling with Mistral web search support Implement optional `resolveToolCall` method in IAIClass interface to allow providers to handle specific tools before falling through to AIToolService. Add Mistral web search capability using native agent API, enabling dynamic tool resolution at the provider level. --- AIClasses/IAIClass.ts | 5 + AIClasses/Mistral/Mistral.ts | 68 +++++++++---- AIClasses/Mistral/MistralAgent.ts | 114 ++++++++++++++++++++++ AIClasses/Mistral/MistralTypes.ts | 38 ++++++++ Enums/AITool.ts | 2 +- Enums/ApiProvider.ts | 5 + Enums/Path.ts | 2 +- Services/AIServices/BaseAgent.ts | 9 ++ Services/AIServices/ExecutionAgent.ts | 2 +- Services/AIServices/MainAgent.ts | 2 +- Services/AIServices/OrchestrationAgent.ts | 2 +- Services/AIServices/PlanningAgent.ts | 2 +- 12 files changed, 226 insertions(+), 25 deletions(-) create mode 100644 AIClasses/Mistral/MistralAgent.ts diff --git a/AIClasses/IAIClass.ts b/AIClasses/IAIClass.ts index 45b2d60..6c01dc6 100644 --- a/AIClasses/IAIClass.ts +++ b/AIClasses/IAIClass.ts @@ -5,6 +5,8 @@ import type { IAIToolDefinition } from "./ToolDefinitions/IAIToolDefinition"; import type { AIProvider } from "Enums/ApiProvider"; import type { AgentType } from "Enums/AgentType"; import type { AIToolUsageMode } from "Enums/AIToolUsageMode"; +import type { AIToolCall } from "./AIToolCall"; +import type { AIToolResponse } from "./ToolDefinitions/AIToolResponse"; export interface IAIClass { get currentProvider(): AIProvider; @@ -16,4 +18,7 @@ export interface IAIClass { streamRequest(conversation: Conversation): AsyncGenerator; formatBinaryFiles(attachments: Attachment[]): string; + + // Optional provider-level tool call handler. Called before AIToolService. + resolveToolCall?(toolCall: AIToolCall): Promise; } \ No newline at end of file diff --git a/AIClasses/Mistral/Mistral.ts b/AIClasses/Mistral/Mistral.ts index 7e6bb07..6eb4eac 100644 --- a/AIClasses/Mistral/Mistral.ts +++ b/AIClasses/Mistral/Mistral.ts @@ -4,7 +4,8 @@ import type { Conversation } from "Conversations/Conversation"; import type { Attachment } from "Conversations/Attachment"; import { AIProvider, AIProviderURL } from "Enums/ApiProvider"; import { AIToolCall } from "AIClasses/AIToolCall"; -import { fromString as aiToolFromString } from "Enums/AITool"; +import { AIToolResponse } from "AIClasses/ToolDefinitions/AIToolResponse"; +import { fromString as aiToolFromString, AITool } from "Enums/AITool"; import type { IAIToolDefinition } from "AIClasses/ToolDefinitions/IAIToolDefinition"; import type { ConversationContent } from "Conversations/ConversationContent"; import { Role } from "Enums/Role"; @@ -18,6 +19,7 @@ import { AIToolUsageMode } from "Enums/AIToolUsageMode"; import type { MistralStreamChunk, MistralToolDefinition, MistralMessage, MistralContentPart } from "./MistralTypes"; import type { MistralFileService } from "./MistralFileService"; import { Copy, replaceCopy } from "Enums/Copy"; +import { MistralAgent } from "./MistralAgent"; export class Mistral extends BaseAIClass { @@ -32,6 +34,8 @@ export class Mistral extends BaseAIClass { MimeType.IMAGE_WEBP ]; + private readonly agent: MistralAgent = new MistralAgent(); + // Accumulation state for streaming tool calls private accumulatedToolCalls: Map = new Map(); @@ -40,30 +44,30 @@ export class Mistral extends BaseAIClass { } public async* streamRequest(conversation: Conversation): AsyncGenerator { + const messages = await this.buildMessages(conversation); - this.accumulatedToolCalls.clear(); - - // Refresh file cache only if conversation has attachments - if (conversation.hasAttachments()) { - await this.aiFileService.refreshCache(); - } - - const systemPrompt = `${this.systemPrompt}\n\n${this.userInstruction}`; - - const messages = await this.extractContents(conversation.contents); - - // Add system message at the beginning - const allMessages: MistralMessage[] = [ - { role: "system", content: systemPrompt }, - ...messages + const tools = [ + { + type: "function" as const, + function: { + name: AITool.RequestWebSearch, + description: `Use this function when you need to search the web for current information, recent events, news, or facts that may have changed.`, + parameters: { + type: "object" as const, + properties: { + query: { type: "string", description: "The search query to look up on the web." } + }, + required: ["query"] + } + } + }, + ...this.mapFunctionDefinitions(this.aiToolDefinitions) ]; - const tools = this.mapFunctionDefinitions(this.aiToolDefinitions); - const requestBody: Record = { model: this.model(), max_tokens: 16384, - messages: allMessages, + messages: messages, stream: true }; @@ -87,6 +91,32 @@ export class Mistral extends BaseAIClass { ); } + public async resolveToolCall(toolCall: AIToolCall): Promise { + if (toolCall.name !== AITool.RequestWebSearch) { + return null; + } + const query = (toolCall.arguments as Record).query ?? ""; + const result = await this.agent.search(query); + return new AIToolResponse(toolCall.name, { result }, toolCall.toolId); + } + + private async buildMessages(conversation: Conversation): Promise { + this.accumulatedToolCalls.clear(); + + // Refresh file cache only if conversation has attachments + if (conversation.hasAttachments()) { + await this.aiFileService.refreshCache(); + } + + const systemPrompt = `${this.systemPrompt}\n\n${this.userInstruction}`; + const messages = await this.extractContents(conversation.contents); + + return [ + { role: "system", content: systemPrompt }, + ...messages + ]; + } + protected parseStreamChunk(chunk: string): IStreamChunk { try { // Mistral sends "[DONE]" as the final message diff --git a/AIClasses/Mistral/MistralAgent.ts b/AIClasses/Mistral/MistralAgent.ts new file mode 100644 index 0000000..5fed933 --- /dev/null +++ b/AIClasses/Mistral/MistralAgent.ts @@ -0,0 +1,114 @@ +import { Resolve } from "Services/DependencyService"; +import { Services } from "Services/Services"; +import type { SettingsService } from "Services/SettingsService"; +import { AIProvider, AIProviderModel, MistralAgentEndpoint } from "Enums/ApiProvider"; +import { Exception } from "Helpers/Exception"; +import type { + MistralAgentCreateRequest, + MistralAgentCreateResponse, + MistralAgentListResponse, + MistralConversationRequest, + MistralConversationResponse +} from "./MistralTypes"; + +/** + * Performs a one-shot web search using the Mistral Agents API. + * Creates and caches a search agent on first use, then reuses it for subsequent calls. + */ +export class MistralAgent { + + private readonly apiKey: string; + private agentId: string | undefined; + + public constructor() { + const settingsService = Resolve(Services.SettingsService); + this.apiKey = settingsService.getApiKeyForProvider(AIProvider.Mistral); + } + + public async search(query: string): Promise { + if (!this.agentId) { + this.agentId = await this.getOrCreateAgent(); + } + + return await this.complete(this.agentId, query); + } + + private async getOrCreateAgent(): Promise { + const listResponse = await fetch(MistralAgentEndpoint.Url, { + headers: { "Authorization": `Bearer ${this.apiKey}` } + }); + + if (listResponse.ok) { + const list = await listResponse.json() as MistralAgentListResponse; + const existing = list.find(agent => agent.name === "VaultKeeper Web Search"); + if (existing) { + return existing.id; + } + } + + return await this.createAgent(); + } + + private async createAgent(): Promise { + const requestBody: MistralAgentCreateRequest = { + model: AIProviderModel.MistralMedium, + name: "VaultKeeper Web Search", + instructions: "You are a web search assistant. **Always** search the web to look up current information before responding and never answer from memory alone.", + tools: [{ type: "web_search" }] + }; + + const response = await fetch(MistralAgentEndpoint.Url, { + method: "POST", + headers: { + "Authorization": `Bearer ${this.apiKey}`, + "Content-Type": "application/json" + }, + body: JSON.stringify(requestBody) + }); + + if (!response.ok) { + Exception.throw(`Mistral create agent failed: ${response.status} ${response.statusText} - ${await response.text()}`); + } + + const data = await response.json() as MistralAgentCreateResponse; + return data.id; + } + + private async complete(agentId: string, query: string): Promise { + const requestBody: MistralConversationRequest = { + agent_id: agentId, + inputs: query, + stream: false + }; + + const response = await fetch(MistralAgentEndpoint.ConversationsUrl, { + method: "POST", + headers: { + "Authorization": `Bearer ${this.apiKey}`, + "Content-Type": "application/json" + }, + body: JSON.stringify(requestBody) + }); + + if (!response.ok) { + const error = `Mistral agent completion failed: ${response.status} ${response.statusText} - ${await response.text()}`; + Exception.log(error); + return error; + } + + const data = await response.json() as MistralConversationResponse; + const messageOutput = data.outputs?.find(output => output.type === "message.output"); + const content = messageOutput?.content; + const textOutput = Array.isArray(content) + ? content.filter(content => content.type === "text").map(content => content.text ?? "").join() + : typeof content === "string" ? content : undefined; + + if (!textOutput) { + const error = "Mistral web agent returned no content"; + Exception.log(error); + return error; + } + + return textOutput; + } +} \ No newline at end of file diff --git a/AIClasses/Mistral/MistralTypes.ts b/AIClasses/Mistral/MistralTypes.ts index c454b81..2fc74b3 100644 --- a/AIClasses/Mistral/MistralTypes.ts +++ b/AIClasses/Mistral/MistralTypes.ts @@ -93,6 +93,44 @@ export interface MistralContentPart { document_url?: string; } +// Agents API types +export interface MistralAgentCreateRequest { + model: string; + name?: string; + description?: string; + instructions?: string; + tools: Array<{ type: "web_search" | "function" }>; +} + +export interface MistralAgentCreateResponse { + id: string; + object: "agent"; + created_at: string; + name: string | null; + model: string; +} + +export type MistralAgentListResponse = MistralAgentCreateResponse[]; + +export interface MistralConversationRequest { + agent_id: string; + inputs: string; + stream?: boolean; +} + +export interface MistralConversationResponse { + object: "conversation.response"; + conversation_id: string; + outputs: Array<{ + type: "message.output" | "tool.execution"; + role?: string; + content?: string | Array<{ + type: "text" | "tool_reference"; + text?: string; + }>; + }>; +} + // File API types export interface MistralFile { id: string; diff --git a/Enums/AITool.ts b/Enums/AITool.ts index e90ddb8..2bbe797 100644 --- a/Enums/AITool.ts +++ b/Enums/AITool.ts @@ -9,7 +9,7 @@ export enum AITool { ReadMemories = "read_memories", UpdateMemories = "update_memories", - // only used by gemini + // used by gemini and mistral RequestWebSearch = "request_web_search", // multi agent calls diff --git a/Enums/ApiProvider.ts b/Enums/ApiProvider.ts index d280ad8..0feca3e 100644 --- a/Enums/ApiProvider.ts +++ b/Enums/ApiProvider.ts @@ -96,4 +96,9 @@ export enum AIFileServiceURL { GeminiUpload = "https://generativelanguage.googleapis.com/upload/v1beta", OpenAI = "https://api.openai.com/v1/files", Mistral = "https://api.mistral.ai/v1/files", +} + +export enum MistralAgentEndpoint { + Url = "https://api.mistral.ai/v1/agents", + ConversationsUrl = "https://api.mistral.ai/v1/conversations" } \ No newline at end of file diff --git a/Enums/Path.ts b/Enums/Path.ts index 5e6236e..f124e9a 100644 --- a/Enums/Path.ts +++ b/Enums/Path.ts @@ -5,5 +5,5 @@ export enum Path { Attachments = `${Path.Conversations}/Attachments`, UserInstructions = `${Path.VaultkeeperAIDir}/User Instructions`, ExampleUserInstructions = `${Path.UserInstructions}/EXAMPLE_INSTRUCTIONS.md`, - Memories = `${Path.VaultkeeperAIDir}/MEMORIES.md` + Memories = `${Path.VaultkeeperAIDir}/Memories.md` }; \ No newline at end of file diff --git a/Services/AIServices/BaseAgent.ts b/Services/AIServices/BaseAgent.ts index bd5e258..0a3491b 100644 --- a/Services/AIServices/BaseAgent.ts +++ b/Services/AIServices/BaseAgent.ts @@ -1,4 +1,5 @@ import type { AIToolCall } from "AIClasses/AIToolCall"; +import type { AIToolResponse } from "AIClasses/ToolDefinitions/AIToolResponse"; import type { IAIClass } from "AIClasses/IAIClass"; import type { IPrompt } from "AIPrompts/IPrompt"; import type { Conversation } from "Conversations/Conversation"; @@ -203,6 +204,14 @@ export abstract class BaseAgent { return { toolCall: capturedToolCall, shouldContinue: capturedShouldContinue }; } + protected async performAITool(toolCall: AIToolCall): Promise { + const providerResult = await this.ai?.resolveToolCall?.(toolCall) ?? null; + if (providerResult !== null) { + return providerResult; + } + return this.aiToolService.performAITool(toolCall); + } + private async withToolCallingDisabled(callback: () => Promise): Promise { if (!this.ai) { // this shouldn't ever happen Exception.throw("Error: No AI provider has been set!"); diff --git a/Services/AIServices/ExecutionAgent.ts b/Services/AIServices/ExecutionAgent.ts index 55a5772..f30f7d1 100644 --- a/Services/AIServices/ExecutionAgent.ts +++ b/Services/AIServices/ExecutionAgent.ts @@ -71,7 +71,7 @@ export class ExecutionAgent extends BaseAgent { this.debugService?.log("ExecutionAgent", `Executing function: ${toolCall.name}`); this.updateThought(toolCall, callbacks); - const functionResponse = await this.aiToolService.performAITool(toolCall); + const functionResponse = await this.performAITool(toolCall); this.conversation.addFunctionResponse(functionResponse); return { shouldExit: false }; }); diff --git a/Services/AIServices/MainAgent.ts b/Services/AIServices/MainAgent.ts index 356b4ba..fd12384 100644 --- a/Services/AIServices/MainAgent.ts +++ b/Services/AIServices/MainAgent.ts @@ -77,7 +77,7 @@ export class MainAgent extends BaseAgent { this.debugService?.log("MainAgent", `Executing function: ${toolCall.name}`); this.updateThought(toolCall, callbacks); - const functionResponse = await this.aiToolService.performAITool(toolCall); + const functionResponse = await this.performAITool(toolCall); conversation.addFunctionResponse(functionResponse); return { shouldExit: false }; }); diff --git a/Services/AIServices/OrchestrationAgent.ts b/Services/AIServices/OrchestrationAgent.ts index 226edb2..f8d6473 100644 --- a/Services/AIServices/OrchestrationAgent.ts +++ b/Services/AIServices/OrchestrationAgent.ts @@ -178,7 +178,7 @@ export class OrchestrationAgent extends BaseAgent { isAITool(toolCallName, AITool.ListVaultFiles)) { this.debugService?.log("Orchestration", `Vault tool called for recovery: ${toolCallName}`); this.updateThought(toolCall, callbacks); - const toolResponse = await this.aiToolService.performAITool(toolCall); + const toolResponse = await this.performAITool(toolCall); planningConversation.addFunctionResponse(toolResponse); return Promise.resolve({ shouldExit: false }); } diff --git a/Services/AIServices/PlanningAgent.ts b/Services/AIServices/PlanningAgent.ts index ce11379..c6f73ac 100644 --- a/Services/AIServices/PlanningAgent.ts +++ b/Services/AIServices/PlanningAgent.ts @@ -88,7 +88,7 @@ export class PlanningAgent extends BaseAgent { } this.updateThought(toolCall, callbacks); - const functionResponse = await this.aiToolService.performAITool(toolCall); + const functionResponse = await this.performAITool(toolCall); conversation.addFunctionResponse(functionResponse); return { shouldExit: false }; });