From e22cd8698afe039af5f9b99f5372215fa2fe74f5 Mon Sep 17 00:00:00 2001 From: Andrew Beal Date: Sun, 4 Jan 2026 18:52:44 +0000 Subject: [PATCH] refactor: implement planning mode with user questions and cross-provider function call improvements - Add planning mode toggle and AskUserQuestion function for interactive planning - Fix Gemini cross-provider function call detection using toolId presence - Update OpenAI naming service to handle new response format - Improve system prompts by removing complexity gate, streamlining to action-first principle - Add InputDisplay component and question mode to ChatInput - Refactor execution plan error messages to show all incomplete steps - Update tests to reflect cross-provider function call format changes --- .../AIFunctionDefinitions.ts | 12 +- .../Functions/AskUserQuestion.ts | 30 +++ AIClasses/Gemini/Gemini.ts | 25 ++- .../OpenAI/OpenAIConversationNamingService.ts | 28 +-- AIClasses/Schemas/AIFunctionSchemas.ts | 6 + AIPrompts/IPrompt.ts | 7 +- AIPrompts/PlanningEnabledAppendix.ts | 8 + AIPrompts/SystemPrompt.ts | 140 ++++--------- Components/ChatInput.svelte | 186 +++++++++++++++--- Components/ChatPlanArea.svelte | 2 +- Components/ChatWindow.svelte | 22 ++- Components/InputDisplay.svelte | 64 ++++++ Enums/AIFunction.ts | 3 +- Enums/ApiProvider.ts | 2 +- Enums/Copy.ts | 18 +- Enums/InputMode.ts | 5 + Services/AIControllerService.ts | 47 ++++- Services/AIFunctionService.ts | 1 + Services/ChatService.ts | 6 +- Types/ExecutionPlan.ts | 12 +- .../CrossProviderIntegration.test.ts | 76 ++++--- __tests__/AIClasses/Gemini.test.ts | 17 +- 22 files changed, 500 insertions(+), 217 deletions(-) create mode 100644 AIClasses/FunctionDefinitions/Functions/AskUserQuestion.ts create mode 100644 AIPrompts/PlanningEnabledAppendix.ts create mode 100644 Components/InputDisplay.svelte create mode 100644 Enums/InputMode.ts diff --git a/AIClasses/FunctionDefinitions/AIFunctionDefinitions.ts b/AIClasses/FunctionDefinitions/AIFunctionDefinitions.ts index 3b270f7..f3b2071 100644 --- a/AIClasses/FunctionDefinitions/AIFunctionDefinitions.ts +++ b/AIClasses/FunctionDefinitions/AIFunctionDefinitions.ts @@ -11,6 +11,7 @@ import { Replan } from "./Functions/Replan"; import { CompleteStep } from "./Functions/CompleteStep"; import { SubmitPlan } from "./Functions/SubmitPlan"; import { CancelPlan } from "./Functions/CancelPlan"; +import { AskUserQuestion } from "./Functions/AskUserQuestion"; export abstract class AIFunctionDefinitions { @@ -18,7 +19,11 @@ export abstract class AIFunctionDefinitions { private static readonly definitionsList = [SearchVaultFiles, ReadVaultFiles, ListVaultFiles, WriteVaultFile, PatchVaultFile, DeleteVaultFiles, MoveVaultFiles]; // Definitions for the main agent - public static agentDefinitions(destructive: boolean): IAIFunctionDefinition[] { + public static agentDefinitions(destructive: boolean, planning: boolean): IAIFunctionDefinition[] { + if (planning) { + return [CreatePlan]; + } + let actions = [ SearchVaultFiles, ReadVaultFiles, @@ -30,8 +35,7 @@ export abstract class AIFunctionDefinitions { WriteVaultFile, PatchVaultFile, DeleteVaultFiles, - MoveVaultFiles, - CreatePlan + MoveVaultFiles ]); } @@ -40,7 +44,7 @@ export abstract class AIFunctionDefinitions { // Definitions for the planning agent public static planningAgentDefinitions(): IAIFunctionDefinition[] { - return [SearchVaultFiles, ReadVaultFiles, ListVaultFiles, SubmitPlan]; + return [SearchVaultFiles, ReadVaultFiles, ListVaultFiles, AskUserQuestion, SubmitPlan]; } // Definitions for the main agent during plan execution diff --git a/AIClasses/FunctionDefinitions/Functions/AskUserQuestion.ts b/AIClasses/FunctionDefinitions/Functions/AskUserQuestion.ts new file mode 100644 index 0000000..d863d0d --- /dev/null +++ b/AIClasses/FunctionDefinitions/Functions/AskUserQuestion.ts @@ -0,0 +1,30 @@ +import { AIFunction } from "Enums/AIFunction"; +import type { IAIFunctionDefinition } from "../IAIFunctionDefinition"; + +export const AskUserQuestion: IAIFunctionDefinition = { + name: AIFunction.AskUserQuestion, + description: `Asks the user a question during the planning phase and waits for their response. +Use this function when you need clarification, user input, or decisions while creating the plan. +This allows interactive planning where user feedback shapes the plan before it's submitted. + +Common use cases: +- Clarifying ambiguous requirements before creating the plan +- Getting user preferences when multiple valid approaches exist for the plan +- Asking which files or folders the user wants to target +- Confirming assumptions about the user's intent before planning`, + + parameters: { + type: "object", + properties: { + question: { + type: "string", + description: "The question to ask the user. Should be clear, specific, and provide enough context for the user to give a meaningful answer. Example: 'I found 3 files matching your criteria. Should I update all of them or would you like to review them first?'" + }, + user_message: { + type: "string", + description: "A short message to be displayed to the user explaining what action you are taking. Examples: 'Asking the users preference for note naming', 'Querying the user about desired vault structure'" + } + }, + required: ["question", "user_message"] + } +}; diff --git a/AIClasses/Gemini/Gemini.ts b/AIClasses/Gemini/Gemini.ts index b73f2e2..8f449f3 100644 --- a/AIClasses/Gemini/Gemini.ts +++ b/AIClasses/Gemini/Gemini.ts @@ -228,21 +228,28 @@ export class Gemini extends BaseAIClass { const parsedContent = parseFunctionCall(content.functionCall); if (parsedContent) { - if (content.thoughtSignature && content.thoughtSignature.trim() !== "") { - // Has signature - use proper function call format + // Check if this is a cross-provider function call (has toolId in the stored format) + // Gemini never uses toolId, so presence of toolId indicates Claude/OpenAI origin + const isCrossProvider = parsedContent.functionCall.id && parsedContent.functionCall.id.trim() !== ""; + + if (isCrossProvider) { + // Cross-provider function call (from Claude/OpenAI) - use legacy text format + parts.push({ + text: this.convertFunctionCallToText(parsedContent) + }); + } else { + // Native Gemini function call - use proper function call format const part: Part = { functionCall: { name: parsedContent.functionCall.name, args: parsedContent.functionCall.args - }, - thoughtSignature: content.thoughtSignature + } }; + // Include thoughtSignature if present (optional Gemini feature) + if (content.thoughtSignature && content.thoughtSignature.trim() !== "") { + part.thoughtSignature = content.thoughtSignature; + } parts.push(part); - } else { - // No signature (cross-provider scenario) - use legacy text format - parts.push({ - text: this.convertFunctionCallToText(parsedContent) - }); } } else { parts.push({ diff --git a/AIClasses/OpenAI/OpenAIConversationNamingService.ts b/AIClasses/OpenAI/OpenAIConversationNamingService.ts index 5873ec2..105fd99 100644 --- a/AIClasses/OpenAI/OpenAIConversationNamingService.ts +++ b/AIClasses/OpenAI/OpenAIConversationNamingService.ts @@ -10,7 +10,6 @@ import { Exception } from "Helpers/Exception"; import type { AbortService } from "Services/AbortService"; export class OpenAIConversationNamingService implements IConversationNamingService { - private readonly apiKey: string; private readonly abortService: AbortService; @@ -24,7 +23,6 @@ export class OpenAIConversationNamingService implements IConversationNamingServi return await this.abortService.abortableOperation(async () => { const requestBody = { model: AIProviderModel.OpenAINamer, - max_output_tokens: 100, instructions: NamePrompt, input: [ { @@ -49,21 +47,23 @@ export class OpenAIConversationNamingService implements IConversationNamingServi Exception.throw(`OpenAI API error: ${response.status} ${response.statusText} - ${await response.text()}`); } - const data = await response.json() as OpenAI.Responses.Response; + const data = await response.json(); - // Try to get the name from output_text first (most common case) - if (data.output_text && data.output_text.trim()) { - return data.output_text.trim(); + // Find text from any message-type output + let generatedName: string | undefined; + + for (const item of data.output ?? []) { + if (item.type === 'message' && Array.isArray(item.content)) { + for (const content of item.content) { + if (content.type === 'output_text' && content.text) { + generatedName = content.text.trim(); + break; + } + } + } + if (generatedName) break; } - // Fall back to checking the output array - const firstOutput = data.output?.[0]; - const generatedName = firstOutput && 'content' in firstOutput - ? firstOutput.content?.[0]?.type === 'output_text' - ? firstOutput.content[0].text - : undefined - : undefined; - if (!generatedName) { Exception.throw("Failed to generate conversation name"); } diff --git a/AIClasses/Schemas/AIFunctionSchemas.ts b/AIClasses/Schemas/AIFunctionSchemas.ts index 979eeb4..dd6b5c6 100644 --- a/AIClasses/Schemas/AIFunctionSchemas.ts +++ b/AIClasses/Schemas/AIFunctionSchemas.ts @@ -60,6 +60,11 @@ export const ReplanArgsSchema = z.object({ user_message: z.string() }); +export const AskUserQuestionArgsSchema = z.object({ + question: z.string(), + user_message: z.string() +}); + export const CancelPlanArgsSchema = z.object({ confirm_cancellation: z.boolean() }); @@ -86,6 +91,7 @@ export type MoveVaultFilesArgs = z.infer; export type ListVaultFilesArgs = z.infer; export type CreatePlanArgs = z.infer; export type ReplanArgs = z.infer; +export type AskUserQuestionArgs = z.infer; export type CancelPlanArgs = z.infer; export type CompleteStepArgs = z.infer; export type SubmitPlanArgs = z.infer; diff --git a/AIPrompts/IPrompt.ts b/AIPrompts/IPrompt.ts index 2bbfceb..23d31c4 100644 --- a/AIPrompts/IPrompt.ts +++ b/AIPrompts/IPrompt.ts @@ -4,9 +4,10 @@ import { SystemInstruction } from "./SystemPrompt"; import type { FileSystemService } from "Services/FileSystemService"; import type { SettingsService } from "Services/SettingsService"; import { PlanningAgentSystemPrompt } from "AIPrompts/PlanningAgentSystemPrompt"; +import { PlanningEnabledAppendix } from "./PlanningEnabledAppendix"; export interface IPrompt { - systemInstruction(): string; + systemInstruction(planningMode?: boolean): string; planningInstruction(): string; userInstruction(): Promise; } @@ -21,8 +22,8 @@ export class AIPrompt implements IPrompt { this.fileSystemService = Resolve(Services.FileSystemService); } - public systemInstruction(): string { - return SystemInstruction; + public systemInstruction(planningMode: boolean = false): string { + return planningMode ? SystemInstruction + PlanningEnabledAppendix : SystemInstruction; } public planningInstruction(): string { diff --git a/AIPrompts/PlanningEnabledAppendix.ts b/AIPrompts/PlanningEnabledAppendix.ts new file mode 100644 index 0000000..e998008 --- /dev/null +++ b/AIPrompts/PlanningEnabledAppendix.ts @@ -0,0 +1,8 @@ +// Appended to the system prompt when planning mode is enabled +export const PlanningEnabledAppendix = ` + +Planning mode is ENABLED. You MUST request a plan before executing any other tools. + +Do not execute actions directly—first create a strategic plan that will be reviewed and approved. Once you receive an approved plan, execute it step by step. + +`; \ No newline at end of file diff --git a/AIPrompts/SystemPrompt.ts b/AIPrompts/SystemPrompt.ts index 358f4f4..a69e7ce 100644 --- a/AIPrompts/SystemPrompt.ts +++ b/AIPrompts/SystemPrompt.ts @@ -3,43 +3,13 @@ export const SystemInstruction: string = ` You are a specialized AI assistant with direct access to the user's Obsidian vault. Your core strength is helping users leverage their personal knowledge base while providing general assistance when needed. -## Critical Operating Principles +## Core Operating Principles -### 0. MANDATORY COMPLEXITY GATE (Evaluate Before Every Action) +### 1. ACTION-FIRST PRINCIPLE -**You MUST classify every request before acting. This gate is non-negotiable.** - -#### AUTOMATIC PLANNING TRIGGERS - -**If ANY of these conditions are present, you MUST request planning. Do not proceed without a plan.** - -| Trigger | Examples | -|---------|----------| -| **Structural keywords** | "organize", "restructure", "set up", "create a system/structure", "reorganize", "clean up", "overhaul" | -| **Vault-wide or multi-folder scope** | "my vault", "all my notes", "everything about", "across my folders" | -| **Unknown or unbounded scope** | Cannot determine how many files/folders are affected without exploration | -| **Ambiguous structural decisions** | Request requires choices about naming, hierarchy, or organization that user hasn't specified | -| **Synthesis + action combination** | "analyze X and then create/reorganize Y based on findings" | - -#### DIRECT EXECUTION CRITERIA - -**Execute directly ONLY when ALL of these are true:** - -- Single file or explicitly bounded scope (e.g., "create a note called X") -- No structural decisions required—path is obvious -- Clear, unambiguous success criteria -- No exploration needed to understand what to do - -#### WHEN UNCERTAIN - -**Plan.** The cost of unnecessary planning is one extra step. The cost of botched vault reorganization is significant. - ---- - -### 1. ACTION-FIRST PRINCIPLE (Applies Only After Passing Gate) - -**Once you've confirmed direct execution is appropriate, act immediately.** +**User requests are commands, not proposals. Execute immediately.** +- Complete ALL necessary operations before concluding your turn - User requests are commands, not proposals - Tool availability implies intended use - Execute first, explain after @@ -50,38 +20,33 @@ You are a specialized AI assistant with direct access to the user's Obsidian vau - Outcome requests ("Show me Y") → Use tools to retrieve/generate Y - Image/PDF references → Read the file first -**Example (simple, bounded request):** +**Example:** User: "Create a note about today's meeting with Sarah" ❌ Wrong: "I can create a note for you. Would you like me to proceed?" ✅ Correct: [Immediately calls write_vault_file with appropriate content] -**Example (triggers planning gate):** -User: "Organize my vault for my D&D campaign" -❌ Wrong: [Starts creating folders immediately] -✅ Correct: [Requests strategic plan—structural keyword + ambiguous scope + decisions required] - --- -### 2. PLAN-THEN-EXECUTE PROTOCOL +### 2. PLAN EXECUTION PROTOCOL -**When the complexity gate triggers planning, follow this protocol exactly.** +**When the user has enabled planning mode and you receive a plan, follow this protocol.** #### Requesting a Plan -Provide the planning agent with: +When planning mode is enabled, provide the planning agent with: 1. **Goal**: What the user wants to accomplish 2. **Context**: Relevant vault state, user preferences, constraints 3. **Unknowns**: What exploration is needed before committing to an approach #### Executing a Plan -1. **Treat plan steps as directives**—execute them, don't reinterpret +1. **Treat plan steps as directives** — execute them, don't reinterpret 2. **Signal completion** after each step to receive the next 3. **Continue until all steps are finished** or a replan is needed #### Mandatory Replanning Triggers -**You MUST request a replan when:** +**Request a replan when:** - Execution reveals the plan's assumptions were wrong - Required files/folders don't exist as expected - User provides new information that changes the goal @@ -94,47 +59,24 @@ Provide the planning agent with: --- -### 3. COMPLEXITY SIGNAL REFERENCE - -Use this reference when the gate decision isn't obvious. **Two or more signals = plan.** - -| Signal | Indicators | -|--------|-----------| -| **Ambiguity** | Multiple valid interpretations; user hasn't specified preferences | -| **Exploration required** | Must search/read before knowing the right approach | -| **Dependency chains** | Later actions depend on earlier outcomes | -| **State changes** | Early actions affect what later actions should do | -| **Outcome uncertainty** | Success criteria are subjective or emergent | - -**Linguistic triggers to watch for:** -- Possessive scope: "all my", "my whole vault", "everything" -- Synthesis verbs: "analyze", "compare", "evaluate", "understand" -- Quality language: "improve", "optimize", "clean up", "fix" -- Uncertainty: "somewhere in my vault", "I think I have" - ---- - -### 4. HISTORICAL CONTEXT INTERPRETATION +### 3. HISTORICAL CONTEXT INTERPRETATION **Tool call history from previous sessions may appear with HTML comment markers.** -These represent completed actions—NOT patterns to reproduce: +These represent completed actions — NOT patterns to reproduce: - ✅ Treat as contextual information about what was previously done - ✅ Use native function calling for any NEW tool operations -- ❌ Do NOT output text mimicking this format (e.g., "...") +- ❌ Do NOT output text mimicking this format - ❌ Do NOT treat historical formats as syntax to follow If you see JSON preceded by "Historical tool call/result", it documents a past action. Use your native function calling for new operations. -### 5. Request Completion -- Execute ALL necessary operations before concluding your turn -- Ensure the user's complete request is fulfilled, not just the first step -- For multi-step tasks, gather all information before presenting findings -- When executing a plan, complete all steps or explicitly pause at decision points +--- -### 6. Wiki-Link Everything from the Vault +### 4. WIKI-LINK EVERYTHING FROM THE VAULT **ALWAYS use [[wiki-link]] notation when referencing any information from the user's notes.** + - Every mention of a note, concept, person, or topic from the vault must be linked - This builds the knowledge graph and helps users navigate their information - Use the exact note name as it appears in the vault @@ -144,7 +86,9 @@ Examples: - "[[Sarah]] mentioned this in her meeting with [[John]]" - "This relates to your ideas about [[Machine Learning]] in [[Research Notes]]" -### 7. Vault-First Decision Framework +--- + +### 5. VAULT-FIRST DECISION FRAMEWORK **The cost of an unnecessary search is negligible. Missing relevant information is costly.** @@ -166,6 +110,8 @@ Examples: Acknowledge the search, then provide general assistance: "I searched your vault but didn't find notes about [topic]. Here's what I can tell you: [general information]. Would you like me to create a note about this?" +--- + ## Search Strategy ### Regex Pattern Matching @@ -206,9 +152,11 @@ When searches return or reference images or PDFs: - Extract relevant information to answer the user's query - Reference the source file with [[wiki-links]] as usual +--- + ## Multi-Tool Workflow Architecture -### Direct Execution (Simple & Moderate Tasks) +### Direct Execution (Default Mode) For straightforward operations: 1. **Intent Analysis**: Understand what the user needs @@ -216,9 +164,9 @@ For straightforward operations: 3. **Progressive Fallback**: If initial approach fails, try alternatives 4. **Complete Delivery**: Present findings with proper wiki-links -### Planned Execution (Complex & Research Tasks) +### Planned Execution (When Planning Mode is Enabled) -For tasks requiring coordination: +For tasks where the user has enabled planning: 1. **Request Planning**: Provide goal, context, and any known constraints 2. **Receive Plan**: Get structured steps with dependencies and success criteria 3. **Execute Sequentially**: Work through steps, gathering ground truth @@ -233,6 +181,8 @@ After multi-step execution: 3. **Universal Wiki-Linking**: Apply [[wiki-links]] to ALL vault references 4. **Gap Identification**: Note missing connections or suggest new notes +--- + ## Core Capabilities **Knowledge Operations** @@ -252,18 +202,12 @@ After multi-step execution: - Problem-solving and explanations across any domain - Programming, writing, and creative tasks with vault context +--- + ## Anti-Patterns to Avoid -❌ Equating "few steps" with "simple task" -❌ Executing without assessing underlying user intent -❌ Treating all single-item requests as trivially simple -❌ Ignoring ambiguity signals in user language -❌ Committing to an approach before understanding vault structure -❌ Assuming you know what the user wants without checking -❌ Planning for truly atomic operations (over-engineering) -❌ Refusing to plan because "it's just one thing" when that thing is complex ❌ Referencing vault content without [[wiki-links]] -❌ Giving up after first failed search—always use progressive strategies +❌ Giving up after first failed search — always use progressive strategies ❌ Searching literal phrases instead of extracting key entities ❌ Asking permission when user intent is clear ("Would you like me to...") ❌ Describing what you'd create instead of creating it @@ -275,23 +219,23 @@ After multi-step execution: ❌ Blindly following a plan when execution reveals it's no longer valid ❌ Replanning for minor issues you can handle directly +--- + ## Decision Framework Summary **Always ask yourself:** 1. "What is the user actually trying to accomplish?" → Look beyond the literal request -2. "Are there complexity signals in this request?" → Check for ambiguity, exploration needs, dependencies -3. "Is this simple enough to execute directly, or do I need strategic planning?" → Default to direct execution, but recognize when planning helps -4. "Have I completed the user's full request?" → Reflect on what can still be achieved -5. "Am I using [[wiki-links]] for every vault reference?" → Always required -6. "Could this information exist in the user's notes?" → Search vault first -7. "Did my search fail? Have I tried all progressive tiers?" → Keep searching -8. "Can I infer the answer from related content I found?" → Read and reason -9. "Has something changed that invalidates my current plan?" → Consider replanning -10. "Am I adapting or do I need strategic guidance?" → Replan only for significant pivots +2. "Have I completed the user's full request?" → Ensure all operations are done before concluding +3. "Am I using [[wiki-links]] for every vault reference?" → Always required +4. "Could this information exist in the user's notes?" → Search vault first +5. "Did my search fail? Have I tried all progressive tiers?" → Keep searching +6. "Can I infer the answer from related content I found?" → Read and reason +7. "Has something changed that invalidates my current plan?" → Consider replanning +8. "Am I adapting or do I need strategic guidance?" → Replan only for significant pivots -**When uncertain**: Always search the vault first. Always try alternative strategies before concluding "not found." Scale complexity assessment to match the query. Complete the full request before concluding. +**When uncertain**: Always search the vault first. Always try alternative strategies before concluding "not found." Complete the full request before concluding. --- -**Core Philosophy**: Gate first, then act decisively. Every request passes through the complexity gate before execution. Planning is not overhead—it's the correct action for structural, ambiguous, or unbounded tasks. For bounded, unambiguous requests, execute immediately without hesitation. Always use [[wiki-links]] for vault references. Search the vault proactively with progressive strategies—never accept a single failed search as final. When executing plans, stay adaptive: replan when reality diverges from assumptions, but handle minor adjustments yourself. +**Core Philosophy**: Act decisively on user requests. Always use [[wiki-links]] for vault references. Search the vault proactively with progressive strategies — never accept a single failed search as final. When executing plans, stay adaptive: replan when reality diverges from assumptions, but handle minor adjustments yourself. `; \ No newline at end of file diff --git a/Components/ChatInput.svelte b/Components/ChatInput.svelte index adc4138..5515d1a 100644 --- a/Components/ChatInput.svelte +++ b/Components/ChatInput.svelte @@ -16,14 +16,17 @@ import type { DiffService } from "Services/DiffService"; import type { Attachment } from "Conversations/Attachment"; import ChatAttachments from "./ChatAttachments.svelte"; + import InputDisplay from "./InputDisplay.svelte"; + import { InputMode } from "Enums/InputMode"; + import { Copy } from "Enums/Copy"; export let attachments: Attachment[] = []; export let hasNoApiKey: boolean; export let isSubmitting: boolean; export let editModeActive: boolean; + export let planningModeActive: boolean; export let onSubmit: (userRequest: string, formattedRequest: string) => void; - export let onTogglEeditMode: () => void; export let onStop: () => void; const inputService: InputService = Resolve(Services.InputService); @@ -34,19 +37,22 @@ const searchState: Writable = searchStateStore.searchState; + let inputDisplay: InputDisplay; let textareaElement: HTMLDivElement; let userInstructionButton: HTMLButtonElement; let submitButton: HTMLButtonElement; let editModeButton: HTMLButtonElement; + let planningModeButton: HTMLButtonElement; let userInstructionActive: boolean = false; let userRequest: string = ""; - let diffOpen: boolean = false; - - const diffOpenedRef: EventRef = eventService.on(Event.DiffOpened, () => { diffOpen = true; focusInput(); }); - const diffClosedRef: EventRef = eventService.on(Event.DiffClosed, () => { diffOpen = false; focusInput(); }); + let inputMode: InputMode = InputMode.Normal; + let questionResolver: ((answer: string) => void) | null = null; + + const diffOpenedRef: EventRef = eventService.on(Event.DiffOpened, () => { inputMode = InputMode.Diff; focusInput(); }); + const diffClosedRef: EventRef = eventService.on(Event.DiffClosed, () => { inputMode = InputMode.Normal; focusInput(); }); onDestroy(() => { eventService.offref(diffOpenedRef); @@ -62,12 +68,27 @@ } } + export function setDisplayItem(element: HTMLElement) { + inputDisplay.setDisplayItem(element); + } + + export function clearDisplayItem() { + inputDisplay.clearDisplayItem(); + inputMode = InputMode.Normal; + } + + export function enterQuestionMode(resolver: (answer: string) => void) { + questionResolver = resolver; + inputMode = InputMode.Question; + focusInput(); + } + $: if (userInstructionButton) { setIcon(userInstructionButton, "user-round-pen"); } $: if (submitButton) { - if (diffOpen) { + if (inputMode === InputMode.Question || inputMode === InputMode.Diff) { setIcon(submitButton, userRequest.trim() === "" ? "square" : "send-horizontal"); } else { setIcon(submitButton, isSubmitting ? "square" : "send-horizontal"); @@ -78,6 +99,33 @@ setIcon(editModeButton, editModeActive ? "pencil" : "pencil-off"); } + $: if (planningModeButton) { + setIcon(planningModeButton, planningModeActive ? "list-ordered" : "list-x"); + } + + $: inputPlaceholder = (() => { + if (inputMode === InputMode.Question) return Copy.InputPlaceholderQuestion; + if (inputMode === InputMode.Diff) return Copy.InputPlaceholderDiff; + return Copy.InputPlaceholderNormal; + })(); + + $: submitDisabled = (() => { + if (inputMode === InputMode.Diff || inputMode === InputMode.Question) { + return false; + } + return !isSubmitting && userRequest.trim() === ""; + })(); + + $: submitAriaLabel = (() => { + if (inputMode === InputMode.Question) { + return userRequest.trim() === "" ? Copy.ButtonCancel : Copy.ButtonSubmitAnswer; + } + if (inputMode === InputMode.Diff) { + return userRequest.trim() === "" ? Copy.ButtonCancel : Copy.ButtonMakeSuggestion; + } + return isSubmitting ? Copy.ButtonCancel : Copy.ButtonSendMessage; + })(); + function handleStop() { onStop(); } @@ -91,13 +139,27 @@ } function handleSuggestion() { - if (userRequest.trim() === "" || !diffOpen) { + if (userRequest.trim() === "" || inputMode !== InputMode.Diff) { return; } const suggestion = requestFromInput(); diffService.onSuggest(suggestion.formattedRequest); } + function handleAnswer() { + if (userRequest.trim() === "" || inputMode !== InputMode.Question) { + return; + } + const answer = requestFromInput(); + + if (questionResolver) { + questionResolver(answer.formattedRequest); + questionResolver = null; + } + + clearDisplayItem(); + } + function requestFromInput() { const request = textareaElement.innerHTML; const formattedRequest = triggerToText(request); @@ -115,7 +177,23 @@ } function toggleEditMode() { - onTogglEeditMode(); + if (planningModeActive) { + planningModeActive = false + } + editModeActive = !editModeActive; + focusInput(); + } + + function togglePlanningMode() { + if (planningModeActive) { + planningModeActive = false; + } else { + if (!editModeActive) { + toggleEditMode(); // Mandatory for planning mode + } + planningModeActive = true; + } + focusInput(); } async function handleKeydown(e: KeyboardEvent) { @@ -130,7 +208,13 @@ return; } e.preventDefault(); - diffOpen ? handleSuggestion() : handleSubmit(); + if (inputMode === InputMode.Question) { + handleAnswer(); + } else if (inputMode === InputMode.Diff) { + handleSuggestion(); + } else { + handleSubmit(); + } } } @@ -302,12 +386,16 @@
+
0 ? "var(--size-4-2)" : 0}> + +
+
0 ? "var(--size-4-2)" : 0}>
-
- +
+
0 ? "var(--size-4-2)" : 0}> @@ -344,7 +432,7 @@ on:click={handleCursorPositionChange} on:keyup={handleCursorPositionChange} on:focusout={handleFocusOut} - data-placeholder={diffOpen ? "Make a suggestion..." : "Type a message..."} + data-placeholder={inputPlaceholder} role="textbox" aria-multiline="true" tabindex="0"> @@ -356,7 +444,16 @@ bind:this={editModeButton} on:click={() => { toggleEditMode() }} disabled={isSubmitting} - aria-label={editModeActive ? "Turn off Agent Mode" : "Turn on Agent Mode"}> + aria-label={editModeActive ? Copy.ButtonTurnOffAgentMode : Copy.ButtonTurnOnAgentMode}> + + +
@@ -380,8 +479,8 @@ grid-row: 3; grid-column: 1; display: grid; - grid-template-rows: auto auto auto var(--size-4-3) 1fr var(--size-4-3); - grid-template-columns: var(--size-4-3) auto var(--size-4-2) 1fr var(--size-4-2) auto var(--size-4-2) auto var(--size-4-3); + grid-template-rows: auto auto auto auto var(--size-4-3) 1fr var(--size-4-3); + grid-template-columns: var(--size-4-3) auto var(--size-4-2) 1fr var(--size-4-2) auto var(--size-4-2) auto var(--size-4-2) auto var(--size-4-3); border-radius: var(--modal-radius); background-color: var(--background-primary); } @@ -391,28 +490,33 @@ transition: border-color 0.5s ease-out; } - #input-attachments-container { + #input-display-container { grid-row: 1; - grid-column: 2 / 9; + grid-column: 2 / 11; + } + + #input-attachments-container { + grid-row: 2; + grid-column: 2 / 11; } #diff-controls-container { - grid-row: 2; - grid-column: 2 / 9; + grid-row: 3; + grid-column: 2 / 11; } #input-search-results-container { - grid-row: 3; - grid-column: 2 / 9; + grid-row: 4; + grid-column: 2 / 11; } #user-instruction-container { - grid-row: 3; - grid-column: 2 / 9; + grid-row: 4; + grid-column: 2 / 11; } #user-instruction-button { - grid-row: 5; + grid-row: 6; grid-column: 2; border-radius: var(--button-radius); align-self: end; @@ -428,7 +532,7 @@ } #input-field { - grid-row: 5; + grid-row: 6; grid-column: 4; height: 100%; max-height: 30vh; @@ -489,21 +593,41 @@ } #edit-mode-button { - grid-row: 5; + grid-row: 6; grid-column: 6; border-radius: var(--button-radius); align-self: end; transition-duration: 0.5s; } + #edit-mode-button.edit-mode { + box-shadow: inset 0px 0px 1px 1px var(--alt-interactive-accent); + } + :global(.is-mobile) #edit-mode-button { max-height: 2rem; } - #submit-button { - grid-row: 5; + #planning-mode-button { + grid-row: 6; grid-column: 8; border-radius: var(--button-radius); + align-self: end; + transition-duration: 0.5s; + } + + #planning-mode-button.planning-mode { + box-shadow: inset 0px 0px 1px 1px var(--alt-interactive-accent); + } + + :global(.is-mobile) #planning-mode-button { + max-height: 2rem; + } + + #submit-button { + grid-row: 6; + grid-column: 10; + border-radius: var(--button-radius); padding-left: var(--size-4-5); padding-right: var(--size-4-5); align-self: end; diff --git a/Components/ChatPlanArea.svelte b/Components/ChatPlanArea.svelte index cbc1285..9f3c765 100644 --- a/Components/ChatPlanArea.svelte +++ b/Components/ChatPlanArea.svelte @@ -215,7 +215,7 @@ position: absolute; left: 0; right: 0; - height: var(--size-4-2); + height: var(--size-4-1); border-radius: var(--radius-m); pointer-events: none; z-index: 1; diff --git a/Components/ChatWindow.svelte b/Components/ChatWindow.svelte index 7c539a8..9b02eb3 100644 --- a/Components/ChatWindow.svelte +++ b/Components/ChatWindow.svelte @@ -35,6 +35,7 @@ let isSubmitting = false; let busyPlanning = false; let editModeActive = false; + let planningModeActive = false; let currentStreamingMessageId: string | null = null; let conversation: Conversation = new Conversation(); @@ -85,11 +86,6 @@ return hasNoApiKey; } - function toggleEditMode() { - editModeActive = !editModeActive; - focusInput(); - } - function handleStop() { chatService.stop(); currentThought = null; @@ -102,7 +98,7 @@ const currentRequest = userRequest; - await chatService.submit(conversation, editModeActive, currentRequest, formattedRequest, attachments, { + await chatService.submit(conversation, editModeActive, planningModeActive, currentRequest, formattedRequest, attachments, { onSubmit: () => { isSubmitting = true; attachments = []; @@ -126,6 +122,12 @@ onPlanningFinished: () => { busyPlanning = false; }, + onPlanningQuestion: async (question) => { + chatInput.setDisplayItem(createEl("span", { text: question })); + return new Promise((resolve) => { + chatInput.enterQuestionMode(resolve); + }); + }, onPlanUpdate: (executionPlan) => { executionPlanStore.setPlan(executionPlan); }, @@ -140,6 +142,7 @@ busyPlanning = false; currentThought = null; executionPlanStore.clearPlan(); + chatInput.clearDisplayItem(); abortService.reset(); tick().then(() => { chatArea.updateChatAreaLayout("smooth", true); @@ -150,6 +153,7 @@ $: if ($conversationStore.shouldReset) { conversation = new Conversation(); + conversationService.resetCurrentConversation(); isSubmitting = false; currentStreamingMessageId = null; @@ -195,12 +199,12 @@ diff --git a/Components/InputDisplay.svelte b/Components/InputDisplay.svelte new file mode 100644 index 0000000..fac66a7 --- /dev/null +++ b/Components/InputDisplay.svelte @@ -0,0 +1,64 @@ + + +{#if displayItem} +
+
+
+
+{/if} + + diff --git a/Enums/AIFunction.ts b/Enums/AIFunction.ts index 3caa970..ed04c84 100644 --- a/Enums/AIFunction.ts +++ b/Enums/AIFunction.ts @@ -17,7 +17,8 @@ export enum AIFunction { Replan = "replan", SubmitPlan = "submit_plan", CompleteStep = "complete_step", - CancelPlan = "cancel_plan" + CancelPlan = "cancel_plan", + AskUserQuestion = "ask_user_question" } export function fromString(functionName: string): AIFunction { diff --git a/Enums/ApiProvider.ts b/Enums/ApiProvider.ts index 0717918..5125648 100644 --- a/Enums/ApiProvider.ts +++ b/Enums/ApiProvider.ts @@ -74,7 +74,7 @@ export enum AIProviderModel { // Conversation naming models (aliases to existing models) ClaudeNamer = ClaudeHaiku_4_5, GeminiNamer = GeminiFlash_2_5_Lite, - OpenAINamer = GPT_4o_Mini, + OpenAINamer = GPT_5_Nano, } export enum AIProviderURL { diff --git a/Enums/Copy.ts b/Enums/Copy.ts index ab4f189..e07f3b2 100644 --- a/Enums/Copy.ts +++ b/Enums/Copy.ts @@ -68,19 +68,35 @@ export enum Copy { SafeContinue= "Continue", + // Chat Input Placeholders + InputPlaceholderQuestion = "Provide an answer...", + InputPlaceholderDiff = "Make a suggestion...", + InputPlaceholderNormal = "Type a message...", + + // Chat Input Button Labels + ButtonCancel = "Cancel", + ButtonSubmitAnswer = "Submit answer", + ButtonMakeSuggestion = "Make Suggestion", + ButtonSendMessage = "Send Message", + ButtonTurnOffAgentMode = "Turn off Agent Mode", + ButtonTurnOnAgentMode = "Turn on Agent Mode", + ButtonTurnOffPlanningMode = "Turn off Planning Mode", + ButtonTurnOnPlanningMode = "Turn on Planning Mode", + // Execution Plan Messages PlanningFailedError = `Failed to generate plan. You should attempt to recover from this. ### Next Actions - Create your own simplified plan - Follow your own simplified plan`, StepDoesNotExistError = "Step {stepNumber} does not exist in the execution plan. Valid step numbers are 1-{totalSteps}.", - StepMustBeCompletedInOrderError = "Cannot complete step {stepNumber}. Step \"{incompletStep}\" is not yet completed. Steps must be completed in order.", + StepMustBeCompletedInOrderError = "Cannot complete step {stepNumber}. The following steps have not yet been marked as completed: {incompleteSteps}. Steps must be completed in order.", StepCompletedWithNextStep = "Step {stepNumber} completed successfully. Now proceed with step {nextStepNumber}.", AllStepsCompleted = "Step {stepNumber} completed successfully. All steps in the execution plan have been completed. Provide a final summary to the user based on the completed steps and overall success criteria.", MaxPlanningIterationsReached = "I've attempted multiple planning iterations but encountered persistent issues completing the task. It may need to be broken down further or requires additional clarification.", PlanExecutionCancelled = "Plan execution cancelled. Provide a summary to the user explaining what happened and any partial progress made.", PlanExecutionNotCancelled = "Confirmation was false, no action taken.", PlanningToolDenial = "Invalid tool call - this is an execution tool and cannot be called during the planning phase.", + PlanningModeError = "You must first create a plan before executing any functions!", // Execution Plan Request Templates ContextTags = ` diff --git a/Enums/InputMode.ts b/Enums/InputMode.ts new file mode 100644 index 0000000..656d703 --- /dev/null +++ b/Enums/InputMode.ts @@ -0,0 +1,5 @@ +export enum InputMode { + Normal = "normal", + Diff = "diff", + Question = "question" +} diff --git a/Services/AIControllerService.ts b/Services/AIControllerService.ts index 850eec5..0b25ab1 100644 --- a/Services/AIControllerService.ts +++ b/Services/AIControllerService.ts @@ -14,7 +14,7 @@ import { AIFunctionDefinitions } from "AIClasses/FunctionDefinitions/AIFunctionD import { AIFunction, isAIFunction } from "Enums/AIFunction"; import { AIFunctionResponse } from "AIClasses/FunctionDefinitions/AIFunctionResponse"; import { Exception } from "Helpers/Exception"; -import { CancelPlanArgsSchema, CompleteStepArgsSchema, CreatePlanArgsSchema, ReplanArgsSchema, SubmitPlanArgsSchema, type CreatePlanArgs, type ReplanArgs } from "AIClasses/Schemas/AIFunctionSchemas"; +import { AskUserQuestionArgsSchema, CancelPlanArgsSchema, CompleteStepArgsSchema, CreatePlanArgsSchema, ReplanArgsSchema, SubmitPlanArgsSchema, type CreatePlanArgs, type ReplanArgs } from "AIClasses/Schemas/AIFunctionSchemas"; import { ExecutionPlan } from "Types/ExecutionPlan"; export class AIControllerService { @@ -41,20 +41,22 @@ export class AIControllerService { this.onSaveConversation = callback; } - public async runMainAgent(conversation: Conversation, allowDestructiveActions: boolean, callbacks: IChatServiceCallbacks) { + public async runMainAgent(conversation: Conversation, allowDestructiveActions: boolean, planningMode: boolean, callbacks: IChatServiceCallbacks) { if (!this.ai) { // this shouldn't ever happen Exception.throw("Error: No AI provider has been set!"); } // Setup initial prompts & tools - this.ai.systemPrompt = this.aiPrompt.systemInstruction(); + this.ai.systemPrompt = this.aiPrompt.systemInstruction(planningMode); this.ai.userInstruction = await this.aiPrompt.userInstruction(); - this.ai.toolDefinitions = AIFunctionDefinitions.agentDefinitions(allowDestructiveActions); + this.ai.toolDefinitions = AIFunctionDefinitions.agentDefinitions(allowDestructiveActions, planningMode); + let planRequested = false; await this.runAgentLoop(conversation, callbacks, async (functionCall) => { const functionCallName = functionCall.name; if (isAIFunction(functionCallName, AIFunction.CreatePlan)) { try { + planRequested = true; const completedSuccessfully = await this.handlePlanningWorkflow(conversation, functionCall, callbacks); return { shouldExit: completedSuccessfully }; } finally { @@ -62,6 +64,15 @@ export class AIControllerService { } } + if (planningMode && !planRequested) { + conversation.addFunctionResponse(new AIFunctionResponse( + functionCallName, + { error: Copy.PlanningModeError }, + functionCall.toolId + )); + return { shouldExit: false }; + } + this.updateThought(functionCall, callbacks); const functionResponse = await this.aiFunctionService.performAIFunction(functionCall); conversation.addFunctionResponse(functionResponse); @@ -141,10 +152,11 @@ export class AIControllerService { // Handle max iterations reached if (planningIteration >= AIControllerService.MAX_PLANNING_ITERATIONS && continueExecution) { - conversation.contents.push(new ConversationContent({ + const conversationContent = new ConversationContent({ role: Role.Assistant, content: Copy.MaxPlanningIterationsReached - })); + }); + conversation.contents.push(conversationContent); } // If plan was cancelled, return false to give control back to main agent for summary @@ -167,6 +179,26 @@ export class AIControllerService { return { shouldExit: false }; } + if (isAIFunction(functionCallName, AIFunction.AskUserQuestion)) { + const parseResult = AskUserQuestionArgsSchema.safeParse(functionCall.arguments); + if (!parseResult.success) { + planningConversation.addFunctionResponse(new AIFunctionResponse( + functionCallName, + { error: `Invalid arguments for ${AIFunction.AskUserQuestion}: ${parseResult.error.message}` }, + functionCall.toolId + )); + return { shouldExit: false }; + } + this.updateThought(functionCall, callbacks); + const answer = await callbacks.onPlanningQuestion(parseResult.data.question); + planningConversation.addFunctionResponse(new AIFunctionResponse( + functionCallName, + { answer: answer }, + functionCall.toolId + )); + return { shouldExit: false }; + } + if (isAIFunction(functionCallName, AIFunction.SubmitPlan)) { const parseResult = SubmitPlanArgsSchema.safeParse(functionCall.arguments); if (!parseResult.success) { @@ -393,6 +425,7 @@ export class AIControllerService { if (capturedFunctionCall) { conversationContent.functionCall = capturedFunctionCall.toConversationString(); conversationContent.toolId = capturedFunctionCall.toolId; + //conversationContent.shouldDisplayContent = false; if (capturedFunctionCall.thoughtSignature) { conversationContent.thoughtSignature = capturedFunctionCall.thoughtSignature; } @@ -402,6 +435,8 @@ export class AIControllerService { if (conversationContent.content?.trim() !== "") { callbacks.onStreamingUpdate(conversationContent.timestamp.getTime().toString()); + } else { + conversationContent.shouldDisplayContent = false; } } diff --git a/Services/AIFunctionService.ts b/Services/AIFunctionService.ts index 6300797..3e9646c 100644 --- a/Services/AIFunctionService.ts +++ b/Services/AIFunctionService.ts @@ -124,6 +124,7 @@ export class AIFunctionService { case AIFunction.CreatePlan: case AIFunction.Replan: case AIFunction.SubmitPlan: + case AIFunction.AskUserQuestion: case AIFunction.CompleteStep: case AIFunction.CancelPlan: { Exception.log(`Multi-agent function ${functionCall.name} should not be handled by AIFunctionService`); diff --git a/Services/ChatService.ts b/Services/ChatService.ts index 00456c3..4797bd2 100644 --- a/Services/ChatService.ts +++ b/Services/ChatService.ts @@ -23,6 +23,7 @@ export interface IChatServiceCallbacks { onThoughtUpdate: (thought: string | null) => void; onPlanningStarted: () => void; onPlanningFinished: () => void; + onPlanningQuestion: (question: string) => Promise; onPlanUpdate: (executionPlan: ExecutionPlan) => void; onPlanStepUpdate: () => void; onPlanComplete: () => void; @@ -57,7 +58,7 @@ export class ChatService { public onNameChanged: ((name: string) => void) | undefined = undefined; - public async submit(conversation: Conversation, allowDestructiveActions: boolean, userRequest: string, formattedRequest: string, attachments: Attachment[], callbacks: IChatServiceCallbacks) { + public async submit(conversation: Conversation, allowDestructiveActions: boolean, planningMode: boolean, userRequest: string, formattedRequest: string, attachments: Attachment[], callbacks: IChatServiceCallbacks) { if (!await this.semaphore.wait()) { return; } @@ -82,6 +83,7 @@ export class ChatService { conversation.contents.push(conversationContent); await this.saveConversation(conversation); + callbacks.onStreamingUpdate(conversationContent.timestamp.getTime().toString()); if (firstMessage) { this.onNameChanged?.(conversation.title); // on change for initial conversation name @@ -104,7 +106,7 @@ export class ChatService { callbacks.onSubmit(); callbacks.onStreamingUpdate(null); - await this.aiControllerService.runMainAgent(conversation, allowDestructiveActions, callbacks); + await this.aiControllerService.runMainAgent(conversation, allowDestructiveActions, planningMode, callbacks); }); } catch (error) { if (!AbortService.isAbortError(error)) { diff --git a/Types/ExecutionPlan.ts b/Types/ExecutionPlan.ts index e4f079f..0f0c92c 100644 --- a/Types/ExecutionPlan.ts +++ b/Types/ExecutionPlan.ts @@ -59,7 +59,7 @@ export class ExecutionPlan { return { error: replaceCopy(Copy.StepMustBeCompletedInOrderError, [ stepNumber.toString(), - `${i + 1}. ${this.executionSteps[i].description}` + this.incompleteSteps().join(",") ]) }; } @@ -108,4 +108,14 @@ export class ExecutionPlan { }; } + private incompleteSteps(): number[] { + const incompleteSteps = []; + for (const step of this.executionSteps) { + if (step.status !== ExecutionStatus.Completed) { + incompleteSteps.push(step.step); + } + } + return incompleteSteps; + } + } \ No newline at end of file diff --git a/__tests__/AIClasses/CrossProviderIntegration.test.ts b/__tests__/AIClasses/CrossProviderIntegration.test.ts index 75c2563..683c0cb 100644 --- a/__tests__/AIClasses/CrossProviderIntegration.test.ts +++ b/__tests__/AIClasses/CrossProviderIntegration.test.ts @@ -100,6 +100,7 @@ describe('Cross-Provider Integration - Thought Signature Support', () => { displayContent: '', functionCall: JSON.stringify({ functionCall: { + id: 'call_abc123', // AIFunctionCall.toConversationString() includes id in JSON name: 'search_vault_files', args: { query: 'meeting notes' } } @@ -127,6 +128,7 @@ describe('Cross-Provider Integration - Thought Signature Support', () => { displayContent: '', functionCall: JSON.stringify({ functionCall: { + id: 'call_xyz789', // AIFunctionCall.toConversationString() includes id in JSON name: 'read_file', args: { path: 'project.md' } } @@ -229,6 +231,7 @@ describe('Cross-Provider Integration - Thought Signature Support', () => { displayContent: '', functionCall: JSON.stringify({ functionCall: { + id: 'call_claude_123', // AIFunctionCall.toConversationString() includes id name: 'search_vault_files', args: { query: 'project' } } @@ -337,6 +340,7 @@ describe('Cross-Provider Integration - Thought Signature Support', () => { displayContent: '', functionCall: JSON.stringify({ functionCall: { + id: 'call_openai_456', // AIFunctionCall.toConversationString() includes id name: 'list_files', args: {} } @@ -389,7 +393,11 @@ describe('Cross-Provider Integration - Thought Signature Support', () => { content: '', displayContent: '', functionCall: JSON.stringify({ - functionCall: { name: 'func1', args: { a: 1 } } + functionCall: { + id: 'call-1', // AIFunctionCall.toConversationString() includes id + name: 'func1', + args: { a: 1 } + } }), timestamp: new Date(), shouldDisplayContent: true, @@ -1512,13 +1520,13 @@ describe('Cross-Provider Integration - Thought Signature Support', () => { }); describe('Edge Cases', () => { - it('should handle empty thoughtSignature as missing signature (legacy format)', async () => { + it('should handle empty thoughtSignature as missing signature (use native format)', async () => { const content = new ConversationContent({ role: Role.Assistant, content: '', displayContent: '', functionCall: JSON.stringify({ - functionCall: { name: 'test_func', args: {} } + functionCall: { name: 'test_func', args: {} } // No id = native Gemini }), timestamp: new Date(), shouldDisplayContent: true, @@ -1527,18 +1535,19 @@ describe('Cross-Provider Integration - Thought Signature Support', () => { const result = await (gemini as any).extractContents([content]); - expect(result[0].parts[0]).toHaveProperty('text'); - expect(result[0].parts[0].text).toContain('