diff --git a/CLAUDE.md b/CLAUDE.md index 1ea34af2..146938cc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -192,6 +192,15 @@ For detailed architecture diagrams and documentation, see [`MESSAGE_ARCHITECTURE - `logError()` for errors - Import from logger: `import { logInfo, logWarn, logError } from "@/logger"` +### CSS & Styling + +- **NEVER edit `styles.css` directly** - This is a generated file +- **Source file**: `src/styles/tailwind.css` - Edit this file for custom CSS +- **Build process**: `npm run build:tailwind` compiles `src/styles/tailwind.css` → `styles.css` +- **Tailwind classes**: Use Tailwind utility classes in components (see `tailwind.config.js` for available classes) +- **Custom CSS**: Add custom styles to `src/styles/tailwind.css` after the `@import` statements +- After editing CSS, always run `npm run build` to regenerate `styles.css` + ## Testing Guidelines - Unit tests use Jest with TypeScript support diff --git a/src/LLMProviders/ChatOpenRouter.ts b/src/LLMProviders/ChatOpenRouter.ts new file mode 100644 index 00000000..dfad8b6e --- /dev/null +++ b/src/LLMProviders/ChatOpenRouter.ts @@ -0,0 +1,485 @@ +import { BaseChatModelParams } from "@langchain/core/language_models/chat_models"; +import { AIMessageChunk } from "@langchain/core/messages"; +import type { UsageMetadata } from "@langchain/core/messages"; +import { ChatGenerationChunk } from "@langchain/core/outputs"; +import { ChatOpenAI } from "@langchain/openai"; +import OpenAI from "openai"; +import { logInfo } from "@/logger"; + +type OpenRouterChatChunk = OpenAI.ChatCompletionChunk; +type OpenRouterUsage = NonNullable; +type OpenRouterMessageParam = OpenAI.ChatCompletionMessageParam; + +/** + * ChatOpenRouter extends ChatOpenAI to support OpenRouter-specific features, + * particularly reasoning/thinking tokens. + * + * OpenRouter exposes thinking tokens via the `reasoning` request parameter + * and responds with `reasoning_details` in both streaming and non-streaming modes. + * + * @see https://openrouter.ai/docs/use-cases/reasoning-tokens + */ +export interface ChatOpenRouterInput extends BaseChatModelParams { + /** + * Enable reasoning/thinking tokens from OpenRouter + * When true, requests will include reasoning parameters + */ + enableReasoning?: boolean; + + /** + * Reasoning effort level: "minimal", "low", "medium", or "high" + * Controls the amount of reasoning the model uses + * Note: "minimal" will be treated as "low" for OpenRouter + */ + reasoningEffort?: "minimal" | "low" | "medium" | "high"; + + // All other ChatOpenAI parameters + modelName?: string; + apiKey?: string; + configuration?: any; + temperature?: number; + maxTokens?: number; + topP?: number; + frequencyPenalty?: number; + streaming?: boolean; + maxRetries?: number; + maxConcurrency?: number; + [key: string]: any; +} + +export class ChatOpenRouter extends ChatOpenAI { + private enableReasoning: boolean; + private reasoningEffort?: "minimal" | "low" | "medium" | "high"; + private openaiClient: OpenAI; + + constructor(fields: ChatOpenRouterInput) { + const { enableReasoning = false, reasoningEffort, ...rest } = fields; + + // Pass all other parameters to ChatOpenAI + super(rest); + + this.enableReasoning = enableReasoning; + this.reasoningEffort = reasoningEffort; + + // Create our own OpenAI client for raw access + this.openaiClient = new OpenAI({ + apiKey: fields.apiKey, + baseURL: fields.configuration?.baseURL || "https://openrouter.ai/api/v1", + defaultHeaders: fields.configuration?.defaultHeaders, + fetch: fields.configuration?.fetch, + dangerouslyAllowBrowser: true, + }); + } + + /** + * Override the invocation parameters to include reasoning when enabled + */ + override invocationParams(options?: this["ParsedCallOptions"]): any { + const baseParams = super.invocationParams(options); + + // Add reasoning parameter if enabled + if (this.enableReasoning) { + // Per OpenRouter docs: + // - For Anthropic models: MUST use reasoning.max_tokens or reasoning.effort + // - For other models: Can use reasoning.enabled + // - max_tokens must be strictly higher than reasoning budget + + // Prefer effort if provided, otherwise fall back to max_tokens + if (this.reasoningEffort) { + // Map "minimal" to "low" since OpenRouter doesn't support "minimal" + const effort = this.reasoningEffort === "minimal" ? "low" : this.reasoningEffort; + logInfo(`OpenRouter reasoning enabled with effort: ${effort}`); + return { + ...baseParams, + reasoning: { + effort, + }, + }; + } else { + logInfo(`OpenRouter reasoning enabled with max_tokens: 1024`); + return { + ...baseParams, + reasoning: { + max_tokens: 1024, + }, + }; + } + } + + return baseParams; + } + + /** + * Override to use raw OpenAI SDK to access reasoning_details + * LangChain filters out reasoning_details, so we bypass it completely + */ + override async *_streamResponseChunks( + messages: any[], + options: this["ParsedCallOptions"], + _runManager?: any + ): AsyncGenerator { + const params = this.invocationParams(options); + const openaiMessages = this.toOpenRouterMessages(messages); + + const stream = (await this.openaiClient.chat.completions.create({ + ...params, + messages: openaiMessages, + stream: true, + stream_options: { + ...(params.stream_options ?? {}), + include_usage: true, + }, + })) as unknown as AsyncIterable; + + let usageSummary: OpenRouterUsage | undefined; + + for await (const rawChunk of stream as AsyncIterable) { + if (rawChunk.usage) { + usageSummary = rawChunk.usage; + } + + const choice = rawChunk.choices?.[0]; + const delta = choice?.delta; + if (!choice || !delta) { + continue; + } + + const reasoningText = this.normalizeReasoningChunk( + (delta as Record)?.reasoning + ); + const reasoningDetails = this.extractReasoningDetails(choice); + const content = this.extractDeltaContent(delta.content); + + const messageChunk = this.buildMessageChunk({ + rawChunk, + delta, + content, + finishReason: choice.finish_reason, + reasoningDetails, + reasoningText, + }); + + const generationChunk = new ChatGenerationChunk({ + message: messageChunk, + text: typeof messageChunk.content === "string" ? messageChunk.content : "", + generationInfo: { + finish_reason: choice.finish_reason, + system_fingerprint: rawChunk.system_fingerprint, + model: rawChunk.model, + }, + }); + + yield generationChunk; + if (generationChunk.text) { + await _runManager?.handleLLMNewToken(generationChunk.text); + } + } + + if (usageSummary) { + yield this.buildUsageGenerationChunk(usageSummary); + } + + if (options.signal?.aborted) { + throw new Error("AbortError"); + } + } + + /** + * Convert LangChain messages to OpenRouter-ready messages. + * + * @param messages LangChain messages passed into the model + * @returns Messages formatted for the OpenRouter API + */ + private toOpenRouterMessages(messages: any[]): OpenRouterMessageParam[] { + return messages.map((msg) => { + const role = typeof msg._getType === "function" ? msg._getType() : (msg.role ?? "user"); + const mappedRole = + role === "human" + ? "user" + : role === "ai" + ? "assistant" + : (role as OpenAI.ChatCompletionRole); + + if (msg.tool_call_id) { + return { + role: "tool", + content: msg.content, + tool_call_id: msg.tool_call_id, + } as OpenRouterMessageParam; + } + + if (msg.additional_kwargs?.function_call) { + return { + role: mappedRole, + content: msg.content, + function_call: msg.additional_kwargs.function_call, + } as OpenRouterMessageParam; + } + + return { + role: mappedRole, + content: msg.content, + } as OpenRouterMessageParam; + }); + } + + /** + * Build an `AIMessageChunk` enriched with reasoning metadata. + * + * @param config Chunk configuration values extracted from the stream + * @returns AI message chunk ready for downstream streaming utilities + */ + private buildMessageChunk(config: { + rawChunk: OpenRouterChatChunk; + delta: Record; + content: string; + finishReason: string | null | undefined; + reasoningText?: string; + reasoningDetails?: unknown[]; + }): AIMessageChunk { + const { rawChunk, delta, content, finishReason, reasoningText, reasoningDetails } = config; + const toolCallChunks = this.extractToolCallChunks(delta.tool_calls); + + const additionalKwargs: Record = {}; + + if (delta.function_call) { + additionalKwargs.function_call = delta.function_call; + } + + if (Array.isArray(delta.tool_calls)) { + additionalKwargs.tool_calls = delta.tool_calls; + } + + const deltaPayload: Record = {}; + if (reasoningText) { + deltaPayload.reasoning = reasoningText; + } + if (reasoningDetails && reasoningDetails.length > 0) { + deltaPayload.reasoning_details = reasoningDetails; + } + + if (Object.keys(deltaPayload).length > 0) { + additionalKwargs.delta = { + ...(additionalKwargs.delta as Record), + ...deltaPayload, + }; + } + + if (reasoningDetails && reasoningDetails.length > 0) { + additionalKwargs.reasoning_details = reasoningDetails; + } + + const responseMetadata = this.buildResponseMetadata(rawChunk, finishReason); + + return new AIMessageChunk({ + content, + additional_kwargs: additionalKwargs, + tool_call_chunks: toolCallChunks, + response_metadata: responseMetadata, + id: rawChunk.id, + }); + } + + /** + * Normalize streamed reasoning payloads into plain text for the UI. + * + * @param reasoning Arbitrary reasoning payload returned by OpenRouter + * @returns Normalized reasoning text or undefined + */ + private normalizeReasoningChunk(reasoning: unknown): string | undefined { + if (!reasoning) { + return undefined; + } + + if (typeof reasoning === "string") { + return reasoning; + } + + if (Array.isArray(reasoning)) { + return reasoning + .map((item) => this.normalizeReasoningChunk(item)) + .filter((item): item is string => Boolean(item)) + .join(""); + } + + if (typeof reasoning === "object") { + const record = reasoning as Record; + const candidates = [ + record.output_text, + record.text, + record.reasoning, + record.thinking, + record.content, + ]; + + const normalized = candidates.find((value) => typeof value === "string"); + if (typeof normalized === "string") { + return normalized; + } + } + + return undefined; + } + + /** + * Extract reasoning details arrays from the streamed choice payload. + * + * @param choice Chunk choice object from the OpenRouter stream + * @returns Array of reasoning detail entries, if present + */ + private extractReasoningDetails(choice: any): unknown[] | undefined { + const candidate = + choice?.delta?.reasoning_details ?? + choice?.message?.reasoning_details ?? + choice?.reasoning_details; + + if (!Array.isArray(candidate)) { + return undefined; + } + + return candidate.filter((detail) => detail !== undefined && detail !== null); + } + + /** + * Flatten OpenRouter delta content into a single text string. + * + * @param content Delta content payload + * @returns Text representation for downstream streaming + */ + private extractDeltaContent(content: unknown): string { + if (typeof content === "string") { + return content; + } + + if (Array.isArray(content)) { + return content + .map((part) => { + if (typeof part === "string") { + return part; + } + if (part && typeof part === "object" && typeof part.text === "string") { + return part.text; + } + return ""; + }) + .join(""); + } + + return ""; + } + + /** + * Map raw OpenRouter tool call deltas into LangChain tool call chunks. + * + * @param toolCalls Tool call deltas returned by OpenRouter + * @returns Tool call chunk array compatible with LangChain + */ + private extractToolCallChunks( + toolCalls: any + ): + | Array<{ name?: string; args?: string; id?: string; index?: number; type: "tool_call_chunk" }> + | undefined { + if (!Array.isArray(toolCalls)) { + return undefined; + } + + return toolCalls.map((call) => ({ + name: call?.function?.name, + args: call?.function?.arguments, + id: call?.id, + index: call?.index, + type: "tool_call_chunk" as const, + })); + } + + /** + * Build response metadata payload with finish reason and usage info. + * + * @param rawChunk Raw streaming chunk from OpenRouter + * @param finishReason Stop reason reported by the model + * @returns Metadata object attached to each AI message chunk + */ + private buildResponseMetadata( + rawChunk: OpenRouterChatChunk, + finishReason: string | null | undefined + ): Record { + const metadata: Record = { + model_provider: "openrouter", + }; + + if (finishReason) { + metadata.finish_reason = finishReason; + } + + if (rawChunk.model) { + metadata.model = rawChunk.model; + } + + if (rawChunk.system_fingerprint) { + metadata.system_fingerprint = rawChunk.system_fingerprint; + } + + if (rawChunk.usage) { + metadata.usage = { ...rawChunk.usage }; + metadata.tokenUsage = { + promptTokens: rawChunk.usage.prompt_tokens, + completionTokens: rawChunk.usage.completion_tokens, + totalTokens: rawChunk.usage.total_tokens, + }; + } + + return metadata; + } + + /** + * Create a terminal usage chunk so downstream consumers can capture token usage. + * + * @param usage Usage payload returned by the streaming API + * @returns Chat generation chunk containing usage metadata + */ + private buildUsageGenerationChunk(usage: OpenRouterUsage): ChatGenerationChunk { + const inputTokenDetails: Record = {}; + const outputTokenDetails: Record = {}; + + const promptDetails = usage.prompt_tokens_details ?? {}; + if (typeof promptDetails.audio_tokens === "number") { + inputTokenDetails.audio = promptDetails.audio_tokens; + } + if (typeof promptDetails.cached_tokens === "number") { + inputTokenDetails.cache_read = promptDetails.cached_tokens; + } + + const completionDetails = usage.completion_tokens_details ?? {}; + if (typeof completionDetails.audio_tokens === "number") { + outputTokenDetails.audio = completionDetails.audio_tokens; + } + if (typeof completionDetails.reasoning_tokens === "number") { + outputTokenDetails.reasoning = completionDetails.reasoning_tokens; + } + + const usageMetadata: UsageMetadata = { + input_tokens: usage.prompt_tokens ?? 0, + output_tokens: usage.completion_tokens ?? 0, + total_tokens: usage.total_tokens ?? 0, + }; + + if (Object.keys(inputTokenDetails).length > 0) { + usageMetadata.input_token_details = inputTokenDetails; + } + + if (Object.keys(outputTokenDetails).length > 0) { + usageMetadata.output_token_details = outputTokenDetails; + } + + const messageChunk = new AIMessageChunk({ + content: "", + response_metadata: { usage: { ...usage } }, + usage_metadata: usageMetadata, + }); + + return new ChatGenerationChunk({ + message: messageChunk, + text: "", + }); + } +} diff --git a/src/LLMProviders/chainRunner/utils/ThinkBlockStreamer.test.ts b/src/LLMProviders/chainRunner/utils/ThinkBlockStreamer.test.ts new file mode 100644 index 00000000..18bd5929 --- /dev/null +++ b/src/LLMProviders/chainRunner/utils/ThinkBlockStreamer.test.ts @@ -0,0 +1,548 @@ +import { ThinkBlockStreamer } from "./ThinkBlockStreamer"; + +describe("ThinkBlockStreamer", () => { + describe("OpenRouter reasoning_details format", () => { + it("should handle reasoning_details with text content", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer((msg) => { + currentMessage = msg; + }); + + // First chunk with reasoning_details containing text + streamer.processChunk({ + content: "", + additional_kwargs: { + reasoning_details: [ + { + text: "Let me think about this problem...", + }, + ], + }, + }); + + expect(currentMessage).toBe("\nLet me think about this problem..."); + + // Second chunk with more reasoning + streamer.processChunk({ + content: "", + additional_kwargs: { + reasoning_details: [ + { + text: " I need to consider several factors.", + }, + ], + }, + }); + + expect(currentMessage).toBe( + "\nLet me think about this problem... I need to consider several factors." + ); + + // Third chunk with regular content (should close think block) + streamer.processChunk({ + content: "Here is my answer.", + additional_kwargs: {}, + }); + + expect(currentMessage).toBe( + "\nLet me think about this problem... I need to consider several factors.Here is my answer." + ); + + const result = streamer.close(); + expect(result.content).toBe( + "\nLet me think about this problem... I need to consider several factors.Here is my answer." + ); + }); + + it("should handle reasoning_details with summary content", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer((msg) => { + currentMessage = msg; + }); + + streamer.processChunk({ + content: "", + additional_kwargs: { + reasoning_details: [ + { + summary: "Analyzed the problem systematically", + }, + ], + }, + }); + + expect(currentMessage).toBe("\nAnalyzed the problem systematically"); + + streamer.processChunk({ + content: "Based on my analysis, the answer is 42.", + additional_kwargs: {}, + }); + + expect(currentMessage).toBe( + "\nAnalyzed the problem systematicallyBased on my analysis, the answer is 42." + ); + }); + + it("should handle encrypted reasoning_details", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer((msg) => { + currentMessage = msg; + }); + + streamer.processChunk({ + content: "", + additional_kwargs: { + reasoning_details: [ + { + encrypted: true, + }, + ], + }, + }); + + expect(currentMessage).toBe("\n[Encrypted reasoning]"); + + streamer.processChunk({ + content: "The answer is available.", + additional_kwargs: {}, + }); + + expect(currentMessage).toBe("\n[Encrypted reasoning]The answer is available."); + }); + + it("should NOT treat empty reasoning_details array as thinking content", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer((msg) => { + currentMessage = msg; + }); + + // This was the bug: empty reasoning_details array should not trigger thinking mode + streamer.processChunk({ + content: "Regular content", + additional_kwargs: { + reasoning_details: [], + }, + }); + + // Should NOT have tags since reasoning_details is empty + expect(currentMessage).toBe("Regular content"); + expect(currentMessage).not.toContain(""); + }); + + it("should handle delta.reasoning for streaming", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer((msg) => { + currentMessage = msg; + }); + + // First chunk with delta.reasoning + streamer.processChunk({ + content: "", + additional_kwargs: { + delta: { + reasoning: "Thinking step 1: ", + }, + }, + }); + + expect(currentMessage).toBe("\nThinking step 1: "); + + // Second chunk with more delta.reasoning + streamer.processChunk({ + content: "", + additional_kwargs: { + delta: { + reasoning: "Thinking step 2.", + }, + }, + }); + + expect(currentMessage).toBe("\nThinking step 1: Thinking step 2."); + + // Regular content should close think block + streamer.processChunk({ + content: "Here's the result.", + additional_kwargs: {}, + }); + + expect(currentMessage).toBe( + "\nThinking step 1: Thinking step 2.Here's the result." + ); + }); + }); + + describe("Proper tag placement", () => { + it("should close tag BEFORE regular content, not after", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer((msg) => { + currentMessage = msg; + }); + + // Thinking content + streamer.processChunk({ + content: "", + additional_kwargs: { + reasoning_details: [{ text: "Analyzing..." }], + }, + }); + + expect(currentMessage).toBe("\nAnalyzing..."); + + // Regular content with empty reasoning_details + // This should close BEFORE adding "20 minutes" + streamer.processChunk({ + content: "20 minutes", + additional_kwargs: { + reasoning_details: [], + }, + }); + + // Bug was: "(20 minutes)" + // Correct: "20 minutes" or "20 minutes" (if properly closed) + expect(currentMessage).toBe("\nAnalyzing...20 minutes"); + expect(currentMessage).not.toMatch(/\(.*<\/think>.*\)/); + }); + + it("should handle multiple transitions between thinking and regular content", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer((msg) => { + currentMessage = msg; + }); + + // First thinking block + streamer.processChunk({ + content: "", + additional_kwargs: { + reasoning_details: [{ text: "First thought" }], + }, + }); + + // Regular content + streamer.processChunk({ + content: "First answer. ", + additional_kwargs: {}, + }); + + expect(currentMessage).toBe("\nFirst thoughtFirst answer. "); + + // Second thinking block + streamer.processChunk({ + content: "", + additional_kwargs: { + reasoning_details: [{ text: "Second thought" }], + }, + }); + + // More regular content + streamer.processChunk({ + content: "Second answer.", + additional_kwargs: {}, + }); + + expect(currentMessage).toBe( + "\nFirst thoughtFirst answer. \nSecond thoughtSecond answer." + ); + }); + }); + + describe("Claude array-based format", () => { + it("should handle Claude's content array with thinking type", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer((msg) => { + currentMessage = msg; + }); + + // Claude format with content array + streamer.processChunk({ + content: [ + { + type: "thinking", + thinking: "Let me analyze this...", + }, + ], + }); + + expect(currentMessage).toBe("\nLet me analyze this..."); + + // Text content in array + streamer.processChunk({ + content: [ + { + type: "text", + text: "Based on my analysis, ", + }, + ], + }); + + expect(currentMessage).toBe("\nLet me analyze this...Based on my analysis, "); + }); + + it("should guard against undefined thinking content in Claude format", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer((msg) => { + currentMessage = msg; + }); + + // Malformed chunk with undefined thinking + streamer.processChunk({ + content: [ + { + type: "thinking", + // thinking property is undefined + }, + ], + }); + + // Should not crash, and should not add undefined to response + expect(currentMessage).toBe("\n"); + expect(currentMessage).not.toContain("undefined"); + }); + }); + + describe("Deepseek format", () => { + it("should handle Deepseek reasoning_content", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer((msg) => { + currentMessage = msg; + }); + + streamer.processChunk({ + content: "", + additional_kwargs: { + reasoning_content: "Deepseek is thinking...", + }, + }); + + expect(currentMessage).toBe("\nDeepseek is thinking..."); + + streamer.processChunk({ + content: "The answer is here.", + additional_kwargs: {}, + }); + + expect(currentMessage).toBe("\nDeepseek is thinking...The answer is here."); + }); + + it("should guard against undefined reasoning_content in Deepseek format", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer((msg) => { + currentMessage = msg; + }); + + // Malformed chunk with undefined reasoning_content + streamer.processChunk({ + content: "", + additional_kwargs: { + reasoning_content: undefined, + }, + }); + + // Should not crash, not open think block, and not add undefined + expect(currentMessage).toBe(""); + expect(currentMessage).not.toContain("undefined"); + expect(currentMessage).not.toContain(""); + }); + + it("should handle streaming Deepseek reasoning_content without premature closure", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer((msg) => { + currentMessage = msg; + }); + + // First chunk with reasoning_content + streamer.processChunk({ + content: "", + additional_kwargs: { + reasoning_content: "Thinking step 1...", + }, + }); + + expect(currentMessage).toBe("\nThinking step 1..."); + + // Second chunk with MORE reasoning_content (streaming) + // This should NOT close and reopen the think block + streamer.processChunk({ + content: "", + additional_kwargs: { + reasoning_content: " Step 2...", + }, + }); + + // Should be continuous, NOT "\n" + expect(currentMessage).toBe("\nThinking step 1... Step 2..."); + expect(currentMessage).not.toContain("\n"); + + // Third chunk with regular content + streamer.processChunk({ + content: "Final answer.", + additional_kwargs: {}, + }); + + expect(currentMessage).toBe("\nThinking step 1... Step 2...Final answer."); + }); + }); + + describe("excludeThinking option", () => { + it("should skip thinking content when excludeThinking is true", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer( + (msg) => { + currentMessage = msg; + }, + undefined, + true // excludeThinking = true + ); + + // Thinking content should be skipped + streamer.processChunk({ + content: "", + additional_kwargs: { + reasoning_details: [{ text: "This should be skipped" }], + }, + }); + + expect(currentMessage).toBe(""); + + // Regular content should still be processed + streamer.processChunk({ + content: "This should be included", + additional_kwargs: {}, + }); + + expect(currentMessage).toBe("This should be included"); + }); + + it("should skip Claude thinking content when excludeThinking is true", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer( + (msg) => { + currentMessage = msg; + }, + undefined, + true + ); + + streamer.processChunk({ + content: [ + { + type: "thinking", + thinking: "Claude thinking", + }, + ], + }); + + expect(currentMessage).toBe(""); + expect(currentMessage).not.toContain(""); + }); + }); + + describe("close() method", () => { + it("should close any open think block at the end", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer((msg) => { + currentMessage = msg; + }); + + streamer.processChunk({ + content: "", + additional_kwargs: { + reasoning_details: [{ text: "Thinking..." }], + }, + }); + + expect(currentMessage).toBe("\nThinking..."); + + const result = streamer.close(); + expect(result.content).toBe("\nThinking..."); + }); + + it("should not add extra closing tag if already closed", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer((msg) => { + currentMessage = msg; + }); + + streamer.processChunk({ + content: "", + additional_kwargs: { + reasoning_details: [{ text: "Thinking..." }], + }, + }); + + streamer.processChunk({ + content: "Done", + additional_kwargs: {}, + }); + + expect(currentMessage).toBe("\nThinking...Done"); + + const result = streamer.close(); + expect(result.content).toBe("\nThinking...Done"); + // Should not have double closing tags + expect(result.content.match(/<\/think>/g)?.length).toBe(1); + }); + }); + + describe("mixed content scenarios", () => { + it("should handle chunks with both reasoning_details and regular content", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer((msg) => { + currentMessage = msg; + }); + + // Chunk with both reasoning and content + streamer.processChunk({ + content: "Some text here", + additional_kwargs: { + reasoning_details: [{ text: "Thinking first" }], + }, + }); + + // Reasoning should be wrapped, content should be outside + expect(currentMessage).toBe("\nThinking firstSome text here"); + }); + + it("should handle rapid alternation between thinking and regular content", () => { + let currentMessage = ""; + const streamer = new ThinkBlockStreamer((msg) => { + currentMessage = msg; + }); + + const chunks = [ + { thinking: "Think 1", content: "" }, + { thinking: "", content: "Text 1" }, + { thinking: "Think 2", content: "" }, + { thinking: "", content: "Text 2" }, + { thinking: "Think 3", content: "" }, + { thinking: "", content: "Text 3" }, + ]; + + chunks.forEach((chunk) => { + if (chunk.thinking) { + streamer.processChunk({ + content: "", + additional_kwargs: { + reasoning_details: [{ text: chunk.thinking }], + }, + }); + } else { + streamer.processChunk({ + content: chunk.content, + additional_kwargs: {}, + }); + } + }); + + // Should have three separate think blocks + const thinkMatches = currentMessage.match(//g); + const thinkCloseMatches = currentMessage.match(/<\/think>/g); + expect(thinkMatches?.length).toBe(3); + expect(thinkCloseMatches?.length).toBe(3); + + // Each text should be outside think blocks + expect(currentMessage).toContain("Text 1"); + expect(currentMessage).toContain("Text 2"); + expect(currentMessage).toContain("Text 3"); + }); + }); +}); diff --git a/src/LLMProviders/chainRunner/utils/ThinkBlockStreamer.ts b/src/LLMProviders/chainRunner/utils/ThinkBlockStreamer.ts index 69020dec..ec72a1f9 100644 --- a/src/LLMProviders/chainRunner/utils/ThinkBlockStreamer.ts +++ b/src/LLMProviders/chainRunner/utils/ThinkBlockStreamer.ts @@ -46,6 +46,11 @@ export class ThinkBlockStreamer { break; } } + // Close think block before adding text content + if (textContent && this.hasOpenThinkBlock) { + this.fullResponse += ""; + this.hasOpenThinkBlock = false; + } if (textContent) { this.fullResponse += textContent; } @@ -77,6 +82,81 @@ export class ThinkBlockStreamer { return false; // No thinking chunk handled } + /** + * Handle OpenRouter reasoning/thinking content + * OpenRouter exposes reasoning via: + * - delta.reasoning (streaming) + * - reasoning_details array (final response) + */ + private handleOpenRouterChunk(chunk: any) { + let handledThinking = false; + + // Handle streaming reasoning from delta + if (chunk.additional_kwargs?.delta?.reasoning) { + // Skip thinking content if excludeThinking is enabled + if (this.excludeThinking) { + return true; + } + if (!this.hasOpenThinkBlock) { + this.fullResponse += "\n"; + this.hasOpenThinkBlock = true; + } + this.fullResponse += chunk.additional_kwargs.delta.reasoning; + handledThinking = true; + } + + // Handle reasoning_details from final response or accumulated chunks + // IMPORTANT: Only treat as thinking if the array has actual content + if (chunk.additional_kwargs?.reasoning_details) { + const details = chunk.additional_kwargs.reasoning_details; + if (Array.isArray(details) && details.length > 0) { + // Skip thinking content if excludeThinking is enabled + if (this.excludeThinking) { + return true; + } + + for (const detail of details) { + if (detail.encrypted) { + // Show placeholder for encrypted reasoning + if (!this.hasOpenThinkBlock) { + this.fullResponse += "\n"; + this.hasOpenThinkBlock = true; + } + this.fullResponse += "[Encrypted reasoning]"; + handledThinking = true; + } else if (detail.text) { + if (!this.hasOpenThinkBlock) { + this.fullResponse += "\n"; + this.hasOpenThinkBlock = true; + } + this.fullResponse += detail.text; + handledThinking = true; + } else if (detail.summary) { + if (!this.hasOpenThinkBlock) { + this.fullResponse += "\n"; + this.hasOpenThinkBlock = true; + } + this.fullResponse += detail.summary; + handledThinking = true; + } + } + } + } + + // Close think block before adding regular content + if (typeof chunk.content === "string" && chunk.content && this.hasOpenThinkBlock) { + this.fullResponse += ""; + this.hasOpenThinkBlock = false; + } + + // Handle standard string content (this is the actual response, not thinking) + if (typeof chunk.content === "string" && chunk.content) { + this.fullResponse += chunk.content; + } + + return handledThinking; + } + processChunk(chunk: any) { // If we've already decided to truncate, don't process more chunks if (this.shouldTruncate) { @@ -95,22 +175,37 @@ export class ThinkBlockStreamer { this.tokenUsage = usage; } - let handledThinking = false; + // Determine if this chunk will handle thinking content + const isThinkingChunk = + Array.isArray(chunk.content) || + chunk.additional_kwargs?.delta?.reasoning || + (chunk.additional_kwargs?.reasoning_details && + Array.isArray(chunk.additional_kwargs.reasoning_details) && + chunk.additional_kwargs.reasoning_details.length > 0) || + chunk.additional_kwargs?.reasoning_content; // Deepseek format - // Handle Claude 3.7 array-based content - if (Array.isArray(chunk.content)) { - handledThinking = this.handleClaude37Chunk(chunk.content); - } else { - // Handle deepseek format - handledThinking = this.handleDeepseekChunk(chunk); - } - - // Close think block if we have one open and didn't handle thinking content - if (this.hasOpenThinkBlock && !handledThinking) { + // Close think block BEFORE processing non-thinking content + if (this.hasOpenThinkBlock && !isThinkingChunk) { this.fullResponse += ""; this.hasOpenThinkBlock = false; } + // Now process the chunk + // Route based on the actual chunk format + if (Array.isArray(chunk.content)) { + // Claude format with content array + this.handleClaude37Chunk(chunk.content); + } else if (chunk.additional_kwargs?.reasoning_content) { + // Deepseek format with reasoning_content + this.handleDeepseekChunk(chunk); + } else if (isThinkingChunk) { + // OpenRouter format with delta.reasoning or reasoning_details + this.handleOpenRouterChunk(chunk); + } else { + // Default case: regular content or other formats + this.handleDeepseekChunk(chunk); + } + // Check if we should truncate streaming based on model adapter if (this.modelAdapter?.shouldTruncateStreaming?.(this.fullResponse)) { this.shouldTruncate = true; diff --git a/src/LLMProviders/chatModelManager.ts b/src/LLMProviders/chatModelManager.ts index f1f8e2a3..d18a4269 100644 --- a/src/LLMProviders/chatModelManager.ts +++ b/src/LLMProviders/chatModelManager.ts @@ -3,6 +3,7 @@ import { BREVILABS_MODELS_BASE_URL, BUILTIN_CHAT_MODELS, ChatModelProviders, + ModelCapability, ProviderInfo, } from "@/constants"; import { getDecryptedKey } from "@/encryptionService"; @@ -32,6 +33,7 @@ import { ChatOllama } from "@langchain/ollama"; import { ChatOpenAI } from "@langchain/openai"; import { ChatXAI } from "@langchain/xai"; import { Notice } from "obsidian"; +import { ChatOpenRouter } from "./ChatOpenRouter"; type ChatConstructorType = { new (config: any): any; @@ -44,7 +46,7 @@ const CHAT_PROVIDER_CONSTRUCTORS = { [ChatModelProviders.COHEREAI]: ChatCohere, [ChatModelProviders.GOOGLE]: ChatGoogleGenerativeAI, [ChatModelProviders.XAI]: ChatXAI, - [ChatModelProviders.OPENROUTERAI]: ChatOpenAI, + [ChatModelProviders.OPENROUTERAI]: ChatOpenRouter, [ChatModelProviders.OLLAMA]: ChatOllama, [ChatModelProviders.LM_STUDIO]: ChatOpenAI, [ChatModelProviders.GROQ]: ChatGroq, @@ -243,6 +245,14 @@ export default class ChatModelManager { "X-Title": "Obsidian Copilot", }, }, + // Enable reasoning if the model has the reasoning capability + enableReasoning: customModel.capabilities?.includes(ModelCapability.REASONING) ?? false, + // Pass reasoning effort if configured and reasoning capability is enabled + reasoningEffort: + customModel.capabilities?.includes(ModelCapability.REASONING) && + customModel.reasoningEffort + ? customModel.reasoningEffort + : undefined, }, [ChatModelProviders.GROQ]: { apiKey: await getDecryptedKey(customModel.apiKey || settings.groqApiKey), diff --git a/src/settings/v2/components/ModelEditDialog.tsx b/src/settings/v2/components/ModelEditDialog.tsx index d8e4de64..ea6d1485 100644 --- a/src/settings/v2/components/ModelEditDialog.tsx +++ b/src/settings/v2/components/ModelEditDialog.tsx @@ -387,6 +387,43 @@ export const ModelEditModalContent: React.FC = ({ )} )} + + {/* Reasoning Effort for OpenRouter models */} + {localModel.provider === "openrouterai" && ( + + handleLocalUpdate("reasoningEffort", value)} + disableFn={() => handleLocalReset("reasoningEffort")} + defaultValue="low" + options={[ + { value: "low", label: "Low" }, + { value: "medium", label: "Medium" }, + { value: "high", label: "High" }, + ]} + helpText={ + <> +

+ Controls the amount of reasoning effort the model uses. Higher effort + provides more thorough reasoning but takes longer. +

+
    +
  • Low: Faster responses, basic reasoning (default)
    • +
  • Medium: Balanced performance
    • +
  • High: Thorough reasoning, slower responses
    • +
+ {!localModel.capabilities?.includes(ModelCapability.REASONING) && ( +

+ Enable the "Reasoning" capability above to use this feature. +

+ )} + + } + /> + + )} )} diff --git a/src/styles/tailwind.css b/src/styles/tailwind.css index ec0b8b8e..68295fdc 100644 --- a/src/styles/tailwind.css +++ b/src/styles/tailwind.css @@ -827,9 +827,6 @@ If your plugin does not need CSS, delete this file. margin-right: 12px; } -.model-card-title { -} - .model-provider-wrapper { display: flex; flex-direction: column; @@ -953,6 +950,7 @@ If your plugin does not need CSS, delete this file. font-weight: var(--font-medium); min-width: var(--size-4-3); text-align: right; + white-space: nowrap; } .copilot-sources__text {