From eb830dfa6fcbb2f4a4d634dab9e738484845de39 Mon Sep 17 00:00:00 2001 From: Logan Yang Date: Mon, 25 Aug 2025 19:59:57 -0700 Subject: [PATCH] Support image and chat context menu in free modes (#1746) --- .../chainRunner/LLMChainRunner.ts | 17 +- .../chainRunner/VaultQAChainRunner.ts | 17 +- src/components/Chat.tsx | 23 +- src/components/chat-components/ChatInput.tsx | 214 +++++++++++------- .../chat-components/ContextControl.tsx | 9 +- src/components/modals/AddContextNoteModal.tsx | 15 +- src/components/modals/BaseNoteModal.tsx | 17 +- src/constants.ts | 11 +- src/contextProcessor.ts | 14 +- src/utils.ts | 32 +++ 10 files changed, 246 insertions(+), 123 deletions(-) diff --git a/src/LLMProviders/chainRunner/LLMChainRunner.ts b/src/LLMProviders/chainRunner/LLMChainRunner.ts index 13b667f8..44473414 100644 --- a/src/LLMProviders/chainRunner/LLMChainRunner.ts +++ b/src/LLMProviders/chainRunner/LLMChainRunner.ts @@ -45,11 +45,18 @@ export class LLMChainRunner extends BaseChainRunner { messages.push({ role: entry.role, content: entry.content }); } - // Add current user message - messages.push({ - role: "user", - content: userMessage.message, - }); + // Add current user message - support multimodal content if available + if (userMessage.content && Array.isArray(userMessage.content)) { + messages.push({ + role: "user", + content: userMessage.content, + }); + } else { + messages.push({ + role: "user", + content: userMessage.message, + }); + } logInfo("==== Final Request to AI ====\n", messages); diff --git a/src/LLMProviders/chainRunner/VaultQAChainRunner.ts b/src/LLMProviders/chainRunner/VaultQAChainRunner.ts index b80e19b6..ec4f8295 100644 --- a/src/LLMProviders/chainRunner/VaultQAChainRunner.ts +++ b/src/LLMProviders/chainRunner/VaultQAChainRunner.ts @@ -99,11 +99,18 @@ export class VaultQAChainRunner extends BaseChainRunner { messages.push({ role: entry.role, content: entry.content }); } - // Add current user question - messages.push({ - role: "user", - content: userMessage.message, - }); + // Add current user question - support multimodal content if available + if (userMessage.content && Array.isArray(userMessage.content)) { + messages.push({ + role: "user", + content: userMessage.content, + }); + } else { + messages.push({ + role: "user", + content: userMessage.message, + }); + } logInfo("==== Final Request to AI ====\n", messages); diff --git a/src/components/Chat.tsx b/src/components/Chat.tsx index 6c9efd90..7ec012c2 100644 --- a/src/components/Chat.tsx +++ b/src/components/Chat.tsx @@ -18,7 +18,13 @@ import ChatMessages from "@/components/chat-components/ChatMessages"; import { NewVersionBanner } from "@/components/chat-components/NewVersionBanner"; import { ProjectList } from "@/components/chat-components/ProjectList"; import ProgressCard from "@/components/project/progress-card"; -import { ABORT_REASON, EVENT_NAMES, LOADING_MESSAGES, USER_SENDER } from "@/constants"; +import { + ABORT_REASON, + EVENT_NAMES, + LOADING_MESSAGES, + USER_SENDER, + RESTRICTION_MESSAGES, +} from "@/constants"; import { AppContext, EventTargetContext } from "@/context"; import { useChatManager } from "@/hooks/useChatManager"; import { getAIResponse } from "@/langchainStream"; @@ -29,7 +35,7 @@ import { useIsPlusUser } from "@/plusUtils"; import { updateSetting, useSettingsValue } from "@/settings/model"; import { ChatUIState } from "@/state/ChatUIState"; import { FileParserManager } from "@/tools/FileParserManager"; -import { err2String } from "@/utils"; +import { err2String, isPlusChain } from "@/utils"; import { Buffer } from "buffer"; import { Notice, TFile } from "obsidian"; import React, { useCallback, useContext, useEffect, useMemo, useRef, useState } from "react"; @@ -152,6 +158,15 @@ const Chat: React.FC = ({ } = {}) => { if (!inputMessage && selectedImages.length === 0) return; + // Check for URL restrictions in non-Plus chains and show notice, but continue processing + const hasUrlsInContext = urls && urls.length > 0; + const hasUrlsInMessage = inputMessage && mention.extractAllUrls(inputMessage).length > 0; + + if ((hasUrlsInContext || hasUrlsInMessage) && !isPlusChain(currentChain)) { + // Show notice but continue processing the message without URL context + new Notice(RESTRICTION_MESSAGES.URL_PROCESSING_RESTRICTED); + } + try { // Create message content array const content: any[] = []; @@ -190,10 +205,10 @@ const Chat: React.FC = ({ displayText += " " + toolCalls.join("\n"); } - // Create message context + // Create message context - filter out URLs for non-Plus chains const context = { notes, - urls: urls || [], + urls: isPlusChain(currentChain) ? urls || [] : [], selectedTextContexts, }; diff --git a/src/components/chat-components/ChatInput.tsx b/src/components/chat-components/ChatInput.tsx index ba021507..489661e0 100644 --- a/src/components/chat-components/ChatInput.tsx +++ b/src/components/chat-components/ChatInput.tsx @@ -15,10 +15,12 @@ import { AddImageModal } from "@/components/modals/AddImageModal"; import { ListPromptModal } from "@/components/modals/ListPromptModal"; import { Button } from "@/components/ui/button"; import { ModelSelector } from "@/components/ui/ModelSelector"; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { ContextProcessor } from "@/contextProcessor"; import { cn } from "@/lib/utils"; import { COPILOT_TOOL_NAMES } from "@/LLMProviders/intentAnalyzer"; import { Mention } from "@/mentions/Mention"; +import { isPlusChain } from "@/utils"; import { updateSetting, useSettingsValue } from "@/settings/model"; import { SelectedTextContext } from "@/types/message"; @@ -110,8 +112,7 @@ const ChatInput = forwardRef<{ focus: () => void }, ChatInputProps>( return isAllowedFileForContext(activeFile) ? activeFile : null; }); const [selectedProject, setSelectedProject] = useState(null); - const isCopilotPlus = - currentChain === ChainType.COPILOT_PLUS_CHAIN || currentChain === ChainType.PROJECT_CHAIN; + const isCopilotPlus = isPlusChain(currentChain); // Toggle states for vault, web search, composer, and autonomous agent const [vaultToggle, setVaultToggle] = useState(false); @@ -225,7 +226,8 @@ const ChatInput = forwardRef<{ focus: () => void }, ChatInputProps>( // Update URLs in context, ensuring uniqueness const newUrls = urls.filter((url) => !contextUrls.includes(url)); - if (newUrls.length > 0) { + if (newUrls.length > 0 && isPlusChain(currentChain)) { + // Only add URLs to context for Plus chains // Use Set to ensure uniqueness setContextUrls((prev) => Array.from(new Set([...prev, ...newUrls]))); } @@ -379,7 +381,7 @@ const ChatInput = forwardRef<{ focus: () => void }, ChatInputProps>( const handlePaste = useCallback( async (e: React.ClipboardEvent) => { const items = e.clipboardData?.items; - if (!items || !isCopilotPlus) return; + if (!items) return; const imageItems = Array.from(items).filter((item) => item.type.indexOf("image") !== -1); @@ -400,7 +402,7 @@ const ChatInput = forwardRef<{ focus: () => void }, ChatInputProps>( } } }, - [onAddImage, isCopilotPlus] + [onAddImage] ); useEffect(() => { @@ -447,8 +449,22 @@ const ChatInput = forwardRef<{ focus: () => void }, ChatInputProps>( ); // Remove any URLs that are no longer present in the input - setContextUrls((prev) => prev.filter((url) => currentUrls.includes(url))); - }, [inputMessage, includeActiveNote, currentActiveNote, mention, setContextNotes, app.vault]); + // Only keep URLs if URL processing is supported for the current chain + if (isPlusChain(currentChain)) { + setContextUrls((prev) => prev.filter((url) => currentUrls.includes(url))); + } else { + // Clear all URLs for non-Plus chains + setContextUrls([]); + } + }, [ + inputMessage, + includeActiveNote, + currentActiveNote, + mention, + setContextNotes, + app.vault, + currentChain, + ]); // Update the current active note whenever it changes useEffect(() => { @@ -535,7 +551,7 @@ const ChatInput = forwardRef<{ focus: () => void }, ChatInputProps>( )} -
+
{isProjectLoading && (
@@ -557,16 +573,12 @@ const ChatInput = forwardRef<{ focus: () => void }, ChatInputProps>( onPaste={handlePaste} disabled={isProjectLoading} /> - {isCopilotPlus && ( - <> - - {/* Overlay that appears when dragging */} - {isDragActive && ( -
- Drop images here... -
- )} - + + {/* Overlay that appears when dragging */} + {isDragActive && ( +
+ Drop images here... +
)}
@@ -604,85 +616,111 @@ const ChatInput = forwardRef<{ focus: () => void }, ChatInputProps>( Stop ) : ( - <> + {/* Autonomous Agent button - only show in Copilot Plus mode and NOT in Projects mode */} {isCopilotPlus && currentChain !== ChainType.PROJECT_CHAIN && ( - + + + + + + Toggle autonomous agent mode + + )} {/* Toggle buttons for vault, web search, and composer - show when Autonomous Agent is off */} {!autonomousAgentToggle && isCopilotPlus && ( <> - - - + + + + + + Toggle vault search + + + + + + + + Toggle web search + + + + + + + + Toggle composer (note editing) + + )} - {isCopilotPlus && ( - - )} + + + + + Add image(s) + - + )}
diff --git a/src/components/chat-components/ContextControl.tsx b/src/components/chat-components/ContextControl.tsx index 795610ca..dc5e7f96 100644 --- a/src/components/chat-components/ContextControl.tsx +++ b/src/components/chat-components/ContextControl.tsx @@ -1,8 +1,7 @@ -import { useChainType } from "@/aiParams"; import { App } from "obsidian"; import React from "react"; -import { ChainType } from "@/chainFactory"; +import { useChainType } from "@/aiParams"; import { AddContextNoteModal } from "@/components/modals/AddContextNoteModal"; import { SelectedTextContext } from "@/types/message"; import { TFile } from "obsidian"; @@ -38,7 +37,6 @@ const ContextControl: React.FC = ({ showProgressCard, }) => { const [selectedChain] = useChainType(); - const handleAddContext = () => { new AddContextNoteModal({ app, @@ -53,6 +51,7 @@ const ContextControl: React.FC = ({ } }, excludeNotePaths, + chainType: selectedChain, }).open(); }; @@ -74,9 +73,7 @@ const ContextControl: React.FC = ({ } }; - if (selectedChain !== ChainType.COPILOT_PLUS_CHAIN && selectedChain !== ChainType.PROJECT_CHAIN) { - return null; - } + // Context menu is now available for all chain types return ( void; excludeNotePaths: string[]; titleOnly?: boolean; + chainType?: ChainType; } export class AddContextNoteModal extends BaseNoteModal { @@ -17,8 +21,9 @@ export class AddContextNoteModal extends BaseNoteModal { onNoteSelect, excludeNotePaths, titleOnly = false, + chainType = ChainType.COPILOT_PLUS_CHAIN, }: AddContextNoteModalProps) { - super(app); + super(app, chainType); this.onNoteSelect = onNoteSelect; this.availableNotes = this.getOrderedNotes(excludeNotePaths); this.titleOnly = titleOnly; @@ -42,6 +47,12 @@ export class AddContextNoteModal extends BaseNoteModal { } onChooseItem(note: TFile, evt: MouseEvent | KeyboardEvent) { + // Check if the file is allowed for the current chain type + if (!isAllowedFileForChainContext(note, this.chainType)) { + new Notice(RESTRICTION_MESSAGES.NON_MARKDOWN_FILES_RESTRICTED); + return; + } + this.onNoteSelect(note); } diff --git a/src/components/modals/BaseNoteModal.tsx b/src/components/modals/BaseNoteModal.tsx index e1501800..d418ad17 100644 --- a/src/components/modals/BaseNoteModal.tsx +++ b/src/components/modals/BaseNoteModal.tsx @@ -1,13 +1,16 @@ import { App, FuzzySuggestModal, TFile } from "obsidian"; -import { isAllowedFileForContext } from "@/utils"; +import { isAllowedFileForChainContext } from "@/utils"; +import { ChainType } from "@/chainFactory"; export abstract class BaseNoteModal extends FuzzySuggestModal { protected activeNote: TFile | null; protected availableNotes: T[]; + protected chainType: ChainType; - constructor(app: App) { + constructor(app: App, chainType: ChainType = ChainType.COPILOT_PLUS_CHAIN) { super(app); this.activeNote = app.workspace.getActiveFile(); + this.chainType = chainType; } protected getOrderedNotes(excludeNotePaths: string[] = []): TFile[] { @@ -18,13 +21,15 @@ export abstract class BaseNoteModal extends FuzzySuggestModal { .filter( (file): file is TFile => file instanceof TFile && - isAllowedFileForContext(file) && + isAllowedFileForChainContext(file, this.chainType) && !excludeNotePaths.includes(file.path) && file.path !== this.activeNote?.path ); // Get all other files that weren't recently opened - const allFiles = this.app.vault.getFiles().filter((file) => isAllowedFileForContext(file)); + const allFiles = this.app.vault + .getFiles() + .filter((file) => isAllowedFileForChainContext(file, this.chainType)); const otherFiles = allFiles.filter( (file) => @@ -35,7 +40,9 @@ export abstract class BaseNoteModal extends FuzzySuggestModal { // Combine active note (if exists and is allowed type) with recent files and other files const activeNoteArray = - this.activeNote && isAllowedFileForContext(this.activeNote) ? [this.activeNote] : []; + this.activeNote && isAllowedFileForChainContext(this.activeNote, this.chainType) + ? [this.activeNote] + : []; return [...activeNoteArray, ...recentFiles, ...otherFiles]; } diff --git a/src/constants.ts b/src/constants.ts index a5fe5cd6..97c971b4 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -99,7 +99,7 @@ export const EMPTY_INDEX_ERROR_MESSAGE = export const CHUNK_SIZE = 6000; export const TEXT_WEIGHT = 0.4; export const MAX_CHARS_FOR_LOCAL_SEARCH_CONTEXT = 448000; -export const LLM_TIMEOUT_MS = 10000; // 10 seconds timeout for LLM operations +export const LLM_TIMEOUT_MS = 30000; // 30 seconds timeout for LLM operations export const LOADING_MESSAGES = { DEFAULT: "", READING_FILES: "Reading files", @@ -658,6 +658,15 @@ export const AUTOCOMPLETE_CONFIG = { KEYBIND: "Tab" as AcceptKeyOption, } as const; +export const RESTRICTION_MESSAGES = { + NON_MARKDOWN_FILES_RESTRICTED: + "Non-markdown files are only available in Copilot Plus mode. Please upgrade to access this file type.", + URL_PROCESSING_RESTRICTED: + "URL processing is only available in Copilot Plus mode. URLs will not be processed for context.", + UNSUPPORTED_FILE_TYPE: (extension: string) => + `${extension.toUpperCase()} files are not supported in the current mode.`, +} as const; + export const DEFAULT_SETTINGS: CopilotSettings = { userId: uuidv4(), isPlusUser: false, diff --git a/src/contextProcessor.ts b/src/contextProcessor.ts index 0d62afa9..38ea8fbd 100644 --- a/src/contextProcessor.ts +++ b/src/contextProcessor.ts @@ -1,7 +1,9 @@ import { getSelectedTextContexts } from "@/aiParams"; import { ChainType } from "@/chainFactory"; +import { RESTRICTION_MESSAGES } from "@/constants"; import { FileParserManager } from "@/tools/FileParserManager"; -import { TFile, Vault } from "obsidian"; +import { isPlusChain } from "@/utils"; +import { TFile, Vault, Notice } from "obsidian"; import { NOTE_CONTEXT_PROMPT_TAG, EMBEDDED_PDF_TAG, SELECTED_TEXT_TAG } from "./constants"; export class ContextProcessor { @@ -93,15 +95,13 @@ export class ContextProcessor { } // 2. Apply chain restrictions only to supported files that are NOT md or canvas - if ( - currentChain !== ChainType.COPILOT_PLUS_CHAIN && - note.extension !== "md" && - note.extension !== "canvas" - ) { + if (!isPlusChain(currentChain) && note.extension !== "md" && note.extension !== "canvas") { // This file type is supported, but requires Plus mode (e.g., PDF) console.warn( `File type ${note.extension} requires Copilot Plus mode for context processing.` ); + // Show user-facing notice about the restriction + new Notice(RESTRICTION_MESSAGES.NON_MARKDOWN_FILES_RESTRICTED); return; } @@ -109,7 +109,7 @@ export class ContextProcessor { let content = await fileParserManager.parseFile(note, vault); // Special handling for embedded PDFs within markdown (only in Plus mode) - if (note.extension === "md" && currentChain === ChainType.COPILOT_PLUS_CHAIN) { + if (note.extension === "md" && isPlusChain(currentChain)) { content = await this.processEmbeddedPDFs(content, vault, fileParserManager); } diff --git a/src/utils.ts b/src/utils.ts index b4a6893d..a5e7672d 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -290,6 +290,38 @@ export function isAllowedFileForContext(file: TFile | null): boolean { return file.extension === "md" || file.extension === "pdf" || file.extension === "canvas"; } +/** + * Checks if a chain type is a Plus mode chain (Copilot Plus or Project Chain). + * Plus mode chains have access to premium features like PDF processing and URL processing. + * @param chainType The chain type to check + * @returns true if this is a Plus mode chain, false otherwise + */ +export function isPlusChain(chainType: ChainType): boolean { + return chainType === ChainType.COPILOT_PLUS_CHAIN || chainType === ChainType.PROJECT_CHAIN; +} + +/** + * Checks if a file extension is allowed for context based on the chain type. + * All chains support markdown and canvas files. + * Plus chains support all file types (PDF, EPUB, PPT, DOCX, etc.). + * Free chains only support markdown and canvas files. + * @param file The file to check + * @param chainType The current chain type + * @returns true if the file is allowed for this chain type, false otherwise + */ +export function isAllowedFileForChainContext(file: TFile | null, chainType: ChainType): boolean { + if (!file) return false; + + // All chains support markdown and canvas files + if (file.extension === "md" || file.extension === "canvas") { + return true; + } + + // Plus chains support all other file types (PDF, EPUB, PPT, DOCX, etc.) + // Free chains only support markdown and canvas + return isPlusChain(chainType); +} + export async function getAllNotesContent(vault: Vault): Promise { const vaultNotes: string[] = [];