diff --git a/jest.setup.js b/jest.setup.js index bc3f3fb2..fe5d2f3a 100644 --- a/jest.setup.js +++ b/jest.setup.js @@ -22,3 +22,15 @@ if (typeof Node !== "undefined" && !Object.prototype.hasOwnProperty.call(Node.pr configurable: true, }); } + +// Obsidian exposes `activeDocument` / `activeWindow` globals pointing at the +// focused popout's document/window. Under jsdom there's only one document, so +// alias them onto `window` (the jsdom global object) — plugin code that portals +// into `activeDocument.body` (e.g. the Radix tooltip) would otherwise throw +// `activeDocument is not defined`. +if (typeof window.activeDocument === "undefined") { + window.activeDocument = window.document; +} +if (typeof window.activeWindow === "undefined") { + window.activeWindow = window; +} diff --git a/src/agentMode/backends/opencode/opencodeModelResolve.ts b/src/agentMode/backends/opencode/opencodeModelResolve.ts index 770e5b87..72deeb68 100644 --- a/src/agentMode/backends/opencode/opencodeModelResolve.ts +++ b/src/agentMode/backends/opencode/opencodeModelResolve.ts @@ -1,6 +1,6 @@ import type { CopilotSettings } from "@/settings/model"; import type { ConfiguredModel, Provider } from "@/modelManagement"; -import { providerRequiresApiKey } from "@/modelManagement"; +import { capabilitiesFromConfiguredInfo, providerRequiresApiKey } from "@/modelManagement"; import type { EnabledModelCredentialState, EnabledModelEntry } from "@/agentMode/session/types"; export interface OpencodeProviderMapping { @@ -124,6 +124,7 @@ export function opencodeEnabledModelEntries( description: configuredModel.info.description, credentialState: credentialStateFor(provider, mapping.native), isFree: isOpencodeZenWireId(baseModelId), + capabilities: capabilitiesFromConfiguredInfo(configuredModel.info), }); } return out.length === 0 ? EMPTY_ENABLED_ENTRIES : out; diff --git a/src/agentMode/backends/shared/agentEnabledModels.test.ts b/src/agentMode/backends/shared/agentEnabledModels.test.ts index 6b970275..161cb97a 100644 --- a/src/agentMode/backends/shared/agentEnabledModels.test.ts +++ b/src/agentMode/backends/shared/agentEnabledModels.test.ts @@ -1,6 +1,7 @@ import { agentOriginEnabledModelEntries } from "./agentEnabledModels"; import type { CopilotSettings } from "@/settings/model"; import type { ConfiguredModel } from "@/modelManagement"; +import { ModelCapability } from "@/constants"; /** Bare descriptor-style decode (claude): the wire id IS the baseModelId. */ const bareDecode = (wireId: string): { selection: { baseModelId: string } } => ({ @@ -74,6 +75,27 @@ describe("agentOriginEnabledModelEntries", () => { expect(entries.map((e) => e.baseModelId)).toEqual(["claude-sonnet-4-5"]); }); + it("derives capabilities from info.modalities, and leaves them undefined when unknown", () => { + const visionModel: ConfiguredModel = { + configuredModelId: "cm1", + providerId: "p1", + info: { + id: "claude-sonnet-4-5", + displayName: "claude-sonnet-4-5", + modalities: { input: ["text", "image"] }, + }, + configuredAt: 0, + }; + const settings = settingsWith("claude", ["cm1", "cm2"], [visionModel, model("cm2", "legacy")]); + const entries = agentOriginEnabledModelEntries(settings, "claude", bareDecode); + const vision = entries.find((e) => e.baseModelId === "claude-sonnet-4-5"); + const unknown = entries.find((e) => e.baseModelId === "legacy"); + expect(vision?.capabilities).toContain(ModelCapability.VISION); + // `model()` builds info without `modalities` → "unknown", so the picker can + // fall back to the catalog rather than asserting no vision. + expect(unknown?.capabilities).toBeUndefined(); + }); + it("only reads the requested agentType's enabledModels", () => { const settings = { backends: { diff --git a/src/agentMode/backends/shared/agentEnabledModels.ts b/src/agentMode/backends/shared/agentEnabledModels.ts index eab68738..9b646078 100644 --- a/src/agentMode/backends/shared/agentEnabledModels.ts +++ b/src/agentMode/backends/shared/agentEnabledModels.ts @@ -1,5 +1,5 @@ import type { CopilotSettings } from "@/settings/model"; -import type { ConfiguredModel } from "@/modelManagement"; +import { type ConfiguredModel, capabilitiesFromConfiguredInfo } from "@/modelManagement"; import type { EnabledModelEntry } from "@/agentMode/session/types"; /** See AGENTS.md → "Referential stability". */ @@ -41,6 +41,7 @@ export function agentOriginEnabledModelEntries( name: configuredModel.info.displayName || configuredModel.info.id, description: configuredModel.info.description, credentialState: "ok", + capabilities: capabilitiesFromConfiguredInfo(configuredModel.info), }); } diff --git a/src/agentMode/session/types.ts b/src/agentMode/session/types.ts index 434f23db..0b96213e 100644 --- a/src/agentMode/session/types.ts +++ b/src/agentMode/session/types.ts @@ -1,4 +1,5 @@ import type React from "react"; +import type { ModelCapability } from "@/constants"; import type { FormattedDateTime, MessageContext } from "@/types/message"; // `import type` keeps the cycle with `fanoutTypes` compile-time only. import type { FanoutTurn } from "@/agentMode/session/fanout/fanoutTypes"; @@ -141,6 +142,14 @@ export interface EnabledModelEntry { name: string; description?: string; credentialState: EnabledModelCredentialState; + /** + * Capabilities derived from the model's persisted `ConfiguredModel.info` + * (the modality snapshot taken at setup) — the reliable source the picker + * prefers over a live catalog lookup. `undefined` means "unknown" (the + * stored info carried no modality data), letting the picker fall back to the + * catalog rather than asserting a model has no vision. + */ + capabilities?: ModelCapability[]; /** * `true` for an opencode Zen model (opencode's self-hosted free tier). The * opencode backend computes this; the UI just renders a privacy warning, so diff --git a/src/agentMode/ui/AgentChatInput.tsx b/src/agentMode/ui/AgentChatInput.tsx index ef82ff3d..82ea6e05 100644 --- a/src/agentMode/ui/AgentChatInput.tsx +++ b/src/agentMode/ui/AgentChatInput.tsx @@ -37,6 +37,8 @@ import { type SelectedTextContext, type WebTabContext, } from "@/types/message"; +import { getModelKeyFromModel } from "@/settings/model"; +import { modelSupportsVision } from "@/utils"; import { arrayBufferToBase64 } from "@/utils/base64"; import { mergeWebTabContexts } from "@/utils/urlNormalization"; import { Clock, Sparkles, X } from "lucide-react"; @@ -344,6 +346,25 @@ export const AgentChatInput = memo(function AgentChatInput({ shouldIncludeActiveWebTab ); + // Hard-block sending images to a model that is KNOWN to lack vision. We + // only block when the active entry's capabilities are populated (an empty + // array still means "known"); undefined means "unknown" and must not + // block. An undefined `modelPickerOverride` (model switching disabled) can't + // resolve an active entry, so it's also treated as unknown. Inputs are left + // intact (guard precedes resetCompose) so the user can switch models. + if (selectedImages.length > 0) { + const activeEntry = modelPickerOverride?.models.find( + (m) => getModelKeyFromModel(m) === modelPickerOverride.value + ); + if (Array.isArray(activeEntry?.capabilities) && !modelSupportsVision(activeEntry)) { + const modelLabel = activeEntry.displayName || activeEntry.name; + new Notice( + `${modelLabel} doesn't support images. Switch to a vision-capable model to send images.` + ); + return; + } + } + const content: PromptContent[] = []; // Convert attached images to base64 image content blocks. @@ -404,6 +425,7 @@ export const AgentChatInput = memo(function AgentChatInput({ selectedTextContexts, loading, isStarting, + modelPickerOverride, holdForContext, disabled, resetCompose, diff --git a/src/agentMode/ui/ModelEnableList.tsx b/src/agentMode/ui/ModelEnableList.tsx index ff395f6d..532295b3 100644 --- a/src/agentMode/ui/ModelEnableList.tsx +++ b/src/agentMode/ui/ModelEnableList.tsx @@ -1,8 +1,10 @@ import { Badge } from "@/components/ui/badge"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import { FreeModelWarningIcon } from "@/components/ui/FreeModelWarningIcon"; +import { ModelCapabilityIcons, hasCapabilityIcons } from "@/components/ui/model-display"; import { SearchBar } from "@/components/ui/SearchBar"; import { SettingSwitch } from "@/components/ui/setting-switch"; +import type { ModelCapability } from "@/constants"; import { cn } from "@/lib/utils"; import { ChevronRight } from "lucide-react"; import React from "react"; @@ -19,6 +21,8 @@ export interface ModelEnableRow { wireId?: string; /** Whether the model is currently enabled. */ enabled: boolean; + /** Modality icons (vision/websearch) shown beside the label; reasoning is not rendered. */ + capabilities?: ModelCapability[]; /** * `true` for a free model (zero catalog cost) routed through a third party. * Renders a privacy-warning icon + tooltip beside the label, since such @@ -100,6 +104,11 @@ export const ModelEnableList: React.FC = ({
{row.label} {row.isFree && } + {hasCapabilityIcons(row.capabilities) && ( + + + + )}
{row.description && (
{row.description}
diff --git a/src/agentMode/ui/agentModelPickerHelpers.test.ts b/src/agentMode/ui/agentModelPickerHelpers.test.ts index 9934402e..c6ee17d3 100644 --- a/src/agentMode/ui/agentModelPickerHelpers.test.ts +++ b/src/agentMode/ui/agentModelPickerHelpers.test.ts @@ -7,6 +7,7 @@ import { resolveActiveDisplayState, synthesizeAgentEntry, } from "./agentModelPickerHelpers"; +import { ModelCapability } from "@/constants"; import { getModelKeyFromModel } from "@/settings/model"; import type { ModelSelectorEntry } from "@/components/ui/ModelSelector"; import type { @@ -723,3 +724,68 @@ describe("buildEffortOptionsByModelKey", () => { expect(out[getModelKeyFromModel(entries[0])]).toEqual([]); }); }); + +// ---- capability propagation -------------------------------------------- + +describe("buildPickerEntries — persisted capability propagation", () => { + function reported(baseModelId: string, provider: string | null): ModelEntry { + return { baseModelId, name: baseModelId, provider, effortOptions: [] }; + } + + function ctxFor(backendId: "codex" | "claude" | "opencode"): ModelActiveContext { + return { + activeSession: { backendId } as unknown as AgentSession, + activeChatUIState: null, + activeBackendId: backendId, + activeDescriptor: makeDescriptor(backendId), + activeSessionHasHistory: false, + activeModelState: null, + activeCurrentEntry: undefined, + }; + } + + it("surfaces an enabled model's persisted capabilities on its picker entry", () => { + const claude = { + ...makeDescriptor("claude"), + getEnabledModelEntries: () => [ + { + baseModelId: "claude-sonnet-4-5", + name: "Sonnet", + credentialState: "ok" as const, + capabilities: [ModelCapability.VISION, ModelCapability.REASONING], + }, + ], + } as unknown as BackendDescriptor; + const manager = makeManager({ + cachedStateById: { + claude: { + model: makeModelState("claude-sonnet-4-5", [reported("claude-sonnet-4-5", "anthropic")]), + mode: null, + }, + }, + }); + const { entries } = buildPickerEntries(manager, [claude], ctxFor("claude"), emptySettings); + const entry = entries.find((e) => e.name === "claude-sonnet-4-5"); + expect(entry?.capabilities).toEqual([ModelCapability.VISION, ModelCapability.REASONING]); + }); + + it("leaves capabilities undefined when the enabled entry carries none", () => { + const claude = { + ...makeDescriptor("claude"), + getEnabledModelEntries: () => [ + { baseModelId: "claude-sonnet-4-5", name: "Sonnet", credentialState: "ok" as const }, + ], + } as unknown as BackendDescriptor; + const manager = makeManager({ + cachedStateById: { + claude: { + model: makeModelState("claude-sonnet-4-5", [reported("claude-sonnet-4-5", "anthropic")]), + mode: null, + }, + }, + }); + const { entries } = buildPickerEntries(manager, [claude], ctxFor("claude"), emptySettings); + const entry = entries.find((e) => e.name === "claude-sonnet-4-5"); + expect(entry?.capabilities).toBeUndefined(); + }); +}); diff --git a/src/agentMode/ui/agentModelPickerHelpers.ts b/src/agentMode/ui/agentModelPickerHelpers.ts index 2a64e7b8..7cf3a1f2 100644 --- a/src/agentMode/ui/agentModelPickerHelpers.ts +++ b/src/agentMode/ui/agentModelPickerHelpers.ts @@ -1,5 +1,6 @@ import { Notice } from "obsidian"; import { logError } from "@/logger"; +import type { ModelCapability } from "@/constants"; import type { ModelSelectorEntry } from "@/components/ui/ModelSelector"; import type { AgentSession } from "@/agentMode/session/AgentSession"; import type { AgentChatUIState } from "@/agentMode/session/AgentChatUIState"; @@ -130,12 +131,17 @@ function appendFromEnabledEntries( const reported = reportedById.get(enabled.baseModelId); const name = reported?.name || enabled.name || enabled.baseModelId; const subtitle = reported?.description ?? enabled.description; + // Capabilities come from the model's persisted `ConfiguredModel.info` + // modality snapshot. `undefined` (no snapshot) means "unknown" — no icon, + // and the image-send guard leaves the model unblocked. + const capabilities = enabled.capabilities; const entry = synthesizeAgentEntry( enabled.baseModelId, name, descriptor, subtitle, - enabled.isFree + enabled.isFree, + capabilities ); const reason = credentialDisabledReason(enabled.credentialState, !!reported); if (reason) entry._disabledReason = reason; @@ -147,7 +153,14 @@ function appendFromEnabledEntries( const reported = reportedById.get(keepBaseModelId); if (reported) { entries.push( - synthesizeAgentEntry(reported.baseModelId, reported.name, descriptor, reported.description) + synthesizeAgentEntry( + reported.baseModelId, + reported.name, + descriptor, + reported.description, + undefined, + undefined + ) ); } } @@ -173,7 +186,14 @@ export function synthesizeAgentEntry( humanName: string, descriptor: BackendDescriptor, subtitle?: string, - isFree?: boolean + isFree?: boolean, + /** + * Capabilities from the model's persisted `ConfiguredModel.info` snapshot, or + * `undefined` when unknown (no snapshot). Left off the entry entirely when + * undefined so the image-send guard treats the model as "unknown" — never + * blocked. + */ + capabilities?: ModelCapability[] ): ModelSelectorEntry { return { name: baseModelId, @@ -181,6 +201,7 @@ export function synthesizeAgentEntry( enabled: true, isBuiltIn: false, displayName: humanName || baseModelId, + capabilities, _group: descriptor.displayName, _backendId: descriptor.id, _subtitle: subtitle, @@ -312,7 +333,9 @@ export function buildPickerEntries( baseId, ctx.activeCurrentEntry.name, ctx.activeDescriptor, - ctx.activeCurrentEntry.description + ctx.activeCurrentEntry.description, + undefined, + undefined ); entries.unshift(synth); valueKey = getModelKeyFromModel(synth); diff --git a/src/agentMode/ui/useAgentModelPicker.ts b/src/agentMode/ui/useAgentModelPicker.ts index 7d380474..18c7cf0b 100644 --- a/src/agentMode/ui/useAgentModelPicker.ts +++ b/src/agentMode/ui/useAgentModelPicker.ts @@ -100,7 +100,7 @@ export function useAgentModelPicker( const descriptors = useMemo(() => listBackendDescriptors(), []); const signal = useAgentModelSignal(manager, descriptors); return useMemo(() => { - // `signal` is the memo invalidator — referenced here so + // `signal` is a memo invalidator — referenced here so // react-hooks/exhaustive-deps accepts it in the dep array. void signal; return buildAgentModelPicker({ manager, descriptors, settings }); diff --git a/src/components/Chat.tsx b/src/components/Chat.tsx index 1838a6dc..e7b1acc7 100644 --- a/src/components/Chat.tsx +++ b/src/components/Chat.tsx @@ -45,11 +45,11 @@ import CopilotPlugin from "@/main"; import { useIsPlusUser } from "@/plusUtils"; import { ProjectFileManager } from "@/projects/ProjectFileManager"; import { useProjects } from "@/projects/state"; -import { useSettingsValue } from "@/settings/model"; +import { getModelKeyFromModel, useSettingsValue } from "@/settings/model"; import { ChatManagerChatUIState } from "@/state/ChatUIState"; import { FileParserManager } from "@/tools/FileParserManager"; import { ChatMessage } from "@/types/message"; -import { err2String, isPlusChain } from "@/utils"; +import { err2String, isPlusChain, modelSupportsVision } from "@/utils"; import { arrayBufferToBase64 } from "@/utils/base64"; import { appendUniqueFiles } from "@/utils/fileListUtils"; import { Notice, TFile } from "obsidian"; @@ -312,6 +312,23 @@ const ChatInternal: React.FC 0) { + const activeModel = chatModelPicker.models.find( + (m) => getModelKeyFromModel(m) === chatModelPicker.value + ); + if (Array.isArray(activeModel?.capabilities) && !modelSupportsVision(activeModel)) { + const modelLabel = activeModel.displayName || activeModel.name; + new Notice( + `${modelLabel} doesn't support images. Switch to a vision-capable model to send images.` + ); + return; + } + } + try { // Create message content array type MessageContentItem = diff --git a/src/components/chat-components/useChatModelPicker.ts b/src/components/chat-components/useChatModelPicker.ts index 8546c00e..be77f08d 100644 --- a/src/components/chat-components/useChatModelPicker.ts +++ b/src/components/chat-components/useChatModelPicker.ts @@ -1,6 +1,6 @@ -import { ModelCapability } from "@/constants"; import { backendPickerAtomFamily, + capabilitiesFromConfiguredInfo, mapProviderTypeToChatModelProvider, providerRequiresApiKey, resolveChatModelSelectionId, @@ -62,11 +62,10 @@ export function useChatModelPicker(params: { for (const entry of entries) { if (entry.state !== "ok") continue; const { configuredModel, provider, configuredModelId } = entry; - const capabilities: ModelCapability[] = []; - if (configuredModel.info.reasoning) capabilities.push(ModelCapability.REASONING); - if (configuredModel.info.modalities?.input?.includes("image")) { - capabilities.push(ModelCapability.VISION); - } + // Leave capabilities `undefined` when the snapshot carries no modality + // data so unknown models stay unblocked; only a populated array (which may + // be empty) asserts "known". See the image guard in `Chat.tsx`. + const capabilities = capabilitiesFromConfiguredInfo(configuredModel.info); const needsKey = providerRequiresApiKey(provider) && !provider.apiKeyKeychainId; const modelEntry: ModelSelectorEntry = { name: configuredModelId, diff --git a/src/components/ui/model-display.test.tsx b/src/components/ui/model-display.test.tsx new file mode 100644 index 00000000..6a61d9fb --- /dev/null +++ b/src/components/ui/model-display.test.tsx @@ -0,0 +1,92 @@ +import { render } from "@testing-library/react"; +import React from "react"; +import { + ModelCapabilityIcons, + getModelDisplayWithIcons, + hasCapabilityIcons, +} from "./model-display"; +import type { CustomModel } from "@/aiParams"; +import { ModelCapability } from "@/constants"; + +const NO_VISION = "model-cap-no-vision"; + +describe("ModelCapabilityIcons", () => { + it("renders nothing for unknown capabilities (undefined)", () => { + // `undefined` = no modality snapshot. We must NOT assert a missing + // capability — render nothing, distinct from the eye-off below. + const { container, queryByTestId } = render(); + expect(container.querySelectorAll("svg").length).toBe(0); + expect(queryByTestId(NO_VISION)).toBeNull(); + }); + + it("renders the eye-off for a known model that lacks vision (empty array)", () => { + const { container, queryByTestId } = render(); + expect(queryByTestId(NO_VISION)).not.toBeNull(); + expect(container.querySelectorAll("svg").length).toBe(1); + }); + + it("renders no icon for a vision-capable model", () => { + // Vision is the norm — we don't badge it (mirrors the hidden reasoning icon). + const { container, queryByTestId } = render( + + ); + expect(container.querySelectorAll("svg").length).toBe(0); + expect(queryByTestId(NO_VISION)).toBeNull(); + }); + + it("renders no icon for a vision + web-search model (web search is not badged)", () => { + const { container, queryByTestId } = render( + + ); + expect(container.querySelectorAll("svg").length).toBe(0); + expect(queryByTestId(NO_VISION)).toBeNull(); + }); + + it("renders only the eye-off for a web-search-only model (no vision)", () => { + const { container, queryByTestId } = render( + + ); + expect(container.querySelectorAll("svg").length).toBe(1); + expect(queryByTestId(NO_VISION)).not.toBeNull(); + }); + + it("renders the eye-off when the only capability is reasoning (no vision)", () => { + const { container, queryByTestId } = render( + + ); + expect(container.querySelectorAll("svg").length).toBe(1); + expect(queryByTestId(NO_VISION)).not.toBeNull(); + }); +}); + +describe("hasCapabilityIcons", () => { + it("is false for unknown (undefined) and any vision-capable model", () => { + expect(hasCapabilityIcons(undefined)).toBe(false); + expect(hasCapabilityIcons([ModelCapability.VISION])).toBe(false); + expect(hasCapabilityIcons([ModelCapability.VISION, ModelCapability.REASONING])).toBe(false); + expect(hasCapabilityIcons([ModelCapability.VISION, ModelCapability.WEB_SEARCH])).toBe(false); + }); + + it("is true when the model is known to lack vision", () => { + expect(hasCapabilityIcons([])).toBe(true); + expect(hasCapabilityIcons([ModelCapability.REASONING])).toBe(true); + expect(hasCapabilityIcons([ModelCapability.WEB_SEARCH])).toBe(true); + }); +}); + +describe("getModelDisplayWithIcons", () => { + it("shows name and provider, never a modality label", () => { + const model: CustomModel = { + name: "omni", + provider: "openai", + enabled: true, + capabilities: [ModelCapability.VISION, ModelCapability.REASONING, ModelCapability.WEB_SEARCH], + }; + const text = getModelDisplayWithIcons(model); + expect(text).toContain("omni"); + expect(text).toContain("OpenAI"); + expect(text).not.toContain("Websearch"); + expect(text).not.toContain("Vision"); + expect(text).not.toContain("Reasoning"); + }); +}); diff --git a/src/components/ui/model-display.tsx b/src/components/ui/model-display.tsx index 198d7125..6d365bb8 100644 --- a/src/components/ui/model-display.tsx +++ b/src/components/ui/model-display.tsx @@ -1,8 +1,9 @@ import React from "react"; import { CustomModel } from "@/aiParams"; import { getProviderLabel } from "@/utils"; -import { Lightbulb, Eye, Globe } from "lucide-react"; +import { EyeOff } from "lucide-react"; import { ModelCapability } from "@/constants"; +import { HelpTooltip } from "@/components/ui/help-tooltip"; interface ModelDisplayProps { model: CustomModel; @@ -14,47 +15,40 @@ interface ModelCapabilityIconsProps { iconSize?: number; } -const EMPTY_CAPABILITIES: ModelCapability[] = []; +const NO_VISION_LABEL = "This model does not support image inputs."; +/** + * Whether {@link ModelCapabilityIcons} would render the eye-off. Drives the + * surrounding wrapper so it never renders empty (a vision-capable model shows no + * icon) and never hides the eye-off for a model known to lack vision (`[]`). + */ +export function hasCapabilityIcons(capabilities: ModelCapability[] | undefined): boolean { + if (capabilities === undefined) return false; + return !capabilities.includes(ModelCapability.VISION); +} + +/** + * We badge only the exception, not the norm. `undefined` means "unknown" (no + * modality snapshot — e.g. an agent-provided model) and renders nothing; we never + * assert a missing capability we don't actually know about. A defined array is + * "known": flag the absence of vision with a muted eye-off. Vision and reasoning + * themselves render nothing — they're ubiquitous on modern models, so the only + * signal we surface is the warning that a model can't take images. + */ export const ModelCapabilityIcons: React.FC = ({ - capabilities = EMPTY_CAPABILITIES, + capabilities, iconSize = 16, }) => { + if (capabilities === undefined) return null; + if (capabilities.includes(ModelCapability.VISION)) return null; return ( - <> - {capabilities - .sort((a, b) => a.localeCompare(b)) - .map((cap) => { - switch (cap) { - case ModelCapability.REASONING: - return ( - - ); - case ModelCapability.VISION: - return ( - - ); - case ModelCapability.WEB_SEARCH: - return ( - - ); - default: - return null; - } - })} - + + + ); }; @@ -63,7 +57,7 @@ export const ModelDisplay: React.FC = ({ model, iconSize = 14 return (
{displayName} - {model.capabilities && model.capabilities.length > 0 && ( + {hasCapabilityIcons(model.capabilities) && (
@@ -81,20 +75,5 @@ export const getModelDisplayText = (model: CustomModel): string => { export const getModelDisplayWithIcons = (model: CustomModel): string => { const displayName = model.displayName || model.name; const provider = `(${getProviderLabel(model.provider, model)})`; - const icons = - model.capabilities - ?.map((cap) => { - switch (cap) { - case ModelCapability.REASONING: - return "Reasoning"; - case ModelCapability.VISION: - return "Vision"; - case ModelCapability.WEB_SEARCH: - return "Websearch"; - default: - return ""; - } - }) - .join("|") || ""; - return `${displayName} ${provider} ${icons}`; + return `${displayName} ${provider}`; }; diff --git a/src/modelManagement/chatModel/modelCapabilityFlags.test.ts b/src/modelManagement/chatModel/modelCapabilityFlags.test.ts new file mode 100644 index 00000000..f2551250 --- /dev/null +++ b/src/modelManagement/chatModel/modelCapabilityFlags.test.ts @@ -0,0 +1,32 @@ +import { ModelCapability } from "@/constants"; +import type { ModelInfo } from "@/modelManagement/types/catalog"; +import { capabilityListFromModelInfo } from "./modelCapabilityFlags"; + +describe("capabilityListFromModelInfo", () => { + it("maps reasoning + vision to the capability enum values", () => { + const info: ModelInfo = { + id: "m", + displayName: "M", + modalities: { input: ["text", "image"] }, + reasoning: true, + }; + expect(capabilityListFromModelInfo(info)).toEqual([ + ModelCapability.REASONING, + ModelCapability.VISION, + ]); + }); + + it("reads vision from an image input modality only", () => { + const info: ModelInfo = { id: "m", displayName: "M", modalities: { input: ["text", "image"] } }; + expect(capabilityListFromModelInfo(info)).toEqual([ModelCapability.VISION]); + }); + + it("is empty for an input without image", () => { + const info: ModelInfo = { id: "m", displayName: "M", modalities: { input: ["text"] } }; + expect(capabilityListFromModelInfo(info)).toEqual([]); + }); + + it("returns an empty list for a plain model", () => { + expect(capabilityListFromModelInfo({ id: "m", displayName: "M" })).toEqual([]); + }); +}); diff --git a/src/modelManagement/chatModel/modelCapabilityFlags.ts b/src/modelManagement/chatModel/modelCapabilityFlags.ts new file mode 100644 index 00000000..2de38c05 --- /dev/null +++ b/src/modelManagement/chatModel/modelCapabilityFlags.ts @@ -0,0 +1,36 @@ +/** + * Pure helpers deriving a model's capabilities from its `ModelInfo` + * modality/reasoning metadata. `ModelInfo` is the single source of truth: + * vision is `modalities.input` containing `"image"`, reasoning is the + * `reasoning` flag. The derivation here MUST match `configuredModelToCustomModel` + * so what the pickers show equals what the chat bridge derives into + * `CustomModel.capabilities`. + */ +import { ModelCapability } from "@/constants"; +import type { ModelInfo } from "@/modelManagement/types/catalog"; + +const IMAGE_MODALITY = "image"; + +const EMPTY_CAPABILITY_LIST: ModelCapability[] = Object.freeze([]) as unknown as ModelCapability[]; + +export function capabilityListFromModelInfo(info: ModelInfo): ModelCapability[] { + const reasoning = !!info.reasoning; + const vision = !!info.modalities?.input?.includes(IMAGE_MODALITY); + if (!reasoning && !vision) return EMPTY_CAPABILITY_LIST; + const list: ModelCapability[] = []; + if (reasoning) list.push(ModelCapability.REASONING); + if (vision) list.push(ModelCapability.VISION); + return list; +} + +/** + * Capabilities for a persisted/configured model, or `undefined` when the + * snapshot carries no modality data at all. The presence of `modalities` + * is what distinguishes "known to lack vision" (returns a list without + * `VISION`) from "unknown" (`undefined`) — the latter lets a caller fall + * back to the live catalog instead of asserting no vision. Used by the agent + * pickers so a model's vision icon comes from its own `info` first. + */ +export function capabilitiesFromConfiguredInfo(info: ModelInfo): ModelCapability[] | undefined { + return info.modalities ? capabilityListFromModelInfo(info) : undefined; +} diff --git a/src/modelManagement/index.ts b/src/modelManagement/index.ts index 81b7dd83..b1f177c3 100644 --- a/src/modelManagement/index.ts +++ b/src/modelManagement/index.ts @@ -49,6 +49,10 @@ export { export type { ResolvedChatBackendEntry } from "./chatModel/chatModelSelection"; export { resolveChatBackendModel } from "./chatModel/resolveChatBackendModel"; export type { ChatBackendResolution } from "./chatModel/resolveChatBackendModel"; +export { + capabilityListFromModelInfo, + capabilitiesFromConfiguredInfo, +} from "./chatModel/modelCapabilityFlags"; // --------------------------------------------------------------------------- // Provider adapter contract diff --git a/src/modelManagement/ui/components/ByokGlobalTable.test.tsx b/src/modelManagement/ui/components/ByokGlobalTable.test.tsx index e1b21808..37308ffe 100644 --- a/src/modelManagement/ui/components/ByokGlobalTable.test.tsx +++ b/src/modelManagement/ui/components/ByokGlobalTable.test.tsx @@ -73,6 +73,47 @@ describe("ByokGlobalTable", () => { expect(screen.getByText("Embedding")).toBeTruthy(); }); + it("badges the no-vision exception on text-only rows but not vision rows", () => { + const withModalities: ByokTableGroup = { + ...group, + models: [ + { + configuredModelId: "text-only", + providerId: "p1", + info: { + id: "text-only", + displayName: "Text Only", + modalities: { input: ["text"] }, + }, + configuredAt: 0, + }, + { + configuredModelId: "vision", + providerId: "p1", + info: { + id: "vision", + displayName: "Vision Model", + modalities: { input: ["text", "image"] }, + }, + configuredAt: 0, + }, + ], + }; + render( + + ); + // A model KNOWN to lack image input shows the muted eye-off. + expect( + screen + .getByTestId("byok-model-text-only") + .querySelector('[data-testid="model-cap-no-vision"]') + ).not.toBeNull(); + // A vision-capable model is the norm and shows no capability icon. + expect( + screen.getByTestId("byok-model-vision").querySelector('[data-testid="model-cap-no-vision"]') + ).toBeNull(); + }); + it("collapses the model rows when the section header is clicked", () => { render(); expect(screen.getByText("Claude Sonnet 4.5")).toBeTruthy(); diff --git a/src/modelManagement/ui/components/ByokGlobalTable.tsx b/src/modelManagement/ui/components/ByokGlobalTable.tsx index d23e8df6..54ada9d1 100644 --- a/src/modelManagement/ui/components/ByokGlobalTable.tsx +++ b/src/modelManagement/ui/components/ByokGlobalTable.tsx @@ -15,7 +15,9 @@ import { DropdownMenuItem, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; +import { ModelCapabilityIcons, hasCapabilityIcons } from "@/components/ui/model-display"; import { cn } from "@/lib/utils"; +import { capabilitiesFromConfiguredInfo } from "@/modelManagement/chatModel/modelCapabilityFlags"; import type { ConfiguredModel, Provider } from "@/modelManagement/types/persisted"; import { formatContextWindow, @@ -193,6 +195,7 @@ const ProviderSection: React.FC = ({ const ModelRow: React.FC<{ model: ConfiguredModel }> = ({ model }) => { const contextLabel = formatContextWindow(model.info.limits?.context); const releaseLabel = formatReleaseDate(model.info.releaseDate); + const capabilities = capabilitiesFromConfiguredInfo(model.info); return (
= ({ model }) => { Embedding )} + {hasCapabilityIcons(capabilities) && ( + + + + )}
{contextLabel} diff --git a/src/modelManagement/ui/components/ModelChecklist.test.tsx b/src/modelManagement/ui/components/ModelChecklist.test.tsx index 811300cf..a70f4d99 100644 --- a/src/modelManagement/ui/components/ModelChecklist.test.tsx +++ b/src/modelManagement/ui/components/ModelChecklist.test.tsx @@ -47,6 +47,38 @@ describe("ModelChecklist", () => { expect(screen.getByText("Embedding")).toBeTruthy(); }); + it("badges only the no-vision exception: nothing for vision or unknown, an eye-off for known text-only", () => { + const VISION_REASON: ModelInfo = { + id: "omni", + displayName: "Omni", + modalities: { input: ["text", "image"] }, + reasoning: true, + }; + const TEXT_ONLY: ModelInfo = { + id: "text-only", + displayName: "Text Only", + modalities: { input: ["text"] }, + }; + render( + ()} + onToggle={jest.fn()} + onAddId={jest.fn()} + /> + ); + // Vision is the norm (not badged) and reasoning is hidden — a vision-capable + // model shows no capability icon. + expect(screen.getByTestId("model-row-omni").querySelectorAll("svg").length).toBe(0); + // A model KNOWN to lack image input shows the muted eye-off. + expect( + screen.getByTestId("model-row-text-only").querySelector('[data-testid="model-cap-no-vision"]') + ).not.toBeNull(); + // A model whose snapshot has no modality data is "unknown" — render nothing, + // never assert a missing capability. + expect(screen.getByTestId("model-row-gpt-5").querySelectorAll("svg").length).toBe(0); + }); + it("emits onToggle with the wire id when a checkbox is clicked", () => { const onToggle = jest.fn(); renderList({ availableModels: [PLAIN], onToggle }); diff --git a/src/modelManagement/ui/components/ModelChecklist.tsx b/src/modelManagement/ui/components/ModelChecklist.tsx index 0f7b7002..c1167b33 100644 --- a/src/modelManagement/ui/components/ModelChecklist.tsx +++ b/src/modelManagement/ui/components/ModelChecklist.tsx @@ -15,7 +15,9 @@ import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; import { Input } from "@/components/ui/input"; +import { ModelCapabilityIcons, hasCapabilityIcons } from "@/components/ui/model-display"; import { cn } from "@/lib/utils"; +import { capabilitiesFromConfiguredInfo } from "@/modelManagement/chatModel/modelCapabilityFlags"; import type { ModelInfo } from "@/modelManagement/types/catalog"; import { formatContextWindow, @@ -117,6 +119,7 @@ export const ModelChecklist: React.FC = ({ const releaseLabel = formatReleaseDate(model.releaseDate); const isLastChecked = index === checkedCount - 1 && checkedCount < filtered.length; const removable = onRemoveId !== undefined && customIds?.has(model.id) === true; + const capabilities = capabilitiesFromConfiguredInfo(model); return ( {contextLabel} diff --git a/src/modelManagement/ui/dialogs/ConfigureProviderDialog.test.tsx b/src/modelManagement/ui/dialogs/ConfigureProviderDialog.test.tsx index 1663075d..8e0e6e48 100644 --- a/src/modelManagement/ui/dialogs/ConfigureProviderDialog.test.tsx +++ b/src/modelManagement/ui/dialogs/ConfigureProviderDialog.test.tsx @@ -29,7 +29,11 @@ jest.mock("@/modelManagement/ui/ModelManagementContext", () => ({ configuredModelRegistry: { bulkSet: mockBulkSet }, backendConfigRegistry: { enableModel: mockEnableModel, removeRefs: mockRemoveRefs }, coordinator: { removeProvider: jest.fn() }, - catalogService: { getProvider: mockGetProvider }, + catalogService: { + getProvider: mockGetProvider, + ensureLoaded: jest.fn().mockResolvedValue(undefined), + onChange: jest.fn().mockReturnValue(() => {}), + }, }), })); // eslint-disable-next-line @eslint-react/hooks-extra/no-unnecessary-use-prefix -- mocks the real `useApp` hook; the name must match the export diff --git a/src/modelManagement/ui/dialogs/ConfigureProviderDialog.tsx b/src/modelManagement/ui/dialogs/ConfigureProviderDialog.tsx index e639c65b..206d95a6 100644 --- a/src/modelManagement/ui/dialogs/ConfigureProviderDialog.tsx +++ b/src/modelManagement/ui/dialogs/ConfigureProviderDialog.tsx @@ -182,13 +182,36 @@ const ConfigureProviderBody: React.FC = ({ ? provider.origin.catalogProviderId : undefined; + // Warm the catalog and re-render when it (re)populates, so a cold first open + // enriches rows the moment `models.dev` lands instead of capturing an empty + // snapshot forever (the memo below would otherwise never recompute). Mirrors + // the load/subscribe pattern in `ByokPanel`. + const [catalogVersion, setCatalogVersion] = useState(0); + useEffect(() => { + let cancelled = false; + const bump = (): void => { + if (!cancelled) setCatalogVersion((v) => v + 1); + }; + const unsub = api.catalogService.onChange(bump); + api.catalogService + .ensureLoaded() + .then(bump) + .catch((err) => logError("[ConfigureProviderDialog] catalog ensureLoaded failed", err)); + return () => { + cancelled = true; + unsub(); + }; + }, [api]); + // Catalog metadata for row enrichment only — never seeds the candidate // pool. Live catalog wins; on miss (offline, legacy id) we fall back to // an empty record and rows render id-only until metadata loads. const catalogMetadata = useMemo>(() => { if (!catalogProviderId) return EMPTY_METADATA; return api.catalogService.getProvider(catalogProviderId)?.models ?? EMPTY_METADATA; - }, [catalogProviderId, api]); + // `catalogVersion` re-runs this once the catalog lands. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [catalogProviderId, api, catalogVersion]); const [displayName, setDisplayName] = useState(() => state.mode === "new" ? state.source.displayName : (provider?.displayName ?? "") diff --git a/src/modelManagement/ui/dialogs/useModelCandidatePool.ts b/src/modelManagement/ui/dialogs/useModelCandidatePool.ts index 973b547a..e59c1862 100644 --- a/src/modelManagement/ui/dialogs/useModelCandidatePool.ts +++ b/src/modelManagement/ui/dialogs/useModelCandidatePool.ts @@ -239,10 +239,12 @@ export function useModelCandidatePool({ [existingByWireId] ); - // Resolved `ModelInfo`s for the selected ids — same precedence the picker - // rendered, so what gets persisted matches what the user saw. + // Resolved `ModelInfo`s for the selected ids — same catalog → snapshot → + // synthesize precedence the picker rendered, so what gets persisted matches + // what the user saw. Capabilities ride along on each `ModelInfo` (its + // `modalities` / `reasoning`); there's no separate overlay. const buildSelectedModelInfos = useCallback( - (): ModelInfo[] => [...selectedWireIds].map(resolveModelInfo), + (): ModelInfo[] => [...selectedWireIds].map((id) => resolveModelInfo(id)), [selectedWireIds, resolveModelInfo] ); diff --git a/src/settings/v2/components/configuredModelGrouping.test.ts b/src/settings/v2/components/configuredModelGrouping.test.ts index 1e4b1513..021449cb 100644 --- a/src/settings/v2/components/configuredModelGrouping.test.ts +++ b/src/settings/v2/components/configuredModelGrouping.test.ts @@ -7,6 +7,7 @@ import { type Candidate, } from "./configuredModelGrouping"; import type { ConfiguredModel, Provider } from "@/modelManagement"; +import { ModelCapability } from "@/constants"; function byokProvider(id: string, displayName: string): Provider { return { @@ -225,6 +226,70 @@ describe("toRow", () => { expect(row.description).toBe("Opus 4.7 with 1M context · Most capable for complex work"); }); + it("derives the vision capability from info.modalities (for the row's icon)", () => { + const provider = agentProvider("claude", "claude", "Claude"); + const visionRow = toRow({ + configuredModel: { + configuredModelId: "cm", + providerId: "claude", + info: { + id: "claude-sonnet-4-5", + displayName: "Claude Sonnet 4.5", + modalities: { input: ["text", "image"] }, + }, + configuredAt: 0, + }, + provider, + enabled: true, + }); + expect(visionRow.capabilities).toContain(ModelCapability.VISION); + + // A model whose snapshot lacks image input carries no vision icon. + const noVisionRow = toRow({ + configuredModel: { + configuredModelId: "cm2", + providerId: "claude", + info: { id: "text-only", displayName: "Text Only", modalities: { input: ["text"] } }, + configuredAt: 0, + }, + provider, + enabled: true, + }); + expect(noVisionRow.capabilities ?? []).not.toContain(ModelCapability.VISION); + }); + + it("leaves capabilities undefined for an unknown snapshot, defined for a known one", () => { + const provider = agentProvider("claude", "claude", "Claude"); + // No `modalities` at all — we don't know, so the row stays "unknown" + // (undefined). The picker renders nothing rather than asserting no vision. + const unknownRow = toRow({ + configuredModel: { + configuredModelId: "cm", + providerId: "claude", + info: { id: "claude-sonnet-4-5", displayName: "Claude Sonnet 4.5" }, + configuredAt: 0, + }, + provider, + enabled: true, + }); + expect(unknownRow.capabilities).toBeUndefined(); + + // A snapshot WITH modalities but no image input is "known to lack vision" — + // a defined array, so the picker renders the eye-off rather than nothing. + const knownNoVisionRow = toRow({ + configuredModel: { + configuredModelId: "cm2", + providerId: "claude", + info: { id: "text-only", displayName: "Text Only", modalities: { input: ["text"] } }, + configuredAt: 0, + }, + provider, + enabled: true, + }); + expect(knownNoVisionRow.capabilities).toBeDefined(); + expect(knownNoVisionRow.capabilities).not.toContain(ModelCapability.VISION); + }); + it("flags opencode Zen models (opencode/ wire id) as free, others not", () => { const provider = agentProvider("oc", "opencode", "opencode"); const zen = toRow({ diff --git a/src/settings/v2/components/configuredModelGrouping.ts b/src/settings/v2/components/configuredModelGrouping.ts index d57c121c..71c1e20b 100644 --- a/src/settings/v2/components/configuredModelGrouping.ts +++ b/src/settings/v2/components/configuredModelGrouping.ts @@ -1,7 +1,11 @@ /** Pure grouping logic for `ConfiguredModelEnableList`, split from the React container so it's testable with plain data. */ import { isOpencodeZenWireId, type ModelEnableGroup, type ModelEnableRow } from "@/agentMode"; -import type { ConfiguredModel, Provider } from "@/modelManagement"; +import { + capabilitiesFromConfiguredInfo, + type ConfiguredModel, + type Provider, +} from "@/modelManagement"; /** One candidate model joined to its provider, plus current enabled state. */ export interface Candidate { @@ -119,6 +123,7 @@ export function toRow(candidate: Candidate): ModelEnableRow { wireId: id, enabled, isFree: isOpencodeZenWireId(id), + capabilities: capabilitiesFromConfiguredInfo(configuredModel.info), }; } diff --git a/src/utils.ts b/src/utils.ts index 794734b3..b957ea98 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -8,6 +8,7 @@ import { ALLOWED_NOTE_CONTEXT_EXTENSIONS, ChatModelProviders, EmbeddingModelProviders, + ModelCapability, NOMIC_EMBED_TEXT, Provider, ProviderInfo, @@ -822,6 +823,12 @@ export function findCustomModel(modelKey: string, activeModels: CustomModel[]): return model; } +// Capabilities can be undefined when a model's vision support is simply unknown; +// callers that hard-block on missing vision must treat undefined as "unknown", not "no". +export function modelSupportsVision(model: CustomModel): boolean { + return !!model.capabilities?.includes(ModelCapability.VISION); +} + export function getProviderInfo(provider: string): ProviderMetadata { const info = ProviderInfo[provider as Provider]; return {