fix(agent-mode): show attached images in the posted user message (#2609)

* fix(agent-mode): show attached images in the posted user message

Attached images were delivered to the model but never appeared in the posted
user bubble: sendPrompt stored the user message without a `content` field and
forwarded the image promptContent blocks only to the backend. The renderer
(ChatSingleMessage) shows images only from `content` entries shaped
`{ type: "image_url", image_url: { url } }`.

Project the outgoing image promptContent blocks into that display shape
(base64 -> data URL) and store them as the user message's `content`, leaving
the backend promptContent path untouched. Markdown autosave still serializes
only message text, so saved notes are not bloated with base64.

Closes logancyang/obsidian-copilot-preview#157

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* fix(agent-mode): keep prompt text visible alongside attached images

Address Codex review: when a user message has a content array, ChatSingleMessage
renders text only from a `type: "text"` content item. buildUserDisplayContent
emitted image_url entries only, so a mixed text+image prompt hid the user's
text. Include the prompt text as the first content entry (text-first, then
images, mirroring the legacy chat composer); image-only messages still omit the
text entry.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Logan Yang 2026-06-13 22:53:09 -07:00 committed by GitHub
parent 894c21d056
commit 4c3044f4b5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 91 additions and 2 deletions

View file

@ -1,6 +1,11 @@
import { AI_SENDER, USER_SENDER } from "@/constants";
import type { TFile } from "obsidian";
import { AgentSession, buildPromptBlocks, tryReadExitPlanModeCall } from "./AgentSession";
import {
AgentSession,
buildPromptBlocks,
buildUserDisplayContent,
tryReadExitPlanModeCall,
} from "./AgentSession";
import { AuthRequiredError, MethodUnsupportedError } from "./errors";
import type {
AgentToolCallOutput,
@ -318,6 +323,33 @@ describe("AgentSession.restoreLabel", () => {
});
});
describe("buildUserDisplayContent", () => {
it("returns undefined when there are no images", () => {
expect(buildUserDisplayContent("hi")).toBeUndefined();
expect(buildUserDisplayContent("hi", [])).toBeUndefined();
expect(buildUserDisplayContent("hi", [{ type: "text", text: "x" }])).toBeUndefined();
});
it("puts the prompt text first, then an image_url entry per image", () => {
expect(
buildUserDisplayContent("describe these", [
{ type: "image", mimeType: "image/png", data: "AAA=" },
{ type: "image", mimeType: "image/jpeg", data: "BBB=" },
])
).toEqual([
{ type: "text", text: "describe these" },
{ type: "image_url", image_url: { url: "data:image/png;base64,AAA=" } },
{ type: "image_url", image_url: { url: "data:image/jpeg;base64,BBB=" } },
]);
});
it("omits the text entry for an image-only message", () => {
expect(
buildUserDisplayContent(" ", [{ type: "image", mimeType: "image/png", data: "AAA=" }])
).toEqual([{ type: "image_url", image_url: { url: "data:image/png;base64,AAA=" } }]);
});
});
describe("AgentSession.sendPrompt", () => {
it("appends user + placeholder synchronously and resolves on stopReason", async () => {
const mock = makeMockBackend();
@ -361,7 +393,13 @@ describe("AgentSession.sendPrompt", () => {
]).turn;
const messages = session.store.getDisplayMessages();
expect(messages[0].message).toBe("describe");
expect(messages[0].content).toBeUndefined();
// The posted user bubble carries the prompt text plus the image as a
// renderable data-URL entry, while the backend still receives the original
// base64 image block.
expect(messages[0].content).toEqual([
{ type: "text", text: "describe" },
{ type: "image_url", image_url: { url: "data:image/png;base64,aGVsbG8=" } },
]);
expect(mock.prompt).toHaveBeenCalledWith({
sessionId: "acp-1",
prompt: [
@ -371,6 +409,18 @@ describe("AgentSession.sendPrompt", () => {
});
});
it("leaves the user message content unset when no images are attached", () => {
const mock = makeMockBackend();
const session = new AgentSession({
backend: mock.asBackend,
backendSessionId: "acp-1",
internalId: "internal-1",
backendId: "opencode",
});
session.sendPrompt("just text");
expect(session.store.getDisplayMessages()[0].content).toBeUndefined();
});
it("rejects if a turn is already in flight", () => {
const mock = makeMockBackend();
const session = new AgentSession({

View file

@ -750,6 +750,9 @@ export class AgentSession {
timestamp: formatDateTime(new Date()),
isVisible: true,
context,
// Surface attached images in the posted bubble. The backend consumes the
// original `promptContent` image blocks; this is a display-only projection.
content: buildUserDisplayContent(displayText, promptContent),
};
const userMessageId = this.store.addMessage(userMessage);
@ -1506,6 +1509,42 @@ function tryParseJsonObject(s: string): Record<string, unknown> | null {
}
}
type DisplayContentItem =
| { type: "text"; text: string }
| { type: "image_url"; image_url: { url: string } };
/**
* Project a user prompt's text + outgoing image blocks into the display
* `content` shape the chat renderer understands: `ChatSingleMessage` renders a
* `content` array of `text` and `image_url` entries (text first, then images,
* mirroring the legacy chat composer). The backend still consumes the original
* `PromptContent` image blocks (base64); this is a display-only projection so
* attached images show in the posted user bubble.
*
* Returns undefined when there are no images text-only messages render via the
* renderer's plain `message` fallback and don't need a `content` array. When
* images are present, the text entry is included only if there is prompt text,
* so an image-only message doesn't render an empty text line.
*/
export function buildUserDisplayContent(
displayText: string,
promptContent?: PromptContent[]
): DisplayContentItem[] | undefined {
const images = (promptContent ?? []).filter(
(p): p is Extract<PromptContent, { type: "image" }> => p.type === "image"
);
if (images.length === 0) return undefined;
const content: DisplayContentItem[] = [];
if (displayText.trim()) content.push({ type: "text", text: displayText });
for (const img of images) {
content.push({
type: "image_url",
image_url: { url: `data:${img.mimeType};base64,${img.data}` },
});
}
return content;
}
export function buildPromptBlocks(
displayText: string,
context?: MessageContext,