mirror of
https://github.com/logancyang/obsidian-copilot.git
synced 2026-07-22 07:50:24 +00:00
fix(agent-mode): show attached images in the posted user message (#2609)
* fix(agent-mode): show attached images in the posted user message
Attached images were delivered to the model but never appeared in the posted
user bubble: sendPrompt stored the user message without a `content` field and
forwarded the image promptContent blocks only to the backend. The renderer
(ChatSingleMessage) shows images only from `content` entries shaped
`{ type: "image_url", image_url: { url } }`.
Project the outgoing image promptContent blocks into that display shape
(base64 -> data URL) and store them as the user message's `content`, leaving
the backend promptContent path untouched. Markdown autosave still serializes
only message text, so saved notes are not bloated with base64.
Closes logancyang/obsidian-copilot-preview#157
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
* fix(agent-mode): keep prompt text visible alongside attached images
Address Codex review: when a user message has a content array, ChatSingleMessage
renders text only from a `type: "text"` content item. buildUserDisplayContent
emitted image_url entries only, so a mixed text+image prompt hid the user's
text. Include the prompt text as the first content entry (text-first, then
images, mirroring the legacy chat composer); image-only messages still omit the
text entry.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---------
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
894c21d056
commit
4c3044f4b5
2 changed files with 91 additions and 2 deletions
|
|
@ -1,6 +1,11 @@
|
|||
import { AI_SENDER, USER_SENDER } from "@/constants";
|
||||
import type { TFile } from "obsidian";
|
||||
import { AgentSession, buildPromptBlocks, tryReadExitPlanModeCall } from "./AgentSession";
|
||||
import {
|
||||
AgentSession,
|
||||
buildPromptBlocks,
|
||||
buildUserDisplayContent,
|
||||
tryReadExitPlanModeCall,
|
||||
} from "./AgentSession";
|
||||
import { AuthRequiredError, MethodUnsupportedError } from "./errors";
|
||||
import type {
|
||||
AgentToolCallOutput,
|
||||
|
|
@ -318,6 +323,33 @@ describe("AgentSession.restoreLabel", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("buildUserDisplayContent", () => {
|
||||
it("returns undefined when there are no images", () => {
|
||||
expect(buildUserDisplayContent("hi")).toBeUndefined();
|
||||
expect(buildUserDisplayContent("hi", [])).toBeUndefined();
|
||||
expect(buildUserDisplayContent("hi", [{ type: "text", text: "x" }])).toBeUndefined();
|
||||
});
|
||||
|
||||
it("puts the prompt text first, then an image_url entry per image", () => {
|
||||
expect(
|
||||
buildUserDisplayContent("describe these", [
|
||||
{ type: "image", mimeType: "image/png", data: "AAA=" },
|
||||
{ type: "image", mimeType: "image/jpeg", data: "BBB=" },
|
||||
])
|
||||
).toEqual([
|
||||
{ type: "text", text: "describe these" },
|
||||
{ type: "image_url", image_url: { url: "data:image/png;base64,AAA=" } },
|
||||
{ type: "image_url", image_url: { url: "data:image/jpeg;base64,BBB=" } },
|
||||
]);
|
||||
});
|
||||
|
||||
it("omits the text entry for an image-only message", () => {
|
||||
expect(
|
||||
buildUserDisplayContent(" ", [{ type: "image", mimeType: "image/png", data: "AAA=" }])
|
||||
).toEqual([{ type: "image_url", image_url: { url: "data:image/png;base64,AAA=" } }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("AgentSession.sendPrompt", () => {
|
||||
it("appends user + placeholder synchronously and resolves on stopReason", async () => {
|
||||
const mock = makeMockBackend();
|
||||
|
|
@ -361,7 +393,13 @@ describe("AgentSession.sendPrompt", () => {
|
|||
]).turn;
|
||||
const messages = session.store.getDisplayMessages();
|
||||
expect(messages[0].message).toBe("describe");
|
||||
expect(messages[0].content).toBeUndefined();
|
||||
// The posted user bubble carries the prompt text plus the image as a
|
||||
// renderable data-URL entry, while the backend still receives the original
|
||||
// base64 image block.
|
||||
expect(messages[0].content).toEqual([
|
||||
{ type: "text", text: "describe" },
|
||||
{ type: "image_url", image_url: { url: "data:image/png;base64,aGVsbG8=" } },
|
||||
]);
|
||||
expect(mock.prompt).toHaveBeenCalledWith({
|
||||
sessionId: "acp-1",
|
||||
prompt: [
|
||||
|
|
@ -371,6 +409,18 @@ describe("AgentSession.sendPrompt", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("leaves the user message content unset when no images are attached", () => {
|
||||
const mock = makeMockBackend();
|
||||
const session = new AgentSession({
|
||||
backend: mock.asBackend,
|
||||
backendSessionId: "acp-1",
|
||||
internalId: "internal-1",
|
||||
backendId: "opencode",
|
||||
});
|
||||
session.sendPrompt("just text");
|
||||
expect(session.store.getDisplayMessages()[0].content).toBeUndefined();
|
||||
});
|
||||
|
||||
it("rejects if a turn is already in flight", () => {
|
||||
const mock = makeMockBackend();
|
||||
const session = new AgentSession({
|
||||
|
|
|
|||
|
|
@ -750,6 +750,9 @@ export class AgentSession {
|
|||
timestamp: formatDateTime(new Date()),
|
||||
isVisible: true,
|
||||
context,
|
||||
// Surface attached images in the posted bubble. The backend consumes the
|
||||
// original `promptContent` image blocks; this is a display-only projection.
|
||||
content: buildUserDisplayContent(displayText, promptContent),
|
||||
};
|
||||
const userMessageId = this.store.addMessage(userMessage);
|
||||
|
||||
|
|
@ -1506,6 +1509,42 @@ function tryParseJsonObject(s: string): Record<string, unknown> | null {
|
|||
}
|
||||
}
|
||||
|
||||
type DisplayContentItem =
|
||||
| { type: "text"; text: string }
|
||||
| { type: "image_url"; image_url: { url: string } };
|
||||
|
||||
/**
|
||||
* Project a user prompt's text + outgoing image blocks into the display
|
||||
* `content` shape the chat renderer understands: `ChatSingleMessage` renders a
|
||||
* `content` array of `text` and `image_url` entries (text first, then images,
|
||||
* mirroring the legacy chat composer). The backend still consumes the original
|
||||
* `PromptContent` image blocks (base64); this is a display-only projection so
|
||||
* attached images show in the posted user bubble.
|
||||
*
|
||||
* Returns undefined when there are no images — text-only messages render via the
|
||||
* renderer's plain `message` fallback and don't need a `content` array. When
|
||||
* images are present, the text entry is included only if there is prompt text,
|
||||
* so an image-only message doesn't render an empty text line.
|
||||
*/
|
||||
export function buildUserDisplayContent(
|
||||
displayText: string,
|
||||
promptContent?: PromptContent[]
|
||||
): DisplayContentItem[] | undefined {
|
||||
const images = (promptContent ?? []).filter(
|
||||
(p): p is Extract<PromptContent, { type: "image" }> => p.type === "image"
|
||||
);
|
||||
if (images.length === 0) return undefined;
|
||||
const content: DisplayContentItem[] = [];
|
||||
if (displayText.trim()) content.push({ type: "text", text: displayText });
|
||||
for (const img of images) {
|
||||
content.push({
|
||||
type: "image_url",
|
||||
image_url: { url: `data:${img.mimeType};base64,${img.data}` },
|
||||
});
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
export function buildPromptBlocks(
|
||||
displayText: string,
|
||||
context?: MessageContext,
|
||||
|
|
|
|||
Loading…
Reference in a new issue