diff --git a/docs/TOOLS.md b/docs/TOOLS.md index 952fa1e0..4f75dada 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -71,6 +71,14 @@ The system uses a three-layer approach for providing tool instructions to LLMs: - Only for persistent model-specific failures - Keep minimal and targeted +### localSearch CiC Prompting Flow + +- CiC: Corpus in Context https://arxiv.org/pdf/2406.13121 +- **Instruction First**: `CopilotPlusChainRunner` now assembles the localSearch payload via `buildLocalSearchInnerContent`, ensuring citation guidance (e.g., `` rules) tops the XML block before any documents. +- **Documents Next**: Search hits are serialized once through `formatSearchResultsForLLM`; the helper simply appends them after guidance, keeping the documents section untouched but clearly separated. +- **Question Last**: `renderCiCMessage` formats the final prompt so any context precedes the user's original query, optionally labeling it (`Question:`) when only vault search was used; this matches the CiC recommendation for instruction → context → query ordering. +- **Reusable Wrapping**: `wrapLocalSearchPayload` centralizes the `` tag creation (including optional `timeRange`), making the layout reusable for future chains without copying string glue. + ## Current Implementation ### Core Files diff --git a/src/LLMProviders/chainRunner/CopilotPlusChainRunner.ts b/src/LLMProviders/chainRunner/CopilotPlusChainRunner.ts index d4bbd9dd..6d191300 100644 --- a/src/LLMProviders/chainRunner/CopilotPlusChainRunner.ts +++ b/src/LLMProviders/chainRunner/CopilotPlusChainRunner.ts @@ -1,4 +1,5 @@ import { getStandaloneQuestion } from "@/chainUtils"; +import { AVAILABLE_TOOLS } from "@/components/chat-components/constants/tools"; import { ABORT_REASON, COMPOSER_OUTPUT_INSTRUCTIONS, @@ -23,28 +24,32 @@ import { getApiErrorMessage, getMessageRole, withSuppressedTokenWarnings } from import { BaseChatModel } from "@langchain/core/language_models/chat_models"; import { IntentAnalyzer } from "../intentAnalyzer"; import { BaseChainRunner } from "./BaseChainRunner"; -import { - formatSourceCatalog, - getCitationInstructions, - sanitizeContentForCitations, - addFallbackSources, - type SourceCatalogEntry, -} from "./utils/citationUtils"; import { ActionBlockStreamer } from "./utils/ActionBlockStreamer"; import { addChatHistoryToMessages, processedMessagesToTextOnly, processRawChatHistory, } from "./utils/chatHistoryUtils"; +import { + addFallbackSources, + formatSourceCatalog, + getCitationInstructions, + sanitizeContentForCitations, + type SourceCatalogEntry, +} from "./utils/citationUtils"; import { extractSourcesFromSearchResults, formatSearchResultsForLLM, formatSearchResultStringForLLM, logSearchResultsDebugTable, } from "./utils/searchResultUtils"; +import { + buildLocalSearchInnerContent, + renderCiCMessage, + wrapLocalSearchPayload, +} from "./utils/cicPromptUtils"; import { ThinkBlockStreamer } from "./utils/ThinkBlockStreamer"; import { deduplicateSources } from "./utils/toolExecution"; -import { AVAILABLE_TOOLS } from "@/components/chat-components/constants/tools"; export class CopilotPlusChainRunner extends BaseChainRunner { private async processImageUrls(urls: string[]): Promise { @@ -616,46 +621,47 @@ export class CopilotPlusChainRunner extends BaseChainRunner { } } - if (toolOutputs.length > 0) { - const validOutputs = toolOutputs.filter((output) => output.output != null); - if (validOutputs.length > 0) { - // Don't add "Additional context" header if only localSearch with results to maintain QA format - const contextHeader = - hasLocalSearchWithResults && validOutputs.length === 1 - ? "" - : "\n\n# Additional context:\n\n"; - context = - contextHeader + - validOutputs - .map((output) => { - let content = output.output; + const validOutputs = toolOutputs.filter((output) => output.output != null); - // localSearch results are already formatted by executeToolCalls - // No need for special processing here + if (validOutputs.length > 0) { + // Don't add "Additional context" header if only localSearch with results to maintain QA format + const contextHeader = + hasLocalSearchWithResults && validOutputs.length === 1 + ? "" + : "\n\n# Additional context:\n\n"; + const rawContext = + contextHeader + + validOutputs + .map((output) => { + let content = output.output; - // Ensure content is string - if (typeof content !== "string") { - content = JSON.stringify(content); - } + // localSearch results are already formatted by executeToolCalls + // No need for special processing here - // localSearch is already wrapped in XML tags, don't double-wrap - if (output.tool === "localSearch") { - return content; - } + // Ensure content is string + if (typeof content !== "string") { + content = JSON.stringify(content); + } - // All other tools get wrapped consistently - return `<${output.tool}>\n${content}\n`; - }) - .join("\n\n"); - } + // localSearch is already wrapped in XML tags, don't double-wrap + if (output.tool === "localSearch") { + return content; + } + + // All other tools get wrapped consistently + return `<${output.tool}>\n${content}\n`; + }) + .join("\n\n"); + + context = rawContext.trim(); } - // For QA format when only localSearch with results is present - if (hasLocalSearchWithResults && toolOutputs.filter((o) => o.output != null).length === 1) { - return `${context}\n\nQuestion: ${userMessage}`; + if (!context) { + return userMessage; } - return `${userMessage}${context}`; + const shouldLabelQuestion = hasLocalSearchWithResults && validOutputs.length === 1; + return renderCiCMessage(context, userMessage, shouldLabelQuestion); } protected getTimeExpression(toolCalls: any[]): string { @@ -724,10 +730,10 @@ export class CopilotPlusChainRunner extends BaseChainRunner { const settings = getSettings(); const guidance = getCitationInstructions(settings.enableInlineCitations, catalogLines); + const innerContent = buildLocalSearchInnerContent(guidance, formattedContent); + // Wrap in XML-like tags for better LLM understanding - return timeExpression - ? `\n${formattedContent}${guidance}\n` - : `\n${formattedContent}${guidance}\n`; + return wrapLocalSearchPayload(innerContent, timeExpression); } /** diff --git a/src/LLMProviders/chainRunner/utils/cicPromptUtils.test.ts b/src/LLMProviders/chainRunner/utils/cicPromptUtils.test.ts new file mode 100644 index 00000000..72079226 --- /dev/null +++ b/src/LLMProviders/chainRunner/utils/cicPromptUtils.test.ts @@ -0,0 +1,50 @@ +import { + buildLocalSearchInnerContent, + renderCiCMessage, + wrapLocalSearchPayload, +} from "./cicPromptUtils"; + +describe("cicPromptUtils", () => { + describe("buildLocalSearchInnerContent", () => { + it("orders guidance before documents and trims whitespace", () => { + const guidance = "\nRules\n"; + const documents = "\nDoc\n"; + + const inner = buildLocalSearchInnerContent(guidance, documents); + + expect(inner).toBe("Rules\n\nDoc"); + }); + + it("returns empty string when both inputs are blank", () => { + expect(buildLocalSearchInnerContent("", "")).toBe(""); + }); + }); + + describe("wrapLocalSearchPayload", () => { + it("wraps content with localSearch tag and preserves time range", () => { + const content = "Rules\n\nDoc"; + + const wrapped = wrapLocalSearchPayload(content, "last week"); + + expect(wrapped).toBe( + '\nRules\n\nDoc\n' + ); + }); + + it("omits payload whitespace when content is empty", () => { + expect(wrapLocalSearchPayload("", "")).toBe(""); + }); + }); + + describe("renderCiCMessage", () => { + it("places context before question and adds label when requested", () => { + const combined = renderCiCMessage("context block", "What?", true); + + expect(combined).toBe("context block\n\nQuestion: What?"); + }); + + it("returns the original question when context is blank", () => { + expect(renderCiCMessage(" \n ", "What?", false)).toBe("What?"); + }); + }); +}); diff --git a/src/LLMProviders/chainRunner/utils/cicPromptUtils.ts b/src/LLMProviders/chainRunner/utils/cicPromptUtils.ts new file mode 100644 index 00000000..7692724e --- /dev/null +++ b/src/LLMProviders/chainRunner/utils/cicPromptUtils.ts @@ -0,0 +1,50 @@ +/** + * Helper utilities for assembling Corpus-in-Context (CiC) ordered tool payloads. + */ + +/** + * Builds the ordered inner payload for a localSearch result, placing guidance before documents. + * @param guidance Citation guidance or instructions associated with the search results. + * @param formattedContent Serialized documents selected for inclusion. + * @returns Combined payload string with minimal whitespace. + */ +export function buildLocalSearchInnerContent(guidance: string, formattedContent: string): string { + const sections = [guidance, formattedContent] + .map((section) => section?.trim()) + .filter((section): section is string => Boolean(section)); + + return sections.join("\n\n"); +} + +/** + * Wraps localSearch payload content in XML, preserving optional time range metadata. + * @param innerContent Ordered combination of guidance and documents. + * @param timeExpression Optional natural-language time range expression. + * @returns XML-wrapped localSearch payload ready for LLM consumption. + */ +export function wrapLocalSearchPayload(innerContent: string, timeExpression: string): string { + const payload = innerContent ? `\n${innerContent}\n` : ""; + const timeAttribute = timeExpression ? ` timeRange="${timeExpression}"` : ""; + return `${payload}`; +} + +/** + * Produces a CiC-aligned prompt by placing context first and the user question last. + * @param contextSection Prepared instruction/context block. + * @param userQuestion Original user message. + * @param labelQuestion Whether to prefix the question with a clarifying label. + * @returns String formatted according to CiC ordering. + */ +export function renderCiCMessage( + contextSection: string, + userQuestion: string, + labelQuestion: boolean +): string { + const contextBlock = contextSection.trim(); + if (!contextBlock) { + return userQuestion; + } + + const questionBlock = labelQuestion ? `Question: ${userQuestion}` : userQuestion; + return `${contextBlock}\n\n${questionBlock}`; +}