From 8cea1a6f173c3eadfd4bb18fbf989f1a4372c8ef Mon Sep 17 00:00:00 2001 From: Logan Yang Date: Sat, 7 Feb 2026 21:18:25 -0800 Subject: [PATCH] Audit context envelope, tag alignment, artifact dedup, and logging (#2164) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: align context block tags and make parseContextIntoSegments registry-driven The context envelope L2 layer was silently dropping URL/media context across turns due to three compounding issues: 1. Tag mismatch: Mention.ts created blocks but the contextBlockRegistry expected . Fixed the tag and inner element ( → ) to match the registry. 2. Missing registration: twitter_content blocks had no registry entry, so they were invisible to compaction and segment parsing. Added the entry. 3. Hardcoded parser: parseContextIntoSegments only matched , , and via hardcoded regexes, silently ignoring all URL/media block types. Rewrote it to dynamically build its regex from CONTEXT_BLOCK_TYPES, ensuring any future block type added to the registry is automatically handled. Extracted parseContextIntoSegments into src/context/parseContextSegments.ts as a standalone pure function for direct testability. Added 19 new tests covering all registered block types, mixed blocks, stable/unstable flags, and a registry completeness guard test. Co-Authored-By: Claude Opus 4.6 * fix: context envelope improvements - regeneration, dedup, per-artifact IDs, XML escaping Phase A: Fix regeneration from loaded chats by lazily reprocessing context when contextEnvelope is missing (e.g., messages loaded from disk). Phase B: Replace static segment IDs ("urls", "selected_text", "web_tabs") with per-artifact IDs from parseContextIntoSegments, enabling accurate smart referencing in LayerToMessagesConverter. Phase C: Deduplicate L2 content by segment ID (last-write-wins) to prevent linear growth when the same artifact appears across multiple turns. Phase D: Escape note title/basename in XML context blocks for consistency with URL/YouTube content. Keep path unescaped as it's used as identifier. Additional fixes from Codex review: - Unique IDs for blocks without source extractors (selected_text counter) - Keep note.path raw to avoid dedup mismatch with TFile.path Co-Authored-By: Claude Opus 4.6 * fix: prevent context artifact duplication in LLM payload and L4 memory CopilotPlusChainRunner was re-injecting processedText (which includes context XML artifacts) into the user message via ensureUserQueryLabel, bypassing the envelope's clean L2/L3/L5 separation. This caused context like YouTube transcripts to appear 3x in the LLM payload. Fix: use L5_USER envelope text (expanded user query without context XML) instead of processedText for cleanedUserMessage, trimmedQuestion fallback, and messageForAnalysis. Also fix BaseChainRunner.handleResponse to save L5 text to L4 memory instead of processedText. Other changes: - Remove dead updateMemoryWithLoadedMessages from chainManager - Update CONTEXT_ENGINEERING.md with example walkthrough, L4 behavior docs, and Phase 6 integration test suite roadmap Co-Authored-By: Claude Opus 4.6 * docs: add chain runner envelope usage section and fix Step 0 log Document per-runner envelope behavior, CopilotPlus/Agent tool flows, and token efficiency audit in CONTEXT_ENGINEERING.md. Fix Step 0 log to show L5 user query instead of processedText with context XML. Co-Authored-By: Claude Opus 4.6 * fix: trim trailing whitespace from chat input display text Trailing newlines from Lexical editor were preserved in displayText and rendered as empty lines in chat bubbles due to whitespace-pre-wrap. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- docs/CONTEXT_ENGINEERING.md | 897 +++++++----------- src/LLMProviders/chainManager.ts | 18 +- .../chainRunner/BaseChainRunner.ts | 14 +- .../chainRunner/CopilotPlusChainRunner.ts | 15 +- src/components/Chat.tsx | 2 +- src/context/contextBlockRegistry.test.ts | 24 + src/context/contextBlockRegistry.ts | 3 + src/context/parseContextSegments.ts | 76 ++ src/contextProcessor.ts | 4 +- src/core/ChatManager.test.ts | 58 +- src/core/ChatManager.ts | 27 +- src/core/ContextManager.parseSegments.test.ts | 270 ++++++ src/core/ContextManager.ts | 154 +-- src/mentions/Mention.ts | 2 +- 14 files changed, 873 insertions(+), 691 deletions(-) create mode 100644 src/context/parseContextSegments.ts create mode 100644 src/core/ContextManager.parseSegments.test.ts diff --git a/docs/CONTEXT_ENGINEERING.md b/docs/CONTEXT_ENGINEERING.md index 810be008..678ef713 100644 --- a/docs/CONTEXT_ENGINEERING.md +++ b/docs/CONTEXT_ENGINEERING.md @@ -2,678 +2,463 @@ ## Table of Contents -1. [Overview](#overview) -2. [Layered Prefix Architecture (L1-L5)](#layered-prefix-architecture-l1-l5) -3. [Current Implementation Status](#current-implementation-status) -4. [Architecture Components](#architecture-components) -5. [Tool System Design](#tool-system-design) -6. [Testing & Migration](#testing--migration) -7. [Historical Context](#historical-context) +1. [Purpose](#purpose) +2. [First-Principles Goals](#first-principles-goals) +3. [Current Architecture (Verified)](#current-architecture-verified) +4. [Example Chat Walkthrough](#example-chat-walkthrough) +5. [Chain Runner Envelope Usage](#chain-runner-envelope-usage) +6. [Strengths](#strengths) +7. [Known Gaps](#known-gaps) +8. [Improvement Roadmap](#improvement-roadmap) +9. [Testing and Observability](#testing-and-observability) +10. [References](#references) --- -## Overview +## Purpose -### Intent +The context envelope system is the canonical prompt-construction pipeline for chat turns. -Copilot's layered context system (L1-L5) enables provider-side prefix caching by organizing prompt content from most stable (L1) to most volatile (L5). This prevents duplicate context transmission across turns and allows deterministic token budget control. +It exists to guarantee: -### Design Goals +- reproducible prompt assembly, +- minimal duplication across L2/L3/L4, +- safe compaction behavior on large context, +- and cache-friendly request prefixes for major providers. -- **Layered prefix** (L1–L5) ordered from most stable to most volatile for cache hits -- **Model-agnostic** baseline: OpenAI/Gemini implicit caching, Anthropic long-context guidance -- **Provider-aware optimizations** (optional): Gemini explicit cache, Anthropic `cache_control` -- **No contract changes** for `ChainRunner` or tool execution APIs -- **Backward compatibility** with existing chat markdown exports - -### Key Benefits - -- **Maximum cache stability**: L2 grows monotonically → cache hits maximized -- **Minimal redundancy**: Items in L2 referenced by ID in L3 → fewer tokens -- **Smart referencing**: Clear instructions to find context in Context Library -- **Simple logic**: Set membership test decides ID vs. full content +This document is an implementation audit and roadmap based on current production code. --- -## Layered Prefix Architecture (L1-L5) +## First-Principles Goals + +### 1. Reproducibility + +For the same turn inputs, envelope construction should be deterministic and byte-stable. + +### 2. Token Efficiency + +Context artifacts should appear once in canonical form (or as references), not duplicated across layers. + +### 3. Prefix Cache Stability + +Stable content should stay in early request tokens (L1/L2) so implicit provider caching has maximal hit rate. + +### 4. Compaction Safety + +Compaction must preserve answerability and recovery affordances, especially for non-recoverable context. + +### 5. Persistence Parity + +Loading chat history should preserve envelope quality (or deterministically reconstruct it), so behavior after load matches in-session behavior. + +--- + +## Current Architecture (Verified) ### Layer Definitions -| Layer | Source | Update Trigger | Purpose | -| ------------------------- | --------------------------------------- | -------------------------------- | ---------------------------------------------- | -| **L1: System & Policies** | System prompt + user memory | Only when settings/memory change | Cacheable prefix with instructions | -| **L2: Context Library** | ALL previous context items (cumulative) | Grows with new context | Stable reference library | -| **L3: Smart References** | Current turn context | Every turn | References L2 by ID, new items as full content | -| **L4: Chat History** | LangChain memory (raw messages) | Each Q/A pair | Conversation continuity | -| **L5: User Message** | Raw user input | Every turn | Minimal user query | +| Layer | Current Source | Update Trigger | Stability | +| --------------- | ------------------------------------------------------------ | ------------------------------- | --------- | +| **L1_SYSTEM** | `ChatManager.getSystemPromptForMessage()` | settings/memory/project changes | High | +| **L2_PREVIOUS** | Auto-promoted previous user-turn L3 segments | per user turn | Medium | +| **L3_TURN** | Current-turn context artifacts (notes/URLs/tags/folders/etc) | every user turn | Low | +| **L4_STRIP** | Deferred in envelope, injected from LangChain memory | every turn | Low | +| **L5_USER** | processed user query (templated user text) | every user turn | Lowest | -### L2: Context Library (Cumulative Design) +### End-to-End Flow -**Key Insight**: L2 is a simple cumulative library of all context ever seen in the conversation. +1. `ChatManager.sendMessage()` creates a user message and resolves L1 system prompt. +2. `ContextManager.processMessageContext()`: + - builds L2 from previous user messages' stored envelopes, + - processes current-turn context artifacts, + - optionally compacts large context, + - builds `PromptContextEnvelope` via `PromptContextEngine`. +3. `MessageRepository.updateProcessedText()` stores both legacy `processedText` and `contextEnvelope`. +4. Chain runners require `contextEnvelope`, convert with `LayerToMessagesConverter`, inject L4 from memory, then append tool context into user-side payload. -**Rules**: +### L2/L3 Smart Referencing -1. **Cumulative**: Grows monotonically, never shrinks -2. **Auto-promotion**: Context from previous turns automatically moves to L2 -3. **Smart referencing**: Items in L2 referenced by path/ID in L3 -4. **New items only**: L3 includes full content only for brand-new items +- L2 is now deduplicated by segment ID with last-write-wins content updates and stable first-seen ordering. +- L3 segments whose IDs already exist in L2 are rendered as references; new IDs include full content. +- Segment parsing is centralized in `parseContextIntoSegments()` using `contextBlockRegistry` tags. -**Example Flow**: +### Tool Placement Model -``` -Turn 1: User adds project-spec.md - L1: System prompt - L2: (empty) - L3: project-spec.md (full content) ← NEW - L5: "Summarize this" +- System message contains only L1 + L2. +- Tool outputs remain turn-scoped and are prepended to user-side content (`CiC` ordering). +- This keeps cacheable prefix isolated from tool variability. -Turn 2: User keeps project-spec.md, adds api-docs.md - L1: System prompt (stable ✅) - L2: project-spec.md (full content) ← Promoted to library - L3: "Context attached: - - Piano Lessons/project-spec.md - Find them in the Context Library above." - api-docs.md (full content) ← NEW - L5: "What's the API?" +### Persistence Behavior -Turn 3: User keeps project-spec.md only - L1: System prompt (stable ✅) - L2: project-spec.md - api-docs.md ← STABLE! Cache hit! - L3: "Context attached: - - Piano Lessons/project-spec.md - Find them in the Context Library above." ← Just ID reference - L5: "Explain that" -``` +- Chat markdown persists message text plus context references (`[Context: ...]`), not full envelopes. +- On load, messages are restored without `contextEnvelope`. +- Regeneration now has lazy reprocessing: if envelope is missing, `ChatManager.regenerateMessage()` reprocesses the target user message before running the chain. +- Continuing chat after load still does not automatically reconstruct historical envelopes for prior turns. -### Unique Identifiers +### Compaction Stack -- **Notes**: Full file path (`note.path`) -- **URLs**: Full URL string -- **Selected text**: Source file path + line range -- **Dataview blocks**: Source file path + query hash +- **Turn-time compaction** (`ContextCompactor`): map-reduce summarization when total context exceeds threshold. +- **L2 carry-forward compaction** (`compactSegmentForL2` + `L2ContextCompactor`): deterministic structure+preview compression for promoted previous context. + +### L4 Memory Behavior + +- L4 (chat history) is injected by chain runners from LangChain `BufferWindowMemory`. +- **Only `displayText` (raw user message) is saved to memory** — context artifacts are NOT included. +- This prevents duplication: context artifacts already live in L2/L3 via the envelope; baking them into L4 would cause triple-inclusion and waste tokens. +- Assistant responses are saved as-is (or with tool-call formatting stripped). --- -## Current Implementation Status +## Example Chat Walkthrough -### ✅ Completed (Production) +This shows the concrete layer contents across a 3-turn conversation. The user attaches `project-spec.md` in Turn 1, adds `api-docs.md` in Turn 2, then drops `api-docs.md` in Turn 3. -1. **Phase 1: Foundations** +### Turn 1: User adds `project-spec.md` - - ✅ `PromptContextEngine` with deterministic L3/L5 rendering - - ✅ `ContextManager` returns structured `ContextProcessingResult` - - ✅ `MessageRepository` persists `contextEnvelope` per message - - ✅ Feature flag removed - system active for all conversations +``` +L1 (System): + [system prompt + user memory + project instructions] -2. **Phase 2: L2 Auto-Promotion** +L2 (Previous Context Library): + (empty — first turn, no prior context) - - ✅ `buildL2ContextFromPreviousTurns()` collects previous context - - ✅ Cumulative L2 (no deduplication from current turn) - - ✅ Segment-based smart referencing with path IDs - - ✅ `LayerToMessagesConverter` handles smart references +L3 (Current Turn Context): + + project-spec + project-spec.md + ... full note content ... + + → Segment ID: "project-spec.md" (NEW — full content included) -3. **Phase 3: ChainRunner Migration** - - ✅ `LLMChainRunner` envelope-based - - ✅ `VaultQAChainRunner` envelope-based - - ✅ `CopilotPlusChainRunner` envelope-based with uniform tool handling - - ✅ `AutonomousAgentChainRunner` envelope-based with iterative tool loop - - ✅ `ProjectChainRunner` envelope-based with project context in L1 +L4 (Chat History): + (empty — first turn) -### 🎯 Current Design: Uniform Tool Placement +L5 (User Message): + "Summarize this" +``` -**System Message** = L1 (system prompt) + L2 (Context Library) ONLY -**User Message** = Tool results + L3 (smart refs) + L5 (user query) +After Turn 1, `BaseChainRunner.handleResponse()` saves to memory: -All tools (`localSearch`, `webSearch`, `getFileTree`, etc.) are treated uniformly: +- Input: `"Summarize this"` (displayText only — no context XML) +- Output: `"Here is a summary of the project spec..."` -- Wrapped as `content` -- Prepended to user message using CiC (Context in Context) format -- Turn-specific, never promoted to L2 +### Turn 2: User keeps `project-spec.md`, adds `api-docs.md` -### 🔄 In Progress / Deferred +``` +L1 (System): + [system prompt — stable ✅, cache-friendly] -**Next Phase**: +L2 (Previous Context Library): + + Structure: project-spec (project-spec.md) | Preview: ...first 200 chars... + + → Segment ID: "project-spec.md" (promoted from Turn 1 L3, compacted for L2) -- Cache stability monitoring +L3 (Current Turn Context): + Context attached to this message: + - project-spec.md -**Future Enhancements** (deferred): + Find them in the Context Library in the system prompt above. -- Provider-specific cache controls (Anthropic `cache_control`, Gemini explicit cache) -- L4 conversation strip with summarization -- Optional manual pinning UI + + api-docs + docs/api-docs.md + ... full note content ... + + → "project-spec.md" rendered as REFERENCE (already in L2) + → "docs/api-docs.md" is NEW — full content included + +L4 (Chat History): + Human: "Summarize this" + AI: "Here is a summary of the project spec..." + → Only displayText — no context XML in L4 + +L5 (User Message): + "What endpoints does the API support?" +``` + +### Turn 3: User keeps `project-spec.md` only (drops `api-docs.md`) + +``` +L1 (System): + [system prompt — stable ✅] + +L2 (Previous Context Library): + + Structure: project-spec (project-spec.md) | Preview: ...first 200 chars... + + + Structure: api-docs (docs/api-docs.md) | Preview: ...first 200 chars... + + → Both deduplicated by segment ID. "project-spec.md" retains its + first-seen position; "docs/api-docs.md" added after. + → L2 is CUMULATIVE and STABLE — cache hit for the prefix ✅ + +L3 (Current Turn Context): + Context attached to this message: + - project-spec.md + + Find them in the Context Library in the system prompt above. + → "project-spec.md" is a REFERENCE (in L2) + → "docs/api-docs.md" is NOT referenced (user didn't attach it this turn) + but it remains in L2 for cache stability and potential follow-up use + +L4 (Chat History): + Human: "Summarize this" + AI: "Here is a summary of the project spec..." + Human: "What endpoints does the API support?" + AI: "The API supports the following endpoints..." + → Clean displayText only — no bloat + +L5 (User Message): + "Explain the auth flow from the spec" +``` + +### Key Behaviors Demonstrated + +| Behavior | Where | Example | +| ------------------------------- | ----------- | ----------------------------------------------------------------- | +| **Per-artifact segment IDs** | L3 parsing | `"project-spec.md"`, `"docs/api-docs.md"` — not generic `"notes"` | +| **L2 dedup (last-write-wins)** | L2 build | Same ID across turns → content updated, position preserved | +| **Smart referencing** | L3 render | Items in L2 become `- project-spec.md` references | +| **L2 cumulative growth** | L2 library | `api-docs.md` stays in L2 even when dropped from L3 | +| **L2 carry-forward compaction** | L2 content | Full `` → `` with structure+preview | +| **L4 displayText only** | Memory save | `"Summarize this"` — no `` XML | +| **Prefix cache stability** | L1+L2 | L1 stable across turns; L2 grows monotonically, doesn't shrink | --- -## Chain Integration Summary +## Chain Runner Envelope Usage -### LLMChainRunner +All four chain runners use the context envelope for LLM message construction. Each delegates final response handling to `BaseChainRunner.handleResponse()`, which saves only L5 text (expanded user query, no context XML) to L4 memory. -- Replaced legacy prompt assembly with `LayerToMessagesConverter` output. -- System message = envelope L1 + L2 only; no tool payloads injected. -- User message = L3 smart references + L5; supports composer instructions via shared helper. -- Payload recorder logs layered view for every request. +### Per-Runner Behavior -### VaultQAChainRunner +| Runner | Envelope Construction | Tool Results | User Message Source | +| ------------------------------ | ------------------------------------------------------------------------------------- | ------------------------------------------------------------- | -------------------- | +| **LLMChainRunner** | `LayerToMessagesConverter.convert()` → system (L1+L2), user (L3 refs + L5) | None | Envelope only | +| **CopilotPlusChainRunner** | Same converter, then `ensureUserQueryLabel` adds `[User query]:` separator | Prepended to user message in CiC order | L5 text via envelope | +| **AutonomousAgentChainRunner** | Same converter for initial messages; ReAct loop appends AI + ToolMessages iteratively | Native tool calling — each result is a separate `ToolMessage` | L5 text via envelope | +| **VaultQAChainRunner** | Same converter | Retrieval results via hybrid/lexical retriever | Envelope only | -- Uses envelope-derived system message (L1/L2) plus chat history before user turn. -- Vault RAG results and citation guidance prepended to user message, keeping system cacheable. -- Applies CiC ordering so the user question follows retrieved documents. -- Multimodal handling respects envelope rules (images only from active note). +### CopilotPlus: Single-Shot Tool Flow -### CopilotPlusChainRunner +1. Planning phase analyzes L5 text to determine which `@commands` to execute. +2. Tool results (localSearch, web fetch, etc.) are formatted and prepended to the user message using CiC ordering: `[tool results] → [L3 references + L5 with User query label]`. +3. Single LLM call with the complete message array: `[system (L1+L2)] → [L4 history] → [user (tools + L3 + L5)]`. -- Intent analysis still Broca-based (ToolCallPlanner pending), but prompt assembly is envelope-first. -- All tool outputs (localSearch, webSearch, file tree, etc.) formatted uniformly and prepended to user message. -- LocalSearch payload is self-contained: includes RAG instruction, documents, and citation guidance all within the `` block. -- Each localSearch call carries its own guidance block (source catalog + citation rules), ensuring correct citations in multi-search turns. -- Composer instructions appended once to avoid duplication; payload recorder captures tool XML alongside L3/L5. +### Autonomous Agent: ReAct Loop Flow -### AutonomousAgentChainRunner +1. Initial message array built identically to CopilotPlus: `[system (L1+L2+tool guidelines)] → [L4 history] → [user (L3 refs + L5)]`. +2. Model responds with native tool calls (e.g., `localSearch`, `readFile`). +3. Each tool result becomes a `ToolMessage` appended to the growing messages array. +4. `localSearch` results get CiC ordering: the user's question (from L5 `originalUserPrompt`) is appended after the search payload via `ensureCiCOrderingWithQuestion`. +5. Loop repeats until model responds without tool calls (final answer). -- Validates envelope presence and reuses converter for baseline messages. -- System prompt combines envelope L1/L2 with tool descriptions from model adapters (no duplicate base prompt). -- L5 text flows through adapter hints so GPT/Claude reminders continue to fire. -- Iterative tool loop reuses existing Think/Action streamers; prompt recorder receives envelope for each run. +### Token Efficiency Audit -### ProjectChainRunner +**Verified efficient (no action needed):** -- Fully migrated to envelope-based context construction. -- Project context automatically added to L1 via `ChatManager.getSystemPromptForMessage()`. -- No special-case logic needed - inherits all behavior from `CopilotPlusChainRunner`. +- L1+L2 prefix is stable and cacheable across turns — tool results never enter the system message. +- L3 uses smart references for artifacts already in L2 — no content duplication in the user message. +- L4 contains only displayText (via L5 extraction in `handleResponse`) — no context XML leakage. +- Chain runners extract L5 text from the envelope for `cleanedUserMessage` and `originalUserPrompt`, never using `processedText` (which contains L2+L3+L5 concatenated). -#### Implementation +**Known inefficiencies (accepted tradeoffs):** -1. **Centralized L1 assembly with helper method.** - Added `ChatManager.getSystemPromptForMessage(chainType)` that calls `await getSystemPromptWithMemory(...)`, then (for `PROJECT_CHAIN` only) appends project system/context blocks: - - ```ts - async getSystemPromptForMessage(chainType: ChainType): Promise { - const basePrompt = await getSystemPromptWithMemory(this.chainManager.userMemoryManager); - - // Special case: Add project context for project chain - if (chainType === ChainType.PROJECT_CHAIN) { - const project = getCurrentProject(); - if (project) { - const context = await ProjectManager.instance.getProjectContext(project.id); - let result = `${basePrompt}\n\n\n${project.systemPrompt}\n`; - - // Only add project_context block if context exists (guards against null) - if (context) { - result += `\n\n\n${context}\n`; - } - - return result; - } - } - return basePrompt; - } - ``` - -2. **Null guard for project context.** - When `ProjectManager.instance.getProjectContext()` returns `null` (e.g., context still loading or cache miss), the `` block is omitted entirely rather than interpolating the literal string "null" into L1. - -3. **Envelope integration.** - The helper's return value is passed into `processMessageContext` / `reprocessMessageContext`. `PromptContextEngine` serializes the full string into the L1 layer, so `ProjectChainRunner` drops its bespoke `getSystemPrompt` override and reuses the envelope just like Copilot Plus. - -4. **Simplified ProjectChainRunner.** - The class is now just a pass-through to `CopilotPlusChainRunner` with no overrides - project context automatically appears in L1 via `ChatManager`. +| Issue | Severity | Tokens Wasted | Rationale | +| ------------------------------------------------------------------------- | -------- | --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| CiC user-question repeated per `localSearch` tool call in agent loop | LOW | ~50 tokens x N searches | Intentional — each `ToolMessage` is independent; the model needs the question for grounding across ReAct iterations | +| L2 content may overlap with `localSearch` results | MEDIUM | Variable (L2 has compacted preview, search has relevant chunks) | Structural limitation — search doesn't know about L2. Overlap is partial since L2 is compacted to structure+preview while search returns precision chunks | +| Legacy `processedText` stores L2+L3+L5 concatenation in MessageRepository | LOW | 0 (not sent to LLM) | Storage-only waste. Field is unused by envelope-based chain runners but persisted for backward compatibility | --- -## Architecture Components +## Strengths -### 1. PromptContextEngine - -**Purpose**: Centralizes prompt assembly, replacing "context baked into message" workflow. - -**Input**: `PromptContextRequest` - -- `chatId`, `currentMessageId` -- User text, structured attachments -- Tool outputs, provider metadata -- Token budget hints - -**Output**: `PromptContextEnvelope` - -- `layers` (L1-L5 structured) -- `llmMessages` (formatted for provider) -- `renderedSegments` (serialized layers) -- Cache hints (`layerHashes`) -- Back-compat `processedUserText` - -### 2. Layer Builders - -**L2 Builder**: - -- Collects context from all previous messages -- Deduplicated by unique ID (note path, URL) -- Sorted by first appearance for stability -- Excludes current turn's L3 - -**L3 TurnContextAssembler**: - -- Reuses `ContextProcessor` for current message only -- Emits structured entries: `{ id, type, sha256, content }` -- Smart referencing logic (ID vs. full content) - -**L4 History**: - -- Provided by LangChain memory (`MessageRepository.getLLMMessages()`) -- Returns raw messages without context - -**Renderer**: - -- Deterministically serializes each layer -- Whitespace normalization for byte-identical outputs -- Hash generation for cache validation - -### 3. LayerToMessagesConverter - -**Purpose**: Converts envelope to provider-ready messages with smart referencing. - -**Smart Referencing Logic**: - -```typescript -// Parse L2 into segments by unique ID -const l2SegmentIds = new Set(parseContextIntoSegments(l2Text).map((s) => s.id)); - -// For each item in L3: -if (l2SegmentIds.has(itemId)) { - // Reference by ID: "Find Piano Lessons/project-spec.md in Context Library above" -} else { - // Include full content: full content -} -``` - -### 4. Data Persistence - -**MessageRepository**: - -- Stores `contextEnvelope?: SerializedLayerRef` per message -- Contains L3 entries + pointers to L1/L2/L4 snapshots -- Backward compatible with legacy `processedText` - -**ChatPersistenceManager**: - -- Saves `displayText` + context metadata to markdown -- Hidden JSON frontmatter for layer reconstruction: `` -- On load: rebuild envelope from metadata (legacy mode if missing) - -**ProcessedText Evolution**: - -- **Phase 1 (Current)**: Dual storage - both `processedText` and `contextEnvelope` -- **Phase 2 (After migration)**: `processedText` becomes computed from envelope -- **Phase 3 (Future)**: Drop `processedText` entirely +1. Envelope-first prompt construction is now consistent across all active chain runners. +2. L2 deduplication by artifact ID prevents linear repeated growth for repeated attachments. +3. Segment parsing is centralized and registry-driven, avoiding ad-hoc per-chain parsing logic. +4. Uniform tool placement removes previous cache-boundary ambiguity. +5. Memory-side assistant compaction reduces L4 bloat from tool payloads. +6. Regeneration path is now resilient to missing envelopes on loaded history. --- -## Tool System Design +## Known Gaps -### Uniform Tool Placement +### P0: Persistence Parity Is Still Incomplete -**All tools go to user message** - no special cases: +- Loaded chats do not rehydrate historical envelopes. +- Result: follow-up turns after load do not benefit from historical L2 library unless messages are individually reprocessed. -```typescript -// System message: L1 + L2 only -messages.push({ - role: "system", - content: systemMessage.content, // Just L1 + L2, no tools -}); +### P0: Non-Deterministic Fallback Segment IDs -// User message: Tools + L3 + L5 -const toolContext = formatAllToolOutputs(allToolOutputs); // Uniform for all tools -const finalUserContent = renderCiCMessage( - toolContext, // ...\n... - userMessageContent.content, // L3 smart refs + L5 - false -); -``` +- `appendParsedSegments()` uses `unparsed-${Date.now()}` when parsing fails. +- This breaks deterministic envelope identity and degrades cache behavior in edge cases. -### Tool Output Formatting +### P1: Parser Still Depends on Regex Over Rendered XML -**All tools use consistent XML wrapping**: +- `parseContextIntoSegments()` is significantly better than earlier local regex logic, but it still parses serialized XML strings rather than typed artifacts. +- Malformed/nested edge cases can still create parse misses and fallback behavior. -```xml -# Additional context: +### P1: Compaction Semantics Need Stronger Invariants - -Answer the question based only on the following context: +- Two compaction modes exist (LLM summarization vs deterministic L2 preview compaction), but explicit invariants on what must remain verbatim are not enforced centrally. +- Non-recoverable context (e.g., selected text) needs stricter protection policies under heavy compaction. -[Retrieved documents...] +### P1: L2 Mutation Tradeoff Not Formalized - -[Citation rules and source catalog...] - - +- Current policy is ID-dedup + content overwrite. +- This is token-efficient, but mutable artifacts can invalidate long cached prefixes when content changes. +- The system needs an explicit "freshness vs cache stability" policy. - -[Web search results...] - -``` +### P2: Envelope Metadata Is Underused -**Key Points**: +- `conversationId` is currently `null`. +- Missing stable conversation-level identity weakens observability and future caching strategies. -- Each tool wrapped as `content` -- Prepended to user message via `renderCiCMessage()` -- RAG instruction ("Answer based on context") included in localSearch output -- Citation guidance included directly within each `` block (self-contained) -- Multi-search turns: each localSearch has its own guidance block with source mappings -- Turn-specific, never cached in L2 +### P2: Documentation Drift Risk -### Intent Analysis - -**Current**: `IntentAnalyzer` analyzes L5 (raw user query) for tool selection - -**Future (Deferred)**: `ToolCallPlanner` - -- Uses user's chat model instead of Broca API -- Input: L5 + L3 summary (metadata only, not full content) -- Output: JSON tool-call array with schema validation -- Shared between CopilotPlus and Agent chains +- Prior docs contained stale statements (e.g., fallback-to-processedText behavior, on-load envelope reconstruction). +- This doc now reflects current code; future changes should keep this aligned. --- -## Testing & Migration +## Improvement Roadmap -### Testing Strategy +### Phase 1: Correctness and Determinism -**Unit Tests**: +1. Replace timestamp fallback IDs with deterministic IDs: + - `unparsed:${sha256(content)}` (plus optional short prefix by source type). +2. Add envelope invariants checker (debug + tests): + - no duplicate segment IDs inside a layer, + - no L3 full-content block when ID exists in L2 unless explicitly marked override, + - stable layer ordering and hash consistency. +3. Add robust parse-failure telemetry: + - count parse misses, + - log failing tag/source metadata, + - capture hash-only samples in debug mode. -- Layer renderer snapshot tests (byte-identical output) -- L2 auto-promotion tests (deterministic collection) -- Smart referencing tests (ID vs. full content logic) -- Deduplication tests (L3 priority over L2) +### Phase 2: Persistence Parity -**Integration Tests**: +1. Add lazy historical envelope reconstruction on first post-load send: + - reprocess only prior user messages that have context references and missing envelopes, + - skip URL/web-tab refetch by policy where needed. +2. Optional long-term path: + - persist compact envelope metadata (or typed artifact snapshots) alongside markdown history for deterministic restoration. -- Golden prompt fixtures (no context, heavy context, tool-heavy, multimodal) -- ChainRunner streaming smoke tests -- Tool marker parsing validation +### Phase 3: Compaction Safety and Policy -**Observability**: +1. Define explicit compaction classes: + - recoverable artifacts: can be summarized with re-fetch instructions, + - non-recoverable artifacts: preserve verbatim or bounded extractive compaction only. +2. Add post-compaction validation: + - each compacted artifact must retain deterministic source identity and recoverability hints. +3. Make compaction strategy configurable by chain type and context source type. -- Log layer hashes + cached token counts (debug mode) -- Warn if L1/L2 hash changes mid-conversation without trigger -- Prompt payload recorder with layered view +### Phase 4: Cache Optimization -### Migration Caveats +1. Split L1 into stable and mutable subsections (for example: + - static system contract, + - user memory and project overlays) to reduce unnecessary prefix invalidation. +2. Introduce provider-aware cache hooks (opt-in): + - Anthropic `cache_control`, + - Gemini explicit cache primitives, + - keep model-agnostic baseline unchanged. +3. Add per-turn prefix hash diff reporting: + - `L1 hash`, `L2 hash`, combined prefix hash, + - classify why prefix changed (settings, context attach, file change, memory update). -#### Multimodal (Images) +### Phase 5: Typed Artifact Pipeline (Strategic) -**Constraint**: Image extraction must read from envelope, not raw message. +Move from "render XML then parse XML" to a typed artifact graph: -**Solution** (CopilotPlusChainRunner): +- `ContextProcessor` emits typed artifacts directly (`artifactKey`, `sourceType`, `recoverable`, `payload`, `contentHash`). +- Envelope stores typed segments as canonical source-of-truth. +- XML remains a rendering format, not parsing substrate. -```typescript -// Extract images from active note ONLY (in L3) -const l3Turn = envelope.layers.find((l) => l.id === "L3_TURN"); -if (l3Turn) { - const activeNoteRegex = /([\s\S]*?)<\/active_note>/; - const activeNoteMatch = activeNoteRegex.exec(l3Turn.text); - if (activeNoteMatch) { - const sourcePath = extractPath(activeNoteMatch[1]); - const content = extractContent(activeNoteMatch[1]); - const images = await this.extractEmbeddedImages(content, sourcePath); - } -} -``` +This is the highest-leverage change for long-term reproducibility and parser robustness. -**Rules**: +### Phase 6: Context Envelope Integration Test Suite -- Extract from `` only (not ``) -- Never promote image-bearing notes to L2 -- Legacy behavior unchanged (no envelope fallback) +Build a comprehensive test suite that validates multi-turn envelope behavior without requiring manual UI testing: -#### Chat Persistence +1. **Multi-turn envelope simulation tests**: -**Saving** (ChatPersistenceManager.formatChatContent): + - Simulate 3+ turn conversations with various artifact combinations (notes, URLs, YouTube, PDFs, selected text). + - Assert correct L2 promotion, dedup, smart referencing, and compaction at each turn. + - Validate that L4 memory contains only displayText (no context XML leakage). -1. Save `displayText` (UI view) to markdown -2. Save context metadata with **full vault-relative paths**: - - Notes: `[Context: Notes: Piano Lessons/Lesson 4.md, DailyNotes/2025-01-27.md]` - - URLs, tags, folders saved as-is -3. `contextEnvelope` NOT saved (deliberate - want fresh content on load) +2. **Layer composition snapshot tests**: -**Loading** (ChatPersistenceManager.parseContextString): + - For canonical conversation trajectories, snapshot the full `[L1, L2, L3, L4, L5]` payload sent to the LLM. + - Detect unintended regressions in layer ordering, dedup behavior, or content placement. -1. Parse context metadata from markdown -2. Resolve note paths via vault lookup: - - **Primary**: Resolve by full path (`app.vault.getAbstractFileByPath()`) - - **Fallback**: Resolve by basename if unique (backward compatibility) - - **Skip**: If deleted or ambiguous, log warning and continue -3. Return resolved TFile[] for context.notes -4. When conversation continues: `buildL2ContextFromPreviousTurns()` processes resolved notes with fresh content +3. **Round-trip persistence tests**: -**Stale Context Handling**: + - Save a conversation to markdown, reload it, send a follow-up turn. + - Assert that lazy reprocessing reconstructs envelopes and L2 correctly. -- **Deleted note**: Skip with warning, continue without it -- **Changed content**: Use current content (expected - provides fresh context) -- **Moved note**: Resolves via basename fallback if unique, otherwise skips -- **Ambiguous basename**: Multiple matches → skip with warning listing all matches +4. **Edge-case regression tests**: -**Backward Compatibility** (2025-01-27 update): + - Same artifact attached across 5+ turns (dedup stability). + - Artifact added, removed, re-added (L2 cumulative behavior). + - Multiple `selected_text` blocks in same turn (unique ID generation). + - Malformed XML blocks (graceful fallback, no silent data loss). + - Very large context triggering compaction (invariants preserved). -- **Old chats** (basenames only): Basename fallback resolution attempts vault-wide search -- **New chats** (full paths): Direct resolution, faster and unambiguous -- **Migration**: Automatic - no user action needed +5. **Property-based tests** (optional, aspirational): + - Generate random artifact sequences and assert envelope invariants hold: + no duplicate segment IDs within a layer, L3 references only exist if ID is in L2, + L4 never contains XML block tags. -### Backward Compatibility - -**ChainRunner Contracts**: - -- `ChainRunner.run()` signatures unchanged -- Envelope passed via `userMessage.contextEnvelope` -- Fallback to `processedText` if envelope missing - -**Markdown Archives**: - -- Continue storing full `processedText` -- Layer metadata additive and optional -- Legacy chats rebuild on load (compat mode) - -**Settings**: - -- No new toggles required -- System always active, backward compatible +This suite replaces the need for manual multi-turn chat testing in the UI and provides a safety net for all future envelope changes. --- -## Historical Context +## Testing and Observability -### Design Simplifications +### Core Tests to Add/Strengthen -#### 1. Simplified Layer Serialization +1. Post-load follow-up turn should rebuild/rehydrate envelope behavior deterministically. +2. Deterministic fallback ID behavior (no wall-clock dependence). +3. Property tests for parser with malformed/nested blocks. +4. Compaction invariants: + - non-recoverable blocks never become unrecoverable summaries without explicit guardrails. +5. Prefix-hash stability tests across common conversation trajectories. -- **Original**: XML tags wrapping each layer (`...`) -- **Simplification**: Clean double-newline separation -- **Rationale**: XML redundant, simpler text cleaner for LLM +### Runtime Metrics (Debug Mode) -#### 2. Deferred L4 Conversation Strip - -- **Original**: Summary + last K turns with deterministic truncation -- **Simplification**: Continue using existing LangChain memory -- **Rationale**: Token optimization, not required for cache correctness - -#### 3. Deferred Provider-Specific Caching - -- **Original**: Gemini explicit cache, Anthropic `cache_control` -- **Simplification**: Model-agnostic baseline only (stable prefixes) -- **Rationale**: Baseline first, optimizations incremental - -### Bug Fixes - -#### L2 Context Deduplication Bug (2025-01-25) - -**Problem**: `buildL2ContextFromPreviousTurns()` removed items from L2 if they appeared in current turn, breaking cache stability. - -**Symptom**: Same note sent twice across consecutive turns instead of being referenced by ID. - -**Root Cause**: - -```typescript -// WRONG: Prevents L2 accumulation -if (currentTurnContext) { - const currentTurnNotePaths = new Set((currentTurnContext.notes || []).map((note) => note.path)); - for (const notePath of currentTurnNotePaths) { - uniqueNotes.delete(notePath); // BUG - } -} -``` - -**Fix**: - -1. Removed deduplication logic from `buildL2ContextFromPreviousTurns()` -2. L2 contains ALL previous context (no filtering) -3. `LayerToMessagesConverter` compares segment IDs for smart referencing - -**Result**: L2 grows monotonically ✅ Cache hits maximized ✅ - -#### Uniform Tool Placement Refactor (2025-01-27) - -**Problem**: Architectural complexity with localSearch in system message vs. other tools in user message. - -**Old Design**: - -- `localSearch` → system message (RAG as authoritative knowledge) -- Other tools → user message (supplementary info) -- Complex branching logic, special-casing - -**New Design**: - -- ALL tools → user message uniformly -- System = L1 + L2 only (cacheable prefix) -- Single code path: `formatAllToolOutputs()` - -**Benefits**: - -- Eliminated ~100 lines of special-case code -- Clear cache boundary -- Automatic tool detection via `ToolRegistry` - -### Implementation Progress Phases - -**Phase 1** (Completed): - -- Canonical Layered Prefix types -- `PromptContextEngine` singleton -- `ContextManager` structured output -- `MessageRepository` envelope persistence - -**Phase 2** (Completed): - -- L2 auto-promotion implementation -- Cumulative L2 design (no deduplication) -- Segment-based smart referencing -- Stable ordering by first appearance - -**Phase 3** (Completed): - -- LLM chain envelope migration -- VaultQA chain envelope migration -- CopilotPlus chain envelope migration + uniform tools -- Agent chain envelope migration + iterative tool loop -- Payload logging with layered view - -**Phase 4** (Deferred): - -- Provider-specific cache controls -- Optional manual pinning UI -- L4 conversation strip with summarization - -### Agent Chain Runner Implementation - -**Goal**: Integrate the autonomous agent chain with the layered context system, following the same envelope-first architecture as CopilotPlus while preserving the iterative tool execution loop. - -**Implementation** (Completed 2025-01-27): - -The agent chain integration follows the CopilotPlus pattern with minimal changes to support the unique iterative tool loop: - -**1. Envelope Extraction & Validation** - -- Added envelope validation at start of `run()` method -- Fails fast with clear error if envelope missing -- Logs envelope-based context construction for debugging - -**2. System Message Construction** - -```typescript -// Extract L1+L2 from envelope via LayerToMessagesConverter -const baseMessages = LayerToMessagesConverter.convert(envelope, { - includeSystemMessage: true, - mergeUserContent: true, -}); - -// Combine with tool descriptions from model adapter -const systemContent = [ - systemMessage?.content || "", // L1 + L2 from envelope - toolDescriptionsPrompt || "", // Tool descriptions + guidelines -] - .filter(Boolean) - .join("\n\n"); -``` - -**3. Initial User Message Assembly** - -- Extract L3 (smart references) + L5 (user query) from converter -- Build multimodal content if images present (inherited from CopilotPlus) -- No adapter enhancement needed (envelope contains formatted content) - -**4. Original Prompt Extraction** - -- Extract L5 text for CiC ordering in tool results -- Used by `applyCiCOrderingToLocalSearchResult()` to append question - -**5. Agent Loop (Unchanged)** - -- Tool execution, parsing, and result formatting remain identical -- Tool results added to conversation as assistant/user message pairs -- CiC ordering applied to localSearch results as before -- ThinkBlockStreamer and ActionBlockStreamer preserved -- Iteration history and source collection unchanged - -**Key Design Points**: - -- **Minimal changes**: Only `prepareAgentConversation()` and envelope extraction modified -- **Tool system preserved**: All tool execution, streaming, and display logic unchanged -- **Message structure**: System (L1+L2+tools) → History → User (L3+L5) → Agent loop -- **Tool results**: Continue to be added to conversation messages, never promoted to L2 -- **Multimodal support**: Image extraction inherited from CopilotPlus base class - -**Differences from CopilotPlus**: - -| Aspect | CopilotPlus | Agent | -| ------------------ | ----------------------------------------- | ------------------------------------------- | -| **Tool Execution** | Pre-run (single batch via IntentAnalyzer) | Iterative loop (multi-turn) | -| **Tool Results** | Formatted once, prepended to user message | Accumulated across iterations | -| **System Prompt** | L1 + L2 | L1 + L2 + tool descriptions + guidelines | -| **Conversation** | Single turn | Multiple assistant/user pairs per iteration | -| **Streaming** | ThinkBlockStreamer only | ThinkBlockStreamer + ActionBlockStreamer | - ---- - -## Open Questions - -1. ~~L2 storage location?~~ **RESOLVED**: Auto-promotion from message history -2. ~~Pin/unpin UI?~~ **RESOLVED**: Auto-promotion, no UI needed -3. Memory snapshot UI affordance? (Deferred) -4. Layer metadata format in markdown exports? (HTML comment vs. YAML frontmatter) -5. Dataview query caching by note + mtime? (Optimization, not critical) +- Envelope build time by phase (L2 build, context processing, compaction, render). +- Segment counts per layer and dedup ratio. +- Prefix hash change reason classification. +- Parse-failure count and compacted-context proportion. --- ## References -### Key Files +### Primary Implementation Files -- **Core**: `src/context/PromptContextEngine.ts`, `src/core/ContextManager.ts` -- **Layer Conversion**: `src/context/LayerToMessagesConverter.ts` -- **Persistence**: `src/core/MessageRepository.ts`, `src/core/ChatPersistenceManager.ts` -- **Chain Runners**: `src/LLMProviders/chainRunner/*.ts` -- **Utilities**: `src/LLMProviders/chainRunner/utils/cicPromptUtils.ts` +- `src/core/ChatManager.ts` +- `src/core/ContextManager.ts` +- `src/context/PromptContextTypes.ts` +- `src/context/PromptContextEngine.ts` +- `src/context/parseContextSegments.ts` +- `src/context/LayerToMessagesConverter.ts` +- `src/core/MessageRepository.ts` +- `src/core/ChatPersistenceManager.ts` +- `src/LLMProviders/chainRunner/LLMChainRunner.ts` +- `src/LLMProviders/chainRunner/VaultQAChainRunner.ts` +- `src/LLMProviders/chainRunner/CopilotPlusChainRunner.ts` +- `src/LLMProviders/chainRunner/AutonomousAgentChainRunner.ts` -### Related Documentation +### Related Docs -- Message Architecture: `docs/MESSAGE_ARCHITECTURE.md` -- Technical Debt: `docs/TECHDEBT.md` -- Current Tasks: `TODO.md` +- `docs/MESSAGE_ARCHITECTURE.md` +- `docs/TOOLS.md` +- `docs/NATIVE_TOOL_CALLING_MIGRATION.md` +- `docs/TECHDEBT.md` +- `TODO.md` diff --git a/src/LLMProviders/chainManager.ts b/src/LLMProviders/chainManager.ts index 44b055a2..59595568 100644 --- a/src/LLMProviders/chainManager.ts +++ b/src/LLMProviders/chainManager.ts @@ -333,7 +333,11 @@ export default class ChainManager { ) { const { ignoreSystemMessage = false } = options; - logInfo("Step 0: Initial user message:\n", userMessage.message); + const l5Text = userMessage.contextEnvelope?.layers.find((l) => l.id === "L5_USER")?.text; + logInfo( + "Step 0: Initial user message:\n", + l5Text || userMessage.originalMessage || userMessage.message + ); this.validateChatModel(); this.validateChainInitialization(); @@ -371,16 +375,4 @@ export default class ChainManager { options ); } - - async updateMemoryWithLoadedMessages(messages: ChatMessage[]) { - await this.memoryManager.clearChatMemory(); - // Use memoryManager.saveContext to apply compaction for any old uncompacted messages - for (let i = 0; i < messages.length; i += 2) { - const userMsg = messages[i]; - const aiMsg = messages[i + 1]; - if (userMsg && aiMsg && userMsg.sender === USER_SENDER) { - await this.memoryManager.saveContext({ input: userMsg.message }, { output: aiMsg.message }); - } - } - } } diff --git a/src/LLMProviders/chainRunner/BaseChainRunner.ts b/src/LLMProviders/chainRunner/BaseChainRunner.ts index 1deb8461..8b9c8d95 100644 --- a/src/LLMProviders/chainRunner/BaseChainRunner.ts +++ b/src/LLMProviders/chainRunner/BaseChainRunner.ts @@ -70,14 +70,18 @@ export abstract class BaseChainRunner implements ChainRunner { !(abortController.signal.aborted && abortController.signal.reason === ABORT_REASON.NEW_CHAT); if (shouldAddMessage) { - // Use MemoryManager.saveContext for atomic operation and proper memory management - // This applies chat history compaction to reduce memory bloat from tool results - // Note: LangChain's memory expects text content, not multimodal arrays - // For truncated empty responses, save a placeholder message + // Save the expanded user message (L5) to memory — NOT the full processedText. + // processedText includes context artifact XML (L3), which already lives in the + // envelope's L2/L3 layers. Baking it into L4 chat history would cause + // triple-inclusion and waste tokens. + // L5 preserves prompt-expanded content (e.g. {include_note_content} placeholders) + // while excluding context artifact blocks. + const l5Text = userMessage.contextEnvelope?.layers.find((l) => l.id === "L5_USER")?.text; + const inputForMemory = l5Text || userMessage.originalMessage || userMessage.message; const outputForMemory = llmFormattedOutput || fullAIResponse || "[Response truncated - no content generated]"; await this.chainManager.memoryManager.saveContext( - { input: userMessage.message }, + { input: inputForMemory }, { output: outputForMemory } ); diff --git a/src/LLMProviders/chainRunner/CopilotPlusChainRunner.ts b/src/LLMProviders/chainRunner/CopilotPlusChainRunner.ts index 78b6e364..ac0dfb5e 100644 --- a/src/LLMProviders/chainRunner/CopilotPlusChainRunner.ts +++ b/src/LLMProviders/chainRunner/CopilotPlusChainRunner.ts @@ -606,10 +606,7 @@ Include your extracted terms as: [SALIENT_TERMS: term1, term2, term3]`; } const trimmedQuestion = - originalUserQuestion.trim() || - userMessage.message?.trim() || - userMessage.originalMessage?.trim() || - ""; + originalUserQuestion.trim() || userMessage.originalMessage?.trim() || ""; if (trimmedQuestion.length > 0) { sections.push(`${userQueryLabel}\n${trimmedQuestion}`); } else { @@ -740,7 +737,7 @@ Include your extracted terms as: [SALIENT_TERMS: term1, term2, term3]`; ); } const l5User = envelope.layers.find((l) => l.id === "L5_USER"); - const messageForAnalysis = l5User?.text || userMessage.originalMessage || userMessage.message; + const messageForAnalysis = l5User?.text || userMessage.originalMessage || ""; try { // Use model-based planning instead of Broca @@ -816,7 +813,13 @@ Include your extracted terms as: [SALIENT_TERMS: term1, term2, term3]`; } // Clean user message by removing @command tokens - const cleanedUserMessage = this.removeAtCommands(userMessage.message); + // Use L5 text (expanded user query without context XML) or displayText as fallback. + // userMessage.message is processedText which includes context artifact XML — using it + // here would re-inject L3 context into the user message, bypassing envelope separation. + const l5Text = userMessage.contextEnvelope?.layers.find((l) => l.id === "L5_USER")?.text; + const cleanedUserMessage = this.removeAtCommands( + l5Text || userMessage.originalMessage || userMessage.message + ); const { toolOutputs, sources: toolSources } = await this.executeToolCalls( toolCalls, diff --git a/src/components/Chat.tsx b/src/components/Chat.tsx index c6bf48cf..91944c55 100644 --- a/src/components/Chat.tsx +++ b/src/components/Chat.tsx @@ -259,7 +259,7 @@ const ChatInternal: React.FC { expect(tags).toContain("active_note"); expect(tags).toContain("url_content"); expect(tags).toContain("youtube_video_context"); + expect(tags).toContain("twitter_content"); expect(tags).toContain("selected_text"); expect(tags).toContain("localSearch"); }); @@ -45,6 +46,7 @@ describe("contextBlockRegistry", () => { it("should return correct source type for URL blocks", () => { expect(getSourceType("url_content")).toBe("url"); expect(getSourceType("web_tab_context")).toBe("url"); + expect(getSourceType("twitter_content")).toBe("url"); }); it("should return correct source type for YouTube blocks", () => { @@ -61,6 +63,7 @@ describe("contextBlockRegistry", () => { expect(isRecoverable("note_context")).toBe(true); expect(isRecoverable("url_content")).toBe(true); expect(isRecoverable("youtube_video_context")).toBe(true); + expect(isRecoverable("twitter_content")).toBe(true); }); it("should return false for non-recoverable types", () => { @@ -108,6 +111,14 @@ describe("contextBlockRegistry", () => { expect(extractSourceFromBlock(xml, "embedded_pdf")).toBe("document.pdf"); }); + it("should extract URL from twitter_content blocks", () => { + const xml = ` +https://x.com/user/status/123 +Tweet +`; + expect(extractSourceFromBlock(xml, "twitter_content")).toBe("https://x.com/user/status/123"); + }); + it("should return empty string for blocks without source extractor", () => { const xml = `Just text`; expect(extractSourceFromBlock(xml, "selected_text")).toBe(""); @@ -140,4 +151,17 @@ describe("contextBlockRegistry", () => { expect(detectBlockTag("")).toBeNull(); }); }); + + describe("tag alignment", () => { + it("should have all URL-producing tags registered (prevents tag mismatch bugs)", () => { + // These are the tags that Mention.ts and other URL processors create. + // If a new tag is added to URL processing, it MUST be registered here. + const registeredTags = new Set(CONTEXT_BLOCK_TYPES.map((bt) => bt.tag)); + const urlProducerTags = ["url_content", "youtube_video_context", "twitter_content"]; + + for (const tag of urlProducerTags) { + expect(registeredTags.has(tag)).toBe(true); + } + }); + }); }); diff --git a/src/context/contextBlockRegistry.ts b/src/context/contextBlockRegistry.ts index 05b8e63b..303184ff 100644 --- a/src/context/contextBlockRegistry.ts +++ b/src/context/contextBlockRegistry.ts @@ -56,6 +56,9 @@ export const CONTEXT_BLOCK_TYPES: ContextBlockType[] = [ sourceExtractor: "url", }, + // Twitter/X (recoverable via URL processing) + { tag: "twitter_content", sourceType: "url", recoverable: true, sourceExtractor: "url" }, + // PDF (partially recoverable - needs the file in vault) { tag: "embedded_pdf", sourceType: "pdf", recoverable: true, sourceExtractor: "name" }, diff --git a/src/context/parseContextSegments.ts b/src/context/parseContextSegments.ts new file mode 100644 index 00000000..a613e12d --- /dev/null +++ b/src/context/parseContextSegments.ts @@ -0,0 +1,76 @@ +import { PromptLayerSegment } from "@/context/PromptContextTypes"; +import { + CONTEXT_BLOCK_TYPES, + extractSourceFromBlock, + getSourceType, +} from "@/context/contextBlockRegistry"; + +/** + * Parse context XML string into individual segments (one per context item). + * Uses the contextBlockRegistry to dynamically match ALL registered block types, + * plus blocks (compaction artifacts from L2). + */ +export function parseContextIntoSegments( + contextXml: string, + stable: boolean +): PromptLayerSegment[] { + if (!contextXml.trim()) { + return []; + } + + const segments: PromptLayerSegment[] = []; + + // Build regex dynamically from all registered block types + prior_context + const registeredTags = CONTEXT_BLOCK_TYPES.map((bt) => bt.tag); + const allTags = [...registeredTags, "prior_context"]; + const allBlocksRegex = new RegExp(`<(${allTags.join("|")})(\\s[^>]*)?>[\\s\\S]*?`, "g"); + + // Track tag-based fallback IDs to ensure uniqueness for blocks without source extractors + // (e.g., multiple selected_text blocks should not share the same ID) + const tagIdCounts = new Map(); + + let match: RegExpExecArray | null; + while ((match = allBlocksRegex.exec(contextXml)) !== null) { + const block = match[0]; + const tag = match[1]; + + if (tag === "prior_context") { + // Compacted blocks have source in attribute: + const sourceMatch = /source="([^"]+)"/.exec(block); + const source = sourceMatch?.[1] ?? "prior_context"; + segments.push({ + id: source, + content: block, + stable, + metadata: { + source: "previous_turns_compacted", + notePath: source, + }, + }); + } else { + // Use registry to extract the source identifier + const extractedId = extractSourceFromBlock(block, tag); + let sourceId: string; + if (extractedId) { + sourceId = extractedId; + } else { + // Fallback: use tag name with counter to ensure uniqueness + const count = (tagIdCounts.get(tag) || 0) + 1; + tagIdCounts.set(tag, count); + sourceId = count === 1 ? tag : `${tag}:${count}`; + } + const isNote = getSourceType(tag) === "note"; + segments.push({ + id: sourceId, + content: block, + stable, + metadata: { + source: stable ? "previous_turns" : "current_turn", + ...(isNote ? { notePath: sourceId } : {}), + }, + }); + } + } + + return segments; +} diff --git a/src/contextProcessor.ts b/src/contextProcessor.ts index e897d29d..9c9f8db1 100644 --- a/src/contextProcessor.ts +++ b/src/contextProcessor.ts @@ -573,10 +573,10 @@ export class ContextProcessor { const ctime = stats ? new Date(stats.ctime).toISOString() : "Unknown"; const mtime = stats ? new Date(stats.mtime).toISOString() : "Unknown"; - additionalContext += `\n\n<${prompt_tag}>\n${note.basename}\n${note.path}\n${ctime}\n${mtime}\n\n${content}\n\n`; + additionalContext += `\n\n<${prompt_tag}>\n${escapeXml(note.basename)}\n${note.path}\n${ctime}\n${mtime}\n\n${content}\n\n`; } catch (error) { logError(`Error processing file ${note.path}:`, error); - additionalContext += `\n\n<${prompt_tag}_error>\n${note.basename}\n${note.path}\n[Error: Could not process file]\n`; + additionalContext += `\n\n<${prompt_tag}_error>\n${escapeXml(note.basename)}\n${note.path}\n[Error: Could not process file]\n`; } }; diff --git a/src/core/ChatManager.test.ts b/src/core/ChatManager.test.ts index b21b5b15..bbfc9bbc 100644 --- a/src/core/ChatManager.test.ts +++ b/src/core/ChatManager.test.ts @@ -28,6 +28,7 @@ jest.mock("./ChatPersistenceManager", () => ({ jest.mock("@/aiParams", () => ({ getCurrentProject: jest.fn().mockReturnValue(null), + getChainType: jest.fn().mockReturnValue("copilot_plus_chain"), })); jest.mock("@/LLMProviders/projectManager", () => { @@ -329,7 +330,10 @@ describe("ChatManager", () => { it("should regenerate AI message successfully", async () => { const mockAiMessage = createMockMessage("msg-2", "AI response", "AI"); const mockUserMessage = createMockMessage("msg-1", "Hello", USER_SENDER); - const mockLLMMessage = createMockMessage("msg-1", "Hello with context", USER_SENDER); + const mockLLMMessage = { + ...createMockMessage("msg-1", "Hello with context", USER_SENDER), + contextEnvelope: { layers: [] } as any, // Has envelope, no lazy reprocessing needed + }; mockMessageRepo.getMessage.mockReturnValue(mockAiMessage); mockMessageRepo.getDisplayMessages.mockReturnValue([mockUserMessage, mockAiMessage]); @@ -357,6 +361,53 @@ describe("ChatManager", () => { ); }); + it("should lazily reprocess context when envelope is missing (loaded from disk)", async () => { + const mockAiMessage = createMockMessage("msg-2", "AI response", "AI"); + const mockUserMessage = createMockMessage("msg-1", "Hello", USER_SENDER); + // First call: no envelope (loaded from disk) + const mockLLMMessageNoEnvelope = createMockMessage( + "msg-1", + "Hello with context", + USER_SENDER + ); + // Second call: after reprocessing, has envelope + const mockLLMMessageWithEnvelope = { + ...createMockMessage("msg-1", "Hello with context", USER_SENDER), + contextEnvelope: { layers: [] } as any, + }; + + mockMessageRepo.getMessage.mockReturnValue(mockAiMessage); + mockMessageRepo.getDisplayMessages.mockReturnValue([mockUserMessage, mockAiMessage]); + mockMessageRepo.getLLMMessage + .mockReturnValueOnce(mockLLMMessageNoEnvelope) + .mockReturnValueOnce(mockLLMMessageWithEnvelope); + mockMessageRepo.truncateAfter.mockReturnValue(undefined); + mockChainManager.runChain.mockResolvedValue(undefined); + mockContextManager.reprocessMessageContext.mockResolvedValue(undefined); + + const result = await chatManager.regenerateMessage("msg-2", jest.fn(), jest.fn()); + + expect(result).toBe(true); + expect(mockContextManager.reprocessMessageContext).toHaveBeenCalledWith( + "msg-1", + expect.anything(), // messageRepo + expect.anything(), // fileParserManager + undefined, // vault (undefined in mock) + "copilot_plus_chain", // chainType + false, // includeActiveNote + undefined, // activeNote + "Test system prompt", // systemPrompt + [] // systemPromptIncludedFiles + ); + expect(mockChainManager.runChain).toHaveBeenCalledWith( + mockLLMMessageWithEnvelope, + expect.any(AbortController), + expect.any(Function), + expect.any(Function), + expect.any(Object) + ); + }); + it("should return false when message not found", async () => { mockMessageRepo.getMessage.mockReturnValue(undefined); @@ -621,7 +672,10 @@ describe("ChatManager", () => { it("should handle regeneration with proper message truncation", async () => { const mockAiMessage = createMockMessage("msg-2", "AI response", "AI"); const mockUserMessage = createMockMessage("msg-1", "Hello", USER_SENDER); - const mockLLMMessage = createMockMessage("msg-1", "Hello with context", USER_SENDER); + const mockLLMMessage = { + ...createMockMessage("msg-1", "Hello with context", USER_SENDER), + contextEnvelope: { layers: [] } as any, + }; mockMessageRepo.getMessage.mockReturnValue(mockAiMessage); mockMessageRepo.getDisplayMessages.mockReturnValue([mockUserMessage, mockAiMessage]); diff --git a/src/core/ChatManager.ts b/src/core/ChatManager.ts index 60096050..7d3a8dd1 100644 --- a/src/core/ChatManager.ts +++ b/src/core/ChatManager.ts @@ -5,7 +5,7 @@ import { getSystemPromptWithMemory, } from "@/system-prompts/systemPromptBuilder"; import { ChainType } from "@/chainFactory"; -import { getCurrentProject } from "@/aiParams"; +import { getChainType, getCurrentProject } from "@/aiParams"; import { logInfo, logWarn } from "@/logger"; import { ChatMessage, MessageContext, WebTabContext } from "@/types/message"; import { processPrompt, type ProcessedPromptResult } from "@/commands/customCommandUtils"; @@ -589,12 +589,35 @@ export class ChatManager { return false; } - const llmMessage = currentRepo.getLLMMessage(userMessage.id); + let llmMessage = currentRepo.getLLMMessage(userMessage.id); if (!llmMessage) { logInfo(`[ChatManager] LLM message not found for regeneration`); return false; } + // Lazy reprocess: if contextEnvelope is missing (e.g., loaded from disk), + // reprocess context before running the chain + if (!llmMessage.contextEnvelope) { + logInfo(`[ChatManager] Context envelope missing, reprocessing context for regeneration`); + const chainType = getChainType(); + const activeNote = this.plugin.app.workspace.getActiveFile(); + const { processedPrompt: systemPrompt, includedFiles: systemPromptIncludedFiles } = + await this.getSystemPromptForMessage(chainType, this.plugin.app.vault, activeNote); + await this.contextManager.reprocessMessageContext( + userMessage.id, + currentRepo, + this.fileParserManager, + this.plugin.app.vault, + chainType, + false, + activeNote, + systemPrompt, + systemPromptIncludedFiles + ); + // Re-fetch the LLM message with the newly created envelope + llmMessage = currentRepo.getLLMMessage(userMessage.id)!; + } + // Run the chain to regenerate the response const abortController = new AbortController(); await this.chainManager.runChain( diff --git a/src/core/ContextManager.parseSegments.test.ts b/src/core/ContextManager.parseSegments.test.ts new file mode 100644 index 00000000..3731a77d --- /dev/null +++ b/src/core/ContextManager.parseSegments.test.ts @@ -0,0 +1,270 @@ +import { parseContextIntoSegments } from "@/context/parseContextSegments"; +import { CONTEXT_BLOCK_TYPES } from "@/context/contextBlockRegistry"; + +describe("parseContextIntoSegments", () => { + describe("empty input", () => { + it("should return empty array for empty string", () => { + expect(parseContextIntoSegments("", false)).toEqual([]); + }); + + it("should return empty array for whitespace-only string", () => { + expect(parseContextIntoSegments(" \n ", false)).toEqual([]); + }); + }); + + describe("note blocks", () => { + it("should parse note_context blocks with path as ID", () => { + const xml = ` +My Note +folder/my-note.md +Some content +`; + const segments = parseContextIntoSegments(xml, false); + expect(segments).toHaveLength(1); + expect(segments[0].id).toBe("folder/my-note.md"); + expect(segments[0].content).toBe(xml); + expect(segments[0].stable).toBe(false); + expect(segments[0].metadata?.source).toBe("current_turn"); + expect(segments[0].metadata?.notePath).toBe("folder/my-note.md"); + }); + + it("should parse active_note blocks", () => { + const xml = ` +Active +active.md +Active content +`; + const segments = parseContextIntoSegments(xml, true); + expect(segments).toHaveLength(1); + expect(segments[0].id).toBe("active.md"); + expect(segments[0].metadata?.source).toBe("previous_turns"); + expect(segments[0].metadata?.notePath).toBe("active.md"); + }); + }); + + describe("URL blocks", () => { + it("should parse url_content blocks with URL as ID", () => { + const xml = ` +https://example.com/page +Page content here +`; + const segments = parseContextIntoSegments(xml, false); + expect(segments).toHaveLength(1); + expect(segments[0].id).toBe("https://example.com/page"); + expect(segments[0].content).toBe(xml); + expect(segments[0].metadata?.source).toBe("current_turn"); + expect(segments[0].metadata?.notePath).toBeUndefined(); + }); + + it("should parse web_tab_context blocks", () => { + const xml = ` +https://docs.example.com +Documentation +`; + const segments = parseContextIntoSegments(xml, false); + expect(segments).toHaveLength(1); + expect(segments[0].id).toBe("https://docs.example.com"); + }); + }); + + describe("YouTube blocks", () => { + it("should parse youtube_video_context blocks with URL as ID", () => { + const xml = ` +https://www.youtube.com/watch?v=abc123 +Transcript of the video +`; + const segments = parseContextIntoSegments(xml, false); + expect(segments).toHaveLength(1); + expect(segments[0].id).toBe("https://www.youtube.com/watch?v=abc123"); + expect(segments[0].content).toBe(xml); + expect(segments[0].metadata?.source).toBe("current_turn"); + expect(segments[0].metadata?.notePath).toBeUndefined(); + }); + }); + + describe("Twitter blocks", () => { + it("should parse twitter_content blocks with URL as ID", () => { + const xml = ` +https://x.com/user/status/123 +Tweet content +`; + const segments = parseContextIntoSegments(xml, false); + expect(segments).toHaveLength(1); + expect(segments[0].id).toBe("https://x.com/user/status/123"); + expect(segments[0].metadata?.source).toBe("current_turn"); + }); + }); + + describe("PDF blocks", () => { + it("should parse embedded_pdf blocks with name as ID", () => { + const xml = ` +document.pdf +PDF text content +`; + const segments = parseContextIntoSegments(xml, false); + expect(segments).toHaveLength(1); + expect(segments[0].id).toBe("document.pdf"); + }); + }); + + describe("selected_text blocks", () => { + it("should parse selected_text blocks with tag as ID (no source extractor)", () => { + const xml = ` +User selected this text +`; + const segments = parseContextIntoSegments(xml, false); + expect(segments).toHaveLength(1); + expect(segments[0].id).toBe("selected_text"); + }); + + it("should assign unique IDs to multiple selected_text blocks", () => { + const xml = ` +First selection + + + +Second selection +`; + const segments = parseContextIntoSegments(xml, false); + expect(segments).toHaveLength(2); + expect(segments[0].id).toBe("selected_text"); + expect(segments[1].id).toBe("selected_text:2"); + }); + }); + + describe("prior_context blocks (compaction artifacts)", () => { + it("should parse prior_context blocks with source attribute as ID", () => { + const xml = ` +Compacted summary of old note +`; + const segments = parseContextIntoSegments(xml, true); + expect(segments).toHaveLength(1); + expect(segments[0].id).toBe("folder/old-note.md"); + expect(segments[0].metadata?.source).toBe("previous_turns_compacted"); + expect(segments[0].metadata?.notePath).toBe("folder/old-note.md"); + }); + + it("should fall back to 'prior_context' as ID when no source attribute", () => { + const xml = ` +Compacted content +`; + const segments = parseContextIntoSegments(xml, true); + expect(segments).toHaveLength(1); + expect(segments[0].id).toBe("prior_context"); + }); + }); + + describe("stable flag", () => { + it("should set metadata.source to 'previous_turns' when stable=true", () => { + const xml = ` +https://example.com +Content +`; + const segments = parseContextIntoSegments(xml, true); + expect(segments[0].stable).toBe(true); + expect(segments[0].metadata?.source).toBe("previous_turns"); + }); + + it("should set metadata.source to 'current_turn' when stable=false", () => { + const xml = ` +https://example.com +Content +`; + const segments = parseContextIntoSegments(xml, false); + expect(segments[0].stable).toBe(false); + expect(segments[0].metadata?.source).toBe("current_turn"); + }); + }); + + describe("mixed block types", () => { + it("should parse multiple different block types from a single string", () => { + const xml = ` +Note +notes/test.md +Note content + + + +https://example.com +URL content + + + +https://youtube.com/watch?v=xyz +Transcript + + + +Compacted note +`; + + const segments = parseContextIntoSegments(xml, false); + expect(segments).toHaveLength(4); + expect(segments[0].id).toBe("notes/test.md"); + expect(segments[1].id).toBe("https://example.com"); + expect(segments[2].id).toBe("https://youtube.com/watch?v=xyz"); + expect(segments[3].id).toBe("old/note.md"); + }); + + it("should parse multiple blocks of the same type", () => { + const xml = ` +Note 1 +note1.md +Content 1 + + + +Note 2 +note2.md +Content 2 +`; + + const segments = parseContextIntoSegments(xml, false); + expect(segments).toHaveLength(2); + expect(segments[0].id).toBe("note1.md"); + expect(segments[1].id).toBe("note2.md"); + }); + }); + + describe("registry completeness", () => { + it("should parse every block type registered in CONTEXT_BLOCK_TYPES", () => { + for (const blockType of CONTEXT_BLOCK_TYPES) { + const tag = blockType.tag; + let xml: string; + + // Build a valid XML block based on the source extractor type + switch (blockType.sourceExtractor) { + case "path": + xml = `<${tag}>test/file.mdtest`; + break; + case "url": + xml = `<${tag}>https://example.comtest`; + break; + case "name": + xml = `<${tag}>file.pdftest`; + break; + default: + xml = `<${tag}>test`; + break; + } + + const segments = parseContextIntoSegments(xml, false); + expect(segments).toHaveLength(1); + expect(segments[0].content).toBe(xml); + } + }); + }); + + describe("non-matching content", () => { + it("should return empty array for plain text without XML blocks", () => { + const segments = parseContextIntoSegments("Just some plain text", false); + expect(segments).toEqual([]); + }); + + it("should return empty array for unregistered XML tags", () => { + const xml = `Something`; + const segments = parseContextIntoSegments(xml, false); + expect(segments).toEqual([]); + }); + }); +}); diff --git a/src/core/ContextManager.ts b/src/core/ContextManager.ts index 1605645d..f3e86edc 100644 --- a/src/core/ContextManager.ts +++ b/src/core/ContextManager.ts @@ -5,6 +5,7 @@ import { LOADING_MESSAGES } from "@/constants"; import { PromptContextEngine } from "@/context/PromptContextEngine"; import { compactXmlBlock, getL2RefetchInstruction } from "@/context/L2ContextCompactor"; import { CONTEXT_BLOCK_TYPES, detectBlockTag } from "@/context/contextBlockRegistry"; +import { parseContextIntoSegments } from "@/context/parseContextSegments"; import { PromptContextEnvelope, PromptLayerId, @@ -359,10 +360,9 @@ export class ContextManager { * - Comparing current file mtime with stored mtime when building L2 * - Skipping path deduplication for stale compacted segments * - * TODO: Deduplicate L2 content by notePath when building l2Context. - * Currently, if the same note is included across multiple turns (e.g., auto-added active note), - * its content is appended to L2 once per turn, causing linear growth. Consider deduplicating - * by notePath when concatenating L3 segments, not just collecting paths for exclusions. + * L2 deduplication: Segments are deduplicated by ID (last-write-wins). When the same + * artifact (note, URL, etc.) appears in multiple turns, only the most recent version + * is kept in L2, preventing linear growth. */ private buildL2ContextFromPreviousTurns( currentMessageId: string, @@ -379,7 +379,10 @@ export class ContextManager { .slice(0, currentIndex) .filter((msg) => msg.sender === "user"); - const l2Parts: string[] = []; + // Deduplicated map: segment ID → compacted content (last-write-wins) + const l2SegmentMap = new Map(); + // Insertion order tracking for stable output + const l2SegmentOrder: string[] = []; const l2Paths = new Set(); // Find the most recent compacted message index. @@ -404,18 +407,16 @@ export class ContextManager { // Only include L3 content from the most recent compacted message onwards. // Earlier messages' content is already included in the compacted L3. if (i >= mostRecentCompactedIndex) { - const segmentContent: string[] = []; for (const segment of l3Layer.segments || []) { if (segment.content) { - // Compact large L3 segments for L2 to reduce context size - // This extracts structure + preview with tool hints for re-fetching const compacted = this.compactSegmentForL2(segment.content); - segmentContent.push(compacted); + // Deduplicate by segment ID: later turns overwrite earlier ones + if (!l2SegmentMap.has(segment.id)) { + l2SegmentOrder.push(segment.id); + } + l2SegmentMap.set(segment.id, compacted); } } - if (segmentContent.length > 0) { - l2Parts.push(segmentContent.join("\n")); - } } // Track paths from ALL messages for deduplication @@ -423,13 +424,11 @@ export class ContextManager { if (segment.metadata?.notePath) { l2Paths.add(segment.metadata.notePath as string); } - // Handle compacted segments if (segment.metadata?.compactedPaths) { for (const path of segment.metadata.compactedPaths as string[]) { l2Paths.add(path); } } - // Handle tag/folder segments that store paths in notePaths if (segment.metadata?.notePaths) { for (const path of segment.metadata.notePaths as string[]) { l2Paths.add(path); @@ -437,24 +436,10 @@ export class ContextManager { } } } - // Note: Messages without envelopes (pre-envelope chat history or loaded from disk) - // are intentionally NOT tracked for L2 deduplication to avoid filtering notes - // without their content appearing in L2. - // - // TODO: Rebuild L2 context for loaded chats. - // ChatPersistenceManager saves context metadata (note paths, tags, folders) but not - // the full contextEnvelope. When a chat is loaded, messages have context but no envelope, - // so L2 context is lost. This means: - // 1. AI loses the "context library" from turns before the save - // 2. Same notes can be duplicated if re-attached after loading - // Fix options: - // - Persist envelopes to disk (increases storage, requires migration) - // - Rebuild L2 content by re-reading files from message.context on load (expensive) - // - Lazy rebuild: process context.notes when first accessed after load } - // Build the L2 context string - const l2Content = l2Parts.join("\n"); + // Build the L2 context string in insertion order (deduplicated) + const l2Content = l2SegmentOrder.map((id) => l2SegmentMap.get(id)!).join("\n"); // Append re-fetch instruction if there's any L2 content with prior_context blocks const hasCompactedContent = l2Content.includes(" 0) { layerSegments.L3_TURN = turnSegments; @@ -620,78 +600,46 @@ export class ContextManager { } /** - * Parse context XML string into individual segments (one per note/context item) - * Extracts , , and blocks and creates segments with path as ID + * Parse context XML string into individual segments (one per context item). + * Delegates to the standalone parseContextIntoSegments function. */ private parseContextIntoSegments(contextXml: string, stable: boolean): PromptLayerSegment[] { - if (!contextXml.trim()) { - return []; - } - - const segments: PromptLayerSegment[] = []; - - // Match , , and blocks - // prior_context has attributes: - const noteContextRegex = - /<(?:note_context|active_note)>[\s\S]*?<\/(?:note_context|active_note)>/g; - const priorContextRegex = /]*>[\s\S]*?<\/prior_context>/g; - - let match: RegExpExecArray | null; - - // Parse original note blocks - while ((match = noteContextRegex.exec(contextXml)) !== null) { - const block = match[0]; - const pathMatch = /([^<]+)<\/path>/.exec(block); - if (!pathMatch) continue; - - const notePath = pathMatch[1]; - segments.push({ - id: notePath, - content: block, - stable, - metadata: { - source: stable ? "previous_turns" : "current_turn", - notePath, - }, - }); - } - - // Parse compacted prior_context blocks (L2 from previous turns) - while ((match = priorContextRegex.exec(contextXml)) !== null) { - const block = match[0]; - const source = match[1]; // Extracted from source="..." attribute - - segments.push({ - id: source, - content: block, - stable, - metadata: { - source: "previous_turns_compacted", - notePath: source, - }, - }); - } - - return segments; + return parseContextIntoSegments(contextXml, stable); } - private appendTurnContextSegment( + /** + * Parse context XML into individual per-artifact segments and append to target. + * Each XML block gets its own segment with a content-derived ID (URL, path, etc.) + * from the registry. Extra metadata (e.g., notePaths for tags/folders) is merged + * into each resulting segment. + */ + private appendParsedSegments( target: PromptLayerSegment[], - segmentId: string, content: string, - metadata: Record + extraMetadata?: Record ) { const normalized = (content || "").trim(); if (!normalized) { return; } - target.push({ - id: `${segmentId}`, - content: normalized, - stable: false, - metadata, - }); + const segments = this.parseContextIntoSegments(normalized, false); + if (segments.length > 0) { + for (const seg of segments) { + if (extraMetadata) { + seg.metadata = { ...seg.metadata, ...extraMetadata }; + } + target.push(seg); + } + } else { + // Fallback: content didn't parse into known XML blocks, keep as single segment + target.push({ + id: `unparsed-${Date.now()}`, + content: normalized, + stable: false, + metadata: { source: "unparsed", ...extraMetadata }, + }); + } } /** diff --git a/src/mentions/Mention.ts b/src/mentions/Mention.ts index fcb4a26a..5d2434dd 100644 --- a/src/mentions/Mention.ts +++ b/src/mentions/Mention.ts @@ -160,7 +160,7 @@ export class Mention { if (urlData.processed) { if (result.type === "youtube") { - urlContext += `\n\n\n${urlData.original}\n\n${urlData.processed}\n\n`; + urlContext += `\n\n\n${urlData.original}\n\n${urlData.processed}\n\n`; } else if (result.type === "twitter") { urlContext += `\n\n\n${urlData.original}\n\n${urlData.processed}\n\n`; } else {