Support embedded note (#1995)

* Update bug report template to enhance clarity and requirements

- Changed the required checklist item from a screenshot to a log file generated via the "Copilot: Create Log File" command.
- Made the screenshot of the note and Copilot chat pane optional.
- Clarified the statement regarding bug report requirements for better understanding.

* feat: Implement embedded note processing in ContextProcessor

- Added support for processing embedded notes within markdown content, allowing for structured `<embedded_note>` blocks.
- Introduced new methods for building and formatting embedded note blocks, including error handling for missing notes.
- Enhanced logging to replace console statements with appropriate logging functions for better error tracking.
- Updated the `buildMarkdownContextContent` method to streamline content processing for markdown files.
- Added a new constant `EMBEDDED_NOTE_TAG` to manage embedded note structures.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,11 +7,13 @@ assignees: ""
 ---
 
 - [ ] Disable all other plugins besides Copilot **(required)**
-- [ ] Screenshot of note + Copilot chat pane + dev console added **(required)**
+- [ ] Log file generated via "Copilot: Create Log File" command or Settings -> Advanced -> Create Log File **(required)**
+- [ ] Screenshot of note + Copilot chat pane + dev console added **(optional)**
 
 Copilot version:
+Model used:
 
-(Bug report without the above will be closed)
+(Bug reports missing the required items above will be closed)
 
 **Describe how to reproduce**
 A clear and concise description of what the bug is. Clear steps to reproduce the behavior

src/constants.ts

@@ -104,6 +104,7 @@ export const SELECTED_TEXT_TAG = "selected_text";
 export const VARIABLE_TAG = "variable";
 export const VARIABLE_NOTE_TAG = "variable_note";
 export const EMBEDDED_PDF_TAG = "embedded_pdf";
+export const EMBEDDED_NOTE_TAG = "embedded_note";
 export const DATAVIEW_BLOCK_TAG = "dataview_block";
 export const VAULT_NOTE_TAG = "vault_note";
 export const RETRIEVED_DOCUMENT_TAG = "retrieved_document";

src/contextProcessor.embeds.test.ts

@@ -0,0 +1,203 @@
+jest.mock("@/chainFactory", () => ({
+  ChainType: {
+    LLM_CHAIN: "llm_chain",
+    COPILOT_PLUS_CHAIN: "copilot_plus",
+    PROJECT_CHAIN: "project_chain",
+  },
+}));
+
+import { ContextProcessor } from "@/contextProcessor";
+import { EMBEDDED_NOTE_TAG } from "@/constants";
+import { ChainType } from "@/chainFactory";
+import { TFile, Vault } from "obsidian";
+
+type FileCacheMap = Record<string, any>;
+type FileContentMap = Record<string, string>;
+
+const createMockFile = (path: string): TFile => new (TFile as any)(path);
+
+describe("ContextProcessor - Embedded Notes", () => {
+  let contextProcessor: ContextProcessor;
+  let vault: Vault;
+  let fileParserManager: any;
+  let fileCaches: FileCacheMap;
+  let fileContents: FileContentMap;
+  let fileIndex: Map<string, TFile>;
+
+  beforeEach(() => {
+    contextProcessor = ContextProcessor.getInstance();
+
+    fileCaches = {};
+    fileContents = {};
+    fileIndex = new Map<string, TFile>();
+
+    const metadataCacheMock = {
+      getFirstLinkpathDest: jest.fn((link: string, _sourcePath: string) => {
+        const normalized = link.endsWith(".md") ? link : `${link}.md`;
+        return fileIndex.get(normalized) ?? null;
+      }),
+      getFileCache: jest.fn((file: TFile) => fileCaches[file.path] ?? {}),
+    };
+
+    (global as any).app = {
+      metadataCache: metadataCacheMock,
+    };
+
+    vault = {
+      adapter: {
+        stat: jest.fn().mockResolvedValue({ ctime: 0, mtime: 0 }),
+      },
+    } as unknown as Vault;
+
+    (vault as any).getAbstractFileByPath = jest.fn();
+
+    fileParserManager = {
+      supportsExtension: jest.fn(
+        (extension: string) => extension === "md" || extension === "canvas"
+      ),
+      parseFile: jest.fn(async (file: TFile) => {
+        const content = fileContents[file.path];
+        if (content === undefined) {
+          throw new Error(`Missing mock content for ${file.path}`);
+        }
+        return content;
+      }),
+    };
+  });
+
+  const registerFile = (file: TFile, content: string, cache: any = {}): void => {
+    fileIndex.set(file.path, file);
+    fileContents[file.path] = content;
+    fileCaches[file.path] = cache;
+  };
+
+  it("should include embedded note content in the context payload", async () => {
+    const source = createMockFile("Source.md");
+    const embedded = createMockFile("Embedded.md");
+
+    registerFile(source, "Introduction\n![[Embedded]]\nConclusion");
+    registerFile(embedded, "Embedded note body");
+
+    const result = await contextProcessor.processContextNotes(
+      new Set(),
+      fileParserManager,
+      vault,
+      [source],
+      false,
+      null,
+      ChainType.LLM_CHAIN
+    );
+
+    expect(result).toContain(`<${EMBEDDED_NOTE_TAG}>`);
+    expect(result).toContain("Embedded note body");
+  });
+
+  it("should extract a heading section when the embedded note targets a heading", async () => {
+    const source = createMockFile("Source.md");
+    const embedded = createMockFile("Embedded.md");
+    const embeddedContent = "## Section\nImportant details\n\n## Other\nOther details";
+
+    registerFile(source, "Root\n![[Embedded#Section]]\nTail");
+    registerFile(embedded, embeddedContent, {
+      headings: [
+        {
+          heading: "Section",
+          level: 2,
+          position: { start: { offset: 0 } },
+        },
+        {
+          heading: "Other",
+          level: 2,
+          position: { start: { offset: embeddedContent.indexOf("## Other") } },
+        },
+      ],
+    });
+
+    const result = await contextProcessor.processContextNotes(
+      new Set(),
+      fileParserManager,
+      vault,
+      [source],
+      false,
+      null,
+      ChainType.LLM_CHAIN
+    );
+
+    expect(result).toContain("<heading>Section</heading>");
+    expect(result).toContain("Important details");
+    expect(result).not.toContain("Other details");
+  });
+
+  it("should extract block reference content when embedding a block", async () => {
+    const source = createMockFile("Source.md");
+    const embedded = createMockFile("Embedded.md");
+    const embeddedContent = "Paragraph 1\nParagraph 2 ^block-ref\nParagraph 3\n";
+    const blockStart = embeddedContent.indexOf("Paragraph 2");
+    const blockEnd = embeddedContent.indexOf("Paragraph 3");
+
+    registerFile(source, "![[Embedded#^block-ref]]");
+    registerFile(embedded, embeddedContent, {
+      blocks: {
+        "block-ref": {
+          position: {
+            start: { offset: blockStart },
+            end: { offset: blockEnd },
+          },
+        },
+      },
+    });
+
+    const result = await contextProcessor.processContextNotes(
+      new Set(),
+      fileParserManager,
+      vault,
+      [source],
+      false,
+      null,
+      ChainType.LLM_CHAIN
+    );
+
+    expect(result).toContain("<block_id>block-ref</block_id>");
+    expect(result).toContain("Paragraph 2 ^block-ref");
+    expect(result).not.toContain("Paragraph 3");
+  });
+
+  it("should leave nested embeds untouched for recursive references", async () => {
+    const source = createMockFile("Source.md");
+    const embedded = createMockFile("Embedded.md");
+
+    registerFile(source, "Parent\n![[Embedded]]", {});
+    registerFile(embedded, "Child\n![[Source]]", {});
+
+    const result = await contextProcessor.processContextNotes(
+      new Set(),
+      fileParserManager,
+      vault,
+      [source],
+      false,
+      null,
+      ChainType.LLM_CHAIN
+    );
+
+    expect(result).toContain("<content>");
+    expect(result).toContain("![[Source]]");
+  });
+
+  it("should surface an error when the embedded note cannot be resolved", async () => {
+    const source = createMockFile("Source.md");
+
+    registerFile(source, "Missing\n![[Absent]]", {});
+
+    const result = await contextProcessor.processContextNotes(
+      new Set(),
+      fileParserManager,
+      vault,
+      [source],
+      false,
+      null,
+      ChainType.LLM_CHAIN
+    );
+
+    expect(result).toContain("<error>Embedded note not found</error>");
+  });
+});

src/contextProcessor.ts

@@ -1,16 +1,29 @@
 import { getSelectedTextContexts } from "@/aiParams";
 import { ChainType } from "@/chainFactory";
 import { RESTRICTION_MESSAGES } from "@/constants";
+import { logWarn, logInfo, logError } from "@/logger";
 import { FileParserManager } from "@/tools/FileParserManager";
 import { isPlusChain } from "@/utils";
 import { TFile, Vault, Notice } from "obsidian";
 import {
   NOTE_CONTEXT_PROMPT_TAG,
   EMBEDDED_PDF_TAG,
+  EMBEDDED_NOTE_TAG,
   SELECTED_TEXT_TAG,
   DATAVIEW_BLOCK_TAG,
 } from "./constants";
 
+interface EmbeddedLinkTarget {
+  path: string | null;
+  heading?: string;
+  blockId?: string;
+}
+
+interface MarkdownSegment {
+  content: string;
+  found: boolean;
+}
+
 export class ContextProcessor {
   private static instance: ContextProcessor;
 
@@ -43,7 +56,7 @@ export class ContextProcessor {
             `\n\n<${EMBEDDED_PDF_TAG}>\n<name>${pdfName}</name>\n<content>\n${pdfContent}\n</content>\n</${EMBEDDED_PDF_TAG}>\n\n`
           );
         } catch (error) {
-          console.error(`Error processing embedded PDF ${pdfName}:`, error);
+          logError(`Error processing embedded PDF ${pdfName}:`, error);
           content = content.replace(
             match[0],
             `\n\n<${EMBEDDED_PDF_TAG}>\n<name>${pdfName}</name>\n<error>Could not process PDF</error>\n</${EMBEDDED_PDF_TAG}>\n\n`
@@ -94,7 +107,7 @@ export class ContextProcessor {
         const replacement = `\n\n<${DATAVIEW_BLOCK_TAG}>\n<query_type>${queryType}</query_type>\n<original_query>\n${query}\n</original_query>\n<executed_result>\n${result}\n</executed_result>\n</${DATAVIEW_BLOCK_TAG}>\n\n`;
         content = content.slice(0, matchStart) + replacement + content.slice(matchEnd);
       } catch (error) {
-        console.error(`Error executing Dataview query:`, error);
+        logError(`Error executing Dataview query:`, error);
         // On error, include query with error message
         const replacement = `\n\n<${DATAVIEW_BLOCK_TAG}>\n<query_type>${queryType}</query_type>\n<original_query>\n${query}\n</original_query>\n<error>${error instanceof Error ? error.message : "Query execution failed"}</error>\n</${DATAVIEW_BLOCK_TAG}>\n\n`;
         content = content.slice(0, matchStart) + replacement + content.slice(matchEnd);
@@ -216,6 +229,281 @@ export class ContextProcessor {
     return String(value);
   }
 
+  /**
+   * Build markdown content for context inclusion by resolving embeds, PDFs, and Dataview blocks.
+   */
+  private async buildMarkdownContextContent(
+    note: TFile,
+    vault: Vault,
+    fileParserManager: FileParserManager,
+    chainType: ChainType
+  ): Promise<string> {
+    let content = await fileParserManager.parseFile(note, vault);
+
+    content = await this.processEmbeddedNotes(content, note, vault, fileParserManager, chainType);
+
+    if (isPlusChain(chainType)) {
+      content = await this.processEmbeddedPDFs(content, vault, fileParserManager);
+    }
+
+    return await this.processDataviewBlocks(content, note.path);
+  }
+
+  /**
+   * Replace embedded note syntax within markdown content.
+   *
+   * Scans the content for all `![[...]]` embed patterns and expands them once
+   * into structured `<embedded_note>` blocks. Nested embeds are left as-is to
+   * keep processing predictable and lightweight.
+   *
+   * @param content - The markdown content to process
+   * @param sourceNote - The note containing this content (for relative link resolution)
+   * @param vault - Obsidian vault instance
+   * @param fileParserManager - Manager for parsing different file types
+   * @param chainType - Current chain type (affects feature availability)
+   * @returns Content with top-level embeds replaced by structured blocks
+   */
+  private async processEmbeddedNotes(
+    content: string,
+    sourceNote: TFile,
+    vault: Vault,
+    fileParserManager: FileParserManager,
+    chainType: ChainType
+  ): Promise<string> {
+    const embedRegex = /!\[\[([^\]]+)\]\]/g;
+    let match: RegExpExecArray | null;
+    let lastIndex = 0;
+    let result = "";
+
+    while ((match = embedRegex.exec(content)) !== null) {
+      result += content.slice(lastIndex, match.index);
+      const rawTarget = match[1].trim();
+      const replacement = await this.buildEmbeddedNoteBlock(
+        rawTarget,
+        match[0],
+        sourceNote,
+        vault,
+        fileParserManager,
+        chainType
+      );
+      result += replacement;
+      lastIndex = match.index + match[0].length;
+    }
+
+    result += content.slice(lastIndex);
+    return result;
+  }
+
+  /**
+   * Build a rendered embedded note block for the given target.
+   */
+  private async buildEmbeddedNoteBlock(
+    rawTarget: string,
+    rawMatch: string,
+    sourceNote: TFile,
+    vault: Vault,
+    fileParserManager: FileParserManager,
+    chainType: ChainType
+  ): Promise<string> {
+    const target = this.parseEmbeddedLinkTarget(rawTarget);
+    if (!target) {
+      return rawMatch;
+    }
+
+    const resolvedFile =
+      target.path === null
+        ? sourceNote
+        : app.metadataCache.getFirstLinkpathDest(target.path, sourceNote.path);
+
+    if (!(resolvedFile instanceof TFile)) {
+      return this.formatEmbeddedNoteBlock({
+        title: target.path ?? sourceNote.basename,
+        path: target.path ?? sourceNote.path,
+        heading: target.heading,
+        blockId: target.blockId,
+        error: "Embedded note not found",
+      });
+    }
+
+    if (resolvedFile.extension !== "md") {
+      return rawMatch;
+    }
+
+    try {
+      let embeddedContent = await fileParserManager.parseFile(resolvedFile, vault);
+
+      if (target.heading || target.blockId) {
+        const segment = this.extractMarkdownSegment(resolvedFile, embeddedContent, target);
+        if (!segment.found) {
+          const targetDescription = target.blockId
+            ? `block reference "${target.blockId}"`
+            : `heading "${target.heading ?? ""}"`;
+          throw new Error(`Embedded note ${targetDescription} not found in ${resolvedFile.path}`);
+        }
+        embeddedContent = segment.content;
+      }
+
+      if (isPlusChain(chainType)) {
+        embeddedContent = await this.processEmbeddedPDFs(embeddedContent, vault, fileParserManager);
+      }
+
+      embeddedContent = await this.processDataviewBlocks(embeddedContent, resolvedFile.path);
+
+      return this.formatEmbeddedNoteBlock({
+        title: resolvedFile.basename,
+        path: resolvedFile.path,
+        heading: target.heading,
+        blockId: target.blockId,
+        content: embeddedContent,
+      });
+    } catch (error) {
+      logWarn("Failed to process embedded note", error);
+      const message = error instanceof Error ? error.message : "Could not process embedded note";
+      return this.formatEmbeddedNoteBlock({
+        title: resolvedFile.basename,
+        path: resolvedFile.path,
+        heading: target.heading,
+        blockId: target.blockId,
+        error: message,
+      });
+    }
+  }
+
+  /**
+   * Parse embedded note syntax into a structured target.
+   */
+  private parseEmbeddedLinkTarget(rawTarget: string): EmbeddedLinkTarget | null {
+    if (!rawTarget) {
+      return null;
+    }
+
+    const aliasIndex = rawTarget.indexOf("|");
+    const linkTarget = aliasIndex >= 0 ? rawTarget.slice(0, aliasIndex) : rawTarget;
+    let cleanedTarget = linkTarget.trim();
+
+    if (!cleanedTarget) {
+      return { path: null };
+    }
+
+    let blockId: string | undefined;
+    let heading: string | undefined;
+
+    const blockIndex = cleanedTarget.indexOf("#^");
+    if (blockIndex !== -1) {
+      blockId = cleanedTarget.slice(blockIndex + 2).trim();
+      cleanedTarget = cleanedTarget.slice(0, blockIndex);
+    }
+
+    const headingIndex = cleanedTarget.indexOf("#");
+    if (headingIndex !== -1) {
+      heading = cleanedTarget.slice(headingIndex + 1).trim();
+      cleanedTarget = cleanedTarget.slice(0, headingIndex);
+    }
+
+    const path = cleanedTarget.length > 0 ? cleanedTarget : null;
+
+    return {
+      path,
+      heading: heading && heading.length > 0 ? heading : undefined,
+      blockId: blockId && blockId.length > 0 ? blockId : undefined,
+    };
+  }
+
+  /**
+   * Extract a markdown segment representing a heading section or block reference.
+   */
+  private extractMarkdownSegment(
+    note: TFile,
+    fileContent: string,
+    focus: EmbeddedLinkTarget
+  ): MarkdownSegment {
+    const cache = app.metadataCache.getFileCache(note);
+
+    if (focus.blockId) {
+      const block = cache?.blocks?.[focus.blockId];
+      const startOffset = block?.position?.start?.offset;
+      const endOffset = block?.position?.end?.offset;
+
+      if (startOffset === undefined || endOffset === undefined) {
+        return { content: "", found: false };
+      }
+
+      return {
+        content: fileContent.slice(startOffset, endOffset),
+        found: true,
+      };
+    }
+
+    if (focus.heading) {
+      const headings = cache?.headings ?? [];
+      const normalizedTarget = this.normalizeHeadingForMatch(focus.heading);
+      const targetIndex = headings.findIndex(
+        (headingCache) => this.normalizeHeadingForMatch(headingCache.heading) === normalizedTarget
+      );
+
+      if (targetIndex === -1) {
+        return { content: "", found: false };
+      }
+
+      const currentHeading = headings[targetIndex];
+      const startOffset = currentHeading.position?.start?.offset ?? 0;
+      let endOffset = fileContent.length;
+
+      for (let i = targetIndex + 1; i < headings.length; i++) {
+        if (headings[i].level <= currentHeading.level) {
+          endOffset = headings[i].position?.start?.offset ?? endOffset;
+          break;
+        }
+      }
+
+      return {
+        content: fileContent.slice(startOffset, endOffset),
+        found: true,
+      };
+    }
+
+    return { content: fileContent, found: true };
+  }
+
+  /**
+   * Normalize heading text for comparison.
+   */
+  private normalizeHeadingForMatch(heading: string): string {
+    return heading.trim().toLowerCase().replace(/\s+/g, " ");
+  }
+
+  /**
+   * Format an embedded note payload using the shared XML-like structure.
+   */
+  private formatEmbeddedNoteBlock(params: {
+    title: string;
+    path: string;
+    heading?: string;
+    blockId?: string;
+    content?: string;
+    error?: string;
+  }): string {
+    const { title, path, heading, blockId, content, error } = params;
+    let block = `\n\n<${EMBEDDED_NOTE_TAG}>\n<title>${title}</title>\n<path>${path}</path>`;
+
+    if (heading) {
+      block += `\n<heading>${heading}</heading>`;
+    }
+
+    if (blockId) {
+      block += `\n<block_id>${blockId}</block_id>`;
+    }
+
+    if (error) {
+      block += `\n<error>${error}</error>`;
+    } else {
+      block += `\n<content>\n${content ?? ""}\n</content>`;
+    }
+
+    block += `\n</${EMBEDDED_NOTE_TAG}>\n\n`;
+    return block;
+  }
+
   /**
    * Processes context notes, excluding any already handled by custom prompts.
    *
@@ -247,44 +535,34 @@ export class ContextProcessor {
       try {
         // Check if this note was already processed (via custom prompt)
         if (excludedNotePaths.has(note.path)) {
-          console.log(`Skipping note ${note.path} as it was included via custom prompt.`);
+          logInfo(`Skipping note ${note.path} as it was included via custom prompt.`);
           return;
         }
 
-        console.log(
+        logInfo(
           `Processing note: ${note.path}, extension: ${note.extension}, chain: ${currentChain}`
         );
 
         // 1. Check if the file extension is supported by any parser
         if (!fileParserManager.supportsExtension(note.extension)) {
-          console.warn(`Unsupported file type: ${note.extension}`);
+          logWarn(`Unsupported file type: ${note.extension}`);
           return;
         }
 
         // 2. Apply chain restrictions only to supported files that are NOT md or canvas
         if (!isPlusChain(currentChain) && note.extension !== "md" && note.extension !== "canvas") {
           // This file type is supported, but requires Plus mode (e.g., PDF)
-          console.warn(
-            `File type ${note.extension} requires Copilot Plus mode for context processing.`
-          );
+          logWarn(`File type ${note.extension} requires Copilot Plus mode for context processing.`);
           // Show user-facing notice about the restriction
           new Notice(RESTRICTION_MESSAGES.NON_MARKDOWN_FILES_RESTRICTED);
           return;
         }
 
         // 3. If we reach here, parse the file (md, canvas, or other supported type in Plus mode)
-        let content = await fileParserManager.parseFile(note, vault);
-
-        // Special handling for markdown files
-        if (note.extension === "md") {
-          // Process embedded PDFs (only in Plus mode)
-          if (isPlusChain(currentChain)) {
-            content = await this.processEmbeddedPDFs(content, vault, fileParserManager);
-          }
-
-          // Process Dataview blocks (all modes)
-          content = await this.processDataviewBlocks(content, note.path);
-        }
+        const content =
+          note.extension === "md"
+            ? await this.buildMarkdownContextContent(note, vault, fileParserManager, currentChain)
+            : await fileParserManager.parseFile(note, vault);
 
         // Get file metadata
         const stats = await vault.adapter.stat(note.path);
@@ -293,7 +571,7 @@ export class ContextProcessor {
 
         additionalContext += `\n\n<${prompt_tag}>\n<title>${note.basename}</title>\n<path>${note.path}</path>\n<ctime>${ctime}</ctime>\n<mtime>${mtime}</mtime>\n<content>\n${content}\n</content>\n</${prompt_tag}>`;
       } catch (error) {
-        console.error(`Error processing file ${note.path}:`, error);
+        logError(`Error processing file ${note.path}:`, error);
         additionalContext += `\n\n<${prompt_tag}_error>\n<title>${note.basename}</title>\n<path>${note.path}</path>\n<error>[Error: Could not process file]</error>\n</${prompt_tag}_error>`;
       }
     };