From d58e9ef1ff19b73c812ee8e7ebd097d12531eb94 Mon Sep 17 00:00:00 2001 From: Andrew Beal Date: Tue, 14 Oct 2025 19:50:45 +0100 Subject: [PATCH] refactor: enhance vault search to return content snippets with context - Update SearchVaultFiles function to return contextual snippets instead of just file lists - Implement snippet extraction with configurable context length (300 chars) - Add snippet merging logic to consolidate overlapping matches - Limit results to 20 randomly sampled matches when exceeding threshold - Update return types across service layer to use SearchMatch/SearchSnippet - Change regex flags to 'ig' for global case-insensitive matching --- .../Functions/SearchVaultFiles.ts | 10 +- Helpers/SearchTypes.ts | 18 +++ Services/AIFunctionService.ts | 14 +- Services/FileSystemService.ts | 3 +- Services/VaultService.ts | 149 +++++++++++++++++- 5 files changed, 176 insertions(+), 18 deletions(-) create mode 100644 Helpers/SearchTypes.ts diff --git a/AIClasses/FunctionDefinitions/Functions/SearchVaultFiles.ts b/AIClasses/FunctionDefinitions/Functions/SearchVaultFiles.ts index a1e01e7..a0fe266 100644 --- a/AIClasses/FunctionDefinitions/Functions/SearchVaultFiles.ts +++ b/AIClasses/FunctionDefinitions/Functions/SearchVaultFiles.ts @@ -4,11 +4,11 @@ import type { IAIFunctionDefinition } from "../IAIFunctionDefinition"; export const SearchVaultFiles: IAIFunctionDefinition = { name: AIFunction.SearchVaultFiles, description: `Searches through all files in the user's Obsidian vault for the given search term. - Uses regex pattern matching to search both file names and file content. - Returns list of matching vault files with metadata (names, paths) and contextual snippets showing matched content with surrounding text. - Call this whenever you need to know what files exist in the vault to answer questions, - verify file presence, or to perform further agentic functions. - Use proactively when vault contents would inform your response.`, + Uses regex pattern matching to search file content. + Returns matching vault files with metadata (names, paths) and contextual snippets showing matched content with surrounding text to enable relevance assessment. + Call this whenever you need to know what files exist in the vault to answer questions, + verify file presence, or to perform further agentic functions. + Use proactively when vault contents would inform your response.`, parameters: { type: "object", properties: { diff --git a/Helpers/SearchTypes.ts b/Helpers/SearchTypes.ts new file mode 100644 index 0000000..8f288ac --- /dev/null +++ b/Helpers/SearchTypes.ts @@ -0,0 +1,18 @@ +import { TFile } from "obsidian"; + +/** + * Represents a single snippet of matched content from a file + */ +export interface SearchSnippet { + text: string; + matchIndex: number; + matchLength: number; +} + +/** + * Represents all matches found in a single file + */ +export interface SearchMatch { + file: TFile; + snippets: SearchSnippet[]; +} diff --git a/Services/AIFunctionService.ts b/Services/AIFunctionService.ts index 413f4be..c951255 100644 --- a/Services/AIFunctionService.ts +++ b/Services/AIFunctionService.ts @@ -1,10 +1,10 @@ import { Resolve } from "./DependencyService"; import { Services } from "./Services"; import type { FileSystemService } from "./FileSystemService"; -import { TFile } from "obsidian"; import { AIFunction } from "Enums/AIFunction"; import { AIFunctionResponse } from "AIClasses/FunctionDefinitions/AIFunctionResponse"; import type { AIFunctionCall } from "AIClasses/AIFunctionCall"; +import type { SearchMatch } from "../Helpers/SearchTypes"; export class AIFunctionService { @@ -33,10 +33,14 @@ export class AIFunctionService { } private async searchVaultFiles(searchTerm: string): Promise { - const files: TFile[] = await this.fileSystemService.searchVaultFiles(searchTerm); - return files.map((file) => ({ - name: file.basename, - path: file.path + const matches: SearchMatch[] = await this.fileSystemService.searchVaultFiles(searchTerm); + return matches.map((match) => ({ + name: match.file.basename, + path: match.file.path, + snippets: match.snippets.map((snippet) => ({ + text: snippet.text, + matchPosition: snippet.matchIndex + })) })); } diff --git a/Services/FileSystemService.ts b/Services/FileSystemService.ts index 5601322..7d4e0ab 100644 --- a/Services/FileSystemService.ts +++ b/Services/FileSystemService.ts @@ -5,6 +5,7 @@ import { Services } from "./Services"; import { isValidJson } from "Helpers/Helpers"; import { Path } from "Enums/Path"; import type { VaultService } from "./VaultService"; +import type { SearchMatch } from "../Helpers/SearchTypes"; export class FileSystemService { @@ -96,7 +97,7 @@ export class FileSystemService { } } - public async searchVaultFiles(searchTerm: string): Promise { + public async searchVaultFiles(searchTerm: string): Promise { return await this.vaultService.searchVaultFiles(searchTerm); } diff --git a/Services/VaultService.ts b/Services/VaultService.ts index 07bfb47..3d0f63d 100644 --- a/Services/VaultService.ts +++ b/Services/VaultService.ts @@ -4,6 +4,7 @@ import { Services } from "./Services"; import type AIAgentPlugin from "main"; import { Path } from "Enums/Path"; import { escapeRegex } from "Helpers/Helpers"; +import type { SearchMatch, SearchSnippet } from "../Helpers/SearchTypes"; /* This service protects the users vault through their exclusions. The plugin root is excluded by default */ export class VaultService { @@ -91,25 +92,159 @@ export class VaultService { return files.filter(file => !this.isExclusion(file.path, allowAccessToPluginRoot)); } - public async searchVaultFiles(searchTerm: string): Promise { + public async searchVaultFiles(searchTerm: string): Promise { let regex: RegExp; try { - regex = new RegExp(searchTerm, "i"); + regex = new RegExp(searchTerm, "ig"); // Added 'g' flag for global matching } catch { - regex = new RegExp(escapeRegex(searchTerm), "i"); + regex = new RegExp(escapeRegex(searchTerm), "ig"); } const files: TFile[] = this.vault.getFiles().filter(file => !this.isExclusion(file.path)); - const matches: TFile[] = []; + // Collect all matches from all files + const allMatches: SearchMatch[] = []; + for (const file of files) { const content = await this.vault.cachedRead(file); - if (regex.test(content)) { - matches.push(file); + const snippets = this.extractSnippets(content, regex); + + if (snippets.length > 0) { + allMatches.push({ file, snippets }); } } - return matches; + // Flatten matches for random sampling + const flatMatches: { file: TFile; snippet: SearchSnippet }[] = []; + for (const match of allMatches) { + for (const snippet of match.snippets) { + flatMatches.push({ file: match.file, snippet }); + } + } + + // If more than 20 matches, randomly sample 20 + let selectedMatches: { file: TFile; snippet: SearchSnippet }[]; + if (flatMatches.length > 20) { + selectedMatches = this.randomSample(flatMatches, 20); + } else { + selectedMatches = flatMatches; + } + + // Regroup by file + const resultMap = new Map(); + for (const match of selectedMatches) { + const existing = resultMap.get(match.file); + if (existing) { + existing.push(match.snippet); + } else { + resultMap.set(match.file, [match.snippet]); + } + } + + // Convert map to array of SearchMatch objects + const results: SearchMatch[] = []; + for (const [file, snippets] of resultMap.entries()) { + results.push({ file, snippets }); + } + + return results; + } + + /** + * Extracts snippets from content based on regex matches. + * Merges overlapping snippets to avoid duplication. + */ + private extractSnippets(content: string, regex: RegExp): SearchSnippet[] { + const snippets: SearchSnippet[] = []; + const maxContextLength = 300; // Characters before and after the match + + let match: RegExpExecArray | null; + + // Find all matches + while ((match = regex.exec(content)) !== null) { + const matchIndex = match.index; + const matchLength = match[0].length; + + // Calculate snippet boundaries + const snippetStart = Math.max(0, matchIndex - maxContextLength); + const snippetEnd = Math.min(content.length, matchIndex + matchLength + maxContextLength); + + snippets.push({ + text: content.substring(snippetStart, snippetEnd), + matchIndex, + matchLength + }); + } + + // Reset regex lastIndex + regex.lastIndex = 0; + + // Merge overlapping snippets + return this.mergeOverlappingSnippets(snippets, content); + } + + /** + * Merges snippets that overlap in the original content + */ + private mergeOverlappingSnippets(snippets: SearchSnippet[], content: string): SearchSnippet[] { + if (snippets.length === 0) return snippets; + + // Sort snippets by match index + snippets.sort((a, b) => a.matchIndex - b.matchIndex); + + const merged: SearchSnippet[] = []; + let current = snippets[0]; + const maxContextLength = 300; + + for (let i = 1; i < snippets.length; i++) { + const next = snippets[i]; + + // Calculate the actual boundaries of current and next snippets in the original content + const currentStart = Math.max(0, current.matchIndex - maxContextLength); + const currentEnd = Math.min(content.length, current.matchIndex + current.matchLength + maxContextLength); + const nextStart = Math.max(0, next.matchIndex - maxContextLength); + const nextEnd = Math.min(content.length, next.matchIndex + next.matchLength + maxContextLength); + + // Check if snippets overlap + if (nextStart <= currentEnd) { + // Merge: extend current to include next + const mergedStart = Math.min(currentStart, nextStart); + const mergedEnd = Math.max(currentEnd, nextEnd); + + current = { + text: content.substring(mergedStart, mergedEnd), + matchIndex: current.matchIndex, // Keep the first match index + matchLength: next.matchIndex + next.matchLength - current.matchIndex // Total span + }; + } else { + // No overlap, save current and move to next + merged.push(current); + current = next; + } + } + + // Don't forget the last snippet + merged.push(current); + + return merged; + } + + /** + * Randomly samples n items from an array + */ + private randomSample(array: T[], n: number): T[] { + const result: T[] = []; + const taken = new Set(); + + while (result.length < n && result.length < array.length) { + const index = Math.floor(Math.random() * array.length); + if (!taken.has(index)) { + taken.add(index); + result.push(array[index]); + } + } + + return result; } private isExclusion(filePath: string, allowAccessToPluginRoot: boolean = false): boolean {