+ {/* Enable Semantic Search (v3) */}
+ updateSetting("enableSemanticSearchV3", checked)}
+ />
+
{/* Auto-Index Strategy */}
{
onChange={(value) => updateSetting("maxSourceChunks", value)}
/>
+ {/* Graph Hops */}
+ updateSetting("graphHops", value)}
+ />
+
{/* Requests per Minute */}
{
onChange={(value) => updateSetting("embeddingBatchSize", value)}
/>
- {/* Number of Partitions */}
- ({
- label: it,
- value: it,
- }))}
- />
+ {/* Number of Partitions removed (auto-managed in v3) */}
{/* Exclusions */}
{
updateSetting("enableIndexSync", checked)}
/>
diff --git a/src/tools/SearchTools.ts b/src/tools/SearchTools.ts
index 6bb24e59..6146122b 100644
--- a/src/tools/SearchTools.ts
+++ b/src/tools/SearchTools.ts
@@ -1,13 +1,8 @@
import { getStandaloneQuestion } from "@/chainUtils";
-import {
- EMPTY_INDEX_ERROR_MESSAGE,
- PLUS_MODE_DEFAULT_SOURCE_CHUNKS,
- TEXT_WEIGHT,
-} from "@/constants";
-import { CustomError } from "@/error";
+import { PLUS_MODE_DEFAULT_SOURCE_CHUNKS, TEXT_WEIGHT } from "@/constants";
import { BrevilabsClient } from "@/LLMProviders/brevilabsClient";
-import { HybridRetriever } from "@/search/hybridRetriever";
-import VectorStoreManager from "@/search/vectorStoreManager";
+import { logInfo } from "@/logger";
+import { TieredLexicalRetriever } from "@/search/v3/TieredLexicalRetriever";
import { getSettings } from "@/settings/model";
import { z } from "zod";
import { createTool, SimpleTool } from "./SimpleTool";
@@ -30,24 +25,22 @@ const localSearchTool = createTool({
description: "Search for notes based on the time range and query",
schema: localSearchSchema,
handler: async ({ timeRange, query, salientTerms }) => {
- const indexEmpty = await VectorStoreManager.getInstance().isIndexEmpty();
- if (indexEmpty) {
- throw new CustomError(EMPTY_INDEX_ERROR_MESSAGE);
- }
+ const settings = getSettings();
const returnAll = timeRange !== undefined;
- const maxSourceChunks =
- getSettings().maxSourceChunks < PLUS_MODE_DEFAULT_SOURCE_CHUNKS
+ const baseMax =
+ settings.maxSourceChunks < PLUS_MODE_DEFAULT_SOURCE_CHUNKS
? PLUS_MODE_DEFAULT_SOURCE_CHUNKS
- : getSettings().maxSourceChunks;
+ : settings.maxSourceChunks;
+ // For time-based queries, ensure a healthy cap to avoid starving recall when users set a very low max
+ const effectiveMaxK = returnAll ? Math.max(baseMax, 200) : baseMax;
- if (getSettings().debug) {
- console.log("returnAll:", returnAll);
- }
+ logInfo(`returnAll: ${returnAll}`);
- const hybridRetriever = new HybridRetriever({
+ // Use tiered lexical retriever for multi-stage search
+ const retriever = new TieredLexicalRetriever(app, {
minSimilarityScore: returnAll ? 0.0 : 0.1,
- maxK: returnAll ? 1000 : maxSourceChunks,
+ maxK: effectiveMaxK,
salientTerms,
timeRange: timeRange
? {
@@ -57,53 +50,56 @@ const localSearchTool = createTool({
: undefined,
textWeight: TEXT_WEIGHT,
returnAll: returnAll,
- // Voyage AI reranker did worse than Orama in some cases, so only use it if
- // Orama did not return anything higher than this threshold
useRerankerThreshold: 0.5,
});
// Perform the search
- const documents = await hybridRetriever.getRelevantDocuments(query);
+ const documents = await retriever.getRelevantDocuments(query);
- // Format the results
- const formattedResults = documents.map((doc) => ({
- title: doc.metadata.title,
- content: doc.pageContent,
- path: doc.metadata.path,
- score: doc.metadata.score,
- rerank_score: doc.metadata.rerank_score,
- includeInContext: doc.metadata.includeInContext,
- }));
+ logInfo(`localSearch found ${documents.length} documents for query: "${query}"`);
+ if (timeRange) {
+ logInfo(
+ `Time range search from ${new Date(timeRange.startTime.epoch).toISOString()} to ${new Date(timeRange.endTime.epoch).toISOString()}`
+ );
+ }
+
+ // Format the results - only include snippet, not full content
+ const formattedResults = documents.map((doc) => {
+ const scored = doc.metadata.rerank_score ?? doc.metadata.score ?? 0;
+ return {
+ title: doc.metadata.title || "Untitled",
+ // Only include a snippet for display (first 200 chars)
+ content: doc.pageContent.substring(0, 200),
+ path: doc.metadata.path || "",
+ // Ensure both fields reflect the same final fused score when present
+ score: scored,
+ rerank_score: scored,
+ includeInContext: doc.metadata.includeInContext ?? true,
+ source: doc.metadata.source, // Pass through source for proper labeling
+ // Show actual modified time for time-based queries
+ mtime: doc.metadata.mtime ?? null,
+ };
+ });
return JSON.stringify(formattedResults);
},
});
+// Note: indexTool is kept for backward compatibility but is no longer used with v3 search
const indexTool = createTool({
name: "indexVault",
description: "Index the vault to the Copilot index",
schema: z.void(), // No parameters
handler: async () => {
- try {
- const indexedCount = await VectorStoreManager.getInstance().indexVaultToVectorStore();
- const indexResultPrompt = `Please report whether the indexing was successful.\nIf success is true, just say it is successful. If 0 files is indexed, say there are no new files to index.`;
- return (
- indexResultPrompt +
- JSON.stringify({
- success: true,
- message:
- indexedCount === 0
- ? "No new files to index."
- : `Indexed ${indexedCount} files in the vault.`,
- })
- );
- } catch (error) {
- console.error("Error indexing vault:", error);
- return JSON.stringify({
- success: false,
- message: "An error occurred while indexing the vault.",
- });
- }
+ // Tiered lexical retriever doesn't require manual indexing - it builds ephemeral indexes on demand
+ const indexResultPrompt = `The tiered lexical retriever builds indexes on demand and doesn't require manual indexing.\n`;
+ return (
+ indexResultPrompt +
+ JSON.stringify({
+ success: true,
+ message: "Tiered lexical retriever uses on-demand indexing. No manual indexing required.",
+ })
+ );
},
isBackground: true,
});
diff --git a/src/tools/ToolResultFormatter.ts b/src/tools/ToolResultFormatter.ts
index 76d92a73..0d67dc34 100644
--- a/src/tools/ToolResultFormatter.ts
+++ b/src/tools/ToolResultFormatter.ts
@@ -1,3 +1,5 @@
+import { logWarn } from "@/logger";
+
/**
* Format tool results for display in the UI
* Each formatter should return a user-friendly representation of the tool result
@@ -18,13 +20,24 @@ export class ToolResultFormatter {
}
static format(toolName: string, result: string): string {
try {
- // Try to parse the result as JSON first
+ // Decode tool marker encoding if present (ENC:...)
+ let normalized = result;
+ if (typeof normalized === "string" && normalized.startsWith("ENC:")) {
+ try {
+ normalized = decodeURIComponent(normalized.slice(4));
+ } catch {
+ // fall back to original
+ }
+ }
+
+ // Try to parse the (potentially decoded) result as JSON first
let parsedResult: any;
try {
- parsedResult = JSON.parse(result);
- } catch {
+ parsedResult = JSON.parse(normalized);
+ } catch (e) {
// If not JSON, use the raw string
- parsedResult = result;
+ logWarn(`ToolResultFormatter: Failed to parse JSON for ${toolName}:`, e);
+ parsedResult = normalized;
}
// Route to specific formatter based on tool name
@@ -51,146 +64,89 @@ export class ToolResultFormatter {
}
private static formatLocalSearch(result: any): string {
- // If already formatted or not a string, return as is
- if (typeof result !== "string") {
- return typeof result === "object" ? JSON.stringify(result, null, 2) : String(result);
+ const searchResults = this.parseSearchResults(result);
+
+ if (!Array.isArray(searchResults)) {
+ return typeof result === "string" ? result : JSON.stringify(result, null, 2);
}
- // Check if it looks like JSON (array or object)
- const trimmedResult = result.trim();
- if (!trimmedResult.startsWith("[") && !trimmedResult.startsWith("{")) {
- return result;
+ const count = searchResults.length;
+ if (count === 0) {
+ return "ð Found 0 relevant notes\n\nNo matching notes found.";
}
- // Try standard JSON parsing first
- let searchResults = this.tryParseJson(result);
+ const topResults = searchResults.slice(0, 10);
+ const formattedItems = topResults
+ .map((item, index) => this.formatSearchItem(item, index))
+ .join("\n\n");
- // If parsing failed, try regex extraction for malformed JSON
- try {
- if (searchResults.length === 0) {
- // Match individual JSON objects (works for arrays or malformed JSON)
- const objectRegex = /\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}/g;
- const matches = result.match(objectRegex);
+ const footer = count > 10 ? `\n\n... and ${count - 10} more results` : "";
- if (matches) {
- const regexResults: any[] = [];
- for (const match of matches) {
- try {
- // Parse each object individually
- const obj = JSON.parse(match);
- regexResults.push(obj);
- } catch {
- // Skip malformed objects
- }
- }
- searchResults = regexResults;
+ return `ð Found ${count} relevant notes\n\nTop results:\n\n${formattedItems}${footer}`;
+ }
+
+ private static parseSearchResults(result: any): any[] {
+ if (Array.isArray(result)) return result;
+ if (typeof result === "object" && result !== null) return [result];
+ if (typeof result === "string") {
+ const trimmed = result.trim();
+ if (!trimmed.startsWith("[") && !trimmed.startsWith("{")) {
+ return [];
+ }
+ return this.tryParseJson(result);
+ }
+ return [];
+ }
+
+ private static formatSearchItem(item: any, index: number): string {
+ const filename = item.path?.split("/").pop()?.replace(/\.md$/, "") || item.title || "Untitled";
+ const score = item.rerank_score || item.score || 0;
+ const scoreDisplay = typeof score === "number" ? score.toFixed(4) : score;
+
+ // For time-filtered results, show as "Recency" instead of "Relevance"
+ const scoreLabel = item.source === "time-filtered" ? "Recency" : "Relevance";
+
+ const lines = [`${index + 1}. ${filename}`];
+
+ // For time-filtered queries, show actual modified time instead of a recency score
+ if (item.source === "time-filtered") {
+ if (item.mtime) {
+ try {
+ const d = new Date(item.mtime);
+ const iso = isNaN(d.getTime()) ? String(item.mtime) : d.toISOString();
+ lines.push(` ð Modified: ${iso}${item.includeInContext ? " â" : ""}`);
+ } catch {
+ lines.push(` ð Modified: ${String(item.mtime)}${item.includeInContext ? " â" : ""}`);
}
}
-
- if (searchResults.length === 0) {
- // Fallback: try to extract key information using regex
- const titleRegex = /"title"\s*:\s*"([^"]+)"/g;
- const pathRegex = /"path"\s*:\s*"([^"]+)"/g;
- const scoreRegex = /"score"\s*:\s*([\d.]+)/g;
-
- let titleMatch;
- const results = [];
- while ((titleMatch = titleRegex.exec(result))) {
- const title = titleMatch[1];
- pathRegex.lastIndex = titleMatch.index;
- const pathMatch = pathRegex.exec(result);
- scoreRegex.lastIndex = titleMatch.index;
- const scoreMatch = scoreRegex.exec(result);
-
- results.push({
- title: title,
- path: pathMatch ? pathMatch[1] : "",
- score: scoreMatch ? parseFloat(scoreMatch[1]) : 0,
- content: "", // Content is too complex to extract reliably
- });
- }
-
- if (results.length > 0) {
- searchResults = results;
- }
- }
- } catch {
- // If extraction fails, return original
- return result;
+ } else if (item.source === "title-match") {
+ // For title matches, avoid misleading numeric scores; mark as a title match
+ lines.push(` ð Title match${item.includeInContext ? " â" : ""}`);
+ } else {
+ // Default: show relevance-like score line
+ lines.push(` ð ${scoreLabel}: ${scoreDisplay}${item.includeInContext ? " â" : ""}`);
}
- // Check if it's an array of search results
- if (Array.isArray(searchResults)) {
- const output: string[] = [`ð Found ${searchResults.length} relevant notes`];
-
- if (searchResults.length === 0) {
- output.push("\nNo matching notes found.");
- return output.join("");
- }
-
- output.push("");
- output.push("Top results:");
- output.push("");
-
- // Show top 10 results
- searchResults.slice(0, 10).forEach((item, index) => {
- const filename =
- item.path?.split("/").pop()?.replace(/\.md$/, "") || item.title || "Untitled";
- const score = item.rerank_score || item.score || 0;
- const scoreDisplay = typeof score === "number" ? score.toFixed(3) : score;
-
- output.push(`${index + 1}. ${filename}`);
- output.push(` ð Relevance: ${scoreDisplay}${item.includeInContext ? " â" : ""}`);
-
- // Show a snippet of content if available
- if (item.content) {
- // Extract actual content from the formatted string
- let cleanContent = item.content;
-
- // Remove NOTE TITLE section
- cleanContent = cleanContent.replace(
- /NOTE TITLE:[\s\S]*?(?=METADATA:|NOTE BLOCK CONTENT:)/,
- ""
- );
-
- // Remove METADATA section
- cleanContent = cleanContent.replace(/METADATA:\{[\s\S]*?\}/, "");
-
- // Extract content after NOTE BLOCK CONTENT:
- const contentMatch = cleanContent.match(/NOTE BLOCK CONTENT:\s*([\s\S]*)/);
- if (contentMatch) {
- cleanContent = contentMatch[1];
- }
-
- // Clean up the content
- const snippet = cleanContent
- .substring(0, 150)
- .replace(/\n+/g, " ")
- .replace(/\s+/g, " ")
- .trim();
-
- if (snippet) {
- output.push(` ðŽ "${snippet}${cleanContent.length > 150 ? "..." : ""}"`);
- }
- }
-
- // Show path if different from filename
- if (item.path && !item.path.endsWith(`/${filename}.md`)) {
- output.push(` ð ${item.path}`);
- }
-
- output.push("");
- });
-
- if (searchResults.length > 10) {
- output.push(`... and ${searchResults.length - 10} more results`);
- }
-
- return output.join("\n");
+ const snippet = this.extractContentSnippet(item.content);
+ if (snippet) {
+ lines.push(` ðŽ "${snippet}${item.content?.length > 150 ? "..." : ""}"`);
}
- // If not an array, return original
- return typeof result === "string" ? result : JSON.stringify(result, null, 2);
+ if (item.path && !item.path.endsWith(`/${filename}.md`)) {
+ lines.push(` ð ${item.path}`);
+ }
+
+ return lines.join("\n");
+ }
+
+ private static extractContentSnippet(content: string, maxLength = 150): string {
+ if (!content) return "";
+
+ // Try to extract content after NOTE BLOCK CONTENT: pattern
+ const contentMatch = content.match(/NOTE BLOCK CONTENT:\s*([\s\S]*)/);
+ const cleanContent = contentMatch?.[1] || content;
+
+ return cleanContent.substring(0, maxLength).replace(/\s+/g, " ").trim();
}
private static formatWebSearch(result: any): string {