mirror of
https://github.com/logancyang/obsidian-copilot.git
synced 2026-07-22 07:50:24 +00:00
Fix search v3 ranking (#2101)
* Replace FlexSearch with MiniSearch for improved search functionality and scoring * Enhance QueryExpander and README for improved recall and ranking clarity - Update QueryExpander to clearly separate salient and expanded terms for recall and ranking. - Modify README to reflect changes in the search system's architecture and clarify the distinction between recall and ranking processes. - Adjust FullTextEngine comments to emphasize BM25's native multi-term scoring capabilities. * Implement weighted query expansion in FullTextEngine with 90/10 scoring for salient and expanded terms
This commit is contained in:
Logan Yang
GitHub
parent
13523c9fc1
commit
ccbe039a86
9 changed files with 549 additions and 585 deletions
40
package-lock.json
generated
40
package-lock.json
generated
|
|
@ -59,7 +59,6 @@
|
|||
"diff": "^7.0.0",
|
||||
"esbuild-plugin-svg": "^0.1.0",
|
||||
"eventsource-parser": "^1.0.0",
|
||||
"flexsearch": "^0.8.205",
|
||||
"fuzzysort": "^3.1.0",
|
||||
"jotai": "^2.10.3",
|
||||
"koa": "^2.14.2",
|
||||
|
|
@ -69,6 +68,7 @@
|
|||
"lodash.debounce": "^4.0.8",
|
||||
"lucide-react": "^0.462.0",
|
||||
"luxon": "^3.5.0",
|
||||
"minisearch": "^7.2.0",
|
||||
"next-i18next": "^13.2.2",
|
||||
"p-queue": "^8.1.0",
|
||||
"prop-types": "^15.8.1",
|
||||
|
|
@ -13683,38 +13683,6 @@
|
|||
"integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/flexsearch": {
|
||||
"version": "0.8.205",
|
||||
"resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.8.205.tgz",
|
||||
"integrity": "sha512-REFjMqy86DKkCTJ4gIE42c9MVm9t1vUWfEub/8taixYuhvyu4jd4XmFALk5VuKW4GH4VLav8A4BJboTsslHF1w==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/ts-thomas"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/flexsearch"
|
||||
},
|
||||
{
|
||||
"type": "patreon",
|
||||
"url": "https://patreon.com/user?u=96245532"
|
||||
},
|
||||
{
|
||||
"type": "liberapay",
|
||||
"url": "https://liberapay.com/ts-thomas"
|
||||
},
|
||||
{
|
||||
"type": "paypal",
|
||||
"url": "https://www.paypal.com/donate/?hosted_button_id=GEVR88FC9BWRW"
|
||||
},
|
||||
{
|
||||
"type": "bountysource",
|
||||
"url": "https://salt.bountysource.com/teams/ts-thomas"
|
||||
}
|
||||
],
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/follow-redirects": {
|
||||
"version": "1.15.6",
|
||||
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz",
|
||||
|
|
@ -18602,6 +18570,12 @@
|
|||
"node": ">=16 || 14 >=14.17"
|
||||
}
|
||||
},
|
||||
"node_modules/minisearch": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/minisearch/-/minisearch-7.2.0.tgz",
|
||||
"integrity": "sha512-dqT2XBYUOZOiC5t2HRnwADjhNS2cecp9u+TJRiJ1Qp/f5qjkeT5APcGPjHw+bz89Ms8Jp+cG4AlE+QZ/QnDglg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/moment": {
|
||||
"version": "2.29.4",
|
||||
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz",
|
||||
|
|
|
|||
|
|
@ -127,7 +127,6 @@
|
|||
"diff": "^7.0.0",
|
||||
"esbuild-plugin-svg": "^0.1.0",
|
||||
"eventsource-parser": "^1.0.0",
|
||||
"flexsearch": "^0.8.205",
|
||||
"fuzzysort": "^3.1.0",
|
||||
"jotai": "^2.10.3",
|
||||
"koa": "^2.14.2",
|
||||
|
|
@ -137,6 +136,7 @@
|
|||
"lodash.debounce": "^4.0.8",
|
||||
"lucide-react": "^0.462.0",
|
||||
"luxon": "^3.5.0",
|
||||
"minisearch": "^7.2.0",
|
||||
"next-i18next": "^13.2.2",
|
||||
"p-queue": "^8.1.0",
|
||||
"prop-types": "^15.8.1",
|
||||
|
|
|
|||
|
|
@ -101,6 +101,8 @@ describe("QueryExpander", () => {
|
|||
|
||||
const result = await expander.expand("#project updates about project deadlines");
|
||||
|
||||
// In fallback mode (LLM error), terms are extracted from original query
|
||||
// "about" is included because there's no stopword filtering in fallback
|
||||
expect(result.salientTerms).toEqual(
|
||||
expect.arrayContaining(["#project", "updates", "about", "project", "deadlines"])
|
||||
);
|
||||
|
|
@ -237,39 +239,48 @@ describe("QueryExpander", () => {
|
|||
});
|
||||
|
||||
it("should parse different response formats", async () => {
|
||||
// Test with XML format
|
||||
// Test with new XML format (salient vs expanded separation)
|
||||
mockChatModel.invoke.mockResolvedValue({
|
||||
content: `<queries>
|
||||
content: `<salient>
|
||||
<term>test1</term>
|
||||
</salient>
|
||||
<queries>
|
||||
<query>xml variant</query>
|
||||
</queries>
|
||||
<terms>
|
||||
<expanded>
|
||||
<term>important</term>
|
||||
<term>keyword</term>
|
||||
</terms>`,
|
||||
</expanded>`,
|
||||
});
|
||||
let result = await expander.expand("test1");
|
||||
expect(result.queries).toContain("test1");
|
||||
expect(result.queries).toContain("xml variant");
|
||||
expect(result.salientTerms).toContain("test1"); // Only from original query
|
||||
expect(result.expandedTerms).toContain("important"); // LLM-generated terms
|
||||
// salientTerms come from <salient> section (original query terms only)
|
||||
expect(result.salientTerms).toContain("test1");
|
||||
// expandedTerms come from <expanded> section (for recall)
|
||||
expect(result.expandedTerms).toContain("important");
|
||||
expect(result.expandedTerms).toContain("keyword");
|
||||
|
||||
// Clear cache for next test
|
||||
expander.clearCache();
|
||||
|
||||
// Test legacy format (backward compatibility)
|
||||
// Test old <terms> format (backward compatibility - treats as expanded)
|
||||
mockChatModel.invoke.mockResolvedValue({
|
||||
content: `QUERIES:
|
||||
- legacy variant
|
||||
TERMS:
|
||||
- legacy
|
||||
- term`,
|
||||
content: `<queries>
|
||||
<query>old format variant</query>
|
||||
</queries>
|
||||
<terms>
|
||||
<term>related</term>
|
||||
<term>term</term>
|
||||
</terms>`,
|
||||
});
|
||||
result = await expander.expand("test2");
|
||||
expect(result.queries).toContain("test2");
|
||||
expect(result.queries).toContain("legacy variant");
|
||||
expect(result.salientTerms).toContain("test2"); // Only from original query
|
||||
expect(result.expandedTerms).toContain("legacy"); // LLM-generated terms
|
||||
expect(result.queries).toContain("old format variant");
|
||||
// With old format, salientTerms fallback to extracting from original query
|
||||
expect(result.salientTerms).toContain("test2");
|
||||
// Old <terms> go to expandedTerms
|
||||
expect(result.expandedTerms).toContain("related");
|
||||
expect(result.expandedTerms).toContain("term");
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -29,49 +29,44 @@ export class QueryExpander {
|
|||
private cache = new Map<string, ExpandedQuery>();
|
||||
private readonly config;
|
||||
|
||||
private static readonly PROMPT_TEMPLATE = `Generate alternative search queries and semantically related terms for the following query:
|
||||
"{query}"
|
||||
private static readonly PROMPT_TEMPLATE = `Analyze this search query and provide:
|
||||
1. SALIENT TERMS from the original query (for ranking)
|
||||
2. Alternative queries and related terms (for finding more results)
|
||||
|
||||
Query: "{query}"
|
||||
|
||||
Instructions:
|
||||
1. Generate {count} alternative search queries that capture the same intent
|
||||
2. Extract semantically related terms that someone might use when searching for this topic
|
||||
3. Include:
|
||||
- Keywords from the original query
|
||||
- Synonyms and related concepts
|
||||
- Domain-specific terminology
|
||||
- Associated terms someone might use
|
||||
4. Keep the SAME LANGUAGE as the original query
|
||||
5. Focus on NOUNS and meaningful concepts
|
||||
6. EXCLUDE common action verbs in ANY language (find, search, get, 查找, chercher, buscar, etc.)
|
||||
|
||||
SALIENT TERMS (for ranking - ONLY from original query):
|
||||
- Extract meaningful words FROM THE ORIGINAL QUERY ONLY
|
||||
- Include: nouns, proper nouns, technical terms, domain concepts
|
||||
- Exclude: action verbs (find, search, get), pronouns (my, your), articles (the, a), prepositions (in, on, for), conjunctions (and, or)
|
||||
- These terms determine search ranking - must be from original query
|
||||
|
||||
EXPANDED (for recall - can add new related terms):
|
||||
- Alternative phrasings of the query
|
||||
- Related/synonym terms to find more documents
|
||||
|
||||
Example: "find my piano notes"
|
||||
- Queries: "piano lesson notes", "piano practice sheets"
|
||||
- Terms: piano, notes, music, sheet, practice, lesson, piece, scales, exercises
|
||||
- Salient (from original): piano, notes
|
||||
- Expanded queries: "piano lesson notes", "piano practice sheets"
|
||||
- Expanded terms: music, sheet, practice, lesson
|
||||
|
||||
Example: "typescript interfaces"
|
||||
- Queries: "typescript type definitions", "typescript contracts"
|
||||
- Terms: typescript, interfaces, types, definitions, contracts, typing, declarations
|
||||
Example: "查找我的学习笔记"
|
||||
- Salient (from original): 学习, 笔记
|
||||
- Expanded queries: "个人笔记文档"
|
||||
- Expanded terms: 文档, 记录, 资料
|
||||
|
||||
Example: "查找我的笔记" (Chinese)
|
||||
- Queries: "我的学习笔记", "个人笔记文档"
|
||||
- Terms: 笔记, 文档, 记录, 资料, 学习, 备忘录 (keep in Chinese)
|
||||
|
||||
Example: "rechercher documents projet" (French)
|
||||
- Queries: "documents de projet", "fichiers projet"
|
||||
- Terms: documents, projet, fichiers, dossiers, archives (keep in French)
|
||||
|
||||
Format your response using XML tags:
|
||||
Format:
|
||||
<salient>
|
||||
<term>word_from_original_query</term>
|
||||
</salient>
|
||||
<queries>
|
||||
<query>alternative query 1</query>
|
||||
<query>alternative query 2</query>
|
||||
<query>alternative query</query>
|
||||
</queries>
|
||||
<terms>
|
||||
<term>keyword1</term>
|
||||
<term>keyword2</term>
|
||||
<term>keyword3</term>
|
||||
<term>related_term1</term>
|
||||
<term>related_term2</term>
|
||||
</terms>`;
|
||||
<expanded>
|
||||
<term>related_term_for_recall</term>
|
||||
</expanded>`;
|
||||
|
||||
constructor(private readonly options: QueryExpanderOptions = {}) {
|
||||
this.config = {
|
||||
|
|
@ -230,7 +225,8 @@ Format your response using XML tags:
|
|||
*/
|
||||
private parseXMLResponse(content: string, originalQuery: string): ExpandedQuery {
|
||||
const queries: string[] = [originalQuery]; // Always include original
|
||||
const llmExpandedTerms = new Set<string>(); // LLM-generated terms for recall only
|
||||
const salientFromLLM = new Set<string>(); // Salient terms from original query (for ranking)
|
||||
const expandedFromLLM = new Set<string>(); // Expanded terms (for recall only)
|
||||
|
||||
// Extract queries from XML tags
|
||||
const queryRegex = /<query>(.*?)<\/query>/g;
|
||||
|
|
@ -242,31 +238,64 @@ Format your response using XML tags:
|
|||
}
|
||||
}
|
||||
|
||||
// Extract LLM-generated terms from XML tags (for recall only)
|
||||
const termRegex = /<term>(.*?)<\/term>/g;
|
||||
let termMatch;
|
||||
while ((termMatch = termRegex.exec(content)) !== null) {
|
||||
const term = termMatch[1]?.trim().toLowerCase();
|
||||
if (term && this.isValidTerm(term)) {
|
||||
llmExpandedTerms.add(term);
|
||||
// Extract SALIENT terms (from original query, for ranking)
|
||||
const salientSection = content.match(/<salient>([\s\S]*?)<\/salient>/);
|
||||
if (salientSection) {
|
||||
const termRegex = /<term>(.*?)<\/term>/g;
|
||||
let termMatch;
|
||||
while ((termMatch = termRegex.exec(salientSection[1])) !== null) {
|
||||
const term = termMatch[1]?.trim().toLowerCase();
|
||||
if (term && this.isValidTerm(term)) {
|
||||
salientFromLLM.add(term);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If no XML tags found, try legacy parsing
|
||||
if (queries.length === 1 && llmExpandedTerms.size === 0) {
|
||||
// Extract EXPANDED terms (for recall, can include new related terms)
|
||||
const expandedSection = content.match(/<expanded>([\s\S]*?)<\/expanded>/);
|
||||
if (expandedSection) {
|
||||
const termRegex = /<term>(.*?)<\/term>/g;
|
||||
let termMatch;
|
||||
while ((termMatch = termRegex.exec(expandedSection[1])) !== null) {
|
||||
const term = termMatch[1]?.trim().toLowerCase();
|
||||
if (term && this.isValidTerm(term)) {
|
||||
expandedFromLLM.add(term);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: if no salient/expanded sections, try old <terms> format
|
||||
if (salientFromLLM.size === 0 && expandedFromLLM.size === 0) {
|
||||
const termRegex = /<term>(.*?)<\/term>/g;
|
||||
let termMatch;
|
||||
while ((termMatch = termRegex.exec(content)) !== null) {
|
||||
const term = termMatch[1]?.trim().toLowerCase();
|
||||
if (term && this.isValidTerm(term)) {
|
||||
// Old format: treat all terms as expanded (for recall)
|
||||
expandedFromLLM.add(term);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If no XML tags found at all, try legacy parsing
|
||||
if (queries.length === 1 && salientFromLLM.size === 0 && expandedFromLLM.size === 0) {
|
||||
return this.parseLegacyFormat(content, originalQuery);
|
||||
}
|
||||
|
||||
// Extract salient terms ONLY from the original query (for scoring)
|
||||
const salientTerms = this.extractSalientTermsFromOriginal(originalQuery);
|
||||
// Salient terms: from LLM's <salient> section, or fallback to extracting from original query
|
||||
const tagTerms = extractTagsFromQuery(originalQuery);
|
||||
const salientTerms =
|
||||
salientFromLLM.size > 0
|
||||
? Array.from(new Set([...salientFromLLM, ...tagTerms]))
|
||||
: this.extractSalientTermsFromOriginal(originalQuery);
|
||||
|
||||
const expandedQueries = queries.slice(1); // Exclude the original query
|
||||
const expandedQueries = queries.slice(1);
|
||||
return {
|
||||
queries: queries.slice(0, this.config.maxVariants + 1), // +1 for original
|
||||
salientTerms: salientTerms, // Only terms from original query
|
||||
queries: queries.slice(0, this.config.maxVariants + 1),
|
||||
salientTerms: salientTerms, // From original query only (for ranking)
|
||||
originalQuery: originalQuery,
|
||||
expandedQueries: expandedQueries.slice(0, this.config.maxVariants),
|
||||
expandedTerms: Array.from(llmExpandedTerms), // LLM-generated terms for recall
|
||||
expandedTerms: Array.from(expandedFromLLM), // New related terms (for recall)
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -280,7 +309,7 @@ Format your response using XML tags:
|
|||
// Fallback parser for non-XML responses
|
||||
const lines = content.split("\n").map((line) => line.trim());
|
||||
const queries: string[] = [originalQuery];
|
||||
const llmExpandedTerms = new Set<string>(); // LLM-generated terms
|
||||
const llmTerms = new Set<string>(); // LLM-generated terms (stopwords filtered by LLM)
|
||||
|
||||
let section: "queries" | "terms" | null = null;
|
||||
|
||||
|
|
@ -309,13 +338,13 @@ Format your response using XML tags:
|
|||
.trim()
|
||||
.toLowerCase();
|
||||
if (cleanTerm && this.isValidTerm(cleanTerm)) {
|
||||
llmExpandedTerms.add(cleanTerm); // Store as expanded terms
|
||||
llmTerms.add(cleanTerm);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If no sections found, treat lines as queries
|
||||
if (queries.length === 1 && llmExpandedTerms.size === 0) {
|
||||
if (queries.length === 1 && llmTerms.size === 0) {
|
||||
for (const line of lines.slice(0, this.config.maxVariants)) {
|
||||
if (line && !line.toUpperCase().includes("QUERY")) {
|
||||
queries.push(line);
|
||||
|
|
@ -323,16 +352,18 @@ Format your response using XML tags:
|
|||
}
|
||||
}
|
||||
|
||||
// Extract salient terms ONLY from the original query
|
||||
// Legacy format doesn't distinguish salient vs expanded terms
|
||||
// Salient terms must come from original query only (for ranking)
|
||||
// LLM terms go to expandedTerms (for recall)
|
||||
const salientTerms = this.extractSalientTermsFromOriginal(originalQuery);
|
||||
|
||||
const expandedQueries = queries.slice(1);
|
||||
return {
|
||||
queries: queries.slice(0, this.config.maxVariants + 1),
|
||||
salientTerms: salientTerms, // Only from original query
|
||||
salientTerms: salientTerms, // Always from original query
|
||||
originalQuery: originalQuery,
|
||||
expandedQueries: expandedQueries.slice(0, this.config.maxVariants),
|
||||
expandedTerms: Array.from(llmExpandedTerms), // LLM-generated terms
|
||||
expandedTerms: Array.from(llmTerms), // LLM terms for recall only
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -425,6 +456,7 @@ Format your response using XML tags:
|
|||
/**
|
||||
* Validates if a term should be included in salient terms.
|
||||
* Filters out terms that are too short or contain only special characters.
|
||||
* Note: Stopword filtering is handled by the LLM prompt, not here.
|
||||
* @param term - The term to validate
|
||||
* @returns true if the term is valid for inclusion
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -6,12 +6,12 @@ A high-performance, memory-bounded lexical search system for Obsidian that uses
|
|||
|
||||
```mermaid
|
||||
graph TD
|
||||
A[User Query] --> B[QueryExpander<br/>variants + salient terms]
|
||||
A[User Query] --> B[QueryExpander<br/>salient + expanded terms]
|
||||
B --> C[SearchCore<br/>pipeline orchestrator]
|
||||
|
||||
C --> D[Grep Scan<br/>recall up to 200 paths]
|
||||
C --> E[ChunkManager<br/>heading-first chunks]
|
||||
E --> F[FullTextEngine<br/>chunk index – title · path · tags · body]
|
||||
E --> F[MiniSearch Engine<br/>BM25+ scoring]
|
||||
F --> G[Boosters<br/>folder / graph / tag bonus]
|
||||
G --> H[Score Normalization]
|
||||
H --> I[Deduplicate + order]
|
||||
|
|
@ -27,6 +27,24 @@ graph TD
|
|||
style J fill:#dcedc8
|
||||
```
|
||||
|
||||
## Recall vs Ranking: Core Design
|
||||
|
||||
The search system clearly separates **recall** (finding candidates) from **ranking** (scoring precision):
|
||||
|
||||
```
|
||||
Original query: "find my piano notes"
|
||||
|
||||
FOR RECALL (finding candidate documents):
|
||||
├── Original query: "find my piano notes"
|
||||
├── Expanded queries: "piano lesson notes", "piano practice sheets"
|
||||
└── Expanded terms: music, sheet, practice, lesson
|
||||
|
||||
FOR RANKING (scoring/precision):
|
||||
└── Salient terms: piano, notes (from original query only, stopwords filtered)
|
||||
```
|
||||
|
||||
**Key principle**: LLM expansion improves recall (finding more documents), but ranking uses only terms from the original query to maintain precision.
|
||||
|
||||
## Chunk ID Mapping and Result Assembly
|
||||
|
||||
### How Chunk IDs Work
|
||||
|
|
@ -43,7 +61,7 @@ The lexical search engine operates on individual chunks and returns consistent c
|
|||
```
|
||||
|
||||
2. **Search Result Assembly**:
|
||||
- **Lexical Search**: Returns chunk IDs from FlexSearch index with lexical match explanations
|
||||
- **Lexical Search**: Returns chunk IDs from MiniSearch index with lexical match explanations
|
||||
- **Final Assembly**: ChunkManager retrieves chunk text for LLM context generation
|
||||
|
||||
### Benefits of Chunk Architecture
|
||||
|
|
@ -57,7 +75,8 @@ The lexical search engine operates on individual chunks and returns consistent c
|
|||
|
||||
- **Memory-Bounded**: No persistent full-text index, everything ephemeral
|
||||
- **Progressive Refinement**: Fast grep → full-text lexical search
|
||||
- **Multilingual**: Supports English and CJK languages
|
||||
- **Multilingual**: Supports English and CJK languages via custom tokenizer
|
||||
- **BM25+ Scoring**: Proper multi-term relevance scoring via MiniSearch
|
||||
- **Explainable**: Tracks why documents ranked highly
|
||||
- **Fault-Tolerant**: Graceful fallbacks at each stage
|
||||
- **Chunk-Based**: Operates on intelligent document chunks for precise context
|
||||
|
|
@ -69,17 +88,31 @@ The lexical search engine operates on individual chunks and returns consistent c
|
|||
|
||||
### 1. Query Expansion
|
||||
|
||||
The LLM extracts salient terms and generates expanded terms/queries:
|
||||
|
||||
```
|
||||
Input: "Need to deploy #project/alpha after fixing #mobile-app sync"
|
||||
Output: ["Need to deploy #project/alpha after fixing #mobile-app sync",
|
||||
"deploy #project/alpha",
|
||||
"mobile app sync fix"]
|
||||
Terms: ["#project/alpha", "#mobile-app", "deploy", "sync"]
|
||||
|
||||
LLM Response:
|
||||
<salient> ← From original query (for ranking)
|
||||
<term>deploy</term>
|
||||
<term>sync</term>
|
||||
<term>#project/alpha</term>
|
||||
<term>#mobile-app</term>
|
||||
</salient>
|
||||
<queries> ← Alternative phrasings (for recall)
|
||||
<query>deploy project alpha</query>
|
||||
<query>mobile app sync fix</query>
|
||||
</queries>
|
||||
<expanded> ← Related terms (for recall only)
|
||||
<term>release</term>
|
||||
<term>synchronization</term>
|
||||
</expanded>
|
||||
```
|
||||
|
||||
### 2. Grep Scan (L0)
|
||||
|
||||
Searches for all recall terms (original + expanded + salient):
|
||||
Searches for all recall terms (original + expanded queries + expanded terms + salient terms):
|
||||
|
||||
```
|
||||
Finds: ["projects/alpha/deploy.md", "projects/mobile/sync.md", "notes/release-checklist.md"]
|
||||
|
|
@ -98,14 +131,25 @@ Output: ["projects/alpha/deploy.md#0", "projects/alpha/deploy.md#1", "projects/m
|
|||
|
||||
### 4. Full-Text Search Execution
|
||||
|
||||
**Two-phase approach**:
|
||||
**MiniSearch with BM25+ scoring and weighted query expansion**:
|
||||
|
||||
- **Recall**: Uses the original query, expanded variants, and salient/tag terms to locate candidate chunks in the FlexSearch index.
|
||||
- **Ranking**: Scores with salient/tag terms when they are available; if the expander supplies none, the original user query is used instead. Weighting comes from the FlexSearch scorer plus our field weights, and we taper the impact of tokens that hit most candidates so everyday words (e.g., “note”, “meeting”) can’t overpower rarer matches.
|
||||
- **Primary scoring (90% weight)**: Salient terms from original query drive ranking
|
||||
- **Secondary scoring (10% weight)**: Expanded terms provide small recall boost
|
||||
- Field weights: Title (3x), Heading (2.5x), Path (1.5x), Tags (4x), Body (1x)
|
||||
|
||||
- Builds an ephemeral FlexSearch index from chunks (not full notes)
|
||||
- **Frontmatter Replication**: Extracts note-level frontmatter once and replicates property values across all chunks from that note
|
||||
- Field weights: Title (3x), Heading (2.5x), Path (2x), Tags (4x), Body (1x)
|
||||
**Weighted Query Expansion**: A safe, common IR strategy where expanded terms influence ranking by a small amount:
|
||||
|
||||
```
|
||||
finalScore = 0.9 × salientScore + 0.1 × expandedScore
|
||||
```
|
||||
|
||||
This approach:
|
||||
|
||||
- Preserves original query intent (salient terms dominate)
|
||||
- Breaks ties between documents with equal salient matches
|
||||
- Gives secondary-only results (found via expansion) a small but non-zero score
|
||||
|
||||
**Why BM25+?**: Documents matching more salient terms naturally score higher. A query "hk milk tea home recipe" will rank "HK Milk Tea - Home Recipe" highly because all 5 terms match in the title field (3x weight).
|
||||
|
||||
### 5. Lexical Reranking (Boosting Stage)
|
||||
|
||||
|
|
@ -119,15 +163,15 @@ Applied to lexical results (when `enableLexicalBoosts: true`):
|
|||
|
||||
Min-max normalization prevents auto-1.0 scores
|
||||
|
||||
- `baseScore`: The raw score after lexical search and all boosts are applied
|
||||
- `baseScore`: The raw BM25+ score after all boosts are applied
|
||||
- `finalScore`: The normalized score in [0.02, 0.98] range that users see
|
||||
|
||||
### 7. Final Results
|
||||
|
||||
```
|
||||
1. projects/alpha/deploy.md#0 (0.94) - Matched `#project/alpha` + “deploy”, folder boost
|
||||
2. projects/mobile/sync.md#0 (0.46) - Matched `#mobile-app` + “sync”, folder boost
|
||||
3. notes/release-checklist.md#1 (0.23) - Matched “deploy” and shared tags
|
||||
1. projects/alpha/deploy.md#0 (0.94) - Matched `#project/alpha` + "deploy", folder boost
|
||||
2. projects/mobile/sync.md#0 (0.46) - Matched `#mobile-app` + "sync", folder boost
|
||||
3. notes/release-checklist.md#1 (0.23) - Matched "deploy" and shared tags
|
||||
```
|
||||
|
||||
## Core Components
|
||||
|
|
@ -149,12 +193,27 @@ Min-max normalization prevents auto-1.0 scores
|
|||
- `getChunks()`: Creates chunks from note paths with size constraints
|
||||
- `getChunkText()`: Retrieves chunk content by ID for LLM context
|
||||
|
||||
### Query Expander
|
||||
### QueryExpander
|
||||
|
||||
- Generates alternative phrasings using LLM (5s timeout)
|
||||
- Separates recall terms (all) from ranking terms (salient only) to prevent stopword pollution
|
||||
- Extracts salient terms (nouns) from original user query
|
||||
- Falls back to original query if LLM unavailable
|
||||
Generates three distinct outputs for recall vs ranking:
|
||||
|
||||
```typescript
|
||||
interface ExpandedQuery {
|
||||
originalQuery: string; // "find my piano notes"
|
||||
queries: string[]; // [original + expanded queries] for recall
|
||||
salientTerms: string[]; // ["piano", "notes"] for ranking (from original only)
|
||||
expandedTerms: string[]; // ["music", "sheet"] for recall only
|
||||
expandedQueries: string[]; // ["piano lesson notes"] for recall only
|
||||
}
|
||||
```
|
||||
|
||||
**LLM Prompt Design**:
|
||||
|
||||
- `<salient>` section: Terms FROM the original query (stopwords filtered)
|
||||
- `<expanded>` section: NEW related terms (for recall only)
|
||||
- `<queries>` section: Alternative phrasings (for recall only)
|
||||
|
||||
Falls back to extracting terms from original query if LLM unavailable.
|
||||
|
||||
### Grep Scanner
|
||||
|
||||
|
|
@ -163,16 +222,15 @@ Min-max normalization prevents auto-1.0 scores
|
|||
- Searches both phrases and individual terms
|
||||
- Path-first optimization for faster matching
|
||||
|
||||
### Full-Text Engine
|
||||
### MiniSearch Engine (Full-Text Engine)
|
||||
|
||||
- Ephemeral FlexSearch index built from chunks per-query
|
||||
- **Two-phase search**: Recall uses all terms, ranking uses only salient terms
|
||||
- Custom tokenizer for ASCII words + CJK bigrams
|
||||
- Multi-field indexing with weights: title (3x), heading (2.5x), path (2x), body (1x)
|
||||
- **Frontmatter Property Indexing**: Extracts and indexes frontmatter property values for searchability
|
||||
- **Note-Level Metadata, Chunk-Level Indexing**: Frontmatter extracted once per note, replicated across all chunks
|
||||
- **BM25+ Scoring**: Proper multi-term relevance ranking (not position-based)
|
||||
- Ephemeral index built from chunks per-query
|
||||
- Custom `tokenizeMixed` tokenizer for ASCII words + CJK bigrams
|
||||
- Multi-field indexing with weights: title (3x), heading (2.5x), path (1.5x), tags (4x), body (1x)
|
||||
- **Frontmatter Property Indexing**: Extracts and indexes frontmatter property values
|
||||
- Note-level metadata replicated across all chunks from that note
|
||||
- Supports primitive values, arrays, and Date objects
|
||||
- **Performance Optimization**: Per-note metadata caching
|
||||
- Memory-efficient: chunk content retrieved from ChunkManager when needed
|
||||
|
||||
### Folder & Graph Boost Calculators
|
||||
|
|
@ -271,7 +329,7 @@ Search: "priority 1" → Finds all 3 chunks, each containing "1" in searchable c
|
|||
```typescript
|
||||
interface NoteIdRank {
|
||||
id: string; // Chunk ID (note_path#chunk_index) or note path for legacy
|
||||
score: number; // Relevance score [0-1]
|
||||
score: number; // BM25+ relevance score [0-1]
|
||||
engine?: string; // Source engine
|
||||
explanation?: {
|
||||
// Why it ranked high
|
||||
|
|
@ -285,7 +343,7 @@ interface NoteIdRank {
|
|||
score: number;
|
||||
boostMultiplier: number;
|
||||
};
|
||||
baseScore: number; // Score before normalization (after RRF fusion and boosts)
|
||||
baseScore: number; // BM25+ score before normalization
|
||||
finalScore: number; // Score after normalization (final 0-1 range score)
|
||||
};
|
||||
}
|
||||
|
|
@ -299,17 +357,25 @@ interface Chunk {
|
|||
heading: string; // section heading
|
||||
mtime: number; // note modification time
|
||||
}
|
||||
|
||||
interface ExpandedQuery {
|
||||
originalQuery: string; // The user's original query
|
||||
queries: string[]; // [original + expanded queries] for recall
|
||||
salientTerms: string[]; // Terms from original query (for ranking)
|
||||
expandedTerms: string[]; // LLM-generated related terms (for recall)
|
||||
expandedQueries: string[]; // LLM-generated alternative queries (for recall)
|
||||
}
|
||||
```
|
||||
|
||||
## Performance Characteristics
|
||||
|
||||
- **Grep scan**: < 50ms for 1k files
|
||||
- **Chunking**: < 50ms for 500 candidates → ~1000 chunks
|
||||
- **Full-text build**: < 100ms for 1000 chunks (memory-bounded)
|
||||
- **Lexical search**: Fast in-memory FlexSearch queries
|
||||
- **MiniSearch build**: < 100ms for 1000 chunks (memory-bounded)
|
||||
- **BM25+ search**: Fast in-memory MiniSearch queries
|
||||
- **Total latency**: < 200ms P95 (chunking + lexical search)
|
||||
- **Memory peak**: < 20MB mobile, < 100MB desktop
|
||||
- **Memory split**: 35% chunk cache, 65% FlexSearch index
|
||||
- **Memory split**: 35% chunk cache, 65% MiniSearch index
|
||||
|
||||
## Configuration & Usage
|
||||
|
||||
|
|
@ -330,49 +396,50 @@ interface Chunk {
|
|||
- Max Candidates: 10 (performance cap)
|
||||
- Boost Strength: 0.1 (connection influence)
|
||||
- Max Boost Multiplier: 1.15x (prevents over-boosting)
|
||||
- **Memory Management**: RAM usage split between chunks (35%) and FlexSearch (65%)
|
||||
- **Memory Management**: RAM usage split between chunks (35%) and MiniSearch (65%)
|
||||
- **File Filtering**: Pattern-based inclusions and exclusions for search
|
||||
|
||||
## Full Pipeline Overview
|
||||
|
||||
```
|
||||
┌────────────┐ ┌──────────────────┐ ┌─────────────────────────┐
|
||||
│ User Query │───────▶│ QueryExpander │─────────▶│ Salient Terms │
|
||||
│ User Query │───────▶│ QueryExpander │─────────▶│ Salient + Expanded │
|
||||
└────────────┘ └──────┬───────────┘ └────────┬──────────────────┘
|
||||
│ keep language / keep #tags │
|
||||
▼ ▼
|
||||
│ <salient> for ranking │
|
||||
│ <expanded> for recall │
|
||||
▼ ▼
|
||||
┌──────────────────┐ ┌──────────────────┐ ┌─────────────────────────┐
|
||||
│ Grep (recall) │◀──│ SearchCore │─────────▶│ FlexSearch (ranking pool)│
|
||||
│ Grep (recall) │◀──│ SearchCore │─────────▶│ MiniSearch (BM25+ rank) │
|
||||
└──────┬───────────┘ └──────┬───────────┘ └────────┬──────────────────┘
|
||||
│ candidate list │ returnAll? cap=200 │ normalized scores
|
||||
▼ ▼ ▼
|
||||
│ candidate list │ returnAll? cap=200 │ BM25+ scores
|
||||
▼ ▼ ▼
|
||||
┌──────────────────┐ ┌──────────────────┐ ┌─────────────────────────┐
|
||||
│ ChunkManager │──▶│ FullTextEngine │─────────▶│ Scoring + Boosting │
|
||||
│ ChunkManager │──▶│ MiniSearch Engine│─────────▶│ Scoring + Boosting │
|
||||
└──────────────────┘ └──────┬───────────┘ └────────┬──────────────────┘
|
||||
│ index title/path/body/tags/props │ folder/graph/tag bonuses
|
||||
▼ ▼
|
||||
┌─────────────────────────┐ ┌─────────────────────────┐
|
||||
│ Dedup + Ordering │────────────▶│ Final Results (≤200) │
|
||||
└─────────────────────────┘ └─────────────────────────┘
|
||||
│ index title/path/body/tags/props │ folder/graph bonuses
|
||||
▼ ▼
|
||||
┌─────────────────────────┐ ┌─────────────────────────┐
|
||||
│ Dedup + Ordering │────────▶│ Final Results (≤200) │
|
||||
└─────────────────────────┘ └─────────────────────────┘
|
||||
```
|
||||
|
||||
- **Query & tags**: Expansion keeps hash-prefixed terms exactly as typed (`#project/alpha`). We do not generate plain variants; tag recall is handled by metadata indexing.
|
||||
- **Recall**: Grep seeds the candidate list with the original query + salient terms while honouring the 200-chunk ceiling when `returnAll` is enabled (time range or tag query).
|
||||
- **Chunking**: Notes are split once through `ChunkManager`, and frontmatter properties/tags are copied to every chunk while inline tags stay local.
|
||||
- **Indexing**: `FullTextEngine` indexes title, path, body, and the normalised tag/prop fields. Tags receive their own field but we avoid partial matches for hyphenated tags, keeping tokens intact.
|
||||
- **Scoring**: FlexSearch results flow through lexical scoring, folder/graph boosts, and a modest tag bonus (`tags` weight = 4) so true tag hits win without overwhelming other evidence.
|
||||
- **Return-all mode**: When a time range or tag query is detected we raise the cap to 100 chunks (`RETURN_ALL_LIMIT`), guaranteeing tag/time windows fan out widely while staying bounded for downstream ranking.
|
||||
- **Query & tags**: Expansion keeps hash-prefixed terms exactly as typed (`#project/alpha`). Salient terms extracted from original query only.
|
||||
- **Recall**: Grep seeds the candidate list with original query + expanded queries + expanded terms + salient terms.
|
||||
- **Chunking**: Notes are split once through `ChunkManager`, and frontmatter properties/tags are copied to every chunk.
|
||||
- **Indexing**: MiniSearch indexes title, path, body, and the normalized tag/prop fields with field weights.
|
||||
- **Scoring**: MiniSearch's BM25+ algorithm scores documents based on salient terms only. Documents matching more terms score higher naturally.
|
||||
- **Return-all mode**: When a time range or tag query is detected, cap raised to 100 chunks (`RETURN_ALL_LIMIT`).
|
||||
|
||||
**Example** — `#project/alpha bugfix`
|
||||
|
||||
1. QueryExpander preserves the tag (`#project/alpha`) and extracts `bugfix` as a salient term.
|
||||
2. SearchCore runs grep + FlexSearch with `#project/alpha` and `bugfix`, raising the cap to 200.
|
||||
3. FullTextEngine finds chunks where the metadata already lists that tag (inline or frontmatter) and indexes them under the `tags` field.
|
||||
4. Scoring boosts those chunks via the tag field weight, so the bugfix notes with the exact tag rise above generic “project alpha” mentions.
|
||||
1. QueryExpander extracts salient terms: `#project/alpha`, `bugfix` (from original query)
|
||||
2. LLM may provide expanded terms: `fix`, `issue`, `patch` (for recall only)
|
||||
3. SearchCore runs grep with all terms, MiniSearch scores using only salient terms
|
||||
4. BM25+ naturally ranks documents with both `#project/alpha` and `bugfix` higher than those with only one term
|
||||
|
||||
## Semantic + Lexical Fusion (Design)
|
||||
|
||||
Semantic retrieval (HybridRetriever/Orama) is strong at paraphrase recall, while Search v3’s TieredLexicalRetriever excels at deliberate keyword/tag matching. When the **Semantic Search** toggle is enabled we can fuse both without widening the retriever surface area.
|
||||
Semantic retrieval (HybridRetriever/Orama) is strong at paraphrase recall, while Search v3's TieredLexicalRetriever excels at deliberate keyword/tag matching. When the **Semantic Search** toggle is enabled we can fuse both without widening the retriever surface area.
|
||||
|
||||
- **MergedSemanticRetriever**
|
||||
- Implements the same `getRelevantDocuments()` API.
|
||||
|
|
@ -392,7 +459,7 @@ Semantic retrieval (HybridRetriever/Orama) is strong at paraphrase recall, while
|
|||
- **Fallback Behaviour**
|
||||
- With semantic search disabled nothing changes—the TieredLexicalRetriever path remains intact.
|
||||
|
||||
This approach keeps callers unaware of the fusion mechanics while giving users semantic coverage plus Search v3’s deterministic tag/keyword strength.
|
||||
This approach keeps callers unaware of the fusion mechanics while giving users semantic coverage plus Search v3's deterministic tag/keyword strength.
|
||||
|
||||
### TODO
|
||||
|
||||
|
|
@ -402,13 +469,15 @@ This approach keeps callers unaware of the fusion mechanics while giving users s
|
|||
## Key Design Decisions
|
||||
|
||||
1. **Lexical-Only Architecture**: Fast, reliable keyword-based search with intelligent boosting
|
||||
2. **Heading-First Algorithm**: Preserves document structure while respecting size limits
|
||||
3. **No Persistent Full-Text Index**: Grep provides fast initial seeding, chunks built per-query
|
||||
4. **Ephemeral Everything**: Eliminates maintenance overhead
|
||||
5. **Memory-Efficient Indexing**: FlexSearch stores metadata only, chunk content retrieved when needed
|
||||
6. **Frontmatter Property Integration**: Extract note-level frontmatter once and replicate across all chunks for seamless search
|
||||
7. **Chunk Sequence Preservation**: Chunks from same note served in order for LLM context
|
||||
8. **Min-Max Normalization**: Prevents artificial perfect scores while preserving monotonicity
|
||||
9. **Explainable Rankings**: Track contributing factors for transparency including chunk-level details
|
||||
10. **Memory Budget Split**: Fixed 35%/65% allocation between chunk cache and FlexSearch index
|
||||
11. **Semantic Search Separation**: Semantic capabilities now handled by dedicated Orama integration
|
||||
2. **MiniSearch with BM25+**: Proper multi-term relevance scoring (replaced FlexSearch's position-based scoring)
|
||||
3. **Recall vs Ranking Separation**: Expanded terms improve recall, salient terms (from original query) drive ranking
|
||||
4. **Heading-First Algorithm**: Preserves document structure while respecting size limits
|
||||
5. **No Persistent Full-Text Index**: Grep provides fast initial seeding, chunks built per-query
|
||||
6. **Ephemeral Everything**: Eliminates maintenance overhead
|
||||
7. **Memory-Efficient Indexing**: MiniSearch stores metadata only, chunk content retrieved when needed
|
||||
8. **Frontmatter Property Integration**: Extract note-level frontmatter once and replicate across all chunks for seamless search
|
||||
9. **Chunk Sequence Preservation**: Chunks from same note served in order for LLM context
|
||||
10. **Min-Max Normalization**: Prevents artificial perfect scores while preserving monotonicity
|
||||
11. **Explainable Rankings**: Track contributing factors for transparency including chunk-level details
|
||||
12. **Memory Budget Split**: Fixed 35%/65% allocation between chunk cache and MiniSearch index
|
||||
13. **Semantic Search Separation**: Semantic capabilities now handled by dedicated Orama integration
|
||||
|
|
|
|||
|
|
@ -150,7 +150,8 @@ export class SearchCore {
|
|||
salientTerms,
|
||||
maxResults,
|
||||
expanded.originalQuery,
|
||||
returnAll
|
||||
returnAll,
|
||||
expanded.expandedTerms
|
||||
);
|
||||
|
||||
// 6. Apply boosts to lexical results (if enabled)
|
||||
|
|
@ -291,6 +292,8 @@ export class SearchCore {
|
|||
* @param salientTerms - Salient terms for scoring (extracted from original query)
|
||||
* @param maxResults - Maximum number of results
|
||||
* @param originalQuery - The original user query for scoring
|
||||
* @param returnAll - Whether to return all results up to RETURN_ALL_LIMIT
|
||||
* @param expandedTerms - LLM-generated related terms (secondary scoring for recall boost)
|
||||
* @returns Ranked list of documents from lexical search
|
||||
*/
|
||||
private async executeLexicalSearch(
|
||||
|
|
@ -299,7 +302,8 @@ export class SearchCore {
|
|||
salientTerms: string[],
|
||||
maxResults: number,
|
||||
originalQuery?: string,
|
||||
returnAll: boolean = false
|
||||
returnAll: boolean = false,
|
||||
expandedTerms: string[] = []
|
||||
): Promise<NoteIdRank[]> {
|
||||
try {
|
||||
// Build ephemeral full-text index
|
||||
|
|
@ -321,7 +325,8 @@ export class SearchCore {
|
|||
recallQueries,
|
||||
searchLimit,
|
||||
salientTerms,
|
||||
originalQuery
|
||||
originalQuery,
|
||||
expandedTerms
|
||||
);
|
||||
const searchTime = Date.now() - searchStartTime;
|
||||
|
||||
|
|
|
|||
|
|
@ -670,11 +670,13 @@ describe("FullTextEngine", () => {
|
|||
expect(tagMatchScore).toBeGreaterThan(otherScore);
|
||||
});
|
||||
|
||||
it("should downweight ubiquitous terms during ranking", async () => {
|
||||
it("should return valid BM25 scores for search results", async () => {
|
||||
await engine.buildFromCandidates(["common.md"]);
|
||||
const results = engine.search(["meeting"], 10, ["meeting"], "meeting");
|
||||
|
||||
expect(results[0].explanation?.baseScore).toBeCloseTo(0.4, 2);
|
||||
// MiniSearch uses BM25+ scoring - verify score is positive and reasonable
|
||||
expect(results[0].explanation?.baseScore).toBeGreaterThan(0);
|
||||
expect(results[0].explanation?.baseScore).toBeLessThan(10);
|
||||
});
|
||||
|
||||
it("should keep rare terms dominant when combined with common terms", async () => {
|
||||
|
|
@ -689,14 +691,101 @@ describe("FullTextEngine", () => {
|
|||
expect(results[0].id).toBe("unique.md#0");
|
||||
});
|
||||
|
||||
it("should display hashless explanation for non-tag matches", async () => {
|
||||
it("should find matches for tag-like queries in body content", async () => {
|
||||
await engine.buildFromCandidates(["unique.md"]);
|
||||
const results = engine.search(["#rareterm"], 10, ["#rareterm"], "#rareterm");
|
||||
const explanation = results[0].explanation?.lexicalMatches?.find(
|
||||
(match) => match.field === "body"
|
||||
|
||||
// MiniSearch should find the document containing "rareterm" in body
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
expect(results[0].id).toBe("unique.md#0");
|
||||
|
||||
// The explanation should contain lexical matches (MiniSearch reports which terms matched)
|
||||
expect(results[0].explanation?.lexicalMatches).toBeDefined();
|
||||
expect(results[0].explanation?.lexicalMatches?.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("weighted query expansion", () => {
|
||||
it("should apply 90/10 weight split between salient and expanded terms", async () => {
|
||||
// Use Piano Lessons files which contain "piano" in content
|
||||
await engine.buildFromCandidates(["Piano Lessons/Lesson 1.md", "projects/music.md"]);
|
||||
|
||||
// Search with salient terms only (no expanded)
|
||||
const salientOnly = engine.search(["piano"], 10, ["piano"], "piano", []);
|
||||
|
||||
// Search with both salient and expanded terms
|
||||
const withExpanded = engine.search(["piano"], 10, ["piano"], "piano", ["music", "lesson"]);
|
||||
|
||||
// Both should find results (piano content exists in these files)
|
||||
expect(salientOnly.length).toBeGreaterThan(0);
|
||||
expect(withExpanded.length).toBeGreaterThan(0);
|
||||
|
||||
// Results with expanded terms may have different scores due to 10% secondary boost
|
||||
const lessonResultSalient = salientOnly.find((r) =>
|
||||
r.id.startsWith("Piano Lessons/Lesson 1.md")
|
||||
);
|
||||
const lessonResultExpanded = withExpanded.find((r) =>
|
||||
r.id.startsWith("Piano Lessons/Lesson 1.md")
|
||||
);
|
||||
|
||||
expect(explanation?.query).toBe("rareterm");
|
||||
expect(lessonResultSalient).toBeDefined();
|
||||
expect(lessonResultExpanded).toBeDefined();
|
||||
});
|
||||
|
||||
it("should include expanded boost in explanation when expanded terms match", async () => {
|
||||
await engine.buildFromCandidates(["common.md"]);
|
||||
|
||||
// common.md contains "meeting" - use it as expanded term
|
||||
// Use "note" as salient term since common.md contains "Meeting note summary"
|
||||
const results = engine.search(
|
||||
["note", "meeting"], // queries
|
||||
10,
|
||||
["note"], // salient terms
|
||||
"note", // original query
|
||||
["meeting"] // expanded terms
|
||||
);
|
||||
|
||||
// Should find results (common.md has "meeting" and "note" in content)
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
|
||||
// Check that expandedBoost is present in explanation when expanded terms contributed
|
||||
const matchingResult = results.find((r) => r.explanation?.expandedBoost !== undefined);
|
||||
if (matchingResult) {
|
||||
expect(matchingResult.explanation?.expandedBoost).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it("should give secondary-only results (found via expansion only) lower scores", async () => {
|
||||
// common.md has "agenda" (unique to it), unique.md has "rareterm" (unique to it)
|
||||
await engine.buildFromCandidates(["common.md", "unique.md"]);
|
||||
|
||||
// "rareterm" only exists in unique.md, "agenda" only exists in common.md
|
||||
// Search with salient term "agenda" (matches only common.md)
|
||||
// and expanded term "rareterm" (matches only unique.md)
|
||||
const results = engine.search(
|
||||
["agenda", "rareterm"], // queries
|
||||
10,
|
||||
["agenda"], // salient (matches only common.md)
|
||||
"agenda",
|
||||
["rareterm"] // expanded (matches only unique.md)
|
||||
);
|
||||
|
||||
// Should find results
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
|
||||
const commonResult = results.find((r) => r.id.startsWith("common.md"));
|
||||
const uniqueResult = results.find((r) => r.id.startsWith("unique.md"));
|
||||
|
||||
// Common should be found (matches salient term "agenda")
|
||||
expect(commonResult).toBeDefined();
|
||||
|
||||
// Unique should be found via expansion only
|
||||
expect(uniqueResult).toBeDefined();
|
||||
|
||||
// Salient match (90% weight) should score higher than expansion-only (10% weight)
|
||||
if (uniqueResult && commonResult) {
|
||||
expect(commonResult.score).toBeGreaterThan(uniqueResult.score);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -980,36 +1069,26 @@ describe("FullTextEngine", () => {
|
|||
expect((engine as any).indexedChunks.size).toBe(0);
|
||||
});
|
||||
|
||||
it("should handle index with destroy method", async () => {
|
||||
it("should nullify MiniSearch index on clear", async () => {
|
||||
await engine.buildFromCandidates(["note1.md"]);
|
||||
|
||||
// Mock index with destroy method
|
||||
const mockDestroy = jest.fn();
|
||||
(engine as any).index = {
|
||||
destroy: mockDestroy,
|
||||
};
|
||||
// Verify index exists
|
||||
expect((engine as any).index).not.toBeNull();
|
||||
|
||||
engine.clear();
|
||||
|
||||
// Should call destroy method
|
||||
expect(mockDestroy).toHaveBeenCalledTimes(1);
|
||||
// MiniSearch cleanup is just nullifying the reference
|
||||
expect((engine as any).index).toBeNull();
|
||||
});
|
||||
|
||||
it("should handle index with clear method when destroy is not available", async () => {
|
||||
it("should allow new index to be created after clear", async () => {
|
||||
await engine.buildFromCandidates(["note1.md"]);
|
||||
|
||||
// Mock index with only clear method
|
||||
const mockClear = jest.fn();
|
||||
(engine as any).index = {
|
||||
clear: mockClear,
|
||||
};
|
||||
|
||||
engine.clear();
|
||||
|
||||
// Should call clear method
|
||||
expect(mockClear).toHaveBeenCalledTimes(1);
|
||||
expect((engine as any).index).toBeNull();
|
||||
// Should be able to create a new index after clearing
|
||||
await engine.buildFromCandidates(["note2.md"]);
|
||||
expect((engine as any).index).not.toBeNull();
|
||||
expect((engine as any).indexedChunks.size).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("should handle index without destroy or clear methods", async () => {
|
||||
|
|
|
|||
|
|
@ -1,36 +1,32 @@
|
|||
import { logInfo, logWarn } from "@/logger";
|
||||
import { CHUNK_SIZE } from "@/constants";
|
||||
import FlexSearch from "flexsearch";
|
||||
import MiniSearch, { SearchResult } from "minisearch";
|
||||
import { App, TFile, getAllTags } from "obsidian";
|
||||
import { ChunkManager } from "../chunks";
|
||||
import { NoteDoc, NoteIdRank, SearchExplanation } from "../interfaces";
|
||||
import { NoteIdRank } from "../interfaces";
|
||||
import { MemoryManager } from "../utils/MemoryManager";
|
||||
|
||||
type ScoreAccumulator = {
|
||||
score: number;
|
||||
fieldMatches: Set<string>;
|
||||
queriesMatched: Set<string>;
|
||||
lexicalMatches: { field: string; query: string; weight: number }[];
|
||||
tagQueryMatches: Set<string>;
|
||||
tagFieldMatches: Set<string>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Full-text search engine using ephemeral FlexSearch index built per-query
|
||||
* Full-text search engine using ephemeral MiniSearch index built per-query.
|
||||
* Uses BM25+ scoring for proper multi-term relevance ranking.
|
||||
*/
|
||||
export class FullTextEngine {
|
||||
private index: any; // FlexSearch.Document
|
||||
private index: MiniSearch | null = null;
|
||||
private memoryManager: MemoryManager;
|
||||
private indexedChunks = new Set<string>();
|
||||
private chunkManager: ChunkManager;
|
||||
|
||||
// Configuration constants
|
||||
private static readonly MAX_CONTENT_SIZE = 10 * 1024 * 1024; // 10MB security limit
|
||||
private static readonly BATCH_SIZE = 10; // Chunk indexing batch size for UI yielding
|
||||
private static readonly CHUNK_MEMORY_PERCENTAGE = 0.35; // 35% of memory budget for chunks
|
||||
private static readonly MAX_ARRAY_ITEMS = 10; // Max items to process from arrays
|
||||
private static readonly MAX_EXTRACTION_DEPTH = 2; // Max recursion depth for property extraction
|
||||
|
||||
// Weighted query expansion constants
|
||||
// Salient terms (from original query) dominate ranking, expanded terms provide small boost
|
||||
private static readonly SALIENT_WEIGHT = 0.9; // 90% weight for original query terms
|
||||
private static readonly EXPANDED_WEIGHT = 0.1; // 10% weight for expanded terms
|
||||
|
||||
// Field weights for search scoring
|
||||
private static readonly FIELD_WEIGHTS = {
|
||||
title: 3,
|
||||
|
|
@ -43,44 +39,33 @@ export class FullTextEngine {
|
|||
body: 1,
|
||||
} as const;
|
||||
|
||||
private static readonly TAG_PRIMARY_FIELD_BOOST = 6;
|
||||
private static readonly TAG_SECONDARY_FIELD_BOOST = 3;
|
||||
private static readonly TAG_BASE_MATCH_BONUS = 2;
|
||||
private static readonly TAG_METADATA_MATCH_BOOST = 2.5;
|
||||
private static readonly TAG_DIVERSITY_BONUS = 0.4;
|
||||
private static readonly TAG_METADATA_SCORE_BONUS = 5;
|
||||
|
||||
constructor(
|
||||
private app: App,
|
||||
chunkManager?: ChunkManager
|
||||
) {
|
||||
this.memoryManager = new MemoryManager();
|
||||
this.chunkManager = chunkManager || new ChunkManager(app);
|
||||
// Defer index creation to avoid blocking UI on initialization
|
||||
this.index = null as any;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new FlexSearch index with multilingual tokenization
|
||||
* Updated for chunk-based indexing
|
||||
* Create a new MiniSearch index with multilingual tokenization and BM25 scoring
|
||||
*/
|
||||
private createIndex(): any {
|
||||
const Document = (FlexSearch as any).Document;
|
||||
const tokenizer = this.tokenizeMixed.bind(this);
|
||||
return new Document({
|
||||
encode: false,
|
||||
tokenize: tokenizer,
|
||||
cache: false,
|
||||
document: {
|
||||
id: "id",
|
||||
index: [
|
||||
{ field: "title", tokenize: tokenizer, weight: 3 }, // Note title
|
||||
{ field: "heading", tokenize: tokenizer, weight: 2.5 }, // Section heading
|
||||
{ field: "path", tokenize: tokenizer, weight: 2 }, // Path components
|
||||
{ field: "tags", tokenize: tokenizer, weight: 4 }, // Note tags and hierarchies
|
||||
{ field: "body", tokenize: tokenizer, weight: 1 }, // Chunk content
|
||||
],
|
||||
store: ["id", "notePath", "title", "heading", "chunkIndex"], // Store metadata only, not body
|
||||
private createIndex(): MiniSearch {
|
||||
return new MiniSearch({
|
||||
fields: ["title", "heading", "path", "tags", "body"],
|
||||
storeFields: ["id", "notePath", "title", "heading", "chunkIndex"],
|
||||
tokenize: this.tokenizeMixed.bind(this),
|
||||
searchOptions: {
|
||||
boost: {
|
||||
title: FullTextEngine.FIELD_WEIGHTS.title,
|
||||
heading: FullTextEngine.FIELD_WEIGHTS.heading,
|
||||
path: FullTextEngine.FIELD_WEIGHTS.path,
|
||||
tags: FullTextEngine.FIELD_WEIGHTS.tags,
|
||||
body: FullTextEngine.FIELD_WEIGHTS.body,
|
||||
},
|
||||
prefix: true,
|
||||
fuzzy: false, // Disable fuzzy for CJK compatibility
|
||||
combineWith: "OR",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
|
@ -167,13 +152,11 @@ export class FullTextEngine {
|
|||
this.memoryManager.reset();
|
||||
|
||||
// Create new index
|
||||
if (!this.index) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
const startTime = Date.now();
|
||||
this.index = this.createIndex();
|
||||
const createTime = Date.now() - startTime;
|
||||
logInfo(`FullTextEngine: FlexSearch index created in ${createTime}ms`);
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
const startTime = Date.now();
|
||||
this.index = this.createIndex();
|
||||
const createTime = Date.now() - startTime;
|
||||
logInfo(`FullTextEngine: MiniSearch index created in ${createTime}ms`);
|
||||
|
||||
// Convert note paths to chunks
|
||||
const chunkOptions = {
|
||||
|
|
@ -242,19 +225,18 @@ export class FullTextEngine {
|
|||
}
|
||||
}
|
||||
|
||||
// Add chunk to index (body indexed but not stored)
|
||||
// Add chunk to index
|
||||
// Include frontmatter properties in body for searchability
|
||||
const bodyWithProps = [chunk.content, ...noteMetadata.props].join(" ");
|
||||
|
||||
// MiniSearch expects string fields, so join tags array
|
||||
this.index.add({
|
||||
id: chunk.id,
|
||||
title: chunk.title,
|
||||
heading: chunk.heading,
|
||||
path: pathComponents,
|
||||
body: bodyWithProps, // Include frontmatter values in searchable content
|
||||
tags: noteMetadata.tags,
|
||||
links: noteMetadata.links,
|
||||
props: noteMetadata.props.join(" "), // Keep props as string for potential future use
|
||||
body: bodyWithProps,
|
||||
tags: noteMetadata.tags.join(" "), // MiniSearch expects strings
|
||||
notePath: chunk.notePath,
|
||||
chunkIndex: chunk.chunkIndex,
|
||||
});
|
||||
|
|
@ -275,57 +257,6 @@ export class FullTextEngine {
|
|||
return indexed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create NoteDoc from TFile using metadata cache
|
||||
* @param file - Obsidian TFile
|
||||
* @returns NoteDoc or null if file can't be processed
|
||||
*/
|
||||
private async createNoteDoc(file: TFile): Promise<NoteDoc | null> {
|
||||
try {
|
||||
const cache = this.app.metadataCache.getFileCache(file);
|
||||
let content = await this.app.vault.cachedRead(file);
|
||||
|
||||
// Security: Limit content size to prevent memory exhaustion
|
||||
if (content.length > FullTextEngine.MAX_CONTENT_SIZE) {
|
||||
logInfo(
|
||||
`FullText: File ${file.path} exceeds size limit (${content.length} bytes), truncating`
|
||||
);
|
||||
content = content.substring(0, FullTextEngine.MAX_CONTENT_SIZE);
|
||||
}
|
||||
|
||||
// Extract metadata
|
||||
const allTags = cache ? (getAllTags(cache) ?? []) : [];
|
||||
const headings = cache?.headings?.map((h) => h.heading) ?? [];
|
||||
const props = cache?.frontmatter ?? {};
|
||||
|
||||
// Get links (using full paths for accuracy)
|
||||
const outgoing = this.app.metadataCache.resolvedLinks[file.path] ?? {};
|
||||
const backlinks = this.app.metadataCache.getBacklinksForFile(file)?.data ?? {};
|
||||
|
||||
// Store full paths for link information
|
||||
const linksOut = Object.keys(outgoing);
|
||||
const linksIn = Object.keys(backlinks);
|
||||
|
||||
// Get title from frontmatter or filename
|
||||
const frontmatter = props as Record<string, any>;
|
||||
const title = frontmatter?.title || frontmatter?.name || file.basename;
|
||||
|
||||
return {
|
||||
id: file.path,
|
||||
title,
|
||||
headings,
|
||||
tags: allTags,
|
||||
props,
|
||||
linksOut,
|
||||
linksIn,
|
||||
body: content,
|
||||
};
|
||||
} catch (error) {
|
||||
logInfo(`FullText: Skipped ${file.path}: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract frontmatter property values for search indexing.
|
||||
* Only indexes primitive values (strings, numbers, booleans) and arrays of primitives.
|
||||
|
|
@ -466,303 +397,187 @@ export class FullTextEngine {
|
|||
}
|
||||
|
||||
/**
|
||||
* Search the ephemeral index with multiple query variants
|
||||
* Search the ephemeral index using MiniSearch's native BM25+ scoring with weighted query expansion.
|
||||
*
|
||||
* IMPORTANT: Expanded queries are used ONLY for recall (finding documents).
|
||||
* Only the original query AND salient terms contribute to ranking/scoring.
|
||||
* Uses a two-phase scoring approach:
|
||||
* 1. Primary score from salient terms (90% weight) - reflects original query intent
|
||||
* 2. Secondary score from expanded terms (10% weight) - provides small recall boost
|
||||
*
|
||||
* @param queries - Array of query strings (original + expanded for recall)
|
||||
* @param limit - Maximum results per query
|
||||
* @param salientTerms - Salient terms extracted from original query (used for scoring)
|
||||
* @param originalQuery - The original user query (used for scoring)
|
||||
* @returns Array of NoteIdRank results
|
||||
* @param queries - Array of query strings [original, ...expanded] (expanded queries used for secondary scoring)
|
||||
* @param limit - Maximum results to return
|
||||
* @param salientTerms - Salient terms extracted from original query (primary scoring)
|
||||
* @param originalQuery - The original user query (fallback for primary scoring)
|
||||
* @param expandedTerms - LLM-generated related terms (secondary scoring for recall boost)
|
||||
* @returns Array of NoteIdRank results sorted by weighted BM25 relevance
|
||||
*/
|
||||
search(
|
||||
queries: string[],
|
||||
limit: number = 30,
|
||||
salientTerms: string[] = [],
|
||||
originalQuery?: string
|
||||
originalQuery?: string,
|
||||
expandedTerms: string[] = []
|
||||
): NoteIdRank[] {
|
||||
// Return empty results if index hasn't been created yet
|
||||
if (!this.index) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// First, use ALL queries to find documents (recall phase)
|
||||
const candidateDocs = new Set<string>();
|
||||
// Build the primary scoring query from salient terms or original query
|
||||
const primaryQuery =
|
||||
salientTerms.length > 0 ? salientTerms.join(" ") : originalQuery || queries[0] || "";
|
||||
|
||||
for (const query of queries) {
|
||||
const normalizedQuery = this.normalizeQueryTerm(query);
|
||||
if (!normalizedQuery) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const results = this.index.search(normalizedQuery, { limit: limit * 3, enrich: true });
|
||||
if (Array.isArray(results)) {
|
||||
for (const fieldResult of results) {
|
||||
if (!fieldResult?.result) continue;
|
||||
for (const item of fieldResult.result) {
|
||||
const id = typeof item === "string" ? item : item?.id;
|
||||
if (id) candidateDocs.add(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logInfo(`FullText: Search failed for "${query}": ${error}`);
|
||||
}
|
||||
if (!primaryQuery.trim()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
logInfo(
|
||||
`FullText: Found ${candidateDocs.size} unique documents from all queries (recall phase)`
|
||||
);
|
||||
// Build secondary scoring query from expanded terms and expanded queries
|
||||
const expandedQueries = queries.slice(1); // Skip original query
|
||||
const secondaryQueryParts = [...expandedTerms, ...expandedQueries];
|
||||
const secondaryQuery = secondaryQueryParts.join(" ").trim();
|
||||
|
||||
// Now, score using ONLY original query AND salient terms (not expanded queries)
|
||||
const scoreMap = new Map<string, ScoreAccumulator>();
|
||||
|
||||
// Build list of scoring queries: ONLY salient terms (original query contains stopwords)
|
||||
// If no salient terms provided, fallback to original query for backward compatibility
|
||||
const scoringQueries: string[] =
|
||||
salientTerms.length > 0 ? [...salientTerms] : originalQuery ? [originalQuery] : [];
|
||||
|
||||
// Score documents that were found in recall phase
|
||||
if (scoringQueries.length > 0 && candidateDocs.size > 0) {
|
||||
for (const query of scoringQueries) {
|
||||
this.scoreWithQuery(query, candidateDocs, scoreMap, limit);
|
||||
}
|
||||
logInfo(
|
||||
`FullText: Scored with ${scoringQueries.length} terms (${salientTerms.length > 0 ? "salient terms" : "original query fallback"})`
|
||||
);
|
||||
}
|
||||
|
||||
// Convert score map to final results with bonuses applied
|
||||
return this.buildFinalResults(scoreMap, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Score documents using a salient term query
|
||||
* This ensures expanded queries and stopwords don't affect ranking, only recall
|
||||
*/
|
||||
private scoreWithQuery(
|
||||
query: string,
|
||||
candidateDocs: Set<string>,
|
||||
scoreMap: Map<string, ScoreAccumulator>,
|
||||
limit: number
|
||||
): void {
|
||||
const normalizedQuery = this.normalizeQueryTerm(query);
|
||||
if (!normalizedQuery) {
|
||||
return;
|
||||
}
|
||||
const searchOptions = {
|
||||
boost: {
|
||||
title: FullTextEngine.FIELD_WEIGHTS.title,
|
||||
heading: FullTextEngine.FIELD_WEIGHTS.heading,
|
||||
path: FullTextEngine.FIELD_WEIGHTS.path,
|
||||
tags: FullTextEngine.FIELD_WEIGHTS.tags,
|
||||
body: FullTextEngine.FIELD_WEIGHTS.body,
|
||||
},
|
||||
prefix: true,
|
||||
fuzzy: false,
|
||||
combineWith: "OR" as const,
|
||||
};
|
||||
|
||||
try {
|
||||
const results = this.index.search(normalizedQuery, { limit: limit * 3, enrich: true });
|
||||
if (!Array.isArray(results)) {
|
||||
return;
|
||||
// Primary search with salient terms
|
||||
const primaryResults = this.index.search(primaryQuery, searchOptions);
|
||||
|
||||
logInfo(
|
||||
`FullText: Primary search found ${primaryResults.length} results for "${primaryQuery.substring(0, 50)}..."`
|
||||
);
|
||||
|
||||
// If no secondary query, return primary results directly
|
||||
if (!secondaryQuery) {
|
||||
return primaryResults.slice(0, limit).map((result) => ({
|
||||
id: result.id as string,
|
||||
score: result.score,
|
||||
engine: "fulltext",
|
||||
explanation: {
|
||||
lexicalMatches: this.extractLexicalMatches(result),
|
||||
baseScore: result.score,
|
||||
finalScore: result.score,
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
const trimmedQuery = normalizedQuery;
|
||||
const isPhrase = trimmedQuery.includes(" ");
|
||||
const isTagQuery = trimmedQuery.startsWith("#");
|
||||
const normalizedTag = isTagQuery ? trimmedQuery : null;
|
||||
const queryWeight = isPhrase ? 1.5 : 1.0;
|
||||
// Secondary search with expanded terms for recall boost
|
||||
const secondaryResults = this.index.search(secondaryQuery, searchOptions);
|
||||
|
||||
const docMatchesForQuery = new Set<string>();
|
||||
for (const fieldResult of results) {
|
||||
if (!fieldResult?.result || !fieldResult?.field) {
|
||||
continue;
|
||||
}
|
||||
for (const item of fieldResult.result) {
|
||||
const id = typeof item === "string" ? item : item?.id;
|
||||
if (id && candidateDocs.has(id)) {
|
||||
docMatchesForQuery.add(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
logInfo(
|
||||
`FullText: Secondary search found ${secondaryResults.length} results for expanded terms`
|
||||
);
|
||||
|
||||
const docMatchRatio =
|
||||
docMatchesForQuery.size === 0 || candidateDocs.size === 0
|
||||
? 0
|
||||
: docMatchesForQuery.size / candidateDocs.size;
|
||||
const rarityWeight = isTagQuery ? 1 : 1 - Math.min(0.6, docMatchRatio * 0.6);
|
||||
const weightedQueryFactor = queryWeight * rarityWeight;
|
||||
// Combine scores with weighted combination
|
||||
const combined = this.combineWeightedScores(primaryResults, secondaryResults);
|
||||
|
||||
for (const fieldResult of results) {
|
||||
if (!fieldResult?.result || !fieldResult?.field) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fieldName = fieldResult.field;
|
||||
const fieldWeight = this.getFieldWeight(fieldName);
|
||||
|
||||
for (let idx = 0; idx < fieldResult.result.length; idx++) {
|
||||
const item = fieldResult.result[idx];
|
||||
const id = typeof item === "string" ? item : item?.id;
|
||||
|
||||
if (!id || !candidateDocs.has(id)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const positionScore = 1 / (idx + 1);
|
||||
let adjustedScore = positionScore * fieldWeight * weightedQueryFactor;
|
||||
|
||||
const existing: ScoreAccumulator = scoreMap.get(id) ?? {
|
||||
score: 0,
|
||||
fieldMatches: new Set<string>(),
|
||||
queriesMatched: new Set<string>(),
|
||||
lexicalMatches: [],
|
||||
tagQueryMatches: new Set<string>(),
|
||||
tagFieldMatches: new Set<string>(),
|
||||
};
|
||||
|
||||
const fieldMatches = new Set(existing.fieldMatches);
|
||||
fieldMatches.add(fieldName);
|
||||
|
||||
const queriesMatched = new Set(existing.queriesMatched);
|
||||
queriesMatched.add(trimmedQuery);
|
||||
|
||||
const explanationQuery =
|
||||
fieldName === "tags" || !trimmedQuery.startsWith("#")
|
||||
? trimmedQuery
|
||||
: trimmedQuery.replace(/^#/, "");
|
||||
|
||||
const lexicalMatches = [
|
||||
...existing.lexicalMatches,
|
||||
{
|
||||
field: fieldName,
|
||||
query: explanationQuery,
|
||||
weight: fieldWeight,
|
||||
},
|
||||
];
|
||||
|
||||
const tagQueryMatches = new Set(existing.tagQueryMatches);
|
||||
const tagFieldMatches = new Set(existing.tagFieldMatches);
|
||||
|
||||
if (isTagQuery && normalizedTag) {
|
||||
tagQueryMatches.add(normalizedTag);
|
||||
const matchedField = fieldName === "tags" ? "metadata" : fieldName;
|
||||
tagFieldMatches.add(matchedField);
|
||||
adjustedScore *=
|
||||
fieldName === "tags"
|
||||
? FullTextEngine.TAG_PRIMARY_FIELD_BOOST
|
||||
: FullTextEngine.TAG_SECONDARY_FIELD_BOOST;
|
||||
if (fieldName === "tags") {
|
||||
adjustedScore += FullTextEngine.TAG_METADATA_SCORE_BONUS;
|
||||
}
|
||||
}
|
||||
|
||||
const updated: ScoreAccumulator = {
|
||||
score: existing.score + adjustedScore,
|
||||
fieldMatches,
|
||||
queriesMatched,
|
||||
lexicalMatches,
|
||||
tagQueryMatches,
|
||||
tagFieldMatches,
|
||||
};
|
||||
|
||||
scoreMap.set(id, updated);
|
||||
}
|
||||
}
|
||||
return combined.slice(0, limit);
|
||||
} catch (error) {
|
||||
logInfo(`FullText: Scoring failed for query "${query}": ${error}`);
|
||||
logWarn(`FullText: Search failed for "${primaryQuery}": ${error}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build final results with bonuses applied
|
||||
* Combine primary and secondary search results with weighted scoring.
|
||||
* Primary results (salient terms) get 90% weight, secondary (expanded) get 10%.
|
||||
*
|
||||
* @param primaryResults - Results from salient term search
|
||||
* @param secondaryResults - Results from expanded term search
|
||||
* @returns Combined and sorted NoteIdRank results
|
||||
*/
|
||||
private buildFinalResults(scoreMap: Map<string, ScoreAccumulator>, limit: number): NoteIdRank[] {
|
||||
const finalResults: NoteIdRank[] = [];
|
||||
private combineWeightedScores(
|
||||
primaryResults: SearchResult[],
|
||||
secondaryResults: SearchResult[]
|
||||
): NoteIdRank[] {
|
||||
// Build a map of secondary scores for quick lookup
|
||||
const secondaryScoreMap = new Map<string, { score: number; result: SearchResult }>();
|
||||
for (const result of secondaryResults) {
|
||||
secondaryScoreMap.set(result.id as string, { score: result.score, result });
|
||||
}
|
||||
|
||||
for (const [id, data] of scoreMap.entries()) {
|
||||
// Calculate bonuses
|
||||
const multiFieldBonus = 1 + (data.fieldMatches.size - 1) * 0.2;
|
||||
const coverageBonus = 1 + Math.max(0, data.queriesMatched.size - 1) * 0.1;
|
||||
const tagBonus = this.calculateTagBonus(data.tagQueryMatches, data.tagFieldMatches);
|
||||
let finalScore = data.score * multiFieldBonus * coverageBonus * tagBonus;
|
||||
// Track all seen IDs to include secondary-only results
|
||||
const seenIds = new Set<string>();
|
||||
const combined: NoteIdRank[] = [];
|
||||
|
||||
// Apply phrase-in-path bonus
|
||||
finalScore = this.applyPhraseInPathBonus(id, data.queriesMatched, finalScore);
|
||||
// Process primary results with weighted combination
|
||||
for (const primary of primaryResults) {
|
||||
const id = primary.id as string;
|
||||
seenIds.add(id);
|
||||
|
||||
const explanation: SearchExplanation = {
|
||||
lexicalMatches: data.lexicalMatches,
|
||||
baseScore: data.score,
|
||||
finalScore,
|
||||
};
|
||||
const secondary = secondaryScoreMap.get(id);
|
||||
const primaryScore = primary.score * FullTextEngine.SALIENT_WEIGHT;
|
||||
const secondaryScore = secondary ? secondary.score * FullTextEngine.EXPANDED_WEIGHT : 0;
|
||||
const combinedScore = primaryScore + secondaryScore;
|
||||
|
||||
finalResults.push({
|
||||
combined.push({
|
||||
id,
|
||||
score: finalScore,
|
||||
score: combinedScore,
|
||||
engine: "fulltext",
|
||||
explanation,
|
||||
explanation: {
|
||||
lexicalMatches: this.extractLexicalMatches(primary),
|
||||
baseScore: primary.score,
|
||||
finalScore: combinedScore,
|
||||
expandedBoost: secondaryScore > 0 ? secondaryScore : undefined,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Sort and return top results
|
||||
finalResults.sort((a, b) => b.score - a.score);
|
||||
return finalResults.slice(0, limit);
|
||||
// Add secondary-only results (found via expansion but not salient terms)
|
||||
// These get only the 10% expanded weight
|
||||
for (const [id, { score, result }] of secondaryScoreMap) {
|
||||
if (!seenIds.has(id)) {
|
||||
const secondaryScore = score * FullTextEngine.EXPANDED_WEIGHT;
|
||||
combined.push({
|
||||
id,
|
||||
score: secondaryScore,
|
||||
engine: "fulltext",
|
||||
explanation: {
|
||||
lexicalMatches: this.extractLexicalMatches(result),
|
||||
baseScore: 0, // No primary match
|
||||
finalScore: secondaryScore,
|
||||
expandedBoost: secondaryScore,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by combined score descending
|
||||
combined.sort((a, b) => b.score - a.score);
|
||||
|
||||
return combined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes a final score multiplier when tag queries are matched.
|
||||
* Rewards documents that satisfy multiple tag queries and surface metadata tag hits preferentially.
|
||||
*
|
||||
* @param tagQueryMatches - Set of matched tag queries (lowercase, hash-prefixed)
|
||||
* @param tagFieldMatches - Fields that satisfied the tag query (metadata vs. content/path)
|
||||
* @returns Multiplicative boost applied to the final score (>= 1)
|
||||
* Extract lexical match information from MiniSearch result for explanation
|
||||
*/
|
||||
private calculateTagBonus(tagQueryMatches?: Set<string>, tagFieldMatches?: Set<string>): number {
|
||||
if (!tagQueryMatches || tagQueryMatches.size === 0) {
|
||||
return 1;
|
||||
}
|
||||
private extractLexicalMatches(
|
||||
result: SearchResult
|
||||
): { field: string; query: string; weight: number }[] {
|
||||
const matches: { field: string; query: string; weight: number }[] = [];
|
||||
|
||||
const baseBoost = 1 + tagQueryMatches.size * FullTextEngine.TAG_BASE_MATCH_BONUS;
|
||||
|
||||
const diversityCount = tagFieldMatches ? tagFieldMatches.size : 0;
|
||||
const diversityBoost =
|
||||
diversityCount > 1 ? 1 + (diversityCount - 1) * FullTextEngine.TAG_DIVERSITY_BONUS : 1;
|
||||
|
||||
const metadataBoost = tagFieldMatches?.has("metadata")
|
||||
? FullTextEngine.TAG_METADATA_MATCH_BOOST
|
||||
: 1;
|
||||
|
||||
return baseBoost * diversityBoost * metadataBoost;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes a query term for case-insensitive matching.
|
||||
* Trims whitespace and lowercases content to align with tokenizer behavior.
|
||||
* @param query - Raw query string
|
||||
* @returns Lowercase trimmed query or null when empty
|
||||
*/
|
||||
private normalizeQueryTerm(query: string): string | null {
|
||||
if (!query) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const trimmed = query.trim();
|
||||
if (trimmed.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return trimmed.toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply bonus if phrase query matches in path
|
||||
*/
|
||||
private applyPhraseInPathBonus(id: string, queriesMatched: Set<string>, score: number): number {
|
||||
const pathIndexString = id.replace(/\.md$/, "").split("/").join(" ").toLowerCase();
|
||||
|
||||
for (const q of queriesMatched) {
|
||||
if (q.includes(" ")) {
|
||||
const ql = q.toLowerCase();
|
||||
if (pathIndexString.includes(ql)) {
|
||||
return score * 1.5;
|
||||
if (result.match) {
|
||||
for (const [field, terms] of Object.entries(result.match)) {
|
||||
for (const term of terms) {
|
||||
matches.push({
|
||||
field,
|
||||
query: term,
|
||||
weight: this.getFieldWeight(field),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return score;
|
||||
return matches;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -779,31 +594,9 @@ export class FullTextEngine {
|
|||
*/
|
||||
clear(): void {
|
||||
try {
|
||||
// Simple: destroy index if it exists
|
||||
if (this.index) {
|
||||
try {
|
||||
// Ultra-defensive cleanup: handle all possible index states
|
||||
const indexValue = this.index;
|
||||
|
||||
if (indexValue != null && typeof indexValue === "object") {
|
||||
try {
|
||||
// Check for methods in prototype chain (not just own properties)
|
||||
if ("destroy" in indexValue && typeof indexValue.destroy === "function") {
|
||||
indexValue.destroy();
|
||||
} else if ("clear" in indexValue && typeof indexValue.clear === "function") {
|
||||
indexValue.clear();
|
||||
}
|
||||
} catch (methodError) {
|
||||
// Even method calls can fail, so handle that too
|
||||
logWarn(`FullTextEngine: Index method call error: ${methodError}`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Log index cleanup error but continue with state reset
|
||||
logWarn(`FullTextEngine: Index cleanup error (type: ${typeof this.index}): ${error}`);
|
||||
}
|
||||
this.index = null;
|
||||
}
|
||||
// MiniSearch doesn't have explicit cleanup methods
|
||||
// Just nullify the reference and let GC handle it
|
||||
this.index = null;
|
||||
// Clear collections
|
||||
this.indexedChunks.clear();
|
||||
this.memoryManager.reset();
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ export interface SearchExplanation {
|
|||
connections: number;
|
||||
boostFactor: number;
|
||||
};
|
||||
expandedBoost?: number; // score contribution from expanded terms (10% weight)
|
||||
baseScore: number; // score before boosts
|
||||
finalScore: number; // score after all adjustments
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue