2023-03-31 00:15:32 +00:00
|
|
|
{
|
2023-04-09 06:30:35 +00:00
|
|
|
"name": "obsidian-copilot",
|
2025-08-31 05:59:37 +00:00
|
|
|
"version": "3.0.2",
|
2025-05-31 03:06:25 +00:00
|
|
|
"description": "Your AI Copilot: Chat with Your Second Brain, Learn Faster, Work Smarter.",
|
2023-03-31 00:30:47 +00:00
|
|
|
"main": "main.js",
|
|
|
|
|
"scripts": {
|
2024-12-28 18:41:08 +00:00
|
|
|
"dev": "npm-run-all --parallel dev:*",
|
|
|
|
|
"dev:tailwind": "npx tailwindcss -i src/styles/tailwind.css -o styles.css --watch",
|
|
|
|
|
"dev:esbuild": "node esbuild.config.mjs",
|
|
|
|
|
"build": "npm run build:tailwind && npm run build:esbuild || exit 1",
|
|
|
|
|
"build:tailwind": "npx tailwindcss -i src/styles/tailwind.css -o styles.css --minify",
|
|
|
|
|
"build:esbuild": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production",
|
2024-07-22 20:22:04 +00:00
|
|
|
"lint": "eslint . --ext .js,.jsx,.ts,.tsx",
|
|
|
|
|
"lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix",
|
2024-08-21 22:16:22 +00:00
|
|
|
"format": "prettier --write 'src/**/*.{js,ts,tsx,md}'",
|
|
|
|
|
"format:check": "prettier --check 'src/**/*.{js,ts,tsx,md}'",
|
2023-04-09 05:57:45 +00:00
|
|
|
"version": "node version-bump.mjs && git add manifest.json versions.json",
|
2025-05-31 03:06:25 +00:00
|
|
|
"test": "jest --testPathIgnorePatterns=src/integration_tests/",
|
|
|
|
|
"test:integration": "jest src/integration_tests/",
|
2024-08-21 04:53:09 +00:00
|
|
|
"prepare": "husky"
|
2023-03-31 00:30:47 +00:00
|
|
|
},
|
|
|
|
|
"keywords": [],
|
2024-08-21 04:53:09 +00:00
|
|
|
"author": "Logan Yang",
|
|
|
|
|
"husky": {
|
|
|
|
|
"hooks": {
|
|
|
|
|
"pre-commit": "lint-staged"
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
"lint-staged": {
|
|
|
|
|
"*.{js,jsx,ts,tsx,json,css,md}": [
|
|
|
|
|
"prettier --write",
|
|
|
|
|
"git add"
|
|
|
|
|
]
|
|
|
|
|
},
|
2023-04-22 03:40:53 +00:00
|
|
|
"license": "AGPL-3.0",
|
2023-03-31 00:30:47 +00:00
|
|
|
"devDependencies": {
|
2025-04-18 06:19:35 +00:00
|
|
|
"@langchain/ollama": "^0.2.0",
|
2024-12-30 21:55:30 +00:00
|
|
|
"@tailwindcss/container-queries": "^0.1.1",
|
2023-04-09 05:57:45 +00:00
|
|
|
"@testing-library/jest-dom": "^5.16.5",
|
|
|
|
|
"@testing-library/react": "^14.0.0",
|
2023-05-28 22:28:28 +00:00
|
|
|
"@types/crypto-js": "^4.1.1",
|
2025-05-31 03:06:25 +00:00
|
|
|
"@types/diff": "^7.0.1",
|
2023-04-18 03:58:13 +00:00
|
|
|
"@types/events": "^3.0.0",
|
2024-01-28 06:51:54 +00:00
|
|
|
"@types/jest": "^29.5.11",
|
2023-08-02 07:34:33 +00:00
|
|
|
"@types/koa": "^2.13.7",
|
|
|
|
|
"@types/koa__cors": "^4.0.0",
|
2025-06-19 05:03:25 +00:00
|
|
|
"@types/lodash.debounce": "^4.0.9",
|
2024-11-22 01:36:52 +00:00
|
|
|
"@types/luxon": "^3.4.2",
|
2023-03-31 00:30:47 +00:00
|
|
|
"@types/node": "^16.11.6",
|
2025-05-31 03:06:25 +00:00
|
|
|
"@types/node-fetch": "^2.6.12",
|
2023-04-04 02:17:23 +00:00
|
|
|
"@types/react": "^18.0.33",
|
|
|
|
|
"@types/react-dom": "^18.0.11",
|
2023-04-05 20:14:26 +00:00
|
|
|
"@types/react-syntax-highlighter": "^15.5.6",
|
2025-01-07 06:45:12 +00:00
|
|
|
"@typescript-eslint/eslint-plugin": "^8.19.1",
|
|
|
|
|
"@typescript-eslint/parser": "^8.19.1",
|
2023-03-31 00:30:47 +00:00
|
|
|
"builtin-modules": "3.3.0",
|
2024-02-22 05:47:55 +00:00
|
|
|
"electron": "^27.3.2",
|
2025-02-20 19:17:36 +00:00
|
|
|
"esbuild": "^0.25.0",
|
2024-07-22 20:22:04 +00:00
|
|
|
"eslint": "^8.57.0",
|
2024-08-23 00:15:03 +00:00
|
|
|
"eslint-plugin-json": "^4.0.1",
|
2025-01-07 06:45:12 +00:00
|
|
|
"eslint-plugin-react": "^7.37.3",
|
|
|
|
|
"eslint-plugin-react-hooks": "^5.1.0",
|
2025-06-01 05:30:54 +00:00
|
|
|
"eslint-plugin-tailwindcss": "^3.18.0",
|
2024-08-21 04:53:09 +00:00
|
|
|
"husky": "^9.1.5",
|
2024-01-28 06:51:54 +00:00
|
|
|
"jest": "^29.7.0",
|
2023-04-09 05:57:45 +00:00
|
|
|
"jest-environment-jsdom": "^29.5.0",
|
2024-08-21 04:53:09 +00:00
|
|
|
"lint-staged": "^15.2.9",
|
2025-05-31 03:06:25 +00:00
|
|
|
"node-fetch": "^2.7.0",
|
2024-12-28 18:41:08 +00:00
|
|
|
"npm-run-all": "^4.1.5",
|
2024-08-25 06:29:44 +00:00
|
|
|
"obsidian": "^1.2.5",
|
2024-07-22 20:22:04 +00:00
|
|
|
"prettier": "^3.3.3",
|
2024-12-28 18:41:08 +00:00
|
|
|
"tailwindcss": "^3.4.15",
|
2023-04-09 05:57:45 +00:00
|
|
|
"ts-jest": "^29.1.0",
|
2023-03-31 00:30:47 +00:00
|
|
|
"tslib": "2.4.0",
|
2024-12-28 18:41:28 +00:00
|
|
|
"typescript": "^5.7.2",
|
2024-01-28 06:51:54 +00:00
|
|
|
"web-streams-polyfill": "^3.3.2"
|
2023-04-02 20:41:20 +00:00
|
|
|
},
|
|
|
|
|
"dependencies": {
|
2025-02-12 21:11:09 +00:00
|
|
|
"@dnd-kit/core": "^6.3.1",
|
|
|
|
|
"@dnd-kit/sortable": "^10.0.0",
|
|
|
|
|
"@dnd-kit/utilities": "^3.2.2",
|
2025-01-26 20:53:21 +00:00
|
|
|
"@google/generative-ai": "^0.21.0",
|
2024-01-11 06:16:19 +00:00
|
|
|
"@huggingface/inference": "^2.6.4",
|
|
|
|
|
"@koa/cors": "^5.0.0",
|
2025-05-31 03:06:25 +00:00
|
|
|
"@langchain/anthropic": "^0.3.20",
|
2024-09-30 23:03:32 +00:00
|
|
|
"@langchain/cohere": "^0.3.0",
|
2025-01-04 03:35:46 +00:00
|
|
|
"@langchain/community": "^0.3.22",
|
2025-04-20 22:52:42 +00:00
|
|
|
"@langchain/core": "^0.3.45",
|
2025-03-27 21:04:33 +00:00
|
|
|
"@langchain/deepseek": "^0.0.1",
|
2025-01-26 20:53:21 +00:00
|
|
|
"@langchain/google-genai": "^0.1.6",
|
2024-09-30 23:03:32 +00:00
|
|
|
"@langchain/groq": "^0.1.2",
|
2025-02-12 20:37:39 +00:00
|
|
|
"@langchain/mistralai": "^0.2.0",
|
2025-08-08 16:30:46 +00:00
|
|
|
"@langchain/openai": "^0.6.6",
|
2025-04-20 22:52:42 +00:00
|
|
|
"@langchain/xai": "^0.0.2",
|
2024-10-05 18:13:34 +00:00
|
|
|
"@orama/orama": "^3.0.0-rc-2",
|
2025-01-14 06:32:20 +00:00
|
|
|
"@radix-ui/react-checkbox": "^1.1.3",
|
2024-12-30 21:55:30 +00:00
|
|
|
"@radix-ui/react-collapsible": "^1.1.2",
|
2025-01-14 06:32:20 +00:00
|
|
|
"@radix-ui/react-dialog": "^1.1.3",
|
2025-01-22 06:03:22 +00:00
|
|
|
"@radix-ui/react-dropdown-menu": "^2.1.4",
|
2025-01-14 06:32:20 +00:00
|
|
|
"@radix-ui/react-label": "^2.1.0",
|
2025-01-22 06:03:22 +00:00
|
|
|
"@radix-ui/react-popover": "^1.1.4",
|
2025-08-08 16:59:00 +00:00
|
|
|
"@radix-ui/react-progress": "^1.1.7",
|
2025-06-24 19:35:48 +00:00
|
|
|
"@radix-ui/react-scroll-area": "^1.2.9",
|
2025-01-14 06:32:20 +00:00
|
|
|
"@radix-ui/react-select": "^2.1.2",
|
2025-06-24 19:35:48 +00:00
|
|
|
"@radix-ui/react-separator": "^1.1.7",
|
2025-08-16 04:32:29 +00:00
|
|
|
"@radix-ui/react-slider": "^1.3.5",
|
2025-06-24 19:35:48 +00:00
|
|
|
"@radix-ui/react-slot": "^1.2.3",
|
2025-01-14 06:32:20 +00:00
|
|
|
"@radix-ui/react-switch": "^1.1.1",
|
2025-05-31 03:06:25 +00:00
|
|
|
"@radix-ui/react-tabs": "^1.1.3",
|
2025-06-24 19:35:48 +00:00
|
|
|
"@radix-ui/react-tooltip": "^1.2.7",
|
2023-04-05 20:14:26 +00:00
|
|
|
"@tabler/icons-react": "^2.14.0",
|
2025-03-05 06:30:56 +00:00
|
|
|
"async-mutex": "^0.5.0",
|
2023-04-04 02:17:23 +00:00
|
|
|
"axios": "^1.3.4",
|
2025-03-03 03:04:05 +00:00
|
|
|
"buffer": "^6.0.3",
|
2024-11-22 01:36:52 +00:00
|
|
|
"chrono-node": "^2.7.7",
|
2024-12-28 18:41:08 +00:00
|
|
|
"class-variance-authority": "^0.7.1",
|
|
|
|
|
"clsx": "^2.1.1",
|
2025-05-31 03:06:25 +00:00
|
|
|
"codemirror-companion-extension": "^0.0.11",
|
2024-09-08 21:50:38 +00:00
|
|
|
"cohere-ai": "^7.13.0",
|
2023-05-28 22:28:28 +00:00
|
|
|
"crypto-js": "^4.1.1",
|
2025-05-31 03:06:25 +00:00
|
|
|
"diff": "^7.0.0",
|
2023-04-04 20:09:45 +00:00
|
|
|
"esbuild-plugin-svg": "^0.1.0",
|
2023-04-05 16:14:01 +00:00
|
|
|
"eventsource-parser": "^1.0.0",
|
Implement vault search v3 (#1703)
* Add LexicalEngine and related interfaces for enhanced search functionality
- Implement LexicalEngine class utilizing FlexSearch for efficient document indexing and searching.
- Create Hit, SearchResult, SearchOptions, and RetrieverEngine interfaces to standardize search operations.
- Update package.json and package-lock.json to include new dependencies: flexsearch and p-queue.
* Add QueryExpander class and tests for enhanced search query expansion
* Add README for Tiered Note-Level Lexical Retrieval with multilingual support and enhanced search pipeline
* Implement v3 tiered search with GrepScanner, FullTextEngine, and GraphExpander
- Add GrepScanner for fast substring search (L0)
- Implement FullTextEngine with ephemeral FlexSearch index (L1)
- Add GraphExpander for link-based candidate expansion
- Create MemoryManager with platform-aware limits
- Implement weighted RRF fusion for result combination
- Add SemanticReranker placeholder for future integration
- Include comprehensive tests for core components
- Simplify code based on review: use TextEncoder, extract methods, streamline RRF
- Add clear logging for debugging search pipeline
The architecture follows a tiered approach:
1. Grep scan for initial candidates
2. Graph expansion to increase recall
3. Full-text search on expanded set
4. Optional semantic reranking
5. RRF fusion to combine signals
🤖 Generated with [Claude Code](https://claude.ai/code)
Co-Authored-By: Claude <noreply@anthropic.com>
* Enhance tiered retrieval recall with both rewritten queries and the expanded salient terms
- Added detailed example of end-to-end query processing in README.md
- Updated TieredRetriever to log salient terms during query expansion
- Modified FullTextEngine to index link basenames for improved searchability
- Adjusted NoteDoc interface to clarify link handling and indexing
* Refactor QueryExpander configuration and caching logic for improved clarity and performance; enhance GraphExpander documentation and testing; remove unused fields from NoteDoc interface.
* Refactor search retrieval system to wire in TieredLexicalRetriever
- Replaced HybridRetriever with TieredLexicalRetriever in ChainManager, VaultQAChainRunner, and main.ts for improved multi-stage retrieval.
- Updated README.md to reflect new integration tasks and performance benchmarks.
- Introduced TieredLexicalRetriever class to handle on-demand indexing and retrieval.
- Enhanced GrepScanner to support inclusion/exclusion patterns for file indexing.
- Modified TieredRetriever to combine expanded salient terms and improved logging for final results.
- Removed legacy index management in favor of ephemeral indexing with TieredLexicalRetriever.
- Updated interfaces and utility functions to support new retrieval logic.
* feat: Implement first working TieredLexicalRetriever and refactor search scoring
- Added comprehensive tests for TieredLexicalRetriever, covering folder boosting and result combination.
- Refactored TieredLexicalRetriever to utilize SearchCore instead of TieredRetriever.
- Enhanced FullTextEngine with improved scoring mechanisms, including field weighting and multi-field match bonuses.
- Introduced FuzzyMatcher utility for fuzzy matching capabilities, including Levenshtein distance and variant generation.
- Updated GrepScanner to prioritize path matching for faster search results.
- Implemented normalized scoring in RRF for better ranking consistency.
* feat: Update vault search result display to show snippet of content instead of full text
* feat: Enhance FullTextEngine to index frontmatter property values and improve search scoring
* feat: Improve scoring and logging
* feat: Refactor TieredLexicalRetriever to support time-based queries and improve document retrieval logic
* Remove TODO.md from tracking and add to .gitignore
- TODO.md is now a local-only development session tracker
- Prevents accidental commits of work-in-progress task lists
- Each developer can maintain their own TODO.md without conflicts
* feat: Update dependencies and enhance search functionality
- Upgraded axios to version 1.11.0 and electron to version 27.3.11 for improved performance and security.
- Refactored QueryExpander to limit queries to the original for strict fallback tests.
- Enhanced SearchCore to rank grep hits by evidence quality before fusion, improving retrieval accuracy.
- Implemented a new method in SearchCore to rank grep hits based on evidence strength.
- Updated TieredLexicalRetriever tests to ensure proper integration with mocked SearchCore.
- Improved FuzzyMatcher to include plural/singular normalization for better fuzzy matching.
* feat: Add tool call marker encoding/decoding
- Introduced new utility functions for encoding and decoding tool call markers to ensure safe embedding in HTML comments.
- Updated `updateChatMemory` to handle encoded tool call markers, preserving their integrity during memory storage.
- Enhanced logging to decode tool marker results for better readability while maintaining encoded formats for storage.
- Added comprehensive tests for tool call marker functionality to ensure correct encoding and decoding behavior.
- Refactored related components to integrate the new encoding/decoding logic seamlessly.
* feat: Enhance FullTextEngine search to support low-weight terms and improve scoring
- Updated FullTextEngine to accept low-weight terms in the search method, allowing for better handling of salient terms.
- Implemented downweighting for boolean and numeric tokens in the properties field to reduce noise in search results.
- Added a new test case to validate the downweighting behavior for boolean and numeric queries in FullTextEngine.
- Improved logging for full-text search results to provide clearer insights into the search process.
* feat: Update GraphExpander to enhance search recall with adaptive hop logic
- Implemented guardrails in GraphExpander to adjust hop depth based on the number of grep hits: allows +1 hop for small sets (<5) and limits to 1 hop for large sets (≥50).
- Updated README.md to reflect new guardrail logic and scoring normalization in search results.
- Enhanced test cases to validate the new behavior of skipping co-citations for large result sets and allowing additional hops for smaller sets.
* feat: Filter out background tools during streaming to enhance user experience
- Added logic to determine and filter out background tools, preventing their names from appearing in the tool call display during streaming.
- Implemented a mechanism to include partial tool names only if they meet a specified length threshold, improving clarity in tool call presentations.
* feat: Enhance search tools to support time-based queries and display modified time
- Updated localSearchTool to ensure a healthy cap on max source chunks for time-based queries, improving recall.
- Modified ToolResultFormatter to display actual modified time for time-filtered results, enhancing clarity in search results.
- Adjusted output formatting to differentiate between recency and relevance scores based on query type.
* feat: Introduce semantic search capabilities with Memory Index support
- Added `enableSemanticSearchV3` setting to control the new semantic search feature.
- Implemented `MemoryIndexManager` for managing in-memory vector indexing with JSONL persistence.
- Enhanced `SearchCore` to utilize semantic retrieval based on the new memory index.
- Introduced command to build the semantic memory index, improving search accuracy and retrieval efficiency.
- Updated relevant components to integrate semantic search functionality seamlessly.
* feat: Add unit tests for MemoryIndexManager to validate functionality
- Introduced comprehensive tests for the MemoryIndexManager, covering scenarios such as loading from a file, building a vector store, and indexing vault contents.
- Implemented a mock embeddings API to facilitate testing without external dependencies.
- Added a utility function to reset the MemoryIndexManager state between tests, ensuring isolation and reliability of test outcomes.
* feat: Enhance MemoryIndexManager and related components for improved semantic indexing
- Introduced incremental indexing capabilities in MemoryIndexManager to update the JSONL index with new or modified files, enhancing efficiency.
- Updated commands to utilize the new incremental indexing method, providing users with real-time updates to the semantic memory index.
- Refactored existing indexing logic to support partitioned writes, improving performance and manageability of large datasets.
- Enhanced user notifications during indexing processes to provide better feedback on progress and status.
- Added a setting to enable semantic search, allowing users to blend semantic similarity into search results seamlessly.
* feat: Refine scoring display and enhance search result handling
- Updated SourcesModal to display relevance scores with four decimal places for consistency with SearchCore logs.
- Enhanced TieredLexicalRetriever to include a new `rerank_score` field for better score management and consistency across search results.
- Modified the combineResults method to ensure that title matches retain their original score while incorporating a fused score as `rerank_score`.
- Adjusted formatting in SearchTools to ensure both `score` and `rerank_score` reflect the same final score when present.
- Updated ToolResultFormatter to display scores with four decimal places, improving clarity in search result presentations.
* refactor: Simplify relevant notes retrieval by removing VectorStoreManager dependency
- Removed the use of VectorStoreManager in the relevant notes fetching logic, streamlining the process.
- Integrated MemoryIndexManager for improved handling of in-memory indexing and note retrieval.
- Enhanced error handling to ensure relevant notes are only fetched when the embedding index is available.
- Updated related functions to utilize the new memory index approach, improving performance and reliability.
* refactor: Remove VectorStoreManager dependency and streamline indexing logic
- Eliminated the VectorStoreManager from various components, transitioning to MemoryIndexManager for indexing and retrieval.
- Updated project and chain managers to remove unnecessary dependencies, enhancing code clarity and maintainability.
- Improved error handling and indexing conditions, particularly for mobile users, ensuring a more robust initialization process.
- Refactored settings components to utilize the new memory indexing approach, simplifying the user experience and reducing legacy code.
* chore: Mark legacy components as deprecated in preparation for v3 transition
- Annotated various files including DebugSearchModal, OramaSearchModal, chunkedStorage, dbOperations, hybridRetriever, and vectorStoreManager as deprecated, indicating they are obsolete in v3.
- Updated README.md to reflect the current implementation status and migration notes, emphasizing the removal of Orama-based modules and the transition to MemoryIndexManager for indexing and retrieval.
* feat: Implement file tracking and reindexing for modified files
- Introduced a new FileTrackingState interface to manage the last active file and its modification time.
- Enhanced the CopilotPlugin to opportunistically reindex the previous active file if it was modified while active, contingent on semantic search settings.
- Updated MemoryIndexManager to support reindexing of single modified files, improving efficiency in handling changes.
- Added unit tests for reindexSingleFileIfModified to ensure correct functionality and performance.
* feat: Add graph hops setting for enhanced search result expansion
- Introduced a new `graphHops` setting in `CopilotSettings` to control the number of hops for graph expansion during search, with a default value of 1 and a range of 1-3.
- Updated the `sanitizeSettings` function to validate the `graphHops` value.
- Enhanced the `SearchCore` and `TieredLexicalRetriever` classes to utilize the `graphHops` setting for improved search result relevance.
- Updated documentation in `README.md` to reflect the new feature and its security optimizations.
* refactor: Improve error handling and indexing logic in MemoryIndexManager and related components
- Replaced console error logging with structured logging using logError and logWarn for better error tracking.
- Enhanced the refresh and reindexing functions to ensure proper user notifications and error handling.
- Updated the logic for managing indexed files, including handling exclusions and ensuring accurate reporting of indexed and unindexed files.
- Implemented rate limiting in the MemoryIndexManager to optimize embedding requests and prevent overloading the service.
- Refactored chunk processing to improve efficiency and clarity in the indexing workflow.
* chore: Update README.md to reflect final session completion and key fixes
- Expanded the final session summary with a date and detailed list of completed features and fixes, including UI enhancements and improved logging practices.
- Clarified the status of deferred features and provided migration notes for better user guidance.
- Ensured documentation aligns with the latest implementation changes and optimizations.
* chore: Update memory limits in README.md and MemoryManager.ts for improved performance
- Adjusted memory limits for mobile and desktop platforms in both README.md and MemoryManager.ts to reflect increased capacity (20MB mobile, 100MB desktop).
- Ensured documentation aligns with the latest implementation changes for better clarity on resource management.
* feat: Integrate HyDE document generation into SearchCore for enhanced semantic search
- Implemented a new method to generate hypothetical documents (HyDE) to improve semantic search capabilities.
- Added timeout handling for HyDE generation to ensure graceful error management.
- Updated SearchCore to utilize HyDE documents in search queries, enhancing the relevance of results when semantic search is enabled.
- Introduced unit tests to validate the integration and functionality of HyDE generation within the search process.
* refactor: Enhance MemoryIndexManager with public methods for better indexing management
- Introduced public methods in MemoryIndexManager to streamline access to indexed file paths, check if a file is indexed, retrieve embeddings, and clear the index.
- Updated related components to utilize these new methods, improving code clarity and reducing direct access to internal properties.
- Enhanced error handling and user notifications during memory index operations.
* feat: Implement indexing notification and progress management
- Introduced the IndexingNotificationManager to handle UI notifications during indexing operations, allowing users to pause or stop the process.
- Added the IndexingProgressTracker to track progress across multiple files, enhancing user feedback during indexing.
- Developed the IndexingPipeline to manage the chunking and processing of files into embeddings, improving the overall indexing workflow.
- Created the IndexPersistenceManager for managing the persistence of indexed data, ensuring efficient storage and retrieval of index records.
- Refactored MemoryIndexManager to utilize the new components, streamlining the indexing process and improving code organization.
---------
Co-authored-by: Claude <noreply@anthropic.com>
2025-08-11 23:58:04 +00:00
|
|
|
"flexsearch": "^0.8.205",
|
2024-12-01 06:36:23 +00:00
|
|
|
"jotai": "^2.10.3",
|
2023-08-02 07:34:33 +00:00
|
|
|
"koa": "^2.14.2",
|
|
|
|
|
"koa-proxies": "^0.12.3",
|
2024-09-30 23:03:32 +00:00
|
|
|
"langchain": "^0.3.2",
|
2025-06-19 05:03:25 +00:00
|
|
|
"lodash.debounce": "^4.0.8",
|
2024-11-27 20:15:58 +00:00
|
|
|
"lucide-react": "^0.462.0",
|
2024-11-22 01:36:52 +00:00
|
|
|
"luxon": "^3.5.0",
|
2023-04-05 20:14:26 +00:00
|
|
|
"next-i18next": "^13.2.2",
|
Implement vault search v3 (#1703)
* Add LexicalEngine and related interfaces for enhanced search functionality
- Implement LexicalEngine class utilizing FlexSearch for efficient document indexing and searching.
- Create Hit, SearchResult, SearchOptions, and RetrieverEngine interfaces to standardize search operations.
- Update package.json and package-lock.json to include new dependencies: flexsearch and p-queue.
* Add QueryExpander class and tests for enhanced search query expansion
* Add README for Tiered Note-Level Lexical Retrieval with multilingual support and enhanced search pipeline
* Implement v3 tiered search with GrepScanner, FullTextEngine, and GraphExpander
- Add GrepScanner for fast substring search (L0)
- Implement FullTextEngine with ephemeral FlexSearch index (L1)
- Add GraphExpander for link-based candidate expansion
- Create MemoryManager with platform-aware limits
- Implement weighted RRF fusion for result combination
- Add SemanticReranker placeholder for future integration
- Include comprehensive tests for core components
- Simplify code based on review: use TextEncoder, extract methods, streamline RRF
- Add clear logging for debugging search pipeline
The architecture follows a tiered approach:
1. Grep scan for initial candidates
2. Graph expansion to increase recall
3. Full-text search on expanded set
4. Optional semantic reranking
5. RRF fusion to combine signals
🤖 Generated with [Claude Code](https://claude.ai/code)
Co-Authored-By: Claude <noreply@anthropic.com>
* Enhance tiered retrieval recall with both rewritten queries and the expanded salient terms
- Added detailed example of end-to-end query processing in README.md
- Updated TieredRetriever to log salient terms during query expansion
- Modified FullTextEngine to index link basenames for improved searchability
- Adjusted NoteDoc interface to clarify link handling and indexing
* Refactor QueryExpander configuration and caching logic for improved clarity and performance; enhance GraphExpander documentation and testing; remove unused fields from NoteDoc interface.
* Refactor search retrieval system to wire in TieredLexicalRetriever
- Replaced HybridRetriever with TieredLexicalRetriever in ChainManager, VaultQAChainRunner, and main.ts for improved multi-stage retrieval.
- Updated README.md to reflect new integration tasks and performance benchmarks.
- Introduced TieredLexicalRetriever class to handle on-demand indexing and retrieval.
- Enhanced GrepScanner to support inclusion/exclusion patterns for file indexing.
- Modified TieredRetriever to combine expanded salient terms and improved logging for final results.
- Removed legacy index management in favor of ephemeral indexing with TieredLexicalRetriever.
- Updated interfaces and utility functions to support new retrieval logic.
* feat: Implement first working TieredLexicalRetriever and refactor search scoring
- Added comprehensive tests for TieredLexicalRetriever, covering folder boosting and result combination.
- Refactored TieredLexicalRetriever to utilize SearchCore instead of TieredRetriever.
- Enhanced FullTextEngine with improved scoring mechanisms, including field weighting and multi-field match bonuses.
- Introduced FuzzyMatcher utility for fuzzy matching capabilities, including Levenshtein distance and variant generation.
- Updated GrepScanner to prioritize path matching for faster search results.
- Implemented normalized scoring in RRF for better ranking consistency.
* feat: Update vault search result display to show snippet of content instead of full text
* feat: Enhance FullTextEngine to index frontmatter property values and improve search scoring
* feat: Improve scoring and logging
* feat: Refactor TieredLexicalRetriever to support time-based queries and improve document retrieval logic
* Remove TODO.md from tracking and add to .gitignore
- TODO.md is now a local-only development session tracker
- Prevents accidental commits of work-in-progress task lists
- Each developer can maintain their own TODO.md without conflicts
* feat: Update dependencies and enhance search functionality
- Upgraded axios to version 1.11.0 and electron to version 27.3.11 for improved performance and security.
- Refactored QueryExpander to limit queries to the original for strict fallback tests.
- Enhanced SearchCore to rank grep hits by evidence quality before fusion, improving retrieval accuracy.
- Implemented a new method in SearchCore to rank grep hits based on evidence strength.
- Updated TieredLexicalRetriever tests to ensure proper integration with mocked SearchCore.
- Improved FuzzyMatcher to include plural/singular normalization for better fuzzy matching.
* feat: Add tool call marker encoding/decoding
- Introduced new utility functions for encoding and decoding tool call markers to ensure safe embedding in HTML comments.
- Updated `updateChatMemory` to handle encoded tool call markers, preserving their integrity during memory storage.
- Enhanced logging to decode tool marker results for better readability while maintaining encoded formats for storage.
- Added comprehensive tests for tool call marker functionality to ensure correct encoding and decoding behavior.
- Refactored related components to integrate the new encoding/decoding logic seamlessly.
* feat: Enhance FullTextEngine search to support low-weight terms and improve scoring
- Updated FullTextEngine to accept low-weight terms in the search method, allowing for better handling of salient terms.
- Implemented downweighting for boolean and numeric tokens in the properties field to reduce noise in search results.
- Added a new test case to validate the downweighting behavior for boolean and numeric queries in FullTextEngine.
- Improved logging for full-text search results to provide clearer insights into the search process.
* feat: Update GraphExpander to enhance search recall with adaptive hop logic
- Implemented guardrails in GraphExpander to adjust hop depth based on the number of grep hits: allows +1 hop for small sets (<5) and limits to 1 hop for large sets (≥50).
- Updated README.md to reflect new guardrail logic and scoring normalization in search results.
- Enhanced test cases to validate the new behavior of skipping co-citations for large result sets and allowing additional hops for smaller sets.
* feat: Filter out background tools during streaming to enhance user experience
- Added logic to determine and filter out background tools, preventing their names from appearing in the tool call display during streaming.
- Implemented a mechanism to include partial tool names only if they meet a specified length threshold, improving clarity in tool call presentations.
* feat: Enhance search tools to support time-based queries and display modified time
- Updated localSearchTool to ensure a healthy cap on max source chunks for time-based queries, improving recall.
- Modified ToolResultFormatter to display actual modified time for time-filtered results, enhancing clarity in search results.
- Adjusted output formatting to differentiate between recency and relevance scores based on query type.
* feat: Introduce semantic search capabilities with Memory Index support
- Added `enableSemanticSearchV3` setting to control the new semantic search feature.
- Implemented `MemoryIndexManager` for managing in-memory vector indexing with JSONL persistence.
- Enhanced `SearchCore` to utilize semantic retrieval based on the new memory index.
- Introduced command to build the semantic memory index, improving search accuracy and retrieval efficiency.
- Updated relevant components to integrate semantic search functionality seamlessly.
* feat: Add unit tests for MemoryIndexManager to validate functionality
- Introduced comprehensive tests for the MemoryIndexManager, covering scenarios such as loading from a file, building a vector store, and indexing vault contents.
- Implemented a mock embeddings API to facilitate testing without external dependencies.
- Added a utility function to reset the MemoryIndexManager state between tests, ensuring isolation and reliability of test outcomes.
* feat: Enhance MemoryIndexManager and related components for improved semantic indexing
- Introduced incremental indexing capabilities in MemoryIndexManager to update the JSONL index with new or modified files, enhancing efficiency.
- Updated commands to utilize the new incremental indexing method, providing users with real-time updates to the semantic memory index.
- Refactored existing indexing logic to support partitioned writes, improving performance and manageability of large datasets.
- Enhanced user notifications during indexing processes to provide better feedback on progress and status.
- Added a setting to enable semantic search, allowing users to blend semantic similarity into search results seamlessly.
* feat: Refine scoring display and enhance search result handling
- Updated SourcesModal to display relevance scores with four decimal places for consistency with SearchCore logs.
- Enhanced TieredLexicalRetriever to include a new `rerank_score` field for better score management and consistency across search results.
- Modified the combineResults method to ensure that title matches retain their original score while incorporating a fused score as `rerank_score`.
- Adjusted formatting in SearchTools to ensure both `score` and `rerank_score` reflect the same final score when present.
- Updated ToolResultFormatter to display scores with four decimal places, improving clarity in search result presentations.
* refactor: Simplify relevant notes retrieval by removing VectorStoreManager dependency
- Removed the use of VectorStoreManager in the relevant notes fetching logic, streamlining the process.
- Integrated MemoryIndexManager for improved handling of in-memory indexing and note retrieval.
- Enhanced error handling to ensure relevant notes are only fetched when the embedding index is available.
- Updated related functions to utilize the new memory index approach, improving performance and reliability.
* refactor: Remove VectorStoreManager dependency and streamline indexing logic
- Eliminated the VectorStoreManager from various components, transitioning to MemoryIndexManager for indexing and retrieval.
- Updated project and chain managers to remove unnecessary dependencies, enhancing code clarity and maintainability.
- Improved error handling and indexing conditions, particularly for mobile users, ensuring a more robust initialization process.
- Refactored settings components to utilize the new memory indexing approach, simplifying the user experience and reducing legacy code.
* chore: Mark legacy components as deprecated in preparation for v3 transition
- Annotated various files including DebugSearchModal, OramaSearchModal, chunkedStorage, dbOperations, hybridRetriever, and vectorStoreManager as deprecated, indicating they are obsolete in v3.
- Updated README.md to reflect the current implementation status and migration notes, emphasizing the removal of Orama-based modules and the transition to MemoryIndexManager for indexing and retrieval.
* feat: Implement file tracking and reindexing for modified files
- Introduced a new FileTrackingState interface to manage the last active file and its modification time.
- Enhanced the CopilotPlugin to opportunistically reindex the previous active file if it was modified while active, contingent on semantic search settings.
- Updated MemoryIndexManager to support reindexing of single modified files, improving efficiency in handling changes.
- Added unit tests for reindexSingleFileIfModified to ensure correct functionality and performance.
* feat: Add graph hops setting for enhanced search result expansion
- Introduced a new `graphHops` setting in `CopilotSettings` to control the number of hops for graph expansion during search, with a default value of 1 and a range of 1-3.
- Updated the `sanitizeSettings` function to validate the `graphHops` value.
- Enhanced the `SearchCore` and `TieredLexicalRetriever` classes to utilize the `graphHops` setting for improved search result relevance.
- Updated documentation in `README.md` to reflect the new feature and its security optimizations.
* refactor: Improve error handling and indexing logic in MemoryIndexManager and related components
- Replaced console error logging with structured logging using logError and logWarn for better error tracking.
- Enhanced the refresh and reindexing functions to ensure proper user notifications and error handling.
- Updated the logic for managing indexed files, including handling exclusions and ensuring accurate reporting of indexed and unindexed files.
- Implemented rate limiting in the MemoryIndexManager to optimize embedding requests and prevent overloading the service.
- Refactored chunk processing to improve efficiency and clarity in the indexing workflow.
* chore: Update README.md to reflect final session completion and key fixes
- Expanded the final session summary with a date and detailed list of completed features and fixes, including UI enhancements and improved logging practices.
- Clarified the status of deferred features and provided migration notes for better user guidance.
- Ensured documentation aligns with the latest implementation changes and optimizations.
* chore: Update memory limits in README.md and MemoryManager.ts for improved performance
- Adjusted memory limits for mobile and desktop platforms in both README.md and MemoryManager.ts to reflect increased capacity (20MB mobile, 100MB desktop).
- Ensured documentation aligns with the latest implementation changes for better clarity on resource management.
* feat: Integrate HyDE document generation into SearchCore for enhanced semantic search
- Implemented a new method to generate hypothetical documents (HyDE) to improve semantic search capabilities.
- Added timeout handling for HyDE generation to ensure graceful error management.
- Updated SearchCore to utilize HyDE documents in search queries, enhancing the relevance of results when semantic search is enabled.
- Introduced unit tests to validate the integration and functionality of HyDE generation within the search process.
* refactor: Enhance MemoryIndexManager with public methods for better indexing management
- Introduced public methods in MemoryIndexManager to streamline access to indexed file paths, check if a file is indexed, retrieve embeddings, and clear the index.
- Updated related components to utilize these new methods, improving code clarity and reducing direct access to internal properties.
- Enhanced error handling and user notifications during memory index operations.
* feat: Implement indexing notification and progress management
- Introduced the IndexingNotificationManager to handle UI notifications during indexing operations, allowing users to pause or stop the process.
- Added the IndexingProgressTracker to track progress across multiple files, enhancing user feedback during indexing.
- Developed the IndexingPipeline to manage the chunking and processing of files into embeddings, improving the overall indexing workflow.
- Created the IndexPersistenceManager for managing the persistence of indexed data, ensuring efficient storage and retrieval of index records.
- Refactored MemoryIndexManager to utilize the new components, streamlining the indexing process and improving code organization.
---------
Co-authored-by: Claude <noreply@anthropic.com>
2025-08-11 23:58:04 +00:00
|
|
|
"p-queue": "^8.1.0",
|
2023-04-05 20:14:26 +00:00
|
|
|
"prop-types": "^15.8.1",
|
2023-04-04 02:17:23 +00:00
|
|
|
"react": "^18.2.0",
|
2023-04-05 19:13:34 +00:00
|
|
|
"react-dom": "^18.2.0",
|
2025-01-22 06:58:02 +00:00
|
|
|
"react-dropzone": "^14.3.5",
|
2024-08-25 06:29:44 +00:00
|
|
|
"react-markdown": "^9.0.1",
|
2025-06-24 19:35:48 +00:00
|
|
|
"react-resizable-panels": "^3.0.2",
|
2023-04-21 03:30:14 +00:00
|
|
|
"react-syntax-highlighter": "^15.5.0",
|
2024-12-28 18:41:08 +00:00
|
|
|
"sse": "github:mpetazzoni/sse.js",
|
|
|
|
|
"tailwind-merge": "^2.5.5",
|
2025-05-31 03:06:25 +00:00
|
|
|
"tailwindcss-animate": "^1.0.7",
|
|
|
|
|
"trie-search": "^2.2.0"
|
2023-03-31 00:30:47 +00:00
|
|
|
}
|
2025-06-07 21:18:24 +00:00
|
|
|
}
|