mirror of
https://github.com/logancyang/obsidian-copilot.git
synced 2026-07-22 07:50:24 +00:00
Move to Orama from PouchDB, remove long note QA (#702)
This commit is contained in:
Logan Yang
GitHub
parent
f6ac2430be
commit
97ea286bed
15 changed files with 445 additions and 720 deletions
|
|
@ -22,12 +22,6 @@ SPECIAL THANKS TO OUR TOP SPONSORS:
|
|||
|
||||
[Changelog](https://github.com/logancyang/obsidian-copilot/releases)
|
||||
|
||||
#### Announcement 🚨
|
||||
|
||||
We are migrating off of PouchDB for better Obsidian Sync and mobile support (starting from v2.6.3). Your existing custom prompts MUST be dumped to markdown using the command _"Copilot: Dump custom prompts to markdown files"_. After running it you should be able to use your Add/Edit/Apply/Delete custom prompts as usual.
|
||||
|
||||
Please make sure you run it, or you will lose all your old prompts when PouchDB is removed!
|
||||
|
||||
#### v2.6.0: MOBILE SUPPORT is here! 🎉🎉🎉
|
||||
|
||||
<a href="https://youtu.be/l-x_QwSNghE" target="_blank"><img src="./images/thumbnail-260.png" width="700" /></a>
|
||||
|
|
@ -67,7 +61,6 @@ What's more, with Ollama **local embeddings** and **local chat models**, this mo
|
|||
- No usage cap for GPT-4 like ChatGPT Plus.
|
||||
- One-click copying any message as markdown.
|
||||
- One-click saving the entire conversation as a note.
|
||||
- Use a super long note as context, and start a discussion around it by switching to "Long Note QA" in the Mode Selection menu.
|
||||
- **Chat with your whole vault** by selecting "Vault QA" mode. Ask questions and get **cited responses**!
|
||||
- All QA modes are powered by retrieval augmentation with a **local vector store**. No sending your data to a cloud-based vector search service!
|
||||
- Easy commands to **simplify, emojify, summarize, translate, change tone, fix grammar, rewrite into a tweet/thread, count tokens** and more.
|
||||
|
|
@ -134,7 +127,6 @@ Now you can see the chat icon in your leftside ribbon, clicking on it will open
|
|||
|
||||
- The chat history is not saved by default. Please use "**Save as Note**" to save it. The note will have a title `Chat-Year_Month_Day-Hour_Minute_Second`, you can change its name as needed.
|
||||
- "**New Chat**" clears all previous chat history. Again, please use "**Save as Note**" if you would like to save the chat.
|
||||
- "**Use Long Note as Context**" creates a local vector index for the active long note so that you can chat with note longer than the model's context window! To start the QA, please switch from "Chat" to "QA" in the Mode Selection dropdown.
|
||||
- You can set a very long context in the setting "**Conversation turns in context**" if needed.
|
||||
|
||||
#### 📣 Again, please always be mindful of the API cost if you use GPT-4 with a long context!
|
||||
|
|
|
|||
202
package-lock.json
generated
202
package-lock.json
generated
|
|
@ -18,8 +18,8 @@
|
|||
"@langchain/core": "^0.3.3",
|
||||
"@langchain/google-genai": "^0.1.0",
|
||||
"@langchain/groq": "^0.1.2",
|
||||
"@orama/orama": "^3.0.0-rc-2",
|
||||
"@tabler/icons-react": "^2.14.0",
|
||||
"@types/pouchdb": "^6.4.0",
|
||||
"axios": "^1.3.4",
|
||||
"cohere-ai": "^7.13.0",
|
||||
"crypto-js": "^4.1.1",
|
||||
|
|
@ -30,7 +30,6 @@
|
|||
"koa-proxies": "^0.12.3",
|
||||
"langchain": "^0.3.2",
|
||||
"next-i18next": "^13.2.2",
|
||||
"pouchdb-browser": "^9.0.0",
|
||||
"prop-types": "^15.8.1",
|
||||
"react": "^18.2.0",
|
||||
"react-dom": "^18.2.0",
|
||||
|
|
@ -3987,6 +3986,14 @@
|
|||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/@orama/orama": {
|
||||
"version": "3.0.0-rc-2",
|
||||
"resolved": "https://registry.npmjs.org/@orama/orama/-/orama-3.0.0-rc-2.tgz",
|
||||
"integrity": "sha512-Yklb1Yj0SoZrPlPV0dCDQKFPGbSIEm9788IVfR4sbv5SiCc47KBKEJJdYG/J8dWGE9pIzYLmUY78fLSBCWXa5g==",
|
||||
"engines": {
|
||||
"node": ">= 16.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@sinclair/typebox": {
|
||||
"version": "0.27.8",
|
||||
"resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz",
|
||||
|
|
@ -5521,169 +5528,6 @@
|
|||
"form-data": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pouchdb": {
|
||||
"version": "6.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/pouchdb/-/pouchdb-6.4.0.tgz",
|
||||
"integrity": "sha512-eGCpX+NXhd5VLJuJMzwe3L79fa9+IDTrAG3CPaf4s/31PD56hOrhDJTSmRELSXuiqXr6+OHzzP0PldSaWsFt7w==",
|
||||
"dependencies": {
|
||||
"@types/pouchdb-adapter-cordova-sqlite": "*",
|
||||
"@types/pouchdb-adapter-fruitdown": "*",
|
||||
"@types/pouchdb-adapter-http": "*",
|
||||
"@types/pouchdb-adapter-idb": "*",
|
||||
"@types/pouchdb-adapter-leveldb": "*",
|
||||
"@types/pouchdb-adapter-localstorage": "*",
|
||||
"@types/pouchdb-adapter-memory": "*",
|
||||
"@types/pouchdb-adapter-node-websql": "*",
|
||||
"@types/pouchdb-adapter-websql": "*",
|
||||
"@types/pouchdb-browser": "*",
|
||||
"@types/pouchdb-core": "*",
|
||||
"@types/pouchdb-http": "*",
|
||||
"@types/pouchdb-mapreduce": "*",
|
||||
"@types/pouchdb-node": "*",
|
||||
"@types/pouchdb-replication": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pouchdb-adapter-cordova-sqlite": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/pouchdb-adapter-cordova-sqlite/-/pouchdb-adapter-cordova-sqlite-1.0.1.tgz",
|
||||
"integrity": "sha512-nqlXpW1ho3KBg1mUQvZgH2755y3z/rw4UA7ZJCPMRTHofxGMY8izRVw5rHBL4/7P615or0J2udpRYxgkT3D02g==",
|
||||
"dependencies": {
|
||||
"@types/pouchdb-core": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pouchdb-adapter-fruitdown": {
|
||||
"version": "6.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/pouchdb-adapter-fruitdown/-/pouchdb-adapter-fruitdown-6.1.3.tgz",
|
||||
"integrity": "sha512-Wz1Z1JLOW1hgmFQjqnSkmyyfH7by/iWb4abKn684WMvQfmxx6BxKJpJ4+eulkVPQzzgMMSgU1MpnQOm9FgRkbw==",
|
||||
"dependencies": {
|
||||
"@types/pouchdb-core": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pouchdb-adapter-http": {
|
||||
"version": "6.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/pouchdb-adapter-http/-/pouchdb-adapter-http-6.1.3.tgz",
|
||||
"integrity": "sha512-9Z4TLbF/KJWy/D2sWRPBA+RNU0odQimfdvlDX+EY7rGcd3aVoH8qjD/X0Xcd/0dfBH5pKrNIMFFQgW/TylRCmA==",
|
||||
"dependencies": {
|
||||
"@types/pouchdb-core": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pouchdb-adapter-idb": {
|
||||
"version": "6.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/pouchdb-adapter-idb/-/pouchdb-adapter-idb-6.1.4.tgz",
|
||||
"integrity": "sha512-KIAXbkF4uYUz0ZwfNEFLtEkK44mEWopAsD76UhucH92XnJloBysav+TjI4FFfYQyTjoW3S1s6V+Z14CUJZ0F6w==",
|
||||
"dependencies": {
|
||||
"@types/pouchdb-core": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pouchdb-adapter-leveldb": {
|
||||
"version": "6.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/pouchdb-adapter-leveldb/-/pouchdb-adapter-leveldb-6.1.3.tgz",
|
||||
"integrity": "sha512-ex8NFqQGFwEpFi7AaZ5YofmuemfZNsL3nTFZBUCAKYMBkazQij1pe2ILLStSvJr0XS0qxgXjCEW19T5Wqiiskg==",
|
||||
"dependencies": {
|
||||
"@types/pouchdb-core": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pouchdb-adapter-localstorage": {
|
||||
"version": "6.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/pouchdb-adapter-localstorage/-/pouchdb-adapter-localstorage-6.1.3.tgz",
|
||||
"integrity": "sha512-oor040tye1KKiGLWYtIy7rRT7C2yoyX3Tf6elEJRpjOA7Ja/H8lKc4LaSh9ATbptIcES6MRqZDxtp7ly9hsW3Q==",
|
||||
"dependencies": {
|
||||
"@types/pouchdb-core": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pouchdb-adapter-memory": {
|
||||
"version": "6.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/pouchdb-adapter-memory/-/pouchdb-adapter-memory-6.1.3.tgz",
|
||||
"integrity": "sha512-gVbsIMzDzgZYThFVT4eVNsmuZwVm/4jDxP1sjlgc3qtDIxbtBhGgyNfcskwwz9Zu5Lv1avkDsIWvcxQhnvRlHg==",
|
||||
"dependencies": {
|
||||
"@types/pouchdb-core": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pouchdb-adapter-node-websql": {
|
||||
"version": "6.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/pouchdb-adapter-node-websql/-/pouchdb-adapter-node-websql-6.1.3.tgz",
|
||||
"integrity": "sha512-F/P+os6Jsa7CgHtH64+Z0HfwIcj0hIRB5z8gNhF7L7dxPWoAfkopK5H2gydrP3sQrlGyN4WInF+UJW/Zu1+FKg==",
|
||||
"dependencies": {
|
||||
"@types/pouchdb-adapter-websql": "*",
|
||||
"@types/pouchdb-core": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pouchdb-adapter-websql": {
|
||||
"version": "6.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/pouchdb-adapter-websql/-/pouchdb-adapter-websql-6.1.4.tgz",
|
||||
"integrity": "sha512-zMJQCtXC40hBsIDRn0GhmpeGMK0f9l/OGWfLguvczROzxxcOD7REI+e6SEmX7gJKw5JuMvlfuHzkQwjmvSJbtg==",
|
||||
"dependencies": {
|
||||
"@types/pouchdb-core": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pouchdb-browser": {
|
||||
"version": "6.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/pouchdb-browser/-/pouchdb-browser-6.1.3.tgz",
|
||||
"integrity": "sha512-EdYowrWxW9SWBMX/rux2eq7dbHi5Zeyzz+FF/IAsgQKnUxgeCO5VO2j4zTzos0SDyJvAQU+EYRc11r7xGn5tvA==",
|
||||
"dependencies": {
|
||||
"@types/pouchdb-adapter-http": "*",
|
||||
"@types/pouchdb-adapter-idb": "*",
|
||||
"@types/pouchdb-adapter-websql": "*",
|
||||
"@types/pouchdb-core": "*",
|
||||
"@types/pouchdb-mapreduce": "*",
|
||||
"@types/pouchdb-replication": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pouchdb-core": {
|
||||
"version": "7.0.11",
|
||||
"resolved": "https://registry.npmjs.org/@types/pouchdb-core/-/pouchdb-core-7.0.11.tgz",
|
||||
"integrity": "sha512-KTKj0Taf8pLTj8eW3qBtCd1Fh+/yhEyVo6/1czTN46MnseobdmnqgXtzaKgHMbuLouh+SHK1vM++aMPaG1qTTA==",
|
||||
"dependencies": {
|
||||
"@types/debug": "*",
|
||||
"@types/pouchdb-find": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pouchdb-find": {
|
||||
"version": "7.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/pouchdb-find/-/pouchdb-find-7.3.0.tgz",
|
||||
"integrity": "sha512-sFPli5tBjGX9UfXioik1jUzPdcN84eV82n0lmEFuoPepWqkLjQcyri0eOa++HYOaNPyMDhKFBqEALEZivK2dRg==",
|
||||
"dependencies": {
|
||||
"@types/pouchdb-core": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pouchdb-http": {
|
||||
"version": "6.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/pouchdb-http/-/pouchdb-http-6.1.3.tgz",
|
||||
"integrity": "sha512-0e9E5SqNOyPl/3FnEIbENssB4FlJsNYuOy131nxrZk36S+y1R/6qO7ZVRypWpGTqBWSuVd7gCsq2UDwO/285+w==",
|
||||
"dependencies": {
|
||||
"@types/pouchdb-adapter-http": "*",
|
||||
"@types/pouchdb-core": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pouchdb-mapreduce": {
|
||||
"version": "6.1.7",
|
||||
"resolved": "https://registry.npmjs.org/@types/pouchdb-mapreduce/-/pouchdb-mapreduce-6.1.7.tgz",
|
||||
"integrity": "sha512-WzBwm7tmO9QhfRzVaWT4v6JQSS/fG2OoUDrWrhX87rPe2Pn6laPvdK5li6myNRxCoI/l5e8Jd+oYBAFnaiFucA==",
|
||||
"dependencies": {
|
||||
"@types/pouchdb-core": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pouchdb-node": {
|
||||
"version": "6.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/pouchdb-node/-/pouchdb-node-6.1.4.tgz",
|
||||
"integrity": "sha512-wnTCH8X1JOPpNOfVhz8HW0AvmdHh6pt40MuRj0jQnK7QEHsHS79WujsKTKSOF8QXtPwpvCNSsI7ut7H7tfxxJQ==",
|
||||
"dependencies": {
|
||||
"@types/pouchdb-adapter-http": "*",
|
||||
"@types/pouchdb-adapter-leveldb": "*",
|
||||
"@types/pouchdb-core": "*",
|
||||
"@types/pouchdb-mapreduce": "*",
|
||||
"@types/pouchdb-replication": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pouchdb-replication": {
|
||||
"version": "6.4.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/pouchdb-replication/-/pouchdb-replication-6.4.4.tgz",
|
||||
"integrity": "sha512-BsE5LKpjJK4iAf6Fx5kyrMw+33V+Ip7uWldUnU2BYrrvtR+MLD22dcImm7DZN1st2wPPb91i0XEnQzvP0w1C/Q==",
|
||||
"dependencies": {
|
||||
"@types/pouchdb-core": "*",
|
||||
"@types/pouchdb-find": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/prop-types": {
|
||||
"version": "15.7.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz",
|
||||
|
|
@ -13469,24 +13313,6 @@
|
|||
"node": "^10 || ^12 || >=14"
|
||||
}
|
||||
},
|
||||
"node_modules/pouchdb-browser": {
|
||||
"version": "9.0.0",
|
||||
"resolved": "https://registry.npmjs.org/pouchdb-browser/-/pouchdb-browser-9.0.0.tgz",
|
||||
"integrity": "sha512-0uKFWhsTtiVOF0+aGo7mvtCTP40f6dlsLNmJUvc/lwjsX1C3v+eBfVbvykyxpFl7UTAoJkXl+g/GOzNvyMtV1g==",
|
||||
"dependencies": {
|
||||
"spark-md5": "3.0.2",
|
||||
"uuid": "8.3.2",
|
||||
"vuvuzela": "1.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/pouchdb-browser/node_modules/uuid": {
|
||||
"version": "8.3.2",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
|
||||
"integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
|
||||
"bin": {
|
||||
"uuid": "dist/bin/uuid"
|
||||
}
|
||||
},
|
||||
"node_modules/prelude-ls": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
|
||||
|
|
@ -14365,11 +14191,6 @@
|
|||
"url": "https://github.com/sponsors/wooorm"
|
||||
}
|
||||
},
|
||||
"node_modules/spark-md5": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/spark-md5/-/spark-md5-3.0.2.tgz",
|
||||
"integrity": "sha512-wcFzz9cDfbuqe0FZzfi2or1sgyIrsDwmPwfZC4hiNidPdPINjeUwNfv5kldczoEAcjl9Y1L3SM7Uz2PUEQzxQw=="
|
||||
},
|
||||
"node_modules/sprintf-js": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
|
||||
|
|
@ -15172,11 +14993,6 @@
|
|||
"integrity": "sha512-AyFQ0EVmsOZOlAnxoFOGOq1SQDWAB7C6aqMGS23svWAllfOaxbuFvcT8D1i8z3Gyn8fraVeZNNmN6e9bxxXkKw==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/vuvuzela": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/vuvuzela/-/vuvuzela-1.0.3.tgz",
|
||||
"integrity": "sha512-Tm7jR1xTzBbPW+6y1tknKiEhz04Wf/1iZkcTJjSFcpNko43+dFW6+OOeQe9taJIug3NdfUAjFKgUSyQrIKaDvQ=="
|
||||
},
|
||||
"node_modules/w3c-keyname": {
|
||||
"version": "2.2.6",
|
||||
"resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.6.tgz",
|
||||
|
|
|
|||
|
|
@ -69,8 +69,8 @@
|
|||
"@langchain/core": "^0.3.3",
|
||||
"@langchain/google-genai": "^0.1.0",
|
||||
"@langchain/groq": "^0.1.2",
|
||||
"@orama/orama": "^3.0.0-rc-2",
|
||||
"@tabler/icons-react": "^2.14.0",
|
||||
"@types/pouchdb": "^6.4.0",
|
||||
"axios": "^1.3.4",
|
||||
"cohere-ai": "^7.13.0",
|
||||
"crypto-js": "^4.1.1",
|
||||
|
|
@ -81,7 +81,6 @@
|
|||
"koa-proxies": "^0.12.3",
|
||||
"langchain": "^0.3.2",
|
||||
"next-i18next": "^13.2.2",
|
||||
"pouchdb-browser": "^9.0.0",
|
||||
"prop-types": "^15.8.1",
|
||||
"react": "^18.2.0",
|
||||
"react-dom": "^18.2.0",
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import {
|
|||
formatDateTime,
|
||||
isSupportedChain,
|
||||
} from "@/utils";
|
||||
import VectorDBManager, { MemoryVector, NoteFile, VectorStoreDocument } from "@/vectorDBManager";
|
||||
import VectorDBManager from "@/vectorDBManager";
|
||||
import VectorStoreManager from "@/VectorStoreManager";
|
||||
import {
|
||||
ChatPromptTemplate,
|
||||
|
|
@ -20,7 +20,6 @@ import {
|
|||
} from "@langchain/core/prompts";
|
||||
import { RunnableSequence } from "@langchain/core/runnables";
|
||||
import { BaseChatMemory } from "langchain/memory";
|
||||
import { MultiQueryRetriever } from "langchain/retrievers/multi_query";
|
||||
import { MemoryVectorStore } from "langchain/vectorstores/memory";
|
||||
import { App, Notice } from "obsidian";
|
||||
import ChatModelManager from "./chatModelManager";
|
||||
|
|
@ -82,7 +81,7 @@ export default class ChainManager {
|
|||
this.langChainParams[key] = value;
|
||||
}
|
||||
|
||||
private setNoteFile(noteFile: NoteFile): void {
|
||||
private setNoteFile(noteFile: any): void {
|
||||
this.setLangChainParam("options", { ...this.getLangChainParams().options, noteFile });
|
||||
}
|
||||
|
||||
|
|
@ -152,7 +151,7 @@ export default class ChainManager {
|
|||
}
|
||||
this.validateChainType(chainType);
|
||||
// MUST set embeddingsManager when switching to QA mode
|
||||
if (chainType === ChainType.LONG_NOTE_QA_CHAIN || chainType === ChainType.VAULT_QA_CHAIN) {
|
||||
if (chainType === ChainType.VAULT_QA_CHAIN) {
|
||||
this.embeddingsManager = EmbeddingsManager.getInstance(
|
||||
() => this.getLangChainParams(),
|
||||
this.encryptionService,
|
||||
|
|
@ -188,84 +187,6 @@ export default class ChainManager {
|
|||
this.setLangChainParam("chainType", ChainType.LLM_CHAIN);
|
||||
break;
|
||||
}
|
||||
case ChainType.LONG_NOTE_QA_CHAIN: {
|
||||
if (!options.noteFile) {
|
||||
new Notice("No note content provided");
|
||||
throw new Error("No note content provided");
|
||||
}
|
||||
|
||||
this.setNoteFile(options.noteFile);
|
||||
const docHash = VectorDBManager.getDocumentHash(options.noteFile.path);
|
||||
const parsedMemoryVectors: MemoryVector[] | undefined =
|
||||
await VectorDBManager.getMemoryVectors(
|
||||
this.vectorStoreManager.getDbVectorStores(),
|
||||
docHash
|
||||
);
|
||||
const embeddingsAPI = this.embeddingsManager.getEmbeddingsAPI();
|
||||
if (!embeddingsAPI) {
|
||||
console.error("Error getting embeddings API. Please check your settings.");
|
||||
return;
|
||||
}
|
||||
if (parsedMemoryVectors) {
|
||||
// Index already exists
|
||||
const vectorStore = await VectorDBManager.rebuildMemoryVectorStore(
|
||||
parsedMemoryVectors,
|
||||
embeddingsAPI
|
||||
);
|
||||
|
||||
// Create new conversational retrieval chain
|
||||
ChainManager.retrievalChain = ChainFactory.createConversationalRetrievalChain(
|
||||
{
|
||||
llm: chatModel,
|
||||
retriever: vectorStore.asRetriever(undefined, (doc) => {
|
||||
return doc.metadata.path === options.noteFile?.path;
|
||||
}),
|
||||
systemMessage: this.getLangChainParams().systemMessage,
|
||||
},
|
||||
ChainManager.storeRetrieverDocuments.bind(ChainManager),
|
||||
options.debug
|
||||
);
|
||||
console.log("Existing vector store for document hash: ", docHash);
|
||||
} else {
|
||||
// Index doesn't exist
|
||||
const vectorStoreDoc = await this.indexFile(options.noteFile);
|
||||
this.vectorStore = await VectorDBManager.getMemoryVectorStore(
|
||||
this.vectorStoreManager.getDbVectorStores(),
|
||||
embeddingsAPI,
|
||||
vectorStoreDoc?._id
|
||||
);
|
||||
if (!this.vectorStore) {
|
||||
console.error("Error creating vector store.");
|
||||
return;
|
||||
}
|
||||
|
||||
const retriever = MultiQueryRetriever.fromLLM({
|
||||
llm: chatModel,
|
||||
retriever: this.vectorStore.asRetriever(undefined, (doc) => {
|
||||
return doc.metadata.path === options.noteFile?.path;
|
||||
}),
|
||||
verbose: false,
|
||||
});
|
||||
|
||||
ChainManager.retrievalChain = ChainFactory.createConversationalRetrievalChain(
|
||||
{
|
||||
llm: chatModel,
|
||||
retriever: retriever,
|
||||
systemMessage: this.getLangChainParams().systemMessage,
|
||||
},
|
||||
ChainManager.storeRetrieverDocuments.bind(ChainManager),
|
||||
options.debug
|
||||
);
|
||||
console.log(
|
||||
"New Long Note QA chain with multi-query retriever created for " + "document hash: ",
|
||||
docHash
|
||||
);
|
||||
}
|
||||
|
||||
this.setLangChainParam("chainType", ChainType.LONG_NOTE_QA_CHAIN);
|
||||
console.log("Set chain:", ChainType.LONG_NOTE_QA_CHAIN);
|
||||
break;
|
||||
}
|
||||
|
||||
case ChainType.VAULT_QA_CHAIN: {
|
||||
const embeddingsAPI = this.embeddingsManager.getEmbeddingsAPI();
|
||||
|
|
@ -273,20 +194,16 @@ export default class ChainManager {
|
|||
console.error("Error getting embeddings API. Please check your settings.");
|
||||
return;
|
||||
}
|
||||
const vectorStore = await VectorDBManager.getMemoryVectorStore(
|
||||
this.vectorStoreManager.getDbVectorStores(),
|
||||
embeddingsAPI
|
||||
);
|
||||
|
||||
const retriever = new HybridRetriever(
|
||||
this.vectorStoreManager.getDbVectorStores(),
|
||||
this.vectorStoreManager.getDb(),
|
||||
this.app.vault,
|
||||
{
|
||||
vectorStore: vectorStore,
|
||||
minSimilarityScore: 0.3, // TODO: Make this a setting
|
||||
maxK: this.settings.maxSourceChunks, // The maximum number of docs (chunks) to retrieve
|
||||
kIncrement: 2,
|
||||
},
|
||||
chatModel,
|
||||
embeddingsAPI,
|
||||
{
|
||||
minSimilarityScore: 0.3,
|
||||
maxK: this.settings.maxSourceChunks,
|
||||
},
|
||||
options.debug
|
||||
);
|
||||
|
||||
|
|
@ -400,7 +317,6 @@ export default class ChainManager {
|
|||
updateCurrentAiMessage(fullAIResponse);
|
||||
}
|
||||
break;
|
||||
case ChainType.LONG_NOTE_QA_CHAIN:
|
||||
case ChainType.VAULT_QA_CHAIN:
|
||||
if (debug) {
|
||||
console.log(
|
||||
|
|
@ -508,7 +424,7 @@ export default class ChainManager {
|
|||
return fullAIResponse;
|
||||
}
|
||||
|
||||
async indexFile(noteFile: NoteFile): Promise<VectorStoreDocument | undefined> {
|
||||
async indexFile(noteFile: any): Promise<any | undefined> {
|
||||
const embeddingsAPI = this.embeddingsManager.getEmbeddingsAPI();
|
||||
if (!embeddingsAPI) {
|
||||
const errorMsg =
|
||||
|
|
@ -518,7 +434,7 @@ export default class ChainManager {
|
|||
return;
|
||||
}
|
||||
return await VectorDBManager.indexFile(
|
||||
this.vectorStoreManager.getDbVectorStores(),
|
||||
this.vectorStoreManager.getDb(),
|
||||
embeddingsAPI,
|
||||
noteFile
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1,22 +1,22 @@
|
|||
// src/VectorStoreManager.ts
|
||||
|
||||
import { LangChainParams } from "@/aiParams";
|
||||
import { VAULT_VECTOR_STORE_STRATEGY } from "@/constants";
|
||||
import EncryptionService from "@/encryptionService";
|
||||
import { CustomError } from "@/error";
|
||||
import EmbeddingsManager from "@/LLMProviders/embeddingManager";
|
||||
import { CopilotSettings } from "@/settings/SettingsPage";
|
||||
import { areEmbeddingModelsSame, getNotesFromTags, isPathInList } from "@/utils";
|
||||
import VectorDBManager, { VectorStoreDocument } from "@/vectorDBManager";
|
||||
import VectorDBManager from "@/vectorDBManager";
|
||||
import { Embeddings } from "@langchain/core/embeddings";
|
||||
import { create, load, Orama, remove, removeMultiple, save, search } from "@orama/orama";
|
||||
import { MD5 } from "crypto-js";
|
||||
import { App, Notice } from "obsidian";
|
||||
import PouchDB from "pouchdb-browser";
|
||||
import { VAULT_VECTOR_STORE_STRATEGY } from "./constants";
|
||||
|
||||
class VectorStoreManager {
|
||||
private app: App;
|
||||
private settings: CopilotSettings;
|
||||
private encryptionService: EncryptionService;
|
||||
private dbVectorStores: PouchDB.Database<VectorStoreDocument>;
|
||||
private oramaDb: Orama<any>;
|
||||
private dbPath: string;
|
||||
private embeddingsManager: EmbeddingsManager;
|
||||
private getLangChainParams: () => LangChainParams;
|
||||
|
||||
|
|
@ -26,7 +26,7 @@ class VectorStoreManager {
|
|||
private indexNoticeMessage: HTMLSpanElement | null = null;
|
||||
private indexedCount = 0;
|
||||
private totalFilesToIndex = 0;
|
||||
|
||||
private initializationPromise: Promise<void>;
|
||||
constructor(
|
||||
app: App,
|
||||
settings: CopilotSettings,
|
||||
|
|
@ -38,32 +38,142 @@ class VectorStoreManager {
|
|||
this.encryptionService = encryptionService;
|
||||
this.getLangChainParams = getLangChainParams;
|
||||
|
||||
this.initializeDB();
|
||||
this.dbPath = this.getDbPath();
|
||||
this.embeddingsManager = EmbeddingsManager.getInstance(
|
||||
this.getLangChainParams,
|
||||
this.encryptionService,
|
||||
this.settings.activeEmbeddingModels
|
||||
);
|
||||
|
||||
// Initialize the database asynchronously
|
||||
this.initializationPromise = this.initializeDB()
|
||||
.then((db) => {
|
||||
this.oramaDb = db;
|
||||
console.log("Database initialized successfully:", this.oramaDb);
|
||||
|
||||
// Perform any operations that depend on the initialized database here
|
||||
this.performPostInitializationTasks();
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error("Failed to initialize database:", error);
|
||||
});
|
||||
|
||||
// Initialize the rate limiter
|
||||
VectorDBManager.initialize({
|
||||
getEmbeddingRequestsPerSecond: () => this.settings.embeddingRequestsPerSecond,
|
||||
debug: this.settings.debug,
|
||||
});
|
||||
}
|
||||
|
||||
private async performPostInitializationTasks() {
|
||||
// Optionally index the vault on startup
|
||||
if (this.settings.indexVaultToVectorStore === VAULT_VECTOR_STORE_STRATEGY.ON_STARTUP) {
|
||||
this.indexVaultToVectorStore().catch((err) => {
|
||||
try {
|
||||
await this.indexVaultToVectorStore();
|
||||
} catch (err) {
|
||||
console.error("Error indexing vault to vector store on startup:", err);
|
||||
new Notice("An error occurred while indexing vault to vector store.");
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private initializeDB() {
|
||||
this.dbVectorStores = new PouchDB<VectorStoreDocument>(
|
||||
`copilot_vector_stores_${this.getVaultIdentifier()}`
|
||||
private getDbPath(): string {
|
||||
return `${this.app.vault.configDir}/copilot-index-${this.getVaultIdentifier()}.json`;
|
||||
}
|
||||
|
||||
private createDynamicSchema(vectorLength: number) {
|
||||
return {
|
||||
id: "string",
|
||||
title: "string", // basename of the TFile
|
||||
path: "string", // path of the TFile
|
||||
content: "string",
|
||||
embedding: `vector[${vectorLength}]`,
|
||||
embeddingModel: "string",
|
||||
created_at: "number",
|
||||
ctime: "number",
|
||||
mtime: "number",
|
||||
tags: "string[]",
|
||||
extension: "string",
|
||||
};
|
||||
}
|
||||
|
||||
private async initializeDB(): Promise<Orama<any>> {
|
||||
this.dbPath = this.getDbPath();
|
||||
// Ensure the config directory exists
|
||||
const configDir = this.app.vault.configDir;
|
||||
if (!(await this.app.vault.adapter.exists(configDir))) {
|
||||
console.log(`Config directory does not exist. Creating: ${configDir}`);
|
||||
await this.app.vault.adapter.mkdir(configDir);
|
||||
}
|
||||
|
||||
try {
|
||||
if (await this.app.vault.adapter.exists(this.dbPath)) {
|
||||
// Load existing database
|
||||
const savedDb = await this.app.vault.adapter.read(this.dbPath);
|
||||
const parsedDb = JSON.parse(savedDb);
|
||||
|
||||
// Create a new database with the same schema as the saved one
|
||||
const schema = parsedDb.schema;
|
||||
const newDb = await create({ schema });
|
||||
|
||||
// Load the data into the new database
|
||||
await load(newDb, parsedDb);
|
||||
|
||||
console.log(`Loaded existing Orama database for ${this.dbPath} from disk.`);
|
||||
return newDb;
|
||||
} else {
|
||||
// Create new database
|
||||
return await this.createNewDb();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error initializing Orama database:`, error);
|
||||
return await this.createNewDb();
|
||||
}
|
||||
}
|
||||
|
||||
private async createNewDb(): Promise<Orama<any>> {
|
||||
const embeddingInstance = this.embeddingsManager.getEmbeddingsAPI();
|
||||
if (!embeddingInstance) {
|
||||
throw new CustomError("Embedding instance not found.");
|
||||
}
|
||||
|
||||
const vectorLength = await this.getVectorLength(embeddingInstance);
|
||||
const schema = this.createDynamicSchema(vectorLength);
|
||||
|
||||
const db = await create({
|
||||
schema,
|
||||
components: {
|
||||
tokenizer: {
|
||||
stemmer: undefined,
|
||||
stopWords: undefined,
|
||||
},
|
||||
},
|
||||
});
|
||||
console.log(
|
||||
`Created new Orama database for ${this.dbPath}. ` +
|
||||
`Embedding model: ${EmbeddingsManager.getModelName(embeddingInstance)} with vector length ${vectorLength}.`
|
||||
);
|
||||
return db;
|
||||
}
|
||||
|
||||
private async getVectorLength(embeddingInstance: Embeddings): Promise<number> {
|
||||
const sampleText = "Sample text for embedding";
|
||||
const sampleEmbedding = await embeddingInstance.embedQuery(sampleText);
|
||||
return sampleEmbedding.length;
|
||||
}
|
||||
|
||||
private async saveDB() {
|
||||
try {
|
||||
const rawData = await save(this.oramaDb);
|
||||
const dataToSave = {
|
||||
schema: this.oramaDb.schema,
|
||||
...rawData,
|
||||
};
|
||||
await this.app.vault.adapter.write(this.dbPath, JSON.stringify(dataToSave));
|
||||
console.log(`Saved Orama database to ${this.dbPath}.`);
|
||||
} catch (error) {
|
||||
console.error(`Error saving Orama database to ${this.dbPath}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
private getVaultIdentifier(): string {
|
||||
|
|
@ -71,8 +181,8 @@ class VectorStoreManager {
|
|||
return MD5(vaultName).toString();
|
||||
}
|
||||
|
||||
public getDbVectorStores(): PouchDB.Database<VectorStoreDocument> {
|
||||
return this.dbVectorStores;
|
||||
public getDb(): Orama<any> {
|
||||
return this.oramaDb;
|
||||
}
|
||||
|
||||
public getEmbeddingsManager(): EmbeddingsManager {
|
||||
|
|
@ -160,38 +270,40 @@ class VectorStoreManager {
|
|||
throw new CustomError("Embedding instance not found.");
|
||||
}
|
||||
|
||||
// Check if embedding model has changed
|
||||
const prevEmbeddingModel = await VectorDBManager.checkEmbeddingModel(this.dbVectorStores);
|
||||
// TODO: Remove this when Ollama model is dynamically set
|
||||
const currEmbeddingModel = EmbeddingsManager.getModelName(embeddingInstance);
|
||||
const singleDoc = await search(this.getDb(), {
|
||||
term: "",
|
||||
limit: 1,
|
||||
});
|
||||
|
||||
if (this.settings.debug) {
|
||||
console.log(
|
||||
`\nVault QA exclusions: ${this.settings.qaExclusions ? this.settings.qaExclusions : "None"}`
|
||||
);
|
||||
console.log("Prev vs Current embedding models:", prevEmbeddingModel, currEmbeddingModel);
|
||||
}
|
||||
let prevEmbeddingModel: string | undefined;
|
||||
|
||||
if (!areEmbeddingModelsSame(prevEmbeddingModel, currEmbeddingModel)) {
|
||||
// Model has changed, clear DB and reindex from scratch
|
||||
overwrite = true;
|
||||
// Clear the current vector store with mixed embeddings
|
||||
try {
|
||||
// Clear the vectorstore db
|
||||
await this.dbVectorStores.destroy();
|
||||
// Reinitialize the database
|
||||
this.dbVectorStores = new PouchDB<VectorStoreDocument>(
|
||||
`copilot_vector_stores_${this.getVaultIdentifier()}`
|
||||
);
|
||||
new Notice("Detected change in embedding model. Rebuild vector store from scratch.");
|
||||
console.log("Detected change in embedding model. Rebuild vector store from scratch.");
|
||||
} catch (err) {
|
||||
console.error("Error clearing vector store for reindexing:", err);
|
||||
new Notice("Error clearing vector store for reindexing.");
|
||||
if (singleDoc.hits.length > 0) {
|
||||
const oramaDocSample = singleDoc.hits[0];
|
||||
if (
|
||||
typeof oramaDocSample === "object" &&
|
||||
oramaDocSample !== null &&
|
||||
"document" in oramaDocSample
|
||||
) {
|
||||
const document = oramaDocSample.document as { embeddingModel?: string };
|
||||
prevEmbeddingModel = document.embeddingModel;
|
||||
}
|
||||
}
|
||||
|
||||
const latestMtime = await VectorDBManager.getLatestFileMtime(this.dbVectorStores);
|
||||
if (prevEmbeddingModel) {
|
||||
const currEmbeddingModel = EmbeddingsManager.getModelName(embeddingInstance);
|
||||
|
||||
if (!areEmbeddingModelsSame(prevEmbeddingModel, currEmbeddingModel)) {
|
||||
// Model has changed, reinitialize DB
|
||||
await this.initializeDB();
|
||||
overwrite = true;
|
||||
new Notice("Detected change in embedding model. Rebuilding vector store from scratch.");
|
||||
console.log("Detected change in embedding model. Rebuilding vector store from scratch.");
|
||||
}
|
||||
} else {
|
||||
console.log("No previous embedding model found in the database.");
|
||||
}
|
||||
|
||||
const latestMtime = await VectorDBManager.getLatestFileMtime(this.oramaDb);
|
||||
// Initialize indexing state
|
||||
this.isIndexingPaused = false;
|
||||
this.isIndexingCancelled = false;
|
||||
|
|
@ -237,14 +349,18 @@ class VectorStoreManager {
|
|||
const file = files[index];
|
||||
|
||||
try {
|
||||
const noteFile = {
|
||||
basename: file.basename,
|
||||
const fileToSave = {
|
||||
title: file.basename,
|
||||
path: file.path,
|
||||
mtime: file.stat.mtime,
|
||||
content: fileContents[index],
|
||||
embeddingModel: EmbeddingsManager.getModelName(embeddingInstance),
|
||||
ctime: file.stat.ctime,
|
||||
mtime: file.stat.mtime,
|
||||
tags: fileMetadatas[index]?.tags ?? [], // Assuming tags are in the metadata
|
||||
extension: file.extension,
|
||||
metadata: fileMetadatas[index]?.frontmatter ?? {},
|
||||
};
|
||||
await VectorDBManager.indexFile(this.dbVectorStores, embeddingInstance, noteFile);
|
||||
await VectorDBManager.indexFile(this.oramaDb, embeddingInstance, fileToSave);
|
||||
|
||||
this.indexedCount++;
|
||||
this.updateIndexingNoticeMessage();
|
||||
|
|
@ -289,6 +405,7 @@ class VectorStoreManager {
|
|||
this.indexNoticeMessage = null;
|
||||
this.isIndexingPaused = false;
|
||||
this.isIndexingCancelled = false;
|
||||
this.saveDB();
|
||||
}, 3000);
|
||||
|
||||
if (errors.length > 0) {
|
||||
|
|
@ -314,8 +431,11 @@ class VectorStoreManager {
|
|||
|
||||
public async clearVectorStore(): Promise<void> {
|
||||
try {
|
||||
await this.dbVectorStores.destroy();
|
||||
this.initializeDB();
|
||||
// Create a new, empty database instance
|
||||
this.oramaDb = await this.createNewDb();
|
||||
|
||||
// Save the new, empty database
|
||||
await this.saveDB();
|
||||
new Notice("Local vector store cleared successfully.");
|
||||
console.log("Local vector store cleared successfully, new instance created.");
|
||||
} catch (err) {
|
||||
|
|
@ -328,19 +448,32 @@ class VectorStoreManager {
|
|||
public async garbageCollectVectorStore(): Promise<void> {
|
||||
try {
|
||||
const files = this.app.vault.getMarkdownFiles();
|
||||
const filePaths = files.map((file) => file.path);
|
||||
const indexedFiles = await VectorDBManager.getNoteFiles(this.dbVectorStores);
|
||||
const indexedFilePaths = indexedFiles.map((file) => file.path);
|
||||
const filesToDelete = indexedFilePaths.filter((filePath) => !filePaths.includes(filePath));
|
||||
const filePaths = new Set(files.map((file) => file.path));
|
||||
|
||||
const deletePromises = filesToDelete.map(async (filePath) => {
|
||||
VectorDBManager.removeMemoryVectors(
|
||||
this.dbVectorStores,
|
||||
VectorDBManager.getDocumentHash(filePath)
|
||||
);
|
||||
// Search for all documents in the database
|
||||
const result = await search(this.oramaDb, {
|
||||
term: "",
|
||||
properties: ["path"],
|
||||
limit: Infinity,
|
||||
});
|
||||
|
||||
await Promise.all(deletePromises);
|
||||
const docsToRemove = result.hits
|
||||
.filter((hit) => !filePaths.has(hit.document.path))
|
||||
.map((hit) => hit.id);
|
||||
|
||||
if (docsToRemove.length === 0) {
|
||||
console.log("No documents to remove during garbage collection.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (docsToRemove.length === 1) {
|
||||
await remove(this.oramaDb, docsToRemove[0]);
|
||||
} else {
|
||||
const removedCount = await removeMultiple(this.oramaDb, docsToRemove, 500);
|
||||
console.log(`Removed ${removedCount} documents during garbage collection.`);
|
||||
}
|
||||
|
||||
await this.saveDB();
|
||||
|
||||
new Notice("Local vector store garbage collected successfully.");
|
||||
console.log("Local vector store garbage collected successfully.");
|
||||
|
|
@ -350,15 +483,40 @@ class VectorStoreManager {
|
|||
}
|
||||
}
|
||||
|
||||
public registerEventHandlers() {
|
||||
public async removeDocs(filePath: string) {
|
||||
// Handle file deletion
|
||||
this.app.vault.on("delete", (file) => {
|
||||
const docHash = VectorDBManager.getDocumentHash(file.path);
|
||||
VectorDBManager.removeMemoryVectors(this.dbVectorStores, docHash);
|
||||
});
|
||||
try {
|
||||
const searchResult = await search(this.oramaDb, {
|
||||
term: filePath,
|
||||
properties: ["path"],
|
||||
tolerance: 1,
|
||||
});
|
||||
if (searchResult.hits.length > 0) {
|
||||
await removeMultiple(
|
||||
this.oramaDb,
|
||||
searchResult.hits.map((hit) => hit.id),
|
||||
500
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Error deleting document from local Copilotindex:", err);
|
||||
}
|
||||
}
|
||||
|
||||
// Add other methods as needed...
|
||||
public async waitForInitialization() {
|
||||
await this.initializationPromise;
|
||||
}
|
||||
|
||||
// Test query to retrieve record by id from the database
|
||||
public async getDocById(id: string): Promise<any | undefined> {
|
||||
const result = await search(this.oramaDb, {
|
||||
term: id,
|
||||
properties: ["id"],
|
||||
limit: 1,
|
||||
includeVectors: true,
|
||||
});
|
||||
return result.hits[0]?.document;
|
||||
}
|
||||
}
|
||||
|
||||
export default VectorStoreManager;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import { ChainType } from "@/chainFactory";
|
||||
import { ChatPromptTemplate } from "@langchain/core/prompts";
|
||||
import { NoteFile } from "./vectorDBManager";
|
||||
|
||||
export interface ModelConfig {
|
||||
modelName: string;
|
||||
|
|
@ -55,7 +54,7 @@ export interface LangChainParams {
|
|||
|
||||
export interface SetChainOptions {
|
||||
prompt?: ChatPromptTemplate;
|
||||
noteFile?: NoteFile;
|
||||
noteFile?: any;
|
||||
forceNewCreation?: boolean;
|
||||
abortController?: AbortController;
|
||||
debug?: boolean;
|
||||
|
|
|
|||
|
|
@ -51,7 +51,6 @@ type ConversationalRetrievalQAChainInput = {
|
|||
// Add new chain types here
|
||||
export enum ChainType {
|
||||
LLM_CHAIN = "llm_chain",
|
||||
LONG_NOTE_QA_CHAIN = "long_note_qa",
|
||||
VAULT_QA_CHAIN = "vault_qa",
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import ChainManager from "@/LLMProviders/chainManager";
|
||||
import { useAIState } from "@/aiState";
|
||||
import { ChainType } from "@/chainFactory";
|
||||
import { updateChatMemory } from "@/chatUtils";
|
||||
import ChatIcons from "@/components/ChatComponents/ChatIcons";
|
||||
import ChatInput from "@/components/ChatComponents/ChatInput";
|
||||
|
|
@ -360,10 +359,6 @@ ${chatContent}`;
|
|||
timestamp: formatDateTime(new Date()),
|
||||
};
|
||||
|
||||
if (currentChain === ChainType.LONG_NOTE_QA_CHAIN) {
|
||||
setChain(ChainType.LONG_NOTE_QA_CHAIN, { noteFile, debug });
|
||||
}
|
||||
|
||||
addMessage(activeNoteOnMessage);
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { AI_SENDER, VAULT_VECTOR_STORE_STRATEGY } from "@/constants";
|
|||
import { CustomError } from "@/error";
|
||||
import { CopilotSettings } from "@/settings/SettingsPage";
|
||||
import { ChatMessage } from "@/sharedState";
|
||||
import { formatDateTime, getFileContent, getFileName } from "@/utils";
|
||||
import { formatDateTime } from "@/utils";
|
||||
import { Notice, Vault } from "obsidian";
|
||||
import React, { useEffect, useState } from "react";
|
||||
|
||||
|
|
@ -69,38 +69,7 @@ const ChatIcons: React.FC<ChatIconsProps> = ({
|
|||
return;
|
||||
}
|
||||
|
||||
if (selectedChain === ChainType.LONG_NOTE_QA_CHAIN) {
|
||||
const file = app.workspace.getActiveFile();
|
||||
if (!file) {
|
||||
new Notice("No active note found.");
|
||||
console.error("No active note found.");
|
||||
return;
|
||||
}
|
||||
|
||||
const noteContent = await getFileContent(file, vault);
|
||||
const fileMetadata = app.metadataCache.getFileCache(file);
|
||||
const noteFile = {
|
||||
path: file.path,
|
||||
basename: file.basename,
|
||||
mtime: file.stat.mtime,
|
||||
content: noteContent ?? "",
|
||||
metadata: fileMetadata?.frontmatter ?? {},
|
||||
};
|
||||
|
||||
const noteName = getFileName(file);
|
||||
|
||||
const activeNoteOnMessage: ChatMessage = {
|
||||
sender: AI_SENDER,
|
||||
message: `OK Feel free to ask me questions about [[${noteName}]]. \n\nPlease note that this is a retrieval-based QA for notes longer than the model context window. Specific questions are encouraged. For generic questions like 'give me a summary', 'brainstorm based on the content', Chat mode with *Send Note to Prompt* button used with a *long context model* is a more suitable choice.`,
|
||||
isVisible: true,
|
||||
timestamp: formatDateTime(new Date()),
|
||||
};
|
||||
addMessage(activeNoteOnMessage);
|
||||
if (noteContent) {
|
||||
setCurrentChain(selectedChain, { noteFile, debug });
|
||||
}
|
||||
return;
|
||||
} else if (selectedChain === ChainType.VAULT_QA_CHAIN) {
|
||||
if (selectedChain === ChainType.VAULT_QA_CHAIN) {
|
||||
if (vault_qa_strategy === VAULT_VECTOR_STORE_STRATEGY.ON_MODE_SWITCH) {
|
||||
await onRefreshVaultContext();
|
||||
}
|
||||
|
|
@ -173,7 +142,6 @@ const ChatIcons: React.FC<ChatIconsProps> = ({
|
|||
onChange={handleChainChange}
|
||||
>
|
||||
<option value="llm_chain">Chat</option>
|
||||
<option value="long_note_qa">Long Note QA</option>
|
||||
<option value="vault_qa">Vault QA (BETA)</option>
|
||||
</select>
|
||||
<span className="tooltip-text">Mode Selection</span>
|
||||
|
|
@ -193,19 +161,6 @@ const ChatIcons: React.FC<ChatIconsProps> = ({
|
|||
</span>
|
||||
</button>
|
||||
)}
|
||||
{selectedChain === "long_note_qa" && (
|
||||
<button
|
||||
className="chat-icon-button clickable-icon"
|
||||
onClick={onForceRebuildActiveNoteContext}
|
||||
>
|
||||
<UseActiveNoteAsContextIcon className="icon-scaler" />
|
||||
<span className="tooltip-text">
|
||||
Refresh Index
|
||||
<br />
|
||||
for Active Note
|
||||
</span>
|
||||
</button>
|
||||
)}
|
||||
{selectedChain === "vault_qa" && (
|
||||
<button className="chat-icon-button clickable-icon" onClick={onRefreshVaultContext}>
|
||||
<UseActiveNoteAsContextIcon className="icon-scaler" />
|
||||
|
|
|
|||
|
|
@ -12,13 +12,6 @@ import {
|
|||
} from "@/utils";
|
||||
import { normalizePath, Notice, TFile, Vault } from "obsidian";
|
||||
|
||||
// TODO: To be deprecated once PouchDB is removed
|
||||
export interface CustomPromptDB {
|
||||
_id: string;
|
||||
_rev?: string;
|
||||
prompt: string;
|
||||
}
|
||||
|
||||
export interface CustomPrompt {
|
||||
title: string;
|
||||
content: string;
|
||||
|
|
|
|||
|
|
@ -330,9 +330,8 @@ export default class CopilotPlugin extends Plugin {
|
|||
});
|
||||
|
||||
this.registerEvent(
|
||||
this.app.vault.on("delete", (file) => {
|
||||
const docHash = VectorDBManager.getDocumentHash(file.path);
|
||||
VectorDBManager.removeMemoryVectors(this.vectorStoreManager.getDbVectorStores(), docHash);
|
||||
this.app.vault.on("delete", async (file) => {
|
||||
await this.vectorStoreManager.removeDocs(file.path);
|
||||
})
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,33 +1,35 @@
|
|||
import { extractNoteTitles, getNoteFileFromTitle } from "@/utils";
|
||||
import VectorDBManager from "@/vectorDBManager";
|
||||
import { Document } from "@langchain/core/documents";
|
||||
import { Embeddings } from "@langchain/core/embeddings";
|
||||
import { BaseLanguageModel } from "@langchain/core/language_models/base";
|
||||
import { ChatPromptTemplate } from "@langchain/core/prompts";
|
||||
import { BaseRetriever } from "@langchain/core/retrievers";
|
||||
import { VectorStore } from "@langchain/core/vectorstores";
|
||||
import {
|
||||
ScoreThresholdRetriever,
|
||||
ScoreThresholdRetrieverInput,
|
||||
} from "langchain/retrievers/score_threshold";
|
||||
import { Orama, search } from "@orama/orama";
|
||||
import { Vault } from "obsidian";
|
||||
|
||||
export class HybridRetriever<V extends VectorStore> extends BaseRetriever {
|
||||
export class HybridRetriever extends BaseRetriever {
|
||||
public lc_namespace = ["hybrid_retriever"];
|
||||
|
||||
private llm: BaseLanguageModel;
|
||||
private queryRewritePrompt: ChatPromptTemplate;
|
||||
|
||||
private embeddingsInstance: Embeddings;
|
||||
constructor(
|
||||
private db: PouchDB.Database,
|
||||
private db: Orama<any>,
|
||||
private vault: Vault,
|
||||
private options: ScoreThresholdRetrieverInput<V>,
|
||||
llm: BaseLanguageModel,
|
||||
embeddingsInstance: Embeddings,
|
||||
private options: {
|
||||
minSimilarityScore: number;
|
||||
maxK: number;
|
||||
},
|
||||
private debug?: boolean
|
||||
) {
|
||||
super();
|
||||
this.llm = llm;
|
||||
this.embeddingsInstance = embeddingsInstance;
|
||||
this.queryRewritePrompt = ChatPromptTemplate.fromTemplate(
|
||||
"Please write a passage to answer the question.\nQuestion: {question}\nPassage:"
|
||||
"Please write a passage to answer the question. If you don't know the answer, just make up a passage. \nQuestion: {question}\nPassage:"
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -36,17 +38,16 @@ export class HybridRetriever<V extends VectorStore> extends BaseRetriever {
|
|||
const noteTitles = extractNoteTitles(query);
|
||||
// Retrieve chunks for explicitly mentioned note titles
|
||||
const explicitChunks = await this.getExplicitChunks(noteTitles);
|
||||
|
||||
// Generate a hypothetical answer passage
|
||||
const rewrittenQuery = await this.rewriteQuery(query);
|
||||
// Perform vector similarity search using ScoreThresholdRetriever
|
||||
const vectorChunks = await this.getVectorChunks(rewrittenQuery);
|
||||
const oramaChunks = await this.getOramaChunks(rewrittenQuery);
|
||||
|
||||
// Combine explicit and vector chunks, removing duplicates while maintaining order
|
||||
const uniqueChunks = new Set<string>(explicitChunks.map((chunk) => chunk.pageContent));
|
||||
const combinedChunks: Document[] = [...explicitChunks];
|
||||
|
||||
for (const chunk of vectorChunks) {
|
||||
for (const chunk of oramaChunks) {
|
||||
const chunkContent = chunk.pageContent;
|
||||
if (!uniqueChunks.has(chunkContent)) {
|
||||
uniqueChunks.add(chunkContent);
|
||||
|
|
@ -65,8 +66,8 @@ export class HybridRetriever<V extends VectorStore> extends BaseRetriever {
|
|||
noteTitles,
|
||||
"\n\nExplicit Chunks:",
|
||||
explicitChunks,
|
||||
"\n\nVector Chunks:",
|
||||
vectorChunks,
|
||||
"\n\nOrama Chunks:",
|
||||
oramaChunks,
|
||||
"\n\nCombined Chunks:",
|
||||
combinedChunks
|
||||
);
|
||||
|
|
@ -98,14 +99,26 @@ export class HybridRetriever<V extends VectorStore> extends BaseRetriever {
|
|||
const explicitChunks: Document[] = [];
|
||||
for (const noteTitle of noteTitles) {
|
||||
const noteFile = await getNoteFileFromTitle(this.vault, noteTitle);
|
||||
const docHash = VectorDBManager.getDocumentHash(noteFile?.path ?? "");
|
||||
const memoryVectors = await VectorDBManager.getMemoryVectors(this.db, docHash);
|
||||
if (memoryVectors) {
|
||||
const matchingChunks = memoryVectors.map(
|
||||
(memoryVector) =>
|
||||
const hits = await VectorDBManager.getDocsByPath(this.db, noteFile?.path ?? "");
|
||||
if (hits) {
|
||||
const matchingChunks = hits.map(
|
||||
(hit: any) =>
|
||||
new Document({
|
||||
pageContent: memoryVector.content,
|
||||
metadata: memoryVector.metadata,
|
||||
pageContent: hit.document.content,
|
||||
metadata: {
|
||||
...hit.document.metadata,
|
||||
score: hit.score,
|
||||
path: hit.document.path,
|
||||
mtime: hit.document.mtime,
|
||||
ctime: hit.document.ctime,
|
||||
title: hit.document.title,
|
||||
id: hit.document.id,
|
||||
embeddingModel: hit.document.embeddingModel,
|
||||
tags: hit.document.tags,
|
||||
extension: hit.document.extension,
|
||||
created_at: hit.document.created_at,
|
||||
nchars: hit.document.nchars,
|
||||
},
|
||||
})
|
||||
);
|
||||
explicitChunks.push(...matchingChunks);
|
||||
|
|
@ -114,13 +127,46 @@ export class HybridRetriever<V extends VectorStore> extends BaseRetriever {
|
|||
return explicitChunks;
|
||||
}
|
||||
|
||||
private async getVectorChunks(query: string): Promise<Document[]> {
|
||||
const retriever = ScoreThresholdRetriever.fromVectorStore(this.options.vectorStore, {
|
||||
minSimilarityScore: this.options.minSimilarityScore,
|
||||
maxK: this.options.maxK,
|
||||
kIncrement: this.options.kIncrement,
|
||||
private async getOramaChunks(query: string): Promise<Document[]> {
|
||||
// Assuming you have a method to convert the query to a vector
|
||||
const queryVector = await this.convertQueryToVector(query);
|
||||
|
||||
const searchResults = await search(this.db, {
|
||||
mode: "vector",
|
||||
vector: {
|
||||
value: queryVector,
|
||||
property: "embedding",
|
||||
},
|
||||
similarity: this.options.minSimilarityScore,
|
||||
limit: this.options.maxK,
|
||||
includeVectors: true,
|
||||
});
|
||||
const vectorChunks = await retriever.getRelevantDocuments(query);
|
||||
|
||||
// Convert Orama search results to Document objects
|
||||
const vectorChunks = searchResults.hits.map(
|
||||
(hit) =>
|
||||
new Document({
|
||||
pageContent: hit.document.content,
|
||||
metadata: {
|
||||
...hit.document.metadata,
|
||||
score: hit.score,
|
||||
path: hit.document.path,
|
||||
mtime: hit.document.mtime,
|
||||
ctime: hit.document.ctime,
|
||||
title: hit.document.title,
|
||||
id: hit.document.id,
|
||||
embeddingModel: hit.document.embeddingModel,
|
||||
tags: hit.document.tags,
|
||||
extension: hit.document.extension,
|
||||
created_at: hit.document.created_at,
|
||||
nchars: hit.document.nchars,
|
||||
},
|
||||
})
|
||||
);
|
||||
return vectorChunks;
|
||||
}
|
||||
|
||||
private async convertQueryToVector(query: string): Promise<number[]> {
|
||||
return await this.embeddingsInstance.embedQuery(query);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -44,19 +44,9 @@ const QASettings: React.FC<QASettingsProps> = ({
|
|||
<br />
|
||||
<br />
|
||||
<h1>QA Settings</h1>
|
||||
<div className="warning-message">
|
||||
Vault QA is in BETA and may not be stable. If you have issues please report in the github
|
||||
repo.
|
||||
</div>
|
||||
<p>
|
||||
QA mode relies on a <em>local</em> vector index.
|
||||
</p>
|
||||
<h2>Long Note QA vs. Vault QA (BETA)</h2>
|
||||
<p>
|
||||
Long Note QA mode uses the Active Note as context. Vault QA (BETA) uses your entire vault as
|
||||
context. Please ask questions as specific as possible, avoid vague questions to get better
|
||||
results.
|
||||
</p>
|
||||
<h2>Local Embedding Model</h2>
|
||||
<p>
|
||||
Check the{" "}
|
||||
|
|
@ -85,9 +75,6 @@ const QASettings: React.FC<QASettingsProps> = ({
|
|||
If you are using a paid embedding provider, beware of costs for large vaults!
|
||||
</div>
|
||||
<p>
|
||||
When you switch to <strong>Long Note QA</strong> mode, your active note is indexed
|
||||
automatically upon mode switch.
|
||||
<br />
|
||||
When you switch to <strong>Vault QA</strong> mode, your vault is indexed{" "}
|
||||
<em>based on the auto-index strategy you select below</em>.
|
||||
<br />
|
||||
|
|
|
|||
|
|
@ -157,8 +157,6 @@ export const stringToChainType = (chain: string): ChainType => {
|
|||
switch (chain) {
|
||||
case "llm_chain":
|
||||
return ChainType.LLM_CHAIN;
|
||||
case "long_note_qa":
|
||||
return ChainType.LONG_NOTE_QA_CHAIN;
|
||||
case "vault_qa":
|
||||
return ChainType.VAULT_QA_CHAIN;
|
||||
default:
|
||||
|
|
|
|||
|
|
@ -1,40 +1,25 @@
|
|||
import EmbeddingManager from "@/LLMProviders/embeddingManager";
|
||||
import { RateLimiter } from "@/rateLimiter";
|
||||
import { Embeddings } from "@langchain/core/embeddings";
|
||||
import { insert, Orama, search, update } from "@orama/orama";
|
||||
import { MD5 } from "crypto-js";
|
||||
import { Document } from "langchain/document";
|
||||
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
|
||||
import { MemoryVectorStore } from "langchain/vectorstores/memory";
|
||||
|
||||
// TODOs
|
||||
// 1. Use embeddingModel rather than embeddingProvider
|
||||
// 2. embeddingModel should be on MemoryVector metadata
|
||||
// 3. VectorDBManager should have a public method checking existing embeddingModel on a MemoryVector
|
||||
export interface VectorStoreDocument {
|
||||
_id: string;
|
||||
_rev?: string;
|
||||
memory_vectors: string;
|
||||
file_mtime: number;
|
||||
embeddingModel: string;
|
||||
created_at: number;
|
||||
}
|
||||
|
||||
export interface MemoryVector {
|
||||
export interface OramaDocument {
|
||||
id: string;
|
||||
title: string;
|
||||
content: string;
|
||||
embedding: number[];
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
metadata: Record<string, any>;
|
||||
}
|
||||
|
||||
export interface NoteFile {
|
||||
path: string;
|
||||
basename: string;
|
||||
embeddingModel: string;
|
||||
created_at: number;
|
||||
ctime: number;
|
||||
mtime: number;
|
||||
content: string;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
tags: string[];
|
||||
extension: string;
|
||||
nchars: number;
|
||||
metadata: Record<string, any>;
|
||||
}
|
||||
|
||||
interface VectorDBConfig {
|
||||
getEmbeddingRequestsPerSecond: () => number;
|
||||
debug: boolean;
|
||||
|
|
@ -58,106 +43,16 @@ class VectorDBManager {
|
|||
}
|
||||
return this.rateLimiter;
|
||||
}
|
||||
public static getDocumentHash(sourceDocument: string): string {
|
||||
|
||||
public static getDocHash(sourceDocument: string): string {
|
||||
return MD5(sourceDocument).toString();
|
||||
}
|
||||
|
||||
public static async rebuildMemoryVectorStore(
|
||||
memoryVectors: MemoryVector[],
|
||||
embeddingsAPI: Embeddings
|
||||
) {
|
||||
if (!Array.isArray(memoryVectors)) {
|
||||
throw new TypeError("Expected memoryVectors to be an array");
|
||||
}
|
||||
// Extract the embeddings and documents from the deserialized memoryVectors
|
||||
const embeddingsArray: number[][] = memoryVectors.map((memoryVector) => memoryVector.embedding);
|
||||
const documentsArray = memoryVectors.map(
|
||||
(memoryVector) =>
|
||||
new Document({
|
||||
pageContent: memoryVector.content,
|
||||
metadata: memoryVector.metadata,
|
||||
})
|
||||
);
|
||||
|
||||
// Create a new MemoryVectorStore instance
|
||||
const memoryVectorStore = new MemoryVectorStore(embeddingsAPI);
|
||||
await memoryVectorStore.addVectors(embeddingsArray, documentsArray);
|
||||
return memoryVectorStore;
|
||||
}
|
||||
|
||||
public static async getMemoryVectorStore(
|
||||
db: PouchDB.Database,
|
||||
embeddingsAPI: Embeddings,
|
||||
docHash?: string
|
||||
): Promise<MemoryVectorStore> {
|
||||
if (!db) throw new Error("DB not initialized");
|
||||
|
||||
let allDocsResponse;
|
||||
if (docHash) {
|
||||
// Fetch a single document by its _id
|
||||
const doc = await db.get(docHash);
|
||||
allDocsResponse = { rows: [{ doc }] };
|
||||
} else {
|
||||
// Fetch all documents
|
||||
allDocsResponse = await db.allDocs({ include_docs: true });
|
||||
}
|
||||
|
||||
const embeddingModel = EmbeddingManager.getModelName(embeddingsAPI);
|
||||
if (!embeddingModel) console.error("EmbeddingManager could not determine model name!");
|
||||
const allDocs = allDocsResponse.rows
|
||||
.map((row) => row.doc as VectorStoreDocument)
|
||||
.filter((doc) => doc.embeddingModel === embeddingModel);
|
||||
const memoryVectors = allDocs
|
||||
.map((doc) => JSON.parse(doc.memory_vectors) as MemoryVector[])
|
||||
.flat();
|
||||
const embeddingsArray: number[][] = memoryVectors.map((memoryVector) => memoryVector.embedding);
|
||||
const documentsArray = memoryVectors.map(
|
||||
(memoryVector) =>
|
||||
new Document({
|
||||
pageContent: memoryVector.content,
|
||||
metadata: memoryVector.metadata,
|
||||
})
|
||||
);
|
||||
// Create a new MemoryVectorStore instance
|
||||
const memoryVectorStore = new MemoryVectorStore(embeddingsAPI);
|
||||
await memoryVectorStore.addVectors(embeddingsArray, documentsArray);
|
||||
return memoryVectorStore;
|
||||
}
|
||||
|
||||
public static async setMemoryVectors(
|
||||
db: PouchDB.Database,
|
||||
memoryVectors: MemoryVector[],
|
||||
docHash: string
|
||||
): Promise<void> {
|
||||
if (!db) throw new Error("DB not initialized");
|
||||
if (!Array.isArray(memoryVectors)) {
|
||||
throw new TypeError("Expected memoryVectors to be an array");
|
||||
}
|
||||
const serializedMemoryVectors = JSON.stringify(memoryVectors);
|
||||
try {
|
||||
// Attempt to fetch the existing document, if it exists.
|
||||
const existingDoc = await db.get(docHash).catch((err) => null);
|
||||
|
||||
// Prepare the document to be saved.
|
||||
const docToSave = {
|
||||
_id: docHash,
|
||||
memory_vectors: serializedMemoryVectors,
|
||||
created_at: Date.now(),
|
||||
_rev: existingDoc?._rev, // Add the current revision if the document exists.
|
||||
};
|
||||
|
||||
// Save the document.
|
||||
await db.put(docToSave);
|
||||
} catch (err) {
|
||||
console.error("Error storing vectors in VectorDB:", err);
|
||||
}
|
||||
}
|
||||
|
||||
public static async indexFile(
|
||||
db: PouchDB.Database,
|
||||
db: Orama<any>,
|
||||
embeddingsAPI: Embeddings,
|
||||
noteFile: NoteFile
|
||||
): Promise<VectorStoreDocument | undefined> {
|
||||
fileToSave: any
|
||||
): Promise<any | undefined> {
|
||||
if (!db) throw new Error("DB not initialized");
|
||||
if (!this.config) throw new Error("VectorDBManager not initialized");
|
||||
|
||||
|
|
@ -170,151 +65,129 @@ class VectorDBManager {
|
|||
});
|
||||
// Add note title as contextual chunk headers
|
||||
// https://js.langchain.com/docs/modules/data_connection/document_transformers/contextual_chunk_headers
|
||||
const splitDocument = await textSplitter.createDocuments([noteFile.content], [], {
|
||||
chunkHeader: "[[" + noteFile.basename + "]]" + "\n\n---\n\n",
|
||||
const chunks = await textSplitter.createDocuments([fileToSave.content], [], {
|
||||
chunkHeader: "[[" + fileToSave.title + "]]" + "\n\n---\n\n",
|
||||
appendChunkOverlapHeader: true,
|
||||
});
|
||||
|
||||
// Apply rate limiting before making the API call
|
||||
await this.getRateLimiter().wait();
|
||||
const docVectors = await embeddingsAPI.embedDocuments(
|
||||
splitDocument.map((doc) => doc.pageContent)
|
||||
);
|
||||
|
||||
const memoryVectors = docVectors.map((docVector, i) => ({
|
||||
content: splitDocument[i].pageContent,
|
||||
metadata: {
|
||||
...noteFile.metadata,
|
||||
title: noteFile.basename,
|
||||
path: noteFile.path,
|
||||
},
|
||||
embedding: docVector,
|
||||
}));
|
||||
const docHash = VectorDBManager.getDocumentHash(noteFile.path);
|
||||
|
||||
const serializedMemoryVectors = JSON.stringify(memoryVectors);
|
||||
const docVectors: number[][] = [];
|
||||
try {
|
||||
// Attempt to fetch the existing document, if it exists.
|
||||
const existingDoc = await db.get(docHash).catch((err) => null);
|
||||
|
||||
// Prepare the document to be saved.
|
||||
const docToSave: VectorStoreDocument = {
|
||||
_id: docHash,
|
||||
memory_vectors: serializedMemoryVectors,
|
||||
file_mtime: noteFile.mtime,
|
||||
embeddingModel: embeddingModel,
|
||||
created_at: Date.now(),
|
||||
_rev: existingDoc?._rev, // Add the current revision if the document exists.
|
||||
};
|
||||
|
||||
// Save the document.
|
||||
await db.put(docToSave);
|
||||
return docToSave;
|
||||
} catch (err) {
|
||||
console.error("Error storing vectors in VectorDB:", err);
|
||||
}
|
||||
}
|
||||
|
||||
public static async getNoteFiles(db: PouchDB.Database): Promise<NoteFile[]> {
|
||||
if (!db) throw new Error("DB not initialized");
|
||||
try {
|
||||
const allDocsResponse = await db.allDocs<VectorStoreDocument>({
|
||||
include_docs: true,
|
||||
});
|
||||
const allDocs = allDocsResponse.rows.map((row) => row.doc as VectorStoreDocument);
|
||||
const memoryVectors = allDocs
|
||||
.map((doc) => JSON.parse(doc.memory_vectors) as MemoryVector[])
|
||||
.filter((memoryVectors) => memoryVectors.length > 0);
|
||||
|
||||
const noteFiles = memoryVectors
|
||||
.map((memoryVectors, i) => {
|
||||
const doc = allDocs[i];
|
||||
const noteFile: NoteFile = {
|
||||
path: memoryVectors[0].metadata.path,
|
||||
basename: memoryVectors[0].metadata.title,
|
||||
mtime: doc.file_mtime,
|
||||
content: memoryVectors[0].content,
|
||||
metadata: memoryVectors[0].metadata,
|
||||
};
|
||||
return noteFile;
|
||||
})
|
||||
.filter((noteFile): noteFile is NoteFile => noteFile !== undefined);
|
||||
return noteFiles;
|
||||
} catch (err) {
|
||||
console.error("Error getting note files from VectorDB:", err);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
public static async removeMemoryVectors(db: PouchDB.Database, docHash: string): Promise<void> {
|
||||
if (!db) throw new Error("DB not initialized");
|
||||
try {
|
||||
const doc = await db.get(docHash);
|
||||
if (doc) {
|
||||
await db.remove(doc);
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
try {
|
||||
// Apply rate limiting before making the API call
|
||||
await this.getRateLimiter().wait();
|
||||
const embedding = await embeddingsAPI.embedDocuments([chunks[i].pageContent]);
|
||||
if (embedding.length > 0) {
|
||||
docVectors.push(embedding[0]);
|
||||
} else {
|
||||
console.error("indexFile - Empty embedding for chunk:", {
|
||||
index: i,
|
||||
length: chunks[i].pageContent.length,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("indexFile - Error during embeddings API call for chunk:", {
|
||||
index: i,
|
||||
length: chunks[i].pageContent.length,
|
||||
error: error,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("indexFile - Unexpected error during embedding process:", error);
|
||||
}
|
||||
|
||||
const chunkWithVectors =
|
||||
docVectors.length > 0
|
||||
? chunks.map((chunk, i) => ({
|
||||
id: VectorDBManager.getDocHash(chunk.pageContent),
|
||||
content: chunk.pageContent,
|
||||
embedding: docVectors[i],
|
||||
}))
|
||||
: [];
|
||||
|
||||
for (const chunkWithVector of chunkWithVectors) {
|
||||
try {
|
||||
// Prepare the document to be saved.
|
||||
const docToSave: OramaDocument = {
|
||||
id: chunkWithVector.id,
|
||||
title: fileToSave.title,
|
||||
content: chunkWithVector.content,
|
||||
embedding: chunkWithVector.embedding,
|
||||
path: fileToSave.path,
|
||||
embeddingModel: fileToSave.embeddingModel,
|
||||
created_at: Date.now(),
|
||||
ctime: fileToSave.ctime,
|
||||
mtime: fileToSave.mtime,
|
||||
tags: Array.isArray(fileToSave.tags) ? fileToSave.tags : [],
|
||||
extension: fileToSave.extension,
|
||||
nchars: chunkWithVector.content.length,
|
||||
metadata: fileToSave.metadata,
|
||||
};
|
||||
|
||||
// Ensure tags are strings
|
||||
docToSave.tags = docToSave.tags.map((tag: any) => String(tag));
|
||||
// Save the document.
|
||||
await this.upsert(db, docToSave);
|
||||
} catch (err) {
|
||||
console.error("Error storing vectors in VectorDB:", err);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Error removing file from VectorDB:", err);
|
||||
}
|
||||
}
|
||||
|
||||
public static async getLatestFileMtime(db: PouchDB.Database): Promise<number> {
|
||||
public static async upsert(db: Orama<any>, docToSave: any): Promise<any | undefined> {
|
||||
if (!db) throw new Error("DB not initialized");
|
||||
if (!this.config) throw new Error("VectorDBManager not initialized");
|
||||
|
||||
// If the document already exists, update it.
|
||||
// Otherwise, insert it.
|
||||
try {
|
||||
await insert(db, docToSave);
|
||||
} catch (err) {
|
||||
console.error(`Error inserting document ${docToSave.id} in VectorDB:`, err);
|
||||
await update(db, docToSave, {
|
||||
id: docToSave.id,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public static async getDocsByPath(db: Orama<any>, path: string): Promise<any | undefined> {
|
||||
if (!db) throw new Error("DB not initialized");
|
||||
if (!this.config) throw new Error("VectorDBManager not initialized");
|
||||
|
||||
const result = await search(db, {
|
||||
term: path,
|
||||
properties: ["path"],
|
||||
limit: 100,
|
||||
includeVectors: true,
|
||||
});
|
||||
return result.hits;
|
||||
}
|
||||
|
||||
public static async getLatestFileMtime(db: Orama<any>): Promise<number> {
|
||||
if (!db) throw new Error("DB not initialized");
|
||||
|
||||
try {
|
||||
const allDocsResponse = await db.allDocs<VectorStoreDocument>({
|
||||
include_docs: true,
|
||||
const result = await search(db, {
|
||||
term: "",
|
||||
limit: 1,
|
||||
sortBy: {
|
||||
property: "mtime",
|
||||
order: "DESC",
|
||||
},
|
||||
});
|
||||
const allDocs = allDocsResponse.rows.map((row) => row.doc as VectorStoreDocument);
|
||||
const newestFileMtime = allDocs.map((doc) => doc.file_mtime).sort((a, b) => b - a)[0];
|
||||
return newestFileMtime;
|
||||
|
||||
if (result.hits.length > 0) {
|
||||
const latestDoc = result.hits[0].document as any;
|
||||
return latestDoc.mtime;
|
||||
}
|
||||
|
||||
return 0; // Return 0 if no documents found
|
||||
} catch (err) {
|
||||
console.error("Error getting newest file mtime from VectorDB:", err);
|
||||
console.error("Error getting latest file mtime from VectorDB:", err);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
public static async checkEmbeddingModel(db: PouchDB.Database): Promise<string | undefined> {
|
||||
if (!db) throw new Error("DB not initialized");
|
||||
|
||||
try {
|
||||
// Fetch all documents
|
||||
const allDocsResponse = await db.allDocs<VectorStoreDocument>({
|
||||
include_docs: true,
|
||||
});
|
||||
const allDocs = allDocsResponse.rows.map((row) => row.doc as VectorStoreDocument);
|
||||
|
||||
// Check if there are any documents
|
||||
if (allDocs.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Extract the embeddingModel from all documents
|
||||
const embeddingModels = allDocs.map((doc) => doc.embeddingModel);
|
||||
|
||||
// Check if all documents have the same embeddingModel
|
||||
const allSame = embeddingModels.every((model) => model === embeddingModels[0]);
|
||||
return allSame ? embeddingModels[0] : undefined;
|
||||
} catch (err) {
|
||||
console.error("Error checking last embedding model from VectorDB:", err);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
public static async getMemoryVectors(
|
||||
db: PouchDB.Database,
|
||||
docHash: string
|
||||
): Promise<MemoryVector[] | undefined> {
|
||||
if (!db) throw new Error("DB not initialized");
|
||||
try {
|
||||
const doc: VectorStoreDocument = await db.get(docHash);
|
||||
if (doc && doc.memory_vectors) {
|
||||
return JSON.parse(doc.memory_vectors);
|
||||
}
|
||||
} catch (err) {
|
||||
console.log("No vectors found in VectorDB for dochash:", docHash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default VectorDBManager;
|
||||
|
|
|
|||
Loading…
Reference in a new issue