= ({
QA Settings
-
- Vault QA is in BETA and may not be stable. If you have issues please report in the github
- repo.
-
QA mode relies on a local vector index.
- Long Note QA vs. Vault QA (BETA)
-
- Long Note QA mode uses the Active Note as context. Vault QA (BETA) uses your entire vault as
- context. Please ask questions as specific as possible, avoid vague questions to get better
- results.
-
Local Embedding Model
Check the{" "}
@@ -85,9 +75,6 @@ const QASettings: React.FC = ({
If you are using a paid embedding provider, beware of costs for large vaults!
- When you switch to Long Note QA mode, your active note is indexed
- automatically upon mode switch.
-
When you switch to Vault QA mode, your vault is indexed{" "}
based on the auto-index strategy you select below.
diff --git a/src/utils.ts b/src/utils.ts
index 39a0af92..a83c3546 100644
--- a/src/utils.ts
+++ b/src/utils.ts
@@ -157,8 +157,6 @@ export const stringToChainType = (chain: string): ChainType => {
switch (chain) {
case "llm_chain":
return ChainType.LLM_CHAIN;
- case "long_note_qa":
- return ChainType.LONG_NOTE_QA_CHAIN;
case "vault_qa":
return ChainType.VAULT_QA_CHAIN;
default:
diff --git a/src/vectorDBManager.ts b/src/vectorDBManager.ts
index 9a624028..c1c8bae4 100644
--- a/src/vectorDBManager.ts
+++ b/src/vectorDBManager.ts
@@ -1,40 +1,25 @@
import EmbeddingManager from "@/LLMProviders/embeddingManager";
import { RateLimiter } from "@/rateLimiter";
import { Embeddings } from "@langchain/core/embeddings";
+import { insert, Orama, search, update } from "@orama/orama";
import { MD5 } from "crypto-js";
-import { Document } from "langchain/document";
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
-import { MemoryVectorStore } from "langchain/vectorstores/memory";
-// TODOs
-// 1. Use embeddingModel rather than embeddingProvider
-// 2. embeddingModel should be on MemoryVector metadata
-// 3. VectorDBManager should have a public method checking existing embeddingModel on a MemoryVector
-export interface VectorStoreDocument {
- _id: string;
- _rev?: string;
- memory_vectors: string;
- file_mtime: number;
- embeddingModel: string;
- created_at: number;
-}
-
-export interface MemoryVector {
+export interface OramaDocument {
+ id: string;
+ title: string;
content: string;
embedding: number[];
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- metadata: Record;
-}
-
-export interface NoteFile {
path: string;
- basename: string;
+ embeddingModel: string;
+ created_at: number;
+ ctime: number;
mtime: number;
- content: string;
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ tags: string[];
+ extension: string;
+ nchars: number;
metadata: Record;
}
-
interface VectorDBConfig {
getEmbeddingRequestsPerSecond: () => number;
debug: boolean;
@@ -58,106 +43,16 @@ class VectorDBManager {
}
return this.rateLimiter;
}
- public static getDocumentHash(sourceDocument: string): string {
+
+ public static getDocHash(sourceDocument: string): string {
return MD5(sourceDocument).toString();
}
- public static async rebuildMemoryVectorStore(
- memoryVectors: MemoryVector[],
- embeddingsAPI: Embeddings
- ) {
- if (!Array.isArray(memoryVectors)) {
- throw new TypeError("Expected memoryVectors to be an array");
- }
- // Extract the embeddings and documents from the deserialized memoryVectors
- const embeddingsArray: number[][] = memoryVectors.map((memoryVector) => memoryVector.embedding);
- const documentsArray = memoryVectors.map(
- (memoryVector) =>
- new Document({
- pageContent: memoryVector.content,
- metadata: memoryVector.metadata,
- })
- );
-
- // Create a new MemoryVectorStore instance
- const memoryVectorStore = new MemoryVectorStore(embeddingsAPI);
- await memoryVectorStore.addVectors(embeddingsArray, documentsArray);
- return memoryVectorStore;
- }
-
- public static async getMemoryVectorStore(
- db: PouchDB.Database,
- embeddingsAPI: Embeddings,
- docHash?: string
- ): Promise {
- if (!db) throw new Error("DB not initialized");
-
- let allDocsResponse;
- if (docHash) {
- // Fetch a single document by its _id
- const doc = await db.get(docHash);
- allDocsResponse = { rows: [{ doc }] };
- } else {
- // Fetch all documents
- allDocsResponse = await db.allDocs({ include_docs: true });
- }
-
- const embeddingModel = EmbeddingManager.getModelName(embeddingsAPI);
- if (!embeddingModel) console.error("EmbeddingManager could not determine model name!");
- const allDocs = allDocsResponse.rows
- .map((row) => row.doc as VectorStoreDocument)
- .filter((doc) => doc.embeddingModel === embeddingModel);
- const memoryVectors = allDocs
- .map((doc) => JSON.parse(doc.memory_vectors) as MemoryVector[])
- .flat();
- const embeddingsArray: number[][] = memoryVectors.map((memoryVector) => memoryVector.embedding);
- const documentsArray = memoryVectors.map(
- (memoryVector) =>
- new Document({
- pageContent: memoryVector.content,
- metadata: memoryVector.metadata,
- })
- );
- // Create a new MemoryVectorStore instance
- const memoryVectorStore = new MemoryVectorStore(embeddingsAPI);
- await memoryVectorStore.addVectors(embeddingsArray, documentsArray);
- return memoryVectorStore;
- }
-
- public static async setMemoryVectors(
- db: PouchDB.Database,
- memoryVectors: MemoryVector[],
- docHash: string
- ): Promise {
- if (!db) throw new Error("DB not initialized");
- if (!Array.isArray(memoryVectors)) {
- throw new TypeError("Expected memoryVectors to be an array");
- }
- const serializedMemoryVectors = JSON.stringify(memoryVectors);
- try {
- // Attempt to fetch the existing document, if it exists.
- const existingDoc = await db.get(docHash).catch((err) => null);
-
- // Prepare the document to be saved.
- const docToSave = {
- _id: docHash,
- memory_vectors: serializedMemoryVectors,
- created_at: Date.now(),
- _rev: existingDoc?._rev, // Add the current revision if the document exists.
- };
-
- // Save the document.
- await db.put(docToSave);
- } catch (err) {
- console.error("Error storing vectors in VectorDB:", err);
- }
- }
-
public static async indexFile(
- db: PouchDB.Database,
+ db: Orama,
embeddingsAPI: Embeddings,
- noteFile: NoteFile
- ): Promise {
+ fileToSave: any
+ ): Promise {
if (!db) throw new Error("DB not initialized");
if (!this.config) throw new Error("VectorDBManager not initialized");
@@ -170,151 +65,129 @@ class VectorDBManager {
});
// Add note title as contextual chunk headers
// https://js.langchain.com/docs/modules/data_connection/document_transformers/contextual_chunk_headers
- const splitDocument = await textSplitter.createDocuments([noteFile.content], [], {
- chunkHeader: "[[" + noteFile.basename + "]]" + "\n\n---\n\n",
+ const chunks = await textSplitter.createDocuments([fileToSave.content], [], {
+ chunkHeader: "[[" + fileToSave.title + "]]" + "\n\n---\n\n",
appendChunkOverlapHeader: true,
});
- // Apply rate limiting before making the API call
- await this.getRateLimiter().wait();
- const docVectors = await embeddingsAPI.embedDocuments(
- splitDocument.map((doc) => doc.pageContent)
- );
-
- const memoryVectors = docVectors.map((docVector, i) => ({
- content: splitDocument[i].pageContent,
- metadata: {
- ...noteFile.metadata,
- title: noteFile.basename,
- path: noteFile.path,
- },
- embedding: docVector,
- }));
- const docHash = VectorDBManager.getDocumentHash(noteFile.path);
-
- const serializedMemoryVectors = JSON.stringify(memoryVectors);
+ const docVectors: number[][] = [];
try {
- // Attempt to fetch the existing document, if it exists.
- const existingDoc = await db.get(docHash).catch((err) => null);
-
- // Prepare the document to be saved.
- const docToSave: VectorStoreDocument = {
- _id: docHash,
- memory_vectors: serializedMemoryVectors,
- file_mtime: noteFile.mtime,
- embeddingModel: embeddingModel,
- created_at: Date.now(),
- _rev: existingDoc?._rev, // Add the current revision if the document exists.
- };
-
- // Save the document.
- await db.put(docToSave);
- return docToSave;
- } catch (err) {
- console.error("Error storing vectors in VectorDB:", err);
- }
- }
-
- public static async getNoteFiles(db: PouchDB.Database): Promise {
- if (!db) throw new Error("DB not initialized");
- try {
- const allDocsResponse = await db.allDocs({
- include_docs: true,
- });
- const allDocs = allDocsResponse.rows.map((row) => row.doc as VectorStoreDocument);
- const memoryVectors = allDocs
- .map((doc) => JSON.parse(doc.memory_vectors) as MemoryVector[])
- .filter((memoryVectors) => memoryVectors.length > 0);
-
- const noteFiles = memoryVectors
- .map((memoryVectors, i) => {
- const doc = allDocs[i];
- const noteFile: NoteFile = {
- path: memoryVectors[0].metadata.path,
- basename: memoryVectors[0].metadata.title,
- mtime: doc.file_mtime,
- content: memoryVectors[0].content,
- metadata: memoryVectors[0].metadata,
- };
- return noteFile;
- })
- .filter((noteFile): noteFile is NoteFile => noteFile !== undefined);
- return noteFiles;
- } catch (err) {
- console.error("Error getting note files from VectorDB:", err);
- return [];
- }
- }
-
- public static async removeMemoryVectors(db: PouchDB.Database, docHash: string): Promise {
- if (!db) throw new Error("DB not initialized");
- try {
- const doc = await db.get(docHash);
- if (doc) {
- await db.remove(doc);
+ for (let i = 0; i < chunks.length; i++) {
+ try {
+ // Apply rate limiting before making the API call
+ await this.getRateLimiter().wait();
+ const embedding = await embeddingsAPI.embedDocuments([chunks[i].pageContent]);
+ if (embedding.length > 0) {
+ docVectors.push(embedding[0]);
+ } else {
+ console.error("indexFile - Empty embedding for chunk:", {
+ index: i,
+ length: chunks[i].pageContent.length,
+ });
+ }
+ } catch (error) {
+ console.error("indexFile - Error during embeddings API call for chunk:", {
+ index: i,
+ length: chunks[i].pageContent.length,
+ error: error,
+ });
+ }
+ }
+ } catch (error) {
+ console.error("indexFile - Unexpected error during embedding process:", error);
+ }
+
+ const chunkWithVectors =
+ docVectors.length > 0
+ ? chunks.map((chunk, i) => ({
+ id: VectorDBManager.getDocHash(chunk.pageContent),
+ content: chunk.pageContent,
+ embedding: docVectors[i],
+ }))
+ : [];
+
+ for (const chunkWithVector of chunkWithVectors) {
+ try {
+ // Prepare the document to be saved.
+ const docToSave: OramaDocument = {
+ id: chunkWithVector.id,
+ title: fileToSave.title,
+ content: chunkWithVector.content,
+ embedding: chunkWithVector.embedding,
+ path: fileToSave.path,
+ embeddingModel: fileToSave.embeddingModel,
+ created_at: Date.now(),
+ ctime: fileToSave.ctime,
+ mtime: fileToSave.mtime,
+ tags: Array.isArray(fileToSave.tags) ? fileToSave.tags : [],
+ extension: fileToSave.extension,
+ nchars: chunkWithVector.content.length,
+ metadata: fileToSave.metadata,
+ };
+
+ // Ensure tags are strings
+ docToSave.tags = docToSave.tags.map((tag: any) => String(tag));
+ // Save the document.
+ await this.upsert(db, docToSave);
+ } catch (err) {
+ console.error("Error storing vectors in VectorDB:", err);
}
- } catch (err) {
- console.error("Error removing file from VectorDB:", err);
}
}
- public static async getLatestFileMtime(db: PouchDB.Database): Promise {
+ public static async upsert(db: Orama, docToSave: any): Promise {
+ if (!db) throw new Error("DB not initialized");
+ if (!this.config) throw new Error("VectorDBManager not initialized");
+
+ // If the document already exists, update it.
+ // Otherwise, insert it.
+ try {
+ await insert(db, docToSave);
+ } catch (err) {
+ console.error(`Error inserting document ${docToSave.id} in VectorDB:`, err);
+ await update(db, docToSave, {
+ id: docToSave.id,
+ });
+ }
+ }
+
+ public static async getDocsByPath(db: Orama, path: string): Promise {
+ if (!db) throw new Error("DB not initialized");
+ if (!this.config) throw new Error("VectorDBManager not initialized");
+
+ const result = await search(db, {
+ term: path,
+ properties: ["path"],
+ limit: 100,
+ includeVectors: true,
+ });
+ return result.hits;
+ }
+
+ public static async getLatestFileMtime(db: Orama): Promise {
if (!db) throw new Error("DB not initialized");
try {
- const allDocsResponse = await db.allDocs({
- include_docs: true,
+ const result = await search(db, {
+ term: "",
+ limit: 1,
+ sortBy: {
+ property: "mtime",
+ order: "DESC",
+ },
});
- const allDocs = allDocsResponse.rows.map((row) => row.doc as VectorStoreDocument);
- const newestFileMtime = allDocs.map((doc) => doc.file_mtime).sort((a, b) => b - a)[0];
- return newestFileMtime;
+
+ if (result.hits.length > 0) {
+ const latestDoc = result.hits[0].document as any;
+ return latestDoc.mtime;
+ }
+
+ return 0; // Return 0 if no documents found
} catch (err) {
- console.error("Error getting newest file mtime from VectorDB:", err);
+ console.error("Error getting latest file mtime from VectorDB:", err);
return 0;
}
}
-
- public static async checkEmbeddingModel(db: PouchDB.Database): Promise {
- if (!db) throw new Error("DB not initialized");
-
- try {
- // Fetch all documents
- const allDocsResponse = await db.allDocs({
- include_docs: true,
- });
- const allDocs = allDocsResponse.rows.map((row) => row.doc as VectorStoreDocument);
-
- // Check if there are any documents
- if (allDocs.length === 0) {
- return undefined;
- }
-
- // Extract the embeddingModel from all documents
- const embeddingModels = allDocs.map((doc) => doc.embeddingModel);
-
- // Check if all documents have the same embeddingModel
- const allSame = embeddingModels.every((model) => model === embeddingModels[0]);
- return allSame ? embeddingModels[0] : undefined;
- } catch (err) {
- console.error("Error checking last embedding model from VectorDB:", err);
- return undefined;
- }
- }
-
- public static async getMemoryVectors(
- db: PouchDB.Database,
- docHash: string
- ): Promise {
- if (!db) throw new Error("DB not initialized");
- try {
- const doc: VectorStoreDocument = await db.get(docHash);
- if (doc && doc.memory_vectors) {
- return JSON.parse(doc.memory_vectors);
- }
- } catch (err) {
- console.log("No vectors found in VectorDB for dochash:", docHash);
- }
- }
}
export default VectorDBManager;
