Fix query embedding overflow (#707)

src/VectorStoreManager.ts

@@ -162,6 +162,18 @@ class VectorStoreManager {
     return sampleEmbedding.length;
   }
 
+  private async ensureCorrectSchema(db: Orama<any>, embeddingInstance: Embeddings): Promise<void> {
+    const currentVectorLength = await this.getVectorLength(embeddingInstance);
+    const dbSchema = db.schema;
+
+    if (dbSchema.embedding !== `vector[${currentVectorLength}]`) {
+      console.log(
+        `Schema mismatch detected. Rebuilding database with new vector length: ${currentVectorLength}`
+      );
+      await this.clearVectorStore();
+    }
+  }
+
   private async saveDB() {
     try {
       const rawData = await save(this.oramaDb);
@@ -269,6 +281,7 @@ class VectorStoreManager {
       if (!embeddingInstance) {
         throw new CustomError("Embedding instance not found.");
       }
+      await this.ensureCorrectSchema(this.oramaDb, embeddingInstance);
 
       const singleDoc = await search(this.getDb(), {
         term: "",

src/constants.ts

@@ -6,6 +6,7 @@ export const USER_SENDER = "user";
 export const AI_SENDER = "ai";
 export const DEFAULT_SYSTEM_PROMPT =
   "You are Obsidian Copilot, a helpful assistant that integrates AI to Obsidian note-taking.";
+export const CHUNK_SIZE = 5000;
 
 export enum ChatModels {
   GPT_4o = "gpt-4o",

src/main.ts

@@ -14,6 +14,7 @@ import {
   BUILTIN_CHAT_MODELS,
   BUILTIN_EMBEDDING_MODELS,
   CHAT_VIEWTYPE,
+  CHUNK_SIZE,
   DEFAULT_SETTINGS,
   DEFAULT_SYSTEM_PROMPT,
   EVENT_NAMES,
@@ -696,7 +697,10 @@ export default class CopilotPlugin extends Plugin {
       this.settings.debug
     );
 
-    const similarDocs = await hybridRetriever.getRelevantDocuments(content, { runName: "no_hyde" });
+    const truncatedContent = content.length > CHUNK_SIZE ? content.slice(0, CHUNK_SIZE) : content;
+    const similarDocs = await hybridRetriever.getRelevantDocuments(truncatedContent, {
+      runName: "no_hyde",
+    });
     return similarDocs
       .filter((doc) => doc.metadata.path !== activeFilePath)
       .map((doc) => ({

src/search/hybridRetriever.ts

@@ -135,8 +135,18 @@ export class HybridRetriever extends BaseRetriever {
   }
 
   private async getOramaChunks(query: string): Promise<Document[]> {
-    // Assuming you have a method to convert the query to a vector
-    const queryVector = await this.convertQueryToVector(query);
+    let queryVector: number[];
+    try {
+      queryVector = await this.convertQueryToVector(query);
+    } catch (error) {
+      console.error(
+        "Error in convertQueryToVector, please ensure your embedding model is working and has an adequate context length:",
+        error,
+        "\nQuery:",
+        query
+      );
+      throw error;
+    }
 
     const searchResults = await search(this.db, {
       mode: "vector",

src/vectorDBManager.ts

@@ -1,9 +1,11 @@
+import { CHUNK_SIZE } from "@/constants";
 import EmbeddingManager from "@/LLMProviders/embeddingManager";
 import { RateLimiter } from "@/rateLimiter";
 import { Embeddings } from "@langchain/core/embeddings";
 import { insert, Orama, search, update } from "@orama/orama";
 import { MD5 } from "crypto-js";
 import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
+import { Notice } from "obsidian";
 
 export interface OramaDocument {
   id: string;
@@ -61,7 +63,7 @@ class VectorDBManager {
 
     // Markdown splitter: https://js.langchain.com/docs/modules/data_connection/document_transformers/code_splitter#markdown
     const textSplitter = RecursiveCharacterTextSplitter.fromLanguage("markdown", {
-      chunkSize: 5000,
+      chunkSize: CHUNK_SIZE,
     });
     // Add note title as contextual chunk headers
     // https://js.langchain.com/docs/modules/data_connection/document_transformers/contextual_chunk_headers
@@ -91,10 +93,22 @@ class VectorDBManager {
             length: chunks[i].pageContent.length,
             error: error,
           });
+
+          // Check if the error is related to context length
+          if (error instanceof Error) {
+            new Notice(
+              `Embedding error: please check your embedding model context length, and consider switching to a model with a larger context length: ${error.message}`
+            );
+          }
+
+          // Rethrow the error to stop the indexing process
+          throw error;
         }
       }
     } catch (error) {
       console.error("indexFile - Unexpected error during embedding process:", error);
+      // Rethrow the error to be handled by the caller
+      throw error;
     }
 
     const chunkWithVectors =
@@ -139,15 +153,32 @@ class VectorDBManager {
     if (!db) throw new Error("DB not initialized");
     if (!this.config) throw new Error("VectorDBManager not initialized");
 
-    // If the document already exists, update it.
-    // Otherwise, insert it.
     try {
-      await insert(db, docToSave);
-    } catch (err) {
-      console.error(`Error inserting document ${docToSave.id} in VectorDB:`, err);
-      await update(db, docToSave, {
-        id: docToSave.id,
+      // Check if the document already exists
+      const existingDoc = await search(db, {
+        term: docToSave.id,
+        properties: ["id"],
+        limit: 1,
       });
+
+      if (existingDoc.hits.length > 0) {
+        // Document exists, update it
+        await update(db, docToSave, {
+          id: docToSave.id,
+        });
+        if (this.config.debug) {
+          console.log(`Updated document ${docToSave.id} in VectorDB`);
+        }
+      } else {
+        // Document doesn't exist, insert it
+        await insert(db, docToSave);
+        if (this.config.debug) {
+          console.log(`Inserted document ${docToSave.id} in VectorDB`);
+        }
+      }
+    } catch (err) {
+      console.error(`Error upserting document ${docToSave.id} in VectorDB:`, err);
+      throw err; // Rethrow the error to be handled by the caller
     }
   }