From eb47b7bcd970e6fd79c357725cb7fdc2e214f201 Mon Sep 17 00:00:00 2001
From: Logan Yang <logancyang@gmail.com>
Date: Thu, 19 Dec 2024 12:01:49 -0800
Subject: [PATCH] Add metadata context (#946)

* Do not throw errors and allow partial features
* Add created and modified time to chunk
---
 src/main.ts                      |  1 -
 src/search/dbOperations.ts       |  9 +++++----
 src/search/indexOperations.ts    | 17 ++++++++++++++---
 src/search/vectorStoreManager.ts |  7 +++----
 4 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/src/main.ts b/src/main.ts
index 0a4047ea..4ae8b804 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -294,7 +294,6 @@ export default class CopilotPlugin extends Plugin {
       name: "Force re-index vault for QA",
       callback: async () => {
         try {
-          await this.vectorStoreManager.clearIndex();
           const indexedFileCount = await this.vectorStoreManager.indexVaultToVectorStore(true);
 
           new Notice(`${indexedFileCount} vault files re-indexed to Copilot index.`);
diff --git a/src/search/dbOperations.ts b/src/search/dbOperations.ts
index 6ae9d86b..2be5f28f 100644
--- a/src/search/dbOperations.ts
+++ b/src/search/dbOperations.ts
@@ -104,10 +104,8 @@ export class DBOperations {
       return newDb;
     } catch (error) {
       console.error(`Error initializing Orama database:`, error);
-      if (error instanceof CustomError) {
-        throw error;
-      }
-      throw new CustomError(`Failed to initialize database: ${error.message}`);
+      new Notice("Failed to initialize Copilot database. Some features may be limited.");
+      return undefined;
     }
   }
 
@@ -181,6 +179,9 @@ export class DBOperations {
   }
 
   public getDb(): Orama<any> | undefined {
+    if (!this.oramaDb) {
+      console.warn("Database not initialized. Some features may be limited.");
+    }
     return this.oramaDb;
   }
 
diff --git a/src/search/indexOperations.ts b/src/search/indexOperations.ts
index f5332d64..14908f16 100644
--- a/src/search/indexOperations.ts
+++ b/src/search/indexOperations.ts
@@ -3,6 +3,7 @@ import { CustomError } from "@/error";
 import EmbeddingsManager from "@/LLMProviders/embeddingManager";
 import { RateLimiter } from "@/rateLimiter";
 import { getSettings, subscribeToSettingsChange } from "@/settings/model";
+import { formatDateTime } from "@/utils";
 import { Embeddings } from "@langchain/core/embeddings";
 import { MD5 } from "crypto-js";
 import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
@@ -62,7 +63,11 @@ export class IndexOperations {
       mtime: file.stat.mtime,
       tags: fileCache?.tags?.map((tag) => tag.tag) ?? [],
       extension: file.extension,
-      metadata: fileCache?.frontmatter ?? {},
+      metadata: {
+        ...(fileCache?.frontmatter ?? {}),
+        created: formatDateTime(new Date(file.stat.ctime)).display,
+        modified: formatDateTime(new Date(file.stat.mtime)).display,
+      },
     };
 
     await this.indexDocument(embeddingInstance, fileToSave);
@@ -79,7 +84,9 @@ export class IndexOperations {
     // Add note title as contextual chunk headers
     // https://js.langchain.com/docs/modules/data_connection/document_transformers/contextual_chunk_headers
     const chunks = await textSplitter.createDocuments([fileToSave.content], [], {
-      chunkHeader: `\n\nNOTE TITLE: [[${fileToSave.title}]]\n\nNOTE BLOCK CONTENT:\n\n`,
+      chunkHeader: `\n\nNOTE TITLE: [[${fileToSave.title}]]\n\nMETADATA:${JSON.stringify(
+        fileToSave.metadata
+      )}\n\nNOTE BLOCK CONTENT:\n\n`,
       appendChunkOverlapHeader: true,
     });
 
@@ -401,7 +408,11 @@ export class IndexOperations {
         mtime: file.stat.mtime,
         tags: fileCache?.tags?.map((tag) => tag.tag) ?? [],
         extension: file.extension,
-        metadata: fileCache?.frontmatter ?? {},
+        metadata: {
+          ...(fileCache?.frontmatter ?? {}),
+          created: file.stat.ctime,
+          modified: file.stat.mtime,
+        },
       };
 
       await this.indexDocument(embeddingInstance, fileToSave);
diff --git a/src/search/vectorStoreManager.ts b/src/search/vectorStoreManager.ts
index d01a130b..c8a28f5b 100644
--- a/src/search/vectorStoreManager.ts
+++ b/src/search/vectorStoreManager.ts
@@ -57,7 +57,6 @@ export default class VectorStoreManager {
 
   private async initialize(): Promise<void> {
     try {
-      // Add retry logic for initialization
       let retries = 3;
       while (retries > 0) {
         try {
@@ -70,17 +69,17 @@ export default class VectorStoreManager {
           ) {
             retries--;
             if (retries > 0) {
-              await new Promise((resolve) => setTimeout(resolve, 100)); // Wait 0.1 second before retry
+              await new Promise((resolve) => setTimeout(resolve, 100));
               continue;
             }
           }
-          throw error;
+          console.error("Failed to initialize vector store:", error);
+          break;
         }
       }
       this.eventHandler.initializeEventListeners();
     } catch (error) {
       console.error("Failed to initialize vector store:", error);
-      throw error;
     }
   }