Merge pull request #5 from Beim/feat/hash-file

Incremental update embedding data

manifest.json

@@ -1,7 +1,7 @@
 {
 	"id": "gpt-assistant",
 	"name": "GPT Assistant",
-	"version": "0.1.2",
+	"version": "0.1.3",
 	"minAppVersion": "0.15.0",
 	"description": "Use a GPT-3 based model on your notes and get personalized answers from your knowledge base.",
 	"author": "M7mdisk",

src/assistant.ts

@@ -5,10 +5,16 @@ import { cosineSimilarity } from "./utils";
 export interface chunkData {
 	text: string;
 	embeddings: number[];
+	sha1: string;
 }
 
 export type EmbeddedData = chunkData[];
 
+export interface CachedData {
+	searchable: EmbeddedData;
+	sha: Array<string>;
+}
+
 export type Answer = { error: boolean; text: string };
 export class Assistant {
 	MAX_TOKENS = 500;
@@ -78,13 +84,20 @@ export class Assistant {
 		};
 	}
 
-	prepareTexts(texts: string[]): string[] {
+	prepareTexts(texts: EmbeddedData): EmbeddedData {
-		let shortened: string[] = [];
+		let shortened: EmbeddedData = [];
-		texts.forEach((text) => {
+		texts.forEach((item) => {
+			const text = item.text;
 			if (this.tokenizer.encode(text).bpe.length > this.MAX_TOKENS) {
-				shortened = shortened.concat(this.splitIntoMany(text));
+				shortened = shortened.concat(this.splitIntoMany(text).map(t => {
+					return {
+						text: t,
+						embeddings: [],
+						sha1: item.sha1,
+					}
+				}));
 			} else {
-				shortened.push(text);
+				shortened.push(item);
 			}
 		});
 		return shortened;
@@ -112,14 +125,15 @@ export class Assistant {
 		});
 		return chunks;
 	}
-	async createEmbeddings(data: string[]): Promise<EmbeddedData> {
+	async createEmbeddings(data: EmbeddedData): Promise<EmbeddedData> {
 		const embeddings = await this.openai.createEmbedding({
-			input: data,
+			input: data.map((d) => d.text),
 			model: "text-embedding-ada-002",
 		});
-		return data.map((text, idx) => ({
+		return data.map((d, idx) => ({
-			text,
+			text: d.text,
 			embeddings: embeddings.data.data[idx].embedding,
+			sha1: d.sha1,
 		}));
 	}
 }

src/main.ts

@@ -1,4 +1,5 @@
-import { Answer, Assistant } from "./assistant";
+import { Answer, Assistant, EmbeddedData, CachedData } from "./assistant";
+import { sha1File } from "./utils";
 import {
 	App,
 	MarkdownRenderer,
@@ -12,10 +13,12 @@ import {
 
 interface PluginSettings {
 	apiKey: string;
+	autoUpdate: boolean;
 }
 
 const DEFAULT_SETTINGS: PluginSettings = {
 	apiKey: "",
+	autoUpdate: false,
 };
 
 export default class GPTAssistantPlugin extends Plugin {
@@ -28,7 +31,6 @@ export default class GPTAssistantPlugin extends Plugin {
 		this.assistant = new Assistant(this.settings.apiKey);
 		if (await this.hasCachedData()) {
 			const { searchable } = await this.loadData();
-			this.saveNamedData("searchable", searchable);
 			this.assistant.setData(searchable);
 		}
 
@@ -46,14 +48,42 @@ export default class GPTAssistantPlugin extends Plugin {
 					new Notice("Please provide an API Key in the settings");
 					return;
 				}
+				if (this.settings.autoUpdate) {
+					this.loadEmbeddingsToAssistant(); // async update embedding
+				}
 				new AskAssistantModal(this.app, async (question) => {
+					try {
 						const answer = await this.assistant.answerQuestion(
 							question
 						);
-					return answer ?? "";
+						return answer;
+					} catch (e) {
+						if (e.response) {
+							console.error(e.response)
+							new Notice("❌ " + e.response.data.error.message)
+						}
+						return { error: true, text: "" }
+					}
 				}).open();
 			},
 		});
+
+		this.addCommand({
+			id: "update-assistant",
+			name: "Update assistant",
+			callback: async () => {
+				if (!this.settings.apiKey) {
+					new Notice("Please provide an API Key in the settings");
+					return;
+				}
+				new Notice(
+					"Loading data into model. this could take a while..."
+				);
+				await this.loadEmbeddingsToAssistant();
+				new Notice("Your data has been loaded into the model.");
+
+			},
+		});
 	}
 
 	private async hasCachedData(): Promise<boolean> {
@@ -61,22 +91,68 @@ export default class GPTAssistantPlugin extends Plugin {
 		return data && data.searchable && data.searchable.length;
 	}
 
+	private async loadCachedData(): Promise<CachedData> {
+		const data = await this.loadData();
+		if (data && data.searchable && data.searchable.length &&
+			data.sha && data.sha.length) {
+			return {
+				searchable: data.searchable,
+				sha: data.sha,
+			}
+		}
+		return {
+			searchable: [],
+			sha: [],
+		}
+	}
+
 	async loadEmbeddingsToAssistant() {
 		const { vault } = this.app;
-		const fileContents: string[] = await Promise.all(
+		const cachedData = await this.loadCachedData();
+		const oldSearchable = cachedData.searchable;
+		const oldSha = new Set<string>(cachedData.sha);
+		const newSha = new Set<string>();
+
+		// Load new/updated file contents
+		const fileContents: EmbeddedData = (await Promise.all(
 			vault
 				.getMarkdownFiles()
-				.map((file) =>
+				.map((file) => {
-					vault.cachedRead(file).then((res) => file.name + res)
+					const sha1 = sha1File(file);
-				)
+					newSha.add(sha1);
-		);
+					if (oldSha.has(sha1)) { // file doesn't change
-		const chunks = await this.assistant.prepareTexts(fileContents);
+						return { text: '', embeddings: [], sha1: sha1 };
-		const searchable = await this.assistant.createEmbeddings(chunks);
+					}
-		this.saveNamedData("searchable", searchable);
+					return vault.cachedRead(file).then((res) => {
+						return { text: file.name + res, embeddings: [], sha1: sha1 }
+					})
+				})
+		)).filter(f => f.text.length);
+
+		let searchable = oldSearchable.filter((e) => oldSha.has(e.sha1) && newSha.has(e.sha1));
+		if (fileContents.length) { // create embeddings for new/updated files
+			const chunks = this.assistant.prepareTexts(fileContents);
+			try {
+				const newSearchable = await this.assistant.createEmbeddings(chunks);
+				searchable = newSearchable.concat(searchable);
+				new Notice("Your data has been loaded into the model.");
+
+			} catch (e) {
+				if (e.response) {
+					console.error(e.response)
+					new Notice("❌ " + e.response.data.error.message)
+				}
+			}
+		}
+
+		this.saveNamedData({
+			"searchable": searchable,
+			"sha": Array.from(newSha),
+		});
 		this.assistant.setData(searchable);
 	}
 
-	onunload() {}
+	onunload() { }
 
 	async loadSettings() {
 		this.settings = Object.assign(
@@ -94,8 +170,8 @@ export default class GPTAssistantPlugin extends Plugin {
 		});
 	}
 
-	async saveNamedData(name: string, data: unknown) {
+	async saveNamedData(data: CachedData) {
-		await this.saveData({ ...(await this.loadData()), [name]: data });
+		await this.saveData({ ...(await this.loadData()), ...data });
 	}
 }
 
@@ -178,6 +254,17 @@ class AssistantSettings extends PluginSettingTab {
 					})
 			);
 
+		new Setting(containerEl)
+			.setName("Automatically update")
+			.setDesc("Automatically load new notes into the assistant")
+			.addToggle((tg) => {
+				tg.setValue(this.plugin.settings.autoUpdate);
+				tg.onChange(async (value) => {
+					this.plugin.settings.autoUpdate = value;
+					await this.plugin.saveSettings();
+				});
+			})
+
 		new Setting(containerEl)
 			.setName("Process notes")
 			.setDesc("Load all your notes into the assistant")
@@ -194,7 +281,6 @@ class AssistantSettings extends PluginSettingTab {
 					);
 					await this.plugin.loadEmbeddingsToAssistant();
 
-					new Notice("Your data has been loaded into the model.");
 				});
 			});
 	}

src/utils.ts

@@ -1,3 +1,6 @@
+import { createHash } from 'crypto'
+import { TFile } from "obsidian";
+
 function dotProduct(vecA: number[], vecB: number[]) {
 	let product = 0;
 	for (let i = 0; i < vecA.length; i++) {
@@ -17,3 +20,7 @@ function magnitude(vec: number[]) {
 export function cosineSimilarity(vecA: number[], vecB: number[]) {
 	return dotProduct(vecA, vecB) / (magnitude(vecA) * magnitude(vecB));
 }
+
+export function sha1File(file: TFile) {
+	return createHash('sha1').update(`${file.path}-${file.stat.ctime}-${file.stat.mtime}-${file.stat.size}`).digest('hex')
+}

versions.json

@@ -1,3 +1,3 @@
 {
-	"1.0.0": "0.15.0"
+	"0.1.3": "0.15.0"
 }