Hash file for incremental update

manifest.json

@@ -1,7 +1,7 @@
 {
 	"id": "gpt-assistant",
 	"name": "GPT Assistant",
-	"version": "0.1.2",
+	"version": "0.1.3",
 	"minAppVersion": "0.15.0",
 	"description": "Use a GPT-3 based model on your notes and get personalized answers from your knowledge base.",
 	"author": "M7mdisk",

src/assistant.ts

@@ -5,10 +5,16 @@ import { cosineSimilarity } from "./utils";
 export interface chunkData {
 	text: string;
 	embeddings: number[];
+	sha1: string;
 }
 
 export type EmbeddedData = chunkData[];
 
+export interface CachedData {
+	searchable: EmbeddedData;
+	sha: Array<string>;
+}
+
 export type Answer = { error: boolean; text: string };
 export class Assistant {
 	MAX_TOKENS = 500;
@@ -78,13 +84,20 @@ export class Assistant {
 		};
 	}
 
-	prepareTexts(texts: string[]): string[] {
-		let shortened: string[] = [];
-		texts.forEach((text) => {
+	prepareTexts(texts: EmbeddedData): EmbeddedData {
+		let shortened: EmbeddedData = [];
+		texts.forEach((item) => {
+			const text = item.text;
 			if (this.tokenizer.encode(text).bpe.length > this.MAX_TOKENS) {
-				shortened = shortened.concat(this.splitIntoMany(text));
+				shortened = shortened.concat(this.splitIntoMany(text).map(t => {
+					return {
+						text: t,
+						embeddings: [],
+						sha1: item.sha1,
+					}
+				}));
 			} else {
-				shortened.push(text);
+				shortened.push(item);
 			}
 		});
 		return shortened;
@@ -112,14 +125,15 @@ export class Assistant {
 		});
 		return chunks;
 	}
-	async createEmbeddings(data: string[]): Promise<EmbeddedData> {
+	async createEmbeddings(data: EmbeddedData): Promise<EmbeddedData> {
 		const embeddings = await this.openai.createEmbedding({
-			input: data,
+			input: data.map((d) => d.text),
 			model: "text-embedding-ada-002",
 		});
-		return data.map((text, idx) => ({
-			text,
+		return data.map((d, idx) => ({
+			text: d.text,
 			embeddings: embeddings.data.data[idx].embedding,
+			sha1: d.sha1,
 		}));
 	}
 }

src/main.ts

@@ -1,4 +1,5 @@
-import { Answer, Assistant } from "./assistant";
+import { Answer, Assistant, EmbeddedData, CachedData } from "./assistant";
+import { sha1File } from "./utils";
 import {
 	App,
 	MarkdownRenderer,
@@ -28,7 +29,6 @@ export default class GPTAssistantPlugin extends Plugin {
 		this.assistant = new Assistant(this.settings.apiKey);
 		if (await this.hasCachedData()) {
 			const { searchable } = await this.loadData();
-			this.saveNamedData("searchable", searchable);
 			this.assistant.setData(searchable);
 		}
 
@@ -46,6 +46,7 @@ export default class GPTAssistantPlugin extends Plugin {
 					new Notice("Please provide an API Key in the settings");
 					return;
 				}
+				this.loadEmbeddingsToAssistant(); // async update embedding
 				new AskAssistantModal(this.app, async (question) => {
 					const answer = await this.assistant.answerQuestion(
 						question
@@ -54,6 +55,23 @@ export default class GPTAssistantPlugin extends Plugin {
 				}).open();
 			},
 		});
+
+		this.addCommand({
+			id: "update-assistant",
+			name: "Update assistant",
+			callback: async () => {
+				if (!this.settings.apiKey) {
+					new Notice("Please provide an API Key in the settings");
+					return;
+				}
+				new Notice(
+					"Loading data into model. this could take a while..."
+				);
+				await this.loadEmbeddingsToAssistant();
+				new Notice("Your data has been loaded into the model.");
+
+			},
+		});
 	}
 
 	private async hasCachedData(): Promise<boolean> {
@@ -61,22 +79,59 @@ export default class GPTAssistantPlugin extends Plugin {
 		return data && data.searchable && data.searchable.length;
 	}
 
+	private async loadCachedData(): Promise<CachedData> {
+		const data = await this.loadData();
+		if (data && data.searchable && data.searchable.length &&
+			data.sha && data.sha.length) {
+			return {
+				searchable: data.searchable,
+				sha: data.sha,
+			}
+		}
+		return {
+			searchable: [],
+			sha: [],
+		}
+	}
+
 	async loadEmbeddingsToAssistant() {
 		const { vault } = this.app;
-		const fileContents: string[] = await Promise.all(
+		const cachedData = await this.loadCachedData();
+		const oldSearchable = cachedData.searchable;
+		const oldSha = new Set<string>(cachedData.sha);
+		const newSha = new Set<string>();
+
+		// Load new/updated file contents
+		const fileContents: EmbeddedData = (await Promise.all(
 			vault
 				.getMarkdownFiles()
-				.map((file) =>
-					vault.cachedRead(file).then((res) => file.name + res)
-				)
-		);
-		const chunks = await this.assistant.prepareTexts(fileContents);
-		const searchable = await this.assistant.createEmbeddings(chunks);
-		this.saveNamedData("searchable", searchable);
+				.map((file) => {
+					const sha1 = sha1File(file);
+					newSha.add(sha1);
+					if (oldSha.has(sha1)) { // file doesn't change
+						return { text: '', embeddings: [], sha1: sha1 };
+					}
+					return vault.cachedRead(file).then((res) => {
+						return { text: file.name + res, embeddings: [], sha1: sha1 }
+					})
+				})
+		)).filter(f => f.text.length);
+
+		let searchable = oldSearchable.filter((e) => oldSha.has(e.sha1) && newSha.has(e.sha1));
+		if (fileContents.length) { // create embeddings for new/updated files
+			const chunks = this.assistant.prepareTexts(fileContents);
+			const newSearchable = await this.assistant.createEmbeddings(chunks);
+			searchable = newSearchable.concat(searchable);
+		}
+
+		this.saveNamedDataV2({
+			"searchable": searchable,
+			"sha": Array.from(newSha),
+		});
 		this.assistant.setData(searchable);
 	}
 
-	onunload() {}
+	onunload() { }
 
 	async loadSettings() {
 		this.settings = Object.assign(
@@ -97,6 +152,10 @@ export default class GPTAssistantPlugin extends Plugin {
 	async saveNamedData(name: string, data: unknown) {
 		await this.saveData({ ...(await this.loadData()), [name]: data });
 	}
+
+	async saveNamedDataV2(data: CachedData) {
+		await this.saveData({ ...(await this.loadData()), ...data });
+	}
 }
 
 class AskAssistantModal extends Modal {

src/utils.ts

@@ -1,3 +1,6 @@
+import { createHash } from 'crypto'
+import { TFile } from "obsidian";
+
 function dotProduct(vecA: number[], vecB: number[]) {
 	let product = 0;
 	for (let i = 0; i < vecA.length; i++) {
@@ -17,3 +20,7 @@ function magnitude(vec: number[]) {
 export function cosineSimilarity(vecA: number[], vecB: number[]) {
 	return dotProduct(vecA, vecB) / (magnitude(vecA) * magnitude(vecB));
 }
+
+export function sha1File(file: TFile) {
+	return createHash('sha1').update(`${file.path}-${file.stat.ctime}-${file.stat.mtime}-${file.stat.size}`).digest('hex')
+}

versions.json

@@ -1,3 +1,3 @@
 {
-	"1.0.0": "0.15.0"
+	"0.1.3": "0.15.0"
 }