Merge pull request #5 from Beim/feat/hash-file

Incremental update embedding data

.github/workflows/release.yml

@@ -0,0 +1,87 @@
+name: Release Obsidian plugin
+
+on:
+    push:
+        tags:
+            - "*"
+
+env:
+    PLUGIN_NAME: obsidian-gpt-assistant
+
+jobs:
+    build:
+        runs-on: ubuntu-latest
+
+        steps:
+            - uses: actions/checkout@v2
+            - name: Use Node.js
+              uses: actions/setup-node@v1
+              with:
+                  node-version: "14.x"
+
+            - name: Build
+              id: build
+              run: |
+                  npm install
+                  npm run build
+                  mkdir ${{ env.PLUGIN_NAME }}
+                  cp main.js manifest.json styles.css ${{ env.PLUGIN_NAME }}
+                  zip -r ${{ env.PLUGIN_NAME }}.zip ${{ env.PLUGIN_NAME }}
+                  ls
+                  echo "::set-output name=tag_name::$(git tag --sort version:refname | tail -n 1)"
+
+            - name: Create Release
+              id: create_release
+              uses: actions/create-release@v1
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+                  VERSION: ${{ github.ref }}
+              with:
+                  tag_name: ${{ github.ref }}
+                  release_name: ${{ github.ref }}
+                  draft: false
+                  prerelease: false
+
+            - name: Upload zip file
+              id: upload-zip
+              uses: actions/upload-release-asset@v1
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              with:
+                  upload_url: ${{ steps.create_release.outputs.upload_url }}
+                  asset_path: ./${{ env.PLUGIN_NAME }}.zip
+                  asset_name: ${{ env.PLUGIN_NAME }}-${{ steps.build.outputs.tag_name }}.zip
+                  asset_content_type: application/zip
+
+            - name: Upload main.js
+              id: upload-main
+              uses: actions/upload-release-asset@v1
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              with:
+                  upload_url: ${{ steps.create_release.outputs.upload_url }}
+                  asset_path: ./main.js
+                  asset_name: main.js
+                  asset_content_type: text/javascript
+
+            - name: Upload manifest.json
+              id: upload-manifest
+              uses: actions/upload-release-asset@v1
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              with:
+                  upload_url: ${{ steps.create_release.outputs.upload_url }}
+                  asset_path: ./manifest.json
+                  asset_name: manifest.json
+                  asset_content_type: application/json
+
+            - name: Upload styles.css
+              id: upload-css
+              uses: actions/upload-release-asset@v1
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              with:
+                  upload_url: ${{ steps.create_release.outputs.upload_url }}
+                  asset_path: ./styles.css
+                  asset_name: styles.css
+                  asset_content_type: text/css

esbuild.config.mjs

@@ -2,20 +2,19 @@ import esbuild from "esbuild";
 import process from "process";
 import builtins from "builtin-modules";
 
-const banner =
-`/*
+const banner = `/*
 THIS IS A GENERATED/BUNDLED FILE BY ESBUILD
 if you want to view the source, please visit the github repository of this plugin
 */
 `;
 
-const prod = (process.argv[2] === "production");
+const prod = process.argv[2] === "production";
 
 const context = await esbuild.context({
 	banner: {
 		js: banner,
 	},
-	entryPoints: ["main.ts"],
+	entryPoints: ["src/main.ts"],
 	bundle: true,
 	external: [
 		"obsidian",
@@ -31,7 +30,8 @@ const context = await esbuild.context({
 		"@lezer/common",
 		"@lezer/highlight",
 		"@lezer/lr",
-		...builtins],
+		...builtins,
+	],
 	format: "cjs",
 	target: "es2018",
 	logLevel: "info",
@@ -45,4 +45,4 @@ if (prod) {
 	process.exit(0);
 } else {
 	await context.watch();
-}
+}

manifest.json

@@ -1,9 +1,9 @@
 {
-	"id": "obsidian-gpt-assistant",
+	"id": "gpt-assistant",
 	"name": "GPT Assistant",
-	"version": "0.1.0",
+	"version": "0.1.3",
 	"minAppVersion": "0.15.0",
-	"description": "Train a GPT-3 based model on your notes and get personalized answers based on your knowledge base.",
+	"description": "Use a GPT-3 based model on your notes and get personalized answers from your knowledge base.",
 	"author": "M7mdisk",
 	"authorUrl": "https://github.com/M7mdisk",
 	"fundingUrl": "https://github.com/M7mdisk",

src/assistant.ts

@@ -1,14 +1,20 @@
 import { Configuration, OpenAIApi } from "openai";
 import GPT3Tokenizer from "gpt3-tokenizer";
-import { cosineSimilarity } from "utils";
+import { cosineSimilarity } from "./utils";
 
 export interface chunkData {
 	text: string;
 	embeddings: number[];
+	sha1: string;
 }
 
 export type EmbeddedData = chunkData[];
 
+export interface CachedData {
+	searchable: EmbeddedData;
+	sha: Array<string>;
+}
+
 export type Answer = { error: boolean; text: string };
 export class Assistant {
 	MAX_TOKENS = 500;
@@ -78,13 +84,20 @@ export class Assistant {
 		};
 	}
 
-	prepareTexts(texts: string[]): string[] {
-		let shortened: string[] = [];
-		texts.forEach((text) => {
+	prepareTexts(texts: EmbeddedData): EmbeddedData {
+		let shortened: EmbeddedData = [];
+		texts.forEach((item) => {
+			const text = item.text;
 			if (this.tokenizer.encode(text).bpe.length > this.MAX_TOKENS) {
-				shortened = shortened.concat(this.splitIntoMany(text));
+				shortened = shortened.concat(this.splitIntoMany(text).map(t => {
+					return {
+						text: t,
+						embeddings: [],
+						sha1: item.sha1,
+					}
+				}));
 			} else {
-				shortened.push(text);
+				shortened.push(item);
 			}
 		});
 		return shortened;
@@ -112,14 +125,15 @@ export class Assistant {
 		});
 		return chunks;
 	}
-	async createEmbeddings(data: string[]): Promise<EmbeddedData> {
+	async createEmbeddings(data: EmbeddedData): Promise<EmbeddedData> {
 		const embeddings = await this.openai.createEmbedding({
-			input: data,
+			input: data.map((d) => d.text),
 			model: "text-embedding-ada-002",
 		});
-		return data.map((text, idx) => ({
-			text,
+		return data.map((d, idx) => ({
+			text: d.text,
 			embeddings: embeddings.data.data[idx].embedding,
+			sha1: d.sha1,
 		}));
 	}
 }

src/main.ts

@@ -1,8 +1,7 @@
-import { Answer, Assistant } from "assistant";
+import { Answer, Assistant, EmbeddedData, CachedData } from "./assistant";
+import { sha1File } from "./utils";
 import {
 	App,
-	Component,
-	MarkdownPreviewRenderer,
 	MarkdownRenderer,
 	Modal,
 	Notice,
@@ -11,18 +10,19 @@ import {
 	Setting,
 	TextAreaComponent,
 } from "obsidian";
-// TODO: Remember to rename these classes and interfaces!
 
-interface MyPluginSettings {
+interface PluginSettings {
 	apiKey: string;
+	autoUpdate: boolean;
 }
 
-const DEFAULT_SETTINGS: MyPluginSettings = {
+const DEFAULT_SETTINGS: PluginSettings = {
 	apiKey: "",
+	autoUpdate: false,
 };
 
-export default class MyPlugin extends Plugin {
-	settings: MyPluginSettings;
+export default class GPTAssistantPlugin extends Plugin {
+	settings: PluginSettings;
 	assistant: Assistant;
 	async onload() {
 		this.addSettingTab(new AssistantSettings(this.app, this));
@@ -30,8 +30,7 @@ export default class MyPlugin extends Plugin {
 
 		this.assistant = new Assistant(this.settings.apiKey);
 		if (await this.hasCachedData()) {
-			let { searchable } = await this.loadData();
-			this.saveNamedData("searchable", searchable);
+			const { searchable } = await this.loadData();
 			this.assistant.setData(searchable);
 		}
 
@@ -49,43 +48,117 @@ export default class MyPlugin extends Plugin {
 					new Notice("Please provide an API Key in the settings");
 					return;
 				}
+				if (this.settings.autoUpdate) {
+					this.loadEmbeddingsToAssistant(); // async update embedding
+				}
 				new AskAssistantModal(this.app, async (question) => {
-					const answer = await this.assistant.answerQuestion(
-						question
-					);
-					return answer ?? "";
+					try {
+						const answer = await this.assistant.answerQuestion(
+							question
+						);
+						return answer;
+					} catch (e) {
+						if (e.response) {
+							console.error(e.response)
+							new Notice("❌ " + e.response.data.error.message)
+						}
+						return { error: true, text: "" }
+					}
 				}).open();
 			},
 		});
+
+		this.addCommand({
+			id: "update-assistant",
+			name: "Update assistant",
+			callback: async () => {
+				if (!this.settings.apiKey) {
+					new Notice("Please provide an API Key in the settings");
+					return;
+				}
+				new Notice(
+					"Loading data into model. this could take a while..."
+				);
+				await this.loadEmbeddingsToAssistant();
+				new Notice("Your data has been loaded into the model.");
+
+			},
+		});
 	}
 
 	private async hasCachedData(): Promise<boolean> {
 		const data = await this.loadData();
-		return data.searchable && data.searchable.length;
+		return data && data.searchable && data.searchable.length;
+	}
+
+	private async loadCachedData(): Promise<CachedData> {
+		const data = await this.loadData();
+		if (data && data.searchable && data.searchable.length &&
+			data.sha && data.sha.length) {
+			return {
+				searchable: data.searchable,
+				sha: data.sha,
+			}
+		}
+		return {
+			searchable: [],
+			sha: [],
+		}
 	}
 
 	async loadEmbeddingsToAssistant() {
 		const { vault } = this.app;
-		const fileContents: string[] = await Promise.all(
+		const cachedData = await this.loadCachedData();
+		const oldSearchable = cachedData.searchable;
+		const oldSha = new Set<string>(cachedData.sha);
+		const newSha = new Set<string>();
+
+		// Load new/updated file contents
+		const fileContents: EmbeddedData = (await Promise.all(
 			vault
 				.getMarkdownFiles()
-				.map((file) =>
-					vault.cachedRead(file).then((res) => file.name + res)
-				)
-		);
-		const chunks = await this.assistant.prepareTexts(fileContents);
-		const searchable = await this.assistant.createEmbeddings(chunks);
-		this.saveNamedData("searchable", searchable);
+				.map((file) => {
+					const sha1 = sha1File(file);
+					newSha.add(sha1);
+					if (oldSha.has(sha1)) { // file doesn't change
+						return { text: '', embeddings: [], sha1: sha1 };
+					}
+					return vault.cachedRead(file).then((res) => {
+						return { text: file.name + res, embeddings: [], sha1: sha1 }
+					})
+				})
+		)).filter(f => f.text.length);
+
+		let searchable = oldSearchable.filter((e) => oldSha.has(e.sha1) && newSha.has(e.sha1));
+		if (fileContents.length) { // create embeddings for new/updated files
+			const chunks = this.assistant.prepareTexts(fileContents);
+			try {
+				const newSearchable = await this.assistant.createEmbeddings(chunks);
+				searchable = newSearchable.concat(searchable);
+				new Notice("Your data has been loaded into the model.");
+
+			} catch (e) {
+				if (e.response) {
+					console.error(e.response)
+					new Notice("❌ " + e.response.data.error.message)
+				}
+			}
+		}
+
+		this.saveNamedData({
+			"searchable": searchable,
+			"sha": Array.from(newSha),
+		});
 		this.assistant.setData(searchable);
 	}
 
-	onunload() {}
+	onunload() { }
 
 	async loadSettings() {
 		this.settings = Object.assign(
 			{},
 			DEFAULT_SETTINGS,
-			(await this.loadData()).settings
+			(await this.loadData())?.settings ?? {}
 		);
 	}
 
@@ -97,8 +170,8 @@ export default class MyPlugin extends Plugin {
 		});
 	}
 
-	async saveNamedData(name: string, data: unknown) {
-		await this.saveData({ ...(await this.loadData()), [name]: data });
+	async saveNamedData(data: CachedData) {
+		await this.saveData({ ...(await this.loadData()), ...data });
 	}
 }
 
@@ -152,9 +225,9 @@ class AskAssistantModal extends Modal {
 }
 
 class AssistantSettings extends PluginSettingTab {
-	plugin: MyPlugin;
+	plugin: GPTAssistantPlugin;
 
-	constructor(app: App, plugin: MyPlugin) {
+	constructor(app: App, plugin: GPTAssistantPlugin) {
 		super(app, plugin);
 		this.plugin = plugin;
 	}
@@ -181,6 +254,17 @@ class AssistantSettings extends PluginSettingTab {
 					})
 			);
 
+		new Setting(containerEl)
+			.setName("Automatically update")
+			.setDesc("Automatically load new notes into the assistant")
+			.addToggle((tg) => {
+				tg.setValue(this.plugin.settings.autoUpdate);
+				tg.onChange(async (value) => {
+					this.plugin.settings.autoUpdate = value;
+					await this.plugin.saveSettings();
+				});
+			})
+
 		new Setting(containerEl)
 			.setName("Process notes")
 			.setDesc("Load all your notes into the assistant")
@@ -188,7 +272,7 @@ class AssistantSettings extends PluginSettingTab {
 				btn.setButtonText("Process").setCta();
 
 				btn.onClick(async (e) => {
-					if (this.plugin.settings.apiKey == "") {
+					if (this.plugin?.settings?.apiKey == "") {
 						new Notice("Please provide an API Key");
 						return;
 					}
@@ -197,7 +281,6 @@ class AssistantSettings extends PluginSettingTab {
 					);
 					await this.plugin.loadEmbeddingsToAssistant();
 
-					new Notice("Your data has been loaded into the model.");
 				});
 			});
 	}

src/utils.ts

@@ -1,3 +1,6 @@
+import { createHash } from 'crypto'
+import { TFile } from "obsidian";
+
 function dotProduct(vecA: number[], vecB: number[]) {
 	let product = 0;
 	for (let i = 0; i < vecA.length; i++) {
@@ -17,3 +20,7 @@ function magnitude(vec: number[]) {
 export function cosineSimilarity(vecA: number[], vecB: number[]) {
 	return dotProduct(vecA, vecB) / (magnitude(vecA) * magnitude(vecB));
 }
+
+export function sha1File(file: TFile) {
+	return createHash('sha1').update(`${file.path}-${file.stat.ctime}-${file.stat.mtime}-${file.stat.size}`).digest('hex')
+}

versions.json

@@ -1,3 +1,3 @@
 {
-	"1.0.0": "0.15.0"
+	"0.1.3": "0.15.0"
 }