.github/workflows/release.yml

@@ -1,87 +0,0 @@
-name: Release Obsidian plugin
-
-on:
-    push:
-        tags:
-            - "*"
-
-env:
-    PLUGIN_NAME: obsidian-gpt-assistant
-
-jobs:
-    build:
-        runs-on: ubuntu-latest
-
-        steps:
-            - uses: actions/checkout@v2
-            - name: Use Node.js
-              uses: actions/setup-node@v1
-              with:
-                  node-version: "14.x"
-
-            - name: Build
-              id: build
-              run: |
-                  npm install
-                  npm run build
-                  mkdir ${{ env.PLUGIN_NAME }}
-                  cp main.js manifest.json styles.css ${{ env.PLUGIN_NAME }}
-                  zip -r ${{ env.PLUGIN_NAME }}.zip ${{ env.PLUGIN_NAME }}
-                  ls
-                  echo "::set-output name=tag_name::$(git tag --sort version:refname | tail -n 1)"
-
-            - name: Create Release
-              id: create_release
-              uses: actions/create-release@v1
-              env:
-                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-                  VERSION: ${{ github.ref }}
-              with:
-                  tag_name: ${{ github.ref }}
-                  release_name: ${{ github.ref }}
-                  draft: false
-                  prerelease: false
-
-            - name: Upload zip file
-              id: upload-zip
-              uses: actions/upload-release-asset@v1
-              env:
-                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-              with:
-                  upload_url: ${{ steps.create_release.outputs.upload_url }}
-                  asset_path: ./${{ env.PLUGIN_NAME }}.zip
-                  asset_name: ${{ env.PLUGIN_NAME }}-${{ steps.build.outputs.tag_name }}.zip
-                  asset_content_type: application/zip
-
-            - name: Upload main.js
-              id: upload-main
-              uses: actions/upload-release-asset@v1
-              env:
-                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-              with:
-                  upload_url: ${{ steps.create_release.outputs.upload_url }}
-                  asset_path: ./main.js
-                  asset_name: main.js
-                  asset_content_type: text/javascript
-
-            - name: Upload manifest.json
-              id: upload-manifest
-              uses: actions/upload-release-asset@v1
-              env:
-                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-              with:
-                  upload_url: ${{ steps.create_release.outputs.upload_url }}
-                  asset_path: ./manifest.json
-                  asset_name: manifest.json
-                  asset_content_type: application/json
-
-            - name: Upload styles.css
-              id: upload-css
-              uses: actions/upload-release-asset@v1
-              env:
-                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-              with:
-                  upload_url: ${{ steps.create_release.outputs.upload_url }}
-                  asset_path: ./styles.css
-                  asset_name: styles.css
-                  asset_content_type: text/css

assistant.ts

@@ -1,20 +1,14 @@
 import { Configuration, OpenAIApi } from "openai";
 import GPT3Tokenizer from "gpt3-tokenizer";
-import { cosineSimilarity } from "./utils";
+import { cosineSimilarity } from "utils";
 
 export interface chunkData {
 	text: string;
 	embeddings: number[];
-	sha1: string;
 }
 
 export type EmbeddedData = chunkData[];
 
-export interface CachedData {
-	searchable: EmbeddedData;
-	sha: Array<string>;
-}
-
 export type Answer = { error: boolean; text: string };
 export class Assistant {
 	MAX_TOKENS = 500;
@@ -84,20 +78,13 @@ export class Assistant {
 		};
 	}
 
-	prepareTexts(texts: EmbeddedData): EmbeddedData {
-		let shortened: EmbeddedData = [];
-		texts.forEach((item) => {
-			const text = item.text;
+	prepareTexts(texts: string[]): string[] {
+		let shortened: string[] = [];
+		texts.forEach((text) => {
 			if (this.tokenizer.encode(text).bpe.length > this.MAX_TOKENS) {
-				shortened = shortened.concat(this.splitIntoMany(text).map(t => {
-					return {
-						text: t,
-						embeddings: [],
-						sha1: item.sha1,
-					}
-				}));
+				shortened = shortened.concat(this.splitIntoMany(text));
 			} else {
-				shortened.push(item);
+				shortened.push(text);
 			}
 		});
 		return shortened;
@@ -125,15 +112,14 @@ export class Assistant {
 		});
 		return chunks;
 	}
-	async createEmbeddings(data: EmbeddedData): Promise<EmbeddedData> {
+	async createEmbeddings(data: string[]): Promise<EmbeddedData> {
 		const embeddings = await this.openai.createEmbedding({
-			input: data.map((d) => d.text),
+			input: data,
 			model: "text-embedding-ada-002",
 		});
-		return data.map((d, idx) => ({
-			text: d.text,
+		return data.map((text, idx) => ({
+			text,
 			embeddings: embeddings.data.data[idx].embedding,
-			sha1: d.sha1,
 		}));
 	}
 }

esbuild.config.mjs

@@ -2,19 +2,20 @@ import esbuild from "esbuild";
 import process from "process";
 import builtins from "builtin-modules";
 
-const banner = `/*
+const banner =
+`/*
 THIS IS A GENERATED/BUNDLED FILE BY ESBUILD
 if you want to view the source, please visit the github repository of this plugin
 */
 `;
 
-const prod = process.argv[2] === "production";
+const prod = (process.argv[2] === "production");
 
 const context = await esbuild.context({
 	banner: {
 		js: banner,
 	},
-	entryPoints: ["src/main.ts"],
+	entryPoints: ["main.ts"],
 	bundle: true,
 	external: [
 		"obsidian",
@@ -30,8 +31,7 @@ const context = await esbuild.context({
 		"@lezer/common",
 		"@lezer/highlight",
 		"@lezer/lr",
-		...builtins,
-	],
+		...builtins],
 	format: "cjs",
 	target: "es2018",
 	logLevel: "info",
@@ -45,4 +45,4 @@ if (prod) {
 	process.exit(0);
 } else {
 	await context.watch();
-}
+}

main.ts

@@ -1,7 +1,8 @@
-import { Answer, Assistant, EmbeddedData, CachedData } from "./assistant";
-import { sha1File } from "./utils";
+import { Answer, Assistant } from "assistant";
 import {
 	App,
+	Component,
+	MarkdownPreviewRenderer,
 	MarkdownRenderer,
 	Modal,
 	Notice,
@@ -10,19 +11,18 @@ import {
 	Setting,
 	TextAreaComponent,
 } from "obsidian";
+// TODO: Remember to rename these classes and interfaces!
 
-interface PluginSettings {
+interface MyPluginSettings {
 	apiKey: string;
-	autoUpdate: boolean;
 }
 
-const DEFAULT_SETTINGS: PluginSettings = {
+const DEFAULT_SETTINGS: MyPluginSettings = {
 	apiKey: "",
-	autoUpdate: false,
 };
 
-export default class GPTAssistantPlugin extends Plugin {
-	settings: PluginSettings;
+export default class MyPlugin extends Plugin {
+	settings: MyPluginSettings;
 	assistant: Assistant;
 	async onload() {
 		this.addSettingTab(new AssistantSettings(this.app, this));
@@ -30,7 +30,8 @@ export default class GPTAssistantPlugin extends Plugin {
 
 		this.assistant = new Assistant(this.settings.apiKey);
 		if (await this.hasCachedData()) {
-			const { searchable } = await this.loadData();
+			let { searchable } = await this.loadData();
+			this.saveNamedData("searchable", searchable);
 			this.assistant.setData(searchable);
 		}
 
@@ -48,117 +49,43 @@ export default class GPTAssistantPlugin extends Plugin {
 					new Notice("Please provide an API Key in the settings");
 					return;
 				}
-				if (this.settings.autoUpdate) {
-					this.loadEmbeddingsToAssistant(); // async update embedding
-				}
 				new AskAssistantModal(this.app, async (question) => {
-					try {
-						const answer = await this.assistant.answerQuestion(
-							question
-						);
-						return answer;
-					} catch (e) {
-						if (e.response) {
-							console.error(e.response)
-							new Notice("❌ " + e.response.data.error.message)
-						}
-						return { error: true, text: "" }
-					}
+					const answer = await this.assistant.answerQuestion(
+						question
+					);
+					return answer ?? "";
 				}).open();
 			},
 		});
-
-		this.addCommand({
-			id: "update-assistant",
-			name: "Update assistant",
-			callback: async () => {
-				if (!this.settings.apiKey) {
-					new Notice("Please provide an API Key in the settings");
-					return;
-				}
-				new Notice(
-					"Loading data into model. this could take a while..."
-				);
-				await this.loadEmbeddingsToAssistant();
-				new Notice("Your data has been loaded into the model.");
-
-			},
-		});
 	}
 
 	private async hasCachedData(): Promise<boolean> {
 		const data = await this.loadData();
-		return data && data.searchable && data.searchable.length;
-	}
-
-	private async loadCachedData(): Promise<CachedData> {
-		const data = await this.loadData();
-		if (data && data.searchable && data.searchable.length &&
-			data.sha && data.sha.length) {
-			return {
-				searchable: data.searchable,
-				sha: data.sha,
-			}
-		}
-		return {
-			searchable: [],
-			sha: [],
-		}
+		return data.searchable && data.searchable.length;
 	}
 
 	async loadEmbeddingsToAssistant() {
 		const { vault } = this.app;
-		const cachedData = await this.loadCachedData();
-		const oldSearchable = cachedData.searchable;
-		const oldSha = new Set<string>(cachedData.sha);
-		const newSha = new Set<string>();
-
-		// Load new/updated file contents
-		const fileContents: EmbeddedData = (await Promise.all(
+		const fileContents: string[] = await Promise.all(
 			vault
 				.getMarkdownFiles()
-				.map((file) => {
-					const sha1 = sha1File(file);
-					newSha.add(sha1);
-					if (oldSha.has(sha1)) { // file doesn't change
-						return { text: '', embeddings: [], sha1: sha1 };
-					}
-					return vault.cachedRead(file).then((res) => {
-						return { text: file.name + res, embeddings: [], sha1: sha1 }
-					})
-				})
-		)).filter(f => f.text.length);
-
-		let searchable = oldSearchable.filter((e) => oldSha.has(e.sha1) && newSha.has(e.sha1));
-		if (fileContents.length) { // create embeddings for new/updated files
-			const chunks = this.assistant.prepareTexts(fileContents);
-			try {
-				const newSearchable = await this.assistant.createEmbeddings(chunks);
-				searchable = newSearchable.concat(searchable);
-				new Notice("Your data has been loaded into the model.");
-
-			} catch (e) {
-				if (e.response) {
-					console.error(e.response)
-					new Notice("❌ " + e.response.data.error.message)
-				}
-			}
-		}
-
-		this.saveNamedData({
-			"searchable": searchable,
-			"sha": Array.from(newSha),
-		});
+				.map((file) =>
+					vault.cachedRead(file).then((res) => file.name + res)
+				)
+		);
+		const chunks = await this.assistant.prepareTexts(fileContents);
+		const searchable = await this.assistant.createEmbeddings(chunks);
+		this.saveNamedData("searchable", searchable);
 		this.assistant.setData(searchable);
 	}
 
-	onunload() { }
+	onunload() {}
 
 	async loadSettings() {
 		this.settings = Object.assign(
 			{},
 			DEFAULT_SETTINGS,
-			(await this.loadData())?.settings ?? {}
+			(await this.loadData()).settings
 		);
 	}
 
@@ -170,8 +97,8 @@ export default class GPTAssistantPlugin extends Plugin {
 		});
 	}
 
-	async saveNamedData(data: CachedData) {
-		await this.saveData({ ...(await this.loadData()), ...data });
+	async saveNamedData(name: string, data: unknown) {
+		await this.saveData({ ...(await this.loadData()), [name]: data });
 	}
 }
 
@@ -225,9 +152,9 @@ class AskAssistantModal extends Modal {
 }
 
 class AssistantSettings extends PluginSettingTab {
-	plugin: GPTAssistantPlugin;
+	plugin: MyPlugin;
 
-	constructor(app: App, plugin: GPTAssistantPlugin) {
+	constructor(app: App, plugin: MyPlugin) {
 		super(app, plugin);
 		this.plugin = plugin;
 	}
@@ -254,17 +181,6 @@ class AssistantSettings extends PluginSettingTab {
 					})
 			);
 
-		new Setting(containerEl)
-			.setName("Automatically update")
-			.setDesc("Automatically load new notes into the assistant")
-			.addToggle((tg) => {
-				tg.setValue(this.plugin.settings.autoUpdate);
-				tg.onChange(async (value) => {
-					this.plugin.settings.autoUpdate = value;
-					await this.plugin.saveSettings();
-				});
-			})
-
 		new Setting(containerEl)
 			.setName("Process notes")
 			.setDesc("Load all your notes into the assistant")
@@ -272,7 +188,7 @@ class AssistantSettings extends PluginSettingTab {
 				btn.setButtonText("Process").setCta();
 
 				btn.onClick(async (e) => {
-					if (this.plugin?.settings?.apiKey == "") {
+					if (this.plugin.settings.apiKey == "") {
 						new Notice("Please provide an API Key");
 						return;
 					}
@@ -281,6 +197,7 @@ class AssistantSettings extends PluginSettingTab {
 					);
 					await this.plugin.loadEmbeddingsToAssistant();
 
+					new Notice("Your data has been loaded into the model.");
 				});
 			});
 	}

manifest.json

@@ -1,9 +1,9 @@
 {
-	"id": "gpt-assistant",
+	"id": "obsidian-gpt-assistant",
 	"name": "GPT Assistant",
-	"version": "0.1.3",
+	"version": "0.1.0",
 	"minAppVersion": "0.15.0",
-	"description": "Use a GPT-3 based model on your notes and get personalized answers from your knowledge base.",
+	"description": "Train a GPT-3 based model on your notes and get personalized answers based on your knowledge base.",
 	"author": "M7mdisk",
 	"authorUrl": "https://github.com/M7mdisk",
 	"fundingUrl": "https://github.com/M7mdisk",

utils.ts

@@ -1,6 +1,3 @@
-import { createHash } from 'crypto'
-import { TFile } from "obsidian";
-
 function dotProduct(vecA: number[], vecB: number[]) {
 	let product = 0;
 	for (let i = 0; i < vecA.length; i++) {
@@ -20,7 +17,3 @@ function magnitude(vec: number[]) {
 export function cosineSimilarity(vecA: number[], vecB: number[]) {
 	return dotProduct(vecA, vecB) / (magnitude(vecA) * magnitude(vecB));
 }
-
-export function sha1File(file: TFile) {
-	return createHash('sha1').update(`${file.path}-${file.stat.ctime}-${file.stat.mtime}-${file.stat.size}`).digest('hex')
-}

versions.json

@@ -1,3 +1,3 @@
 {
-	"0.1.3": "0.15.0"
+	"1.0.0": "0.15.0"
 }