From c02ef1e16828036e2c8b23b75218e93c09540d71 Mon Sep 17 00:00:00 2001 From: xingkun liu Date: Wed, 17 May 2023 04:43:01 +1200 Subject: [PATCH 1/3] Hash file for incremental update --- manifest.json | 2 +- src/assistant.ts | 32 +++++++++++++------ src/main.ts | 81 +++++++++++++++++++++++++++++++++++++++++------- src/utils.ts | 7 +++++ versions.json | 2 +- 5 files changed, 102 insertions(+), 22 deletions(-) diff --git a/manifest.json b/manifest.json index b28085f..2cb0027 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "gpt-assistant", "name": "GPT Assistant", - "version": "0.1.2", + "version": "0.1.3", "minAppVersion": "0.15.0", "description": "Use a GPT-3 based model on your notes and get personalized answers from your knowledge base.", "author": "M7mdisk", diff --git a/src/assistant.ts b/src/assistant.ts index d09e71e..7f1ac7a 100644 --- a/src/assistant.ts +++ b/src/assistant.ts @@ -5,10 +5,16 @@ import { cosineSimilarity } from "./utils"; export interface chunkData { text: string; embeddings: number[]; + sha1: string; } export type EmbeddedData = chunkData[]; +export interface CachedData { + searchable: EmbeddedData; + sha: Array; +} + export type Answer = { error: boolean; text: string }; export class Assistant { MAX_TOKENS = 500; @@ -78,13 +84,20 @@ export class Assistant { }; } - prepareTexts(texts: string[]): string[] { - let shortened: string[] = []; - texts.forEach((text) => { + prepareTexts(texts: EmbeddedData): EmbeddedData { + let shortened: EmbeddedData = []; + texts.forEach((item) => { + const text = item.text; if (this.tokenizer.encode(text).bpe.length > this.MAX_TOKENS) { - shortened = shortened.concat(this.splitIntoMany(text)); + shortened = shortened.concat(this.splitIntoMany(text).map(t => { + return { + text: t, + embeddings: [], + sha1: item.sha1, + } + })); } else { - shortened.push(text); + shortened.push(item); } }); return shortened; @@ -112,14 +125,15 @@ export class Assistant { }); return chunks; } - async createEmbeddings(data: string[]): Promise { + async createEmbeddings(data: EmbeddedData): Promise { const embeddings = await this.openai.createEmbedding({ - input: data, + input: data.map((d) => d.text), model: "text-embedding-ada-002", }); - return data.map((text, idx) => ({ - text, + return data.map((d, idx) => ({ + text: d.text, embeddings: embeddings.data.data[idx].embedding, + sha1: d.sha1, })); } } diff --git a/src/main.ts b/src/main.ts index 0e3b056..4153b27 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,4 +1,5 @@ -import { Answer, Assistant } from "./assistant"; +import { Answer, Assistant, EmbeddedData, CachedData } from "./assistant"; +import { sha1File } from "./utils"; import { App, MarkdownRenderer, @@ -28,7 +29,6 @@ export default class GPTAssistantPlugin extends Plugin { this.assistant = new Assistant(this.settings.apiKey); if (await this.hasCachedData()) { const { searchable } = await this.loadData(); - this.saveNamedData("searchable", searchable); this.assistant.setData(searchable); } @@ -46,6 +46,7 @@ export default class GPTAssistantPlugin extends Plugin { new Notice("Please provide an API Key in the settings"); return; } + this.loadEmbeddingsToAssistant(); // async update embedding new AskAssistantModal(this.app, async (question) => { const answer = await this.assistant.answerQuestion( question @@ -54,6 +55,23 @@ export default class GPTAssistantPlugin extends Plugin { }).open(); }, }); + + this.addCommand({ + id: "update-assistant", + name: "Update assistant", + callback: async () => { + if (!this.settings.apiKey) { + new Notice("Please provide an API Key in the settings"); + return; + } + new Notice( + "Loading data into model. this could take a while..." + ); + await this.loadEmbeddingsToAssistant(); + new Notice("Your data has been loaded into the model."); + + }, + }); } private async hasCachedData(): Promise { @@ -61,22 +79,59 @@ export default class GPTAssistantPlugin extends Plugin { return data && data.searchable && data.searchable.length; } + private async loadCachedData(): Promise { + const data = await this.loadData(); + if (data && data.searchable && data.searchable.length && + data.sha && data.sha.length) { + return { + searchable: data.searchable, + sha: data.sha, + } + } + return { + searchable: [], + sha: [], + } + } + async loadEmbeddingsToAssistant() { const { vault } = this.app; - const fileContents: string[] = await Promise.all( + const cachedData = await this.loadCachedData(); + const oldSearchable = cachedData.searchable; + const oldSha = new Set(cachedData.sha); + const newSha = new Set(); + + // Load new/updated file contents + const fileContents: EmbeddedData = (await Promise.all( vault .getMarkdownFiles() - .map((file) => - vault.cachedRead(file).then((res) => file.name + res) - ) - ); - const chunks = await this.assistant.prepareTexts(fileContents); - const searchable = await this.assistant.createEmbeddings(chunks); - this.saveNamedData("searchable", searchable); + .map((file) => { + const sha1 = sha1File(file); + newSha.add(sha1); + if (oldSha.has(sha1)) { // file doesn't change + return { text: '', embeddings: [], sha1: sha1 }; + } + return vault.cachedRead(file).then((res) => { + return { text: file.name + res, embeddings: [], sha1: sha1 } + }) + }) + )).filter(f => f.text.length); + + let searchable = oldSearchable.filter((e) => oldSha.has(e.sha1) && newSha.has(e.sha1)); + if (fileContents.length) { // create embeddings for new/updated files + const chunks = this.assistant.prepareTexts(fileContents); + const newSearchable = await this.assistant.createEmbeddings(chunks); + searchable = newSearchable.concat(searchable); + } + + this.saveNamedDataV2({ + "searchable": searchable, + "sha": Array.from(newSha), + }); this.assistant.setData(searchable); } - onunload() {} + onunload() { } async loadSettings() { this.settings = Object.assign( @@ -97,6 +152,10 @@ export default class GPTAssistantPlugin extends Plugin { async saveNamedData(name: string, data: unknown) { await this.saveData({ ...(await this.loadData()), [name]: data }); } + + async saveNamedDataV2(data: CachedData) { + await this.saveData({ ...(await this.loadData()), ...data }); + } } class AskAssistantModal extends Modal { diff --git a/src/utils.ts b/src/utils.ts index 8b97a83..9253f15 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,3 +1,6 @@ +import { createHash } from 'crypto' +import { TFile } from "obsidian"; + function dotProduct(vecA: number[], vecB: number[]) { let product = 0; for (let i = 0; i < vecA.length; i++) { @@ -17,3 +20,7 @@ function magnitude(vec: number[]) { export function cosineSimilarity(vecA: number[], vecB: number[]) { return dotProduct(vecA, vecB) / (magnitude(vecA) * magnitude(vecB)); } + +export function sha1File(file: TFile) { + return createHash('sha1').update(`${file.path}-${file.stat.ctime}-${file.stat.mtime}-${file.stat.size}`).digest('hex') +} diff --git a/versions.json b/versions.json index 26382a1..8213519 100644 --- a/versions.json +++ b/versions.json @@ -1,3 +1,3 @@ { - "1.0.0": "0.15.0" + "0.1.3": "0.15.0" } From 6a51dc9e7dc297a4e009435a184f0f84c94efef8 Mon Sep 17 00:00:00 2001 From: xingkun liu Date: Sun, 21 May 2023 17:27:31 +1200 Subject: [PATCH 2/3] Add toggle for auto update files --- src/main.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/main.ts b/src/main.ts index 4153b27..b108114 100644 --- a/src/main.ts +++ b/src/main.ts @@ -13,10 +13,12 @@ import { interface PluginSettings { apiKey: string; + autoUpdate: boolean; } const DEFAULT_SETTINGS: PluginSettings = { apiKey: "", + autoUpdate: false, }; export default class GPTAssistantPlugin extends Plugin { @@ -46,7 +48,9 @@ export default class GPTAssistantPlugin extends Plugin { new Notice("Please provide an API Key in the settings"); return; } - this.loadEmbeddingsToAssistant(); // async update embedding + if (this.settings.autoUpdate) { + this.loadEmbeddingsToAssistant(); // async update embedding + } new AskAssistantModal(this.app, async (question) => { const answer = await this.assistant.answerQuestion( question @@ -237,6 +241,18 @@ class AssistantSettings extends PluginSettingTab { }) ); + new Setting(containerEl) + .setName("Automatically update") + .setDesc("Automatically load new notes into the assistant") + .addToggle((tg) => { + tg.setValue(this.plugin.settings.autoUpdate); + tg.onChange(async (value) => { + this.plugin.settings.autoUpdate = value; + await this.plugin.saveSettings(); + new Notice("updated"); + }); + }) + new Setting(containerEl) .setName("Process notes") .setDesc("Load all your notes into the assistant") From 150f2a45820774fe4652501d2b1b5fd5d74bdbad Mon Sep 17 00:00:00 2001 From: Mohammad Iskandarani Date: Sun, 21 May 2023 20:06:48 +0300 Subject: [PATCH 3/3] Better Error feedback --- src/main.ts | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/src/main.ts b/src/main.ts index b108114..2c17d61 100644 --- a/src/main.ts +++ b/src/main.ts @@ -52,10 +52,18 @@ export default class GPTAssistantPlugin extends Plugin { this.loadEmbeddingsToAssistant(); // async update embedding } new AskAssistantModal(this.app, async (question) => { - const answer = await this.assistant.answerQuestion( - question - ); - return answer ?? ""; + try { + const answer = await this.assistant.answerQuestion( + question + ); + return answer; + } catch (e) { + if (e.response) { + console.log(e.response) + new Notice("❌ " + e.response.data.error.message) + } + return { error: true, text: "" } + } }).open(); }, }); @@ -124,11 +132,20 @@ export default class GPTAssistantPlugin extends Plugin { let searchable = oldSearchable.filter((e) => oldSha.has(e.sha1) && newSha.has(e.sha1)); if (fileContents.length) { // create embeddings for new/updated files const chunks = this.assistant.prepareTexts(fileContents); - const newSearchable = await this.assistant.createEmbeddings(chunks); - searchable = newSearchable.concat(searchable); + try { + const newSearchable = await this.assistant.createEmbeddings(chunks); + searchable = newSearchable.concat(searchable); + new Notice("Your data has been loaded into the model."); + + } catch (e) { + if (e.response) { + console.log(e.response) + new Notice("❌ " + e.response.data.error.message) + } + } } - this.saveNamedDataV2({ + this.saveNamedData({ "searchable": searchable, "sha": Array.from(newSha), }); @@ -153,11 +170,7 @@ export default class GPTAssistantPlugin extends Plugin { }); } - async saveNamedData(name: string, data: unknown) { - await this.saveData({ ...(await this.loadData()), [name]: data }); - } - - async saveNamedDataV2(data: CachedData) { + async saveNamedData(data: CachedData) { await this.saveData({ ...(await this.loadData()), ...data }); } } @@ -249,10 +262,9 @@ class AssistantSettings extends PluginSettingTab { tg.onChange(async (value) => { this.plugin.settings.autoUpdate = value; await this.plugin.saveSettings(); - new Notice("updated"); }); }) - + new Setting(containerEl) .setName("Process notes") .setDesc("Load all your notes into the assistant") @@ -269,7 +281,6 @@ class AssistantSettings extends PluginSettingTab { ); await this.plugin.loadEmbeddingsToAssistant(); - new Notice("Your data has been loaded into the model."); }); }); }