diff --git a/manifest.json b/manifest.json index b28085f..2cb0027 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "gpt-assistant", "name": "GPT Assistant", - "version": "0.1.2", + "version": "0.1.3", "minAppVersion": "0.15.0", "description": "Use a GPT-3 based model on your notes and get personalized answers from your knowledge base.", "author": "M7mdisk", diff --git a/src/assistant.ts b/src/assistant.ts index d09e71e..7f1ac7a 100644 --- a/src/assistant.ts +++ b/src/assistant.ts @@ -5,10 +5,16 @@ import { cosineSimilarity } from "./utils"; export interface chunkData { text: string; embeddings: number[]; + sha1: string; } export type EmbeddedData = chunkData[]; +export interface CachedData { + searchable: EmbeddedData; + sha: Array; +} + export type Answer = { error: boolean; text: string }; export class Assistant { MAX_TOKENS = 500; @@ -78,13 +84,20 @@ export class Assistant { }; } - prepareTexts(texts: string[]): string[] { - let shortened: string[] = []; - texts.forEach((text) => { + prepareTexts(texts: EmbeddedData): EmbeddedData { + let shortened: EmbeddedData = []; + texts.forEach((item) => { + const text = item.text; if (this.tokenizer.encode(text).bpe.length > this.MAX_TOKENS) { - shortened = shortened.concat(this.splitIntoMany(text)); + shortened = shortened.concat(this.splitIntoMany(text).map(t => { + return { + text: t, + embeddings: [], + sha1: item.sha1, + } + })); } else { - shortened.push(text); + shortened.push(item); } }); return shortened; @@ -112,14 +125,15 @@ export class Assistant { }); return chunks; } - async createEmbeddings(data: string[]): Promise { + async createEmbeddings(data: EmbeddedData): Promise { const embeddings = await this.openai.createEmbedding({ - input: data, + input: data.map((d) => d.text), model: "text-embedding-ada-002", }); - return data.map((text, idx) => ({ - text, + return data.map((d, idx) => ({ + text: d.text, embeddings: embeddings.data.data[idx].embedding, + sha1: d.sha1, })); } } diff --git a/src/main.ts b/src/main.ts index 0e3b056..e0a9da4 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,4 +1,5 @@ -import { Answer, Assistant } from "./assistant"; +import { Answer, Assistant, EmbeddedData, CachedData } from "./assistant"; +import { sha1File } from "./utils"; import { App, MarkdownRenderer, @@ -12,10 +13,12 @@ import { interface PluginSettings { apiKey: string; + autoUpdate: boolean; } const DEFAULT_SETTINGS: PluginSettings = { apiKey: "", + autoUpdate: false, }; export default class GPTAssistantPlugin extends Plugin { @@ -28,7 +31,6 @@ export default class GPTAssistantPlugin extends Plugin { this.assistant = new Assistant(this.settings.apiKey); if (await this.hasCachedData()) { const { searchable } = await this.loadData(); - this.saveNamedData("searchable", searchable); this.assistant.setData(searchable); } @@ -46,14 +48,42 @@ export default class GPTAssistantPlugin extends Plugin { new Notice("Please provide an API Key in the settings"); return; } + if (this.settings.autoUpdate) { + this.loadEmbeddingsToAssistant(); // async update embedding + } new AskAssistantModal(this.app, async (question) => { - const answer = await this.assistant.answerQuestion( - question - ); - return answer ?? ""; + try { + const answer = await this.assistant.answerQuestion( + question + ); + return answer; + } catch (e) { + if (e.response) { + console.error(e.response) + new Notice("❌ " + e.response.data.error.message) + } + return { error: true, text: "" } + } }).open(); }, }); + + this.addCommand({ + id: "update-assistant", + name: "Update assistant", + callback: async () => { + if (!this.settings.apiKey) { + new Notice("Please provide an API Key in the settings"); + return; + } + new Notice( + "Loading data into model. this could take a while..." + ); + await this.loadEmbeddingsToAssistant(); + new Notice("Your data has been loaded into the model."); + + }, + }); } private async hasCachedData(): Promise { @@ -61,22 +91,68 @@ export default class GPTAssistantPlugin extends Plugin { return data && data.searchable && data.searchable.length; } + private async loadCachedData(): Promise { + const data = await this.loadData(); + if (data && data.searchable && data.searchable.length && + data.sha && data.sha.length) { + return { + searchable: data.searchable, + sha: data.sha, + } + } + return { + searchable: [], + sha: [], + } + } + async loadEmbeddingsToAssistant() { const { vault } = this.app; - const fileContents: string[] = await Promise.all( + const cachedData = await this.loadCachedData(); + const oldSearchable = cachedData.searchable; + const oldSha = new Set(cachedData.sha); + const newSha = new Set(); + + // Load new/updated file contents + const fileContents: EmbeddedData = (await Promise.all( vault .getMarkdownFiles() - .map((file) => - vault.cachedRead(file).then((res) => file.name + res) - ) - ); - const chunks = await this.assistant.prepareTexts(fileContents); - const searchable = await this.assistant.createEmbeddings(chunks); - this.saveNamedData("searchable", searchable); + .map((file) => { + const sha1 = sha1File(file); + newSha.add(sha1); + if (oldSha.has(sha1)) { // file doesn't change + return { text: '', embeddings: [], sha1: sha1 }; + } + return vault.cachedRead(file).then((res) => { + return { text: file.name + res, embeddings: [], sha1: sha1 } + }) + }) + )).filter(f => f.text.length); + + let searchable = oldSearchable.filter((e) => oldSha.has(e.sha1) && newSha.has(e.sha1)); + if (fileContents.length) { // create embeddings for new/updated files + const chunks = this.assistant.prepareTexts(fileContents); + try { + const newSearchable = await this.assistant.createEmbeddings(chunks); + searchable = newSearchable.concat(searchable); + new Notice("Your data has been loaded into the model."); + + } catch (e) { + if (e.response) { + console.error(e.response) + new Notice("❌ " + e.response.data.error.message) + } + } + } + + this.saveNamedData({ + "searchable": searchable, + "sha": Array.from(newSha), + }); this.assistant.setData(searchable); } - onunload() {} + onunload() { } async loadSettings() { this.settings = Object.assign( @@ -94,8 +170,8 @@ export default class GPTAssistantPlugin extends Plugin { }); } - async saveNamedData(name: string, data: unknown) { - await this.saveData({ ...(await this.loadData()), [name]: data }); + async saveNamedData(data: CachedData) { + await this.saveData({ ...(await this.loadData()), ...data }); } } @@ -178,6 +254,17 @@ class AssistantSettings extends PluginSettingTab { }) ); + new Setting(containerEl) + .setName("Automatically update") + .setDesc("Automatically load new notes into the assistant") + .addToggle((tg) => { + tg.setValue(this.plugin.settings.autoUpdate); + tg.onChange(async (value) => { + this.plugin.settings.autoUpdate = value; + await this.plugin.saveSettings(); + }); + }) + new Setting(containerEl) .setName("Process notes") .setDesc("Load all your notes into the assistant") @@ -194,7 +281,6 @@ class AssistantSettings extends PluginSettingTab { ); await this.plugin.loadEmbeddingsToAssistant(); - new Notice("Your data has been loaded into the model."); }); }); } diff --git a/src/utils.ts b/src/utils.ts index 8b97a83..9253f15 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,3 +1,6 @@ +import { createHash } from 'crypto' +import { TFile } from "obsidian"; + function dotProduct(vecA: number[], vecB: number[]) { let product = 0; for (let i = 0; i < vecA.length; i++) { @@ -17,3 +20,7 @@ function magnitude(vec: number[]) { export function cosineSimilarity(vecA: number[], vecB: number[]) { return dotProduct(vecA, vecB) / (magnitude(vecA) * magnitude(vecB)); } + +export function sha1File(file: TFile) { + return createHash('sha1').update(`${file.path}-${file.stat.ctime}-${file.stat.mtime}-${file.stat.size}`).digest('hex') +} diff --git a/versions.json b/versions.json index 26382a1..8213519 100644 --- a/versions.json +++ b/versions.json @@ -1,3 +1,3 @@ { - "1.0.0": "0.15.0" + "0.1.3": "0.15.0" }