mirror of
https://github.com/logancyang/obsidian-copilot.git
synced 2026-07-22 07:50:24 +00:00
[Epic Feature] Vault QA BETA (#285)
This commit is contained in:
Logan Yang
GitHub
parent
15b3b7c304
commit
b16b677abc
18 changed files with 950 additions and 291 deletions
|
|
@ -32,7 +32,7 @@ The `ollama run mistral` command downloads and starts a chat with Mistral 7B rig
|
|||
|
||||
#### Important note about setting context window
|
||||
|
||||
AFAIK `ollama serve` doesn't have a consolidated way to configure context window for all the models at a single place. The current best way is to run `ollama run <modelname>` and then `/set parameter num_ctx 32768` (this is the max for Mistral, set it based on your model requirement), and don't forget to `/save` for each model individually.
|
||||
AFAIK `ollama serve` doesn't have a consolidated way to configure context window for all the models at a single place. The current best way is to run `ollama run <modelname>` and then `/set parameter num_ctx 32768` (this is the max for Mistral, set it based on your model requirement), and don't forget to `/save <modelname>` for each model individually.
|
||||
|
||||
Remember that you MUST set this parameter for Ollama models, or they will silently fail and you will think your long prompt successfully reached the model!
|
||||
|
||||
|
|
@ -56,7 +56,7 @@ Pick OLLAMA (LOCAL) in the model dropdown and start chatting!
|
|||
|
||||
#### Ollama for Windows (Preview)
|
||||
|
||||
Ollama has released a Windows preview version! Just download it to your Windows machine, open your PowerShell, and
|
||||
Ollama has released a Windows preview version! Just download it to your Windows machine, open your PowerShell, and
|
||||
|
||||
```
|
||||
ollama pull <model>
|
||||
|
|
@ -64,4 +64,15 @@ ollama pull <model>
|
|||
$env:OLLAMA_ORIGINS="app://obsidian.md*"; ollama serve
|
||||
```
|
||||
|
||||
#### Now, go crazy with local models using your custom prompts!
|
||||
## Ollama for Local Embeddings
|
||||
Ollama has added support for local embeddings for RAG since v0.1.26! It's super easy to setup, just run
|
||||
|
||||
```
|
||||
ollama pull nomic-embed-text
|
||||
```
|
||||
|
||||
and start your local Ollama server as before. Now you can set your embedding model in Copilot settings as `ollama-nomic-embed-text`, and it will use your local embeddings!
|
||||
|
||||
With this one Ollama server running, you can set your Chat model as Ollama too, meaning it handles both chat streaming and embedding! You can then have a **completely offline QA** experience!
|
||||
|
||||
#### Now, go crazy with local models in Chat mode and QA modes!
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { LangChainParams, SetChainOptions } from '@/aiParams';
|
||||
import ChainFactory, {
|
||||
ChainType
|
||||
ChainType,
|
||||
Document,
|
||||
} from '@/chainFactory';
|
||||
import {
|
||||
AI_SENDER,
|
||||
|
|
@ -8,9 +9,15 @@ import {
|
|||
} from '@/constants';
|
||||
import EncryptionService from '@/encryptionService';
|
||||
import { ProxyChatOpenAI } from '@/langchainWrappers';
|
||||
import { CopilotSettings } from "@/settings/SettingsPage";
|
||||
import { ChatMessage } from '@/sharedState';
|
||||
import { extractChatHistory, getModelName, isSupportedChain } from '@/utils';
|
||||
import VectorDBManager, { MemoryVector } from '@/vectorDBManager';
|
||||
import {
|
||||
extractChatHistory,
|
||||
extractUniqueTitlesFromDocs,
|
||||
getModelName,
|
||||
isSupportedChain
|
||||
} from '@/utils';
|
||||
import VectorDBManager, { MemoryVector, NoteFile, VectorStoreDocument } from '@/vectorDBManager';
|
||||
import { ChatOllama } from "@langchain/community/chat_models/ollama";
|
||||
import { RunnableSequence } from "@langchain/core/runnables";
|
||||
import { BaseChatMemory } from "langchain/memory";
|
||||
|
|
@ -20,9 +27,9 @@ import {
|
|||
MessagesPlaceholder
|
||||
} from "langchain/prompts";
|
||||
import { MultiQueryRetriever } from "langchain/retrievers/multi_query";
|
||||
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
|
||||
import { ScoreThresholdRetriever } from "langchain/retrievers/score_threshold";
|
||||
import { MemoryVectorStore } from "langchain/vectorstores/memory";
|
||||
import { Notice } from 'obsidian';
|
||||
import { App, Notice } from 'obsidian';
|
||||
import ChatModelManager from './chatModelManager';
|
||||
import EmbeddingsManager from './embeddingManager';
|
||||
import MemoryManager from './memoryManager';
|
||||
|
|
@ -31,10 +38,13 @@ import PromptManager from './promptManager';
|
|||
export default class ChainManager {
|
||||
private static chain: RunnableSequence;
|
||||
private static retrievalChain: RunnableSequence;
|
||||
private static retrievedDocuments: Document[] = [];
|
||||
|
||||
private static isOllamaModelActive = false;
|
||||
private static isOpenRouterModelActive = false;
|
||||
|
||||
private app: App;
|
||||
private settings: CopilotSettings;
|
||||
private vectorStore: MemoryVectorStore;
|
||||
private promptManager: PromptManager;
|
||||
private embeddingsManager: EmbeddingsManager;
|
||||
|
|
@ -50,11 +60,15 @@ export default class ChainManager {
|
|||
* @return {void}
|
||||
*/
|
||||
constructor(
|
||||
app: App,
|
||||
langChainParams: LangChainParams,
|
||||
encryptionService: EncryptionService,
|
||||
settings: CopilotSettings,
|
||||
) {
|
||||
// Instantiate singletons
|
||||
this.app = app;
|
||||
this.langChainParams = langChainParams;
|
||||
this.settings = settings;
|
||||
this.memoryManager = MemoryManager.getInstance(this.langChainParams);
|
||||
this.encryptionService = encryptionService;
|
||||
this.chatModelManager = ChatModelManager.getInstance(this.langChainParams, encryptionService);
|
||||
|
|
@ -62,14 +76,18 @@ export default class ChainManager {
|
|||
this.createChainWithNewModel(this.langChainParams.modelDisplayName);
|
||||
}
|
||||
|
||||
private setNoteContent(noteContent: string): void {
|
||||
this.langChainParams.options.noteContent = noteContent;
|
||||
private setNoteFile(noteFile: NoteFile): void {
|
||||
this.langChainParams.options.noteFile = noteFile;
|
||||
}
|
||||
|
||||
private validateChainType(chainType: ChainType): void {
|
||||
if (chainType === undefined || chainType === null) throw new Error('No chain type set');
|
||||
}
|
||||
|
||||
static storeRetrieverDocuments(documents: Document[]) {
|
||||
ChainManager.retrievedDocuments = documents;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the active model and create a new chain
|
||||
* with the specified model display name.
|
||||
|
|
@ -104,7 +122,7 @@ export default class ChainManager {
|
|||
// Create a new chain with the new chatModel
|
||||
this.createChain(
|
||||
this.langChainParams.chainType,
|
||||
{...this.langChainParams.options, forceNewCreation: true},
|
||||
{ ...this.langChainParams.options, forceNewCreation: true },
|
||||
)
|
||||
console.log(`Setting model to ${newModelDisplayName}: ${newModel}`);
|
||||
} catch (error) {
|
||||
|
|
@ -138,7 +156,7 @@ export default class ChainManager {
|
|||
}
|
||||
this.validateChainType(chainType);
|
||||
// MUST set embeddingsManager when switching to QA mode
|
||||
if (chainType === ChainType.RETRIEVAL_QA_CHAIN) {
|
||||
if (chainType === ChainType.LONG_NOTE_QA_CHAIN || chainType === ChainType.VAULT_QA_CHAIN) {
|
||||
this.embeddingsManager = EmbeddingsManager.getInstance(this.langChainParams, this.encryptionService);
|
||||
}
|
||||
|
||||
|
|
@ -177,36 +195,48 @@ export default class ChainManager {
|
|||
this.langChainParams.chainType = ChainType.LLM_CHAIN;
|
||||
break;
|
||||
}
|
||||
case ChainType.RETRIEVAL_QA_CHAIN: {
|
||||
if (!options.noteContent) {
|
||||
case ChainType.LONG_NOTE_QA_CHAIN: {
|
||||
if (!options.noteFile) {
|
||||
new Notice('No note content provided');
|
||||
throw new Error('No note content provided');
|
||||
}
|
||||
|
||||
this.setNoteContent(options.noteContent);
|
||||
const docHash = VectorDBManager.getDocumentHash(options.noteContent);
|
||||
this.setNoteFile(options.noteFile);
|
||||
const docHash = VectorDBManager.getDocumentHash(options.noteFile.path);
|
||||
const parsedMemoryVectors: MemoryVector[] | undefined = await VectorDBManager.getMemoryVectors(docHash);
|
||||
const embeddingsAPI = this.embeddingsManager.getEmbeddingsAPI();
|
||||
if (!embeddingsAPI) {
|
||||
console.error('Error getting embeddings API. Please check your settings.');
|
||||
return;
|
||||
}
|
||||
if (parsedMemoryVectors) {
|
||||
// Index already exists
|
||||
const embeddingsAPI = this.embeddingsManager.getEmbeddingsAPI();
|
||||
if (!embeddingsAPI) {
|
||||
console.error('Error getting embeddings API. Please check your settings.');
|
||||
return;
|
||||
}
|
||||
const vectorStore = await VectorDBManager.rebuildMemoryVectorStore(
|
||||
parsedMemoryVectors,
|
||||
embeddingsAPI,
|
||||
);
|
||||
|
||||
// Create new conversational retrieval chain
|
||||
ChainManager.retrievalChain = ChainFactory.createConversationalRetrievalChain({
|
||||
llm: chatModel,
|
||||
retriever: vectorStore.asRetriever(),
|
||||
})
|
||||
ChainManager.retrievalChain = ChainFactory.createConversationalRetrievalChain(
|
||||
{
|
||||
llm: chatModel,
|
||||
retriever: vectorStore.asRetriever(
|
||||
undefined,
|
||||
(doc) => {
|
||||
return doc.metadata.path === options.noteFile?.path;
|
||||
}
|
||||
),
|
||||
},
|
||||
ChainManager.storeRetrieverDocuments.bind(ChainManager),
|
||||
)
|
||||
console.log('Existing vector store for document hash: ', docHash);
|
||||
} else {
|
||||
// Index doesn't exist
|
||||
await this.buildIndex(options.noteContent, docHash);
|
||||
const vectorStoreDoc = await this.indexFile(options.noteFile);
|
||||
this.vectorStore = await VectorDBManager.getMemoryVectorStore(
|
||||
embeddingsAPI,
|
||||
vectorStoreDoc?._id,
|
||||
);
|
||||
if (!this.vectorStore) {
|
||||
console.error('Error creating vector store.');
|
||||
return;
|
||||
|
|
@ -214,24 +244,63 @@ export default class ChainManager {
|
|||
|
||||
const retriever = MultiQueryRetriever.fromLLM({
|
||||
llm: chatModel,
|
||||
retriever: this.vectorStore.asRetriever(),
|
||||
retriever: this.vectorStore.asRetriever(
|
||||
undefined,
|
||||
(doc) => {
|
||||
return doc.metadata.path === options.noteFile?.path;
|
||||
}
|
||||
),
|
||||
verbose: false,
|
||||
});
|
||||
|
||||
ChainManager.retrievalChain = ChainFactory.createConversationalRetrievalChain({
|
||||
llm: chatModel,
|
||||
retriever: retriever,
|
||||
})
|
||||
ChainManager.retrievalChain = ChainFactory.createConversationalRetrievalChain(
|
||||
{
|
||||
llm: chatModel,
|
||||
retriever: retriever,
|
||||
},
|
||||
ChainManager.storeRetrieverDocuments.bind(ChainManager)
|
||||
)
|
||||
console.log(
|
||||
'New conversational retrieval qa chain with multi-query retriever created for '
|
||||
'New conversational retrieval QA chain with multi-query retriever created for '
|
||||
+ 'document hash: ', docHash
|
||||
);
|
||||
}
|
||||
|
||||
this.langChainParams.chainType = ChainType.RETRIEVAL_QA_CHAIN;
|
||||
console.log('Set chain:', ChainType.RETRIEVAL_QA_CHAIN);
|
||||
this.langChainParams.chainType = ChainType.LONG_NOTE_QA_CHAIN;
|
||||
console.log('Set chain:', ChainType.LONG_NOTE_QA_CHAIN);
|
||||
break;
|
||||
}
|
||||
|
||||
case ChainType.VAULT_QA_CHAIN: {
|
||||
const embeddingsAPI = this.embeddingsManager.getEmbeddingsAPI();
|
||||
if (!embeddingsAPI) {
|
||||
console.error('Error getting embeddings API. Please check your settings.');
|
||||
return;
|
||||
}
|
||||
const vectorStore = await VectorDBManager.getMemoryVectorStore(embeddingsAPI)
|
||||
const retriever = ScoreThresholdRetriever.fromVectorStore(vectorStore, {
|
||||
minSimilarityScore: 0.3, // TODO: Make this a user setting
|
||||
maxK: this.settings.maxSourceChunks, // The maximum number of docs (chunks) to retrieve
|
||||
kIncrement: 2,
|
||||
});
|
||||
|
||||
// Create new conversational retrieval chain
|
||||
ChainManager.retrievalChain = ChainFactory.createConversationalRetrievalChain(
|
||||
{
|
||||
llm: chatModel,
|
||||
retriever: retriever,
|
||||
},
|
||||
ChainManager.storeRetrieverDocuments.bind(ChainManager)
|
||||
)
|
||||
console.log(
|
||||
'New conversational retrieval QA chain with multi-query retriever created for entire vault'
|
||||
);
|
||||
|
||||
this.langChainParams.chainType = ChainType.VAULT_QA_CHAIN;
|
||||
console.log('Set chain:', ChainType.VAULT_QA_CHAIN);
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
this.validateChainType(chainType);
|
||||
break;
|
||||
|
|
@ -285,9 +354,9 @@ export default class ChainManager {
|
|||
if (ignoreSystemMessage) {
|
||||
const effectivePrompt = ignoreSystemMessage
|
||||
? ChatPromptTemplate.fromMessages([
|
||||
new MessagesPlaceholder("history"),
|
||||
HumanMessagePromptTemplate.fromTemplate("{input}"),
|
||||
])
|
||||
new MessagesPlaceholder("history"),
|
||||
HumanMessagePromptTemplate.fromTemplate("{input}"),
|
||||
])
|
||||
: chatPrompt;
|
||||
|
||||
this.setChain(chainType, {
|
||||
|
|
@ -305,7 +374,7 @@ export default class ChainManager {
|
|||
const chatStream = await ChainManager.chain.stream({ input: userMessage } as any);
|
||||
|
||||
try {
|
||||
switch(chainType) {
|
||||
switch (chainType) {
|
||||
case ChainType.LLM_CHAIN:
|
||||
if (debug) {
|
||||
console.log(`*** DEBUG INFO ***\n`
|
||||
|
|
@ -328,7 +397,8 @@ export default class ChainManager {
|
|||
updateCurrentAiMessage(fullAIResponse);
|
||||
}
|
||||
break;
|
||||
case ChainType.RETRIEVAL_QA_CHAIN:
|
||||
case ChainType.LONG_NOTE_QA_CHAIN:
|
||||
case ChainType.VAULT_QA_CHAIN:
|
||||
if (debug) {
|
||||
console.log(`*** DEBUG INFO ***\n`
|
||||
+ `user message: ${userMessage}\n`
|
||||
|
|
@ -340,10 +410,10 @@ export default class ChainManager {
|
|||
+ `chat context turns: ${chatContextTurns}\n`,
|
||||
);
|
||||
console.log('chain RunnableSequence:', ChainManager.chain);
|
||||
console.log('embedding provider:', this.langChainParams.embeddingProvider);
|
||||
console.log('embedding model:', this.langChainParams.embeddingModel);
|
||||
}
|
||||
fullAIResponse = await this.runRetrievalChain(
|
||||
userMessage, memory, updateCurrentAiMessage, abortController
|
||||
userMessage, memory, updateCurrentAiMessage, abortController, { debug },
|
||||
);
|
||||
break;
|
||||
default:
|
||||
|
|
@ -383,6 +453,9 @@ export default class ChainManager {
|
|||
memory: BaseChatMemory,
|
||||
updateCurrentAiMessage: (message: string) => void,
|
||||
abortController: AbortController,
|
||||
options: {
|
||||
debug?: boolean,
|
||||
} = {},
|
||||
): Promise<string> {
|
||||
const memoryVariables = await memory.loadMemoryVariables({});
|
||||
const chatHistory = extractChatHistory(memoryVariables);
|
||||
|
|
@ -399,33 +472,40 @@ export default class ChainManager {
|
|||
fullAIResponse += chunk.content;
|
||||
updateCurrentAiMessage(fullAIResponse);
|
||||
}
|
||||
|
||||
if (options.debug) {
|
||||
console.log('Max source chunks:', this.settings.maxSourceChunks);
|
||||
console.log('Retrieved chunks:', ChainManager.retrievedDocuments);
|
||||
}
|
||||
|
||||
// TODO: This only returns unique note titles, but actual retrieved docs are chunks.
|
||||
// That means multiple chunks can be from the same note. A more advanced logic is needed
|
||||
// to show specific chunks in the future. E.g. collapsed note title when clicked,
|
||||
// expand and reveal the chunk
|
||||
if (this.langChainParams.chainType === ChainType.VAULT_QA_CHAIN) {
|
||||
const docTitles = extractUniqueTitlesFromDocs(ChainManager.retrievedDocuments);
|
||||
const markdownLinks = docTitles
|
||||
.map(
|
||||
(title) =>
|
||||
`[${title}](obsidian://open?vault=${this.app.vault.getName()}&file=${encodeURIComponent(
|
||||
title
|
||||
)})`
|
||||
)
|
||||
.join("\n");
|
||||
fullAIResponse += "\n\n**Sources**:\n" + markdownLinks;
|
||||
}
|
||||
|
||||
return fullAIResponse;
|
||||
}
|
||||
|
||||
async buildIndex(noteContent: string, docHash: string): Promise<void> {
|
||||
// Note: HF can give 503 errors frequently (it's free)
|
||||
console.log('Creating vector store...');
|
||||
try {
|
||||
const textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000 });
|
||||
|
||||
const docs = await textSplitter.createDocuments([noteContent]);
|
||||
const embeddingsAPI = this.embeddingsManager.getEmbeddingsAPI();
|
||||
if (!embeddingsAPI) {
|
||||
const errorMsg = 'Failed to create vector store, embedding API is not set correctly, please check your settings.';
|
||||
new Notice(errorMsg);
|
||||
console.error(errorMsg);
|
||||
return;
|
||||
}
|
||||
this.vectorStore = await MemoryVectorStore.fromDocuments(
|
||||
docs, embeddingsAPI,
|
||||
);
|
||||
// Serialize and save vector store to PouchDB
|
||||
VectorDBManager.setMemoryVectors(this.vectorStore.memoryVectors, docHash);
|
||||
console.log('Vector store created successfully.');
|
||||
new Notice('Vector store created successfully.');
|
||||
} catch (error) {
|
||||
new Notice('Failed to create vector store, please try again:', error);
|
||||
console.error('Failed to create vector store, please try again.:', error);
|
||||
async indexFile(noteFile: NoteFile): Promise<VectorStoreDocument | undefined> {
|
||||
const embeddingsAPI = this.embeddingsManager.getEmbeddingsAPI();
|
||||
if (!embeddingsAPI) {
|
||||
const errorMsg = 'Failed to load file, embedding API is not set correctly, please check your settings.';
|
||||
new Notice(errorMsg);
|
||||
console.error(errorMsg);
|
||||
return;
|
||||
}
|
||||
return await VectorDBManager.indexFile(noteFile, embeddingsAPI);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,14 @@
|
|||
import { LangChainParams } from '@/aiParams';
|
||||
import { ModelProviders } from '@/constants';
|
||||
import {
|
||||
EMBEDDING_MODEL_TO_PROVIDERS,
|
||||
ModelProviders,
|
||||
NOMIC_EMBED_TEXT,
|
||||
} from '@/constants';
|
||||
import EncryptionService from '@/encryptionService';
|
||||
import { ProxyOpenAIEmbeddings } from '@/langchainWrappers';
|
||||
import { CohereEmbeddings } from "@langchain/cohere";
|
||||
import { Embeddings } from "langchain/embeddings/base";
|
||||
import { HuggingFaceInferenceEmbeddings } from "langchain/embeddings/hf";
|
||||
import { OllamaEmbeddings } from "langchain/embeddings/ollama";
|
||||
import { OpenAIEmbeddings } from "langchain/embeddings/openai";
|
||||
|
||||
export default class EmbeddingManager {
|
||||
|
|
@ -24,6 +28,17 @@ export default class EmbeddingManager {
|
|||
return EmbeddingManager.instance;
|
||||
}
|
||||
|
||||
static getModelName(embeddingsInstance: Embeddings): string {
|
||||
const emb = embeddingsInstance as any;
|
||||
if ('model' in emb && emb.model) {
|
||||
return emb.model as string;
|
||||
} else if ('modelName' in emb && emb.modelName) {
|
||||
return emb.modelName as string;
|
||||
} else {
|
||||
throw new Error(`Embeddings instance missing model or modelName properties: ${embeddingsInstance}`);
|
||||
}
|
||||
}
|
||||
|
||||
getEmbeddingsAPI(): Embeddings | undefined {
|
||||
const decrypt = (key: string) => this.encryptionService.getDecryptedKey(key);
|
||||
const {
|
||||
|
|
@ -55,19 +70,15 @@ export default class EmbeddingManager {
|
|||
})
|
||||
) : null;
|
||||
|
||||
switch(this.langChainParams.embeddingProvider) {
|
||||
const embeddingProvder = EMBEDDING_MODEL_TO_PROVIDERS[this.langChainParams.embeddingModel];
|
||||
|
||||
switch(embeddingProvder) {
|
||||
case ModelProviders.OPENAI:
|
||||
if (OpenAIEmbeddingsAPI) {
|
||||
return OpenAIEmbeddingsAPI;
|
||||
}
|
||||
console.error('OpenAI API key is not provided for the embedding model.');
|
||||
break;
|
||||
case ModelProviders.HUGGINGFACE:
|
||||
return new HuggingFaceInferenceEmbeddings({
|
||||
apiKey: decrypt(this.langChainParams.huggingfaceApiKey),
|
||||
maxRetries: 3,
|
||||
maxConcurrency: 3,
|
||||
});
|
||||
case ModelProviders.COHEREAI:
|
||||
return new CohereEmbeddings({
|
||||
apiKey: decrypt(this.langChainParams.cohereApiKey),
|
||||
|
|
@ -87,6 +98,12 @@ export default class EmbeddingManager {
|
|||
}
|
||||
console.error('Azure OpenAI API key is not provided for the embedding model.');
|
||||
break;
|
||||
case ModelProviders.OLLAMA:
|
||||
return new OllamaEmbeddings({
|
||||
...(this.langChainParams.ollamaBaseUrl ? { baseUrl: this.langChainParams.ollamaBaseUrl } : {}),
|
||||
// TODO: Add custom ollama embedding model setting once they have other models
|
||||
model: NOMIC_EMBED_TEXT,
|
||||
})
|
||||
default:
|
||||
console.error('No embedding provider set or no valid API key provided. Defaulting to OpenAI.');
|
||||
return OpenAIEmbeddingsAPI || new OpenAIEmbeddings({
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { ChainType } from '@/chainFactory';
|
||||
import { ChatPromptTemplate } from "langchain/prompts";
|
||||
import { NoteFile } from './vectorDBManager';
|
||||
|
||||
export interface ModelConfig {
|
||||
modelName: string,
|
||||
|
|
@ -43,7 +44,6 @@ export interface LangChainParams {
|
|||
maxTokens: number,
|
||||
systemMessage: string,
|
||||
chatContextTurns: number,
|
||||
embeddingProvider: string,
|
||||
chainType: ChainType, // Default ChainType is set in main.ts getChainManagerParams
|
||||
options: SetChainOptions,
|
||||
ollamaModel: string,
|
||||
|
|
@ -58,7 +58,7 @@ export interface LangChainParams {
|
|||
|
||||
export interface SetChainOptions {
|
||||
prompt?: ChatPromptTemplate;
|
||||
noteContent?: string;
|
||||
noteFile?: NoteFile;
|
||||
forceNewCreation?: boolean;
|
||||
abortController?: AbortController;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,12 +28,19 @@ export interface ConversationalRetrievalChainParams {
|
|||
llm: BaseLanguageModel;
|
||||
retriever: BaseRetriever;
|
||||
options?: {
|
||||
returnSourceDocuments?: boolean;
|
||||
questionGeneratorTemplate?: string;
|
||||
qaTemplate?: string;
|
||||
returnSourceDocuments?: boolean;
|
||||
questionGeneratorTemplate?: string;
|
||||
qaTemplate?: string;
|
||||
}
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
export interface Document<T = Record<string, any>> {
|
||||
// Structure of Document, possibly including pageContent, metadata, etc.
|
||||
pageContent: string;
|
||||
metadata: T;
|
||||
}
|
||||
|
||||
type ConversationalRetrievalQAChainInput = {
|
||||
question: string;
|
||||
chat_history: [string, string][];
|
||||
|
|
@ -46,9 +53,8 @@ type ConversationalRetrievalQAChainInput = {
|
|||
// Add new chain types here
|
||||
export enum ChainType {
|
||||
LLM_CHAIN = 'llm_chain',
|
||||
RETRIEVAL_QA_CHAIN = 'retrieval_qa',
|
||||
// TODO: Wait for official fix and use conversational retrieval chain instead of retrieval qa.
|
||||
CONVERSATIONAL_RETRIEVAL_QA_CHAIN = 'conversational_retrieval_chain',
|
||||
LONG_NOTE_QA_CHAIN = 'long_note_qa',
|
||||
VAULT_QA_CHAIN = 'vault_qa',
|
||||
}
|
||||
|
||||
class ChainFactory {
|
||||
|
|
@ -122,7 +128,8 @@ class ChainFactory {
|
|||
* @return {RunnableSequence} a new conversational retrieval chain
|
||||
*/
|
||||
public static createConversationalRetrievalChain(
|
||||
args: ConversationalRetrievalChainParams
|
||||
args: ConversationalRetrievalChainParams,
|
||||
onDocumentsRetrieved: (documents: Document[]) => void,
|
||||
): RunnableSequence {
|
||||
const { llm, retriever } = args;
|
||||
|
||||
|
|
@ -161,9 +168,15 @@ class ChainFactory {
|
|||
new StringOutputParser(),
|
||||
]);
|
||||
|
||||
const formatDocumentsAsStringAndStore = async (documents: Document[]) => {
|
||||
// Store or log documents for debugging
|
||||
onDocumentsRetrieved(documents);
|
||||
return formatDocumentsAsString(documents);
|
||||
};
|
||||
|
||||
const answerChain = RunnableSequence.from([
|
||||
{
|
||||
context: retriever.pipe(formatDocumentsAsString),
|
||||
context: retriever.pipe(formatDocumentsAsStringAndStore),
|
||||
question: new RunnablePassthrough(),
|
||||
},
|
||||
ANSWER_PROMPT,
|
||||
|
|
@ -171,8 +184,6 @@ class ChainFactory {
|
|||
]);
|
||||
|
||||
const conversationalRetrievalQAChain = standaloneQuestionChain.pipe(answerChain);
|
||||
console.log('New Conversational Retrieval QA Chain created.');
|
||||
|
||||
return conversationalRetrievalQAChain as RunnableSequence;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -143,4 +143,13 @@ export function registerBuiltInCommands(plugin: CopilotPlugin) {
|
|||
plugin.processSelection(editor, "countTokensSelection");
|
||||
},
|
||||
});
|
||||
|
||||
plugin.addCommand({
|
||||
id: "count-total-vault-tokens",
|
||||
name: "Count total tokens in your vault",
|
||||
editorCallback: async (editor: Editor) => {
|
||||
const totalTokens = await plugin.countTotalTokens();
|
||||
new Notice(`Total tokens in your vault: ${totalTokens}`);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import { AI_SENDER, USER_SENDER } from '@/constants';
|
|||
import { AppContext } from '@/context';
|
||||
import { CustomPromptProcessor } from '@/customPromptProcessor';
|
||||
import { getAIResponse } from '@/langchainStream';
|
||||
import CopilotPlugin from '@/main';
|
||||
import { CopilotSettings } from '@/settings/SettingsPage';
|
||||
import SharedState, {
|
||||
ChatMessage, useSharedState,
|
||||
|
|
@ -37,9 +38,8 @@ import {
|
|||
summarizePrompt,
|
||||
tocPrompt
|
||||
} from '@/utils';
|
||||
import VectorDBManager from '@/vectorDBManager';
|
||||
import { EventEmitter } from 'events';
|
||||
import { Notice, TFile, Vault } from 'obsidian';
|
||||
import { Notice, TFile } from 'obsidian';
|
||||
import React, {
|
||||
useContext,
|
||||
useEffect,
|
||||
|
|
@ -59,7 +59,7 @@ interface ChatProps {
|
|||
emitter: EventEmitter;
|
||||
getChatVisibility: () => Promise<boolean>;
|
||||
defaultSaveFolder: string;
|
||||
vault: Vault;
|
||||
plugin: CopilotPlugin;
|
||||
debug: boolean;
|
||||
}
|
||||
|
||||
|
|
@ -70,7 +70,7 @@ const Chat: React.FC<ChatProps> = ({
|
|||
emitter,
|
||||
getChatVisibility,
|
||||
defaultSaveFolder,
|
||||
vault,
|
||||
plugin,
|
||||
debug
|
||||
}) => {
|
||||
const [
|
||||
|
|
@ -84,7 +84,7 @@ const Chat: React.FC<ChatProps> = ({
|
|||
const [abortController, setAbortController] = useState<AbortController | null>(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
|
||||
const app = useContext(AppContext);
|
||||
const app = plugin.app || useContext(AppContext);
|
||||
|
||||
const handleSendMessage = async () => {
|
||||
if (!inputMessage) return;
|
||||
|
|
@ -159,13 +159,13 @@ const Chat: React.FC<ChatProps> = ({
|
|||
}
|
||||
if (settings.chatNoteContextPath) {
|
||||
// Recursively get all note TFiles in the path
|
||||
noteFiles = await getNotesFromPath(vault, settings.chatNoteContextPath);
|
||||
noteFiles = await getNotesFromPath(app.vault, settings.chatNoteContextPath);
|
||||
}
|
||||
if (settings.chatNoteContextTags?.length > 0) {
|
||||
// Get all notes with the specified tags
|
||||
// If path is provided, get all notes with the specified tags in the path
|
||||
// If path is not provided, get all notes with the specified tags
|
||||
noteFiles = await getNotesFromTags(vault, settings.chatNoteContextTags, noteFiles);
|
||||
noteFiles = await getNotesFromTags(app.vault, settings.chatNoteContextTags, noteFiles);
|
||||
}
|
||||
const file = app.workspace.getActiveFile();
|
||||
// If no note context provided, default to the active note
|
||||
|
|
@ -182,8 +182,8 @@ const Chat: React.FC<ChatProps> = ({
|
|||
const notes = [];
|
||||
for (const file of noteFiles) {
|
||||
// Get the content of the note
|
||||
const content = await getFileContent(file, vault);
|
||||
const tags = await getTagsFromNote(file, vault);
|
||||
const content = await getFileContent(file, app.vault);
|
||||
const tags = await getTagsFromNote(file, app.vault);
|
||||
if (content) {
|
||||
notes.push({ name: getFileName(file), content, tags});
|
||||
}
|
||||
|
|
@ -236,7 +236,7 @@ const Chat: React.FC<ChatProps> = ({
|
|||
console.error('No active note found.');
|
||||
return;
|
||||
}
|
||||
const noteContent = await getFileContent(file, vault);
|
||||
const noteContent = await getFileContent(file, app.vault);
|
||||
const noteName = getFileName(file);
|
||||
if (!noteContent) {
|
||||
new Notice('No note content found.');
|
||||
|
|
@ -244,21 +244,38 @@ const Chat: React.FC<ChatProps> = ({
|
|||
return;
|
||||
}
|
||||
|
||||
const docHash = VectorDBManager.getDocumentHash(noteContent);
|
||||
await chainManager.buildIndex(noteContent, docHash);
|
||||
const fileMetadata = app.metadataCache.getFileCache(file)
|
||||
const noteFile = {
|
||||
path: file.path,
|
||||
basename: file.basename,
|
||||
mtime: file.stat.mtime,
|
||||
content: noteContent,
|
||||
metadata: fileMetadata?.frontmatter ?? {},
|
||||
};
|
||||
await chainManager.indexFile(noteFile);
|
||||
const activeNoteOnMessage: ChatMessage = {
|
||||
sender: AI_SENDER,
|
||||
message: `Indexing [[${noteName}]]...\n\n Please switch to "QA" in Mode Selection to ask questions about it.`,
|
||||
isVisible: true,
|
||||
};
|
||||
|
||||
if (currentChain === ChainType.RETRIEVAL_QA_CHAIN) {
|
||||
setChain(ChainType.RETRIEVAL_QA_CHAIN, { noteContent });
|
||||
if (currentChain === ChainType.LONG_NOTE_QA_CHAIN) {
|
||||
setChain(ChainType.LONG_NOTE_QA_CHAIN, { noteFile });
|
||||
}
|
||||
|
||||
addMessage(activeNoteOnMessage);
|
||||
};
|
||||
|
||||
const refreshVaultContext = async () => {
|
||||
if (!app) {
|
||||
console.error('App instance is not available.');
|
||||
return;
|
||||
}
|
||||
|
||||
await plugin.indexVaultToVectorStore();
|
||||
new Notice('Vault index refreshed.');
|
||||
}
|
||||
|
||||
const clearCurrentAiMessage = () => {
|
||||
setCurrentAiMessage('');
|
||||
};
|
||||
|
|
@ -381,7 +398,7 @@ const Chat: React.FC<ChatProps> = ({
|
|||
[]
|
||||
);
|
||||
|
||||
const customPromptProcessor = CustomPromptProcessor.getInstance(vault);
|
||||
const customPromptProcessor = CustomPromptProcessor.getInstance(app.vault);
|
||||
useEffect(
|
||||
createEffect(
|
||||
'applyCustomPrompt',
|
||||
|
|
@ -435,8 +452,10 @@ const Chat: React.FC<ChatProps> = ({
|
|||
onSaveAsNote={handleSaveAsNote}
|
||||
onSendActiveNoteToPrompt={handleSendActiveNoteToPrompt}
|
||||
onForceRebuildActiveNoteContext={forceRebuildActiveNoteContext}
|
||||
onRefreshVaultContext={refreshVaultContext}
|
||||
addMessage={addMessage}
|
||||
vault={vault}
|
||||
vault={app.vault}
|
||||
vault_qa_strategy={plugin.settings.indexVaultToVectorStore}
|
||||
/>
|
||||
<ChatInput
|
||||
inputMessage={inputMessage}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { SetChainOptions } from '@/aiParams';
|
|||
import {
|
||||
AI_SENDER,
|
||||
ChatModelDisplayNames,
|
||||
VAULT_VECTOR_STORE_STRATEGY,
|
||||
} from '@/constants';
|
||||
import {
|
||||
ChatMessage
|
||||
|
|
@ -36,8 +37,10 @@ interface ChatIconsProps {
|
|||
onSaveAsNote: () => void;
|
||||
onSendActiveNoteToPrompt: () => void;
|
||||
onForceRebuildActiveNoteContext: () => void;
|
||||
onRefreshVaultContext: () => void;
|
||||
addMessage: (message: ChatMessage) => void;
|
||||
vault: Vault;
|
||||
vault_qa_strategy: string;
|
||||
}
|
||||
|
||||
const ChatIcons: React.FC<ChatIconsProps> = ({
|
||||
|
|
@ -50,8 +53,10 @@ const ChatIcons: React.FC<ChatIconsProps> = ({
|
|||
onSaveAsNote,
|
||||
onSendActiveNoteToPrompt,
|
||||
onForceRebuildActiveNoteContext,
|
||||
onRefreshVaultContext,
|
||||
addMessage,
|
||||
vault,
|
||||
vault_qa_strategy,
|
||||
}) => {
|
||||
const [selectedChain, setSelectedChain] = useState<ChainType>(currentChain);
|
||||
|
||||
|
|
@ -64,38 +69,58 @@ const ChatIcons: React.FC<ChatIconsProps> = ({
|
|||
}
|
||||
|
||||
useEffect(() => {
|
||||
const handleRetrievalQAChain = async () => {
|
||||
if (selectedChain !== ChainType.RETRIEVAL_QA_CHAIN) {
|
||||
setCurrentChain(selectedChain);
|
||||
return;
|
||||
}
|
||||
|
||||
const handleChainSelection = async () => {
|
||||
if (!app) {
|
||||
console.error('App instance is not available.');
|
||||
return;
|
||||
}
|
||||
|
||||
const file = app.workspace.getActiveFile();
|
||||
if (!file) {
|
||||
new Notice('No active note found.');
|
||||
console.error('No active note found.');
|
||||
if (selectedChain === ChainType.LONG_NOTE_QA_CHAIN) {
|
||||
const file = app.workspace.getActiveFile();
|
||||
if (!file) {
|
||||
new Notice('No active note found.');
|
||||
console.error('No active note found.');
|
||||
return;
|
||||
}
|
||||
|
||||
const noteContent = await getFileContent(file, vault);
|
||||
const fileMetadata = app.metadataCache.getFileCache(file)
|
||||
const noteFile = {
|
||||
path: file.path,
|
||||
basename: file.basename,
|
||||
mtime: file.stat.mtime,
|
||||
content: noteContent ?? "",
|
||||
metadata: fileMetadata?.frontmatter ?? {},
|
||||
};
|
||||
|
||||
const noteName = getFileName(file);
|
||||
|
||||
const activeNoteOnMessage: ChatMessage = {
|
||||
sender: AI_SENDER,
|
||||
message: `OK Feel free to ask me questions about [[${noteName}]]. \n\nPlease note that this is a retrieval-based QA for notes longer than the model context window. Specific questions are encouraged. For generic questions like 'give me a summary', 'brainstorm based on the content', Chat mode with *Send Note to Prompt* button used with a *long context model* is a more suitable choice.`,
|
||||
isVisible: true,
|
||||
};
|
||||
addMessage(activeNoteOnMessage);
|
||||
if (noteContent) {
|
||||
setCurrentChain(selectedChain, { noteFile });
|
||||
}
|
||||
return;
|
||||
} else if (selectedChain === ChainType.VAULT_QA_CHAIN) {
|
||||
if (vault_qa_strategy === VAULT_VECTOR_STORE_STRATEGY.ON_MODE_SWITCH) {
|
||||
await onRefreshVaultContext();
|
||||
}
|
||||
const activeNoteOnMessage: ChatMessage = {
|
||||
sender: AI_SENDER,
|
||||
message: `OK Feel free to ask me questions about your vault: **${app.vault.getName()}**. \n\nIf you have *NEVER* as your auto-index strategy, you must click the *Refresh Index* button below, or run Copilot command: *Index vault for QA* first before you proceed!\n\nPlease note that this is a retrieval-based QA. Specific questions are encouraged. For generic questions like 'give me a summary', 'brainstorm based on the content', Chat mode with *Send Note to Prompt* button used with a *long context model* is a more suitable choice.`,
|
||||
isVisible: true,
|
||||
};
|
||||
addMessage(activeNoteOnMessage);
|
||||
}
|
||||
const noteContent = await getFileContent(file, vault);
|
||||
const noteName = getFileName(file);
|
||||
|
||||
const activeNoteOnMessage: ChatMessage = {
|
||||
sender: AI_SENDER,
|
||||
message: `OK Feel free to ask me questions about [[${noteName}]]. \n\nPlease note that this is a retrieval-based QA for notes longer than the model context window. Specific questions are encouraged. For generic questions like 'give me a summary', 'brainstorm based on the content', Chat mode with *Send Note to Prompt* button used with a *long context model* is a more suitable choice. \n\n(A new mode will be added to work on the entire vault next)`,
|
||||
isVisible: true,
|
||||
};
|
||||
addMessage(activeNoteOnMessage);
|
||||
if (noteContent) {
|
||||
setCurrentChain(selectedChain, { noteContent });
|
||||
}
|
||||
};
|
||||
setCurrentChain(selectedChain);
|
||||
};
|
||||
|
||||
handleRetrievalQAChain();
|
||||
handleChainSelection();
|
||||
}, [selectedChain]);
|
||||
|
||||
return (
|
||||
|
|
@ -143,7 +168,8 @@ const ChatIcons: React.FC<ChatIconsProps> = ({
|
|||
onChange={handleChainChange}
|
||||
>
|
||||
<option value='llm_chain'>Chat</option>
|
||||
<option value='retrieval_qa'>QA</option>
|
||||
<option value='long_note_qa'>Long Note QA</option>
|
||||
<option value='vault_qa'>Vault QA (BETA)</option>
|
||||
</select>
|
||||
<span className="tooltip-text">Mode Selection</span>
|
||||
</div>
|
||||
|
|
@ -154,10 +180,16 @@ const ChatIcons: React.FC<ChatIconsProps> = ({
|
|||
<span className="tooltip-text">Send Note(s) to Prompt<br/>(Set with Copilot command: <br/>set note context <br/>in Chat mode.<br/>Default is active note)</span>
|
||||
</button>
|
||||
)}
|
||||
{selectedChain === 'retrieval_qa' && (
|
||||
{selectedChain === 'long_note_qa' && (
|
||||
<button className='chat-icon-button' onClick={onForceRebuildActiveNoteContext}>
|
||||
<UseActiveNoteAsContextIcon className='icon-scaler' />
|
||||
<span className="tooltip-text">Rebuild Index for Active Note</span>
|
||||
<span className="tooltip-text">Refresh Index<br/>for Active Note</span>
|
||||
</button>
|
||||
)}
|
||||
{selectedChain === 'vault_qa' && (
|
||||
<button className='chat-icon-button' onClick={onRefreshVaultContext}>
|
||||
<UseActiveNoteAsContextIcon className='icon-scaler' />
|
||||
<span className="tooltip-text">Refresh Index<br/>for Vault</span>
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -40,7 +40,9 @@ const ChatSingleMessage: React.FC<ChatSingleMessageProps> = ({ message }) => {
|
|||
{message.sender === USER_SENDER ? (
|
||||
<span>{message.message}</span>
|
||||
) : (
|
||||
<ReactMarkdown>{message.message}</ReactMarkdown>
|
||||
<ReactMarkdown
|
||||
transformLinkUri={null}
|
||||
>{message.message}</ReactMarkdown>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import CopilotPlugin from '@/main';
|
|||
import { CopilotSettings } from '@/settings/SettingsPage';
|
||||
import SharedState from '@/sharedState';
|
||||
import { EventEmitter } from 'events';
|
||||
import { ItemView, Vault, WorkspaceLeaf } from 'obsidian';
|
||||
import { ItemView, WorkspaceLeaf } from 'obsidian';
|
||||
import * as React from 'react';
|
||||
import { Root, createRoot } from 'react-dom/client';
|
||||
|
||||
|
|
@ -15,7 +15,6 @@ export default class CopilotView extends ItemView {
|
|||
private sharedState: SharedState;
|
||||
private chainManager: ChainManager;
|
||||
private root: Root | null = null;
|
||||
private vault: Vault;
|
||||
private settings: CopilotSettings;
|
||||
private defaultSaveFolder: string;
|
||||
private debug = false;
|
||||
|
|
@ -32,7 +31,7 @@ export default class CopilotView extends ItemView {
|
|||
this.emitter = new EventEmitter();
|
||||
this.getChatVisibility = this.getChatVisibility.bind(this);
|
||||
this.userSystemPrompt = plugin.settings.userSystemPrompt;
|
||||
this.vault = plugin.app.vault;
|
||||
this.plugin = plugin;
|
||||
this.defaultSaveFolder = plugin.settings.defaultSaveFolder;
|
||||
}
|
||||
|
||||
|
|
@ -73,7 +72,7 @@ export default class CopilotView extends ItemView {
|
|||
emitter={this.emitter}
|
||||
getChatVisibility={this.getChatVisibility}
|
||||
defaultSaveFolder={this.defaultSaveFolder}
|
||||
vault={this.vault}
|
||||
plugin={this.plugin}
|
||||
debug={this.debug}
|
||||
/>
|
||||
</React.StrictMode>
|
||||
|
|
|
|||
|
|
@ -29,12 +29,12 @@ export enum ChatModelDisplayNames {
|
|||
}
|
||||
|
||||
export const OPENAI_MODELS = new Set([
|
||||
ChatModelDisplayNames.GPT_35_TURBO,
|
||||
ChatModelDisplayNames.GPT_35_TURBO_16K,
|
||||
ChatModelDisplayNames.GPT_4,
|
||||
ChatModelDisplayNames.GPT_4_TURBO,
|
||||
ChatModelDisplayNames.GPT_4_32K,
|
||||
ChatModelDisplayNames.LM_STUDIO,
|
||||
ChatModelDisplayNames.GPT_35_TURBO,
|
||||
ChatModelDisplayNames.GPT_35_TURBO_16K,
|
||||
ChatModelDisplayNames.GPT_4,
|
||||
ChatModelDisplayNames.GPT_4_TURBO,
|
||||
ChatModelDisplayNames.GPT_4_32K,
|
||||
ChatModelDisplayNames.LM_STUDIO,
|
||||
]);
|
||||
|
||||
export const AZURE_MODELS = new Set([
|
||||
|
|
@ -94,21 +94,44 @@ export const EMBEDDING_PROVIDERS = [
|
|||
ModelProviders.AZURE_OPENAI,
|
||||
ModelProviders.COHEREAI,
|
||||
ModelProviders.HUGGINGFACE,
|
||||
ModelProviders.OLLAMA,
|
||||
];
|
||||
|
||||
// Embedding Models
|
||||
export const ADA_V2 = 'text-embedding-ada-002';
|
||||
export const OPENAI_EMBEDDING_SMALL = 'text-embedding-3-small';
|
||||
export const OPENAI_EMBEDDING_LARGE = 'text-embedding-3-large';
|
||||
export const DISTILBERT_NLI = 'sentence-transformers/distilbert-base-nli-mean-tokens';
|
||||
export const INSTRUCTOR_XL = 'hkunlp/instructor-xl'; // Inference API is off for this
|
||||
export const MPNET_V2 = 'sentence-transformers/all-mpnet-base-v2'; // Inference API returns 400
|
||||
export enum EmbeddingModels {
|
||||
OPENAI_EMBEDDING_ADA_V2 = 'text-embedding-ada-002',
|
||||
OPENAI_EMBEDDING_SMALL = 'text-embedding-3-small',
|
||||
OPENAI_EMBEDDING_LARGE = 'text-embedding-3-large',
|
||||
AZURE_OPENAI = 'azure-openai',
|
||||
COHEREAI = 'cohereai',
|
||||
OLLAMA_NOMIC = 'ollama-nomic-embed-text',
|
||||
}
|
||||
|
||||
export const OPENAI_EMBEDDING_MODELS = [
|
||||
ADA_V2,
|
||||
OPENAI_EMBEDDING_SMALL,
|
||||
OPENAI_EMBEDDING_LARGE,
|
||||
]
|
||||
export const EMBEDDING_MODEL_TO_PROVIDERS: Record<string, string> = {
|
||||
[EmbeddingModels.OPENAI_EMBEDDING_ADA_V2]: ModelProviders.OPENAI,
|
||||
[EmbeddingModels.OPENAI_EMBEDDING_SMALL]: ModelProviders.OPENAI,
|
||||
[EmbeddingModels.OPENAI_EMBEDDING_LARGE]: ModelProviders.OPENAI,
|
||||
[EmbeddingModels.AZURE_OPENAI]: ModelProviders.AZURE_OPENAI,
|
||||
[EmbeddingModels.COHEREAI]: ModelProviders.COHEREAI,
|
||||
[EmbeddingModels.OLLAMA_NOMIC]: ModelProviders.OLLAMA,
|
||||
}
|
||||
|
||||
// Embedding Models
|
||||
export const NOMIC_EMBED_TEXT = 'nomic-embed-text';
|
||||
// export const DISTILBERT_NLI = 'sentence-transformers/distilbert-base-nli-mean-tokens';
|
||||
// export const INSTRUCTOR_XL = 'hkunlp/instructor-xl'; // Inference API is off for this
|
||||
// export const MPNET_V2 = 'sentence-transformers/all-mpnet-base-v2'; // Inference API returns 400
|
||||
|
||||
export enum VAULT_VECTOR_STORE_STRATEGY {
|
||||
NEVER = 'NEVER',
|
||||
ON_STARTUP = 'ON STARTUP',
|
||||
ON_MODE_SWITCH = 'ON MODE SWITCH',
|
||||
}
|
||||
|
||||
export const VAULT_VECTOR_STORE_STRATEGIES = [
|
||||
VAULT_VECTOR_STORE_STRATEGY.NEVER,
|
||||
VAULT_VECTOR_STORE_STRATEGY.ON_STARTUP,
|
||||
VAULT_VECTOR_STORE_STRATEGY.ON_MODE_SWITCH,
|
||||
];
|
||||
|
||||
export const DEFAULT_SETTINGS: CopilotSettings = {
|
||||
openAIApiKey: '',
|
||||
|
|
@ -125,7 +148,7 @@ export const DEFAULT_SETTINGS: CopilotSettings = {
|
|||
openRouterModel: 'cognitivecomputations/dolphin-mixtral-8x7b',
|
||||
defaultModel: ChatModels.GPT_4_TURBO,
|
||||
defaultModelDisplayName: ChatModelDisplayNames.GPT_4_TURBO,
|
||||
embeddingModel: OPENAI_EMBEDDING_SMALL,
|
||||
embeddingModel: EmbeddingModels.OPENAI_EMBEDDING_SMALL,
|
||||
temperature: 0.1,
|
||||
maxTokens: 1000,
|
||||
contextTurns: 15,
|
||||
|
|
@ -137,12 +160,12 @@ export const DEFAULT_SETTINGS: CopilotSettings = {
|
|||
ollamaModel: 'llama2',
|
||||
ollamaBaseUrl: '',
|
||||
lmStudioBaseUrl: 'http://localhost:1234/v1',
|
||||
ttlDays: 30,
|
||||
stream: true,
|
||||
embeddingProvider: ModelProviders.OPENAI,
|
||||
defaultSaveFolder: 'copilot-conversations',
|
||||
indexVaultToVectorStore: VAULT_VECTOR_STORE_STRATEGY.NEVER,
|
||||
chatNoteContextPath: '',
|
||||
chatNoteContextTags: [],
|
||||
debug: false,
|
||||
enableEncryption: false,
|
||||
maxSourceChunks: 3,
|
||||
};
|
||||
|
|
|
|||
242
src/main.ts
242
src/main.ts
|
|
@ -1,4 +1,5 @@
|
|||
import ChainManager from "@/LLMProviders/chainManager";
|
||||
import EmbeddingsManager from "@/LLMProviders/embeddingManager";
|
||||
import { LangChainParams, SetChainOptions } from "@/aiParams";
|
||||
import { ChainType } from "@/chainFactory";
|
||||
import { registerBuiltInCommands } from "@/commands";
|
||||
|
|
@ -11,15 +12,16 @@ import {
|
|||
CHAT_VIEWTYPE,
|
||||
DEFAULT_SETTINGS,
|
||||
DEFAULT_SYSTEM_PROMPT,
|
||||
VAULT_VECTOR_STORE_STRATEGY
|
||||
} from "@/constants";
|
||||
import { CustomPrompt } from "@/customPromptProcessor";
|
||||
import EncryptionService from "@/encryptionService";
|
||||
import { CopilotSettingTab, CopilotSettings } from "@/settings/SettingsPage";
|
||||
import SharedState from "@/sharedState";
|
||||
import { sanitizeSettings } from "@/utils";
|
||||
import { areEmbeddingModelsSame, getAllNotesContent, sanitizeSettings } from "@/utils";
|
||||
import VectorDBManager, { VectorStoreDocument } from "@/vectorDBManager";
|
||||
import { Server } from "http";
|
||||
import { Editor, Notice, Plugin, WorkspaceLeaf } from "obsidian";
|
||||
import { Editor, Notice, Plugin, TFile, WorkspaceLeaf } from "obsidian";
|
||||
import PouchDB from "pouchdb";
|
||||
|
||||
export default class CopilotPlugin extends Plugin {
|
||||
|
|
@ -32,6 +34,7 @@ export default class CopilotPlugin extends Plugin {
|
|||
chatIsVisible = false;
|
||||
dbPrompts: PouchDB.Database;
|
||||
dbVectorStores: PouchDB.Database;
|
||||
embeddingsManager: EmbeddingsManager;
|
||||
encryptionService: EncryptionService;
|
||||
server: Server | null = null;
|
||||
|
||||
|
|
@ -45,25 +48,27 @@ export default class CopilotPlugin extends Plugin {
|
|||
const langChainParams = this.getChainManagerParams();
|
||||
this.encryptionService = new EncryptionService(this.settings);
|
||||
this.chainManager = new ChainManager(
|
||||
this.app,
|
||||
langChainParams,
|
||||
this.encryptionService
|
||||
this.encryptionService,
|
||||
this.settings,
|
||||
);
|
||||
|
||||
if (this.settings.enableEncryption) {
|
||||
await this.saveSettings();
|
||||
}
|
||||
|
||||
this.dbPrompts = new PouchDB<CustomPrompt>("copilot_custom_prompts");
|
||||
|
||||
this.dbVectorStores = new PouchDB<VectorStoreDocument>(
|
||||
"copilot_vector_stores"
|
||||
);
|
||||
this.embeddingsManager = EmbeddingsManager.getInstance(
|
||||
langChainParams,
|
||||
this.encryptionService
|
||||
);
|
||||
this.dbPrompts = new PouchDB<CustomPrompt>("copilot_custom_prompts");
|
||||
|
||||
VectorDBManager.initializeDB(this.dbVectorStores);
|
||||
// Remove documents older than TTL days on load
|
||||
VectorDBManager.removeOldDocuments(
|
||||
this.settings.ttlDays * 24 * 60 * 60 * 1000
|
||||
);
|
||||
VectorDBManager.setEmbeddingModel(this.settings.embeddingModel);
|
||||
|
||||
this.registerView(
|
||||
CHAT_VIEWTYPE,
|
||||
|
|
@ -308,6 +313,82 @@ export default class CopilotPlugin extends Plugin {
|
|||
},
|
||||
});
|
||||
|
||||
this.addCommand({
|
||||
id: "garbage-collect-vector-store",
|
||||
name: "Garbage collect vector store (remove files that no longer exist in vault)",
|
||||
callback: async () => {
|
||||
try {
|
||||
const files = this.app.vault.getMarkdownFiles();
|
||||
const filePaths = files.map((file) => file.path);
|
||||
const indexedFiles = await VectorDBManager.getNoteFiles();
|
||||
const indexedFilePaths = indexedFiles.map((file) => file.path);
|
||||
const filesToDelete = indexedFilePaths.filter(
|
||||
(filePath) => !filePaths.includes(filePath)
|
||||
);
|
||||
|
||||
const deletePromises = filesToDelete.map(async (filePath) => {
|
||||
VectorDBManager.removeMemoryVectors(
|
||||
VectorDBManager.getDocumentHash(filePath)
|
||||
);
|
||||
});
|
||||
|
||||
await Promise.all(deletePromises);
|
||||
|
||||
new Notice("Local vector store garbage collected successfully.");
|
||||
console.log(
|
||||
"Local vector store garbage collected successfully, new instance created."
|
||||
);
|
||||
} catch (err) {
|
||||
console.error("Error clearing the local vector store:", err);
|
||||
new Notice(
|
||||
"An error occurred while clearing the local vector store."
|
||||
);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
this.addCommand({
|
||||
id: "index-vault-to-vector-store",
|
||||
name: "Index (refresh) vault for QA",
|
||||
callback: async () => {
|
||||
try {
|
||||
const indexedFileCount = await this.indexVaultToVectorStore();
|
||||
|
||||
new Notice(
|
||||
`${indexedFileCount} vault files indexed to vector store.`
|
||||
);
|
||||
console.log(
|
||||
`${indexedFileCount} vault files indexed to vector store.`
|
||||
);
|
||||
} catch (err) {
|
||||
console.error("Error indexing vault to vector store:", err);
|
||||
new Notice("An error occurred while indexing vault to vector store.");
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
this.addCommand({
|
||||
id: "force-reindex-vault-to-vector-store",
|
||||
name: "Force re-index vault for QA",
|
||||
callback: async () => {
|
||||
try {
|
||||
const indexedFileCount = await this.indexVaultToVectorStore(true);
|
||||
|
||||
new Notice(
|
||||
`${indexedFileCount} vault files indexed to vector store.`
|
||||
);
|
||||
console.log(
|
||||
`${indexedFileCount} vault files indexed to vector store.`
|
||||
);
|
||||
} catch (err) {
|
||||
console.error("Error re-indexing vault to vector store:", err);
|
||||
new Notice(
|
||||
"An error occurred while re-indexing vault to vector store."
|
||||
);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
this.addCommand({
|
||||
id: "set-chat-note-context",
|
||||
name: "Set note context for Chat mode",
|
||||
|
|
@ -324,6 +405,136 @@ export default class CopilotPlugin extends Plugin {
|
|||
).open();
|
||||
},
|
||||
});
|
||||
|
||||
this.registerEvent(
|
||||
this.app.vault.on("delete", (file) => {
|
||||
const docHash = VectorDBManager.getDocumentHash(file.path);
|
||||
VectorDBManager.removeMemoryVectors(docHash);
|
||||
})
|
||||
);
|
||||
|
||||
// Index vault to vector store on startup and after loading all commands
|
||||
// This can take a while, so we don't want to block the startup process
|
||||
if (
|
||||
this.settings.indexVaultToVectorStore ===
|
||||
VAULT_VECTOR_STORE_STRATEGY.ON_STARTUP
|
||||
) {
|
||||
try {
|
||||
await this.indexVaultToVectorStore();
|
||||
} catch (err) {
|
||||
console.error("Error saving vault to vector store:", err);
|
||||
new Notice("An error occurred while saving vault to vector store.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async saveFileToVectorStore(file: TFile): Promise<void> {
|
||||
const embeddingInstance = this.embeddingsManager.getEmbeddingsAPI();
|
||||
if (!embeddingInstance) {
|
||||
new Notice("Embedding instance not found.");
|
||||
return;
|
||||
}
|
||||
const fileContent = await this.app.vault.cachedRead(file);
|
||||
const fileMetadata = this.app.metadataCache.getFileCache(file);
|
||||
const noteFile = {
|
||||
basename: file.basename,
|
||||
path: file.path,
|
||||
mtime: file.stat.mtime,
|
||||
content: fileContent,
|
||||
metadata: fileMetadata?.frontmatter ?? {},
|
||||
};
|
||||
VectorDBManager.indexFile(noteFile, embeddingInstance);
|
||||
}
|
||||
|
||||
async indexVaultToVectorStore(overwrite?: boolean): Promise<number> {
|
||||
const embeddingInstance = this.embeddingsManager.getEmbeddingsAPI();
|
||||
if (!embeddingInstance) {
|
||||
throw new Error("Embedding instance not found.");
|
||||
}
|
||||
|
||||
// Check if embedding model has changed
|
||||
const prevEmbeddingModel = await VectorDBManager.checkEmbeddingModel();
|
||||
// TODO: Remove this when Ollama model is dynamically set
|
||||
const currEmbeddingModel = EmbeddingsManager.getModelName(embeddingInstance);
|
||||
|
||||
console.log(
|
||||
'Prev vs Current embedding models:', prevEmbeddingModel, currEmbeddingModel
|
||||
);
|
||||
|
||||
if (!areEmbeddingModelsSame(prevEmbeddingModel, currEmbeddingModel)) {
|
||||
// Model has changed, clear DB and reindex from scratch
|
||||
overwrite = true;
|
||||
// Clear the current vector store with mixed embeddings
|
||||
try {
|
||||
// Clear the vectorstore db
|
||||
await this.dbVectorStores.destroy();
|
||||
// Reinitialize the database
|
||||
this.dbVectorStores = new PouchDB<VectorStoreDocument>(
|
||||
"copilot_vector_stores"
|
||||
);
|
||||
// Make sure to update the instance with VectorDBManager
|
||||
VectorDBManager.updateDBInstance(this.dbVectorStores);
|
||||
new Notice("Detected change in embedding model. Rebuild vector store from scratch.");
|
||||
console.log(
|
||||
"Detected change in embedding model. Rebuild vector store from scratch."
|
||||
);
|
||||
} catch (err) {
|
||||
console.error("Error clearing vector store for reindexing:", err);
|
||||
new Notice(
|
||||
"Error clearing vector store for reindexing."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const latestMtime = await VectorDBManager.getLatestFileMtime();
|
||||
|
||||
const files = this.app.vault.getMarkdownFiles().filter((file) => {
|
||||
if (!latestMtime || overwrite) return true;
|
||||
return file.stat.mtime > latestMtime;
|
||||
});
|
||||
const fileContents: string[] = await Promise.all(
|
||||
files.map((file) => this.app.vault.cachedRead(file))
|
||||
);
|
||||
const fileMetadatas = files.map((file) =>
|
||||
this.app.metadataCache.getFileCache(file)
|
||||
);
|
||||
|
||||
const totalFiles = files.length;
|
||||
if (totalFiles === 0) {
|
||||
new Notice("Copilot vault index is up-to-date.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
let indexedCount = 0;
|
||||
const indexNotice = new Notice(
|
||||
`Copilot is indexing your vault... 0/${totalFiles} files processed.`,
|
||||
0
|
||||
);
|
||||
|
||||
const loadPromises = files.map(async (file, index) => {
|
||||
const noteFile = {
|
||||
basename: file.basename,
|
||||
path: file.path,
|
||||
mtime: file.stat.mtime,
|
||||
content: fileContents[index],
|
||||
metadata: fileMetadatas[index]?.frontmatter ?? {},
|
||||
};
|
||||
const result = await VectorDBManager.indexFile(
|
||||
noteFile,
|
||||
embeddingInstance
|
||||
);
|
||||
indexedCount++;
|
||||
indexNotice.setMessage(
|
||||
`Copilot is indexing your vault... ${indexedCount}/${totalFiles} files processed.`
|
||||
);
|
||||
return result;
|
||||
});
|
||||
|
||||
await Promise.all(loadPromises);
|
||||
setTimeout(() => {
|
||||
indexNotice.hide();
|
||||
}, 2000);
|
||||
return files.length;
|
||||
}
|
||||
|
||||
async processText(
|
||||
|
|
@ -422,6 +633,17 @@ export default class CopilotPlugin extends Plugin {
|
|||
.filter(Boolean) as string[];
|
||||
}
|
||||
|
||||
async countTotalTokens(): Promise<number> {
|
||||
try {
|
||||
const allContent = await getAllNotesContent(this.app.vault);
|
||||
const totalTokens = await this.chainManager.chatModelManager.countTokens(allContent);
|
||||
return totalTokens;
|
||||
} catch (error) {
|
||||
console.error('Error counting tokens: ', error);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
getChainManagerParams(): LangChainParams {
|
||||
const {
|
||||
openAIApiKey,
|
||||
|
|
@ -440,7 +662,6 @@ export default class CopilotPlugin extends Plugin {
|
|||
temperature,
|
||||
maxTokens,
|
||||
contextTurns,
|
||||
embeddingProvider,
|
||||
ollamaModel,
|
||||
ollamaBaseUrl,
|
||||
lmStudioBaseUrl,
|
||||
|
|
@ -468,7 +689,6 @@ export default class CopilotPlugin extends Plugin {
|
|||
maxTokens: Number(maxTokens),
|
||||
systemMessage: this.settings.userSystemPrompt || DEFAULT_SYSTEM_PROMPT,
|
||||
chatContextTurns: Number(contextTurns),
|
||||
embeddingProvider: embeddingProvider,
|
||||
chainType: ChainType.LLM_CHAIN, // Set LLM_CHAIN as default ChainType
|
||||
options: { forceNewCreation: true } as SetChainOptions,
|
||||
openAIProxyBaseUrl: this.settings.openAIProxyBaseUrl,
|
||||
|
|
|
|||
|
|
@ -31,14 +31,14 @@ export interface CopilotSettings {
|
|||
ollamaModel: string;
|
||||
ollamaBaseUrl: string;
|
||||
lmStudioBaseUrl: string;
|
||||
ttlDays: number;
|
||||
stream: boolean;
|
||||
embeddingProvider: string;
|
||||
defaultSaveFolder: string;
|
||||
indexVaultToVectorStore: string;
|
||||
chatNoteContextPath: string;
|
||||
chatNoteContextTags: string[];
|
||||
debug: boolean;
|
||||
enableEncryption: boolean;
|
||||
maxSourceChunks: number;
|
||||
}
|
||||
|
||||
export class CopilotSettingTab extends PluginSettingTab {
|
||||
|
|
@ -112,4 +112,4 @@ export class CopilotSettingTab extends PluginSettingTab {
|
|||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,68 +1,93 @@
|
|||
import { EMBEDDING_PROVIDERS, OPENAI_EMBEDDING_MODELS } from '@/constants';
|
||||
import { EmbeddingModels, VAULT_VECTOR_STORE_STRATEGIES } from '@/constants';
|
||||
import React from 'react';
|
||||
import ApiSetting from './ApiSetting';
|
||||
import Collapsible from './Collapsible';
|
||||
import { DropdownComponent, SliderComponent } from './SettingBlocks';
|
||||
|
||||
interface QASettingsProps {
|
||||
embeddingProvider: string;
|
||||
setEmbeddingProvider: (value: string) => void;
|
||||
embeddingModel: string;
|
||||
setEmbeddingModel: (value: string) => void;
|
||||
ttlDays: number;
|
||||
setTtlDays: (value: number) => void;
|
||||
cohereApiKey: string;
|
||||
setCohereApiKey: (value: string) => void;
|
||||
huggingfaceApiKey: string;
|
||||
setHuggingfaceApiKey: (value: string) => void;
|
||||
indexVaultToVectorStore: string;
|
||||
setIndexVaultToVectorStore: (value: string) => void;
|
||||
maxSourceChunks: number;
|
||||
setMaxSourceChunks: (value: number) => void;
|
||||
}
|
||||
|
||||
const QASettings: React.FC<QASettingsProps> = ({
|
||||
embeddingProvider,
|
||||
setEmbeddingProvider,
|
||||
embeddingModel,
|
||||
setEmbeddingModel,
|
||||
ttlDays,
|
||||
setTtlDays,
|
||||
cohereApiKey,
|
||||
setCohereApiKey,
|
||||
huggingfaceApiKey,
|
||||
setHuggingfaceApiKey,
|
||||
indexVaultToVectorStore,
|
||||
setIndexVaultToVectorStore,
|
||||
maxSourceChunks,
|
||||
setMaxSourceChunks,
|
||||
}) => {
|
||||
return (
|
||||
<div>
|
||||
<br/>
|
||||
<br/>
|
||||
<br />
|
||||
<br />
|
||||
<h1>QA Settings</h1>
|
||||
<div className="warning-message">
|
||||
YOU MUST REBUILD YOUR INDEX AFTER SWITCHING EMBEDDING PROVIDERS!
|
||||
Vault QA is in BETA and may not be stable. If you have issues please report in the github repo.
|
||||
</div>
|
||||
<p>QA mode relies a <em>local</em> vector index (experimental)
|
||||
<br />
|
||||
OpenAI embeddings currently has the best retrieval quality. CohereAI embeddings are free during trial and are decent. With Huggingface Inference API, your mileage may vary.
|
||||
</p>
|
||||
<p>QA mode relies a <em>local</em> vector index.</p>
|
||||
<h2>Long Note QA vs. Vault QA (BETA)</h2>
|
||||
<p>Long Note QA mode uses the Active Note as context. Vault QA (BETA) uses your entire vault as context. Please ask questions as specific as possible, avoid vague questions to get better results.</p>
|
||||
<h2>Local Embedding Model</h2>
|
||||
<p>Check the <a href='https://github.com/logancyang/obsidian-copilot/blob/master/local_copilot.md'>local copilot</a> setup guide to setup Ollama's local embedding model (requires Ollama v0.1.26 or above).</p>
|
||||
<DropdownComponent
|
||||
name="Embedding Provider"
|
||||
description="The embedding API to call"
|
||||
value={embeddingProvider}
|
||||
onChange={setEmbeddingProvider}
|
||||
options={EMBEDDING_PROVIDERS}
|
||||
/>
|
||||
<DropdownComponent
|
||||
name="OpenAI Embedding Model"
|
||||
description="(for when embedding provider is OpenAI)"
|
||||
name="Embedding Models"
|
||||
description="The embedding API/model to use"
|
||||
value={embeddingModel}
|
||||
onChange={setEmbeddingModel}
|
||||
options={OPENAI_EMBEDDING_MODELS}
|
||||
options={Object.values(EmbeddingModels)}
|
||||
/>
|
||||
<h1>Auto-Index Strategy</h1>
|
||||
<div className="warning-message">
|
||||
If you are using a paid embedding provider, beware of costs for large vaults!
|
||||
</div>
|
||||
<p>
|
||||
When you switch to <strong>Long Note QA</strong> mode, your active note is indexed automatically upon mode switch.
|
||||
<br />
|
||||
When you switch to <strong>Vault QA</strong> mode, your vault is indexed <em>based on the auto-index strategy you select below</em>.
|
||||
<br />
|
||||
</p>
|
||||
<DropdownComponent
|
||||
name="Auto-index vault strategy"
|
||||
description="Decide when you want the vault to be indexed."
|
||||
value={indexVaultToVectorStore}
|
||||
onChange={setIndexVaultToVectorStore}
|
||||
options={VAULT_VECTOR_STORE_STRATEGIES}
|
||||
/>
|
||||
<br />
|
||||
<p>
|
||||
<strong>NEVER</strong>: Notes are never indexed to the vector store unless users run the command <em>Index vault for QA</em> explicitly, or hit the <em>Refresh Index</em> button.
|
||||
<br /><br />
|
||||
<strong>ON STARTUP</strong>: Vault index is refreshed on plugin load/reload.
|
||||
<br /><br />
|
||||
<strong>ON MODE SWITCH (Recommended)</strong>: Vault index is refreshed when switching to Vault QA mode.
|
||||
<br /><br />
|
||||
By "refreshed", it means the vault index is not rebuilt from scratch but rather updated incrementally with new/modified notes since the last index. If you need a complete rebuild, run the commands "Clear vector store" and "Force re-index for QA" manually. This helps reduce costs when using paid embedding models.<br /><br />
|
||||
Beware of the cost if you are using a paid embedding model and have a large vault! You can run Copilot command <em>Count total tokens in your vault</em> and refer to your selected embedding model pricing to estimate indexing costs.
|
||||
</p>
|
||||
<br />
|
||||
<SliderComponent
|
||||
name="TTL Days"
|
||||
description="The number of days to keep embeddings in the index"
|
||||
value={ttlDays}
|
||||
onChange={setTtlDays}
|
||||
name="Max Sources"
|
||||
description="Default is 3 (Recommended). Increase if you want more sources from your notes. A higher number can lead to irrelevant sources and lower quality responses, it also fills up the context window faster."
|
||||
min={1}
|
||||
max={365}
|
||||
max={10}
|
||||
step={1}
|
||||
value={maxSourceChunks}
|
||||
onChange={async (value) => {
|
||||
setMaxSourceChunks(value);
|
||||
}}
|
||||
/>
|
||||
<br />
|
||||
<Collapsible title="Cohere API Settings">
|
||||
|
|
@ -83,26 +108,8 @@ const QASettings: React.FC<QASettingsProps> = ({
|
|||
</a>
|
||||
</p>
|
||||
</Collapsible>
|
||||
<Collapsible title="Huggingface Inference API Settings">
|
||||
<ApiSetting
|
||||
title="Huggingface Inference API Key"
|
||||
value={huggingfaceApiKey}
|
||||
setValue={setHuggingfaceApiKey}
|
||||
placeholder="Enter Huggingface Inference API key"
|
||||
/>
|
||||
<p>
|
||||
Get your Huggingface Inference API key{' '}
|
||||
<a
|
||||
href="https://hf.co/settings/tokens"
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
>
|
||||
here
|
||||
</a>
|
||||
</p>
|
||||
</Collapsible>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default QASettings;
|
||||
export default QASettings;
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ const DropdownComponent: React.FC<DropdownComponentProps> = ({ name, description
|
|||
>
|
||||
{options.map((option, index) => (
|
||||
<option key={index} value={option}>
|
||||
{option.replace(/_/g, ' ').toUpperCase()}
|
||||
{option}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
|
|
|
|||
|
|
@ -34,11 +34,11 @@ export default function SettingsMain({ plugin, reloadPlugin }: SettingsMainProps
|
|||
const [azureOpenAIApiEmbeddingDeploymentName, setAzureOpenAIApiEmbeddingDeploymentName] = useState(plugin.settings.azureOpenAIApiEmbeddingDeploymentName);
|
||||
|
||||
// QA settings
|
||||
const [embeddingProvider, setEmbeddingProvider] = useState(plugin.settings.embeddingProvider);
|
||||
const [embeddingModel, setEmbeddingModel] = useState(plugin.settings.embeddingModel);
|
||||
const [ttlDays, setTtlDays] = useState(plugin.settings.ttlDays);
|
||||
const [cohereApiKey, setCohereApiKey] = useState(plugin.settings.cohereApiKey);
|
||||
const [huggingfaceApiKey, setHuggingfaceApiKey] = useState(plugin.settings.huggingfaceApiKey);
|
||||
const [indexVaultToVectorStore, setIndexVaultToVectorStore] = useState(plugin.settings.indexVaultToVectorStore);
|
||||
const [maxSourceChunks, setMaxSourceChunks] = useState(plugin.settings.maxSourceChunks);
|
||||
|
||||
// Advanced settings
|
||||
const [userSystemPrompt, setUserSystemPrompt] = useState(plugin.settings.userSystemPrompt);
|
||||
|
|
@ -73,11 +73,11 @@ export default function SettingsMain({ plugin, reloadPlugin }: SettingsMainProps
|
|||
plugin.settings.azureOpenAIApiEmbeddingDeploymentName = azureOpenAIApiEmbeddingDeploymentName;
|
||||
|
||||
// QA settings
|
||||
plugin.settings.embeddingProvider = embeddingProvider;
|
||||
plugin.settings.embeddingModel = embeddingModel;
|
||||
plugin.settings.ttlDays = ttlDays;
|
||||
plugin.settings.cohereApiKey = cohereApiKey;
|
||||
plugin.settings.huggingfaceApiKey = huggingfaceApiKey;
|
||||
plugin.settings.indexVaultToVectorStore = indexVaultToVectorStore;
|
||||
plugin.settings.maxSourceChunks = maxSourceChunks;
|
||||
|
||||
// Advanced settings
|
||||
plugin.settings.userSystemPrompt = userSystemPrompt;
|
||||
|
|
@ -196,16 +196,16 @@ export default function SettingsMain({ plugin, reloadPlugin }: SettingsMainProps
|
|||
setAzureOpenAIApiEmbeddingDeploymentName={setAzureOpenAIApiEmbeddingDeploymentName}
|
||||
/>
|
||||
<QASettings
|
||||
embeddingProvider={embeddingProvider}
|
||||
setEmbeddingProvider={setEmbeddingProvider}
|
||||
embeddingModel={embeddingModel}
|
||||
setEmbeddingModel={setEmbeddingModel}
|
||||
ttlDays={ttlDays}
|
||||
setTtlDays={setTtlDays}
|
||||
cohereApiKey={cohereApiKey}
|
||||
setCohereApiKey={setCohereApiKey}
|
||||
huggingfaceApiKey={huggingfaceApiKey}
|
||||
setHuggingfaceApiKey={setHuggingfaceApiKey}
|
||||
indexVaultToVectorStore={indexVaultToVectorStore}
|
||||
setIndexVaultToVectorStore={setIndexVaultToVectorStore}
|
||||
maxSourceChunks={maxSourceChunks}
|
||||
setMaxSourceChunks={setMaxSourceChunks}
|
||||
/>
|
||||
<AdvancedSettings
|
||||
openAIProxyBaseUrl={openAIProxyBaseUrl}
|
||||
|
|
|
|||
59
src/utils.ts
59
src/utils.ts
|
|
@ -1,8 +1,9 @@
|
|||
import { ChainType } from '@/chainFactory';
|
||||
import { ChainType, Document } from '@/chainFactory';
|
||||
import {
|
||||
DEFAULT_SETTINGS,
|
||||
DISPLAY_NAME_TO_MODEL,
|
||||
USER_SENDER
|
||||
NOMIC_EMBED_TEXT,
|
||||
USER_SENDER,
|
||||
} from '@/constants';
|
||||
import { CopilotSettings } from '@/settings/SettingsPage';
|
||||
import { ChatMessage } from '@/sharedState';
|
||||
|
|
@ -71,11 +72,13 @@ export async function getNotesFromTags(
|
|||
}
|
||||
|
||||
export const stringToChainType = (chain: string): ChainType => {
|
||||
switch(chain) {
|
||||
switch (chain) {
|
||||
case 'llm_chain':
|
||||
return ChainType.LLM_CHAIN;
|
||||
case 'retrieval_qa':
|
||||
return ChainType.RETRIEVAL_QA_CHAIN;
|
||||
case 'long_note_qa':
|
||||
return ChainType.LONG_NOTE_QA_CHAIN;
|
||||
case 'vault_qa':
|
||||
return ChainType.VAULT_QA_CHAIN
|
||||
default:
|
||||
throw new Error(`Unknown chain type: ${chain}`);
|
||||
}
|
||||
|
|
@ -92,8 +95,8 @@ export const isRetrievalQAChain = (chain: BaseChain): chain is RetrievalQAChain
|
|||
}
|
||||
|
||||
export const isSupportedChain = (chain: RunnableSequence): chain is RunnableSequence => {
|
||||
return isLLMChain(chain) || isRetrievalQAChain(chain);
|
||||
}
|
||||
return isLLMChain(chain) || isRetrievalQAChain(chain);
|
||||
}
|
||||
|
||||
export const getModelName = (modelDisplayName: string): string => {
|
||||
return DISPLAY_NAME_TO_MODEL[modelDisplayName];
|
||||
|
|
@ -135,6 +138,37 @@ export function getFileName(file: TFile): string {
|
|||
return file.basename;
|
||||
}
|
||||
|
||||
export async function getAllNotesContent(vault: Vault): Promise<string> {
|
||||
let allContent = '';
|
||||
|
||||
const markdownFiles = vault.getMarkdownFiles();
|
||||
|
||||
for (const file of markdownFiles) {
|
||||
const fileContent = await vault.cachedRead(file);
|
||||
allContent += fileContent + ' ';
|
||||
}
|
||||
|
||||
return allContent;
|
||||
}
|
||||
|
||||
export function areEmbeddingModelsSame(
|
||||
model1: string | undefined,
|
||||
model2: string | undefined,
|
||||
): boolean {
|
||||
if (!model1 || !model2) return false;
|
||||
// TODO: Hacks to handle different embedding model names for the same model. Need better handling.
|
||||
if (model1.includes(NOMIC_EMBED_TEXT) && model2.includes(NOMIC_EMBED_TEXT)) {
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
(model1 === "small" && model2 === "cohereai") ||
|
||||
(model1 === "cohereai" && model2 === "small")
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return model1 === model2;
|
||||
}
|
||||
|
||||
export function sanitizeSettings(settings: CopilotSettings): CopilotSettings {
|
||||
const sanitizedSettings: CopilotSettings = { ...settings };
|
||||
|
||||
|
|
@ -314,3 +348,14 @@ export function processVariableNameForNotePath(variableName: string): string {
|
|||
// It's a path, so we just return it as is
|
||||
return variableName;
|
||||
}
|
||||
|
||||
export function extractUniqueTitlesFromDocs(docs: Document[]): string[] {
|
||||
const titlesSet = new Set<string>();
|
||||
docs.forEach(doc => {
|
||||
if (doc.metadata?.title) {
|
||||
titlesSet.add(doc.metadata?.title);
|
||||
}
|
||||
});
|
||||
|
||||
return Array.from(titlesSet);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,23 +1,41 @@
|
|||
import { MD5 } from 'crypto-js';
|
||||
import { Document } from "langchain/document";
|
||||
import { Embeddings } from 'langchain/embeddings/base';
|
||||
import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
|
||||
import { MemoryVectorStore } from "langchain/vectorstores/memory";
|
||||
|
||||
// TODOs
|
||||
// 1. Use embeddingModel rather than embeddingProvider
|
||||
// 2. embeddingModel should be on MemoryVector metadata
|
||||
// 3. VectorDBManager should have a public method checking existing embeddingModel on a MemoryVector
|
||||
export interface VectorStoreDocument {
|
||||
_id: string;
|
||||
_rev?: string;
|
||||
memory_vectors: string;
|
||||
created_at: number;
|
||||
_id: string;
|
||||
_rev?: string;
|
||||
memory_vectors: string;
|
||||
file_mtime: number;
|
||||
embeddingModel: string;
|
||||
created_at: number;
|
||||
}
|
||||
|
||||
export interface MemoryVector {
|
||||
content: string;
|
||||
embedding: number[];
|
||||
metadata: Record<string, any>;
|
||||
content: string;
|
||||
embedding: number[];
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
metadata: Record<string, any>;
|
||||
}
|
||||
|
||||
export interface NoteFile {
|
||||
path: string;
|
||||
basename: string;
|
||||
mtime: number;
|
||||
content: string;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
metadata: Record<string, any>;
|
||||
}
|
||||
|
||||
class VectorDBManager {
|
||||
public static db: PouchDB.Database | null = null;
|
||||
public static embeddingModel: string | null = null;
|
||||
|
||||
public static initializeDB(db: PouchDB.Database): void {
|
||||
this.db = db;
|
||||
|
|
@ -27,25 +45,31 @@ class VectorDBManager {
|
|||
this.db = newDb;
|
||||
}
|
||||
|
||||
public static setEmbeddingModel(embeddingModel: string): void {
|
||||
this.embeddingModel = embeddingModel;
|
||||
}
|
||||
|
||||
public static getDocumentHash(sourceDocument: string): string {
|
||||
return MD5(sourceDocument).toString();
|
||||
}
|
||||
|
||||
public static async rebuildMemoryVectorStore(
|
||||
memoryVectors: MemoryVector[], embeddingsAPI: Embeddings
|
||||
memoryVectors: MemoryVector[],
|
||||
embeddingsAPI: Embeddings
|
||||
) {
|
||||
if (!Array.isArray(memoryVectors)) {
|
||||
throw new TypeError("Expected memoryVectors to be an array");
|
||||
}
|
||||
// Extract the embeddings and documents from the deserialized memoryVectors
|
||||
const embeddingsArray: number[][] = memoryVectors.map(
|
||||
memoryVector => memoryVector.embedding
|
||||
(memoryVector) => memoryVector.embedding
|
||||
);
|
||||
const documentsArray = memoryVectors.map(
|
||||
memoryVector => new Document({
|
||||
pageContent: memoryVector.content,
|
||||
metadata: memoryVector.metadata
|
||||
})
|
||||
(memoryVector) =>
|
||||
new Document({
|
||||
pageContent: memoryVector.content,
|
||||
metadata: memoryVector.metadata,
|
||||
})
|
||||
);
|
||||
|
||||
// Create a new MemoryVectorStore instance
|
||||
|
|
@ -54,7 +78,49 @@ class VectorDBManager {
|
|||
return memoryVectorStore;
|
||||
}
|
||||
|
||||
public static async setMemoryVectors(memoryVectors: MemoryVector[], docHash: string): Promise<void> {
|
||||
public static async getMemoryVectorStore(
|
||||
embeddingsAPI: Embeddings,
|
||||
docHash?: string
|
||||
): Promise<MemoryVectorStore> {
|
||||
if (!this.db) throw new Error("DB not initialized");
|
||||
if (!this.embeddingModel) throw new Error("Embedding model not set");
|
||||
|
||||
let allDocsResponse;
|
||||
if (docHash) {
|
||||
// Fetch a single document by its _id
|
||||
const doc = await this.db.get(docHash);
|
||||
allDocsResponse = { rows: [{ doc }] };
|
||||
} else {
|
||||
// Fetch all documents
|
||||
allDocsResponse = await this.db.allDocs({ include_docs: true });
|
||||
}
|
||||
|
||||
const allDocs = allDocsResponse.rows
|
||||
.map((row) => row.doc as VectorStoreDocument)
|
||||
.filter((doc) => doc.embeddingModel === this.embeddingModel);
|
||||
const memoryVectors = allDocs
|
||||
.map((doc) => JSON.parse(doc.memory_vectors) as MemoryVector[])
|
||||
.flat();
|
||||
const embeddingsArray: number[][] = memoryVectors.map(
|
||||
(memoryVector) => memoryVector.embedding
|
||||
);
|
||||
const documentsArray = memoryVectors.map(
|
||||
(memoryVector) =>
|
||||
new Document({
|
||||
pageContent: memoryVector.content,
|
||||
metadata: memoryVector.metadata,
|
||||
})
|
||||
);
|
||||
// Create a new MemoryVectorStore instance
|
||||
const memoryVectorStore = new MemoryVectorStore(embeddingsAPI);
|
||||
await memoryVectorStore.addVectors(embeddingsArray, documentsArray);
|
||||
return memoryVectorStore;
|
||||
}
|
||||
|
||||
public static async setMemoryVectors(
|
||||
memoryVectors: MemoryVector[],
|
||||
docHash: string
|
||||
): Promise<void> {
|
||||
if (!this.db) throw new Error("DB not initialized");
|
||||
if (!Array.isArray(memoryVectors)) {
|
||||
throw new TypeError("Expected memoryVectors to be an array");
|
||||
|
|
@ -62,14 +128,14 @@ class VectorDBManager {
|
|||
const serializedMemoryVectors = JSON.stringify(memoryVectors);
|
||||
try {
|
||||
// Attempt to fetch the existing document, if it exists.
|
||||
const existingDoc = await this.db.get(docHash).catch(err => null);
|
||||
const existingDoc = await this.db.get(docHash).catch((err) => null);
|
||||
|
||||
// Prepare the document to be saved.
|
||||
const docToSave = {
|
||||
_id: docHash,
|
||||
memory_vectors: serializedMemoryVectors,
|
||||
created_at: Date.now(),
|
||||
_rev: existingDoc?._rev // Add the current revision if the document exists.
|
||||
_rev: existingDoc?._rev, // Add the current revision if the document exists.
|
||||
};
|
||||
|
||||
// Save the document.
|
||||
|
|
@ -79,7 +145,164 @@ class VectorDBManager {
|
|||
}
|
||||
}
|
||||
|
||||
public static async getMemoryVectors(docHash: string): Promise<MemoryVector[] | undefined> {
|
||||
public static async indexFile(noteFile: NoteFile, embeddingsAPI: Embeddings) {
|
||||
if (!this.db) throw new Error("DB not initialized");
|
||||
if (!this.embeddingModel) throw new Error("Embedding model not set");
|
||||
|
||||
// Markdown splitter: https://js.langchain.com/docs/modules/data_connection/document_transformers/code_splitter#markdown
|
||||
const textSplitter = RecursiveCharacterTextSplitter.fromLanguage(
|
||||
"markdown",
|
||||
{
|
||||
chunkSize: 5000,
|
||||
}
|
||||
);
|
||||
// Add note title as contextual chunk headers
|
||||
// https://js.langchain.com/docs/modules/data_connection/document_transformers/contextual_chunk_headers
|
||||
const splitDocument = await textSplitter.createDocuments(
|
||||
[noteFile.content],
|
||||
[],
|
||||
{
|
||||
chunkHeader: "[[" + noteFile.basename + "]]" + "\n\n---\n\n",
|
||||
appendChunkOverlapHeader: true,
|
||||
}
|
||||
);
|
||||
const docVectors = await embeddingsAPI.embedDocuments(
|
||||
splitDocument.map((doc) => doc.pageContent)
|
||||
);
|
||||
const memoryVectors = docVectors.map((docVector, i) => ({
|
||||
content: splitDocument[i].pageContent,
|
||||
metadata: {
|
||||
...noteFile.metadata,
|
||||
title: noteFile.basename,
|
||||
path: noteFile.path,
|
||||
},
|
||||
embedding: docVector,
|
||||
}));
|
||||
const docHash = VectorDBManager.getDocumentHash(noteFile.path);
|
||||
|
||||
const serializedMemoryVectors = JSON.stringify(memoryVectors);
|
||||
try {
|
||||
// Attempt to fetch the existing document, if it exists.
|
||||
const existingDoc = await this.db.get(docHash).catch((err) => null);
|
||||
|
||||
// Prepare the document to be saved.
|
||||
const docToSave: VectorStoreDocument = {
|
||||
_id: docHash,
|
||||
memory_vectors: serializedMemoryVectors,
|
||||
file_mtime: noteFile.mtime,
|
||||
embeddingModel: this.embeddingModel,
|
||||
created_at: Date.now(),
|
||||
_rev: existingDoc?._rev, // Add the current revision if the document exists.
|
||||
};
|
||||
|
||||
// Save the document.
|
||||
await this.db.put(docToSave);
|
||||
return docToSave;
|
||||
} catch (err) {
|
||||
console.error("Error storing vectors in VectorDB:", err);
|
||||
}
|
||||
}
|
||||
|
||||
public static async getNoteFiles(): Promise<NoteFile[]> {
|
||||
if (!this.db) throw new Error("DB not initialized");
|
||||
try {
|
||||
const allDocsResponse = await this.db.allDocs<VectorStoreDocument>({
|
||||
include_docs: true,
|
||||
});
|
||||
const allDocs = allDocsResponse.rows.map(
|
||||
(row) => row.doc as VectorStoreDocument
|
||||
);
|
||||
const memoryVectors = allDocs
|
||||
.map((doc) => JSON.parse(doc.memory_vectors) as MemoryVector[])
|
||||
.filter((memoryVectors) => memoryVectors.length > 0);
|
||||
|
||||
const noteFiles = memoryVectors
|
||||
.map((memoryVectors, i) => {
|
||||
const doc = allDocs[i];
|
||||
const noteFile: NoteFile = {
|
||||
path: memoryVectors[0].metadata.path,
|
||||
basename: memoryVectors[0].metadata.title,
|
||||
mtime: doc.file_mtime,
|
||||
content: memoryVectors[0].content,
|
||||
metadata: memoryVectors[0].metadata,
|
||||
};
|
||||
return noteFile;
|
||||
})
|
||||
.filter((noteFile): noteFile is NoteFile => noteFile !== undefined);
|
||||
return noteFiles;
|
||||
} catch (err) {
|
||||
console.error("Error getting note files from VectorDB:", err);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
public static async removeMemoryVectors(docHash: string): Promise<void> {
|
||||
if (!this.db) throw new Error("DB not initialized");
|
||||
try {
|
||||
const doc = await this.db.get(docHash);
|
||||
if (doc) {
|
||||
await this.db.remove(doc);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Error removing file from VectorDB:", err);
|
||||
}
|
||||
}
|
||||
|
||||
public static async getLatestFileMtime(): Promise<number> {
|
||||
if (!this.db) throw new Error("DB not initialized");
|
||||
if (!this.embeddingModel) throw new Error("Embedding provider not set");
|
||||
|
||||
try {
|
||||
const allDocsResponse = await this.db.allDocs<VectorStoreDocument>({
|
||||
include_docs: true,
|
||||
});
|
||||
const allDocs = allDocsResponse.rows
|
||||
.map((row) => row.doc as VectorStoreDocument)
|
||||
.filter((doc) => doc.embeddingModel === this.embeddingModel);
|
||||
const newestFileMtime = allDocs
|
||||
.map((doc) => doc.file_mtime)
|
||||
.sort((a, b) => b - a)[0];
|
||||
return newestFileMtime;
|
||||
} catch (err) {
|
||||
console.error("Error getting newest file mtime from VectorDB:", err);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
public static async checkEmbeddingModel(): Promise<string | undefined> {
|
||||
if (!this.db) throw new Error("DB not initialized");
|
||||
|
||||
try {
|
||||
// Fetch all documents
|
||||
const allDocsResponse = await this.db.allDocs<VectorStoreDocument>({
|
||||
include_docs: true,
|
||||
});
|
||||
const allDocs = allDocsResponse.rows.map(
|
||||
(row) => row.doc as VectorStoreDocument
|
||||
);
|
||||
|
||||
// Check if there are any documents
|
||||
if (allDocs.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Extract the embeddingModel from all documents
|
||||
const embeddingModels = allDocs.map((doc) => doc.embeddingModel);
|
||||
|
||||
// Check if all documents have the same embeddingModel
|
||||
const allSame = embeddingModels.every(
|
||||
(model) => model === embeddingModels[0]
|
||||
);
|
||||
return allSame ? embeddingModels[0] : undefined;
|
||||
} catch (err) {
|
||||
console.error("Error checking last embedding model from VectorDB:", err);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
public static async getMemoryVectors(
|
||||
docHash: string
|
||||
): Promise<MemoryVector[] | undefined> {
|
||||
if (!this.db) throw new Error("DB not initialized");
|
||||
try {
|
||||
const doc: VectorStoreDocument = await this.db.get(docHash);
|
||||
|
|
@ -90,45 +313,6 @@ class VectorDBManager {
|
|||
console.log("No vectors found in VectorDB for dochash:", docHash);
|
||||
}
|
||||
}
|
||||
|
||||
public static async removeOldDocuments(ttl: number): Promise<void> {
|
||||
if (!this.db) throw new Error("DB not initialized");
|
||||
|
||||
try {
|
||||
const thresholdTime = Date.now() - ttl;
|
||||
|
||||
// Fetch all documents from the database
|
||||
const allDocsResponse = await this.db.allDocs<{ created_at: number }>({ include_docs: true });
|
||||
|
||||
// Filter out the documents older than 2 weeks
|
||||
const oldDocs = allDocsResponse.rows.filter(row => {
|
||||
// Assert the doc type
|
||||
const doc = row.doc as VectorStoreDocument;
|
||||
return doc && doc.created_at < thresholdTime;
|
||||
});
|
||||
|
||||
if (oldDocs.length === 0) {
|
||||
return;
|
||||
}
|
||||
// Prepare the documents for deletion
|
||||
const docsToDelete = oldDocs.map(row => ({
|
||||
_id: row.id,
|
||||
_rev: (row.doc as VectorStoreDocument)._rev,
|
||||
_deleted: true
|
||||
}));
|
||||
|
||||
// Delete the old documents
|
||||
await this.db.bulkDocs(docsToDelete);
|
||||
console.log("Deleted old documents from VectorDB");
|
||||
}
|
||||
catch (err) {
|
||||
console.error("Error removing old documents from VectorDB:", err);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Implement advanced stale document removal.
|
||||
// NOTE: Cannot just rely on note title + ts because a "document" here is a chunk from
|
||||
// the original note. Need a better strategy.
|
||||
}
|
||||
|
||||
export default VectorDBManager;
|
||||
|
|
|
|||
Loading…
Reference in a new issue