obsidian-plugin/logancyang_obsidian-copilot
1
0
Fork 0
mirror of https://github.com/logancyang/obsidian-copilot.git synced 2026-07-22 07:50:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implement index exclusion list and tests (#352)

Browse source
This commit is contained in:
Logan Yang 2024-03-09 21:29:35 -08:00 committed by GitHub
parent 17f41500f3
commit da59d0a7ac
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 250 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
src/components/QAExclusionModal.tsx Normal file
View file

@ -0,0 +1,50 @@
import { CopilotSettings } from "@/settings/SettingsPage";
import { App, Modal } from "obsidian";
export class QAExclusionModal extends Modal {
private settings: CopilotSettings;
private onSubmit: (paths: string) => void;
constructor(app: App, settings: CopilotSettings, onSubmit: (paths: string) => void) {
super(app);
this.settings = settings;
this.onSubmit = onSubmit;
}
onOpen() {
const formContainer = this.contentEl.createEl('div', { cls: 'copilot-command-modal' });
const pathContainer = formContainer.createEl('div', { cls: 'copilot-command-input-container' });
pathContainer.createEl('h3', { text: 'Exclude by Folder Path or Note Title', cls: 'copilot-command-header' });
const descFragment = createFragment((frag) => {
frag.appendText('All notes under the paths will be excluded from indexing');
});
pathContainer.appendChild(descFragment);
const pathField = pathContainer.createEl(
'input',
{
type: 'text',
cls: 'copilot-command-input',
value: this.settings.qaExclusionPaths,
placeholder: 'Enter /folderPath, [[note title]] separated by commas',
}
);
pathField.setAttribute('name', 'folderPath');
const submitButtonContainer = formContainer.createEl('div', { cls: 'copilot-command-save-btn-container' });
const submitButton = submitButtonContainer.createEl('button', { text: 'Submit', cls: 'copilot-command-save-btn' });
submitButton.addEventListener('click', () => {
// Parse the input list
const pathsValue = pathField.value
.split(',')
.map(pathValue => pathValue.trim())
.filter(pathValue => pathValue !== '')
.join(',');
this.onSubmit(pathsValue);
this.close();
});
}
}

158
src/constants.ts
View file

@ -1,32 +1,33 @@
import { CopilotSettings } from '@/settings/SettingsPage';
import { CopilotSettings } from "@/settings/SettingsPage";
export const CHAT_VIEWTYPE = 'copilot-chat-view';
export const USER_SENDER = 'user';
export const AI_SENDER = 'ai';
export const DEFAULT_SYSTEM_PROMPT = 'You are Obsidian Copilot, a helpful assistant that integrates AI to Obsidian note-taking.';
export const CHAT_VIEWTYPE = "copilot-chat-view";
export const USER_SENDER = "user";
export const AI_SENDER = "ai";
export const DEFAULT_SYSTEM_PROMPT =
"You are Obsidian Copilot, a helpful assistant that integrates AI to Obsidian note-taking.";
export enum ChatModels {
GPT_35_TURBO = 'gpt-3.5-turbo',
GPT_35_TURBO_16K = 'gpt-3.5-turbo-16k',
GPT_4 = 'gpt-4',
GPT_4_TURBO = 'gpt-4-turbo-preview',
GPT_4_32K = 'gpt-4-32k',
GEMINI_PRO = 'gemini-pro',
OLLAMA = 'ollama',
GPT_35_TURBO = "gpt-3.5-turbo",
GPT_35_TURBO_16K = "gpt-3.5-turbo-16k",
GPT_4 = "gpt-4",
GPT_4_TURBO = "gpt-4-turbo-preview",
GPT_4_32K = "gpt-4-32k",
GEMINI_PRO = "gemini-pro",
OLLAMA = "ollama",
}
export enum ChatModelDisplayNames {
GPT_35_TURBO = 'GPT-3.5',
GPT_35_TURBO_16K = 'GPT-3.5 16K',
GPT_4 = 'GPT-4',
GPT_4_TURBO = 'GPT-4 TURBO',
GPT_4_32K = 'GPT-4 32K',
AZURE_OPENAI = 'AZURE OPENAI',
CLAUDE = 'CLAUDE 3',
GEMINI_PRO = 'GEMINI PRO',
OPENROUTERAI = 'OPENROUTER.AI',
OLLAMA = 'OLLAMA (LOCAL)',
LM_STUDIO = 'LM STUDIO (LOCAL)',
GPT_35_TURBO = "GPT-3.5",
GPT_35_TURBO_16K = "GPT-3.5 16K",
GPT_4 = "GPT-4",
GPT_4_TURBO = "GPT-4 TURBO",
GPT_4_32K = "GPT-4 32K",
AZURE_OPENAI = "AZURE OPENAI",
CLAUDE = "CLAUDE 3",
GEMINI_PRO = "GEMINI PRO",
OPENROUTERAI = "OPENROUTER.AI",
OLLAMA = "OLLAMA (LOCAL)",
LM_STUDIO = "LM STUDIO (LOCAL)",
}
export const OPENAI_MODELS = new Set([
@ -38,29 +39,19 @@ export const OPENAI_MODELS = new Set([
ChatModelDisplayNames.LM_STUDIO,
]);
export const AZURE_MODELS = new Set([
ChatModelDisplayNames.AZURE_OPENAI,
]);
export const AZURE_MODELS = new Set([ChatModelDisplayNames.AZURE_OPENAI]);
export const GOOGLE_MODELS = new Set([
ChatModelDisplayNames.GEMINI_PRO,
]);
export const GOOGLE_MODELS = new Set([ChatModelDisplayNames.GEMINI_PRO]);
export const ANTHROPIC_MODELS = new Set([
ChatModelDisplayNames.CLAUDE,
]);
export const ANTHROPIC_MODELS = new Set([ChatModelDisplayNames.CLAUDE]);
export const OPENROUTERAI_MODELS = new Set([
ChatModelDisplayNames.OPENROUTERAI,
])
export const OLLAMA_MODELS = new Set([
ChatModelDisplayNames.OLLAMA,
]);
export const LM_STUDIO_MODELS = new Set([
ChatModelDisplayNames.LM_STUDIO,
]);
export const OLLAMA_MODELS = new Set([ChatModelDisplayNames.OLLAMA]);
export const LM_STUDIO_MODELS = new Set([ChatModelDisplayNames.LM_STUDIO]);
export const DISPLAY_NAME_TO_MODEL: Record<string, string> = {
[ChatModelDisplayNames.GPT_35_TURBO]: ChatModels.GPT_35_TURBO,
@ -68,21 +59,21 @@ export const DISPLAY_NAME_TO_MODEL: Record<string, string> = {
[ChatModelDisplayNames.GPT_4]: ChatModels.GPT_4,
[ChatModelDisplayNames.GPT_4_TURBO]: ChatModels.GPT_4_TURBO,
[ChatModelDisplayNames.GPT_4_32K]: ChatModels.GPT_4_32K,
[ChatModelDisplayNames.AZURE_OPENAI]: 'azure_openai',
[ChatModelDisplayNames.AZURE_OPENAI]: "azure_openai",
[ChatModelDisplayNames.GEMINI_PRO]: ChatModels.GEMINI_PRO,
};
// Model Providers
export enum ModelProviders {
OPENAI = 'openai',
HUGGINGFACE = 'huggingface',
COHEREAI = 'cohereai',
AZURE_OPENAI = 'azure_openai',
ANTHROPIC = 'anthropic',
GOOGLE = 'google',
OPENROUTERAI = 'openrouterai',
LM_STUDIO = 'lm_studio',
OLLAMA = 'ollama',
OPENAI = "openai",
HUGGINGFACE = "huggingface",
COHEREAI = "cohereai",
AZURE_OPENAI = "azure_openai",
ANTHROPIC = "anthropic",
GOOGLE = "google",
OPENROUTERAI = "openrouterai",
LM_STUDIO = "lm_studio",
OLLAMA = "ollama",
}
export const VENDOR_MODELS: Record<string, Set<string>> = {
@ -104,12 +95,12 @@ export const EMBEDDING_PROVIDERS = [
];
export enum EmbeddingModels {
OPENAI_EMBEDDING_ADA_V2 = 'text-embedding-ada-002',
OPENAI_EMBEDDING_SMALL = 'text-embedding-3-small',
OPENAI_EMBEDDING_LARGE = 'text-embedding-3-large',
AZURE_OPENAI = 'azure-openai',
COHEREAI = 'cohereai',
OLLAMA_NOMIC = 'ollama-nomic-embed-text',
OPENAI_EMBEDDING_ADA_V2 = "text-embedding-ada-002",
OPENAI_EMBEDDING_SMALL = "text-embedding-3-small",
OPENAI_EMBEDDING_LARGE = "text-embedding-3-large",
AZURE_OPENAI = "azure-openai",
COHEREAI = "cohereai",
OLLAMA_NOMIC = "ollama-nomic-embed-text",
}
export const EMBEDDING_MODEL_TO_PROVIDERS: Record<string, string> = {
@ -119,18 +110,18 @@ export const EMBEDDING_MODEL_TO_PROVIDERS: Record<string, string> = {
[EmbeddingModels.AZURE_OPENAI]: ModelProviders.AZURE_OPENAI,
[EmbeddingModels.COHEREAI]: ModelProviders.COHEREAI,
[EmbeddingModels.OLLAMA_NOMIC]: ModelProviders.OLLAMA,
}
};
// Embedding Models
export const NOMIC_EMBED_TEXT = 'nomic-embed-text';
export const NOMIC_EMBED_TEXT = "nomic-embed-text";
// export const DISTILBERT_NLI = 'sentence-transformers/distilbert-base-nli-mean-tokens';
// export const INSTRUCTOR_XL = 'hkunlp/instructor-xl'; // Inference API is off for this
// export const MPNET_V2 = 'sentence-transformers/all-mpnet-base-v2'; // Inference API returns 400
export enum VAULT_VECTOR_STORE_STRATEGY {
NEVER = 'NEVER',
ON_STARTUP = 'ON STARTUP',
ON_MODE_SWITCH = 'ON MODE SWITCH',
NEVER = "NEVER",
ON_STARTUP = "ON STARTUP",
ON_MODE_SWITCH = "ON MODE SWITCH",
}
export const VAULT_VECTOR_STORE_STRATEGIES = [
@ -142,37 +133,38 @@ export const VAULT_VECTOR_STORE_STRATEGIES = [
export const PROXY_SERVER_PORT = 53001;
export const DEFAULT_SETTINGS: CopilotSettings = {
openAIApiKey: '',
huggingfaceApiKey: '',
cohereApiKey: '',
anthropicApiKey: '',
anthropicModel: 'claude-3-sonnet-20240229',
azureOpenAIApiKey: '',
azureOpenAIApiInstanceName: '',
azureOpenAIApiDeploymentName: '',
azureOpenAIApiVersion: '',
azureOpenAIApiEmbeddingDeploymentName: '',
googleApiKey: '',
openRouterAiApiKey: '',
openRouterModel: 'cognitivecomputations/dolphin-mixtral-8x7b',
openAIApiKey: "",
huggingfaceApiKey: "",
cohereApiKey: "",
anthropicApiKey: "",
anthropicModel: "claude-3-sonnet-20240229",
azureOpenAIApiKey: "",
azureOpenAIApiInstanceName: "",
azureOpenAIApiDeploymentName: "",
azureOpenAIApiVersion: "",
azureOpenAIApiEmbeddingDeploymentName: "",
googleApiKey: "",
openRouterAiApiKey: "",
openRouterModel: "cognitivecomputations/dolphin-mixtral-8x7b",
defaultModel: ChatModels.GPT_4_TURBO,
defaultModelDisplayName: ChatModelDisplayNames.GPT_4_TURBO,
embeddingModel: EmbeddingModels.OPENAI_EMBEDDING_SMALL,
temperature: 0.1,
maxTokens: 1000,
contextTurns: 15,
userSystemPrompt: '',
openAIProxyBaseUrl: '',
openAIProxyModelName: '',
openAIEmbeddingProxyBaseUrl: '',
openAIEmbeddingProxyModelName: '',
ollamaModel: 'llama2',
ollamaBaseUrl: '',
lmStudioBaseUrl: 'http://localhost:1234/v1',
userSystemPrompt: "",
openAIProxyBaseUrl: "",
openAIProxyModelName: "",
openAIEmbeddingProxyBaseUrl: "",
openAIEmbeddingProxyModelName: "",
ollamaModel: "llama2",
ollamaBaseUrl: "",
lmStudioBaseUrl: "http://localhost:1234/v1",
stream: true,
defaultSaveFolder: 'copilot-conversations',
defaultSaveFolder: "copilot-conversations",
indexVaultToVectorStore: VAULT_VECTOR_STORE_STRATEGY.NEVER,
chatNoteContextPath: '',
qaExclusionPaths: "",
chatNoteContextPath: "",
chatNoteContextTags: [],
debug: false,
enableEncryption: false,

43
src/main.ts
View file

@ -8,6 +8,7 @@ import { AdhocPromptModal } from "@/components/AdhocPromptModal";
import { ChatNoteContextModal } from "@/components/ChatNoteContextModal";
import CopilotView from "@/components/CopilotView";
import { ListPromptModal } from "@/components/ListPromptModal";
import { QAExclusionModal } from "@/components/QAExclusionModal";
import {
CHAT_VIEWTYPE,
DEFAULT_SETTINGS,
@ -23,6 +24,7 @@ import SharedState from "@/sharedState";
import {
areEmbeddingModelsSame,
getAllNotesContent,
isPathInList,
sanitizeSettings,
} from "@/utils";
import VectorDBManager, { VectorStoreDocument } from "@/vectorDBManager";
@ -412,6 +414,18 @@ export default class CopilotPlugin extends Plugin {
},
});
this.addCommand({
id: "set-vault-qa-exclusion",
name: "Set exclusion for Vault QA mode",
callback: async () => {
new QAExclusionModal(this.app, this.settings, async (paths: string) => {
// Store the path in the plugin's settings, default to empty string
this.settings.qaExclusionPaths = paths;
await this.saveSettings();
}).open();
},
});
this.registerEvent(
this.app.vault.on("delete", (file) => {
const docHash = VectorDBManager.getDocumentHash(file.path);
@ -471,11 +485,12 @@ export default class CopilotPlugin extends Plugin {
const currEmbeddingModel =
EmbeddingsManager.getModelName(embeddingInstance);
console.log(
"Prev vs Current embedding models:",
prevEmbeddingModel,
currEmbeddingModel,
);
if (this.settings.debug)
console.log(
"Prev vs Current embedding models:",
prevEmbeddingModel,
currEmbeddingModel,
);
if (!areEmbeddingModelsSame(prevEmbeddingModel, currEmbeddingModel)) {
// Model has changed, clear DB and reindex from scratch
@ -504,10 +519,18 @@ export default class CopilotPlugin extends Plugin {
this.dbVectorStores,
);
const files = this.app.vault.getMarkdownFiles().filter((file) => {
if (!latestMtime || overwrite) return true;
return file.stat.mtime > latestMtime;
});
const files = this.app.vault
.getMarkdownFiles()
.filter((file) => {
if (!latestMtime || overwrite) return true;
return file.stat.mtime > latestMtime;
})
// file not in qaExclusionPaths
.filter((file) => {
if (!this.settings.qaExclusionPaths) return true;
return !isPathInList(file.path, this.settings.qaExclusionPaths);
});
const fileContents: string[] = await Promise.all(
files.map((file) => this.app.vault.cachedRead(file)),
);
@ -540,6 +563,8 @@ export default class CopilotPlugin extends Plugin {
embeddingInstance,
noteFile,
);
if (this.settings.debug) console.log(`Indexed: [[${file.basename}]].`);
indexedCount++;
indexNotice.setMessage(
`Copilot is indexing your vault...\n${indexedCount}/${totalFiles} files processed.`,

1
src/settings/SettingsPage.tsx
View file

@ -40,6 +40,7 @@ export interface CopilotSettings {
debug: boolean;
enableEncryption: boolean;
maxSourceChunks: number;
qaExclusionPaths: string;
}
export class CopilotSettingTab extends PluginSettingTab {

33
src/utils.ts
View file

@ -101,6 +101,39 @@ export async function getNotesFromTags(
return filesWithTag;
}
export function isPathInList(filePath: string, pathList: string): boolean {
if (!pathList) return false;
// Extract the file name from the filePath
const fileName = filePath.split("/").pop()?.toLowerCase();
// Normalize the file path for case-insensitive comparison
const normalizedFilePath = filePath.toLowerCase();
return pathList
.split(",")
.map(
(path) =>
path
.trim() // Trim whitespace
.replace(/^\[\[|\]\]$/g, "") // Remove surrounding [[ and ]]
.replace(/^\//, "") // Remove leading slash
.toLowerCase(), // Convert to lowercase for case-insensitive comparison
)
.some((normalizedPath) => {
// Check for exact match or proper segmentation
const isExactMatch =
normalizedFilePath === normalizedPath ||
normalizedFilePath.startsWith(normalizedPath + "/") ||
normalizedFilePath.endsWith("/" + normalizedPath) ||
normalizedFilePath.includes("/" + normalizedPath + "/");
// Check for file name match (for cases like [[note1]])
const isFileNameMatch = fileName === normalizedPath + ".md";
return isExactMatch || isFileNameMatch;
});
}
export const stringToChainType = (chain: string): ChainType => {
switch (chain) {
case "llm_chain":

57
tests/utils.test.ts
View file

@ -4,6 +4,7 @@ import {
getNotesFromPath,
getNotesFromTags,
isFolderMatch,
isPathInList,
processVariableNameForNotePath,
} from "../src/utils";
@ -213,3 +214,59 @@ describe("getNotesFromTags", () => {
expect(resultPaths.length).toEqual(expectedPaths.length);
});
});
describe("isPathInList", () => {
it("should exclude a file path that exactly matches an excluded path", () => {
const result = isPathInList("test/folder/note.md", "test/folder");
expect(result).toBe(true);
});
it("should not exclude a file path if there is no match", () => {
const result = isPathInList("test/folder/note.md", "another/folder");
expect(result).toBe(false);
});
it("should exclude a file path that matches an excluded path with leading slash", () => {
const result = isPathInList("test/folder/note.md", "/test/folder");
expect(result).toBe(true);
});
it("should exclude a note title that matches an excluded path with surrounding [[ and ]]", () => {
const result = isPathInList("test/folder/note1.md", "[[note1]]");
expect(result).toBe(true);
});
it("should be case insensitive when excluding a file path", () => {
const result = isPathInList("Test/Folder/Note.md", "test/folder");
expect(result).toBe(true);
});
it("should handle multiple excluded paths separated by commas", () => {
const result = isPathInList(
"test/folder/note.md",
"another/folder,test/folder",
);
expect(result).toBe(true);
});
it("should not exclude a file path if it partially matches an excluded path without proper segmentation", () => {
const result = isPathInList("test/folder123/note.md", "test/folder");
expect(result).toBe(false);
});
it("should exclude a file path that matches any one of multiple excluded paths", () => {
const result = isPathInList(
"test/folder/note.md",
"another/folder, test/folder, yet/another/folder",
);
expect(result).toBe(true);
});
it("should trim spaces around excluded paths", () => {
const result = isPathInList(
"test/folder/note.md",
" another/folder , test/folder ",
);
expect(result).toBe(true);
});
});