obsidian-plugin/aldo-g_obsidian-llm-test
1
0
Fork 0
mirror of https://github.com/aldo-g/obsidian-llm-test.git synced 2026-07-22 05:42:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

update: added multi language support, and adding new models

Browse source
This commit is contained in:
Alastair Grant 2026-01-14 21:57:29 +11:00
parent 4c61c6ca5d
commit 72772a2041
5 changed files with 499 additions and 380 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
main.ts
View file

@ -54,11 +54,11 @@ const DEFAULT_SETTINGS: ObsidianTestPluginSettings = {
ollama: ""
},
models: {
openai: "gpt-4",
anthropic: "claude-3-opus-20240229",
openai: "gpt-4o",
anthropic: "claude-3-5-sonnet-latest",
deepseek: "deepseek-chat",
gemini: "gemini-pro",
mistral: "mistral-medium",
gemini: "gemini-1.5-pro",
mistral: "mistral-large-latest",
ollama: "llama3"
},
ollamaSettings: {
@ -88,13 +88,13 @@ export default class ObsidianTestPlugin extends Plugin {
async onload() {
// Load CSS
await this.loadStyles();
await this.loadSettings();
this.registerView(DASHBOARD_VIEW_TYPE, (leaf) => new TestDashboardView(leaf, this.app, this.indexedNotes, this));
this.registerView(QUESTION_VIEW_TYPE, (leaf) => new QuestionDocumentView(leaf, this.app, this, { description: "", questions: [] }));
this.addRibbonIcon("flask-conical", "Test dashboard", () => this.openTestDashboard());
this.addCommand({
id: "open-test-dashboard",
name: "Open test dashboard",
@ -102,19 +102,19 @@ export default class ObsidianTestPlugin extends Plugin {
this.openTestDashboard();
},
});
this.addSettingTab(new SettingsTab(this.app, this));
this.registerInterval(window.setInterval(() => {}, 5 * 60 * 1000));
this.registerInterval(window.setInterval(() => { }, 5 * 60 * 1000));
}
onunload() {
// Do not detach leaves in onunload - this is considered an antipattern
}
async loadStyles() {
// Load styles directly from the CSS file via Obsidian API
await this.loadData();
}
async loadStyles() {
// Load styles directly from the CSS file via Obsidian API
await this.loadData();
}
async loadSettings() {
const data = (await this.loadData()) as ObsidianTestPluginData | null;
@ -141,28 +141,28 @@ export default class ObsidianTestPlugin extends Plugin {
async indexTestNotes() {
this.indexedNotes = [];
const markdownFiles = this.app.vault.getFiles();
for (const file of markdownFiles) {
if (!file.path.endsWith(".md")) continue;
const content = await this.app.vault.read(file);
this.indexedNotes.push({
filePath: file.path,
content,
testStatus: { testsReady: true, passed: 0, total: 0 }
});
}
await this.saveSettings();
const dashLeaf = this.app.workspace.getLeavesOfType(DASHBOARD_VIEW_TYPE)[0];
if (dashLeaf?.view instanceof TestDashboardView) {
dashLeaf.view.pluginData = this.indexedNotes;
dashLeaf.view.render();
}
new Notice(`Indexed ${this.indexedNotes.length} notes`);
return this.indexedNotes;
return this.indexedNotes;
}
openTestDashboard() {
@ -171,7 +171,7 @@ export default class ObsidianTestPlugin extends Plugin {
this.app.workspace.revealLeaf(existingLeaves[0]);
return;
}
const leaf = this.app.workspace.getLeftLeaf(false);
if (!leaf) {
new Notice("Could not obtain workspace leaf.");
@ -194,19 +194,19 @@ export default class ObsidianTestPlugin extends Plugin {
return;
}
const response = this.testDocuments[filePath];
const existingLeaves = this.app.workspace.getLeavesOfType(QUESTION_VIEW_TYPE);
let leaf: WorkspaceLeaf;
if (existingLeaves.length > 0) {
leaf = existingLeaves[0];
} else {
leaf = this.app.workspace.getLeaf("tab");
}
leaf.setViewState({ type: QUESTION_VIEW_TYPE, active: true });
this.app.workspace.revealLeaf(leaf);
setTimeout(() => {
const view = leaf.view as QuestionDocumentView;
if (view) {
@ -214,26 +214,26 @@ export default class ObsidianTestPlugin extends Plugin {
view.description = response.description;
view.generatedTests = response.questions;
view.answers = response.answers || {};
if (response.markResults && response.markResults.length > 0) {
view.markResults = response.markResults;
if (typeof response.score === "number") {
const markResults = response.markResults || [];
let totalEarnedMarks = 0;
let totalPossibleMarks = 0;
markResults.forEach(result => {
if (result) {
totalEarnedMarks += result.marks;
totalPossibleMarks += result.maxMarks;
}
});
view.scoreSummary = `You scored ${totalEarnedMarks} / ${totalPossibleMarks} marks (${response.score.toFixed(1)}%)`;
}
}
view.render();
}
}, 200);

2
manifest.json
View file

@ -1,7 +1,7 @@
{
"id": "llm-test-gen",
"name": "LLM Test Generator",
"version": "1.1.0",
"version": "1.2.2",
"minAppVersion": "0.15.0",
"description": "Generate AI-powered test questions from your notes with multiple LLM providers (OpenAI, Mistral, Gemini, DeepSeek, Ollama) to enhance your learning and retention.",
"author": "Aldo E George",

12
package-lock.json generated
View file

@ -914,9 +914,9 @@
"peer": true
},
"node_modules/brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"version": "1.1.12",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
"integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
"dev": true,
"license": "MIT",
"peer": true,
@ -1729,9 +1729,9 @@
"peer": true
},
"node_modules/js-yaml": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
"integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz",
"integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==",
"dev": true,
"license": "MIT",
"peer": true,

392
src/services/llm.ts
View file

@ -1,4 +1,5 @@
import { formatNotesForLLM } from "./formatter";
import { requestUrl } from "obsidian";
import type {
IndexedNote,
LLMResponse,
@ -9,7 +10,6 @@ import type { LLMProvider } from "../../main";
const OPENAI_API_URL = "https://api.openai.com/v1/chat/completions";
const ANTHROPIC_API_URL = "https://api.anthropic.com/v1/messages";
const DEEPSEEK_API_URL = "https://api.deepseek.com/v1/chat/completions";
const GEMINI_API_URL = "https://generativelanguage.googleapis.com/v1beta/models/gemini-pro:generateContent";
const MISTRAL_API_URL = "https://api.mistral.ai/v1/chat/completions";
export class ContextLengthExceededError extends Error {
@ -32,7 +32,7 @@ export async function generateTestQuestions(
): Promise<TestQuestionsResponse> {
const apiKey = getApiKey(provider, apiKeys);
const model = models?.[provider] || getDefaultModel(provider);
// Skip API key check for Ollama
if (provider !== "ollama" && !apiKey) {
throw new Error(`Missing API key for ${provider}! Please set it in the plugin settings.`);
@ -41,6 +41,8 @@ export async function generateTestQuestions(
const notesPrompt = formatNotesForLLM(indexedNotes);
const systemInstructions = `
You are a helpful AI that generates test questions from user study notes.
IMPORTANT: You must generate the questions and description in the SAME LANGUAGE as the study notes provided.
We want each question to end with "(1)", "(2)", or "(3)" to show how many marks it is worth.
Correspondingly, the "type" field is "short" (1 mark), "long" (2 marks), or "extended" (3 marks).
@ -57,7 +59,7 @@ Return JSON in this shape (no extra keys, no markdown fences):
try {
let responseData;
switch (provider) {
case "openai":
responseData = await callOpenAI(systemInstructions, notesPrompt, apiKey, model);
@ -87,15 +89,15 @@ Return JSON in this shape (no extra keys, no markdown fences):
if (error instanceof ContextLengthExceededError) {
throw error;
}
if (error instanceof Error &&
(error.message.includes("context_length_exceeded") ||
error.message.includes("maximum context length"))) {
if (error instanceof Error &&
(error.message.includes("context_length_exceeded") ||
error.message.includes("maximum context length"))) {
throw new ContextLengthExceededError(
"The document is too large for the model's context window. Please split your document into smaller sections."
);
}
throw error;
}
}
@ -114,7 +116,7 @@ export async function markTestAnswers(
): Promise<Array<{ questionNumber: number; marks: number; maxMarks: number; feedback: string }>> {
const apiKey = getApiKey(provider, apiKeys);
const model = models?.[provider] || getDefaultModel(provider);
// Skip API key check for Ollama
if (provider !== "ollama" && !apiKey) {
throw new Error(`Missing API key for ${provider}! Please set it in the plugin settings.`);
@ -122,6 +124,8 @@ export async function markTestAnswers(
const systemMessage = `
You are a helpful AI that grades user answers based on the provided source text.
IMPORTANT: You must provide the "feedback" in the SAME LANGUAGE as the source text and the user's answers.
A question might have (1), (2), or (3) to indicate mark weighting. For each answer, you need to return:
- questionNumber: The question number (starting from 1)
- marks: How many marks earned (0 to maxMarks)
@ -142,7 +146,7 @@ Output must be a JSON array with these fields only.
} else if (pair.type === "extended") {
maxMarks = 3;
}
userPrompt += `Q${index + 1} (maxMarks=${maxMarks}): ${pair.question}\nAnswer: ${pair.answer || "[No answer provided]"}\n\n`;
});
userPrompt += `Please return a JSON array of objects, each with { "questionNumber", "marks", "maxMarks", "feedback" }.
@ -151,7 +155,7 @@ No extra fields, no markdown code blocks.`;
try {
let responseData;
switch (provider) {
case "openai":
responseData = await callOpenAI(systemMessage, userPrompt, apiKey, model, 1000);
@ -181,15 +185,15 @@ No extra fields, no markdown code blocks.`;
if (error instanceof ContextLengthExceededError) {
throw error;
}
if (error instanceof Error &&
(error.message.includes("context_length_exceeded") ||
error.message.includes("maximum context length"))) {
if (error instanceof Error &&
(error.message.includes("context_length_exceeded") ||
error.message.includes("maximum context length"))) {
throw new ContextLengthExceededError(
"The document is too large for the model's context window. Please split your document into smaller sections."
);
}
throw error;
}
}
@ -197,15 +201,15 @@ No extra fields, no markdown code blocks.`;
function getDefaultModel(provider: LLMProvider): string {
switch (provider) {
case "openai":
return "gpt-4";
return "gpt-4o";
case "anthropic":
return "claude-3-opus-20240229";
return "claude-3-5-sonnet-latest";
case "deepseek":
return "deepseek-chat";
case "gemini":
return "gemini-pro";
return "gemini-1.5-pro";
case "mistral":
return "mistral-medium";
return "mistral-large-latest";
case "ollama":
return "llama3";
default:
@ -264,23 +268,41 @@ function parseMarkingResponse(feedback: string): Array<{ questionNumber: number;
}
async function callOpenAI(
systemMessage: string,
userPrompt: string,
apiKey: string,
model: string = "gpt-4",
systemMessage: string,
userPrompt: string,
apiKey: string,
model: string = "gpt-4o",
maxTokens = 500
): Promise<string> {
const requestBody = {
const isReasoningModel = model.startsWith("o1-") || model.startsWith("o3-") || model.startsWith("gpt-5");
const messages = [];
// For o1-preview and o1-mini, system role is not supported. Best to merge into user.
// For newer models (o1, o3), system is supported but often mapped to 'developer' role.
if (isReasoningModel && (model.includes("preview") || model.includes("mini") && !model.includes("o3"))) {
messages.push({ role: "user", content: `${systemMessage}\n\n${userPrompt}` });
} else {
messages.push({ role: "system", content: systemMessage });
messages.push({ role: "user", content: userPrompt });
}
const requestBody: any = {
model: model,
messages: [
{ role: "system", content: systemMessage },
{ role: "user", content: userPrompt }
],
temperature: 0.7,
max_tokens: maxTokens
messages: messages,
};
const response = await fetch(OPENAI_API_URL, {
// Reasoning models (o1/o3/gpt-5) use max_completion_tokens and don't support temperature.
// CRITICAL: max_completion_tokens counts both thinking (internal) and output tokens.
// We must increase this significantly for reasoning models, or they cut off before the JSON starts.
if (isReasoningModel) {
requestBody.max_completion_tokens = Math.max(maxTokens * 4, 4000);
} else {
requestBody.max_tokens = maxTokens;
requestBody.temperature = 0.7;
}
const response = await requestUrl({
url: OPENAI_API_URL,
method: "POST",
headers: {
Authorization: `Bearer ${apiKey}`,
@ -289,27 +311,34 @@ async function callOpenAI(
body: JSON.stringify(requestBody)
});
if (!response.ok) {
const errorData = await response.json();
if (response.status !== 200) {
const errorData = response.json;
if (errorData.error?.code === "context_length_exceeded") {
throw new ContextLengthExceededError(
`The document is too large for ${model}'s context window. Please split your document into smaller sections.`
);
}
throw new Error(`OpenAI API Error: ${response.status} - ${errorData.error?.message || response.statusText}`);
throw new Error(`OpenAI API Error: ${response.status} - ${errorData.error?.message || "Unknown error"}`);
}
const responseData = await response.json();
return responseData.choices?.[0]?.message?.content || "";
const content = response.json.choices?.[0]?.message?.content || "";
if (!content && isReasoningModel) {
// Sometimes reasoning models return their reasoning in a separate field or fail to output if tokens are low.
const reasoning = response.json.choices?.[0]?.message?.reasoning_content;
if (reasoning && !content) {
throw new Error(`Reasoning model (${model}) spent all tokens thinking and didn't generate an answer. Try a shorter note or check your usage tiers.`);
}
}
return content;
}
async function callAnthropic(
systemMessage: string,
userPrompt: string,
apiKey: string,
model: string = "claude-3-opus-20240229",
systemMessage: string,
userPrompt: string,
apiKey: string,
model: string = "claude-3-5-sonnet-latest",
maxTokens = 500
): Promise<string> {
const requestBody = {
@ -328,7 +357,8 @@ async function callAnthropic(
max_tokens: maxTokens
};
const response = await fetch(ANTHROPIC_API_URL, {
const response = await requestUrl({
url: ANTHROPIC_API_URL,
method: "POST",
headers: {
"x-api-key": apiKey,
@ -338,28 +368,27 @@ async function callAnthropic(
body: JSON.stringify(requestBody)
});
if (!response.ok) {
const errorData = await response.json();
if (errorData.error?.type === "context_length_exceeded" ||
errorData.error?.message?.includes("context window")) {
if (response.status !== 200) {
const errorData = response.json;
if (errorData.error?.type === "context_length_exceeded" ||
errorData.error?.message?.includes("context window")) {
throw new ContextLengthExceededError(
`The document is too large for ${model}'s context window. Please split your document into smaller sections.`
);
}
throw new Error(`Anthropic API Error: ${response.status} - ${errorData.error?.message || response.statusText}`);
throw new Error(`Anthropic API Error: ${response.status} - ${errorData.error?.message || "Unknown error"}`);
}
const responseData = await response.json();
return responseData.content?.[0]?.text || "";
return response.json.content?.[0]?.text || "";
}
async function callDeepSeek(
systemMessage: string,
userPrompt: string,
apiKey: string,
model: string = "deepseek-chat",
systemMessage: string,
userPrompt: string,
apiKey: string,
model: string = "deepseek-chat",
maxTokens = 500
): Promise<string> {
const requestBody = {
@ -371,7 +400,8 @@ async function callDeepSeek(
max_tokens: maxTokens
};
const response = await fetch(DEEPSEEK_API_URL, {
const response = await requestUrl({
url: DEEPSEEK_API_URL,
method: "POST",
headers: {
Authorization: `Bearer ${apiKey}`,
@ -380,35 +410,34 @@ async function callDeepSeek(
body: JSON.stringify(requestBody)
});
if (!response.ok) {
const errorData = await response.json();
if (errorData.error?.code === "context_window_exceeded" ||
errorData.error?.message?.includes("context length")) {
if (response.status !== 200) {
const errorData = response.json;
if (errorData.error?.code === "context_window_exceeded" ||
errorData.error?.message?.includes("context length")) {
throw new ContextLengthExceededError(
`The document is too large for ${model}'s context window. Please split your document into smaller sections.`
);
}
throw new Error(`DeepSeek API Error: ${response.status} - ${errorData.error?.message || response.statusText}`);
throw new Error(`DeepSeek API Error: ${response.status} - ${errorData.error?.message || "Unknown error"}`);
}
const responseData = await response.json();
return responseData.choices?.[0]?.message?.content || "";
return response.json.choices?.[0]?.message?.content || "";
}
async function callGemini(
systemMessage: string,
userPrompt: string,
apiKey: string,
model: string = "gemini-pro",
systemMessage: string,
userPrompt: string,
apiKey: string,
model: string = "gemini-1.5-pro",
maxTokens = 500
): Promise<string> {
const fullPrompt = `${systemMessage}\n\n${userPrompt}`;
const modelEndpoint = model === "gemini-pro" ? "gemini-pro" : model;
const apiUrl = `https://generativelanguage.googleapis.com/v1beta/models/${modelEndpoint}:generateContent?key=${apiKey}`;
const requestBody = {
contents: [
{
@ -423,7 +452,8 @@ async function callGemini(
}
};
const response = await fetch(apiUrl, {
const response = await requestUrl({
url: apiUrl,
method: "POST",
headers: {
"Content-Type": "application/json"
@ -431,28 +461,27 @@ async function callGemini(
body: JSON.stringify(requestBody)
});
if (!response.ok) {
const errorData = await response.json();
if (errorData.error?.message?.includes("context length") ||
errorData.error?.message?.includes("token limit")) {
if (response.status !== 200) {
const errorData = response.json;
if (errorData.error?.message?.includes("context length") ||
errorData.error?.message?.includes("token limit")) {
throw new ContextLengthExceededError(
`The document is too large for ${model}'s context window. Please split your document into smaller sections.`
);
}
throw new Error(`Google Gemini API Error: ${response.status} - ${errorData.error?.message || response.statusText}`);
throw new Error(`Google Gemini API Error: ${response.status} - ${errorData.error?.message || "Unknown error"}`);
}
const responseData = await response.json();
return responseData.candidates?.[0]?.content?.parts?.[0]?.text || "";
return response.json.candidates?.[0]?.content?.parts?.[0]?.text || "";
}
async function callMistral(
systemMessage: string,
userPrompt: string,
apiKey: string,
model: string = "mistral-medium",
systemMessage: string,
userPrompt: string,
apiKey: string,
model: string = "mistral-large-latest",
maxTokens = 500
): Promise<string> {
const requestBody = {
@ -465,7 +494,8 @@ async function callMistral(
max_tokens: maxTokens
};
const response = await fetch(MISTRAL_API_URL, {
const response = await requestUrl({
url: MISTRAL_API_URL,
method: "POST",
headers: {
Authorization: `Bearer ${apiKey}`,
@ -474,36 +504,35 @@ async function callMistral(
body: JSON.stringify(requestBody)
});
if (!response.ok) {
const errorData = await response.json();
if (errorData.error?.type === "context_length_exceeded" ||
errorData.error?.message?.includes("context") ||
errorData.error?.message?.includes("token limit")) {
if (response.status !== 200) {
const errorData = response.json;
if (errorData.error?.type === "context_length_exceeded" ||
errorData.error?.message?.includes("context") ||
errorData.error?.message?.includes("token limit")) {
throw new ContextLengthExceededError(
`The document is too large for ${model}'s context window. Please split your document into smaller sections.`
);
}
throw new Error(`Mistral API Error: ${response.status} - ${errorData.error?.message || response.statusText}`);
throw new Error(`Mistral API Error: ${response.status} - ${errorData.error?.message || "Unknown error"}`);
}
const responseData = await response.json();
return responseData.choices?.[0]?.message?.content || "";
return response.json.choices?.[0]?.message?.content || "";
}
async function callOllama(
systemMessage: string,
userPrompt: string,
model: string = "llama3",
systemMessage: string,
userPrompt: string,
model: string = "llama3",
maxTokens = 500,
ollamaUrl = "http://localhost:11434"
): Promise<string> {
const fullPrompt = `${systemMessage}\n\n${userPrompt}`;
// Ollama API endpoint for generate
const generateUrl = `${ollamaUrl}/api/generate`;
const requestBody = {
model: model,
prompt: fullPrompt,
@ -514,7 +543,8 @@ async function callOllama(
};
try {
const response = await fetch(generateUrl, {
const response = await requestUrl({
url: generateUrl,
method: "POST",
headers: {
"Content-Type": "application/json"
@ -522,36 +552,156 @@ async function callOllama(
body: JSON.stringify(requestBody)
});
if (!response.ok) {
const errorText = await response.text();
let errorData = {};
try {
errorData = JSON.parse(errorText);
} catch (e) {
// If it's not valid JSON, use the text as is
}
if (response.status !== 200) {
const errorText = response.text;
if (errorText.includes("context window") || errorText.includes("context length")) {
throw new ContextLengthExceededError(
`The document is too large for ${model}'s context window. Please split your document into smaller sections.`
);
}
throw new Error(`Ollama API Error: ${response.status} - ${(errorData as any).error || errorText || response.statusText}`);
throw new Error(`Ollama API Error: ${response.status} - ${errorText || "Unknown error"}`);
}
const responseData = await response.json();
return responseData.response || "";
return response.json.response || "";
} catch (error) {
if (error instanceof ContextLengthExceededError) {
throw error;
}
if (error.message?.includes("Failed to fetch")) {
if (error.message?.includes("Failed to fetch") || error.message?.includes("net::ERR_CONNECTION_REFUSED")) {
throw new Error(`Could not connect to Ollama server at ${ollamaUrl}. Is it running?`);
}
throw error;
}
}
export async function fetchProviderModels(
provider: LLMProvider,
apiKey: string,
ollamaUrl: string = "http://localhost:11434"
): Promise<Array<{ id: string; name: string }>> {
try {
switch (provider) {
case "openai": {
const response = await requestUrl({
url: "https://api.openai.com/v1/models",
headers: { Authorization: `Bearer ${apiKey}` }
});
const data = response.json;
return data.data
.filter((m: any) => {
const id = m.id.toLowerCase();
// 1. MUST start with a known chat-capable prefix
const isChatPrefix = id.startsWith("gpt-") || id.startsWith("o1-") || id.startsWith("o3-");
if (!isChatPrefix) return false;
// 2. EXCLUDE specialized/incompatible utility models
const isSpecialized =
id.includes("-instruct") ||
id.includes("-realtime") ||
id.includes("-audio") ||
id.includes("-tts") ||
id.includes("-embedding") ||
id.includes("-moderation") ||
id.includes("-transcribe") ||
id.includes("-diarize") ||
id.includes("-image") ||
id.includes("-search") ||
id.includes("-codex") ||
id.includes("-edit");
if (isSpecialized) return false;
// 3. EXCLUDE legacy fixed-version legacy models
const isLegacy = ["gpt-3.5-turbo-0301", "gpt-4-0314", "gpt-4-0613"].includes(id);
if (isLegacy) return false;
// 4. HANDLE -PREVIEW models (Hide most, but keep important ones like o1/o3/vision)
if (id.includes("-preview")) {
const allowedPreviews = ["gpt-4-vision-preview", "o1-preview", "o1-mini-preview", "o3-mini"];
return allowedPreviews.some(allowed => id.includes(allowed));
}
return true;
})
.map((m: any) => ({ id: m.id, name: m.id }));
}
case "anthropic": {
const response = await requestUrl({
url: "https://api.anthropic.com/v1/models",
headers: {
"x-api-key": apiKey,
"anthropic-version": "2023-06-01"
}
});
const data = response.json;
return data.data
.filter((m: any) => m.id.startsWith("claude-"))
.map((m: any) => ({ id: m.id, name: m.display_name || m.id }));
}
case "deepseek": {
const response = await requestUrl({
url: "https://api.deepseek.com/v1/models",
headers: { Authorization: `Bearer ${apiKey}` }
});
const data = response.json;
return data.data.map((m: any) => ({ id: m.id, name: m.id }));
}
case "gemini": {
const response = await requestUrl({
url: `https://generativelanguage.googleapis.com/v1beta/models?key=${apiKey}`
});
const data = response.json;
return data.models
.filter((m: any) => m.supportedGenerationMethods.includes("generateContent"))
.map((m: any) => ({
id: m.name.replace("models/", ""),
name: m.displayName || m.name.replace("models/", "")
}));
}
case "mistral": {
const response = await requestUrl({
url: "https://api.mistral.ai/v1/models",
headers: { Authorization: `Bearer ${apiKey}` }
});
const data = response.json;
return data.data
.filter((m: any) => !m.id.includes("embed") && !m.id.includes("moderation"))
.map((m: any) => ({ id: m.id, name: m.id }));
}
case "ollama": {
const response = await requestUrl({
url: `${ollamaUrl}/api/tags`
});
const data = response.json;
return data.models.map((m: any) => ({ id: m.name, name: m.name }));
}
default:
return [];
}
} catch (error) {
console.error(`Error fetching models for ${provider}:`, error);
return [];
}
}
export async function fetchCommunityModels(
provider: LLMProvider
): Promise<Array<{ id: string; name: string }>> {
try {
const url = `https://raw.githubusercontent.com/aldo-g/obsidian-llm-test/master/community-models.json`;
const response = await requestUrl({ url });
if (response.status === 200) {
const data = response.json;
return data[provider] || [];
}
} catch (error) {
// Fail silently
}
return [];
}

415
src/ui/SettingsTab.ts
View file

@ -1,19 +1,61 @@
import { App, PluginSettingTab, Setting, DropdownComponent, Notice } from "obsidian";
import type MyPlugin from "../../main";
import type { LLMProvider } from "../../main";
import { fetchProviderModels, fetchCommunityModels } from "../services/llm";
export default class SettingsTab extends PluginSettingTab {
plugin: MyPlugin;
fetchedModels: Record<string, Array<{ id: string; name: string }>> = {};
modelFilter: string = "";
static readonly CORE_MODELS: Record<string, Array<{ id: string; name: string }>> = {
openai: [
{ id: "gpt-4o", name: "GPT-4o (Latest)" },
{ id: "gpt-4o-2024-08-06", name: "GPT-4o (2024-08-06)" },
{ id: "gpt-4o-2024-05-13", name: "GPT-4o (2024-05-13)" },
{ id: "gpt-4o-mini", name: "GPT-4o Mini" },
{ id: "o1-preview", name: "o1 Preview" },
{ id: "o1-mini", name: "o1 Mini" },
{ id: "o3-mini", name: "o3 Mini" },
{ id: "gpt-4-turbo", name: "GPT-4 Turbo" },
{ id: "gpt-4", name: "GPT-4" },
{ id: "gpt-3.5-turbo", name: "GPT-3.5 Turbo" }
],
anthropic: [
{ id: "claude-3-5-sonnet-latest", name: "Claude 3.5 Sonnet (Latest)" },
{ id: "claude-3-5-sonnet-20240620", name: "Claude 3.5 Sonnet (2024-06-20)" },
{ id: "claude-3-opus-20240229", name: "Claude 3 Opus" },
{ id: "claude-3-sonnet-20240229", name: "Claude 3 Sonnet" },
{ id: "claude-3-haiku-20240307", name: "Claude 3 Haiku" }
],
deepseek: [
{ id: "deepseek-chat", name: "DeepSeek Chat (DeepSeek-V3)" },
{ id: "deepseek-coder", name: "DeepSeek Coder" }
],
gemini: [
{ id: "gemini-1.5-pro", name: "Gemini 1.5 Pro" },
{ id: "gemini-1.5-flash", name: "Gemini 1.5 Flash" },
{ id: "gemini-1.5-flash-8b", name: "Gemini 1.5 Flash-8B" },
{ id: "gemini-1.0-pro", name: "Gemini 1.0 Pro" }
],
mistral: [
{ id: "mistral-large-latest", name: "Mistral Large (Latest)" },
{ id: "mistral-medium-latest", name: "Mistral Medium (Latest)" },
{ id: "mistral-small-latest", name: "Mistral Small (Latest)" },
{ id: "codestral-latest", name: "Codestral" },
{ id: "pixtral-12b-2409", name: "Pixtral 12B" }
]
};
constructor(app: App, plugin: MyPlugin) {
super(app, plugin);
this.plugin = plugin;
}
display(): void {
const { containerEl } = this;
containerEl.empty();
new Setting(containerEl)
.setName("LLM provider")
.setDesc("Select which LLM provider you want to use for generating and marking tests")
@ -31,35 +73,140 @@ export default class SettingsTab extends PluginSettingTab {
this.display();
});
});
// OpenAI Settings
if (this.plugin.settings.llmProvider === "openai") {
// Model selection
// Cloud Provider Model Settings
if (["openai", "anthropic", "deepseek", "gemini", "mistral"].includes(this.plugin.settings.llmProvider)) {
const provider = this.plugin.settings.llmProvider;
const apiKey = this.plugin.settings.apiKeys[provider];
// Model Filter
new Setting(containerEl)
.setName("OpenAI model")
.setDesc("Select which OpenAI model to use")
.addDropdown((dropdown: DropdownComponent) => {
dropdown
.addOption("gpt-3.5-turbo", "GPT-3.5 Turbo")
.addOption("gpt-4", "GPT-4")
.addOption("gpt-4-turbo", "GPT-4 Turbo")
.addOption("gpt-4o", "GPT-4o")
.setValue(this.plugin.settings.models.openai || "gpt-4")
.onChange(async (value: string) => {
this.plugin.settings.models.openai = value;
await this.plugin.saveSettings();
.setName("Filter models")
.setDesc("Type to filter the available models list")
.addSearch((search) => {
search
.setPlaceholder("Filter models...")
.setValue(this.modelFilter)
.onChange((value) => {
this.modelFilter = value.toLowerCase();
this.display();
});
})
.addExtraButton((button) => {
button
.setIcon("x-circle")
.setTooltip("Clear filter")
.onClick(() => {
this.modelFilter = "";
this.display();
});
});
// Model selection
const modelSetting = new Setting(containerEl)
.setName(`${provider.charAt(0).toUpperCase() + provider.slice(1)} model`)
.setDesc(`Select which ${provider} model to use`);
modelSetting.addDropdown(async (dropdown: DropdownComponent) => {
const currentModel = this.plugin.settings.models[provider];
// Combined list: defaults + any newly fetched models
const coreModels = SettingsTab.CORE_MODELS[provider] || [];
const fetched = this.fetchedModels[provider] || [];
// Create a unique list of models
const modelMap = new Map<string, string>();
coreModels.forEach(m => modelMap.set(m.id, m.name));
fetched.forEach(m => modelMap.set(m.id, m.name));
if (currentModel && !modelMap.has(currentModel)) {
modelMap.set(currentModel, currentModel);
}
// Apply filter
let visibleModelsCount = 0;
modelMap.forEach((name, id) => {
if (!this.modelFilter || id.toLowerCase().includes(this.modelFilter) || name.toLowerCase().includes(this.modelFilter)) {
dropdown.addOption(id, name);
visibleModelsCount++;
}
});
if (visibleModelsCount === 0 && this.modelFilter) {
dropdown.addOption("", "No models match filter");
dropdown.setDisabled(true);
} else {
dropdown.setDisabled(false);
dropdown.setValue(currentModel);
}
// Auto-fetch in background if we have an API key and haven't fetched yet
if (fetched.length === 0 && apiKey) {
fetchProviderModels(provider, apiKey).then(newModels => {
if (newModels && newModels.length > 0) {
this.fetchedModels[provider] = newModels;
// We don't call this.display() here to avoid infinite loops,
// but the next time they open the tab it will be populated.
}
});
}
dropdown.onChange(async (value: string) => {
this.plugin.settings.models[provider] = value;
await this.plugin.saveSettings();
});
});
modelSetting.addExtraButton(button => {
button.setIcon("refresh-cw")
.setTooltip("Refresh models list")
.onClick(async () => {
if (provider === "ollama") {
new Notice(`Fetching local models from Ollama...`);
const models = await fetchProviderModels(provider, "", this.plugin.settings.ollamaSettings?.url);
if (models.length > 0) {
this.fetchedModels[provider] = models;
this.display();
new Notice(`Updated models from Ollama`);
} else {
new Notice(`Could not connect to Ollama. Check your URL.`);
}
return;
}
if (!apiKey) {
new Notice(`Fetching latest community models list...`);
const models = await fetchCommunityModels(provider);
if (models.length > 0) {
this.fetchedModels[provider] = models;
this.display();
new Notice(`Updated with community model list`);
} else {
new Notice(`Could not fetch community list. Enter an API key to fetch directly from ${provider}.`);
}
return;
}
new Notice(`Fetching live models from ${provider}...`);
const models = await fetchProviderModels(provider, apiKey);
if (models.length > 0) {
this.fetchedModels[provider] = models;
this.display();
new Notice(`Updated model list from ${provider}`);
} else {
new Notice(`Failed to fetch models from ${provider}. Check your API key.`);
}
});
});
// API Key
new Setting(containerEl)
.setName("OpenAI API key")
.setDesc("Enter your OpenAI API key for test generation and marking")
.setName(`${provider.charAt(0).toUpperCase() + provider.slice(1)} API key`)
.setDesc(`Enter your ${provider} API key`)
.addText((text) =>
text
.setPlaceholder("sk-...")
.setValue(this.plugin.settings.apiKeys.openai)
.inputEl.type = "password" // Make this a password field
.setValue(apiKey)
.inputEl.type = "password"
)
.addExtraButton((button) => {
button
@ -85,188 +232,10 @@ export default class SettingsTab extends PluginSettingTab {
.onClick(async () => {
const inputEl = button.extraSettingsEl.parentElement?.querySelector("input");
if (inputEl) {
this.plugin.settings.apiKeys.openai = inputEl.value;
this.plugin.settings.apiKeys[provider] = inputEl.value;
await this.plugin.saveSettings();
new Notice("OpenAI API key saved");
}
});
});
}
// DeepSeek Settings
if (this.plugin.settings.llmProvider === "deepseek") {
// Model selection
new Setting(containerEl)
.setName("DeepSeek model")
.setDesc("Select which DeepSeek model to use")
.addDropdown((dropdown: DropdownComponent) => {
dropdown
.addOption("deepseek-chat", "DeepSeek Chat")
.addOption("deepseek-coder", "DeepSeek Coder")
.setValue(this.plugin.settings.models.deepseek || "deepseek-chat")
.onChange(async (value: string) => {
this.plugin.settings.models.deepseek = value;
await this.plugin.saveSettings();
});
});
// API Key
new Setting(containerEl)
.setName("DeepSeek API key")
.setDesc("Enter your DeepSeek API key")
.addText((text) =>
text
.setPlaceholder("sk-...")
.setValue(this.plugin.settings.apiKeys.deepseek)
.inputEl.type = "password" // Make this a password field
)
.addExtraButton((button) => {
button
.setIcon("eye")
.setTooltip("Toggle visibility")
.onClick(() => {
const inputEl = button.extraSettingsEl.parentElement?.querySelector("input");
if (inputEl) {
if (inputEl.type === "password") {
inputEl.type = "text";
button.setIcon("eye-off");
} else {
inputEl.type = "password";
button.setIcon("eye");
}
}
});
})
.addExtraButton((button) => {
button
.setIcon("save")
.setTooltip("Save")
.onClick(async () => {
const inputEl = button.extraSettingsEl.parentElement?.querySelector("input");
if (inputEl) {
this.plugin.settings.apiKeys.deepseek = inputEl.value;
await this.plugin.saveSettings();
new Notice("DeepSeek API key saved");
}
});
});
}
// Gemini Settings
if (this.plugin.settings.llmProvider === "gemini") {
// Model selection
new Setting(containerEl)
.setName("Gemini model")
.setDesc("Select which Gemini model to use")
.addDropdown((dropdown: DropdownComponent) => {
dropdown
.addOption("gemini-1.5-pro", "Gemini 1.5 Pro")
.addOption("gemini-1.5-flash", "Gemini 1.5 Flash")
.setValue(this.plugin.settings.models.gemini || "gemini-pro")
.onChange(async (value: string) => {
this.plugin.settings.models.gemini = value;
await this.plugin.saveSettings();
});
});
// API Key
new Setting(containerEl)
.setName("Google Gemini API key")
.setDesc("Enter your Google API key for Gemini")
.addText((text) =>
text
.setPlaceholder("API key...")
.setValue(this.plugin.settings.apiKeys.gemini)
.inputEl.type = "password" // Make this a password field
)
.addExtraButton((button) => {
button
.setIcon("eye")
.setTooltip("Toggle visibility")
.onClick(() => {
const inputEl = button.extraSettingsEl.parentElement?.querySelector("input");
if (inputEl) {
if (inputEl.type === "password") {
inputEl.type = "text";
button.setIcon("eye-off");
} else {
inputEl.type = "password";
button.setIcon("eye");
}
}
});
})
.addExtraButton((button) => {
button
.setIcon("save")
.setTooltip("Save")
.onClick(async () => {
const inputEl = button.extraSettingsEl.parentElement?.querySelector("input");
if (inputEl) {
this.plugin.settings.apiKeys.gemini = inputEl.value;
await this.plugin.saveSettings();
new Notice("Gemini API key saved");
}
});
});
}
// Mistral Settings
if (this.plugin.settings.llmProvider === "mistral") {
// Model selection
new Setting(containerEl)
.setName("Mistral model")
.setDesc("Select which Mistral model to use")
.addDropdown((dropdown: DropdownComponent) => {
dropdown
.addOption("mistral-tiny", "Mistral Tiny")
.addOption("mistral-small", "Mistral Small")
.addOption("mistral-medium", "Mistral Medium")
.addOption("mistral-large-latest", "Mistral Large (Latest)")
.setValue(this.plugin.settings.models.mistral || "mistral-medium")
.onChange(async (value: string) => {
this.plugin.settings.models.mistral = value;
await this.plugin.saveSettings();
});
});
// API Key
new Setting(containerEl)
.setName("Mistral API key")
.setDesc("Enter your Mistral API key")
.addText((text) =>
text
.setPlaceholder("...")
.setValue(this.plugin.settings.apiKeys.mistral)
.inputEl.type = "password" // Make this a password field
)
.addExtraButton((button) => {
button
.setIcon("eye")
.setTooltip("Toggle visibility")
.onClick(() => {
const inputEl = button.extraSettingsEl.parentElement?.querySelector("input");
if (inputEl) {
if (inputEl.type === "password") {
inputEl.type = "text";
button.setIcon("eye-off");
} else {
inputEl.type = "password";
button.setIcon("eye");
}
}
});
})
.addExtraButton((button) => {
button
.setIcon("save")
.setTooltip("Save")
.onClick(async () => {
const inputEl = button.extraSettingsEl.parentElement?.querySelector("input");
if (inputEl) {
this.plugin.settings.apiKeys.mistral = inputEl.value;
await this.plugin.saveSettings();
new Notice("Mistral API key saved");
new Notice(`${provider.charAt(0).toUpperCase() + provider.slice(1)} API key saved`);
this.display(); // Refresh to trigger model fetch
}
});
});
@ -291,7 +260,7 @@ export default class SettingsTab extends PluginSettingTab {
await this.plugin.saveSettings();
})
);
// Model selection
new Setting(containerEl)
.setName("Ollama model")
@ -305,59 +274,59 @@ export default class SettingsTab extends PluginSettingTab {
await this.plugin.saveSettings();
})
);
// Add information about Ollama models
const providerInfoDiv = containerEl.createDiv({ cls: "provider-info" });
providerInfoDiv.createEl("p", {
providerInfoDiv.createEl("p", {
text: "Ollama lets you run LLMs locally. Make sure Ollama is installed and running before using this option."
});
providerInfoDiv.createEl("p", {
providerInfoDiv.createEl("p", {
text: "Download Ollama from: https://ollama.com/"
});
providerInfoDiv.createEl("p", {
providerInfoDiv.createEl("p", {
text: "Common models: llama3, mistral, gemma, codellama, llama3:8b, etc."
});
providerInfoDiv.createEl("p", {
providerInfoDiv.createEl("p", {
text: "Run 'ollama pull [model]' in your terminal to download models before using them with this plugin."
});
}
// About section
new Setting(containerEl).setName("About API keys").setHeading();
const apiInfoDiv = containerEl.createDiv({ cls: "api-key-info" });
apiInfoDiv.createEl("p", {
apiInfoDiv.createEl("p", {
text: "Your API keys are stored locally in your vault and are only used to communicate with the selected LLM provider."
});
// Provider-specific info
const providerInfoDiv = containerEl.createDiv({ cls: "provider-info" });
if (this.plugin.settings.llmProvider === "openai") {
providerInfoDiv.createEl("p", {
providerInfoDiv.createEl("p", {
text: "OpenAI API keys can be obtained from: https://platform.openai.com/account/api-keys"
});
providerInfoDiv.createEl("p", {
providerInfoDiv.createEl("p", {
text: "GPT-4 and GPT-4o provide the best results but require a higher API usage tier."
});
} else if (this.plugin.settings.llmProvider === "deepseek") {
providerInfoDiv.createEl("p", {
providerInfoDiv.createEl("p", {
text: "DeepSeek API keys can be obtained from the DeepSeek website."
});
} else if (this.plugin.settings.llmProvider === "gemini") {
providerInfoDiv.createEl("p", {
providerInfoDiv.createEl("p", {
text: "Google Gemini API keys can be obtained from Google AI Studio: https://makersuite.google.com/app/apikey"
});
providerInfoDiv.createEl("p", {
providerInfoDiv.createEl("p", {
text: "Gemini 1.5 Pro offers a larger context window and improved capabilities over Gemini Pro."
});
} else if (this.plugin.settings.llmProvider === "mistral") {
providerInfoDiv.createEl("p", {
providerInfoDiv.createEl("p", {
text: "WARNING: Mistral is currently not effective at marking answers"
});
providerInfoDiv.createEl("p", {
providerInfoDiv.createEl("p", {
text: "Mistral API keys can be obtained from: https://console.mistral.ai/api-keys/"
});
providerInfoDiv.createEl("p", {
providerInfoDiv.createEl("p", {
text: "Mistral offers several model sizes with different capabilities and pricing. Mistral Medium provides a good balance of performance and cost."
});
}