mirror of
https://github.com/aldo-g/obsidian-llm-test.git
synced 2026-07-22 05:42:19 +00:00
marking complete
This commit is contained in:
Alastair Grant
parent
ebb53e347b
commit
c4d0c1bc98
4 changed files with 442 additions and 120 deletions
20
main.ts
20
main.ts
|
|
@ -55,12 +55,15 @@ export default class MyPlugin extends Plugin {
|
|||
console.log("onload: Loaded settings:", this.settings);
|
||||
console.log("onload: Loaded testDocuments keys:", Object.keys(this.testDocuments));
|
||||
|
||||
// Register the dashboard and question doc views
|
||||
this.registerView(DASHBOARD_VIEW_TYPE, (leaf) => new TestDashboardView(leaf, this.app, this.indexedNotes));
|
||||
this.registerView(QUESTION_VIEW_TYPE, (leaf) => new QuestionDocumentView(leaf, this.app, this, {
|
||||
description: "",
|
||||
questions: []
|
||||
}));
|
||||
this.registerView(DASHBOARD_VIEW_TYPE, (leaf) =>
|
||||
new TestDashboardView(leaf, this.app, this.indexedNotes)
|
||||
);
|
||||
this.registerView(QUESTION_VIEW_TYPE, (leaf) =>
|
||||
new QuestionDocumentView(leaf, this.app, this, {
|
||||
description: "",
|
||||
questions: []
|
||||
})
|
||||
);
|
||||
|
||||
this.addRibbonIcon("dice", "Test Dashboard", () => this.openTestDashboard());
|
||||
this.addStatusBarItem().setText("RAG Test Plugin Active");
|
||||
|
|
@ -75,7 +78,7 @@ export default class MyPlugin extends Plugin {
|
|||
|
||||
this.addSettingTab(new SettingsTab(this.app, this));
|
||||
|
||||
// Interval log for debugging
|
||||
// Example interval log
|
||||
this.registerInterval(window.setInterval(() => console.log("Interval log"), 5 * 60 * 1000));
|
||||
}
|
||||
|
||||
|
|
@ -148,7 +151,8 @@ export default class MyPlugin extends Plugin {
|
|||
}
|
||||
|
||||
/**
|
||||
* Called after a user typed an answer. Refreshes the dashboard so it can update icon color.
|
||||
* Called after user typed an answer in the question doc.
|
||||
* Refreshes the dashboard so it can recolor the icon.
|
||||
*/
|
||||
public markFileAnswered(filePath: string): void {
|
||||
console.log(`File marked as answered: ${filePath}`);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { ItemView, WorkspaceLeaf } from "obsidian";
|
||||
import { ItemView, WorkspaceLeaf, Notice } from "obsidian";
|
||||
import type MyPlugin from "../../main";
|
||||
import { markTestAnswers } from "../services/llm";
|
||||
|
||||
export const QUESTION_VIEW_TYPE = "question-document-view";
|
||||
|
||||
|
|
@ -10,6 +11,17 @@ export default class QuestionDocumentView extends ItemView {
|
|||
filePath: string;
|
||||
answers: { [key: number]: string };
|
||||
|
||||
/**
|
||||
* markResults => For each question i, we store { correct: boolean, feedback: string } after LLM marking.
|
||||
* If null => not yet marked. This is cleared if we “Reset.”
|
||||
*/
|
||||
markResults: Array<{ correct: boolean; feedback: string } | null> = [];
|
||||
|
||||
/**
|
||||
* We also keep track of the final score after marking. E.g. "You scored X% (Y / Z correct)."
|
||||
*/
|
||||
scoreSummary: string = "";
|
||||
|
||||
constructor(
|
||||
leaf: WorkspaceLeaf,
|
||||
app: any,
|
||||
|
|
@ -30,10 +42,9 @@ export default class QuestionDocumentView extends ItemView {
|
|||
}
|
||||
|
||||
getDisplayText(): string {
|
||||
if (this.filePath) {
|
||||
return `Test: ${this.filePath}`;
|
||||
}
|
||||
return "Generated Test Questions";
|
||||
return this.filePath
|
||||
? `Test: ${this.filePath}`
|
||||
: "Generated Test Questions";
|
||||
}
|
||||
|
||||
async onOpen(): Promise<void> {
|
||||
|
|
@ -46,51 +57,217 @@ export default class QuestionDocumentView extends ItemView {
|
|||
}
|
||||
|
||||
render(): void {
|
||||
console.log(`QuestionDocumentView render: filePath='${this.filePath}', #questions=${this.generatedTests?.length || 0}`);
|
||||
console.log(
|
||||
`QuestionDocumentView render: filePath='${this.filePath}', #questions=${this.generatedTests?.length || 0}`
|
||||
);
|
||||
|
||||
const container = this.containerEl;
|
||||
container.empty();
|
||||
|
||||
// Scrolling
|
||||
container.style.overflowY = "auto";
|
||||
container.style.maxHeight = "calc(100vh - 100px)";
|
||||
|
||||
if (!this.generatedTests || !this.generatedTests.length) {
|
||||
console.log("QuestionDocumentView render: no questions found.");
|
||||
container.createEl("p", { text: "No test questions available." });
|
||||
return;
|
||||
}
|
||||
|
||||
// Show description
|
||||
const descEl = container.createEl("p", { text: this.description });
|
||||
descEl.style.fontStyle = "italic";
|
||||
descEl.style.marginBottom = "1em";
|
||||
|
||||
// Create a form for the Q&A
|
||||
const formEl = container.createEl("form");
|
||||
|
||||
this.generatedTests.forEach((test, index) => {
|
||||
const qDiv = formEl.createEl("div", { cls: "question-item" });
|
||||
qDiv.style.marginBottom = "1em";
|
||||
const questionDiv = formEl.createEl("div", { cls: "question-item" });
|
||||
questionDiv.style.marginBottom = "1em";
|
||||
|
||||
const label = qDiv.createEl("label", {
|
||||
text: `Q${index + 1}: ${test.question}`,
|
||||
const label = questionDiv.createEl("label", {
|
||||
text: `Q${index + 1}: ${test.question}`
|
||||
});
|
||||
label.style.display = "block";
|
||||
label.style.fontWeight = "bold";
|
||||
|
||||
const input = qDiv.createEl("input", { type: "text" }) as HTMLInputElement;
|
||||
// The user’s typed answer
|
||||
const input = questionDiv.createEl("input", { type: "text" }) as HTMLInputElement;
|
||||
input.placeholder = "Type your answer here";
|
||||
input.style.width = "100%";
|
||||
input.style.marginTop = "0.5em";
|
||||
input.dataset.questionIndex = index.toString();
|
||||
|
||||
// If we have a saved answer, restore it
|
||||
if (this.answers[index]) {
|
||||
input.value = this.answers[index];
|
||||
}
|
||||
|
||||
// On input, save answers & mark answered
|
||||
let borderColor = "";
|
||||
let feedbackColor = "";
|
||||
let feedbackText = "";
|
||||
|
||||
// Only color if we have a marking result
|
||||
const result = this.markResults[index];
|
||||
if (result) {
|
||||
// If markResults is non-null, we have a boolean correct + feedback
|
||||
if (result.correct === true) {
|
||||
borderColor = "green";
|
||||
feedbackColor = "green";
|
||||
} else {
|
||||
borderColor = "red";
|
||||
feedbackColor = "red";
|
||||
}
|
||||
feedbackText = result.feedback;
|
||||
}
|
||||
if (borderColor) {
|
||||
input.style.border = `2px solid ${borderColor}`;
|
||||
} else {
|
||||
input.style.border = "";
|
||||
}
|
||||
|
||||
input.addEventListener("input", () => {
|
||||
this.answers[index] = input.value;
|
||||
this.saveAnswers();
|
||||
});
|
||||
|
||||
// Feedback paragraph
|
||||
const feedbackEl = questionDiv.createEl("p");
|
||||
feedbackEl.style.marginTop = "0.25em";
|
||||
feedbackEl.style.color = feedbackColor;
|
||||
feedbackEl.style.fontWeight = feedbackText ? "bold" : "normal";
|
||||
feedbackEl.textContent = feedbackText;
|
||||
});
|
||||
|
||||
// Container for Mark/Reset buttons
|
||||
const buttonRow = formEl.createEl("div", { cls: "test-document-actions" });
|
||||
buttonRow.style.marginTop = "1em";
|
||||
buttonRow.style.display = "flex";
|
||||
buttonRow.style.gap = "1em";
|
||||
|
||||
// “Mark” button
|
||||
const markButton = buttonRow.createEl("button", { text: "Mark" });
|
||||
markButton.type = "button";
|
||||
markButton.addEventListener("click", async () => {
|
||||
await this.handleMarkButtonClick();
|
||||
});
|
||||
|
||||
// “Reset” button
|
||||
const resetButton = buttonRow.createEl("button", { text: "Reset" });
|
||||
resetButton.type = "button";
|
||||
resetButton.addEventListener("click", () => {
|
||||
this.handleResetButtonClick();
|
||||
});
|
||||
|
||||
// A place to display the final score summary after marking
|
||||
const scoreEl = formEl.createEl("p", { text: this.scoreSummary });
|
||||
scoreEl.style.marginTop = "1em";
|
||||
scoreEl.style.fontWeight = "bold";
|
||||
|
||||
container.appendChild(formEl);
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when user clicks “Mark”:
|
||||
* 1. Retrieve note content
|
||||
* 2. Build Q&A
|
||||
* 3. Call markTestAnswers
|
||||
* 4. Store results => color-coded highlight
|
||||
* 5. Compute overall score => show a summary
|
||||
*/
|
||||
private async handleMarkButtonClick(): Promise<void> {
|
||||
if (!this.filePath) {
|
||||
new Notice("No file path found for this test document.");
|
||||
return;
|
||||
}
|
||||
|
||||
const indexedNote = this.plugin.indexedNotes.find(n => n.filePath === this.filePath);
|
||||
if (!indexedNote) {
|
||||
new Notice("No indexed content found for this file. Cannot mark answers.");
|
||||
return;
|
||||
}
|
||||
const noteContent = indexedNote.content;
|
||||
|
||||
const qnaPairs: { question: string; answer: string }[] = [];
|
||||
this.generatedTests.forEach((test, idx) => {
|
||||
qnaPairs.push({
|
||||
question: test.question,
|
||||
answer: this.answers[idx] || ""
|
||||
});
|
||||
});
|
||||
|
||||
const apiKey = this.plugin.settings.apiKey;
|
||||
if (!apiKey) {
|
||||
new Notice("OpenAI API key missing. Please set it in plugin settings.");
|
||||
return;
|
||||
}
|
||||
|
||||
new Notice("Marking in progress...");
|
||||
try {
|
||||
const feedbackArray = await markTestAnswers(noteContent, qnaPairs, apiKey);
|
||||
console.log("LLM Marking feedback array:", feedbackArray);
|
||||
|
||||
// Clear old results
|
||||
this.markResults = new Array(this.generatedTests.length).fill(null);
|
||||
|
||||
// E.g. feedbackArray => [{ questionNumber, correct, feedback }, ...]
|
||||
feedbackArray.forEach(item => {
|
||||
const qIndex = item.questionNumber - 1;
|
||||
if (qIndex >= 0 && qIndex < this.generatedTests.length) {
|
||||
this.markResults[qIndex] = {
|
||||
correct: item.correct,
|
||||
feedback: item.feedback
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
// Compute overall score
|
||||
let correctCount = 0;
|
||||
this.markResults.forEach((result) => {
|
||||
if (result?.correct === true) {
|
||||
correctCount++;
|
||||
}
|
||||
});
|
||||
const total = this.generatedTests.length;
|
||||
const percentage = ((correctCount / total) * 100).toFixed(1); // e.g. "66.7"
|
||||
this.scoreSummary = `You scored ${percentage}% (${correctCount} / ${total} correct)`;
|
||||
|
||||
this.render(); // Re-render with color-coded results + score summary
|
||||
new Notice("Marking complete!");
|
||||
} catch (err) {
|
||||
console.error("Error marking answers:", err);
|
||||
new Notice("Error marking answers. Check console for details.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when user clicks “Reset”:
|
||||
* Clears typed answers & marking results => re-render unmarked doc
|
||||
*/
|
||||
private handleResetButtonClick(): void {
|
||||
// Clear all user answers
|
||||
this.answers = {};
|
||||
// Clear marking results
|
||||
this.markResults = [];
|
||||
// Clear score summary
|
||||
this.scoreSummary = "";
|
||||
// Re-save to plugin docs so we don't reload with old data
|
||||
if (this.filePath) {
|
||||
if (!this.plugin.testDocuments[this.filePath]) {
|
||||
this.plugin.testDocuments[this.filePath] = {
|
||||
description: this.description,
|
||||
questions: this.generatedTests,
|
||||
answers: {}
|
||||
};
|
||||
} else {
|
||||
this.plugin.testDocuments[this.filePath].answers = {};
|
||||
}
|
||||
this.plugin.saveSettings();
|
||||
}
|
||||
this.render();
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves user answers so they're not lost upon leaving/re-entering the doc.
|
||||
*/
|
||||
saveAnswers(): void {
|
||||
if (!this.filePath) {
|
||||
console.error("saveAnswers: No file path set.");
|
||||
|
|
@ -107,7 +284,6 @@ export default class QuestionDocumentView extends ItemView {
|
|||
this.plugin.saveSettings();
|
||||
console.log("saveAnswers: Saved answers for", this.filePath, this.answers);
|
||||
|
||||
// Mark answered => triggers re-render in dashboard to update icon color
|
||||
this.plugin.markFileAnswered(this.filePath);
|
||||
}
|
||||
}
|
||||
|
|
@ -10,7 +10,10 @@ export const VIEW_TYPE = "rag-test-view";
|
|||
* - Gray: no tests generated
|
||||
* - White: tests generated, no answers
|
||||
* - Orange: partial answers
|
||||
* - Green: all answers
|
||||
* - Green: all answered
|
||||
*
|
||||
* The gray icon is NOT clickable;
|
||||
* for white/orange/green, it's wrapped in a <button> so the user can open the doc.
|
||||
*/
|
||||
export default class TestDashboardView extends ItemView {
|
||||
pluginData: IndexedNote[];
|
||||
|
|
@ -60,55 +63,59 @@ export default class TestDashboardView extends ItemView {
|
|||
checkbox.dataset.filePath = note.filePath;
|
||||
itemEl.createEl("span", { text: ` ${note.filePath}` });
|
||||
|
||||
// Make a clickable button for the icon
|
||||
const docState = plugin.testDocuments[note.filePath];
|
||||
let fillColor = "gray"; // no tests => gray
|
||||
let clickable = false; // not clickable if gray
|
||||
|
||||
if (docState) {
|
||||
// tests exist => white/orange/green
|
||||
const totalQ = docState.questions.length;
|
||||
const answersArr = Object.values(docState.answers || {});
|
||||
const answeredCount = answersArr.filter(a => a.trim().length > 0).length;
|
||||
|
||||
if (answeredCount === 0) {
|
||||
// no answers => white
|
||||
fillColor = "white";
|
||||
} else if (answeredCount < totalQ) {
|
||||
// partial => orange
|
||||
fillColor = "orange";
|
||||
} else {
|
||||
// all answered => green
|
||||
fillColor = "green";
|
||||
}
|
||||
clickable = true; // user can open doc
|
||||
}
|
||||
|
||||
const statusIconEl = itemEl.createEl("span", { cls: "status-icon" });
|
||||
statusIconEl.style.marginLeft = "0.5em";
|
||||
|
||||
// Determine icon color based on docState
|
||||
let fillColor = "gray";
|
||||
const docState = plugin.testDocuments[note.filePath];
|
||||
if (docState) {
|
||||
const totalQuestions = docState.questions.length;
|
||||
const allAnswers = Object.values(docState.answers || {});
|
||||
// Count how many are non-blank
|
||||
const answeredCount = allAnswers.filter(ans => ans.trim().length > 0).length;
|
||||
|
||||
if (answeredCount === 0) {
|
||||
// tests generated, no answers
|
||||
fillColor = "white";
|
||||
} else if (answeredCount < totalQuestions) {
|
||||
// partial
|
||||
fillColor = "orange";
|
||||
if (!clickable) {
|
||||
// Gray icon, no tests => not a button
|
||||
statusIconEl.innerHTML = `
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="${fillColor}" class="bi bi-file-text" viewBox="0 0 16 16">
|
||||
<path d="M4 0a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h8a2 2 0 0 0 2-2V5.5L9.5 0H4z"/>
|
||||
<path d="M9.5 0v4a1 1 0 0 0 1 1h4"/>
|
||||
<path d="M4.5 7a.5.5 0 0 1 .5.5v.5h5v-1h-5V7.5a.5.5 0 0 1 .5-.5z"/>
|
||||
</svg>`;
|
||||
} else {
|
||||
// all answered
|
||||
fillColor = "green";
|
||||
}
|
||||
}
|
||||
|
||||
statusIconEl.innerHTML = `
|
||||
<button class="view-tests-icon" title="View/Generate Tests">
|
||||
// White/orange/green => clickable <button>
|
||||
statusIconEl.innerHTML = `
|
||||
<button class="view-tests-icon" title="Open Test Document">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="${fillColor}" class="bi bi-file-text" viewBox="0 0 16 16">
|
||||
<path d="M4 0a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h8a2 2 0 0 0 2-2V5.5L9.5 0H4z"/>
|
||||
<path d="M9.5 0v4a1 1 0 0 0 1 1h4"/>
|
||||
<path d="M4.5 7a.5.5 0 0 1 .5.5v.5h5v-1h-5V7.5a.5.5 0 0 1 .5-.5z"/>
|
||||
</svg>
|
||||
</button>
|
||||
`;
|
||||
</button>`;
|
||||
const iconBtn = statusIconEl.querySelector("button.view-tests-icon") as HTMLButtonElement;
|
||||
iconBtn.addEventListener("click", () => {
|
||||
plugin.openQuestionDoc(note.filePath);
|
||||
});
|
||||
}
|
||||
|
||||
checkbox.addEventListener("change", () => {
|
||||
this.updateCreateTestsButtonState(createTestsButton, listEl);
|
||||
});
|
||||
|
||||
// Make the icon clickable to open the doc if it exists
|
||||
const iconBtn = statusIconEl.querySelector("button.view-tests-icon") as HTMLButtonElement;
|
||||
iconBtn.addEventListener("click", () => {
|
||||
// If doc doesn't exist, user needs to generate tests first
|
||||
if (!plugin.testDocuments[note.filePath]) {
|
||||
new Notice("No tests found. Generate them first or check your selection.");
|
||||
} else {
|
||||
plugin.openQuestionDoc(note.filePath);
|
||||
}
|
||||
});
|
||||
});
|
||||
this.updateCreateTestsButtonState(createTestsButton, listEl);
|
||||
}
|
||||
|
|
@ -124,75 +131,91 @@ export default class TestDashboardView extends ItemView {
|
|||
button.disabled = !anyChecked;
|
||||
}
|
||||
|
||||
/**
|
||||
* For each selected note, sends an OpenAI request to generate tests CONCURRENTLY,
|
||||
* instead of waiting for each request to finish before starting the next.
|
||||
*/
|
||||
async createTests(): Promise<void> {
|
||||
const plugin = (this.app as any).plugins.plugins["obsidian-rag-test-plugin"];
|
||||
if (!plugin) {
|
||||
new Notice("❌ RAG Test Plugin not found.");
|
||||
return;
|
||||
}
|
||||
|
||||
const apiKey = plugin.settings?.apiKey;
|
||||
if (!apiKey) {
|
||||
new Notice("❌ OpenAI API Key is missing! Please set it in the plugin settings.");
|
||||
return;
|
||||
}
|
||||
|
||||
// Gather all selected checkboxes
|
||||
const container = this.containerEl;
|
||||
const checkboxes = container.querySelectorAll('input[type="checkbox"]');
|
||||
const tasks: Promise<void>[] = []; // We'll store each note's "test generation" promise here
|
||||
|
||||
for (const checkbox of Array.from(checkboxes)) {
|
||||
const input = checkbox as HTMLInputElement;
|
||||
if (input.checked) {
|
||||
const filePath = input.dataset.filePath;
|
||||
const note = this.pluginData.find((n) => n.filePath === filePath);
|
||||
if (!note) continue;
|
||||
console.log(`Generating tests for: ${filePath}`);
|
||||
|
||||
console.log(`Preparing to generate tests concurrently for: ${filePath}`);
|
||||
|
||||
const listItem = input.parentElement;
|
||||
const statusIconEl = listItem?.querySelector(".status-icon");
|
||||
if (statusIconEl) {
|
||||
statusIconEl.innerHTML = `<div class="spinner"></div>`;
|
||||
}
|
||||
try {
|
||||
const response = await generateTestQuestions([note], apiKey);
|
||||
console.log(`Generated tests for ${filePath}`, response);
|
||||
|
||||
plugin.testDocuments[filePath] = {
|
||||
description: response.description,
|
||||
questions: response.questions,
|
||||
answers: {}
|
||||
};
|
||||
await plugin.saveSettings();
|
||||
console.log("createTests: Saved test doc under path:", filePath);
|
||||
// Create an async task for *this* file
|
||||
const task = (async () => {
|
||||
try {
|
||||
const response = await generateTestQuestions([note], apiKey);
|
||||
console.log(`Generated tests for ${filePath}`, response);
|
||||
|
||||
if (statusIconEl) {
|
||||
// tests created, but no answers => icon is a white file
|
||||
statusIconEl.innerHTML = `
|
||||
<button class="view-tests-icon" title="View Generated Tests">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="white" class="bi bi-file-text" viewBox="0 0 16 16">
|
||||
<path d="M4 0a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h8a2 2 0 0 0 2-2V5.5L9.5 0H4z"/>
|
||||
<path d="M9.5 0v4a1 1 0 0 0 1 1h4"/>
|
||||
<path d="M4.5 7a.5.5 0 0 1 .5.5v.5h5v-1h-5V7.5a.5.5 0 0 1 .5-.5z"/>
|
||||
</svg>
|
||||
</button>
|
||||
`;
|
||||
plugin.testDocuments[filePath] = {
|
||||
description: response.description,
|
||||
questions: response.questions,
|
||||
answers: {}
|
||||
};
|
||||
await plugin.saveSettings();
|
||||
console.log("createTests (concurrent): Saved test doc under path:", filePath);
|
||||
|
||||
const iconBtn = statusIconEl.querySelector("button.view-tests-icon") as HTMLButtonElement;
|
||||
iconBtn.addEventListener("click", () => {
|
||||
plugin.openQuestionDoc(filePath);
|
||||
});
|
||||
if (statusIconEl) {
|
||||
// By default, no answers => white
|
||||
statusIconEl.innerHTML = `
|
||||
<button class="view-tests-icon" title="Open Test Document">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="white" class="bi bi-file-text" viewBox="0 0 16 16">
|
||||
<path d="M4 0a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h8a2 2 0 0 0 2-2V5.5L9.5 0H4z"/>
|
||||
<path d="M9.5 0v4a1 1 0 0 0 1 1h4"/>
|
||||
<path d="M4.5 7a.5.5 0 0 1 .5.5v.5h5v-1h-5V7.5a.5.5 0 0 1 .5-.5z"/>
|
||||
</svg>
|
||||
</button>
|
||||
`;
|
||||
const iconBtn = statusIconEl.querySelector("button.view-tests-icon") as HTMLButtonElement;
|
||||
iconBtn.addEventListener("click", () => {
|
||||
plugin.openQuestionDoc(filePath);
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error generating tests for ${filePath}`, error);
|
||||
if (statusIconEl) {
|
||||
statusIconEl.innerHTML = `
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="gray" class="bi bi-file-text" viewBox="0 0 16 16">
|
||||
<path d="M4 0a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h8a2 2 0 0 0 2-2V5.5L9.5 0H4z"/>
|
||||
<path d="M9.5 0v4a1 1 0 0 0 1 1h4"/>
|
||||
<path d="M4.5 7a.5.5 0 0 1 .5.5v.5h5v-1h-5V7.5a.5.5 0 0 1 .5-.5z"/>
|
||||
</svg>`;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error generating tests for ${filePath}`, error);
|
||||
if (statusIconEl) {
|
||||
statusIconEl.innerHTML = `
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="gray" class="bi bi-file-text" viewBox="0 0 16 16">
|
||||
<path d="M4 0a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h8a2 2 0 0 0 2-2V5.5L9.5 0H4z"/>
|
||||
<path d="M9.5 0v4a1 1 0 0 0 1 1h4"/>
|
||||
<path d="M4.5 7a.5.5 0 0 1 .5.5v.5h5v-1h-5V7.5a.5.5 0 0 1 .5-.5z"/>
|
||||
</svg>`;
|
||||
}
|
||||
}
|
||||
})(); // immediately invoke
|
||||
|
||||
tasks.push(task);
|
||||
}
|
||||
}
|
||||
|
||||
// Now wait for all tasks concurrently with Promise.all
|
||||
await Promise.all(tasks);
|
||||
console.log("All concurrent test generation tasks finished!");
|
||||
}
|
||||
}
|
||||
|
|
@ -1,32 +1,69 @@
|
|||
/*
|
||||
This file handles communication with OpenAI to generate test questions.
|
||||
It now expects a JSON-formatted response that returns an object with two keys:
|
||||
'description' (a brief context for the tests) and 'questions' (an array of objects, each with a single key 'question').
|
||||
If the JSON is incomplete or wrapped in markdown fences, the code attempts to clean the output.
|
||||
Services for LLM calls:
|
||||
1) generateTestQuestions(...)
|
||||
2) markTestAnswers(...)
|
||||
*/
|
||||
|
||||
import { formatNotesForLLM } from "./formatter";
|
||||
import type { IndexedNote, LLMResponse, TestQuestionsResponse } from "../models/types";
|
||||
import type { IndexedNote } from "../models/types";
|
||||
|
||||
export interface LLMResponse {
|
||||
id: string;
|
||||
object: string;
|
||||
created: number;
|
||||
model: string;
|
||||
choices: {
|
||||
index: number;
|
||||
message: {
|
||||
role: string;
|
||||
content: string;
|
||||
};
|
||||
}[];
|
||||
usage: {
|
||||
prompt_tokens: number;
|
||||
completion_tokens: number;
|
||||
total_tokens: number;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The shape of the JSON returned for test questions:
|
||||
* {
|
||||
* "description": string,
|
||||
* "questions": [{ question: string }, ...]
|
||||
* }
|
||||
*/
|
||||
export interface TestQuestionsResponse {
|
||||
description: string;
|
||||
questions: { question: string }[];
|
||||
}
|
||||
|
||||
const API_URL = "https://api.openai.com/v1/chat/completions";
|
||||
|
||||
/**
|
||||
* Sends formatted note data to OpenAI and retrieves test questions as JSON.
|
||||
* @param indexedNotes - The indexed notes to generate questions from.
|
||||
* @param apiKey - The OpenAI API key.
|
||||
* @returns A promise that resolves to a TestQuestionsResponse object.
|
||||
* The response must have two keys:
|
||||
* "description": string
|
||||
* "questions": [{ question: string }, ...]
|
||||
*/
|
||||
export async function generateTestQuestions(indexedNotes: IndexedNote[], apiKey: string): Promise<TestQuestionsResponse> {
|
||||
export async function generateTestQuestions(
|
||||
indexedNotes: IndexedNote[],
|
||||
apiKey: string
|
||||
): Promise<TestQuestionsResponse> {
|
||||
if (!apiKey) {
|
||||
throw new Error("Missing OpenAI API key! Please set it in the plugin settings.");
|
||||
}
|
||||
const prompt = formatNotesForLLM(indexedNotes);
|
||||
const requestBody = {
|
||||
model: "gpt-4-turbo",
|
||||
model: "gpt-4",
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: "You are a helpful AI that generates test questions from study notes. Respond ONLY with a JSON object with two keys: 'description' and 'questions'. 'description' should be a brief summary of what the tests cover. 'questions' should be an array of objects, each having a single key 'question'. Do not include any additional text or markdown formatting."
|
||||
{
|
||||
role: "system",
|
||||
content: `You are a helpful AI that generates test questions from study notes.
|
||||
Respond ONLY with a JSON object with two keys: 'description' and 'questions'.
|
||||
'description' is a brief summary of what the tests cover.
|
||||
'questions' is an array of objects, each with a single key 'question'.
|
||||
Do not include any additional text or markdown formatting.`
|
||||
},
|
||||
{ role: "user", content: prompt }
|
||||
],
|
||||
|
|
@ -37,7 +74,7 @@ export async function generateTestQuestions(indexedNotes: IndexedNote[], apiKey:
|
|||
const response = await fetch(API_URL, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Authorization": `Bearer ${apiKey}`,
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
body: JSON.stringify(requestBody)
|
||||
|
|
@ -47,15 +84,15 @@ export async function generateTestQuestions(indexedNotes: IndexedNote[], apiKey:
|
|||
throw new Error(`OpenAI API Error: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const responseData: LLMResponse = await response.json();
|
||||
const responseData = await response.json() as LLMResponse;
|
||||
const output = responseData.choices?.[0]?.message?.content;
|
||||
if (!output) {
|
||||
throw new Error("Invalid response from OpenAI");
|
||||
}
|
||||
|
||||
console.log("Raw LLM output:", output);
|
||||
console.log("Raw LLM output for questions:", output);
|
||||
|
||||
// Clean the output by trimming whitespace and removing markdown fences if present.
|
||||
// Clean the JSON
|
||||
let jsonString = output.trim();
|
||||
if (jsonString.startsWith("```json")) {
|
||||
jsonString = jsonString.slice(7).trim();
|
||||
|
|
@ -63,18 +100,15 @@ export async function generateTestQuestions(indexedNotes: IndexedNote[], apiKey:
|
|||
if (jsonString.endsWith("```")) {
|
||||
jsonString = jsonString.slice(0, -3).trim();
|
||||
}
|
||||
// If the JSON might be truncated, try to extract up to the last closing brace.
|
||||
|
||||
const lastBrace = jsonString.lastIndexOf("}");
|
||||
if (lastBrace !== -1) {
|
||||
jsonString = jsonString.slice(0, lastBrace + 1);
|
||||
}
|
||||
// Ensure the JSON string ends with a closing curly brace.
|
||||
if (!jsonString.endsWith("}")) {
|
||||
jsonString += "}";
|
||||
}
|
||||
|
||||
console.log("Cleaned JSON string:", jsonString);
|
||||
|
||||
try {
|
||||
const parsed: TestQuestionsResponse = JSON.parse(jsonString);
|
||||
return parsed;
|
||||
|
|
@ -82,4 +116,89 @@ export async function generateTestQuestions(indexedNotes: IndexedNote[], apiKey:
|
|||
console.error("Error parsing JSON from LLM response:", err, "Cleaned JSON string:", jsonString);
|
||||
throw new Error("Failed to parse JSON response from OpenAI");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* markTestAnswers sends your note content and question–answer pairs to the LLM,
|
||||
* returning a concise JSON array, e.g.:
|
||||
* [
|
||||
* { "questionNumber": 1, "correct": true, "feedback": "..." },
|
||||
* { "questionNumber": 2, "correct": false, "feedback": "..." },
|
||||
* ...
|
||||
* ]
|
||||
*/
|
||||
export async function markTestAnswers(
|
||||
noteContent: string,
|
||||
qnaPairs: { question: string; answer: string }[],
|
||||
apiKey: string
|
||||
): Promise<Array<{ questionNumber: number; correct: boolean; feedback: string }>> {
|
||||
if (!apiKey) {
|
||||
throw new Error("No API key provided for markTestAnswers.");
|
||||
}
|
||||
|
||||
const systemMessage = `
|
||||
You are a helpful AI that grades user answers based on the provided source text.
|
||||
Return your feedback ONLY as a JSON array, with each element containing:
|
||||
- "questionNumber": the question index (1-based)
|
||||
- "correct": true or false
|
||||
- "feedback": a concise explanation
|
||||
No extra text or markdown fences.
|
||||
`.trim();
|
||||
|
||||
let userPrompt = `SOURCE DOCUMENT:\n${noteContent}\n\nUSER'S ANSWERS:\n`;
|
||||
qnaPairs.forEach((pair, idx) => {
|
||||
userPrompt += `Q${idx + 1}: ${pair.question}\nAnswer: ${pair.answer}\n\n`;
|
||||
});
|
||||
userPrompt += `Please return a JSON array where each object has "questionNumber", "correct", and "feedback".
|
||||
No extra text or formatting.
|
||||
If you cannot judge correctness, please set "correct": false and provide minimal feedback.`;
|
||||
|
||||
const reqBody = {
|
||||
model: "gpt-4",
|
||||
messages: [
|
||||
{ role: "system", content: systemMessage },
|
||||
{ role: "user", content: userPrompt }
|
||||
],
|
||||
max_tokens: 1000,
|
||||
temperature: 0.0
|
||||
};
|
||||
|
||||
const resp = await fetch(API_URL, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
body: JSON.stringify(reqBody)
|
||||
});
|
||||
|
||||
if (!resp.ok) {
|
||||
throw new Error(`OpenAI API error: ${resp.status} - ${resp.statusText}`);
|
||||
}
|
||||
|
||||
const data = await resp.json() as LLMResponse;
|
||||
let feedback = data.choices?.[0]?.message?.content;
|
||||
if (!feedback) {
|
||||
throw new Error("No content returned by LLM for markTestAnswers.");
|
||||
}
|
||||
|
||||
feedback = feedback.trim();
|
||||
console.log("Raw LLM Marking Output:", feedback);
|
||||
|
||||
// Clean potential triple-backticks
|
||||
if (feedback.startsWith("```")) {
|
||||
feedback = feedback.replace(/^```[a-z]*\n?/, "").replace(/```$/, "").trim();
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(feedback) as Array<{
|
||||
questionNumber: number;
|
||||
correct: boolean;
|
||||
feedback: string;
|
||||
}>;
|
||||
return parsed;
|
||||
} catch (err) {
|
||||
console.error("Error parsing JSON from LLM marking response:", err, "Raw:", feedback);
|
||||
throw new Error("Failed to parse LLM marking JSON output.");
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue