Enhance configuration options

- Introduced new configuration options for vision and notes LLM prompts
- Added category detection feature toggle in ConfigModal
- Updated ScreenshotProcessor to utilize new prompt configurations
- Cleaned up HelpModal by removing outdated forum links
This commit is contained in:
Travis Van Nimwegen 2025-03-20 17:22:57 -04:00
parent 317d3c2f6f
commit 773cb610e6
10 changed files with 163 additions and 38 deletions

View file

@ -1,7 +1,7 @@
{ {
"id": "vision-recall", "id": "vision-recall",
"name": "Vision Recall", "name": "Vision Recall",
"version": "1.1.0", "version": "1.2.0",
"minAppVersion": "1.8.3", "minAppVersion": "1.8.3",
"description": "Transform screenshots into searchable notes using AI vision and text analysis.", "description": "Transform screenshots into searchable notes using AI vision and text analysis.",
"author": "Travis Van Nimwegen", "author": "Travis Van Nimwegen",

View file

@ -1,6 +1,6 @@
{ {
"name": "obsidian-vision-recall", "name": "obsidian-vision-recall",
"version": "1.1.0", "version": "1.2.0",
"description": "Transform screenshots into searchable Obsidian notes using AI vision and text analysis.", "description": "Transform screenshots into searchable Obsidian notes using AI vision and text analysis.",
"main": "main.js", "main": "main.js",
"scripts": { "scripts": {

View file

@ -7,7 +7,7 @@ import { DocViewerModal } from '@/components/modals/DocViewerModal';
import { ConfigModal } from '@/components/modals/ConfigModal'; import { ConfigModal } from '@/components/modals/ConfigModal';
import { cn } from '@/lib/utils'; import { cn } from '@/lib/utils';
import { useDataContext } from '@/data/DataContext'; import { useDataContext } from '@/data/DataContext';
import { TestConfigModal } from './modals/TestConfigModal'; import { TestSetupModal } from './modals/TestSetupModal';
import { ProcessingQueueModal } from './modals/ProcessingQueueModal'; import { ProcessingQueueModal } from './modals/ProcessingQueueModal';
import { useQueueStore } from '@/stores/queueStore'; import { useQueueStore } from '@/stores/queueStore';
import { DebugOperationsModal } from './modals/DebugOperationsModal'; import { DebugOperationsModal } from './modals/DebugOperationsModal';
@ -166,10 +166,10 @@ export const MainViewHeader = ({ metadata, refreshMetadata, viewMode = 'list', o
</button> </button>
<button <button
aria-label='Test config' aria-label='Test setup'
className='cursor-pointer flex flex-row items-center gap-2' className='cursor-pointer flex flex-row items-center gap-2'
onClick={() => { onClick={() => {
new TestConfigModal( new TestSetupModal(
app, app,
plugin, plugin,
dataManager dataManager
@ -178,7 +178,7 @@ export const MainViewHeader = ({ metadata, refreshMetadata, viewMode = 'list', o
> >
<Bug className='w-4 h-4' /> <Bug className='w-4 h-4' />
<div className='hidden @3xl/subheader:block'> <div className='hidden @3xl/subheader:block'>
Test config Test setup
</div> </div>
</button> </button>
@ -264,7 +264,7 @@ export const MainViewHeader = ({ metadata, refreshMetadata, viewMode = 'list', o
</div> </div>
<button <button
aria-label='Advanced settings' aria-label='Advanced configuration'
className='cursor-pointer flex flex-row items-center gap-2' className='cursor-pointer flex flex-row items-center gap-2'
onClick={() => { onClick={() => {
new ConfigModal(app, plugin, plugin.dataManager).open(); new ConfigModal(app, plugin, plugin.dataManager).open();
@ -272,7 +272,7 @@ export const MainViewHeader = ({ metadata, refreshMetadata, viewMode = 'list', o
> >
<Settings2 className='w-4 h-4' /> <Settings2 className='w-4 h-4' />
<div className='hidden @3xl/subheader:block'> <div className='hidden @3xl/subheader:block'>
Settings Config
</div> </div>
</button> </button>
</div> </div>

View file

@ -1,9 +1,10 @@
import React, { useEffect, useState } from 'react'; import React, { useEffect, useRef, useState } from 'react';
import { Modal } from 'obsidian'; import { Modal } from 'obsidian';
import { DataManager } from '@/data/DataManager'; import { DataManager } from '@/data/DataManager';
import { Plugin } from 'obsidian'; import { Plugin } from 'obsidian';
import { Config, DefaultConfig } from '@/types/config-types'; import { Config, DefaultConfig } from '@/types/config-types';
import { createRoot } from 'react-dom/client'; import { createRoot } from 'react-dom/client';
import { cn } from '@/lib/utils';
const ConfigModalContent = ({ const ConfigModalContent = ({
dataManager, dataManager,
@ -16,11 +17,17 @@ const ConfigModalContent = ({
const [config, setConfig] = useState<Config>({}); const [config, setConfig] = useState<Config>({});
const [dirty, setDirty] = useState(false); const [dirty, setDirty] = useState(false);
const [enableCategoryDetection, setEnableCategoryDetection] = useState(false);
const promptTextareaRef = useRef<HTMLTextAreaElement>(null);
const notesPromptTextareaRef = useRef<HTMLTextAreaElement>(null);
useEffect(() => { useEffect(() => {
if (initialized) return; if (initialized) return;
const loadConfig = async () => { const loadConfig = async () => {
const currentConfig = dataManager.getConfig(); const currentConfig = dataManager.getConfig();
setConfig(currentConfig); setConfig(currentConfig);
setEnableCategoryDetection(currentConfig.enableCategoryDetection);
setInitialized(true); setInitialized(true);
}; };
loadConfig(); loadConfig();
@ -34,9 +41,26 @@ const ConfigModalContent = ({
const updateConfig = (updates: Partial<Config>) => { const updateConfig = (updates: Partial<Config>) => {
setConfig(prev => ({ ...prev, ...updates })); setConfig(prev => ({ ...prev, ...updates }));
if (updates.enableCategoryDetection !== undefined) {
setEnableCategoryDetection(updates.enableCategoryDetection);
}
setDirty(true); setDirty(true);
}; };
const resetVisionLLMPrompt = () => {
if (promptTextareaRef.current) {
promptTextareaRef.current.value = DefaultConfig.visionLLMPrompt;
}
updateConfig({ visionLLMPrompt: DefaultConfig.visionLLMPrompt });
};
const resetNotesLLMPrompt = () => {
if (notesPromptTextareaRef.current) {
notesPromptTextareaRef.current.value = DefaultConfig.notesLLMPrompt;
}
updateConfig({ notesLLMPrompt: DefaultConfig.notesLLMPrompt });
};
if (!initialized) { if (!initialized) {
return <div>Loading...</div>; return <div>Loading...</div>;
} }
@ -96,6 +120,83 @@ const ConfigModalContent = ({
</label> </label>
</div> </div>
<div className='config-field'>
<div className="flex flex-col gap-1 w-full">
<div className='flex flex-row gap-1 w-full items-center'>
<label className='text-sm font-bold flex-1'>
Vision LLM prompt
</label>
<button
aria-label="Reset vision LLM prompt to default"
type="button"
className="mod-warning cursor-pointer opacity-50 hover:opacity-100 duration-300 text-xs px-2 py-1"
onClick={resetVisionLLMPrompt}
>
Reset
</button>
</div>
<textarea
ref={promptTextareaRef}
className='w-full flex-1'
defaultValue={config.visionLLMPrompt || DefaultConfig.visionLLMPrompt}
onChange={e => updateConfig({ visionLLMPrompt: e.target.value })}
/>
</div>
</div>
<div className="config-field">
<label
className="config-field-checkbox-label"
aria-label={`Attempts to categorize screenshots based on vision LLM response. If enabled, the notes LLM prompt will be modified to include the category.`}
>
<input
type="checkbox"
defaultChecked={config.enableCategoryDetection || DefaultConfig.enableCategoryDetection}
onChange={e => updateConfig({ enableCategoryDetection: e.target.checked })}
/>
Enable category detection
</label>
</div>
<div className='config-field'>
<div className="flex flex-col gap-1 w-full">
<div className='flex flex-row gap-1 w-full items-center'>
<label
className={cn(
'text-sm font-bold flex-1',
enableCategoryDetection && 'opacity-30 hover:opacity-70 duration-300'
)}
aria-label={!enableCategoryDetection ? 'Notes LLM prompt' : 'Notes LLM prompt (disabled because category detection is enabled)'}
>
Notes LLM prompt
</label>
<button
aria-label="Reset notes LLM prompt to default"
type="button"
className="mod-warning cursor-pointer opacity-30 hover:opacity-70 duration-300 text-xs px-2 py-1"
onClick={resetNotesLLMPrompt}
>
Reset
</button>
</div>
<textarea
ref={notesPromptTextareaRef}
aria-label={!enableCategoryDetection ? 'Notes LLM prompt' : 'Notes LLM prompt (disabled because category detection is enabled)'}
className={cn(
'w-full flex-1',
enableCategoryDetection && 'opacity-30 hover:opacity-70 duration-300'
)}
disabled={enableCategoryDetection}
defaultValue={config.notesLLMPrompt || DefaultConfig.notesLLMPrompt}
onChange={e => updateConfig({ notesLLMPrompt: e.target.value })}
/>
</div>
</div>
{/* <div className="config-field"> {/* <div className="config-field">
<label className="config-field-checkbox-label"> <label className="config-field-checkbox-label">

View file

@ -41,18 +41,6 @@ const HelpView: React.FC<HelpModalProps> = ({ plugin, onClose }) => {
> >
Join the new Discord server <ExternalLink className='w-3 h-3' /> Join the new Discord server <ExternalLink className='w-3 h-3' />
</a> </a>
<div className='text-text-muted'>or</div>
<a
href='https://forum.visionrecall.com'
target='_blank'
rel='noopener noreferrer'
className='cursor-pointer inline-flex flex-row items-center gap-1'
aria-label={'Open the forum'}
>
Visit & chat on the forum <ExternalLink className='w-3 h-3' />
</a>
</div> </div>
<div className="flex flex-row justify-end items-center w-full"> <div className="flex flex-row justify-end items-center w-full">

View file

@ -6,13 +6,13 @@ import { DateTime } from 'luxon';
import { DataManager } from '@/data/DataManager'; import { DataManager } from '@/data/DataManager';
import { getModels } from '@/services/llm-service'; import { getModels } from '@/services/llm-service';
interface TestConfigModalProps { interface TestSetupModalProps {
dataManager: DataManager; dataManager: DataManager;
plugin: VisionRecallPlugin; plugin: VisionRecallPlugin;
onClose: () => void; onClose: () => void;
} }
const TestConfigView: React.FC<TestConfigModalProps> = ({ dataManager, plugin, onClose }) => { const TestSetupView: React.FC<TestSetupModalProps> = ({ dataManager, plugin, onClose }) => {
const settings = plugin.settings; const settings = plugin.settings;
const [models, setModels] = useState<string[]>([]); const [models, setModels] = useState<string[]>([]);
@ -127,7 +127,7 @@ const TestConfigView: React.FC<TestConfigModalProps> = ({ dataManager, plugin, o
); );
}; };
export class TestConfigModal extends Modal { export class TestSetupModal extends Modal {
private metadata: any; private metadata: any;
private plugin: VisionRecallPlugin; private plugin: VisionRecallPlugin;
@ -139,11 +139,11 @@ export class TestConfigModal extends Modal {
onOpen() { onOpen() {
const { contentEl, titleEl } = this; const { contentEl, titleEl } = this;
titleEl.setText('Test config'); titleEl.setText('Test your setup');
const root = createRoot(contentEl); const root = createRoot(contentEl);
root.render( root.render(
<TestConfigView <TestSetupView
dataManager={this.plugin.dataManager} dataManager={this.plugin.dataManager}
plugin={this.plugin} plugin={this.plugin}
onClose={() => this.close()} onClose={() => this.close()}

View file

@ -1,11 +1,30 @@
import { OPENROUTER_HEADERS } from '@/constants'; import { OPENROUTER_HEADERS } from '@/constants';
import { getLanguagePromptModifierIfIndicated } from '@/lib/languages'; import { getLanguagePromptModifierIfIndicated } from '@/lib/languages';
import { tagsJsonSchema, TagsSchema } from '@/lib/tag-utils'; import { tagsJsonSchema, TagsSchema } from '@/lib/tag-utils';
import { Config } from '@/types/config-types';
import { VisionRecallPluginSettings } from '@/types/settings-types'; import { VisionRecallPluginSettings } from '@/types/settings-types';
import { requestUrl, RequestUrlParam, RequestUrlResponse } from 'obsidian'; import { requestUrl, RequestUrlParam, RequestUrlResponse } from 'obsidian';
export const VISION_LLM_PROMPT = "Analyze this screenshot and describe its content and identify the type of screenshot if possible."; export const VISION_LLM_PROMPT = "Analyze this screenshot and describe its content and identify the type of screenshot if possible.";
export const NOTES_LLM_PROMPT = "The following OCR text and vision analysis are from a screenshot. Summarize and synthesize the text and vision analysis and identify key information.";
export const getVisionLLMPrompt = (config: Config) => {
if (config.visionLLMPrompt && config.visionLLMPrompt.trim() !== '') {
return config.visionLLMPrompt;
}
return VISION_LLM_PROMPT;
}
export const getNotesLLMPrompt = (config: Config) => {
if (config.notesLLMPrompt && config.notesLLMPrompt.trim() !== '') {
return config.notesLLMPrompt;
}
return NOTES_LLM_PROMPT;
}
export function adjustEndpoint(apiEndpointUrl: string, shouldRemove: boolean): string { export function adjustEndpoint(apiEndpointUrl: string, shouldRemove: boolean): string {
const suffix = '/v1' const suffix = '/v1'

View file

@ -1,7 +1,7 @@
import { App, FileManager, Notice, TFile, TFolder, arrayBufferToBase64, normalizePath } from 'obsidian'; import { App, FileManager, Notice, TFile, TFolder, arrayBufferToBase64, normalizePath } from 'obsidian';
import Tesseract, { createWorker, Worker } from 'tesseract.js'; import Tesseract, { createWorker, Worker } from 'tesseract.js';
import { VisionRecallPluginSettings } from '@/types/settings-types'; import { VisionRecallPluginSettings } from '@/types/settings-types';
import { DEFAULT_TAGS_AND_TITLE, TagsAndTitle, VISION_LLM_PROMPT, callLLMAPI } from '@/services/llm-service'; import { DEFAULT_TAGS_AND_TITLE, TagsAndTitle, VISION_LLM_PROMPT, callLLMAPI, getNotesLLMPrompt, getVisionLLMPrompt } from '@/services/llm-service';
import VisionRecallPlugin from '@/main'; import VisionRecallPlugin from '@/main';
import { checkOCRText } from '@/lib/ocr-validation'; import { checkOCRText } from '@/lib/ocr-validation';
import { formatTags, sanitizeObsidianTag, tagsToCommaString } from '@/lib/tag-utils'; import { formatTags, sanitizeObsidianTag, tagsToCommaString } from '@/lib/tag-utils';
@ -171,7 +171,7 @@ export class ScreenshotProcessor {
if (!ocrText || this.progressManager.isStoppedByUser()) return null; if (!ocrText || this.progressManager.isStoppedByUser()) return null;
ocrText = cleanOCRResultLanguageSpecific(ocrText, currentLanguageSetting || 'eng') || ''; ocrText = cleanOCRResultLanguageSpecific(ocrText, currentLanguageSetting || 'eng') || '';
console.log('ocrText (after cleaning)', ocrText); this.plugin.logger.debug('ocrText (after cleaning)', ocrText);
if (!ocrText) return null; if (!ocrText) return null;
let validOCRText = await checkOCRText(ocrText, currentLanguageSetting || 'eng'); let validOCRText = await checkOCRText(ocrText, currentLanguageSetting || 'eng');
@ -363,7 +363,7 @@ export class ScreenshotProcessor {
languagePromptModifier = getLanguagePromptModifierIfIndicated(this.settings, true); languagePromptModifier = getLanguagePromptModifierIfIndicated(this.settings, true);
} }
let textPayload = VISION_LLM_PROMPT + languagePromptModifier; let textPayload = getVisionLLMPrompt(this.plugin.dataManager.getConfig()) + languagePromptModifier;
this.plugin.logger.debug('Vision LLM prompt:', textPayload); this.plugin.logger.debug('Vision LLM prompt:', textPayload);
const visionPayload = { const visionPayload = {
@ -399,19 +399,23 @@ export class ScreenshotProcessor {
private async generateNotes(ocrText: string, visionLLMResponse: string, settings: VisionRecallPluginSettings): Promise<string | null> { private async generateNotes(ocrText: string, visionLLMResponse: string, settings: VisionRecallPluginSettings): Promise<string | null> {
try { try {
const config = this.plugin.dataManager.getConfig();
const languagePromptModifier = getLanguagePromptModifierIfIndicated(settings, true); const languagePromptModifier = getLanguagePromptModifierIfIndicated(settings, true);
let endpointPrompt = `The following text is from a screenshot. Summarize the text and identify key information.`; let endpointPrompt = getNotesLLMPrompt(config);
const visionLLMCategories = Object.keys(visionLLMResponseCategoriesMap); if (config.enableCategoryDetection) {
const visionLLMCategories = Object.keys(visionLLMResponseCategoriesMap);
const visionLLMCategory = visionLLMCategories.find(category => visionLLMResponse.toLowerCase().includes(category)); const visionLLMCategory = visionLLMCategories.find(category => visionLLMResponse.toLowerCase().includes(category));
if (visionLLMCategory) { if (visionLLMCategory) {
endpointPrompt = visionLLMResponseCategoriesMap[visionLLMCategory]; endpointPrompt = visionLLMResponseCategoriesMap[visionLLMCategory];
}
} }
this.plugin.logger.debug('Generate Notes LLM prompt (without added language modifier, OCR text, or vision analysis that are added later):', endpointPrompt);
endpointPrompt += `${languagePromptModifier}\n\nOCR text:\n${ocrText}\n\nVision analysis:\n${visionLLMResponse}` endpointPrompt += `${languagePromptModifier}\n\nOCR text:\n${ocrText}\n\nVision analysis:\n${visionLLMResponse}`
const endpointPayload = { const endpointPayload = {
@ -430,8 +434,8 @@ export class ScreenshotProcessor {
); );
this.plugin.logger.debug('Generated notes:', generatedNotes ? generatedNotes.substring(0, 300) + '...' : 'API call failed or no notes generated'); this.plugin.logger.debug('Generated notes:', generatedNotes ? generatedNotes.substring(0, 300) + '...' : 'API call failed or no notes generated');
return generatedNotes;
return generatedNotes;
} catch (error) { } catch (error) {
this.plugin.logger.error('Endpoint LLM API error:', error); this.plugin.logger.error('Endpoint LLM API error:', error);
new Notice('Endpoint LLM API call failed. See console for details.'); new Notice('Endpoint LLM API call failed. See console for details.');
@ -502,6 +506,7 @@ export class ScreenshotProcessor {
let counter = 1; let counter = 1;
while (this.app.vault.getAbstractFileByPath(notePath) && counter < 100) { while (this.app.vault.getAbstractFileByPath(notePath) && counter < 100) {
this.plugin.logger.debug(`Note title already exists: ${noteTitle}. Incrementing counter and trying again.`);
noteTitle = `${noteTitleBase} Notes (${counter}).md`; noteTitle = `${noteTitleBase} Notes (${counter}).md`;
notePath = normalizePath(`${outputNotesFolder}/${noteTitle}`); notePath = normalizePath(`${outputNotesFolder}/${noteTitle}`);
counter++; counter++;

View file

@ -1,3 +1,5 @@
import { VISION_LLM_PROMPT, NOTES_LLM_PROMPT } from '@/services/llm-service';
export interface Config { export interface Config {
enableAutoIntakeFolderProcessing?: boolean; enableAutoIntakeFolderProcessing?: boolean;
@ -10,6 +12,12 @@ export interface Config {
experimentalFeatures?: string[]; experimentalFeatures?: string[];
visionLLMPrompt?: string;
enableCategoryDetection?: boolean; // this is rudimentarily implemented, so defaulting to false
notesLLMPrompt?: string;
[key: string]: unknown; [key: string]: unknown;
} }
@ -18,5 +26,8 @@ export const DefaultConfig: Config = {
enablePeriodicIntakeFolderProcessing: false, enablePeriodicIntakeFolderProcessing: false,
intakeFolderPollingInterval: 300, // 300 seconds, or 5 minutes intakeFolderPollingInterval: 300, // 300 seconds, or 5 minutes
defaultMinimizedProgressDisplay: false, // Currently not functional defaultMinimizedProgressDisplay: false, // Currently not functional
experimentalFeatures: [] experimentalFeatures: [],
visionLLMPrompt: VISION_LLM_PROMPT,
enableCategoryDetection: false,
notesLLMPrompt: NOTES_LLM_PROMPT,
}; };

View file

@ -14,5 +14,6 @@
"1.0.12": "1.8.3", "1.0.12": "1.8.3",
"1.0.13": "1.8.3", "1.0.13": "1.8.3",
"1.0.14": "1.8.3", "1.0.14": "1.8.3",
"1.1.0": "1.8.3" "1.1.0": "1.8.3",
"1.2.0": "1.8.3"
} }