mirror of
https://github.com/travisvn/obsidian-vision-recall.git
synced 2026-07-22 05:38:13 +00:00
Compare commits
6 commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6b99aafa81 | ||
|
|
ea199c4d50 | ||
|
|
50a079968a | ||
|
|
773cb610e6 | ||
|
|
317d3c2f6f | ||
|
|
a183e1e12c |
18 changed files with 590 additions and 65 deletions
|
|
@ -10,7 +10,9 @@
|
|||
<a href="https://github.com/travisvn/obsidian-vision-recall/issues">
|
||||
<img src="https://img.shields.io/github/issues/travisvn/obsidian-vision-recall" alt="GitHub issues"></a>
|
||||
<img src="https://img.shields.io/github/last-commit/travisvn/obsidian-vision-recall?color=red" alt="GitHub last commit">
|
||||
<img src="https://hits.seeyoufarm.com/api/count/incr/badge.svg?url=https%3A%2F%2Fgithub.com%2Ftravisvn%2Fobsidian-vision-recall&count_bg=%2379C83D&title_bg=%23555555&icon=&icon_color=%23E7E7E7&title=hits&edge_flat=false" alt="Hits">
|
||||
<a href="https://visionrecall.com/discord">
|
||||
<img src="https://img.shields.io/badge/Discord-Vision_Recall-blue?logo=discord&logoColor=white" alt="Discord">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
<h1 align="center">Screenshot anything you want to recall later. Let AI handle the rest.</h1>
|
||||
|
|
@ -81,4 +83,4 @@
|
|||
|
||||
Visit the [wiki](https://github.com/travisvn/obsidian-vision-recall/wiki)
|
||||
|
||||
If you still need help, [join our new Discord](https://visionrecall.com/discord) or [post on the forum](https://forum.visionrecall.com)
|
||||
If you still need help, [join our new Discord](https://visionrecall.com/discord)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"id": "vision-recall",
|
||||
"name": "Vision Recall",
|
||||
"version": "1.0.13",
|
||||
"version": "1.2.0",
|
||||
"minAppVersion": "1.8.3",
|
||||
"description": "Transform screenshots into searchable notes using AI vision and text analysis.",
|
||||
"author": "Travis Van Nimwegen",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "obsidian-vision-recall",
|
||||
"version": "1.0.13",
|
||||
"version": "1.2.0",
|
||||
"description": "Transform screenshots into searchable Obsidian notes using AI vision and text analysis.",
|
||||
"main": "main.js",
|
||||
"scripts": {
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import { DocViewerModal } from '@/components/modals/DocViewerModal';
|
|||
import { ConfigModal } from '@/components/modals/ConfigModal';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { useDataContext } from '@/data/DataContext';
|
||||
import { TestConfigModal } from './modals/TestConfigModal';
|
||||
import { TestSetupModal } from './modals/TestSetupModal';
|
||||
import { ProcessingQueueModal } from './modals/ProcessingQueueModal';
|
||||
import { useQueueStore } from '@/stores/queueStore';
|
||||
import { DebugOperationsModal } from './modals/DebugOperationsModal';
|
||||
|
|
@ -166,10 +166,10 @@ export const MainViewHeader = ({ metadata, refreshMetadata, viewMode = 'list', o
|
|||
</button>
|
||||
|
||||
<button
|
||||
aria-label='Test config'
|
||||
aria-label='Test setup'
|
||||
className='cursor-pointer flex flex-row items-center gap-2'
|
||||
onClick={() => {
|
||||
new TestConfigModal(
|
||||
new TestSetupModal(
|
||||
app,
|
||||
plugin,
|
||||
dataManager
|
||||
|
|
@ -178,7 +178,7 @@ export const MainViewHeader = ({ metadata, refreshMetadata, viewMode = 'list', o
|
|||
>
|
||||
<Bug className='w-4 h-4' />
|
||||
<div className='hidden @3xl/subheader:block'>
|
||||
Test config
|
||||
Test setup
|
||||
</div>
|
||||
</button>
|
||||
|
||||
|
|
@ -264,7 +264,7 @@ export const MainViewHeader = ({ metadata, refreshMetadata, viewMode = 'list', o
|
|||
</div>
|
||||
|
||||
<button
|
||||
aria-label='Advanced settings'
|
||||
aria-label='Advanced configuration'
|
||||
className='cursor-pointer flex flex-row items-center gap-2'
|
||||
onClick={() => {
|
||||
new ConfigModal(app, plugin, plugin.dataManager).open();
|
||||
|
|
@ -272,7 +272,7 @@ export const MainViewHeader = ({ metadata, refreshMetadata, viewMode = 'list', o
|
|||
>
|
||||
<Settings2 className='w-4 h-4' />
|
||||
<div className='hidden @3xl/subheader:block'>
|
||||
Settings
|
||||
Config
|
||||
</div>
|
||||
</button>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import React, { useEffect, useState } from 'react';
|
||||
import React, { useEffect, useRef, useState } from 'react';
|
||||
import { Modal } from 'obsidian';
|
||||
import { DataManager } from '@/data/DataManager';
|
||||
import { Plugin } from 'obsidian';
|
||||
import { Config, DefaultConfig } from '@/types/config-types';
|
||||
import { createRoot } from 'react-dom/client';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
const ConfigModalContent = ({
|
||||
dataManager,
|
||||
|
|
@ -16,11 +17,17 @@ const ConfigModalContent = ({
|
|||
const [config, setConfig] = useState<Config>({});
|
||||
const [dirty, setDirty] = useState(false);
|
||||
|
||||
const [enableCategoryDetection, setEnableCategoryDetection] = useState(false);
|
||||
|
||||
const promptTextareaRef = useRef<HTMLTextAreaElement>(null);
|
||||
const notesPromptTextareaRef = useRef<HTMLTextAreaElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (initialized) return;
|
||||
const loadConfig = async () => {
|
||||
const currentConfig = dataManager.getConfig();
|
||||
setConfig(currentConfig);
|
||||
setEnableCategoryDetection(currentConfig.enableCategoryDetection);
|
||||
setInitialized(true);
|
||||
};
|
||||
loadConfig();
|
||||
|
|
@ -34,9 +41,26 @@ const ConfigModalContent = ({
|
|||
|
||||
const updateConfig = (updates: Partial<Config>) => {
|
||||
setConfig(prev => ({ ...prev, ...updates }));
|
||||
if (updates.enableCategoryDetection !== undefined) {
|
||||
setEnableCategoryDetection(updates.enableCategoryDetection);
|
||||
}
|
||||
setDirty(true);
|
||||
};
|
||||
|
||||
const resetVisionLLMPrompt = () => {
|
||||
if (promptTextareaRef.current) {
|
||||
promptTextareaRef.current.value = DefaultConfig.visionLLMPrompt;
|
||||
}
|
||||
updateConfig({ visionLLMPrompt: DefaultConfig.visionLLMPrompt });
|
||||
};
|
||||
|
||||
const resetNotesLLMPrompt = () => {
|
||||
if (notesPromptTextareaRef.current) {
|
||||
notesPromptTextareaRef.current.value = DefaultConfig.notesLLMPrompt;
|
||||
}
|
||||
updateConfig({ notesLLMPrompt: DefaultConfig.notesLLMPrompt });
|
||||
};
|
||||
|
||||
if (!initialized) {
|
||||
return <div>Loading...</div>;
|
||||
}
|
||||
|
|
@ -96,6 +120,83 @@ const ConfigModalContent = ({
|
|||
</label>
|
||||
</div>
|
||||
|
||||
<div className='config-field'>
|
||||
<div className="flex flex-col gap-1 w-full">
|
||||
<div className='flex flex-row gap-1 w-full items-center'>
|
||||
<label className='text-sm font-bold flex-1'>
|
||||
Vision LLM prompt
|
||||
</label>
|
||||
<button
|
||||
aria-label="Reset vision LLM prompt to default"
|
||||
type="button"
|
||||
className="mod-warning cursor-pointer opacity-50 hover:opacity-100 duration-300 text-xs px-2 py-1"
|
||||
onClick={resetVisionLLMPrompt}
|
||||
>
|
||||
Reset
|
||||
</button>
|
||||
</div>
|
||||
<textarea
|
||||
ref={promptTextareaRef}
|
||||
className='w-full flex-1'
|
||||
defaultValue={config.visionLLMPrompt || DefaultConfig.visionLLMPrompt}
|
||||
onChange={e => updateConfig({ visionLLMPrompt: e.target.value })}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div className="config-field">
|
||||
<label
|
||||
className="config-field-checkbox-label"
|
||||
aria-label={`Attempts to categorize screenshots based on vision LLM response. If enabled, the notes LLM prompt will be modified to include the category.`}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
defaultChecked={config.enableCategoryDetection || DefaultConfig.enableCategoryDetection}
|
||||
onChange={e => updateConfig({ enableCategoryDetection: e.target.checked })}
|
||||
/>
|
||||
Enable category detection
|
||||
</label>
|
||||
</div>
|
||||
|
||||
|
||||
<div className='config-field'>
|
||||
<div className="flex flex-col gap-1 w-full">
|
||||
<div className='flex flex-row gap-1 w-full items-center'>
|
||||
<label
|
||||
className={cn(
|
||||
'text-sm font-bold flex-1',
|
||||
enableCategoryDetection && 'opacity-30 hover:opacity-70 duration-300'
|
||||
)}
|
||||
aria-label={!enableCategoryDetection ? 'Notes LLM prompt' : 'Notes LLM prompt (disabled because category detection is enabled)'}
|
||||
>
|
||||
Notes LLM prompt
|
||||
</label>
|
||||
<button
|
||||
aria-label="Reset notes LLM prompt to default"
|
||||
type="button"
|
||||
className="mod-warning cursor-pointer opacity-30 hover:opacity-70 duration-300 text-xs px-2 py-1"
|
||||
onClick={resetNotesLLMPrompt}
|
||||
>
|
||||
Reset
|
||||
</button>
|
||||
</div>
|
||||
<textarea
|
||||
ref={notesPromptTextareaRef}
|
||||
aria-label={!enableCategoryDetection ? 'Notes LLM prompt' : 'Notes LLM prompt (disabled because category detection is enabled)'}
|
||||
className={cn(
|
||||
'w-full flex-1',
|
||||
enableCategoryDetection && 'opacity-30 hover:opacity-70 duration-300'
|
||||
)}
|
||||
disabled={enableCategoryDetection}
|
||||
defaultValue={config.notesLLMPrompt || DefaultConfig.notesLLMPrompt}
|
||||
onChange={e => updateConfig({ notesLLMPrompt: e.target.value })}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
{/* <div className="config-field">
|
||||
<label className="config-field-checkbox-label">
|
||||
|
|
|
|||
|
|
@ -41,18 +41,6 @@ const HelpView: React.FC<HelpModalProps> = ({ plugin, onClose }) => {
|
|||
>
|
||||
Join the new Discord server <ExternalLink className='w-3 h-3' />
|
||||
</a>
|
||||
|
||||
<div className='text-text-muted'>or</div>
|
||||
|
||||
<a
|
||||
href='https://forum.visionrecall.com'
|
||||
target='_blank'
|
||||
rel='noopener noreferrer'
|
||||
className='cursor-pointer inline-flex flex-row items-center gap-1'
|
||||
aria-label={'Open the forum'}
|
||||
>
|
||||
Visit & chat on the forum <ExternalLink className='w-3 h-3' />
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-row justify-end items-center w-full">
|
||||
|
|
|
|||
|
|
@ -6,13 +6,13 @@ import { DateTime } from 'luxon';
|
|||
import { DataManager } from '@/data/DataManager';
|
||||
import { getModels } from '@/services/llm-service';
|
||||
|
||||
interface TestConfigModalProps {
|
||||
interface TestSetupModalProps {
|
||||
dataManager: DataManager;
|
||||
plugin: VisionRecallPlugin;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
const TestConfigView: React.FC<TestConfigModalProps> = ({ dataManager, plugin, onClose }) => {
|
||||
const TestSetupView: React.FC<TestSetupModalProps> = ({ dataManager, plugin, onClose }) => {
|
||||
const settings = plugin.settings;
|
||||
|
||||
const [models, setModels] = useState<string[]>([]);
|
||||
|
|
@ -127,7 +127,7 @@ const TestConfigView: React.FC<TestConfigModalProps> = ({ dataManager, plugin, o
|
|||
);
|
||||
};
|
||||
|
||||
export class TestConfigModal extends Modal {
|
||||
export class TestSetupModal extends Modal {
|
||||
private metadata: any;
|
||||
private plugin: VisionRecallPlugin;
|
||||
|
||||
|
|
@ -139,11 +139,11 @@ export class TestConfigModal extends Modal {
|
|||
|
||||
onOpen() {
|
||||
const { contentEl, titleEl } = this;
|
||||
titleEl.setText('Test config');
|
||||
titleEl.setText('Test your setup');
|
||||
const root = createRoot(contentEl);
|
||||
|
||||
root.render(
|
||||
<TestConfigView
|
||||
<TestSetupView
|
||||
dataManager={this.plugin.dataManager}
|
||||
plugin={this.plugin}
|
||||
onClose={() => this.close()}
|
||||
263
src/lib/languages.ts
Normal file
263
src/lib/languages.ts
Normal file
|
|
@ -0,0 +1,263 @@
|
|||
import { VisionRecallPluginSettings } from '@/types/settings-types'
|
||||
|
||||
export type Language = {
|
||||
code: string
|
||||
name: string
|
||||
nativeName: string
|
||||
}
|
||||
|
||||
export const languages: Record<string, Language> = {
|
||||
AFR: { code: 'afr', name: 'Afrikaans', nativeName: 'Afrikaans' },
|
||||
AMH: { code: 'amh', name: 'Amharic', nativeName: 'አማርኛ' },
|
||||
ARA: { code: 'ara', name: 'Arabic', nativeName: 'العربية' },
|
||||
ASM: { code: 'asm', name: 'Assamese', nativeName: 'অসমীয়া' },
|
||||
AZE: { code: 'aze', name: 'Azerbaijani', nativeName: 'Azərbaycanca' },
|
||||
AZE_CYRL: { code: 'aze_cyrl', name: 'Azerbaijani - Cyrillic', nativeName: 'Азәрбајҹан' },
|
||||
BEL: { code: 'bel', name: 'Belarusian', nativeName: 'Беларуская' },
|
||||
BEN: { code: 'ben', name: 'Bengali', nativeName: 'বাংলা' },
|
||||
BOD: { code: 'bod', name: 'Tibetan', nativeName: 'བོད་སྐད་' },
|
||||
BOS: { code: 'bos', name: 'Bosnian', nativeName: 'Bosanski' },
|
||||
BUL: { code: 'bul', name: 'Bulgarian', nativeName: 'Български' },
|
||||
CAT: { code: 'cat', name: 'Catalan', nativeName: 'Català' },
|
||||
CEB: { code: 'ceb', name: 'Cebuano', nativeName: 'Cebuano' },
|
||||
CES: { code: 'ces', name: 'Czech', nativeName: 'Čeština' },
|
||||
CHI_SIM: { code: 'chi_sim', name: 'Chinese - Simplified', nativeName: '简体中文' },
|
||||
CHI_TRA: { code: 'chi_tra', name: 'Chinese - Traditional', nativeName: '繁體中文' },
|
||||
CHR: { code: 'chr', name: 'Cherokee', nativeName: 'ᏣᎳᎩ' },
|
||||
CYM: { code: 'cym', name: 'Welsh', nativeName: 'Cymraeg' },
|
||||
DAN: { code: 'dan', name: 'Danish', nativeName: 'Dansk' },
|
||||
DEU: { code: 'deu', name: 'German', nativeName: 'Deutsch' },
|
||||
ELL: { code: 'ell', name: 'Greek', nativeName: 'Ελληνικά' },
|
||||
ENG: { code: 'eng', name: 'English', nativeName: 'English' },
|
||||
EPO: { code: 'epo', name: 'Esperanto', nativeName: 'Esperanto' },
|
||||
EST: { code: 'est', name: 'Estonian', nativeName: 'Eesti' },
|
||||
FAS: { code: 'fas', name: 'Persian', nativeName: 'فارسی' },
|
||||
FIN: { code: 'fin', name: 'Finnish', nativeName: 'Suomi' },
|
||||
FRA: { code: 'fra', name: 'French', nativeName: 'Français' },
|
||||
GLG: { code: 'glg', name: 'Galician', nativeName: 'Galego' },
|
||||
HEB: { code: 'heb', name: 'Hebrew', nativeName: 'עברית' },
|
||||
HIN: { code: 'hin', name: 'Hindi', nativeName: 'हिन्दी' },
|
||||
HRV: { code: 'hrv', name: 'Croatian', nativeName: 'Hrvatski' },
|
||||
HUN: { code: 'hun', name: 'Hungarian', nativeName: 'Magyar' },
|
||||
IND: { code: 'ind', name: 'Indonesian', nativeName: 'Bahasa Indonesia' },
|
||||
ISL: { code: 'isl', name: 'Icelandic', nativeName: 'Íslenska' },
|
||||
ITA: { code: 'ita', name: 'Italian', nativeName: 'Italiano' },
|
||||
JPN: { code: 'jpn', name: 'Japanese', nativeName: '日本語' },
|
||||
KOR: { code: 'kor', name: 'Korean', nativeName: '한국어' },
|
||||
LAV: { code: 'lav', name: 'Latvian', nativeName: 'Latviešu' },
|
||||
LIT: { code: 'lit', name: 'Lithuanian', nativeName: 'Lietuvių' },
|
||||
MAL: { code: 'mal', name: 'Malayalam', nativeName: 'മലയാളം' },
|
||||
MAR: { code: 'mar', name: 'Marathi', nativeName: 'मराठी' },
|
||||
MKD: { code: 'mkd', name: 'Macedonian', nativeName: 'Македонски' },
|
||||
MSA: { code: 'msa', name: 'Malay', nativeName: 'Bahasa Melayu' },
|
||||
MYA: { code: 'mya', name: 'Burmese', nativeName: 'မြန်မာစာ' },
|
||||
NLD: { code: 'nld', name: 'Dutch', nativeName: 'Nederlands' },
|
||||
NOR: { code: 'nor', name: 'Norwegian', nativeName: 'Norsk' },
|
||||
POL: { code: 'pol', name: 'Polish', nativeName: 'Polski' },
|
||||
POR: { code: 'por', name: 'Portuguese', nativeName: 'Português' },
|
||||
RON: { code: 'ron', name: 'Romanian', nativeName: 'Română' },
|
||||
RUS: { code: 'rus', name: 'Russian', nativeName: 'Русский' },
|
||||
SLK: { code: 'slk', name: 'Slovak', nativeName: 'Slovenčina' },
|
||||
SLV: { code: 'slv', name: 'Slovenian', nativeName: 'Slovenščina' },
|
||||
SPA: { code: 'spa', name: 'Spanish', nativeName: 'Español' },
|
||||
SWE: { code: 'swe', name: 'Swedish', nativeName: 'Svenska' },
|
||||
TUR: { code: 'tur', name: 'Turkish', nativeName: 'Türkçe' },
|
||||
UKR: { code: 'ukr', name: 'Ukrainian', nativeName: 'Українська' },
|
||||
URD: { code: 'urd', name: 'Urdu', nativeName: 'اردو' },
|
||||
VIE: { code: 'vie', name: 'Vietnamese', nativeName: 'Tiếng Việt' },
|
||||
YID: { code: 'yid', name: 'Yiddish', nativeName: 'ייִדיש' }
|
||||
}
|
||||
|
||||
export type SelectOption = {
|
||||
value: string
|
||||
label: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Transforms languages into a format suitable for a select component.
|
||||
* Format: { value: "eng", label: "English (English)" }
|
||||
*/
|
||||
export const getLanguagesForSelect = (): SelectOption[] => {
|
||||
return getLanguagesArray().map((lang: Language) => ({
|
||||
value: lang.code,
|
||||
label: `${lang.name} (${lang.nativeName})`,
|
||||
}))
|
||||
}
|
||||
|
||||
export const getLanguagesForObsidianSettingsDropdown = (): Record<string, string> => {
|
||||
return getLanguagesArray().reduce((acc, lang) => {
|
||||
acc[lang.code] = `${lang.name} (${lang.nativeName})`;
|
||||
return acc;
|
||||
}, {} as Record<string, string>);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an array of languages for selection.
|
||||
*/
|
||||
export const getLanguagesArray = (): Language[] => {
|
||||
return Object.values(languages)
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves a language object by its Tesseract.js code.
|
||||
*/
|
||||
export const getLanguageByCode = (code: string): Language | undefined => {
|
||||
return Object.values(languages).find(lang => lang.code === code)
|
||||
}
|
||||
|
||||
|
||||
export const getLanguagePromptModifierIfIndicated = (config: VisionRecallPluginSettings, extended: boolean = false): string => {
|
||||
if (config.addLanguageConvertToPrompt && config.tesseractLanguage && config.tesseractLanguage !== 'eng') {
|
||||
const language = getLanguageByCode(config.tesseractLanguage as string)
|
||||
if (language) {
|
||||
let extendedModifier = '';
|
||||
if (extended) {
|
||||
extendedModifier = ` (${language.nativeName})`;
|
||||
}
|
||||
// Maybe "language.name (language.nativeName)"
|
||||
return `\n\nGenerate the response in the following language: ${language.name}${extendedModifier}\n\n`
|
||||
}
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
export const languageRegexMap_ScriptRegex: Record<string, RegExp> = {
|
||||
afr: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:; ]+$/,
|
||||
amh: /^[\p{Script=Ethiopic}0-9.,!?'"()\-:; ]+$/u,
|
||||
ara: /^[\p{Script=Arabic}0-9.,!?'"()\-:;،؛ ]+$/u,
|
||||
asm: /^[\p{Script=Bengali}0-9.,!?'"()\-:; ]+$/u,
|
||||
aze: /^[a-zA-ZƏəĞğİıÖöŞşÜüÇç0-9.,!?'"()\-:; ]+$/,
|
||||
aze_cyrl: /^[\p{Script=Cyrillic}0-9.,!?'"()\-:; ]+$/u,
|
||||
bel: /^[\p{Script=Cyrillic}0-9.,!?'"()\-:;«» ]+$/u,
|
||||
ben: /^[\p{Script=Bengali}0-9.,!?'"()\-:; ]+$/u,
|
||||
bod: /^[\p{Script=Tibetan}0-9.,!?'"()\-:; ]+$/u,
|
||||
bos: /^[a-zA-ZČčĆćĐ𩹮ž0-9.,!?'"()\-:; ]+$/,
|
||||
bul: /^[\p{Script=Cyrillic}0-9.,!?'"()\-:;«» ]+$/u,
|
||||
cat: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:;«» ]+$/,
|
||||
ceb: /^[a-zA-ZÑñ0-9.,!?'"()\-:; ]+$/,
|
||||
ces: /^[a-zA-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽáčďéěíňóřšťúůýž0-9.,!?'"()\-:; ]+$/,
|
||||
chi_sim: /^[\p{Script=Han}0-9.,!?'"()\-:;、。《》]+$/u,
|
||||
chi_tra: /^[\p{Script=Han}0-9.,!?'"()\-:;、。《》]+$/u,
|
||||
chr: /^[\p{Script=Cherokee}0-9.,!?'"()\-:; ]+$/u,
|
||||
cym: /^[a-zA-Zŵŷáéíóúàèìòùâêîôû0-9.,!?'"()\-:; ]+$/,
|
||||
dan: /^[a-zA-ZÆØÅæøå0-9.,!?'"()\-:; ]+$/,
|
||||
deu: /^[a-zA-ZÄÖÜäöüß0-9.,!?'"()\-:;„“– ]+$/,
|
||||
ell: /^[\p{Script=Greek}0-9.,!?'"()\-:;«» ]+$/u,
|
||||
eng: /^[a-zA-Z0-9.,!?'"()\-:; ]+$/,
|
||||
epo: /^[a-zA-ZĈĉĜĝĤĥĴĵŜŝŬŭ0-9.,!?'"()\-:; ]+$/,
|
||||
est: /^[a-zA-ZÄÖÜäöüÕõŠšŽž0-9.,!?'"()\-:; ]+$/,
|
||||
fas: /^[\p{Script=Arabic}0-9.,!?'"()\-:;،؛ ]+$/u,
|
||||
fin: /^[a-zA-ZÄÖäöÅå0-9.,!?'"()\-:; ]+$/,
|
||||
fra: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:;«» ]+$/,
|
||||
glg: /^[a-zA-ZÁÉÍÓÚÜÑáéíóúüñ0-9.,!?'"()\-:; ]+$/,
|
||||
heb: /^[\p{Script=Hebrew}0-9.,!?'"()\-:;״ ]+$/u,
|
||||
hin: /^[\p{Script=Devanagari}0-9.,!?'"()\-:; ]+$/u,
|
||||
hrv: /^[a-zA-ZČčĆćĐ𩹮ž0-9.,!?'"()\-:; ]+$/,
|
||||
hun: /^[a-zA-ZÁÉÍÓÖŐÚÜŰáéíóöőúüű0-9.,!?'"()\-:; ]+$/,
|
||||
ind: /^[a-zA-Z0-9.,!?'"()\-:; ]+$/,
|
||||
isl: /^[a-zA-ZÁÉÍÓÚÝÞÆÐÖáéíóúýþæðö0-9.,!?'"()\-:; ]+$/,
|
||||
ita: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:;«» ]+$/,
|
||||
jpn: /^[\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Han}0-9.,!?'"()\-:;「」『』]+$/u,
|
||||
kor: /^[\p{Script=Hangul}0-9.,!?'"()\-:; ]+$/u,
|
||||
lav: /^[a-zA-ZĀČĒĢĪĶĻŅŌŖŠŪŽāčēģīķļņōŗšūž0-9.,!?'"()\-:; ]+$/,
|
||||
lit: /^[a-zA-ZĄČĘĖĮŠŲŪŽąčęėįšųūž0-9.,!?'"()\-:; ]+$/,
|
||||
mal: /^[\p{Script=Malayalam}0-9.,!?'"()\-:; ]+$/u,
|
||||
mar: /^[\p{Script=Devanagari}0-9.,!?'"()\-:; ]+$/u,
|
||||
mkd: /^[\p{Script=Cyrillic}0-9.,!?'"()\-:;«» ]+$/u,
|
||||
msa: /^[a-zA-Z0-9.,!?'"()\-:; ]+$/,
|
||||
mya: /^[\p{Script=Myanmar}0-9.,!?'"()\-:; ]+$/u,
|
||||
nld: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:; ]+$/,
|
||||
nor: /^[a-zA-ZÆØÅæøå0-9.,!?'"()\-:; ]+$/,
|
||||
pol: /^[a-zA-ZĄĆĘŁŃÓŚŹŻąćęłńóśźż0-9.,!?'"()\-:; ]+$/,
|
||||
por: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:;«» ]+$/,
|
||||
ron: /^[a-zA-ZĂÂÎȘȚăâîșț0-9.,!?'"()\-:; ]+$/,
|
||||
rus: /^[\p{Script=Cyrillic}0-9.,!?'"()\-:;«» ]+$/u,
|
||||
slk: /^[a-zA-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽáäčďéíĺľňóôŕšťúýž0-9.,!?'"()\-:; ]+$/,
|
||||
slv: /^[a-zA-ZČŠŽčšž0-9.,!?'"()\-:; ]+$/,
|
||||
spa: /^[a-zA-ZÁÉÍÓÚÑÜáéíóúñü0-9.,!?'"()\-:;«»¡¿ ]+$/,
|
||||
swe: /^[a-zA-ZÅÄÖåäö0-9.,!?'"()\-:; ]+$/,
|
||||
tur: /^[a-zA-ZÇĞİÖŞÜçğıöşü0-9.,!?'"()\-:; ]+$/,
|
||||
ukr: /^[\p{Script=Cyrillic}0-9.,!?'"()\-:;«» ]+$/u,
|
||||
urd: /^[\p{Script=Arabic}0-9.,!?'"()\-:;،؛ ]+$/u,
|
||||
vie: /^[a-zA-ZÀ-ỹ0-9.,!?'"()\-:; ]+$/,
|
||||
yid: /^[\p{Script=Hebrew}0-9.,!?'"()\-:;״ ]+$/u,
|
||||
all: /^[\p{L}\p{N}.,!?'"()\-:;«»„“‘’「」『』¡¿،؛《》 ]+$/u,
|
||||
};
|
||||
|
||||
export const languageRegexMap: Record<string, RegExp> = {
|
||||
afr: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:; ]+$/, // Afrikaans
|
||||
amh: /^[\u1200-\u137F0-9.,!?'"()\-:; ]+$/, // Amharic (Ethiopic)
|
||||
ara: /^[\u0600-\u06FF0-9.,!?'"()\-:;،؛ ]+$/, // Arabic
|
||||
asm: /^[\u0980-\u09FF0-9.,!?'"()\-:; ]+$/, // Assamese (Bengali script)
|
||||
aze: /^[a-zA-ZƏəĞğİıÖöŞşÜüÇç0-9.,!?'"()\-:; ]+$/, // Azerbaijani (Latin)
|
||||
aze_cyrl: /^[\u0400-\u04FF0-9.,!?'"()\-:; ]+$/, // Azerbaijani (Cyrillic)
|
||||
bel: /^[\u0400-\u04FF0-9.,!?'"()\-:;«» ]+$/, // Belarusian (Cyrillic)
|
||||
ben: /^[\u0980-\u09FF0-9.,!?'"()\-:; ]+$/, // Bengali
|
||||
bod: /^[\u0F00-\u0FFF0-9.,!?'"()\-:; ]+$/, // Tibetan
|
||||
bos: /^[a-zA-ZČčĆćĐ𩹮ž0-9.,!?'"()\-:; ]+$/, // Bosnian
|
||||
bul: /^[\u0400-\u04FF0-9.,!?'"()\-:;«» ]+$/, // Bulgarian (Cyrillic)
|
||||
cat: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:;«» ]+$/, // Catalan
|
||||
ceb: /^[a-zA-ZÑñ0-9.,!?'"()\-:; ]+$/, // Cebuano
|
||||
ces: /^[a-zA-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽáčďéěíňóřšťúůýž0-9.,!?'"()\-:; ]+$/, // Czech
|
||||
chi_sim: /^[\u4E00-\u9FFF0-9.,!?'"()\-:;、。《》]+$/, // Chinese - Simplified
|
||||
chi_tra: /^[\u4E00-\u9FFF0-9.,!?'"()\-:;、。《》]+$/, // Chinese - Traditional
|
||||
chr: /^[\u13A0-\u13FF0-9.,!?'"()\-:; ]+$/, // Cherokee
|
||||
cym: /^[a-zA-Zŵŷáéíóúàèìòùâêîôû0-9.,!?'"()\-:; ]+$/, // Welsh
|
||||
dan: /^[a-zA-ZÆØÅæøå0-9.,!?'"()\-:; ]+$/, // Danish
|
||||
deu: /^[a-zA-ZÄÖÜäöüß0-9.,!?'"()\-:;„“– ]+$/, // German
|
||||
ell: /^[\u0370-\u03FF0-9.,!?'"()\-:;«» ]+$/, // Greek
|
||||
eng: /^[a-zA-Z0-9.,!?'"()\-:; ]+$/, // English
|
||||
epo: /^[a-zA-ZĈĉĜĝĤĥĴĵŜŝŬŭ0-9.,!?'"()\-:; ]+$/, // Esperanto
|
||||
est: /^[a-zA-ZÄÖÜäöüÕõŠšŽž0-9.,!?'"()\-:; ]+$/, // Estonian
|
||||
fas: /^[\u0600-\u06FF0-9.,!?'"()\-:;،؛ ]+$/, // Persian (Arabic script)
|
||||
fin: /^[a-zA-ZÄÖäöÅå0-9.,!?'"()\-:; ]+$/, // Finnish
|
||||
fra: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:;«» ]+$/, // French
|
||||
glg: /^[a-zA-ZÁÉÍÓÚÜÑáéíóúüñ0-9.,!?'"()\-:; ]+$/, // Galician
|
||||
heb: /^[\u0590-\u05FF0-9.,!?'"()\-:;״ ]+$/, // Hebrew
|
||||
hin: /^[\u0900-\u097F0-9.,!?'"()\-:; ]+$/, // Hindi (Devanagari)
|
||||
hrv: /^[a-zA-ZČčĆćĐ𩹮ž0-9.,!?'"()\-:; ]+$/, // Croatian
|
||||
hun: /^[a-zA-ZÁÉÍÓÖŐÚÜŰáéíóöőúüű0-9.,!?'"()\-:; ]+$/, // Hungarian
|
||||
isl: /^[a-zA-ZÁÉÍÓÚÝÞÆÐÖáéíóúýþæðö0-9.,!?'"()\-:; ]+$/, // Icelandic
|
||||
ita: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:;«» ]+$/, // Italian
|
||||
jpn: /^[\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\uFF01-\uFF5E0-9.,!?'"()\-:;「」『』]+$/, // Japanese
|
||||
kor: /^[\uAC00-\uD7AF0-9.,!?'"()\-:; ]+$/, // Korean (Hangul)
|
||||
msa: /^[a-zA-Z0-9.,!?'"()\-:; ]+$/, // Malay
|
||||
mya: /^[\u1000-\u109F0-9.,!?'"()\-:; ]+$/, // Burmese (Myanmar script)
|
||||
pol: /^[a-zA-ZĄĆĘŁŃÓŚŹŻąćęłńóśźż0-9.,!?'"()\-:; ]+$/, // Polish
|
||||
por: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:;«» ]+$/, // Portuguese
|
||||
ron: /^[a-zA-ZĂÂÎȘȚăâîșț0-9.,!?'"()\-:; ]+$/, // Romanian
|
||||
rus: /^[\u0400-\u04FF0-9.,!?'"()\-:;«» ]+$/, // Russian
|
||||
slk: /^[a-zA-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽáäčďéíĺľňóôŕšťúýž0-9.,!?'"()\-:; ]+$/, // Slovak
|
||||
slv: /^[a-zA-ZČŠŽčšž0-9.,!?'"()\-:; ]+$/, // Slovenian
|
||||
spa: /^[a-zA-ZÁÉÍÓÚÑÜáéíóúñü0-9.,!?'"()\-:;«»¡¿ ]+$/, // Spanish
|
||||
swe: /^[a-zA-ZÅÄÖåäö0-9.,!?'"()\-:; ]+$/, // Swedish
|
||||
tur: /^[a-zA-ZÇĞİÖŞÜçğıöşü0-9.,!?'"()\-:; ]+$/, // Turkish
|
||||
ukr: /^[\u0400-\u04FF0-9.,!?'"()\-:;«» ]+$/, // Ukrainian
|
||||
urd: /^[\u0600-\u06FF0-9.,!?'"()\-:;،؛ ]+$/, // Urdu
|
||||
vie: /^[a-zA-ZÀ-ỹ0-9.,!?'"()\-:; ]+$/, // Vietnamese
|
||||
yid: /^[\u0590-\u05FF0-9.,!?'"()\-:;״ ]+$/, // Yiddish (Hebrew script)
|
||||
all: /^[\p{L}\p{N}.,!?'"()\-:;«»„“‘’「」『』¡¿،؛《》 ]+$/u, // Default: all scripts
|
||||
};
|
||||
|
||||
export function cleanOCRResultLanguageSpecificOld(ocrText: string, language: string = 'all'): string {
|
||||
const allowedRegex = languageRegexMap[language] || languageRegexMap['all'];
|
||||
return ocrText.split('\n').filter(line => !!line.trim() && allowedRegex.test(line.trim())).join('\n');
|
||||
}
|
||||
|
||||
// Checks characters individually to see if they're within our RegEx for the language
|
||||
export function cleanOCRResultLanguageSpecific(text: string, language: string = 'all'): string {
|
||||
// Get the regex for the specified language or use the default 'all' regex
|
||||
const allowedRegex = languageRegexMap_ScriptRegex[language] || languageRegexMap_ScriptRegex['all'];
|
||||
|
||||
// Filter out characters that do not match the regex
|
||||
const filteredText = [...text].filter(char => char === '\n' || allowedRegex.test(char)).join('');
|
||||
|
||||
return filteredText;
|
||||
}
|
||||
|
||||
export function getLanguageSetting(settings: VisionRecallPluginSettings): string | null {
|
||||
if (settings.addLanguageConvertToPrompt && settings.tesseractLanguage) {
|
||||
return settings.tesseractLanguage;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
|
@ -1,37 +1,53 @@
|
|||
import { languageRegexMap } from '@/lib/languages';
|
||||
import { franc } from 'franc';
|
||||
|
||||
export function isValidLanguage(text: string): boolean {
|
||||
export function isValidLanguage(text: string, expectedLanguage?: string): boolean {
|
||||
const detectedLanguage = franc(text); // Detect language
|
||||
if (expectedLanguage) {
|
||||
return detectedLanguage === expectedLanguage; // Ensure it matches expected language
|
||||
}
|
||||
return detectedLanguage !== 'und'; // 'und' means undefined
|
||||
}
|
||||
|
||||
export function isMostlyValidCharacters(text: string): boolean {
|
||||
const validCharRegex = /^[\w\s.,!?'"-–—()]+$/u; // Adjust regex for more languages if needed
|
||||
const validChars = text.split('').filter(char => validCharRegex.test(char)).length;
|
||||
export function isMostlyValidCharacters(text: string, language: string = 'all'): boolean {
|
||||
// Use the regex from `cleanOCRResult` based on the language
|
||||
// const languageRegexMap: Record<string, RegExp> = {
|
||||
// en: /^[a-zA-Z0-9.,!?'"()\-:; ]+$/,
|
||||
// fra: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:; ]+$/,
|
||||
// rus: /^[\p{Script=Cyrillic}0-9.,!?'"()\-:; ]+$/u,
|
||||
// jpn: /^[\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Han}0-9.,!?'"()\-:; ]+$/u,
|
||||
// all: /^[\p{L}\p{N}.,!?'"()\-:; ]+$/u, // Default: any Unicode letter or number
|
||||
// };
|
||||
|
||||
const allowedRegex = languageRegexMap[language] || languageRegexMap['all'];
|
||||
|
||||
const validChars = text.split('').filter(char => allowedRegex.test(char)).length;
|
||||
return validChars / text.length > 0.8; // At least 80% valid characters
|
||||
}
|
||||
|
||||
export function isValidTextLength(text: string): boolean {
|
||||
return text.length > 5 && text.length < 5000; // Arbitrary thresholds
|
||||
export function isValidTextLength(text: string, minLength = 5, maxLength = 5000): boolean {
|
||||
return text.length >= minLength && text.length <= maxLength;
|
||||
}
|
||||
|
||||
export async function checkOCRText(text: string): Promise<string | null> {
|
||||
export async function checkOCRText(text: string, expectedLanguage?: string): Promise<string | null> {
|
||||
// Clean the text first
|
||||
// const cleanedText = cleanOCRResult(text, expectedLanguage || 'all');
|
||||
|
||||
// Combine validation checks
|
||||
if (!isValidTextLength(text)) {
|
||||
console.warn('Text length is invalid.');
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!isMostlyValidCharacters(text)) {
|
||||
if (!isMostlyValidCharacters(text, expectedLanguage || 'all')) {
|
||||
console.warn('Text contains too many invalid characters.');
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!isValidLanguage(text)) {
|
||||
// maybe remove this
|
||||
if (!isValidLanguage(text, expectedLanguage)) {
|
||||
console.warn('Language detection failed.');
|
||||
return null;
|
||||
}
|
||||
|
||||
return text; // Process further if valid
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
import { z } from 'zod';
|
||||
import { zodToJsonSchema } from 'zod-to-json-schema';
|
||||
import { cleanOCRResultLanguageSpecific, getLanguageSetting } from './languages';
|
||||
import { VisionRecallPluginSettings } from '@/types/settings-types';
|
||||
// Define the Zod schema
|
||||
export const TagsSchema = z.object({
|
||||
title: z.string(),
|
||||
|
|
@ -39,7 +41,7 @@ export function tagsFromCommaString(tags: string): string[] {
|
|||
return tags.split(',').map(tag => tag.trim());
|
||||
}
|
||||
|
||||
export function sanitizeObsidianTag(tag: string): string | null {
|
||||
export function sanitizeObsidianTagOLD(tag: string): string | null {
|
||||
// Replace spaces with underscores to maintain readability
|
||||
tag = tag.replace(/\s+/g, '_');
|
||||
|
||||
|
|
@ -53,5 +55,26 @@ export function sanitizeObsidianTag(tag: string): string | null {
|
|||
|
||||
// Convert to lowercase to ensure case insensitivity
|
||||
// return tag.toLowerCase();
|
||||
return tag;
|
||||
}
|
||||
|
||||
export function sanitizeObsidianTag(tag: string, settings: VisionRecallPluginSettings): string | null {
|
||||
// Replace spaces with underscores to maintain readability
|
||||
tag = tag.replace(/\s+/g, '_');
|
||||
|
||||
const language = getLanguageSetting(settings);
|
||||
|
||||
if (language) {
|
||||
tag = cleanOCRResultLanguageSpecific(tag, language);
|
||||
} else {
|
||||
// Remove all disallowed characters (keep letters, numbers, _, -, /)
|
||||
tag = tag.replace(/[^a-zA-Z0-9_\/-]/g, '');
|
||||
|
||||
// Ensure there's at least one non-numeric character
|
||||
if (!/[a-zA-Z_\/-]/.test(tag)) {
|
||||
return null; // Return null if there's no valid non-numeric character
|
||||
}
|
||||
}
|
||||
|
||||
return tag;
|
||||
}
|
||||
|
|
@ -1,10 +1,30 @@
|
|||
import { OPENROUTER_HEADERS } from '@/constants';
|
||||
import { getLanguagePromptModifierIfIndicated } from '@/lib/languages';
|
||||
import { tagsJsonSchema, TagsSchema } from '@/lib/tag-utils';
|
||||
import { Config } from '@/types/config-types';
|
||||
import { VisionRecallPluginSettings } from '@/types/settings-types';
|
||||
import { requestUrl, RequestUrlParam, RequestUrlResponse } from 'obsidian';
|
||||
|
||||
export const VISION_LLM_PROMPT = "Analyze this screenshot and describe its content and identify the type of screenshot if possible.";
|
||||
|
||||
export const NOTES_LLM_PROMPT = "The following OCR text and vision analysis are from a screenshot. Summarize and synthesize the text and vision analysis and identify key information.";
|
||||
|
||||
export const getVisionLLMPrompt = (config: Config) => {
|
||||
if (config.visionLLMPrompt && config.visionLLMPrompt.trim() !== '') {
|
||||
return config.visionLLMPrompt;
|
||||
}
|
||||
|
||||
return VISION_LLM_PROMPT;
|
||||
}
|
||||
|
||||
export const getNotesLLMPrompt = (config: Config) => {
|
||||
if (config.notesLLMPrompt && config.notesLLMPrompt.trim() !== '') {
|
||||
return config.notesLLMPrompt;
|
||||
}
|
||||
|
||||
return NOTES_LLM_PROMPT;
|
||||
}
|
||||
|
||||
export function adjustEndpoint(apiEndpointUrl: string, shouldRemove: boolean): string {
|
||||
const suffix = '/v1'
|
||||
|
||||
|
|
@ -127,7 +147,8 @@ export async function llmSuggestTagsAndTitle(settings: VisionRecallPluginSetting
|
|||
}
|
||||
|
||||
try {
|
||||
const tagPrompt = `Please suggest exactly 5 relevant tags or keywords to categorize the following notes as well as a title for the notes. Return ONLY a JSON object with the following properties: tags (array of strings), title (string). Example format: { title: 'Title of the notes', tags: ['tag1', 'tag2', 'tag3', 'tag4', 'tag5'] }\n\nNotes:\n${notesText}`;
|
||||
const languagePromptModifier = getLanguagePromptModifierIfIndicated(settings)
|
||||
const tagPrompt = `Please suggest exactly 5 relevant tags or keywords to categorize the following notes as well as a title for the notes. Return ONLY a JSON object with the following properties: tags (array of strings), title (string). Example format: { title: 'Title of the notes', tags: ['tag1', 'tag2', 'tag3', 'tag4', 'tag5'] }${languagePromptModifier}\n\nNotes:\n${notesText}`;
|
||||
|
||||
const tagPayload = {
|
||||
model: settings.endpointLlmModelName,
|
||||
|
|
|
|||
|
|
@ -1,16 +1,17 @@
|
|||
import { App, FileManager, Notice, TFile, TFolder, arrayBufferToBase64, normalizePath } from 'obsidian';
|
||||
import Tesseract, { createWorker, Worker } from 'tesseract.js';
|
||||
import { VisionRecallPluginSettings } from '@/types/settings-types';
|
||||
import { DEFAULT_TAGS_AND_TITLE, TagsAndTitle, VISION_LLM_PROMPT, callLLMAPI } from '@/services/llm-service';
|
||||
import { DEFAULT_TAGS_AND_TITLE, TagsAndTitle, VISION_LLM_PROMPT, callLLMAPI, getNotesLLMPrompt, getVisionLLMPrompt } from '@/services/llm-service';
|
||||
import VisionRecallPlugin from '@/main';
|
||||
import { checkOCRText } from '@/lib/ocr-validation';
|
||||
import { formatTags, sanitizeObsidianTag, tagsToCommaString } from '@/lib/tag-utils';
|
||||
import { useQueueStore } from '@/stores/queueStore';
|
||||
import { visionLLMResponseCategoriesMap } from '@/data/reference';
|
||||
import { computeFileHash, shouldProcessImage } from '@/lib/image-utils';
|
||||
import { sanitizeFilename } from './shared-functions';
|
||||
import { sanitizeFilename, sanitizeObsidianTitle } from './shared-functions';
|
||||
import { generateTagsWithRetries } from './tag-service';
|
||||
import { IMAGE_EXTENSIONS } from '@/constants';
|
||||
import { cleanOCRResultLanguageSpecific, getLanguagePromptModifierIfIndicated, getLanguageSetting } from '@/lib/languages';
|
||||
|
||||
export type DeleteScreenshotMetadataParams = {
|
||||
identity: string;
|
||||
|
|
@ -78,17 +79,21 @@ class ProgressManager {
|
|||
export class ScreenshotProcessor {
|
||||
app: App;
|
||||
plugin: VisionRecallPlugin;
|
||||
settings: VisionRecallPluginSettings;
|
||||
tesseractLanguage: string | null;
|
||||
private worker: Worker | null = null;
|
||||
private progressManager: ProgressManager;
|
||||
// private logger: PluginLogger;
|
||||
|
||||
constructor(
|
||||
app: App,
|
||||
private settings: VisionRecallPluginSettings,
|
||||
settings: VisionRecallPluginSettings,
|
||||
plugin: VisionRecallPlugin
|
||||
) {
|
||||
this.app = app;
|
||||
this.plugin = plugin;
|
||||
this.settings = settings;
|
||||
this.tesseractLanguage = settings.addLanguageConvertToPrompt ? settings.tesseractLanguage : 'eng';
|
||||
// this.plugin.logger = plugin.logger;
|
||||
this.progressManager = new ProgressManager(plugin);
|
||||
// this.initializeWorker();
|
||||
|
|
@ -99,9 +104,11 @@ export class ScreenshotProcessor {
|
|||
}
|
||||
|
||||
async initializeWorker() {
|
||||
let currentTesseractLanguage = this.settings.addLanguageConvertToPrompt ? this.settings.tesseractLanguage : 'eng';
|
||||
try {
|
||||
this.worker = await createWorker();
|
||||
this.worker = await createWorker(currentTesseractLanguage);
|
||||
this.plugin.logger.debug('Tesseract worker initialized.');
|
||||
this.tesseractLanguage = currentTesseractLanguage;
|
||||
} catch (error) {
|
||||
this.plugin.logger.error('Error initializing Tesseract worker:', error);
|
||||
new Notice('Failed to initialize OCR worker. See console for details.');
|
||||
|
|
@ -117,6 +124,11 @@ export class ScreenshotProcessor {
|
|||
}
|
||||
}
|
||||
|
||||
async reinitializeWorker() {
|
||||
await this.terminateWorker();
|
||||
await this.initializeWorker();
|
||||
}
|
||||
|
||||
private async initializeProcessing(imageFile: TFile): Promise<boolean> {
|
||||
if (this.worker === null) {
|
||||
new Notice('Initializing Tesseract worker...');
|
||||
|
|
@ -149,26 +161,33 @@ export class ScreenshotProcessor {
|
|||
tagsAndTitle: TagsAndTitle;
|
||||
formattedTags: string;
|
||||
} | null> {
|
||||
const currentLanguageSetting = getLanguageSetting(this.settings);
|
||||
this.plugin.logger.debug('Current language setting:', currentLanguageSetting);
|
||||
|
||||
this.progressManager.updateProgress('Performing OCR...', 10);
|
||||
if (this.progressManager.isStoppedByUser()) return null;
|
||||
|
||||
const ocrText = await this.performOCR(imageFile);
|
||||
let ocrText = await this.performOCR(imageFile, currentLanguageSetting);
|
||||
if (!ocrText || this.progressManager.isStoppedByUser()) return null;
|
||||
|
||||
let validOCRText = await checkOCRText(ocrText);
|
||||
ocrText = cleanOCRResultLanguageSpecific(ocrText, currentLanguageSetting || 'eng') || '';
|
||||
this.plugin.logger.debug('ocrText (after cleaning)', ocrText);
|
||||
if (!ocrText) return null;
|
||||
|
||||
let validOCRText = await checkOCRText(ocrText, currentLanguageSetting || 'eng');
|
||||
if (!validOCRText) validOCRText = '';
|
||||
if (this.progressManager.isStoppedByUser()) return null;
|
||||
|
||||
this.progressManager.updateProgress('Analyzing image...', 30);
|
||||
if (this.progressManager.isStoppedByUser()) return null;
|
||||
|
||||
const visionLLMResponse = await this.performVisionAnalysis(imageFile, validOCRText);
|
||||
const visionLLMResponse = await this.performVisionAnalysis(imageFile, validOCRText, this.settings.tesseractLanguage);
|
||||
if (!visionLLMResponse || this.progressManager.isStoppedByUser()) return null;
|
||||
|
||||
this.progressManager.updateProgress('Generating notes...', 20);
|
||||
if (this.progressManager.isStoppedByUser()) return null;
|
||||
|
||||
const generatedNotes = await this.generateNotes(validOCRText, visionLLMResponse);
|
||||
const generatedNotes = await this.generateNotes(validOCRText, visionLLMResponse, this.settings);
|
||||
if (!generatedNotes || this.progressManager.isStoppedByUser()) return null;
|
||||
|
||||
this.progressManager.updateProgress('Generating tags...', 20);
|
||||
|
|
@ -253,7 +272,7 @@ export class ScreenshotProcessor {
|
|||
this.progressManager.updateProgress('Saving results...', 20);
|
||||
if (this.progressManager.isStoppedByUser()) return false;
|
||||
|
||||
const tagPath = sanitizeObsidianTag(paths.uniqueName);
|
||||
const tagPath = sanitizeObsidianTag(paths.uniqueName, this.settings);
|
||||
const linkingTag = `#${this.settings.tagPrefix}/${tagPath}`;
|
||||
|
||||
const noteInfo = await this.createObsidianNote(
|
||||
|
|
@ -305,13 +324,18 @@ export class ScreenshotProcessor {
|
|||
}
|
||||
}
|
||||
|
||||
private async performOCR(imageFile: TFile): Promise<string | null> {
|
||||
private async performOCR(imageFile: TFile, language: string | null = null): Promise<string | null> {
|
||||
if (!this.worker) {
|
||||
this.plugin.logger.warn('Tesseract worker not initialized. OCR will not be performed.');
|
||||
new Notice('OCR worker is not ready.');
|
||||
return null;
|
||||
}
|
||||
|
||||
if ((language != null && language != this.tesseractLanguage) || (language == null && this.tesseractLanguage != 'eng')) {
|
||||
this.plugin.logger.warn('Tesseract language mismatch. Reinitializing Tesseract worker.');
|
||||
await this.reinitializeWorker();
|
||||
}
|
||||
|
||||
try {
|
||||
const imageBuffer = await this.app.vault.readBinary(imageFile);
|
||||
if (this.progressManager.isStoppedByUser()) return null;
|
||||
|
|
@ -328,18 +352,27 @@ export class ScreenshotProcessor {
|
|||
}
|
||||
}
|
||||
|
||||
private async performVisionAnalysis(imageFile: TFile, ocrText: string): Promise<string | null> {
|
||||
private async performVisionAnalysis(imageFile: TFile, ocrText: string, language: string | null = null): Promise<string | null> {
|
||||
try {
|
||||
const imageBuffer = await this.app.vault.readBinary(imageFile);
|
||||
const base64Image = arrayBufferToBase64(imageBuffer);
|
||||
|
||||
let languagePromptModifier = '';
|
||||
if (language) {
|
||||
// languagePromptModifier = `\n\nThe response should be in ${language}.`;
|
||||
languagePromptModifier = getLanguagePromptModifierIfIndicated(this.settings, true);
|
||||
}
|
||||
|
||||
let textPayload = getVisionLLMPrompt(this.plugin.dataManager.getConfig()) + languagePromptModifier;
|
||||
this.plugin.logger.debug('Vision LLM prompt:', textPayload);
|
||||
|
||||
const visionPayload = {
|
||||
model: this.settings.visionModelName,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: VISION_LLM_PROMPT },
|
||||
{ type: "text", text: textPayload },
|
||||
{ type: "image_url", image_url: { url: `data:image/png;base64,${base64Image}`, detail: "high" } },
|
||||
],
|
||||
}
|
||||
|
|
@ -364,19 +397,26 @@ export class ScreenshotProcessor {
|
|||
}
|
||||
}
|
||||
|
||||
private async generateNotes(ocrText: string, visionLLMResponse: string): Promise<string | null> {
|
||||
private async generateNotes(ocrText: string, visionLLMResponse: string, settings: VisionRecallPluginSettings): Promise<string | null> {
|
||||
try {
|
||||
let endpointPrompt = `The following text is from a screenshot. Summarize the text and identify key information.`;
|
||||
const config = this.plugin.dataManager.getConfig();
|
||||
const languagePromptModifier = getLanguagePromptModifierIfIndicated(settings, true);
|
||||
|
||||
const visionLLMCategories = Object.keys(visionLLMResponseCategoriesMap);
|
||||
let endpointPrompt = getNotesLLMPrompt(config);
|
||||
|
||||
const visionLLMCategory = visionLLMCategories.find(category => visionLLMResponse.toLowerCase().includes(category));
|
||||
if (config.enableCategoryDetection) {
|
||||
const visionLLMCategories = Object.keys(visionLLMResponseCategoriesMap);
|
||||
|
||||
if (visionLLMCategory) {
|
||||
endpointPrompt = visionLLMResponseCategoriesMap[visionLLMCategory];
|
||||
const visionLLMCategory = visionLLMCategories.find(category => visionLLMResponse.toLowerCase().includes(category));
|
||||
|
||||
if (visionLLMCategory) {
|
||||
endpointPrompt = visionLLMResponseCategoriesMap[visionLLMCategory];
|
||||
}
|
||||
}
|
||||
|
||||
endpointPrompt += `\n\nOCR text:\n${ocrText}\n\nVision analysis:\n${visionLLMResponse}`
|
||||
this.plugin.logger.debug('Generate Notes LLM prompt (without added language modifier, OCR text, or vision analysis that are added later):', endpointPrompt);
|
||||
|
||||
endpointPrompt += `${languagePromptModifier}\n\nOCR text:\n${ocrText}\n\nVision analysis:\n${visionLLMResponse}`
|
||||
|
||||
const endpointPayload = {
|
||||
model: this.settings.endpointLlmModelName,
|
||||
|
|
@ -394,8 +434,8 @@ export class ScreenshotProcessor {
|
|||
);
|
||||
|
||||
this.plugin.logger.debug('Generated notes:', generatedNotes ? generatedNotes.substring(0, 300) + '...' : 'API call failed or no notes generated');
|
||||
return generatedNotes;
|
||||
|
||||
return generatedNotes;
|
||||
} catch (error) {
|
||||
this.plugin.logger.error('Endpoint LLM API error:', error);
|
||||
new Notice('Endpoint LLM API call failed. See console for details.');
|
||||
|
|
@ -460,12 +500,13 @@ export class ScreenshotProcessor {
|
|||
try {
|
||||
const outputNotesFolder = normalizePath(await this.plugin.getFolderFromSettingsKey('outputNotesFolderPath'));
|
||||
|
||||
const noteTitleBase = tagsAndTitle.title || imageFile.basename;
|
||||
const noteTitleBase = sanitizeObsidianTitle(tagsAndTitle.title || imageFile.basename);
|
||||
let noteTitle = `${noteTitleBase} Notes.md`;
|
||||
let notePath = normalizePath(`${outputNotesFolder}/${noteTitle}`);
|
||||
|
||||
let counter = 1;
|
||||
while (this.app.vault.getAbstractFileByPath(notePath) && counter < 100) {
|
||||
this.plugin.logger.debug(`Note title already exists: ${noteTitle}. Incrementing counter and trying again.`);
|
||||
noteTitle = `${noteTitleBase} Notes (${counter}).md`;
|
||||
notePath = normalizePath(`${outputNotesFolder}/${noteTitle}`);
|
||||
counter++;
|
||||
|
|
@ -476,6 +517,9 @@ export class ScreenshotProcessor {
|
|||
return null;
|
||||
}
|
||||
|
||||
const titleWithoutExtension = noteTitle.replace('.md', '').replace(/ /g, '_');
|
||||
const uniqueTag = `#${this.settings.tagPrefix}/${titleWithoutExtension}`;
|
||||
|
||||
// Truncate text if needed
|
||||
const truncatedOcrText = this.settings.truncateOcrText > 0
|
||||
? ocrText.substring(0, this.settings.truncateOcrText)
|
||||
|
|
@ -496,7 +540,7 @@ export class ScreenshotProcessor {
|
|||
// Main note content
|
||||
const noteContent = `# Notes from screenshot: ${noteTitleBase}\n\n${generatedNotes}`;
|
||||
|
||||
const metadataContent = `\n\n---\n*Screenshot filename:* [[${newScreenshotPath}]]\n*${ocrTitle}*:\n\`\`\`\n${truncatedOcrText}...\n\`\`\`\n*${visionTitle}*:\n\`\`\`\n${truncatedVisionLLMResponse}...\n\`\`\`\n\n*Tags:* ${formattedTags}\n\n${linkingTag}\n`;
|
||||
const metadataContent = `\n\n---\n*Screenshot filename:* [[${newScreenshotPath}]]\n*${ocrTitle}*:\n\`\`\`\n${truncatedOcrText}...\n\`\`\`\n*${visionTitle}*:\n\`\`\`\n${truncatedVisionLLMResponse}...\n\`\`\`\n\n*Tags:* ${formattedTags}\n\n${uniqueTag}\n`;
|
||||
|
||||
const finalNoteContent = this.settings.includeMetadataInNote
|
||||
? `${noteContent}\n\n${metadataContent}`
|
||||
|
|
|
|||
|
|
@ -44,3 +44,14 @@ export function sanitizeFilename(input: string): string {
|
|||
// Replace them with an empty string
|
||||
return normalizePath(input.replace(forbiddenChars, ''))
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitizes a string to be used as an Obsidian title
|
||||
* (may be redundant wrt above function)
|
||||
*
|
||||
* @param title - The title to sanitize
|
||||
* @returns The sanitized title
|
||||
*/
|
||||
export function sanitizeObsidianTitle(title: string): string {
|
||||
return title.replace(/[\\/:]/g, '');
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@ import { z } from 'zod';
|
|||
import { zodToJsonSchema } from 'zod-to-json-schema';
|
||||
import { callLLMAPI } from '@/services/llm-service';
|
||||
import { VisionRecallPluginSettings } from '@/types/settings-types';
|
||||
import { getLanguagePromptModifierIfIndicated } from '@/lib/languages';
|
||||
|
||||
// Define the Zod schema for tags and title
|
||||
export const TagsSchema = z.object({
|
||||
|
|
@ -120,10 +121,12 @@ export async function generateTagsWithRetries(
|
|||
*/
|
||||
export async function generateTags(generatedNotes: string, settings: VisionRecallPluginSettings): Promise<TagsAndTitle> {
|
||||
try {
|
||||
const languagePromptModifier = getLanguagePromptModifierIfIndicated(settings)
|
||||
const prompt = `
|
||||
Based on the following notes, suggest a concise title and relevant tags.
|
||||
The title should be brief but descriptive.
|
||||
The tags should be single words or short phrases that categorize the content.
|
||||
${languagePromptModifier}
|
||||
|
||||
Notes:
|
||||
${generatedNotes}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { getLanguagesForObsidianSettingsDropdown } from '@/lib/languages';
|
||||
import VisionRecallPlugin from '@/main';
|
||||
import { App, PluginSettingTab, Setting } from 'obsidian';
|
||||
|
||||
|
|
@ -93,6 +94,38 @@ export default class VisionRecallSettingTab extends PluginSettingTab {
|
|||
})
|
||||
);
|
||||
|
||||
|
||||
new Setting(containerEl).setName('Language (experimental)').setHeading();
|
||||
|
||||
new Setting(containerEl)
|
||||
.setName('Add language to prompt')
|
||||
.setDesc('Add a line at the end of the prompt to generate the response in the selected language (experimental).')
|
||||
.addToggle(toggle => toggle
|
||||
.setValue(this.plugin.settings.addLanguageConvertToPrompt)
|
||||
.onChange(async (value) => {
|
||||
this.plugin.settings.addLanguageConvertToPrompt = value;
|
||||
await this.plugin.saveSettings();
|
||||
this.display();
|
||||
})
|
||||
);
|
||||
|
||||
const languageOptions = getLanguagesForObsidianSettingsDropdown();
|
||||
|
||||
if (this.plugin.settings.addLanguageConvertToPrompt) {
|
||||
new Setting(containerEl)
|
||||
.setName('Language for OCR and LLM processing')
|
||||
.setDesc('Requested language to use for note generation and screenshot ingestion.')
|
||||
.addDropdown(dropdown => dropdown
|
||||
.addOptions(languageOptions)
|
||||
.setValue(this.plugin.settings.tesseractLanguage)
|
||||
.onChange(async (value: string) => {
|
||||
this.plugin.settings.tesseractLanguage = value;
|
||||
await this.plugin.saveSettings();
|
||||
this.display();
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
new Setting(containerEl).setName('Storage').setHeading();
|
||||
|
||||
new Setting(containerEl)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import { VISION_LLM_PROMPT, NOTES_LLM_PROMPT } from '@/services/llm-service';
|
||||
|
||||
export interface Config {
|
||||
enableAutoIntakeFolderProcessing?: boolean;
|
||||
|
||||
|
|
@ -10,6 +12,12 @@ export interface Config {
|
|||
|
||||
experimentalFeatures?: string[];
|
||||
|
||||
visionLLMPrompt?: string;
|
||||
|
||||
enableCategoryDetection?: boolean; // this is rudimentarily implemented, so defaulting to false
|
||||
|
||||
notesLLMPrompt?: string;
|
||||
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
|
|
@ -18,5 +26,8 @@ export const DefaultConfig: Config = {
|
|||
enablePeriodicIntakeFolderProcessing: false,
|
||||
intakeFolderPollingInterval: 300, // 300 seconds, or 5 minutes
|
||||
defaultMinimizedProgressDisplay: false, // Currently not functional
|
||||
experimentalFeatures: []
|
||||
experimentalFeatures: [],
|
||||
visionLLMPrompt: VISION_LLM_PROMPT,
|
||||
enableCategoryDetection: false,
|
||||
notesLLMPrompt: NOTES_LLM_PROMPT,
|
||||
};
|
||||
|
|
@ -7,6 +7,9 @@ export interface VisionRecallPluginSettings {
|
|||
visionModelName: string;
|
||||
endpointLlmModelName: string;
|
||||
|
||||
addLanguageConvertToPrompt: boolean;
|
||||
tesseractLanguage: string;
|
||||
|
||||
useParentFolder: boolean;
|
||||
parentFolderPath: string;
|
||||
|
||||
|
|
@ -41,6 +44,9 @@ export const DEFAULT_SETTINGS: VisionRecallPluginSettings = {
|
|||
visionModelName: 'gpt-4o-mini',
|
||||
endpointLlmModelName: 'gpt-4o-mini',
|
||||
|
||||
addLanguageConvertToPrompt: false,
|
||||
tesseractLanguage: 'eng',
|
||||
|
||||
useParentFolder: true,
|
||||
parentFolderPath: 'VisionRecall',
|
||||
|
||||
|
|
|
|||
|
|
@ -12,5 +12,8 @@
|
|||
"1.0.10": "1.8.3",
|
||||
"1.0.11": "1.8.3",
|
||||
"1.0.12": "1.8.3",
|
||||
"1.0.13": "1.8.3"
|
||||
"1.0.13": "1.8.3",
|
||||
"1.0.14": "1.8.3",
|
||||
"1.1.0": "1.8.3",
|
||||
"1.2.0": "1.8.3"
|
||||
}
|
||||
Loading…
Reference in a new issue