From 317d3c2f6fcfb8021239b209cae818e2c9679d77 Mon Sep 17 00:00:00 2001 From: Travis Van Nimwegen Date: Thu, 20 Mar 2025 01:09:31 -0400 Subject: [PATCH] Implement language support features - Added language support functionality, including validation - Introduced new language-related utility functions in languages.ts - Enhanced OCR validation to consider expected language - Updated settings page to allow users to select a language for OCR and LLM processing - Integrated language settings into tag generation and note creation processes --- manifest.json | 2 +- package.json | 2 +- src/lib/languages.ts | 263 +++++++++++++++++++++++++++ src/lib/ocr-validation.ts | 38 ++-- src/lib/tag-utils.ts | 25 ++- src/services/llm-service.ts | 4 +- src/services/screenshot-processor.ts | 65 +++++-- src/services/tag-service.ts | 3 + src/settings/SettingsPage.tsx | 33 ++++ src/types/settings-types.ts | 6 + versions.json | 3 +- 11 files changed, 415 insertions(+), 29 deletions(-) create mode 100644 src/lib/languages.ts diff --git a/manifest.json b/manifest.json index 3da0a75..5027d5a 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "vision-recall", "name": "Vision Recall", - "version": "1.0.14", + "version": "1.1.0", "minAppVersion": "1.8.3", "description": "Transform screenshots into searchable notes using AI vision and text analysis.", "author": "Travis Van Nimwegen", diff --git a/package.json b/package.json index 21dd5f3..b216938 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "obsidian-vision-recall", - "version": "1.0.14", + "version": "1.1.0", "description": "Transform screenshots into searchable Obsidian notes using AI vision and text analysis.", "main": "main.js", "scripts": { diff --git a/src/lib/languages.ts b/src/lib/languages.ts new file mode 100644 index 0000000..9692cd1 --- /dev/null +++ b/src/lib/languages.ts @@ -0,0 +1,263 @@ +import { VisionRecallPluginSettings } from '@/types/settings-types' + +export type Language = { + code: string + name: string + nativeName: string +} + +export const languages: Record = { + AFR: { code: 'afr', name: 'Afrikaans', nativeName: 'Afrikaans' }, + AMH: { code: 'amh', name: 'Amharic', nativeName: 'አማርኛ' }, + ARA: { code: 'ara', name: 'Arabic', nativeName: 'العربية' }, + ASM: { code: 'asm', name: 'Assamese', nativeName: 'অসমীয়া' }, + AZE: { code: 'aze', name: 'Azerbaijani', nativeName: 'Azərbaycanca' }, + AZE_CYRL: { code: 'aze_cyrl', name: 'Azerbaijani - Cyrillic', nativeName: 'Азәрбајҹан' }, + BEL: { code: 'bel', name: 'Belarusian', nativeName: 'Беларуская' }, + BEN: { code: 'ben', name: 'Bengali', nativeName: 'বাংলা' }, + BOD: { code: 'bod', name: 'Tibetan', nativeName: 'བོད་སྐད་' }, + BOS: { code: 'bos', name: 'Bosnian', nativeName: 'Bosanski' }, + BUL: { code: 'bul', name: 'Bulgarian', nativeName: 'Български' }, + CAT: { code: 'cat', name: 'Catalan', nativeName: 'Català' }, + CEB: { code: 'ceb', name: 'Cebuano', nativeName: 'Cebuano' }, + CES: { code: 'ces', name: 'Czech', nativeName: 'Čeština' }, + CHI_SIM: { code: 'chi_sim', name: 'Chinese - Simplified', nativeName: '简体中文' }, + CHI_TRA: { code: 'chi_tra', name: 'Chinese - Traditional', nativeName: '繁體中文' }, + CHR: { code: 'chr', name: 'Cherokee', nativeName: 'ᏣᎳᎩ' }, + CYM: { code: 'cym', name: 'Welsh', nativeName: 'Cymraeg' }, + DAN: { code: 'dan', name: 'Danish', nativeName: 'Dansk' }, + DEU: { code: 'deu', name: 'German', nativeName: 'Deutsch' }, + ELL: { code: 'ell', name: 'Greek', nativeName: 'Ελληνικά' }, + ENG: { code: 'eng', name: 'English', nativeName: 'English' }, + EPO: { code: 'epo', name: 'Esperanto', nativeName: 'Esperanto' }, + EST: { code: 'est', name: 'Estonian', nativeName: 'Eesti' }, + FAS: { code: 'fas', name: 'Persian', nativeName: 'فارسی' }, + FIN: { code: 'fin', name: 'Finnish', nativeName: 'Suomi' }, + FRA: { code: 'fra', name: 'French', nativeName: 'Français' }, + GLG: { code: 'glg', name: 'Galician', nativeName: 'Galego' }, + HEB: { code: 'heb', name: 'Hebrew', nativeName: 'עברית' }, + HIN: { code: 'hin', name: 'Hindi', nativeName: 'हिन्दी' }, + HRV: { code: 'hrv', name: 'Croatian', nativeName: 'Hrvatski' }, + HUN: { code: 'hun', name: 'Hungarian', nativeName: 'Magyar' }, + IND: { code: 'ind', name: 'Indonesian', nativeName: 'Bahasa Indonesia' }, + ISL: { code: 'isl', name: 'Icelandic', nativeName: 'Íslenska' }, + ITA: { code: 'ita', name: 'Italian', nativeName: 'Italiano' }, + JPN: { code: 'jpn', name: 'Japanese', nativeName: '日本語' }, + KOR: { code: 'kor', name: 'Korean', nativeName: '한국어' }, + LAV: { code: 'lav', name: 'Latvian', nativeName: 'Latviešu' }, + LIT: { code: 'lit', name: 'Lithuanian', nativeName: 'Lietuvių' }, + MAL: { code: 'mal', name: 'Malayalam', nativeName: 'മലയാളം' }, + MAR: { code: 'mar', name: 'Marathi', nativeName: 'मराठी' }, + MKD: { code: 'mkd', name: 'Macedonian', nativeName: 'Македонски' }, + MSA: { code: 'msa', name: 'Malay', nativeName: 'Bahasa Melayu' }, + MYA: { code: 'mya', name: 'Burmese', nativeName: 'မြန်မာစာ' }, + NLD: { code: 'nld', name: 'Dutch', nativeName: 'Nederlands' }, + NOR: { code: 'nor', name: 'Norwegian', nativeName: 'Norsk' }, + POL: { code: 'pol', name: 'Polish', nativeName: 'Polski' }, + POR: { code: 'por', name: 'Portuguese', nativeName: 'Português' }, + RON: { code: 'ron', name: 'Romanian', nativeName: 'Română' }, + RUS: { code: 'rus', name: 'Russian', nativeName: 'Русский' }, + SLK: { code: 'slk', name: 'Slovak', nativeName: 'Slovenčina' }, + SLV: { code: 'slv', name: 'Slovenian', nativeName: 'Slovenščina' }, + SPA: { code: 'spa', name: 'Spanish', nativeName: 'Español' }, + SWE: { code: 'swe', name: 'Swedish', nativeName: 'Svenska' }, + TUR: { code: 'tur', name: 'Turkish', nativeName: 'Türkçe' }, + UKR: { code: 'ukr', name: 'Ukrainian', nativeName: 'Українська' }, + URD: { code: 'urd', name: 'Urdu', nativeName: 'اردو' }, + VIE: { code: 'vie', name: 'Vietnamese', nativeName: 'Tiếng Việt' }, + YID: { code: 'yid', name: 'Yiddish', nativeName: 'ייִדיש' } +} + +export type SelectOption = { + value: string + label: string +} + +/** + * Transforms languages into a format suitable for a select component. + * Format: { value: "eng", label: "English (English)" } + */ +export const getLanguagesForSelect = (): SelectOption[] => { + return getLanguagesArray().map((lang: Language) => ({ + value: lang.code, + label: `${lang.name} (${lang.nativeName})`, + })) +} + +export const getLanguagesForObsidianSettingsDropdown = (): Record => { + return getLanguagesArray().reduce((acc, lang) => { + acc[lang.code] = `${lang.name} (${lang.nativeName})`; + return acc; + }, {} as Record); +} + +/** + * Returns an array of languages for selection. + */ +export const getLanguagesArray = (): Language[] => { + return Object.values(languages) +} + +/** + * Retrieves a language object by its Tesseract.js code. + */ +export const getLanguageByCode = (code: string): Language | undefined => { + return Object.values(languages).find(lang => lang.code === code) +} + + +export const getLanguagePromptModifierIfIndicated = (config: VisionRecallPluginSettings, extended: boolean = false): string => { + if (config.addLanguageConvertToPrompt && config.tesseractLanguage && config.tesseractLanguage !== 'eng') { + const language = getLanguageByCode(config.tesseractLanguage as string) + if (language) { + let extendedModifier = ''; + if (extended) { + extendedModifier = ` (${language.nativeName})`; + } + // Maybe "language.name (language.nativeName)" + return `\n\nGenerate the response in the following language: ${language.name}${extendedModifier}\n\n` + } + } + return '' +} + +export const languageRegexMap_ScriptRegex: Record = { + afr: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:; ]+$/, + amh: /^[\p{Script=Ethiopic}0-9.,!?'"()\-:; ]+$/u, + ara: /^[\p{Script=Arabic}0-9.,!?'"()\-:;،؛ ]+$/u, + asm: /^[\p{Script=Bengali}0-9.,!?'"()\-:; ]+$/u, + aze: /^[a-zA-ZƏəĞğİıÖöŞşÜüÇç0-9.,!?'"()\-:; ]+$/, + aze_cyrl: /^[\p{Script=Cyrillic}0-9.,!?'"()\-:; ]+$/u, + bel: /^[\p{Script=Cyrillic}0-9.,!?'"()\-:;«» ]+$/u, + ben: /^[\p{Script=Bengali}0-9.,!?'"()\-:; ]+$/u, + bod: /^[\p{Script=Tibetan}0-9.,!?'"()\-:; ]+$/u, + bos: /^[a-zA-ZČčĆćĐ𩹮ž0-9.,!?'"()\-:; ]+$/, + bul: /^[\p{Script=Cyrillic}0-9.,!?'"()\-:;«» ]+$/u, + cat: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:;«» ]+$/, + ceb: /^[a-zA-ZÑñ0-9.,!?'"()\-:; ]+$/, + ces: /^[a-zA-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽáčďéěíňóřšťúůýž0-9.,!?'"()\-:; ]+$/, + chi_sim: /^[\p{Script=Han}0-9.,!?'"()\-:;、。《》]+$/u, + chi_tra: /^[\p{Script=Han}0-9.,!?'"()\-:;、。《》]+$/u, + chr: /^[\p{Script=Cherokee}0-9.,!?'"()\-:; ]+$/u, + cym: /^[a-zA-Zŵŷáéíóúàèìòùâêîôû0-9.,!?'"()\-:; ]+$/, + dan: /^[a-zA-ZÆØÅæøå0-9.,!?'"()\-:; ]+$/, + deu: /^[a-zA-ZÄÖÜäöüß0-9.,!?'"()\-:;„“– ]+$/, + ell: /^[\p{Script=Greek}0-9.,!?'"()\-:;«» ]+$/u, + eng: /^[a-zA-Z0-9.,!?'"()\-:; ]+$/, + epo: /^[a-zA-ZĈĉĜĝĤĥĴĵŜŝŬŭ0-9.,!?'"()\-:; ]+$/, + est: /^[a-zA-ZÄÖÜäöüÕõŠšŽž0-9.,!?'"()\-:; ]+$/, + fas: /^[\p{Script=Arabic}0-9.,!?'"()\-:;،؛ ]+$/u, + fin: /^[a-zA-ZÄÖäöÅå0-9.,!?'"()\-:; ]+$/, + fra: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:;«» ]+$/, + glg: /^[a-zA-ZÁÉÍÓÚÜÑáéíóúüñ0-9.,!?'"()\-:; ]+$/, + heb: /^[\p{Script=Hebrew}0-9.,!?'"()\-:;״ ]+$/u, + hin: /^[\p{Script=Devanagari}0-9.,!?'"()\-:; ]+$/u, + hrv: /^[a-zA-ZČčĆćĐ𩹮ž0-9.,!?'"()\-:; ]+$/, + hun: /^[a-zA-ZÁÉÍÓÖŐÚÜŰáéíóöőúüű0-9.,!?'"()\-:; ]+$/, + ind: /^[a-zA-Z0-9.,!?'"()\-:; ]+$/, + isl: /^[a-zA-ZÁÉÍÓÚÝÞÆÐÖáéíóúýþæðö0-9.,!?'"()\-:; ]+$/, + ita: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:;«» ]+$/, + jpn: /^[\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Han}0-9.,!?'"()\-:;「」『』]+$/u, + kor: /^[\p{Script=Hangul}0-9.,!?'"()\-:; ]+$/u, + lav: /^[a-zA-ZĀČĒĢĪĶĻŅŌŖŠŪŽāčēģīķļņōŗšūž0-9.,!?'"()\-:; ]+$/, + lit: /^[a-zA-ZĄČĘĖĮŠŲŪŽąčęėįšųūž0-9.,!?'"()\-:; ]+$/, + mal: /^[\p{Script=Malayalam}0-9.,!?'"()\-:; ]+$/u, + mar: /^[\p{Script=Devanagari}0-9.,!?'"()\-:; ]+$/u, + mkd: /^[\p{Script=Cyrillic}0-9.,!?'"()\-:;«» ]+$/u, + msa: /^[a-zA-Z0-9.,!?'"()\-:; ]+$/, + mya: /^[\p{Script=Myanmar}0-9.,!?'"()\-:; ]+$/u, + nld: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:; ]+$/, + nor: /^[a-zA-ZÆØÅæøå0-9.,!?'"()\-:; ]+$/, + pol: /^[a-zA-ZĄĆĘŁŃÓŚŹŻąćęłńóśźż0-9.,!?'"()\-:; ]+$/, + por: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:;«» ]+$/, + ron: /^[a-zA-ZĂÂÎȘȚăâîșț0-9.,!?'"()\-:; ]+$/, + rus: /^[\p{Script=Cyrillic}0-9.,!?'"()\-:;«» ]+$/u, + slk: /^[a-zA-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽáäčďéíĺľňóôŕšťúýž0-9.,!?'"()\-:; ]+$/, + slv: /^[a-zA-ZČŠŽčšž0-9.,!?'"()\-:; ]+$/, + spa: /^[a-zA-ZÁÉÍÓÚÑÜáéíóúñü0-9.,!?'"()\-:;«»¡¿ ]+$/, + swe: /^[a-zA-ZÅÄÖåäö0-9.,!?'"()\-:; ]+$/, + tur: /^[a-zA-ZÇĞİÖŞÜçğıöşü0-9.,!?'"()\-:; ]+$/, + ukr: /^[\p{Script=Cyrillic}0-9.,!?'"()\-:;«» ]+$/u, + urd: /^[\p{Script=Arabic}0-9.,!?'"()\-:;،؛ ]+$/u, + vie: /^[a-zA-ZÀ-ỹ0-9.,!?'"()\-:; ]+$/, + yid: /^[\p{Script=Hebrew}0-9.,!?'"()\-:;״ ]+$/u, + all: /^[\p{L}\p{N}.,!?'"()\-:;«»„“‘’「」『』¡¿،؛《》 ]+$/u, +}; + +export const languageRegexMap: Record = { + afr: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:; ]+$/, // Afrikaans + amh: /^[\u1200-\u137F0-9.,!?'"()\-:; ]+$/, // Amharic (Ethiopic) + ara: /^[\u0600-\u06FF0-9.,!?'"()\-:;،؛ ]+$/, // Arabic + asm: /^[\u0980-\u09FF0-9.,!?'"()\-:; ]+$/, // Assamese (Bengali script) + aze: /^[a-zA-ZƏəĞğİıÖöŞşÜüÇç0-9.,!?'"()\-:; ]+$/, // Azerbaijani (Latin) + aze_cyrl: /^[\u0400-\u04FF0-9.,!?'"()\-:; ]+$/, // Azerbaijani (Cyrillic) + bel: /^[\u0400-\u04FF0-9.,!?'"()\-:;«» ]+$/, // Belarusian (Cyrillic) + ben: /^[\u0980-\u09FF0-9.,!?'"()\-:; ]+$/, // Bengali + bod: /^[\u0F00-\u0FFF0-9.,!?'"()\-:; ]+$/, // Tibetan + bos: /^[a-zA-ZČčĆćĐ𩹮ž0-9.,!?'"()\-:; ]+$/, // Bosnian + bul: /^[\u0400-\u04FF0-9.,!?'"()\-:;«» ]+$/, // Bulgarian (Cyrillic) + cat: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:;«» ]+$/, // Catalan + ceb: /^[a-zA-ZÑñ0-9.,!?'"()\-:; ]+$/, // Cebuano + ces: /^[a-zA-ZÁČĎÉĚÍŇÓŘŠŤÚŮÝŽáčďéěíňóřšťúůýž0-9.,!?'"()\-:; ]+$/, // Czech + chi_sim: /^[\u4E00-\u9FFF0-9.,!?'"()\-:;、。《》]+$/, // Chinese - Simplified + chi_tra: /^[\u4E00-\u9FFF0-9.,!?'"()\-:;、。《》]+$/, // Chinese - Traditional + chr: /^[\u13A0-\u13FF0-9.,!?'"()\-:; ]+$/, // Cherokee + cym: /^[a-zA-Zŵŷáéíóúàèìòùâêîôû0-9.,!?'"()\-:; ]+$/, // Welsh + dan: /^[a-zA-ZÆØÅæøå0-9.,!?'"()\-:; ]+$/, // Danish + deu: /^[a-zA-ZÄÖÜäöüß0-9.,!?'"()\-:;„“– ]+$/, // German + ell: /^[\u0370-\u03FF0-9.,!?'"()\-:;«» ]+$/, // Greek + eng: /^[a-zA-Z0-9.,!?'"()\-:; ]+$/, // English + epo: /^[a-zA-ZĈĉĜĝĤĥĴĵŜŝŬŭ0-9.,!?'"()\-:; ]+$/, // Esperanto + est: /^[a-zA-ZÄÖÜäöüÕõŠšŽž0-9.,!?'"()\-:; ]+$/, // Estonian + fas: /^[\u0600-\u06FF0-9.,!?'"()\-:;،؛ ]+$/, // Persian (Arabic script) + fin: /^[a-zA-ZÄÖäöÅå0-9.,!?'"()\-:; ]+$/, // Finnish + fra: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:;«» ]+$/, // French + glg: /^[a-zA-ZÁÉÍÓÚÜÑáéíóúüñ0-9.,!?'"()\-:; ]+$/, // Galician + heb: /^[\u0590-\u05FF0-9.,!?'"()\-:;״ ]+$/, // Hebrew + hin: /^[\u0900-\u097F0-9.,!?'"()\-:; ]+$/, // Hindi (Devanagari) + hrv: /^[a-zA-ZČčĆćĐ𩹮ž0-9.,!?'"()\-:; ]+$/, // Croatian + hun: /^[a-zA-ZÁÉÍÓÖŐÚÜŰáéíóöőúüű0-9.,!?'"()\-:; ]+$/, // Hungarian + isl: /^[a-zA-ZÁÉÍÓÚÝÞÆÐÖáéíóúýþæðö0-9.,!?'"()\-:; ]+$/, // Icelandic + ita: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:;«» ]+$/, // Italian + jpn: /^[\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\uFF01-\uFF5E0-9.,!?'"()\-:;「」『』]+$/, // Japanese + kor: /^[\uAC00-\uD7AF0-9.,!?'"()\-:; ]+$/, // Korean (Hangul) + msa: /^[a-zA-Z0-9.,!?'"()\-:; ]+$/, // Malay + mya: /^[\u1000-\u109F0-9.,!?'"()\-:; ]+$/, // Burmese (Myanmar script) + pol: /^[a-zA-ZĄĆĘŁŃÓŚŹŻąćęłńóśźż0-9.,!?'"()\-:; ]+$/, // Polish + por: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:;«» ]+$/, // Portuguese + ron: /^[a-zA-ZĂÂÎȘȚăâîșț0-9.,!?'"()\-:; ]+$/, // Romanian + rus: /^[\u0400-\u04FF0-9.,!?'"()\-:;«» ]+$/, // Russian + slk: /^[a-zA-ZÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽáäčďéíĺľňóôŕšťúýž0-9.,!?'"()\-:; ]+$/, // Slovak + slv: /^[a-zA-ZČŠŽčšž0-9.,!?'"()\-:; ]+$/, // Slovenian + spa: /^[a-zA-ZÁÉÍÓÚÑÜáéíóúñü0-9.,!?'"()\-:;«»¡¿ ]+$/, // Spanish + swe: /^[a-zA-ZÅÄÖåäö0-9.,!?'"()\-:; ]+$/, // Swedish + tur: /^[a-zA-ZÇĞİÖŞÜçğıöşü0-9.,!?'"()\-:; ]+$/, // Turkish + ukr: /^[\u0400-\u04FF0-9.,!?'"()\-:;«» ]+$/, // Ukrainian + urd: /^[\u0600-\u06FF0-9.,!?'"()\-:;،؛ ]+$/, // Urdu + vie: /^[a-zA-ZÀ-ỹ0-9.,!?'"()\-:; ]+$/, // Vietnamese + yid: /^[\u0590-\u05FF0-9.,!?'"()\-:;״ ]+$/, // Yiddish (Hebrew script) + all: /^[\p{L}\p{N}.,!?'"()\-:;«»„“‘’「」『』¡¿،؛《》 ]+$/u, // Default: all scripts +}; + +export function cleanOCRResultLanguageSpecificOld(ocrText: string, language: string = 'all'): string { + const allowedRegex = languageRegexMap[language] || languageRegexMap['all']; + return ocrText.split('\n').filter(line => !!line.trim() && allowedRegex.test(line.trim())).join('\n'); +} + +// Checks characters individually to see if they're within our RegEx for the language +export function cleanOCRResultLanguageSpecific(text: string, language: string = 'all'): string { + // Get the regex for the specified language or use the default 'all' regex + const allowedRegex = languageRegexMap_ScriptRegex[language] || languageRegexMap_ScriptRegex['all']; + + // Filter out characters that do not match the regex + const filteredText = [...text].filter(char => char === '\n' || allowedRegex.test(char)).join(''); + + return filteredText; +} + +export function getLanguageSetting(settings: VisionRecallPluginSettings): string | null { + if (settings.addLanguageConvertToPrompt && settings.tesseractLanguage) { + return settings.tesseractLanguage; + } + return null; +} diff --git a/src/lib/ocr-validation.ts b/src/lib/ocr-validation.ts index 6dbb93a..a4c02bc 100644 --- a/src/lib/ocr-validation.ts +++ b/src/lib/ocr-validation.ts @@ -1,37 +1,53 @@ +import { languageRegexMap } from '@/lib/languages'; import { franc } from 'franc'; -export function isValidLanguage(text: string): boolean { +export function isValidLanguage(text: string, expectedLanguage?: string): boolean { const detectedLanguage = franc(text); // Detect language + if (expectedLanguage) { + return detectedLanguage === expectedLanguage; // Ensure it matches expected language + } return detectedLanguage !== 'und'; // 'und' means undefined } -export function isMostlyValidCharacters(text: string): boolean { - const validCharRegex = /^[\w\s.,!?'"-–—()]+$/u; // Adjust regex for more languages if needed - const validChars = text.split('').filter(char => validCharRegex.test(char)).length; +export function isMostlyValidCharacters(text: string, language: string = 'all'): boolean { + // Use the regex from `cleanOCRResult` based on the language + // const languageRegexMap: Record = { + // en: /^[a-zA-Z0-9.,!?'"()\-:; ]+$/, + // fra: /^[a-zA-ZÀ-ÿ0-9.,!?'"()\-:; ]+$/, + // rus: /^[\p{Script=Cyrillic}0-9.,!?'"()\-:; ]+$/u, + // jpn: /^[\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Han}0-9.,!?'"()\-:; ]+$/u, + // all: /^[\p{L}\p{N}.,!?'"()\-:; ]+$/u, // Default: any Unicode letter or number + // }; + + const allowedRegex = languageRegexMap[language] || languageRegexMap['all']; + + const validChars = text.split('').filter(char => allowedRegex.test(char)).length; return validChars / text.length > 0.8; // At least 80% valid characters } -export function isValidTextLength(text: string): boolean { - return text.length > 5 && text.length < 5000; // Arbitrary thresholds +export function isValidTextLength(text: string, minLength = 5, maxLength = 5000): boolean { + return text.length >= minLength && text.length <= maxLength; } -export async function checkOCRText(text: string): Promise { +export async function checkOCRText(text: string, expectedLanguage?: string): Promise { + // Clean the text first + // const cleanedText = cleanOCRResult(text, expectedLanguage || 'all'); - // Combine validation checks if (!isValidTextLength(text)) { console.warn('Text length is invalid.'); return null; } - if (!isMostlyValidCharacters(text)) { + if (!isMostlyValidCharacters(text, expectedLanguage || 'all')) { console.warn('Text contains too many invalid characters.'); return null; } - if (!isValidLanguage(text)) { + // maybe remove this + if (!isValidLanguage(text, expectedLanguage)) { console.warn('Language detection failed.'); return null; } return text; // Process further if valid -} +} \ No newline at end of file diff --git a/src/lib/tag-utils.ts b/src/lib/tag-utils.ts index f76bc9f..c771944 100644 --- a/src/lib/tag-utils.ts +++ b/src/lib/tag-utils.ts @@ -1,5 +1,7 @@ import { z } from 'zod'; import { zodToJsonSchema } from 'zod-to-json-schema'; +import { cleanOCRResultLanguageSpecific, getLanguageSetting } from './languages'; +import { VisionRecallPluginSettings } from '@/types/settings-types'; // Define the Zod schema export const TagsSchema = z.object({ title: z.string(), @@ -39,7 +41,7 @@ export function tagsFromCommaString(tags: string): string[] { return tags.split(',').map(tag => tag.trim()); } -export function sanitizeObsidianTag(tag: string): string | null { +export function sanitizeObsidianTagOLD(tag: string): string | null { // Replace spaces with underscores to maintain readability tag = tag.replace(/\s+/g, '_'); @@ -53,5 +55,26 @@ export function sanitizeObsidianTag(tag: string): string | null { // Convert to lowercase to ensure case insensitivity // return tag.toLowerCase(); + return tag; +} + +export function sanitizeObsidianTag(tag: string, settings: VisionRecallPluginSettings): string | null { + // Replace spaces with underscores to maintain readability + tag = tag.replace(/\s+/g, '_'); + + const language = getLanguageSetting(settings); + + if (language) { + tag = cleanOCRResultLanguageSpecific(tag, language); + } else { + // Remove all disallowed characters (keep letters, numbers, _, -, /) + tag = tag.replace(/[^a-zA-Z0-9_\/-]/g, ''); + + // Ensure there's at least one non-numeric character + if (!/[a-zA-Z_\/-]/.test(tag)) { + return null; // Return null if there's no valid non-numeric character + } + } + return tag; } \ No newline at end of file diff --git a/src/services/llm-service.ts b/src/services/llm-service.ts index b65abe6..d9e8d9c 100644 --- a/src/services/llm-service.ts +++ b/src/services/llm-service.ts @@ -1,4 +1,5 @@ import { OPENROUTER_HEADERS } from '@/constants'; +import { getLanguagePromptModifierIfIndicated } from '@/lib/languages'; import { tagsJsonSchema, TagsSchema } from '@/lib/tag-utils'; import { VisionRecallPluginSettings } from '@/types/settings-types'; import { requestUrl, RequestUrlParam, RequestUrlResponse } from 'obsidian'; @@ -127,7 +128,8 @@ export async function llmSuggestTagsAndTitle(settings: VisionRecallPluginSetting } try { - const tagPrompt = `Please suggest exactly 5 relevant tags or keywords to categorize the following notes as well as a title for the notes. Return ONLY a JSON object with the following properties: tags (array of strings), title (string). Example format: { title: 'Title of the notes', tags: ['tag1', 'tag2', 'tag3', 'tag4', 'tag5'] }\n\nNotes:\n${notesText}`; + const languagePromptModifier = getLanguagePromptModifierIfIndicated(settings) + const tagPrompt = `Please suggest exactly 5 relevant tags or keywords to categorize the following notes as well as a title for the notes. Return ONLY a JSON object with the following properties: tags (array of strings), title (string). Example format: { title: 'Title of the notes', tags: ['tag1', 'tag2', 'tag3', 'tag4', 'tag5'] }${languagePromptModifier}\n\nNotes:\n${notesText}`; const tagPayload = { model: settings.endpointLlmModelName, diff --git a/src/services/screenshot-processor.ts b/src/services/screenshot-processor.ts index ea1aa8f..0be8f5a 100644 --- a/src/services/screenshot-processor.ts +++ b/src/services/screenshot-processor.ts @@ -11,6 +11,7 @@ import { computeFileHash, shouldProcessImage } from '@/lib/image-utils'; import { sanitizeFilename, sanitizeObsidianTitle } from './shared-functions'; import { generateTagsWithRetries } from './tag-service'; import { IMAGE_EXTENSIONS } from '@/constants'; +import { cleanOCRResultLanguageSpecific, getLanguagePromptModifierIfIndicated, getLanguageSetting } from '@/lib/languages'; export type DeleteScreenshotMetadataParams = { identity: string; @@ -78,17 +79,21 @@ class ProgressManager { export class ScreenshotProcessor { app: App; plugin: VisionRecallPlugin; + settings: VisionRecallPluginSettings; + tesseractLanguage: string | null; private worker: Worker | null = null; private progressManager: ProgressManager; // private logger: PluginLogger; constructor( app: App, - private settings: VisionRecallPluginSettings, + settings: VisionRecallPluginSettings, plugin: VisionRecallPlugin ) { this.app = app; this.plugin = plugin; + this.settings = settings; + this.tesseractLanguage = settings.addLanguageConvertToPrompt ? settings.tesseractLanguage : 'eng'; // this.plugin.logger = plugin.logger; this.progressManager = new ProgressManager(plugin); // this.initializeWorker(); @@ -99,9 +104,11 @@ export class ScreenshotProcessor { } async initializeWorker() { + let currentTesseractLanguage = this.settings.addLanguageConvertToPrompt ? this.settings.tesseractLanguage : 'eng'; try { - this.worker = await createWorker(); + this.worker = await createWorker(currentTesseractLanguage); this.plugin.logger.debug('Tesseract worker initialized.'); + this.tesseractLanguage = currentTesseractLanguage; } catch (error) { this.plugin.logger.error('Error initializing Tesseract worker:', error); new Notice('Failed to initialize OCR worker. See console for details.'); @@ -117,6 +124,11 @@ export class ScreenshotProcessor { } } + async reinitializeWorker() { + await this.terminateWorker(); + await this.initializeWorker(); + } + private async initializeProcessing(imageFile: TFile): Promise { if (this.worker === null) { new Notice('Initializing Tesseract worker...'); @@ -149,26 +161,33 @@ export class ScreenshotProcessor { tagsAndTitle: TagsAndTitle; formattedTags: string; } | null> { + const currentLanguageSetting = getLanguageSetting(this.settings); + this.plugin.logger.debug('Current language setting:', currentLanguageSetting); + this.progressManager.updateProgress('Performing OCR...', 10); if (this.progressManager.isStoppedByUser()) return null; - const ocrText = await this.performOCR(imageFile); + let ocrText = await this.performOCR(imageFile, currentLanguageSetting); if (!ocrText || this.progressManager.isStoppedByUser()) return null; - let validOCRText = await checkOCRText(ocrText); + ocrText = cleanOCRResultLanguageSpecific(ocrText, currentLanguageSetting || 'eng') || ''; + console.log('ocrText (after cleaning)', ocrText); + if (!ocrText) return null; + + let validOCRText = await checkOCRText(ocrText, currentLanguageSetting || 'eng'); if (!validOCRText) validOCRText = ''; if (this.progressManager.isStoppedByUser()) return null; this.progressManager.updateProgress('Analyzing image...', 30); if (this.progressManager.isStoppedByUser()) return null; - const visionLLMResponse = await this.performVisionAnalysis(imageFile, validOCRText); + const visionLLMResponse = await this.performVisionAnalysis(imageFile, validOCRText, this.settings.tesseractLanguage); if (!visionLLMResponse || this.progressManager.isStoppedByUser()) return null; this.progressManager.updateProgress('Generating notes...', 20); if (this.progressManager.isStoppedByUser()) return null; - const generatedNotes = await this.generateNotes(validOCRText, visionLLMResponse); + const generatedNotes = await this.generateNotes(validOCRText, visionLLMResponse, this.settings); if (!generatedNotes || this.progressManager.isStoppedByUser()) return null; this.progressManager.updateProgress('Generating tags...', 20); @@ -253,7 +272,7 @@ export class ScreenshotProcessor { this.progressManager.updateProgress('Saving results...', 20); if (this.progressManager.isStoppedByUser()) return false; - const tagPath = sanitizeObsidianTag(paths.uniqueName); + const tagPath = sanitizeObsidianTag(paths.uniqueName, this.settings); const linkingTag = `#${this.settings.tagPrefix}/${tagPath}`; const noteInfo = await this.createObsidianNote( @@ -305,13 +324,18 @@ export class ScreenshotProcessor { } } - private async performOCR(imageFile: TFile): Promise { + private async performOCR(imageFile: TFile, language: string | null = null): Promise { if (!this.worker) { this.plugin.logger.warn('Tesseract worker not initialized. OCR will not be performed.'); new Notice('OCR worker is not ready.'); return null; } + if ((language != null && language != this.tesseractLanguage) || (language == null && this.tesseractLanguage != 'eng')) { + this.plugin.logger.warn('Tesseract language mismatch. Reinitializing Tesseract worker.'); + await this.reinitializeWorker(); + } + try { const imageBuffer = await this.app.vault.readBinary(imageFile); if (this.progressManager.isStoppedByUser()) return null; @@ -328,18 +352,27 @@ export class ScreenshotProcessor { } } - private async performVisionAnalysis(imageFile: TFile, ocrText: string): Promise { + private async performVisionAnalysis(imageFile: TFile, ocrText: string, language: string | null = null): Promise { try { const imageBuffer = await this.app.vault.readBinary(imageFile); const base64Image = arrayBufferToBase64(imageBuffer); + let languagePromptModifier = ''; + if (language) { + // languagePromptModifier = `\n\nThe response should be in ${language}.`; + languagePromptModifier = getLanguagePromptModifierIfIndicated(this.settings, true); + } + + let textPayload = VISION_LLM_PROMPT + languagePromptModifier; + this.plugin.logger.debug('Vision LLM prompt:', textPayload); + const visionPayload = { model: this.settings.visionModelName, messages: [ { role: "user", content: [ - { type: "text", text: VISION_LLM_PROMPT }, + { type: "text", text: textPayload }, { type: "image_url", image_url: { url: `data:image/png;base64,${base64Image}`, detail: "high" } }, ], } @@ -364,8 +397,11 @@ export class ScreenshotProcessor { } } - private async generateNotes(ocrText: string, visionLLMResponse: string): Promise { + private async generateNotes(ocrText: string, visionLLMResponse: string, settings: VisionRecallPluginSettings): Promise { try { + + const languagePromptModifier = getLanguagePromptModifierIfIndicated(settings, true); + let endpointPrompt = `The following text is from a screenshot. Summarize the text and identify key information.`; const visionLLMCategories = Object.keys(visionLLMResponseCategoriesMap); @@ -376,7 +412,7 @@ export class ScreenshotProcessor { endpointPrompt = visionLLMResponseCategoriesMap[visionLLMCategory]; } - endpointPrompt += `\n\nOCR text:\n${ocrText}\n\nVision analysis:\n${visionLLMResponse}` + endpointPrompt += `${languagePromptModifier}\n\nOCR text:\n${ocrText}\n\nVision analysis:\n${visionLLMResponse}` const endpointPayload = { model: this.settings.endpointLlmModelName, @@ -476,6 +512,9 @@ export class ScreenshotProcessor { return null; } + const titleWithoutExtension = noteTitle.replace('.md', '').replace(/ /g, '_'); + const uniqueTag = `#${this.settings.tagPrefix}/${titleWithoutExtension}`; + // Truncate text if needed const truncatedOcrText = this.settings.truncateOcrText > 0 ? ocrText.substring(0, this.settings.truncateOcrText) @@ -496,7 +535,7 @@ export class ScreenshotProcessor { // Main note content const noteContent = `# Notes from screenshot: ${noteTitleBase}\n\n${generatedNotes}`; - const metadataContent = `\n\n---\n*Screenshot filename:* [[${newScreenshotPath}]]\n*${ocrTitle}*:\n\`\`\`\n${truncatedOcrText}...\n\`\`\`\n*${visionTitle}*:\n\`\`\`\n${truncatedVisionLLMResponse}...\n\`\`\`\n\n*Tags:* ${formattedTags}\n\n${linkingTag}\n`; + const metadataContent = `\n\n---\n*Screenshot filename:* [[${newScreenshotPath}]]\n*${ocrTitle}*:\n\`\`\`\n${truncatedOcrText}...\n\`\`\`\n*${visionTitle}*:\n\`\`\`\n${truncatedVisionLLMResponse}...\n\`\`\`\n\n*Tags:* ${formattedTags}\n\n${uniqueTag}\n`; const finalNoteContent = this.settings.includeMetadataInNote ? `${noteContent}\n\n${metadataContent}` diff --git a/src/services/tag-service.ts b/src/services/tag-service.ts index 30a2525..f00acf8 100644 --- a/src/services/tag-service.ts +++ b/src/services/tag-service.ts @@ -2,6 +2,7 @@ import { z } from 'zod'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { callLLMAPI } from '@/services/llm-service'; import { VisionRecallPluginSettings } from '@/types/settings-types'; +import { getLanguagePromptModifierIfIndicated } from '@/lib/languages'; // Define the Zod schema for tags and title export const TagsSchema = z.object({ @@ -120,10 +121,12 @@ export async function generateTagsWithRetries( */ export async function generateTags(generatedNotes: string, settings: VisionRecallPluginSettings): Promise { try { + const languagePromptModifier = getLanguagePromptModifierIfIndicated(settings) const prompt = ` Based on the following notes, suggest a concise title and relevant tags. The title should be brief but descriptive. The tags should be single words or short phrases that categorize the content. +${languagePromptModifier} Notes: ${generatedNotes} diff --git a/src/settings/SettingsPage.tsx b/src/settings/SettingsPage.tsx index 490fa87..d69c797 100644 --- a/src/settings/SettingsPage.tsx +++ b/src/settings/SettingsPage.tsx @@ -1,3 +1,4 @@ +import { getLanguagesForObsidianSettingsDropdown } from '@/lib/languages'; import VisionRecallPlugin from '@/main'; import { App, PluginSettingTab, Setting } from 'obsidian'; @@ -93,6 +94,38 @@ export default class VisionRecallSettingTab extends PluginSettingTab { }) ); + + new Setting(containerEl).setName('Language (experimental)').setHeading(); + + new Setting(containerEl) + .setName('Add language to prompt') + .setDesc('Add a line at the end of the prompt to generate the response in the selected language (experimental).') + .addToggle(toggle => toggle + .setValue(this.plugin.settings.addLanguageConvertToPrompt) + .onChange(async (value) => { + this.plugin.settings.addLanguageConvertToPrompt = value; + await this.plugin.saveSettings(); + this.display(); + }) + ); + + const languageOptions = getLanguagesForObsidianSettingsDropdown(); + + if (this.plugin.settings.addLanguageConvertToPrompt) { + new Setting(containerEl) + .setName('Language for OCR and LLM processing') + .setDesc('Requested language to use for note generation and screenshot ingestion.') + .addDropdown(dropdown => dropdown + .addOptions(languageOptions) + .setValue(this.plugin.settings.tesseractLanguage) + .onChange(async (value: string) => { + this.plugin.settings.tesseractLanguage = value; + await this.plugin.saveSettings(); + this.display(); + }) + ); + } + new Setting(containerEl).setName('Storage').setHeading(); new Setting(containerEl) diff --git a/src/types/settings-types.ts b/src/types/settings-types.ts index c7443f8..32598f7 100644 --- a/src/types/settings-types.ts +++ b/src/types/settings-types.ts @@ -7,6 +7,9 @@ export interface VisionRecallPluginSettings { visionModelName: string; endpointLlmModelName: string; + addLanguageConvertToPrompt: boolean; + tesseractLanguage: string; + useParentFolder: boolean; parentFolderPath: string; @@ -41,6 +44,9 @@ export const DEFAULT_SETTINGS: VisionRecallPluginSettings = { visionModelName: 'gpt-4o-mini', endpointLlmModelName: 'gpt-4o-mini', + addLanguageConvertToPrompt: false, + tesseractLanguage: 'eng', + useParentFolder: true, parentFolderPath: 'VisionRecall', diff --git a/versions.json b/versions.json index 1e6bbf4..923d4b8 100644 --- a/versions.json +++ b/versions.json @@ -13,5 +13,6 @@ "1.0.11": "1.8.3", "1.0.12": "1.8.3", "1.0.13": "1.8.3", - "1.0.14": "1.8.3" + "1.0.14": "1.8.3", + "1.1.0": "1.8.3" } \ No newline at end of file