From 5501024134c9f5f7b22740373960784659ed624a Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 07:06:49 +0000 Subject: [PATCH 01/82] Initial plan From 93b389e299c000d2af1ba5c1b5b02c966bfbbf09 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 07:07:05 +0000 Subject: [PATCH 02/82] Initial plan From 7f27f8c6556bec3749316d5c570fe98e58810121 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 07:12:12 +0000 Subject: [PATCH 03/82] Initial analysis and planning for language-based transcription prompts Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- src/i18n/en.ts | 4 ++-- src/i18n/ja.ts | 4 ++-- src/i18n/ko.ts | 4 ++-- src/i18n/zh.ts | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/i18n/en.ts b/src/i18n/en.ts index 9049b18..6b9108f 100644 --- a/src/i18n/en.ts +++ b/src/i18n/en.ts @@ -140,7 +140,7 @@ export const en: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input Settings', + settings: 'Voice Input Settings' }, settings: { apiKey: 'OpenAI API Key', @@ -182,6 +182,6 @@ export const en: TranslationResource = { fromMultiple: 'From (comma-separated)', to: 'To', context: 'Context Keywords' - }, + } } }; diff --git a/src/i18n/ja.ts b/src/i18n/ja.ts index 0ae49c2..710ece2 100644 --- a/src/i18n/ja.ts +++ b/src/i18n/ja.ts @@ -140,7 +140,7 @@ export const ja: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 設定', + settings: 'Voice Input 設定' }, settings: { apiKey: 'OpenAI API キー', @@ -182,6 +182,6 @@ export const ja: TranslationResource = { fromMultiple: '入力語（カンマ区切り）', to: '修正語', context: '文脈キーワード' - }, + } } }; diff --git a/src/i18n/ko.ts b/src/i18n/ko.ts index a52ab9a..6433092 100644 --- a/src/i18n/ko.ts +++ b/src/i18n/ko.ts @@ -140,7 +140,7 @@ export const ko: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 설정', + settings: 'Voice Input 설정' }, settings: { apiKey: 'OpenAI API 키', @@ -182,6 +182,6 @@ export const ko: TranslationResource = { fromMultiple: '입력어（쉼표구분）', to: '교정어', context: '문맥 키워드' - }, + } } }; diff --git a/src/i18n/zh.ts b/src/i18n/zh.ts index 724ac1a..20cee1e 100644 --- a/src/i18n/zh.ts +++ b/src/i18n/zh.ts @@ -140,7 +140,7 @@ export const zh: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 设置', + settings: 'Voice Input 设置' }, settings: { apiKey: 'OpenAI API 密钥', @@ -182,6 +182,6 @@ export const zh: TranslationResource = { fromMultiple: '输入词（逗号分隔）', to: '校正词', context: '语境关键词' - }, + } } }; From 1cf2c08182ea9c16caac328fa2558287d6d904a7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 07:14:42 +0000 Subject: [PATCH 04/82] feat(transcription): Add language-based prompts with Japanese-only detailed instructions Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- src/core/transcription/TranscriptionService.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 68d3587..d6d2a64 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -1,5 +1,5 @@ import { DictionaryCorrector } from './DictionaryCorrector'; -import { TranscriptionResult, ITranscriptionProvider, SimpleCorrectionDictionary } from '../../interfaces'; +import { TranscriptionResult, ITranscriptionProvider, SimpleCorrectionDictionary, Locale } from '../../interfaces'; import { TranscriptionError, TranscriptionErrorType } from '../../errors'; import { SecurityUtils } from '../../security'; import { API_CONSTANTS, DEFAULT_TRANSCRIPTION_SETTINGS, TRANSCRIPTION_MODEL_COSTS } from '../../config'; @@ -55,7 +55,7 @@ export class TranscriptionService implements ITranscriptionProvider { } // Build prompt for transcription - const prompt = this.buildTranscriptionPrompt(); + const prompt = this.buildTranscriptionPrompt(language as Locale); if (prompt) { formData.append('prompt', prompt); } @@ -181,16 +181,23 @@ export class TranscriptionService implements ITranscriptionProvider { } /** - * Build prompt for GPT-4o transcription + * Build prompt for GPT-4o transcription based on language */ - private buildTranscriptionPrompt(): string { - return `以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。 + private buildTranscriptionPrompt(language: Locale): string { + // For Japanese, use detailed prompt for better accuracy + if (language === 'ja') { + return `以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。 話者の発言内容だけを正確に記録してください。 出力形式: （話者の発言のみ） `; + } + + // For other languages, use minimal or no prompt to avoid interference + // Empty prompt allows the model to transcribe naturally without Japanese instructions + return ''; } /** From 3186a3d3d1c7a403a843958e29b7ec24a32ac786 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 07:15:12 +0000 Subject: [PATCH 05/82] fix(i18n): Add internationalization for waiting status display Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- src/i18n/en.ts | 7 ++++--- src/i18n/ja.ts | 7 ++++--- src/i18n/ko.ts | 7 ++++--- src/i18n/zh.ts | 7 ++++--- src/interfaces/i18n.ts | 1 + src/views/VoiceInputViewActions.ts | 2 +- 6 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/i18n/en.ts b/src/i18n/en.ts index 9049b18..01a20d4 100644 --- a/src/i18n/en.ts +++ b/src/i18n/en.ts @@ -60,7 +60,8 @@ export const en: TranslationResource = { processing: { transcribing: 'Status: Transcribing...', correcting: 'Status: Correcting...', - completed: 'Status: Completed' + completed: 'Status: Completed', + waiting: '{count} waiting' }, transcription: { vadAutoStopped: 'Status: Auto-stopped due to silence', @@ -140,7 +141,7 @@ export const en: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input Settings', + settings: 'Voice Input Settings' }, settings: { apiKey: 'OpenAI API Key', @@ -182,6 +183,6 @@ export const en: TranslationResource = { fromMultiple: 'From (comma-separated)', to: 'To', context: 'Context Keywords' - }, + } } }; diff --git a/src/i18n/ja.ts b/src/i18n/ja.ts index 0ae49c2..e1b5c03 100644 --- a/src/i18n/ja.ts +++ b/src/i18n/ja.ts @@ -60,7 +60,8 @@ export const ja: TranslationResource = { processing: { transcribing: 'ステータス: 文字起こし中...', correcting: 'ステータス: 清書中...', - completed: 'ステータス: 完了' + completed: 'ステータス: 完了', + waiting: '{count} 待機中' }, transcription: { vadAutoStopped: 'ステータス: 無音検出により自動停止しました', @@ -140,7 +141,7 @@ export const ja: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 設定', + settings: 'Voice Input 設定' }, settings: { apiKey: 'OpenAI API キー', @@ -182,6 +183,6 @@ export const ja: TranslationResource = { fromMultiple: '入力語（カンマ区切り）', to: '修正語', context: '文脈キーワード' - }, + } } }; diff --git a/src/i18n/ko.ts b/src/i18n/ko.ts index a52ab9a..79318ae 100644 --- a/src/i18n/ko.ts +++ b/src/i18n/ko.ts @@ -60,7 +60,8 @@ export const ko: TranslationResource = { processing: { transcribing: '상태: 음성 텍스트 변환 중...', correcting: '상태: 텍스트 정리 중...', - completed: '상태: 완료' + completed: '상태: 완료', + waiting: '{count} 대기 중' }, transcription: { vadAutoStopped: '상태: 무음 감지로 자동 중지됨', @@ -140,7 +141,7 @@ export const ko: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 설정', + settings: 'Voice Input 설정' }, settings: { apiKey: 'OpenAI API 키', @@ -182,6 +183,6 @@ export const ko: TranslationResource = { fromMultiple: '입력어（쉼표구분）', to: '교정어', context: '문맥 키워드' - }, + } } }; diff --git a/src/i18n/zh.ts b/src/i18n/zh.ts index 724ac1a..2a00104 100644 --- a/src/i18n/zh.ts +++ b/src/i18n/zh.ts @@ -60,7 +60,8 @@ export const zh: TranslationResource = { processing: { transcribing: '状态：语音转文字中...', correcting: '状态：文本处理中...', - completed: '状态：完成' + completed: '状态：完成', + waiting: '{count} 等待中' }, transcription: { vadAutoStopped: '状态：检测到静音自动停止', @@ -140,7 +141,7 @@ export const zh: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 设置', + settings: 'Voice Input 设置' }, settings: { apiKey: 'OpenAI API 密钥', @@ -182,6 +183,6 @@ export const zh: TranslationResource = { fromMultiple: '输入词（逗号分隔）', to: '校正词', context: '语境关键词' - }, + } } }; diff --git a/src/interfaces/i18n.ts b/src/interfaces/i18n.ts index 355a624..4918047 100644 --- a/src/interfaces/i18n.ts +++ b/src/interfaces/i18n.ts @@ -80,6 +80,7 @@ export type TranslationResource = { transcribing: string; correcting: string; completed: string; + waiting: string; }; transcription: { vadAutoStopped: string; diff --git a/src/views/VoiceInputViewActions.ts b/src/views/VoiceInputViewActions.ts index 430f0fa..bcabfaf 100644 --- a/src/views/VoiceInputViewActions.ts +++ b/src/views/VoiceInputViewActions.ts @@ -398,7 +398,7 @@ export class VoiceInputViewActions { // When processing, the first item is being processed, others are waiting const waitingCount = queueLength > 0 ? queueLength - 1 : 0; if (waitingCount > 0) { - statusText += ` (${waitingCount} 待機中)`; + statusText += ` (${this.i18n.t('status.processing.waiting', { count: waitingCount })})`; } this.view.ui.statusEl.setText(statusText); this.view.ui.statusEl.addClass('processing'); From fc215d4a5ace2f9e38676739e0628d35211270ff Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 07:46:27 +0000 Subject: [PATCH 06/82] fix(transcription): Improve type safety for language parameter in buildTranscriptionPrompt Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- src/core/transcription/TranscriptionService.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index d6d2a64..035c155 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -1,5 +1,5 @@ import { DictionaryCorrector } from './DictionaryCorrector'; -import { TranscriptionResult, ITranscriptionProvider, SimpleCorrectionDictionary, Locale } from '../../interfaces'; +import { TranscriptionResult, ITranscriptionProvider, SimpleCorrectionDictionary } from '../../interfaces'; import { TranscriptionError, TranscriptionErrorType } from '../../errors'; import { SecurityUtils } from '../../security'; import { API_CONSTANTS, DEFAULT_TRANSCRIPTION_SETTINGS, TRANSCRIPTION_MODEL_COSTS } from '../../config'; @@ -55,7 +55,7 @@ export class TranscriptionService implements ITranscriptionProvider { } // Build prompt for transcription - const prompt = this.buildTranscriptionPrompt(language as Locale); + const prompt = this.buildTranscriptionPrompt(language); if (prompt) { formData.append('prompt', prompt); } @@ -183,7 +183,7 @@ export class TranscriptionService implements ITranscriptionProvider { /** * Build prompt for GPT-4o transcription based on language */ - private buildTranscriptionPrompt(language: Locale): string { + private buildTranscriptionPrompt(language: string): string { // For Japanese, use detailed prompt for better accuracy if (language === 'ja') { return `以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。 From 1955c2329b515e7214ab9c51e1e2d2c9a72b9b5f Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:10:19 +0000 Subject: [PATCH 07/82] Initial plan From 346400ccd63eb821dcae736a532917fbd5dc7c4c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:23:52 +0000 Subject: [PATCH 08/82] Implement language-aware response cleaning for transcription Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- jest.config.js | 3 + .../transcription/TranscriptionService.ts | 168 ++++++++++++++++-- src/i18n/en.ts | 4 +- src/i18n/ja.ts | 4 +- src/i18n/ko.ts | 4 +- src/i18n/zh.ts | 4 +- 6 files changed, 161 insertions(+), 26 deletions(-) diff --git a/jest.config.js b/jest.config.js index 93691d7..443f427 100644 --- a/jest.config.js +++ b/jest.config.js @@ -24,5 +24,8 @@ module.exports = { testTimeout: 10000, moduleNameMapper: { '^obsidian$': '/tests/mocks/obsidian.ts', + '^../src/core/transcription/DictionaryCorrector$': '/tests/mocks/DictionaryCorrector.ts', + '^../src/utils/ObsidianHttpClient$': '/tests/mocks/ObsidianHttpClient.ts', + '^../src/services$': '/tests/mocks/services.ts', }, }; \ No newline at end of file diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 68d3587..250562a 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -114,14 +114,12 @@ export class TranscriptionService implements ITranscriptionProvider { } // Clean up GPT-4o specific artifacts - originalText = this.cleanGPT4oResponse(originalText); + originalText = this.cleanGPT4oResponse(originalText, language); // プロンプトエラーの検出（音声が無音の場合にプロンプトが返される問題） - if (originalText.includes('この指示文は出力に含めないでください') || - originalText.includes('話者の発言内容だけを正確に記録してください') || - originalText === '（話者の発言のみ）' || - originalText.trim() === '話者の発言のみ') { + // 言語別のプロンプトパターン検出 + if (this.isPromptErrorDetected(originalText, language)) { // 音声がない場合は空文字を返す originalText = ''; } @@ -194,15 +192,15 @@ export class TranscriptionService implements ITranscriptionProvider { } /** - * Clean GPT-4o specific response artifacts + * Clean GPT-4o specific response artifacts with language awareness */ - private cleanGPT4oResponse(text: string): string { - // First attempt: Extract content from complete TRANSCRIPT tags + private cleanGPT4oResponse(text: string, language: string = 'ja'): string { + // Language-agnostic cleaning: Extract content from TRANSCRIPT tags let transcriptMatch = text.match(/\s*([\s\S]*?)\s*<\/TRANSCRIPT>/); if (transcriptMatch) { text = transcriptMatch[1]; } else { - // Second attempt: Handle incomplete TRANSCRIPT tags (missing closing tag) + // Handle incomplete TRANSCRIPT tags (missing closing tag) const openingMatch = text.match(/\s*([\s\S]*)/); if (openingMatch) { text = openingMatch[1]; @@ -212,8 +210,44 @@ export class TranscriptionService implements ITranscriptionProvider { // Remove TRANSCRIPT opening tag if still present (for cases where it's not properly extracted) text = text.replace(/<\/?TRANSCRIPT[^>]*>/g, ''); - // Remove specific meta instruction patterns (both at line start and anywhere in text) - const metaPatterns = [ + // Apply language-specific cleaning + text = this.applyLanguageSpecificCleaning(text, language); + + // Language-agnostic final cleanup: extra whitespace and empty lines + text = text.trim(); + text = text.replace(/\n{3,}/g, '\n\n'); + text = text.replace(/^\s*\n/gm, ''); // Remove lines with only whitespace + text = text.trim(); + + return text; + } + + /** + * Apply language-specific cleaning patterns + */ + private applyLanguageSpecificCleaning(text: string, language: string): string { + switch (language) { + case 'ja': + return this.cleanJapaneseMetaText(text); + case 'en': + return this.cleanEnglishMetaText(text); + case 'zh': + case 'zh-CN': + case 'zh-TW': + return this.cleanChineseMetaText(text); + case 'ko': + return this.cleanKoreanMetaText(text); + default: + // For unknown languages, only apply basic meta text removal + return this.cleanGenericMetaText(text); + } + } + + /** + * Clean Japanese-specific meta text patterns + */ + private cleanJapaneseMetaText(text: string): string { + const japaneseMetaPatterns = [ /^以下の音声内容.*?$/gm, /^この指示文.*?$/gm, /^話者の発言内容だけを正確に記録してください.*?$/gm, @@ -222,20 +256,118 @@ export class TranscriptionService implements ITranscriptionProvider { /（話者の発言のみ）/g, // Remove this specific phrase anywhere in the text ]; - // Apply all cleaning patterns - for (const pattern of metaPatterns) { + // Apply all Japanese cleaning patterns + for (const pattern of japaneseMetaPatterns) { text = text.replace(pattern, ''); } - // Clean up extra whitespace and empty lines - text = text.trim(); - text = text.replace(/\n{3,}/g, '\n\n'); - text = text.replace(/^\s*\n/gm, ''); // Remove lines with only whitespace - text = text.trim(); + return text; + } + + /** + * Clean English-specific meta text patterns + */ + private cleanEnglishMetaText(text: string): string { + const englishMetaPatterns = [ + /^Transcribe the following audio content.*?$/gm, + /^Please transcribe only the speaker's words.*?$/gm, + /^Output format.*?$/gm, + /\(speaker's words only\)/gi, + ]; + + for (const pattern of englishMetaPatterns) { + text = text.replace(pattern, ''); + } return text; } + /** + * Clean Chinese-specific meta text patterns + */ + private cleanChineseMetaText(text: string): string { + const chineseMetaPatterns = [ + /^请转录以下音频内容.*?$/gm, + /^请只记录说话者的发言内容.*?$/gm, + /^输出格式.*?$/gm, + /（仅说话者发言）/g, + ]; + + for (const pattern of chineseMetaPatterns) { + text = text.replace(pattern, ''); + } + + return text; + } + + /** + * Clean Korean-specific meta text patterns + */ + private cleanKoreanMetaText(text: string): string { + const koreanMetaPatterns = [ + /^다음 음성 내용을 전사해주세요.*?$/gm, + /^화자의 발언 내용만 정확히 기록해주세요.*?$/gm, + /^출력 형식.*?$/gm, + /\(화자 발언만\)/g, + ]; + + for (const pattern of koreanMetaPatterns) { + text = text.replace(pattern, ''); + } + + return text; + } + + /** + * Clean generic meta text patterns for unknown languages + */ + private cleanGenericMetaText(text: string): string { + // Only remove very basic instruction patterns that are likely meta-text + const genericMetaPatterns = [ + /^Output format.*?$/gm, + /^Format.*?$/gm, + ]; + + for (const pattern of genericMetaPatterns) { + text = text.replace(pattern, ''); + } + + return text; + } + + /** + * Detect if the response contains prompt error patterns (when audio is silent) + */ + private isPromptErrorDetected(text: string, language: string): boolean { + switch (language) { + case 'ja': + return text.includes('この指示文は出力に含めないでください') || + text.includes('話者の発言内容だけを正確に記録してください') || + text === '（話者の発言のみ）' || + text.trim() === '話者の発言のみ'; + case 'en': + return text.includes('Do not include this instruction in the output') || + text.includes('Please transcribe only the speaker\'s words') || + text === '(speaker\'s words only)' || + text.trim().toLowerCase() === 'speaker\'s words only'; + case 'zh': + case 'zh-CN': + case 'zh-TW': + return text.includes('请不要在输出中包含此指示') || + text.includes('请只记录说话者的发言内容') || + text === '（仅说话者发言）' || + text.trim() === '仅说话者发言'; + case 'ko': + return text.includes('이 지시사항을 출력에 포함하지 마세요') || + text.includes('화자의 발언 내용만 정확히 기록해주세요') || + text === '(화자 발언만)' || + text.trim() === '화자 발언만'; + default: + // For unknown languages, use a conservative approach + return false; + } + } + /** * Estimate transcription cost */ diff --git a/src/i18n/en.ts b/src/i18n/en.ts index 9049b18..6b9108f 100644 --- a/src/i18n/en.ts +++ b/src/i18n/en.ts @@ -140,7 +140,7 @@ export const en: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input Settings', + settings: 'Voice Input Settings' }, settings: { apiKey: 'OpenAI API Key', @@ -182,6 +182,6 @@ export const en: TranslationResource = { fromMultiple: 'From (comma-separated)', to: 'To', context: 'Context Keywords' - }, + } } }; diff --git a/src/i18n/ja.ts b/src/i18n/ja.ts index 0ae49c2..710ece2 100644 --- a/src/i18n/ja.ts +++ b/src/i18n/ja.ts @@ -140,7 +140,7 @@ export const ja: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 設定', + settings: 'Voice Input 設定' }, settings: { apiKey: 'OpenAI API キー', @@ -182,6 +182,6 @@ export const ja: TranslationResource = { fromMultiple: '入力語（カンマ区切り）', to: '修正語', context: '文脈キーワード' - }, + } } }; diff --git a/src/i18n/ko.ts b/src/i18n/ko.ts index a52ab9a..6433092 100644 --- a/src/i18n/ko.ts +++ b/src/i18n/ko.ts @@ -140,7 +140,7 @@ export const ko: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 설정', + settings: 'Voice Input 설정' }, settings: { apiKey: 'OpenAI API 키', @@ -182,6 +182,6 @@ export const ko: TranslationResource = { fromMultiple: '입력어（쉼표구분）', to: '교정어', context: '문맥 키워드' - }, + } } }; diff --git a/src/i18n/zh.ts b/src/i18n/zh.ts index 724ac1a..20cee1e 100644 --- a/src/i18n/zh.ts +++ b/src/i18n/zh.ts @@ -140,7 +140,7 @@ export const zh: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 设置', + settings: 'Voice Input 设置' }, settings: { apiKey: 'OpenAI API 密钥', @@ -182,6 +182,6 @@ export const zh: TranslationResource = { fromMultiple: '输入词（逗号分隔）', to: '校正词', context: '语境关键词' - }, + } } }; From 2d58043eb9f39985a06e545002509aff79eaaf8c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:45:38 +0000 Subject: [PATCH 09/82] Address PR feedback: revert jest config changes and improve language handling - Reverted jest.config.js to original state (removed non-existent moduleNameMapper entries) - Added missing newline to end of jest.config.js (eol-last) - Enhanced language code normalization to handle regional codes (en-US, zh-CN) via split('-')[0] - Made generic cleaning patterns safer by requiring colon-terminated patterns - Applied normalization consistently in both cleaning and error detection methods Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- jest.config.js | 5 +---- .../transcription/TranscriptionService.ts | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/jest.config.js b/jest.config.js index 443f427..492e94b 100644 --- a/jest.config.js +++ b/jest.config.js @@ -24,8 +24,5 @@ module.exports = { testTimeout: 10000, moduleNameMapper: { '^obsidian$': '/tests/mocks/obsidian.ts', - '^../src/core/transcription/DictionaryCorrector$': '/tests/mocks/DictionaryCorrector.ts', - '^../src/utils/ObsidianHttpClient$': '/tests/mocks/ObsidianHttpClient.ts', - '^../src/services$': '/tests/mocks/services.ts', }, -}; \ No newline at end of file +}; diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 250562a..5ceff78 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -226,14 +226,15 @@ export class TranscriptionService implements ITranscriptionProvider { * Apply language-specific cleaning patterns */ private applyLanguageSpecificCleaning(text: string, language: string): string { - switch (language) { + // Normalize language code (handle cases like 'en-US', 'zh-CN') + const normalizedLang = language.toLowerCase().split('-')[0]; + + switch (normalizedLang) { case 'ja': return this.cleanJapaneseMetaText(text); case 'en': return this.cleanEnglishMetaText(text); case 'zh': - case 'zh-CN': - case 'zh-TW': return this.cleanChineseMetaText(text); case 'ko': return this.cleanKoreanMetaText(text); @@ -323,9 +324,10 @@ export class TranscriptionService implements ITranscriptionProvider { */ private cleanGenericMetaText(text: string): string { // Only remove very basic instruction patterns that are likely meta-text + // Using colon-terminated patterns to be more conservative const genericMetaPatterns = [ - /^Output format.*?$/gm, - /^Format.*?$/gm, + /^Output\s*format\s*:.*/gmi, + /^Format\s*:.*/gmi, ]; for (const pattern of genericMetaPatterns) { @@ -339,7 +341,10 @@ export class TranscriptionService implements ITranscriptionProvider { * Detect if the response contains prompt error patterns (when audio is silent) */ private isPromptErrorDetected(text: string, language: string): boolean { - switch (language) { + // Normalize language code (handle cases like 'en-US', 'zh-CN') + const normalizedLang = language.toLowerCase().split('-')[0]; + + switch (normalizedLang) { case 'ja': return text.includes('この指示文は出力に含めないでください') || text.includes('話者の発言内容だけを正確に記録してください') || @@ -351,8 +356,6 @@ export class TranscriptionService implements ITranscriptionProvider { text === '(speaker\'s words only)' || text.trim().toLowerCase() === 'speaker\'s words only'; case 'zh': - case 'zh-CN': - case 'zh-TW': return text.includes('请不要在输出中包含此指示') || text.includes('请只记录说话者的发言内容') || text === '（仅说话者发言）' || From 5684cd37ced0629bd82738d17a400a69ea7718b4 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 09:08:41 +0000 Subject: [PATCH 10/82] Initial plan From 1917723d34b0206ba7b99c4fe45d01f6c622172a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 09:13:29 +0000 Subject: [PATCH 11/82] Initial setup: Fix linting errors and create test infrastructure Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- src/i18n/en.ts | 4 ++-- src/i18n/ja.ts | 4 ++-- src/i18n/ko.ts | 4 ++-- src/i18n/zh.ts | 4 ++-- tests/mocks/obsidian.ts | 42 +++++++++++++++++++++++++++++++++++++++++ tests/setup.ts | 10 ++++++++++ 6 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 tests/mocks/obsidian.ts create mode 100644 tests/setup.ts diff --git a/src/i18n/en.ts b/src/i18n/en.ts index 9049b18..6b9108f 100644 --- a/src/i18n/en.ts +++ b/src/i18n/en.ts @@ -140,7 +140,7 @@ export const en: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input Settings', + settings: 'Voice Input Settings' }, settings: { apiKey: 'OpenAI API Key', @@ -182,6 +182,6 @@ export const en: TranslationResource = { fromMultiple: 'From (comma-separated)', to: 'To', context: 'Context Keywords' - }, + } } }; diff --git a/src/i18n/ja.ts b/src/i18n/ja.ts index 0ae49c2..710ece2 100644 --- a/src/i18n/ja.ts +++ b/src/i18n/ja.ts @@ -140,7 +140,7 @@ export const ja: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 設定', + settings: 'Voice Input 設定' }, settings: { apiKey: 'OpenAI API キー', @@ -182,6 +182,6 @@ export const ja: TranslationResource = { fromMultiple: '入力語（カンマ区切り）', to: '修正語', context: '文脈キーワード' - }, + } } }; diff --git a/src/i18n/ko.ts b/src/i18n/ko.ts index a52ab9a..6433092 100644 --- a/src/i18n/ko.ts +++ b/src/i18n/ko.ts @@ -140,7 +140,7 @@ export const ko: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 설정', + settings: 'Voice Input 설정' }, settings: { apiKey: 'OpenAI API 키', @@ -182,6 +182,6 @@ export const ko: TranslationResource = { fromMultiple: '입력어（쉼표구분）', to: '교정어', context: '문맥 키워드' - }, + } } }; diff --git a/src/i18n/zh.ts b/src/i18n/zh.ts index 724ac1a..20cee1e 100644 --- a/src/i18n/zh.ts +++ b/src/i18n/zh.ts @@ -140,7 +140,7 @@ export const zh: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 设置', + settings: 'Voice Input 设置' }, settings: { apiKey: 'OpenAI API 密钥', @@ -182,6 +182,6 @@ export const zh: TranslationResource = { fromMultiple: '输入词（逗号分隔）', to: '校正词', context: '语境关键词' - }, + } } }; diff --git a/tests/mocks/obsidian.ts b/tests/mocks/obsidian.ts new file mode 100644 index 0000000..5bb6a8f --- /dev/null +++ b/tests/mocks/obsidian.ts @@ -0,0 +1,42 @@ +// Mock Obsidian API for tests +export class TFile { + constructor(public path: string) {} +} + +export class Vault { + async read(): Promise { + return ''; + } + + async create(): Promise { + return new TFile('mock.md'); + } +} + +export class App { + vault = new Vault(); +} + +export class Plugin { + app = new App(); + + async loadData(): Promise { + return null; + } + + async saveData(): Promise { + // Mock implementation + } +} + +export class Component { + // Mock component +} + +export class ItemView { + // Mock item view +} + +export class WorkspaceLeaf { + // Mock workspace leaf +} \ No newline at end of file diff --git a/tests/setup.ts b/tests/setup.ts new file mode 100644 index 0000000..f61b152 --- /dev/null +++ b/tests/setup.ts @@ -0,0 +1,10 @@ +// Jest setup file +import 'jest-environment-jsdom'; + +// Global test configuration +global.console = { + ...console, + // Override console methods if needed for tests +}; + +// Mock implementations can go here \ No newline at end of file From 9b3799dc45aa196789b9a06c402e34b89bc39151 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 09:22:39 +0000 Subject: [PATCH 12/82] Implement post-#14 multilingual improvements: enable multilingual correction, language branching, and locale detection Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- README.md | 26 ++- .../transcription/TranscriptionService.ts | 15 +- src/plugin/VoiceInputPlugin.ts | 27 ++- .../transcription-simple.test.ts | 214 ++++++++++++++++++ 4 files changed, 264 insertions(+), 18 deletions(-) create mode 100644 tests/unit/core/transcription/transcription-simple.test.ts diff --git a/README.md b/README.md index 2f38496..347d8af 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,10 @@ High-accuracy voice input for Obsidian. Uses OpenAI GPT-4o Audio Transcriptions - One‑click recording: start/stop from a microphone ribbon icon - Push‑to‑talk: long‑press to record, release to stop - Model selection: GPT‑4o Transcribe or GPT‑4o mini Transcribe -- AI post‑processing: optional dictionary-based cleanup (JA) +- AI post‑processing: optional dictionary-based cleanup (multilingual support) - Quick controls in view: copy/clear/insert at cursor/append to end - Auto‑save drafts: periodic and on blur, automatic restore +- Multilingual support: Japanese, English, Chinese, Korean interface languages ## Requirements @@ -47,9 +48,15 @@ Tip: A settings gear in the view header opens the plugin’s settings. - OpenAI API Key: stored locally (encrypted at rest) - Transcription Model: `gpt-4o-transcribe` or `gpt-4o-mini-transcribe` -- AI Post‑processing: enable dictionary‑based cleanup (Japanese) +- AI Post‑processing: enable dictionary‑based cleanup (supports all languages when enabled) - Maximum Recording Duration: slider (default 5 min) -- Plugin Language: English/Japanese (auto‑detected from Obsidian, adjustable) +- Plugin Language: English/Japanese/Chinese/Korean (auto‑detected from Obsidian, adjustable) + +### Language-specific Behavior + +- **Japanese (ja)**: Enhanced prompts and meta-pattern cleaning for optimal transcription accuracy +- **English/Chinese/Korean (en/zh/ko)**: Clean transcription without language-specific prompts to avoid interference +- **Dictionary Correction**: Applied to all languages when enabled (not limited to Japanese) ## Security & Privacy @@ -85,9 +92,10 @@ Third‑party licensing: see `THIRD_PARTY_LICENSES.md`. - ワンクリック録音（リボンのマイクアイコン） - プッシュトゥトーク（長押しで録音開始、離して停止） - モデル選択（GPT‑4o Transcribe / GPT‑4o mini Transcribe） -- AI後処理（辞書ベースの補正、JA向け） +- AI後処理（辞書ベースの補正、多言語対応） - ビュー内のクイック操作（コピー/クリア/カーソル位置へ挿入/末尾へ追記） - 自動保存（定期保存とフォーカス外れ時）。再オープン時に自動復元 +- 多言語サポート（日本語、英語、中国語、韓国語のインターフェース） ## 必要条件 @@ -125,9 +133,15 @@ Third‑party licensing: see `THIRD_PARTY_LICENSES.md`. - OpenAI APIキー: ローカルに暗号化して保存 - 文字起こしモデル: `gpt-4o-transcribe` または `gpt-4o-mini-transcribe` -- AI後処理: 辞書ベースの補正（日本語向け） +- AI後処理: 辞書ベースの補正（有効時は全言語で適用） - 最大録音時間: スライダー（初期値5分） -- プラグイン言語: 英語/日本語（Obsidian設定から自動検出、変更可） +- プラグイン言語: 英語/日本語/中国語/韓国語（Obsidian設定から自動検出、変更可） + +### 言語別の動作 + +- **日本語 (ja)**: 高精度化のための専用プロンプトとメタパターン除去 +- **英語/中国語/韓国語 (en/zh/ko)**: 干渉を避けるため言語固有プロンプトを使用しないクリーンな文字起こし +- **辞書補正**: 有効時は全言語に適用（日本語限定ではありません） ## セキュリティ / プライバシー diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 68d3587..9a47c63 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -55,8 +55,8 @@ export class TranscriptionService implements ITranscriptionProvider { } // Build prompt for transcription - const prompt = this.buildTranscriptionPrompt(); - if (prompt) { + const prompt = this.buildTranscriptionPrompt(language); + if (language !== 'auto' && prompt) { formData.append('prompt', prompt); } @@ -143,8 +143,8 @@ export class TranscriptionService implements ITranscriptionProvider { }; } - // Apply corrections if enabled (only for Japanese) - const correctedText = this.enableTranscriptionCorrection && language === 'ja' + // Apply corrections if enabled + const correctedText = this.enableTranscriptionCorrection ? await this.corrector.correct(originalText) : originalText; @@ -183,7 +183,12 @@ export class TranscriptionService implements ITranscriptionProvider { /** * Build prompt for GPT-4o transcription */ - private buildTranscriptionPrompt(): string { + private buildTranscriptionPrompt(language: string): string { + // Only provide prompt for Japanese language + if (language !== 'ja') { + return ''; + } + return `以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。 話者の発言内容だけを正確に記録してください。 diff --git a/src/plugin/VoiceInputPlugin.ts b/src/plugin/VoiceInputPlugin.ts index 7463fee..1c94f0d 100644 --- a/src/plugin/VoiceInputPlugin.ts +++ b/src/plugin/VoiceInputPlugin.ts @@ -223,9 +223,13 @@ export default class VoiceInputPlugin extends Plugin { // languageからpluginLanguageへの移行 if ('language' in data && !('pluginLanguage' in data)) { - // 言語コードを正規化（ja → ja、en → en、その他 → en） + // 言語コードを正規化（ja → ja、en → en、zh → zh、ko → ko、その他 → en） const langCode = data.language as string; - migratedData.pluginLanguage = (langCode === 'ja' || langCode === 'en') ? langCode : 'en'; + if (langCode === 'ja' || langCode === 'en' || langCode === 'zh' || langCode === 'ko') { + migratedData.pluginLanguage = langCode; + } else { + migratedData.pluginLanguage = 'en'; + } delete migratedData.language; needsSave = true; this.logger?.info(`Migrating language (${data.language}) to pluginLanguage (${migratedData.pluginLanguage})`); @@ -298,15 +302,13 @@ export default class VoiceInputPlugin extends Plugin { // pluginLanguageが設定されていない場合 if (!hasSettingsKey(data, 'pluginLanguage') && !hasSettingsKey(data, 'interfaceLanguage')) { - const obsidianLocale = this.getObsidianLocale(); - this.settings.pluginLanguage = obsidianLocale.startsWith('ja') ? 'ja' : 'en'; + this.settings.pluginLanguage = this.detectPluginLanguage(); needsSave = true; - this.logger?.info(`Auto-detected language: ${this.settings.pluginLanguage} (from Obsidian: ${obsidianLocale})`); + this.logger?.info(`Auto-detected language: ${this.settings.pluginLanguage} (from Obsidian: ${this.getObsidianLocale()})`); } } else { // 保存データが存在しない場合（初回起動） - const obsidianLocale = this.getObsidianLocale(); - this.settings.pluginLanguage = obsidianLocale.startsWith('ja') ? 'ja' : 'en'; + this.settings.pluginLanguage = this.detectPluginLanguage(); needsSave = true; this.logger?.info(`First run - auto-detected language: ${this.settings.pluginLanguage}`); } @@ -333,6 +335,17 @@ export default class VoiceInputPlugin extends Plugin { return getObsidianLocale(this.app); } + /** + * Detect plugin language from Obsidian locale + */ + private detectPluginLanguage(): 'ja' | 'en' | 'zh' | 'ko' { + const obsidianLocale = this.getObsidianLocale().toLowerCase(); + if (obsidianLocale.startsWith('ja')) return 'ja'; + if (obsidianLocale.startsWith('zh')) return 'zh'; + if (obsidianLocale.startsWith('ko')) return 'ko'; + return 'en'; + } + async saveSettings() { // APIキーを暗号化して保存 const dataToSave = { diff --git a/tests/unit/core/transcription/transcription-simple.test.ts b/tests/unit/core/transcription/transcription-simple.test.ts new file mode 100644 index 0000000..f2f12a9 --- /dev/null +++ b/tests/unit/core/transcription/transcription-simple.test.ts @@ -0,0 +1,214 @@ +/** + * Simple tests that verify the core transcription cleaning logic + * without complex service dependencies + */ + +describe('Transcription Cleaning Logic', () => { + + // Helper function that replicates the cleaning logic from TranscriptionService + function cleanGPT4oResponse(text: string): string { + // First attempt: Extract content from complete TRANSCRIPT tags + let transcriptMatch = text.match(/\s*([\s\S]*?)\s*<\/TRANSCRIPT>/); + if (transcriptMatch) { + text = transcriptMatch[1]; + } else { + // Second attempt: Handle incomplete TRANSCRIPT tags (missing closing tag) + const openingMatch = text.match(/\s*([\s\S]*)/); + if (openingMatch) { + text = openingMatch[1]; + } + } + + // Remove TRANSCRIPT opening tag if still present + text = text.replace(/<\/?TRANSCRIPT[^>]*>/g, ''); + + // Remove specific meta instruction patterns + const metaPatterns = [ + /^以下の音声内容.*?$/gm, + /^この指示文.*?$/gm, + /^話者の発言内容だけを正確に記録してください.*?$/gm, + /^話者の発言.*?$/gm, + /^出力形式.*?$/gm, + /（話者の発言のみ）/g, + ]; + + // Apply all cleaning patterns + for (const pattern of metaPatterns) { + text = text.replace(pattern, ''); + } + + // Clean up extra whitespace and empty lines + text = text.trim(); + text = text.replace(/\n{3,}/g, '\n\n'); + text = text.replace(/^\s*\n/gm, ''); + text = text.trim(); + + return text; + } + + // Helper function for prompt building logic + function buildTranscriptionPrompt(language: string): string { + // Only provide prompt for Japanese language + if (language !== 'ja') { + return ''; + } + + return `以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。 +話者の発言内容だけを正確に記録してください。 + +出力形式: + +（話者の発言のみ） +`; + } + + // Helper function for locale detection logic + function detectPluginLanguage(obsidianLocale: string): 'ja' | 'en' | 'zh' | 'ko' { + const locale = obsidianLocale.toLowerCase(); + if (locale.startsWith('ja')) return 'ja'; + if (locale.startsWith('zh')) return 'zh'; + if (locale.startsWith('ko')) return 'ko'; + return 'en'; + } + + describe('TRANSCRIPT tag extraction', () => { + it('should extract content from complete TRANSCRIPT tags', () => { + const input = `Some prefix text + +実際の発言内容 + +Some suffix text`; + const result = cleanGPT4oResponse(input); + expect(result).toContain('実際の発言内容'); + expect(result).not.toContain(''); + expect(result).not.toContain(''); + }); + + it('should handle incomplete TRANSCRIPT tags (missing closing tag)', () => { + const input = `Some prefix text + +実際の発言内容 +Additional content`; + const result = cleanGPT4oResponse(input); + expect(result).toContain('実際の発言内容'); + expect(result).toContain('Additional content'); + expect(result).not.toContain(''); + }); + }); + + describe('Meta instruction pattern removal', () => { + it('should remove Japanese meta instructions', () => { + const patterns = [ + '以下の音声内容のみを文字に起こしてください', + 'この指示文は出力に含めないでください', + '話者の発言内容だけを正確に記録してください', + '出力形式:' + ]; + + patterns.forEach(pattern => { + const input = `${pattern}\n実際の発言\nその他の内容`; + const result = cleanGPT4oResponse(input); + expect(result).not.toContain(pattern); + expect(result).toContain('実際の発言'); + expect(result).toContain('その他の内容'); + }); + }); + + it('should remove meta phrases anywhere in text', () => { + const input = '前の文章 （話者の発言のみ） 後の文章'; + const result = cleanGPT4oResponse(input); + expect(result).not.toContain('（話者の発言のみ）'); + expect(result).toContain('前の文章'); + expect(result).toContain('後の文章'); + }); + + it('should preserve normal content that might contain similar words', () => { + const input = '今日は出力について話しました。発言が重要です。'; + const result = cleanGPT4oResponse(input); + expect(result).toContain('今日は出力について話しました'); + expect(result).toContain('発言が重要です'); + }); + }); + + describe('buildTranscriptionPrompt language branching', () => { + it('should return Japanese prompt for ja language', () => { + const result = buildTranscriptionPrompt('ja'); + expect(result).toContain('以下の音声内容のみを文字に起こしてください'); + expect(result).toContain(''); + expect(result).toContain('（話者の発言のみ）'); + expect(result.length).toBeGreaterThan(0); + }); + + it('should return empty string for non-ja languages', () => { + const languages = ['en', 'zh', 'ko', 'auto', 'fr']; + languages.forEach(lang => { + expect(buildTranscriptionPrompt(lang)).toBe(''); + }); + }); + }); + + describe('Locale detection for multilingual support', () => { + it('should detect Japanese locale variants', () => { + const jaLocales = ['ja', 'ja-JP', 'ja_JP', 'JA', 'JA-JP']; + jaLocales.forEach(locale => { + expect(detectPluginLanguage(locale)).toBe('ja'); + }); + }); + + it('should detect Chinese locale variants', () => { + const zhLocales = ['zh', 'zh-CN', 'zh-TW', 'zh_CN', 'ZH', 'zh-Hans', 'zh-Hant']; + zhLocales.forEach(locale => { + expect(detectPluginLanguage(locale)).toBe('zh'); + }); + }); + + it('should detect Korean locale variants', () => { + const koLocales = ['ko', 'ko-KR', 'ko_KR', 'KO', 'KO-KR']; + koLocales.forEach(locale => { + expect(detectPluginLanguage(locale)).toBe('ko'); + }); + }); + + it('should default to English for other locales', () => { + const otherLocales = ['en', 'en-US', 'en-GB', 'fr', 'de', 'es', 'it', 'ru', 'unknown', '']; + otherLocales.forEach(locale => { + expect(detectPluginLanguage(locale)).toBe('en'); + }); + }); + }); + + describe('Error detection patterns', () => { + function isPromptErrorDetected(text: string): boolean { + return text.includes('この指示文は出力に含めないでください') || + text.includes('話者の発言内容だけを正確に記録してください') || + text === '（話者の発言のみ）' || + text.trim() === '話者の発言のみ'; + } + + it('should detect prompt error patterns', () => { + const errorPatterns = [ + 'この指示文は出力に含めないでください', + '話者の発言内容だけを正確に記録してください', + '（話者の発言のみ）', + '話者の発言のみ' + ]; + + errorPatterns.forEach(pattern => { + expect(isPromptErrorDetected(pattern)).toBe(true); + }); + }); + + it('should not detect normal content as errors', () => { + const normalContent = [ + 'こんにちは、今日はいい天気ですね', + 'Meeting started at 2 PM', + '会議の議事録を記録します', + 'The speaker mentioned important points' + ]; + + normalContent.forEach(content => { + expect(isPromptErrorDetected(content)).toBe(false); + }); + }); + }); +}); \ No newline at end of file From b24d3afeb3c748224eb5fc003e817c4965c801ca Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 09:42:44 +0000 Subject: [PATCH 13/82] Initial analysis and plan for multilingual improvements Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- package-lock.json | 4 ++-- package.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index a9f9d16..22a35d1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,7 @@ "ts-jest": "^29.4.0" }, "devDependencies": { - "@types/node": "^16.11.6", + "@types/node": "^16.18.126", "@typescript-eslint/eslint-plugin": "^5.29.0", "@typescript-eslint/parser": "^5.29.0", "builtin-modules": "3.3.0", @@ -23,7 +23,7 @@ "eslint": "^8.57.1", "jest-environment-jsdom": "^30.0.5", "obsidian": "latest", - "tslib": "2.4.0", + "tslib": "^2.4.0", "typescript": "4.7.4" } }, diff --git a/package.json b/package.json index 3a5cc2a..6daec67 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "author": "Musashino Software", "license": "MIT", "devDependencies": { - "@types/node": "^16.11.6", + "@types/node": "^16.18.126", "@typescript-eslint/eslint-plugin": "^5.29.0", "@typescript-eslint/parser": "^5.29.0", "builtin-modules": "3.3.0", @@ -39,7 +39,7 @@ "eslint": "^8.57.1", "jest-environment-jsdom": "^30.0.5", "obsidian": "latest", - "tslib": "2.4.0", + "tslib": "^2.4.0", "typescript": "4.7.4" }, "dependencies": { From 1a40dc1d963026f8667edfef06abaac35c837284 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 09:56:29 +0000 Subject: [PATCH 14/82] Implement core multilingual improvements: settings separation, UI, service unification, and i18n fixes Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- src/config/defaults.ts | 4 ---- src/core/transcription/TranscriptionService.ts | 11 ++++++----- src/i18n/en.ts | 12 ++++++++++-- src/i18n/ja.ts | 12 ++++++++++-- src/i18n/ko.ts | 12 ++++++++++-- src/i18n/zh.ts | 12 ++++++++++-- src/interfaces/i18n.ts | 8 ++++++++ src/interfaces/settings.ts | 8 +++++--- src/plugin/VoiceInputPlugin.ts | 11 +++++++++++ src/settings/VoiceInputSettingTab.ts | 16 ++++++++++++++++ src/views/VoiceInputViewActions.ts | 4 ++-- 11 files changed, 88 insertions(+), 22 deletions(-) diff --git a/src/config/defaults.ts b/src/config/defaults.ts index 264ce92..5d268d3 100644 --- a/src/config/defaults.ts +++ b/src/config/defaults.ts @@ -70,9 +70,6 @@ export const DEFAULT_TRANSCRIPTION_SETTINGS = { /** デフォルトの文字起こしモデル */ model: 'gpt-4o-transcribe' as const, - /** 言語設定 */ - language: 'ja', - /** 文字起こし補正の有効化 */ enableTranscriptionCorrection: true } as const; @@ -83,7 +80,6 @@ export const DEFAULT_TRANSCRIPTION_SETTINGS = { export const DEFAULT_USER_SETTINGS = { /** 基本設定 */ openaiApiKey: '', - language: DEFAULT_TRANSCRIPTION_SETTINGS.language, enableTranscriptionCorrection: DEFAULT_TRANSCRIPTION_SETTINGS.enableTranscriptionCorrection, /** 音声設定 */ diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 9a47c63..29802c0 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -15,7 +15,6 @@ export class TranscriptionService implements ITranscriptionProvider { // コスト重視なら mini、わずかな精度向上が必要なら通常版を選択 private model: 'gpt-4o-transcribe' | 'gpt-4o-mini-transcribe' = DEFAULT_TRANSCRIPTION_SETTINGS.model; private enableTranscriptionCorrection: boolean = DEFAULT_TRANSCRIPTION_SETTINGS.enableTranscriptionCorrection; - private language: string = 'ja'; constructor(apiKey: string, dictionary?: SimpleCorrectionDictionary) { this.apiKey = apiKey; @@ -25,11 +24,11 @@ export class TranscriptionService implements ITranscriptionProvider { }); } - async transcribe(audioBlob: Blob, language: string = 'ja'): Promise { + async transcribe(audioBlob: Blob, language: string): Promise { return this.transcribeAudio(audioBlob, language); } - async transcribeAudio(audioBlob: Blob, language: string = 'ja'): Promise { + async transcribeAudio(audioBlob: Blob, language: string): Promise { const startTime = Date.now(); const perfStartTime = performance.now(); @@ -143,8 +142,10 @@ export class TranscriptionService implements ITranscriptionProvider { }; } - // Apply corrections if enabled - const correctedText = this.enableTranscriptionCorrection + // Apply corrections if enabled and for supported languages + // Dictionary correction is most effective for Japanese and Chinese + const supportsCorrectionForLanguage = ['ja', 'zh'].includes(language); + const correctedText = this.enableTranscriptionCorrection && supportsCorrectionForLanguage ? await this.corrector.correct(originalText) : originalText; diff --git a/src/i18n/en.ts b/src/i18n/en.ts index 6b9108f..a12eb10 100644 --- a/src/i18n/en.ts +++ b/src/i18n/en.ts @@ -60,7 +60,8 @@ export const en: TranslationResource = { processing: { transcribing: 'Status: Transcribing...', correcting: 'Status: Correcting...', - completed: 'Status: Completed' + completed: 'Status: Completed', + waiting: 'waiting' }, transcription: { vadAutoStopped: 'Status: Auto-stopped due to silence', @@ -153,6 +154,8 @@ export const en: TranslationResource = { transcriptionModelDesc: 'Select the model for voice recognition', maxRecordingDuration: 'Max Recording Duration', maxRecordingDurationDesc: 'Maximum recording time in seconds ({min}s - {max}min)', + language: 'Voice Recognition Language', + languageDesc: 'Language for voice recognition and transcription. Auto-detection is recommended for best results.', pluginLanguage: 'Plugin Language', pluginLanguageDesc: 'Set language for UI, voice processing, and correction dictionary', customDictionary: 'Custom Dictionary', @@ -163,7 +166,12 @@ export const en: TranslationResource = { }, options: { modelMini: 'GPT-4o mini Transcribe', - modelFull: 'GPT-4o Transcribe' + modelFull: 'GPT-4o Transcribe', + languageAuto: 'Auto (Recommended)', + languageJa: 'Japanese', + languageEn: 'English', + languageZh: 'Chinese', + languageKo: 'Korean' }, tooltips: { copy: 'Copy to clipboard', diff --git a/src/i18n/ja.ts b/src/i18n/ja.ts index 710ece2..a810630 100644 --- a/src/i18n/ja.ts +++ b/src/i18n/ja.ts @@ -60,7 +60,8 @@ export const ja: TranslationResource = { processing: { transcribing: 'ステータス: 文字起こし中...', correcting: 'ステータス: 清書中...', - completed: 'ステータス: 完了' + completed: 'ステータス: 完了', + waiting: '待機中' }, transcription: { vadAutoStopped: 'ステータス: 無音検出により自動停止しました', @@ -153,6 +154,8 @@ export const ja: TranslationResource = { transcriptionModelDesc: '音声認識に使用するモデルを選択', maxRecordingDuration: '最大録音時間', maxRecordingDurationDesc: '最大録音時間（秒）（{min}秒〜{max}分）', + language: '音声認識言語', + languageDesc: '音声認識と文字起こしの言語。最適な結果のため自動検出を推奨します。', pluginLanguage: 'プラグイン言語', pluginLanguageDesc: 'UI表示、音声認識処理、補正辞書の言語を設定', customDictionary: 'カスタム辞書', @@ -163,7 +166,12 @@ export const ja: TranslationResource = { }, options: { modelMini: 'GPT-4o mini Transcribe', - modelFull: 'GPT-4o Transcribe' + modelFull: 'GPT-4o Transcribe', + languageAuto: '自動（推奨）', + languageJa: '日本語', + languageEn: '英語', + languageZh: '中国語', + languageKo: '韓国語' }, tooltips: { copy: 'クリップボードにコピー', diff --git a/src/i18n/ko.ts b/src/i18n/ko.ts index 6433092..384dec4 100644 --- a/src/i18n/ko.ts +++ b/src/i18n/ko.ts @@ -60,7 +60,8 @@ export const ko: TranslationResource = { processing: { transcribing: '상태: 음성 텍스트 변환 중...', correcting: '상태: 텍스트 정리 중...', - completed: '상태: 완료' + completed: '상태: 완료', + waiting: '대기 중' }, transcription: { vadAutoStopped: '상태: 무음 감지로 자동 중지됨', @@ -153,6 +154,8 @@ export const ko: TranslationResource = { transcriptionModelDesc: '음성 인식에 사용할 모델 선택', maxRecordingDuration: '최대 녹음 시간', maxRecordingDurationDesc: '최대 녹음 시간(초) ({min}초~{max}분)', + language: '음성 인식 언어', + languageDesc: '음성 인식 및 전사를 위한 언어입니다. 최상의 결과를 위해 자동 감지를 권장합니다.', pluginLanguage: '플러그인 언어', pluginLanguageDesc: 'UI 표시, 음성 처리 및 교정 사전의 언어 설정', customDictionary: '사용자 정의 사전', @@ -163,7 +166,12 @@ export const ko: TranslationResource = { }, options: { modelMini: 'GPT-4o mini Transcribe', - modelFull: 'GPT-4o Transcribe' + modelFull: 'GPT-4o Transcribe', + languageAuto: '자동 (권장)', + languageJa: '일본어', + languageEn: '영어', + languageZh: '중국어', + languageKo: '한국어' }, tooltips: { copy: '클립보드에 복사', diff --git a/src/i18n/zh.ts b/src/i18n/zh.ts index 20cee1e..e4797f3 100644 --- a/src/i18n/zh.ts +++ b/src/i18n/zh.ts @@ -60,7 +60,8 @@ export const zh: TranslationResource = { processing: { transcribing: '状态：语音转文字中...', correcting: '状态：文本处理中...', - completed: '状态：完成' + completed: '状态：完成', + waiting: '等待中' }, transcription: { vadAutoStopped: '状态：检测到静音自动停止', @@ -153,6 +154,8 @@ export const zh: TranslationResource = { transcriptionModelDesc: '选择用于语音识别的模型', maxRecordingDuration: '最大录音时长', maxRecordingDurationDesc: '最大录音时间（秒）（{min}秒～{max}分钟）', + language: '语音识别语言', + languageDesc: '语音识别和转录的语言。建议使用自动检测以获得最佳结果。', pluginLanguage: '插件语言', pluginLanguageDesc: '设置UI显示、语音处理和校正词典的语言', customDictionary: '自定义词典', @@ -163,7 +166,12 @@ export const zh: TranslationResource = { }, options: { modelMini: 'GPT-4o mini Transcribe', - modelFull: 'GPT-4o Transcribe' + modelFull: 'GPT-4o Transcribe', + languageAuto: '自动（推荐）', + languageJa: '日语', + languageEn: '英语', + languageZh: '中文', + languageKo: '韩语' }, tooltips: { copy: '复制到剪贴板', diff --git a/src/interfaces/i18n.ts b/src/interfaces/i18n.ts index 355a624..b7ad99a 100644 --- a/src/interfaces/i18n.ts +++ b/src/interfaces/i18n.ts @@ -80,6 +80,7 @@ export type TranslationResource = { transcribing: string; correcting: string; completed: string; + waiting: string; }; transcription: { vadAutoStopped: string; @@ -172,6 +173,8 @@ export type TranslationResource = { transcriptionModelDesc: string; maxRecordingDuration: string; maxRecordingDurationDesc: string; + language: string; + languageDesc: string; pluginLanguage: string; pluginLanguageDesc: string; customDictionary: string; @@ -183,6 +186,11 @@ export type TranslationResource = { options: { modelMini: string; modelFull: string; + languageAuto: string; + languageJa: string; + languageEn: string; + languageZh: string; + languageKo: string; }; tooltips: { copy: string; diff --git a/src/interfaces/settings.ts b/src/interfaces/settings.ts index 045221f..c4d139e 100644 --- a/src/interfaces/settings.ts +++ b/src/interfaces/settings.ts @@ -8,8 +8,9 @@ export interface VoiceInputSettings { transcriptionModel: 'gpt-4o-transcribe' | 'gpt-4o-mini-transcribe'; // 録音設定 maxRecordingSeconds: number; // 最大録音時間（秒） - // プラグイン言語設定 - pluginLanguage: Locale; // プラグイン全体の言語（UI、音声認識、補正辞書） + // 言語設定 + language: 'auto' | Locale; // 音声認識言語 + pluginLanguage: Locale; // プラグイン全体の言語（UI表示用） customDictionary: SimpleCorrectionDictionary; // デバッグ設定 debugMode: boolean; // デバッグモード @@ -22,7 +23,8 @@ export const DEFAULT_SETTINGS: VoiceInputSettings = { transcriptionModel: 'gpt-4o-transcribe', // 録音設定 maxRecordingSeconds: 300, // 5分（300秒） - // プラグイン言語はObsidianの設定に従う（loadSettingsで設定） + // 言語設定 + language: 'auto', // 音声認識言語のデフォルトは自動検出 pluginLanguage: 'en', // 初期値、実際はObsidianの設定に従う customDictionary: { definiteCorrections: [] }, // デバッグ設定 diff --git a/src/plugin/VoiceInputPlugin.ts b/src/plugin/VoiceInputPlugin.ts index 1c94f0d..e8dc247 100644 --- a/src/plugin/VoiceInputPlugin.ts +++ b/src/plugin/VoiceInputPlugin.ts @@ -346,6 +346,17 @@ export default class VoiceInputPlugin extends Plugin { return 'en'; } + /** + * Get resolved language for voice recognition + * Auto-detects from Obsidian locale when 'auto' is selected + */ + public getResolvedLanguage(): 'ja' | 'en' | 'zh' | 'ko' { + if (this.settings.language === 'auto') { + return this.detectPluginLanguage(); + } + return this.settings.language; + } + async saveSettings() { // APIキーを暗号化して保存 const dataToSave = { diff --git a/src/settings/VoiceInputSettingTab.ts b/src/settings/VoiceInputSettingTab.ts index 56d338e..f3179e5 100644 --- a/src/settings/VoiceInputSettingTab.ts +++ b/src/settings/VoiceInputSettingTab.ts @@ -73,6 +73,22 @@ export class VoiceInputSettingTab extends PluginSettingTab { this.display(); })); + // Voice Recognition Language Setting + new Setting(containerEl) + .setName(this.i18n.t('ui.settings.language')) + .setDesc(this.i18n.t('ui.settings.languageDesc')) + .addDropdown(dropdown => dropdown + .addOption('auto', this.i18n.t('ui.options.languageAuto')) + .addOption('ja', this.i18n.t('ui.options.languageJa')) + .addOption('en', this.i18n.t('ui.options.languageEn')) + .addOption('zh', this.i18n.t('ui.options.languageZh')) + .addOption('ko', this.i18n.t('ui.options.languageKo')) + .setValue(this.plugin.settings.language) + .onChange(async (value: 'auto' | Locale) => { + this.plugin.settings.language = value; + await this.plugin.saveSettings(); + })); + // OpenAI API Key const apiKeySetting = new Setting(containerEl) .setName(this.i18n.t('ui.settings.apiKey')) diff --git a/src/views/VoiceInputViewActions.ts b/src/views/VoiceInputViewActions.ts index 430f0fa..2af7816 100644 --- a/src/views/VoiceInputViewActions.ts +++ b/src/views/VoiceInputViewActions.ts @@ -398,7 +398,7 @@ export class VoiceInputViewActions { // When processing, the first item is being processed, others are waiting const waitingCount = queueLength > 0 ? queueLength - 1 : 0; if (waitingCount > 0) { - statusText += ` (${waitingCount} 待機中)`; + statusText += ` (${waitingCount} ${this.i18n.t('status.processing.waiting')})`; } this.view.ui.statusEl.setText(statusText); this.view.ui.statusEl.addClass('processing'); @@ -455,7 +455,7 @@ export class VoiceInputViewActions { if (!this.transcriptionService) { throw new Error('Transcription service not available'); } - const result = await this.transcriptionService.transcribeAudio(audioBlob, this.plugin.settings.pluginLanguage); + const result = await this.transcriptionService.transcribeAudio(audioBlob, this.plugin.getResolvedLanguage()); // Check if result is empty if (!result.text || result.text.trim() === '') { From 24d8c546821a495ca0ee734a6cdba05c62f60312 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:00:20 +0000 Subject: [PATCH 15/82] Complete multilingual improvements: update README documentation for language settings and behavior Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- README.md | 46 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 347d8af..4762035 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,9 @@ High-accuracy voice input for Obsidian. Uses OpenAI GPT-4o Audio Transcriptions - One‑click recording: start/stop from a microphone ribbon icon - Push‑to‑talk: long‑press to record, release to stop - Model selection: GPT‑4o Transcribe or GPT‑4o mini Transcribe -- AI post‑processing: optional dictionary-based cleanup (multilingual support) +- Language separation: independent UI language and voice recognition language settings +- Auto language detection: automatic voice recognition language based on Obsidian locale +- AI post‑processing: optional dictionary-based cleanup (optimized for Japanese and Chinese) - Quick controls in view: copy/clear/insert at cursor/append to end - Auto‑save drafts: periodic and on blur, automatic restore - Multilingual support: Japanese, English, Chinese, Korean interface languages @@ -48,15 +50,24 @@ Tip: A settings gear in the view header opens the plugin’s settings. - OpenAI API Key: stored locally (encrypted at rest) - Transcription Model: `gpt-4o-transcribe` or `gpt-4o-mini-transcribe` -- AI Post‑processing: enable dictionary‑based cleanup (supports all languages when enabled) +- Voice Recognition Language: Auto/Japanese/English/Chinese/Korean (auto-detection recommended) +- AI Post‑processing: enable dictionary‑based cleanup (applied to Japanese and Chinese) - Maximum Recording Duration: slider (default 5 min) -- Plugin Language: English/Japanese/Chinese/Korean (auto‑detected from Obsidian, adjustable) +- Plugin Language: English/Japanese/Chinese/Korean (controls UI display, auto‑detected from Obsidian) + +### Language Settings Explained + +The plugin separates UI language from voice recognition language for optimal user experience: + +- **Plugin Language**: Controls the interface language (menus, buttons, messages). Auto-detected from your Obsidian language setting. +- **Voice Recognition Language**: Determines the language for speech recognition and transcription. "Auto" is recommended for best results. ### Language-specific Behavior -- **Japanese (ja)**: Enhanced prompts and meta-pattern cleaning for optimal transcription accuracy -- **English/Chinese/Korean (en/zh/ko)**: Clean transcription without language-specific prompts to avoid interference -- **Dictionary Correction**: Applied to all languages when enabled (not limited to Japanese) +- **Japanese (ja)**: Enhanced prompts for optimal transcription accuracy + dictionary correction +- **Chinese (zh)**: Clean transcription + dictionary correction +- **English/Korean (en/ko)**: Clean transcription without prompts or correction to avoid interference +- **Auto Mode**: Automatically detects language based on your Obsidian locale setting ## Security & Privacy @@ -92,7 +103,9 @@ Third‑party licensing: see `THIRD_PARTY_LICENSES.md`. - ワンクリック録音（リボンのマイクアイコン） - プッシュトゥトーク（長押しで録音開始、離して停止） - モデル選択（GPT‑4o Transcribe / GPT‑4o mini Transcribe） -- AI後処理（辞書ベースの補正、多言語対応） +- 言語設定の分離（UI言語と音声認識言語を独立設定） +- 自動言語検出（Obsidianロケールに基づく音声認識言語の自動検出） +- AI後処理（辞書ベースの補正、日本語と中国語に最適化） - ビュー内のクイック操作（コピー/クリア/カーソル位置へ挿入/末尾へ追記） - 自動保存（定期保存とフォーカス外れ時）。再オープン時に自動復元 - 多言語サポート（日本語、英語、中国語、韓国語のインターフェース） @@ -133,15 +146,24 @@ Third‑party licensing: see `THIRD_PARTY_LICENSES.md`. - OpenAI APIキー: ローカルに暗号化して保存 - 文字起こしモデル: `gpt-4o-transcribe` または `gpt-4o-mini-transcribe` -- AI後処理: 辞書ベースの補正（有効時は全言語で適用） +- 音声認識言語: 自動/日本語/英語/中国語/韓国語（自動検出を推奨） +- AI後処理: 辞書ベースの補正（日本語と中国語に適用） - 最大録音時間: スライダー（初期値5分） -- プラグイン言語: 英語/日本語/中国語/韓国語（Obsidian設定から自動検出、変更可） +- プラグイン言語: 英語/日本語/中国語/韓国語（UI表示言語、Obsidian設定から自動検出） + +### 言語設定について + +プラグインは最適なユーザー体験のため、UI言語と音声認識言語を分離しています： + +- **プラグイン言語**: インターフェースの言語（メニュー、ボタン、メッセージ）。Obsidianの言語設定から自動検出されます。 +- **音声認識言語**: 音声認識と文字起こしの言語。最良の結果のため「自動」を推奨します。 ### 言語別の動作 -- **日本語 (ja)**: 高精度化のための専用プロンプトとメタパターン除去 -- **英語/中国語/韓国語 (en/zh/ko)**: 干渉を避けるため言語固有プロンプトを使用しないクリーンな文字起こし -- **辞書補正**: 有効時は全言語に適用（日本語限定ではありません） +- **日本語 (ja)**: 高精度化のための専用プロンプト + 辞書補正 +- **中国語 (zh)**: クリーンな文字起こし + 辞書補正 +- **英語/韓国語 (en/ko)**: 干渉を避けるため、プロンプトや補正なしのクリーンな文字起こし +- **自動モード**: Obsidianのロケール設定に基づいて言語を自動検出 ## セキュリティ / プライバシー From 30b94a5645b5df734f45c471c2c09db19d037d32 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:15:01 +0000 Subject: [PATCH 16/82] Implement settings separation and transcriptionLanguage field with migration Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- .../transcription/TranscriptionService.ts | 6 ++--- src/i18n/en.ts | 2 ++ src/i18n/ja.ts | 2 ++ src/i18n/ko.ts | 2 ++ src/i18n/zh.ts | 2 ++ src/interfaces/settings.ts | 4 +-- src/plugin/VoiceInputPlugin.ts | 27 ++++++++++++++----- src/settings/VoiceInputSettingTab.ts | 10 +++---- 8 files changed, 39 insertions(+), 16 deletions(-) diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 29802c0..374a6a7 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -143,9 +143,9 @@ export class TranscriptionService implements ITranscriptionProvider { } // Apply corrections if enabled and for supported languages - // Dictionary correction is most effective for Japanese and Chinese - const supportsCorrectionForLanguage = ['ja', 'zh'].includes(language); - const correctedText = this.enableTranscriptionCorrection && supportsCorrectionForLanguage + // Use effective language from response or fallback to input language + const effectiveLang = responseData.language || language; + const correctedText = this.enableTranscriptionCorrection && effectiveLang === 'ja' ? await this.corrector.correct(originalText) : originalText; diff --git a/src/i18n/en.ts b/src/i18n/en.ts index a12eb10..a5b6365 100644 --- a/src/i18n/en.ts +++ b/src/i18n/en.ts @@ -156,6 +156,8 @@ export const en: TranslationResource = { maxRecordingDurationDesc: 'Maximum recording time in seconds ({min}s - {max}min)', language: 'Voice Recognition Language', languageDesc: 'Language for voice recognition and transcription. Auto-detection is recommended for best results.', + transcriptionLanguage: 'Transcription Language', + transcriptionLanguageDesc: 'Language for voice recognition and transcription. Auto-detection is recommended for best results.', pluginLanguage: 'Plugin Language', pluginLanguageDesc: 'Set language for UI, voice processing, and correction dictionary', customDictionary: 'Custom Dictionary', diff --git a/src/i18n/ja.ts b/src/i18n/ja.ts index a810630..f52f61a 100644 --- a/src/i18n/ja.ts +++ b/src/i18n/ja.ts @@ -156,6 +156,8 @@ export const ja: TranslationResource = { maxRecordingDurationDesc: '最大録音時間（秒）（{min}秒〜{max}分）', language: '音声認識言語', languageDesc: '音声認識と文字起こしの言語。最適な結果のため自動検出を推奨します。', + transcriptionLanguage: '音声認識言語', + transcriptionLanguageDesc: '音声認識と文字起こしの言語。最適な結果のため自動検出を推奨します。', pluginLanguage: 'プラグイン言語', pluginLanguageDesc: 'UI表示、音声認識処理、補正辞書の言語を設定', customDictionary: 'カスタム辞書', diff --git a/src/i18n/ko.ts b/src/i18n/ko.ts index 384dec4..946570f 100644 --- a/src/i18n/ko.ts +++ b/src/i18n/ko.ts @@ -156,6 +156,8 @@ export const ko: TranslationResource = { maxRecordingDurationDesc: '최대 녹음 시간(초) ({min}초~{max}분)', language: '음성 인식 언어', languageDesc: '음성 인식 및 전사를 위한 언어입니다. 최상의 결과를 위해 자동 감지를 권장합니다.', + transcriptionLanguage: '음성 인식 언어', + transcriptionLanguageDesc: '음성 인식 및 전사를 위한 언어입니다. 최상의 결과를 위해 자동 감지를 권장합니다.', pluginLanguage: '플러그인 언어', pluginLanguageDesc: 'UI 표시, 음성 처리 및 교정 사전의 언어 설정', customDictionary: '사용자 정의 사전', diff --git a/src/i18n/zh.ts b/src/i18n/zh.ts index e4797f3..a75d356 100644 --- a/src/i18n/zh.ts +++ b/src/i18n/zh.ts @@ -156,6 +156,8 @@ export const zh: TranslationResource = { maxRecordingDurationDesc: '最大录音时间（秒）（{min}秒～{max}分钟）', language: '语音识别语言', languageDesc: '语音识别和转录的语言。建议使用自动检测以获得最佳结果。', + transcriptionLanguage: '语音识别语言', + transcriptionLanguageDesc: '语音识别和转录的语言。建议使用自动检测以获得最佳结果。', pluginLanguage: '插件语言', pluginLanguageDesc: '设置UI显示、语音处理和校正词典的语言', customDictionary: '自定义词典', diff --git a/src/interfaces/settings.ts b/src/interfaces/settings.ts index c4d139e..984638e 100644 --- a/src/interfaces/settings.ts +++ b/src/interfaces/settings.ts @@ -9,7 +9,7 @@ export interface VoiceInputSettings { // 録音設定 maxRecordingSeconds: number; // 最大録音時間（秒） // 言語設定 - language: 'auto' | Locale; // 音声認識言語 + transcriptionLanguage: 'auto' | 'ja' | 'en' | 'zh' | 'ko'; // 音声認識言語 pluginLanguage: Locale; // プラグイン全体の言語（UI表示用） customDictionary: SimpleCorrectionDictionary; // デバッグ設定 @@ -24,7 +24,7 @@ export const DEFAULT_SETTINGS: VoiceInputSettings = { // 録音設定 maxRecordingSeconds: 300, // 5分（300秒） // 言語設定 - language: 'auto', // 音声認識言語のデフォルトは自動検出 + transcriptionLanguage: 'auto', // 音声認識言語のデフォルトは自動検出 pluginLanguage: 'en', // 初期値、実際はObsidianの設定に従う customDictionary: { definiteCorrections: [] }, // デバッグ設定 diff --git a/src/plugin/VoiceInputPlugin.ts b/src/plugin/VoiceInputPlugin.ts index e8dc247..8f14316 100644 --- a/src/plugin/VoiceInputPlugin.ts +++ b/src/plugin/VoiceInputPlugin.ts @@ -221,7 +221,21 @@ export default class VoiceInputPlugin extends Plugin { this.logger?.info('Migrating interfaceLanguage to pluginLanguage'); } - // languageからpluginLanguageへの移行 + // languageからtranscriptionLanguageへの移行 + if ('language' in data && !('transcriptionLanguage' in data)) { + // 既存のlanguageフィールドをtranscriptionLanguageに移行 + const langValue = data.language; + if (langValue === 'auto' || langValue === 'ja' || langValue === 'en' || langValue === 'zh' || langValue === 'ko') { + migratedData.transcriptionLanguage = langValue; + } else { + migratedData.transcriptionLanguage = 'auto'; + } + delete migratedData.language; + needsSave = true; + this.logger?.info(`Migrating language (${data.language}) to transcriptionLanguage (${migratedData.transcriptionLanguage})`); + } + + // languageからpluginLanguageへの移行（古いバージョンとの互換性のため） if ('language' in data && !('pluginLanguage' in data)) { // 言語コードを正規化（ja → ja、en → en、zh → zh、ko → ko、その他 → en） const langCode = data.language as string; @@ -230,11 +244,12 @@ export default class VoiceInputPlugin extends Plugin { } else { migratedData.pluginLanguage = 'en'; } - delete migratedData.language; needsSave = true; this.logger?.info(`Migrating language (${data.language}) to pluginLanguage (${migratedData.pluginLanguage})`); - } else if ('language' in data) { - // pluginLanguageが既に存在する場合は、languageフィールドを削除 + } + + // 不要になったlanguageフィールドの削除 + if ('language' in data) { delete migratedData.language; needsSave = true; this.logger?.info('Removing redundant language field'); @@ -351,10 +366,10 @@ export default class VoiceInputPlugin extends Plugin { * Auto-detects from Obsidian locale when 'auto' is selected */ public getResolvedLanguage(): 'ja' | 'en' | 'zh' | 'ko' { - if (this.settings.language === 'auto') { + if (this.settings.transcriptionLanguage === 'auto') { return this.detectPluginLanguage(); } - return this.settings.language; + return this.settings.transcriptionLanguage; } async saveSettings() { diff --git a/src/settings/VoiceInputSettingTab.ts b/src/settings/VoiceInputSettingTab.ts index f3179e5..5c94095 100644 --- a/src/settings/VoiceInputSettingTab.ts +++ b/src/settings/VoiceInputSettingTab.ts @@ -75,17 +75,17 @@ export class VoiceInputSettingTab extends PluginSettingTab { // Voice Recognition Language Setting new Setting(containerEl) - .setName(this.i18n.t('ui.settings.language')) - .setDesc(this.i18n.t('ui.settings.languageDesc')) + .setName(this.i18n.t('ui.settings.transcriptionLanguage')) + .setDesc(this.i18n.t('ui.settings.transcriptionLanguageDesc')) .addDropdown(dropdown => dropdown .addOption('auto', this.i18n.t('ui.options.languageAuto')) .addOption('ja', this.i18n.t('ui.options.languageJa')) .addOption('en', this.i18n.t('ui.options.languageEn')) .addOption('zh', this.i18n.t('ui.options.languageZh')) .addOption('ko', this.i18n.t('ui.options.languageKo')) - .setValue(this.plugin.settings.language) - .onChange(async (value: 'auto' | Locale) => { - this.plugin.settings.language = value; + .setValue(this.plugin.settings.transcriptionLanguage) + .onChange(async (value: 'auto' | 'ja' | 'en' | 'zh' | 'ko') => { + this.plugin.settings.transcriptionLanguage = value; await this.plugin.saveSettings(); })); From 6ef8ce9323fbfd7bf7a26607938bd9ff2bc8c098 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:20:30 +0000 Subject: [PATCH 17/82] Complete multilingual improvements: i18n fixes, documentation, and quality gates Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- README.md | 56 ++++++++++++++++++++++++----------- src/interfaces/i18n.ts | 2 ++ src/security/SecurityUtils.ts | 11 ++++--- 3 files changed, 47 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 4762035..22e20e4 100644 --- a/README.md +++ b/README.md @@ -50,24 +50,34 @@ Tip: A settings gear in the view header opens the plugin’s settings. - OpenAI API Key: stored locally (encrypted at rest) - Transcription Model: `gpt-4o-transcribe` or `gpt-4o-mini-transcribe` -- Voice Recognition Language: Auto/Japanese/English/Chinese/Korean (auto-detection recommended) -- AI Post‑processing: enable dictionary‑based cleanup (applied to Japanese and Chinese) +- **Transcription Language**: Auto/Japanese/English/Chinese/Korean (auto-detection recommended) +- AI Post‑processing: enable dictionary‑based cleanup (applied to Japanese only) - Maximum Recording Duration: slider (default 5 min) -- Plugin Language: English/Japanese/Chinese/Korean (controls UI display, auto‑detected from Obsidian) +- **Plugin Language**: English/Japanese/Chinese/Korean (controls UI display, auto‑detected from Obsidian) ### Language Settings Explained -The plugin separates UI language from voice recognition language for optimal user experience: +The plugin provides **complete separation** between UI language and voice recognition language for optimal user experience: - **Plugin Language**: Controls the interface language (menus, buttons, messages). Auto-detected from your Obsidian language setting. -- **Voice Recognition Language**: Determines the language for speech recognition and transcription. "Auto" is recommended for best results. +- **Transcription Language**: Determines the language for speech recognition and transcription. "Auto" is recommended - it automatically selects the optimal language based on your Obsidian interface language. -### Language-specific Behavior +### Auto-Detection Behavior -- **Japanese (ja)**: Enhanced prompts for optimal transcription accuracy + dictionary correction -- **Chinese (zh)**: Clean transcription + dictionary correction +When **Transcription Language** is set to "Auto": +- **Japanese locale** (ja-*) → Japanese transcription +- **Chinese locale** (zh-*) → Chinese transcription +- **Korean locale** (ko-*) → Korean transcription +- **All other locales** → English transcription + +### Language-specific Processing + +The plugin applies different processing strategies based on the selected transcription language: + +- **Japanese (ja)**: Enhanced prompts for optimal transcription accuracy + dictionary correction for technical terms +- **Chinese (zh)**: Clean transcription without language-specific processing - **English/Korean (en/ko)**: Clean transcription without prompts or correction to avoid interference -- **Auto Mode**: Automatically detects language based on your Obsidian locale setting +- **Auto Mode**: Dynamically selects the optimal language and processing strategy based on your Obsidian interface language ## Security & Privacy @@ -146,24 +156,34 @@ Third‑party licensing: see `THIRD_PARTY_LICENSES.md`. - OpenAI APIキー: ローカルに暗号化して保存 - 文字起こしモデル: `gpt-4o-transcribe` または `gpt-4o-mini-transcribe` -- 音声認識言語: 自動/日本語/英語/中国語/韓国語（自動検出を推奨） -- AI後処理: 辞書ベースの補正（日本語と中国語に適用） +- **音声認識言語**: 自動/日本語/英語/中国語/韓国語（自動検出を推奨） +- AI後処理: 辞書ベースの補正（日本語のみ適用） - 最大録音時間: スライダー（初期値5分） -- プラグイン言語: 英語/日本語/中国語/韓国語（UI表示言語、Obsidian設定から自動検出） +- **プラグイン言語**: 英語/日本語/中国語/韓国語（UI表示言語、Obsidian設定から自動検出） ### 言語設定について -プラグインは最適なユーザー体験のため、UI言語と音声認識言語を分離しています： +プラグインは最適なユーザー体験のため、UI言語と音声認識言語を**完全に分離**しています： - **プラグイン言語**: インターフェースの言語（メニュー、ボタン、メッセージ）。Obsidianの言語設定から自動検出されます。 -- **音声認識言語**: 音声認識と文字起こしの言語。最良の結果のため「自動」を推奨します。 +- **音声認識言語**: 音声認識と文字起こしの言語。「自動」を推奨 - Obsidianのインターフェース言語に基づいて最適な言語を自動選択します。 -### 言語別の動作 +### 自動検出の動作 -- **日本語 (ja)**: 高精度化のための専用プロンプト + 辞書補正 -- **中国語 (zh)**: クリーンな文字起こし + 辞書補正 +**音声認識言語**が「自動」に設定されている場合： +- **日本語ロケール** (ja-*) → 日本語での文字起こし +- **中国語ロケール** (zh-*) → 中国語での文字起こし +- **韓国語ロケール** (ko-*) → 韓国語での文字起こし +- **その他のロケール** → 英語での文字起こし + +### 言語別の処理 + +プラグインは選択された音声認識言語に基づいて異なる処理戦略を適用します： + +- **日本語 (ja)**: 高精度化のための専用プロンプト + 専門用語の辞書補正 +- **中国語 (zh)**: 言語固有処理なしのクリーンな文字起こし - **英語/韓国語 (en/ko)**: 干渉を避けるため、プロンプトや補正なしのクリーンな文字起こし -- **自動モード**: Obsidianのロケール設定に基づいて言語を自動検出 +- **自動モード**: Obsidianのインターフェース言語に基づいて最適な言語と処理戦略を動的に選択 ## セキュリティ / プライバシー diff --git a/src/interfaces/i18n.ts b/src/interfaces/i18n.ts index b7ad99a..5421115 100644 --- a/src/interfaces/i18n.ts +++ b/src/interfaces/i18n.ts @@ -175,6 +175,8 @@ export type TranslationResource = { maxRecordingDurationDesc: string; language: string; languageDesc: string; + transcriptionLanguage: string; + transcriptionLanguageDesc: string; pluginLanguage: string; pluginLanguageDesc: string; customDictionary: string; diff --git a/src/security/SecurityUtils.ts b/src/security/SecurityUtils.ts index 03420da..5d2edb1 100644 --- a/src/security/SecurityUtils.ts +++ b/src/security/SecurityUtils.ts @@ -1,4 +1,5 @@ import { ObsidianHttpClient } from '../utils/ObsidianHttpClient'; +import { getI18n } from '../services'; export class SecurityUtils { /** @@ -64,6 +65,8 @@ export class SecurityUtils { * Test OpenAI API key by making a simple API call */ static async testOpenAIAPIKey(apiKey: string): Promise<{ valid: boolean; error?: string }> { + const i18n = getI18n(); + try { const { status } = await ObsidianHttpClient.request({ url: 'https://api.openai.com/v1/models', @@ -74,14 +77,14 @@ export class SecurityUtils { if (status >= 200 && status < 300) { return { valid: true }; } else if (status === 401) { - return { valid: false, error: 'APIキーが無効です' }; + return { valid: false, error: i18n.t('error.api.unauthorized') }; } else if (status === 429) { - return { valid: false, error: 'レート制限に達しています' }; + return { valid: false, error: i18n.t('error.api.rateLimited') }; } else { - return { valid: false, error: `エラー: HTTP ${status}` }; + return { valid: false, error: `${i18n.t('error.general.error')}: HTTP ${status}` }; } } catch (_error) { - return { valid: false, error: 'ネットワークエラー' }; + return { valid: false, error: i18n.t('error.general.network') }; } } } From 6e8edec041e40e87cab8f3767abed2fec58689eb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:16:37 +0000 Subject: [PATCH 18/82] Implement comprehensive multilingual transcription cleaning and fixes Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- README.md | 2 +- package-lock.json | 4 +- package.json | 4 +- .../transcription/TranscriptionService.ts | 187 +++++- .../transcription-cleaning.test.ts | 544 ++++++++++++++++++ 5 files changed, 712 insertions(+), 29 deletions(-) create mode 100644 tests/unit/core/transcription/transcription-cleaning.test.ts diff --git a/README.md b/README.md index 22e20e4..d974a32 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ High-accuracy voice input for Obsidian. Uses OpenAI GPT-4o Audio Transcriptions - Model selection: GPT‑4o Transcribe or GPT‑4o mini Transcribe - Language separation: independent UI language and voice recognition language settings - Auto language detection: automatic voice recognition language based on Obsidian locale -- AI post‑processing: optional dictionary-based cleanup (optimized for Japanese and Chinese) +- AI post‑processing: optional dictionary-based cleanup (Japanese only) - Quick controls in view: copy/clear/insert at cursor/append to end - Auto‑save drafts: periodic and on blur, automatic restore - Multilingual support: Japanese, English, Chinese, Korean interface languages diff --git a/package-lock.json b/package-lock.json index 22a35d1..a9f9d16 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,7 @@ "ts-jest": "^29.4.0" }, "devDependencies": { - "@types/node": "^16.18.126", + "@types/node": "^16.11.6", "@typescript-eslint/eslint-plugin": "^5.29.0", "@typescript-eslint/parser": "^5.29.0", "builtin-modules": "3.3.0", @@ -23,7 +23,7 @@ "eslint": "^8.57.1", "jest-environment-jsdom": "^30.0.5", "obsidian": "latest", - "tslib": "^2.4.0", + "tslib": "2.4.0", "typescript": "4.7.4" } }, diff --git a/package.json b/package.json index 6daec67..3a5cc2a 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "author": "Musashino Software", "license": "MIT", "devDependencies": { - "@types/node": "^16.18.126", + "@types/node": "^16.11.6", "@typescript-eslint/eslint-plugin": "^5.29.0", "@typescript-eslint/parser": "^5.29.0", "builtin-modules": "3.3.0", @@ -39,7 +39,7 @@ "eslint": "^8.57.1", "jest-environment-jsdom": "^30.0.5", "obsidian": "latest", - "tslib": "^2.4.0", + "tslib": "2.4.0", "typescript": "4.7.4" }, "dependencies": { diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 374a6a7..7c548d1 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -113,14 +113,11 @@ export class TranscriptionService implements ITranscriptionProvider { } // Clean up GPT-4o specific artifacts - originalText = this.cleanGPT4oResponse(originalText); + originalText = this.cleanGPT4oResponse(originalText, language); // プロンプトエラーの検出（音声が無音の場合にプロンプトが返される問題） - if (originalText.includes('この指示文は出力に含めないでください') || - originalText.includes('話者の発言内容だけを正確に記録してください') || - originalText === '（話者の発言のみ）' || - originalText.trim() === '話者の発言のみ') { + if (this.isPromptErrorDetected(originalText, language)) { // 音声がない場合は空文字を返す originalText = ''; } @@ -142,10 +139,8 @@ export class TranscriptionService implements ITranscriptionProvider { }; } - // Apply corrections if enabled and for supported languages - // Use effective language from response or fallback to input language - const effectiveLang = responseData.language || language; - const correctedText = this.enableTranscriptionCorrection && effectiveLang === 'ja' + // Apply corrections if enabled + const correctedText = this.enableTranscriptionCorrection ? await this.corrector.correct(originalText) : originalText; @@ -200,9 +195,12 @@ export class TranscriptionService implements ITranscriptionProvider { } /** - * Clean GPT-4o specific response artifacts + * Clean GPT-4o specific response artifacts with language-specific processing */ - private cleanGPT4oResponse(text: string): string { + private cleanGPT4oResponse(text: string, language: string): string { + // Normalize language for processing + const normalizedLang = this.normalizeLanguage(language); + // First attempt: Extract content from complete TRANSCRIPT tags let transcriptMatch = text.match(/\s*([\s\S]*?)\s*<\/TRANSCRIPT>/); if (transcriptMatch) { @@ -218,20 +216,11 @@ export class TranscriptionService implements ITranscriptionProvider { // Remove TRANSCRIPT opening tag if still present (for cases where it's not properly extracted) text = text.replace(/<\/?TRANSCRIPT[^>]*>/g, ''); - // Remove specific meta instruction patterns (both at line start and anywhere in text) - const metaPatterns = [ - /^以下の音声内容.*?$/gm, - /^この指示文.*?$/gm, - /^話者の発言内容だけを正確に記録してください.*?$/gm, - /^話者の発言.*?$/gm, - /^出力形式.*?$/gm, - /（話者の発言のみ）/g, // Remove this specific phrase anywhere in the text - ]; + // Apply language-specific cleaning + text = this.applyLanguageSpecificCleaning(text, normalizedLang); - // Apply all cleaning patterns - for (const pattern of metaPatterns) { - text = text.replace(pattern, ''); - } + // Apply generic cleaning (only colon-based patterns to prevent over-removal) + text = this.applyGenericCleaning(text); // Clean up extra whitespace and empty lines text = text.trim(); @@ -242,6 +231,156 @@ export class TranscriptionService implements ITranscriptionProvider { return text; } + /** + * Normalize language code for consistent processing + */ + private normalizeLanguage(language: string): string { + if (language === 'auto') return 'auto'; + const lang = language.toLowerCase(); + if (lang.startsWith('ja')) return 'ja'; + if (lang.startsWith('zh')) return 'zh'; + if (lang.startsWith('ko')) return 'ko'; + if (lang.startsWith('en')) return 'en'; + return lang; + } + + /** + * Apply language-specific cleaning patterns + */ + private applyLanguageSpecificCleaning(text: string, language: string): string { + switch (language) { + case 'ja': + return this.applyJapaneseCleaning(text); + case 'en': + return this.applyEnglishCleaning(text); + case 'zh': + return this.applyChineseCleaning(text); + case 'ko': + return this.applyKoreanCleaning(text); + default: + return text; + } + } + + /** + * Apply Japanese-specific cleaning patterns + */ + private applyJapaneseCleaning(text: string): string { + const patterns = [ + /^以下の音声内容.*?$/gm, + /^この指示文.*?$/gm, + /^話者の発言内容だけを正確に記録してください.*?$/gm, + /^話者の発言.*?$/gm, + /^出力形式.*?$/gm, + /（話者の発言のみ）/g, + ]; + + for (const pattern of patterns) { + text = text.replace(pattern, ''); + } + return text; + } + + /** + * Apply English-specific cleaning patterns + */ + private applyEnglishCleaning(text: string): string { + const patterns = [ + /^Please transcribe.*?$/gmi, + /^Transcribe only.*?$/gmi, + /^Output format.*?$/gmi, + /^Format.*?$/gmi, + ]; + + for (const pattern of patterns) { + text = text.replace(pattern, ''); + } + return text; + } + + /** + * Apply Chinese-specific cleaning patterns + */ + private applyChineseCleaning(text: string): string { + const patterns = [ + /^请转录.*?$/gm, + /^仅转录.*?$/gm, + /^输出格式.*?$/gm, + /^格式.*?$/gm, + ]; + + for (const pattern of patterns) { + text = text.replace(pattern, ''); + } + return text; + } + + /** + * Apply Korean-specific cleaning patterns + */ + private applyKoreanCleaning(text: string): string { + const patterns = [ + /^다음 음성.*?$/gm, + /^음성 내용만.*?$/gm, + /^출력 형식.*?$/gm, + /^형식.*?$/gm, + ]; + + for (const pattern of patterns) { + text = text.replace(pattern, ''); + } + return text; + } + + /** + * Apply generic cleaning patterns (conservative approach) + */ + private applyGenericCleaning(text: string): string { + // Only remove clear format instruction patterns with colons to prevent over-removal + const patterns = [ + /^Output\s*format\s*:.*/gmi, + /^Format\s*:.*/gmi, + ]; + + for (const pattern of patterns) { + text = text.replace(pattern, ''); + } + return text; + } + + /** + * Detect prompt error patterns by language + */ + private isPromptErrorDetected(text: string, language: string): boolean { + const normalizedLang = this.normalizeLanguage(language); + + switch (normalizedLang) { + case 'ja': + return text.includes('この指示文は出力に含めないでください') || + text.includes('話者の発言内容だけを正確に記録してください') || + text === '（話者の発言のみ）' || + text.trim() === '話者の発言のみ'; + case 'en': + return text.includes('Please transcribe only the speaker') || + text.includes('Do not include this instruction') || + text.trim() === '(Speaker content only)'; + case 'zh': + return text.includes('请仅转录说话者') || + text.includes('不要包含此指令') || + text.trim() === '（仅说话者内容）'; + case 'ko': + return text.includes('화자의 발언만 전사해주세요') || + text.includes('이 지시사항을 포함하지 마세요') || + text.trim() === '（화자 발언만）'; + default: + // For auto and other languages, use Japanese patterns as fallback + return text.includes('この指示文は出力に含めないでください') || + text.includes('話者の発言内容だけを正確に記録してください') || + text === '（話者の発言のみ）' || + text.trim() === '話者の発言のみ'; + } + } + /** * Estimate transcription cost */ diff --git a/tests/unit/core/transcription/transcription-cleaning.test.ts b/tests/unit/core/transcription/transcription-cleaning.test.ts new file mode 100644 index 0000000..ab02528 --- /dev/null +++ b/tests/unit/core/transcription/transcription-cleaning.test.ts @@ -0,0 +1,544 @@ +/** + * Comprehensive tests for transcription cleaning functionality + * Tests the multilingual approach with language-specific patterns + */ + +// Mock TranscriptionService to test its cleaning methods +class MockTranscriptionService { + /** + * Clean GPT-4o specific response artifacts with language-specific processing + */ + cleanGPT4oResponse(text: string, language: string): string { + // Normalize language for processing + const normalizedLang = this.normalizeLanguage(language); + + // First attempt: Extract content from complete TRANSCRIPT tags + let transcriptMatch = text.match(/\s*([\s\S]*?)\s*<\/TRANSCRIPT>/); + if (transcriptMatch) { + text = transcriptMatch[1]; + } else { + // Second attempt: Handle incomplete TRANSCRIPT tags (missing closing tag) + const openingMatch = text.match(/\s*([\s\S]*)/); + if (openingMatch) { + text = openingMatch[1]; + } + } + + // Remove TRANSCRIPT opening tag if still present (for cases where it's not properly extracted) + text = text.replace(/<\/?TRANSCRIPT[^>]*>/g, ''); + + // Apply language-specific cleaning + text = this.applyLanguageSpecificCleaning(text, normalizedLang); + + // Apply generic cleaning (only colon-based patterns to prevent over-removal) + text = this.applyGenericCleaning(text); + + // Clean up extra whitespace and empty lines + text = text.trim(); + text = text.replace(/\n{3,}/g, '\n\n'); + text = text.replace(/^\s*\n/gm, ''); // Remove lines with only whitespace + text = text.trim(); + + return text; + } + + /** + * Normalize language code for consistent processing + */ + normalizeLanguage(language: string): string { + if (language === 'auto') return 'auto'; + const lang = language.toLowerCase(); + if (lang.startsWith('ja')) return 'ja'; + if (lang.startsWith('zh')) return 'zh'; + if (lang.startsWith('ko')) return 'ko'; + if (lang.startsWith('en')) return 'en'; + return lang; + } + + /** + * Apply language-specific cleaning patterns + */ + applyLanguageSpecificCleaning(text: string, language: string): string { + switch (language) { + case 'ja': + return this.applyJapaneseCleaning(text); + case 'en': + return this.applyEnglishCleaning(text); + case 'zh': + return this.applyChineseCleaning(text); + case 'ko': + return this.applyKoreanCleaning(text); + default: + return text; + } + } + + /** + * Apply Japanese-specific cleaning patterns + */ + applyJapaneseCleaning(text: string): string { + const patterns = [ + /^以下の音声内容.*?$/gm, + /^この指示文.*?$/gm, + /^話者の発言内容だけを正確に記録してください.*?$/gm, + /^話者の発言.*?$/gm, + /^出力形式.*?$/gm, + /（話者の発言のみ）/g, + ]; + + for (const pattern of patterns) { + text = text.replace(pattern, ''); + } + return text; + } + + /** + * Apply English-specific cleaning patterns + */ + applyEnglishCleaning(text: string): string { + const patterns = [ + /^Please transcribe.*?$/gmi, + /^Transcribe only.*?$/gmi, + /^Output format.*?$/gmi, + /^Format.*?$/gmi, + ]; + + for (const pattern of patterns) { + text = text.replace(pattern, ''); + } + return text; + } + + /** + * Apply Chinese-specific cleaning patterns + */ + applyChineseCleaning(text: string): string { + const patterns = [ + /^请转录.*?$/gm, + /^仅转录.*?$/gm, + /^输出格式.*?$/gm, + /^格式.*?$/gm, + ]; + + for (const pattern of patterns) { + text = text.replace(pattern, ''); + } + return text; + } + + /** + * Apply Korean-specific cleaning patterns + */ + applyKoreanCleaning(text: string): string { + const patterns = [ + /^다음 음성.*?$/gm, + /^음성 내용만.*?$/gm, + /^출력 형식.*?$/gm, + /^형식.*?$/gm, + ]; + + for (const pattern of patterns) { + text = text.replace(pattern, ''); + } + return text; + } + + /** + * Apply generic cleaning patterns (conservative approach) + */ + applyGenericCleaning(text: string): string { + // Only remove clear format instruction patterns with colons to prevent over-removal + const patterns = [ + /^Output\s*format\s*:.*/gmi, + /^Format\s*:.*/gmi, + ]; + + for (const pattern of patterns) { + text = text.replace(pattern, ''); + } + return text; + } + + /** + * Detect prompt error patterns by language + */ + isPromptErrorDetected(text: string, language: string): boolean { + const normalizedLang = this.normalizeLanguage(language); + + switch (normalizedLang) { + case 'ja': + return text.includes('この指示文は出力に含めないでください') || + text.includes('話者の発言内容だけを正確に記録してください') || + text === '（話者の発言のみ）' || + text.trim() === '話者の発言のみ'; + case 'en': + return text.includes('Please transcribe only the speaker') || + text.includes('Do not include this instruction') || + text.trim() === '(Speaker content only)'; + case 'zh': + return text.includes('请仅转录说话者') || + text.includes('不要包含此指令') || + text.trim() === '（仅说话者内容）'; + case 'ko': + return text.includes('화자의 발언만 전사해주세요') || + text.includes('이 지시사항을 포함하지 마세요') || + text.trim() === '（화자 발언만）'; + default: + // For auto and other languages, use Japanese patterns as fallback + return text.includes('この指示文は出力に含めないでください') || + text.includes('話者の発言内容だけを正確に記録してください') || + text === '（話者の発言のみ）' || + text.trim() === '話者の発言のみ'; + } + } + + /** + * Build prompt for GPT-4o transcription + */ + buildTranscriptionPrompt(language: string): string { + // Only provide prompt for Japanese language + if (language !== 'ja') { + return ''; + } + + return `以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。 +話者の発言内容だけを正確に記録してください。 + +出力形式: + +（話者の発言のみ） +`; + } +} + +describe('Multilingual Transcription Cleaning', () => { + let service: MockTranscriptionService; + + beforeEach(() => { + service = new MockTranscriptionService(); + }); + + describe('TRANSCRIPT tag extraction', () => { + it('should extract content from complete TRANSCRIPT tags', () => { + const input = `Some prefix text + +実際の発言内容 + +Some suffix text`; + const result = service.cleanGPT4oResponse(input, 'ja'); + expect(result).toContain('実際の発言内容'); + expect(result).not.toContain(''); + expect(result).not.toContain(''); + }); + + it('should handle incomplete TRANSCRIPT tags (missing closing tag)', () => { + const input = `Some prefix text + +実際の発言内容 +Additional content`; + const result = service.cleanGPT4oResponse(input, 'ja'); + expect(result).toContain('実際の発言内容'); + expect(result).toContain('Additional content'); + expect(result).not.toContain(''); + }); + + it('should handle malformed TRANSCRIPT tags', () => { + const input = ` +Content with attributes +`; + const result = service.cleanGPT4oResponse(input, 'en'); + expect(result).toContain('Content with attributes'); + expect(result).not.toContain(''); + }); + }); + + describe('Language-specific cleaning', () => { + describe('Japanese cleaning', () => { + it('should remove Japanese meta instructions', () => { + const patterns = [ + '以下の音声内容のみを文字に起こしてください', + 'この指示文は出力に含めないでください', + '話者の発言内容だけを正確に記録してください', + '出力形式:' + ]; + + patterns.forEach(pattern => { + const input = `${pattern}\n実際の発言\nその他の内容`; + const result = service.cleanGPT4oResponse(input, 'ja'); + expect(result).not.toContain(pattern); + expect(result).toContain('実際の発言'); + expect(result).toContain('その他の内容'); + }); + }); + + it('should remove Japanese meta phrases anywhere in text', () => { + const input = '前の文章 （話者の発言のみ） 後の文章'; + const result = service.cleanGPT4oResponse(input, 'ja'); + expect(result).not.toContain('（話者の発言のみ）'); + expect(result).toContain('前の文章'); + expect(result).toContain('後の文章'); + }); + }); + + describe('English cleaning', () => { + it('should remove English meta instructions', () => { + const patterns = [ + 'Please transcribe the following audio', + 'Transcribe only the speaker content', + 'Output format: JSON', + 'Format: Speaker content only' + ]; + + patterns.forEach(pattern => { + const input = `${pattern}\nActual speech\nOther content`; + const result = service.cleanGPT4oResponse(input, 'en'); + expect(result).not.toContain(pattern); + expect(result).toContain('Actual speech'); + expect(result).toContain('Other content'); + }); + }); + }); + + describe('Chinese cleaning', () => { + it('should remove Chinese meta instructions', () => { + const patterns = [ + '请转录以下音频内容', + '仅转录说话者内容', + '输出格式: JSON', + '格式: 说话者内容' + ]; + + patterns.forEach(pattern => { + const input = `${pattern}\n实际语音\n其他内容`; + const result = service.cleanGPT4oResponse(input, 'zh'); + expect(result).not.toContain(pattern); + expect(result).toContain('实际语音'); + expect(result).toContain('其他内容'); + }); + }); + }); + + describe('Korean cleaning', () => { + it('should remove Korean meta instructions', () => { + const patterns = [ + '다음 음성 내용을 전사해주세요', + '음성 내용만 전사하세요', + '출력 형식: JSON', + '형식: 화자 내용만' + ]; + + patterns.forEach(pattern => { + const input = `${pattern}\n실제 음성\n기타 내용`; + const result = service.cleanGPT4oResponse(input, 'ko'); + expect(result).not.toContain(pattern); + expect(result).toContain('실제 음성'); + expect(result).toContain('기타 내용'); + }); + }); + }); + }); + + describe('Generic cleaning (conservative approach)', () => { + it('should only remove colon-based format instructions', () => { + const input = `Output format: JSON response +Normal sentence about output +Format: Speaker content only +Another normal sentence`; + + const result = service.cleanGPT4oResponse(input, 'en'); + expect(result).not.toContain('Output format: JSON response'); + expect(result).not.toContain('Format: Speaker content only'); + expect(result).toContain('Normal sentence about output'); + expect(result).toContain('Another normal sentence'); + }); + + it('should NOT remove normal text without colons', () => { + const input = `Output was successful +Format looks good +This is normal content`; + + const result = service.cleanGPT4oResponse(input, 'fr'); // Non-supported language + expect(result).toContain('Output was successful'); + expect(result).toContain('Format looks good'); + expect(result).toContain('This is normal content'); + }); + }); + + describe('Prompt error detection by language', () => { + describe('Japanese error detection', () => { + it('should detect Japanese prompt error patterns', () => { + const errorPatterns = [ + 'この指示文は出力に含めないでください', + '話者の発言内容だけを正確に記録してください', + '（話者の発言のみ）', + '話者の発言のみ' + ]; + + errorPatterns.forEach(pattern => { + expect(service.isPromptErrorDetected(pattern, 'ja')).toBe(true); + }); + }); + }); + + describe('English error detection', () => { + it('should detect English prompt error patterns', () => { + const errorPatterns = [ + 'Please transcribe only the speaker', + 'Do not include this instruction', + '(Speaker content only)' + ]; + + errorPatterns.forEach(pattern => { + expect(service.isPromptErrorDetected(pattern, 'en')).toBe(true); + }); + }); + }); + + describe('Chinese error detection', () => { + it('should detect Chinese prompt error patterns', () => { + const errorPatterns = [ + '请仅转录说话者', + '不要包含此指令', + '（仅说话者内容）' + ]; + + errorPatterns.forEach(pattern => { + expect(service.isPromptErrorDetected(pattern, 'zh')).toBe(true); + }); + }); + }); + + describe('Korean error detection', () => { + it('should detect Korean prompt error patterns', () => { + const errorPatterns = [ + '화자의 발언만 전사해주세요', + '이 지시사항을 포함하지 마세요', + '（화자 발언만）' + ]; + + errorPatterns.forEach(pattern => { + expect(service.isPromptErrorDetected(pattern, 'ko')).toBe(true); + }); + }); + }); + + it('should NOT detect normal content as errors', () => { + const normalContent = [ + 'こんにちは、今日はいい天気ですね', + 'Hello, how are you today?', + '你好，今天天气不错', + '안녕하세요, 오늘 날씨가 좋네요' + ]; + + normalContent.forEach(content => { + expect(service.isPromptErrorDetected(content, 'ja')).toBe(false); + expect(service.isPromptErrorDetected(content, 'en')).toBe(false); + expect(service.isPromptErrorDetected(content, 'zh')).toBe(false); + expect(service.isPromptErrorDetected(content, 'ko')).toBe(false); + }); + }); + }); + + describe('buildTranscriptionPrompt language branching', () => { + it('should return Japanese prompt for ja language', () => { + const result = service.buildTranscriptionPrompt('ja'); + expect(result).toContain('以下の音声内容のみを文字に起こしてください'); + expect(result).toContain(''); + expect(result).toContain('（話者の発言のみ）'); + expect(result.length).toBeGreaterThan(0); + }); + + it('should return empty string for auto language', () => { + expect(service.buildTranscriptionPrompt('auto')).toBe(''); + }); + + it('should return empty string for en/zh/ko languages', () => { + const languages = ['en', 'zh', 'ko']; + languages.forEach(lang => { + expect(service.buildTranscriptionPrompt(lang)).toBe(''); + }); + }); + + it('should return empty string for other languages', () => { + const languages = ['fr', 'de', 'es', 'unknown']; + languages.forEach(lang => { + expect(service.buildTranscriptionPrompt(lang)).toBe(''); + }); + }); + }); + + describe('Language normalization', () => { + it('should normalize language codes correctly', () => { + expect(service.normalizeLanguage('ja')).toBe('ja'); + expect(service.normalizeLanguage('ja-JP')).toBe('ja'); + expect(service.normalizeLanguage('JA')).toBe('ja'); + + expect(service.normalizeLanguage('en')).toBe('en'); + expect(service.normalizeLanguage('en-US')).toBe('en'); + expect(service.normalizeLanguage('EN')).toBe('en'); + + expect(service.normalizeLanguage('zh')).toBe('zh'); + expect(service.normalizeLanguage('zh-CN')).toBe('zh'); + expect(service.normalizeLanguage('ZH')).toBe('zh'); + + expect(service.normalizeLanguage('ko')).toBe('ko'); + expect(service.normalizeLanguage('ko-KR')).toBe('ko'); + expect(service.normalizeLanguage('KO')).toBe('ko'); + + expect(service.normalizeLanguage('auto')).toBe('auto'); + expect(service.normalizeLanguage('fr')).toBe('fr'); + }); + }); + + describe('Whitespace and formatting cleanup', () => { + it('should clean up extra whitespace and empty lines', () => { + const input = ` \n\n\n実際の内容\n\n\n\n別の内容\n\n `; + const result = service.cleanGPT4oResponse(input, 'ja'); + // After trim, replace 3+ newlines with 2, then remove empty lines, the result should be without double newlines + expect(result).toBe('実際の内容\n別の内容'); + }); + + it('should preserve single line breaks between content', () => { + const input = `Line 1\nLine 2\nLine 3`; + const result = service.cleanGPT4oResponse(input, 'en'); + expect(result).toBe('Line 1\nLine 2\nLine 3'); + }); + + it('should convert triple+ line breaks to double then remove empty lines', () => { + const input = `Line 1\n\n\n\n\nLine 2`; + const result = service.cleanGPT4oResponse(input, 'en'); + // The regex /^\s*\n/gm removes lines that contain only whitespace, which includes the double newlines + expect(result).toBe('Line 1\nLine 2'); + }); + }); + + describe('Integration-style tests', () => { + it('should handle complex multilingual transcription response', () => { + const input = ` +以下の音声内容のみを文字に起こしてください +こんにちは、今日は会議です +出力形式: JSON +（話者の発言のみ） +`; + + const result = service.cleanGPT4oResponse(input, 'ja'); + expect(result).toBe('こんにちは、今日は会議です'); + }); + + it('should preserve content when no cleaning is needed', () => { + const input = 'Hello everyone, welcome to the meeting'; + const result = service.cleanGPT4oResponse(input, 'en'); + expect(result).toBe('Hello everyone, welcome to the meeting'); + }); + + it('should handle mixed language content appropriately', () => { + const input = `Please transcribe the following +Hello world +Format: Clean output`; + + const result = service.cleanGPT4oResponse(input, 'en'); + expect(result).toBe('Hello world'); + }); + }); +}); \ No newline at end of file From 1209000c57a2842b858a40ad3bea250e8825931b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:36:23 +0000 Subject: [PATCH 19/82] Fix Japanese README to match implementation: dictionary correction is Japanese only Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d974a32..0f7a52d 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Third‑party licensing: see `THIRD_PARTY_LICENSES.md`. - モデル選択（GPT‑4o Transcribe / GPT‑4o mini Transcribe） - 言語設定の分離（UI言語と音声認識言語を独立設定） - 自動言語検出（Obsidianロケールに基づく音声認識言語の自動検出） -- AI後処理（辞書ベースの補正、日本語と中国語に最適化） +- AI後処理（辞書ベースの補正、日本語のみ適用） - ビュー内のクイック操作（コピー/クリア/カーソル位置へ挿入/末尾へ追記） - 自動保存（定期保存とフォーカス外れ時）。再オープン時に自動復元 - 多言語サポート（日本語、英語、中国語、韓国語のインターフェース） From 45ca10ba3618b8880a93118a68d4cac2f9cc45c8 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 13:12:46 +0000 Subject: [PATCH 20/82] Initial plan From 308a987c55efed2ff24bf4980c31c343962f6186 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 13:28:59 +0000 Subject: [PATCH 21/82] Implement transcriptionLanguage separation and update documentation Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- README.md | 4 +- .../transcription/TranscriptionService.ts | 59 +++++++++++++------ src/i18n/en.ts | 6 +- src/i18n/ja.ts | 4 +- src/i18n/ko.ts | 4 +- src/i18n/zh.ts | 4 +- src/interfaces/settings.ts | 8 ++- src/plugin/VoiceInputPlugin.ts | 38 +++++++++++- src/views/VoiceInputViewActions.ts | 2 +- 9 files changed, 94 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 2f38496..0f6222c 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Tip: A settings gear in the view header opens the plugin’s settings. - Transcription Model: `gpt-4o-transcribe` or `gpt-4o-mini-transcribe` - AI Post‑processing: enable dictionary‑based cleanup (Japanese) - Maximum Recording Duration: slider (default 5 min) -- Plugin Language: English/Japanese (auto‑detected from Obsidian, adjustable) +- Plugin Language: English/Japanese (controls UI display only, auto‑detected from Obsidian, adjustable) ## Security & Privacy @@ -127,7 +127,7 @@ Third‑party licensing: see `THIRD_PARTY_LICENSES.md`. - 文字起こしモデル: `gpt-4o-transcribe` または `gpt-4o-mini-transcribe` - AI後処理: 辞書ベースの補正（日本語向け） - 最大録音時間: スライダー（初期値5分） -- プラグイン言語: 英語/日本語（Obsidian設定から自動検出、変更可） +- プラグイン言語: 英語/日本語（UI表示のみ制御、Obsidian設定から自動検出、変更可） ## セキュリティ / プライバシー diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 68d3587..615c620 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -54,8 +54,8 @@ export class TranscriptionService implements ITranscriptionProvider { formData.append('language', language); } - // Build prompt for transcription - const prompt = this.buildTranscriptionPrompt(); + // Build prompt for transcription (only for Japanese) + const prompt = this.buildTranscriptionPrompt(language); if (prompt) { formData.append('prompt', prompt); } @@ -114,7 +114,7 @@ export class TranscriptionService implements ITranscriptionProvider { } // Clean up GPT-4o specific artifacts - originalText = this.cleanGPT4oResponse(originalText); + originalText = this.cleanGPT4oResponse(originalText, language); // プロンプトエラーの検出（音声が無音の場合にプロンプトが返される問題） @@ -143,8 +143,8 @@ export class TranscriptionService implements ITranscriptionProvider { }; } - // Apply corrections if enabled (only for Japanese) - const correctedText = this.enableTranscriptionCorrection && language === 'ja' + // Apply corrections if enabled (language-independent) + const correctedText = this.enableTranscriptionCorrection ? await this.corrector.correct(originalText) : originalText; @@ -181,9 +181,14 @@ export class TranscriptionService implements ITranscriptionProvider { } /** - * Build prompt for GPT-4o transcription + * Build prompt for GPT-4o transcription (only for Japanese) */ - private buildTranscriptionPrompt(): string { + private buildTranscriptionPrompt(language: string): string | null { + // Only provide prompts for Japanese language + if (language !== 'ja') { + return null; + } + return `以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。 話者の発言内容だけを正確に記録してください。 @@ -194,9 +199,9 @@ export class TranscriptionService implements ITranscriptionProvider { } /** - * Clean GPT-4o specific response artifacts + * Clean GPT-4o specific response artifacts with language-specific patterns */ - private cleanGPT4oResponse(text: string): string { + private cleanGPT4oResponse(text: string, language: string): string { // First attempt: Extract content from complete TRANSCRIPT tags let transcriptMatch = text.match(/\s*([\s\S]*?)\s*<\/TRANSCRIPT>/); if (transcriptMatch) { @@ -212,18 +217,34 @@ export class TranscriptionService implements ITranscriptionProvider { // Remove TRANSCRIPT opening tag if still present (for cases where it's not properly extracted) text = text.replace(/<\/?TRANSCRIPT[^>]*>/g, ''); - // Remove specific meta instruction patterns (both at line start and anywhere in text) - const metaPatterns = [ - /^以下の音声内容.*?$/gm, - /^この指示文.*?$/gm, - /^話者の発言内容だけを正確に記録してください.*?$/gm, - /^話者の発言.*?$/gm, - /^出力形式.*?$/gm, - /（話者の発言のみ）/g, // Remove this specific phrase anywhere in the text + // Language-specific cleaning patterns + if (language === 'ja') { + // Japanese-specific meta instruction patterns + const jaMetaPatterns = [ + /^以下の音声内容.*?$/gm, + /^この指示文.*?$/gm, + /^話者の発言内容だけを正確に記録してください.*?$/gm, + /^話者の発言.*?$/gm, + /^出力形式.*?$/gm, + /（話者の発言のみ）/g, // Remove this specific phrase anywhere in the text + ]; + + // Apply Japanese-specific cleaning patterns + for (const pattern of jaMetaPatterns) { + text = text.replace(pattern, ''); + } + } + + // Generic patterns (colon-based instructions) for all languages + const genericPatterns = [ + /^Output format:\s*.*?$/gmi, + /^Format:\s*.*?$/gmi, + /^Transcription:\s*.*?$/gmi, + /^Instructions:\s*.*?$/gmi, ]; - // Apply all cleaning patterns - for (const pattern of metaPatterns) { + // Apply generic cleaning patterns + for (const pattern of genericPatterns) { text = text.replace(pattern, ''); } diff --git a/src/i18n/en.ts b/src/i18n/en.ts index 9049b18..765e2f2 100644 --- a/src/i18n/en.ts +++ b/src/i18n/en.ts @@ -140,7 +140,7 @@ export const en: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input Settings', + settings: 'Voice Input Settings' }, settings: { apiKey: 'OpenAI API Key', @@ -154,7 +154,7 @@ export const en: TranslationResource = { maxRecordingDuration: 'Max Recording Duration', maxRecordingDurationDesc: 'Maximum recording time in seconds ({min}s - {max}min)', pluginLanguage: 'Plugin Language', - pluginLanguageDesc: 'Set language for UI, voice processing, and correction dictionary', + pluginLanguageDesc: 'Set language for UI display (voice recognition language is auto-detected separately)', customDictionary: 'Custom Dictionary', customDictionaryDesc: 'Manage corrections used for post-processing', dictionaryDefinite: 'Definite Corrections (max {max})', @@ -182,6 +182,6 @@ export const en: TranslationResource = { fromMultiple: 'From (comma-separated)', to: 'To', context: 'Context Keywords' - }, + } } }; diff --git a/src/i18n/ja.ts b/src/i18n/ja.ts index 0ae49c2..710ece2 100644 --- a/src/i18n/ja.ts +++ b/src/i18n/ja.ts @@ -140,7 +140,7 @@ export const ja: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 設定', + settings: 'Voice Input 設定' }, settings: { apiKey: 'OpenAI API キー', @@ -182,6 +182,6 @@ export const ja: TranslationResource = { fromMultiple: '入力語（カンマ区切り）', to: '修正語', context: '文脈キーワード' - }, + } } }; diff --git a/src/i18n/ko.ts b/src/i18n/ko.ts index a52ab9a..6433092 100644 --- a/src/i18n/ko.ts +++ b/src/i18n/ko.ts @@ -140,7 +140,7 @@ export const ko: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 설정', + settings: 'Voice Input 설정' }, settings: { apiKey: 'OpenAI API 키', @@ -182,6 +182,6 @@ export const ko: TranslationResource = { fromMultiple: '입력어（쉼표구분）', to: '교정어', context: '문맥 키워드' - }, + } } }; diff --git a/src/i18n/zh.ts b/src/i18n/zh.ts index 724ac1a..20cee1e 100644 --- a/src/i18n/zh.ts +++ b/src/i18n/zh.ts @@ -140,7 +140,7 @@ export const zh: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 设置', + settings: 'Voice Input 设置' }, settings: { apiKey: 'OpenAI API 密钥', @@ -182,6 +182,6 @@ export const zh: TranslationResource = { fromMultiple: '输入词（逗号分隔）', to: '校正词', context: '语境关键词' - }, + } } }; diff --git a/src/interfaces/settings.ts b/src/interfaces/settings.ts index 045221f..bbedd5c 100644 --- a/src/interfaces/settings.ts +++ b/src/interfaces/settings.ts @@ -8,8 +8,9 @@ export interface VoiceInputSettings { transcriptionModel: 'gpt-4o-transcribe' | 'gpt-4o-mini-transcribe'; // 録音設定 maxRecordingSeconds: number; // 最大録音時間（秒） - // プラグイン言語設定 - pluginLanguage: Locale; // プラグイン全体の言語（UI、音声認識、補正辞書） + // 言語設定 + transcriptionLanguage: 'auto' | 'ja' | 'en' | 'zh' | 'ko'; // 音声認識の言語 + pluginLanguage: Locale; // プラグインUI表示の言語 customDictionary: SimpleCorrectionDictionary; // デバッグ設定 debugMode: boolean; // デバッグモード @@ -22,7 +23,8 @@ export const DEFAULT_SETTINGS: VoiceInputSettings = { transcriptionModel: 'gpt-4o-transcribe', // 録音設定 maxRecordingSeconds: 300, // 5分（300秒） - // プラグイン言語はObsidianの設定に従う（loadSettingsで設定） + // 言語設定 + transcriptionLanguage: 'auto', // 自動検出 pluginLanguage: 'en', // 初期値、実際はObsidianの設定に従う customDictionary: { definiteCorrections: [] }, // デバッグ設定 diff --git a/src/plugin/VoiceInputPlugin.ts b/src/plugin/VoiceInputPlugin.ts index 7463fee..033b62c 100644 --- a/src/plugin/VoiceInputPlugin.ts +++ b/src/plugin/VoiceInputPlugin.ts @@ -303,12 +303,20 @@ export default class VoiceInputPlugin extends Plugin { needsSave = true; this.logger?.info(`Auto-detected language: ${this.settings.pluginLanguage} (from Obsidian: ${obsidianLocale})`); } + + // transcriptionLanguageが設定されていない場合 + if (!hasSettingsKey(data, 'transcriptionLanguage')) { + this.settings.transcriptionLanguage = 'auto'; // 新規フィールドはデフォルトで自動検出 + needsSave = true; + this.logger?.info('transcriptionLanguage not found, using default: auto'); + } } else { // 保存データが存在しない場合（初回起動） const obsidianLocale = this.getObsidianLocale(); this.settings.pluginLanguage = obsidianLocale.startsWith('ja') ? 'ja' : 'en'; + this.settings.transcriptionLanguage = 'auto'; // 初回起動時は自動検出 needsSave = true; - this.logger?.info(`First run - auto-detected language: ${this.settings.pluginLanguage}`); + this.logger?.info(`First run - auto-detected language: ${this.settings.pluginLanguage}, transcriptionLanguage: auto`); } // 必要に応じて設定を保存 @@ -333,6 +341,34 @@ export default class VoiceInputPlugin extends Plugin { return getObsidianLocale(this.app); } + /** + * 音声認識用の言語を解決する + * transcriptionLanguage が 'auto' の場合は Obsidian のロケールから自動判定 + */ + getResolvedLanguage(): 'ja' | 'en' | 'zh' | 'ko' { + if (this.settings.transcriptionLanguage === 'auto') { + return this.detectPluginLanguage(); + } + return this.settings.transcriptionLanguage as 'ja' | 'en' | 'zh' | 'ko'; + } + + /** + * Obsidian のロケールから対応言語を検出 + */ + detectPluginLanguage(): 'ja' | 'en' | 'zh' | 'ko' { + const obsidianLocale = this.getObsidianLocale(); + + if (obsidianLocale.startsWith('ja')) { + return 'ja'; + } else if (obsidianLocale.startsWith('zh')) { + return 'zh'; + } else if (obsidianLocale.startsWith('ko')) { + return 'ko'; + } else { + return 'en'; // デフォルト + } + } + async saveSettings() { // APIキーを暗号化して保存 const dataToSave = { diff --git a/src/views/VoiceInputViewActions.ts b/src/views/VoiceInputViewActions.ts index 430f0fa..c674dcb 100644 --- a/src/views/VoiceInputViewActions.ts +++ b/src/views/VoiceInputViewActions.ts @@ -455,7 +455,7 @@ export class VoiceInputViewActions { if (!this.transcriptionService) { throw new Error('Transcription service not available'); } - const result = await this.transcriptionService.transcribeAudio(audioBlob, this.plugin.settings.pluginLanguage); + const result = await this.transcriptionService.transcribeAudio(audioBlob, this.plugin.getResolvedLanguage()); // Check if result is empty if (!result.text || result.text.trim() === '') { From 956ebe5ca3bf0233aab7c6799016a0405433ad46 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 16:37:33 +0000 Subject: [PATCH 22/82] fix(transcription): limit generic cleaning patterns to safer colon-based instructions only Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- src/core/transcription/TranscriptionService.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 615c620..6691662 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -237,10 +237,8 @@ export class TranscriptionService implements ITranscriptionProvider { // Generic patterns (colon-based instructions) for all languages const genericPatterns = [ - /^Output format:\s*.*?$/gmi, - /^Format:\s*.*?$/gmi, - /^Transcription:\s*.*?$/gmi, - /^Instructions:\s*.*?$/gmi, + /^Output\s*format\s*:.*/i, + /^Format\s*:.*/i, ]; // Apply generic cleaning patterns From 53c99144d417538cb97a528b024e93e5b54b970f Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Wed, 13 Aug 2025 02:10:40 +0900 Subject: [PATCH 23/82] merge(PR#20): resolve conflicts per agreed rules (docs + service + settings) From 23c50663ebbf98428c7685605da0aacb0fef9972 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 17:19:51 +0000 Subject: [PATCH 24/82] Initial plan From b523991dc82e36f2e9ff1f3036f64ae9916fd20e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 17:33:30 +0000 Subject: [PATCH 25/82] Implement zh/ko language auto-detection with getResolvedLanguage() Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- src/i18n/en.ts | 4 +- src/i18n/ja.ts | 4 +- src/i18n/ko.ts | 4 +- src/i18n/zh.ts | 4 +- src/plugin/VoiceInputPlugin.ts | 45 ++++++++++-- src/views/VoiceInputViewActions.ts | 2 +- tests/language-detection.test.ts | 114 +++++++++++++++++++++++++++++ tests/mocks/obsidian.ts | 20 +++++ tests/setup.ts | 1 + 9 files changed, 184 insertions(+), 14 deletions(-) create mode 100644 tests/language-detection.test.ts create mode 100644 tests/mocks/obsidian.ts create mode 100644 tests/setup.ts diff --git a/src/i18n/en.ts b/src/i18n/en.ts index 9049b18..6b9108f 100644 --- a/src/i18n/en.ts +++ b/src/i18n/en.ts @@ -140,7 +140,7 @@ export const en: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input Settings', + settings: 'Voice Input Settings' }, settings: { apiKey: 'OpenAI API Key', @@ -182,6 +182,6 @@ export const en: TranslationResource = { fromMultiple: 'From (comma-separated)', to: 'To', context: 'Context Keywords' - }, + } } }; diff --git a/src/i18n/ja.ts b/src/i18n/ja.ts index 0ae49c2..710ece2 100644 --- a/src/i18n/ja.ts +++ b/src/i18n/ja.ts @@ -140,7 +140,7 @@ export const ja: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 設定', + settings: 'Voice Input 設定' }, settings: { apiKey: 'OpenAI API キー', @@ -182,6 +182,6 @@ export const ja: TranslationResource = { fromMultiple: '入力語（カンマ区切り）', to: '修正語', context: '文脈キーワード' - }, + } } }; diff --git a/src/i18n/ko.ts b/src/i18n/ko.ts index a52ab9a..6433092 100644 --- a/src/i18n/ko.ts +++ b/src/i18n/ko.ts @@ -140,7 +140,7 @@ export const ko: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 설정', + settings: 'Voice Input 설정' }, settings: { apiKey: 'OpenAI API 키', @@ -182,6 +182,6 @@ export const ko: TranslationResource = { fromMultiple: '입력어（쉼표구분）', to: '교정어', context: '문맥 키워드' - }, + } } }; diff --git a/src/i18n/zh.ts b/src/i18n/zh.ts index 724ac1a..20cee1e 100644 --- a/src/i18n/zh.ts +++ b/src/i18n/zh.ts @@ -140,7 +140,7 @@ export const zh: TranslationResource = { }, titles: { main: 'Voice Input', - settings: 'Voice Input 设置', + settings: 'Voice Input 设置' }, settings: { apiKey: 'OpenAI API 密钥', @@ -182,6 +182,6 @@ export const zh: TranslationResource = { fromMultiple: '输入词（逗号分隔）', to: '校正词', context: '语境关键词' - }, + } } }; diff --git a/src/plugin/VoiceInputPlugin.ts b/src/plugin/VoiceInputPlugin.ts index 7463fee..21db2fe 100644 --- a/src/plugin/VoiceInputPlugin.ts +++ b/src/plugin/VoiceInputPlugin.ts @@ -298,15 +298,13 @@ export default class VoiceInputPlugin extends Plugin { // pluginLanguageが設定されていない場合 if (!hasSettingsKey(data, 'pluginLanguage') && !hasSettingsKey(data, 'interfaceLanguage')) { - const obsidianLocale = this.getObsidianLocale(); - this.settings.pluginLanguage = obsidianLocale.startsWith('ja') ? 'ja' : 'en'; + this.settings.pluginLanguage = this.detectPluginLanguage(); needsSave = true; - this.logger?.info(`Auto-detected language: ${this.settings.pluginLanguage} (from Obsidian: ${obsidianLocale})`); + this.logger?.info(`Auto-detected language: ${this.settings.pluginLanguage} (from Obsidian: ${this.getObsidianLocale()})`); } } else { // 保存データが存在しない場合（初回起動） - const obsidianLocale = this.getObsidianLocale(); - this.settings.pluginLanguage = obsidianLocale.startsWith('ja') ? 'ja' : 'en'; + this.settings.pluginLanguage = this.detectPluginLanguage(); needsSave = true; this.logger?.info(`First run - auto-detected language: ${this.settings.pluginLanguage}`); } @@ -333,6 +331,43 @@ export default class VoiceInputPlugin extends Plugin { return getObsidianLocale(this.app); } + /** + * プラグイン言語を自動検出（ja/zh/ko/en） + */ + private detectPluginLanguage(): 'ja' | 'zh' | 'ko' | 'en' { + const obsidianLocale = this.getObsidianLocale().toLowerCase(); + + if (obsidianLocale.startsWith('ja')) { + return 'ja'; + } else if (obsidianLocale.startsWith('zh')) { + return 'zh'; + } else if (obsidianLocale.startsWith('ko')) { + return 'ko'; + } else { + return 'en'; + } + } + + /** + * 解決済み言語を取得（transcriptionLanguage が 'auto' の場合は自動検出） + */ + getResolvedLanguage(): 'ja' | 'zh' | 'ko' | 'en' { + // 現在の実装では pluginLanguage を transcriptionLanguage として使用 + // 'auto' の概念は今後の拡張のために準備 + const transcriptionLanguage = this.settings.pluginLanguage; + + if (transcriptionLanguage === 'auto' as string) { + return this.detectPluginLanguage(); + } + + // pluginLanguage が ja/zh/ko/en 以外の場合は自動検出にフォールバック + if (!['ja', 'zh', 'ko', 'en'].includes(transcriptionLanguage)) { + return this.detectPluginLanguage(); + } + + return transcriptionLanguage as 'ja' | 'zh' | 'ko' | 'en'; + } + async saveSettings() { // APIキーを暗号化して保存 const dataToSave = { diff --git a/src/views/VoiceInputViewActions.ts b/src/views/VoiceInputViewActions.ts index 430f0fa..c674dcb 100644 --- a/src/views/VoiceInputViewActions.ts +++ b/src/views/VoiceInputViewActions.ts @@ -455,7 +455,7 @@ export class VoiceInputViewActions { if (!this.transcriptionService) { throw new Error('Transcription service not available'); } - const result = await this.transcriptionService.transcribeAudio(audioBlob, this.plugin.settings.pluginLanguage); + const result = await this.transcriptionService.transcribeAudio(audioBlob, this.plugin.getResolvedLanguage()); // Check if result is empty if (!result.text || result.text.trim() === '') { diff --git a/tests/language-detection.test.ts b/tests/language-detection.test.ts new file mode 100644 index 0000000..79779e5 --- /dev/null +++ b/tests/language-detection.test.ts @@ -0,0 +1,114 @@ +import { getObsidianLocale } from '../src/types'; + +// Mock to test language detection logic +jest.mock('../src/types', () => ({ + getObsidianLocale: jest.fn() +})); + +// Extract the detection logic for isolated testing +function detectPluginLanguage(getObsidianLocaleFn: () => string): 'ja' | 'zh' | 'ko' | 'en' { + const obsidianLocale = getObsidianLocaleFn().toLowerCase(); + + if (obsidianLocale.startsWith('ja')) { + return 'ja'; + } else if (obsidianLocale.startsWith('zh')) { + return 'zh'; + } else if (obsidianLocale.startsWith('ko')) { + return 'ko'; + } else { + return 'en'; + } +} + +function getResolvedLanguage( + pluginLanguage: string, + detectFn: () => 'ja' | 'zh' | 'ko' | 'en' +): 'ja' | 'zh' | 'ko' | 'en' { + if (pluginLanguage === 'auto') { + return detectFn(); + } + + if (!['ja', 'zh', 'ko', 'en'].includes(pluginLanguage)) { + return detectFn(); + } + + return pluginLanguage as 'ja' | 'zh' | 'ko' | 'en'; +} + +describe('Language Detection Logic', () => { + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('detectPluginLanguage', () => { + test('should detect Japanese locale', () => { + const mockGetObsidianLocale = jest.fn(() => 'ja-JP'); + const result = detectPluginLanguage(mockGetObsidianLocale); + expect(result).toBe('ja'); + }); + + test('should detect Chinese locale', () => { + const mockGetObsidianLocale = jest.fn(() => 'zh-CN'); + const result = detectPluginLanguage(mockGetObsidianLocale); + expect(result).toBe('zh'); + }); + + test('should detect Korean locale', () => { + const mockGetObsidianLocale = jest.fn(() => 'ko-KR'); + const result = detectPluginLanguage(mockGetObsidianLocale); + expect(result).toBe('ko'); + }); + + test('should default to English for unknown locales', () => { + const mockGetObsidianLocale = jest.fn(() => 'fr-FR'); + const result = detectPluginLanguage(mockGetObsidianLocale); + expect(result).toBe('en'); + }); + + test('should handle case insensitive locale matching', () => { + const mockGetObsidianLocale = jest.fn(() => 'ZH-TW'); + const result = detectPluginLanguage(mockGetObsidianLocale); + expect(result).toBe('zh'); + }); + + test('should handle edge cases', () => { + const mockGetObsidianLocale = jest.fn(() => ''); + const result = detectPluginLanguage(mockGetObsidianLocale); + expect(result).toBe('en'); + }); + }); + + describe('getResolvedLanguage', () => { + const mockDetectFn = jest.fn(() => 'ko' as const); + + beforeEach(() => { + mockDetectFn.mockClear(); + }); + + test('should return plugin language when set to valid value', () => { + const result = getResolvedLanguage('ja', mockDetectFn); + expect(result).toBe('ja'); + expect(mockDetectFn).not.toHaveBeenCalled(); + }); + + test('should fall back to auto-detection for invalid plugin language', () => { + const result = getResolvedLanguage('invalid', mockDetectFn); + expect(result).toBe('ko'); + expect(mockDetectFn).toHaveBeenCalledTimes(1); + }); + + test('should handle auto value', () => { + const result = getResolvedLanguage('auto', mockDetectFn); + expect(result).toBe('ko'); + expect(mockDetectFn).toHaveBeenCalledTimes(1); + }); + + test('should work with all supported languages', () => { + expect(getResolvedLanguage('ja', mockDetectFn)).toBe('ja'); + expect(getResolvedLanguage('zh', mockDetectFn)).toBe('zh'); + expect(getResolvedLanguage('ko', mockDetectFn)).toBe('ko'); + expect(getResolvedLanguage('en', mockDetectFn)).toBe('en'); + expect(mockDetectFn).not.toHaveBeenCalled(); + }); + }); +}); \ No newline at end of file diff --git a/tests/mocks/obsidian.ts b/tests/mocks/obsidian.ts new file mode 100644 index 0000000..5e5df20 --- /dev/null +++ b/tests/mocks/obsidian.ts @@ -0,0 +1,20 @@ +// Mock Obsidian API for tests +export const App = jest.fn(); +export const Plugin = jest.fn(); +export const Notice = jest.fn(); +export const Setting = jest.fn(); +export const ItemView = jest.fn(); +export const WorkspaceLeaf = jest.fn(); +export const moment = { + locale: jest.fn(() => 'en') +}; + +export default { + App, + Plugin, + Notice, + Setting, + ItemView, + WorkspaceLeaf, + moment +}; \ No newline at end of file diff --git a/tests/setup.ts b/tests/setup.ts new file mode 100644 index 0000000..91e7e70 --- /dev/null +++ b/tests/setup.ts @@ -0,0 +1 @@ +// Jest setup file for test environment \ No newline at end of file From e35254a15618f5647f782f26b33393f89605f69b Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Wed, 13 Aug 2025 03:10:24 +0900 Subject: [PATCH 26/82] merge(PR#21): resolve conflicts and unify detection logic (zh/ko autodetect), tests mocks setup From cc4d0fc961c98d8a37f1abb616cff7ca9128bf91 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 18:46:12 +0000 Subject: [PATCH 27/82] Initial plan From 538280ab7f8eb1787d24b5816f50e98b2c3671b3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 18:54:56 +0000 Subject: [PATCH 28/82] Fix merge conflicts and build errors - prepare for multilingual dictionary correction analysis Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- .../transcription/TranscriptionService.ts | 31 ------------------- src/plugin/VoiceInputPlugin.ts | 9 +++--- 2 files changed, 4 insertions(+), 36 deletions(-) diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index b09531a..fc894d4 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -213,42 +213,11 @@ export class TranscriptionService implements ITranscriptionProvider { // Remove TRANSCRIPT opening tag if still present (for cases where it's not properly extracted) text = text.replace(/<\/?TRANSCRIPT[^>]*>/g, ''); -<<<<<<< HEAD - // Language-specific cleaning patterns - if (language === 'ja') { - // Japanese-specific meta instruction patterns - const jaMetaPatterns = [ - /^以下の音声内容.*?$/gm, - /^この指示文.*?$/gm, - /^話者の発言内容だけを正確に記録してください.*?$/gm, - /^話者の発言.*?$/gm, - /^出力形式.*?$/gm, - /（話者の発言のみ）/g, // Remove this specific phrase anywhere in the text - ]; - - // Apply Japanese-specific cleaning patterns - for (const pattern of jaMetaPatterns) { - text = text.replace(pattern, ''); - } - } - - // Generic patterns (colon-based instructions) for all languages - const genericPatterns = [ - /^Output\s*format\s*:.*/i, - /^Format\s*:.*/i, - ]; - - // Apply generic cleaning patterns - for (const pattern of genericPatterns) { - text = text.replace(pattern, ''); - } -======= // Apply language-specific cleaning text = this.applyLanguageSpecificCleaning(text, normalizedLang); // Apply generic cleaning (only colon-based patterns to prevent over-removal) text = this.applyGenericCleaning(text); ->>>>>>> origin/feat/multilingual-improvements // Clean up extra whitespace and empty lines text = text.trim(); diff --git a/src/plugin/VoiceInputPlugin.ts b/src/plugin/VoiceInputPlugin.ts index bffeda8..fbdc3d0 100644 --- a/src/plugin/VoiceInputPlugin.ts +++ b/src/plugin/VoiceInputPlugin.ts @@ -320,8 +320,7 @@ export default class VoiceInputPlugin extends Plugin { this.settings.pluginLanguage = this.detectPluginLanguage(); needsSave = true; this.logger?.info(`Auto-detected language: ${this.settings.pluginLanguage} (from Obsidian: ${this.getObsidianLocale()})`); - needsSave = true; - this.logger?.info(`First run - auto-detected language: ${this.settings.pluginLanguage}, transcriptionLanguage: auto`); + } } // 必要に応じて設定を保存 @@ -340,14 +339,14 @@ export default class VoiceInputPlugin extends Plugin { } /** - * Obsidianの言語設定を取得 - */ + * Get Obsidian locale setting + */ private getObsidianLocale(): string { return getObsidianLocale(this.app); } /** - * プラグイン言語を自動検出（ja/zh/ko/en） + * Auto-detect plugin language (ja/zh/ko/en) */ private detectPluginLanguage(): 'ja' | 'zh' | 'ko' | 'en' { const obsidianLocale = this.getObsidianLocale().toLowerCase(); From 249c9943c00083f54e9f2aeeb4d166af04f596f2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 18:59:57 +0000 Subject: [PATCH 29/82] feat(transcription): enable multilingual dictionary correction for all languages - Fix settings synchronization between TranscriptionService and DictionaryCorrector - Add comprehensive test coverage for multilingual dictionary correction - Update README documentation to reflect multilingual support - Ensure enableTranscriptionCorrection setting applies consistently across all components Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- README.md | 4 +- .../transcription/TranscriptionService.ts | 8 +- ...multilingual-dictionary-correction.test.ts | 214 ++++++++++++++++++ ...on-service-multilingual-correction.test.ts | 187 +++++++++++++++ 4 files changed, 409 insertions(+), 4 deletions(-) create mode 100644 tests/unit/core/transcription/multilingual-dictionary-correction.test.ts create mode 100644 tests/unit/core/transcription/transcription-service-multilingual-correction.test.ts diff --git a/README.md b/README.md index 06b554b..1d90491 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ High-accuracy voice input for Obsidian. Uses OpenAI GPT-4o Audio Transcriptions - Model selection: GPT‑4o Transcribe or GPT‑4o mini Transcribe - Language separation: independent UI language and voice recognition language settings - Auto language detection: automatic voice recognition language based on Obsidian locale -- AI post‑processing: optional dictionary-based cleanup (Japanese only) +- AI post‑processing: optional dictionary-based cleanup (applied to all languages when enabled) - Quick controls in view: copy/clear/insert at cursor/append to end - Auto‑save drafts: periodic and on blur, automatic restore - Multilingual support: Japanese, English, Chinese, Korean interface languages @@ -51,7 +51,7 @@ Tip: A settings gear in the view header opens the plugin’s settings. - OpenAI API Key: stored locally (encrypted at rest) - Transcription Model: `gpt-4o-transcribe` or `gpt-4o-mini-transcribe` - **Transcription Language**: Auto/Japanese/English/Chinese/Korean (auto-detection recommended) -- AI Post‑processing: enable dictionary‑based cleanup (applied to Japanese only) +- AI Post‑processing: enable dictionary‑based cleanup (applied to all languages when enabled) - Maximum Recording Duration: slider (default 5 min) - Plugin Language: English/Japanese (controls UI display only, auto‑detected from Obsidian, adjustable) diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index fc894d4..4ba070f 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -419,6 +419,9 @@ export class TranscriptionService implements ITranscriptionProvider { */ setTranscriptionCorrection(enabled: boolean) { this.enableTranscriptionCorrection = enabled; + this.corrector.updateSettings({ + enabled: enabled + }); } /** @@ -428,9 +431,10 @@ export class TranscriptionService implements ITranscriptionProvider { this.apiKey = apiKey; // API key is no longer needed for the simplified corrector // Preserve existing dictionary settings when updating API key - const currentDict = this.corrector.getSettings().correctionDictionary; + const currentSettings = this.corrector.getSettings(); this.corrector = new DictionaryCorrector({ - correctionDictionary: currentDict + enabled: currentSettings.enabled, + correctionDictionary: currentSettings.correctionDictionary }); } diff --git a/tests/unit/core/transcription/multilingual-dictionary-correction.test.ts b/tests/unit/core/transcription/multilingual-dictionary-correction.test.ts new file mode 100644 index 0000000..ad86158 --- /dev/null +++ b/tests/unit/core/transcription/multilingual-dictionary-correction.test.ts @@ -0,0 +1,214 @@ +/** + * Tests for multilingual dictionary correction functionality + * Validates that dictionary correction applies to all languages when enabled + */ + +import { DictionaryCorrector } from '../../../../src/core/transcription/DictionaryCorrector'; +import { SimpleCorrectionDictionary } from '../../../../src/interfaces'; + +describe('Multilingual Dictionary Correction', () => { + let corrector: DictionaryCorrector; + + // Sample dictionary with common corrections + const testDictionary: SimpleCorrectionDictionary = { + definiteCorrections: [ + { from: ['AI'], to: 'artificial intelligence' }, + { from: ['GPT'], to: 'Generative Pre-trained Transformer' }, + { from: ['API'], to: 'Application Programming Interface' }, + { from: ['UI'], to: 'User Interface' }, + { from: ['あなた'], to: 'あなた様' }, // Japanese correction + { from: ['hello'], to: 'Hello' }, // English correction + { from: ['你好'], to: '您好' }, // Chinese correction (informal to formal) + { from: ['안녕'], to: '안녕하세요' }, // Korean correction (informal to formal) + ] + }; + + beforeEach(() => { + corrector = new DictionaryCorrector({ + enabled: true, + correctionDictionary: testDictionary + }); + }); + + describe('Dictionary correction applies to all languages', () => { + it('should apply corrections to English text', async () => { + const input = 'The AI uses GPT technology for API calls with a modern UI.'; + const result = await corrector.correct(input); + + expect(result).toContain('artificial intelligence'); + expect(result).toContain('Generative Pre-trained Transformer'); + expect(result).toContain('Application Programming Interface'); + expect(result).toContain('User Interface'); + expect(result).not.toContain('AI '); + expect(result).not.toContain('GPT '); + expect(result).not.toContain('API '); + expect(result).not.toContain('UI.'); + }); + + it('should apply corrections to Japanese text', async () => { + const input = 'あなたは AI を使って GPT で API を呼び出します。'; + const result = await corrector.correct(input); + + expect(result).toContain('あなた様'); + expect(result).toContain('artificial intelligence'); + expect(result).toContain('Generative Pre-trained Transformer'); + expect(result).toContain('Application Programming Interface'); + expect(result).not.toContain('あなたは'); + }); + + it('should apply corrections to Chinese text', async () => { + const input = '你好，AI 使用 GPT 技术通过 API 调用。'; + const result = await corrector.correct(input); + + expect(result).toContain('您好'); + expect(result).toContain('artificial intelligence'); + expect(result).toContain('Generative Pre-trained Transformer'); + expect(result).toContain('Application Programming Interface'); + expect(result).not.toContain('你好'); + }); + + it('should apply corrections to Korean text', async () => { + const input = '안녕, AI가 GPT 기술로 API를 호출합니다.'; + const result = await corrector.correct(input); + + expect(result).toContain('안녕하세요'); + expect(result).toContain('artificial intelligence'); + expect(result).toContain('Generative Pre-trained Transformer'); + expect(result).toContain('Application Programming Interface'); + expect(result).not.toContain('안녕,'); + }); + + it('should apply corrections to mixed language text', async () => { + const input = 'Hello, あなた! The AI system uses GPT. 你好 API!'; + const result = await corrector.correct(input); + + expect(result).toContain('Hello'); + expect(result).toContain('あなた様'); + expect(result).toContain('artificial intelligence'); + expect(result).toContain('Generative Pre-trained Transformer'); + expect(result).toContain('您好'); + expect(result).toContain('Application Programming Interface'); + }); + }); + + describe('Dictionary correction can be disabled', () => { + it('should not apply corrections when disabled', async () => { + const disabledCorrector = new DictionaryCorrector({ + enabled: false, + correctionDictionary: testDictionary + }); + + const input = 'The AI uses GPT technology for API calls with a modern UI.'; + const result = await disabledCorrector.correct(input); + + // Should return original text unchanged + expect(result).toBe(input); + expect(result).toContain('AI'); + expect(result).toContain('GPT'); + expect(result).toContain('API'); + expect(result).toContain('UI'); + }); + }); + + describe('Empty or missing dictionary handling', () => { + it('should handle empty dictionary gracefully', async () => { + const emptyCorrector = new DictionaryCorrector({ + enabled: true, + correctionDictionary: { definiteCorrections: [] } + }); + + const inputs = [ + 'English text with AI and GPT', + 'あなたは日本語です', + '你好中文', + '안녕하세요 한국어' + ]; + + for (const input of inputs) { + const result = await emptyCorrector.correct(input); + expect(result).toBe(input); // Should return unchanged + } + }); + + it('should handle missing dictionary gracefully', async () => { + const noDictCorrector = new DictionaryCorrector({ + enabled: true + // No correctionDictionary provided + }); + + const inputs = [ + 'English text with AI and GPT', + 'あなたは日本語です', + '你好中文', + '안녕하세요 한국어' + ]; + + for (const input of inputs) { + const result = await noDictCorrector.correct(input); + expect(result).toBe(input); // Should return unchanged + } + }); + }); + + describe('Edge cases', () => { + it('should handle short text correctly', async () => { + const shortTexts = ['AI', 'GPT', 'API', 'UI']; + + for (const text of shortTexts) { + const result = await corrector.correct(text); + expect(result).not.toBe(text); // Should be corrected + expect(result.length).toBeGreaterThan(text.length); + } + }); + + it('should handle empty or very short text', async () => { + const emptyTexts = ['', ' ', 'a']; + + for (const text of emptyTexts) { + const result = await corrector.correct(text); + // Should return as-is for very short text based on implementation + expect(result).toBe(text); + } + }); + + it('should handle text without any corrections needed', async () => { + const cleanTexts = [ + 'This is clean English text.', + 'これは綺麗な日本語です。', + '这是干净的中文文本。', + '이것은 깨끗한 한국어 텍스트입니다.' + ]; + + for (const text of cleanTexts) { + const result = await corrector.correct(text); + expect(result).toBe(text); // Should return unchanged + } + }); + }); + + describe('Case sensitivity and pattern matching', () => { + it('should match corrections case-sensitively', async () => { + const input = 'The ai and AI are different, also gpt vs GPT.'; + const result = await corrector.correct(input); + + // Only 'AI' and 'GPT' (exact case matches) should be corrected + expect(result).toContain('artificial intelligence'); + expect(result).toContain('Generative Pre-trained Transformer'); + expect(result).toContain('ai and'); // lowercase 'ai' should remain + expect(result).toContain('gpt vs'); // lowercase 'gpt' should remain + }); + + it('should handle multiple occurrences of the same pattern', async () => { + const input = 'AI helps with AI development. GPT-3 and GPT-4 are AI models.'; + const result = await corrector.correct(input); + + // All instances of 'AI' and 'GPT' should be corrected + expect(result).not.toContain('AI '); + expect(result).not.toContain('GPT-'); + const aiCount = (result.match(/artificial intelligence/g) || []).length; + const gptCount = (result.match(/Generative Pre-trained Transformer/g) || []).length; + expect(aiCount).toBeGreaterThanOrEqual(2); + expect(gptCount).toBeGreaterThanOrEqual(2); + }); + }); +}); \ No newline at end of file diff --git a/tests/unit/core/transcription/transcription-service-multilingual-correction.test.ts b/tests/unit/core/transcription/transcription-service-multilingual-correction.test.ts new file mode 100644 index 0000000..7c05bb4 --- /dev/null +++ b/tests/unit/core/transcription/transcription-service-multilingual-correction.test.ts @@ -0,0 +1,187 @@ +/** + * Integration tests for TranscriptionService multilingual dictionary correction + * Validates that the service applies dictionary correction to all languages when enabled + */ + +import { TranscriptionService } from '../../../../src/core/transcription/TranscriptionService'; +import { SimpleCorrectionDictionary } from '../../../../src/interfaces'; + +// Mock the ObsidianHttpClient since we're not testing actual API calls +jest.mock('../../../../src/utils/ObsidianHttpClient', () => ({ + ObsidianHttpClient: { + postFormData: jest.fn() + } +})); + +// Mock the logger +jest.mock('../../../../src/services', () => ({ + createServiceLogger: jest.fn(() => ({ + info: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + warn: jest.fn() + })) +})); + +describe('TranscriptionService Multilingual Dictionary Correction Integration', () => { + let service: TranscriptionService; + + const testDictionary: SimpleCorrectionDictionary = { + definiteCorrections: [ + { from: ['AI'], to: 'artificial intelligence' }, + { from: ['GPT'], to: 'Generative Pre-trained Transformer' }, + { from: ['こんにちは'], to: 'こんにちは（修正済み）' }, + { from: ['你好'], to: '您好' }, + { from: ['안녕'], to: '안녕하세요' } + ] + }; + + beforeEach(() => { + service = new TranscriptionService('test-api-key', testDictionary); + // Enable dictionary correction + service.setTranscriptionCorrection(true); + }); + + // Helper to verify correction is applied via the service's corrector + describe('Direct corrector access verification', () => { + it('should provide access to corrector with multilingual capability', async () => { + const corrector = service.getCorrector(); + + // Test multiple languages + const testCases = [ + { input: 'This AI uses GPT', expected: 'artificial intelligence' }, + { input: 'こんにちは、AIです', expected: 'こんにちは（修正済み）' }, + { input: '你好，这是AI', expected: '您好' }, + { input: '안녕, AI 시스템', expected: '안녕하세요' } + ]; + + for (const testCase of testCases) { + const result = await corrector.correct(testCase.input); + expect(result).toContain(testCase.expected); + } + }); + + it('should respect the enableTranscriptionCorrection setting', async () => { + const corrector = service.getCorrector(); + + // Test with correction enabled + service.setTranscriptionCorrection(true); + let result = await corrector.correct('This AI system uses GPT'); + expect(result).toContain('artificial intelligence'); + expect(result).toContain('Generative Pre-trained Transformer'); + + // Test with correction disabled + service.setTranscriptionCorrection(false); + result = await corrector.correct('This AI system uses GPT'); + expect(result).toBe('This AI system uses GPT'); // Should remain unchanged + }); + }); + + describe('Dictionary update functionality', () => { + it('should allow updating dictionary and apply new corrections to all languages', async () => { + const newDictionary: SimpleCorrectionDictionary = { + definiteCorrections: [ + { from: ['ML'], to: 'Machine Learning' }, + { from: ['さようなら'], to: 'さようなら（丁寧）' }, + { from: ['再见'], to: '再見' }, + { from: ['가다'], to: '가시다' } + ] + }; + + // Update dictionary + service.setCustomDictionary(newDictionary); + const corrector = service.getCorrector(); + + // Test that new corrections work for multiple languages + const testCases = [ + { input: 'ML is powerful', expected: 'Machine Learning' }, + { input: 'さようなら、また明日', expected: 'さようなら（丁寧）' }, + { input: '再见朋友', expected: '再見' }, + { input: '가다 오다', expected: '가시다' } + ]; + + for (const testCase of testCases) { + const result = await corrector.correct(testCase.input); + expect(result).toContain(testCase.expected); + } + + // Test that old corrections are replaced + const oldResult = await corrector.correct('This AI uses GPT'); + expect(oldResult).toBe('This AI uses GPT'); // Old corrections should not apply + }); + }); + + describe('Settings update integration', () => { + it('should respect corrector settings updates for all languages', async () => { + const corrector = service.getCorrector(); + + // Initial test - corrections should work + let result = await corrector.correct('AI and GPT systems こんにちは'); + expect(result).toContain('artificial intelligence'); + expect(result).toContain('Generative Pre-trained Transformer'); + expect(result).toContain('こんにちは（修正済み）'); + + // Update settings to disable corrections + service.updateCorrectorSettings({ enabled: false }); + + // Corrections should no longer apply to any language + result = await corrector.correct('AI and GPT systems こんにちは'); + expect(result).toBe('AI and GPT systems こんにちは'); + + // Re-enable corrections + service.updateCorrectorSettings({ enabled: true }); + + // Corrections should work again for all languages + result = await corrector.correct('AI and GPT systems こんにちは'); + expect(result).toContain('artificial intelligence'); + expect(result).toContain('Generative Pre-trained Transformer'); + expect(result).toContain('こんにちは（修正済み）'); + }); + }); + + describe('API key update preservation', () => { + it('should preserve dictionary settings when API key is updated', async () => { + const corrector = service.getCorrector(); + + // Test initial corrections work + let result = await corrector.correct('AI and GPT systems'); + expect(result).toContain('artificial intelligence'); + + // Update API key + service.updateApiKey('new-test-api-key'); + + // Dictionary corrections should still work + const newCorrector = service.getCorrector(); + result = await newCorrector.correct('AI and GPT systems'); + expect(result).toContain('artificial intelligence'); + expect(result).toContain('Generative Pre-trained Transformer'); + }); + }); + + describe('Model update compatibility', () => { + it('should maintain dictionary correction functionality across model changes', async () => { + const corrector = service.getCorrector(); + + // Test with initial model + let result = await corrector.correct('AI uses GPT technology'); + expect(result).toContain('artificial intelligence'); + expect(result).toContain('Generative Pre-trained Transformer'); + + // Change model + service.setModel('gpt-4o-mini-transcribe'); + + // Dictionary corrections should still work + result = await corrector.correct('AI uses GPT technology'); + expect(result).toContain('artificial intelligence'); + expect(result).toContain('Generative Pre-trained Transformer'); + + // Change to other model + service.setModel('gpt-4o-transcribe'); + + // Dictionary corrections should still work + result = await corrector.correct('AI uses GPT technology'); + expect(result).toContain('artificial intelligence'); + expect(result).toContain('Generative Pre-trained Transformer'); + }); + }); +}); \ No newline at end of file From 727f163c853c492fdd9cbebb039f13c0e2872f0b Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Wed, 13 Aug 2025 04:58:59 +0900 Subject: [PATCH 30/82] fix(transcription): enforce conservative cleaning for en/zh/ko; delegate toggling to single path; preserve corrector state on API key update\ndocs(ja): clarify dictionary correction applies to all languages\nfix(plugin): restore first-run init in loadSettings and remove merge remnants # Conflicts: # src/core/transcription/TranscriptionService.ts # src/plugin/VoiceInputPlugin.ts --- .gitignore | 1 + README.md | 38 +++++++---------- .../transcription/TranscriptionService.ts | 42 +++---------------- src/plugin/VoiceInputPlugin.ts | 12 ++++-- 4 files changed, 30 insertions(+), 63 deletions(-) diff --git a/.gitignore b/.gitignore index e8c46e8..ea2951a 100644 --- a/.gitignore +++ b/.gitignore @@ -100,3 +100,4 @@ agents.md # Personal, not for public release scripts/ memo/ +tests/ diff --git a/README.md b/README.md index 1d90491..66cbe50 100644 --- a/README.md +++ b/README.md @@ -133,37 +133,27 @@ Third‑party licensing: see `THIRD_PARTY_LICENSES.md`. - OpenAI APIキー: ローカルに暗号化して保存 - 文字起こしモデル: `gpt-4o-transcribe` または `gpt-4o-mini-transcribe` - **音声認識言語**: 自動/日本語/英語/中国語/韓国語（自動検出を推奨） -- AI後処理: 辞書ベースの補正（日本語のみ適用） +- AI後処理: 辞書ベースの補正（有効時は全言語に適用） - 最大録音時間: スライダー（初期値5分） -<<<<<<< HEAD -- プラグイン言語: 英語/日本語（UI表示のみ制御、Obsidian設定から自動検出、変更可） -======= -- **プラグイン言語**: 英語/日本語/中国語/韓国語（UI表示言語、Obsidian設定から自動検出） +- **プラグイン言語**: UI表示のみを制御。Obsidianの言語設定から自動検出（ja/zh/ko/en）。 -### 言語設定について +### 言語設定 -プラグインは最適なユーザー体験のため、UI言語と音声認識言語を**完全に分離**しています： +- **プラグイン言語**: UI表示のみを制御。Obsidianの言語設定から自動検出（ja/zh/ko/en）。 +- **音声認識言語**: 音声認識/文字起こしの言語。既定は「自動」（Obsidianのロケールから ja/zh/ko/en を自動選択）。 -- **プラグイン言語**: インターフェースの言語（メニュー、ボタン、メッセージ）。Obsidianの言語設定から自動検出されます。 -- **音声認識言語**: 音声認識と文字起こしの言語。「自動」を推奨 - Obsidianのインターフェース言語に基づいて最適な言語を自動選択します。 +#### 自動検出の動作 -### 自動検出の動作 +- ja-* → 日本語 +- zh-* → 中国語 +- ko-* → 韓国語 +- その他 → 英語 -**音声認識言語**が「自動」に設定されている場合： -- **日本語ロケール** (ja-*) → 日本語での文字起こし -- **中国語ロケール** (zh-*) → 中国語での文字起こし -- **韓国語ロケール** (ko-*) → 韓国語での文字起こし -- **その他のロケール** → 英語での文字起こし +#### 言語別の処理 -### 言語別の処理 - -プラグインは選択された音声認識言語に基づいて異なる処理戦略を適用します： - -- **日本語 (ja)**: 高精度化のための専用プロンプト + 専門用語の辞書補正 -- **中国語 (zh)**: 言語固有処理なしのクリーンな文字起こし -- **英語/韓国語 (en/ko)**: 干渉を避けるため、プロンプトや補正なしのクリーンな文字起こし -- **自動モード**: Obsidianのインターフェース言語に基づいて最適な言語と処理戦略を動的に選択 ->>>>>>> origin/feat/multilingual-improvements +- 日本語 (ja): 精度向上のためプロンプトを付与＋辞書補正（有効時） +- 中国語/英語/韓国語 (zh/en/ko): プロンプトは付与しません。クリーニングは言語別＋汎用（コロン付き）で安全側に適用。 +- 自動: 上記の動作を自動選択。 ## セキュリティ / プライバシー diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 4ba070f..5d619dc 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -282,16 +282,7 @@ export class TranscriptionService implements ITranscriptionProvider { * Apply English-specific cleaning patterns */ private applyEnglishCleaning(text: string): string { - const patterns = [ - /^Please transcribe.*?$/gmi, - /^Transcribe only.*?$/gmi, - /^Output format.*?$/gmi, - /^Format.*?$/gmi, - ]; - - for (const pattern of patterns) { - text = text.replace(pattern, ''); - } + // For English, rely on conservative generic cleaning only return text; } @@ -299,16 +290,7 @@ export class TranscriptionService implements ITranscriptionProvider { * Apply Chinese-specific cleaning patterns */ private applyChineseCleaning(text: string): string { - const patterns = [ - /^请转录.*?$/gm, - /^仅转录.*?$/gm, - /^输出格式.*?$/gm, - /^格式.*?$/gm, - ]; - - for (const pattern of patterns) { - text = text.replace(pattern, ''); - } + // For Chinese, rely on conservative generic cleaning only return text; } @@ -316,16 +298,7 @@ export class TranscriptionService implements ITranscriptionProvider { * Apply Korean-specific cleaning patterns */ private applyKoreanCleaning(text: string): string { - const patterns = [ - /^다음 음성.*?$/gm, - /^음성 내용만.*?$/gm, - /^출력 형식.*?$/gm, - /^형식.*?$/gm, - ]; - - for (const pattern of patterns) { - text = text.replace(pattern, ''); - } + // For Korean, rely on conservative generic cleaning only return text; } @@ -418,10 +391,8 @@ export class TranscriptionService implements ITranscriptionProvider { * Set transcription correction enabled/disabled */ setTranscriptionCorrection(enabled: boolean) { - this.enableTranscriptionCorrection = enabled; - this.corrector.updateSettings({ - enabled: enabled - }); + // Delegate to a single code path to keep states in sync + this.updateCorrectorSettings({ enabled }); } /** @@ -429,8 +400,7 @@ export class TranscriptionService implements ITranscriptionProvider { */ updateApiKey(apiKey: string) { this.apiKey = apiKey; - // API key is no longer needed for the simplified corrector - // Preserve existing dictionary settings when updating API key + // Preserve existing corrector settings when updating API key const currentSettings = this.corrector.getSettings(); this.corrector = new DictionaryCorrector({ enabled: currentSettings.enabled, diff --git a/src/plugin/VoiceInputPlugin.ts b/src/plugin/VoiceInputPlugin.ts index fbdc3d0..3a74cf9 100644 --- a/src/plugin/VoiceInputPlugin.ts +++ b/src/plugin/VoiceInputPlugin.ts @@ -321,6 +321,12 @@ export default class VoiceInputPlugin extends Plugin { needsSave = true; this.logger?.info(`Auto-detected language: ${this.settings.pluginLanguage} (from Obsidian: ${this.getObsidianLocale()})`); } + } else { + // 保存データが存在しない場合（初回起動） + this.settings.pluginLanguage = this.detectPluginLanguage(); + this.settings.transcriptionLanguage = 'auto'; + needsSave = true; + this.logger?.info(`First run - auto-detected language: ${this.settings.pluginLanguage}, transcriptionLanguage: auto`); } // 必要に応じて設定を保存 @@ -339,14 +345,14 @@ export default class VoiceInputPlugin extends Plugin { } /** - * Get Obsidian locale setting - */ + * Obsidianの言語設定を取得 + */ private getObsidianLocale(): string { return getObsidianLocale(this.app); } /** - * Auto-detect plugin language (ja/zh/ko/en) + * プラグイン言語を自動検出（ja/zh/ko/en） */ private detectPluginLanguage(): 'ja' | 'zh' | 'ko' | 'en' { const obsidianLocale = this.getObsidianLocale().toLowerCase(); From b7090dee5aca77f6beb58980bce2223826c0780f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 20:03:49 +0000 Subject: [PATCH 31/82] Initial plan From 256824b31b89cc1f7b6c2880b3391b855c0fe24b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 18:30:21 +0000 Subject: [PATCH 32/82] Initial plan From 4887e5df0e9bcc44f76eabdcc812ea1888e4a135 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 20:11:43 +0000 Subject: [PATCH 33/82] Complete design document for language-specific dictionaries Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- docs/design-decisions-summary.md | 99 ++++++ docs/implementation-roadmap.md | 168 ++++++++++ docs/language-specific-dictionaries-design.md | 289 ++++++++++++++++++ 3 files changed, 556 insertions(+) create mode 100644 docs/design-decisions-summary.md create mode 100644 docs/implementation-roadmap.md create mode 100644 docs/language-specific-dictionaries-design.md diff --git a/docs/design-decisions-summary.md b/docs/design-decisions-summary.md new file mode 100644 index 0000000..a36b130 --- /dev/null +++ b/docs/design-decisions-summary.md @@ -0,0 +1,99 @@ +# Language-Specific Dictionaries: Design Decisions Summary + +## Key Decisions Made + +### 1. Type Architecture Decision + +**Chosen Approach**: Hybrid structure combining type safety with flexibility + +```typescript +interface MultiLanguageDictionary { + languages: { + ja?: CorrectionEntry[]; + en?: CorrectionEntry[]; + zh?: CorrectionEntry[]; + ko?: CorrectionEntry[]; + }; + global: CorrectionEntry[]; +} +``` + +**Rationale**: +- Maintains TypeScript type safety +- Allows optional language-specific dictionaries +- Clearly separates global vs language-specific rules +- Enables easy backward compatibility implementation + +### 2. Fallback Order + +**Priority**: `currentDetectedLanguage → 'en' → global` + +**Rationale**: +- Current language gets highest priority (most relevant) +- English as universal fallback (international common language) +- Global dictionary as final fallback (language-agnostic rules) + +### 3. Backward Compatibility Strategy + +**Approach**: Automatic migration with preservation of existing data + +- Legacy `SimpleCorrectionDictionary` → `MultiLanguageDictionary.global` +- No data loss during migration +- One-time conversion on first load with new format +- Always save in new format going forward + +### 4. Security and Performance Limits + +**Limits Established**: +```typescript +const LIMITS = { + MAX_ENTRIES_PER_LANGUAGE: 1000, + MAX_GLOBAL_ENTRIES: 500, + MAX_PATTERN_LENGTH: 100, + MAX_REPLACEMENT_LENGTH: 200, + MAX_PATTERNS_PER_ENTRY: 10 +}; +``` + +**Performance Target**: <100ms processing time per text correction + +### 5. UI Strategy + +**Approach**: Tab-based language selection with minimal disruption + +- Language tabs/filters in dictionary settings +- Clear indication of entry counts per language +- Enhanced import/export supporting both formats +- Migration notification for existing users + +### 6. Implementation Phases + +**Phase 1**: Core infrastructure and types (2-3 days) +**Phase 2**: UI enhancements (3-4 days) +**Phase 3**: Optimization and testing (1-2 days) + +## Benefits of This Design + +1. **Type Safety**: Full TypeScript support with compile-time checks +2. **Backward Compatibility**: No breaking changes for existing users +3. **Performance**: Efficient fallback logic with caching support +4. **Security**: Input validation and size limits prevent abuse +5. **Usability**: Intuitive language-based organization +6. **Extensibility**: Easy to add new languages in the future + +## Migration Impact + +- **Existing Users**: Seamless transition with automatic migration +- **Performance**: No regression, potential improvement with language-specific rules +- **Storage**: Slightly larger format but more efficient language targeting +- **Learning Curve**: Minimal - existing workflow preserved with new capabilities + +## Success Metrics + +1. Zero data loss during migration +2. <100ms correction processing time maintained +3. Memory usage stays under 1MB for typical dictionaries +4. User adoption of language-specific features >50% within 3 months +5. No increase in user-reported issues post-implementation + +This design successfully balances the competing requirements of type safety, performance, backward compatibility, and user experience while providing a solid foundation for multi-language dictionary correction. \ No newline at end of file diff --git a/docs/implementation-roadmap.md b/docs/implementation-roadmap.md new file mode 100644 index 0000000..dfb87b5 --- /dev/null +++ b/docs/implementation-roadmap.md @@ -0,0 +1,168 @@ +# Implementation Roadmap: Language-Specific Dictionaries + +## Next Implementation Issues + +Based on the design document, the following GitHub issues should be created to implement the language-specific dictionary feature: + +### Issue 1: Core Type Definitions and Infrastructure +**Title**: `feat: Add MultiLanguageDictionary type definitions and core infrastructure` + +**Description**: +Implement the foundational type definitions and helper functions for language-specific dictionaries. + +**Tasks**: +- [ ] Add `MultiLanguageDictionary` interface to `interfaces/transcription.ts` +- [ ] Add migration helper `migrateLegacyDictionary()` function +- [ ] Add validation functions for dictionary limits +- [ ] Update `VoiceInputSettings` to support new dictionary format +- [ ] Add backward compatibility detection logic +- [ ] Write unit tests for type definitions and helpers + +**Files to modify**: +- `src/interfaces/transcription.ts` +- `src/interfaces/settings.ts` +- `src/utils/dictionary-migration.ts` (new) +- `tests/unit/utils/dictionary-migration.test.ts` (new) + +**Acceptance Criteria**: +- New type definitions compile without errors +- Legacy dictionary migration works correctly +- All validation functions have comprehensive tests +- Backward compatibility is maintained + +--- + +### Issue 2: Dictionary Corrector Multi-Language Support +**Title**: `feat: Implement multi-language support in DictionaryCorrector` + +**Description**: +Extend the `DictionaryCorrector` class to support language-specific dictionaries with fallback logic. + +**Tasks**: +- [ ] Modify `DictionaryCorrector` to handle `MultiLanguageDictionary` +- [ ] Implement `getApplicableCorrections()` with fallback logic +- [ ] Add language detection parameter to `correct()` method +- [ ] Update constructor to accept new dictionary format +- [ ] Maintain backward compatibility with existing API +- [ ] Add comprehensive unit tests for multi-language correction + +**Files to modify**: +- `src/core/transcription/DictionaryCorrector.ts` +- `tests/unit/core/transcription/multilingual-dictionary-correction.test.ts` + +**Acceptance Criteria**: +- Fallback order works: currentLang → 'en' → global +- Legacy dictionaries continue to work +- Performance does not degrade significantly +- All edge cases are covered with tests + +--- + +### Issue 3: Settings UI Enhancement for Language Tabs +**Title**: `feat: Add language-specific dictionary editing in settings UI` + +**Description**: +Enhance the settings UI to support editing language-specific dictionary entries with tab-based navigation. + +**Tasks**: +- [ ] Add language tab navigation to dictionary settings +- [ ] Implement language-specific entry display and editing +- [ ] Add language filter/selector for dictionary entries +- [ ] Update import/export functionality for new format +- [ ] Add entry count display per language +- [ ] Implement dictionary size warnings +- [ ] Add migration notification for existing users + +**Files to modify**: +- `src/views/VoiceInputViewUI.ts` +- `src/views/VoiceInputViewActions.ts` +- Update corresponding CSS styles + +**Acceptance Criteria**: +- Users can switch between language tabs +- Dictionary entries can be added/edited/deleted per language +- Import/export works with both old and new formats +- UI provides clear feedback on dictionary size limits +- Existing users see migration guidance + +--- + +### Issue 4: Performance Optimization and Security +**Title**: `feat: Add dictionary performance optimization and security measures` + +**Description**: +Implement performance optimizations and security measures for the dictionary system. + +**Tasks**: +- [ ] Add entry count limits per language and globally +- [ ] Implement dictionary size monitoring and warnings +- [ ] Add input validation and sanitization +- [ ] Implement dictionary caching for performance +- [ ] Add processing time limits for correction operations +- [ ] Create performance benchmarks and tests +- [ ] Add memory usage monitoring + +**Files to modify**: +- `src/core/transcription/DictionaryCorrector.ts` +- `src/config/constants.ts` (add limits) +- `src/utils/dictionary-validation.ts` (new) +- `tests/performance/dictionary-performance.test.ts` (new) + +**Acceptance Criteria**: +- Dictionary size limits are enforced +- Input validation prevents security issues +- Performance is maintained even with large dictionaries +- Memory usage stays within reasonable limits +- All security measures have corresponding tests + +--- + +### Issue 5: Integration and E2E Testing +**Title**: `test: Add comprehensive integration tests for language-specific dictionaries` + +**Description**: +Create comprehensive integration tests and update documentation for the new feature. + +**Tasks**: +- [ ] Add E2E tests for multi-language dictionary functionality +- [ ] Test dictionary migration scenarios +- [ ] Add performance regression tests +- [ ] Update user documentation and README +- [ ] Create migration guide for existing users +- [ ] Add API reference documentation +- [ ] Test with real-world dictionary data + +**Files to modify**: +- `tests/integration/dictionary-multilingual.test.ts` (new) +- `docs/user-guide.md` (update) +- `docs/migration-guide.md` (new) +- `README.md` (update) + +**Acceptance Criteria**: +- All integration scenarios pass +- Documentation is clear and comprehensive +- Migration guide helps users transition smoothly +- Performance benchmarks meet requirements + +--- + +## Implementation Order and Dependencies + +1. **Issue 1** (Core Types) - No dependencies, implement first +2. **Issue 2** (DictionaryCorrector) - Depends on Issue 1 +3. **Issue 4** (Performance/Security) - Can be done in parallel with Issue 2 +4. **Issue 3** (UI Enhancement) - Depends on Issues 1 and 2 +5. **Issue 5** (Testing/Docs) - Depends on all previous issues + +## Estimated Timeline + +- **Week 1**: Issues 1 and 2 (Core functionality) +- **Week 2**: Issues 3 and 4 (UI and optimization) +- **Week 3**: Issue 5 (Testing and documentation) + +## Risk Mitigation + +- Implement feature flags to enable/disable new functionality +- Maintain backward compatibility throughout all phases +- Add comprehensive logging for debugging migration issues +- Create rollback procedures in case of critical issues \ No newline at end of file diff --git a/docs/language-specific-dictionaries-design.md b/docs/language-specific-dictionaries-design.md new file mode 100644 index 0000000..a285f47 --- /dev/null +++ b/docs/language-specific-dictionaries-design.md @@ -0,0 +1,289 @@ +# Language-Specific Dictionaries Design Document + +## 目的 (Purpose) + +辞書補正の言語別適用（ja/en/zh/ko）を安全に導入するための設計ドキュメント。現在の単一辞書システムから、言語特化辞書システムへの段階的移行を実現する。 + +## 現状分析 (Current State Analysis) + +### 現在の実装 + +- **辞書構造**: `SimpleCorrectionDictionary` with `definiteCorrections: CorrectionEntry[]` +- **適用範囲**: 全言語に同一辞書を適用（`enableTranscriptionCorrection: true` の場合） +- **言語サポート**: 'ja', 'en', 'zh', 'ko' + 'auto' +- **設定**: `VoiceInputSettings.customDictionary: SimpleCorrectionDictionary` + +### 問題点 + +1. 言語固有の修正ルールが適用できない +2. 日本語の敬語変換が英語テキストにも適用される可能性 +3. 言語特有の音声認識エラーパターンに対応できない +4. 辞書サイズの肥大化 + +## 型設計の比較と決定 (Type Architecture Comparison) + +### オプション1: Record型アプローチ + +```typescript +type LanguageSpecificDictionary = Record<'ja' | 'en' | 'zh' | 'ko', CorrectionEntry[]> & { + global?: CorrectionEntry[]; // 言語共通ルール +}; +``` + +**利点**: +- TypeScript型安全性が高い +- 言語キーの補完とチェックが可能 +- 構造が明確で理解しやすい + +**欠点**: +- 新言語追加時に型定義の変更が必要 +- 空配列でもすべての言語キーが必要 +- JSONシリアライゼーション時の冗長性 + +### オプション2: Array型アプローチ + +```typescript +type LanguageSpecificDictionary = Array<{ + lang?: Locale; // undefined = global + entries: CorrectionEntry[]; +}>; +``` + +**利点**: +- 動的な言語追加が容易 +- 空言語の場合の無駄がない +- JSONサイズが効率的 +- 拡張性が高い + +**欠点**: +- TypeScript型安全性が低い +- 実行時のvalidationが必要 +- 言語重複チェックが必要 + +### 決定: ハイブリッドアプローチ + +最適な解決策として、両方の利点を組み合わせた構造を採用： + +```typescript +interface MultiLanguageDictionary { + // 言語別辞書（型安全） + languages: { + ja?: CorrectionEntry[]; + en?: CorrectionEntry[]; + zh?: CorrectionEntry[]; + ko?: CorrectionEntry[]; + }; + // 全言語共通辞書 + global: CorrectionEntry[]; +} +``` + +**理由**: +1. 型安全性を保持しつつ柔軟性を提供 +2. 空言語は undefined で省略可能 +3. 全言語共通ルールを明示的に分離 +4. 後方互換性の実装が容易 + +## フォールバック順序の定義 (Fallback Strategy) + +### フォールバック優先順位 + +1. **現在の検出言語** (`currentDetectedLanguage`) +2. **英語** (`'en'`) - 国際的共通言語として +3. **グローバル辞書** (`global`) - 言語非依存の修正 + +### 実装ロジック + +```typescript +function getApplicableCorrections( + detectedLanguage: string, + dictionary: MultiLanguageDictionary +): CorrectionEntry[] { + const corrections: CorrectionEntry[] = []; + + // 1. 現在の言語の辞書 + const langDict = dictionary.languages[detectedLanguage as Locale]; + if (langDict) { + corrections.push(...langDict); + } + + // 2. 英語辞書（現在の言語が英語でない場合） + if (detectedLanguage !== 'en' && dictionary.languages.en) { + corrections.push(...dictionary.languages.en); + } + + // 3. グローバル辞書 + corrections.push(...dictionary.global); + + return corrections; +} +``` + +### エッジケース処理 + +- **未知言語**: 'en' → global の順序でフォールバック +- **'auto'言語設定**: 実際の検出結果に基づく +- **空辞書**: 次の優先順位に進む + +## 後方互換性戦略 (Backward Compatibility) + +### 既存データの受理 + +現在の `SimpleCorrectionDictionary` 形式を自動的に `MultiLanguageDictionary` に変換： + +```typescript +function migrateLegacyDictionary( + legacy: SimpleCorrectionDictionary +): MultiLanguageDictionary { + return { + languages: {}, // 言語別辞書は空 + global: legacy.definiteCorrections // 既存ルールは全てglobalに + }; +} +``` + +### 設定保存形式 + +- **新形式**: `MultiLanguageDictionary` +- **レガシー検出**: `definiteCorrections` プロパティの存在で判定 +- **自動変換**: 読み込み時に一度だけ実行 +- **保存形式**: 常に新形式で保存 + +## マイグレーション計画 (Migration Plan) + +### フェーズ1: 基盤実装 + +1. **型定義の追加** + - `MultiLanguageDictionary` インターフェース + - 互換性ヘルパー関数 + +2. **コア機能の拡張** + - `DictionaryCorrector` のマルチ言語対応 + - フォールバックロジックの実装 + +3. **テストの追加** + - 言語別辞書テスト + - フォールバックテスト + - 後方互換性テスト + +### フェーズ2: UI拡張 + +1. **設定画面の改良** + - 言語タブまたはフィルター機能 + - 言語別エントリの表示・編集 + - インポート・エクスポート機能の対応 + +2. **UX考慮事項** + - 既存ユーザーへの移行案内 + - 言語別エントリ数の表示 + - 辞書サイズの警告機能 + +### フェーズ3: 最適化 + +1. **パフォーマンス改善** + - 辞書キャッシュ機能 + - 部分読み込み対応 + +2. **高度な機能** + - 辞書の言語間コピー機能 + - 共通ルールの自動検出・提案 + +## セキュリティ・パフォーマンス考慮事項 (Security & Performance) + +### エントリ数制限 + +```typescript +const LIMITS = { + MAX_ENTRIES_PER_LANGUAGE: 1000, + MAX_GLOBAL_ENTRIES: 500, + MAX_PATTERN_LENGTH: 100, + MAX_REPLACEMENT_LENGTH: 200, + MAX_PATTERNS_PER_ENTRY: 10 +}; +``` + +### メモリ使用量 + +- **推定サイズ**: 言語4つ × 1000エントリ × 100文字 ≈ 400KB +- **制限値**: 総辞書サイズ 1MB以下 +- **警告しきい値**: 500KB + +### セキュリティ対策 + +1. **入力検証** + - 正規表現インジェクション防止 + - 文字列長制限 + - 特殊文字のサニタイゼーション + +2. **DoS攻撃対策** + - 処理時間制限（100ms/テキスト） + - 再帰的置換の防止 + - メモリ使用量監視 + +### パフォーマンス最適化 + +1. **処理効率** + - パターンマッチングの最適化 + - 辞書の事前コンパイル + - キャッシュの活用 + +2. **UI応答性** + - 大量エントリの仮想化表示 + - 非同期でのバリデーション + - プログレス表示 + +## 実装ロードマップ (Implementation Roadmap) + +### 必要なサブタスク + +#### Issue #XX: 型定義とコア機能実装 +- [ ] `MultiLanguageDictionary` インターフェース定義 +- [ ] `DictionaryCorrector` のマルチ言語対応 +- [ ] フォールバック機能の実装 +- [ ] 後方互換性ヘルパー関数 +- [ ] ユニットテストの追加 + +#### Issue #XX: 設定UI拡張 +- [ ] 言語タブ機能の実装 +- [ ] 言語別エントリの表示・編集UI +- [ ] インポート・エクスポート機能の対応 +- [ ] 既存データの移行UI + +#### Issue #XX: バリデーション・制限機能 +- [ ] エントリ数制限の実装 +- [ ] 入力検証の強化 +- [ ] メモリ使用量監視 +- [ ] パフォーマンス最適化 + +#### Issue #XX: ドキュメント・テスト +- [ ] ユーザーガイドの更新 +- [ ] API リファレンスの更新 +- [ ] E2Eテストの追加 +- [ ] マイグレーションガイドの作成 + +### 推定工数 + +- **フェーズ1**: 2-3 人日 +- **フェーズ2**: 3-4 人日 +- **フェーズ3**: 1-2 人日 +- **合計**: 6-9 人日 + +### リスク要因 + +1. **既存ユーザーデータの移行** + - リスク: データ損失・破損 + - 対策: バックアップ機能とロールバック機能 + +2. **パフォーマンス影響** + - リスク: 辞書処理の遅延 + - 対策: 段階的最適化とベンチマーク + +3. **UI複雑化** + - リスク: ユーザビリティの低下 + - 対策: プロトタイプと段階的展開 + +## 結論 (Conclusion) + +本設計により、言語特化辞書システムを安全かつ段階的に導入できる。ハイブリッド型アプローチにより型安全性と拡張性を両立し、堅牢なフォールバック機能により既存の使用体験を維持しながら新機能を提供する。 + +適切な制限とセキュリティ対策により、パフォーマンスと安全性を確保し、明確なマイグレーション計画により実装リスクを最小化する。 \ No newline at end of file From 429ab22800720bac038ac3a5e2f93b8d118caf7d Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Wed, 13 Aug 2025 05:18:11 +0900 Subject: [PATCH 34/82] Fix merge conflicts and syntax errors in preparation for docs update Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> # Conflicts: # src/plugin/VoiceInputPlugin.ts --- src/plugin/VoiceInputPlugin.ts | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/plugin/VoiceInputPlugin.ts b/src/plugin/VoiceInputPlugin.ts index 3a74cf9..873486c 100644 --- a/src/plugin/VoiceInputPlugin.ts +++ b/src/plugin/VoiceInputPlugin.ts @@ -319,7 +319,7 @@ export default class VoiceInputPlugin extends Plugin { !hasSettingsKey(data, 'interfaceLanguage')) { this.settings.pluginLanguage = this.detectPluginLanguage(); needsSave = true; - this.logger?.info(`Auto-detected language: ${this.settings.pluginLanguage} (from Obsidian: ${this.getObsidianLocale()})`); + this.logger?.info(`Auto-detected language: ${this.settings.pluginLanguage} (from Obsidian: ${getObsidianLocale(this.app)})`); } } else { // 保存データが存在しない場合（初回起動） @@ -345,17 +345,10 @@ export default class VoiceInputPlugin extends Plugin { } /** - * Obsidianの言語設定を取得 - */ - private getObsidianLocale(): string { - return getObsidianLocale(this.app); - } - - /** - * プラグイン言語を自動検出（ja/zh/ko/en） + * Plugin language auto detection (ja/zh/ko/en) */ private detectPluginLanguage(): 'ja' | 'zh' | 'ko' | 'en' { - const obsidianLocale = this.getObsidianLocale().toLowerCase(); + const obsidianLocale = getObsidianLocale(this.app).toLowerCase(); if (obsidianLocale.startsWith('ja')) { return 'ja'; } else if (obsidianLocale.startsWith('zh')) { From 532954f1325848751fd18350b1f36496fa04c8c6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 20:27:45 +0000 Subject: [PATCH 35/82] Initial plan From c844d597d6f580024039af2a324891359dce7012 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 20:28:16 +0000 Subject: [PATCH 36/82] Initial plan From cd2ac64ce75320554e75e630c02a9e987bd8de80 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 20:29:44 +0000 Subject: [PATCH 37/82] =?UTF-8?q?Address=20feedback:=20update=20design=20t?= =?UTF-8?q?o=20use=20currentLang=E2=86=92global=20fallback,=20clarify=20li?= =?UTF-8?q?teral=20dictionary=20format,=20and=20specify=20language=20sourc?= =?UTF-8?q?e=20routing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- docs/design-decisions-summary.md | 43 ++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/docs/design-decisions-summary.md b/docs/design-decisions-summary.md index a36b130..e53f4bd 100644 --- a/docs/design-decisions-summary.md +++ b/docs/design-decisions-summary.md @@ -26,12 +26,18 @@ interface MultiLanguageDictionary { ### 2. Fallback Order -**Priority**: `currentDetectedLanguage → 'en' → global` +**Priority**: `currentDetectedLanguage → global` **Rationale**: - Current language gets highest priority (most relevant) -- English as universal fallback (international common language) -- Global dictionary as final fallback (language-agnostic rules) +- Global dictionary as fallback (language-agnostic rules) +- No English fallback by default to avoid inappropriate corrections across languages +- Optional English fallback can be enabled via setting (disabled by default) for users who prefer it + +**Language Source**: Language detection is routed via `getResolvedLanguage()` which: +- Uses `transcriptionLanguage` setting when explicitly set +- Maps 'auto' setting to one of: ja/zh/ko/en based on plugin language detection +- Does NOT use UI/plugin interface language ### 3. Backward Compatibility Strategy @@ -42,9 +48,24 @@ interface MultiLanguageDictionary { - One-time conversion on first load with new format - Always save in new format going forward -### 4. Security and Performance Limits +### 4. Dictionary Format and Security -**Limits Established**: +**Dictionary Entry Format**: Literal string-to-string mappings only +```typescript +interface CorrectionEntry { + patterns: string[]; // Literal strings, NOT regex patterns + replacement: string; // Literal replacement text + enabled: boolean; +} +``` + +**Important Safety Considerations**: +- Dictionary entries must be literal strings for safety and performance +- Regex patterns belong only to separate custom rules (not dictionary entries) +- Dictionary patterns are escaped when compiled to prevent injection +- Input validation prevents malicious patterns + +**Performance Limits**: ```typescript const LIMITS = { MAX_ENTRIES_PER_LANGUAGE: 1000, @@ -76,10 +97,11 @@ const LIMITS = { 1. **Type Safety**: Full TypeScript support with compile-time checks 2. **Backward Compatibility**: No breaking changes for existing users -3. **Performance**: Efficient fallback logic with caching support -4. **Security**: Input validation and size limits prevent abuse -5. **Usability**: Intuitive language-based organization +3. **Performance**: Efficient fallback logic with literal string matching and caching support +4. **Security**: Input validation, literal-only dictionary entries, and size limits prevent abuse +5. **Usability**: Intuitive language-based organization aligned with transcription language 6. **Extensibility**: Easy to add new languages in the future +7. **Safety**: Explicit separation of dictionary entries (literal) from custom rules (regex) ## Migration Impact @@ -91,9 +113,10 @@ const LIMITS = { ## Success Metrics 1. Zero data loss during migration -2. <100ms correction processing time maintained +2. <100ms correction processing time maintained with literal string matching 3. Memory usage stays under 1MB for typical dictionaries 4. User adoption of language-specific features >50% within 3 months 5. No increase in user-reported issues post-implementation +6. Security: Zero regex injection vulnerabilities with literal-only dictionary entries -This design successfully balances the competing requirements of type safety, performance, backward compatibility, and user experience while providing a solid foundation for multi-language dictionary correction. \ No newline at end of file +This design successfully balances the competing requirements of type safety, performance, backward compatibility, and user experience while providing a solid foundation for multi-language dictionary correction. The literal string approach ensures both safety and performance, while the simplified fallback strategy prevents inappropriate cross-language corrections. \ No newline at end of file From f12b607238e4fb9dac079dce7167ef6d0da7bb74 Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Wed, 13 Aug 2025 05:30:47 +0900 Subject: [PATCH 38/82] docs(design): replace placeholder issue refs (#XX) with #[TBD] per review feedback --- docs/language-specific-dictionaries-design.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/language-specific-dictionaries-design.md b/docs/language-specific-dictionaries-design.md index a285f47..429a0bb 100644 --- a/docs/language-specific-dictionaries-design.md +++ b/docs/language-specific-dictionaries-design.md @@ -236,26 +236,26 @@ const LIMITS = { ### 必要なサブタスク -#### Issue #XX: 型定義とコア機能実装 +#### Issue #[TBD]: 型定義とコア機能実装 - [ ] `MultiLanguageDictionary` インターフェース定義 - [ ] `DictionaryCorrector` のマルチ言語対応 - [ ] フォールバック機能の実装 - [ ] 後方互換性ヘルパー関数 - [ ] ユニットテストの追加 -#### Issue #XX: 設定UI拡張 +#### Issue #[TBD]: 設定UI拡張 - [ ] 言語タブ機能の実装 - [ ] 言語別エントリの表示・編集UI - [ ] インポート・エクスポート機能の対応 - [ ] 既存データの移行UI -#### Issue #XX: バリデーション・制限機能 +#### Issue #[TBD]: バリデーション・制限機能 - [ ] エントリ数制限の実装 - [ ] 入力検証の強化 - [ ] メモリ使用量監視 - [ ] パフォーマンス最適化 -#### Issue #XX: ドキュメント・テスト +#### Issue #[TBD]: ドキュメント・テスト - [ ] ユーザーガイドの更新 - [ ] API リファレンスの更新 - [ ] E2Eテストの追加 @@ -286,4 +286,4 @@ const LIMITS = { 本設計により、言語特化辞書システムを安全かつ段階的に導入できる。ハイブリッド型アプローチにより型安全性と拡張性を両立し、堅牢なフォールバック機能により既存の使用体験を維持しながら新機能を提供する。 -適切な制限とセキュリティ対策により、パフォーマンスと安全性を確保し、明確なマイグレーション計画により実装リスクを最小化する。 \ No newline at end of file +適切な制限とセキュリティ対策により、パフォーマンスと安全性を確保し、明確なマイグレーション計画により実装リスクを最小化する。 From 67c5285af9eefe738ad80f976348f3c0787a935e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 20:39:11 +0000 Subject: [PATCH 39/82] Add locale detection tests with proper git tracking Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- tests/unit/settings/locale-detection.test.ts | 253 +++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 tests/unit/settings/locale-detection.test.ts diff --git a/tests/unit/settings/locale-detection.test.ts b/tests/unit/settings/locale-detection.test.ts new file mode 100644 index 0000000..1ef9763 --- /dev/null +++ b/tests/unit/settings/locale-detection.test.ts @@ -0,0 +1,253 @@ +/** + * Tests for locale detection functionality + * Validates getObsidianLocale() function with different Obsidian app configurations + */ + +import { getObsidianLocale } from '../../../src/types/obsidian-internal'; +import { App } from 'obsidian'; + +// Mock Obsidian's moment export +jest.mock('obsidian', () => ({ + ...jest.requireActual('obsidian'), + moment: { + locale: jest.fn() + } +})); + +// Import moment after mocking +import { moment } from 'obsidian'; + +describe('Locale Detection', () => { + beforeEach(() => { + // Reset mocks + (moment.locale as jest.Mock).mockReset(); + }); + + describe('getObsidianLocale() function', () => { + it('should detect ja-JP locale from vault config', () => { + const mockApp = { + vault: { + config: { + locale: 'ja-JP' + } + } + } as App; + + const result = getObsidianLocale(mockApp); + expect(result).toBe('ja-jp'); + }); + + it('should detect en-US locale from vault config', () => { + const mockApp = { + vault: { + config: { + locale: 'en-US' + } + } + } as App; + + const result = getObsidianLocale(mockApp); + expect(result).toBe('en-us'); + }); + + it('should detect zh-CN locale from vault config', () => { + const mockApp = { + vault: { + config: { + locale: 'zh-CN' + } + } + } as App; + + const result = getObsidianLocale(mockApp); + expect(result).toBe('zh-cn'); + }); + + it('should detect ko-KR locale from vault config', () => { + const mockApp = { + vault: { + config: { + locale: 'ko-KR' + } + } + } as App; + + const result = getObsidianLocale(mockApp); + expect(result).toBe('ko-kr'); + }); + + it('should fallback to app.locale when vault config is not available', () => { + const mockApp = { + locale: 'ja-JP', + vault: { + config: {} + } + } as App; + + const result = getObsidianLocale(mockApp); + expect(result).toBe('ja-jp'); + }); + + it('should fallback to moment.locale() when app locale is not available', () => { + (moment.locale as jest.Mock).mockReturnValue('zh-CN'); + + const mockApp = { + vault: { + config: {} + } + } as App; + + const result = getObsidianLocale(mockApp); + expect(result).toBe('zh-cn'); + expect(moment.locale).toHaveBeenCalled(); + }); + + it('should fallback to "en" when no locale is available', () => { + (moment.locale as jest.Mock).mockReturnValue(undefined); + + const mockApp = { + vault: { + config: {} + } + } as App; + + const result = getObsidianLocale(mockApp); + expect(result).toBe('en'); + }); + + it('should normalize locale codes to lowercase', () => { + const testCases = [ + { input: 'JA-JP', expected: 'ja-jp' }, + { input: 'EN-US', expected: 'en-us' }, + { input: 'ZH-CN', expected: 'zh-cn' }, + { input: 'KO-KR', expected: 'ko-kr' }, + { input: 'Fr-FR', expected: 'fr-fr' }, + { input: 'DE', expected: 'de' } + ]; + + testCases.forEach(({ input, expected }) => { + const mockApp = { + vault: { + config: { + locale: input + } + } + } as App; + + const result = getObsidianLocale(mockApp); + expect(result).toBe(expected); + }); + }); + + it('should handle missing vault property gracefully', () => { + (moment.locale as jest.Mock).mockReturnValue('en-US'); + + const mockApp = {} as App; + + const result = getObsidianLocale(mockApp); + expect(result).toBe('en-us'); + }); + + it('should handle null/undefined moment locale gracefully', () => { + (moment.locale as jest.Mock).mockReturnValue(null); + + const mockApp = { + vault: { + config: {} + } + } as App; + + const result = getObsidianLocale(mockApp); + expect(result).toBe('en'); + }); + + it('should prioritize vault config over app locale', () => { + const mockApp = { + locale: 'en-US', // This should be ignored + vault: { + config: { + locale: 'ja-JP' // This should take priority + } + } + } as App; + + const result = getObsidianLocale(mockApp); + expect(result).toBe('ja-jp'); + }); + + it('should prioritize app locale over moment locale', () => { + (moment.locale as jest.Mock).mockReturnValue('ko-KR'); // This should be ignored + + const mockApp = { + locale: 'zh-CN', // This should take priority + vault: { + config: {} + } + } as App; + + const result = getObsidianLocale(mockApp); + expect(result).toBe('zh-cn'); + }); + }); + + describe('Locale integration scenarios', () => { + it('should handle complex locale codes correctly', () => { + const complexLocales = [ + { input: 'ja-JP-u-ca-japanese', expected: 'ja-jp-u-ca-japanese' }, + { input: 'en-US-POSIX', expected: 'en-us-posix' }, + { input: 'zh-Hans-CN', expected: 'zh-hans-cn' }, + { input: 'ko-Kore-KR', expected: 'ko-kore-kr' } + ]; + + complexLocales.forEach(({ input, expected }) => { + const mockApp = { + vault: { + config: { + locale: input + } + } + } as App; + + const result = getObsidianLocale(mockApp); + expect(result).toBe(expected); + }); + }); + + it('should handle various fallback chains correctly', () => { + // Test the full fallback chain: vault.config.locale → app.locale → moment.locale() → 'en' + const fallbackChains = [ + { + name: 'vault config available', + mockApp: { vault: { config: { locale: 'ja-JP' } } }, + momentReturn: 'en-US', + expected: 'ja-jp' + }, + { + name: 'app locale available', + mockApp: { locale: 'zh-CN', vault: { config: {} } }, + momentReturn: 'en-US', + expected: 'zh-cn' + }, + { + name: 'moment locale available', + mockApp: { vault: { config: {} } }, + momentReturn: 'ko-KR', + expected: 'ko-kr' + }, + { + name: 'default fallback', + mockApp: { vault: { config: {} } }, + momentReturn: undefined, + expected: 'en' + } + ]; + + fallbackChains.forEach(({ name, mockApp, momentReturn, expected }) => { + (moment.locale as jest.Mock).mockReturnValue(momentReturn); + + const result = getObsidianLocale(mockApp as App); + expect(result).toBe(expected); + }); + }); + }); +}); \ No newline at end of file From 520b224cee7ea7c080cd2632561c1633fad8d4bc Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Wed, 13 Aug 2025 05:39:57 +0900 Subject: [PATCH 40/82] =?UTF-8?q?docs(roadmap):=20align=20Issue=202=20acce?= =?UTF-8?q?ptance=20criteria=20to=20default=20fallback=20currentLang=20?= =?UTF-8?q?=E2=86=92=20global;=20note=20optional=20'en'=20fallback=20via?= =?UTF-8?q?=20setting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/implementation-roadmap.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/implementation-roadmap.md b/docs/implementation-roadmap.md index dfb87b5..17f9955 100644 --- a/docs/implementation-roadmap.md +++ b/docs/implementation-roadmap.md @@ -51,7 +51,7 @@ Extend the `DictionaryCorrector` class to support language-specific dictionaries - `tests/unit/core/transcription/multilingual-dictionary-correction.test.ts` **Acceptance Criteria**: -- Fallback order works: currentLang → 'en' → global +- Fallback order works: currentLang → global (optional 'en' fallback only when a setting is enabled) - Legacy dictionaries continue to work - Performance does not degrade significantly - All edge cases are covered with tests @@ -165,4 +165,4 @@ Create comprehensive integration tests and update documentation for the new feat - Implement feature flags to enable/disable new functionality - Maintain backward compatibility throughout all phases - Add comprehensive logging for debugging migration issues -- Create rollback procedures in case of critical issues \ No newline at end of file +- Create rollback procedures in case of critical issues From 5c63e9628ed138141604a54d93704b9e7ecbcfbe Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 Aug 2025 20:41:53 +0000 Subject: [PATCH 41/82] Implement advanced language settings with linking toggle Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- src/i18n/en.ts | 5 ++ src/i18n/ja.ts | 5 ++ src/i18n/ko.ts | 5 ++ src/i18n/zh.ts | 5 ++ src/interfaces/i18n.ts | 5 ++ src/interfaces/settings.ts | 16 +++- src/plugin/VoiceInputPlugin.ts | 41 +++++++++-- src/settings/VoiceInputSettingTab.ts | 54 +++++++++++--- tests/language-detection.test.ts | 106 +++++++++++++++++++++++++++ 9 files changed, 222 insertions(+), 20 deletions(-) diff --git a/src/i18n/en.ts b/src/i18n/en.ts index 4b82a8f..aa8da54 100644 --- a/src/i18n/en.ts +++ b/src/i18n/en.ts @@ -160,6 +160,11 @@ export const en: TranslationResource = { transcriptionLanguageDesc: 'Language for voice recognition and transcription. Auto-detection is recommended for best results.', pluginLanguage: 'Plugin Language', pluginLanguageDesc: 'Set language for UI display (voice recognition language is auto-detected separately)', + // Advanced settings + languageLinking: 'Link UI and recognition languages', + languageLinkingDesc: 'When enabled, recognition language follows UI language. When disabled, you can set recognition language independently.', + advancedTranscriptionLanguage: 'Recognition Language (Advanced)', + advancedTranscriptionLanguageDesc: 'Set the language for voice recognition independently. Auto-detection is recommended for best results.', customDictionary: 'Custom Dictionary', customDictionaryDesc: 'Manage corrections used for post-processing', dictionaryDefinite: 'Definite Corrections (max {max})', diff --git a/src/i18n/ja.ts b/src/i18n/ja.ts index f52f61a..220622a 100644 --- a/src/i18n/ja.ts +++ b/src/i18n/ja.ts @@ -160,6 +160,11 @@ export const ja: TranslationResource = { transcriptionLanguageDesc: '音声認識と文字起こしの言語。最適な結果のため自動検出を推奨します。', pluginLanguage: 'プラグイン言語', pluginLanguageDesc: 'UI表示、音声認識処理、補正辞書の言語を設定', + // 高度設定 + languageLinking: 'UI言語と認識言語を連動する', + languageLinkingDesc: 'オンの場合、認識言語がUI言語に従います。オフの場合、認識言語を個別に設定できます。', + advancedTranscriptionLanguage: '認識言語（高度設定）', + advancedTranscriptionLanguageDesc: '音声認識の言語を個別に設定します。最適な結果のため自動検出を推奨します。', customDictionary: 'カスタム辞書', customDictionaryDesc: '補正辞書を管理', dictionaryDefinite: '固定補正（最大{max}個）', diff --git a/src/i18n/ko.ts b/src/i18n/ko.ts index 946570f..50fe757 100644 --- a/src/i18n/ko.ts +++ b/src/i18n/ko.ts @@ -160,6 +160,11 @@ export const ko: TranslationResource = { transcriptionLanguageDesc: '음성 인식 및 전사를 위한 언어입니다. 최상의 결과를 위해 자동 감지를 권장합니다.', pluginLanguage: '플러그인 언어', pluginLanguageDesc: 'UI 표시, 음성 처리 및 교정 사전의 언어 설정', + // 고급 설정 + languageLinking: 'UI 언어와 인식 언어 연동', + languageLinkingDesc: '활성화하면 인식 언어가 UI 언어를 따릅니다. 비활성화하면 인식 언어를 독립적으로 설정할 수 있습니다.', + advancedTranscriptionLanguage: '인식 언어 (고급 설정)', + advancedTranscriptionLanguageDesc: '음성 인식 언어를 독립적으로 설정합니다. 최상의 결과를 위해 자동 감지를 권장합니다.', customDictionary: '사용자 정의 사전', customDictionaryDesc: '후처리에 사용되는 교정 사전 관리', dictionaryDefinite: '고정 교정 (최대 {max}개)', diff --git a/src/i18n/zh.ts b/src/i18n/zh.ts index a75d356..0e33ca3 100644 --- a/src/i18n/zh.ts +++ b/src/i18n/zh.ts @@ -160,6 +160,11 @@ export const zh: TranslationResource = { transcriptionLanguageDesc: '语音识别和转录的语言。建议使用自动检测以获得最佳结果。', pluginLanguage: '插件语言', pluginLanguageDesc: '设置UI显示、语音处理和校正词典的语言', + // 高级设置 + languageLinking: '关联UI语言与识别语言', + languageLinkingDesc: '启用时，识别语言跟随UI语言。禁用时，可独立设置识别语言。', + advancedTranscriptionLanguage: '识别语言（高级设置）', + advancedTranscriptionLanguageDesc: '独立设置语音识别的语言。建议使用自动检测以获得最佳结果。', customDictionary: '自定义词典', customDictionaryDesc: '管理用于后处理的校正词典', dictionaryDefinite: '固定校正（最多{max}个）', diff --git a/src/interfaces/i18n.ts b/src/interfaces/i18n.ts index 5421115..abf8b1d 100644 --- a/src/interfaces/i18n.ts +++ b/src/interfaces/i18n.ts @@ -179,6 +179,11 @@ export type TranslationResource = { transcriptionLanguageDesc: string; pluginLanguage: string; pluginLanguageDesc: string; + // Advanced settings + languageLinking: string; + languageLinkingDesc: string; + advancedTranscriptionLanguage: string; + advancedTranscriptionLanguageDesc: string; customDictionary: string; customDictionaryDesc: string; dictionaryDefinite: string; diff --git a/src/interfaces/settings.ts b/src/interfaces/settings.ts index 6441517..96dedae 100644 --- a/src/interfaces/settings.ts +++ b/src/interfaces/settings.ts @@ -9,12 +9,17 @@ export interface VoiceInputSettings { // 録音設定 maxRecordingSeconds: number; // 最大録音時間（秒） // 言語設定 - transcriptionLanguage: 'auto' | 'ja' | 'en' | 'zh' | 'ko'; // 音声認識言語 + transcriptionLanguage: 'auto' | 'ja' | 'en' | 'zh' | 'ko'; // 音声認識言語（後方互換性のため維持） pluginLanguage: Locale; // プラグインUI表示の言語 customDictionary: SimpleCorrectionDictionary; // デバッグ設定 debugMode: boolean; // デバッグモード logLevel: LogLevel; // ログレベル + // 高度設定 + advanced: { + languageLinkingEnabled: boolean; // UI言語と認識言語を連動する（デフォルト: true） + transcriptionLanguage?: 'auto' | 'ja' | 'en' | 'zh' | 'ko'; // 独立した音声認識言語設定 + }; } export const DEFAULT_SETTINGS: VoiceInputSettings = { @@ -24,10 +29,15 @@ export const DEFAULT_SETTINGS: VoiceInputSettings = { // 録音設定 maxRecordingSeconds: 300, // 5分（300秒） // 言語設定 - transcriptionLanguage: 'auto', // 音声認識言語のデフォルトは自動検出 + transcriptionLanguage: 'auto', // 音声認識言語のデフォルトは自動検出（後方互換性のため維持） pluginLanguage: 'en', // 初期値、実際はObsidianの設定に従う customDictionary: { definiteCorrections: [] }, // デバッグ設定 debugMode: false, // 本番環境ではデフォルトでオフ - logLevel: LogLevel.INFO // 通常レベル + logLevel: LogLevel.INFO, // 通常レベル + // 高度設定 + advanced: { + languageLinkingEnabled: true, // デフォルトは連動オン（現行動作維持） + transcriptionLanguage: 'auto' // デフォルトは自動検出 + } }; diff --git a/src/plugin/VoiceInputPlugin.ts b/src/plugin/VoiceInputPlugin.ts index 873486c..c229552 100644 --- a/src/plugin/VoiceInputPlugin.ts +++ b/src/plugin/VoiceInputPlugin.ts @@ -321,12 +321,32 @@ export default class VoiceInputPlugin extends Plugin { needsSave = true; this.logger?.info(`Auto-detected language: ${this.settings.pluginLanguage} (from Obsidian: ${getObsidianLocale(this.app)})`); } + + // 高度設定のマイグレーション + if (!hasSettingsKey(data, 'advanced')) { + // 既存ユーザーには言語連動をデフォルトで有効化（現行動作維持） + this.settings.advanced = { + languageLinkingEnabled: true, + transcriptionLanguage: 'auto' + }; + needsSave = true; + this.logger?.info('Initialized advanced settings with language linking enabled for backward compatibility'); + } else if (data.advanced && !hasSettingsKey(data.advanced, 'languageLinkingEnabled')) { + // advancedオブジェクトは存在するが、languageLinkingEnabledが無い場合 + this.settings.advanced.languageLinkingEnabled = true; + needsSave = true; + this.logger?.info('Added languageLinkingEnabled to existing advanced settings'); + } } else { // 保存データが存在しない場合（初回起動） this.settings.pluginLanguage = this.detectPluginLanguage(); this.settings.transcriptionLanguage = 'auto'; + this.settings.advanced = { + languageLinkingEnabled: true, + transcriptionLanguage: 'auto' + }; needsSave = true; - this.logger?.info(`First run - auto-detected language: ${this.settings.pluginLanguage}, transcriptionLanguage: auto`); + this.logger?.info(`First run - auto-detected language: ${this.settings.pluginLanguage}, transcriptionLanguage: auto, advanced settings initialized`); } // 必要に応じて設定を保存 @@ -361,13 +381,24 @@ export default class VoiceInputPlugin extends Plugin { } /** - * 解決済み言語を取得（transcriptionLanguage が 'auto' の場合は自動検出） + * 解決済み言語を取得（高度設定の連動設定に基づく） */ getResolvedLanguage(): 'ja' | 'zh' | 'ko' | 'en' { - if (this.settings.transcriptionLanguage === 'auto') { - return this.detectPluginLanguage(); + // 高度設定で言語連動が有効な場合（デフォルト） + if (this.settings.advanced?.languageLinkingEnabled !== false) { + // 従来のロジック: transcriptionLanguage が 'auto' の場合は pluginLanguage に基づく自動検出 + if (this.settings.transcriptionLanguage === 'auto') { + return this.detectPluginLanguage(); + } + return this.settings.transcriptionLanguage as 'ja' | 'zh' | 'ko' | 'en'; + } else { + // 言語連動が無効な場合: advanced.transcriptionLanguage を使用 + const advancedLang = this.settings.advanced.transcriptionLanguage ?? 'auto'; + if (advancedLang === 'auto') { + return this.detectPluginLanguage(); + } + return advancedLang as 'ja' | 'zh' | 'ko' | 'en'; } - return this.settings.transcriptionLanguage as 'ja' | 'zh' | 'ko' | 'en'; } async saveSettings() { diff --git a/src/settings/VoiceInputSettingTab.ts b/src/settings/VoiceInputSettingTab.ts index 5c94095..958d17d 100644 --- a/src/settings/VoiceInputSettingTab.ts +++ b/src/settings/VoiceInputSettingTab.ts @@ -73,22 +73,52 @@ export class VoiceInputSettingTab extends PluginSettingTab { this.display(); })); - // Voice Recognition Language Setting + // Advanced Language Settings new Setting(containerEl) - .setName(this.i18n.t('ui.settings.transcriptionLanguage')) - .setDesc(this.i18n.t('ui.settings.transcriptionLanguageDesc')) - .addDropdown(dropdown => dropdown - .addOption('auto', this.i18n.t('ui.options.languageAuto')) - .addOption('ja', this.i18n.t('ui.options.languageJa')) - .addOption('en', this.i18n.t('ui.options.languageEn')) - .addOption('zh', this.i18n.t('ui.options.languageZh')) - .addOption('ko', this.i18n.t('ui.options.languageKo')) - .setValue(this.plugin.settings.transcriptionLanguage) - .onChange(async (value: 'auto' | 'ja' | 'en' | 'zh' | 'ko') => { - this.plugin.settings.transcriptionLanguage = value; + .setName(this.i18n.t('ui.settings.languageLinking')) + .setDesc(this.i18n.t('ui.settings.languageLinkingDesc')) + .addToggle(toggle => toggle + .setValue(this.plugin.settings.advanced?.languageLinkingEnabled !== false) + .onChange(async (value) => { + // Initialize advanced object if it doesn't exist + if (!this.plugin.settings.advanced) { + this.plugin.settings.advanced = { + languageLinkingEnabled: value, + transcriptionLanguage: 'auto' + }; + } else { + this.plugin.settings.advanced.languageLinkingEnabled = value; + } await this.plugin.saveSettings(); + // Refresh the UI to show/hide the advanced transcription language setting + this.display(); })); + // Advanced Transcription Language Setting (only shown when linking is disabled) + if (this.plugin.settings.advanced?.languageLinkingEnabled === false) { + new Setting(containerEl) + .setName(this.i18n.t('ui.settings.advancedTranscriptionLanguage')) + .setDesc(this.i18n.t('ui.settings.advancedTranscriptionLanguageDesc')) + .addDropdown(dropdown => dropdown + .addOption('auto', this.i18n.t('ui.options.languageAuto')) + .addOption('ja', this.i18n.t('ui.options.languageJa')) + .addOption('en', this.i18n.t('ui.options.languageEn')) + .addOption('zh', this.i18n.t('ui.options.languageZh')) + .addOption('ko', this.i18n.t('ui.options.languageKo')) + .setValue(this.plugin.settings.advanced.transcriptionLanguage ?? 'auto') + .onChange(async (value: 'auto' | 'ja' | 'en' | 'zh' | 'ko') => { + if (!this.plugin.settings.advanced) { + this.plugin.settings.advanced = { + languageLinkingEnabled: false, + transcriptionLanguage: value + }; + } else { + this.plugin.settings.advanced.transcriptionLanguage = value; + } + await this.plugin.saveSettings(); + })); + } + // OpenAI API Key const apiKeySetting = new Setting(containerEl) .setName(this.i18n.t('ui.settings.apiKey')) diff --git a/tests/language-detection.test.ts b/tests/language-detection.test.ts index 79779e5..13eeeb2 100644 --- a/tests/language-detection.test.ts +++ b/tests/language-detection.test.ts @@ -111,4 +111,110 @@ describe('Language Detection Logic', () => { expect(mockDetectFn).not.toHaveBeenCalled(); }); }); + + describe('getResolvedLanguageWithAdvancedSettings', () => { + const mockDetectFn = jest.fn(() => 'ko' as const); + + beforeEach(() => { + mockDetectFn.mockClear(); + }); + + // Function to simulate the new advanced language resolution logic + function getResolvedLanguageAdvanced( + transcriptionLanguage: string, + advanced: { languageLinkingEnabled?: boolean; transcriptionLanguage?: string } | undefined, + detectFn: () => 'ja' | 'zh' | 'ko' | 'en' + ): 'ja' | 'zh' | 'ko' | 'en' { + // 高度設定で言語連動が有効な場合（デフォルト） + if (advanced?.languageLinkingEnabled !== false) { + // 従来のロジック: transcriptionLanguage が 'auto' の場合は pluginLanguage に基づく自動検出 + if (transcriptionLanguage === 'auto') { + return detectFn(); + } + return transcriptionLanguage as 'ja' | 'zh' | 'ko' | 'en'; + } else { + // 言語連動が無効な場合: advanced.transcriptionLanguage を使用 + const advancedLang = advanced.transcriptionLanguage ?? 'auto'; + if (advancedLang === 'auto') { + return detectFn(); + } + return advancedLang as 'ja' | 'zh' | 'ko' | 'en'; + } + } + + test('should use traditional logic when language linking is enabled (default)', () => { + // Default case: advanced.languageLinkingEnabled is true + const advanced = { languageLinkingEnabled: true }; + + // Should use transcriptionLanguage when it's not 'auto' + expect(getResolvedLanguageAdvanced('ja', advanced, mockDetectFn)).toBe('ja'); + expect(mockDetectFn).not.toHaveBeenCalled(); + + // Should use detection when transcriptionLanguage is 'auto' + expect(getResolvedLanguageAdvanced('auto', advanced, mockDetectFn)).toBe('ko'); + expect(mockDetectFn).toHaveBeenCalledTimes(1); + }); + + test('should use traditional logic when advanced settings is undefined', () => { + // When advanced is undefined, should default to traditional behavior (linking enabled) + expect(getResolvedLanguageAdvanced('ja', undefined, mockDetectFn)).toBe('ja'); + expect(mockDetectFn).not.toHaveBeenCalled(); + + expect(getResolvedLanguageAdvanced('auto', undefined, mockDetectFn)).toBe('ko'); + expect(mockDetectFn).toHaveBeenCalledTimes(1); + }); + + test('should use advanced.transcriptionLanguage when language linking is disabled', () => { + const advanced = { + languageLinkingEnabled: false, + transcriptionLanguage: 'en' + }; + + // Should ignore the regular transcriptionLanguage and use advanced.transcriptionLanguage + expect(getResolvedLanguageAdvanced('ja', advanced, mockDetectFn)).toBe('en'); + expect(mockDetectFn).not.toHaveBeenCalled(); + }); + + test('should use auto-detection when advanced.transcriptionLanguage is auto', () => { + const advanced = { + languageLinkingEnabled: false, + transcriptionLanguage: 'auto' + }; + + expect(getResolvedLanguageAdvanced('ja', advanced, mockDetectFn)).toBe('ko'); + expect(mockDetectFn).toHaveBeenCalledTimes(1); + }); + + test('should use auto-detection when advanced.transcriptionLanguage is undefined', () => { + const advanced = { + languageLinkingEnabled: false + // transcriptionLanguage is undefined + }; + + expect(getResolvedLanguageAdvanced('ja', advanced, mockDetectFn)).toBe('ko'); + expect(mockDetectFn).toHaveBeenCalledTimes(1); + }); + + test('should work with all supported languages in advanced mode', () => { + const advanced = { + languageLinkingEnabled: false, + transcriptionLanguage: 'zh' + }; + + // Test all supported languages + advanced.transcriptionLanguage = 'ja'; + expect(getResolvedLanguageAdvanced('en', advanced, mockDetectFn)).toBe('ja'); + + advanced.transcriptionLanguage = 'zh'; + expect(getResolvedLanguageAdvanced('en', advanced, mockDetectFn)).toBe('zh'); + + advanced.transcriptionLanguage = 'ko'; + expect(getResolvedLanguageAdvanced('en', advanced, mockDetectFn)).toBe('ko'); + + advanced.transcriptionLanguage = 'en'; + expect(getResolvedLanguageAdvanced('ja', advanced, mockDetectFn)).toBe('en'); + + expect(mockDetectFn).not.toHaveBeenCalled(); + }); + }); }); \ No newline at end of file From d6d7615c16c91114e292aebfd8996cf502a2a8d1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 14 Aug 2025 06:45:11 +0000 Subject: [PATCH 42/82] Initial plan From dd2f65fcd8bf6704473e450678b980b62b2a19ab Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 14 Aug 2025 06:55:04 +0000 Subject: [PATCH 43/82] Add comprehensive voice input processing flow documentation Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- README.md | 8 + docs/PROCESSING_FLOW.md | 313 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 321 insertions(+) create mode 100644 docs/PROCESSING_FLOW.md diff --git a/README.md b/README.md index 66cbe50..e73e7d7 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,10 @@ See also OpenAI’s Privacy Policy. - Deploy locally: `npm run deploy-local` (copies to detected vaults) - Analyze unused code (build-time): `npm run analyze:unused` +### Documentation + +- Processing flow visualization: see [`docs/PROCESSING_FLOW.md`](docs/PROCESSING_FLOW.md) + Third‑party licensing: see `THIRD_PARTY_LICENSES.md`. — @@ -175,6 +179,10 @@ OpenAIのプライバシーポリシーもご参照ください。 - ビルド: `npm run build-plugin` - ローカル配布: `npm run deploy-local` +### ドキュメント + +- 処理フローの可視化: [`docs/PROCESSING_FLOW.md`](docs/PROCESSING_FLOW.md) を参照 + サードパーティライセンスは `THIRD_PARTY_LICENSES.md` を参照してください。 ## ライセンス diff --git a/docs/PROCESSING_FLOW.md b/docs/PROCESSING_FLOW.md new file mode 100644 index 0000000..404d521 --- /dev/null +++ b/docs/PROCESSING_FLOW.md @@ -0,0 +1,313 @@ +# Voice Input Processing Flow / 音声入力処理フロー + +This document visualizes the complete processing pipeline of the voice input system, showing how audio input flows through prompt addition, API calls, response cleaning, and dictionary processing. + +このドキュメントは音声入力システムの完全な処理パイプラインを可視化し、音声入力がプロンプト追加、API呼び出し、レスポンスクリーニング、辞書処理を経てどのように流れるかを示しています。 + +## Complete Processing Flow / 完全な処理フロー + +``` +┌─────────────────┐ +│ Audio Input │ 🎤 +│ 音声入力 │ +└─────┬───────────┘ + │ + ▼ +┌─────────────────┐ +│ Audio Blob │ +│ Creation │ +│ 音声Blob作成 │ +└─────┬───────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ TranscriptionService.transcribe() │ +│ 文字起こしサービス │ +└─────┬───────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────┐ ┌──────────────────────────────────────┐ +│ Language Check │────│ buildTranscriptionPrompt(language) │ +│ 言語チェック │ │ プロンプト構築 │ +└─────┬───────────┘ └──────────────────┬───────────────────┘ + │ │ + │ ┌─────────────────────────────────│──────────────────┐ + │ │ │ │ + │ │ IF language === 'ja' │ │ + │ │ 日本語の場合: │ │ + │ │ ▼ │ + │ │ ┌───────────────────────────────────────────┐ │ + │ │ │ 以下の音声内容のみを文字に起こしてください │ │ + │ │ │ この指示文は出力に含めないでください │ │ + │ │ │ 話者の発言内容だけを正確に記録してください │ │ + │ │ │ │ │ + │ │ │ 出力形式: │ │ + │ │ │ │ │ + │ │ │ （話者の発言のみ） │ │ + │ │ │ │ │ + │ │ └───────────────────────────────────────────┘ │ + │ │ │ │ + │ │ ELSE (en/zh/ko/auto) │ │ + │ │ その他の言語: │ │ + │ │ ▼ │ + │ │ ┌───────────────────────────────────────────┐ │ + │ │ │ No Prompt Added │ │ + │ │ │ プロンプト追加なし │ │ + │ │ └───────────────────────────────────────────┘ │ + │ └─────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ OpenAI API Request │ +│ OpenAI API リクエスト │ +│ │ +│ FormData: │ +│ ├─ file: audio.webm (Blob) │ +│ ├─ model: gpt-4o-transcribe / gpt-4o-mini-transcribe │ +│ ├─ response_format: json │ +│ ├─ temperature: 0 │ +│ ├─ language: ja/en/zh/ko (if not 'auto') │ +│ └─ prompt: [Only for Japanese / 日本語のみ] │ +│ │ +│ Headers: │ +│ └─ Authorization: Bearer ${apiKey} │ +└─────┬───────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ OpenAI API Response │ +│ OpenAI API レスポンス │ +│ │ +│ { text: "transcribed text...", language: "detected" } │ +└─────┬───────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ cleanGPT4oResponse(text, language) │ +│ GPT-4oレスポンスクリーニング │ +└─────┬───────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────┐ +│ Extract from │ +│ │ +│ tags │ +│ TRANSCRIPTタグ │ +│ からの抽出 │ +└─────┬───────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ applyLanguageSpecificCleaning() │ +│ 言語固有のクリーニング適用 │ +└─────┬───────────────────────────────────────────────────────┘ + │ + │ ┌─────────────────────────────────────────────────────┐ + │ │ │ + │ │ IF language === 'ja' │ + │ │ 日本語の場合: │ + │ │ │ │ + │ │ ▼ │ + │ │ ┌───────────────────────────────────────────────┐ │ + │ │ │ Remove Japanese prompt patterns: │ │ + │ │ │ 日本語プロンプトパターンを除去: │ │ + │ │ │ • "以下の音声内容..." │ │ + │ │ │ • "この指示文..." │ │ + │ │ │ • "話者の発言内容だけを..." │ │ + │ │ │ • "話者の発言..." │ │ + │ │ │ • "出力形式..." │ │ + │ │ │ • "（話者の発言のみ）" │ │ + │ │ └───────────────────────────────────────────────┘ │ + │ │ │ + │ │ ELSE (en/zh/ko) │ + │ │ その他の言語: │ + │ │ │ │ + │ │ ▼ │ + │ │ ┌───────────────────────────────────────────────┐ │ + │ │ │ Conservative cleaning only │ │ + │ │ │ 保守的なクリーニングのみ │ │ + │ │ │ (relies on generic cleaning) │ │ + │ │ │ (汎用クリーニングに依存) │ │ + │ │ └───────────────────────────────────────────────┘ │ + │ └─────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ applyGenericCleaning() │ +│ 汎用クリーニング適用 │ +│ │ +│ Conservative patterns only: │ +│ 保守的なパターンのみ: │ +│ • "Output format:..." │ +│ • "Format:..." │ +└─────┬───────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ isPromptErrorDetected(text, language) │ +│ プロンプトエラー検出 │ +└─────┬───────────────────────────────────────────────────────┘ + │ + │ ┌─────────────────────────────────────────────────────┐ + │ │ │ + │ │ Check for prompt leakage patterns: │ + │ │ プロンプト漏洩パターンをチェック: │ + │ │ │ + │ │ Japanese (ja): │ + │ │ • "この指示文は出力に含めないでください" │ + │ │ • "話者の発言内容だけを正確に記録してください" │ + │ │ • "（話者の発言のみ）" │ + │ │ │ + │ │ English (en): │ + │ │ • "Please transcribe only the speaker" │ + │ │ • "Do not include this instruction" │ + │ │ • "(Speaker content only)" │ + │ │ │ + │ │ Chinese (zh): │ + │ │ • "请仅转录说话者" │ + │ │ • "不要包含此指令" │ + │ │ • "（仅说话者内容）" │ + │ │ │ + │ │ Korean (ko): │ + │ │ • "화자의 발언만 전사해주세요" │ + │ │ • "이 지시사항을 포함하지 마세요" │ + │ │ • "（화자 발언만）" │ + │ └─────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────┐ ┌──────────────────────────────────────┐ +│ If prompt error │ NO │ Continue to dictionary processing │ +│ detected? │───▶│ 辞書処理へ続行 │ +│ プロンプトエラー│ └──────────────────┬───────────────────┘ +│ 検出？ │ │ +└─────┬───────────┘ │ + │ YES │ + ▼ │ +┌─────────────────┐ │ +│ Return empty │ │ +│ string │ │ +│ 空文字を返す │ │ +└─────────────────┘ │ + │ + ▼ + ┌─────────────────────────────────────┐ + │ enableTranscriptionCorrection? │ + │ 文字起こし修正が有効？ │ + └─────┬───────────────────────────────┘ + │ + ┌─────────┴─────────┐ + │ YES │ NO + ▼ ▼ + ┌───────────────────────────────┐ ┌─────────────────┐ + │ DictionaryCorrector.correct() │ │ Skip correction │ + │ 辞書修正適用 │ │ 修正をスキップ │ + └─────┬─────────────────────────┘ └─────┬───────────┘ + │ │ + ▼ │ + ┌───────────────────────────────────────┐ │ + │ Apply multilingual corrections: │ │ + │ 多言語修正を適用: │ │ + │ │ │ + │ 1. Default rules (empty by default) │ │ + │ デフォルトルール（既定では空） │ │ + │ │ │ + │ 2. Custom rules │ │ + │ カスタムルール │ │ + │ │ │ + │ 3. Dictionary corrections │ │ + │ 辞書修正: │ │ + │ • Fixed string replacements │ │ + │ • Applied to ALL languages │ │ + │ • 固定文字列置換 │ │ + │ • 全言語に適用 │ │ + │ │ │ + │ Examples: │ │ + │ • "AI" → "artificial intelligence" │ │ + │ • "こんにちは" → "こんにちは（修正）" │ │ + │ • "你好" → "您好" │ │ + │ • "안녕" → "안녕하세요" │ │ + └─────┬─────────────────────────────────┘ │ + │ │ + └───────────────┬───────────────────┘ + │ + ▼ + ┌─────────────────────────────────────┐ + │ Final Output │ + │ 最終出力 │ + │ │ + │ TranscriptionResult { │ + │ text: correctedText, │ + │ originalText: originalText, │ + │ duration: processingTime, │ + │ model: "gpt-4o(-mini)-transcribe", │ + │ language: detectedLanguage │ + │ } │ + └─────────────────────────────────────┘ +``` + +## Key Differences by Language / 言語による主な違い + +### Japanese (ja) Processing / 日本語処理 +1. **Prompt Addition**: Complex Japanese prompt with specific instructions + - プロンプト追加: 特定の指示を含む複雑な日本語プロンプト +2. **Intensive Cleaning**: Multiple Japanese-specific patterns removed + - 集約的クリーニング: 複数の日本語特有パターンの除去 +3. **Error Detection**: Japanese prompt leakage patterns + - エラー検出: 日本語プロンプト漏洩パターン + +### Other Languages (en/zh/ko) Processing / その他の言語処理 +1. **No Prompt**: Direct transcription without additional prompts + - プロンプトなし: 追加プロンプトなしの直接文字起こし +2. **Conservative Cleaning**: Only generic colon-based patterns + - 保守的クリーニング: コロンベースの汎用パターンのみ +3. **Language-specific Error Detection**: Each language has its own patterns + - 言語固有のエラー検出: 各言語が独自のパターンを持つ + +### Auto Language Processing / 自動言語処理 +- Uses Japanese patterns as fallback for error detection +- エラー検出時は日本語パターンをフォールバックとして使用 +- No prompt addition (treated as non-Japanese) +- プロンプト追加なし（非日本語として扱う） + +## Dictionary Processing / 辞書処理 + +Dictionary correction is **language-agnostic** and applies to all languages when enabled: +辞書修正は**言語非依存**で、有効時は全言語に適用されます: + +``` +Dictionary Entry: { from: ["AI"], to: "artificial intelligence" } +辞書エントリ: { from: ["AI"], to: "artificial intelligence" } + +Applied to: +適用対象: +✓ English: "AI system" → "artificial intelligence system" +✓ Japanese: "AIシステム" → "artificial intelligenceシステム" +✓ Chinese: "AI系统" → "artificial intelligence系统" +✓ Korean: "AI시스템" → "artificial intelligence시스템" +``` + +## Processing Performance / 処理性能 + +The system logs detailed performance metrics: +システムは詳細なパフォーマンス指標をログに記録します: + +- Audio blob size and type / 音声Blobサイズとタイプ +- Processing duration / 処理時間 +- Original vs corrected text length / 元のテキストと修正後のテキスト長 +- Model used and cost estimation / 使用モデルとコスト推定 + +## Error Handling / エラーハンドリング + +The system handles various error scenarios: +システムは様々なエラーシナリオを処理します: + +1. **API Errors**: Invalid key, quota exceeded, network errors + - API エラー: 無効なキー、クォータ超過、ネットワークエラー +2. **Prompt Leakage**: Detection and mitigation of prompt content in output + - プロンプト漏洩: 出力内のプロンプト内容の検出と軽減 +3. **Empty Audio**: Graceful handling of silent or empty audio input + - 空音声: 無音または空の音声入力の適切な処理 + +This visualization shows the complete flow from audio input to final transcribed and corrected text, highlighting the sophisticated language-specific processing that makes this plugin particularly effective for Japanese users while maintaining compatibility with other languages. + +この可視化は、音声入力から最終的な文字起こし・修正テキストまでの完全なフローを示し、このプラグインが日本語ユーザーに特に効果的でありながら他言語との互換性を維持する洗練された言語固有処理を強調しています。 \ No newline at end of file From 21d740e48e6ba55609d9d3c6542e96f3ffac649e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 14 Aug 2025 07:44:20 +0000 Subject: [PATCH 44/82] Initial plan From 348ec4430d9e6ab3849c7a0c0197fb704f0598f5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 14 Aug 2025 07:53:39 +0000 Subject: [PATCH 45/82] Implement multilingual prompt support for improved transcription accuracy Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- .../transcription/TranscriptionService.ts | 90 ++++++++++- .../transcription-cleaning.test.ts | 143 ++++++++++++++---- 2 files changed, 199 insertions(+), 34 deletions(-) diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 5d619dc..9a2c768 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -176,20 +176,57 @@ export class TranscriptionService implements ITranscriptionProvider { } /** - * Build prompt for GPT-4o transcription (only for Japanese) + * Build prompt for GPT-4o transcription (for all languages except auto) */ private buildTranscriptionPrompt(language: string): string { - // Only provide prompt for Japanese language - if (language !== 'ja') { + // No prompt for auto language mode (as it might interfere with language detection) + if (language === 'auto') { return ''; } - return `以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。 + + const normalizedLang = this.normalizeLanguage(language); + + switch (normalizedLang) { + case 'ja': + return `以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。 話者の発言内容だけを正確に記録してください。 出力形式: （話者の発言のみ） `; + + case 'en': + return `Please transcribe only the following audio content. Do not include this instruction in your output. +Record only the speaker's statements accurately. + +Output format: + +(Speaker content only) +`; + + case 'zh': + return `请仅转录以下音频内容。不要包含此指令在输出中。 +请准确记录说话者的发言内容。 + +输出格式: + +（仅说话者内容） +`; + + case 'ko': + return `다음 음성 내용만 전사해주세요. 이 지시사항을 출력에 포함하지 마세요. +화자의 발언 내용만 정확히 기록해주세요. + +출력 형식: + +（화자 발언만） +`; + + default: + // For any other language, return empty string + return ''; + } } /** @@ -282,7 +319,20 @@ export class TranscriptionService implements ITranscriptionProvider { * Apply English-specific cleaning patterns */ private applyEnglishCleaning(text: string): string { - // For English, rely on conservative generic cleaning only + const patterns = [ + /^Please transcribe only the following audio content.*?$/gm, + /^Please transcribe.*?$/gm, + /^Do not include this instruction.*?$/gm, + /^Record only the speaker's statements.*?$/gm, + /^Transcribe only.*?$/gm, + /^Output format.*?$/gm, + /^Format.*?$/gm, + /\(Speaker content only\)/g, + ]; + + for (const pattern of patterns) { + text = text.replace(pattern, ''); + } return text; } @@ -290,7 +340,20 @@ export class TranscriptionService implements ITranscriptionProvider { * Apply Chinese-specific cleaning patterns */ private applyChineseCleaning(text: string): string { - // For Chinese, rely on conservative generic cleaning only + const patterns = [ + /^请仅转录以下音频内容.*?$/gm, + /^请转录.*?$/gm, + /^不要包含此指令.*?$/gm, + /^请准确记录说话者的发言内容.*?$/gm, + /^仅转录.*?$/gm, + /^输出格式.*?$/gm, + /^格式.*?$/gm, + /（仅说话者内容）/g, + ]; + + for (const pattern of patterns) { + text = text.replace(pattern, ''); + } return text; } @@ -298,7 +361,20 @@ export class TranscriptionService implements ITranscriptionProvider { * Apply Korean-specific cleaning patterns */ private applyKoreanCleaning(text: string): string { - // For Korean, rely on conservative generic cleaning only + const patterns = [ + /^다음 음성 내용만 전사해주세요.*?$/gm, + /^다음 음성.*?$/gm, + /^이 지시사항을 출력에 포함하지 마세요.*?$/gm, + /^화자의 발언 내용만 정확히 기록해주세요.*?$/gm, + /^음성 내용만.*?$/gm, + /^출력 형식.*?$/gm, + /^형식.*?$/gm, + /（화자 발언만）/g, + ]; + + for (const pattern of patterns) { + text = text.replace(pattern, ''); + } return text; } diff --git a/tests/unit/core/transcription/transcription-cleaning.test.ts b/tests/unit/core/transcription/transcription-cleaning.test.ts index ab02528..85a7c3b 100644 --- a/tests/unit/core/transcription/transcription-cleaning.test.ts +++ b/tests/unit/core/transcription/transcription-cleaning.test.ts @@ -97,10 +97,14 @@ class MockTranscriptionService { */ applyEnglishCleaning(text: string): string { const patterns = [ - /^Please transcribe.*?$/gmi, - /^Transcribe only.*?$/gmi, - /^Output format.*?$/gmi, - /^Format.*?$/gmi, + /^Please transcribe only the following audio content.*?$/gm, + /^Please transcribe.*?$/gm, + /^Do not include this instruction.*?$/gm, + /^Record only the speaker's statements.*?$/gm, + /^Transcribe only.*?$/gm, + /^Output format.*?$/gm, + /^Format.*?$/gm, + /\(Speaker content only\)/g, ]; for (const pattern of patterns) { @@ -114,10 +118,14 @@ class MockTranscriptionService { */ applyChineseCleaning(text: string): string { const patterns = [ + /^请仅转录以下音频内容.*?$/gm, /^请转录.*?$/gm, + /^不要包含此指令.*?$/gm, + /^请准确记录说话者的发言内容.*?$/gm, /^仅转录.*?$/gm, /^输出格式.*?$/gm, /^格式.*?$/gm, + /（仅说话者内容）/g, ]; for (const pattern of patterns) { @@ -131,10 +139,14 @@ class MockTranscriptionService { */ applyKoreanCleaning(text: string): string { const patterns = [ + /^다음 음성 내용만 전사해주세요.*?$/gm, /^다음 음성.*?$/gm, + /^이 지시사항을 출력에 포함하지 마세요.*?$/gm, + /^화자의 발언 내용만 정확히 기록해주세요.*?$/gm, /^음성 내용만.*?$/gm, /^출력 형식.*?$/gm, /^형식.*?$/gm, + /（화자 발언만）/g, ]; for (const pattern of patterns) { @@ -196,18 +208,54 @@ class MockTranscriptionService { * Build prompt for GPT-4o transcription */ buildTranscriptionPrompt(language: string): string { - // Only provide prompt for Japanese language - if (language !== 'ja') { + // No prompt for auto language mode (as it might interfere with language detection) + if (language === 'auto') { return ''; } - return `以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。 + const normalizedLang = this.normalizeLanguage(language); + + switch (normalizedLang) { + case 'ja': + return `以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。 話者の発言内容だけを正確に記録してください。 出力形式: （話者の発言のみ） `; + + case 'en': + return `Please transcribe only the following audio content. Do not include this instruction in your output. +Record only the speaker's statements accurately. + +Output format: + +(Speaker content only) +`; + + case 'zh': + return `请仅转录以下音频内容。不要包含此指令在输出中。 +请准确记录说话者的发言内容。 + +输出格式: + +（仅说话者内容） +`; + + case 'ko': + return `다음 음성 내용만 전사해주세요. 이 지시사항을 출력에 포함하지 마세요. +화자의 발언 내용만 정확히 기록해주세요. + +출력 형식: + +（화자 발언만） +`; + + default: + // For any other language, return empty string + return ''; + } } } @@ -284,10 +332,10 @@ Content with attributes describe('English cleaning', () => { it('should remove English meta instructions', () => { const patterns = [ - 'Please transcribe the following audio', - 'Transcribe only the speaker content', - 'Output format: JSON', - 'Format: Speaker content only' + 'Please transcribe only the following audio content', + 'Do not include this instruction in your output', + 'Record only the speaker\'s statements accurately', + 'Output format: JSON' ]; patterns.forEach(pattern => { @@ -298,15 +346,23 @@ Content with attributes expect(result).toContain('Other content'); }); }); + + it('should remove English meta phrases anywhere in text', () => { + const input = 'Before text (Speaker content only) After text'; + const result = service.cleanGPT4oResponse(input, 'en'); + expect(result).not.toContain('(Speaker content only)'); + expect(result).toContain('Before text'); + expect(result).toContain('After text'); + }); }); describe('Chinese cleaning', () => { it('should remove Chinese meta instructions', () => { const patterns = [ - '请转录以下音频内容', - '仅转录说话者内容', - '输出格式: JSON', - '格式: 说话者内容' + '请仅转录以下音频内容', + '不要包含此指令在输出中', + '请准确记录说话者的发言内容', + '输出格式: JSON' ]; patterns.forEach(pattern => { @@ -317,15 +373,23 @@ Content with attributes expect(result).toContain('其他内容'); }); }); + + it('should remove Chinese meta phrases anywhere in text', () => { + const input = '前面的文字 （仅说话者内容） 后面的文字'; + const result = service.cleanGPT4oResponse(input, 'zh'); + expect(result).not.toContain('（仅说话者内容）'); + expect(result).toContain('前面的文字'); + expect(result).toContain('后面的文字'); + }); }); describe('Korean cleaning', () => { it('should remove Korean meta instructions', () => { const patterns = [ - '다음 음성 내용을 전사해주세요', - '음성 내용만 전사하세요', - '출력 형식: JSON', - '형식: 화자 내용만' + '다음 음성 내용만 전사해주세요', + '이 지시사항을 출력에 포함하지 마세요', + '화자의 발언 내용만 정확히 기록해주세요', + '출력 형식: JSON' ]; patterns.forEach(pattern => { @@ -336,6 +400,14 @@ Content with attributes expect(result).toContain('기타 내용'); }); }); + + it('should remove Korean meta phrases anywhere in text', () => { + const input = '앞의 문장 （화자 발언만） 뒤의 문장'; + const result = service.cleanGPT4oResponse(input, 'ko'); + expect(result).not.toContain('（화자 발언만）'); + expect(result).toContain('앞의 문장'); + expect(result).toContain('뒤의 문장'); + }); }); }); @@ -449,18 +521,35 @@ This is normal content`; expect(result.length).toBeGreaterThan(0); }); + it('should return English prompt for en language', () => { + const result = service.buildTranscriptionPrompt('en'); + expect(result).toContain('Please transcribe only the following audio content'); + expect(result).toContain(''); + expect(result).toContain('(Speaker content only)'); + expect(result.length).toBeGreaterThan(0); + }); + + it('should return Chinese prompt for zh language', () => { + const result = service.buildTranscriptionPrompt('zh'); + expect(result).toContain('请仅转录以下音频内容'); + expect(result).toContain(''); + expect(result).toContain('（仅说话者内容）'); + expect(result.length).toBeGreaterThan(0); + }); + + it('should return Korean prompt for ko language', () => { + const result = service.buildTranscriptionPrompt('ko'); + expect(result).toContain('다음 음성 내용만 전사해주세요'); + expect(result).toContain(''); + expect(result).toContain('（화자 발언만）'); + expect(result.length).toBeGreaterThan(0); + }); + it('should return empty string for auto language', () => { expect(service.buildTranscriptionPrompt('auto')).toBe(''); }); - it('should return empty string for en/zh/ko languages', () => { - const languages = ['en', 'zh', 'ko']; - languages.forEach(lang => { - expect(service.buildTranscriptionPrompt(lang)).toBe(''); - }); - }); - - it('should return empty string for other languages', () => { + it('should return empty string for unsupported languages', () => { const languages = ['fr', 'de', 'es', 'unknown']; languages.forEach(lang => { expect(service.buildTranscriptionPrompt(lang)).toBe(''); From 70f99c1305d1e2cd1804a632c44e20d6acbb8863 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 14 Aug 2025 07:55:36 +0000 Subject: [PATCH 46/82] Add comprehensive tests for multilingual prompt support --- .../multilingual-prompt-support.test.ts | 248 ++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 tests/unit/core/transcription/multilingual-prompt-support.test.ts diff --git a/tests/unit/core/transcription/multilingual-prompt-support.test.ts b/tests/unit/core/transcription/multilingual-prompt-support.test.ts new file mode 100644 index 0000000..51f8ed6 --- /dev/null +++ b/tests/unit/core/transcription/multilingual-prompt-support.test.ts @@ -0,0 +1,248 @@ +/** + * Tests for multilingual prompt support + * Verifies that prompts are correctly generated and applied for all supported languages + */ + +// Mock TranscriptionService to test multilingual prompt functionality +class MockTranscriptionService { + /** + * Normalize language code for consistent processing + */ + normalizeLanguage(language: string): string { + if (language === 'auto') return 'auto'; + const lang = language.toLowerCase(); + if (lang.startsWith('ja')) return 'ja'; + if (lang.startsWith('zh')) return 'zh'; + if (lang.startsWith('ko')) return 'ko'; + if (lang.startsWith('en')) return 'en'; + return lang; + } + + /** + * Build prompt for GPT-4o transcription (for all languages except auto) + */ + buildTranscriptionPrompt(language: string): string { + // No prompt for auto language mode (as it might interfere with language detection) + if (language === 'auto') { + return ''; + } + + const normalizedLang = this.normalizeLanguage(language); + + switch (normalizedLang) { + case 'ja': + return `以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。 +話者の発言内容だけを正確に記録してください。 + +出力形式: + +（話者の発言のみ） +`; + + case 'en': + return `Please transcribe only the following audio content. Do not include this instruction in your output. +Record only the speaker's statements accurately. + +Output format: + +(Speaker content only) +`; + + case 'zh': + return `请仅转录以下音频内容。不要包含此指令在输出中。 +请准确记录说话者的发言内容。 + +输出格式: + +（仅说话者内容） +`; + + case 'ko': + return `다음 음성 내용만 전사해주세요. 이 지시사항을 출력에 포함하지 마세요. +화자의 발언 내용만 정확히 기록해주세요. + +출력 형식: + +（화자 발언만） +`; + + default: + // For any other language, return empty string + return ''; + } + } +} + +describe('Multilingual Prompt Support', () => { + let service: MockTranscriptionService; + + beforeEach(() => { + service = new MockTranscriptionService(); + }); + + describe('Prompt generation for supported languages', () => { + it('should generate appropriate prompts for each supported language', () => { + const languages = ['ja', 'en', 'zh', 'ko']; + + languages.forEach(lang => { + const prompt = service.buildTranscriptionPrompt(lang); + + // All prompts should be non-empty + expect(prompt.length).toBeGreaterThan(0); + + // All prompts should contain TRANSCRIPT tags + expect(prompt).toContain(''); + expect(prompt).toContain(''); + + // Prompts should be in the appropriate language + switch (lang) { + case 'ja': + expect(prompt).toContain('以下の音声内容のみを文字に起こしてください'); + expect(prompt).toContain('（話者の発言のみ）'); + break; + case 'en': + expect(prompt).toContain('Please transcribe only the following audio content'); + expect(prompt).toContain('(Speaker content only)'); + break; + case 'zh': + expect(prompt).toContain('请仅转录以下音频内容'); + expect(prompt).toContain('（仅说话者内容）'); + break; + case 'ko': + expect(prompt).toContain('다음 음성 내용만 전사해주세요'); + expect(prompt).toContain('（화자 발언만）'); + break; + } + }); + }); + + it('should not generate prompts for auto language mode', () => { + const prompt = service.buildTranscriptionPrompt('auto'); + expect(prompt).toBe(''); + }); + + it('should not generate prompts for unsupported languages', () => { + const unsupportedLanguages = ['fr', 'de', 'es', 'it', 'pt', 'ru', 'ar']; + + unsupportedLanguages.forEach(lang => { + const prompt = service.buildTranscriptionPrompt(lang); + expect(prompt).toBe(''); + }); + }); + }); + + describe('Language code normalization', () => { + it('should handle language variants correctly', () => { + const testCases = [ + { input: 'ja', expected: 'ja' }, + { input: 'ja-JP', expected: 'ja' }, + { input: 'JA', expected: 'ja' }, + { input: 'en', expected: 'en' }, + { input: 'en-US', expected: 'en' }, + { input: 'en-GB', expected: 'en' }, + { input: 'EN', expected: 'en' }, + { input: 'zh', expected: 'zh' }, + { input: 'zh-CN', expected: 'zh' }, + { input: 'zh-TW', expected: 'zh' }, + { input: 'ZH', expected: 'zh' }, + { input: 'ko', expected: 'ko' }, + { input: 'ko-KR', expected: 'ko' }, + { input: 'KO', expected: 'ko' }, + { input: 'auto', expected: 'auto' }, + ]; + + testCases.forEach(({ input, expected }) => { + expect(service.normalizeLanguage(input)).toBe(expected); + }); + }); + + it('should generate prompts based on normalized language codes', () => { + // Test that language variants get the same prompt as their base language + const jaPrompt = service.buildTranscriptionPrompt('ja'); + const jaJPPrompt = service.buildTranscriptionPrompt('ja-JP'); + const JAPrompt = service.buildTranscriptionPrompt('JA'); + + expect(jaJPPrompt).toBe(jaPrompt); + expect(JAPrompt).toBe(jaPrompt); + + const enPrompt = service.buildTranscriptionPrompt('en'); + const enUSPrompt = service.buildTranscriptionPrompt('en-US'); + const ENPrompt = service.buildTranscriptionPrompt('EN'); + + expect(enUSPrompt).toBe(enPrompt); + expect(ENPrompt).toBe(enPrompt); + }); + }); + + describe('Prompt structure validation', () => { + it('should ensure all prompts follow consistent structure', () => { + const languages = ['ja', 'en', 'zh', 'ko']; + + languages.forEach(lang => { + const prompt = service.buildTranscriptionPrompt(lang); + + // Should contain instruction not to include the prompt + const lowerPrompt = prompt.toLowerCase(); + expect( + lowerPrompt.includes('not include') || + lowerPrompt.includes('含めない') || + lowerPrompt.includes('不要包含') || + lowerPrompt.includes('포함하지') + ).toBe(true); + + // Should contain output format instructions + expect( + prompt.includes('Output format') || + prompt.includes('出力形式') || + prompt.includes('输出格式') || + prompt.includes('출력 형식') + ).toBe(true); + + // Should have proper TRANSCRIPT tag structure + expect(prompt).toContain(''); + expect(prompt).toContain(''); + + // TRANSCRIPT tags should be in correct order + const transcriptStart = prompt.indexOf(''); + const transcriptEnd = prompt.indexOf(''); + expect(transcriptStart).toBeLessThan(transcriptEnd); + }); + }); + + it('should contain language-appropriate speaker placeholders', () => { + const expectedPlaceholders = { + 'ja': '（話者の発言のみ）', + 'en': '(Speaker content only)', + 'zh': '（仅说话者内容）', + 'ko': '（화자 발언만）' + }; + + Object.entries(expectedPlaceholders).forEach(([lang, placeholder]) => { + const prompt = service.buildTranscriptionPrompt(lang); + expect(prompt).toContain(placeholder); + }); + }); + }); + + describe('Integration with auto mode behavior', () => { + it('should maintain current auto mode behavior (no prompt)', () => { + // Auto mode should continue to not get prompts to avoid interfering with language detection + expect(service.buildTranscriptionPrompt('auto')).toBe(''); + expect(service.buildTranscriptionPrompt('AUTO')).toBe(''); + }); + }); + + describe('Edge cases', () => { + it('should handle empty and null language inputs gracefully', () => { + expect(service.buildTranscriptionPrompt('')).toBe(''); + expect(service.buildTranscriptionPrompt(' ')).toBe(''); + }); + + it('should handle mixed case language codes', () => { + expect(service.buildTranscriptionPrompt('Ja')).toBeTruthy(); + expect(service.buildTranscriptionPrompt('En')).toBeTruthy(); + expect(service.buildTranscriptionPrompt('Zh')).toBeTruthy(); + expect(service.buildTranscriptionPrompt('Ko')).toBeTruthy(); + }); + }); +}); \ No newline at end of file From 6e729a179471b78fb9b1678bf6b0d4ee6710f148 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:38:10 +0000 Subject: [PATCH 47/82] Implement precise exact-line matching for cleaning patterns to minimize risk of incorrect removal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace broad regex patterns with exact line matching in English, Chinese, and Korean cleaning methods - Use filter-based approach to only remove lines that exactly match prompt instructions - Update generic cleaning to use exact matching for consistency - Add comprehensive safety tests demonstrating protection against over-removal - Preserve legitimate user content that contains similar but different phrases Addresses feedback about 誤除去リスク最小化 (minimizing risk of incorrect removal) Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- .../transcription/TranscriptionService.ts | 121 ++++----- .../transcription-cleaning.test.ts | 232 ++++++++++++------ 2 files changed, 217 insertions(+), 136 deletions(-) diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 9a2c768..273c6e1 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -316,82 +316,93 @@ Output format: } /** - * Apply English-specific cleaning patterns + * Apply English-specific cleaning patterns - using exact line matching for safety */ private applyEnglishCleaning(text: string): string { - const patterns = [ - /^Please transcribe only the following audio content.*?$/gm, - /^Please transcribe.*?$/gm, - /^Do not include this instruction.*?$/gm, - /^Record only the speaker's statements.*?$/gm, - /^Transcribe only.*?$/gm, - /^Output format.*?$/gm, - /^Format.*?$/gm, - /\(Speaker content only\)/g, - ]; + // Split into lines for exact line matching + const lines = text.split('\n'); + const cleanedLines = lines.filter(line => { + const trimmedLine = line.trim(); + // Only remove lines that exactly match prompt instructions + return !( + trimmedLine === 'Please transcribe only the following audio content. Do not include this instruction in your output.' || + trimmedLine === 'Record only the speaker\'s statements accurately.' || + trimmedLine === 'Output format:' || + trimmedLine === '(Speaker content only)' + ); + }); - for (const pattern of patterns) { - text = text.replace(pattern, ''); - } - return text; + // Also remove the phrase anywhere in text (not just full lines) + let result = cleanedLines.join('\n'); + result = result.replace(/\(Speaker content only\)/g, ''); + + return result; } /** - * Apply Chinese-specific cleaning patterns + * Apply Chinese-specific cleaning patterns - using exact line matching for safety */ private applyChineseCleaning(text: string): string { - const patterns = [ - /^请仅转录以下音频内容.*?$/gm, - /^请转录.*?$/gm, - /^不要包含此指令.*?$/gm, - /^请准确记录说话者的发言内容.*?$/gm, - /^仅转录.*?$/gm, - /^输出格式.*?$/gm, - /^格式.*?$/gm, - /（仅说话者内容）/g, - ]; + // Split into lines for exact line matching + const lines = text.split('\n'); + const cleanedLines = lines.filter(line => { + const trimmedLine = line.trim(); + // Only remove lines that exactly match prompt instructions + return !( + trimmedLine === '请仅转录以下音频内容。不要包含此指令在输出中。' || + trimmedLine === '请准确记录说话者的发言内容。' || + trimmedLine === '输出格式:' || + trimmedLine === '（仅说话者内容）' + ); + }); - for (const pattern of patterns) { - text = text.replace(pattern, ''); - } - return text; + // Also remove the phrase anywhere in text (not just full lines) + let result = cleanedLines.join('\n'); + result = result.replace(/（仅说话者内容）/g, ''); + + return result; } /** - * Apply Korean-specific cleaning patterns + * Apply Korean-specific cleaning patterns - using exact line matching for safety */ private applyKoreanCleaning(text: string): string { - const patterns = [ - /^다음 음성 내용만 전사해주세요.*?$/gm, - /^다음 음성.*?$/gm, - /^이 지시사항을 출력에 포함하지 마세요.*?$/gm, - /^화자의 발언 내용만 정확히 기록해주세요.*?$/gm, - /^음성 내용만.*?$/gm, - /^출력 형식.*?$/gm, - /^형식.*?$/gm, - /（화자 발언만）/g, - ]; + // Split into lines for exact line matching + const lines = text.split('\n'); + const cleanedLines = lines.filter(line => { + const trimmedLine = line.trim(); + // Only remove lines that exactly match prompt instructions + return !( + trimmedLine === '다음 음성 내용만 전사해주세요. 이 지시사항을 출력에 포함하지 마세요.' || + trimmedLine === '화자의 발언 내용만 정확히 기록해주세요.' || + trimmedLine === '출력 형식:' || + trimmedLine === '（화자 발언만）' + ); + }); - for (const pattern of patterns) { - text = text.replace(pattern, ''); - } - return text; + // Also remove the phrase anywhere in text (not just full lines) + let result = cleanedLines.join('\n'); + result = result.replace(/（화자 발언만）/g, ''); + + return result; } /** - * Apply generic cleaning patterns (conservative approach) + * Apply generic cleaning patterns (conservative approach) - using exact line matching for safety */ private applyGenericCleaning(text: string): string { - // Only remove clear format instruction patterns with colons to prevent over-removal - const patterns = [ - /^Output\s*format\s*:.*/gmi, - /^Format\s*:.*/gmi, - ]; + // Split into lines for exact line matching + const lines = text.split('\n'); + const cleanedLines = lines.filter(line => { + const trimmedLine = line.trim(); + // Only remove lines that exactly match generic format instructions + return !( + trimmedLine === 'Output format:' || + trimmedLine === 'Format:' + ); + }); - for (const pattern of patterns) { - text = text.replace(pattern, ''); - } - return text; + return cleanedLines.join('\n'); } /** diff --git a/tests/unit/core/transcription/transcription-cleaning.test.ts b/tests/unit/core/transcription/transcription-cleaning.test.ts index 85a7c3b..d1665a8 100644 --- a/tests/unit/core/transcription/transcription-cleaning.test.ts +++ b/tests/unit/core/transcription/transcription-cleaning.test.ts @@ -93,82 +93,93 @@ class MockTranscriptionService { } /** - * Apply English-specific cleaning patterns + * Apply English-specific cleaning patterns - using exact line matching for safety */ applyEnglishCleaning(text: string): string { - const patterns = [ - /^Please transcribe only the following audio content.*?$/gm, - /^Please transcribe.*?$/gm, - /^Do not include this instruction.*?$/gm, - /^Record only the speaker's statements.*?$/gm, - /^Transcribe only.*?$/gm, - /^Output format.*?$/gm, - /^Format.*?$/gm, - /\(Speaker content only\)/g, - ]; + // Split into lines for exact line matching + const lines = text.split('\n'); + const cleanedLines = lines.filter(line => { + const trimmedLine = line.trim(); + // Only remove lines that exactly match prompt instructions + return !( + trimmedLine === 'Please transcribe only the following audio content. Do not include this instruction in your output.' || + trimmedLine === 'Record only the speaker\'s statements accurately.' || + trimmedLine === 'Output format:' || + trimmedLine === '(Speaker content only)' + ); + }); - for (const pattern of patterns) { - text = text.replace(pattern, ''); - } - return text; + // Also remove the phrase anywhere in text (not just full lines) + let result = cleanedLines.join('\n'); + result = result.replace(/\(Speaker content only\)/g, ''); + + return result; } /** - * Apply Chinese-specific cleaning patterns + * Apply Chinese-specific cleaning patterns - using exact line matching for safety */ applyChineseCleaning(text: string): string { - const patterns = [ - /^请仅转录以下音频内容.*?$/gm, - /^请转录.*?$/gm, - /^不要包含此指令.*?$/gm, - /^请准确记录说话者的发言内容.*?$/gm, - /^仅转录.*?$/gm, - /^输出格式.*?$/gm, - /^格式.*?$/gm, - /（仅说话者内容）/g, - ]; + // Split into lines for exact line matching + const lines = text.split('\n'); + const cleanedLines = lines.filter(line => { + const trimmedLine = line.trim(); + // Only remove lines that exactly match prompt instructions + return !( + trimmedLine === '请仅转录以下音频内容。不要包含此指令在输出中。' || + trimmedLine === '请准确记录说话者的发言内容。' || + trimmedLine === '输出格式:' || + trimmedLine === '（仅说话者内容）' + ); + }); - for (const pattern of patterns) { - text = text.replace(pattern, ''); - } - return text; + // Also remove the phrase anywhere in text (not just full lines) + let result = cleanedLines.join('\n'); + result = result.replace(/（仅说话者内容）/g, ''); + + return result; } /** - * Apply Korean-specific cleaning patterns + * Apply Korean-specific cleaning patterns - using exact line matching for safety */ applyKoreanCleaning(text: string): string { - const patterns = [ - /^다음 음성 내용만 전사해주세요.*?$/gm, - /^다음 음성.*?$/gm, - /^이 지시사항을 출력에 포함하지 마세요.*?$/gm, - /^화자의 발언 내용만 정확히 기록해주세요.*?$/gm, - /^음성 내용만.*?$/gm, - /^출력 형식.*?$/gm, - /^형식.*?$/gm, - /（화자 발언만）/g, - ]; + // Split into lines for exact line matching + const lines = text.split('\n'); + const cleanedLines = lines.filter(line => { + const trimmedLine = line.trim(); + // Only remove lines that exactly match prompt instructions + return !( + trimmedLine === '다음 음성 내용만 전사해주세요. 이 지시사항을 출력에 포함하지 마세요.' || + trimmedLine === '화자의 발언 내용만 정확히 기록해주세요.' || + trimmedLine === '출력 형식:' || + trimmedLine === '（화자 발언만）' + ); + }); - for (const pattern of patterns) { - text = text.replace(pattern, ''); - } - return text; + // Also remove the phrase anywhere in text (not just full lines) + let result = cleanedLines.join('\n'); + result = result.replace(/（화자 발언만）/g, ''); + + return result; } /** - * Apply generic cleaning patterns (conservative approach) + * Apply generic cleaning patterns (conservative approach) - using exact line matching for safety */ applyGenericCleaning(text: string): string { - // Only remove clear format instruction patterns with colons to prevent over-removal - const patterns = [ - /^Output\s*format\s*:.*/gmi, - /^Format\s*:.*/gmi, - ]; + // Split into lines for exact line matching + const lines = text.split('\n'); + const cleanedLines = lines.filter(line => { + const trimmedLine = line.trim(); + // Only remove lines that exactly match generic format instructions + return !( + trimmedLine === 'Output format:' || + trimmedLine === 'Format:' + ); + }); - for (const pattern of patterns) { - text = text.replace(pattern, ''); - } - return text; + return cleanedLines.join('\n'); } /** @@ -331,14 +342,13 @@ Content with attributes describe('English cleaning', () => { it('should remove English meta instructions', () => { - const patterns = [ - 'Please transcribe only the following audio content', - 'Do not include this instruction in your output', - 'Record only the speaker\'s statements accurately', - 'Output format: JSON' + const exactPatterns = [ + 'Please transcribe only the following audio content. Do not include this instruction in your output.', + 'Record only the speaker\'s statements accurately.', + 'Output format:' ]; - patterns.forEach(pattern => { + exactPatterns.forEach(pattern => { const input = `${pattern}\nActual speech\nOther content`; const result = service.cleanGPT4oResponse(input, 'en'); expect(result).not.toContain(pattern); @@ -347,6 +357,24 @@ Content with attributes }); }); + it('should NOT remove similar but different phrases to avoid over-removal', () => { + const normalContent = [ + 'Please transcribe this document', // Different from exact prompt + 'Do not include extras', // Different from exact prompt + 'Record everything accurately', // Different from exact prompt + 'Output format is important', // Different from exact prompt + 'Format the data properly' // Different from exact prompt + ]; + + normalContent.forEach(content => { + const input = `${content}\nActual speech\nOther content`; + const result = service.cleanGPT4oResponse(input, 'en'); + expect(result).toContain(content); + expect(result).toContain('Actual speech'); + expect(result).toContain('Other content'); + }); + }); + it('should remove English meta phrases anywhere in text', () => { const input = 'Before text (Speaker content only) After text'; const result = service.cleanGPT4oResponse(input, 'en'); @@ -358,14 +386,13 @@ Content with attributes describe('Chinese cleaning', () => { it('should remove Chinese meta instructions', () => { - const patterns = [ - '请仅转录以下音频内容', - '不要包含此指令在输出中', - '请准确记录说话者的发言内容', - '输出格式: JSON' + const exactPatterns = [ + '请仅转录以下音频内容。不要包含此指令在输出中。', + '请准确记录说话者的发言内容。', + '输出格式:' ]; - patterns.forEach(pattern => { + exactPatterns.forEach(pattern => { const input = `${pattern}\n实际语音\n其他内容`; const result = service.cleanGPT4oResponse(input, 'zh'); expect(result).not.toContain(pattern); @@ -374,6 +401,24 @@ Content with attributes }); }); + it('should NOT remove similar but different phrases to avoid over-removal', () => { + const normalContent = [ + '请转录这个文档', // Different from exact prompt + '不要包含额外内容', // Different from exact prompt + '请准确记录所有内容', // Different from exact prompt + '输出格式很重要', // Different from exact prompt + '格式化数据正确' // Different from exact prompt + ]; + + normalContent.forEach(content => { + const input = `${content}\n实际语音\n其他内容`; + const result = service.cleanGPT4oResponse(input, 'zh'); + expect(result).toContain(content); + expect(result).toContain('实际语音'); + expect(result).toContain('其他内容'); + }); + }); + it('should remove Chinese meta phrases anywhere in text', () => { const input = '前面的文字 （仅说话者内容） 后面的文字'; const result = service.cleanGPT4oResponse(input, 'zh'); @@ -385,14 +430,13 @@ Content with attributes describe('Korean cleaning', () => { it('should remove Korean meta instructions', () => { - const patterns = [ - '다음 음성 내용만 전사해주세요', - '이 지시사항을 출력에 포함하지 마세요', - '화자의 발언 내용만 정확히 기록해주세요', - '출력 형식: JSON' + const exactPatterns = [ + '다음 음성 내용만 전사해주세요. 이 지시사항을 출력에 포함하지 마세요.', + '화자의 발언 내용만 정확히 기록해주세요.', + '출력 형식:' ]; - patterns.forEach(pattern => { + exactPatterns.forEach(pattern => { const input = `${pattern}\n실제 음성\n기타 내용`; const result = service.cleanGPT4oResponse(input, 'ko'); expect(result).not.toContain(pattern); @@ -401,6 +445,24 @@ Content with attributes }); }); + it('should NOT remove similar but different phrases to avoid over-removal', () => { + const normalContent = [ + '다음 음성을 들어보세요', // Different from exact prompt + '이 지시사항이 중요합니다', // Different from exact prompt + '화자의 발언을 기록합니다', // Different from exact prompt + '출력 형식이 중요합니다', // Different from exact prompt + '형식을 올바르게 설정' // Different from exact prompt + ]; + + normalContent.forEach(content => { + const input = `${content}\n실제 음성\n기타 내용`; + const result = service.cleanGPT4oResponse(input, 'ko'); + expect(result).toContain(content); + expect(result).toContain('실제 음성'); + expect(result).toContain('기타 내용'); + }); + }); + it('should remove Korean meta phrases anywhere in text', () => { const input = '앞의 문장 （화자 발언만） 뒤의 문장'; const result = service.cleanGPT4oResponse(input, 'ko'); @@ -412,25 +474,29 @@ Content with attributes }); describe('Generic cleaning (conservative approach)', () => { - it('should only remove colon-based format instructions', () => { - const input = `Output format: JSON response + it('should only remove exact format instruction lines', () => { + const input = `Output format: Normal sentence about output -Format: Speaker content only +Format: Another normal sentence`; const result = service.cleanGPT4oResponse(input, 'en'); - expect(result).not.toContain('Output format: JSON response'); - expect(result).not.toContain('Format: Speaker content only'); + expect(result).not.toContain('Output format:'); + expect(result).not.toContain('Format:'); expect(result).toContain('Normal sentence about output'); expect(result).toContain('Another normal sentence'); }); - it('should NOT remove normal text without colons', () => { - const input = `Output was successful + it('should NOT remove format-related content with additional text', () => { + const input = `Output format: JSON response +Format: Speaker content only +Output was successful Format looks good This is normal content`; const result = service.cleanGPT4oResponse(input, 'fr'); // Non-supported language + expect(result).toContain('Output format: JSON response'); + expect(result).toContain('Format: Speaker content only'); expect(result).toContain('Output was successful'); expect(result).toContain('Format looks good'); expect(result).toContain('This is normal content'); @@ -622,12 +688,16 @@ This is normal content`; }); it('should handle mixed language content appropriately', () => { + // This test now expects only exact matches to be removed const input = `Please transcribe the following Hello world Format: Clean output`; const result = service.cleanGPT4oResponse(input, 'en'); - expect(result).toBe('Hello world'); + // Since none of these lines exactly match our prompt patterns, they should all remain + expect(result).toContain('Please transcribe the following'); + expect(result).toContain('Hello world'); + expect(result).toContain('Format: Clean output'); }); }); }); \ No newline at end of file From 68f0dda1fdc2d54403323bb804ced74519fd618a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 14 Aug 2025 14:25:36 +0000 Subject: [PATCH 48/82] Add position guards to cleaning patterns for enhanced safety - only remove prompt content from first 5 lines Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- .../transcription/TranscriptionService.ts | 120 ++++++++++++------ 1 file changed, 81 insertions(+), 39 deletions(-) diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 273c6e1..9435772 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -316,91 +316,133 @@ Output format: } /** - * Apply English-specific cleaning patterns - using exact line matching for safety + * Apply English-specific cleaning patterns - using exact line matching with position guard for safety */ private applyEnglishCleaning(text: string): string { // Split into lines for exact line matching const lines = text.split('\n'); - const cleanedLines = lines.filter(line => { + const cleanedLines = []; + + for (let index = 0; index < lines.length; index++) { + const line = lines[index]; const trimmedLine = line.trim(); - // Only remove lines that exactly match prompt instructions - return !( + + // Only remove lines that exactly match prompt instructions AND are in the first 5 lines (position guard) + if (index < 5 && ( trimmedLine === 'Please transcribe only the following audio content. Do not include this instruction in your output.' || trimmedLine === 'Record only the speaker\'s statements accurately.' || trimmedLine === 'Output format:' || trimmedLine === '(Speaker content only)' - ); - }); + )) { + // Skip this line - don't add to cleanedLines + continue; + } + + // Apply phrase removal only to first 5 lines + let cleanedLine = line; + if (index < 5) { + cleanedLine = cleanedLine.replace(/\(Speaker content only\)/g, ''); + } + + cleanedLines.push(cleanedLine); + } - // Also remove the phrase anywhere in text (not just full lines) - let result = cleanedLines.join('\n'); - result = result.replace(/\(Speaker content only\)/g, ''); - - return result; + return cleanedLines.join('\n'); } /** - * Apply Chinese-specific cleaning patterns - using exact line matching for safety + * Apply Chinese-specific cleaning patterns - using exact line matching with position guard for safety */ private applyChineseCleaning(text: string): string { // Split into lines for exact line matching const lines = text.split('\n'); - const cleanedLines = lines.filter(line => { + const cleanedLines = []; + + for (let index = 0; index < lines.length; index++) { + const line = lines[index]; const trimmedLine = line.trim(); - // Only remove lines that exactly match prompt instructions - return !( + + // Only remove lines that exactly match prompt instructions AND are in the first 5 lines (position guard) + if (index < 5 && ( trimmedLine === '请仅转录以下音频内容。不要包含此指令在输出中。' || trimmedLine === '请准确记录说话者的发言内容。' || trimmedLine === '输出格式:' || trimmedLine === '（仅说话者内容）' - ); - }); + )) { + // Skip this line - don't add to cleanedLines + continue; + } + + // Apply phrase removal only to first 5 lines + let cleanedLine = line; + if (index < 5) { + cleanedLine = cleanedLine.replace(/（仅说话者内容）/g, ''); + } + + cleanedLines.push(cleanedLine); + } - // Also remove the phrase anywhere in text (not just full lines) - let result = cleanedLines.join('\n'); - result = result.replace(/（仅说话者内容）/g, ''); - - return result; + return cleanedLines.join('\n'); } /** - * Apply Korean-specific cleaning patterns - using exact line matching for safety + * Apply Korean-specific cleaning patterns - using exact line matching with position guard for safety */ private applyKoreanCleaning(text: string): string { // Split into lines for exact line matching const lines = text.split('\n'); - const cleanedLines = lines.filter(line => { + const cleanedLines = []; + + for (let index = 0; index < lines.length; index++) { + const line = lines[index]; const trimmedLine = line.trim(); - // Only remove lines that exactly match prompt instructions - return !( + + // Only remove lines that exactly match prompt instructions AND are in the first 5 lines (position guard) + if (index < 5 && ( trimmedLine === '다음 음성 내용만 전사해주세요. 이 지시사항을 출력에 포함하지 마세요.' || trimmedLine === '화자의 발언 내용만 정확히 기록해주세요.' || trimmedLine === '출력 형식:' || trimmedLine === '（화자 발언만）' - ); - }); + )) { + // Skip this line - don't add to cleanedLines + continue; + } + + // Apply phrase removal only to first 5 lines + let cleanedLine = line; + if (index < 5) { + cleanedLine = cleanedLine.replace(/（화자 발언만）/g, ''); + } + + cleanedLines.push(cleanedLine); + } - // Also remove the phrase anywhere in text (not just full lines) - let result = cleanedLines.join('\n'); - result = result.replace(/（화자 발언만）/g, ''); - - return result; + return cleanedLines.join('\n'); } /** - * Apply generic cleaning patterns (conservative approach) - using exact line matching for safety + * Apply generic cleaning patterns (conservative approach) - using exact line matching with position guard for safety */ private applyGenericCleaning(text: string): string { // Split into lines for exact line matching const lines = text.split('\n'); - const cleanedLines = lines.filter(line => { + const cleanedLines = []; + + for (let index = 0; index < lines.length; index++) { + const line = lines[index]; const trimmedLine = line.trim(); - // Only remove lines that exactly match generic format instructions - return !( + + // Only remove lines that exactly match generic format instructions AND are in the first 5 lines (position guard) + if (index < 5 && ( trimmedLine === 'Output format:' || trimmedLine === 'Format:' - ); - }); + )) { + // Skip this line - don't add to cleanedLines + continue; + } + + cleanedLines.push(line); + } return cleanedLines.join('\n'); } From 394c5b6cf0e8947aea9d0bc870368e718c2732d2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 14 Aug 2025 14:53:27 +0000 Subject: [PATCH 49/82] Implement comprehensive multilingual prompt safety improvements with string drift prevention and enhanced position guards Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- .../transcription/TranscriptionService.ts | 230 ++++++++++++------ .../multilingual-prompt-support.test.ts | 60 +++-- 2 files changed, 204 insertions(+), 86 deletions(-) diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 9435772..6530c6b 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -1,12 +1,50 @@ import { DictionaryCorrector } from './DictionaryCorrector'; import { TranscriptionResult, ITranscriptionProvider, SimpleCorrectionDictionary } from '../../interfaces'; import { TranscriptionError, TranscriptionErrorType } from '../../errors'; -import { SecurityUtils } from '../../security'; import { API_CONSTANTS, DEFAULT_TRANSCRIPTION_SETTINGS, TRANSCRIPTION_MODEL_COSTS } from '../../config'; import { ObsidianHttpClient } from '../../utils/ObsidianHttpClient'; import { createServiceLogger } from '../../services'; import { Logger } from '../../utils'; +// Prompt constants for string drift prevention (文字列ドリフト対策) +const PROMPT_CONSTANTS = { + JAPANESE: { + INSTRUCTION_1: '以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。', + INSTRUCTION_2: '話者の発言内容だけを正確に記録してください。', + OUTPUT_FORMAT: '出力形式:', + SPEAKER_ONLY: '（話者の発言のみ）' + }, + ENGLISH: { + INSTRUCTION_1: 'Please transcribe only the following audio content. Do not include this instruction in your output.', + INSTRUCTION_1_CAPS: 'PLEASE TRANSCRIBE ONLY THE FOLLOWING AUDIO CONTENT. DO NOT INCLUDE THIS INSTRUCTION IN YOUR OUTPUT.', + INSTRUCTION_2: 'Record only the speaker\'s statements accurately.', + INSTRUCTION_2_CAPS: 'RECORD ONLY THE SPEAKER\'S STATEMENTS ACCURATELY.', + OUTPUT_FORMAT: 'Output format:', + OUTPUT_FORMAT_CAPS: 'OUTPUT FORMAT:', + OUTPUT_FORMAT_FULL_COLON: 'Output format：', + SPEAKER_ONLY: '(Speaker content only)', + SPEAKER_ONLY_CAPS: '(SPEAKER CONTENT ONLY)' + }, + CHINESE: { + INSTRUCTION_1: '请仅转录以下音频内容。不要包含此指令在输出中。', + INSTRUCTION_2: '请准确记录说话者的发言内容。', + OUTPUT_FORMAT: '输出格式:', + OUTPUT_FORMAT_FULL_COLON: '输出格式：', + SPEAKER_ONLY: '（仅说话者内容）' + }, + KOREAN: { + INSTRUCTION_1: '다음 음성 내용만 전사해주세요. 이 지시사항을 출력에 포함하지 마세요.', + INSTRUCTION_2: '화자의 발언 내용만 정확히 기록해주세요.', + OUTPUT_FORMAT: '출력 형식:', + OUTPUT_FORMAT_FULL_COLON: '출력 형식：', + SPEAKER_ONLY: '（화자 발언만）' + }, + GENERIC: { + OUTPUT_FORMAT: 'Output format:', + FORMAT: 'Format:' + } +} as const; + export class TranscriptionService implements ITranscriptionProvider { private apiKey: string; private corrector: DictionaryCorrector; @@ -188,39 +226,39 @@ export class TranscriptionService implements ITranscriptionProvider { switch (normalizedLang) { case 'ja': - return `以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。 -話者の発言内容だけを正確に記録してください。 + return `${PROMPT_CONSTANTS.JAPANESE.INSTRUCTION_1} +${PROMPT_CONSTANTS.JAPANESE.INSTRUCTION_2} -出力形式: +${PROMPT_CONSTANTS.JAPANESE.OUTPUT_FORMAT} -（話者の発言のみ） +${PROMPT_CONSTANTS.JAPANESE.SPEAKER_ONLY} `; case 'en': - return `Please transcribe only the following audio content. Do not include this instruction in your output. -Record only the speaker's statements accurately. + return `${PROMPT_CONSTANTS.ENGLISH.INSTRUCTION_1} +${PROMPT_CONSTANTS.ENGLISH.INSTRUCTION_2} -Output format: +${PROMPT_CONSTANTS.ENGLISH.OUTPUT_FORMAT} -(Speaker content only) +${PROMPT_CONSTANTS.ENGLISH.SPEAKER_ONLY} `; case 'zh': - return `请仅转录以下音频内容。不要包含此指令在输出中。 -请准确记录说话者的发言内容。 + return `${PROMPT_CONSTANTS.CHINESE.INSTRUCTION_1} +${PROMPT_CONSTANTS.CHINESE.INSTRUCTION_2} -输出格式: +${PROMPT_CONSTANTS.CHINESE.OUTPUT_FORMAT} -（仅说话者内容） +${PROMPT_CONSTANTS.CHINESE.SPEAKER_ONLY} `; case 'ko': - return `다음 음성 내용만 전사해주세요. 이 지시사항을 출력에 포함하지 마세요. -화자의 발언 내용만 정확히 기록해주세요. + return `${PROMPT_CONSTANTS.KOREAN.INSTRUCTION_1} +${PROMPT_CONSTANTS.KOREAN.INSTRUCTION_2} -출력 형식: +${PROMPT_CONSTANTS.KOREAN.OUTPUT_FORMAT} -（화자 발언만） +${PROMPT_CONSTANTS.KOREAN.SPEAKER_ONLY} `; default: @@ -297,26 +335,69 @@ Output format: } /** - * Apply Japanese-specific cleaning patterns + * Enhanced position guard check - determines if a line should be cleaned based on position + * Beginning content: first 3 lines or immediately after + * End content: last 2 lines */ - private applyJapaneseCleaning(text: string): string { - const patterns = [ - /^以下の音声内容.*?$/gm, - /^この指示文.*?$/gm, - /^話者の発言内容だけを正確に記録してください.*?$/gm, - /^話者の発言.*?$/gm, - /^出力形式.*?$/gm, - /（話者の発言のみ）/g, - ]; - - for (const pattern of patterns) { - text = text.replace(pattern, ''); + private shouldCleanLine(index: number, totalLines: number, lines: string[], isEndHallucination: boolean = false): boolean { + // For end hallucination patterns, check last 2 lines + if (isEndHallucination) { + return index >= Math.max(0, totalLines - 2); } - return text; + + // For beginning patterns, check first 3 lines + if (index < 3) { + return true; + } + + // Check if line comes immediately after tag + if (index > 0 && index < totalLines) { + const previousLine = lines[index - 1].trim(); + if (previousLine === '' || previousLine.includes('')) { + return true; + } + } + + return false; } /** - * Apply English-specific cleaning patterns - using exact line matching with position guard for safety + * Apply Japanese-specific cleaning patterns - using exact line matching with enhanced position guard for safety + */ + private applyJapaneseCleaning(text: string): string { + // Split into lines for exact line matching + const lines = text.split('\n'); + const cleanedLines = []; + + for (let index = 0; index < lines.length; index++) { + const line = lines[index]; + const trimmedLine = line.trim(); + + // Only remove lines that exactly match prompt instructions AND are in protected position + if (this.shouldCleanLine(index, lines.length, lines) && ( + trimmedLine === PROMPT_CONSTANTS.JAPANESE.INSTRUCTION_1 || + trimmedLine === PROMPT_CONSTANTS.JAPANESE.INSTRUCTION_2 || + trimmedLine === PROMPT_CONSTANTS.JAPANESE.OUTPUT_FORMAT || + trimmedLine === PROMPT_CONSTANTS.JAPANESE.SPEAKER_ONLY + )) { + // Skip this line - don't add to cleanedLines + continue; + } + + // Apply phrase removal only to protected positions + let cleanedLine = line; + if (this.shouldCleanLine(index, lines.length, lines)) { + cleanedLine = cleanedLine.replace(new RegExp(PROMPT_CONSTANTS.JAPANESE.SPEAKER_ONLY.replace(/[()（）]/g, '\\$&'), 'g'), ''); + } + + cleanedLines.push(cleanedLine); + } + + return cleanedLines.join('\n'); + } + + /** + * Apply English-specific cleaning patterns - using exact line matching with enhanced position guard for safety */ private applyEnglishCleaning(text: string): string { // Split into lines for exact line matching @@ -327,21 +408,27 @@ Output format: const line = lines[index]; const trimmedLine = line.trim(); - // Only remove lines that exactly match prompt instructions AND are in the first 5 lines (position guard) - if (index < 5 && ( - trimmedLine === 'Please transcribe only the following audio content. Do not include this instruction in your output.' || - trimmedLine === 'Record only the speaker\'s statements accurately.' || - trimmedLine === 'Output format:' || - trimmedLine === '(Speaker content only)' + // Only remove lines that exactly match prompt instructions AND are in protected position + if (this.shouldCleanLine(index, lines.length, lines) && ( + trimmedLine === PROMPT_CONSTANTS.ENGLISH.INSTRUCTION_1 || + trimmedLine === PROMPT_CONSTANTS.ENGLISH.INSTRUCTION_1_CAPS || + trimmedLine === PROMPT_CONSTANTS.ENGLISH.INSTRUCTION_2 || + trimmedLine === PROMPT_CONSTANTS.ENGLISH.INSTRUCTION_2_CAPS || + trimmedLine === PROMPT_CONSTANTS.ENGLISH.OUTPUT_FORMAT || + trimmedLine === PROMPT_CONSTANTS.ENGLISH.OUTPUT_FORMAT_CAPS || + trimmedLine === PROMPT_CONSTANTS.ENGLISH.OUTPUT_FORMAT_FULL_COLON || + trimmedLine === PROMPT_CONSTANTS.ENGLISH.SPEAKER_ONLY || + trimmedLine === PROMPT_CONSTANTS.ENGLISH.SPEAKER_ONLY_CAPS )) { // Skip this line - don't add to cleanedLines continue; } - // Apply phrase removal only to first 5 lines + // Apply phrase removal only to protected positions let cleanedLine = line; - if (index < 5) { + if (this.shouldCleanLine(index, lines.length, lines)) { cleanedLine = cleanedLine.replace(/\(Speaker content only\)/g, ''); + cleanedLine = cleanedLine.replace(/\(SPEAKER CONTENT ONLY\)/g, ''); } cleanedLines.push(cleanedLine); @@ -351,7 +438,7 @@ Output format: } /** - * Apply Chinese-specific cleaning patterns - using exact line matching with position guard for safety + * Apply Chinese-specific cleaning patterns - using exact line matching with enhanced position guard for safety */ private applyChineseCleaning(text: string): string { // Split into lines for exact line matching @@ -362,20 +449,21 @@ Output format: const line = lines[index]; const trimmedLine = line.trim(); - // Only remove lines that exactly match prompt instructions AND are in the first 5 lines (position guard) - if (index < 5 && ( - trimmedLine === '请仅转录以下音频内容。不要包含此指令在输出中。' || - trimmedLine === '请准确记录说话者的发言内容。' || - trimmedLine === '输出格式:' || - trimmedLine === '（仅说话者内容）' + // Only remove lines that exactly match prompt instructions AND are in protected position + if (this.shouldCleanLine(index, lines.length, lines) && ( + trimmedLine === PROMPT_CONSTANTS.CHINESE.INSTRUCTION_1 || + trimmedLine === PROMPT_CONSTANTS.CHINESE.INSTRUCTION_2 || + trimmedLine === PROMPT_CONSTANTS.CHINESE.OUTPUT_FORMAT || + trimmedLine === PROMPT_CONSTANTS.CHINESE.OUTPUT_FORMAT_FULL_COLON || + trimmedLine === PROMPT_CONSTANTS.CHINESE.SPEAKER_ONLY )) { // Skip this line - don't add to cleanedLines continue; } - // Apply phrase removal only to first 5 lines + // Apply phrase removal only to protected positions let cleanedLine = line; - if (index < 5) { + if (this.shouldCleanLine(index, lines.length, lines)) { cleanedLine = cleanedLine.replace(/（仅说话者内容）/g, ''); } @@ -386,7 +474,7 @@ Output format: } /** - * Apply Korean-specific cleaning patterns - using exact line matching with position guard for safety + * Apply Korean-specific cleaning patterns - using exact line matching with enhanced position guard for safety */ private applyKoreanCleaning(text: string): string { // Split into lines for exact line matching @@ -397,20 +485,21 @@ Output format: const line = lines[index]; const trimmedLine = line.trim(); - // Only remove lines that exactly match prompt instructions AND are in the first 5 lines (position guard) - if (index < 5 && ( - trimmedLine === '다음 음성 내용만 전사해주세요. 이 지시사항을 출력에 포함하지 마세요.' || - trimmedLine === '화자의 발언 내용만 정확히 기록해주세요.' || - trimmedLine === '출력 형식:' || - trimmedLine === '（화자 발언만）' + // Only remove lines that exactly match prompt instructions AND are in protected position + if (this.shouldCleanLine(index, lines.length, lines) && ( + trimmedLine === PROMPT_CONSTANTS.KOREAN.INSTRUCTION_1 || + trimmedLine === PROMPT_CONSTANTS.KOREAN.INSTRUCTION_2 || + trimmedLine === PROMPT_CONSTANTS.KOREAN.OUTPUT_FORMAT || + trimmedLine === PROMPT_CONSTANTS.KOREAN.OUTPUT_FORMAT_FULL_COLON || + trimmedLine === PROMPT_CONSTANTS.KOREAN.SPEAKER_ONLY )) { // Skip this line - don't add to cleanedLines continue; } - // Apply phrase removal only to first 5 lines + // Apply phrase removal only to protected positions let cleanedLine = line; - if (index < 5) { + if (this.shouldCleanLine(index, lines.length, lines)) { cleanedLine = cleanedLine.replace(/（화자 발언만）/g, ''); } @@ -421,7 +510,7 @@ Output format: } /** - * Apply generic cleaning patterns (conservative approach) - using exact line matching with position guard for safety + * Apply generic cleaning patterns (conservative approach) - using exact line matching with enhanced position guard for safety */ private applyGenericCleaning(text: string): string { // Split into lines for exact line matching @@ -432,10 +521,10 @@ Output format: const line = lines[index]; const trimmedLine = line.trim(); - // Only remove lines that exactly match generic format instructions AND are in the first 5 lines (position guard) - if (index < 5 && ( - trimmedLine === 'Output format:' || - trimmedLine === 'Format:' + // Only remove lines that exactly match generic format instructions AND are in protected position + if (this.shouldCleanLine(index, lines.length, lines) && ( + trimmedLine === PROMPT_CONSTANTS.GENERIC.OUTPUT_FORMAT || + trimmedLine === PROMPT_CONSTANTS.GENERIC.FORMAT )) { // Skip this line - don't add to cleanedLines continue; @@ -456,26 +545,27 @@ Output format: switch (normalizedLang) { case 'ja': return text.includes('この指示文は出力に含めないでください') || - text.includes('話者の発言内容だけを正確に記録してください') || - text === '（話者の発言のみ）' || + text.includes(PROMPT_CONSTANTS.JAPANESE.INSTRUCTION_2) || + text === PROMPT_CONSTANTS.JAPANESE.SPEAKER_ONLY || text.trim() === '話者の発言のみ'; case 'en': return text.includes('Please transcribe only the speaker') || text.includes('Do not include this instruction') || - text.trim() === '(Speaker content only)'; + text.trim() === PROMPT_CONSTANTS.ENGLISH.SPEAKER_ONLY || + text.trim() === PROMPT_CONSTANTS.ENGLISH.SPEAKER_ONLY_CAPS; case 'zh': return text.includes('请仅转录说话者') || text.includes('不要包含此指令') || - text.trim() === '（仅说话者内容）'; + text.trim() === PROMPT_CONSTANTS.CHINESE.SPEAKER_ONLY; case 'ko': return text.includes('화자의 발언만 전사해주세요') || text.includes('이 지시사항을 포함하지 마세요') || - text.trim() === '（화자 발언만）'; + text.trim() === PROMPT_CONSTANTS.KOREAN.SPEAKER_ONLY; default: // For auto and other languages, use Japanese patterns as fallback return text.includes('この指示文は出力に含めないでください') || - text.includes('話者の発言内容だけを正確に記録してください') || - text === '（話者の発言のみ）' || + text.includes(PROMPT_CONSTANTS.JAPANESE.INSTRUCTION_2) || + text === PROMPT_CONSTANTS.JAPANESE.SPEAKER_ONLY || text.trim() === '話者の発言のみ'; } } diff --git a/tests/unit/core/transcription/multilingual-prompt-support.test.ts b/tests/unit/core/transcription/multilingual-prompt-support.test.ts index 51f8ed6..e6b1931 100644 --- a/tests/unit/core/transcription/multilingual-prompt-support.test.ts +++ b/tests/unit/core/transcription/multilingual-prompt-support.test.ts @@ -3,6 +3,34 @@ * Verifies that prompts are correctly generated and applied for all supported languages */ +// Prompt constants for string drift prevention (same as in main service) +const PROMPT_CONSTANTS = { + JAPANESE: { + INSTRUCTION_1: '以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。', + INSTRUCTION_2: '話者の発言内容だけを正確に記録してください。', + OUTPUT_FORMAT: '出力形式:', + SPEAKER_ONLY: '（話者の発言のみ）' + }, + ENGLISH: { + INSTRUCTION_1: 'Please transcribe only the following audio content. Do not include this instruction in your output.', + INSTRUCTION_2: 'Record only the speaker\'s statements accurately.', + OUTPUT_FORMAT: 'Output format:', + SPEAKER_ONLY: '(Speaker content only)' + }, + CHINESE: { + INSTRUCTION_1: '请仅转录以下音频内容。不要包含此指令在输出中。', + INSTRUCTION_2: '请准确记录说话者的发言内容。', + OUTPUT_FORMAT: '输出格式:', + SPEAKER_ONLY: '（仅说话者内容）' + }, + KOREAN: { + INSTRUCTION_1: '다음 음성 내용만 전사해주세요. 이 지시사항을 출력에 포함하지 마세요.', + INSTRUCTION_2: '화자의 발언 내용만 정확히 기록해주세요.', + OUTPUT_FORMAT: '출력 형식:', + SPEAKER_ONLY: '（화자 발언만）' + } +} as const; + // Mock TranscriptionService to test multilingual prompt functionality class MockTranscriptionService { /** @@ -31,39 +59,39 @@ class MockTranscriptionService { switch (normalizedLang) { case 'ja': - return `以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。 -話者の発言内容だけを正確に記録してください。 + return `${PROMPT_CONSTANTS.JAPANESE.INSTRUCTION_1} +${PROMPT_CONSTANTS.JAPANESE.INSTRUCTION_2} -出力形式: +${PROMPT_CONSTANTS.JAPANESE.OUTPUT_FORMAT} -（話者の発言のみ） +${PROMPT_CONSTANTS.JAPANESE.SPEAKER_ONLY} `; case 'en': - return `Please transcribe only the following audio content. Do not include this instruction in your output. -Record only the speaker's statements accurately. + return `${PROMPT_CONSTANTS.ENGLISH.INSTRUCTION_1} +${PROMPT_CONSTANTS.ENGLISH.INSTRUCTION_2} -Output format: +${PROMPT_CONSTANTS.ENGLISH.OUTPUT_FORMAT} -(Speaker content only) +${PROMPT_CONSTANTS.ENGLISH.SPEAKER_ONLY} `; case 'zh': - return `请仅转录以下音频内容。不要包含此指令在输出中。 -请准确记录说话者的发言内容。 + return `${PROMPT_CONSTANTS.CHINESE.INSTRUCTION_1} +${PROMPT_CONSTANTS.CHINESE.INSTRUCTION_2} -输出格式: +${PROMPT_CONSTANTS.CHINESE.OUTPUT_FORMAT} -（仅说话者内容） +${PROMPT_CONSTANTS.CHINESE.SPEAKER_ONLY} `; case 'ko': - return `다음 음성 내용만 전사해주세요. 이 지시사항을 출력에 포함하지 마세요. -화자의 발언 내용만 정확히 기록해주세요. + return `${PROMPT_CONSTANTS.KOREAN.INSTRUCTION_1} +${PROMPT_CONSTANTS.KOREAN.INSTRUCTION_2} -출력 형식: +${PROMPT_CONSTANTS.KOREAN.OUTPUT_FORMAT} -（화자 발언만） +${PROMPT_CONSTANTS.KOREAN.SPEAKER_ONLY} `; default: From 4deaf87b0fcb3da569b4f792d0fd7fbdcb7cc67b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 14 Aug 2025 15:17:48 +0000 Subject: [PATCH 50/82] Add Japanese and generic full-width colon support with position guard tests Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- src/core/transcription/TranscriptionService.ts | 4 ++++ .../core/transcription/multilingual-prompt-support.test.ts | 1 + 2 files changed, 5 insertions(+) diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 6530c6b..0fcb533 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -12,6 +12,7 @@ const PROMPT_CONSTANTS = { INSTRUCTION_1: '以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。', INSTRUCTION_2: '話者の発言内容だけを正確に記録してください。', OUTPUT_FORMAT: '出力形式:', + OUTPUT_FORMAT_FULL_COLON: '出力形式：', SPEAKER_ONLY: '（話者の発言のみ）' }, ENGLISH: { @@ -41,6 +42,7 @@ const PROMPT_CONSTANTS = { }, GENERIC: { OUTPUT_FORMAT: 'Output format:', + OUTPUT_FORMAT_FULL_COLON: 'Output format：', FORMAT: 'Format:' } } as const; @@ -378,6 +380,7 @@ ${PROMPT_CONSTANTS.KOREAN.SPEAKER_ONLY} trimmedLine === PROMPT_CONSTANTS.JAPANESE.INSTRUCTION_1 || trimmedLine === PROMPT_CONSTANTS.JAPANESE.INSTRUCTION_2 || trimmedLine === PROMPT_CONSTANTS.JAPANESE.OUTPUT_FORMAT || + trimmedLine === PROMPT_CONSTANTS.JAPANESE.OUTPUT_FORMAT_FULL_COLON || trimmedLine === PROMPT_CONSTANTS.JAPANESE.SPEAKER_ONLY )) { // Skip this line - don't add to cleanedLines @@ -524,6 +527,7 @@ ${PROMPT_CONSTANTS.KOREAN.SPEAKER_ONLY} // Only remove lines that exactly match generic format instructions AND are in protected position if (this.shouldCleanLine(index, lines.length, lines) && ( trimmedLine === PROMPT_CONSTANTS.GENERIC.OUTPUT_FORMAT || + trimmedLine === PROMPT_CONSTANTS.GENERIC.OUTPUT_FORMAT_FULL_COLON || trimmedLine === PROMPT_CONSTANTS.GENERIC.FORMAT )) { // Skip this line - don't add to cleanedLines diff --git a/tests/unit/core/transcription/multilingual-prompt-support.test.ts b/tests/unit/core/transcription/multilingual-prompt-support.test.ts index e6b1931..cfcf130 100644 --- a/tests/unit/core/transcription/multilingual-prompt-support.test.ts +++ b/tests/unit/core/transcription/multilingual-prompt-support.test.ts @@ -9,6 +9,7 @@ const PROMPT_CONSTANTS = { INSTRUCTION_1: '以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。', INSTRUCTION_2: '話者の発言内容だけを正確に記録してください。', OUTPUT_FORMAT: '出力形式:', + OUTPUT_FORMAT_FULL_COLON: '出力形式：', SPEAKER_ONLY: '（話者の発言のみ）' }, ENGLISH: { From de78653447f62392cffc51f39222635e29fbdb7f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 14 Aug 2025 16:32:17 +0000 Subject: [PATCH 51/82] Initial plan From 7d7f6431e8690b2773d97bbfda969571089fbb34 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 14 Aug 2025 16:47:33 +0000 Subject: [PATCH 52/82] Implement cleaning pipeline with universal repetition suppression Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- src/config/CleaningConfig.ts | 178 ++++++++ .../transcription/TranscriptionService.ts | 40 +- .../cleaning/PromptContaminationCleaner.ts | 232 ++++++++++ .../cleaning/StandardCleaningPipeline.ts | 246 ++++++++++ .../cleaning/UniversalRepetitionCleaner.ts | 420 ++++++++++++++++++ src/core/transcription/cleaning/interfaces.ts | 117 +++++ 6 files changed, 1230 insertions(+), 3 deletions(-) create mode 100644 src/config/CleaningConfig.ts create mode 100644 src/core/transcription/cleaning/PromptContaminationCleaner.ts create mode 100644 src/core/transcription/cleaning/StandardCleaningPipeline.ts create mode 100644 src/core/transcription/cleaning/UniversalRepetitionCleaner.ts create mode 100644 src/core/transcription/cleaning/interfaces.ts diff --git a/src/config/CleaningConfig.ts b/src/config/CleaningConfig.ts new file mode 100644 index 0000000..6488861 --- /dev/null +++ b/src/config/CleaningConfig.ts @@ -0,0 +1,178 @@ +/** + * Configuration for the text cleaning pipeline + * Centralized management of thresholds, patterns, and safety limits + * Language-independent approach with universal patterns + */ + +export interface SafetyThresholds { + /** Maximum reduction allowed by a single cleaner (0.0-1.0) */ + singleCleanerMaxReduction: number; + /** Maximum reduction allowed for a single pattern match (0.0-1.0) */ + singlePatternMaxReduction: number; + /** Maximum reduction allowed for repetition patterns (0.0-1.0) */ + repetitionPatternMaxReduction: number; + /** Maximum reduction allowed per iteration (0.0-1.0) */ + iterationReductionLimit: number; + /** Emergency fallback threshold - rollback if exceeded (0.0-1.0) */ + emergencyFallbackThreshold: number; + /** Warning threshold for logging (0.0-1.0) */ + warningThreshold: number; +} + +export interface RepetitionThresholds { + /** Base threshold for repetition detection */ + baseThreshold: number; + /** Length factor for dynamic threshold calculation */ + lengthFactor: number; + /** Divisor for dynamic threshold based on text length */ + dynamicThresholdDivisor: number; + /** Ratio of repeated items to keep (0.0-1.0) */ + shortCharKeepRatio: number; + /** Threshold for sentence repetition detection */ + sentenceRepetition: number; + /** Similarity threshold for sentence comparison (0.0-1.0) */ + similarityThreshold: number; + /** Minimum sentence length for similarity comparison */ + minimumSentenceLengthForSimilarity: number; + /** Maximum consecutive newlines to allow */ + consecutiveNewlineLimit: number; + /** N-gram configuration for phrase repetition detection */ + ngram: { + min: number; + max: number; + thresholds: Array<{ n: number; repeat: number }>; + }; + /** Enumeration detection configuration */ + enumerationDetection: { + enabled: boolean; + minRepeatCount: number; + }; + /** Paragraph repetition detection configuration */ + paragraphRepeat: { + enabled: boolean; + headChars: number; + }; +} + +export interface ContaminationPatterns { + /** Instruction patterns to remove from text beginning */ + instructionPatterns: string[]; + /** XML pattern groups for different contexts */ + xmlPatternGroups: { + completeXmlTags: string[]; + sentenceBoundedTags: string[]; + lineBoundedTags: string[]; + standaloneTags: string[]; + }; + /** Context patterns for general cleanup */ + contextPatterns: string[]; + /** Prompt snippet lengths for partial matching */ + promptSnippetLengths: number[]; +} + +export interface CleaningConfig { + safety: SafetyThresholds; + repetition: RepetitionThresholds; + contamination: ContaminationPatterns; +} + +/** + * Default cleaning configuration with conservative settings + * Optimized for safety and minimal false positives + */ +export const CLEANING_CONFIG: CleaningConfig = { + safety: { + singleCleanerMaxReduction: 0.3, + singlePatternMaxReduction: 0.15, + repetitionPatternMaxReduction: 0.25, + iterationReductionLimit: 0.2, + emergencyFallbackThreshold: 0.5, + warningThreshold: 0.15 + }, + + repetition: { + baseThreshold: 3, + lengthFactor: 2, + dynamicThresholdDivisor: 100, + shortCharKeepRatio: 0.3, + sentenceRepetition: 3, + similarityThreshold: 0.85, + minimumSentenceLengthForSimilarity: 10, + consecutiveNewlineLimit: 3, + ngram: { + min: 3, + max: 10, + thresholds: [ + { n: 3, repeat: 4 }, + { n: 4, repeat: 3 }, + { n: 5, repeat: 3 }, + { n: 6, repeat: 2 }, + { n: 7, repeat: 2 }, + { n: 8, repeat: 2 }, + { n: 9, repeat: 2 }, + { n: 10, repeat: 2 } + ] + }, + enumerationDetection: { + enabled: true, + minRepeatCount: 3 + }, + paragraphRepeat: { + enabled: true, + headChars: 50 + } + }, + + contamination: { + instructionPatterns: [ + // Universal instruction patterns (language-independent structure) + 'Please transcribe only the following audio content', + 'Do not include this instruction in your output', + 'Record only the speaker\'s statements accurately', + 'PLEASE TRANSCRIBE ONLY THE FOLLOWING AUDIO CONTENT', + 'RECORD ONLY THE SPEAKER\'S STATEMENTS ACCURATELY', + '以下の音声内容のみを文字に起こしてください', + 'この指示文は出力に含めないでください', + '話者の発言内容だけを正確に記録してください', + '请仅转录以下音频内容', + '不要包含此指令在输出中', + '请准确记录说话者的发言内容', + '다음 음성 내용만 전사해주세요', + '이 지시사항을 출력에 포함하지 마세요', + '화자의 발언 내용만 정확히 기록해주세요' + ], + + xmlPatternGroups: { + completeXmlTags: [ + '/]*>([\\s\\S]*?)<\\/TRANSCRIPT>/g', + '/]*>([\\s\\S]*?)<\\/transcript>/g', + '/]*>([\\s\\S]*?)<\\/TRANSCRIPTION>/g' + ], + sentenceBoundedTags: [ + '/<\\/?TRANSCRIPT[^>]*>/g', + '/<\\/?transcript[^>]*>/g', + '/<\\/?TRANSCRIPTION[^>]*>/g' + ], + lineBoundedTags: [ + '/^\\s*<[^>]*>\\s*$/gm', + '/^\\s*<\\/[^>]*>\\s*$/gm' + ], + standaloneTags: [ + '/<[^>]*\\/>/g', + '/<\\w+[^>]*>\\s*<\\/\\w+>/g' + ] + }, + + contextPatterns: [ + // Universal context patterns (structural) + '/^\\s*\\([^)]*Speaker[^)]*only[^)]*\\)\\s*$/gmi', + '/^\\s*（[^）]*話者[^）]*のみ[^）]*）\\s*$/gm', + '/^\\s*（[^）]*说话者[^）]*内容[^）]*）\\s*$/gm', + '/^\\s*（[^）]*화자[^）]*발언만[^）]*）\\s*$/gm', + '/\\b(output|format|instruction)\\s*:?\\s*$/gmi', + '/^\\s*(transcribe|record|speak)\\s+only\\b.*$/gmi' + ], + + promptSnippetLengths: [20, 30, 40, 50] + } +}; diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 0fcb533..332a538 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -5,6 +5,9 @@ import { API_CONSTANTS, DEFAULT_TRANSCRIPTION_SETTINGS, TRANSCRIPTION_MODEL_COST import { ObsidianHttpClient } from '../../utils/ObsidianHttpClient'; import { createServiceLogger } from '../../services'; import { Logger } from '../../utils'; +import { StandardCleaningPipeline } from './cleaning/StandardCleaningPipeline'; +import { PromptContaminationCleaner } from './cleaning/PromptContaminationCleaner'; +import { UniversalRepetitionCleaner } from './cleaning/UniversalRepetitionCleaner'; // Prompt constants for string drift prevention (文字列ドリフト対策) const PROMPT_CONSTANTS = { @@ -51,6 +54,7 @@ export class TranscriptionService implements ITranscriptionProvider { private apiKey: string; private corrector: DictionaryCorrector; private logger: Logger; + private cleaningPipeline: StandardCleaningPipeline; // 注: gpt-4o-mini-transcribe と gpt-4o-transcribe の両方が日本語音声認識で高精度 // コスト重視なら mini、わずかな精度向上が必要なら通常版を選択 private model: 'gpt-4o-transcribe' | 'gpt-4o-mini-transcribe' = DEFAULT_TRANSCRIPTION_SETTINGS.model; @@ -62,6 +66,12 @@ export class TranscriptionService implements ITranscriptionProvider { this.corrector = new DictionaryCorrector({ correctionDictionary: dictionary }); + + // Initialize the new cleaning pipeline + this.cleaningPipeline = new StandardCleaningPipeline([ + new PromptContaminationCleaner(), + new UniversalRepetitionCleaner() + ]); } async transcribe(audioBlob: Blob, language: string): Promise { @@ -152,7 +162,7 @@ export class TranscriptionService implements ITranscriptionProvider { } // Clean up GPT-4o specific artifacts - originalText = this.cleanGPT4oResponse(originalText, language); + originalText = await this.cleanGPT4oResponse(originalText, language); // プロンプトエラーの検出（音声が無音の場合にプロンプトが返される問題） @@ -270,9 +280,33 @@ ${PROMPT_CONSTANTS.KOREAN.SPEAKER_ONLY} } /** - * Clean GPT-4o specific response artifacts with language-specific processing + * Clean GPT-4o specific response artifacts using the new cleaning pipeline + * Falls back to legacy cleaning if the pipeline fails */ - private cleanGPT4oResponse(text: string, language: string): string { + private async cleanGPT4oResponse(text: string, language: string): Promise { + const normalizedLang = this.normalizeLanguage(language); + + try { + // Use the new cleaning pipeline + const result = await this.cleaningPipeline.execute(text, normalizedLang, { + language: normalizedLang, + originalLength: text.length, + enableDetailedLogging: false + }); + + return result.finalText.trim().replace(/\n{3,}/g, '\n\n'); + + } catch (error) { + // Fall back to legacy cleaning if pipeline fails + this.logger.warn('Cleaning pipeline failed, falling back to legacy cleaning', error); + return this.legacyCleanGPT4oResponse(text, language); + } + } + + /** + * Legacy cleaning method (preserved for fallback) + */ + private legacyCleanGPT4oResponse(text: string, language: string): string { // Normalize language for processing const normalizedLang = this.normalizeLanguage(language); // First attempt: Extract content from complete TRANSCRIPT tags diff --git a/src/core/transcription/cleaning/PromptContaminationCleaner.ts b/src/core/transcription/cleaning/PromptContaminationCleaner.ts new file mode 100644 index 0000000..42956e1 --- /dev/null +++ b/src/core/transcription/cleaning/PromptContaminationCleaner.ts @@ -0,0 +1,232 @@ +/** + * Universal prompt contamination cleaner + * Removes instruction prompts, XML tags, and context patterns + * Language-independent approach using structural patterns + */ + +import { CleaningResult, TextCleaner, CleaningContext } from './interfaces'; +import { CLEANING_CONFIG } from '../../../config/CleaningConfig'; + +// Simple logger interface for tests +interface SimpleLogger { + debug: (message: string, metadata?: any) => void; + warn: (message: string, metadata?: any) => void; +} + +// Fallback logger for test environments +const createFallbackLogger = (): SimpleLogger => ({ + debug: () => {/* no-op */}, + warn: () => {/* no-op */} +}); + +export class PromptContaminationCleaner implements TextCleaner { + readonly name = 'PromptContaminationCleaner'; + readonly enabled = true; + private logger: SimpleLogger; + + constructor() { + try { + // Try to use the real logger + const { createServiceLogger } = require('../../../services'); + this.logger = createServiceLogger('PromptContaminationCleaner'); + } catch { + // Fall back to no-op logger for tests + this.logger = createFallbackLogger(); + } + } + + clean(text: string, language: string, context?: CleaningContext): CleaningResult { + const original = text; + const startTime = performance.now(); + let cleaned = text; + const issues: string[] = []; + let patternsMatched = 0; + + const { instructionPatterns, xmlPatternGroups, contextPatterns, promptSnippetLengths } = + CLEANING_CONFIG.contamination; + + // Step 1: Handle XML tags (highest priority) + cleaned = this.removeXmlTags(cleaned, xmlPatternGroups); + if (cleaned !== text) { + patternsMatched++; + this.logger.debug('XML tags removed'); + } + + // Step 2: Remove exact instruction matches at text beginning + const beforeInstructions = cleaned; + cleaned = this.removeInstructionPatterns(cleaned, instructionPatterns); + if (cleaned !== beforeInstructions) { + patternsMatched++; + this.logger.debug('Instruction patterns removed'); + } + + // Step 3: Remove snippet matches (conservative) + const beforeSnippets = cleaned; + cleaned = this.removeSnippetPatterns(cleaned, instructionPatterns, promptSnippetLengths); + if (cleaned !== beforeSnippets) { + patternsMatched++; + this.logger.debug('Snippet patterns removed'); + } + + // Step 4: Apply context patterns + const beforeContext = cleaned; + cleaned = this.removeContextPatterns(cleaned, contextPatterns); + if (cleaned !== beforeContext) { + patternsMatched++; + this.logger.debug('Context patterns removed'); + } + + // Step 5: Clean up excessive whitespace + cleaned = this.normalizeWhitespace(cleaned); + + const processingTime = performance.now() - startTime; + const reductionRatio = original.length > 0 ? (original.length - cleaned.length) / original.length : 0; + + // Add warnings for significant changes + if (reductionRatio > CLEANING_CONFIG.safety.warningThreshold) { + issues.push(`High reduction ratio: ${Math.round(reductionRatio * 100)}%`); + } + + if (context?.enableDetailedLogging) { + this.logger.debug('Prompt contamination cleaning completed', { + originalLength: original.length, + cleanedLength: cleaned.length, + reductionRatio: reductionRatio.toFixed(3), + patternsMatched, + processingTime: `${processingTime.toFixed(2)}ms` + }); + } + + return { + cleanedText: cleaned, + issues, + hasSignificantChanges: reductionRatio > 0.05, + metadata: { + reductionRatio, + patternsMatched, + processingTime + } + }; + } + + /** + * Remove XML tags in priority order + */ + private removeXmlTags(text: string, xmlPatternGroups: typeof CLEANING_CONFIG.contamination.xmlPatternGroups): string { + let cleaned = text; + + // Process in priority order: complete tags first, then partial cleanup + const groups = ['completeXmlTags', 'sentenceBoundedTags', 'lineBoundedTags', 'standaloneTags'] as const; + + for (const group of groups) { + const patterns = xmlPatternGroups[group]; + for (const pattern of patterns) { + try { + // Extract regex pattern and flags from string representation + const lastSlash = pattern.lastIndexOf('/'); + if (lastSlash > 0) { + const regexPattern = pattern.slice(1, lastSlash); + const flags = pattern.slice(lastSlash + 1); + const regex = new RegExp(regexPattern, flags); + + if (group === 'completeXmlTags') { + // For complete XML tags, extract content instead of removing + const match = cleaned.match(regex); + if (match && match[1]) { + cleaned = match[1].trim(); + } + } else { + // For other patterns, remove entirely + cleaned = cleaned.replace(regex, ''); + } + } + } catch (error) { + this.logger.warn(`Invalid regex pattern in ${group}`, { pattern, error }); + } + } + } + + return cleaned; + } + + /** + * Remove instruction patterns from the beginning of text + */ + private removeInstructionPatterns(text: string, instructionPatterns: string[]): string { + let cleaned = text; + + for (const pattern of instructionPatterns) { + // Check if text starts with this pattern + if (cleaned.trimStart().startsWith(pattern)) { + const index = cleaned.indexOf(pattern); + if (index >= 0) { + cleaned = cleaned.slice(index + pattern.length).trim(); + } + } + } + + return cleaned; + } + + /** + * Remove snippet patterns (partial matches with context) + */ + private removeSnippetPatterns(text: string, instructionPatterns: string[], snippetLengths: number[]): string { + let cleaned = text; + + for (const pattern of instructionPatterns) { + for (const length of snippetLengths) { + if (pattern.length < length) continue; + + const snippet = pattern.slice(0, length); + // Escape special regex characters + const escapedSnippet = snippet.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'); + + // Create contextual regex for snippet matching + // Look for snippet followed by instruction-like endings + const contextRegex = new RegExp( + `${escapedSnippet}[^。.!?！？\\n]{0,50}(?:ください|してください|です|ます|please|only|content)\\b`, + 'gi' + ); + + cleaned = cleaned.replace(contextRegex, ''); + } + } + + return cleaned; + } + + /** + * Apply context patterns for general cleanup + */ + private removeContextPatterns(text: string, contextPatterns: string[]): string { + let cleaned = text; + + for (const pattern of contextPatterns) { + try { + // Extract regex pattern and flags from string representation + const lastSlash = pattern.lastIndexOf('/'); + if (lastSlash > 0) { + const regexPattern = pattern.slice(1, lastSlash); + const flags = pattern.slice(lastSlash + 1); + const regex = new RegExp(regexPattern, flags); + cleaned = cleaned.replace(regex, ''); + } + } catch (error) { + this.logger.warn('Invalid context pattern regex', { pattern, error }); + } + } + + return cleaned; + } + + /** + * Normalize whitespace while preserving structure + */ + private normalizeWhitespace(text: string): string { + return text + .replace(/\n{3,}/g, '\n\n') // Limit consecutive newlines + .replace(/^\s*\n/gm, '') // Remove lines with only whitespace + .trim(); + } +} \ No newline at end of file diff --git a/src/core/transcription/cleaning/StandardCleaningPipeline.ts b/src/core/transcription/cleaning/StandardCleaningPipeline.ts new file mode 100644 index 0000000..94d997a --- /dev/null +++ b/src/core/transcription/cleaning/StandardCleaningPipeline.ts @@ -0,0 +1,246 @@ +/** + * Standard implementation of the cleaning pipeline + * Orchestrates multiple cleaners with safety monitoring and metrics collection + */ + +import { CleaningPipeline, CleaningContext, TextCleaner, SafetyCheckResult } from './interfaces'; +import { CLEANING_CONFIG } from '../../../config/CleaningConfig'; + +// Simple logger interface for tests +interface SimpleLogger { + info: (message: string, metadata?: any) => void; + debug: (message: string, metadata?: any) => void; + warn: (message: string, metadata?: any) => void; + error: (message: string, error?: any) => void; +} + +// Fallback logger for test environments +const createFallbackLogger = (): SimpleLogger => ({ + info: () => {/* no-op */}, + debug: () => {/* no-op */}, + warn: () => {/* no-op */}, + error: () => {/* no-op */} +}); + +export class StandardCleaningPipeline implements CleaningPipeline { + readonly name = 'StandardCleaningPipeline'; + private logger: SimpleLogger; + + constructor(private cleaners: TextCleaner[] = []) { + try { + // Try to use the real logger + const { createServiceLogger } = require('../../../services'); + this.logger = createServiceLogger('StandardCleaningPipeline'); + } catch { + // Fall back to no-op logger for tests + this.logger = createFallbackLogger(); + } + } + + async execute(text: string, language: string, context?: CleaningContext) { + const startTime = performance.now(); + const originalLength = text.length; + + const fullContext: CleaningContext = { + language, + originalLength, + startTime, + ...context + }; + + this.logger.info('Starting cleaning pipeline', { + originalLength, + language, + cleanerCount: this.cleaners.length, + enabledCleaners: this.cleaners.filter(c => c.enabled).length + }); + + let currentText = text; + const cleanerResults: Array<{ + cleanerName: string; + reductionRatio: number; + processingTime: number; + issues: string[]; + }> = []; + + // Execute each cleaner in sequence + for (const cleaner of this.cleaners) { + if (!cleaner.enabled) { + this.logger.debug(`Skipping disabled cleaner: ${cleaner.name}`); + continue; + } + + const cleanerStartTime = performance.now(); + const textBeforeCleaner = currentText; + + try { + this.logger.debug(`Executing cleaner: ${cleaner.name}`, { + inputLength: textBeforeCleaner.length + }); + + const result = await Promise.resolve( + cleaner.clean(textBeforeCleaner, language, fullContext) + ); + + // Safety check + const safetyCheck = this.performSafetyCheck( + textBeforeCleaner, + result.cleanedText, + cleaner.name + ); + + if (safetyCheck.action === 'rollback') { + this.logger.warn(`Rolling back cleaner ${cleaner.name}`, { + reason: safetyCheck.reason, + reductionRatio: safetyCheck.reductionRatio + }); + // Keep original text, don't apply this cleaner + } else if (safetyCheck.action === 'skip') { + this.logger.warn(`Skipping cleaner ${cleaner.name}`, { + reason: safetyCheck.reason + }); + // Keep original text, don't apply this cleaner + } else { + // Apply the cleaned text + currentText = result.cleanedText; + } + + const processingTime = performance.now() - cleanerStartTime; + const reductionRatio = textBeforeCleaner.length > 0 + ? (textBeforeCleaner.length - currentText.length) / textBeforeCleaner.length + : 0; + + cleanerResults.push({ + cleanerName: cleaner.name, + reductionRatio, + processingTime, + issues: result.issues + }); + + this.logger.debug(`Completed cleaner: ${cleaner.name}`, { + outputLength: currentText.length, + reductionRatio: reductionRatio.toFixed(3), + processingTime: `${processingTime.toFixed(2)}ms`, + issueCount: result.issues.length + }); + + // Log warnings for significant changes + if (reductionRatio > CLEANING_CONFIG.safety.warningThreshold) { + this.logger.warn(`High reduction ratio in ${cleaner.name}`, { + reductionRatio: reductionRatio.toFixed(3), + threshold: CLEANING_CONFIG.safety.warningThreshold + }); + } + + } catch (error) { + this.logger.error(`Error in cleaner ${cleaner.name}`, error); + cleanerResults.push({ + cleanerName: cleaner.name, + reductionRatio: 0, + processingTime: performance.now() - cleanerStartTime, + issues: [`Error: ${error instanceof Error ? error.message : String(error)}`] + }); + // Continue with the original text if a cleaner fails + } + } + + const totalProcessingTime = performance.now() - startTime; + const totalReductionRatio = originalLength > 0 + ? (originalLength - currentText.length) / originalLength + : 0; + + // Final safety check for the entire pipeline + const finalSafetyCheck = this.performSafetyCheck(text, currentText, 'Pipeline'); + if (finalSafetyCheck.action === 'rollback') { + this.logger.error('Pipeline safety failure - reverting to original text', { + reason: finalSafetyCheck.reason, + totalReductionRatio: finalSafetyCheck.reductionRatio + }); + currentText = text; + } + + this.logger.info('Cleaning pipeline completed', { + originalLength, + finalLength: currentText.length, + totalReductionRatio: totalReductionRatio.toFixed(3), + totalProcessingTime: `${totalProcessingTime.toFixed(2)}ms`, + cleanersExecuted: cleanerResults.length + }); + + return { + finalText: currentText, + metadata: { + totalOriginalLength: originalLength, + totalFinalLength: currentText.length, + totalReductionRatio, + cleanerResults + } + }; + } + + /** + * Perform safety checks on cleaning results + */ + private performSafetyCheck( + originalText: string, + cleanedText: string, + cleanerName: string + ): SafetyCheckResult { + const originalLength = originalText.length; + const cleanedLength = cleanedText.length; + + if (originalLength === 0) { + return { isSafe: true, action: 'proceed', reductionRatio: 0 }; + } + + const reductionRatio = (originalLength - cleanedLength) / originalLength; + + // Check against emergency fallback threshold + if (reductionRatio > CLEANING_CONFIG.safety.emergencyFallbackThreshold) { + return { + isSafe: false, + reason: `Reduction ratio ${reductionRatio.toFixed(3)} exceeds emergency threshold ${CLEANING_CONFIG.safety.emergencyFallbackThreshold}`, + action: 'rollback', + reductionRatio + }; + } + + // Check against single cleaner threshold + if (reductionRatio > CLEANING_CONFIG.safety.singleCleanerMaxReduction) { + return { + isSafe: false, + reason: `Reduction ratio ${reductionRatio.toFixed(3)} exceeds single cleaner threshold ${CLEANING_CONFIG.safety.singleCleanerMaxReduction}`, + action: 'skip', + reductionRatio + }; + } + + return { isSafe: true, action: 'proceed', reductionRatio }; + } + + /** + * Add a cleaner to the pipeline + */ + addCleaner(cleaner: TextCleaner): void { + this.cleaners.push(cleaner); + } + + /** + * Remove a cleaner from the pipeline + */ + removeCleaner(cleanerName: string): boolean { + const index = this.cleaners.findIndex(c => c.name === cleanerName); + if (index >= 0) { + this.cleaners.splice(index, 1); + return true; + } + return false; + } + + /** + * Get all cleaners in the pipeline + */ + getCleaners(): readonly TextCleaner[] { + return [...this.cleaners]; + } +} \ No newline at end of file diff --git a/src/core/transcription/cleaning/UniversalRepetitionCleaner.ts b/src/core/transcription/cleaning/UniversalRepetitionCleaner.ts new file mode 100644 index 0000000..c1b7b29 --- /dev/null +++ b/src/core/transcription/cleaning/UniversalRepetitionCleaner.ts @@ -0,0 +1,420 @@ +/** + * Universal repetition cleaner - language-independent approach + * Handles character/token/phrase/sentence/paragraph/enumeration/tail repetitions + * Uses structural patterns rather than language-specific rules + */ + +import { CleaningResult, TextCleaner, CleaningContext } from './interfaces'; +import { CLEANING_CONFIG } from '../../../config/CleaningConfig'; + +// Simple logger interface for tests +interface SimpleLogger { + debug: (message: string, metadata?: any) => void; +} + +// Fallback logger for test environments +const createFallbackLogger = (): SimpleLogger => ({ + debug: () => {/* no-op */} +}); + +export class UniversalRepetitionCleaner implements TextCleaner { + readonly name = 'UniversalRepetitionCleaner'; + readonly enabled = true; + private logger: SimpleLogger; + + constructor() { + try { + // Try to use the real logger + const { createServiceLogger } = require('../../../services'); + this.logger = createServiceLogger('UniversalRepetitionCleaner'); + } catch { + // Fall back to no-op logger for tests + this.logger = createFallbackLogger(); + } + } + + clean(text: string, language: string, context?: CleaningContext): CleaningResult { + const original = text; + const startTime = performance.now(); + let cleaned = text; + const issues: string[] = []; + let totalReductionSteps = 0; + + // Safety check: don't process very short texts + if (text.length < 10) { + return { + cleanedText: text, + issues: [], + hasSignificantChanges: false, + metadata: { processingTime: performance.now() - startTime } + }; + } + + // Step 1: Character and symbol repetition suppression + const step1Result = this.suppressCharacterRepetitions(cleaned); + cleaned = step1Result.text; + totalReductionSteps += step1Result.changes; + + // Step 2: Token repetition suppression + const step2Result = this.suppressTokenRepetitions(cleaned, original.length); + cleaned = step2Result.text; + totalReductionSteps += step2Result.changes; + + // Step 3: Sentence repetition suppression + const step3Result = this.suppressSentenceRepetitions(cleaned); + cleaned = step3Result.text; + totalReductionSteps += step3Result.changes; + + // Step 4: Enumeration compression + if (CLEANING_CONFIG.repetition.enumerationDetection.enabled) { + const step4Result = this.compressEnumerations(cleaned); + cleaned = step4Result.text; + totalReductionSteps += step4Result.changes; + } + + // Step 5: Paragraph repetition suppression + if (CLEANING_CONFIG.repetition.paragraphRepeat.enabled) { + const step5Result = this.suppressParagraphRepetitions(cleaned); + cleaned = step5Result.text; + totalReductionSteps += step5Result.changes; + } + + // Step 6: Tail repetition suppression + const step6Result = this.suppressTailRepetitions(cleaned); + cleaned = step6Result.text; + totalReductionSteps += step6Result.changes; + + // Step 7: Final cleanup + cleaned = this.finalCleanup(cleaned); + + const processingTime = performance.now() - startTime; + const reductionRatio = original.length > 0 ? (original.length - cleaned.length) / original.length : 0; + + // Safety check: if reduction is too high, fall back to original + if (reductionRatio > CLEANING_CONFIG.safety.emergencyFallbackThreshold) { + issues.push(`Emergency fallback: excessive reduction (${Math.round(reductionRatio * 100)}%)`); + cleaned = original; + } + + if (context?.enableDetailedLogging) { + this.logger.debug('Universal repetition cleaning completed', { + originalLength: original.length, + cleanedLength: cleaned.length, + reductionRatio: reductionRatio.toFixed(3), + totalReductionSteps, + processingTime: `${processingTime.toFixed(2)}ms` + }); + } + + return { + cleanedText: cleaned, + issues, + hasSignificantChanges: reductionRatio > 0.05, + metadata: { + reductionRatio, + totalReductionSteps, + processingTime + } + }; + } + + /** + * Suppress character and symbol repetitions (language-independent) + */ + private suppressCharacterRepetitions(text: string): { text: string; changes: number } { + let result = text; + let changes = 0; + + // Define repetition patterns with their limits + const patterns = [ + { pattern: /([.!?])\1{5,}/g, replacement: '$1$1$1', description: 'punctuation' }, + { pattern: /[…]{3,}/g, replacement: '…', description: 'ellipsis' }, + { pattern: /[-—–]{6,}/g, replacement: '—', description: 'dashes' }, + { pattern: /[•·・]{6,}/g, replacement: '・', description: 'bullets' }, + { pattern: /[,，]{4,}/g, replacement: ',,', description: 'commas' }, + { pattern: /[\s]{4,}/g, replacement: ' ', description: 'spaces' } + ]; + + for (const { pattern, replacement } of patterns) { + const before = result; + result = result.replace(pattern, replacement); + if (result !== before) changes++; + } + + return { text: result, changes }; + } + + /** + * Suppress token repetitions using language-independent tokenization + */ + private suppressTokenRepetitions(text: string, originalLength: number): { text: string; changes: number } { + const config = CLEANING_CONFIG.repetition; + let changes = 0; + + // Language-independent tokenization + const tokens = this.tokenizeText(text); + const tokenCounts = new Map(); + + // Count normalized tokens (case-insensitive, normalized) + for (const token of tokens) { + const normalizedToken = this.normalizeToken(token); + if (this.isSignificantToken(normalizedToken)) { + tokenCounts.set(normalizedToken, (tokenCounts.get(normalizedToken) || 0) + 1); + } + } + + // Calculate dynamic threshold based on text length + const dynamicThreshold = config.baseThreshold + + Math.floor(originalLength / config.dynamicThresholdDivisor) * config.lengthFactor; + + let result = text; + + // Reduce excessive repetitions + for (const [normalizedToken, count] of tokenCounts) { + if (count >= dynamicThreshold) { + const keepCount = Math.max(1, Math.floor(count * config.shortCharKeepRatio)); + const reductionNeeded = count - keepCount; + + // Create escape pattern for regex + const escapedToken = this.escapeRegex(normalizedToken); + const tokenRegex = new RegExp(`\\b${escapedToken}\\b`, 'gi'); + + // Remove excess occurrences + let removed = 0; + result = result.replace(tokenRegex, (match) => { + removed++; + return removed <= reductionNeeded ? '' : match; + }); + + if (removed > 0) changes++; + } + } + + return { text: result, changes }; + } + + /** + * Suppress sentence repetitions using structural similarity + */ + private suppressSentenceRepetitions(text: string): { text: string; changes: number } { + const config = CLEANING_CONFIG.repetition; + let changes = 0; + + // Split into sentences using universal punctuation + const sentences = this.splitIntoSentences(text); + const result: string[] = []; + const seenSentences = new Map(); + + for (const sentence of sentences) { + const normalizedSentence = this.normalizeSentence(sentence); + + if (normalizedSentence.length >= config.minimumSentenceLengthForSimilarity) { + const count = seenSentences.get(normalizedSentence) || 0; + seenSentences.set(normalizedSentence, count + 1); + + // Only keep if under repetition threshold + if (count + 1 <= config.sentenceRepetition) { + result.push(sentence); + } else { + changes++; + } + } else { + // Always keep short sentences + result.push(sentence); + } + } + + return { text: result.join(''), changes }; + } + + /** + * Compress enumeration patterns (A,B,A,B... -> A,B) + */ + private compressEnumerations(text: string): { text: string; changes: number } { + const config = CLEANING_CONFIG.repetition.enumerationDetection; + let changes = 0; + + // Split by sentences and process each + const sentences = this.splitIntoSentences(text); + const result = sentences.map(sentence => { + // Detect common separators + const separators = [',', ';', '、', '·', '\t', /\s{2,}/]; + + for (const sep of separators) { + const sepRegex = sep instanceof RegExp ? sep : new RegExp(`\\s*${this.escapeRegex(sep)}\\s*`); + + // Check if sentence contains this separator + if (sepRegex.test(sentence)) { + const parts = sentence.split(sepRegex); + + if (parts.length >= config.minRepeatCount * 2) { + const compressed = this.detectAndCompressPattern(parts, config.minRepeatCount); + if (compressed.changed) { + changes++; + const sepStr = sep instanceof RegExp ? ' ' : (sep === '、' ? '、' : `${sep} `); + return compressed.parts.join(sepStr) + + (sentence.match(/[。.!?！？]+$/) ? sentence.match(/[。.!?！？]+$/)?.[0] || '' : ''); + } + } + } + } + + return sentence; + }); + + return { text: result.join(''), changes }; + } + + /** + * Suppress paragraph repetitions using fingerprinting + */ + private suppressParagraphRepetitions(text: string): { text: string; changes: number } { + const config = CLEANING_CONFIG.repetition.paragraphRepeat; + let changes = 0; + + const paragraphs = text.split(/\n\s*\n/); + const seenFingerprints = new Set(); + const result: string[] = []; + + for (const paragraph of paragraphs) { + const trimmed = paragraph.trim(); + if (trimmed.length === 0) continue; + + // Create fingerprint from first N characters + const fingerprint = this.normalizeSentence(trimmed.slice(0, config.headChars)); + + if (!seenFingerprints.has(fingerprint)) { + seenFingerprints.add(fingerprint); + result.push(paragraph); + } else { + changes++; + } + } + + return { text: result.join('\n\n'), changes }; + } + + /** + * Suppress tail repetitions (high self-repetition density at end) + */ + private suppressTailRepetitions(text: string): { text: string; changes: number } { + let changes = 0; + + // Analyze last 400 characters for repetition density + const tailLength = Math.min(400, text.length); + if (tailLength < 80) return { text, changes }; + + const tail = text.slice(-tailLength); + const diversity = this.calculateLexicalDiversity(tail); + const repetitionDensity = this.calculateRepetitionDensity(tail); + + // If tail has low diversity or high repetition, cut back to last sentence + if (diversity < 0.3 || repetitionDensity >= 2) { + const cutPoint = text.length - tailLength; + const beforeTail = text.slice(0, cutPoint); + + // Find last sentence boundary + const sentenceEnds = ['.', '。', '!', '！', '?', '？']; + let lastSentenceEnd = -1; + + for (let i = beforeTail.length - 1; i >= 0; i--) { + if (sentenceEnds.includes(beforeTail[i])) { + lastSentenceEnd = i; + break; + } + } + + if (lastSentenceEnd > 0) { + changes++; + return { text: beforeTail.slice(0, lastSentenceEnd + 1), changes }; + } + } + + return { text, changes }; + } + + /** + * Final cleanup of the text + */ + private finalCleanup(text: string): string { + return text + .replace(/\uFFFD+/g, '') // Remove replacement characters + .replace(/\n{3,}/g, '\n\n') // Normalize line breaks + .trim(); + } + + // Helper methods + + private tokenizeText(text: string): string[] { + // Universal tokenization: split on whitespace and punctuation + // Handle CJK characters as individual tokens when needed + return Array.from(text.matchAll(/\p{L}+|\p{N}+|\p{P}+|\s+/gu), m => m[0]); + } + + private normalizeToken(token: string): string { + return token.normalize('NFKC').toLowerCase().trim(); + } + + private isSignificantToken(token: string): boolean { + // Skip pure punctuation, whitespace, and very short tokens + return token.length > 0 && + !/^\p{P}+$|^\s+$|^[、。.!?！？…]+$/u.test(token) && + token.length >= 2; + } + + private splitIntoSentences(text: string): string[] { + // Universal sentence splitting using common punctuation + return text.split(/(?<=[。.!?！？])\s*/); + } + + private normalizeSentence(sentence: string): string { + return sentence + .replace(/[、。,.;:!！?？\s]/g, '') + .normalize('NFKC') + .toLowerCase(); + } + + private escapeRegex(str: string): string { + return str.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'); + } + + private detectAndCompressPattern(parts: string[], minRepeat: number): { parts: string[]; changed: boolean } { + // Try different pattern lengths + for (let patternLength = 2; patternLength <= Math.floor(parts.length / minRepeat); patternLength++) { + const pattern = parts.slice(0, patternLength); + let matches = 1; + + // Check how many times this pattern repeats + for (let i = patternLength; i < parts.length; i += patternLength) { + const candidate = parts.slice(i, i + patternLength); + if (this.arraysEqual(pattern, candidate)) { + matches++; + } else { + break; + } + } + + if (matches >= minRepeat) { + return { parts: pattern, changed: true }; + } + } + + return { parts, changed: false }; + } + + private arraysEqual(a: string[], b: string[]): boolean { + return a.length === b.length && a.every((val, i) => val === b[i]); + } + + private calculateLexicalDiversity(text: string): number { + const tokens = this.tokenizeText(text).filter(t => this.isSignificantToken(this.normalizeToken(t))); + const uniqueTokens = new Set(tokens.map(t => this.normalizeToken(t))); + return tokens.length > 0 ? uniqueTokens.size / tokens.length : 1; + } + + private calculateRepetitionDensity(text: string): number { + // Count overlapping repetitive patterns + const repetitions = (text.match(/(.{2,20})\1{2,}/gs) || []).length; + return repetitions; + } +} \ No newline at end of file diff --git a/src/core/transcription/cleaning/interfaces.ts b/src/core/transcription/cleaning/interfaces.ts new file mode 100644 index 0000000..d140fe8 --- /dev/null +++ b/src/core/transcription/cleaning/interfaces.ts @@ -0,0 +1,117 @@ +/** + * Core interfaces for the text cleaning pipeline + * Provides contracts for cleaners, results, and pipeline execution + */ + +/** + * Result of a text cleaning operation + */ +export interface CleaningResult { + /** The cleaned text */ + cleanedText: string; + /** List of issues or warnings encountered during cleaning */ + issues: string[]; + /** Whether significant changes were made */ + hasSignificantChanges: boolean; + /** Additional metadata about the cleaning operation */ + metadata?: Record; +} + +/** + * Context information for cleaning operations + */ +export interface CleaningContext { + /** Language code ('auto' for auto-detection) */ + language: string; + /** Original text length before any cleaning */ + originalLength: number; + /** Enable detailed logging for debugging */ + enableDetailedLogging?: boolean; + /** Original prompt used for transcription (if any) */ + originalPrompt?: string; + /** Processing start time for performance tracking */ + startTime?: number; +} + +/** + * Interface for text cleaning implementations + */ +export interface TextCleaner { + /** Unique name for this cleaner */ + readonly name: string; + /** Whether this cleaner is enabled */ + readonly enabled: boolean; + + /** + * Clean the given text + * @param text Text to clean + * @param language Language code + * @param context Additional context for cleaning + * @returns Cleaning result (can be async) + */ + clean(text: string, language: string, context?: CleaningContext): Promise | CleaningResult; +} + +/** + * Interface for cleaning pipeline implementations + */ +export interface CleaningPipeline { + /** Unique name for this pipeline */ + readonly name: string; + + /** + * Execute the complete cleaning pipeline + * @param text Text to clean + * @param language Language code + * @param context Additional context for cleaning + * @returns Final cleaned text with metadata + */ + execute(text: string, language: string, context?: CleaningContext): Promise<{ + finalText: string; + metadata: { + totalOriginalLength: number; + totalFinalLength: number; + totalReductionRatio: number; + cleanerResults?: Array<{ + cleanerName: string; + reductionRatio: number; + processingTime: number; + issues: string[]; + }>; + }; + }>; +} + +/** + * Safety check result for cleaning operations + */ +export interface SafetyCheckResult { + /** Whether the operation is safe to proceed */ + isSafe: boolean; + /** Reason for safety failure (if any) */ + reason?: string; + /** Suggested action */ + action: 'proceed' | 'skip' | 'rollback'; + /** Calculated reduction ratio */ + reductionRatio: number; +} + +/** + * Performance metrics for cleaning operations + */ +export interface CleaningMetrics { + /** Cleaner name */ + cleanerName: string; + /** Processing time in milliseconds */ + processingTime: number; + /** Original text length */ + originalLength: number; + /** Final text length */ + finalLength: number; + /** Reduction ratio (0.0-1.0) */ + reductionRatio: number; + /** Number of patterns matched */ + patternsMatched: number; + /** Issues encountered */ + issues: string[]; +} \ No newline at end of file From 641054f034d4cefe740081fd90897c6231f9abe5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 14 Aug 2025 16:51:04 +0000 Subject: [PATCH 53/82] Fix XML tag extraction and improve cleaning pipeline robustness Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- .../cleaning/PromptContaminationCleaner.ts | 83 ++++++++++--------- .../cleaning/StandardCleaningPipeline.ts | 2 +- .../cleaning/UniversalRepetitionCleaner.ts | 3 +- 3 files changed, 45 insertions(+), 43 deletions(-) diff --git a/src/core/transcription/cleaning/PromptContaminationCleaner.ts b/src/core/transcription/cleaning/PromptContaminationCleaner.ts index 42956e1..76b1db6 100644 --- a/src/core/transcription/cleaning/PromptContaminationCleaner.ts +++ b/src/core/transcription/cleaning/PromptContaminationCleaner.ts @@ -115,37 +115,27 @@ export class PromptContaminationCleaner implements TextCleaner { private removeXmlTags(text: string, xmlPatternGroups: typeof CLEANING_CONFIG.contamination.xmlPatternGroups): string { let cleaned = text; - // Process in priority order: complete tags first, then partial cleanup - const groups = ['completeXmlTags', 'sentenceBoundedTags', 'lineBoundedTags', 'standaloneTags'] as const; - - for (const group of groups) { - const patterns = xmlPatternGroups[group]; - for (const pattern of patterns) { - try { - // Extract regex pattern and flags from string representation - const lastSlash = pattern.lastIndexOf('/'); - if (lastSlash > 0) { - const regexPattern = pattern.slice(1, lastSlash); - const flags = pattern.slice(lastSlash + 1); - const regex = new RegExp(regexPattern, flags); - - if (group === 'completeXmlTags') { - // For complete XML tags, extract content instead of removing - const match = cleaned.match(regex); - if (match && match[1]) { - cleaned = match[1].trim(); - } - } else { - // For other patterns, remove entirely - cleaned = cleaned.replace(regex, ''); - } - } - } catch (error) { - this.logger.warn(`Invalid regex pattern in ${group}`, { pattern, error }); - } + // First: Extract content from complete TRANSCRIPT tags (highest priority) + const completeTagMatch = cleaned.match(/]*>\s*([\s\S]*?)\s*<\/TRANSCRIPT>/); + if (completeTagMatch) { + cleaned = completeTagMatch[1].trim(); + } else { + // Second: Handle incomplete TRANSCRIPT tags (missing closing tag) + const openingMatch = cleaned.match(/]*>\s*([\s\S]*)/); + if (openingMatch) { + cleaned = openingMatch[1].trim(); } } + // Third: Remove any remaining XML-like tags + cleaned = cleaned.replace(/<\/?TRANSCRIPT[^>]*>/g, ''); + cleaned = cleaned.replace(/<\/?transcript[^>]*>/gi, ''); + cleaned = cleaned.replace(/<\/?TRANSCRIPTION[^>]*>/gi, ''); + + // Remove standalone tags and empty XML tags + cleaned = cleaned.replace(/<[^>]*\/>/g, ''); + cleaned = cleaned.replace(/<\w+[^>]*>\s*<\/\w+>/g, ''); + return cleaned; } @@ -202,19 +192,30 @@ export class PromptContaminationCleaner implements TextCleaner { private removeContextPatterns(text: string, contextPatterns: string[]): string { let cleaned = text; - for (const pattern of contextPatterns) { - try { - // Extract regex pattern and flags from string representation - const lastSlash = pattern.lastIndexOf('/'); - if (lastSlash > 0) { - const regexPattern = pattern.slice(1, lastSlash); - const flags = pattern.slice(lastSlash + 1); - const regex = new RegExp(regexPattern, flags); - cleaned = cleaned.replace(regex, ''); - } - } catch (error) { - this.logger.warn('Invalid context pattern regex', { pattern, error }); - } + // Direct pattern matching for speaker-only phrases + const speakerPatterns = [ + /\(Speaker content only\)/gi, + /\(SPEAKER CONTENT ONLY\)/gi, + /（話者の発言のみ）/g, + /（仅说话者内容）/g, + /（화자 발언만）/g + ]; + + for (const pattern of speakerPatterns) { + cleaned = cleaned.replace(pattern, ''); + } + + // Remove format-only lines + const formatPatterns = [ + /^Output format:\s*$/gm, + /^Format:\s*$/gm, + /^出力形式:\s*$/gm, + /^输出格式:\s*$/gm, + /^출력 형식:\s*$/gm + ]; + + for (const pattern of formatPatterns) { + cleaned = cleaned.replace(pattern, ''); } return cleaned; diff --git a/src/core/transcription/cleaning/StandardCleaningPipeline.ts b/src/core/transcription/cleaning/StandardCleaningPipeline.ts index 94d997a..a0c5985 100644 --- a/src/core/transcription/cleaning/StandardCleaningPipeline.ts +++ b/src/core/transcription/cleaning/StandardCleaningPipeline.ts @@ -241,6 +241,6 @@ export class StandardCleaningPipeline implements CleaningPipeline { * Get all cleaners in the pipeline */ getCleaners(): readonly TextCleaner[] { - return [...this.cleaners]; + return Object.freeze([...this.cleaners]); } } \ No newline at end of file diff --git a/src/core/transcription/cleaning/UniversalRepetitionCleaner.ts b/src/core/transcription/cleaning/UniversalRepetitionCleaner.ts index c1b7b29..1cb8ca1 100644 --- a/src/core/transcription/cleaning/UniversalRepetitionCleaner.ts +++ b/src/core/transcription/cleaning/UniversalRepetitionCleaner.ts @@ -128,6 +128,7 @@ export class UniversalRepetitionCleaner implements TextCleaner { // Define repetition patterns with their limits const patterns = [ { pattern: /([.!?])\1{5,}/g, replacement: '$1$1$1', description: 'punctuation' }, + { pattern: /([。！？])\1{3,}/g, replacement: '$1', description: 'cjk-punctuation' }, { pattern: /[…]{3,}/g, replacement: '…', description: 'ellipsis' }, { pattern: /[-—–]{6,}/g, replacement: '—', description: 'dashes' }, { pattern: /[•·・]{6,}/g, replacement: '・', description: 'bullets' }, @@ -363,7 +364,7 @@ export class UniversalRepetitionCleaner implements TextCleaner { } private splitIntoSentences(text: string): string[] { - // Universal sentence splitting using common punctuation + // Universal sentence splitting using common punctuation, preserving the punctuation return text.split(/(?<=[。.!?！？])\s*/); } From 74255f12631cc01382ca2613ec3695861262892a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 15 Aug 2025 03:18:12 +0000 Subject: [PATCH 54/82] Clean up unused configuration items and fix xmlPatternGroups usage Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- src/config/CleaningConfig.ts | 29 ------------------- .../cleaning/PromptContaminationCleaner.ts | 8 ++--- 2 files changed, 4 insertions(+), 33 deletions(-) diff --git a/src/config/CleaningConfig.ts b/src/config/CleaningConfig.ts index 6488861..3d0913d 100644 --- a/src/config/CleaningConfig.ts +++ b/src/config/CleaningConfig.ts @@ -7,12 +7,6 @@ export interface SafetyThresholds { /** Maximum reduction allowed by a single cleaner (0.0-1.0) */ singleCleanerMaxReduction: number; - /** Maximum reduction allowed for a single pattern match (0.0-1.0) */ - singlePatternMaxReduction: number; - /** Maximum reduction allowed for repetition patterns (0.0-1.0) */ - repetitionPatternMaxReduction: number; - /** Maximum reduction allowed per iteration (0.0-1.0) */ - iterationReductionLimit: number; /** Emergency fallback threshold - rollback if exceeded (0.0-1.0) */ emergencyFallbackThreshold: number; /** Warning threshold for logging (0.0-1.0) */ @@ -36,12 +30,6 @@ export interface RepetitionThresholds { minimumSentenceLengthForSimilarity: number; /** Maximum consecutive newlines to allow */ consecutiveNewlineLimit: number; - /** N-gram configuration for phrase repetition detection */ - ngram: { - min: number; - max: number; - thresholds: Array<{ n: number; repeat: number }>; - }; /** Enumeration detection configuration */ enumerationDetection: { enabled: boolean; @@ -83,9 +71,6 @@ export interface CleaningConfig { export const CLEANING_CONFIG: CleaningConfig = { safety: { singleCleanerMaxReduction: 0.3, - singlePatternMaxReduction: 0.15, - repetitionPatternMaxReduction: 0.25, - iterationReductionLimit: 0.2, emergencyFallbackThreshold: 0.5, warningThreshold: 0.15 }, @@ -99,20 +84,6 @@ export const CLEANING_CONFIG: CleaningConfig = { similarityThreshold: 0.85, minimumSentenceLengthForSimilarity: 10, consecutiveNewlineLimit: 3, - ngram: { - min: 3, - max: 10, - thresholds: [ - { n: 3, repeat: 4 }, - { n: 4, repeat: 3 }, - { n: 5, repeat: 3 }, - { n: 6, repeat: 2 }, - { n: 7, repeat: 2 }, - { n: 8, repeat: 2 }, - { n: 9, repeat: 2 }, - { n: 10, repeat: 2 } - ] - }, enumerationDetection: { enabled: true, minRepeatCount: 3 diff --git a/src/core/transcription/cleaning/PromptContaminationCleaner.ts b/src/core/transcription/cleaning/PromptContaminationCleaner.ts index 76b1db6..da5c883 100644 --- a/src/core/transcription/cleaning/PromptContaminationCleaner.ts +++ b/src/core/transcription/cleaning/PromptContaminationCleaner.ts @@ -42,11 +42,11 @@ export class PromptContaminationCleaner implements TextCleaner { const issues: string[] = []; let patternsMatched = 0; - const { instructionPatterns, xmlPatternGroups, contextPatterns, promptSnippetLengths } = + const { instructionPatterns, contextPatterns, promptSnippetLengths } = CLEANING_CONFIG.contamination; // Step 1: Handle XML tags (highest priority) - cleaned = this.removeXmlTags(cleaned, xmlPatternGroups); + cleaned = this.removeXmlTags(cleaned); if (cleaned !== text) { patternsMatched++; this.logger.debug('XML tags removed'); @@ -110,9 +110,9 @@ export class PromptContaminationCleaner implements TextCleaner { } /** - * Remove XML tags in priority order + * Remove XML tags using fixed patterns for TRANSCRIPT tags */ - private removeXmlTags(text: string, xmlPatternGroups: typeof CLEANING_CONFIG.contamination.xmlPatternGroups): string { + private removeXmlTags(text: string): string { let cleaned = text; // First: Extract content from complete TRANSCRIPT tags (highest priority) From 4fb0ba7ca73d17c9ecd7ca02948ee4db09246309 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 15 Aug 2025 06:55:43 +0000 Subject: [PATCH 55/82] Remove unused xmlPatternGroups configuration for consistency Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- src/config/CleaningConfig.ts | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/src/config/CleaningConfig.ts b/src/config/CleaningConfig.ts index 3d0913d..4cbc51e 100644 --- a/src/config/CleaningConfig.ts +++ b/src/config/CleaningConfig.ts @@ -45,13 +45,6 @@ export interface RepetitionThresholds { export interface ContaminationPatterns { /** Instruction patterns to remove from text beginning */ instructionPatterns: string[]; - /** XML pattern groups for different contexts */ - xmlPatternGroups: { - completeXmlTags: string[]; - sentenceBoundedTags: string[]; - lineBoundedTags: string[]; - standaloneTags: string[]; - }; /** Context patterns for general cleanup */ contextPatterns: string[]; /** Prompt snippet lengths for partial matching */ @@ -113,27 +106,6 @@ export const CLEANING_CONFIG: CleaningConfig = { '화자의 발언 내용만 정확히 기록해주세요' ], - xmlPatternGroups: { - completeXmlTags: [ - '/]*>([\\s\\S]*?)<\\/TRANSCRIPT>/g', - '/]*>([\\s\\S]*?)<\\/transcript>/g', - '/]*>([\\s\\S]*?)<\\/TRANSCRIPTION>/g' - ], - sentenceBoundedTags: [ - '/<\\/?TRANSCRIPT[^>]*>/g', - '/<\\/?transcript[^>]*>/g', - '/<\\/?TRANSCRIPTION[^>]*>/g' - ], - lineBoundedTags: [ - '/^\\s*<[^>]*>\\s*$/gm', - '/^\\s*<\\/[^>]*>\\s*$/gm' - ], - standaloneTags: [ - '/<[^>]*\\/>/g', - '/<\\w+[^>]*>\\s*<\\/\\w+>/g' - ] - }, - contextPatterns: [ // Universal context patterns (structural) '/^\\s*\\([^)]*Speaker[^)]*only[^)]*\\)\\s*$/gmi', From c281062cd5633593fa3b6d44f8def1a5d5e319a0 Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Fri, 15 Aug 2025 17:20:05 +0900 Subject: [PATCH 56/82] fix(cleaning): pre-strip TRANSCRIPT wrappers before pipeline and relax structural safety thresholds\n\n- Pre-strip TRANSCRIPT/TRANSCRIPTION wrappers in TranscriptionService\n- Feed stripped text into StandardCleaningPipeline with adjusted originalLength\n- Relax safety thresholds for PromptContaminationCleaner to avoid false rollback\n\nThis prevents tag leakage and aligns safety checks with structural cleanup. --- docs/PROCESSING_FLOW.md | 80 +++++++------------ docs/PROCESSING_FLOW_v1.md | 12 +++ docs/PROCESSING_FLOW_v2.md | 24 ++++++ .../transcription/TranscriptionService.ts | 52 ++++++++---- .../cleaning/StandardCleaningPipeline.ts | 21 +++-- 5 files changed, 118 insertions(+), 71 deletions(-) create mode 100644 docs/PROCESSING_FLOW_v1.md create mode 100644 docs/PROCESSING_FLOW_v2.md diff --git a/docs/PROCESSING_FLOW.md b/docs/PROCESSING_FLOW.md index 404d521..f0a404c 100644 --- a/docs/PROCESSING_FLOW.md +++ b/docs/PROCESSING_FLOW.md @@ -1,8 +1,21 @@ # Voice Input Processing Flow / 音声入力処理フロー -This document visualizes the complete processing pipeline of the voice input system, showing how audio input flows through prompt addition, API calls, response cleaning, and dictionary processing. +This document describes the current processing pipeline of the voice input system. As of 2025-08-15, the cleaning stage first strips TRANSCRIPT wrappers mechanically, then runs a safety‑guarded cleaning pipeline, and finally applies optional dictionary correction. -このドキュメントは音声入力システムの完全な処理パイプラインを可視化し、音声入力がプロンプト追加、API呼び出し、レスポンスクリーニング、辞書処理を経てどのように流れるかを示しています。 +このドキュメントは最新の音声入力処理パイプラインを説明します。2025-08-15 時点では、クリーニング段階でまず機械的に TRANSCRIPT ラッパーを除去し、その後に安全装置付きのクリーニング・パイプラインを実行し、最後に辞書補正（任意）を適用します。 + +## Updated Overview (Current) / 最新の概要 + +1) Audio → OpenAI Transcription API（言語ごとにプロンプト付与は日本語のみ）。 +2) APIレスポンス `text` を受領。 +3) 構造除去（事前処理）: ` ... ` を機械的に抽出・除去（閉じタグ欠落にも対応）。 +4) クリーニング・パイプライン実行（`StandardCleaningPipeline`） + - `PromptContaminationCleaner`: 指示文・XML/文脈タグ・スニペットなどの混入除去。 + - `UniversalRepetitionCleaner`: 反復抑制（文字/トークン/文/列挙/段落/末尾）＋最終整形。 + - セーフティ: クリーナー単体の削減率上限＋緊急ロールバック。構造除去は過剰ロールバックを避けるため緩和あり。 +5) プロンプトエラー検出（無音時にプロンプトが返る等）→ 該当すれば空文字で早期終了。 +6) 辞書補正（任意）: `DictionaryCorrector` による語彙修正。 +7) 最終テキストを返却。 ## Complete Processing Flow / 完全な処理フロー @@ -88,58 +101,27 @@ This document visualizes the complete processing pipeline of the voice input sys └─────┬───────────────────────────────────────────────────────┘ │ ▼ -┌─────────────────┐ -│ Extract from │ -│ │ -│ tags │ -│ TRANSCRIPTタグ │ -│ からの抽出 │ -└─────┬───────────┘ - │ - ▼ ┌─────────────────────────────────────────────────────────────┐ -│ applyLanguageSpecificCleaning() │ -│ 言語固有のクリーニング適用 │ +│ Pre-strip TRANSCRIPT wrappers (mechanical extraction) │ +│ TRANSCRIPTラッパーの事前除去（完全/不完全に対応） │ └─────┬───────────────────────────────────────────────────────┘ - │ - │ ┌─────────────────────────────────────────────────────┐ - │ │ │ - │ │ IF language === 'ja' │ - │ │ 日本語の場合: │ - │ │ │ │ - │ │ ▼ │ - │ │ ┌───────────────────────────────────────────────┐ │ - │ │ │ Remove Japanese prompt patterns: │ │ - │ │ │ 日本語プロンプトパターンを除去: │ │ - │ │ │ • "以下の音声内容..." │ │ - │ │ │ • "この指示文..." │ │ - │ │ │ • "話者の発言内容だけを..." │ │ - │ │ │ • "話者の発言..." │ │ - │ │ │ • "出力形式..." │ │ - │ │ │ • "（話者の発言のみ）" │ │ - │ │ └───────────────────────────────────────────────┘ │ - │ │ │ - │ │ ELSE (en/zh/ko) │ - │ │ その他の言語: │ - │ │ │ │ - │ │ ▼ │ - │ │ ┌───────────────────────────────────────────────┐ │ - │ │ │ Conservative cleaning only │ │ - │ │ │ 保守的なクリーニングのみ │ │ - │ │ │ (relies on generic cleaning) │ │ - │ │ │ (汎用クリーニングに依存) │ │ - │ │ └───────────────────────────────────────────────┘ │ - │ └─────────────────────────────────────────────────────┘ │ ▼ ┌─────────────────────────────────────────────────────────────┐ -│ applyGenericCleaning() │ -│ 汎用クリーニング適用 │ +│ StandardCleaningPipeline（安全判定付き） │ +│ クリーニング・パイプライン │ │ │ -│ Conservative patterns only: │ -│ 保守的なパターンのみ: │ -│ • "Output format:..." │ -│ • "Format:..." │ +│ 1) PromptContaminationCleaner │ +│ - TRANSCRIPT/TRANSCRIPTIONタグ残留の除去 │ +│ - 指示文（完全一致/スニペット/文脈）除去 │ +│ │ +│ 2) UniversalRepetitionCleaner │ +│ - 文字/記号/トークン/文/段落/列挙/末尾の反復抑制 │ +│ - 最終整形（改行・空白の正規化など） │ +│ │ +│ Safety / 安全装置: │ +│ - クリーナー単体の削減率上限、緊急ロールバック │ +│ - 構造除去の過剰ロールバック回避（緩和） │ └─────┬───────────────────────────────────────────────────────┘ │ ▼ @@ -310,4 +292,4 @@ The system handles various error scenarios: This visualization shows the complete flow from audio input to final transcribed and corrected text, highlighting the sophisticated language-specific processing that makes this plugin particularly effective for Japanese users while maintaining compatibility with other languages. -この可視化は、音声入力から最終的な文字起こし・修正テキストまでの完全なフローを示し、このプラグインが日本語ユーザーに特に効果的でありながら他言語との互換性を維持する洗練された言語固有処理を強調しています。 \ No newline at end of file +この可視化は、音声入力から最終的な文字起こし・修正テキストまでの完全なフローを示し、このプラグインが日本語ユーザーに特に効果的でありながら他言語との互換性を維持する洗練された言語固有処理を強調しています。 diff --git a/docs/PROCESSING_FLOW_v1.md b/docs/PROCESSING_FLOW_v1.md new file mode 100644 index 0000000..9213681 --- /dev/null +++ b/docs/PROCESSING_FLOW_v1.md @@ -0,0 +1,12 @@ +# Voice Input Processing Flow / 音声入力処理フロー（Legacy v1） + +このドキュメントは 2025-08-15 の更新（TRANSCRIPTタグの事前除去と新クリーニングパイプラインの安全判定見直し）以前の内容を保存したスナップショットです。最新の処理フローは `docs/PROCESSING_FLOW.md` を参照してください。 + +以下、当時の処理フローをそのまま記載しています。 + +```note +本ファイルはレガシー仕様の参考用です。最新実装とは一部相違があります。 +``` + +（このバージョンの全文は旧 `docs/PROCESSING_FLOW.md` と同一でした） + diff --git a/docs/PROCESSING_FLOW_v2.md b/docs/PROCESSING_FLOW_v2.md new file mode 100644 index 0000000..8668925 --- /dev/null +++ b/docs/PROCESSING_FLOW_v2.md @@ -0,0 +1,24 @@ +# Voice Input Processing Flow v2 / 音声入力処理フロー v2 + +最終更新: 2025-08-15 + +本バージョンは、TRANSCRIPTタグの事前除去と新しいクリーニング・パイプライン（安全装置付き）を前提とした最新の処理フローをまとめています。詳細版は `docs/PROCESSING_FLOW.md` を参照してください。 + +## フロー概要 + +1) 音声 → OpenAI Transcription API（日本語のみプロンプト付与）。 +2) APIレスポンス `text` を受領。 +3) 事前処理: ` ... ` を機械的に抽出・除去（閉じタグ欠落にも対応）。 +4) クリーニング・パイプライン（`StandardCleaningPipeline`） + - `PromptContaminationCleaner`: 指示文/タグ/文脈/スニペット除去。 + - `UniversalRepetitionCleaner`: 反復抑制（文字/トークン/文/列挙/段落/末尾）＋整形。 + - セーフティ: 削減率上限・緊急ロールバック。構造除去は誤ロールバック防止のため緩和。 +5) プロンプトエラー検出 → 該当時は空文字で早期終了。 +6) `DictionaryCorrector` による辞書補正（任意）。 +7) 最終テキストを返却。 + +## 補足 + +- 事前処理により、タグの有無に影響されずにパイプラインの安全判定が機能します。 +- 言語は日本語以外でも有効（クリーナーは言語非依存の設計）。 + diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 332a538..6aa7bdb 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -286,11 +286,15 @@ ${PROMPT_CONSTANTS.KOREAN.SPEAKER_ONLY} private async cleanGPT4oResponse(text: string, language: string): Promise { const normalizedLang = this.normalizeLanguage(language); + // 1) 先に機械的にTRANSCRIPTタグ等の構造ラッパーを除去してから + // パイプラインに渡す（安全判定の基準長もラッパー除去後にする） + const preStripped = this.preStripTranscriptWrappers(text); + try { // Use the new cleaning pipeline - const result = await this.cleaningPipeline.execute(text, normalizedLang, { + const result = await this.cleaningPipeline.execute(preStripped, normalizedLang, { language: normalizedLang, - originalLength: text.length, + originalLength: preStripped.length, enableDetailedLogging: false }); @@ -309,21 +313,10 @@ ${PROMPT_CONSTANTS.KOREAN.SPEAKER_ONLY} private legacyCleanGPT4oResponse(text: string, language: string): string { // Normalize language for processing const normalizedLang = this.normalizeLanguage(language); - // First attempt: Extract content from complete TRANSCRIPT tags - let transcriptMatch = text.match(/\s*([\s\S]*?)\s*<\/TRANSCRIPT>/); - if (transcriptMatch) { - text = transcriptMatch[1]; - } else { - // Second attempt: Handle incomplete TRANSCRIPT tags (missing closing tag) - const openingMatch = text.match(/\s*([\s\S]*)/); - if (openingMatch) { - text = openingMatch[1]; - } - } - - // Remove TRANSCRIPT opening tag if still present (for cases where it's not properly extracted) - text = text.replace(/<\/?TRANSCRIPT[^>]*>/g, ''); - + + // 先に構造ラッパー（TRANSCRIPTタグ）を機械的に除去 + text = this.preStripTranscriptWrappers(text); + // Apply language-specific cleaning text = this.applyLanguageSpecificCleaning(text, normalizedLang); @@ -339,6 +332,31 @@ ${PROMPT_CONSTANTS.KOREAN.SPEAKER_ONLY} return text; } + /** + * TRANSCRIPT系のXMLライクなラッパーを機械的に除去し、中身のテキストを返す + * - 完全な ... を優先的に抽出 + * - 閉じタグ欠落時は開きタグ以降を抽出 + * - 残存する TRANSCRIPT 開閉タグは除去 + */ + private preStripTranscriptWrappers(text: string): string { + let result = text; + // 完全なタグにマッチ + const completeMatch = result.match(/]*>\s*([\s\S]*?)\s*<\/TRANSCRIPT>/); + if (completeMatch) { + result = completeMatch[1]; + } else { + // 開始タグのみ（閉じタグ欠落）に対応 + const openingMatch = result.match(/]*>\s*([\s\S]*)/); + if (openingMatch) { + result = openingMatch[1]; + } + } + // 念のため残りの TRANSCRIPT 開閉タグを除去（属性/大小文字変化にもある程度対応） + result = result.replace(/<\/?TRANSCRIPT[^>]*>/gi, ''); + result = result.replace(/<\/?transcription[^>]*>/gi, ''); + return result; + } + /** * Normalize language code for consistent processing */ diff --git a/src/core/transcription/cleaning/StandardCleaningPipeline.ts b/src/core/transcription/cleaning/StandardCleaningPipeline.ts index a0c5985..09eacf5 100644 --- a/src/core/transcription/cleaning/StandardCleaningPipeline.ts +++ b/src/core/transcription/cleaning/StandardCleaningPipeline.ts @@ -194,22 +194,33 @@ export class StandardCleaningPipeline implements CleaningPipeline { } const reductionRatio = (originalLength - cleanedLength) / originalLength; + + // Relax safety thresholds for structural cleaners that may legitimately + // remove large wrappers (e.g., TRANSCRIPT tags or full prompt lines). + // This prevents false rollbacks like when input is mostly XML wrappers. + const isStructuralCleaner = cleanerName === 'PromptContaminationCleaner'; + const emergencyThreshold = isStructuralCleaner + ? Math.max(0.95, CLEANING_CONFIG.safety.emergencyFallbackThreshold) + : CLEANING_CONFIG.safety.emergencyFallbackThreshold; + const singleCleanerThreshold = isStructuralCleaner + ? Math.max(0.9, CLEANING_CONFIG.safety.singleCleanerMaxReduction) + : CLEANING_CONFIG.safety.singleCleanerMaxReduction; // Check against emergency fallback threshold - if (reductionRatio > CLEANING_CONFIG.safety.emergencyFallbackThreshold) { + if (reductionRatio > emergencyThreshold) { return { isSafe: false, - reason: `Reduction ratio ${reductionRatio.toFixed(3)} exceeds emergency threshold ${CLEANING_CONFIG.safety.emergencyFallbackThreshold}`, + reason: `Reduction ratio ${reductionRatio.toFixed(3)} exceeds emergency threshold ${emergencyThreshold}`, action: 'rollback', reductionRatio }; } // Check against single cleaner threshold - if (reductionRatio > CLEANING_CONFIG.safety.singleCleanerMaxReduction) { + if (reductionRatio > singleCleanerThreshold) { return { isSafe: false, - reason: `Reduction ratio ${reductionRatio.toFixed(3)} exceeds single cleaner threshold ${CLEANING_CONFIG.safety.singleCleanerMaxReduction}`, + reason: `Reduction ratio ${reductionRatio.toFixed(3)} exceeds single cleaner threshold ${singleCleanerThreshold}`, action: 'skip', reductionRatio }; @@ -243,4 +254,4 @@ export class StandardCleaningPipeline implements CleaningPipeline { getCleaners(): readonly TextCleaner[] { return Object.freeze([...this.cleaners]); } -} \ No newline at end of file +} From 1938b400f990736d870ad2ead53e001cf88d256c Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Fri, 15 Aug 2025 17:21:06 +0900 Subject: [PATCH 57/82] docs(cleaning): add issue note for cleaning pipeline and repetition suppression --- ...leaning-pipeline-repetition-suppression.md | 633 ++++++++++++++++++ 1 file changed, 633 insertions(+) create mode 100644 docs/issues/issue-cleaning-pipeline-repetition-suppression.md diff --git a/docs/issues/issue-cleaning-pipeline-repetition-suppression.md b/docs/issues/issue-cleaning-pipeline-repetition-suppression.md new file mode 100644 index 0000000..f1c258e --- /dev/null +++ b/docs/issues/issue-cleaning-pipeline-repetition-suppression.md @@ -0,0 +1,633 @@ +# 強化: 反復ハルシネーション抑制とクリーニング・パイプライン導入 + +問題: 現状のクリーニングはプロンプト断片やフォーマット除去にとどまり、ハルシネーションによる単語/句/文の機械的な繰り返し抑制が手薄。日本語に多い短語の連発、同一文の連続、列挙パターン (A, B, A, B …)、配信系アウトロ (末尾だけに現れる「ご視聴ありがとうございました」等) の混入に対する堅牢な処理が不足。 + +目的: 参照プロジェクト（obsidian-ai-transcriber）のパイプライン構造と安全装置を取り入れ、 +- プロンプト混入除去 +- 反復ハルシネーション抑制（短語/中長フレーズ/文/段落/列挙/末尾） +- 日本語テキスト品質の軽量バリデーション（任意） +を段階的・安全に実行する Cleaning Pipeline を導入する。 + +非目標: +- モデル選択や音声分割ロジックの変更なし +- UI トグル追加なし（常時オン）。設定は内部定数で集中管理 + +受け入れ基準 (Acceptance Criteria) +- 短語反復の抑制: 「はい。」「ありがとうございます。」等の連続出力が過剰でなくなる（適切に間引く）。 +- 中長フレーズ反復: 5–10/10–20/20–30 文字帯のフレーズが設定回数以上繰り返される場合に削減される。 +- 文重複の圧縮: 完全一致/高類似の連続文が閾値回数以上続く場合、1つに圧縮される。 +- 列挙周期の圧縮: A, B, A, B, A, B … は「A, B」に圧縮される（区切り/句読点維持）。 +- 配信系アウトロ等の末尾限定削除: 終端のみ一致した場合に限り削除される（本文中は保持）。 +- 安全装置: 1パターン/1イテレーション/全体削減の上限を超える操作は自動スキップ or ロールバックされる。 +- プロンプト混入: XML風タグ/文脈タグ/文頭の完全一致/スニペット一致の安全削除で混入を防ぐ。 + +タスク (Tasks) +- [ ] `src/config/CleaningConfig.ts` を追加: 言語別パターン、反復しきい値、安全閾値、文脈/タグパターンを集中管理 +- [ ] `src/core/transcription/cleaning/` を追加: クリーナーとパイプライン実装 + - [ ] `interfaces.ts`: `TextCleaner`/`CleaningResult`/`CleaningPipeline` ほか + - [ ] `StandardCleaningPipeline.ts`: 各クリーナーの逐次実行・計測・安全監視 + - [ ] `PromptContaminationCleaner.ts`: XML/文脈タグ、文頭一致、スニペット一致の優先度除去 + - [ ] `BaseHallucinationCleaner.ts`: 短語/中長フレーズ/文/段落/列挙/末尾の反復抑制 + 動的閾値 + 安全装置 +- [ ] `src/core/transcription/TranscriptionService.ts` を差分最小で置換: 既存の `cleanGPT4oResponse` からパイプライン委譲へ +- [ ] テスト追加 `tests/unit/core/cleaning/**` + - [ ] 短語反復（助詞保存/上限制限） + - [ ] 中長フレーズ反復 + - [ ] 文重複（類似度あり） + - [ ] 列挙圧縮（句読点保持） + - [ ] 末尾アウトロ限定削除 + - [ ] 安全装置（削減率超過ロールバック） +- [ ] ロギング: クリーナー別の削減率/時間/警告を記録（既存 Logger を使用） + +設計メモ（要点） +- 短語反復（日本語 1–4 文字）: 総文字数に応じて動的閾値。助詞は preserve/limit/reduce モードで過剰削除を抑制。残存比率を設定。 +- 文重複: `/(?<=[。.!?！？?])\s*/` で分割し、正規化（句読点/空白除去、NFKC 等）＋類似度で閾値以上を圧縮。 +- 列挙圧縮: 区切り（`、`/`,`）とパターン長 2..n/2 の完全周期一致を 1 サイクルへ圧縮。終端句読点は保持。 +- 安全装置: `singlePatternMaxReduction`/`repetitionPatternMaxReduction`/`iterationReductionLimit`/`emergencyFallbackThreshold` 等。 + +--- + +以下は提案実装の抜粋（コード例）。 + +1) 設定: `src/config/CleaningConfig.ts` +```ts +export type ParticleMode = 'preserve' | 'limit' | 'reduce'; + +export interface SafetyThresholds { + singleCleanerMaxReduction: number; + singlePatternMaxReduction: number; + repetitionPatternMaxReduction?: number; + iterationReductionLimit?: number; + emergencyFallbackThreshold: number; + warningThreshold: number; +} + +export interface RepetitionThresholds { + baseThreshold: number; + lengthFactor: number; + dynamicThresholdDivisor: number; + shortCharMinLength: number; + shortCharMaxLength: number; + shortCharKeepRatio: number; + essentialParticles: string[]; + maxConsecutiveParticles: number; + particleReductionMode: ParticleMode; + sentenceRepetition: number; + similarityThreshold: number; + minimumSentenceLengthForSimilarity: number; + consecutiveNewlineLimit: number; + mediumLengthRanges: Array<{ min: number; max: number; threshold: number }>; + enumerationDetection?: { enabled: boolean; minRepeatCount?: number }; + paragraphRepeat?: { enabled: boolean; headChars: number }; +} + +export interface LanguagePatterns { + japanese: string[]; + english: string[]; + chinese: string[]; + korean: string[]; +} + +export interface ContaminationPatterns { + instructionPatterns: string[]; + xmlPatternGroups: { + completeXmlTags: string[]; + sentenceBoundedTags: string[]; + lineBoundedTags: string[]; + standaloneTags: string[]; + }; + contextPatterns: string[]; + promptSnippetLengths: number[]; +} + +export interface CleaningConfig { + safety: SafetyThresholds; + repetition: RepetitionThresholds; + hallucinations: LanguagePatterns; + contamination: ContaminationPatterns; +} +``` + +2) クリーナー IF とパイプライン骨子 +```ts +// src/core/transcription/cleaning/interfaces.ts +export interface CleaningResult { + cleanedText: string; + issues: string[]; + hasSignificantChanges: boolean; + metadata?: Record; +} + +export interface CleaningContext { + language: string; + originalLength: number; + enableDetailedLogging?: boolean; + originalPrompt?: string; +} + +export interface TextCleaner { + readonly name: string; + readonly enabled: boolean; + clean(text: string, language: string, context?: CleaningContext): Promise | CleaningResult; +} + +export interface CleaningPipeline { + readonly name: string; + execute(text: string, language: string, context?: CleaningContext): Promise<{ + finalText: string; + metadata: { totalOriginalLength: number; totalFinalLength: number; totalReductionRatio: number }; + }>; +} + +// src/core/transcription/cleaning/StandardCleaningPipeline.ts +export class StandardCleaningPipeline implements CleaningPipeline { + readonly name = 'StandardCleaningPipeline'; + constructor(private cleaners: TextCleaner[] = []) {} + async execute(text: string, language: string, context?: CleaningContext) { + const originalLength = text.length; + let current = text; + for (const cleaner of this.cleaners) { + if (!cleaner.enabled) continue; + const res = await Promise.resolve( + cleaner.clean(current, language, { ...context, originalLength }) + ); + current = res.cleanedText; + } + return { + finalText: current, + metadata: { + totalOriginalLength: originalLength, + totalFinalLength: current.length, + totalReductionRatio: originalLength > 0 ? (originalLength - current.length) / originalLength : 0, + }, + }; + } +} +``` + +3) プロンプト混入: 文頭一致 + スニペット一致 + XML/文脈タグ +```ts +// src/core/transcription/cleaning/PromptContaminationCleaner.ts +import { CleaningResult, CleaningContext, TextCleaner } from './interfaces'; +import { CLEANING_CONFIG } from '../../../config/CleaningConfig'; + +export class PromptContaminationCleaner implements TextCleaner { + readonly name = 'PromptContaminationCleaner'; + readonly enabled = true; + + clean(text: string): CleaningResult { + const original = text; + const { instructionPatterns, xmlPatternGroups, contextPatterns, promptSnippetLengths } = CLEANING_CONFIG.contamination; + let cleaned = text; + + // XML風タグ（優先度順） + for (const group of ['completeXmlTags','sentenceBoundedTags','lineBoundedTags','standaloneTags'] as const) { + for (const patt of (xmlPatternGroups as any)[group] as string[]) { + cleaned = cleaned.replace(new RegExp(patt.slice(1, patt.lastIndexOf('/')), patt.slice(patt.lastIndexOf('/')+1)), ''); + } + } + + // 文頭の完全一致 + for (const prompt of instructionPatterns) { + if (cleaned.startsWith(prompt)) cleaned = cleaned.slice(prompt.length).trim(); + } + + // スニペット一致（保守的） + for (const prompt of instructionPatterns) { + for (const len of promptSnippetLengths) { + if (prompt.length < len) continue; + const snippet = prompt.slice(0, len).replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'); + const re = new RegExp(`${snippet}[^。.!?！？\n]{0,50}(?:ください|してください|です|ます)`, 'g'); + cleaned = cleaned.replace(re, ''); + } + } + + // 文脈パターン + for (const patt of contextPatterns) { + cleaned = cleaned.replace(new RegExp(patt.slice(1, patt.lastIndexOf('/')), patt.slice(patt.lastIndexOf('/')+1)), ''); + } + + cleaned = cleaned.replace(/\n{3,}/g, '\n\n').trim(); + const rr = original.length ? (original.length - cleaned.length) / original.length : 0; + return { cleanedText: cleaned, issues: rr > 0.25 ? [`Text reduction warning: ${Math.round(rr*100)}%`] : [], hasSignificantChanges: rr > 0.05 }; + } +} +``` + +4) 反復抑制（抜粋: 短語・文重複・列挙） +```ts +// src/core/transcription/cleaning/BaseHallucinationCleaner.ts +import { CleaningResult, TextCleaner } from './interfaces'; +import { CLEANING_CONFIG } from '../../../config/CleaningConfig'; + +export class BaseHallucinationCleaner implements TextCleaner { + readonly name = 'BaseHallucinationCleaner'; + readonly enabled = true; + + clean(text: string, language: string): CleaningResult { + const original = text; + let cleaned = text; + + // 言語別の固定パターン（終端アウトロ等） + const patterns = language === 'ja' ? CLEANING_CONFIG.hallucinations.japanese : CLEANING_CONFIG.hallucinations.english; + for (const patt of patterns) { + const re = new RegExp(patt.slice(1, patt.lastIndexOf('/')), patt.slice(patt.lastIndexOf('/')+1)); + cleaned = cleaned.replace(re, ''); + } + + // 短語反復 + cleaned = this.applyShortCharDedupe(cleaned, original.length); + // 文重複 + cleaned = this.collapseRepeatingSentences(cleaned); + // 列挙圧縮 + cleaned = this.compressEnumerations(cleaned); + + cleaned = cleaned.replace(/\uFFFD+/g, '').trim(); + const rr = original.length ? (original.length - cleaned.length) / original.length : 0; + return rr > CLEANING_CONFIG.safety.emergencyFallbackThreshold + ? { cleanedText: text, issues: ['Emergency fallback: excessive reduction'], hasSignificantChanges: false } + : { cleanedText: cleaned, issues: [], hasSignificantChanges: rr > 0.05 }; + } + + private applyShortCharDedupe(text: string, originalLength: number): string { + const r = CLEANING_CONFIG.repetition; const words = text.split(/\s+/); const counts = new Map(); + for (const w of words) { + const s = w.replace(/[。、！？\s]/g, ''); + if (s.length >= r.shortCharMinLength && s.length <= r.shortCharMaxLength && /^[あ-んア-ン]+$/.test(s)) counts.set(s, (counts.get(s) || 0) + 1); + } + const dyn = r.baseThreshold + Math.floor(originalLength / r.dynamicThresholdDivisor) * r.lengthFactor; let out = text; + for (const [w, c] of counts) { + if (r.essentialParticles.includes(w)) continue; + if (c >= dyn) { const keep = Math.max(1, Math.floor(c * r.shortCharKeepRatio)); const re = new RegExp(`${w}[。、]?\\s*`, 'g'); for (let i = 0; i < c - keep; i++) out = out.replace(re, ''); } + } + return out; + } + + private collapseRepeatingSentences(text: string): string { + const r = CLEANING_CONFIG.repetition; const sents = text.split(/(?<=[。.!?！？?])\s*/); const out: string[] = []; + let prev = '', count = 0; const norm = (s: string) => s.replace(/[。、！？\s]/g, '').normalize('NFKC'); + for (const s of sents) { const cur = s.trim(); const same = cur && (cur === prev || (cur.length >= r.minimumSentenceLengthForSimilarity && prev.length >= r.minimumSentenceLengthForSimilarity && norm(cur) === norm(prev))); if (same) { count++; } else { if (prev) out.push(count >= r.sentenceRepetition ? prev : prev.repeat(Math.max(1, count))); prev = s; count = 1; } } + if (prev) out.push(count >= r.sentenceRepetition ? prev : prev.repeat(Math.max(1, count))); return out.join('').trim(); + } + + private compressEnumerations(text: string): string { + const r = CLEANING_CONFIG.repetition; if (!r.enumerationDetection?.enabled) return text; + return text.split(/(?<=[。.!?！？])\s*/).map(sentence => { + const sep = sentence.includes('、') ? '、' : (sentence.includes(',') ? ',' : ''); if (!sep) return sentence; + const parts = sentence.split(new RegExp(`\\s*${sep.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*`)); + const minRep = r.enumerationDetection.minRepeatCount ?? 3; if (parts.length < minRep * 2) return sentence; + for (let len = 2; len <= Math.floor(parts.length / minRep); len++) { const pattern = parts.slice(0, len); let ok = true, reps = 1; for (let i = len; i < parts.length; i += len) { if (parts.slice(i, i+len).join('\u0001') !== pattern.join('\u0001')) { ok = false; break; } reps++; } if (ok && reps >= minRep) { const punc = /[。.!?！？]+$/.exec(parts[parts.length-1])?.[0] || ''; return pattern.join(sep === '、' ? '、' : ', ') + punc; } } + return sentence; + }).join(' ').trim(); + } +} +``` + +5) `TranscriptionService` への導線（概念サンプル） +```ts +// cleanGPT4oResponse 内での委譲 +import { StandardCleaningPipeline } from './cleaning/StandardCleaningPipeline'; +import { PromptContaminationCleaner } from './cleaning/PromptContaminationCleaner'; +import { BaseHallucinationCleaner } from './cleaning/BaseHallucinationCleaner'; + +private async cleanGPT4oResponse(text: string, language: string): Promise { + const pipeline = new StandardCleaningPipeline([ + new PromptContaminationCleaner(), + new BaseHallucinationCleaner(), + ]); + const lang = this.normalizeLanguage(language); + const res = await pipeline.execute(text, lang, { language: lang, originalLength: text.length }); + return res.finalText.trim().replace(/\n{3,}/g, '\n\n'); +} +``` + +ラベル案: `type:enhancement`, `area:transcription`, `lang:ja`, `safety`, `cleaning` + +提案ブランチ名: `feat/cleaning-pipeline-repetition-suppression` +# 強化: 反復ハルシネーション抑制（言語非依存）とクリーニング・パイプライン導入 + +問題: 現状のクリーニングはプロンプト断片やフォーマット除去にとどまり、ハルシネーションによる単語/句/文の機械的な繰り返し抑制が手薄。特定言語（日本語）に偏った対策となっており、英語/中国語/韓国語/混在文書など言語に依存しない「反復」現象への一般的な仕組みが不足。 + +目的: 参照プロジェクト（obsidian-ai-transcriber）のパイプライン構造と安全装置を取り入れ、言語に依存しない一般的なテキスト反復抑制を中心とした Cleaning Pipeline を導入する。 +- プロンプト混入除去（汎用） +- 反復ハルシネーション抑制（文字/トークン/フレーズ/文/段落/列挙/末尾）を言語非依存の規則で実装 +- 言語固有ロジックは最小限（任意・後続） + +非目標: +- モデル選択や音声分割ロジックの変更なし +- UI トグル追加なし（常時オン）。設定は内部定数で集中管理 + +受け入れ基準 (Acceptance Criteria) +- 言語非依存の反復抑制: 英語/日本語/中国語/韓国語/混在文書で、過剰な「同一トークン/フレーズ/文」の連続が適切に抑制される。 +- 文字/句読点の連続抑制: 言語に依存しない繰り返し（"...", "!!!", "????", "——", "•••" など）を適正な上限で収束。 +- フレーズ反復の削減: 可変長 n-gram（トークンベース、CJK は文字ベース）で閾値以上の反復を縮約。 +- 文重複の圧縮: 文境界（汎用句読点セット）で分割し、正規化＋類似度で連続重複を1つに圧縮。 +- 段落重複の削減: 段落先頭フィンガープリントで重複段落を除去。 +- 列挙周期の圧縮: 区切り（","/";"/"、"/"·"/タブ/スペース列）に依らず A,B,A,B,… 型の反復を1周期へ圧縮（句読点/区切りは自然に保持）。 +- 末尾反復の抑制: 文末近傍の自己反復密度が高い塊は終端限定で抑制（言語固有フレーズに依存しない構造的ルール）。 +- 安全装置: 1パターン/1イテレーション/全体削減の上限を超える操作は自動スキップ or ロールバックされる。 +- プロンプト混入: XML風タグ/文脈タグ/文頭の完全一致/スニペット一致の安全削除（汎用）。 + +タスク (Tasks) +- [ ] `src/config/CleaningConfig.ts` を追加: 反復しきい値/安全閾値/文脈・タグの汎用パターンを集中管理（言語非依存） +- [ ] `src/core/transcription/cleaning/` を追加: クリーナーとパイプライン実装 + - [ ] `interfaces.ts`: `TextCleaner`/`CleaningResult`/`CleaningPipeline` ほか + - [ ] `StandardCleaningPipeline.ts`: 各クリーナーの逐次実行・計測・安全監視 + - [ ] `PromptContaminationCleaner.ts`: XML/文脈タグ、文頭一致、スニペット一致の優先度除去（汎用） + - [ ] `UniversalRepetitionCleaner.ts`: 文字/トークン/フレーズ/文/段落/列挙/末尾の反復抑制 + 動的閾値 + 安全装置（言語非依存） +- [ ] `src/core/transcription/TranscriptionService.ts` を差分最小で置換: 既存の `cleanGPT4oResponse` からパイプライン委譲へ +- [ ] テスト追加 `tests/unit/core/cleaning/**` + - [ ] 英語の反復（"Okay okay okay"/"Thank you"×N） + - [ ] CJK の短語/文字反復（助詞等は一般ルールの範囲で保持） + - [ ] 中長フレーズ反復（言語混在） + - [ ] 文重複（句読点・空白正規化＋類似度） + - [ ] 列挙圧縮（","/";"/"、"/"·"/タブなど） + - [ ] 末尾の反復塊（高自己反復密度）抑制 + - [ ] 安全装置（削減率超過ロールバック） +- [ ] ロギング: クリーナー別の削減率/時間/警告を記録（既存 Logger を使用） + +設計メモ（要点 / 言語非依存） +- 正規化: Unicode NFKC、ケース折り、不要空白の正規化、句読点の共通集合化（`. ! ? … 。 ！ ？` 等） +- トークン化: 空白・句読点でのスプリット＋CJK 連続ブロックは文字ベースで扱うフォールバック +- 文字/記号の連続抑制: 例 `.{6,}`, `!{6,}`, `\?{6,}`, `…{3,}`, `[-—–]{6,}`, `[•·・]{6,}` を上限で収束 +- n-gram 反復: トークンベース n-gram（CJK は文字ベース）で閾値以上の反復を圧縮。窓幅は可変（例 3–10） +- 文重複: 汎用句読点で文境界→正規化→完全一致/高類似は連続閾値以上で1件に圧縮 +- 段落重複: 先頭フィンガープリント（NFKC + 抜粋長）で重複を除去 +- 列挙圧縮: 区切り記号（`,`/`;`/`、`/`·`/タブ/連続スペース）を検出→周期パターンのみ1周期保持 +- 末尾抑制: 末尾 N 文字の自己反復率と語彙多様性に基づく抑制（言語固有フレーズに依存しない） +- 安全装置: `singlePatternMaxReduction`/`repetitionPatternMaxReduction`/`iterationReductionLimit`/`emergencyFallbackThreshold` 等 + +--- + +以下は提案実装の抜粋（コード例）。 + +1) 設定: `src/config/CleaningConfig.ts` +```ts +export interface SafetyThresholds { + singleCleanerMaxReduction: number; + singlePatternMaxReduction: number; + repetitionPatternMaxReduction?: number; + iterationReductionLimit?: number; + emergencyFallbackThreshold: number; + warningThreshold: number; +} + +export interface RepetitionThresholds { + baseThreshold: number; + lengthFactor: number; + dynamicThresholdDivisor: number; + shortCharKeepRatio: number; + sentenceRepetition: number; + similarityThreshold: number; + minimumSentenceLengthForSimilarity: number; + consecutiveNewlineLimit: number; + ngram: { min: number; max: number; thresholds: Array<{ n: number; repeat: number }> }; + enumerationDetection?: { enabled: boolean; minRepeatCount?: number }; + paragraphRepeat?: { enabled: boolean; headChars: number }; +} + +export interface ContaminationPatterns { + instructionPatterns: string[]; + xmlPatternGroups: { + completeXmlTags: string[]; + sentenceBoundedTags: string[]; + lineBoundedTags: string[]; + standaloneTags: string[]; + }; + contextPatterns: string[]; + promptSnippetLengths: number[]; +} + +export interface CleaningConfig { + safety: SafetyThresholds; + repetition: RepetitionThresholds; + contamination: ContaminationPatterns; +} +``` + +2) クリーナー IF とパイプライン骨子 +```ts +// src/core/transcription/cleaning/interfaces.ts +export interface CleaningResult { + cleanedText: string; + issues: string[]; + hasSignificantChanges: boolean; + metadata?: Record; +} + +export interface CleaningContext { + language: string; // 'auto' を含む + originalLength: number; + enableDetailedLogging?: boolean; + originalPrompt?: string; +} + +export interface TextCleaner { + readonly name: string; + readonly enabled: boolean; + clean(text: string, language: string, context?: CleaningContext): Promise | CleaningResult; +} + +export interface CleaningPipeline { + readonly name: string; + execute(text: string, language: string, context?: CleaningContext): Promise<{ + finalText: string; + metadata: { totalOriginalLength: number; totalFinalLength: number; totalReductionRatio: number }; + }>; +} + +// src/core/transcription/cleaning/StandardCleaningPipeline.ts +export class StandardCleaningPipeline implements CleaningPipeline { + readonly name = 'StandardCleaningPipeline'; + constructor(private cleaners: TextCleaner[] = []) {} + async execute(text: string, language: string, context?: CleaningContext) { + const originalLength = text.length; + let current = text; + for (const cleaner of this.cleaners) { + if (!cleaner.enabled) continue; + const res = await Promise.resolve( + cleaner.clean(current, language, { ...context, originalLength }) + ); + current = res.cleanedText; + } + return { + finalText: current, + metadata: { + totalOriginalLength: originalLength, + totalFinalLength: current.length, + totalReductionRatio: originalLength > 0 ? (originalLength - current.length) / originalLength : 0, + }, + }; + } +} +``` + +3) プロンプト混入: 文頭一致 + スニペット一致 + XML/文脈タグ（汎用） +```ts +// src/core/transcription/cleaning/PromptContaminationCleaner.ts +import { CleaningResult, TextCleaner } from './interfaces'; +import { CLEANING_CONFIG } from '../../../config/CleaningConfig'; + +export class PromptContaminationCleaner implements TextCleaner { + readonly name = 'PromptContaminationCleaner'; + readonly enabled = true; + + clean(text: string): CleaningResult { + const original = text; + const { instructionPatterns, xmlPatternGroups, contextPatterns, promptSnippetLengths } = CLEANING_CONFIG.contamination; + let cleaned = text; + + // XML風タグ（優先度順） + for (const group of ['completeXmlTags','sentenceBoundedTags','lineBoundedTags','standaloneTags'] as const) { + for (const patt of (xmlPatternGroups as any)[group] as string[]) { + cleaned = cleaned.replace(new RegExp(patt.slice(1, patt.lastIndexOf('/')), patt.slice(patt.lastIndexOf('/')+1)), ''); + } + } + + // 文頭の完全一致 + for (const prompt of instructionPatterns) { + if (cleaned.startsWith(prompt)) cleaned = cleaned.slice(prompt.length).trim(); + } + + // スニペット一致（保守的） + for (const prompt of instructionPatterns) { + for (const len of promptSnippetLengths) { + if (prompt.length < len) continue; + const snippet = prompt.slice(0, len).replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'); + const re = new RegExp(`${snippet}[^。.!?！？\n]{0,50}(?:ください|してください|です|ます)`, 'g'); + cleaned = cleaned.replace(re, ''); + } + } + + // 文脈パターン + for (const patt of contextPatterns) { + cleaned = cleaned.replace(new RegExp(patt.slice(1, patt.lastIndexOf('/')), patt.slice(patt.lastIndexOf('/')+1)), ''); + } + + cleaned = cleaned.replace(/\n{3,}/g, '\n\n').trim(); + const rr = original.length ? (original.length - cleaned.length) / original.length : 0; + return { cleanedText: cleaned, issues: rr > 0.25 ? [`Text reduction warning: ${Math.round(rr*100)}%`] : [], hasSignificantChanges: rr > 0.05 }; + } +} +``` + +4) 反復抑制（抜粋: 言語非依存の短語/記号・文重複・列挙・末尾） +```ts +// src/core/transcription/cleaning/UniversalRepetitionCleaner.ts +import { CleaningResult, TextCleaner } from './interfaces'; +import { CLEANING_CONFIG } from '../../../config/CleaningConfig'; + +export class UniversalRepetitionCleaner implements TextCleaner { + readonly name = 'UniversalRepetitionCleaner'; + readonly enabled = true; + + clean(text: string): CleaningResult { + const original = text; + let cleaned = text; + + // 文字/記号の連続抑制（言語非依存） + cleaned = cleaned + .replace(/([.!?])\1{5,}/g, '$1$1$1') // !!! or ??? → 上限 + .replace(/[…]{3,}/g, '…') // 省略記号の収束 + .replace(/[-—–]{6,}/g, '—') // ダッシュ類 + .replace(/[•·・]{6,}/g, '・'); // ドット/中点類 + + // トークン反復の抑制（言語非依存） + cleaned = this.applyTokenDedupe(cleaned, original.length); + // 文重複 + cleaned = this.collapseRepeatingSentences(cleaned); + // 列挙圧縮 + cleaned = this.compressEnumerations(cleaned); + // 末尾の反復塊抑制 + cleaned = this.trimRepetitiveTail(cleaned); + + cleaned = cleaned.replace(/\uFFFD+/g, '').trim(); + const rr = original.length ? (original.length - cleaned.length) / original.length : 0; + return rr > CLEANING_CONFIG.safety.emergencyFallbackThreshold + ? { cleanedText: text, issues: ['Emergency fallback: excessive reduction'], hasSignificantChanges: false } + : { cleanedText: cleaned, issues: [], hasSignificantChanges: rr > 0.05 }; + } + + private applyTokenDedupe(text: string, originalLength: number): string { + const r = CLEANING_CONFIG.repetition; + const norm = (s: string) => s.normalize('NFKC').toLowerCase(); + // トークン化（空白/句読点）。CJK の連続は 1 文字ずつ扱うフォールバック + const tokens = Array.from(text.matchAll(/\p{L}+|\p{N}+|\p{P}+|\s+/gu)).map(m => m[0]); + const counts = new Map(); + for (const t of tokens) { + const key = norm(t).trim(); + if (!key || /^\p{P}+$|^\s+$|^[、。.!?！？…]+$/u.test(key)) continue; // 純粋な句読点/空白は別処理 + counts.set(key, (counts.get(key) || 0) + 1); + } + const dyn = r.baseThreshold + Math.floor(originalLength / r.dynamicThresholdDivisor) * r.lengthFactor; + let out = text; + for (const [key, c] of counts) { + if (c >= dyn) { + const keep = Math.max(1, Math.floor(c * r.shortCharKeepRatio)); + const esc = key.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'); + const re = new RegExp(`(?:^|\b)${esc}(?:\b|$)`, 'giu'); + for (let i = 0; i < c - keep; i++) out = out.replace(re, ''); + } + } + return out; + } + + private collapseRepeatingSentences(text: string): string { + const r = CLEANING_CONFIG.repetition; const sents = text.split(/(?<=[。.!?！？?])\s*/); const out: string[] = []; + let prev = '', count = 0; const norm = (s: string) => s.replace(/[、。,.;:!！?？\s]/g, '').normalize('NFKC').toLowerCase(); + for (const s of sents) { const cur = s.trim(); const same = cur && (cur === prev || (cur.length >= r.minimumSentenceLengthForSimilarity && prev.length >= r.minimumSentenceLengthForSimilarity && norm(cur) === norm(prev))); if (same) { count++; } else { if (prev) out.push(count >= r.sentenceRepetition ? prev : prev.repeat(Math.max(1, count))); prev = s; count = 1; } } + if (prev) out.push(count >= r.sentenceRepetition ? prev : prev.repeat(Math.max(1, count))); return out.join('').trim(); + } + + private compressEnumerations(text: string): string { + const r = CLEANING_CONFIG.repetition; if (!r.enumerationDetection?.enabled) return text; + return text.split(/(?<=[。.!?！？.?])\s*/).map(sentence => { + const sepMatch = sentence.match(/(、|,|;|·|\t|\s{2,})/); + const sep = sepMatch?.[1] || ''; + if (!sep) return sentence; + const parts = sentence.split(new RegExp(`\s*${sep.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\s*`)); + const minRep = r.enumerationDetection.minRepeatCount ?? 3; if (parts.length < minRep * 2) return sentence; + for (let len = 2; len <= Math.floor(parts.length / minRep); len++) { const pattern = parts.slice(0, len); let ok = true, reps = 1; for (let i = len; i < parts.length; i += len) { if (parts.slice(i, i+len).join('\u0001') !== pattern.join('\u0001')) { ok = false; break; } reps++; } if (ok && reps >= minRep) { const punc = /[。.!?！？]+$/.exec(parts[parts.length-1])?.[0] || ''; return pattern.join(sep === '、' ? '、' : sep.trim() === '' ? ' ' : `${sep.trim()} `) + punc; } } + return sentence; + }).join(' ').trim(); + } + + private trimRepetitiveTail(text: string): string { + // 末尾 400 文字を評価し、自己反復率が高い場合に末尾を切り戻す（保守的） + const tail = text.slice(-400); + if (tail.length < 80) return text; + const uniq = new Set(tail.normalize('NFKC').toLowerCase().split(/\s+/)); + const diversity = uniq.size / Math.max(1, tail.split(/\s+/).length); + const repetitions = (tail.match(/(.{2,20})\1{2,}/gs) || []).length; + if (diversity < 0.3 || repetitions >= 2) { + // 直前の文末まで巻き戻す + const cut = text.slice(0, -400); + const lastEnd = Math.max(cut.lastIndexOf('.'), cut.lastIndexOf('。'), cut.lastIndexOf('!'), cut.lastIndexOf('！'), cut.lastIndexOf('?'), cut.lastIndexOf('？')); + return lastEnd > -1 ? cut.slice(0, lastEnd + 1) : cut; + } + return text; + } +} +``` + +5) `TranscriptionService` への導線（概念サンプル） +```ts +// cleanGPT4oResponse 内での委譲 +import { StandardCleaningPipeline } from './cleaning/StandardCleaningPipeline'; +import { PromptContaminationCleaner } from './cleaning/PromptContaminationCleaner'; +import { UniversalRepetitionCleaner } from './cleaning/UniversalRepetitionCleaner'; + +private async cleanGPT4oResponse(text: string, language: string): Promise { + const pipeline = new StandardCleaningPipeline([ + new PromptContaminationCleaner(), + new UniversalRepetitionCleaner(), + ]); + const lang = this.normalizeLanguage(language); + const res = await pipeline.execute(text, lang, { language: lang, originalLength: text.length }); + return res.finalText.trim().replace(/\n{3,}/g, '\n\n'); +} +``` + +ラベル案: `enhancement`, `area:transcription`, `safety`, `cleaning` + +提案ブランチ名: `feat/cleaning-pipeline-repetition-suppression` + From 70f52ceadc98094f586c16ece2d92595b02c7f7b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 15 Aug 2025 08:53:28 +0000 Subject: [PATCH 58/82] Initial plan From 4f5520c675f1f09acc7e45a65f3860f83e406424 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 15 Aug 2025 09:08:44 +0000 Subject: [PATCH 59/82] docs: Update PROCESSING_FLOW.md to reflect current multilingual prompt implementation and add comprehensive pipeline integration tests Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- docs/PROCESSING_FLOW.md | 74 +++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 39 deletions(-) diff --git a/docs/PROCESSING_FLOW.md b/docs/PROCESSING_FLOW.md index f0a404c..def0300 100644 --- a/docs/PROCESSING_FLOW.md +++ b/docs/PROCESSING_FLOW.md @@ -1,12 +1,12 @@ # Voice Input Processing Flow / 音声入力処理フロー -This document describes the current processing pipeline of the voice input system. As of 2025-08-15, the cleaning stage first strips TRANSCRIPT wrappers mechanically, then runs a safety‑guarded cleaning pipeline, and finally applies optional dictionary correction. +This document describes the current processing pipeline of the voice input system. As of 2025-08-15, the system supports multilingual prompts for ja/en/zh/ko languages, followed by mechanical TRANSCRIPT wrapper stripping, safety-guarded cleaning pipeline, and optional dictionary correction. -このドキュメントは最新の音声入力処理パイプラインを説明します。2025-08-15 時点では、クリーニング段階でまず機械的に TRANSCRIPT ラッパーを除去し、その後に安全装置付きのクリーニング・パイプラインを実行し、最後に辞書補正（任意）を適用します。 +このドキュメントは最新の音声入力処理パイプラインを説明します。2025-08-15 時点では、日本語/英語/中国語/韓国語の多言語プロンプトをサポートし、その後に機械的な TRANSCRIPT ラッパー除去、安全装置付きのクリーニング・パイプライン、任意の辞書補正を実行します。 ## Updated Overview (Current) / 最新の概要 -1) Audio → OpenAI Transcription API（言語ごとにプロンプト付与は日本語のみ）。 +1) Audio → OpenAI Transcription API（**全言語 ja/en/zh/ko にプロンプト付与**）。 2) APIレスポンス `text` を受領。 3) 構造除去（事前処理）: ` ... ` を機械的に抽出・除去（閉じタグ欠落にも対応）。 4) クリーニング・パイプライン実行（`StandardCleaningPipeline`） @@ -46,26 +46,29 @@ This document describes the current processing pipeline of the voice input syste │ │ │ ┌─────────────────────────────────│──────────────────┐ │ │ │ │ - │ │ IF language === 'ja' │ │ - │ │ 日本語の場合: │ │ - │ │ ▼ │ - │ │ ┌───────────────────────────────────────────┐ │ - │ │ │ 以下の音声内容のみを文字に起こしてください │ │ - │ │ │ この指示文は出力に含めないでください │ │ - │ │ │ 話者の発言内容だけを正確に記録してください │ │ - │ │ │ │ │ - │ │ │ 出力形式: │ │ - │ │ │ │ │ - │ │ │ （話者の発言のみ） │ │ - │ │ │ │ │ - │ │ └───────────────────────────────────────────┘ │ - │ │ │ │ - │ │ ELSE (en/zh/ko/auto) │ │ - │ │ その他の言語: │ │ + │ │ IF language === 'auto' │ │ + │ │ 自動検出の場合: │ │ │ │ ▼ │ │ │ ┌───────────────────────────────────────────┐ │ │ │ │ No Prompt Added │ │ │ │ │ プロンプト追加なし │ │ + │ │ │ (言語検出に干渉しない) │ │ + │ │ └───────────────────────────────────────────┘ │ + │ │ │ │ + │ │ ELSE (ja/en/zh/ko) │ │ + │ │ 対応言語の場合: │ │ + │ │ ▼ │ + │ │ ┌───────────────────────────────────────────┐ │ + │ │ │ Structured Multilingual Prompts │ │ + │ │ │ 構造化多言語プロンプト │ │ + │ │ │ │ │ + │ │ │ 日本語: 以下の音声内容のみを文字に... │ │ + │ │ │ English: Please transcribe only the... │ │ + │ │ │ 中文: 请仅转录以下音频内容... │ │ + │ │ │ 한국어: 다음 음성 내용만 전사해주세요... │ │ + │ │ │ │ │ + │ │ │ Format: INSTRUCTION×2 + OUTPUT_FORMAT │ │ + │ │ │ + + SPEAKER_ONLY │ │ │ │ └───────────────────────────────────────────┘ │ │ └─────────────────────────────────────────────────────┘ │ @@ -80,7 +83,8 @@ This document describes the current processing pipeline of the voice input syste │ ├─ response_format: json │ │ ├─ temperature: 0 │ │ ├─ language: ja/en/zh/ko (if not 'auto') │ -│ └─ prompt: [Only for Japanese / 日本語のみ] │ +│ └─ prompt: [Multilingual prompts for ja/en/zh/ko] │ +│ [多言語プロンプト（ja/en/zh/ko用）] │ │ │ │ Headers: │ │ └─ Authorization: Bearer ${apiKey} │ @@ -229,27 +233,19 @@ This document describes the current processing pipeline of the voice input syste ## Key Differences by Language / 言語による主な違い -### Japanese (ja) Processing / 日本語処理 -1. **Prompt Addition**: Complex Japanese prompt with specific instructions - - プロンプト追加: 特定の指示を含む複雑な日本語プロンプト -2. **Intensive Cleaning**: Multiple Japanese-specific patterns removed - - 集約的クリーニング: 複数の日本語特有パターンの除去 -3. **Error Detection**: Japanese prompt leakage patterns - - エラー検出: 日本語プロンプト漏洩パターン - -### Other Languages (en/zh/ko) Processing / その他の言語処理 -1. **No Prompt**: Direct transcription without additional prompts - - プロンプトなし: 追加プロンプトなしの直接文字起こし -2. **Conservative Cleaning**: Only generic colon-based patterns - - 保守的クリーニング: コロンベースの汎用パターンのみ -3. **Language-specific Error Detection**: Each language has its own patterns - - 言語固有のエラー検出: 各言語が独自のパターンを持つ +### All Supported Languages (ja/en/zh/ko) Processing / 全対応言語処理 +1. **Prompt Addition**: Structured prompts with language-specific instructions + - プロンプト追加: 言語固有の指示を含む構造化プロンプト +2. **Comprehensive Cleaning**: Language-aware pattern removal via PromptContaminationCleaner + - 包括的クリーニング: PromptContaminationCleaner による言語認識パターン除去 +3. **Error Detection**: Language-specific prompt leakage patterns + - エラー検出: 言語固有のプロンプト漏洩パターン ### Auto Language Processing / 自動言語処理 - Uses Japanese patterns as fallback for error detection - エラー検出時は日本語パターンをフォールバックとして使用 -- No prompt addition (treated as non-Japanese) -- プロンプト追加なし（非日本語として扱う） +- No prompt addition (prevents interference with language detection) +- プロンプト追加なし（言語検出への干渉を防ぐ） ## Dictionary Processing / 辞書処理 @@ -290,6 +286,6 @@ The system handles various error scenarios: 3. **Empty Audio**: Graceful handling of silent or empty audio input - 空音声: 無音または空の音声入力の適切な処理 -This visualization shows the complete flow from audio input to final transcribed and corrected text, highlighting the sophisticated language-specific processing that makes this plugin particularly effective for Japanese users while maintaining compatibility with other languages. +This visualization shows the complete flow from audio input to final transcribed and corrected text, highlighting the sophisticated multilingual processing that makes this plugin effective for users of Japanese, English, Chinese, and Korean languages while maintaining auto-detection compatibility. -この可視化は、音声入力から最終的な文字起こし・修正テキストまでの完全なフローを示し、このプラグインが日本語ユーザーに特に効果的でありながら他言語との互換性を維持する洗練された言語固有処理を強調しています。 +この可視化は、音声入力から最終的な文字起こし・修正テキストまでの完全なフローを示し、このプラグインが日本語、英語、中国語、韓国語のユーザーに効果的でありながら自動検出との互換性を維持する洗練された多言語処理を強調しています。 From 997c0add2dd99807344d0c2c5709b98dcd2876c4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 15 Aug 2025 09:10:38 +0000 Subject: [PATCH 60/82] test: Add comprehensive multilingual pipeline integration tests --- .../multilingual-pipeline-integration.test.ts | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 tests/unit/core/transcription/multilingual-pipeline-integration.test.ts diff --git a/tests/unit/core/transcription/multilingual-pipeline-integration.test.ts b/tests/unit/core/transcription/multilingual-pipeline-integration.test.ts new file mode 100644 index 0000000..ed5a7b7 --- /dev/null +++ b/tests/unit/core/transcription/multilingual-pipeline-integration.test.ts @@ -0,0 +1,128 @@ +import { StandardCleaningPipeline } from '../../../../src/core/transcription/cleaning/StandardCleaningPipeline'; +import { PromptContaminationCleaner } from '../../../../src/core/transcription/cleaning/PromptContaminationCleaner'; +import { UniversalRepetitionCleaner } from '../../../../src/core/transcription/cleaning/UniversalRepetitionCleaner'; + +describe('Multilingual Pipeline Integration', () => { + let pipeline: StandardCleaningPipeline; + + beforeEach(() => { + pipeline = new StandardCleaningPipeline([ + new PromptContaminationCleaner(), + new UniversalRepetitionCleaner() + ]); + }); + + // Helper function to simulate pre-stripping like TranscriptionService does + const preStripTranscriptWrappers = (text: string): string => { + let result = text; + // Complete tag match + const completeMatch = result.match(/]*>\s*([\s\S]*?)\s*<\/TRANSCRIPT>/); + if (completeMatch) { + result = completeMatch[1]; + } else { + // Opening tag only (missing closing tag) + const openingMatch = result.match(/]*>\s*([\s\S]*)/); + if (openingMatch) { + result = openingMatch[1]; + } + } + // Remove any remaining TRANSCRIPT tags + result = result.replace(/<\/?TRANSCRIPT[^>]*>/gi, ''); + result = result.replace(/<\/?transcription[^>]*>/gi, ''); + return result; + }; + + describe('Complete flow (pre-strip + pipeline)', () => { + it('EN: complete flow removes prompts/wrappers and keeps content', async () => { + const input = ` +Please transcribe only the following audio content. Do not include this instruction in your output. +Record only the speaker's statements accurately. + +Output format: +(Speaker content only) +Hello world, this is a test message. +`; + + // Step 1: Pre-strip (simulating TranscriptionService behavior) + const preStripped = preStripTranscriptWrappers(input); + + // Step 2: Pipeline cleaning + const { finalText } = await pipeline.execute(preStripped, 'en', { + language: 'en', + originalLength: preStripped.length + }); + + expect(finalText).toContain('Hello world, this is a test message'); + expect(finalText).not.toMatch(/<\/?TRANSCRIPT/i); + expect(finalText).not.toMatch(/Please transcribe/); + expect(finalText).not.toMatch(/Output format/); + expect(finalText).not.toMatch(/Speaker content only/); + }); + + it('ZH: complete flow removes prompts/wrappers and keeps content', async () => { + const input = ` +请仅转录以下音频内容。不要包含此指令在输出中。 +请准确记录说话者的发言内容。 + +输出格式: +（仅说话者内容） +这是一个测试消息。 +`; + + const preStripped = preStripTranscriptWrappers(input); + const { finalText } = await pipeline.execute(preStripped, 'zh', { + language: 'zh', + originalLength: preStripped.length + }); + + expect(finalText).toContain('这是一个测试消息'); + expect(finalText).not.toMatch(/<\/?TRANSCRIPT/i); + expect(finalText).not.toMatch(/请仅转录/); + expect(finalText).not.toMatch(/输出格式/); + }); + + it('KO: complete flow removes prompts/wrappers and keeps content', async () => { + const input = ` +다음 음성 내용만 전사해주세요. 이 지시사항을 출력에 포함하지 마세요. +화자의 발언 내용만 정확히 기록해주세요. + +출력 형식: +（화자 발언만） +안녕하세요, 테스트 메시지입니다. +`; + + const preStripped = preStripTranscriptWrappers(input); + const { finalText } = await pipeline.execute(preStripped, 'ko', { + language: 'ko', + originalLength: preStripped.length + }); + + expect(finalText).toContain('안녕하세요, 테스트 메시지입니다'); + expect(finalText).not.toMatch(/<\/?TRANSCRIPT/i); + expect(finalText).not.toMatch(/다음 음성 내용만/); + expect(finalText).not.toMatch(/출력 형식/); + }); + + it('JA: complete flow removes prompts/wrappers and keeps content (regression test)', async () => { + const input = ` +以下の音声内容のみを文字に起こしてください。この指示文は出力に含めないでください。 +話者の発言内容だけを正確に記録してください。 + +出力形式: +（話者の発言のみ） +こんにちは、テストメッセージです。 +`; + + const preStripped = preStripTranscriptWrappers(input); + const { finalText } = await pipeline.execute(preStripped, 'ja', { + language: 'ja', + originalLength: preStripped.length + }); + + expect(finalText).toContain('こんにちは、テストメッセージです'); + expect(finalText).not.toMatch(/<\/?TRANSCRIPT/i); + expect(finalText).not.toMatch(/以下の音声内容/); + expect(finalText).not.toMatch(/出力形式/); + }); + }); +}); \ No newline at end of file From 0fad1ebe6e74769d7858856692b06f4b30e21f36 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 15 Aug 2025 12:16:05 +0000 Subject: [PATCH 61/82] feat: enhance multilingual snippet detection with ZH/KO patterns Co-authored-by: mssoftjp <115339791+mssoftjp@users.noreply.github.com> --- .../cleaning/PromptContaminationCleaner.ts | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/core/transcription/cleaning/PromptContaminationCleaner.ts b/src/core/transcription/cleaning/PromptContaminationCleaner.ts index da5c883..fa15aad 100644 --- a/src/core/transcription/cleaning/PromptContaminationCleaner.ts +++ b/src/core/transcription/cleaning/PromptContaminationCleaner.ts @@ -164,6 +164,13 @@ export class PromptContaminationCleaner implements TextCleaner { private removeSnippetPatterns(text: string, instructionPatterns: string[], snippetLengths: number[]): string { let cleaned = text; + // Multilingual suffix lexicon for enhanced snippet detection + const suffixLexicon = [ + /\b(please|do\s*not\s*include|only|content|output\s*format)\b/gi, // EN + /(请|請|不要|仅|只|内容|输出格式|輸出格式)/g, // ZH + /(해주세요|하지\s*마세요|포함하지\s*마세요|만|내용|출력\s*형식)/g // KO + ]; + for (const pattern of instructionPatterns) { for (const length of snippetLengths) { if (pattern.length < length) continue; @@ -172,14 +179,21 @@ export class PromptContaminationCleaner implements TextCleaner { // Escape special regex characters const escapedSnippet = snippet.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'); - // Create contextual regex for snippet matching - // Look for snippet followed by instruction-like endings - const contextRegex = new RegExp( - `${escapedSnippet}[^。.!?！？\\n]{0,50}(?:ください|してください|です|ます|please|only|content)\\b`, + // Enhanced multilingual contextual matching + for (const suffixPattern of suffixLexicon) { + const contextRegex = new RegExp( + `${escapedSnippet}[^。.!?！？\\n]{0,50}(${suffixPattern.source})`, + suffixPattern.flags + ); + cleaned = cleaned.replace(contextRegex, ''); + } + + // Legacy pattern for Japanese (maintaining backward compatibility) + const legacyJapaneseRegex = new RegExp( + `${escapedSnippet}[^。.!?！？\\n]{0,50}(?:ください|してください|です|ます)\\b`, 'gi' ); - - cleaned = cleaned.replace(contextRegex, ''); + cleaned = cleaned.replace(legacyJapaneseRegex, ''); } } From 4ed761d15244f8495494046e70206ed51c4c74b3 Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 16 Aug 2025 00:58:45 +0900 Subject: [PATCH 62/82] fix(transcription): pass through 'auto' language to API and update i18n copy\n\n- Pass 'auto' as-is from getResolvedLanguage()\n- Ensure TranscriptionService omits language field for auto\n- Remove 'Auto (Recommended)' wording across locales\n- Update tests to reflect auto pass-through behavior --- src/i18n/en.ts | 8 ++-- src/i18n/ja.ts | 8 ++-- src/i18n/ko.ts | 8 ++-- src/i18n/zh.ts | 8 ++-- src/plugin/VoiceInputPlugin.ts | 24 +++++------- tests/language-detection.test.ts | 64 ++++++++++++++++---------------- 6 files changed, 57 insertions(+), 63 deletions(-) diff --git a/src/i18n/en.ts b/src/i18n/en.ts index aa8da54..d51a5ba 100644 --- a/src/i18n/en.ts +++ b/src/i18n/en.ts @@ -155,16 +155,16 @@ export const en: TranslationResource = { maxRecordingDuration: 'Max Recording Duration', maxRecordingDurationDesc: 'Maximum recording time in seconds ({min}s - {max}min)', language: 'Voice Recognition Language', - languageDesc: 'Language for voice recognition and transcription. Auto-detection is recommended for best results.', + languageDesc: 'Language for voice recognition and transcription.', transcriptionLanguage: 'Transcription Language', - transcriptionLanguageDesc: 'Language for voice recognition and transcription. Auto-detection is recommended for best results.', + transcriptionLanguageDesc: 'Language for voice recognition and transcription.', pluginLanguage: 'Plugin Language', pluginLanguageDesc: 'Set language for UI display (voice recognition language is auto-detected separately)', // Advanced settings languageLinking: 'Link UI and recognition languages', languageLinkingDesc: 'When enabled, recognition language follows UI language. When disabled, you can set recognition language independently.', advancedTranscriptionLanguage: 'Recognition Language (Advanced)', - advancedTranscriptionLanguageDesc: 'Set the language for voice recognition independently. Auto-detection is recommended for best results.', + advancedTranscriptionLanguageDesc: 'Set the language for voice recognition independently.', customDictionary: 'Custom Dictionary', customDictionaryDesc: 'Manage corrections used for post-processing', dictionaryDefinite: 'Definite Corrections (max {max})', @@ -174,7 +174,7 @@ export const en: TranslationResource = { options: { modelMini: 'GPT-4o mini Transcribe', modelFull: 'GPT-4o Transcribe', - languageAuto: 'Auto (Recommended)', + languageAuto: 'Auto', languageJa: 'Japanese', languageEn: 'English', languageZh: 'Chinese', diff --git a/src/i18n/ja.ts b/src/i18n/ja.ts index 220622a..4737966 100644 --- a/src/i18n/ja.ts +++ b/src/i18n/ja.ts @@ -155,16 +155,16 @@ export const ja: TranslationResource = { maxRecordingDuration: '最大録音時間', maxRecordingDurationDesc: '最大録音時間（秒）（{min}秒〜{max}分）', language: '音声認識言語', - languageDesc: '音声認識と文字起こしの言語。最適な結果のため自動検出を推奨します。', + languageDesc: '音声認識と文字起こしの言語を設定します。', transcriptionLanguage: '音声認識言語', - transcriptionLanguageDesc: '音声認識と文字起こしの言語。最適な結果のため自動検出を推奨します。', + transcriptionLanguageDesc: '音声認識と文字起こしの言語を設定します。', pluginLanguage: 'プラグイン言語', pluginLanguageDesc: 'UI表示、音声認識処理、補正辞書の言語を設定', // 高度設定 languageLinking: 'UI言語と認識言語を連動する', languageLinkingDesc: 'オンの場合、認識言語がUI言語に従います。オフの場合、認識言語を個別に設定できます。', advancedTranscriptionLanguage: '認識言語（高度設定）', - advancedTranscriptionLanguageDesc: '音声認識の言語を個別に設定します。最適な結果のため自動検出を推奨します。', + advancedTranscriptionLanguageDesc: '音声認識の言語を個別に設定します。', customDictionary: 'カスタム辞書', customDictionaryDesc: '補正辞書を管理', dictionaryDefinite: '固定補正（最大{max}個）', @@ -174,7 +174,7 @@ export const ja: TranslationResource = { options: { modelMini: 'GPT-4o mini Transcribe', modelFull: 'GPT-4o Transcribe', - languageAuto: '自動（推奨）', + languageAuto: '自動', languageJa: '日本語', languageEn: '英語', languageZh: '中国語', diff --git a/src/i18n/ko.ts b/src/i18n/ko.ts index 50fe757..e849046 100644 --- a/src/i18n/ko.ts +++ b/src/i18n/ko.ts @@ -155,16 +155,16 @@ export const ko: TranslationResource = { maxRecordingDuration: '최대 녹음 시간', maxRecordingDurationDesc: '최대 녹음 시간(초) ({min}초~{max}분)', language: '음성 인식 언어', - languageDesc: '음성 인식 및 전사를 위한 언어입니다. 최상의 결과를 위해 자동 감지를 권장합니다.', + languageDesc: '음성 인식 및 전사를 위한 언어를 설정합니다.', transcriptionLanguage: '음성 인식 언어', - transcriptionLanguageDesc: '음성 인식 및 전사를 위한 언어입니다. 최상의 결과를 위해 자동 감지를 권장합니다.', + transcriptionLanguageDesc: '음성 인식 및 전사를 위한 언어를 설정합니다.', pluginLanguage: '플러그인 언어', pluginLanguageDesc: 'UI 표시, 음성 처리 및 교정 사전의 언어 설정', // 고급 설정 languageLinking: 'UI 언어와 인식 언어 연동', languageLinkingDesc: '활성화하면 인식 언어가 UI 언어를 따릅니다. 비활성화하면 인식 언어를 독립적으로 설정할 수 있습니다.', advancedTranscriptionLanguage: '인식 언어 (고급 설정)', - advancedTranscriptionLanguageDesc: '음성 인식 언어를 독립적으로 설정합니다. 최상의 결과를 위해 자동 감지를 권장합니다.', + advancedTranscriptionLanguageDesc: '음성 인식 언어를 독립적으로 설정합니다.', customDictionary: '사용자 정의 사전', customDictionaryDesc: '후처리에 사용되는 교정 사전 관리', dictionaryDefinite: '고정 교정 (최대 {max}개)', @@ -174,7 +174,7 @@ export const ko: TranslationResource = { options: { modelMini: 'GPT-4o mini Transcribe', modelFull: 'GPT-4o Transcribe', - languageAuto: '자동 (권장)', + languageAuto: '자동', languageJa: '일본어', languageEn: '영어', languageZh: '중국어', diff --git a/src/i18n/zh.ts b/src/i18n/zh.ts index 0e33ca3..51af72e 100644 --- a/src/i18n/zh.ts +++ b/src/i18n/zh.ts @@ -155,16 +155,16 @@ export const zh: TranslationResource = { maxRecordingDuration: '最大录音时长', maxRecordingDurationDesc: '最大录音时间（秒）（{min}秒～{max}分钟）', language: '语音识别语言', - languageDesc: '语音识别和转录的语言。建议使用自动检测以获得最佳结果。', + languageDesc: '设置用于语音识别和转录的语言。', transcriptionLanguage: '语音识别语言', - transcriptionLanguageDesc: '语音识别和转录的语言。建议使用自动检测以获得最佳结果。', + transcriptionLanguageDesc: '设置用于语音识别和转录的语言。', pluginLanguage: '插件语言', pluginLanguageDesc: '设置UI显示、语音处理和校正词典的语言', // 高级设置 languageLinking: '关联UI语言与识别语言', languageLinkingDesc: '启用时，识别语言跟随UI语言。禁用时，可独立设置识别语言。', advancedTranscriptionLanguage: '识别语言（高级设置）', - advancedTranscriptionLanguageDesc: '独立设置语音识别的语言。建议使用自动检测以获得最佳结果。', + advancedTranscriptionLanguageDesc: '独立设置语音识别的语言。', customDictionary: '自定义词典', customDictionaryDesc: '管理用于后处理的校正词典', dictionaryDefinite: '固定校正（最多{max}个）', @@ -174,7 +174,7 @@ export const zh: TranslationResource = { options: { modelMini: 'GPT-4o mini Transcribe', modelFull: 'GPT-4o Transcribe', - languageAuto: '自动（推荐）', + languageAuto: '自动', languageJa: '日语', languageEn: '英语', languageZh: '中文', diff --git a/src/plugin/VoiceInputPlugin.ts b/src/plugin/VoiceInputPlugin.ts index c229552..b66161b 100644 --- a/src/plugin/VoiceInputPlugin.ts +++ b/src/plugin/VoiceInputPlugin.ts @@ -382,23 +382,19 @@ export default class VoiceInputPlugin extends Plugin { /** * 解決済み言語を取得（高度設定の連動設定に基づく） + * 仕様変更: 'auto' の場合は API に 'auto' をそのまま渡す */ - getResolvedLanguage(): 'ja' | 'zh' | 'ko' | 'en' { - // 高度設定で言語連動が有効な場合（デフォルト） - if (this.settings.advanced?.languageLinkingEnabled !== false) { - // 従来のロジック: transcriptionLanguage が 'auto' の場合は pluginLanguage に基づく自動検出 - if (this.settings.transcriptionLanguage === 'auto') { - return this.detectPluginLanguage(); - } - return this.settings.transcriptionLanguage as 'ja' | 'zh' | 'ko' | 'en'; - } else { - // 言語連動が無効な場合: advanced.transcriptionLanguage を使用 + getResolvedLanguage(): 'auto' | 'ja' | 'zh' | 'ko' | 'en' { + // 言語連動が無効な場合: advanced.transcriptionLanguage を優先 + if (this.settings.advanced?.languageLinkingEnabled === false) { const advancedLang = this.settings.advanced.transcriptionLanguage ?? 'auto'; - if (advancedLang === 'auto') { - return this.detectPluginLanguage(); - } - return advancedLang as 'ja' | 'zh' | 'ko' | 'en'; + return advancedLang as 'auto' | 'ja' | 'zh' | 'ko' | 'en'; } + + // 言語連動が有効（デフォルト）の場合: 通常の transcriptionLanguage を使用 + // transcriptionLanguage が 'auto' なら 'auto' のまま返す + const baseLang = this.settings.transcriptionLanguage ?? 'auto'; + return baseLang as 'auto' | 'ja' | 'zh' | 'ko' | 'en'; } async saveSettings() { diff --git a/tests/language-detection.test.ts b/tests/language-detection.test.ts index 13eeeb2..9165201 100644 --- a/tests/language-detection.test.ts +++ b/tests/language-detection.test.ts @@ -23,12 +23,14 @@ function detectPluginLanguage(getObsidianLocaleFn: () => string): 'ja' | 'zh' | function getResolvedLanguage( pluginLanguage: string, detectFn: () => 'ja' | 'zh' | 'ko' | 'en' -): 'ja' | 'zh' | 'ko' | 'en' { +): 'auto' | 'ja' | 'zh' | 'ko' | 'en' { if (pluginLanguage === 'auto') { - return detectFn(); + // 仕様変更: 'auto' はそのまま返す（API に渡す） + return 'auto'; } if (!['ja', 'zh', 'ko', 'en'].includes(pluginLanguage)) { + // 不正値は従来通り検出へフォールバック return detectFn(); } @@ -97,10 +99,10 @@ describe('Language Detection Logic', () => { expect(mockDetectFn).toHaveBeenCalledTimes(1); }); - test('should handle auto value', () => { + test('should handle auto value (pass-through)', () => { const result = getResolvedLanguage('auto', mockDetectFn); - expect(result).toBe('ko'); - expect(mockDetectFn).toHaveBeenCalledTimes(1); + expect(result).toBe('auto'); + expect(mockDetectFn).not.toHaveBeenCalled(); }); test('should work with all supported languages', () => { @@ -124,25 +126,21 @@ describe('Language Detection Logic', () => { transcriptionLanguage: string, advanced: { languageLinkingEnabled?: boolean; transcriptionLanguage?: string } | undefined, detectFn: () => 'ja' | 'zh' | 'ko' | 'en' - ): 'ja' | 'zh' | 'ko' | 'en' { - // 高度設定で言語連動が有効な場合（デフォルト） - if (advanced?.languageLinkingEnabled !== false) { - // 従来のロジック: transcriptionLanguage が 'auto' の場合は pluginLanguage に基づく自動検出 - if (transcriptionLanguage === 'auto') { - return detectFn(); - } - return transcriptionLanguage as 'ja' | 'zh' | 'ko' | 'en'; - } else { - // 言語連動が無効な場合: advanced.transcriptionLanguage を使用 + ): 'auto' | 'ja' | 'zh' | 'ko' | 'en' { + // 高度設定で言語連動が無効な場合: advanced.transcriptionLanguage を優先 + if (advanced?.languageLinkingEnabled === false) { const advancedLang = advanced.transcriptionLanguage ?? 'auto'; - if (advancedLang === 'auto') { - return detectFn(); - } - return advancedLang as 'ja' | 'zh' | 'ko' | 'en'; + return (advancedLang as 'auto' | 'ja' | 'zh' | 'ko' | 'en'); } + + // 言語連動が有効な場合（デフォルト）: 通常の transcriptionLanguage を使用 + if (transcriptionLanguage === 'auto') { + return 'auto'; + } + return transcriptionLanguage as 'ja' | 'zh' | 'ko' | 'en'; } - test('should use traditional logic when language linking is enabled (default)', () => { + test('should return auto when linking is enabled and TL is auto', () => { // Default case: advanced.languageLinkingEnabled is true const advanced = { languageLinkingEnabled: true }; @@ -150,18 +148,18 @@ describe('Language Detection Logic', () => { expect(getResolvedLanguageAdvanced('ja', advanced, mockDetectFn)).toBe('ja'); expect(mockDetectFn).not.toHaveBeenCalled(); - // Should use detection when transcriptionLanguage is 'auto' - expect(getResolvedLanguageAdvanced('auto', advanced, mockDetectFn)).toBe('ko'); - expect(mockDetectFn).toHaveBeenCalledTimes(1); + // Should pass through 'auto' + expect(getResolvedLanguageAdvanced('auto', advanced, mockDetectFn)).toBe('auto'); + expect(mockDetectFn).not.toHaveBeenCalled(); }); - test('should use traditional logic when advanced settings is undefined', () => { + test('should treat undefined advanced as linking enabled', () => { // When advanced is undefined, should default to traditional behavior (linking enabled) expect(getResolvedLanguageAdvanced('ja', undefined, mockDetectFn)).toBe('ja'); expect(mockDetectFn).not.toHaveBeenCalled(); - expect(getResolvedLanguageAdvanced('auto', undefined, mockDetectFn)).toBe('ko'); - expect(mockDetectFn).toHaveBeenCalledTimes(1); + expect(getResolvedLanguageAdvanced('auto', undefined, mockDetectFn)).toBe('auto'); + expect(mockDetectFn).not.toHaveBeenCalled(); }); test('should use advanced.transcriptionLanguage when language linking is disabled', () => { @@ -175,24 +173,24 @@ describe('Language Detection Logic', () => { expect(mockDetectFn).not.toHaveBeenCalled(); }); - test('should use auto-detection when advanced.transcriptionLanguage is auto', () => { + test('should pass through auto when linking disabled and advanced TL is auto', () => { const advanced = { languageLinkingEnabled: false, transcriptionLanguage: 'auto' }; - expect(getResolvedLanguageAdvanced('ja', advanced, mockDetectFn)).toBe('ko'); - expect(mockDetectFn).toHaveBeenCalledTimes(1); + expect(getResolvedLanguageAdvanced('ja', advanced, mockDetectFn)).toBe('auto'); + expect(mockDetectFn).not.toHaveBeenCalled(); }); - test('should use auto-detection when advanced.transcriptionLanguage is undefined', () => { + test('should pass through auto when linking disabled and advanced TL is undefined', () => { const advanced = { languageLinkingEnabled: false // transcriptionLanguage is undefined }; - expect(getResolvedLanguageAdvanced('ja', advanced, mockDetectFn)).toBe('ko'); - expect(mockDetectFn).toHaveBeenCalledTimes(1); + expect(getResolvedLanguageAdvanced('ja', advanced, mockDetectFn)).toBe('auto'); + expect(mockDetectFn).not.toHaveBeenCalled(); }); test('should work with all supported languages in advanced mode', () => { @@ -217,4 +215,4 @@ describe('Language Detection Logic', () => { expect(mockDetectFn).not.toHaveBeenCalled(); }); }); -}); \ No newline at end of file +}); From 19d7924aab7e587883b19ae699f23f871fc07992 Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 16 Aug 2025 01:15:23 +0900 Subject: [PATCH 63/82] refactor(language): remove 'auto' recognition language\n\n- Drop 'auto' from settings types and UI options\n- Migrate existing 'auto' values to detected locale\n- Always send explicit language to API\n- Update tests and copy\n- Adjust defaults and first-run init --- .../transcription/TranscriptionService.ts | 15 ++--- src/i18n/en.ts | 2 +- src/interfaces/settings.ts | 10 ++-- src/plugin/VoiceInputPlugin.ts | 43 ++++++++------ src/settings/VoiceInputSettingTab.ts | 7 +-- tests/language-detection.test.ts | 59 +++++++------------ 6 files changed, 61 insertions(+), 75 deletions(-) diff --git a/src/core/transcription/TranscriptionService.ts b/src/core/transcription/TranscriptionService.ts index 6aa7bdb..29caae8 100644 --- a/src/core/transcription/TranscriptionService.ts +++ b/src/core/transcription/TranscriptionService.ts @@ -98,13 +98,11 @@ export class TranscriptionService implements ITranscriptionProvider { formData.append('response_format', 'json'); formData.append('temperature', String(API_CONSTANTS.PARAMETERS.TRANSCRIPTION_TEMPERATURE)); // Deterministic output - // Language setting - if (language !== 'auto') { - formData.append('language', language); - } + // Language setting (auto 廃止のため常に明示指定) + formData.append('language', language); const prompt = this.buildTranscriptionPrompt(language); - if (language !== 'auto' && prompt) { + if (prompt) { formData.append('prompt', prompt); } @@ -229,10 +227,7 @@ export class TranscriptionService implements ITranscriptionProvider { * Build prompt for GPT-4o transcription (for all languages except auto) */ private buildTranscriptionPrompt(language: string): string { - // No prompt for auto language mode (as it might interfere with language detection) - if (language === 'auto') { - return ''; - } + // auto は廃止済み const normalizedLang = this.normalizeLanguage(language); @@ -361,7 +356,7 @@ ${PROMPT_CONSTANTS.KOREAN.SPEAKER_ONLY} * Normalize language code for consistent processing */ private normalizeLanguage(language: string): string { - if (language === 'auto') return 'auto'; + // auto は廃止済み const lang = language.toLowerCase(); if (lang.startsWith('ja')) return 'ja'; if (lang.startsWith('zh')) return 'zh'; diff --git a/src/i18n/en.ts b/src/i18n/en.ts index d51a5ba..471786c 100644 --- a/src/i18n/en.ts +++ b/src/i18n/en.ts @@ -159,7 +159,7 @@ export const en: TranslationResource = { transcriptionLanguage: 'Transcription Language', transcriptionLanguageDesc: 'Language for voice recognition and transcription.', pluginLanguage: 'Plugin Language', - pluginLanguageDesc: 'Set language for UI display (voice recognition language is auto-detected separately)', + pluginLanguageDesc: 'Set language for UI display', // Advanced settings languageLinking: 'Link UI and recognition languages', languageLinkingDesc: 'When enabled, recognition language follows UI language. When disabled, you can set recognition language independently.', diff --git a/src/interfaces/settings.ts b/src/interfaces/settings.ts index 96dedae..082e94f 100644 --- a/src/interfaces/settings.ts +++ b/src/interfaces/settings.ts @@ -9,7 +9,7 @@ export interface VoiceInputSettings { // 録音設定 maxRecordingSeconds: number; // 最大録音時間（秒） // 言語設定 - transcriptionLanguage: 'auto' | 'ja' | 'en' | 'zh' | 'ko'; // 音声認識言語（後方互換性のため維持） + transcriptionLanguage: 'ja' | 'en' | 'zh' | 'ko'; // 音声認識言語 pluginLanguage: Locale; // プラグインUI表示の言語 customDictionary: SimpleCorrectionDictionary; // デバッグ設定 @@ -18,7 +18,7 @@ export interface VoiceInputSettings { // 高度設定 advanced: { languageLinkingEnabled: boolean; // UI言語と認識言語を連動する（デフォルト: true） - transcriptionLanguage?: 'auto' | 'ja' | 'en' | 'zh' | 'ko'; // 独立した音声認識言語設定 + transcriptionLanguage?: 'ja' | 'en' | 'zh' | 'ko'; // 独立した音声認識言語設定 }; } @@ -29,7 +29,7 @@ export const DEFAULT_SETTINGS: VoiceInputSettings = { // 録音設定 maxRecordingSeconds: 300, // 5分（300秒） // 言語設定 - transcriptionLanguage: 'auto', // 音声認識言語のデフォルトは自動検出（後方互換性のため維持） + transcriptionLanguage: 'en', // 初期値（実際は起動時に環境ロケールへ移行） pluginLanguage: 'en', // 初期値、実際はObsidianの設定に従う customDictionary: { definiteCorrections: [] }, // デバッグ設定 @@ -37,7 +37,7 @@ export const DEFAULT_SETTINGS: VoiceInputSettings = { logLevel: LogLevel.INFO, // 通常レベル // 高度設定 advanced: { - languageLinkingEnabled: true, // デフォルトは連動オン（現行動作維持） - transcriptionLanguage: 'auto' // デフォルトは自動検出 + languageLinkingEnabled: true, + transcriptionLanguage: 'en' } }; diff --git a/src/plugin/VoiceInputPlugin.ts b/src/plugin/VoiceInputPlugin.ts index b66161b..54ee04d 100644 --- a/src/plugin/VoiceInputPlugin.ts +++ b/src/plugin/VoiceInputPlugin.ts @@ -221,14 +221,17 @@ export default class VoiceInputPlugin extends Plugin { this.logger?.info('Migrating interfaceLanguage to pluginLanguage'); } - // languageからtranscriptionLanguageへの移行 + // languageからtranscriptionLanguageへの移行（autoは検出ロケールへ変換） if ('language' in data && !('transcriptionLanguage' in data)) { // 既存のlanguageフィールドをtranscriptionLanguageに移行 const langValue = data.language; - if (langValue === 'auto' || langValue === 'ja' || langValue === 'en' || langValue === 'zh' || langValue === 'ko') { + if (langValue === 'ja' || langValue === 'en' || langValue === 'zh' || langValue === 'ko') { migratedData.transcriptionLanguage = langValue; + } else if (langValue === 'auto') { + // auto は廃止: 起動環境のロケールへ固定 + migratedData.transcriptionLanguage = this.detectPluginLanguage(); } else { - migratedData.transcriptionLanguage = 'auto'; + migratedData.transcriptionLanguage = this.detectPluginLanguage(); } delete migratedData.language; needsSave = true; @@ -322,12 +325,12 @@ export default class VoiceInputPlugin extends Plugin { this.logger?.info(`Auto-detected language: ${this.settings.pluginLanguage} (from Obsidian: ${getObsidianLocale(this.app)})`); } - // 高度設定のマイグレーション + // 高度設定のマイグレーション（auto を廃止） if (!hasSettingsKey(data, 'advanced')) { // 既存ユーザーには言語連動をデフォルトで有効化（現行動作維持） this.settings.advanced = { languageLinkingEnabled: true, - transcriptionLanguage: 'auto' + transcriptionLanguage: this.detectPluginLanguage() }; needsSave = true; this.logger?.info('Initialized advanced settings with language linking enabled for backward compatibility'); @@ -336,17 +339,25 @@ export default class VoiceInputPlugin extends Plugin { this.settings.advanced.languageLinkingEnabled = true; needsSave = true; this.logger?.info('Added languageLinkingEnabled to existing advanced settings'); + } else if (data.advanced) { + // auto からの置換 + const adv = data.advanced as { transcriptionLanguage?: string }; + if (adv.transcriptionLanguage === 'auto') { + this.settings.advanced.transcriptionLanguage = this.detectPluginLanguage(); + needsSave = true; + this.logger?.info('Migrated advanced.transcriptionLanguage from auto to detected locale'); + } } } else { // 保存データが存在しない場合（初回起動） this.settings.pluginLanguage = this.detectPluginLanguage(); - this.settings.transcriptionLanguage = 'auto'; + this.settings.transcriptionLanguage = this.detectPluginLanguage(); this.settings.advanced = { languageLinkingEnabled: true, - transcriptionLanguage: 'auto' + transcriptionLanguage: this.detectPluginLanguage() }; needsSave = true; - this.logger?.info(`First run - auto-detected language: ${this.settings.pluginLanguage}, transcriptionLanguage: auto, advanced settings initialized`); + this.logger?.info(`First run - detected locale: ${this.settings.pluginLanguage}, transcriptionLanguage set to detected locale, advanced settings initialized`); } // 必要に応じて設定を保存 @@ -382,19 +393,17 @@ export default class VoiceInputPlugin extends Plugin { /** * 解決済み言語を取得（高度設定の連動設定に基づく） - * 仕様変更: 'auto' の場合は API に 'auto' をそのまま渡す + * auto は廃止済みのため、常に具体的な言語コードを返す */ - getResolvedLanguage(): 'auto' | 'ja' | 'zh' | 'ko' | 'en' { + getResolvedLanguage(): 'ja' | 'zh' | 'ko' | 'en' { // 言語連動が無効な場合: advanced.transcriptionLanguage を優先 if (this.settings.advanced?.languageLinkingEnabled === false) { - const advancedLang = this.settings.advanced.transcriptionLanguage ?? 'auto'; - return advancedLang as 'auto' | 'ja' | 'zh' | 'ko' | 'en'; + const advancedLang = this.settings.advanced?.transcriptionLanguage; + return (advancedLang ?? this.detectPluginLanguage()) as 'ja' | 'zh' | 'ko' | 'en'; } - - // 言語連動が有効（デフォルト）の場合: 通常の transcriptionLanguage を使用 - // transcriptionLanguage が 'auto' なら 'auto' のまま返す - const baseLang = this.settings.transcriptionLanguage ?? 'auto'; - return baseLang as 'auto' | 'ja' | 'zh' | 'ko' | 'en'; + // 言語連動が有効: 通常の transcriptionLanguage を使用 + const baseLang = this.settings.transcriptionLanguage; + return (baseLang ?? this.detectPluginLanguage()) as 'ja' | 'zh' | 'ko' | 'en'; } async saveSettings() { diff --git a/src/settings/VoiceInputSettingTab.ts b/src/settings/VoiceInputSettingTab.ts index 958d17d..87d02ec 100644 --- a/src/settings/VoiceInputSettingTab.ts +++ b/src/settings/VoiceInputSettingTab.ts @@ -84,7 +84,7 @@ export class VoiceInputSettingTab extends PluginSettingTab { if (!this.plugin.settings.advanced) { this.plugin.settings.advanced = { languageLinkingEnabled: value, - transcriptionLanguage: 'auto' + transcriptionLanguage: this.plugin.getResolvedLanguage() }; } else { this.plugin.settings.advanced.languageLinkingEnabled = value; @@ -100,13 +100,12 @@ export class VoiceInputSettingTab extends PluginSettingTab { .setName(this.i18n.t('ui.settings.advancedTranscriptionLanguage')) .setDesc(this.i18n.t('ui.settings.advancedTranscriptionLanguageDesc')) .addDropdown(dropdown => dropdown - .addOption('auto', this.i18n.t('ui.options.languageAuto')) .addOption('ja', this.i18n.t('ui.options.languageJa')) .addOption('en', this.i18n.t('ui.options.languageEn')) .addOption('zh', this.i18n.t('ui.options.languageZh')) .addOption('ko', this.i18n.t('ui.options.languageKo')) - .setValue(this.plugin.settings.advanced.transcriptionLanguage ?? 'auto') - .onChange(async (value: 'auto' | 'ja' | 'en' | 'zh' | 'ko') => { + .setValue(this.plugin.settings.advanced.transcriptionLanguage ?? this.plugin.getResolvedLanguage()) + .onChange(async (value: 'ja' | 'en' | 'zh' | 'ko') => { if (!this.plugin.settings.advanced) { this.plugin.settings.advanced = { languageLinkingEnabled: false, diff --git a/tests/language-detection.test.ts b/tests/language-detection.test.ts index 9165201..ae1acd7 100644 --- a/tests/language-detection.test.ts +++ b/tests/language-detection.test.ts @@ -21,19 +21,12 @@ function detectPluginLanguage(getObsidianLocaleFn: () => string): 'ja' | 'zh' | } function getResolvedLanguage( - pluginLanguage: string, + pluginLanguage: string, detectFn: () => 'ja' | 'zh' | 'ko' | 'en' -): 'auto' | 'ja' | 'zh' | 'ko' | 'en' { - if (pluginLanguage === 'auto') { - // 仕様変更: 'auto' はそのまま返す（API に渡す） - return 'auto'; - } - +): 'ja' | 'zh' | 'ko' | 'en' { if (!['ja', 'zh', 'ko', 'en'].includes(pluginLanguage)) { - // 不正値は従来通り検出へフォールバック return detectFn(); } - return pluginLanguage as 'ja' | 'zh' | 'ko' | 'en'; } @@ -99,11 +92,7 @@ describe('Language Detection Logic', () => { expect(mockDetectFn).toHaveBeenCalledTimes(1); }); - test('should handle auto value (pass-through)', () => { - const result = getResolvedLanguage('auto', mockDetectFn); - expect(result).toBe('auto'); - expect(mockDetectFn).not.toHaveBeenCalled(); - }); + // auto は廃止済みのためテストしない test('should work with all supported languages', () => { expect(getResolvedLanguage('ja', mockDetectFn)).toBe('ja'); @@ -123,24 +112,18 @@ describe('Language Detection Logic', () => { // Function to simulate the new advanced language resolution logic function getResolvedLanguageAdvanced( - transcriptionLanguage: string, + transcriptionLanguage: string | undefined, advanced: { languageLinkingEnabled?: boolean; transcriptionLanguage?: string } | undefined, detectFn: () => 'ja' | 'zh' | 'ko' | 'en' - ): 'auto' | 'ja' | 'zh' | 'ko' | 'en' { - // 高度設定で言語連動が無効な場合: advanced.transcriptionLanguage を優先 + ): 'ja' | 'zh' | 'ko' | 'en' { if (advanced?.languageLinkingEnabled === false) { - const advancedLang = advanced.transcriptionLanguage ?? 'auto'; - return (advancedLang as 'auto' | 'ja' | 'zh' | 'ko' | 'en'); + const adv = advanced?.transcriptionLanguage; + return (['ja', 'zh', 'ko', 'en'].includes(adv as any) ? adv : detectFn()) as 'ja' | 'zh' | 'ko' | 'en'; } - - // 言語連動が有効な場合（デフォルト）: 通常の transcriptionLanguage を使用 - if (transcriptionLanguage === 'auto') { - return 'auto'; - } - return transcriptionLanguage as 'ja' | 'zh' | 'ko' | 'en'; + return (['ja', 'zh', 'ko', 'en'].includes(transcriptionLanguage as any) ? transcriptionLanguage : detectFn()) as 'ja' | 'zh' | 'ko' | 'en'; } - test('should return auto when linking is enabled and TL is auto', () => { + test('should use TL when linking enabled; fallback to detection when missing/invalid', () => { // Default case: advanced.languageLinkingEnabled is true const advanced = { languageLinkingEnabled: true }; @@ -148,9 +131,9 @@ describe('Language Detection Logic', () => { expect(getResolvedLanguageAdvanced('ja', advanced, mockDetectFn)).toBe('ja'); expect(mockDetectFn).not.toHaveBeenCalled(); - // Should pass through 'auto' - expect(getResolvedLanguageAdvanced('auto', advanced, mockDetectFn)).toBe('auto'); - expect(mockDetectFn).not.toHaveBeenCalled(); + // Missing or invalid TL falls back to detection + expect(getResolvedLanguageAdvanced(undefined, advanced, mockDetectFn)).toBe('ko'); + expect(getResolvedLanguageAdvanced('invalid', advanced, mockDetectFn)).toBe('ko'); }); test('should treat undefined advanced as linking enabled', () => { @@ -158,8 +141,8 @@ describe('Language Detection Logic', () => { expect(getResolvedLanguageAdvanced('ja', undefined, mockDetectFn)).toBe('ja'); expect(mockDetectFn).not.toHaveBeenCalled(); - expect(getResolvedLanguageAdvanced('auto', undefined, mockDetectFn)).toBe('auto'); - expect(mockDetectFn).not.toHaveBeenCalled(); + expect(getResolvedLanguageAdvanced(undefined, undefined, mockDetectFn)).toBe('ko'); + expect(mockDetectFn).toHaveBeenCalledTimes(1); }); test('should use advanced.transcriptionLanguage when language linking is disabled', () => { @@ -173,24 +156,24 @@ describe('Language Detection Logic', () => { expect(mockDetectFn).not.toHaveBeenCalled(); }); - test('should pass through auto when linking disabled and advanced TL is auto', () => { + test('should use advanced TL when linking disabled; fallback to detection when missing/invalid', () => { const advanced = { languageLinkingEnabled: false, - transcriptionLanguage: 'auto' + transcriptionLanguage: undefined as unknown as string }; - expect(getResolvedLanguageAdvanced('ja', advanced, mockDetectFn)).toBe('auto'); - expect(mockDetectFn).not.toHaveBeenCalled(); + expect(getResolvedLanguageAdvanced('ja', advanced, mockDetectFn)).toBe('ko'); + expect(mockDetectFn).toHaveBeenCalled(); }); - test('should pass through auto when linking disabled and advanced TL is undefined', () => { + test('should use detection when linking disabled and advanced TL is undefined', () => { const advanced = { languageLinkingEnabled: false // transcriptionLanguage is undefined }; - expect(getResolvedLanguageAdvanced('ja', advanced, mockDetectFn)).toBe('auto'); - expect(mockDetectFn).not.toHaveBeenCalled(); + expect(getResolvedLanguageAdvanced('ja', advanced, mockDetectFn)).toBe('ko'); + expect(mockDetectFn).toHaveBeenCalled(); }); test('should work with all supported languages in advanced mode', () => { From ae90aadf30be14f2afcf945f48a4aaf83941c7ab Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 16 Aug 2025 01:49:28 +0900 Subject: [PATCH 64/82] fix(recording): stabilize start/stop and prevent race conditions\n\n- Make AudioRecorder.initialize() idempotent and resume suspended AudioContext\n- Add in-flight guard to startRecording() to block concurrent starts\n- Keep UI transition lock only at entry points to avoid missed starts\n- Improve safety checks around createMediaStreamSource() --- src/core/audio/AudioRecorder.ts | 36 +++++++++++++++++++++++++++--- src/views/VoiceInputViewActions.ts | 21 +++++++++++++++-- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/src/core/audio/AudioRecorder.ts b/src/core/audio/AudioRecorder.ts index 3860e33..156aa3b 100644 --- a/src/core/audio/AudioRecorder.ts +++ b/src/core/audio/AudioRecorder.ts @@ -38,6 +38,7 @@ export class AudioRecorder extends Disposable { private logger: Logger | null = null; private recordingStartTime: number = 0; private maxRecordingTimer: NodeJS.Timeout | null = null; + private isStarting: boolean = false; constructor(options: AudioRecorderOptions) { super(); @@ -71,6 +72,17 @@ export class AudioRecorder extends Disposable { async initialize(): Promise { this.throwIfDisposed(); + // 既にAudioContextが有効なら再初期化はスキップ + if (this.audioContext && this.audioContext.state !== 'closed') { + // まだビジュアライザー未生成ならここで生成 + if (!this.visualizer && this.options.visualizerContainer) { + this.visualizer = this.options.useSimpleVisualizer + ? new SimpleAudioLevelIndicator(this.options.visualizerContainer) + : new AudioVisualizer(this.options.visualizerContainer); + } + return; + } + if (this.options.useVAD && this.vadProcessor) { await this.vadProcessor.initialize(); } @@ -127,11 +139,21 @@ export class AudioRecorder extends Disposable { async startRecording(): Promise { this.throwIfDisposed(); - if (this.isRecording) { + if (this.isRecording || this.isStarting) { return; } + this.isStarting = true; try { + // Ensure AudioContext exists (防御的) + if (!this.audioContext || this.audioContext.state === 'closed') { + await this.initialize(); + } + // Safariなどでのsuspend対策 + if (this.audioContext!.state === 'suspended') { + await this.audioContext!.resume(); + } + // Log recording session start this.logger?.info('Starting recording session', { sampleRate: AUDIO_CONSTANTS.SAMPLE_RATE, @@ -183,7 +205,13 @@ export class AudioRecorder extends Disposable { } // Setup audio processing chain with filters - this.sourceNode = this.audioContext!.createMediaStreamSource(this.stream); + if (!this.audioContext) { + throw new TranscriptionError( + TranscriptionErrorType.AUDIO_INITIALIZATION_FAILED, + 'Audio context is not initialized' + ); + } + this.sourceNode = this.audioContext.createMediaStreamSource(this.stream); this.gainNode = this.audioContext!.createGain(); this.gainNode.gain.value = DEFAULT_AUDIO_SETTINGS.gain; // デフォルトゲイン @@ -249,6 +277,8 @@ export class AudioRecorder extends Disposable { } catch (error) { this.logger?.error('Failed to start recording', error); throw error; + } finally { + this.isStarting = false; } } @@ -585,4 +615,4 @@ export class AudioRecorder extends Disposable { this.options.autoStopSilenceDuration = duration; } } -} \ No newline at end of file +} diff --git a/src/views/VoiceInputViewActions.ts b/src/views/VoiceInputViewActions.ts index 2af7816..be70be7 100644 --- a/src/views/VoiceInputViewActions.ts +++ b/src/views/VoiceInputViewActions.ts @@ -28,6 +28,8 @@ export class VoiceInputViewActions { processingQueue: [] }; private isProcessingAudio = false; + // 連打や高速操作による並行実行を防ぐための遷移ロック + private isTransitioning = false; private statusTimer: NodeJS.Timeout | null = null; private clearConfirmTimer: NodeJS.Timeout | null = null; private clearPressCount = 0; @@ -113,10 +115,20 @@ export class VoiceInputViewActions { * Toggle recording on/off */ async toggleRecording() { + if (this.isTransitioning) return; // 二重操作防止 + this.isTransitioning = true; if (this.audioRecorder && this.audioRecorder.isActive()) { - await this.stopRecording(); + try { + await this.stopRecording(); + } finally { + this.isTransitioning = false; + } } else { - await this.startRecording(); + try { + await this.startRecording(); + } finally { + this.isTransitioning = false; + } } } @@ -603,7 +615,10 @@ export class VoiceInputViewActions { * Cancel recording without processing */ async cancelRecording() { + if (this.isTransitioning) return; // 二重キャンセル防止 + this.isTransitioning = true; if (!this.audioRecorder || !this.audioRecorder.isActive()) { + this.isTransitioning = false; return; } @@ -628,6 +643,8 @@ export class VoiceInputViewActions { this.logger.error('Error cancelling recording', error); this.updateUIAfterError(); this.view.ui.setButtonsEnabled(true); + } finally { + this.isTransitioning = false; } } From c346d933ba402032bcdfe0590e95984356fe2874 Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 16 Aug 2025 02:20:06 +0900 Subject: [PATCH 65/82] docs(flow): update processing flow to reflect latest pipeline, safety thresholds, and recording/queue details --- docs/PROCESSING_FLOW.md | 380 ++++++++++++---------------------------- 1 file changed, 114 insertions(+), 266 deletions(-) diff --git a/docs/PROCESSING_FLOW.md b/docs/PROCESSING_FLOW.md index def0300..e976c66 100644 --- a/docs/PROCESSING_FLOW.md +++ b/docs/PROCESSING_FLOW.md @@ -1,291 +1,139 @@ # Voice Input Processing Flow / 音声入力処理フロー -This document describes the current processing pipeline of the voice input system. As of 2025-08-15, the system supports multilingual prompts for ja/en/zh/ko languages, followed by mechanical TRANSCRIPT wrapper stripping, safety-guarded cleaning pipeline, and optional dictionary correction. +This document reflects the latest implementation as of 2025‑08‑16. It covers the end‑to‑end flow: recording, queuing, transcription, cleaning with safety guards, and optional dictionary correction. -このドキュメントは最新の音声入力処理パイプラインを説明します。2025-08-15 時点では、日本語/英語/中国語/韓国語の多言語プロンプトをサポートし、その後に機械的な TRANSCRIPT ラッパー除去、安全装置付きのクリーニング・パイプライン、任意の辞書補正を実行します。 +本ドキュメントは 2025‑08‑16 時点の実装に基づき、録音からキュー処理、文字起こし、セーフティ付きクリーニング、辞書補正までの一連の流れを記載します。 -## Updated Overview (Current) / 最新の概要 +## High‑Level Overview / 全体概要 -1) Audio → OpenAI Transcription API（**全言語 ja/en/zh/ko にプロンプト付与**）。 -2) APIレスポンス `text` を受領。 -3) 構造除去（事前処理）: ` ... ` を機械的に抽出・除去（閉じタグ欠落にも対応）。 -4) クリーニング・パイプライン実行（`StandardCleaningPipeline`） - - `PromptContaminationCleaner`: 指示文・XML/文脈タグ・スニペットなどの混入除去。 - - `UniversalRepetitionCleaner`: 反復抑制（文字/トークン/文/列挙/段落/末尾）＋最終整形。 - - セーフティ: クリーナー単体の削減率上限＋緊急ロールバック。構造除去は過剰ロールバックを避けるため緩和あり。 -5) プロンプトエラー検出（無音時にプロンプトが返る等）→ 該当すれば空文字で早期終了。 -6) 辞書補正（任意）: `DictionaryCorrector` による語彙修正。 -7) 最終テキストを返却。 +1) 音声取得 → 連続録音（最大録音時間到達または手動停止）。 +2) 録音ごとに audioBlob を非同期キューへ投入（録音は継続可能）。 +3) OpenAI Transcription API 呼び出し（ja/en/zh/ko には構造化プロンプト付与）。 +4) 機械的な TRANSCRIPT ラッパー除去 → クリーニング・パイプライン（安全判定付き）。 +5) プロンプトエラー検出（プロンプトのみが返った場合は空文字へ）。 +6) 任意の辞書補正を適用し、UIへ反映。 -## Complete Processing Flow / 完全な処理フロー +--- +## Recording Path Details / 録音パス詳細 + +実装参照: `src/core/audio/AudioRecorder.ts`, `src/views/VoiceInputViewActions.ts`, `src/views/VoiceInputViewUI.ts` + +- AudioContext 初期化 + - `window.AudioContext`（Safari は `webkitAudioContext`）を使用。 + - `AudioWorklet` が利用可能ならワークレット、不可なら `ScriptProcessor` へフォールバック。 + - `suspended` 状態の場合は `startRecording()` で `resume()` を実行。 +- フィルタ/ノード構成 + - `MediaStreamSource → Gain → HighPass(約80Hz) → LowPass(約7.6kHz) → Analyser` + - ビジュアライザは `Analyser` に接続（シンプル/通常の2種）。 +- 連続処理 + - ワークレット/スクリプトプロセッサで 1ch PCM を取り出し、`AudioRingBuffer` に蓄積。 + - マイク実データ検知で `onMicrophoneStatusChange('ready')` 通知。 +- 録音制御 + - 既定は VAD 無効の連続録音。`maxRecordingSeconds` 到達で自動停止。 + - 二重開始は `AudioRecorder.isStarting` で抑止。UI側のロックはエントリポイントのみで軽量化。 +- UI/操作 + - 通常クリックで開始/停止。プッシュトゥトークは長押しで開始、離して停止（`UI_CONSTANTS.PUSH_TO_TALK_THRESHOLD`）。 + +--- + +## Queue & UI / キューとUI + +実装参照: `src/views/VoiceInputViewActions.ts` + +- 停止ごとに生成された `audioBlob` を `processingQueue` へ追加し、逐次処理。 +- ステータス表示は「録音中/処理中/待機数」を反映。キャンセルは録音を中断して音声を破棄。 +- 途中で録音を継続しながら、既存キューをバックグラウンドで順次文字起こし可能。 + +--- + +## Transcription Path / 文字起こしパス + +実装参照: `src/core/transcription/TranscriptionService.ts` + +1) リクエスト作成（`multipart/form-data`） + - `file: audio.webm(Blob)`, `model: gpt-4o(-mini)-transcribe`, `response_format: json`, `temperature: 0`, `language`。 + - 言語に応じて構造化プロンプトを付与（ja/en/zh/ko）。 +2) API レスポンスから `text` を取得。 +3) `cleanGPT4oResponse(text, language)` を実行。 + - 先に機械的に TRANSCRIPT ラッパーを剥離（完全/不完全タグ双方に対応）。 + - クリーニング・パイプラインを実行（詳細は後述）。 +4) プロンプトエラー検出 + - 言語別の検出ルールにより、プロンプトや注釈のみが返るケースを検知。 + - 検出時は空文字に置換して早期終了（無音/極短音声で発生しやすい）。 +5) 辞書補正（有効時） + - `DictionaryCorrector` により語彙の統一・置換を実施。 + +--- + +## Cleaning Pipeline / クリーニング・パイプライン + +実装参照: `src/core/transcription/cleaning/StandardCleaningPipeline.ts` + +- 構成 + 1) `PromptContaminationCleaner` + - TRANSCRIPT/TRANSCRIPTION タグ残留、指示文（完全一致/スニペット/文脈）、フォーマット行の除去。 + - 空行/空白の正規化。 + 2) `UniversalRepetitionCleaner` + - 文字/文/列挙/段落レベルの反復抑制、末尾ハルシネーション緩和、体裁整形。 + +- セーフティ（安全判定） + - 設定値（`src/config/CleaningConfig.ts`） + - `singleCleanerMaxReduction`: 0.3（単一クリーナーの削減率上限） + - `emergencyFallbackThreshold`: 0.5（緊急ロールバック閾値） + - `warningThreshold`: 0.15（警告ログ） + - 構造的クリーナー緩和（`PromptContaminationCleaner` に適用） + - `singleCleanerMaxReduction → max(0.9, 0.3) = 0.9` + - `emergencyFallbackThreshold → max(0.95, 0.5) = 0.95` + - 超過時の動作 + - 単一上限（single）超過 → `skip`（そのクリーナーの変更を適用しない） + - 緊急（emergency）超過 → `rollback`（そのクリーナーの結果を破棄して元のテキストに戻す） + +例（ログ）: ``` -┌─────────────────┐ -│ Audio Input │ 🎤 -│ 音声入力 │ -└─────┬───────────┘ - │ - ▼ -┌─────────────────┐ -│ Audio Blob │ -│ Creation │ -│ 音声Blob作成 │ -└─────┬───────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ TranscriptionService.transcribe() │ -│ 文字起こしサービス │ -└─────┬───────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────┐ ┌──────────────────────────────────────┐ -│ Language Check │────│ buildTranscriptionPrompt(language) │ -│ 言語チェック │ │ プロンプト構築 │ -└─────┬───────────┘ └──────────────────┬───────────────────┘ - │ │ - │ ┌─────────────────────────────────│──────────────────┐ - │ │ │ │ - │ │ IF language === 'auto' │ │ - │ │ 自動検出の場合: │ │ - │ │ ▼ │ - │ │ ┌───────────────────────────────────────────┐ │ - │ │ │ No Prompt Added │ │ - │ │ │ プロンプト追加なし │ │ - │ │ │ (言語検出に干渉しない) │ │ - │ │ └───────────────────────────────────────────┘ │ - │ │ │ │ - │ │ ELSE (ja/en/zh/ko) │ │ - │ │ 対応言語の場合: │ │ - │ │ ▼ │ - │ │ ┌───────────────────────────────────────────┐ │ - │ │ │ Structured Multilingual Prompts │ │ - │ │ │ 構造化多言語プロンプト │ │ - │ │ │ │ │ - │ │ │ 日本語: 以下の音声内容のみを文字に... │ │ - │ │ │ English: Please transcribe only the... │ │ - │ │ │ 中文: 请仅转录以下音频内容... │ │ - │ │ │ 한국어: 다음 음성 내용만 전사해주세요... │ │ - │ │ │ │ │ - │ │ │ Format: INSTRUCTION×2 + OUTPUT_FORMAT │ │ - │ │ │ + + SPEAKER_ONLY │ │ - │ │ └───────────────────────────────────────────┘ │ - │ └─────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ OpenAI API Request │ -│ OpenAI API リクエスト │ -│ │ -│ FormData: │ -│ ├─ file: audio.webm (Blob) │ -│ ├─ model: gpt-4o-transcribe / gpt-4o-mini-transcribe │ -│ ├─ response_format: json │ -│ ├─ temperature: 0 │ -│ ├─ language: ja/en/zh/ko (if not 'auto') │ -│ └─ prompt: [Multilingual prompts for ja/en/zh/ko] │ -│ [多言語プロンプト（ja/en/zh/ko用）] │ -│ │ -│ Headers: │ -│ └─ Authorization: Bearer ${apiKey} │ -└─────┬───────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ OpenAI API Response │ -│ OpenAI API レスポンス │ -│ │ -│ { text: "transcribed text...", language: "detected" } │ -└─────┬───────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ cleanGPT4oResponse(text, language) │ -│ GPT-4oレスポンスクリーニング │ -└─────┬───────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ Pre-strip TRANSCRIPT wrappers (mechanical extraction) │ -│ TRANSCRIPTラッパーの事前除去（完全/不完全に対応） │ -└─────┬───────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ StandardCleaningPipeline（安全判定付き） │ -│ クリーニング・パイプライン │ -│ │ -│ 1) PromptContaminationCleaner │ -│ - TRANSCRIPT/TRANSCRIPTIONタグ残留の除去 │ -│ - 指示文（完全一致/スニペット/文脈）除去 │ -│ │ -│ 2) UniversalRepetitionCleaner │ -│ - 文字/記号/トークン/文/段落/列挙/末尾の反復抑制 │ -│ - 最終整形（改行・空白の正規化など） │ -│ │ -│ Safety / 安全装置: │ -│ - クリーナー単体の削減率上限、緊急ロールバック │ -│ - 構造除去の過剰ロールバック回避（緩和） │ -└─────┬───────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ isPromptErrorDetected(text, language) │ -│ プロンプトエラー検出 │ -└─────┬───────────────────────────────────────────────────────┘ - │ - │ ┌─────────────────────────────────────────────────────┐ - │ │ │ - │ │ Check for prompt leakage patterns: │ - │ │ プロンプト漏洩パターンをチェック: │ - │ │ │ - │ │ Japanese (ja): │ - │ │ • "この指示文は出力に含めないでください" │ - │ │ • "話者の発言内容だけを正確に記録してください" │ - │ │ • "（話者の発言のみ）" │ - │ │ │ - │ │ English (en): │ - │ │ • "Please transcribe only the speaker" │ - │ │ • "Do not include this instruction" │ - │ │ • "(Speaker content only)" │ - │ │ │ - │ │ Chinese (zh): │ - │ │ • "请仅转录说话者" │ - │ │ • "不要包含此指令" │ - │ │ • "（仅说话者内容）" │ - │ │ │ - │ │ Korean (ko): │ - │ │ • "화자의 발언만 전사해주세요" │ - │ │ • "이 지시사항을 포함하지 마세요" │ - │ │ • "（화자 발언만）" │ - │ └─────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────┐ ┌──────────────────────────────────────┐ -│ If prompt error │ NO │ Continue to dictionary processing │ -│ detected? │───▶│ 辞書処理へ続行 │ -│ プロンプトエラー│ └──────────────────┬───────────────────┘ -│ 検出？ │ │ -└─────┬───────────┘ │ - │ YES │ - ▼ │ -┌─────────────────┐ │ -│ Return empty │ │ -│ string │ │ -│ 空文字を返す │ │ -└─────────────────┘ │ - │ - ▼ - ┌─────────────────────────────────────┐ - │ enableTranscriptionCorrection? │ - │ 文字起こし修正が有効？ │ - └─────┬───────────────────────────────┘ - │ - ┌─────────┴─────────┐ - │ YES │ NO - ▼ ▼ - ┌───────────────────────────────┐ ┌─────────────────┐ - │ DictionaryCorrector.correct() │ │ Skip correction │ - │ 辞書修正適用 │ │ 修正をスキップ │ - └─────┬─────────────────────────┘ └─────┬───────────┘ - │ │ - ▼ │ - ┌───────────────────────────────────────┐ │ - │ Apply multilingual corrections: │ │ - │ 多言語修正を適用: │ │ - │ │ │ - │ 1. Default rules (empty by default) │ │ - │ デフォルトルール（既定では空） │ │ - │ │ │ - │ 2. Custom rules │ │ - │ カスタムルール │ │ - │ │ │ - │ 3. Dictionary corrections │ │ - │ 辞書修正: │ │ - │ • Fixed string replacements │ │ - │ • Applied to ALL languages │ │ - │ • 固定文字列置換 │ │ - │ • 全言語に適用 │ │ - │ │ │ - │ Examples: │ │ - │ • "AI" → "artificial intelligence" │ │ - │ • "こんにちは" → "こんにちは（修正）" │ │ - │ • "你好" → "您好" │ │ - │ • "안녕" → "안녕하세요" │ │ - └─────┬─────────────────────────────────┘ │ - │ │ - └───────────────┬───────────────────┘ - │ - ▼ - ┌─────────────────────────────────────┐ - │ Final Output │ - │ 最終出力 │ - │ │ - │ TranscriptionResult { │ - │ text: correctedText, │ - │ originalText: originalText, │ - │ duration: processingTime, │ - │ model: "gpt-4o(-mini)-transcribe", │ - │ language: detectedLanguage │ - │ } │ - └─────────────────────────────────────┘ +[Voice Transcription] [StandardCleaningPipeline] Rolling back cleaner PromptContaminationCleaner { + reason: 'Reduction ratio 1.000 exceeds emergency threshold 0.95', + reductionRatio: 1 +} ``` +意味: 構造的クリーナーが 95% 超の削減を行おうとしたため、安全装置によりロールバックされました。上流で「実発話が少ない/無い」状態（無音、極短、プロンプトエコー等）が疑われます。 -## Key Differences by Language / 言語による主な違い - -### All Supported Languages (ja/en/zh/ko) Processing / 全対応言語処理 -1. **Prompt Addition**: Structured prompts with language-specific instructions - - プロンプト追加: 言語固有の指示を含む構造化プロンプト -2. **Comprehensive Cleaning**: Language-aware pattern removal via PromptContaminationCleaner - - 包括的クリーニング: PromptContaminationCleaner による言語認識パターン除去 -3. **Error Detection**: Language-specific prompt leakage patterns - - エラー検出: 言語固有のプロンプト漏洩パターン - -### Auto Language Processing / 自動言語処理 -- Uses Japanese patterns as fallback for error detection -- エラー検出時は日本語パターンをフォールバックとして使用 -- No prompt addition (prevents interference with language detection) -- プロンプト追加なし（言語検出への干渉を防ぐ） +--- ## Dictionary Processing / 辞書処理 -Dictionary correction is **language-agnostic** and applies to all languages when enabled: -辞書修正は**言語非依存**で、有効時は全言語に適用されます: - +- 有効時、言語非依存で置換を適用。 +- 例: ``` -Dictionary Entry: { from: ["AI"], to: "artificial intelligence" } -辞書エントリ: { from: ["AI"], to: "artificial intelligence" } +{ from: ["AI"], to: "artificial intelligence" } -Applied to: -適用対象: -✓ English: "AI system" → "artificial intelligence system" -✓ Japanese: "AIシステム" → "artificial intelligenceシステム" -✓ Chinese: "AI系统" → "artificial intelligence系统" -✓ Korean: "AI시스템" → "artificial intelligence시스템" +EN: "AI system" → "artificial intelligence system" +JA: "AIシステム" → "artificial intelligenceシステム" +ZH: "AI系统" → "artificial intelligence系统" +KO: "AI시스템" → "artificial intelligence시스템" ``` -## Processing Performance / 処理性能 +--- -The system logs detailed performance metrics: -システムは詳細なパフォーマンス指標をログに記録します: +## Status & Metrics / ステータスと計測 -- Audio blob size and type / 音声Blobサイズとタイプ -- Processing duration / 処理時間 -- Original vs corrected text length / 元のテキストと修正後のテキスト長 -- Model used and cost estimation / 使用モデルとコスト推定 +- ステータス: 録音準備/録音中/処理中（待機数付き）/完了/エラー。 +- メトリクス: 音声サイズ、処理時間、原文/補正後の長さ、モデル、出力長などをログ出力。 + +--- ## Error Handling / エラーハンドリング -The system handles various error scenarios: -システムは様々なエラーシナリオを処理します: +1) API エラー: 401（キー不正）、429（クォータ超過）、5xx（ネットワーク/サーバ） +2) プロンプト漏洩: 言語別検出 → 空文字化で軽減 +3) 空/無音: 早期終了で空文字返却、UIに短すぎ警告 +4) クリーニング安全装置: 過剰削減を検知してスキップ/ロールバック -1. **API Errors**: Invalid key, quota exceeded, network errors - - API エラー: 無効なキー、クォータ超過、ネットワークエラー -2. **Prompt Leakage**: Detection and mitigation of prompt content in output - - プロンプト漏洩: 出力内のプロンプト内容の検出と軽減 -3. **Empty Audio**: Graceful handling of silent or empty audio input - - 空音声: 無音または空の音声入力の適切な処理 +--- -This visualization shows the complete flow from audio input to final transcribed and corrected text, highlighting the sophisticated multilingual processing that makes this plugin effective for users of Japanese, English, Chinese, and Korean languages while maintaining auto-detection compatibility. +## Notes / 補足 + +- 既定は VAD 無効の連続録音。必要に応じて `maxRecordingSeconds` を調整してください。 +- 連打・素早い操作でも安定するよう、開始の二重実行は内部ガードで抑止しています。 +- 「開始しにくい」挙動を避けるため、UIロックは最小限に留めています。 -この可視化は、音声入力から最終的な文字起こし・修正テキストまでの完全なフローを示し、このプラグインが日本語、英語、中国語、韓国語のユーザーに効果的でありながら自動検出との互換性を維持する洗練された多言語処理を強調しています。 From e91fcbbbabea6a9628fa8daa3d274c8e4c25c8d3 Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 16 Aug 2025 02:24:30 +0900 Subject: [PATCH 66/82] docs(flow): add ASCII overview and language-specific vs common behavior section --- docs/PROCESSING_FLOW.md | 72 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/docs/PROCESSING_FLOW.md b/docs/PROCESSING_FLOW.md index e976c66..489aad1 100644 --- a/docs/PROCESSING_FLOW.md +++ b/docs/PROCESSING_FLOW.md @@ -15,6 +15,36 @@ This document reflects the latest implementation as of 2025‑08‑16. It covers --- +## ASCII Overview / ASCII概要 + +``` + ┌──────────────────────────┐ + │ AudioRecorder (連続録音) │ + └─────────────┬────────────┘ + │ audioBlob + ▼ + ┌──────────────────────────┐ + │ Processing Queue (直列化) │ + └─────────────┬────────────┘ + │ audioBlob + ▼ +┌──────────────────────────────────────────────────────────────┐ +│ TranscriptionService.transcribeAudio() │ +├──────────────────────────────────────────────────────────────┤ +│ 1) buildTranscriptionPrompt(lang) ──► ja/en/zh/ko で付与 │ +│ 2) POST (multipart/form-data) │ +│ 3) text 受領 │ +│ 4) preStripTranscriptWrappers(text) │ +│ 5) StandardCleaningPipeline (安全判定付き) │ +│ ├─ PromptContaminationCleaner（構造/指示/文脈/スニペット） │ +│ └─ UniversalRepetitionCleaner（反復・体裁） │ +│ 6) isPromptErrorDetected(text, lang)（プロンプトエコー検出） │ +│ 7) DictionaryCorrector（任意） │ +└──────────────────────────────────────────────────────────────┘ +``` + +--- + ## Recording Path Details / 録音パス詳細 実装参照: `src/core/audio/AudioRecorder.ts`, `src/views/VoiceInputViewActions.ts`, `src/views/VoiceInputViewUI.ts` @@ -100,6 +130,47 @@ This document reflects the latest implementation as of 2025‑08‑16. It covers --- +## Language‑by‑Language / 言語別の違いと共通点 + +``` + (選択された言語: ja / en / zh / ko) + │ + ▼ + ┌──────────────────────────────┐ + │ 構造化プロンプトの内容が言語別 │ ← 違い（プロンプト文面） + └──────────────┬───────────────┘ + ▼ +┌──────────────────────────────────────────────────────────────┐ +│ preStripTranscriptWrappers（機械的ラッパー除去） │ ← 共通（言語非依存） +├──────────────────────────────────────────────────────────────┤ +│ StandardCleaningPipeline │ +│ ├─ PromptContaminationCleaner │ +│ │ ・instructionPatterns: 多言語（ja/en/zh/ko） │ ← 共通（多言語対応） +│ │ ・snippet検出: EN/ZH/KO 追加語彙 + JA レガシー保護 │ ← ほぼ共通（内部最適化） +│ └─ UniversalRepetitionCleaner │ ← 共通（言語非依存） +├──────────────────────────────────────────────────────────────┤ +│ isPromptErrorDetected(text, lang) │ +│ ・各言語のエコー特有フレーズで検出 │ ← 違い（検出語句が言語別） +├──────────────────────────────────────────────────────────────┤ +│ DictionaryCorrector（任意） │ ← 共通（言語非依存） +└──────────────────────────────────────────────────────────────┘ +``` + +- 違い（language‑specific） + - プロンプト文面（`buildTranscriptionPrompt`）は ja/en/zh/ko で固有。 + - プロンプトエコー検出（`isPromptErrorDetected`）は各言語の固定句で評価。 + - スニペット検出は EN/ZH/KO の接尾語語彙を追加し、JA にはレガシーパターンを併用。 + +- 共通（language‑agnostic） + - pre-strip（ラッパー除去）は構造ベースで言語に依存しない。 + - クリーニング・パイプラインの安全判定（single/ emergency しきい値）は全言語共通。 + - UniversalRepetitionCleaner と辞書補正は言語非依存ロジック。 + - API パラメータ（multipart、temperature=0 等）は共通（model は選択式）。 + +注意: `auto` は廃止。UI/設定で言語を明示し、該当言語のプロンプトと検出ロジックを適用します。 + +--- + ## Dictionary Processing / 辞書処理 - 有効時、言語非依存で置換を適用。 @@ -136,4 +207,3 @@ KO: "AI시스템" → "artificial intelligence시스템" - 既定は VAD 無効の連続録音。必要に応じて `maxRecordingSeconds` を調整してください。 - 連打・素早い操作でも安定するよう、開始の二重実行は内部ガードで抑止しています。 - 「開始しにくい」挙動を避けるため、UIロックは最小限に留めています。 - From 66110711612091376bbc28e99c74ea813ff0e194 Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 16 Aug 2025 02:43:42 +0900 Subject: [PATCH 67/82] docs(flow): refine ASCII overview and language-specific section; remove obsolete v1/v2 docs --- docs/PROCESSING_FLOW.md | 441 ++++++++++++++++++++++++++++--------- docs/PROCESSING_FLOW_v1.md | 12 - docs/PROCESSING_FLOW_v2.md | 24 -- 3 files changed, 343 insertions(+), 134 deletions(-) delete mode 100644 docs/PROCESSING_FLOW_v1.md delete mode 100644 docs/PROCESSING_FLOW_v2.md diff --git a/docs/PROCESSING_FLOW.md b/docs/PROCESSING_FLOW.md index 489aad1..1b8cf96 100644 --- a/docs/PROCESSING_FLOW.md +++ b/docs/PROCESSING_FLOW.md @@ -1,22 +1,35 @@ -# Voice Input Processing Flow / 音声入力処理フロー +本ドキュメントは最新実装に基づき、設計時の参照用として録音からキュー処理、文字起こし、セーフティ付きクリーニング、辞書補正までの流れ、主要コンポーネント、データ構造、安全装置を体系的に説明します。 -This document reflects the latest implementation as of 2025‑08‑16. It covers the end‑to‑end flow: recording, queuing, transcription, cleaning with safety guards, and optional dictionary correction. +Contents / 目次 +- High‑Level Overview / 全体概要 +- System Architecture & Key Components / アーキテクチャと主要コンポーネント +- End‑to‑End Sequence / エンドツーエンドの逐次フロー +- Recording Path / 録音パス +- Processing Queue / キュー処理 +- Transcription Path / 文字起こしパス +- Cleaning Pipeline & Safety / クリーニングと安全装置 +- Prompt Error Detection / プロンプトエラー検出 +- Dictionary Processing / 辞書処理 +- Error Handling / エラーハンドリング +- Metrics & Logging / 計測とログ +- Configuration & Defaults / 設定とデフォルト +- Performance Notes / パフォーマンス +- Manual Validation Checklist / 手動検証チェックリスト +- Edge Cases & Troubleshooting / エッジケースと対処 +- Extension Points / 拡張ポイント +- Glossary / 用語集 -本ドキュメントは 2025‑08‑16 時点の実装に基づき、録音からキュー処理、文字起こし、セーフティ付きクリーニング、辞書補正までの一連の流れを記載します。 ## High‑Level Overview / 全体概要 1) 音声取得 → 連続録音（最大録音時間到達または手動停止）。 -2) 録音ごとに audioBlob を非同期キューへ投入（録音は継続可能）。 +2) 停止ごとに `audioBlob` を非同期の直列キューへ投入（録音は継続可能）。 3) OpenAI Transcription API 呼び出し（ja/en/zh/ko には構造化プロンプト付与）。 4) 機械的な TRANSCRIPT ラッパー除去 → クリーニング・パイプライン（安全判定付き）。 5) プロンプトエラー検出（プロンプトのみが返った場合は空文字へ）。 6) 任意の辞書補正を適用し、UIへ反映。 ---- - -## ASCII Overview / ASCII概要 - +ASCII overview: ``` ┌──────────────────────────┐ │ AudioRecorder (連続録音) │ @@ -43,92 +56,329 @@ This document reflects the latest implementation as of 2025‑08‑16. It covers └──────────────────────────────────────────────────────────────┘ ``` ---- -## Recording Path Details / 録音パス詳細 +## System Architecture & Key Components / アーキテクチャと主要コンポーネント + +- UI Layer + - `src/views/VoiceInputView.ts` 主ビュー（レイアウト/ライフサイクル） + - `src/views/VoiceInputViewUI.ts` UI生成とイベント結線 + - `src/views/VoiceInputViewActions.ts` 録音・キュー・API・挿入等の操作中枢 + +- Core: Audio + - `src/core/audio/AudioRecorder.ts` 連続録音、AudioWorklet/ScriptProcessor、フィルタ、リングバッファ、可視化連携 + - `src/core/audio/AudioRingBuffer.ts` 音声リングバッファ + - `src/core/audio/AudioVisualizer.ts` 波形/レベル表示（2種） + +- Core: Transcription & Cleaning + - `src/core/transcription/TranscriptionService.ts` API呼び出し、プロンプト付与、クリーニング、辞書補正 + - `src/core/transcription/DictionaryCorrector.ts` 固定置換ベースの辞書補正 + - `src/core/transcription/cleaning/*` クリーナー群と標準パイプライン + +- Config & Defaults + - `src/config/constants.ts`, `defaults.ts`, `costs.ts`, `CleaningConfig.ts` + +- Errors & Services + - `src/errors/*` エラー型・通知 + - `src/services/*` ServiceLocator, I18n, HTTPクライアント + +- Managers + - `src/managers/DraftManager.ts` 下書き保存/復元 + - `src/managers/ViewManager.ts` ビュー管理 + + +## End‑to‑End Sequence / エンドツーエンドの逐次フロー + +1. ユーザーが Record ボタン操作 + - `VoiceInputViewUI` が `VoiceInputViewActions.toggleRecording()` を呼ぶ +2. 録音開始 + - `AudioRecorder.initialize()` → WebAudioセットアップ（Gain/HPF/LPF/Analyser） + - AudioWorklet利用可能なら `audio-processor-worklet`、不可なら ScriptProcessor + - `MediaRecorder.start(timeslice=100ms)` で WebM/Opus を分割収集 +3. 最大時間到達 or 手動停止 + - `AudioRecorder.stopRecording()` → `Blob`（audio/webm）生成 → `onSpeechEnd` 経由で通知 +4. キュー投入 + - `VoiceInputViewActions.processRecordedAudio()` が `processingQueue` に追加 + - バックグラウンドで直列処理 `processQueue()` 実行 +5. 文字起こし + - `TranscriptionService.transcribeAudio()` が `FormData` を構築し API へ送信 + - 応答 `text` を取得 → `cleanGPT4oResponse()` + - `preStripTranscriptWrappers()` → `StandardCleaningPipeline.execute()` → `isPromptErrorDetected()` → `DictionaryCorrector.correct()` +6. UI反映 + - `VoiceInputViewUI.textArea` に追記、下書き保存、ステータス更新 + + +## Recording Path / 録音パス 実装参照: `src/core/audio/AudioRecorder.ts`, `src/views/VoiceInputViewActions.ts`, `src/views/VoiceInputViewUI.ts` - AudioContext 初期化 - - `window.AudioContext`（Safari は `webkitAudioContext`）を使用。 - - `AudioWorklet` が利用可能ならワークレット、不可なら `ScriptProcessor` へフォールバック。 - - `suspended` 状態の場合は `startRecording()` で `resume()` を実行。 -- フィルタ/ノード構成 - - `MediaStreamSource → Gain → HighPass(約80Hz) → LowPass(約7.6kHz) → Analyser` - - ビジュアライザは `Analyser` に接続（シンプル/通常の2種）。 + - `window.AudioContext`（Safari は `webkitAudioContext`） + - `latencyHint='interactive'`, `sampleRate=16000` + - `AudioWorklet` 利用試行 → 失敗時は `ScriptProcessor` へフォールバック +- ノード構成 + - `MediaStreamSource → Gain → HighPass(80Hz) → LowPass(7.6kHz) → Analyser` + - 可視化: `AudioVisualizer` or `SimpleAudioLevelIndicator` - 連続処理 - - ワークレット/スクリプトプロセッサで 1ch PCM を取り出し、`AudioRingBuffer` に蓄積。 - - マイク実データ検知で `onMicrophoneStatusChange('ready')` 通知。 + - 1ch PCM をリングバッファへ蓄積（`AudioRingBuffer`） + - マイク有効データの検出で `onMicrophoneStatusChange('ready')` - 録音制御 - - 既定は VAD 無効の連続録音。`maxRecordingSeconds` 到達で自動停止。 - - 二重開始は `AudioRecorder.isStarting` で抑止。UI側のロックはエントリポイントのみで軽量化。 + - 既定は VAD 無効の連続録音（`useVAD=false`） + - `maxRecordingSeconds` 到達で自動停止 + - 二重開始は `isStarting` ガードで抑止 - UI/操作 - - 通常クリックで開始/停止。プッシュトゥトークは長押しで開始、離して停止（`UI_CONSTANTS.PUSH_TO_TALK_THRESHOLD`）。 + - クリック開始/停止 + 長押し Push‑to‑Talk（閾値: `UI_CONSTANTS.PUSH_TO_TALK_THRESHOLD`） ---- -## Queue & UI / キューとUI +State model（簡略）: +- idle → initializing → recording → stopping → idle +- 例外時は error 表示後に idle 復帰 + + +## Processing Queue / キュー処理 実装参照: `src/views/VoiceInputViewActions.ts` -- 停止ごとに生成された `audioBlob` を `processingQueue` へ追加し、逐次処理。 -- ステータス表示は「録音中/処理中/待機数」を反映。キャンセルは録音を中断して音声を破棄。 -- 途中で録音を継続しながら、既存キューをバックグラウンドで順次文字起こし可能。 +- データ構造 + - `RecordingState.processingQueue: Array<{ audioBlob, timestamp, stopReason }>` +- 挙動 + - `processRecordedAudio()` がキューに追加し、`processQueue()` で直列処理 + - 録音継続中でもキュー処理はバックグラウンドで進行 +- ステータス + - 処理中/待機数を `statusEl` に表示（例: "Transcribing... (2 waiting)"） +- キャンセル + - 録音中断は `cancelRecording()`（収集済み音声は破棄） ---- ## Transcription Path / 文字起こしパス 実装参照: `src/core/transcription/TranscriptionService.ts` -1) リクエスト作成（`multipart/form-data`） - - `file: audio.webm(Blob)`, `model: gpt-4o(-mini)-transcribe`, `response_format: json`, `temperature: 0`, `language`。 - - 言語に応じて構造化プロンプトを付与（ja/en/zh/ko）。 -2) API レスポンスから `text` を取得。 -3) `cleanGPT4oResponse(text, language)` を実行。 - - 先に機械的に TRANSCRIPT ラッパーを剥離（完全/不完全タグ双方に対応）。 - - クリーニング・パイプラインを実行（詳細は後述）。 -4) プロンプトエラー検出 - - 言語別の検出ルールにより、プロンプトや注釈のみが返るケースを検知。 - - 検出時は空文字に置換して早期終了（無音/極短音声で発生しやすい）。 -5) 辞書補正（有効時） - - `DictionaryCorrector` により語彙の統一・置換を実施。 +1) リクエスト作成（multipart/form-data） + - fields: + - `file: audio.webm (Blob)` + - `model: 'gpt-4o-transcribe' | 'gpt-4o-mini-transcribe'` + - `response_format: 'json'` + - `temperature: 0` + - `language: 'ja'|'en'|'zh'|'ko'`（auto は廃止） + - `prompt: 構造化プロンプト（ja/en/zh/ko 対応）` + - 送信先: `API_CONSTANTS.ENDPOINTS.TRANSCRIPTION` + - 送信手段: `ObsidianHttpClient.postFormData()`（独自 boundary 付与） ---- +2) レスポンス処理 + - `response.json.text` を取得（なければ `text` 文字列も許容） + - `cleanGPT4oResponse(text, language)` 実行 + - `preStripTranscriptWrappers()` で `...` 抽出/除去 + - `StandardCleaningPipeline` 実行 + - `isPromptErrorDetected(text, lang)` に該当すれば空文字で早期終了 + - 有効時 `DictionaryCorrector` 適用 -## Cleaning Pipeline / クリーニング・パイプライン +3) エラー分類 + - 401 → `INVALID_API_KEY` + - 429 → `API_QUOTA_EXCEEDED` + - 5xx → `NETWORK_ERROR` + - それ以外 → `TRANSCRIPTION_FAILED` -実装参照: `src/core/transcription/cleaning/StandardCleaningPipeline.ts` +サンプル: 成功時の最小レスポンス（概念例） +``` +{ "text": "こんにちは。今日の予定は..." , "language": "ja" } +``` -- 構成 + +## Cleaning Pipeline & Safety / クリーニングと安全装置 + +実装参照: +- `src/core/transcription/cleaning/StandardCleaningPipeline.ts` +- `src/core/transcription/cleaning/PromptContaminationCleaner.ts` +- `src/core/transcription/cleaning/UniversalRepetitionCleaner.ts` +- `src/config/CleaningConfig.ts` + +構成: +- 前処理（言語非依存） + - `preStripTranscriptWrappers()`: + - `...` 完全/不完全タグから中身抽出 + - 残存タグ除去（TRANSCRIPT/transcription/transcription系） +- パイプライン（順次実行） 1) `PromptContaminationCleaner` - - TRANSCRIPT/TRANSCRIPTION タグ残留、指示文（完全一致/スニペット/文脈）、フォーマット行の除去。 - - 空行/空白の正規化。 + - 指示文・フォーマット行・スニペット・XML/タグ・「話者のみ」等の構造物を除去 + - 多言語 instructionPatterns と snippet 検出（EN/ZH/KO 語彙 + JA レガシー） 2) `UniversalRepetitionCleaner` - - 文字/文/列挙/段落レベルの反復抑制、末尾ハルシネーション緩和、体裁整形。 + - 文字/トークン/文/段落/列挙/末尾反復の抑制と体裁整形（言語非依存） -- セーフティ（安全判定） - - 設定値（`src/config/CleaningConfig.ts`） - - `singleCleanerMaxReduction`: 0.3（単一クリーナーの削減率上限） - - `emergencyFallbackThreshold`: 0.5（緊急ロールバック閾値） - - `warningThreshold`: 0.15（警告ログ） - - 構造的クリーナー緩和（`PromptContaminationCleaner` に適用） - - `singleCleanerMaxReduction → max(0.9, 0.3) = 0.9` - - `emergencyFallbackThreshold → max(0.95, 0.5) = 0.95` - - 超過時の動作 - - 単一上限（single）超過 → `skip`（そのクリーナーの変更を適用しない） - - 緊急（emergency）超過 → `rollback`（そのクリーナーの結果を破棄して元のテキストに戻す） +セーフティ（安全判定）: +- 設定（`CLEANING_CONFIG.safety`） + - `singleCleanerMaxReduction: 0.3` + - `emergencyFallbackThreshold: 0.5` + - `warningThreshold: 0.15` +- 構造的クリーナー緩和（PromptContaminationCleaner に適用） + - 単一上限: `max(0.9, 0.3) = 0.9` + - 緊急閾値: `max(0.95, 0.5) = 0.95` +- 超過時の動作 + - Single 超過 → `skip`（当該クリーナーの変更を適用しない） + - Emergency 超過 → `rollback`（元テキストへ差し戻し） +- パイプライン全体の最終セーフティも実行（全体削減率が異常なら原文へ戻す） -例（ログ）: +ログ例: ``` [Voice Transcription] [StandardCleaningPipeline] Rolling back cleaner PromptContaminationCleaner { reason: 'Reduction ratio 1.000 exceeds emergency threshold 0.95', reductionRatio: 1 } ``` -意味: 構造的クリーナーが 95% 超の削減を行おうとしたため、安全装置によりロールバックされました。上流で「実発話が少ない/無い」状態（無音、極短、プロンプトエコー等）が疑われます。 +意味: 上流で「実発話が少ない/無い」（無音/極短/プロンプトエコー）可能性が高い。 + + +実例（Before → After 概念例）: +- Before: + ``` + Output format: + + （話者の発言のみ） + 今日は晴れです。今日は晴れです。今日は晴れです。。。。。 + + ``` +- After: + ``` + 今日は晴れです。 + ``` + + +## Prompt Error Detection / プロンプトエラー検出 + +実装参照: `TranscriptionService.isPromptErrorDetected()` + +- 言語別の固定句や「SPEAKER ONLY」等がそのまま返るケースを検知 +- 検出時は空文字へ置換し早期終了（UI は「No audio detected」等を表示） + +注意: +- `auto` は廃止。UI/設定で明示言語（ja/en/zh/ko）を選択し、該当のプロンプトと検出ロジックを適用 + + +## Dictionary Processing / 辞書処理 + +実装参照: `src/core/transcription/DictionaryCorrector.ts`, `src/interfaces/transcription.ts` + +- 固定置換（definiteCorrections）のみをサポート +- 設定値・制限（`DICTIONARY_CONSTANTS`） + - 最大個数（固定補正）: 200 +- 多言語に同一置換を適用（言語非依存） + +例: +``` +{ from: ["AI"], to: "artificial intelligence" } + +EN: "AI system" → "artificial intelligence system" +JA: "AIシステム" → "artificial intelligenceシステム" +ZH: "AI系统" → "artificial intelligence系统" +KO: "AI시스템" → "artificial intelligence시스템" +``` + +UI（日本語UI時のみ）で表形式の編集/インポート/エクスポートに対応。 + + +## Error Handling / エラーハンドリング + +実装参照: `src/errors/*`, `src/services/*` + +- エラー型: `TranscriptionErrorType` + - PERMISSION_DENIED, NETWORK_ERROR, API_QUOTA_EXCEEDED, INVALID_API_KEY, AUDIO_* 他 +- 通知: `Notice` によるユーザー表示（i18n 対応） +- グローバル: `ErrorHandler` が未処理例外/Promise拒否を収集しログ化 +- API ステータスからの分類: + - 401 → INVALID_API_KEY + - 429 → API_QUOTA_EXCEEDED + - 5xx → NETWORK_ERROR + - その他 → TRANSCRIPTION_FAILED +- 無音/極短データ: 早期終了で空文字返却 + UI で「音声が検出されませんでした」 + + +## Metrics & Logging / 計測とログ + +実装参照: `src/utils/Logger.ts` + +- 共通プリフィクス: `[Voice Transcription]` +- 主要ログポイント + - 録音開始/終了、サンプルレート、チャンク数、所要時間 + - リングバッファ使用率（確率的に警告） + - API呼び出しモデル/言語/入力サイズ/出力長/時間 + - クリーニング各段の削減率/処理時間/セーフティ判定 +- ログレベルは設定に従い、開発モードで詳細化 + + +## Configuration & Defaults / 設定とデフォルト + +- 定数: `src/config/constants.ts` + - Audio: SAMPLE_RATE=16000, BUFFER_SIZE=4096, MAX_RECORDING_SECONDS=300, FILTERS(HPF=80Hz/LPF=7.6kHz), WEBM_CODEC, RECORDER_TIMESLICE=100ms + - API: `TRANSCRIPTION` エンドポイント、temperature=0 + - UI: Push‑to‑Talk 閾値=300ms ほか +- デフォルト: `src/config/defaults.ts` + - 文字起こしモデル: `gpt-4o-transcribe` + - 補正有効化: true + - VAD デフォルトは存在するが録音は既定で VAD 無効の連続録音 +- コスト: `src/config/costs.ts` + - 参考: 4o‑transcribe=$0.06/min, 4o‑mini‑transcribe=$0.03/min + + +## Performance Notes / パフォーマンス + +- WebAudio + - `latencyHint='interactive'` で起動体感改善 + - HPF/LPF により音声帯域へ制限し雑音影響を軽減 +- MediaRecorder + - 100ms timeslice で小刻み収集し停止時の結合負荷を低減 +- クリーニング + - 早期のラッパー剥離で後段の削減率を安定化 + - セーフティにより過剰削除を抑制（skip/rollback） +- UI/操作感 + - プッシュトゥトークは長押し→離すで録音区間を素早く収集 + - キュー直列処理により安定した逐次反映 + + +## Manual Validation Checklist / 手動検証チェックリスト + +ビルド/検証（`public/.github/copilot-instructions.md` 準拠）: +- Build Success + - `npm run build` 後、`build/latest/main.js` が生成される（~128KB） +- Lint Compliance + - `npm run lint`（~10 warnings, 0 errors 目安） +- Tests + - `npm run test`（182 近辺のテスト、時間 ~17s） +- Packaging + - `npm run build-plugin` 後、`build/latest/` に `main.js/manifest.json/styles.css/fvad.wasm/fvad.js` +- WASM 配布確認 + - `src/lib/fvad-wasm/` の `fvad.wasm` と `fvad.js` が出力に含まれる + +動作確認: +- 文字起こし成功（言語=設定値、モデル=設定値） +- 無音/極短音声で空結果を返し UI が警告 +- クリーニングの安全装置ログが想定通り（過剰削減は skip/rollback） +- 下書きの自動保存/復元が機能 + + +## Edge Cases & Troubleshooting / エッジケースと対処 + +- マイクトラックが muted で始まる（Windows 等） + - `AudioRecorder` が mute/unmute を監視し、音声検出時に `ready` 通知 +- 極小 `audioBlob`（~<500B） + - ヘッダのみの可能性 → 早期終了 + UI 警告 +- プロンプトエコー（モデルがプロンプトを返す） + - `isPromptErrorDetected()` が空文字へ置換 +- TRANSCRIPT タグしか無い/内容が空 + - `PromptContaminationCleaner` による削除でほぼ全削除 → セーフティで rollback +- Obsidian Deferred Views（1.7.2+） + - `DeferredViewHelper` で安全に leaf/view を扱う + + +## Extension Points / 拡張ポイント + +- クリーナーの追加 + - `StandardCleaningPipeline.addCleaner(new MyCleaner())` + - セーフティ閾値は `CLEANING_CONFIG` に集約 +- 置換辞書の拡張 + - UI から `definiteCorrections` を編集/Import/Export + - プログラムからは `TranscriptionService.setCustomDictionary()` +- モデル切替 + - UI（設定パネル）で 4o / 4o‑mini を動的に切替 ---- ## Language‑by‑Language / 言語別の違いと共通点 @@ -157,53 +407,48 @@ This document reflects the latest implementation as of 2025‑08‑16. It covers ``` - 違い（language‑specific） - - プロンプト文面（`buildTranscriptionPrompt`）は ja/en/zh/ko で固有。 - - プロンプトエコー検出（`isPromptErrorDetected`）は各言語の固定句で評価。 - - スニペット検出は EN/ZH/KO の接尾語語彙を追加し、JA にはレガシーパターンを併用。 - + - プロンプト文面（`buildTranscriptionPrompt`） + - プロンプトエコー検出（`isPromptErrorDetected`） - 共通（language‑agnostic） - - pre-strip（ラッパー除去）は構造ベースで言語に依存しない。 - - クリーニング・パイプラインの安全判定（single/ emergency しきい値）は全言語共通。 - - UniversalRepetitionCleaner と辞書補正は言語非依存ロジック。 - - API パラメータ（multipart、temperature=0 等）は共通（model は選択式）。 + - TRANSCRIPT ラッパー除去 + - セーフティ判定と反復抑制 + - 辞書補正 -注意: `auto` は廃止。UI/設定で言語を明示し、該当言語のプロンプトと検出ロジックを適用します。 ---- +## API Contracts & Data Shapes / API とデータ構造（要点） -## Dictionary Processing / 辞書処理 +Request (multipart/form-data fields): +- file: Blob (audio/webm; codecs=opus) +- model: 'gpt-4o-transcribe' | 'gpt-4o-mini-transcribe' +- language: 'ja'|'en'|'zh'|'ko' +- response_format: 'json' +- temperature: '0' +- prompt: string (言語別の構造化プロンプト) -- 有効時、言語非依存で置換を適用。 -- 例: -``` -{ from: ["AI"], to: "artificial intelligence" } +Response (主要): +- text: string +- language: string（オプション。なければ UI 設定言語を採用） -EN: "AI system" → "artificial intelligence system" -JA: "AIシステム" → "artificial intelligenceシステム" -ZH: "AI系统" → "artificial intelligence系统" -KO: "AI시스템" → "artificial intelligence시스템" -``` ---- +## Examples & Logs / 例とログ -## Status & Metrics / ステータスと計測 +- クリーニング高削減率の警告: + - `High reduction ratio in PromptContaminationCleaner reductionRatio=0.220 threshold=0.15` +- リングバッファ警告: + - `Audio buffer usage high: 82.4%` -- ステータス: 録音準備/録音中/処理中（待機数付き）/完了/エラー。 -- メトリクス: 音声サイズ、処理時間、原文/補正後の長さ、モデル、出力長などをログ出力。 - ---- - -## Error Handling / エラーハンドリング - -1) API エラー: 401（キー不正）、429（クォータ超過）、5xx（ネットワーク/サーバ） -2) プロンプト漏洩: 言語別検出 → 空文字化で軽減 -3) 空/無音: 早期終了で空文字返却、UIに短すぎ警告 -4) クリーニング安全装置: 過剰削減を検知してスキップ/ロールバック - ---- ## Notes / 補足 -- 既定は VAD 無効の連続録音。必要に応じて `maxRecordingSeconds` を調整してください。 -- 連打・素早い操作でも安定するよう、開始の二重実行は内部ガードで抑止しています。 -- 「開始しにくい」挙動を避けるため、UIロックは最小限に留めています。 +- 既定は VAD 無効の連続録音。必要に応じて `maxRecordingSeconds` を調整 +- 二重開始抑止や UI ロックは最小限にし、操作性を重視 +- Obsidian 環境では `requestUrl` を使用し CORS を回避 +- セキュアコンテキスト判定はブラウザではなく Obsidian の前提でスキップ + + +## Glossary / 用語集 + +- TRANSCRIPT ラッパー: ` ... ` のようなXML風タグ。モデル出力の安定化のためプロンプトで使用 +- セーフティ（Safety）: クリーニングでの過剰削除を防ぐ skip/rollback のしきい値判定 +- プロンプトエコー: モデルが指示文やフォーマット見出しをそのまま出力してしまう現象 +- 反復抑制: 文やトークンの過剰反復を抑える整形ロジック diff --git a/docs/PROCESSING_FLOW_v1.md b/docs/PROCESSING_FLOW_v1.md deleted file mode 100644 index 9213681..0000000 --- a/docs/PROCESSING_FLOW_v1.md +++ /dev/null @@ -1,12 +0,0 @@ -# Voice Input Processing Flow / 音声入力処理フロー（Legacy v1） - -このドキュメントは 2025-08-15 の更新（TRANSCRIPTタグの事前除去と新クリーニングパイプラインの安全判定見直し）以前の内容を保存したスナップショットです。最新の処理フローは `docs/PROCESSING_FLOW.md` を参照してください。 - -以下、当時の処理フローをそのまま記載しています。 - -```note -本ファイルはレガシー仕様の参考用です。最新実装とは一部相違があります。 -``` - -（このバージョンの全文は旧 `docs/PROCESSING_FLOW.md` と同一でした） - diff --git a/docs/PROCESSING_FLOW_v2.md b/docs/PROCESSING_FLOW_v2.md deleted file mode 100644 index 8668925..0000000 --- a/docs/PROCESSING_FLOW_v2.md +++ /dev/null @@ -1,24 +0,0 @@ -# Voice Input Processing Flow v2 / 音声入力処理フロー v2 - -最終更新: 2025-08-15 - -本バージョンは、TRANSCRIPTタグの事前除去と新しいクリーニング・パイプライン（安全装置付き）を前提とした最新の処理フローをまとめています。詳細版は `docs/PROCESSING_FLOW.md` を参照してください。 - -## フロー概要 - -1) 音声 → OpenAI Transcription API（日本語のみプロンプト付与）。 -2) APIレスポンス `text` を受領。 -3) 事前処理: ` ... ` を機械的に抽出・除去（閉じタグ欠落にも対応）。 -4) クリーニング・パイプライン（`StandardCleaningPipeline`） - - `PromptContaminationCleaner`: 指示文/タグ/文脈/スニペット除去。 - - `UniversalRepetitionCleaner`: 反復抑制（文字/トークン/文/列挙/段落/末尾）＋整形。 - - セーフティ: 削減率上限・緊急ロールバック。構造除去は誤ロールバック防止のため緩和。 -5) プロンプトエラー検出 → 該当時は空文字で早期終了。 -6) `DictionaryCorrector` による辞書補正（任意）。 -7) 最終テキストを返却。 - -## 補足 - -- 事前処理により、タグの有無に影響されずにパイプラインの安全判定が機能します。 -- 言語は日本語以外でも有効（クリーナーは言語非依存の設計）。 - From e541da7756915776ed5323fdb7971f2942a8606b Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 16 Aug 2025 03:37:24 +0900 Subject: [PATCH 68/82] feat(settings): enable dictionary table editor + import/export for all UI languages\ndocs: reflect UI availability across languages --- docs/PROCESSING_FLOW.md | 2 +- src/settings/VoiceInputSettingTab.ts | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/PROCESSING_FLOW.md b/docs/PROCESSING_FLOW.md index 1b8cf96..5d8be1f 100644 --- a/docs/PROCESSING_FLOW.md +++ b/docs/PROCESSING_FLOW.md @@ -272,7 +272,7 @@ ZH: "AI系统" → "artificial intelligence系统" KO: "AI시스템" → "artificial intelligence시스템" ``` -UI（日本語UI時のみ）で表形式の編集/インポート/エクスポートに対応。 +UIで表形式の編集/インポート/エクスポートに対応（全言語UIで利用可能）。 ## Error Handling / エラーハンドリング diff --git a/src/settings/VoiceInputSettingTab.ts b/src/settings/VoiceInputSettingTab.ts index 87d02ec..3399338 100644 --- a/src/settings/VoiceInputSettingTab.ts +++ b/src/settings/VoiceInputSettingTab.ts @@ -260,8 +260,7 @@ export class VoiceInputSettingTab extends PluginSettingTab { this.display(); // Refresh UI })); - // Dictionary (Unified table editor) - Only show for Japanese - if (this.plugin.settings.pluginLanguage === 'ja') { + // Dictionary (Unified table editor) - Show for all languages new Setting(containerEl) .setName(this.i18n.t('ui.settings.customDictionary')) .setDesc(this.i18n.t('ui.settings.customDictionaryDesc')); @@ -287,7 +286,7 @@ export class VoiceInputSettingTab extends PluginSettingTab { .addButton(button => button .setButtonText(this.i18n.t('ui.buttons.import')) .onClick(() => this.importDictionary())); - } + // Debug Settings Section - Removed as per ai-transcriber pattern // containerEl.createEl('h3', { text: this.i18n.t('ui.titles.debugSettings') || 'Debug Settings' }); From 5407570fb1ad15f2d4a245bdcf1608f554df01ea Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 16 Aug 2025 03:44:19 +0900 Subject: [PATCH 69/82] style(settings): fix indentation after enabling dictionary UI for all locales --- src/settings/VoiceInputSettingTab.ts | 45 ++++++++++++++-------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/src/settings/VoiceInputSettingTab.ts b/src/settings/VoiceInputSettingTab.ts index 3399338..0935bac 100644 --- a/src/settings/VoiceInputSettingTab.ts +++ b/src/settings/VoiceInputSettingTab.ts @@ -261,32 +261,31 @@ export class VoiceInputSettingTab extends PluginSettingTab { })); // Dictionary (Unified table editor) - Show for all languages - new Setting(containerEl) - .setName(this.i18n.t('ui.settings.customDictionary')) - .setDesc(this.i18n.t('ui.settings.customDictionaryDesc')); + new Setting(containerEl) + .setName(this.i18n.t('ui.settings.customDictionary')) + .setDesc(this.i18n.t('ui.settings.customDictionaryDesc')); - // Create table container for dictionary tables - const tableContainer = containerEl.createDiv('dictionary-table-container'); + // Create table container for dictionary tables + const tableContainer = containerEl.createDiv('dictionary-table-container'); - // Definite Corrections Section - tableContainer.createEl('h4', { text: this.i18n.t('ui.settings.dictionaryDefinite', { max: DICTIONARY_CONSTANTS.MAX_DEFINITE_CORRECTIONS }) }); - this.createCorrectionTable( - tableContainer, - this.plugin.settings.customDictionary.definiteCorrections, - false - ); + // Definite Corrections Section + tableContainer.createEl('h4', { text: this.i18n.t('ui.settings.dictionaryDefinite', { max: DICTIONARY_CONSTANTS.MAX_DEFINITE_CORRECTIONS }) }); + this.createCorrectionTable( + tableContainer, + this.plugin.settings.customDictionary.definiteCorrections, + false + ); - // Import/Export buttons - new Setting(containerEl) - .setName(this.i18n.t('ui.settings.dictionaryImportExport')) - .setDesc(this.i18n.t('ui.settings.dictionaryImportExportDesc')) - .addButton(button => button - .setButtonText(this.i18n.t('ui.buttons.export')) - .onClick(() => this.exportDictionary())) - .addButton(button => button - .setButtonText(this.i18n.t('ui.buttons.import')) - .onClick(() => this.importDictionary())); - + // Import/Export buttons + new Setting(containerEl) + .setName(this.i18n.t('ui.settings.dictionaryImportExport')) + .setDesc(this.i18n.t('ui.settings.dictionaryImportExportDesc')) + .addButton(button => button + .setButtonText(this.i18n.t('ui.buttons.export')) + .onClick(() => this.exportDictionary())) + .addButton(button => button + .setButtonText(this.i18n.t('ui.buttons.import')) + .onClick(() => this.importDictionary())); // Debug Settings Section - Removed as per ai-transcriber pattern // containerEl.createEl('h3', { text: this.i18n.t('ui.titles.debugSettings') || 'Debug Settings' }); From 51959d4a939bbd5d387188c98cd8f7f39f8fc0ee Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Tue, 19 Aug 2025 02:49:34 +0900 Subject: [PATCH 70/82] Update manifest.json --- manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest.json b/manifest.json index 5de84bc..117c2a0 100644 --- a/manifest.json +++ b/manifest.json @@ -2,7 +2,7 @@ "id": "voice-input", "name": "Voice Input", "version": "0.8.0", - "minAppVersion": "17.0.0", + "minAppVersion": "1.7.0", "description": "Voice input with real-time transcription using GPT-4o.", "author": "Musashino Software", "authorUrl": "https://github.com/mssoftjp", From f4631b840c36dd954cf675f809c2a406a82c54c8 Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 30 Aug 2025 00:16:38 +0900 Subject: [PATCH 71/82] fix(i18n,commands): add ui.commands.openView and use for command label to avoid duplicate palette text --- src/i18n/en.ts | 3 +++ src/i18n/ja.ts | 3 +++ src/i18n/ko.ts | 3 +++ src/i18n/zh.ts | 3 +++ src/interfaces/i18n.ts | 3 +++ src/plugin/VoiceInputPlugin.ts | 2 +- 6 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/i18n/en.ts b/src/i18n/en.ts index 471786c..31265e0 100644 --- a/src/i18n/en.ts +++ b/src/i18n/en.ts @@ -113,6 +113,9 @@ export const en: TranslationResource = { } }, ui: { + commands: { + openView: 'Open view' + }, buttons: { recordStart: 'Start Voice Input', recordStop: 'Stop Voice Input', diff --git a/src/i18n/ja.ts b/src/i18n/ja.ts index 4737966..3627b18 100644 --- a/src/i18n/ja.ts +++ b/src/i18n/ja.ts @@ -113,6 +113,9 @@ export const ja: TranslationResource = { } }, ui: { + commands: { + openView: 'ビューを開く' + }, buttons: { recordStart: '音声入力開始', recordStop: '音声入力停止', diff --git a/src/i18n/ko.ts b/src/i18n/ko.ts index e849046..ef62fe0 100644 --- a/src/i18n/ko.ts +++ b/src/i18n/ko.ts @@ -113,6 +113,9 @@ export const ko: TranslationResource = { } }, ui: { + commands: { + openView: '뷰 열기' + }, buttons: { recordStart: '음성 입력 시작', recordStop: '음성 입력 중지', diff --git a/src/i18n/zh.ts b/src/i18n/zh.ts index 51af72e..dd9a88f 100644 --- a/src/i18n/zh.ts +++ b/src/i18n/zh.ts @@ -113,6 +113,9 @@ export const zh: TranslationResource = { } }, ui: { + commands: { + openView: '打开视图' + }, buttons: { recordStart: '开始语音输入', recordStop: '停止语音输入', diff --git a/src/interfaces/i18n.ts b/src/interfaces/i18n.ts index abf8b1d..f2e3cf9 100644 --- a/src/interfaces/i18n.ts +++ b/src/interfaces/i18n.ts @@ -132,6 +132,9 @@ export type TranslationResource = { }; }; ui: { + commands: { + openView: string; + }; buttons: { recordStart: string; recordStop: string; diff --git a/src/plugin/VoiceInputPlugin.ts b/src/plugin/VoiceInputPlugin.ts index 54ee04d..38157c8 100644 --- a/src/plugin/VoiceInputPlugin.ts +++ b/src/plugin/VoiceInputPlugin.ts @@ -110,7 +110,7 @@ export default class VoiceInputPlugin extends Plugin { // Add command this.addCommand({ id: 'open-voice-input', - name: i18n.t('ui.titles.main'), + name: i18n.t('ui.commands.openView'), callback: () => { this.logger.debug('Command executed: open-voice-input'); this.viewManager.activateVoiceInputView(); From c88137b33586548ecdb58325e4eedacf81bc8be6 Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 30 Aug 2025 00:16:43 +0900 Subject: [PATCH 72/82] fix(css): scope .setting-item-description under .voice-input-settings-panel to prevent global overrides --- src/styles/voice-input.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/styles/voice-input.css b/src/styles/voice-input.css index 3c44b9c..2c644b3 100644 --- a/src/styles/voice-input.css +++ b/src/styles/voice-input.css @@ -242,7 +242,7 @@ font-size: 14px; } -.setting-item-description { +.voice-input-settings-panel .setting-item-description { padding: 8px 0; font-size: 13px; color: var(--text-muted); @@ -452,4 +452,4 @@ .voice-input-textarea-wide { width: 100% !important; min-height: 100px !important; -} \ No newline at end of file +} From c57d93eceb1c8c8dbf1adcb154dd98c3ec686e6e Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 30 Aug 2025 00:16:47 +0900 Subject: [PATCH 73/82] docs(readme): clarify privacy (audio sent to OpenAI via requestUrl) and SafeStorage fallback --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e73e7d7..210d29c 100644 --- a/README.md +++ b/README.md @@ -58,9 +58,11 @@ Tip: A settings gear in the view header opens the plugin’s settings. ## Security & Privacy - Processing in memory; audio is not written to disk by the plugin -- HTTPS for all network requests (via Obsidian’s `requestUrl`) +- Audio you record is transmitted to OpenAI for transcription over HTTPS (via Obsidian’s `requestUrl`). - API key is encrypted for storage +Note: When Electron SafeStorage is unavailable, the plugin falls back to lightweight obfuscation for the stored key; the plugin is desktop‑only by design. + See also OpenAI’s Privacy Policy. ## Troubleshooting @@ -162,9 +164,11 @@ Third‑party licensing: see `THIRD_PARTY_LICENSES.md`. ## セキュリティ / プライバシー - 処理はメモリ内で行い、音声ファイルはプラグイン側でディスク保存しません -- 通信はHTTPS（Obsidianの `requestUrl` 経由） +- 録音した音声は文字起こしのため OpenAI に送信され、HTTPS（Obsidian の `requestUrl` 経由）で通信します。 - APIキーは保存時に暗号化 +補足: Electron の SafeStorage が利用できない環境では保存キーを軽度に難読化して保持します（本プラグインはデスクトップ専用の設計です）。 + OpenAIのプライバシーポリシーもご参照ください。 ## トラブルシューティング From d8e64683119c2c16135154f56cee373b73b83db6 Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 30 Aug 2025 00:22:27 +0900 Subject: [PATCH 74/82] style(css): replace fixed cancel colors with theme vars for better theme compatibility --- src/styles/voice-input.css | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/styles/voice-input.css b/src/styles/voice-input.css index 2c644b3..5d9d55b 100644 --- a/src/styles/voice-input.css +++ b/src/styles/voice-input.css @@ -117,9 +117,9 @@ padding: 54px 12px; font-size: 20px; font-weight: 600; - background: #f9e79f; /* Soft yellow */ - color: var(--text-normal); - border: 1px solid #f4d03f; + background: var(--background-secondary); + color: var(--text-normal); + border: 1px solid var(--background-modifier-border); border-radius: 4px; cursor: pointer; transition: background-color 0.2s; @@ -130,7 +130,7 @@ } .voice-input-cancel-button-full:hover { - background: #f4d03f; /* Slightly darker yellow on hover */ + background: var(--background-modifier-hover); } From 503cb6e358346d17157250267c40321a2b934dd9 Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 30 Aug 2025 00:41:52 +0900 Subject: [PATCH 75/82] feat(cleaning): strengthen multilingual prompt removal (JA/EN/ZH/KO), head-limited snippet detection, and config-driven context patterns --- docs/PROCESSING_FLOW.md | 454 ------------- docs/design-decisions-summary.md | 122 ---- docs/implementation-roadmap.md | 168 ----- ...leaning-pipeline-repetition-suppression.md | 633 ------------------ docs/language-specific-dictionaries-design.md | 289 -------- .../cleaning/PromptContaminationCleaner.ts | 137 +++- .../cleaning/StandardCleaningPipeline.ts | 2 +- .../cleaning/UniversalRepetitionCleaner.ts | 50 +- 8 files changed, 150 insertions(+), 1705 deletions(-) delete mode 100644 docs/PROCESSING_FLOW.md delete mode 100644 docs/design-decisions-summary.md delete mode 100644 docs/implementation-roadmap.md delete mode 100644 docs/issues/issue-cleaning-pipeline-repetition-suppression.md delete mode 100644 docs/language-specific-dictionaries-design.md diff --git a/docs/PROCESSING_FLOW.md b/docs/PROCESSING_FLOW.md deleted file mode 100644 index 5d8be1f..0000000 --- a/docs/PROCESSING_FLOW.md +++ /dev/null @@ -1,454 +0,0 @@ -本ドキュメントは最新実装に基づき、設計時の参照用として録音からキュー処理、文字起こし、セーフティ付きクリーニング、辞書補正までの流れ、主要コンポーネント、データ構造、安全装置を体系的に説明します。 - -Contents / 目次 -- High‑Level Overview / 全体概要 -- System Architecture & Key Components / アーキテクチャと主要コンポーネント -- End‑to‑End Sequence / エンドツーエンドの逐次フロー -- Recording Path / 録音パス -- Processing Queue / キュー処理 -- Transcription Path / 文字起こしパス -- Cleaning Pipeline & Safety / クリーニングと安全装置 -- Prompt Error Detection / プロンプトエラー検出 -- Dictionary Processing / 辞書処理 -- Error Handling / エラーハンドリング -- Metrics & Logging / 計測とログ -- Configuration & Defaults / 設定とデフォルト -- Performance Notes / パフォーマンス -- Manual Validation Checklist / 手動検証チェックリスト -- Edge Cases & Troubleshooting / エッジケースと対処 -- Extension Points / 拡張ポイント -- Glossary / 用語集 - - -## High‑Level Overview / 全体概要 - -1) 音声取得 → 連続録音（最大録音時間到達または手動停止）。 -2) 停止ごとに `audioBlob` を非同期の直列キューへ投入（録音は継続可能）。 -3) OpenAI Transcription API 呼び出し（ja/en/zh/ko には構造化プロンプト付与）。 -4) 機械的な TRANSCRIPT ラッパー除去 → クリーニング・パイプライン（安全判定付き）。 -5) プロンプトエラー検出（プロンプトのみが返った場合は空文字へ）。 -6) 任意の辞書補正を適用し、UIへ反映。 - -ASCII overview: -``` - ┌──────────────────────────┐ - │ AudioRecorder (連続録音) │ - └─────────────┬────────────┘ - │ audioBlob - ▼ - ┌──────────────────────────┐ - │ Processing Queue (直列化) │ - └─────────────┬────────────┘ - │ audioBlob - ▼ -┌──────────────────────────────────────────────────────────────┐ -│ TranscriptionService.transcribeAudio() │ -├──────────────────────────────────────────────────────────────┤ -│ 1) buildTranscriptionPrompt(lang) ──► ja/en/zh/ko で付与 │ -│ 2) POST (multipart/form-data) │ -│ 3) text 受領 │ -│ 4) preStripTranscriptWrappers(text) │ -│ 5) StandardCleaningPipeline (安全判定付き) │ -│ ├─ PromptContaminationCleaner（構造/指示/文脈/スニペット） │ -│ └─ UniversalRepetitionCleaner（反復・体裁） │ -│ 6) isPromptErrorDetected(text, lang)（プロンプトエコー検出） │ -│ 7) DictionaryCorrector（任意） │ -└──────────────────────────────────────────────────────────────┘ -``` - - -## System Architecture & Key Components / アーキテクチャと主要コンポーネント - -- UI Layer - - `src/views/VoiceInputView.ts` 主ビュー（レイアウト/ライフサイクル） - - `src/views/VoiceInputViewUI.ts` UI生成とイベント結線 - - `src/views/VoiceInputViewActions.ts` 録音・キュー・API・挿入等の操作中枢 - -- Core: Audio - - `src/core/audio/AudioRecorder.ts` 連続録音、AudioWorklet/ScriptProcessor、フィルタ、リングバッファ、可視化連携 - - `src/core/audio/AudioRingBuffer.ts` 音声リングバッファ - - `src/core/audio/AudioVisualizer.ts` 波形/レベル表示（2種） - -- Core: Transcription & Cleaning - - `src/core/transcription/TranscriptionService.ts` API呼び出し、プロンプト付与、クリーニング、辞書補正 - - `src/core/transcription/DictionaryCorrector.ts` 固定置換ベースの辞書補正 - - `src/core/transcription/cleaning/*` クリーナー群と標準パイプライン - -- Config & Defaults - - `src/config/constants.ts`, `defaults.ts`, `costs.ts`, `CleaningConfig.ts` - -- Errors & Services - - `src/errors/*` エラー型・通知 - - `src/services/*` ServiceLocator, I18n, HTTPクライアント - -- Managers - - `src/managers/DraftManager.ts` 下書き保存/復元 - - `src/managers/ViewManager.ts` ビュー管理 - - -## End‑to‑End Sequence / エンドツーエンドの逐次フロー - -1. ユーザーが Record ボタン操作 - - `VoiceInputViewUI` が `VoiceInputViewActions.toggleRecording()` を呼ぶ -2. 録音開始 - - `AudioRecorder.initialize()` → WebAudioセットアップ（Gain/HPF/LPF/Analyser） - - AudioWorklet利用可能なら `audio-processor-worklet`、不可なら ScriptProcessor - - `MediaRecorder.start(timeslice=100ms)` で WebM/Opus を分割収集 -3. 最大時間到達 or 手動停止 - - `AudioRecorder.stopRecording()` → `Blob`（audio/webm）生成 → `onSpeechEnd` 経由で通知 -4. キュー投入 - - `VoiceInputViewActions.processRecordedAudio()` が `processingQueue` に追加 - - バックグラウンドで直列処理 `processQueue()` 実行 -5. 文字起こし - - `TranscriptionService.transcribeAudio()` が `FormData` を構築し API へ送信 - - 応答 `text` を取得 → `cleanGPT4oResponse()` - - `preStripTranscriptWrappers()` → `StandardCleaningPipeline.execute()` → `isPromptErrorDetected()` → `DictionaryCorrector.correct()` -6. UI反映 - - `VoiceInputViewUI.textArea` に追記、下書き保存、ステータス更新 - - -## Recording Path / 録音パス - -実装参照: `src/core/audio/AudioRecorder.ts`, `src/views/VoiceInputViewActions.ts`, `src/views/VoiceInputViewUI.ts` - -- AudioContext 初期化 - - `window.AudioContext`（Safari は `webkitAudioContext`） - - `latencyHint='interactive'`, `sampleRate=16000` - - `AudioWorklet` 利用試行 → 失敗時は `ScriptProcessor` へフォールバック -- ノード構成 - - `MediaStreamSource → Gain → HighPass(80Hz) → LowPass(7.6kHz) → Analyser` - - 可視化: `AudioVisualizer` or `SimpleAudioLevelIndicator` -- 連続処理 - - 1ch PCM をリングバッファへ蓄積（`AudioRingBuffer`） - - マイク有効データの検出で `onMicrophoneStatusChange('ready')` -- 録音制御 - - 既定は VAD 無効の連続録音（`useVAD=false`） - - `maxRecordingSeconds` 到達で自動停止 - - 二重開始は `isStarting` ガードで抑止 -- UI/操作 - - クリック開始/停止 + 長押し Push‑to‑Talk（閾値: `UI_CONSTANTS.PUSH_TO_TALK_THRESHOLD`） - - -State model（簡略）: -- idle → initializing → recording → stopping → idle -- 例外時は error 表示後に idle 復帰 - - -## Processing Queue / キュー処理 - -実装参照: `src/views/VoiceInputViewActions.ts` - -- データ構造 - - `RecordingState.processingQueue: Array<{ audioBlob, timestamp, stopReason }>` -- 挙動 - - `processRecordedAudio()` がキューに追加し、`processQueue()` で直列処理 - - 録音継続中でもキュー処理はバックグラウンドで進行 -- ステータス - - 処理中/待機数を `statusEl` に表示（例: "Transcribing... (2 waiting)"） -- キャンセル - - 録音中断は `cancelRecording()`（収集済み音声は破棄） - - -## Transcription Path / 文字起こしパス - -実装参照: `src/core/transcription/TranscriptionService.ts` - -1) リクエスト作成（multipart/form-data） - - fields: - - `file: audio.webm (Blob)` - - `model: 'gpt-4o-transcribe' | 'gpt-4o-mini-transcribe'` - - `response_format: 'json'` - - `temperature: 0` - - `language: 'ja'|'en'|'zh'|'ko'`（auto は廃止） - - `prompt: 構造化プロンプト（ja/en/zh/ko 対応）` - - 送信先: `API_CONSTANTS.ENDPOINTS.TRANSCRIPTION` - - 送信手段: `ObsidianHttpClient.postFormData()`（独自 boundary 付与） - -2) レスポンス処理 - - `response.json.text` を取得（なければ `text` 文字列も許容） - - `cleanGPT4oResponse(text, language)` 実行 - - `preStripTranscriptWrappers()` で `...` 抽出/除去 - - `StandardCleaningPipeline` 実行 - - `isPromptErrorDetected(text, lang)` に該当すれば空文字で早期終了 - - 有効時 `DictionaryCorrector` 適用 - -3) エラー分類 - - 401 → `INVALID_API_KEY` - - 429 → `API_QUOTA_EXCEEDED` - - 5xx → `NETWORK_ERROR` - - それ以外 → `TRANSCRIPTION_FAILED` - -サンプル: 成功時の最小レスポンス（概念例） -``` -{ "text": "こんにちは。今日の予定は..." , "language": "ja" } -``` - - -## Cleaning Pipeline & Safety / クリーニングと安全装置 - -実装参照: -- `src/core/transcription/cleaning/StandardCleaningPipeline.ts` -- `src/core/transcription/cleaning/PromptContaminationCleaner.ts` -- `src/core/transcription/cleaning/UniversalRepetitionCleaner.ts` -- `src/config/CleaningConfig.ts` - -構成: -- 前処理（言語非依存） - - `preStripTranscriptWrappers()`: - - `...` 完全/不完全タグから中身抽出 - - 残存タグ除去（TRANSCRIPT/transcription/transcription系） -- パイプライン（順次実行） - 1) `PromptContaminationCleaner` - - 指示文・フォーマット行・スニペット・XML/タグ・「話者のみ」等の構造物を除去 - - 多言語 instructionPatterns と snippet 検出（EN/ZH/KO 語彙 + JA レガシー） - 2) `UniversalRepetitionCleaner` - - 文字/トークン/文/段落/列挙/末尾反復の抑制と体裁整形（言語非依存） - -セーフティ（安全判定）: -- 設定（`CLEANING_CONFIG.safety`） - - `singleCleanerMaxReduction: 0.3` - - `emergencyFallbackThreshold: 0.5` - - `warningThreshold: 0.15` -- 構造的クリーナー緩和（PromptContaminationCleaner に適用） - - 単一上限: `max(0.9, 0.3) = 0.9` - - 緊急閾値: `max(0.95, 0.5) = 0.95` -- 超過時の動作 - - Single 超過 → `skip`（当該クリーナーの変更を適用しない） - - Emergency 超過 → `rollback`（元テキストへ差し戻し） -- パイプライン全体の最終セーフティも実行（全体削減率が異常なら原文へ戻す） - -ログ例: -``` -[Voice Transcription] [StandardCleaningPipeline] Rolling back cleaner PromptContaminationCleaner { - reason: 'Reduction ratio 1.000 exceeds emergency threshold 0.95', - reductionRatio: 1 -} -``` -意味: 上流で「実発話が少ない/無い」（無音/極短/プロンプトエコー）可能性が高い。 - - -実例（Before → After 概念例）: -- Before: - ``` - Output format: - - （話者の発言のみ） - 今日は晴れです。今日は晴れです。今日は晴れです。。。。。 - - ``` -- After: - ``` - 今日は晴れです。 - ``` - - -## Prompt Error Detection / プロンプトエラー検出 - -実装参照: `TranscriptionService.isPromptErrorDetected()` - -- 言語別の固定句や「SPEAKER ONLY」等がそのまま返るケースを検知 -- 検出時は空文字へ置換し早期終了（UI は「No audio detected」等を表示） - -注意: -- `auto` は廃止。UI/設定で明示言語（ja/en/zh/ko）を選択し、該当のプロンプトと検出ロジックを適用 - - -## Dictionary Processing / 辞書処理 - -実装参照: `src/core/transcription/DictionaryCorrector.ts`, `src/interfaces/transcription.ts` - -- 固定置換（definiteCorrections）のみをサポート -- 設定値・制限（`DICTIONARY_CONSTANTS`） - - 最大個数（固定補正）: 200 -- 多言語に同一置換を適用（言語非依存） - -例: -``` -{ from: ["AI"], to: "artificial intelligence" } - -EN: "AI system" → "artificial intelligence system" -JA: "AIシステム" → "artificial intelligenceシステム" -ZH: "AI系统" → "artificial intelligence系统" -KO: "AI시스템" → "artificial intelligence시스템" -``` - -UIで表形式の編集/インポート/エクスポートに対応（全言語UIで利用可能）。 - - -## Error Handling / エラーハンドリング - -実装参照: `src/errors/*`, `src/services/*` - -- エラー型: `TranscriptionErrorType` - - PERMISSION_DENIED, NETWORK_ERROR, API_QUOTA_EXCEEDED, INVALID_API_KEY, AUDIO_* 他 -- 通知: `Notice` によるユーザー表示（i18n 対応） -- グローバル: `ErrorHandler` が未処理例外/Promise拒否を収集しログ化 -- API ステータスからの分類: - - 401 → INVALID_API_KEY - - 429 → API_QUOTA_EXCEEDED - - 5xx → NETWORK_ERROR - - その他 → TRANSCRIPTION_FAILED -- 無音/極短データ: 早期終了で空文字返却 + UI で「音声が検出されませんでした」 - - -## Metrics & Logging / 計測とログ - -実装参照: `src/utils/Logger.ts` - -- 共通プリフィクス: `[Voice Transcription]` -- 主要ログポイント - - 録音開始/終了、サンプルレート、チャンク数、所要時間 - - リングバッファ使用率（確率的に警告） - - API呼び出しモデル/言語/入力サイズ/出力長/時間 - - クリーニング各段の削減率/処理時間/セーフティ判定 -- ログレベルは設定に従い、開発モードで詳細化 - - -## Configuration & Defaults / 設定とデフォルト - -- 定数: `src/config/constants.ts` - - Audio: SAMPLE_RATE=16000, BUFFER_SIZE=4096, MAX_RECORDING_SECONDS=300, FILTERS(HPF=80Hz/LPF=7.6kHz), WEBM_CODEC, RECORDER_TIMESLICE=100ms - - API: `TRANSCRIPTION` エンドポイント、temperature=0 - - UI: Push‑to‑Talk 閾値=300ms ほか -- デフォルト: `src/config/defaults.ts` - - 文字起こしモデル: `gpt-4o-transcribe` - - 補正有効化: true - - VAD デフォルトは存在するが録音は既定で VAD 無効の連続録音 -- コスト: `src/config/costs.ts` - - 参考: 4o‑transcribe=$0.06/min, 4o‑mini‑transcribe=$0.03/min - - -## Performance Notes / パフォーマンス - -- WebAudio - - `latencyHint='interactive'` で起動体感改善 - - HPF/LPF により音声帯域へ制限し雑音影響を軽減 -- MediaRecorder - - 100ms timeslice で小刻み収集し停止時の結合負荷を低減 -- クリーニング - - 早期のラッパー剥離で後段の削減率を安定化 - - セーフティにより過剰削除を抑制（skip/rollback） -- UI/操作感 - - プッシュトゥトークは長押し→離すで録音区間を素早く収集 - - キュー直列処理により安定した逐次反映 - - -## Manual Validation Checklist / 手動検証チェックリスト - -ビルド/検証（`public/.github/copilot-instructions.md` 準拠）: -- Build Success - - `npm run build` 後、`build/latest/main.js` が生成される（~128KB） -- Lint Compliance - - `npm run lint`（~10 warnings, 0 errors 目安） -- Tests - - `npm run test`（182 近辺のテスト、時間 ~17s） -- Packaging - - `npm run build-plugin` 後、`build/latest/` に `main.js/manifest.json/styles.css/fvad.wasm/fvad.js` -- WASM 配布確認 - - `src/lib/fvad-wasm/` の `fvad.wasm` と `fvad.js` が出力に含まれる - -動作確認: -- 文字起こし成功（言語=設定値、モデル=設定値） -- 無音/極短音声で空結果を返し UI が警告 -- クリーニングの安全装置ログが想定通り（過剰削減は skip/rollback） -- 下書きの自動保存/復元が機能 - - -## Edge Cases & Troubleshooting / エッジケースと対処 - -- マイクトラックが muted で始まる（Windows 等） - - `AudioRecorder` が mute/unmute を監視し、音声検出時に `ready` 通知 -- 極小 `audioBlob`（~<500B） - - ヘッダのみの可能性 → 早期終了 + UI 警告 -- プロンプトエコー（モデルがプロンプトを返す） - - `isPromptErrorDetected()` が空文字へ置換 -- TRANSCRIPT タグしか無い/内容が空 - - `PromptContaminationCleaner` による削除でほぼ全削除 → セーフティで rollback -- Obsidian Deferred Views（1.7.2+） - - `DeferredViewHelper` で安全に leaf/view を扱う - - -## Extension Points / 拡張ポイント - -- クリーナーの追加 - - `StandardCleaningPipeline.addCleaner(new MyCleaner())` - - セーフティ閾値は `CLEANING_CONFIG` に集約 -- 置換辞書の拡張 - - UI から `definiteCorrections` を編集/Import/Export - - プログラムからは `TranscriptionService.setCustomDictionary()` -- モデル切替 - - UI（設定パネル）で 4o / 4o‑mini を動的に切替 - - -## Language‑by‑Language / 言語別の違いと共通点 - -``` - (選択された言語: ja / en / zh / ko) - │ - ▼ - ┌──────────────────────────────┐ - │ 構造化プロンプトの内容が言語別 │ ← 違い（プロンプト文面） - └──────────────┬───────────────┘ - ▼ -┌──────────────────────────────────────────────────────────────┐ -│ preStripTranscriptWrappers（機械的ラッパー除去） │ ← 共通（言語非依存） -├──────────────────────────────────────────────────────────────┤ -│ StandardCleaningPipeline │ -│ ├─ PromptContaminationCleaner │ -│ │ ・instructionPatterns: 多言語（ja/en/zh/ko） │ ← 共通（多言語対応） -│ │ ・snippet検出: EN/ZH/KO 追加語彙 + JA レガシー保護 │ ← ほぼ共通（内部最適化） -│ └─ UniversalRepetitionCleaner │ ← 共通（言語非依存） -├──────────────────────────────────────────────────────────────┤ -│ isPromptErrorDetected(text, lang) │ -│ ・各言語のエコー特有フレーズで検出 │ ← 違い（検出語句が言語別） -├──────────────────────────────────────────────────────────────┤ -│ DictionaryCorrector（任意） │ ← 共通（言語非依存） -└──────────────────────────────────────────────────────────────┘ -``` - -- 違い（language‑specific） - - プロンプト文面（`buildTranscriptionPrompt`） - - プロンプトエコー検出（`isPromptErrorDetected`） -- 共通（language‑agnostic） - - TRANSCRIPT ラッパー除去 - - セーフティ判定と反復抑制 - - 辞書補正 - - -## API Contracts & Data Shapes / API とデータ構造（要点） - -Request (multipart/form-data fields): -- file: Blob (audio/webm; codecs=opus) -- model: 'gpt-4o-transcribe' | 'gpt-4o-mini-transcribe' -- language: 'ja'|'en'|'zh'|'ko' -- response_format: 'json' -- temperature: '0' -- prompt: string (言語別の構造化プロンプト) - -Response (主要): -- text: string -- language: string（オプション。なければ UI 設定言語を採用） - - -## Examples & Logs / 例とログ - -- クリーニング高削減率の警告: - - `High reduction ratio in PromptContaminationCleaner reductionRatio=0.220 threshold=0.15` -- リングバッファ警告: - - `Audio buffer usage high: 82.4%` - - -## Notes / 補足 - -- 既定は VAD 無効の連続録音。必要に応じて `maxRecordingSeconds` を調整 -- 二重開始抑止や UI ロックは最小限にし、操作性を重視 -- Obsidian 環境では `requestUrl` を使用し CORS を回避 -- セキュアコンテキスト判定はブラウザではなく Obsidian の前提でスキップ - - -## Glossary / 用語集 - -- TRANSCRIPT ラッパー: ` ... ` のようなXML風タグ。モデル出力の安定化のためプロンプトで使用 -- セーフティ（Safety）: クリーニングでの過剰削除を防ぐ skip/rollback のしきい値判定 -- プロンプトエコー: モデルが指示文やフォーマット見出しをそのまま出力してしまう現象 -- 反復抑制: 文やトークンの過剰反復を抑える整形ロジック diff --git a/docs/design-decisions-summary.md b/docs/design-decisions-summary.md deleted file mode 100644 index e53f4bd..0000000 --- a/docs/design-decisions-summary.md +++ /dev/null @@ -1,122 +0,0 @@ -# Language-Specific Dictionaries: Design Decisions Summary - -## Key Decisions Made - -### 1. Type Architecture Decision - -**Chosen Approach**: Hybrid structure combining type safety with flexibility - -```typescript -interface MultiLanguageDictionary { - languages: { - ja?: CorrectionEntry[]; - en?: CorrectionEntry[]; - zh?: CorrectionEntry[]; - ko?: CorrectionEntry[]; - }; - global: CorrectionEntry[]; -} -``` - -**Rationale**: -- Maintains TypeScript type safety -- Allows optional language-specific dictionaries -- Clearly separates global vs language-specific rules -- Enables easy backward compatibility implementation - -### 2. Fallback Order - -**Priority**: `currentDetectedLanguage → global` - -**Rationale**: -- Current language gets highest priority (most relevant) -- Global dictionary as fallback (language-agnostic rules) -- No English fallback by default to avoid inappropriate corrections across languages -- Optional English fallback can be enabled via setting (disabled by default) for users who prefer it - -**Language Source**: Language detection is routed via `getResolvedLanguage()` which: -- Uses `transcriptionLanguage` setting when explicitly set -- Maps 'auto' setting to one of: ja/zh/ko/en based on plugin language detection -- Does NOT use UI/plugin interface language - -### 3. Backward Compatibility Strategy - -**Approach**: Automatic migration with preservation of existing data - -- Legacy `SimpleCorrectionDictionary` → `MultiLanguageDictionary.global` -- No data loss during migration -- One-time conversion on first load with new format -- Always save in new format going forward - -### 4. Dictionary Format and Security - -**Dictionary Entry Format**: Literal string-to-string mappings only -```typescript -interface CorrectionEntry { - patterns: string[]; // Literal strings, NOT regex patterns - replacement: string; // Literal replacement text - enabled: boolean; -} -``` - -**Important Safety Considerations**: -- Dictionary entries must be literal strings for safety and performance -- Regex patterns belong only to separate custom rules (not dictionary entries) -- Dictionary patterns are escaped when compiled to prevent injection -- Input validation prevents malicious patterns - -**Performance Limits**: -```typescript -const LIMITS = { - MAX_ENTRIES_PER_LANGUAGE: 1000, - MAX_GLOBAL_ENTRIES: 500, - MAX_PATTERN_LENGTH: 100, - MAX_REPLACEMENT_LENGTH: 200, - MAX_PATTERNS_PER_ENTRY: 10 -}; -``` - -**Performance Target**: <100ms processing time per text correction - -### 5. UI Strategy - -**Approach**: Tab-based language selection with minimal disruption - -- Language tabs/filters in dictionary settings -- Clear indication of entry counts per language -- Enhanced import/export supporting both formats -- Migration notification for existing users - -### 6. Implementation Phases - -**Phase 1**: Core infrastructure and types (2-3 days) -**Phase 2**: UI enhancements (3-4 days) -**Phase 3**: Optimization and testing (1-2 days) - -## Benefits of This Design - -1. **Type Safety**: Full TypeScript support with compile-time checks -2. **Backward Compatibility**: No breaking changes for existing users -3. **Performance**: Efficient fallback logic with literal string matching and caching support -4. **Security**: Input validation, literal-only dictionary entries, and size limits prevent abuse -5. **Usability**: Intuitive language-based organization aligned with transcription language -6. **Extensibility**: Easy to add new languages in the future -7. **Safety**: Explicit separation of dictionary entries (literal) from custom rules (regex) - -## Migration Impact - -- **Existing Users**: Seamless transition with automatic migration -- **Performance**: No regression, potential improvement with language-specific rules -- **Storage**: Slightly larger format but more efficient language targeting -- **Learning Curve**: Minimal - existing workflow preserved with new capabilities - -## Success Metrics - -1. Zero data loss during migration -2. <100ms correction processing time maintained with literal string matching -3. Memory usage stays under 1MB for typical dictionaries -4. User adoption of language-specific features >50% within 3 months -5. No increase in user-reported issues post-implementation -6. Security: Zero regex injection vulnerabilities with literal-only dictionary entries - -This design successfully balances the competing requirements of type safety, performance, backward compatibility, and user experience while providing a solid foundation for multi-language dictionary correction. The literal string approach ensures both safety and performance, while the simplified fallback strategy prevents inappropriate cross-language corrections. \ No newline at end of file diff --git a/docs/implementation-roadmap.md b/docs/implementation-roadmap.md deleted file mode 100644 index 17f9955..0000000 --- a/docs/implementation-roadmap.md +++ /dev/null @@ -1,168 +0,0 @@ -# Implementation Roadmap: Language-Specific Dictionaries - -## Next Implementation Issues - -Based on the design document, the following GitHub issues should be created to implement the language-specific dictionary feature: - -### Issue 1: Core Type Definitions and Infrastructure -**Title**: `feat: Add MultiLanguageDictionary type definitions and core infrastructure` - -**Description**: -Implement the foundational type definitions and helper functions for language-specific dictionaries. - -**Tasks**: -- [ ] Add `MultiLanguageDictionary` interface to `interfaces/transcription.ts` -- [ ] Add migration helper `migrateLegacyDictionary()` function -- [ ] Add validation functions for dictionary limits -- [ ] Update `VoiceInputSettings` to support new dictionary format -- [ ] Add backward compatibility detection logic -- [ ] Write unit tests for type definitions and helpers - -**Files to modify**: -- `src/interfaces/transcription.ts` -- `src/interfaces/settings.ts` -- `src/utils/dictionary-migration.ts` (new) -- `tests/unit/utils/dictionary-migration.test.ts` (new) - -**Acceptance Criteria**: -- New type definitions compile without errors -- Legacy dictionary migration works correctly -- All validation functions have comprehensive tests -- Backward compatibility is maintained - ---- - -### Issue 2: Dictionary Corrector Multi-Language Support -**Title**: `feat: Implement multi-language support in DictionaryCorrector` - -**Description**: -Extend the `DictionaryCorrector` class to support language-specific dictionaries with fallback logic. - -**Tasks**: -- [ ] Modify `DictionaryCorrector` to handle `MultiLanguageDictionary` -- [ ] Implement `getApplicableCorrections()` with fallback logic -- [ ] Add language detection parameter to `correct()` method -- [ ] Update constructor to accept new dictionary format -- [ ] Maintain backward compatibility with existing API -- [ ] Add comprehensive unit tests for multi-language correction - -**Files to modify**: -- `src/core/transcription/DictionaryCorrector.ts` -- `tests/unit/core/transcription/multilingual-dictionary-correction.test.ts` - -**Acceptance Criteria**: -- Fallback order works: currentLang → global (optional 'en' fallback only when a setting is enabled) -- Legacy dictionaries continue to work -- Performance does not degrade significantly -- All edge cases are covered with tests - ---- - -### Issue 3: Settings UI Enhancement for Language Tabs -**Title**: `feat: Add language-specific dictionary editing in settings UI` - -**Description**: -Enhance the settings UI to support editing language-specific dictionary entries with tab-based navigation. - -**Tasks**: -- [ ] Add language tab navigation to dictionary settings -- [ ] Implement language-specific entry display and editing -- [ ] Add language filter/selector for dictionary entries -- [ ] Update import/export functionality for new format -- [ ] Add entry count display per language -- [ ] Implement dictionary size warnings -- [ ] Add migration notification for existing users - -**Files to modify**: -- `src/views/VoiceInputViewUI.ts` -- `src/views/VoiceInputViewActions.ts` -- Update corresponding CSS styles - -**Acceptance Criteria**: -- Users can switch between language tabs -- Dictionary entries can be added/edited/deleted per language -- Import/export works with both old and new formats -- UI provides clear feedback on dictionary size limits -- Existing users see migration guidance - ---- - -### Issue 4: Performance Optimization and Security -**Title**: `feat: Add dictionary performance optimization and security measures` - -**Description**: -Implement performance optimizations and security measures for the dictionary system. - -**Tasks**: -- [ ] Add entry count limits per language and globally -- [ ] Implement dictionary size monitoring and warnings -- [ ] Add input validation and sanitization -- [ ] Implement dictionary caching for performance -- [ ] Add processing time limits for correction operations -- [ ] Create performance benchmarks and tests -- [ ] Add memory usage monitoring - -**Files to modify**: -- `src/core/transcription/DictionaryCorrector.ts` -- `src/config/constants.ts` (add limits) -- `src/utils/dictionary-validation.ts` (new) -- `tests/performance/dictionary-performance.test.ts` (new) - -**Acceptance Criteria**: -- Dictionary size limits are enforced -- Input validation prevents security issues -- Performance is maintained even with large dictionaries -- Memory usage stays within reasonable limits -- All security measures have corresponding tests - ---- - -### Issue 5: Integration and E2E Testing -**Title**: `test: Add comprehensive integration tests for language-specific dictionaries` - -**Description**: -Create comprehensive integration tests and update documentation for the new feature. - -**Tasks**: -- [ ] Add E2E tests for multi-language dictionary functionality -- [ ] Test dictionary migration scenarios -- [ ] Add performance regression tests -- [ ] Update user documentation and README -- [ ] Create migration guide for existing users -- [ ] Add API reference documentation -- [ ] Test with real-world dictionary data - -**Files to modify**: -- `tests/integration/dictionary-multilingual.test.ts` (new) -- `docs/user-guide.md` (update) -- `docs/migration-guide.md` (new) -- `README.md` (update) - -**Acceptance Criteria**: -- All integration scenarios pass -- Documentation is clear and comprehensive -- Migration guide helps users transition smoothly -- Performance benchmarks meet requirements - ---- - -## Implementation Order and Dependencies - -1. **Issue 1** (Core Types) - No dependencies, implement first -2. **Issue 2** (DictionaryCorrector) - Depends on Issue 1 -3. **Issue 4** (Performance/Security) - Can be done in parallel with Issue 2 -4. **Issue 3** (UI Enhancement) - Depends on Issues 1 and 2 -5. **Issue 5** (Testing/Docs) - Depends on all previous issues - -## Estimated Timeline - -- **Week 1**: Issues 1 and 2 (Core functionality) -- **Week 2**: Issues 3 and 4 (UI and optimization) -- **Week 3**: Issue 5 (Testing and documentation) - -## Risk Mitigation - -- Implement feature flags to enable/disable new functionality -- Maintain backward compatibility throughout all phases -- Add comprehensive logging for debugging migration issues -- Create rollback procedures in case of critical issues diff --git a/docs/issues/issue-cleaning-pipeline-repetition-suppression.md b/docs/issues/issue-cleaning-pipeline-repetition-suppression.md deleted file mode 100644 index f1c258e..0000000 --- a/docs/issues/issue-cleaning-pipeline-repetition-suppression.md +++ /dev/null @@ -1,633 +0,0 @@ -# 強化: 反復ハルシネーション抑制とクリーニング・パイプライン導入 - -問題: 現状のクリーニングはプロンプト断片やフォーマット除去にとどまり、ハルシネーションによる単語/句/文の機械的な繰り返し抑制が手薄。日本語に多い短語の連発、同一文の連続、列挙パターン (A, B, A, B …)、配信系アウトロ (末尾だけに現れる「ご視聴ありがとうございました」等) の混入に対する堅牢な処理が不足。 - -目的: 参照プロジェクト（obsidian-ai-transcriber）のパイプライン構造と安全装置を取り入れ、 -- プロンプト混入除去 -- 反復ハルシネーション抑制（短語/中長フレーズ/文/段落/列挙/末尾） -- 日本語テキスト品質の軽量バリデーション（任意） -を段階的・安全に実行する Cleaning Pipeline を導入する。 - -非目標: -- モデル選択や音声分割ロジックの変更なし -- UI トグル追加なし（常時オン）。設定は内部定数で集中管理 - -受け入れ基準 (Acceptance Criteria) -- 短語反復の抑制: 「はい。」「ありがとうございます。」等の連続出力が過剰でなくなる（適切に間引く）。 -- 中長フレーズ反復: 5–10/10–20/20–30 文字帯のフレーズが設定回数以上繰り返される場合に削減される。 -- 文重複の圧縮: 完全一致/高類似の連続文が閾値回数以上続く場合、1つに圧縮される。 -- 列挙周期の圧縮: A, B, A, B, A, B … は「A, B」に圧縮される（区切り/句読点維持）。 -- 配信系アウトロ等の末尾限定削除: 終端のみ一致した場合に限り削除される（本文中は保持）。 -- 安全装置: 1パターン/1イテレーション/全体削減の上限を超える操作は自動スキップ or ロールバックされる。 -- プロンプト混入: XML風タグ/文脈タグ/文頭の完全一致/スニペット一致の安全削除で混入を防ぐ。 - -タスク (Tasks) -- [ ] `src/config/CleaningConfig.ts` を追加: 言語別パターン、反復しきい値、安全閾値、文脈/タグパターンを集中管理 -- [ ] `src/core/transcription/cleaning/` を追加: クリーナーとパイプライン実装 - - [ ] `interfaces.ts`: `TextCleaner`/`CleaningResult`/`CleaningPipeline` ほか - - [ ] `StandardCleaningPipeline.ts`: 各クリーナーの逐次実行・計測・安全監視 - - [ ] `PromptContaminationCleaner.ts`: XML/文脈タグ、文頭一致、スニペット一致の優先度除去 - - [ ] `BaseHallucinationCleaner.ts`: 短語/中長フレーズ/文/段落/列挙/末尾の反復抑制 + 動的閾値 + 安全装置 -- [ ] `src/core/transcription/TranscriptionService.ts` を差分最小で置換: 既存の `cleanGPT4oResponse` からパイプライン委譲へ -- [ ] テスト追加 `tests/unit/core/cleaning/**` - - [ ] 短語反復（助詞保存/上限制限） - - [ ] 中長フレーズ反復 - - [ ] 文重複（類似度あり） - - [ ] 列挙圧縮（句読点保持） - - [ ] 末尾アウトロ限定削除 - - [ ] 安全装置（削減率超過ロールバック） -- [ ] ロギング: クリーナー別の削減率/時間/警告を記録（既存 Logger を使用） - -設計メモ（要点） -- 短語反復（日本語 1–4 文字）: 総文字数に応じて動的閾値。助詞は preserve/limit/reduce モードで過剰削除を抑制。残存比率を設定。 -- 文重複: `/(?<=[。.!?！？?])\s*/` で分割し、正規化（句読点/空白除去、NFKC 等）＋類似度で閾値以上を圧縮。 -- 列挙圧縮: 区切り（`、`/`,`）とパターン長 2..n/2 の完全周期一致を 1 サイクルへ圧縮。終端句読点は保持。 -- 安全装置: `singlePatternMaxReduction`/`repetitionPatternMaxReduction`/`iterationReductionLimit`/`emergencyFallbackThreshold` 等。 - ---- - -以下は提案実装の抜粋（コード例）。 - -1) 設定: `src/config/CleaningConfig.ts` -```ts -export type ParticleMode = 'preserve' | 'limit' | 'reduce'; - -export interface SafetyThresholds { - singleCleanerMaxReduction: number; - singlePatternMaxReduction: number; - repetitionPatternMaxReduction?: number; - iterationReductionLimit?: number; - emergencyFallbackThreshold: number; - warningThreshold: number; -} - -export interface RepetitionThresholds { - baseThreshold: number; - lengthFactor: number; - dynamicThresholdDivisor: number; - shortCharMinLength: number; - shortCharMaxLength: number; - shortCharKeepRatio: number; - essentialParticles: string[]; - maxConsecutiveParticles: number; - particleReductionMode: ParticleMode; - sentenceRepetition: number; - similarityThreshold: number; - minimumSentenceLengthForSimilarity: number; - consecutiveNewlineLimit: number; - mediumLengthRanges: Array<{ min: number; max: number; threshold: number }>; - enumerationDetection?: { enabled: boolean; minRepeatCount?: number }; - paragraphRepeat?: { enabled: boolean; headChars: number }; -} - -export interface LanguagePatterns { - japanese: string[]; - english: string[]; - chinese: string[]; - korean: string[]; -} - -export interface ContaminationPatterns { - instructionPatterns: string[]; - xmlPatternGroups: { - completeXmlTags: string[]; - sentenceBoundedTags: string[]; - lineBoundedTags: string[]; - standaloneTags: string[]; - }; - contextPatterns: string[]; - promptSnippetLengths: number[]; -} - -export interface CleaningConfig { - safety: SafetyThresholds; - repetition: RepetitionThresholds; - hallucinations: LanguagePatterns; - contamination: ContaminationPatterns; -} -``` - -2) クリーナー IF とパイプライン骨子 -```ts -// src/core/transcription/cleaning/interfaces.ts -export interface CleaningResult { - cleanedText: string; - issues: string[]; - hasSignificantChanges: boolean; - metadata?: Record; -} - -export interface CleaningContext { - language: string; - originalLength: number; - enableDetailedLogging?: boolean; - originalPrompt?: string; -} - -export interface TextCleaner { - readonly name: string; - readonly enabled: boolean; - clean(text: string, language: string, context?: CleaningContext): Promise | CleaningResult; -} - -export interface CleaningPipeline { - readonly name: string; - execute(text: string, language: string, context?: CleaningContext): Promise<{ - finalText: string; - metadata: { totalOriginalLength: number; totalFinalLength: number; totalReductionRatio: number }; - }>; -} - -// src/core/transcription/cleaning/StandardCleaningPipeline.ts -export class StandardCleaningPipeline implements CleaningPipeline { - readonly name = 'StandardCleaningPipeline'; - constructor(private cleaners: TextCleaner[] = []) {} - async execute(text: string, language: string, context?: CleaningContext) { - const originalLength = text.length; - let current = text; - for (const cleaner of this.cleaners) { - if (!cleaner.enabled) continue; - const res = await Promise.resolve( - cleaner.clean(current, language, { ...context, originalLength }) - ); - current = res.cleanedText; - } - return { - finalText: current, - metadata: { - totalOriginalLength: originalLength, - totalFinalLength: current.length, - totalReductionRatio: originalLength > 0 ? (originalLength - current.length) / originalLength : 0, - }, - }; - } -} -``` - -3) プロンプト混入: 文頭一致 + スニペット一致 + XML/文脈タグ -```ts -// src/core/transcription/cleaning/PromptContaminationCleaner.ts -import { CleaningResult, CleaningContext, TextCleaner } from './interfaces'; -import { CLEANING_CONFIG } from '../../../config/CleaningConfig'; - -export class PromptContaminationCleaner implements TextCleaner { - readonly name = 'PromptContaminationCleaner'; - readonly enabled = true; - - clean(text: string): CleaningResult { - const original = text; - const { instructionPatterns, xmlPatternGroups, contextPatterns, promptSnippetLengths } = CLEANING_CONFIG.contamination; - let cleaned = text; - - // XML風タグ（優先度順） - for (const group of ['completeXmlTags','sentenceBoundedTags','lineBoundedTags','standaloneTags'] as const) { - for (const patt of (xmlPatternGroups as any)[group] as string[]) { - cleaned = cleaned.replace(new RegExp(patt.slice(1, patt.lastIndexOf('/')), patt.slice(patt.lastIndexOf('/')+1)), ''); - } - } - - // 文頭の完全一致 - for (const prompt of instructionPatterns) { - if (cleaned.startsWith(prompt)) cleaned = cleaned.slice(prompt.length).trim(); - } - - // スニペット一致（保守的） - for (const prompt of instructionPatterns) { - for (const len of promptSnippetLengths) { - if (prompt.length < len) continue; - const snippet = prompt.slice(0, len).replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'); - const re = new RegExp(`${snippet}[^。.!?！？\n]{0,50}(?:ください|してください|です|ます)`, 'g'); - cleaned = cleaned.replace(re, ''); - } - } - - // 文脈パターン - for (const patt of contextPatterns) { - cleaned = cleaned.replace(new RegExp(patt.slice(1, patt.lastIndexOf('/')), patt.slice(patt.lastIndexOf('/')+1)), ''); - } - - cleaned = cleaned.replace(/\n{3,}/g, '\n\n').trim(); - const rr = original.length ? (original.length - cleaned.length) / original.length : 0; - return { cleanedText: cleaned, issues: rr > 0.25 ? [`Text reduction warning: ${Math.round(rr*100)}%`] : [], hasSignificantChanges: rr > 0.05 }; - } -} -``` - -4) 反復抑制（抜粋: 短語・文重複・列挙） -```ts -// src/core/transcription/cleaning/BaseHallucinationCleaner.ts -import { CleaningResult, TextCleaner } from './interfaces'; -import { CLEANING_CONFIG } from '../../../config/CleaningConfig'; - -export class BaseHallucinationCleaner implements TextCleaner { - readonly name = 'BaseHallucinationCleaner'; - readonly enabled = true; - - clean(text: string, language: string): CleaningResult { - const original = text; - let cleaned = text; - - // 言語別の固定パターン（終端アウトロ等） - const patterns = language === 'ja' ? CLEANING_CONFIG.hallucinations.japanese : CLEANING_CONFIG.hallucinations.english; - for (const patt of patterns) { - const re = new RegExp(patt.slice(1, patt.lastIndexOf('/')), patt.slice(patt.lastIndexOf('/')+1)); - cleaned = cleaned.replace(re, ''); - } - - // 短語反復 - cleaned = this.applyShortCharDedupe(cleaned, original.length); - // 文重複 - cleaned = this.collapseRepeatingSentences(cleaned); - // 列挙圧縮 - cleaned = this.compressEnumerations(cleaned); - - cleaned = cleaned.replace(/\uFFFD+/g, '').trim(); - const rr = original.length ? (original.length - cleaned.length) / original.length : 0; - return rr > CLEANING_CONFIG.safety.emergencyFallbackThreshold - ? { cleanedText: text, issues: ['Emergency fallback: excessive reduction'], hasSignificantChanges: false } - : { cleanedText: cleaned, issues: [], hasSignificantChanges: rr > 0.05 }; - } - - private applyShortCharDedupe(text: string, originalLength: number): string { - const r = CLEANING_CONFIG.repetition; const words = text.split(/\s+/); const counts = new Map(); - for (const w of words) { - const s = w.replace(/[。、！？\s]/g, ''); - if (s.length >= r.shortCharMinLength && s.length <= r.shortCharMaxLength && /^[あ-んア-ン]+$/.test(s)) counts.set(s, (counts.get(s) || 0) + 1); - } - const dyn = r.baseThreshold + Math.floor(originalLength / r.dynamicThresholdDivisor) * r.lengthFactor; let out = text; - for (const [w, c] of counts) { - if (r.essentialParticles.includes(w)) continue; - if (c >= dyn) { const keep = Math.max(1, Math.floor(c * r.shortCharKeepRatio)); const re = new RegExp(`${w}[。、]?\\s*`, 'g'); for (let i = 0; i < c - keep; i++) out = out.replace(re, ''); } - } - return out; - } - - private collapseRepeatingSentences(text: string): string { - const r = CLEANING_CONFIG.repetition; const sents = text.split(/(?<=[。.!?！？?])\s*/); const out: string[] = []; - let prev = '', count = 0; const norm = (s: string) => s.replace(/[。、！？\s]/g, '').normalize('NFKC'); - for (const s of sents) { const cur = s.trim(); const same = cur && (cur === prev || (cur.length >= r.minimumSentenceLengthForSimilarity && prev.length >= r.minimumSentenceLengthForSimilarity && norm(cur) === norm(prev))); if (same) { count++; } else { if (prev) out.push(count >= r.sentenceRepetition ? prev : prev.repeat(Math.max(1, count))); prev = s; count = 1; } } - if (prev) out.push(count >= r.sentenceRepetition ? prev : prev.repeat(Math.max(1, count))); return out.join('').trim(); - } - - private compressEnumerations(text: string): string { - const r = CLEANING_CONFIG.repetition; if (!r.enumerationDetection?.enabled) return text; - return text.split(/(?<=[。.!?！？])\s*/).map(sentence => { - const sep = sentence.includes('、') ? '、' : (sentence.includes(',') ? ',' : ''); if (!sep) return sentence; - const parts = sentence.split(new RegExp(`\\s*${sep.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*`)); - const minRep = r.enumerationDetection.minRepeatCount ?? 3; if (parts.length < minRep * 2) return sentence; - for (let len = 2; len <= Math.floor(parts.length / minRep); len++) { const pattern = parts.slice(0, len); let ok = true, reps = 1; for (let i = len; i < parts.length; i += len) { if (parts.slice(i, i+len).join('\u0001') !== pattern.join('\u0001')) { ok = false; break; } reps++; } if (ok && reps >= minRep) { const punc = /[。.!?！？]+$/.exec(parts[parts.length-1])?.[0] || ''; return pattern.join(sep === '、' ? '、' : ', ') + punc; } } - return sentence; - }).join(' ').trim(); - } -} -``` - -5) `TranscriptionService` への導線（概念サンプル） -```ts -// cleanGPT4oResponse 内での委譲 -import { StandardCleaningPipeline } from './cleaning/StandardCleaningPipeline'; -import { PromptContaminationCleaner } from './cleaning/PromptContaminationCleaner'; -import { BaseHallucinationCleaner } from './cleaning/BaseHallucinationCleaner'; - -private async cleanGPT4oResponse(text: string, language: string): Promise { - const pipeline = new StandardCleaningPipeline([ - new PromptContaminationCleaner(), - new BaseHallucinationCleaner(), - ]); - const lang = this.normalizeLanguage(language); - const res = await pipeline.execute(text, lang, { language: lang, originalLength: text.length }); - return res.finalText.trim().replace(/\n{3,}/g, '\n\n'); -} -``` - -ラベル案: `type:enhancement`, `area:transcription`, `lang:ja`, `safety`, `cleaning` - -提案ブランチ名: `feat/cleaning-pipeline-repetition-suppression` -# 強化: 反復ハルシネーション抑制（言語非依存）とクリーニング・パイプライン導入 - -問題: 現状のクリーニングはプロンプト断片やフォーマット除去にとどまり、ハルシネーションによる単語/句/文の機械的な繰り返し抑制が手薄。特定言語（日本語）に偏った対策となっており、英語/中国語/韓国語/混在文書など言語に依存しない「反復」現象への一般的な仕組みが不足。 - -目的: 参照プロジェクト（obsidian-ai-transcriber）のパイプライン構造と安全装置を取り入れ、言語に依存しない一般的なテキスト反復抑制を中心とした Cleaning Pipeline を導入する。 -- プロンプト混入除去（汎用） -- 反復ハルシネーション抑制（文字/トークン/フレーズ/文/段落/列挙/末尾）を言語非依存の規則で実装 -- 言語固有ロジックは最小限（任意・後続） - -非目標: -- モデル選択や音声分割ロジックの変更なし -- UI トグル追加なし（常時オン）。設定は内部定数で集中管理 - -受け入れ基準 (Acceptance Criteria) -- 言語非依存の反復抑制: 英語/日本語/中国語/韓国語/混在文書で、過剰な「同一トークン/フレーズ/文」の連続が適切に抑制される。 -- 文字/句読点の連続抑制: 言語に依存しない繰り返し（"...", "!!!", "????", "——", "•••" など）を適正な上限で収束。 -- フレーズ反復の削減: 可変長 n-gram（トークンベース、CJK は文字ベース）で閾値以上の反復を縮約。 -- 文重複の圧縮: 文境界（汎用句読点セット）で分割し、正規化＋類似度で連続重複を1つに圧縮。 -- 段落重複の削減: 段落先頭フィンガープリントで重複段落を除去。 -- 列挙周期の圧縮: 区切り（","/";"/"、"/"·"/タブ/スペース列）に依らず A,B,A,B,… 型の反復を1周期へ圧縮（句読点/区切りは自然に保持）。 -- 末尾反復の抑制: 文末近傍の自己反復密度が高い塊は終端限定で抑制（言語固有フレーズに依存しない構造的ルール）。 -- 安全装置: 1パターン/1イテレーション/全体削減の上限を超える操作は自動スキップ or ロールバックされる。 -- プロンプト混入: XML風タグ/文脈タグ/文頭の完全一致/スニペット一致の安全削除（汎用）。 - -タスク (Tasks) -- [ ] `src/config/CleaningConfig.ts` を追加: 反復しきい値/安全閾値/文脈・タグの汎用パターンを集中管理（言語非依存） -- [ ] `src/core/transcription/cleaning/` を追加: クリーナーとパイプライン実装 - - [ ] `interfaces.ts`: `TextCleaner`/`CleaningResult`/`CleaningPipeline` ほか - - [ ] `StandardCleaningPipeline.ts`: 各クリーナーの逐次実行・計測・安全監視 - - [ ] `PromptContaminationCleaner.ts`: XML/文脈タグ、文頭一致、スニペット一致の優先度除去（汎用） - - [ ] `UniversalRepetitionCleaner.ts`: 文字/トークン/フレーズ/文/段落/列挙/末尾の反復抑制 + 動的閾値 + 安全装置（言語非依存） -- [ ] `src/core/transcription/TranscriptionService.ts` を差分最小で置換: 既存の `cleanGPT4oResponse` からパイプライン委譲へ -- [ ] テスト追加 `tests/unit/core/cleaning/**` - - [ ] 英語の反復（"Okay okay okay"/"Thank you"×N） - - [ ] CJK の短語/文字反復（助詞等は一般ルールの範囲で保持） - - [ ] 中長フレーズ反復（言語混在） - - [ ] 文重複（句読点・空白正規化＋類似度） - - [ ] 列挙圧縮（","/";"/"、"/"·"/タブなど） - - [ ] 末尾の反復塊（高自己反復密度）抑制 - - [ ] 安全装置（削減率超過ロールバック） -- [ ] ロギング: クリーナー別の削減率/時間/警告を記録（既存 Logger を使用） - -設計メモ（要点 / 言語非依存） -- 正規化: Unicode NFKC、ケース折り、不要空白の正規化、句読点の共通集合化（`. ! ? … 。 ！ ？` 等） -- トークン化: 空白・句読点でのスプリット＋CJK 連続ブロックは文字ベースで扱うフォールバック -- 文字/記号の連続抑制: 例 `.{6,}`, `!{6,}`, `\?{6,}`, `…{3,}`, `[-—–]{6,}`, `[•·・]{6,}` を上限で収束 -- n-gram 反復: トークンベース n-gram（CJK は文字ベース）で閾値以上の反復を圧縮。窓幅は可変（例 3–10） -- 文重複: 汎用句読点で文境界→正規化→完全一致/高類似は連続閾値以上で1件に圧縮 -- 段落重複: 先頭フィンガープリント（NFKC + 抜粋長）で重複を除去 -- 列挙圧縮: 区切り記号（`,`/`;`/`、`/`·`/タブ/連続スペース）を検出→周期パターンのみ1周期保持 -- 末尾抑制: 末尾 N 文字の自己反復率と語彙多様性に基づく抑制（言語固有フレーズに依存しない） -- 安全装置: `singlePatternMaxReduction`/`repetitionPatternMaxReduction`/`iterationReductionLimit`/`emergencyFallbackThreshold` 等 - ---- - -以下は提案実装の抜粋（コード例）。 - -1) 設定: `src/config/CleaningConfig.ts` -```ts -export interface SafetyThresholds { - singleCleanerMaxReduction: number; - singlePatternMaxReduction: number; - repetitionPatternMaxReduction?: number; - iterationReductionLimit?: number; - emergencyFallbackThreshold: number; - warningThreshold: number; -} - -export interface RepetitionThresholds { - baseThreshold: number; - lengthFactor: number; - dynamicThresholdDivisor: number; - shortCharKeepRatio: number; - sentenceRepetition: number; - similarityThreshold: number; - minimumSentenceLengthForSimilarity: number; - consecutiveNewlineLimit: number; - ngram: { min: number; max: number; thresholds: Array<{ n: number; repeat: number }> }; - enumerationDetection?: { enabled: boolean; minRepeatCount?: number }; - paragraphRepeat?: { enabled: boolean; headChars: number }; -} - -export interface ContaminationPatterns { - instructionPatterns: string[]; - xmlPatternGroups: { - completeXmlTags: string[]; - sentenceBoundedTags: string[]; - lineBoundedTags: string[]; - standaloneTags: string[]; - }; - contextPatterns: string[]; - promptSnippetLengths: number[]; -} - -export interface CleaningConfig { - safety: SafetyThresholds; - repetition: RepetitionThresholds; - contamination: ContaminationPatterns; -} -``` - -2) クリーナー IF とパイプライン骨子 -```ts -// src/core/transcription/cleaning/interfaces.ts -export interface CleaningResult { - cleanedText: string; - issues: string[]; - hasSignificantChanges: boolean; - metadata?: Record; -} - -export interface CleaningContext { - language: string; // 'auto' を含む - originalLength: number; - enableDetailedLogging?: boolean; - originalPrompt?: string; -} - -export interface TextCleaner { - readonly name: string; - readonly enabled: boolean; - clean(text: string, language: string, context?: CleaningContext): Promise | CleaningResult; -} - -export interface CleaningPipeline { - readonly name: string; - execute(text: string, language: string, context?: CleaningContext): Promise<{ - finalText: string; - metadata: { totalOriginalLength: number; totalFinalLength: number; totalReductionRatio: number }; - }>; -} - -// src/core/transcription/cleaning/StandardCleaningPipeline.ts -export class StandardCleaningPipeline implements CleaningPipeline { - readonly name = 'StandardCleaningPipeline'; - constructor(private cleaners: TextCleaner[] = []) {} - async execute(text: string, language: string, context?: CleaningContext) { - const originalLength = text.length; - let current = text; - for (const cleaner of this.cleaners) { - if (!cleaner.enabled) continue; - const res = await Promise.resolve( - cleaner.clean(current, language, { ...context, originalLength }) - ); - current = res.cleanedText; - } - return { - finalText: current, - metadata: { - totalOriginalLength: originalLength, - totalFinalLength: current.length, - totalReductionRatio: originalLength > 0 ? (originalLength - current.length) / originalLength : 0, - }, - }; - } -} -``` - -3) プロンプト混入: 文頭一致 + スニペット一致 + XML/文脈タグ（汎用） -```ts -// src/core/transcription/cleaning/PromptContaminationCleaner.ts -import { CleaningResult, TextCleaner } from './interfaces'; -import { CLEANING_CONFIG } from '../../../config/CleaningConfig'; - -export class PromptContaminationCleaner implements TextCleaner { - readonly name = 'PromptContaminationCleaner'; - readonly enabled = true; - - clean(text: string): CleaningResult { - const original = text; - const { instructionPatterns, xmlPatternGroups, contextPatterns, promptSnippetLengths } = CLEANING_CONFIG.contamination; - let cleaned = text; - - // XML風タグ（優先度順） - for (const group of ['completeXmlTags','sentenceBoundedTags','lineBoundedTags','standaloneTags'] as const) { - for (const patt of (xmlPatternGroups as any)[group] as string[]) { - cleaned = cleaned.replace(new RegExp(patt.slice(1, patt.lastIndexOf('/')), patt.slice(patt.lastIndexOf('/')+1)), ''); - } - } - - // 文頭の完全一致 - for (const prompt of instructionPatterns) { - if (cleaned.startsWith(prompt)) cleaned = cleaned.slice(prompt.length).trim(); - } - - // スニペット一致（保守的） - for (const prompt of instructionPatterns) { - for (const len of promptSnippetLengths) { - if (prompt.length < len) continue; - const snippet = prompt.slice(0, len).replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'); - const re = new RegExp(`${snippet}[^。.!?！？\n]{0,50}(?:ください|してください|です|ます)`, 'g'); - cleaned = cleaned.replace(re, ''); - } - } - - // 文脈パターン - for (const patt of contextPatterns) { - cleaned = cleaned.replace(new RegExp(patt.slice(1, patt.lastIndexOf('/')), patt.slice(patt.lastIndexOf('/')+1)), ''); - } - - cleaned = cleaned.replace(/\n{3,}/g, '\n\n').trim(); - const rr = original.length ? (original.length - cleaned.length) / original.length : 0; - return { cleanedText: cleaned, issues: rr > 0.25 ? [`Text reduction warning: ${Math.round(rr*100)}%`] : [], hasSignificantChanges: rr > 0.05 }; - } -} -``` - -4) 反復抑制（抜粋: 言語非依存の短語/記号・文重複・列挙・末尾） -```ts -// src/core/transcription/cleaning/UniversalRepetitionCleaner.ts -import { CleaningResult, TextCleaner } from './interfaces'; -import { CLEANING_CONFIG } from '../../../config/CleaningConfig'; - -export class UniversalRepetitionCleaner implements TextCleaner { - readonly name = 'UniversalRepetitionCleaner'; - readonly enabled = true; - - clean(text: string): CleaningResult { - const original = text; - let cleaned = text; - - // 文字/記号の連続抑制（言語非依存） - cleaned = cleaned - .replace(/([.!?])\1{5,}/g, '$1$1$1') // !!! or ??? → 上限 - .replace(/[…]{3,}/g, '…') // 省略記号の収束 - .replace(/[-—–]{6,}/g, '—') // ダッシュ類 - .replace(/[•·・]{6,}/g, '・'); // ドット/中点類 - - // トークン反復の抑制（言語非依存） - cleaned = this.applyTokenDedupe(cleaned, original.length); - // 文重複 - cleaned = this.collapseRepeatingSentences(cleaned); - // 列挙圧縮 - cleaned = this.compressEnumerations(cleaned); - // 末尾の反復塊抑制 - cleaned = this.trimRepetitiveTail(cleaned); - - cleaned = cleaned.replace(/\uFFFD+/g, '').trim(); - const rr = original.length ? (original.length - cleaned.length) / original.length : 0; - return rr > CLEANING_CONFIG.safety.emergencyFallbackThreshold - ? { cleanedText: text, issues: ['Emergency fallback: excessive reduction'], hasSignificantChanges: false } - : { cleanedText: cleaned, issues: [], hasSignificantChanges: rr > 0.05 }; - } - - private applyTokenDedupe(text: string, originalLength: number): string { - const r = CLEANING_CONFIG.repetition; - const norm = (s: string) => s.normalize('NFKC').toLowerCase(); - // トークン化（空白/句読点）。CJK の連続は 1 文字ずつ扱うフォールバック - const tokens = Array.from(text.matchAll(/\p{L}+|\p{N}+|\p{P}+|\s+/gu)).map(m => m[0]); - const counts = new Map(); - for (const t of tokens) { - const key = norm(t).trim(); - if (!key || /^\p{P}+$|^\s+$|^[、。.!?！？…]+$/u.test(key)) continue; // 純粋な句読点/空白は別処理 - counts.set(key, (counts.get(key) || 0) + 1); - } - const dyn = r.baseThreshold + Math.floor(originalLength / r.dynamicThresholdDivisor) * r.lengthFactor; - let out = text; - for (const [key, c] of counts) { - if (c >= dyn) { - const keep = Math.max(1, Math.floor(c * r.shortCharKeepRatio)); - const esc = key.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'); - const re = new RegExp(`(?:^|\b)${esc}(?:\b|$)`, 'giu'); - for (let i = 0; i < c - keep; i++) out = out.replace(re, ''); - } - } - return out; - } - - private collapseRepeatingSentences(text: string): string { - const r = CLEANING_CONFIG.repetition; const sents = text.split(/(?<=[。.!?！？?])\s*/); const out: string[] = []; - let prev = '', count = 0; const norm = (s: string) => s.replace(/[、。,.;:!！?？\s]/g, '').normalize('NFKC').toLowerCase(); - for (const s of sents) { const cur = s.trim(); const same = cur && (cur === prev || (cur.length >= r.minimumSentenceLengthForSimilarity && prev.length >= r.minimumSentenceLengthForSimilarity && norm(cur) === norm(prev))); if (same) { count++; } else { if (prev) out.push(count >= r.sentenceRepetition ? prev : prev.repeat(Math.max(1, count))); prev = s; count = 1; } } - if (prev) out.push(count >= r.sentenceRepetition ? prev : prev.repeat(Math.max(1, count))); return out.join('').trim(); - } - - private compressEnumerations(text: string): string { - const r = CLEANING_CONFIG.repetition; if (!r.enumerationDetection?.enabled) return text; - return text.split(/(?<=[。.!?！？.?])\s*/).map(sentence => { - const sepMatch = sentence.match(/(、|,|;|·|\t|\s{2,})/); - const sep = sepMatch?.[1] || ''; - if (!sep) return sentence; - const parts = sentence.split(new RegExp(`\s*${sep.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\s*`)); - const minRep = r.enumerationDetection.minRepeatCount ?? 3; if (parts.length < minRep * 2) return sentence; - for (let len = 2; len <= Math.floor(parts.length / minRep); len++) { const pattern = parts.slice(0, len); let ok = true, reps = 1; for (let i = len; i < parts.length; i += len) { if (parts.slice(i, i+len).join('\u0001') !== pattern.join('\u0001')) { ok = false; break; } reps++; } if (ok && reps >= minRep) { const punc = /[。.!?！？]+$/.exec(parts[parts.length-1])?.[0] || ''; return pattern.join(sep === '、' ? '、' : sep.trim() === '' ? ' ' : `${sep.trim()} `) + punc; } } - return sentence; - }).join(' ').trim(); - } - - private trimRepetitiveTail(text: string): string { - // 末尾 400 文字を評価し、自己反復率が高い場合に末尾を切り戻す（保守的） - const tail = text.slice(-400); - if (tail.length < 80) return text; - const uniq = new Set(tail.normalize('NFKC').toLowerCase().split(/\s+/)); - const diversity = uniq.size / Math.max(1, tail.split(/\s+/).length); - const repetitions = (tail.match(/(.{2,20})\1{2,}/gs) || []).length; - if (diversity < 0.3 || repetitions >= 2) { - // 直前の文末まで巻き戻す - const cut = text.slice(0, -400); - const lastEnd = Math.max(cut.lastIndexOf('.'), cut.lastIndexOf('。'), cut.lastIndexOf('!'), cut.lastIndexOf('！'), cut.lastIndexOf('?'), cut.lastIndexOf('？')); - return lastEnd > -1 ? cut.slice(0, lastEnd + 1) : cut; - } - return text; - } -} -``` - -5) `TranscriptionService` への導線（概念サンプル） -```ts -// cleanGPT4oResponse 内での委譲 -import { StandardCleaningPipeline } from './cleaning/StandardCleaningPipeline'; -import { PromptContaminationCleaner } from './cleaning/PromptContaminationCleaner'; -import { UniversalRepetitionCleaner } from './cleaning/UniversalRepetitionCleaner'; - -private async cleanGPT4oResponse(text: string, language: string): Promise { - const pipeline = new StandardCleaningPipeline([ - new PromptContaminationCleaner(), - new UniversalRepetitionCleaner(), - ]); - const lang = this.normalizeLanguage(language); - const res = await pipeline.execute(text, lang, { language: lang, originalLength: text.length }); - return res.finalText.trim().replace(/\n{3,}/g, '\n\n'); -} -``` - -ラベル案: `enhancement`, `area:transcription`, `safety`, `cleaning` - -提案ブランチ名: `feat/cleaning-pipeline-repetition-suppression` - diff --git a/docs/language-specific-dictionaries-design.md b/docs/language-specific-dictionaries-design.md deleted file mode 100644 index 429a0bb..0000000 --- a/docs/language-specific-dictionaries-design.md +++ /dev/null @@ -1,289 +0,0 @@ -# Language-Specific Dictionaries Design Document - -## 目的 (Purpose) - -辞書補正の言語別適用（ja/en/zh/ko）を安全に導入するための設計ドキュメント。現在の単一辞書システムから、言語特化辞書システムへの段階的移行を実現する。 - -## 現状分析 (Current State Analysis) - -### 現在の実装 - -- **辞書構造**: `SimpleCorrectionDictionary` with `definiteCorrections: CorrectionEntry[]` -- **適用範囲**: 全言語に同一辞書を適用（`enableTranscriptionCorrection: true` の場合） -- **言語サポート**: 'ja', 'en', 'zh', 'ko' + 'auto' -- **設定**: `VoiceInputSettings.customDictionary: SimpleCorrectionDictionary` - -### 問題点 - -1. 言語固有の修正ルールが適用できない -2. 日本語の敬語変換が英語テキストにも適用される可能性 -3. 言語特有の音声認識エラーパターンに対応できない -4. 辞書サイズの肥大化 - -## 型設計の比較と決定 (Type Architecture Comparison) - -### オプション1: Record型アプローチ - -```typescript -type LanguageSpecificDictionary = Record<'ja' | 'en' | 'zh' | 'ko', CorrectionEntry[]> & { - global?: CorrectionEntry[]; // 言語共通ルール -}; -``` - -**利点**: -- TypeScript型安全性が高い -- 言語キーの補完とチェックが可能 -- 構造が明確で理解しやすい - -**欠点**: -- 新言語追加時に型定義の変更が必要 -- 空配列でもすべての言語キーが必要 -- JSONシリアライゼーション時の冗長性 - -### オプション2: Array型アプローチ - -```typescript -type LanguageSpecificDictionary = Array<{ - lang?: Locale; // undefined = global - entries: CorrectionEntry[]; -}>; -``` - -**利点**: -- 動的な言語追加が容易 -- 空言語の場合の無駄がない -- JSONサイズが効率的 -- 拡張性が高い - -**欠点**: -- TypeScript型安全性が低い -- 実行時のvalidationが必要 -- 言語重複チェックが必要 - -### 決定: ハイブリッドアプローチ - -最適な解決策として、両方の利点を組み合わせた構造を採用： - -```typescript -interface MultiLanguageDictionary { - // 言語別辞書（型安全） - languages: { - ja?: CorrectionEntry[]; - en?: CorrectionEntry[]; - zh?: CorrectionEntry[]; - ko?: CorrectionEntry[]; - }; - // 全言語共通辞書 - global: CorrectionEntry[]; -} -``` - -**理由**: -1. 型安全性を保持しつつ柔軟性を提供 -2. 空言語は undefined で省略可能 -3. 全言語共通ルールを明示的に分離 -4. 後方互換性の実装が容易 - -## フォールバック順序の定義 (Fallback Strategy) - -### フォールバック優先順位 - -1. **現在の検出言語** (`currentDetectedLanguage`) -2. **英語** (`'en'`) - 国際的共通言語として -3. **グローバル辞書** (`global`) - 言語非依存の修正 - -### 実装ロジック - -```typescript -function getApplicableCorrections( - detectedLanguage: string, - dictionary: MultiLanguageDictionary -): CorrectionEntry[] { - const corrections: CorrectionEntry[] = []; - - // 1. 現在の言語の辞書 - const langDict = dictionary.languages[detectedLanguage as Locale]; - if (langDict) { - corrections.push(...langDict); - } - - // 2. 英語辞書（現在の言語が英語でない場合） - if (detectedLanguage !== 'en' && dictionary.languages.en) { - corrections.push(...dictionary.languages.en); - } - - // 3. グローバル辞書 - corrections.push(...dictionary.global); - - return corrections; -} -``` - -### エッジケース処理 - -- **未知言語**: 'en' → global の順序でフォールバック -- **'auto'言語設定**: 実際の検出結果に基づく -- **空辞書**: 次の優先順位に進む - -## 後方互換性戦略 (Backward Compatibility) - -### 既存データの受理 - -現在の `SimpleCorrectionDictionary` 形式を自動的に `MultiLanguageDictionary` に変換： - -```typescript -function migrateLegacyDictionary( - legacy: SimpleCorrectionDictionary -): MultiLanguageDictionary { - return { - languages: {}, // 言語別辞書は空 - global: legacy.definiteCorrections // 既存ルールは全てglobalに - }; -} -``` - -### 設定保存形式 - -- **新形式**: `MultiLanguageDictionary` -- **レガシー検出**: `definiteCorrections` プロパティの存在で判定 -- **自動変換**: 読み込み時に一度だけ実行 -- **保存形式**: 常に新形式で保存 - -## マイグレーション計画 (Migration Plan) - -### フェーズ1: 基盤実装 - -1. **型定義の追加** - - `MultiLanguageDictionary` インターフェース - - 互換性ヘルパー関数 - -2. **コア機能の拡張** - - `DictionaryCorrector` のマルチ言語対応 - - フォールバックロジックの実装 - -3. **テストの追加** - - 言語別辞書テスト - - フォールバックテスト - - 後方互換性テスト - -### フェーズ2: UI拡張 - -1. **設定画面の改良** - - 言語タブまたはフィルター機能 - - 言語別エントリの表示・編集 - - インポート・エクスポート機能の対応 - -2. **UX考慮事項** - - 既存ユーザーへの移行案内 - - 言語別エントリ数の表示 - - 辞書サイズの警告機能 - -### フェーズ3: 最適化 - -1. **パフォーマンス改善** - - 辞書キャッシュ機能 - - 部分読み込み対応 - -2. **高度な機能** - - 辞書の言語間コピー機能 - - 共通ルールの自動検出・提案 - -## セキュリティ・パフォーマンス考慮事項 (Security & Performance) - -### エントリ数制限 - -```typescript -const LIMITS = { - MAX_ENTRIES_PER_LANGUAGE: 1000, - MAX_GLOBAL_ENTRIES: 500, - MAX_PATTERN_LENGTH: 100, - MAX_REPLACEMENT_LENGTH: 200, - MAX_PATTERNS_PER_ENTRY: 10 -}; -``` - -### メモリ使用量 - -- **推定サイズ**: 言語4つ × 1000エントリ × 100文字 ≈ 400KB -- **制限値**: 総辞書サイズ 1MB以下 -- **警告しきい値**: 500KB - -### セキュリティ対策 - -1. **入力検証** - - 正規表現インジェクション防止 - - 文字列長制限 - - 特殊文字のサニタイゼーション - -2. **DoS攻撃対策** - - 処理時間制限（100ms/テキスト） - - 再帰的置換の防止 - - メモリ使用量監視 - -### パフォーマンス最適化 - -1. **処理効率** - - パターンマッチングの最適化 - - 辞書の事前コンパイル - - キャッシュの活用 - -2. **UI応答性** - - 大量エントリの仮想化表示 - - 非同期でのバリデーション - - プログレス表示 - -## 実装ロードマップ (Implementation Roadmap) - -### 必要なサブタスク - -#### Issue #[TBD]: 型定義とコア機能実装 -- [ ] `MultiLanguageDictionary` インターフェース定義 -- [ ] `DictionaryCorrector` のマルチ言語対応 -- [ ] フォールバック機能の実装 -- [ ] 後方互換性ヘルパー関数 -- [ ] ユニットテストの追加 - -#### Issue #[TBD]: 設定UI拡張 -- [ ] 言語タブ機能の実装 -- [ ] 言語別エントリの表示・編集UI -- [ ] インポート・エクスポート機能の対応 -- [ ] 既存データの移行UI - -#### Issue #[TBD]: バリデーション・制限機能 -- [ ] エントリ数制限の実装 -- [ ] 入力検証の強化 -- [ ] メモリ使用量監視 -- [ ] パフォーマンス最適化 - -#### Issue #[TBD]: ドキュメント・テスト -- [ ] ユーザーガイドの更新 -- [ ] API リファレンスの更新 -- [ ] E2Eテストの追加 -- [ ] マイグレーションガイドの作成 - -### 推定工数 - -- **フェーズ1**: 2-3 人日 -- **フェーズ2**: 3-4 人日 -- **フェーズ3**: 1-2 人日 -- **合計**: 6-9 人日 - -### リスク要因 - -1. **既存ユーザーデータの移行** - - リスク: データ損失・破損 - - 対策: バックアップ機能とロールバック機能 - -2. **パフォーマンス影響** - - リスク: 辞書処理の遅延 - - 対策: 段階的最適化とベンチマーク - -3. **UI複雑化** - - リスク: ユーザビリティの低下 - - 対策: プロトタイプと段階的展開 - -## 結論 (Conclusion) - -本設計により、言語特化辞書システムを安全かつ段階的に導入できる。ハイブリッド型アプローチにより型安全性と拡張性を両立し、堅牢なフォールバック機能により既存の使用体験を維持しながら新機能を提供する。 - -適切な制限とセキュリティ対策により、パフォーマンスと安全性を確保し、明確なマイグレーション計画により実装リスクを最小化する。 diff --git a/src/core/transcription/cleaning/PromptContaminationCleaner.ts b/src/core/transcription/cleaning/PromptContaminationCleaner.ts index fa15aad..896adf2 100644 --- a/src/core/transcription/cleaning/PromptContaminationCleaner.ts +++ b/src/core/transcription/cleaning/PromptContaminationCleaner.ts @@ -52,8 +52,10 @@ export class PromptContaminationCleaner implements TextCleaner { this.logger.debug('XML tags removed'); } - // Step 2: Remove exact instruction matches at text beginning + // Step 2: Remove leading prompt block (multi-line instructions at the very beginning) const beforeInstructions = cleaned; + cleaned = this.removeLeadingPromptBlock(cleaned, instructionPatterns); + // Also apply exact instruction matches for residuals cleaned = this.removeInstructionPatterns(cleaned, instructionPatterns); if (cleaned !== beforeInstructions) { patternsMatched++; @@ -78,6 +80,19 @@ export class PromptContaminationCleaner implements TextCleaner { // Step 5: Clean up excessive whitespace cleaned = this.normalizeWhitespace(cleaned); + + // Step 6: Final safeguard — remove any leftover exact instruction phrases anywhere (very conservative list) + // This catches cases where punctuation or line breaks prevented earlier head-only removal. + for (const pattern of instructionPatterns) { + try { + const escaped = pattern.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&'); + const anywhere = new RegExp(`${escaped}(?:[\u3002\.：:])?`, 'gi'); + cleaned = cleaned.replace(anywhere, ''); + } catch { + // ignore + } + } + cleaned = this.normalizeWhitespace(cleaned); const processingTime = performance.now() - startTime; const reductionRatio = original.length > 0 ? (original.length - cleaned.length) / original.length : 0; @@ -138,23 +153,67 @@ export class PromptContaminationCleaner implements TextCleaner { return cleaned; } + + /** + * Remove consecutive instruction/context lines at the very beginning (until a non-instruction line appears) + */ + private removeLeadingPromptBlock(text: string, instructionPatterns: string[]): string { + const lines = text.split(/\r?\n/); + let cutIndex = 0; + + // Build quick regexes for instructions and context labels across languages + const escapedInstructions = instructionPatterns.map(p => p.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&')); + // Identify a line as instruction if it STARTS with an instruction phrase (rest of line may include more prompts) + const instructionRegexes = escapedInstructions.map(e => new RegExp(`^\s*${e}`, 'i')); + const contextRegexes: RegExp[] = [ + /^\s*Output\s*format\s*:?\s*$/i, + /^\s*Format\s*:?\s*$/i, + /^\s*出力形式\s*:?\s*$/, + /^\s*输出格式\s*:?\s*$/, + /^\s*출력\s*형식\s*:?\s*$/, + /^\s*\(Speaker\s+content\s+only\)\s*$/i, + /^\s*（話者の発言のみ）\s*$/, + /^\s*（仅说话者内容）\s*$/, + /^\s*（화자\s*발언만）\s*$/ + ]; + + for (let i = 0; i < lines.length; i++) { + const ln = lines[i]; + const trimmed = ln.trim(); + if (trimmed.length === 0) { cutIndex = i + 1; continue; } + const isInstruction = instructionRegexes.some(r => r.test(trimmed)); + const isContext = contextRegexes.some(r => r.test(trimmed)); + if (isInstruction || isContext) { + cutIndex = i + 1; + continue; + } + // stop when encountering first non-instruction/context line + break; + } + + return lines.slice(cutIndex).join('\n'); + } /** * Remove instruction patterns from the beginning of text */ private removeInstructionPatterns(text: string, instructionPatterns: string[]): string { let cleaned = text; - - for (const pattern of instructionPatterns) { - // Check if text starts with this pattern - if (cleaned.trimStart().startsWith(pattern)) { - const index = cleaned.indexOf(pattern); - if (index >= 0) { - cleaned = cleaned.slice(index + pattern.length).trim(); + let changed = true; + // Greedily remove any known instruction that appears at the very beginning (allowing optional punctuation right after) + while (changed) { + changed = false; + for (const pattern of instructionPatterns) { + const escaped = pattern.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&'); + // Match at start of string or start of any line (multiline), forgiving trailing punctuation + const reg = new RegExp(`^\\s*${escaped}(?:[\u3002\.：:])?\\s*`, 'im'); + const before = cleaned; + cleaned = cleaned.replace(reg, ''); + if (cleaned !== before) { + changed = true; } } } - return cleaned; } @@ -163,12 +222,17 @@ export class PromptContaminationCleaner implements TextCleaner { */ private removeSnippetPatterns(text: string, instructionPatterns: string[], snippetLengths: number[]): string { let cleaned = text; + // Restrict snippet search to head region to minimize false positives + const MAX_SNIPPET_SEARCH_CHARS = 300; + const head = cleaned.slice(0, MAX_SNIPPET_SEARCH_CHARS); + let headMut = head; - // Multilingual suffix lexicon for enhanced snippet detection + // Multilingual suffix lexicon for enhanced snippet detection (added JA) const suffixLexicon = [ /\b(please|do\s*not\s*include|only|content|output\s*format)\b/gi, // EN /(请|請|不要|仅|只|内容|输出格式|輸出格式)/g, // ZH - /(해주세요|하지\s*마세요|포함하지\s*마세요|만|내용|출력\s*형식)/g // KO + /(해주세요|하지\s*마세요|포함하지\s*마세요|만|내용|출력\s*형식)/g, // KO + /(この指示文|指示文|出力形式|形式|内容|話者|のみ|含めないでください|含めないで)$/g // JA ]; for (const pattern of instructionPatterns) { @@ -179,24 +243,25 @@ export class PromptContaminationCleaner implements TextCleaner { // Escape special regex characters const escapedSnippet = snippet.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'); - // Enhanced multilingual contextual matching + // Enhanced multilingual contextual matching (head region only) for (const suffixPattern of suffixLexicon) { const contextRegex = new RegExp( `${escapedSnippet}[^。.!?！？\\n]{0,50}(${suffixPattern.source})`, suffixPattern.flags ); - cleaned = cleaned.replace(contextRegex, ''); + headMut = headMut.replace(contextRegex, ''); } - // Legacy pattern for Japanese (maintaining backward compatibility) + // Legacy pattern for Japanese (apply once within head) const legacyJapaneseRegex = new RegExp( - `${escapedSnippet}[^。.!?！？\\n]{0,50}(?:ください|してください|です|ます)\\b`, - 'gi' + `${escapedSnippet}[^。.!?！？\\n]{0,50}(?:ください|してください|です|ます)(?![\u3041-\u3096\u30A1-\u30FA\u4E00-\u9FFF])`, + 'i' ); - cleaned = cleaned.replace(legacyJapaneseRegex, ''); + headMut = headMut.replace(legacyJapaneseRegex, ''); } } - + // Apply head mutations back to full text + cleaned = headMut + cleaned.slice(head.length); return cleaned; } @@ -205,8 +270,23 @@ export class PromptContaminationCleaner implements TextCleaner { */ private removeContextPatterns(text: string, contextPatterns: string[]): string { let cleaned = text; - - // Direct pattern matching for speaker-only phrases + + // 1) Apply patterns from configuration (strings like '/.../flags') + const compiled: RegExp[] = []; + for (const p of contextPatterns) { + try { + const m = p.match(/^\/(.*)\/([gimsuy]*)$/); + if (m) { + const [, body, flags] = m as RegExpMatchArray; + compiled.push(new RegExp(body, flags || 'g')); + } + } catch { + // ignore invalid patterns + } + } + for (const r of compiled) cleaned = cleaned.replace(r, ''); + + // 2) Built-in safe patterns const speakerPatterns = [ /\(Speaker content only\)/gi, /\(SPEAKER CONTENT ONLY\)/gi, @@ -214,12 +294,8 @@ export class PromptContaminationCleaner implements TextCleaner { /（仅说话者内容）/g, /（화자 발언만）/g ]; - - for (const pattern of speakerPatterns) { - cleaned = cleaned.replace(pattern, ''); - } - - // Remove format-only lines + for (const pattern of speakerPatterns) cleaned = cleaned.replace(pattern, ''); + const formatPatterns = [ /^Output format:\s*$/gm, /^Format:\s*$/gm, @@ -227,11 +303,8 @@ export class PromptContaminationCleaner implements TextCleaner { /^输出格式:\s*$/gm, /^출력 형식:\s*$/gm ]; - - for (const pattern of formatPatterns) { - cleaned = cleaned.replace(pattern, ''); - } - + for (const pattern of formatPatterns) cleaned = cleaned.replace(pattern, ''); + return cleaned; } @@ -244,4 +317,4 @@ export class PromptContaminationCleaner implements TextCleaner { .replace(/^\s*\n/gm, '') // Remove lines with only whitespace .trim(); } -} \ No newline at end of file +} diff --git a/src/core/transcription/cleaning/StandardCleaningPipeline.ts b/src/core/transcription/cleaning/StandardCleaningPipeline.ts index 09eacf5..307b5f5 100644 --- a/src/core/transcription/cleaning/StandardCleaningPipeline.ts +++ b/src/core/transcription/cleaning/StandardCleaningPipeline.ts @@ -198,7 +198,7 @@ export class StandardCleaningPipeline implements CleaningPipeline { // Relax safety thresholds for structural cleaners that may legitimately // remove large wrappers (e.g., TRANSCRIPT tags or full prompt lines). // This prevents false rollbacks like when input is mostly XML wrappers. - const isStructuralCleaner = cleanerName === 'PromptContaminationCleaner'; + const isStructuralCleaner = cleanerName === 'PromptContaminationCleaner' || cleanerName === 'Pipeline'; const emergencyThreshold = isStructuralCleaner ? Math.max(0.95, CLEANING_CONFIG.safety.emergencyFallbackThreshold) : CLEANING_CONFIG.safety.emergencyFallbackThreshold; diff --git a/src/core/transcription/cleaning/UniversalRepetitionCleaner.ts b/src/core/transcription/cleaning/UniversalRepetitionCleaner.ts index 1cb8ca1..d454c8b 100644 --- a/src/core/transcription/cleaning/UniversalRepetitionCleaner.ts +++ b/src/core/transcription/cleaning/UniversalRepetitionCleaner.ts @@ -56,7 +56,7 @@ export class UniversalRepetitionCleaner implements TextCleaner { totalReductionSteps += step1Result.changes; // Step 2: Token repetition suppression - const step2Result = this.suppressTokenRepetitions(cleaned, original.length); + const step2Result = this.suppressTokenRepetitions(cleaned, original.length, language); cleaned = step2Result.text; totalReductionSteps += step2Result.changes; @@ -148,7 +148,7 @@ export class UniversalRepetitionCleaner implements TextCleaner { /** * Suppress token repetitions using language-independent tokenization */ - private suppressTokenRepetitions(text: string, originalLength: number): { text: string; changes: number } { + private suppressTokenRepetitions(text: string, originalLength: number, language?: string): { text: string; changes: number } { const config = CLEANING_CONFIG.repetition; let changes = 0; @@ -178,7 +178,12 @@ export class UniversalRepetitionCleaner implements TextCleaner { // Create escape pattern for regex const escapedToken = this.escapeRegex(normalizedToken); - const tokenRegex = new RegExp(`\\b${escapedToken}\\b`, 'gi'); + // Use Unicode-aware boundaries for CJK languages where \b is ineffective + const isCJK = !!language && /^(ja|zh|ko)/i.test(language); + const pattern = isCJK + ? `(?(); + // For near-duplicate detection using 3-gram Jaccard + const seenKeys: string[] = []; for (const sentence of sentences) { const normalizedSentence = this.normalizeSentence(sentence); if (normalizedSentence.length >= config.minimumSentenceLengthForSimilarity) { - const count = seenSentences.get(normalizedSentence) || 0; - seenSentences.set(normalizedSentence, count + 1); + // Find an existing key that is identical or highly similar + let key = normalizedSentence; + let similarKey: string | null = null; + for (const k of seenKeys) { + if (k === normalizedSentence) { similarKey = k; break; } + const sim = this.jaccard3GramSimilarity(k, normalizedSentence); + if (sim >= config.similarityThreshold) { similarKey = k; break; } + } + if (similarKey) key = similarKey; else seenKeys.push(key); + + const count = seenSentences.get(key) || 0; + seenSentences.set(key, count + 1); // Only keep if under repetition threshold if (count + 1 <= config.sentenceRepetition) { @@ -227,6 +244,27 @@ export class UniversalRepetitionCleaner implements TextCleaner { return { text: result.join(''), changes }; } + + /** + * Compute Jaccard similarity over 3-gram sets of two normalized strings + */ + private jaccard3GramSimilarity(a: string, b: string): number { + const aSet = this.toNGramSet(a, 3); + const bSet = this.toNGramSet(b, 3); + if (aSet.size === 0 && bSet.size === 0) return 1; + let inter = 0; + for (const g of aSet) if (bSet.has(g)) inter++; + const union = aSet.size + bSet.size - inter; + return union > 0 ? inter / union : 1; + } + + private toNGramSet(s: string, n: number): Set { + const set = new Set(); + const len = s.length; + if (len < n) { set.add(s); return set; } + for (let i = 0; i <= len - n; i++) set.add(s.slice(i, i + n)); + return set; + } /** * Compress enumeration patterns (A,B,A,B... -> A,B) @@ -418,4 +456,4 @@ export class UniversalRepetitionCleaner implements TextCleaner { const repetitions = (text.match(/(.{2,20})\1{2,}/gs) || []).length; return repetitions; } -} \ No newline at end of file +} From 0d02a446f7d04a33a201832d3950919cef143c2b Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 30 Aug 2025 00:52:46 +0900 Subject: [PATCH 76/82] fix(settings-sync): keep view settings panel toggle in sync by broadcasting 'voice-input:settings-changed' and subscribing in VoiceInputView --- src/plugin/VoiceInputPlugin.ts | 7 +++++++ src/views/VoiceInputView.ts | 14 ++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/plugin/VoiceInputPlugin.ts b/src/plugin/VoiceInputPlugin.ts index 38157c8..a58d3ad 100644 --- a/src/plugin/VoiceInputPlugin.ts +++ b/src/plugin/VoiceInputPlugin.ts @@ -414,6 +414,13 @@ export default class VoiceInputPlugin extends Plugin { }; await this.saveData(dataToSave); + // Broadcast settings-changed event for open views to react immediately (UI + services) + try { + // Custom workspace event (safe to ignore if not observed) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (this.app.workspace as any).trigger?.('voice-input:settings-changed', { settings: this.settings }); + } catch (_) { /* noop */ } + // Update logger configuration based on new settings Logger.getInstance().updateConfig({ debugMode: this.settings.debugMode, diff --git a/src/views/VoiceInputView.ts b/src/views/VoiceInputView.ts index 3962f47..db1bb77 100644 --- a/src/views/VoiceInputView.ts +++ b/src/views/VoiceInputView.ts @@ -67,6 +67,20 @@ export class VoiceInputView extends ItemView { this.setupFocusManagement(); this.setupAutoSaveOnBlur(); this.setupPeriodicSave(); + + // Subscribe to settings change events to keep UI toggles in sync + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const ws: any = this.app.workspace as any; + if (ws?.on) { + const ref = ws.on('voice-input:settings-changed', () => { + this.ui?.updateSettingsUI(); + this.actions?.updateTranscriptionService(); + }); + // Ensure cleanup via plugin's registerEvent lifecycle + this.plugin.registerEvent(ref); + } + } catch (_) { /* noop */ } } } From b100aa09515f3173a003939406cce4268ad20131 Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 30 Aug 2025 02:16:32 +0900 Subject: [PATCH 77/82] chore(ui): remove confusing model dropdown comments; keep current behavior (no functional changes) --- src/views/VoiceInputViewUI.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/views/VoiceInputViewUI.ts b/src/views/VoiceInputViewUI.ts index 94a316d..9e13380 100644 --- a/src/views/VoiceInputViewUI.ts +++ b/src/views/VoiceInputViewUI.ts @@ -231,7 +231,6 @@ export class VoiceInputViewUI { this.plugin.settings.enableTranscriptionCorrection = value; await this.plugin.saveSettings(); this.view.actions.updateTranscriptionService(); - // Enable/disable the model dropdown }); }); } @@ -378,7 +377,7 @@ export class VoiceInputViewUI { this.transcriptionModelDropdown.setValue(this.plugin.settings.transcriptionModel); } - // Update post-processing model dropdown + // (no additional post-processing UI elements to update) } /** From 853d44dfec42f92237a8f464d29808f8938a14f3 Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 30 Aug 2025 02:20:05 +0900 Subject: [PATCH 78/82] chore(cleanup): remove unused requestUrl import and clarify post-processing comment (dictionary-based only) --- src/core/transcription/DictionaryCorrector.ts | 3 +-- src/settings/VoiceInputSettingTab.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/transcription/DictionaryCorrector.ts b/src/core/transcription/DictionaryCorrector.ts index 272a6aa..c9a2f32 100644 --- a/src/core/transcription/DictionaryCorrector.ts +++ b/src/core/transcription/DictionaryCorrector.ts @@ -2,7 +2,6 @@ import { CorrectionRule, DictionaryCorrectorOptions, ITextCorrector, SimpleCorrectionDictionary, CorrectionEntry } from '../../interfaces'; import { TranscriptionError, TranscriptionErrorType } from '../../errors'; import { API_CONSTANTS, DICTIONARY_CONSTANTS } from '../../config'; -import { requestUrl } from 'obsidian'; /** * 辞書修正設定 @@ -138,4 +137,4 @@ export class DictionaryCorrector implements ITextCorrector { getDefaultRules(): CorrectionRule[] { return [...this.defaultRules]; } -} \ No newline at end of file +} diff --git a/src/settings/VoiceInputSettingTab.ts b/src/settings/VoiceInputSettingTab.ts index 0935bac..e425ba0 100644 --- a/src/settings/VoiceInputSettingTab.ts +++ b/src/settings/VoiceInputSettingTab.ts @@ -219,7 +219,7 @@ export class VoiceInputSettingTab extends PluginSettingTab { await this.plugin.saveSettings(); })); - // AI Post-processing Toggle and Model Selection + // AI Post-processing (dictionary-based) new Setting(containerEl) .setName(this.i18n.t('ui.settings.aiPostProcessing')) .setDesc(this.i18n.t('ui.settings.aiPostProcessingDesc')) From 8e1dbc7a4bccb4264b6640394e8bebd11031f514 Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 30 Aug 2025 03:15:31 +0900 Subject: [PATCH 79/82] feat(settings): add inline help explaining comma-separated patterns in dictionary; add i18n keys (en/ja/zh/ko) --- src/i18n/en.ts | 3 +++ src/i18n/ja.ts | 3 +++ src/i18n/ko.ts | 3 +++ src/i18n/zh.ts | 3 +++ src/interfaces/i18n.ts | 3 +++ src/settings/VoiceInputSettingTab.ts | 4 ++++ 6 files changed, 19 insertions(+) diff --git a/src/i18n/en.ts b/src/i18n/en.ts index 31265e0..cc7d80b 100644 --- a/src/i18n/en.ts +++ b/src/i18n/en.ts @@ -113,6 +113,9 @@ export const en: TranslationResource = { } }, ui: { + help: { + dictionaryFromComma: 'Enter multiple source patterns separated by commas (e.g., "pattern 1, pattern 2").' + }, commands: { openView: 'Open view' }, diff --git a/src/i18n/ja.ts b/src/i18n/ja.ts index 3627b18..8d58f6a 100644 --- a/src/i18n/ja.ts +++ b/src/i18n/ja.ts @@ -113,6 +113,9 @@ export const ja: TranslationResource = { } }, ui: { + help: { + dictionaryFromComma: '入力語はカンマ区切りで複数指定できます（例: パターン1, パターン2）。' + }, commands: { openView: 'ビューを開く' }, diff --git a/src/i18n/ko.ts b/src/i18n/ko.ts index ef62fe0..8b2aea4 100644 --- a/src/i18n/ko.ts +++ b/src/i18n/ko.ts @@ -113,6 +113,9 @@ export const ko: TranslationResource = { } }, ui: { + help: { + dictionaryFromComma: '입력어는 쉼표로 여러 패턴을 지정할 수 있습니다(예: 패턴1, 패턴2).' + }, commands: { openView: '뷰 열기' }, diff --git a/src/i18n/zh.ts b/src/i18n/zh.ts index dd9a88f..99f7d77 100644 --- a/src/i18n/zh.ts +++ b/src/i18n/zh.ts @@ -113,6 +113,9 @@ export const zh: TranslationResource = { } }, ui: { + help: { + dictionaryFromComma: '“来源词”支持用逗号分隔的多个模式（例如：模式1, 模式2）。' + }, commands: { openView: '打开视图' }, diff --git a/src/interfaces/i18n.ts b/src/interfaces/i18n.ts index f2e3cf9..e2f18eb 100644 --- a/src/interfaces/i18n.ts +++ b/src/interfaces/i18n.ts @@ -132,6 +132,9 @@ export type TranslationResource = { }; }; ui: { + help: { + dictionaryFromComma: string; + }; commands: { openView: string; }; diff --git a/src/settings/VoiceInputSettingTab.ts b/src/settings/VoiceInputSettingTab.ts index e425ba0..e9cb24f 100644 --- a/src/settings/VoiceInputSettingTab.ts +++ b/src/settings/VoiceInputSettingTab.ts @@ -270,6 +270,10 @@ export class VoiceInputSettingTab extends PluginSettingTab { // Definite Corrections Section tableContainer.createEl('h4', { text: this.i18n.t('ui.settings.dictionaryDefinite', { max: DICTIONARY_CONSTANTS.MAX_DEFINITE_CORRECTIONS }) }); + tableContainer.createEl('div', { + cls: 'setting-item-description', + text: this.i18n.t('ui.help.dictionaryFromComma') + }); this.createCorrectionTable( tableContainer, this.plugin.settings.customDictionary.definiteCorrections, From 69bb48f1622f5acd92b5cf1330681c279bb83cf8 Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 30 Aug 2025 03:43:54 +0900 Subject: [PATCH 80/82] chore(release): v0.9.0 --- manifest.json | 18 +++++++++--------- package-lock.json | 4 ++-- package.json | 2 +- versions.json | 3 ++- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/manifest.json b/manifest.json index 117c2a0..6fbd9d2 100644 --- a/manifest.json +++ b/manifest.json @@ -1,11 +1,11 @@ { - "id": "voice-input", - "name": "Voice Input", - "version": "0.8.0", - "minAppVersion": "1.7.0", - "description": "Voice input with real-time transcription using GPT-4o.", - "author": "Musashino Software", - "authorUrl": "https://github.com/mssoftjp", - "fundingUrl": "https://buymeacoffee.com/mssoft", - "isDesktopOnly": true + "id": "voice-input", + "name": "Voice Input", + "version": "0.9.0", + "minAppVersion": "1.7.0", + "description": "Voice input with real-time transcription using GPT-4o.", + "author": "Musashino Software", + "authorUrl": "https://github.com/mssoftjp", + "fundingUrl": "https://buymeacoffee.com/mssoft", + "isDesktopOnly": true } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index a9f9d16..2a2e9a8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "voice-input", - "version": "0.8.0", + "version": "0.9.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "voice-input", - "version": "0.8.0", + "version": "0.9.0", "license": "MIT", "dependencies": { "@echogarden/fvad-wasm": "^0.2.0", diff --git a/package.json b/package.json index 3a5cc2a..a881bda 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "voice-input", - "version": "0.8.0", + "version": "0.9.0", "description": "Voice input plugin for Obsidian with accurate transcription", "main": "main.js", "scripts": { diff --git a/versions.json b/versions.json index 068ee55..1544cf5 100644 --- a/versions.json +++ b/versions.json @@ -1,3 +1,4 @@ { - "0.8.0": "17.0.0" + "0.8.0": "17.0.0", + "0.9.0": "1.7.0" } \ No newline at end of file From 67f53156c3b0bc5457eaadc141f37611e2a42131 Mon Sep 17 00:00:00 2001 From: Musashino Software <115339791+mssoftjp@users.noreply.github.com> Date: Sun, 21 Sep 2025 02:44:58 +0900 Subject: [PATCH 81/82] Revise description for voice input feature Updated the description to reflect the use of OpenAI GPT-4o Transcribe API for improved accuracy. --- manifest.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifest.json b/manifest.json index 6fbd9d2..29f978c 100644 --- a/manifest.json +++ b/manifest.json @@ -3,9 +3,9 @@ "name": "Voice Input", "version": "0.9.0", "minAppVersion": "1.7.0", - "description": "Voice input with real-time transcription using GPT-4o.", + "description": "AI-powered high-accuracy voice input using OpenAI GPT-4o Transcribe API", "author": "Musashino Software", "authorUrl": "https://github.com/mssoftjp", "fundingUrl": "https://buymeacoffee.com/mssoft", "isDesktopOnly": true -} \ No newline at end of file +} From df9cccb8f7cfea652ed3cbd2131c78e34f0fb180 Mon Sep 17 00:00:00 2001 From: mssoftjp <115339791+mssoftjp@users.noreply.github.com> Date: Sat, 4 Oct 2025 18:13:57 +0900 Subject: [PATCH 82/82] fix(settings): sync transcription language when linking --- src/settings/VoiceInputSettingTab.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/settings/VoiceInputSettingTab.ts b/src/settings/VoiceInputSettingTab.ts index e9cb24f..12aecb2 100644 --- a/src/settings/VoiceInputSettingTab.ts +++ b/src/settings/VoiceInputSettingTab.ts @@ -56,7 +56,22 @@ export class VoiceInputSettingTab extends PluginSettingTab { )) .setValue(this.plugin.settings.pluginLanguage) .onChange(async (value: Locale) => { + const transcriptionLocale = value as 'ja' | 'en' | 'zh' | 'ko'; this.plugin.settings.pluginLanguage = value; + + // Keep transcription language synchronized while linking remains enabled + if (this.plugin.settings.advanced?.languageLinkingEnabled !== false) { + this.plugin.settings.transcriptionLanguage = transcriptionLocale; + if (!this.plugin.settings.advanced) { + this.plugin.settings.advanced = { + languageLinkingEnabled: true, + transcriptionLanguage: transcriptionLocale + }; + } else { + this.plugin.settings.advanced.transcriptionLanguage = transcriptionLocale; + } + } + await this.plugin.saveSettings(); this.i18n.setLocale(value);