From c207d5444dccbc189c2fe9839c61e8dad654ec98 Mon Sep 17 00:00:00 2001 From: WiseGuru <42100212+WiseGuru@users.noreply.github.com> Date: Wed, 27 May 2026 13:29:51 -0700 Subject: [PATCH] Various (adding Voxtral, various) --- .claude/settings.local.json | 3 +- CLAUDE.md | 14 +++-- docs/FEATURES.md | 34 ++++++++++++ eslint.config.mts | 2 +- src/audio-transcode.ts | 66 +++++++++++++++++++++++ src/pipeline.ts | 4 +- src/recorder.ts | 8 --- src/settings/tab.ts | 3 ++ src/transcription/index.ts | 3 ++ src/transcription/limits.ts | 81 ++++++++++++++++++++++++++++ src/transcription/mistral-voxtral.ts | 65 ++++++++++++++++++++++ src/transcription/whisper-local.ts | 70 +----------------------- src/types.ts | 1 + src/ui/modal.ts | 8 +-- src/ui/quick-record.ts | 2 +- src/ui/setup-card.ts | 3 ++ 16 files changed, 278 insertions(+), 89 deletions(-) create mode 100644 src/audio-transcode.ts create mode 100644 src/transcription/limits.ts create mode 100644 src/transcription/mistral-voxtral.ts diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 5f15b9d..32a001d 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -1,7 +1,8 @@ { "permissions": { "allow": [ - "Bash(npm run *)" + "Bash(npm run *)", + "WebSearch" ] } } diff --git a/CLAUDE.md b/CLAUDE.md index 99a3574..9ee2bee 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,7 +47,8 @@ src/ ├── http.ts # requestUrl wrappers: jsonPost/jsonGet/multipartPost + ProviderError ├── platform.ts # Active-profile resolver + MediaRecorder/Web Speech availability probes ├── secrets.ts # safeStorage (desktop) + plaintext fallback (mobile) for API keys -├── recorder.ts # MediaRecorder state machine + getBestMimeType + 25 MB size guard +├── recorder.ts # MediaRecorder state machine + getBestMimeType (no size cap; per-provider limits live in transcription/limits.ts) +├── audio-transcode.ts # WebAudio decode + resample to 16 kHz mono PCM WAV (shared by whisper-local and mistral-voxtral) ├── webspeech.ts # SpeechRecognition wrapper ├── pipeline.ts # transcribe → cleanup → insert orchestrator ├── insert.ts # cursor/newFile/append + {{date}}/{{time}} expansion @@ -70,10 +71,12 @@ src/ │ └── whisper-status-bar.ts # Status-bar dot for whisper-host start/stop (desktop + whisper-local profile only) ├── transcription/ │ ├── index.ts # TranscriptionProvider interface + createTranscriptionProvider() +│ ├── limits.ts # Per-provider maxBytes/maxDurationMs + validateRecording (called from pipeline) │ ├── openai.ts # Whisper-shape POST (also used by openai-compatible + groq) │ ├── assemblyai.ts # upload → submit → poll │ ├── deepgram.ts # single POST │ ├── revai.ts # submit → poll → fetch text +│ ├── mistral-voxtral.ts # Mistral Voxtral STT (JSON response; always transcodes to WAV) │ ├── webspeech.ts # adapter; throws "should not be called" (see Gotchas) │ └── whisper-local.ts # Thin shim that POSTs to the WhisperHost-managed local server └── llm/ @@ -88,7 +91,7 @@ src/ [src/pipeline.ts](src/pipeline.ts) runs four stages with `onStage` callbacks for UI: 1. **Persist audio** (`audio` source only): writes the raw `Blob` to the vault via [src/audio-persist.ts](src/audio-persist.ts) before transcription, so the user keeps the recording even if later stages fail. Path resolution: when `settings.attachmentsFolderPath` is set, the file goes under that folder with manual de-collision (`-1`, `-2`, ...); when empty, the path comes from `app.fileManager.getAvailablePathForAttachment(filename)`, which respects Obsidian's own attachments setting. Filename is `ReWrite-YYYY-MM-DD-HHmmss.` with the extension derived from the blob's mime type (`webm` / `m4a` / `ogg` / `wav` / `mp3`, default `webm`). Failure is non-fatal: a Notice fires and transcription proceeds. The resolved path is later prepended to the cleaned output as `![[]]\n\n` before insertion. -2. **Transcribe**: `audio` → `createTranscriptionProvider(profile.transcriptionProvider).transcribe(blob, config)`. Skipped when the source is `paste` or `text` (input passes through). Short-circuited when the source is `webspeech` (the transcript was already captured live by `src/webspeech.ts` during recording). +2. **Transcribe**: `audio` → `createTranscriptionProvider(profile.transcriptionProvider).transcribe(blob, config)`. Skipped when the source is `paste` or `text` (input passes through). Short-circuited when the source is `webspeech` (the transcript was already captured live by `src/webspeech.ts` during recording). Just before dispatching, `validateRecording(blobSize, durationMs, providerId)` from [src/transcription/limits.ts](src/transcription/limits.ts) throws a friendly per-provider error if the recording exceeds the provider's documented byte or duration cap. Because validation runs after `persist-audio`, the user keeps the saved file and can switch providers + reprocess from the vault. 3. **Cleanup**: `createLLMProvider(profile.llmProvider).complete(systemPrompt, transcript, config)`. The system prompt is the template prompt, optionally augmented with an `## Ad-hoc instructions` block when the wake-name scan ([src/wake-name.ts](src/wake-name.ts)) extracts directives from the transcript, and a `## Known nouns` block when `plugin.knownNouns` is non-empty (see Assistant prompt and Known nouns sections below). On error, the (possibly stripped) transcript is copied to the clipboard before re-throwing, so the user keeps their words. 4. **Insert**: `src/insert.ts` routes to `cursor` / `newFile` / `append` per the template. `cursor` falls back to `append` when no editor is active; `append` falls back to `newFile` when no markdown file exists. `{{date}}` / `{{time}}` in filename templates expand via Obsidian's `moment`. The modal's per-invocation Destination control overrides `insertMode` / `newFileFolder` / `newFileNameTemplate` via `PipelineParams.destinationOverride`; the override is shallow-merged onto a copy of the template before the insert call, so the template file on disk is never mutated. @@ -98,7 +101,7 @@ The `PipelineSource` union has four variants: `audio` (recorded blob, optional ` ## Provider system -[src/transcription/index.ts](src/transcription/index.ts) and [src/llm/index.ts](src/llm/index.ts) each define a small interface plus a `create...Provider(id)` factory. Provider families share one adapter file: OpenAI Whisper, `openai-compatible`, and Groq all dispatch into [src/transcription/openai.ts](src/transcription/openai.ts) with different base URLs; OpenAI GPT, `openai-compatible`, and Mistral all dispatch into [src/llm/openai.ts](src/llm/openai.ts). +[src/transcription/index.ts](src/transcription/index.ts) and [src/llm/index.ts](src/llm/index.ts) each define a small interface plus a `create...Provider(id)` factory. Provider families share one adapter file where the API shapes match: OpenAI Whisper, `openai-compatible`, and Groq all dispatch into [src/transcription/openai.ts](src/transcription/openai.ts) with different base URLs; OpenAI GPT, `openai-compatible`, and Mistral all dispatch into [src/llm/openai.ts](src/llm/openai.ts). Mistral Voxtral does NOT share with `openai.ts` — see [src/transcription/mistral-voxtral.ts](src/transcription/mistral-voxtral.ts) — because Voxtral's response is JSON-only (no `response_format=text`), it rejects WebM input (so the blob is always transcoded to 16 kHz mono WAV via [src/audio-transcode.ts](src/audio-transcode.ts)), and `/v1/models` doesn't surface audio-model IDs the same way it surfaces chat models. API keys are stored per profile on `EnvironmentProfile.transcriptionConfig.apiKey` / `llmConfig.apiKey`. Two slots per profile, one for transcription and one for the LLM. No global by-family map; the desktop and mobile profiles each carry their own keys even when both use the same provider (deliberate: per-profile keys make per-function usage tracking easier). Persistence is in [src/secrets.ts](src/secrets.ts) using the key IDs `profile:desktop:transcription`, `profile:desktop:llm`, `profile:mobile:transcription`, `profile:mobile:llm`. @@ -110,7 +113,7 @@ Providers may optionally implement `listModels(config, signal)` returning a stri `start()` validates the binary and model paths exist via `fs.existsSync`, probes the port via `net.createServer().listen(port)` to detect conflicts, spawns the user's whisper-server with `child_process.spawn`, captures stdout/stderr into a 1 MB ring buffer, then polls `net.createConnection` against the port every 250 ms for up to 5 s before declaring `'running'`. Any failure transitions status to `'crashed'` with the log tail surfaced in the error message. `stop()` sends SIGTERM, waits up to 3 s, then SIGKILL. -The `whisper-local` transcription provider ([src/transcription/whisper-local.ts](src/transcription/whisper-local.ts)) is a thin shim that POSTs to `http://127.0.0.1:/inference` (whisper.cpp's native server route, not OpenAI's `/v1/audio/transcriptions`). Before sending, the recorded blob is transcoded to 16 kHz mono 16-bit PCM WAV via Web Audio API (`AudioContext.decodeAudioData` → `OfflineAudioContext` for resample/downmix → hand-written RIFF header). whisper.cpp's server cannot decode WebM/Opus (the MediaRecorder default) without a custom ffmpeg-enabled build, so the transcode is mandatory — it is *not* applied to any other transcription provider. The shim grabs the host reference via `bindWhisperHost(host)` called from [src/main.ts](src/main.ts) on load. If the host status is anything other than `'running'`, the adapter throws a `ProviderError` with a "start it from settings" message; the pipeline surfaces that as a Notice. No API key is collected for this provider (no auth, no settings field). +The `whisper-local` transcription provider ([src/transcription/whisper-local.ts](src/transcription/whisper-local.ts)) is a thin shim that POSTs to `http://127.0.0.1:/inference` (whisper.cpp's native server route, not OpenAI's `/v1/audio/transcriptions`). Before sending, the recorded blob is transcoded to 16 kHz mono 16-bit PCM WAV via `transcodeToWavPcm` in [src/audio-transcode.ts](src/audio-transcode.ts) (Web Audio API: `AudioContext.decodeAudioData` → `OfflineAudioContext` for resample/downmix → hand-written RIFF header). whisper.cpp's server cannot decode WebM/Opus (the MediaRecorder default) without a custom ffmpeg-enabled build, so the transcode is mandatory. The same helper is reused by [src/transcription/mistral-voxtral.ts](src/transcription/mistral-voxtral.ts) (Voxtral also rejects WebM). If you add a third audio-uploading provider that needs WAV, import from `audio-transcode.ts` rather than duplicating the helpers. The shim grabs the host reference via `bindWhisperHost(host)` called from [src/main.ts](src/main.ts) on load. If the host status is anything other than `'running'`, the adapter throws a `ProviderError` with a "start it from settings" message; the pipeline surfaces that as a Notice. No API key is collected for this provider (no auth, no settings field). Mobile compatibility: `WhisperHost` and the `whisper-local` option are guarded everywhere by `Platform.isDesktop`. Node modules (`child_process`, `net`, `fs`) are lazy-required inside the `Platform.isDesktop` branch, mirroring the [src/secrets.ts](src/secrets.ts) `getSafeStorage` pattern; on mobile the host is inert and the provider option is filtered out of dropdowns in [src/settings/tab.ts](src/settings/tab.ts) and [src/ui/setup-card.ts](src/ui/setup-card.ts). @@ -224,6 +227,7 @@ Per [.editorconfig](.editorconfig): tabs (width 4), LF, UTF-8, final newline. Ma - **Frontmatter parsing uses `parseYaml` from Obsidian, not the metadata cache.** The metadata cache is async and may not be populated for newly created files; reading content via `app.vault.read(file)`, splitting off the leading `---...---` block, and parsing it with `parseYaml` is synchronous-enough and works immediately after `app.vault.create`. - **Quick Record uses a custom floating div, not a `Notice`.** Obsidian `Notice` does not support real interactive buttons. The floater is a `position: fixed` div on `document.body`, owned by `QuickRecordController`, with `cancel()` wired into `onunload`. - **`secrets.json.nosync` uses the `.nosync` suffix on purpose**: iCloud Drive natively skips any file or folder whose name ends in `.nosync`. The README documents per-tool sync exclusion for other tools. +- **Per-provider recording limits live in [src/transcription/limits.ts](src/transcription/limits.ts), not the recorder.** [src/recorder.ts](src/recorder.ts) does not cap recordings at any size; `validateRecording(blobSize, durationMs, providerId)` runs in [src/pipeline.ts](src/pipeline.ts) between the `persist-audio` and `transcribe` stages, throwing a friendly provider-attributed error if the recording exceeds the documented byte or duration ceiling. Both modal and Quick Record thread the recorder's `durationMs` onto the `audio` pipeline source so the duration check has data; the reprocess flow ([src/ui/audio-source.ts](src/ui/audio-source.ts)) omits `durationMs` (no cheap way to measure an arbitrary vault file), so reprocess only triggers the byte check. Limits source: `openai`/`groq` 25 MB, `assemblyai` 5 GB/10 h, `deepgram` 2 GB, `revai` 2 GB/17 h, `mistral-voxtral` 1 GB/30 min, `openai-compatible`/`whisper-local`/`webspeech` no client-side cap. - **WhisperHost lazy-requires Node modules** the same way [src/secrets.ts](src/secrets.ts) does for `safeStorage`. Importing `child_process` / `net` / `fs` at module top would crash on mobile load. The cached `nodeApiCache` is `null` on mobile, so any host method that needs it bails with a clear "desktop only" error. - **WhisperHost has three ownership states.** `'spawned'` (this session's child handle is live; log capture works), `'adopted'` (port bound, PID sidecar matches a live PID — we started it in a previous session, no log capture), `'external'` (port bound but no sidecar match — someone else started it). Status enum gains `'external'` alongside `stopped`/`starting`/`running`/`crashed`; `running` means we own it (spawned OR adopted) and Stop works, `external` means we don't and Stop is disabled. `WhisperHost.snapshot()` returns `{ status, baseUrl, ownership, pid }` for UI consumers; `formatWhisperStatus(snap)` produces labels like "Running on http://... (adopted from previous session, pid 12345)." or "External whisper-server on http://... (not started by ReWrite).". - **Transcription never checks `WhisperHost.status()`.** [src/transcription/whisper-local.ts](src/transcription/whisper-local.ts) just asks for `host.baseUrl()` and POSTs. The HTTP API doesn't care who owns the process — if the port is reachable, transcription works. `baseUrl()` returns the URL for both `'running'` and `'external'` states. @@ -235,7 +239,7 @@ Per [.editorconfig](.editorconfig): tabs (width 4), LF, UTF-8, final newline. Ma - **Known nouns frontmatter is NOT sent to the LLM.** The vault file at `GlobalSettings.knownNounsPath` uses YAML frontmatter purely for human-readable guidance (token-cost warning, format hint). Only the body lines are parsed via `loadKnownNounsFromFile` and injected by `buildKnownNounsSystemPromptSection`. If a future change opts frontmatter in, it should be a per-vault opt-in setting, not the default. The body parser treats `#` lines and blank lines as ignored; an entry can be either bare canonical (`Anthropic`) or canonical + misheard alternates (`Hoxhunt: hawks hunt, hocks hunt`). - **PipelineHost decouples the pipeline from `ReWritePlugin`.** [src/pipeline.ts](src/pipeline.ts) reads `params.host.assistantPrompt` and `params.host.knownNouns` through the narrow `PipelineHost` interface in [src/types.ts](src/types.ts). The plugin class `implements PipelineHost`, but the pipeline never imports `ReWritePlugin` directly, which would create a cycle through the UI layer. New cross-cutting cleanup-stage inputs should extend `PipelineHost` rather than reach for the plugin object. - **New-file collisions are resolved by `insert.ts`, not the caller.** `GlobalSettings.newFileCollisionMode` is `'auto'` (silently iterate `name-1.md`, `name-2.md`, ...) or `'prompt'` (open `RenamePromptModal` defaulted to the next free path; Cancel throws `Insert canceled: file already exists.`). Threaded through `InsertParams.collisionMode` from `pipeline.ts`. The path search uses `app.vault.getAbstractFileByPath` and caps at 1000 iterations. `nextFreePath` is local to [src/insert.ts](src/insert.ts); the equivalent `deCollide` in [src/audio-persist.ts](src/audio-persist.ts) is intentionally not shared — audio always auto-iterates regardless of the setting (the file is a side-effect users keep; the new-note path is the *target* the user named). -- **Destination override does not mutate the template object.** [src/ui/modal.ts](src/ui/modal.ts) renders a per-invocation Destination control (insertMode + conditional newFile fields) and threads the result through `PipelineParams.destinationOverride`. [src/pipeline.ts](src/pipeline.ts) shallow-merges the override onto a *copy* of the template via `applyDestinationOverride` before calling `insertOutput`; the cached template and the file on disk remain untouched. The override is ephemeral: it resets when the modal closes and when the template selector changes. Not exposed in Quick Record, `runTextPipeline`, or `runAudioFilePipeline` (no UI surface). The UI is a collapsible `
` whose `` reads `"Default destination: "` (no override) or `"Custom destination: "` (override set, forced open); expand state is tracked on `ReWriteModal.destinationExpanded` so it survives the full-container re-renders that fire when the inner insertMode dropdown changes. `describeDestination(mode, folder, name)` formats the summary value (e.g. `New file: ReWrite Notes/{{date}}-note`). +- **Destination override does not mutate the template object.** [src/ui/modal.ts](src/ui/modal.ts) renders a per-invocation Destination control (insertMode + conditional newFile fields) and threads the result through `PipelineParams.destinationOverride`. [src/pipeline.ts](src/pipeline.ts) shallow-merges the override onto a *copy* of the template via `applyDestinationOverride` before calling `insertOutput`; the cached template and the file on disk remain untouched. The override is ephemeral: it resets when the modal closes and when the template selector changes. Not exposed in Quick Record, `runTextPipeline`, or `runAudioFilePipeline` (no UI surface). The UI is a collapsible `
` whose `` reads `"Destination: Default ()"` (no override) or `"Destination: Custom ()"` (override set, forced open); expand state is tracked on `ReWriteModal.destinationExpanded` so it survives the full-container re-renders that fire when the inner insertMode dropdown changes. `describeDestination(mode, folder, name)` formats the description (e.g. `New file: ReWrite Notes/{{date}}-note`). - **Quick Record floater holds its own popover.** The floater grew a third button between the timer and Stop that opens a popover-style template list ([src/ui/quick-record.ts](src/ui/quick-record.ts)). The popover is a child of the floater div, listens for outside-click via a capture-phase `document` listener and Escape via `document keydown`, and cleans up both listeners on dismiss. The popover dismisses on selection, Escape, outside click, and when `setBusy` runs (the pipeline is in flight). Selecting a template updates `controller.template` but does NOT update `lastUsedTemplateId`; that only happens after a successful completion, matching pre-popover behavior. - **Whisper status bar polls `whisperHost.status()` every 1 s** via `registerInterval`. The host has no event emitter; both [src/settings/tab.ts](src/settings/tab.ts) (re-render on Start/Stop click) and [src/ui/whisper-status-bar.ts](src/ui/whisper-status-bar.ts) (interval poll) re-read the status synchronously. If you add a third consumer, poll the same way; do not bolt on an event system. diff --git a/docs/FEATURES.md b/docs/FEATURES.md index dc454d0..b779791 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -12,6 +12,40 @@ Phase A shipped (see Done). Remaining work: - "Stop when idle for N minutes" toggle (default off; useful for the large-v3 user who doesn't want 1.5 GB resident all day). - Process supervision hardening based on real-world Phase A usage (zombies, orphans, signal handling differences across platforms). +### 2. Mobile API management UX + +Editing keys, base URLs, and model strings on mobile is rough. Two concrete issues plus one visual ask: + +- The two profile sections in [src/settings/tab.ts](../src/settings/tab.ts) render identically, so it's easy to edit the desktop profile while sitting on the phone. Add a visual differentiator (color accent, badge, or border on the active-on-this-device profile) so the mobile vs desktop section is unmistakable. +- The settings tab re-renders the entire container on dropdown changes (provider, insertMode, activeProfileOverride per [CLAUDE.md](../CLAUDE.md)'s Gotchas), which on mobile causes a visible scroll jump after committing a text field that sits next to a dropdown. Text fields already skip the re-render; audit whether any newer fields slipped into the redraw path and consider scrolling the focused row back into view after redraw. +- On mobile the on-screen keyboard frequently covers the input that just received focus. Likely needs a `scrollIntoView({ block: 'center' })` on the active control's focus/blur events, or a `keyboardWillShow`-equivalent hook through Obsidian's API. + +### 3. (Bug) Voxtral transcription model dropdown is empty + +[src/transcription/mistral-voxtral.ts](../src/transcription/mistral-voxtral.ts) does not implement `listModels`, so the Refresh button on the model field is a no-op for the Voxtral transcription provider. Mistral's `/v1/models` returns the full catalog including chat models; filter to audio/Voxtral capability (probably `id.includes('voxtral')` for a v1) when surfacing transcription model IDs. Cross-check: the LLM-side Mistral provider already uses the shared OpenAI adapter's listing, so this gap is transcription-only. + +### 4. Real-time transcription mode (live STT, no cleanup) + +Voxtral exposes a real-time STT model that doesn't accept whole-file uploads, and a few other providers have equivalents (Deepgram streaming, AssemblyAI realtime). Investigate adding an opt-in "Real-time" mode: + +- Bound to a command/shortcut only (no main-modal entry, no template, no LLM cleanup pass). +- Inserts the live transcript at the cursor as it streams, on both mobile and desktop. +- Provider-side gating: only enable for transcription providers that expose a realtime endpoint; document which. +- Honest assessment: this is lower-value than the rest of the plugin (transcript with no cleanup is what Obsidian's existing voice plugins already do). Ship only if users ask. + +### 5. (Bug) Web Speech repeats short phrases indefinitely + +The Web Speech adapter in [src/webspeech.ts](../src/webspeech.ts) currently loops short utterances: the same phrase is repeated over and over instead of producing one transcript line. Likely cause is the interim-result accumulator concatenating without deduping against final results, or the restart-on-end logic re-emitting buffered interim text. Needs investigation. Until fixed, document Web Speech as a "do not use" option in the README (or remove it from the provider dropdown). + +### 6. Daily-note template: how far can the prompt drive a structured fill? + +The current daily-note default template is a freeform cleanup. Investigate whether the system prompt can reliably lay the transcript into a real structured daily-note template with named sections (Mood / Highlights / Tasks / Tomorrow / etc.) rather than emitting prose. References: + +- https://dannb.org/blog/2022/obsidian-daily-note-template/ +- https://www.craft.do/templates/category/daily-notes + +Open questions: do we ship this as an updated default template (prompt-only), or extend `NoteTemplate` to carry a target-template path the LLM should fill into? If the latter, that's a real schema change and needs design work first. Either way, validate on at least one of the linked templates before committing to a format. + --- ## Done diff --git a/eslint.config.mts b/eslint.config.mts index b35df2b..008c445 100644 --- a/eslint.config.mts +++ b/eslint.config.mts @@ -37,7 +37,7 @@ const REWRITE_BRANDS = [ "OpenAI", "Whisper", "Anthropic", "Claude", "Google Gemini", "Gemini", - "Groq", "Mistral", + "Groq", "Mistral", "Voxtral", "Ollama", "LM Studio", "AssemblyAI", "Deepgram", "Rev.ai", "whisper.cpp", "whisper-server", "faster-whisper-server", diff --git a/src/audio-transcode.ts b/src/audio-transcode.ts new file mode 100644 index 0000000..7b6709a --- /dev/null +++ b/src/audio-transcode.ts @@ -0,0 +1,66 @@ +export const TARGET_SAMPLE_RATE = 16000; + +export async function transcodeToWavPcm(audio: Blob, targetSampleRate: number = TARGET_SAMPLE_RATE): Promise { + const input = await audio.arrayBuffer(); + const decoded = await decodeAudio(input); + const resampled = await resampleToMono(decoded, targetSampleRate); + return encodeWav16(resampled, targetSampleRate); +} + +async function decodeAudio(buffer: ArrayBuffer): Promise { + const Ctx = (window.AudioContext ?? (window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext); + if (!Ctx) throw new Error('Web Audio API is unavailable in this environment.'); + const ctx = new Ctx(); + try { + return await ctx.decodeAudioData(buffer.slice(0)); + } finally { + try { await ctx.close(); } catch { /* best effort */ } + } +} + +async function resampleToMono(input: AudioBuffer, targetSampleRate: number): Promise { + const targetLength = Math.max(1, Math.ceil(input.duration * targetSampleRate)); + const offline = new OfflineAudioContext(1, targetLength, targetSampleRate); + const source = offline.createBufferSource(); + source.buffer = input; + source.connect(offline.destination); + source.start(); + const rendered = await offline.startRendering(); + return rendered.getChannelData(0); +} + +function encodeWav16(samples: Float32Array, sampleRate: number): ArrayBuffer { + const numChannels = 1; + const bytesPerSample = 2; + const blockAlign = numChannels * bytesPerSample; + const byteRate = sampleRate * blockAlign; + const dataSize = samples.length * bytesPerSample; + const buffer = new ArrayBuffer(44 + dataSize); + const view = new DataView(buffer); + writeAscii(view, 0, 'RIFF'); + view.setUint32(4, 36 + dataSize, true); + writeAscii(view, 8, 'WAVE'); + writeAscii(view, 12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, numChannels, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, byteRate, true); + view.setUint16(32, blockAlign, true); + view.setUint16(34, bytesPerSample * 8, true); + writeAscii(view, 36, 'data'); + view.setUint32(40, dataSize, true); + let offset = 44; + for (let i = 0; i < samples.length; i++) { + const s = Math.max(-1, Math.min(1, samples[i] ?? 0)); + view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true); + offset += 2; + } + return buffer; +} + +function writeAscii(view: DataView, offset: number, value: string): void { + for (let i = 0; i < value.length; i++) { + view.setUint8(offset + i, value.charCodeAt(i)); + } +} diff --git a/src/pipeline.ts b/src/pipeline.ts index b391649..207a3c8 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,6 +1,7 @@ import { App, Notice } from 'obsidian'; import { DestinationOverride, EnvironmentProfile, GlobalSettings, NoteTemplate, PipelineHost } from './types'; import { createTranscriptionProvider } from './transcription'; +import { validateRecording } from './transcription/limits'; import { createLLMProvider } from './llm'; import { insertOutput, InsertResult } from './insert'; import { persistAudio } from './audio-persist'; @@ -11,7 +12,7 @@ import { buildKnownNounsSystemPromptSection } from './known-nouns'; export type PipelineStage = 'persist-audio' | 'transcribe' | 'cleanup' | 'insert'; export type PipelineSource = - | { kind: 'audio'; audio: Blob; sourcePath?: string } + | { kind: 'audio'; audio: Blob; sourcePath?: string; durationMs?: number } | { kind: 'paste'; text: string } | { kind: 'webspeech'; transcript: string } | { kind: 'text'; text: string }; @@ -89,6 +90,7 @@ async function collectTranscript(params: PipelineParams): Promise { case 'webspeech': return source.transcript; case 'audio': { + validateRecording(source.audio.size, source.durationMs, params.profile.transcriptionProvider); params.onStage?.('transcribe'); const provider = createTranscriptionProvider(params.profile.transcriptionProvider); return provider.transcribe(source.audio, params.profile.transcriptionConfig, params.signal); diff --git a/src/recorder.ts b/src/recorder.ts index 45b5321..cb578d7 100644 --- a/src/recorder.ts +++ b/src/recorder.ts @@ -8,8 +8,6 @@ export interface RecorderResult { durationMs: number; } -const MAX_RECORDING_BYTES = 25 * 1024 * 1024; - const WEBM_FIRST: string[] = [ 'audio/webm;codecs=opus', 'audio/webm', @@ -125,12 +123,6 @@ export class Recorder { const firstChunk = this.chunks[0]; const type = this.mimeType || (firstChunk ? firstChunk.type : '') || 'audio/webm'; const blob = new Blob(this.chunks, { type }); - if (blob.size > MAX_RECORDING_BYTES) { - const mb = Math.round(blob.size / 1024 / 1024); - throw new Error( - `Recording is ${mb}MB which exceeds the 25MB transcription limit. Please record a shorter clip.`, - ); - } return { blob, mimeType: type, durationMs: this.accumulatedMs }; } diff --git a/src/settings/tab.ts b/src/settings/tab.ts index 558870b..f14570c 100644 --- a/src/settings/tab.ts +++ b/src/settings/tab.ts @@ -29,6 +29,7 @@ const TRANSCRIPTION_OPTIONS: Array<{ id: TranscriptionProviderID; label: string; { id: 'assemblyai', label: 'AssemblyAI' }, { id: 'deepgram', label: 'Deepgram' }, { id: 'revai', label: 'Rev.ai' }, + { id: 'mistral-voxtral', label: 'Mistral Voxtral' }, { id: 'webspeech', label: 'Web Speech (browser)' }, { id: 'whisper-local', label: 'Local whisper.cpp (desktop only)', desktopOnly: true }, ]; @@ -871,6 +872,8 @@ function transcriptionModelHint(id: TranscriptionProviderID): string { return 'e.g. nova-2 or nova-3'; case 'revai': return 'Optional transcriber name'; + case 'mistral-voxtral': + return 'e.g. voxtral-mini-latest or voxtral-small-latest'; case 'openai-compatible': return 'Whichever model your local server exposes'; case 'whisper-local': diff --git a/src/transcription/index.ts b/src/transcription/index.ts index 56d7d13..897573e 100644 --- a/src/transcription/index.ts +++ b/src/transcription/index.ts @@ -3,6 +3,7 @@ import { createOpenAITranscription } from './openai'; import { createAssemblyAITranscription } from './assemblyai'; import { createDeepgramTranscription } from './deepgram'; import { createRevAITranscription } from './revai'; +import { createMistralVoxtralTranscription } from './mistral-voxtral'; import { createWebSpeechTranscription } from './webspeech'; import { createWhisperLocalTranscription } from './whisper-local'; @@ -31,6 +32,8 @@ export function createTranscriptionProvider( return createDeepgramTranscription(); case 'revai': return createRevAITranscription(); + case 'mistral-voxtral': + return createMistralVoxtralTranscription(); case 'webspeech': return createWebSpeechTranscription(); case 'whisper-local': diff --git a/src/transcription/limits.ts b/src/transcription/limits.ts new file mode 100644 index 0000000..46db2bb --- /dev/null +++ b/src/transcription/limits.ts @@ -0,0 +1,81 @@ +import { TranscriptionProviderID } from '../types'; + +// Per-provider audio upload limits. Sources: +// - OpenAI Whisper: 25 MB (platform.openai.com/docs/guides/speech-to-text) +// - Groq: 25 MB on free tier; higher on paid tiers but the UI can't tell, so use the conservative number (console.groq.com/docs/speech-to-text) +// - AssemblyAI: 5 GB / 10 h (assemblyai.com/docs/faq) +// - Deepgram: 2 GB sync (developers.deepgram.com) +// - Rev.ai: 2 GB multipart / 17 h (docs.rev.ai/api/asynchronous) +// - Mistral Voxtral: 1 GB / 30 min (docs.mistral.ai/api/endpoint/audio/transcriptions) +// - openai-compatible / whisper-local / webspeech: no client-side cap +export interface TranscriptionLimits { + readonly maxBytes?: number; + readonly maxDurationMs?: number; +} + +const MB = 1024 * 1024; +const GB = 1024 * MB; +const MIN = 60 * 1000; +const HOUR = 60 * MIN; + +export function getTranscriptionLimits(id: TranscriptionProviderID): TranscriptionLimits { + switch (id) { + case 'openai': + return { maxBytes: 25 * MB }; + case 'groq': + return { maxBytes: 25 * MB }; + case 'assemblyai': + return { maxBytes: 5 * GB, maxDurationMs: 10 * HOUR }; + case 'deepgram': + return { maxBytes: 2 * GB }; + case 'revai': + return { maxBytes: 2 * GB, maxDurationMs: 17 * HOUR }; + case 'mistral-voxtral': + return { maxBytes: 1 * GB, maxDurationMs: 30 * MIN }; + case 'openai-compatible': + case 'whisper-local': + case 'webspeech': + return {}; + } +} + +export function transcriptionProviderLabel(id: TranscriptionProviderID): string { + switch (id) { + case 'openai': return 'OpenAI Whisper'; + case 'groq': return 'Groq'; + case 'assemblyai': return 'AssemblyAI'; + case 'deepgram': return 'Deepgram'; + case 'revai': return 'Rev.ai'; + case 'mistral-voxtral': return 'Mistral Voxtral'; + case 'openai-compatible': return 'OpenAI-compatible'; + case 'whisper-local': return 'Local whisper.cpp'; + case 'webspeech': return 'Web Speech'; + } +} + +export function validateRecording( + blobSize: number, + durationMs: number | undefined, + id: TranscriptionProviderID, +): void { + const limits = getTranscriptionLimits(id); + const label = transcriptionProviderLabel(id); + if (limits.maxBytes !== undefined && blobSize > limits.maxBytes) { + const sizeMb = Math.round(blobSize / MB); + const limitMb = Math.round(limits.maxBytes / MB); + throw new Error( + `Recording is ${sizeMb} MB which exceeds the ${label} ${limitMb} MB limit. Save the audio elsewhere or switch transcription provider in settings.`, + ); + } + if ( + limits.maxDurationMs !== undefined && + durationMs !== undefined && + durationMs > limits.maxDurationMs + ) { + const mins = Math.round(durationMs / MIN); + const limitMins = Math.round(limits.maxDurationMs / MIN); + throw new Error( + `Recording is ${mins} min which exceeds the ${label} ${limitMins} min limit. Switch transcription provider in settings.`, + ); + } +} diff --git a/src/transcription/mistral-voxtral.ts b/src/transcription/mistral-voxtral.ts new file mode 100644 index 0000000..72c05cd --- /dev/null +++ b/src/transcription/mistral-voxtral.ts @@ -0,0 +1,65 @@ +import { TranscriptionConfig } from '../types'; +import { MultipartPart, multipartPost, ProviderError } from '../http'; +import { transcodeToWavPcm } from '../audio-transcode'; +import { TranscriptionProvider } from './index'; + +// Mistral Voxtral diverges from the OpenAI Whisper shape on three points, so it +// gets its own adapter rather than dispatching through openai.ts: +// 1. Response is JSON only ({ text, segments, ... }); no response_format=text. +// 2. WebM/Opus is not an accepted input format, so the recorded blob is always +// transcoded to 16 kHz mono WAV before upload (same path as whisper-local). +// 30 min of 16 kHz mono 16-bit PCM is ~57 MB, well under the 1 GB cap. +// 3. /v1/models does not document audio-model surfacing, so listModels is omitted. +const VOXTRAL_ENDPOINT = 'https://api.mistral.ai/v1/audio/transcriptions'; + +interface VoxtralResponse { + text?: unknown; +} + +export function createMistralVoxtralTranscription(): TranscriptionProvider { + return { + id: 'mistral-voxtral', + requiresAudio: true, + async transcribe( + audio: Blob, + config: TranscriptionConfig, + signal?: AbortSignal, + ): Promise { + if (!config.apiKey) throw new Error('mistral-voxtral: API key is not configured'); + if (!config.model) throw new Error('mistral-voxtral: model is not configured'); + let wavBuffer: ArrayBuffer; + try { + wavBuffer = await transcodeToWavPcm(audio); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + throw new ProviderError('mistral-voxtral', 0, '', `Failed to transcode audio to WAV for Voxtral: ${msg}`); + } + const parts: MultipartPart[] = [ + { + type: 'file', + name: 'file', + filename: 'audio.wav', + contentType: 'audio/wav', + data: wavBuffer, + }, + { type: 'text', name: 'model', value: config.model }, + ]; + if (config.language) { + parts.push({ type: 'text', name: 'language', value: config.language }); + } + const res = await multipartPost( + 'mistral-voxtral', + VOXTRAL_ENDPOINT, + parts, + { Authorization: `Bearer ${config.apiKey}` }, + signal, + ); + const body = res.json as VoxtralResponse; + const text = typeof body.text === 'string' ? body.text : ''; + if (!text) { + throw new ProviderError('mistral-voxtral', res.status, res.text, 'Voxtral returned no text.'); + } + return text.trim(); + }, + }; +} diff --git a/src/transcription/whisper-local.ts b/src/transcription/whisper-local.ts index 7e5ac8c..b71e8c1 100644 --- a/src/transcription/whisper-local.ts +++ b/src/transcription/whisper-local.ts @@ -1,5 +1,6 @@ import { TranscriptionConfig } from '../types'; import { MultipartPart, multipartPost, ProviderError } from '../http'; +import { transcodeToWavPcm } from '../audio-transcode'; import { TranscriptionProvider } from './index'; import type { WhisperHost } from '../whisper-host'; @@ -9,8 +10,6 @@ export function bindWhisperHost(h: WhisperHost): void { host = h; } -const TARGET_SAMPLE_RATE = 16000; - export function createWhisperLocalTranscription(): TranscriptionProvider { return { id: 'whisper-local', @@ -29,7 +28,7 @@ export function createWhisperLocalTranscription(): TranscriptionProvider { } let wavBuffer: ArrayBuffer; try { - wavBuffer = await transcodeToWavPcm(audio, TARGET_SAMPLE_RATE); + wavBuffer = await transcodeToWavPcm(audio); } catch (e) { const msg = e instanceof Error ? e.message : String(e); throw new ProviderError('whisper-local', 0, '', `Failed to transcode audio to WAV for whisper.cpp: ${msg}`); @@ -58,68 +57,3 @@ export function createWhisperLocalTranscription(): TranscriptionProvider { }, }; } - -async function transcodeToWavPcm(audio: Blob, targetSampleRate: number): Promise { - const input = await audio.arrayBuffer(); - const decoded = await decodeAudio(input); - const resampled = await resampleToMono(decoded, targetSampleRate); - return encodeWav16(resampled, targetSampleRate); -} - -async function decodeAudio(buffer: ArrayBuffer): Promise { - const Ctx = (window.AudioContext ?? (window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext); - if (!Ctx) throw new Error('Web Audio API is unavailable in this environment.'); - const ctx = new Ctx(); - try { - return await ctx.decodeAudioData(buffer.slice(0)); - } finally { - try { await ctx.close(); } catch { /* best effort */ } - } -} - -async function resampleToMono(input: AudioBuffer, targetSampleRate: number): Promise { - const targetLength = Math.max(1, Math.ceil(input.duration * targetSampleRate)); - const offline = new OfflineAudioContext(1, targetLength, targetSampleRate); - const source = offline.createBufferSource(); - source.buffer = input; - source.connect(offline.destination); - source.start(); - const rendered = await offline.startRendering(); - return rendered.getChannelData(0); -} - -function encodeWav16(samples: Float32Array, sampleRate: number): ArrayBuffer { - const numChannels = 1; - const bytesPerSample = 2; - const blockAlign = numChannels * bytesPerSample; - const byteRate = sampleRate * blockAlign; - const dataSize = samples.length * bytesPerSample; - const buffer = new ArrayBuffer(44 + dataSize); - const view = new DataView(buffer); - writeAscii(view, 0, 'RIFF'); - view.setUint32(4, 36 + dataSize, true); - writeAscii(view, 8, 'WAVE'); - writeAscii(view, 12, 'fmt '); - view.setUint32(16, 16, true); - view.setUint16(20, 1, true); - view.setUint16(22, numChannels, true); - view.setUint32(24, sampleRate, true); - view.setUint32(28, byteRate, true); - view.setUint16(32, blockAlign, true); - view.setUint16(34, bytesPerSample * 8, true); - writeAscii(view, 36, 'data'); - view.setUint32(40, dataSize, true); - let offset = 44; - for (let i = 0; i < samples.length; i++) { - const s = Math.max(-1, Math.min(1, samples[i] ?? 0)); - view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true); - offset += 2; - } - return buffer; -} - -function writeAscii(view: DataView, offset: number, value: string): void { - for (let i = 0; i < value.length; i++) { - view.setUint8(offset + i, value.charCodeAt(i)); - } -} diff --git a/src/types.ts b/src/types.ts index 5e84287..2ed41dc 100644 --- a/src/types.ts +++ b/src/types.ts @@ -5,6 +5,7 @@ export type TranscriptionProviderID = | 'assemblyai' | 'deepgram' | 'revai' + | 'mistral-voxtral' | 'webspeech' | 'whisper-local'; diff --git a/src/ui/modal.ts b/src/ui/modal.ts index 68ab00e..0518d8e 100644 --- a/src/ui/modal.ts +++ b/src/ui/modal.ts @@ -177,11 +177,11 @@ export class ReWriteModal extends Modal { }); const summary = details.createEl('summary', { cls: 'rewrite-destination-summary' }); - const summaryLabel = hasOverride ? 'Custom destination' : 'Default destination'; - summary.createSpan({ cls: 'rewrite-destination-summary-label', text: `${summaryLabel}: ` }); + const summaryQualifier = hasOverride ? 'Custom' : 'Default'; + summary.createSpan({ cls: 'rewrite-destination-summary-label', text: 'Destination: ' }); summary.createSpan({ cls: 'rewrite-destination-summary-value', - text: describeDestination(effectiveMode, effectiveFolder, effectiveName), + text: `${summaryQualifier} (${describeDestination(effectiveMode, effectiveFolder, effectiveName)})`, }); const body = details.createDiv({ cls: 'rewrite-destination-body' }); @@ -404,7 +404,7 @@ export class ReWriteModal extends Modal { if (!this.recorder) throw new Error('No active recording.'); const result = await this.recorder.stop(); this.recorder = null; - return { kind: 'audio', audio: result.blob }; + return { kind: 'audio', audio: result.blob, durationMs: result.durationMs }; } private startTimerLoop(timerEl: HTMLElement, isWebSpeech: boolean): void { diff --git a/src/ui/quick-record.ts b/src/ui/quick-record.ts index a574531..bb99150 100644 --- a/src/ui/quick-record.ts +++ b/src/ui/quick-record.ts @@ -113,7 +113,7 @@ export class QuickRecordController { if (!this.recorder) throw new Error('No active recording.'); const result = await this.recorder.stop(); this.recorder = null; - return { kind: 'audio', audio: result.blob }; + return { kind: 'audio', audio: result.blob, durationMs: result.durationMs }; } private stopTimer(): void { diff --git a/src/ui/setup-card.ts b/src/ui/setup-card.ts index a92932d..46240a5 100644 --- a/src/ui/setup-card.ts +++ b/src/ui/setup-card.ts @@ -8,6 +8,7 @@ const TRANSCRIPTION_OPTIONS: Array<{ id: TranscriptionProviderID; label: string; { id: 'assemblyai', label: 'AssemblyAI' }, { id: 'deepgram', label: 'Deepgram' }, { id: 'revai', label: 'Rev.ai' }, + { id: 'mistral-voxtral', label: 'Mistral Voxtral' }, { id: 'webspeech', label: 'Web Speech (browser)' }, { id: 'whisper-local', label: 'Local whisper.cpp (desktop only)', desktopOnly: true }, ]; @@ -189,6 +190,8 @@ function modelPlaceholderForTranscription(id: TranscriptionProviderID): string { return 'e.g. nova-2 or nova-3'; case 'revai': return 'Optional transcriber name'; + case 'mistral-voxtral': + return 'e.g. voxtral-mini-latest or voxtral-small-latest'; case 'openai-compatible': return 'Whichever model your local server exposes'; case 'whisper-local':