Various (adding Voxtral, various)

This commit is contained in:
WiseGuru 2026-05-27 13:29:51 -07:00
parent f9a6bc5272
commit c207d5444d
16 changed files with 278 additions and 89 deletions

View file

@ -1,7 +1,8 @@
{
"permissions": {
"allow": [
"Bash(npm run *)"
"Bash(npm run *)",
"WebSearch"
]
}
}

View file

@ -47,7 +47,8 @@ src/
├── http.ts # requestUrl wrappers: jsonPost/jsonGet/multipartPost + ProviderError
├── platform.ts # Active-profile resolver + MediaRecorder/Web Speech availability probes
├── secrets.ts # safeStorage (desktop) + plaintext fallback (mobile) for API keys
├── recorder.ts # MediaRecorder state machine + getBestMimeType + 25 MB size guard
├── recorder.ts # MediaRecorder state machine + getBestMimeType (no size cap; per-provider limits live in transcription/limits.ts)
├── audio-transcode.ts # WebAudio decode + resample to 16 kHz mono PCM WAV (shared by whisper-local and mistral-voxtral)
├── webspeech.ts # SpeechRecognition wrapper
├── pipeline.ts # transcribe → cleanup → insert orchestrator
├── insert.ts # cursor/newFile/append + {{date}}/{{time}} expansion
@ -70,10 +71,12 @@ src/
│ └── whisper-status-bar.ts # Status-bar dot for whisper-host start/stop (desktop + whisper-local profile only)
├── transcription/
│ ├── index.ts # TranscriptionProvider interface + createTranscriptionProvider()
│ ├── limits.ts # Per-provider maxBytes/maxDurationMs + validateRecording (called from pipeline)
│ ├── openai.ts # Whisper-shape POST (also used by openai-compatible + groq)
│ ├── assemblyai.ts # upload → submit → poll
│ ├── deepgram.ts # single POST
│ ├── revai.ts # submit → poll → fetch text
│ ├── mistral-voxtral.ts # Mistral Voxtral STT (JSON response; always transcodes to WAV)
│ ├── webspeech.ts # adapter; throws "should not be called" (see Gotchas)
│ └── whisper-local.ts # Thin shim that POSTs to the WhisperHost-managed local server
└── llm/
@ -88,7 +91,7 @@ src/
[src/pipeline.ts](src/pipeline.ts) runs four stages with `onStage` callbacks for UI:
1. **Persist audio** (`audio` source only): writes the raw `Blob` to the vault via [src/audio-persist.ts](src/audio-persist.ts) before transcription, so the user keeps the recording even if later stages fail. Path resolution: when `settings.attachmentsFolderPath` is set, the file goes under that folder with manual de-collision (`-1`, `-2`, ...); when empty, the path comes from `app.fileManager.getAvailablePathForAttachment(filename)`, which respects Obsidian's own attachments setting. Filename is `ReWrite-YYYY-MM-DD-HHmmss.<ext>` with the extension derived from the blob's mime type (`webm` / `m4a` / `ogg` / `wav` / `mp3`, default `webm`). Failure is non-fatal: a Notice fires and transcription proceeds. The resolved path is later prepended to the cleaned output as `![[<path>]]\n\n` before insertion.
2. **Transcribe**: `audio``createTranscriptionProvider(profile.transcriptionProvider).transcribe(blob, config)`. Skipped when the source is `paste` or `text` (input passes through). Short-circuited when the source is `webspeech` (the transcript was already captured live by `src/webspeech.ts` during recording).
2. **Transcribe**: `audio``createTranscriptionProvider(profile.transcriptionProvider).transcribe(blob, config)`. Skipped when the source is `paste` or `text` (input passes through). Short-circuited when the source is `webspeech` (the transcript was already captured live by `src/webspeech.ts` during recording). Just before dispatching, `validateRecording(blobSize, durationMs, providerId)` from [src/transcription/limits.ts](src/transcription/limits.ts) throws a friendly per-provider error if the recording exceeds the provider's documented byte or duration cap. Because validation runs after `persist-audio`, the user keeps the saved file and can switch providers + reprocess from the vault.
3. **Cleanup**: `createLLMProvider(profile.llmProvider).complete(systemPrompt, transcript, config)`. The system prompt is the template prompt, optionally augmented with an `## Ad-hoc instructions` block when the wake-name scan ([src/wake-name.ts](src/wake-name.ts)) extracts directives from the transcript, and a `## Known nouns` block when `plugin.knownNouns` is non-empty (see Assistant prompt and Known nouns sections below). On error, the (possibly stripped) transcript is copied to the clipboard before re-throwing, so the user keeps their words.
4. **Insert**: `src/insert.ts` routes to `cursor` / `newFile` / `append` per the template. `cursor` falls back to `append` when no editor is active; `append` falls back to `newFile` when no markdown file exists. `{{date}}` / `{{time}}` in filename templates expand via Obsidian's `moment`. The modal's per-invocation Destination control overrides `insertMode` / `newFileFolder` / `newFileNameTemplate` via `PipelineParams.destinationOverride`; the override is shallow-merged onto a copy of the template before the insert call, so the template file on disk is never mutated.
@ -98,7 +101,7 @@ The `PipelineSource` union has four variants: `audio` (recorded blob, optional `
## Provider system
[src/transcription/index.ts](src/transcription/index.ts) and [src/llm/index.ts](src/llm/index.ts) each define a small interface plus a `create...Provider(id)` factory. Provider families share one adapter file: OpenAI Whisper, `openai-compatible`, and Groq all dispatch into [src/transcription/openai.ts](src/transcription/openai.ts) with different base URLs; OpenAI GPT, `openai-compatible`, and Mistral all dispatch into [src/llm/openai.ts](src/llm/openai.ts).
[src/transcription/index.ts](src/transcription/index.ts) and [src/llm/index.ts](src/llm/index.ts) each define a small interface plus a `create...Provider(id)` factory. Provider families share one adapter file where the API shapes match: OpenAI Whisper, `openai-compatible`, and Groq all dispatch into [src/transcription/openai.ts](src/transcription/openai.ts) with different base URLs; OpenAI GPT, `openai-compatible`, and Mistral all dispatch into [src/llm/openai.ts](src/llm/openai.ts). Mistral Voxtral does NOT share with `openai.ts` — see [src/transcription/mistral-voxtral.ts](src/transcription/mistral-voxtral.ts) — because Voxtral's response is JSON-only (no `response_format=text`), it rejects WebM input (so the blob is always transcoded to 16 kHz mono WAV via [src/audio-transcode.ts](src/audio-transcode.ts)), and `/v1/models` doesn't surface audio-model IDs the same way it surfaces chat models.
API keys are stored per profile on `EnvironmentProfile.transcriptionConfig.apiKey` / `llmConfig.apiKey`. Two slots per profile, one for transcription and one for the LLM. No global by-family map; the desktop and mobile profiles each carry their own keys even when both use the same provider (deliberate: per-profile keys make per-function usage tracking easier). Persistence is in [src/secrets.ts](src/secrets.ts) using the key IDs `profile:desktop:transcription`, `profile:desktop:llm`, `profile:mobile:transcription`, `profile:mobile:llm`.
@ -110,7 +113,7 @@ Providers may optionally implement `listModels(config, signal)` returning a stri
`start()` validates the binary and model paths exist via `fs.existsSync`, probes the port via `net.createServer().listen(port)` to detect conflicts, spawns the user's whisper-server with `child_process.spawn`, captures stdout/stderr into a 1 MB ring buffer, then polls `net.createConnection` against the port every 250 ms for up to 5 s before declaring `'running'`. Any failure transitions status to `'crashed'` with the log tail surfaced in the error message. `stop()` sends SIGTERM, waits up to 3 s, then SIGKILL.
The `whisper-local` transcription provider ([src/transcription/whisper-local.ts](src/transcription/whisper-local.ts)) is a thin shim that POSTs to `http://127.0.0.1:<port>/inference` (whisper.cpp's native server route, not OpenAI's `/v1/audio/transcriptions`). Before sending, the recorded blob is transcoded to 16 kHz mono 16-bit PCM WAV via Web Audio API (`AudioContext.decodeAudioData``OfflineAudioContext` for resample/downmix → hand-written RIFF header). whisper.cpp's server cannot decode WebM/Opus (the MediaRecorder default) without a custom ffmpeg-enabled build, so the transcode is mandatory — it is *not* applied to any other transcription provider. The shim grabs the host reference via `bindWhisperHost(host)` called from [src/main.ts](src/main.ts) on load. If the host status is anything other than `'running'`, the adapter throws a `ProviderError` with a "start it from settings" message; the pipeline surfaces that as a Notice. No API key is collected for this provider (no auth, no settings field).
The `whisper-local` transcription provider ([src/transcription/whisper-local.ts](src/transcription/whisper-local.ts)) is a thin shim that POSTs to `http://127.0.0.1:<port>/inference` (whisper.cpp's native server route, not OpenAI's `/v1/audio/transcriptions`). Before sending, the recorded blob is transcoded to 16 kHz mono 16-bit PCM WAV via `transcodeToWavPcm` in [src/audio-transcode.ts](src/audio-transcode.ts) (Web Audio API: `AudioContext.decodeAudioData``OfflineAudioContext` for resample/downmix → hand-written RIFF header). whisper.cpp's server cannot decode WebM/Opus (the MediaRecorder default) without a custom ffmpeg-enabled build, so the transcode is mandatory. The same helper is reused by [src/transcription/mistral-voxtral.ts](src/transcription/mistral-voxtral.ts) (Voxtral also rejects WebM). If you add a third audio-uploading provider that needs WAV, import from `audio-transcode.ts` rather than duplicating the helpers. The shim grabs the host reference via `bindWhisperHost(host)` called from [src/main.ts](src/main.ts) on load. If the host status is anything other than `'running'`, the adapter throws a `ProviderError` with a "start it from settings" message; the pipeline surfaces that as a Notice. No API key is collected for this provider (no auth, no settings field).
Mobile compatibility: `WhisperHost` and the `whisper-local` option are guarded everywhere by `Platform.isDesktop`. Node modules (`child_process`, `net`, `fs`) are lazy-required inside the `Platform.isDesktop` branch, mirroring the [src/secrets.ts](src/secrets.ts) `getSafeStorage` pattern; on mobile the host is inert and the provider option is filtered out of dropdowns in [src/settings/tab.ts](src/settings/tab.ts) and [src/ui/setup-card.ts](src/ui/setup-card.ts).
@ -224,6 +227,7 @@ Per [.editorconfig](.editorconfig): tabs (width 4), LF, UTF-8, final newline. Ma
- **Frontmatter parsing uses `parseYaml` from Obsidian, not the metadata cache.** The metadata cache is async and may not be populated for newly created files; reading content via `app.vault.read(file)`, splitting off the leading `---...---` block, and parsing it with `parseYaml` is synchronous-enough and works immediately after `app.vault.create`.
- **Quick Record uses a custom floating div, not a `Notice`.** Obsidian `Notice` does not support real interactive buttons. The floater is a `position: fixed` div on `document.body`, owned by `QuickRecordController`, with `cancel()` wired into `onunload`.
- **`secrets.json.nosync` uses the `.nosync` suffix on purpose**: iCloud Drive natively skips any file or folder whose name ends in `.nosync`. The README documents per-tool sync exclusion for other tools.
- **Per-provider recording limits live in [src/transcription/limits.ts](src/transcription/limits.ts), not the recorder.** [src/recorder.ts](src/recorder.ts) does not cap recordings at any size; `validateRecording(blobSize, durationMs, providerId)` runs in [src/pipeline.ts](src/pipeline.ts) between the `persist-audio` and `transcribe` stages, throwing a friendly provider-attributed error if the recording exceeds the documented byte or duration ceiling. Both modal and Quick Record thread the recorder's `durationMs` onto the `audio` pipeline source so the duration check has data; the reprocess flow ([src/ui/audio-source.ts](src/ui/audio-source.ts)) omits `durationMs` (no cheap way to measure an arbitrary vault file), so reprocess only triggers the byte check. Limits source: `openai`/`groq` 25 MB, `assemblyai` 5 GB/10 h, `deepgram` 2 GB, `revai` 2 GB/17 h, `mistral-voxtral` 1 GB/30 min, `openai-compatible`/`whisper-local`/`webspeech` no client-side cap.
- **WhisperHost lazy-requires Node modules** the same way [src/secrets.ts](src/secrets.ts) does for `safeStorage`. Importing `child_process` / `net` / `fs` at module top would crash on mobile load. The cached `nodeApiCache` is `null` on mobile, so any host method that needs it bails with a clear "desktop only" error.
- **WhisperHost has three ownership states.** `'spawned'` (this session's child handle is live; log capture works), `'adopted'` (port bound, PID sidecar matches a live PID — we started it in a previous session, no log capture), `'external'` (port bound but no sidecar match — someone else started it). Status enum gains `'external'` alongside `stopped`/`starting`/`running`/`crashed`; `running` means we own it (spawned OR adopted) and Stop works, `external` means we don't and Stop is disabled. `WhisperHost.snapshot()` returns `{ status, baseUrl, ownership, pid }` for UI consumers; `formatWhisperStatus(snap)` produces labels like "Running on http://... (adopted from previous session, pid 12345)." or "External whisper-server on http://... (not started by ReWrite).".
- **Transcription never checks `WhisperHost.status()`.** [src/transcription/whisper-local.ts](src/transcription/whisper-local.ts) just asks for `host.baseUrl()` and POSTs. The HTTP API doesn't care who owns the process — if the port is reachable, transcription works. `baseUrl()` returns the URL for both `'running'` and `'external'` states.
@ -235,7 +239,7 @@ Per [.editorconfig](.editorconfig): tabs (width 4), LF, UTF-8, final newline. Ma
- **Known nouns frontmatter is NOT sent to the LLM.** The vault file at `GlobalSettings.knownNounsPath` uses YAML frontmatter purely for human-readable guidance (token-cost warning, format hint). Only the body lines are parsed via `loadKnownNounsFromFile` and injected by `buildKnownNounsSystemPromptSection`. If a future change opts frontmatter in, it should be a per-vault opt-in setting, not the default. The body parser treats `#` lines and blank lines as ignored; an entry can be either bare canonical (`Anthropic`) or canonical + misheard alternates (`Hoxhunt: hawks hunt, hocks hunt`).
- **PipelineHost decouples the pipeline from `ReWritePlugin`.** [src/pipeline.ts](src/pipeline.ts) reads `params.host.assistantPrompt` and `params.host.knownNouns` through the narrow `PipelineHost` interface in [src/types.ts](src/types.ts). The plugin class `implements PipelineHost`, but the pipeline never imports `ReWritePlugin` directly, which would create a cycle through the UI layer. New cross-cutting cleanup-stage inputs should extend `PipelineHost` rather than reach for the plugin object.
- **New-file collisions are resolved by `insert.ts`, not the caller.** `GlobalSettings.newFileCollisionMode` is `'auto'` (silently iterate `name-1.md`, `name-2.md`, ...) or `'prompt'` (open `RenamePromptModal` defaulted to the next free path; Cancel throws `Insert canceled: file already exists.`). Threaded through `InsertParams.collisionMode` from `pipeline.ts`. The path search uses `app.vault.getAbstractFileByPath` and caps at 1000 iterations. `nextFreePath` is local to [src/insert.ts](src/insert.ts); the equivalent `deCollide` in [src/audio-persist.ts](src/audio-persist.ts) is intentionally not shared — audio always auto-iterates regardless of the setting (the file is a side-effect users keep; the new-note path is the *target* the user named).
- **Destination override does not mutate the template object.** [src/ui/modal.ts](src/ui/modal.ts) renders a per-invocation Destination control (insertMode + conditional newFile fields) and threads the result through `PipelineParams.destinationOverride`. [src/pipeline.ts](src/pipeline.ts) shallow-merges the override onto a *copy* of the template via `applyDestinationOverride` before calling `insertOutput`; the cached template and the file on disk remain untouched. The override is ephemeral: it resets when the modal closes and when the template selector changes. Not exposed in Quick Record, `runTextPipeline`, or `runAudioFilePipeline` (no UI surface). The UI is a collapsible `<details>` whose `<summary>` reads `"Default destination: <description>"` (no override) or `"Custom destination: <description>"` (override set, forced open); expand state is tracked on `ReWriteModal.destinationExpanded` so it survives the full-container re-renders that fire when the inner insertMode dropdown changes. `describeDestination(mode, folder, name)` formats the summary value (e.g. `New file: ReWrite Notes/{{date}}-note`).
- **Destination override does not mutate the template object.** [src/ui/modal.ts](src/ui/modal.ts) renders a per-invocation Destination control (insertMode + conditional newFile fields) and threads the result through `PipelineParams.destinationOverride`. [src/pipeline.ts](src/pipeline.ts) shallow-merges the override onto a *copy* of the template via `applyDestinationOverride` before calling `insertOutput`; the cached template and the file on disk remain untouched. The override is ephemeral: it resets when the modal closes and when the template selector changes. Not exposed in Quick Record, `runTextPipeline`, or `runAudioFilePipeline` (no UI surface). The UI is a collapsible `<details>` whose `<summary>` reads `"Destination: Default (<description>)"` (no override) or `"Destination: Custom (<description>)"` (override set, forced open); expand state is tracked on `ReWriteModal.destinationExpanded` so it survives the full-container re-renders that fire when the inner insertMode dropdown changes. `describeDestination(mode, folder, name)` formats the description (e.g. `New file: ReWrite Notes/{{date}}-note`).
- **Quick Record floater holds its own popover.** The floater grew a third button between the timer and Stop that opens a popover-style template list ([src/ui/quick-record.ts](src/ui/quick-record.ts)). The popover is a child of the floater div, listens for outside-click via a capture-phase `document` listener and Escape via `document keydown`, and cleans up both listeners on dismiss. The popover dismisses on selection, Escape, outside click, and when `setBusy` runs (the pipeline is in flight). Selecting a template updates `controller.template` but does NOT update `lastUsedTemplateId`; that only happens after a successful completion, matching pre-popover behavior.
- **Whisper status bar polls `whisperHost.status()` every 1 s** via `registerInterval`. The host has no event emitter; both [src/settings/tab.ts](src/settings/tab.ts) (re-render on Start/Stop click) and [src/ui/whisper-status-bar.ts](src/ui/whisper-status-bar.ts) (interval poll) re-read the status synchronously. If you add a third consumer, poll the same way; do not bolt on an event system.

View file

@ -12,6 +12,40 @@ Phase A shipped (see Done). Remaining work:
- "Stop when idle for N minutes" toggle (default off; useful for the large-v3 user who doesn't want 1.5 GB resident all day).
- Process supervision hardening based on real-world Phase A usage (zombies, orphans, signal handling differences across platforms).
### 2. Mobile API management UX
Editing keys, base URLs, and model strings on mobile is rough. Two concrete issues plus one visual ask:
- The two profile sections in [src/settings/tab.ts](../src/settings/tab.ts) render identically, so it's easy to edit the desktop profile while sitting on the phone. Add a visual differentiator (color accent, badge, or border on the active-on-this-device profile) so the mobile vs desktop section is unmistakable.
- The settings tab re-renders the entire container on dropdown changes (provider, insertMode, activeProfileOverride per [CLAUDE.md](../CLAUDE.md)'s Gotchas), which on mobile causes a visible scroll jump after committing a text field that sits next to a dropdown. Text fields already skip the re-render; audit whether any newer fields slipped into the redraw path and consider scrolling the focused row back into view after redraw.
- On mobile the on-screen keyboard frequently covers the input that just received focus. Likely needs a `scrollIntoView({ block: 'center' })` on the active control's focus/blur events, or a `keyboardWillShow`-equivalent hook through Obsidian's API.
### 3. (Bug) Voxtral transcription model dropdown is empty
[src/transcription/mistral-voxtral.ts](../src/transcription/mistral-voxtral.ts) does not implement `listModels`, so the Refresh button on the model field is a no-op for the Voxtral transcription provider. Mistral's `/v1/models` returns the full catalog including chat models; filter to audio/Voxtral capability (probably `id.includes('voxtral')` for a v1) when surfacing transcription model IDs. Cross-check: the LLM-side Mistral provider already uses the shared OpenAI adapter's listing, so this gap is transcription-only.
### 4. Real-time transcription mode (live STT, no cleanup)
Voxtral exposes a real-time STT model that doesn't accept whole-file uploads, and a few other providers have equivalents (Deepgram streaming, AssemblyAI realtime). Investigate adding an opt-in "Real-time" mode:
- Bound to a command/shortcut only (no main-modal entry, no template, no LLM cleanup pass).
- Inserts the live transcript at the cursor as it streams, on both mobile and desktop.
- Provider-side gating: only enable for transcription providers that expose a realtime endpoint; document which.
- Honest assessment: this is lower-value than the rest of the plugin (transcript with no cleanup is what Obsidian's existing voice plugins already do). Ship only if users ask.
### 5. (Bug) Web Speech repeats short phrases indefinitely
The Web Speech adapter in [src/webspeech.ts](../src/webspeech.ts) currently loops short utterances: the same phrase is repeated over and over instead of producing one transcript line. Likely cause is the interim-result accumulator concatenating without deduping against final results, or the restart-on-end logic re-emitting buffered interim text. Needs investigation. Until fixed, document Web Speech as a "do not use" option in the README (or remove it from the provider dropdown).
### 6. Daily-note template: how far can the prompt drive a structured fill?
The current daily-note default template is a freeform cleanup. Investigate whether the system prompt can reliably lay the transcript into a real structured daily-note template with named sections (Mood / Highlights / Tasks / Tomorrow / etc.) rather than emitting prose. References:
- https://dannb.org/blog/2022/obsidian-daily-note-template/
- https://www.craft.do/templates/category/daily-notes
Open questions: do we ship this as an updated default template (prompt-only), or extend `NoteTemplate` to carry a target-template path the LLM should fill into? If the latter, that's a real schema change and needs design work first. Either way, validate on at least one of the linked templates before committing to a format.
---
## Done

View file

@ -37,7 +37,7 @@ const REWRITE_BRANDS = [
"OpenAI", "Whisper",
"Anthropic", "Claude",
"Google Gemini", "Gemini",
"Groq", "Mistral",
"Groq", "Mistral", "Voxtral",
"Ollama", "LM Studio",
"AssemblyAI", "Deepgram", "Rev.ai",
"whisper.cpp", "whisper-server", "faster-whisper-server",

66
src/audio-transcode.ts Normal file
View file

@ -0,0 +1,66 @@
export const TARGET_SAMPLE_RATE = 16000;
export async function transcodeToWavPcm(audio: Blob, targetSampleRate: number = TARGET_SAMPLE_RATE): Promise<ArrayBuffer> {
const input = await audio.arrayBuffer();
const decoded = await decodeAudio(input);
const resampled = await resampleToMono(decoded, targetSampleRate);
return encodeWav16(resampled, targetSampleRate);
}
async function decodeAudio(buffer: ArrayBuffer): Promise<AudioBuffer> {
const Ctx = (window.AudioContext ?? (window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext);
if (!Ctx) throw new Error('Web Audio API is unavailable in this environment.');
const ctx = new Ctx();
try {
return await ctx.decodeAudioData(buffer.slice(0));
} finally {
try { await ctx.close(); } catch { /* best effort */ }
}
}
async function resampleToMono(input: AudioBuffer, targetSampleRate: number): Promise<Float32Array> {
const targetLength = Math.max(1, Math.ceil(input.duration * targetSampleRate));
const offline = new OfflineAudioContext(1, targetLength, targetSampleRate);
const source = offline.createBufferSource();
source.buffer = input;
source.connect(offline.destination);
source.start();
const rendered = await offline.startRendering();
return rendered.getChannelData(0);
}
function encodeWav16(samples: Float32Array, sampleRate: number): ArrayBuffer {
const numChannels = 1;
const bytesPerSample = 2;
const blockAlign = numChannels * bytesPerSample;
const byteRate = sampleRate * blockAlign;
const dataSize = samples.length * bytesPerSample;
const buffer = new ArrayBuffer(44 + dataSize);
const view = new DataView(buffer);
writeAscii(view, 0, 'RIFF');
view.setUint32(4, 36 + dataSize, true);
writeAscii(view, 8, 'WAVE');
writeAscii(view, 12, 'fmt ');
view.setUint32(16, 16, true);
view.setUint16(20, 1, true);
view.setUint16(22, numChannels, true);
view.setUint32(24, sampleRate, true);
view.setUint32(28, byteRate, true);
view.setUint16(32, blockAlign, true);
view.setUint16(34, bytesPerSample * 8, true);
writeAscii(view, 36, 'data');
view.setUint32(40, dataSize, true);
let offset = 44;
for (let i = 0; i < samples.length; i++) {
const s = Math.max(-1, Math.min(1, samples[i] ?? 0));
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
offset += 2;
}
return buffer;
}
function writeAscii(view: DataView, offset: number, value: string): void {
for (let i = 0; i < value.length; i++) {
view.setUint8(offset + i, value.charCodeAt(i));
}
}

View file

@ -1,6 +1,7 @@
import { App, Notice } from 'obsidian';
import { DestinationOverride, EnvironmentProfile, GlobalSettings, NoteTemplate, PipelineHost } from './types';
import { createTranscriptionProvider } from './transcription';
import { validateRecording } from './transcription/limits';
import { createLLMProvider } from './llm';
import { insertOutput, InsertResult } from './insert';
import { persistAudio } from './audio-persist';
@ -11,7 +12,7 @@ import { buildKnownNounsSystemPromptSection } from './known-nouns';
export type PipelineStage = 'persist-audio' | 'transcribe' | 'cleanup' | 'insert';
export type PipelineSource =
| { kind: 'audio'; audio: Blob; sourcePath?: string }
| { kind: 'audio'; audio: Blob; sourcePath?: string; durationMs?: number }
| { kind: 'paste'; text: string }
| { kind: 'webspeech'; transcript: string }
| { kind: 'text'; text: string };
@ -89,6 +90,7 @@ async function collectTranscript(params: PipelineParams): Promise<string> {
case 'webspeech':
return source.transcript;
case 'audio': {
validateRecording(source.audio.size, source.durationMs, params.profile.transcriptionProvider);
params.onStage?.('transcribe');
const provider = createTranscriptionProvider(params.profile.transcriptionProvider);
return provider.transcribe(source.audio, params.profile.transcriptionConfig, params.signal);

View file

@ -8,8 +8,6 @@ export interface RecorderResult {
durationMs: number;
}
const MAX_RECORDING_BYTES = 25 * 1024 * 1024;
const WEBM_FIRST: string[] = [
'audio/webm;codecs=opus',
'audio/webm',
@ -125,12 +123,6 @@ export class Recorder {
const firstChunk = this.chunks[0];
const type = this.mimeType || (firstChunk ? firstChunk.type : '') || 'audio/webm';
const blob = new Blob(this.chunks, { type });
if (blob.size > MAX_RECORDING_BYTES) {
const mb = Math.round(blob.size / 1024 / 1024);
throw new Error(
`Recording is ${mb}MB which exceeds the 25MB transcription limit. Please record a shorter clip.`,
);
}
return { blob, mimeType: type, durationMs: this.accumulatedMs };
}

View file

@ -29,6 +29,7 @@ const TRANSCRIPTION_OPTIONS: Array<{ id: TranscriptionProviderID; label: string;
{ id: 'assemblyai', label: 'AssemblyAI' },
{ id: 'deepgram', label: 'Deepgram' },
{ id: 'revai', label: 'Rev.ai' },
{ id: 'mistral-voxtral', label: 'Mistral Voxtral' },
{ id: 'webspeech', label: 'Web Speech (browser)' },
{ id: 'whisper-local', label: 'Local whisper.cpp (desktop only)', desktopOnly: true },
];
@ -871,6 +872,8 @@ function transcriptionModelHint(id: TranscriptionProviderID): string {
return 'e.g. nova-2 or nova-3';
case 'revai':
return 'Optional transcriber name';
case 'mistral-voxtral':
return 'e.g. voxtral-mini-latest or voxtral-small-latest';
case 'openai-compatible':
return 'Whichever model your local server exposes';
case 'whisper-local':

View file

@ -3,6 +3,7 @@ import { createOpenAITranscription } from './openai';
import { createAssemblyAITranscription } from './assemblyai';
import { createDeepgramTranscription } from './deepgram';
import { createRevAITranscription } from './revai';
import { createMistralVoxtralTranscription } from './mistral-voxtral';
import { createWebSpeechTranscription } from './webspeech';
import { createWhisperLocalTranscription } from './whisper-local';
@ -31,6 +32,8 @@ export function createTranscriptionProvider(
return createDeepgramTranscription();
case 'revai':
return createRevAITranscription();
case 'mistral-voxtral':
return createMistralVoxtralTranscription();
case 'webspeech':
return createWebSpeechTranscription();
case 'whisper-local':

View file

@ -0,0 +1,81 @@
import { TranscriptionProviderID } from '../types';
// Per-provider audio upload limits. Sources:
// - OpenAI Whisper: 25 MB (platform.openai.com/docs/guides/speech-to-text)
// - Groq: 25 MB on free tier; higher on paid tiers but the UI can't tell, so use the conservative number (console.groq.com/docs/speech-to-text)
// - AssemblyAI: 5 GB / 10 h (assemblyai.com/docs/faq)
// - Deepgram: 2 GB sync (developers.deepgram.com)
// - Rev.ai: 2 GB multipart / 17 h (docs.rev.ai/api/asynchronous)
// - Mistral Voxtral: 1 GB / 30 min (docs.mistral.ai/api/endpoint/audio/transcriptions)
// - openai-compatible / whisper-local / webspeech: no client-side cap
export interface TranscriptionLimits {
readonly maxBytes?: number;
readonly maxDurationMs?: number;
}
const MB = 1024 * 1024;
const GB = 1024 * MB;
const MIN = 60 * 1000;
const HOUR = 60 * MIN;
export function getTranscriptionLimits(id: TranscriptionProviderID): TranscriptionLimits {
switch (id) {
case 'openai':
return { maxBytes: 25 * MB };
case 'groq':
return { maxBytes: 25 * MB };
case 'assemblyai':
return { maxBytes: 5 * GB, maxDurationMs: 10 * HOUR };
case 'deepgram':
return { maxBytes: 2 * GB };
case 'revai':
return { maxBytes: 2 * GB, maxDurationMs: 17 * HOUR };
case 'mistral-voxtral':
return { maxBytes: 1 * GB, maxDurationMs: 30 * MIN };
case 'openai-compatible':
case 'whisper-local':
case 'webspeech':
return {};
}
}
export function transcriptionProviderLabel(id: TranscriptionProviderID): string {
switch (id) {
case 'openai': return 'OpenAI Whisper';
case 'groq': return 'Groq';
case 'assemblyai': return 'AssemblyAI';
case 'deepgram': return 'Deepgram';
case 'revai': return 'Rev.ai';
case 'mistral-voxtral': return 'Mistral Voxtral';
case 'openai-compatible': return 'OpenAI-compatible';
case 'whisper-local': return 'Local whisper.cpp';
case 'webspeech': return 'Web Speech';
}
}
export function validateRecording(
blobSize: number,
durationMs: number | undefined,
id: TranscriptionProviderID,
): void {
const limits = getTranscriptionLimits(id);
const label = transcriptionProviderLabel(id);
if (limits.maxBytes !== undefined && blobSize > limits.maxBytes) {
const sizeMb = Math.round(blobSize / MB);
const limitMb = Math.round(limits.maxBytes / MB);
throw new Error(
`Recording is ${sizeMb} MB which exceeds the ${label} ${limitMb} MB limit. Save the audio elsewhere or switch transcription provider in settings.`,
);
}
if (
limits.maxDurationMs !== undefined &&
durationMs !== undefined &&
durationMs > limits.maxDurationMs
) {
const mins = Math.round(durationMs / MIN);
const limitMins = Math.round(limits.maxDurationMs / MIN);
throw new Error(
`Recording is ${mins} min which exceeds the ${label} ${limitMins} min limit. Switch transcription provider in settings.`,
);
}
}

View file

@ -0,0 +1,65 @@
import { TranscriptionConfig } from '../types';
import { MultipartPart, multipartPost, ProviderError } from '../http';
import { transcodeToWavPcm } from '../audio-transcode';
import { TranscriptionProvider } from './index';
// Mistral Voxtral diverges from the OpenAI Whisper shape on three points, so it
// gets its own adapter rather than dispatching through openai.ts:
// 1. Response is JSON only ({ text, segments, ... }); no response_format=text.
// 2. WebM/Opus is not an accepted input format, so the recorded blob is always
// transcoded to 16 kHz mono WAV before upload (same path as whisper-local).
// 30 min of 16 kHz mono 16-bit PCM is ~57 MB, well under the 1 GB cap.
// 3. /v1/models does not document audio-model surfacing, so listModels is omitted.
const VOXTRAL_ENDPOINT = 'https://api.mistral.ai/v1/audio/transcriptions';
interface VoxtralResponse {
text?: unknown;
}
export function createMistralVoxtralTranscription(): TranscriptionProvider {
return {
id: 'mistral-voxtral',
requiresAudio: true,
async transcribe(
audio: Blob,
config: TranscriptionConfig,
signal?: AbortSignal,
): Promise<string> {
if (!config.apiKey) throw new Error('mistral-voxtral: API key is not configured');
if (!config.model) throw new Error('mistral-voxtral: model is not configured');
let wavBuffer: ArrayBuffer;
try {
wavBuffer = await transcodeToWavPcm(audio);
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
throw new ProviderError('mistral-voxtral', 0, '', `Failed to transcode audio to WAV for Voxtral: ${msg}`);
}
const parts: MultipartPart[] = [
{
type: 'file',
name: 'file',
filename: 'audio.wav',
contentType: 'audio/wav',
data: wavBuffer,
},
{ type: 'text', name: 'model', value: config.model },
];
if (config.language) {
parts.push({ type: 'text', name: 'language', value: config.language });
}
const res = await multipartPost(
'mistral-voxtral',
VOXTRAL_ENDPOINT,
parts,
{ Authorization: `Bearer ${config.apiKey}` },
signal,
);
const body = res.json as VoxtralResponse;
const text = typeof body.text === 'string' ? body.text : '';
if (!text) {
throw new ProviderError('mistral-voxtral', res.status, res.text, 'Voxtral returned no text.');
}
return text.trim();
},
};
}

View file

@ -1,5 +1,6 @@
import { TranscriptionConfig } from '../types';
import { MultipartPart, multipartPost, ProviderError } from '../http';
import { transcodeToWavPcm } from '../audio-transcode';
import { TranscriptionProvider } from './index';
import type { WhisperHost } from '../whisper-host';
@ -9,8 +10,6 @@ export function bindWhisperHost(h: WhisperHost): void {
host = h;
}
const TARGET_SAMPLE_RATE = 16000;
export function createWhisperLocalTranscription(): TranscriptionProvider {
return {
id: 'whisper-local',
@ -29,7 +28,7 @@ export function createWhisperLocalTranscription(): TranscriptionProvider {
}
let wavBuffer: ArrayBuffer;
try {
wavBuffer = await transcodeToWavPcm(audio, TARGET_SAMPLE_RATE);
wavBuffer = await transcodeToWavPcm(audio);
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
throw new ProviderError('whisper-local', 0, '', `Failed to transcode audio to WAV for whisper.cpp: ${msg}`);
@ -58,68 +57,3 @@ export function createWhisperLocalTranscription(): TranscriptionProvider {
},
};
}
async function transcodeToWavPcm(audio: Blob, targetSampleRate: number): Promise<ArrayBuffer> {
const input = await audio.arrayBuffer();
const decoded = await decodeAudio(input);
const resampled = await resampleToMono(decoded, targetSampleRate);
return encodeWav16(resampled, targetSampleRate);
}
async function decodeAudio(buffer: ArrayBuffer): Promise<AudioBuffer> {
const Ctx = (window.AudioContext ?? (window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext);
if (!Ctx) throw new Error('Web Audio API is unavailable in this environment.');
const ctx = new Ctx();
try {
return await ctx.decodeAudioData(buffer.slice(0));
} finally {
try { await ctx.close(); } catch { /* best effort */ }
}
}
async function resampleToMono(input: AudioBuffer, targetSampleRate: number): Promise<Float32Array> {
const targetLength = Math.max(1, Math.ceil(input.duration * targetSampleRate));
const offline = new OfflineAudioContext(1, targetLength, targetSampleRate);
const source = offline.createBufferSource();
source.buffer = input;
source.connect(offline.destination);
source.start();
const rendered = await offline.startRendering();
return rendered.getChannelData(0);
}
function encodeWav16(samples: Float32Array, sampleRate: number): ArrayBuffer {
const numChannels = 1;
const bytesPerSample = 2;
const blockAlign = numChannels * bytesPerSample;
const byteRate = sampleRate * blockAlign;
const dataSize = samples.length * bytesPerSample;
const buffer = new ArrayBuffer(44 + dataSize);
const view = new DataView(buffer);
writeAscii(view, 0, 'RIFF');
view.setUint32(4, 36 + dataSize, true);
writeAscii(view, 8, 'WAVE');
writeAscii(view, 12, 'fmt ');
view.setUint32(16, 16, true);
view.setUint16(20, 1, true);
view.setUint16(22, numChannels, true);
view.setUint32(24, sampleRate, true);
view.setUint32(28, byteRate, true);
view.setUint16(32, blockAlign, true);
view.setUint16(34, bytesPerSample * 8, true);
writeAscii(view, 36, 'data');
view.setUint32(40, dataSize, true);
let offset = 44;
for (let i = 0; i < samples.length; i++) {
const s = Math.max(-1, Math.min(1, samples[i] ?? 0));
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
offset += 2;
}
return buffer;
}
function writeAscii(view: DataView, offset: number, value: string): void {
for (let i = 0; i < value.length; i++) {
view.setUint8(offset + i, value.charCodeAt(i));
}
}

View file

@ -5,6 +5,7 @@ export type TranscriptionProviderID =
| 'assemblyai'
| 'deepgram'
| 'revai'
| 'mistral-voxtral'
| 'webspeech'
| 'whisper-local';

View file

@ -177,11 +177,11 @@ export class ReWriteModal extends Modal {
});
const summary = details.createEl('summary', { cls: 'rewrite-destination-summary' });
const summaryLabel = hasOverride ? 'Custom destination' : 'Default destination';
summary.createSpan({ cls: 'rewrite-destination-summary-label', text: `${summaryLabel}: ` });
const summaryQualifier = hasOverride ? 'Custom' : 'Default';
summary.createSpan({ cls: 'rewrite-destination-summary-label', text: 'Destination: ' });
summary.createSpan({
cls: 'rewrite-destination-summary-value',
text: describeDestination(effectiveMode, effectiveFolder, effectiveName),
text: `${summaryQualifier} (${describeDestination(effectiveMode, effectiveFolder, effectiveName)})`,
});
const body = details.createDiv({ cls: 'rewrite-destination-body' });
@ -404,7 +404,7 @@ export class ReWriteModal extends Modal {
if (!this.recorder) throw new Error('No active recording.');
const result = await this.recorder.stop();
this.recorder = null;
return { kind: 'audio', audio: result.blob };
return { kind: 'audio', audio: result.blob, durationMs: result.durationMs };
}
private startTimerLoop(timerEl: HTMLElement, isWebSpeech: boolean): void {

View file

@ -113,7 +113,7 @@ export class QuickRecordController {
if (!this.recorder) throw new Error('No active recording.');
const result = await this.recorder.stop();
this.recorder = null;
return { kind: 'audio', audio: result.blob };
return { kind: 'audio', audio: result.blob, durationMs: result.durationMs };
}
private stopTimer(): void {

View file

@ -8,6 +8,7 @@ const TRANSCRIPTION_OPTIONS: Array<{ id: TranscriptionProviderID; label: string;
{ id: 'assemblyai', label: 'AssemblyAI' },
{ id: 'deepgram', label: 'Deepgram' },
{ id: 'revai', label: 'Rev.ai' },
{ id: 'mistral-voxtral', label: 'Mistral Voxtral' },
{ id: 'webspeech', label: 'Web Speech (browser)' },
{ id: 'whisper-local', label: 'Local whisper.cpp (desktop only)', desktopOnly: true },
];
@ -189,6 +190,8 @@ function modelPlaceholderForTranscription(id: TranscriptionProviderID): string {
return 'e.g. nova-2 or nova-3';
case 'revai':
return 'Optional transcriber name';
case 'mistral-voxtral':
return 'e.g. voxtral-mini-latest or voxtral-small-latest';
case 'openai-compatible':
return 'Whichever model your local server exposes';
case 'whisper-local':