audio profiles

This commit is contained in:
Michael Gorini 2026-04-29 08:25:26 +02:00
parent d3f4bf8eeb
commit e83e26412e
7 changed files with 496 additions and 10 deletions

View file

@ -0,0 +1,145 @@
import type { CaptureSettings, CaptureProfile } from "./settings";
export interface ResolvedAudioProfile {
micGainDb: number;
systemGainDb: number;
micMixWeight: number;
systemMixWeight: number;
highpassHz: number | null;
lowpassHz: number | null;
limiter: boolean;
limiterCeiling: number;
}
const PROFILES: Record<Exclude<CaptureProfile, "custom">, ResolvedAudioProfile> = {
transcription: {
micGainDb: 6.6,
systemGainDb: 0,
micMixWeight: 1.65,
systemMixWeight: 0.75,
highpassHz: 80,
lowpassHz: 7000,
limiter: true,
limiterCeiling: 0.95,
},
balanced: {
micGainDb: 3.0,
systemGainDb: 0,
micMixWeight: 1.2,
systemMixWeight: 0.9,
highpassHz: 80,
lowpassHz: 12000,
limiter: true,
limiterCeiling: 0.95,
},
natural: {
micGainDb: 0,
systemGainDb: 0,
micMixWeight: 1.0,
systemMixWeight: 1.0,
highpassHz: null,
lowpassHz: null,
limiter: true,
limiterCeiling: 0.98,
},
};
function dbToLinear(db: number): number {
return Math.round(Math.pow(10, db / 20) * 1000) / 1000;
}
export function resolveAudioProfile(settings: CaptureSettings): ResolvedAudioProfile {
if (settings.captureProfile !== "custom") {
return PROFILES[settings.captureProfile];
}
return {
micGainDb: settings.micGainDb,
systemGainDb: settings.systemGainDb,
micMixWeight: 1.0,
systemMixWeight: 1.0,
highpassHz: settings.noiseSuppression ? 80 : null,
lowpassHz: settings.noiseSuppression ? 12000 : null,
limiter: settings.limiter,
limiterCeiling: 0.95,
};
}
function buildMicFilters(profile: ResolvedAudioProfile): string {
const parts: string[] = [];
if (profile.highpassHz !== null) parts.push(`highpass=f=${profile.highpassHz}`);
if (profile.lowpassHz !== null) parts.push(`lowpass=f=${profile.lowpassHz}`);
const linearGain = dbToLinear(profile.micGainDb);
if (linearGain !== 1) parts.push(`volume=${linearGain}`);
return parts.join(",");
}
function buildSystemFilters(profile: ResolvedAudioProfile): string {
const linearGain = dbToLinear(profile.systemGainDb);
if (linearGain !== 1) return `volume=${linearGain}`;
return "";
}
function buildLimiter(profile: ResolvedAudioProfile): string {
if (!profile.limiter) return "";
return `alimiter=limit=${profile.limiterCeiling}`;
}
export interface ResolvedFilterChain {
filterArgs: string[];
mapArgs: string[];
/** Human-readable preview of the effective FFmpeg filter for the settings UI. */
filterPreview: string;
}
export function resolveAudioFilterChain(settings: CaptureSettings, dualInput: boolean): ResolvedFilterChain {
const profile = resolveAudioProfile(settings);
if (dualInput) {
const micFilters = buildMicFilters(profile);
const sysFilters = buildSystemFilters(profile);
const micChain = micFilters ? `[0:a]${micFilters}[mic]` : `[0:a]anull[mic]`;
const sysChain = sysFilters ? `[1:a]${sysFilters}[sys]` : `[1:a]anull[sys]`;
const amix = `[mic][sys]amix=inputs=2:duration=longest:weights='${profile.micMixWeight} ${profile.systemMixWeight}':normalize=0`;
const limiter = buildLimiter(profile);
const tail = limiter ? `,${limiter}[aout]` : `[aout]`;
const filterComplex = `${micChain};${sysChain};${amix}${tail}`;
return {
filterArgs: ["-filter_complex", filterComplex],
mapArgs: ["-map", "[aout]"],
filterPreview: filterComplex,
};
}
const parts: string[] = [];
const micFilters = buildMicFilters(profile);
if (micFilters) parts.push(micFilters);
const limiter = buildLimiter(profile);
if (limiter) parts.push(limiter);
const af = parts.length > 0 ? parts.join(",") : "anull";
return {
filterArgs: ["-af", af],
mapArgs: ["-map", "0:a"],
filterPreview: af,
};
}
export function getCaptureProfileDescription(profile: CaptureProfile): string {
switch (profile) {
case "transcription":
return "Aggressive voice boost and band-pass filtering. Best transcription accuracy, but can sound distorted on some hardware.";
case "balanced":
return "Moderate voice boost with gentle filtering. Good transcription accuracy with cleaner playback.";
case "natural":
return "Minimal processing, clean audio. Best for playback. May reduce transcription accuracy in noisy environments.";
case "custom":
return "Full manual control over gain, filtering, and limiter.";
}
}
export const CAPTURE_PROFILE_LABELS: Record<CaptureProfile, string> = {
transcription: "Transcription (aggressive)",
balanced: "Balanced (recommended)",
natural: "Natural (clean)",
custom: "Custom",
};

View file

@ -1,6 +1,7 @@
import { getSelectedSummaryModel, type SummaryProviderId } from "./providers";
export type CaptureBackend = "auto" | "avfoundation" | "dshow" | "pulse" | "alsa";
export type CaptureProfile = "transcription" | "balanced" | "natural" | "custom";
export interface CaptureSettings {
ffmpegPath: string;
@ -13,6 +14,11 @@ export interface CaptureSettings {
channels: 1 | 2;
bitrateKbps: number;
segmentDurationSeconds: number;
captureProfile: CaptureProfile;
micGainDb: number;
systemGainDb: number;
noiseSuppression: boolean;
limiter: boolean;
}
export interface TranscriptionSettings {
@ -78,6 +84,11 @@ export const DEFAULT_SETTINGS_V2: PluginSettingsV2 = {
channels: 2,
bitrateKbps: 160,
segmentDurationSeconds: 20,
captureProfile: "balanced",
micGainDb: 0,
systemGainDb: 0,
noiseSuppression: true,
limiter: true,
},
transcription: {
whisperRepoPath: "",
@ -122,6 +133,12 @@ function clampInteger(value: unknown, fallback: number, min: number, max: number
return Math.max(min, Math.min(max, Math.floor(parsed)));
}
function clampFloat(value: unknown, fallback: number, min: number, max: number): number {
const parsed = Number(value);
if (!Number.isFinite(parsed)) return fallback;
return Math.round(Math.max(min, Math.min(max, parsed)) * 10) / 10;
}
function asString(value: unknown, fallback = ""): string {
return typeof value === "string" ? value : fallback;
}
@ -133,6 +150,7 @@ function asBoolean(value: unknown, fallback: boolean): boolean {
function normalizeCaptureSettings(raw: unknown): CaptureSettings {
const input = (raw ?? {}) as Partial<CaptureSettings>;
const backend = input.backend;
const profile = input.captureProfile;
return {
ffmpegPath: asString(input.ffmpegPath),
backend:
@ -147,6 +165,14 @@ function normalizeCaptureSettings(raw: unknown): CaptureSettings {
channels: clampInteger(input.channels, DEFAULT_SETTINGS_V2.capture.channels, 1, 2) === 1 ? 1 : 2,
bitrateKbps: clampInteger(input.bitrateKbps, DEFAULT_SETTINGS_V2.capture.bitrateKbps, 64, 320),
segmentDurationSeconds: clampInteger(input.segmentDurationSeconds, DEFAULT_SETTINGS_V2.capture.segmentDurationSeconds, 5, 300),
captureProfile:
profile === "transcription" || profile === "balanced" || profile === "natural" || profile === "custom"
? profile
: DEFAULT_SETTINGS_V2.capture.captureProfile,
micGainDb: clampFloat(input.micGainDb, DEFAULT_SETTINGS_V2.capture.micGainDb, -12, 12),
systemGainDb: clampFloat(input.systemGainDb, DEFAULT_SETTINGS_V2.capture.systemGainDb, -12, 12),
noiseSuppression: asBoolean(input.noiseSuppression, DEFAULT_SETTINGS_V2.capture.noiseSuppression),
limiter: asBoolean(input.limiter, DEFAULT_SETTINGS_V2.capture.limiter),
};
}

View file

@ -1,6 +1,7 @@
import type { CaptureSettings } from "../../domain/settings";
import { requireNodeModule } from "../node";
import { resolveCaptureInputs, type ResolvedCaptureInputs } from "./captureUtils";
import { resolveAudioFilterChain } from "../../domain/captureProfiles";
interface ChildProcessModule {
spawn: typeof import("node:child_process").spawn;
@ -59,16 +60,7 @@ export class AudioCaptureAdapter {
const { spawn } = requireNodeModule<ChildProcessModule>("child_process");
const segmentPattern = path.join(options.segmentsDir, "segment-%04d.mp3");
const shouldMix = Boolean(resolvedInputs.micSpec && resolvedInputs.systemSpec);
const speechPrep = "highpass=f=80,lowpass=f=7000";
const filterArgs = shouldMix
? [
"-filter_complex",
`[0:a]${speechPrep},volume=2.15[mic];` +
`[1:a]volume=1.0[sys];` +
`[mic][sys]amix=inputs=2:duration=longest:weights='1.65 0.75':normalize=0,alimiter=limit=0.95[aout]`,
]
: ["-af", `${speechPrep},volume=2.15,alimiter=limit=0.95`];
const mapArgs = shouldMix ? ["-map", "[aout]"] : ["-map", "0:a"];
const { filterArgs, mapArgs } = resolveAudioFilterChain(options.settings, shouldMix);
const sampleRate = Math.max(8_000, Math.floor(options.settings.sampleRateHz));
const channels = options.settings.channels === 1 ? 1 : 2;
const bitrate = Math.max(64, Math.floor(options.settings.bitrateKbps));

View file

@ -8,6 +8,7 @@ import {
type PluginSettingsV2,
} from "../../domain/settings";
import type { DiagnosticCheck, DiagnosticsReport } from "../../domain/diagnostics";
import { CAPTURE_PROFILE_LABELS } from "../../domain/captureProfiles";
import { requireNodeModule } from "../node";
import { resolveCaptureInputs } from "../adapters/captureUtils";
import { WhisperTranscriptionAdapter } from "../adapters/TranscriptionAdapter";
@ -160,6 +161,19 @@ export class DiagnosticsService {
remediation: "Set a dedicated output folder if you want generated notes grouped together.",
});
const profileLabel = CAPTURE_PROFILE_LABELS[settings.capture.captureProfile];
const transcriptionWithNonStandardRate =
settings.capture.captureProfile === "transcription" && settings.capture.sampleRateHz !== 48000;
addCheck({
id: "capture-profile",
label: "Audio processing profile",
severity: transcriptionWithNonStandardRate ? "warning" : "ok",
detail: transcriptionWithNonStandardRate
? `Profile: ${profileLabel}. Sample rate ${settings.capture.sampleRateHz} Hz with the Transcription profile can cause extra resampling artifacts. Consider switching to 48000 Hz.`
: `Profile: ${profileLabel}. Active on next recording.`,
remediation: "Switch to Balanced or Natural profile if audio sounds robotic, or set sample rate to 48000 Hz.",
});
const blockingIssueIds = checks.filter((check) => check.severity === "error").map((check) => check.id);
const warningIds = checks.filter((check) => check.severity === "warning").map((check) => check.id);
return {

View file

@ -4,6 +4,7 @@ import type { SessionController } from "../application/SessionController";
import type { DashboardSnapshot } from "../domain/dashboard";
import type { SessionListItem } from "../domain/session";
import { isCoreConfigured, resolveCaptureBackend, type PluginSettingsV2 } from "../domain/settings";
import { CAPTURE_PROFILE_LABELS, getCaptureProfileDescription, resolveAudioFilterChain } from "../domain/captureProfiles";
import { VaultAdapter } from "../infrastructure/adapters/VaultAdapter";
import { requireNodeModule } from "../infrastructure/node";
import {
@ -530,6 +531,119 @@ export class ResonanceNextSettingTab extends PluginSettingTab {
}));
})
);
// ── Audio processing ───────────────────────────────────────────────────
const audioSection = this.createGuideSection(container, {
badge: "Audio",
title: uiCopy.capture.profileSection,
intro: uiCopy.capture.profileIntro,
});
// Troubleshooting callout
const tsBox = audioSection.createDiv({ cls: "rxn-inline-note is-info" });
tsBox.createEl("strong", { text: uiCopy.capture.troubleshooting.title });
const tsList = tsBox.createEl("ul", { cls: "rxn-guide-list" });
tsList.createEl("li", { text: uiCopy.capture.troubleshooting.robotic });
tsList.createEl("li", { text: uiCopy.capture.troubleshooting.inaccurate });
tsList.createEl("li", { text: uiCopy.capture.troubleshooting.noisy });
// Profile dropdown
const profileDesc = getCaptureProfileDescription(settings.capture.captureProfile);
new Setting(audioSection)
.setName(uiCopy.capture.profileLabel)
.setDesc(profileDesc)
.addDropdown((dropdown) => {
for (const [value, label] of Object.entries(CAPTURE_PROFILE_LABELS)) {
dropdown.addOption(value, label);
}
dropdown.setValue(settings.capture.captureProfile);
dropdown.onChange(async (value) => {
await this.options.saveSettings((current) => ({
...current,
capture: { ...current.capture, captureProfile: value as PluginSettingsV2["capture"]["captureProfile"] },
}));
await this.display();
});
});
// Custom profile controls (only shown when profile = custom)
if (settings.capture.captureProfile === "custom") {
const customSection = this.createGuideSection(container, {
badge: "Custom",
title: uiCopy.capture.customSection,
intro: uiCopy.capture.customIntro,
});
new Setting(customSection)
.setName(uiCopy.capture.micGainLabel)
.setDesc(uiCopy.capture.micGainDesc)
.addText((text) =>
text
.setPlaceholder("0")
.setValue(String(settings.capture.micGainDb))
.onChange(async (value) => {
const parsed = parseFloat(value);
if (!Number.isFinite(parsed)) return;
await this.options.saveSettings((current) => ({
...current,
capture: { ...current.capture, micGainDb: Math.max(-12, Math.min(12, parsed)) },
}));
})
);
new Setting(customSection)
.setName(uiCopy.capture.systemGainLabel)
.setDesc(uiCopy.capture.systemGainDesc)
.addText((text) =>
text
.setPlaceholder("0")
.setValue(String(settings.capture.systemGainDb))
.onChange(async (value) => {
const parsed = parseFloat(value);
if (!Number.isFinite(parsed)) return;
await this.options.saveSettings((current) => ({
...current,
capture: { ...current.capture, systemGainDb: Math.max(-12, Math.min(12, parsed)) },
}));
})
);
new Setting(customSection)
.setName(uiCopy.capture.noiseSuppressionLabel)
.setDesc(uiCopy.capture.noiseSuppressionDesc)
.addToggle((toggle) =>
toggle.setValue(settings.capture.noiseSuppression).onChange(async (value) => {
await this.options.saveSettings((current) => ({
...current,
capture: { ...current.capture, noiseSuppression: value },
}));
})
);
new Setting(customSection)
.setName(uiCopy.capture.limiterLabel)
.setDesc(uiCopy.capture.limiterDesc)
.addToggle((toggle) =>
toggle.setValue(settings.capture.limiter).onChange(async (value) => {
await this.options.saveSettings((current) => ({
...current,
capture: { ...current.capture, limiter: value },
}));
})
);
}
// Filter preview — always visible
const hasDual = Boolean(settings.capture.systemDevice || settings.capture.systemLabel);
const { filterPreview } = resolveAudioFilterChain(settings.capture, hasDual);
const previewSetting = new Setting(audioSection)
.setName(uiCopy.capture.filterPreviewLabel)
.setDesc(uiCopy.capture.filterPreviewDesc);
const previewEl = previewSetting.controlEl.createEl("code", {
text: filterPreview,
cls: "rxn-filter-preview",
});
previewEl.setAttribute("title", filterPreview);
}
private async renderTranscriptionTab(container: HTMLElement) {

View file

@ -79,4 +79,28 @@ export const uiCopy = {
modelDetected: "Whisper model path updated.",
quickTestFailedPrefix: "Quick test failed",
},
capture: {
profileSection: "Audio processing",
profileIntro: "Choose how the plugin processes your microphone audio before sending it to Whisper.",
profileLabel: "Capture profile",
profileDesc: "Controls how aggressively audio is processed for transcription.",
customSection: "Custom audio controls",
customIntro: "Fine-tune gain, filtering, and limiter when using the Custom profile.",
micGainLabel: "Microphone gain (dB)",
micGainDesc: "Boost or cut mic volume. Range: -12 to +12.",
systemGainLabel: "System audio gain (dB)",
systemGainDesc: "Boost or cut system audio. Range: -12 to +12.",
noiseSuppressionLabel: "Noise suppression",
noiseSuppressionDesc: "Apply highpass and lowpass filters to reduce non-speech frequencies.",
limiterLabel: "Audio limiter",
limiterDesc: "Prevent clipping by capping the output level.",
filterPreviewLabel: "FFmpeg filter preview",
filterPreviewDesc: "Read-only preview of the filter that will be sent to FFmpeg.",
troubleshooting: {
title: "Troubleshooting",
robotic: "Audio sounds robotic or distorted? → Switch to Balanced or Natural.",
inaccurate: "Whisper transcription inaccurate? → Switch to Transcription or increase mic gain.",
noisy: "Too much background noise? → Enable noise suppression or try the Transcription profile.",
},
},
} as const;

View file

@ -0,0 +1,171 @@
import test from "node:test";
import assert from "node:assert/strict";
import { resolveAudioFilterChain, resolveAudioProfile, CAPTURE_PROFILE_LABELS } from "../src/domain/captureProfiles";
import type { CaptureSettings } from "../src/domain/settings";
const baseCapture: CaptureSettings = {
ffmpegPath: "/usr/bin/ffmpeg",
backend: "avfoundation",
microphoneDevice: ":2",
microphoneLabel: "Built-in Microphone",
systemDevice: "",
systemLabel: "",
sampleRateHz: 48000,
channels: 1,
bitrateKbps: 160,
segmentDurationSeconds: 20,
captureProfile: "transcription",
micGainDb: 0,
systemGainDb: 0,
noiseSuppression: true,
limiter: true,
};
// ── resolveAudioProfile ──────────────────────────────────────────────────────
test("resolveAudioProfile returns transcription profile unchanged", () => {
const profile = resolveAudioProfile({ ...baseCapture, captureProfile: "transcription" });
assert.equal(profile.micGainDb, 6.6);
assert.equal(profile.highpassHz, 80);
assert.equal(profile.lowpassHz, 7000);
assert.equal(profile.limiter, true);
assert.equal(profile.limiterCeiling, 0.95);
assert.equal(profile.micMixWeight, 1.65);
assert.equal(profile.systemMixWeight, 0.75);
});
test("resolveAudioProfile returns balanced profile", () => {
const profile = resolveAudioProfile({ ...baseCapture, captureProfile: "balanced" });
assert.equal(profile.micGainDb, 3.0);
assert.equal(profile.highpassHz, 80);
assert.equal(profile.lowpassHz, 12000);
assert.equal(profile.limiterCeiling, 0.95);
});
test("resolveAudioProfile returns natural profile with no filtering", () => {
const profile = resolveAudioProfile({ ...baseCapture, captureProfile: "natural" });
assert.equal(profile.micGainDb, 0);
assert.equal(profile.highpassHz, null);
assert.equal(profile.lowpassHz, null);
assert.equal(profile.limiterCeiling, 0.98);
});
test("resolveAudioProfile custom profile respects user micGainDb", () => {
const profile = resolveAudioProfile({
...baseCapture,
captureProfile: "custom",
micGainDb: 6,
systemGainDb: -3,
noiseSuppression: false,
limiter: false,
});
assert.equal(profile.micGainDb, 6);
assert.equal(profile.systemGainDb, -3);
assert.equal(profile.highpassHz, null);
assert.equal(profile.lowpassHz, null);
assert.equal(profile.limiter, false);
});
test("resolveAudioProfile custom with noiseSuppression=true applies filters", () => {
const profile = resolveAudioProfile({
...baseCapture,
captureProfile: "custom",
noiseSuppression: true,
});
assert.equal(profile.highpassHz, 80);
assert.equal(profile.lowpassHz, 12000);
});
// ── resolveAudioFilterChain — single input ───────────────────────────────────
test("resolveAudioFilterChain transcription single-input produces -af string", () => {
const { filterArgs, mapArgs } = resolveAudioFilterChain(
{ ...baseCapture, captureProfile: "transcription" },
false
);
assert.equal(filterArgs[0], "-af");
// Must include highpass, lowpass, volume boost, and limiter
assert.ok(filterArgs[1].includes("highpass=f=80"), "should have highpass");
assert.ok(filterArgs[1].includes("lowpass=f=7000"), "should have lowpass");
assert.ok(filterArgs[1].includes("volume="), "should have volume");
assert.ok(filterArgs[1].includes("alimiter="), "should have limiter");
assert.deepEqual(mapArgs, ["-map", "0:a"]);
});
test("resolveAudioFilterChain natural single-input produces minimal -af string", () => {
const { filterArgs } = resolveAudioFilterChain(
{ ...baseCapture, captureProfile: "natural" },
false
);
assert.equal(filterArgs[0], "-af");
// natural has 0 dB gain (no volume filter) and no hp/lp, only limiter
assert.ok(!filterArgs[1].includes("highpass"), "should NOT have highpass");
assert.ok(!filterArgs[1].includes("lowpass"), "should NOT have lowpass");
assert.ok(!filterArgs[1].includes("volume="), "should NOT have volume filter when gain=0");
assert.ok(filterArgs[1].includes("alimiter="), "should still have limiter");
});
test("resolveAudioFilterChain custom no-limiter no-noise produces anull or empty chain", () => {
const { filterArgs } = resolveAudioFilterChain(
{ ...baseCapture, captureProfile: "custom", micGainDb: 0, systemGainDb: 0, noiseSuppression: false, limiter: false },
false
);
// With zero gain, no filters, no limiter, should be "anull"
assert.equal(filterArgs[0], "-af");
assert.equal(filterArgs[1], "anull");
});
// ── resolveAudioFilterChain — dual input ─────────────────────────────────────
test("resolveAudioFilterChain transcription dual-input produces -filter_complex", () => {
const { filterArgs, mapArgs } = resolveAudioFilterChain(
{ ...baseCapture, captureProfile: "transcription" },
true
);
assert.equal(filterArgs[0], "-filter_complex");
const fc = filterArgs[1];
assert.ok(fc.includes("[0:a]"), "should reference first input");
assert.ok(fc.includes("[1:a]"), "should reference second input");
assert.ok(fc.includes("amix=inputs=2"), "should mix two inputs");
assert.ok(fc.includes("weights='1.65 0.75'"), "should have transcription mix weights");
assert.ok(fc.includes("[aout]"), "should produce [aout]");
assert.deepEqual(mapArgs, ["-map", "[aout]"]);
});
test("resolveAudioFilterChain balanced dual-input has correct mix weights", () => {
const { filterArgs } = resolveAudioFilterChain(
{ ...baseCapture, captureProfile: "balanced" },
true
);
const fc = filterArgs[1];
assert.ok(fc.includes("weights='1.2 0.9'"), "should have balanced mix weights");
});
test("resolveAudioFilterChain natural dual-input has equal mix weights and no hp/lp", () => {
const { filterArgs } = resolveAudioFilterChain(
{ ...baseCapture, captureProfile: "natural" },
true
);
const fc = filterArgs[1];
assert.ok(fc.includes("weights='1 1'"), "should have equal weights");
assert.ok(!fc.includes("highpass"), "should NOT have highpass in natural profile");
});
// ── filterPreview ─────────────────────────────────────────────────────────────
test("resolveAudioFilterChain filterPreview matches the effective filter string", () => {
const { filterArgs, filterPreview } = resolveAudioFilterChain(
{ ...baseCapture, captureProfile: "balanced" },
false
);
assert.equal(filterPreview, filterArgs[1]);
});
// ── CAPTURE_PROFILE_LABELS ────────────────────────────────────────────────────
test("CAPTURE_PROFILE_LABELS has labels for all four profiles", () => {
assert.ok(CAPTURE_PROFILE_LABELS["transcription"].length > 0);
assert.ok(CAPTURE_PROFILE_LABELS["balanced"].length > 0);
assert.ok(CAPTURE_PROFILE_LABELS["natural"].length > 0);
assert.ok(CAPTURE_PROFILE_LABELS["custom"].length > 0);
});