diff --git a/manifest.json b/manifest.json index 35e85c2..2ba8896 100644 --- a/manifest.json +++ b/manifest.json @@ -5,6 +5,7 @@ "minAppVersion": "1.5.0", "description": "A local first AI Obsidian recorder with resilient live transcription, diagnostics, and session based notes.", "author": "Michael Gorini", - "authorUrl": "", + "authorUrl": "https://buymeacoffee.com/michaelgorini", + "fundingUrl": "https://buymeacoffee.com/michaelgorini", "isDesktopOnly": true } \ No newline at end of file diff --git a/src/application/SessionController.ts b/src/application/SessionController.ts index b70ce94..b6262d9 100644 --- a/src/application/SessionController.ts +++ b/src/application/SessionController.ts @@ -1,18 +1,16 @@ import type { App } from "obsidian"; import { getScenario } from "../domain/scenarios"; import { buildDiagnosticsSummary, type RecordingSessionManifest, type SessionListItem, type SessionRuntimeSnapshot, type SessionState } from "../domain/session"; -import { type PluginSettingsV2 } from "../domain/settings"; +import { resolveCaptureRuntime, type PluginSettingsV2 } from "../domain/settings"; import { OrderedSegmentQueue, type SegmentDescriptor } from "./OrderedSegmentQueue"; import { deriveSessionListItem } from "./dashboard"; -import { collectSegmentDescriptors } from "./segmentFiles"; -import { AudioCaptureAdapter } from "../infrastructure/adapters/AudioCaptureAdapter"; +import { WebCaptureAdapter } from "../infrastructure/adapters/WebCaptureAdapter"; import { WhisperTranscriptionAdapter } from "../infrastructure/adapters/TranscriptionAdapter"; import { SummaryAdapter } from "../infrastructure/adapters/SummaryAdapter"; import { SessionStore } from "../infrastructure/storage/SessionStore"; import { VaultAdapter } from "../infrastructure/adapters/VaultAdapter"; import { DiagnosticsService } from "../infrastructure/system/DiagnosticsService"; import { formatTranscriptChunkMarkdown, normalizeCheckboxes, sanitizeSummary } from "../utils/markdown"; -import { requireNodeModule } from "../infrastructure/node"; interface SessionControllerOptions { app: App; @@ -23,13 +21,18 @@ interface SessionControllerOptions { const STARTABLE_STATES = new Set(["idle", "done", "failed"]); +interface ActiveCaptureRuntime { + stop(): Promise; + isRunning(): boolean; +} + export class SessionController { private readonly store: SessionStore; private readonly vaultAdapter: VaultAdapter; private readonly diagnosticsService: DiagnosticsService; private readonly summaryAdapter = new SummaryAdapter(); - private captureAdapter: AudioCaptureAdapter | null = null; + private captureAdapter: ActiveCaptureRuntime | null = null; private queue: OrderedSegmentQueue | null = null; private manifest: RecordingSessionManifest | null = null; private segmentPollerId: number | null = null; @@ -163,6 +166,7 @@ export class SessionController { this.resetSnapshot(); const settings = this.options.getSettings(); const scenario = getScenario(scenarioKey); + const captureRuntime = resolveCaptureRuntime(settings.capture); this.stopRequested = false; try { @@ -181,20 +185,35 @@ export class SessionController { this.manifest.notes.liveTranscriptNotePath = workspace.liveTranscriptNotePath; await this.store.writeManifest(this.manifest); - this.captureAdapter = new AudioCaptureAdapter(); const transcriptionAdapter = new WhisperTranscriptionAdapter(settings.transcription, settings.capture.ffmpegPath); this.queue = new OrderedSegmentQueue(async (segment) => { await this.commitSegment(segment, transcriptionAdapter); }, Math.max(0, this.manifest.live.lastCommittedSegment + 1)); - await this.captureAdapter.start({ - settings: settings.capture, + const adapter = new WebCaptureAdapter(); + this.captureAdapter = adapter; + await adapter.start({ fullAudioPath: this.manifest.paths.fullAudioPath, segmentsDir: this.manifest.paths.segmentsDir, + segmentDurationSeconds: settings.capture.segmentDurationSeconds, + microphoneDevice: settings.capture.microphoneDevice, + additionalSources: + captureRuntime === "web-multi-input" && settings.capture.systemDevice.trim() + ? [ + { + deviceId: settings.capture.systemDevice.trim(), + label: settings.capture.systemLabel.trim() || "Additional source", + }, + ] + : [], + onSegmentReady: (segment) => { + this.queue?.enqueue([segment]); + this.refreshQueueSnapshot(); + }, onLog: (line) => { if (this.manifest) void this.store.appendDiagnostics(this.manifest, line); }, - onUnexpectedExit: (message) => { + onError: (message) => { void this.failSession(message); }, }); @@ -212,7 +231,6 @@ export class SessionController { this.manifest.runtime.startedAt = new Date(this.startedAtMs).toISOString(); await this.store.writeManifest(this.manifest); this.startElapsedTimer(); - this.startSegmentPolling(); this.transition("segmenting", "Recorder primed. Waiting for the first segment..."); } catch (error) { const message = String((error as Error)?.message ?? error); @@ -256,7 +274,6 @@ export class SessionController { private async finishStoppedSession(): Promise { const manifest = this.requireManifest(); try { - await this.enqueueStableSegments(true); await this.queue?.whenIdle(); this.refreshQueueSnapshot(); await this.store.appendDiagnostics(manifest, "Capture stopped and all live segments committed."); @@ -354,13 +371,6 @@ export class SessionController { } } - private startSegmentPolling() { - this.stopSegmentPolling(); - this.segmentPollerId = window.setInterval(() => { - void this.enqueueStableSegments(false); - }, 900); - } - private stopSegmentPolling() { if (this.segmentPollerId !== null) { window.clearInterval(this.segmentPollerId); @@ -368,29 +378,6 @@ export class SessionController { } } - private async enqueueStableSegments(allowUnstable: boolean) { - const manifest = this.requireManifest(); - const fs = requireNodeModule<{ - readdirSync: (path: string) => string[]; - statSync: (path: string) => { mtimeMs: number; isFile(): boolean }; - }>("fs"); - const path = requireNodeModule<{ join: (...parts: string[]) => string }>("path"); - const now = Date.now(); - const entries = fs.readdirSync(manifest.paths.segmentsDir).map((entry) => { - const fullPath = path.join(manifest.paths.segmentsDir, entry); - const stat = fs.statSync(fullPath); - return { - name: entry, - path: fullPath, - mtimeMs: stat.mtimeMs, - isFile: stat.isFile(), - }; - }); - const segments = collectSegmentDescriptors(entries, now, allowUnstable); - this.queue?.enqueue(segments); - this.refreshQueueSnapshot(); - } - private refreshQueueSnapshot() { if (!this.queue) { this.patchSnapshot({ queuedSegments: 0 }); diff --git a/src/application/dashboard.ts b/src/application/dashboard.ts index 4bd0e38..5035039 100644 --- a/src/application/dashboard.ts +++ b/src/application/dashboard.ts @@ -1,6 +1,7 @@ import type { DashboardSnapshot, DiagnosticsGroups } from "../domain/dashboard"; import type { DiagnosticsReport } from "../domain/diagnostics"; import type { SessionListItem, SessionRuntimeSnapshot, RecordingSessionManifest, SessionHealthBadge } from "../domain/session"; +import type { SystemAudioMode } from "../domain/settings"; import { uiCopy } from "../ui/copy"; export function groupDiagnosticsChecks(report?: DiagnosticsReport): DiagnosticsGroups { @@ -23,10 +24,15 @@ export function deriveSessionHealthBadge(manifest: RecordingSessionManifest): Se } export function deriveSessionListItem(manifest: RecordingSessionManifest, audioSizeBytes: number): SessionListItem { + const captureEngine = manifest.captureEngine ?? "ffmpeg"; + const systemAudioMode = manifest.systemAudioMode ?? inferSystemAudioMode(manifest); return { sessionId: manifest.sessionId, scenarioKey: manifest.scenarioKey, scenarioLabel: manifest.scenarioLabel, + captureEngine, + systemAudioMode, + sourceLabel: describeSessionSource(systemAudioMode), createdAt: manifest.createdAt, updatedAt: manifest.updatedAt, lastActivityAt: manifest.runtime.lastActivityAt, @@ -56,6 +62,22 @@ export function deriveSessionListItem(manifest: RecordingSessionManifest, audioS }; } +function describeSessionSource(systemAudioMode: SystemAudioMode): string { + if (systemAudioMode === "share") { + return "Shared system audio"; + } + if (systemAudioMode === "loopback") { + return "Microphone + additional source"; + } + return "Microphone"; +} + +function inferSystemAudioMode(manifest: RecordingSessionManifest): SystemAudioMode { + if (manifest.captureMode === "microphone+system") return "loopback"; + if (manifest.captureMode === "system") return "share"; + return "off"; +} + export function buildDashboardSnapshot(input: { runtime: SessionRuntimeSnapshot; diagnosticsReport?: DiagnosticsReport; diff --git a/src/application/segmentFiles.ts b/src/application/segmentFiles.ts index d9f2f80..07081e9 100644 --- a/src/application/segmentFiles.ts +++ b/src/application/segmentFiles.ts @@ -10,7 +10,7 @@ export interface SegmentFileEntry { export function collectSegmentDescriptors(entries: SegmentFileEntry[], now: number, allowNewestOpenSegment: boolean): SegmentDescriptor[] { const parsed = entries .map((entry) => { - const match = entry.name.match(/^segment-(\d{4})\.mp3$/i); + const match = entry.name.match(/^segment-(\d{4})\.(mp3|wav)$/i); if (!match || !entry.isFile) return null; return { index: Number(match[1]), diff --git a/src/domain/diagnostics.ts b/src/domain/diagnostics.ts index b3c9ffe..390f322 100644 --- a/src/domain/diagnostics.ts +++ b/src/domain/diagnostics.ts @@ -13,7 +13,7 @@ export interface DiagnosticCheck { export interface DiagnosticsReport { checkedAt: string; provider: SummaryProviderId; - backend: "avfoundation" | "dshow" | "pulse" | "alsa"; + backend: "web" | "avfoundation" | "dshow" | "pulse" | "alsa"; checks: DiagnosticCheck[]; blockingIssueIds: string[]; warningIds: string[]; diff --git a/src/domain/session.ts b/src/domain/session.ts index c98d1b4..eb20fcf 100644 --- a/src/domain/session.ts +++ b/src/domain/session.ts @@ -1,5 +1,6 @@ import type { DiagnosticsReport } from "./diagnostics"; import type { SummaryProviderId } from "./providers"; +import type { CaptureEngine, SystemAudioMode } from "./settings"; export const SESSION_STATES = [ "idle", @@ -37,13 +38,15 @@ export interface RecordingSessionPaths { } export interface RecordingSessionManifest { - schemaVersion: 1; + schemaVersion: 1 | 2 | 3; sessionId: string; createdAt: string; updatedAt: string; scenarioKey: string; scenarioLabel: string; - captureMode: "microphone" | "microphone+system"; + captureEngine?: CaptureEngine; + systemAudioMode?: SystemAudioMode; + captureMode: "microphone" | "microphone+system" | "system"; status: SessionState; paths: RecordingSessionPaths; providerInfo: { @@ -96,6 +99,9 @@ export interface SessionListItem { sessionId: string; scenarioKey: string; scenarioLabel: string; + captureEngine: CaptureEngine; + systemAudioMode: SystemAudioMode; + sourceLabel: string; createdAt: string; updatedAt: string; lastActivityAt: string; diff --git a/src/domain/settings.ts b/src/domain/settings.ts index fae228e..14dbae6 100644 --- a/src/domain/settings.ts +++ b/src/domain/settings.ts @@ -2,8 +2,13 @@ import { getSelectedSummaryModel, type SummaryProviderId } from "./providers"; export type CaptureBackend = "auto" | "avfoundation" | "dshow" | "pulse" | "alsa"; export type CaptureProfile = "transcription" | "balanced" | "natural" | "custom"; +export type CaptureEngine = "web" | "ffmpeg"; +export type SystemAudioMode = "off" | "loopback" | "share"; +export type CaptureRuntimeKind = "web-mic" | "web-multi-input"; export interface CaptureSettings { + captureEngine: CaptureEngine; + systemAudioMode: SystemAudioMode; ffmpegPath: string; backend: CaptureBackend; microphoneDevice: string; @@ -74,6 +79,8 @@ export interface PluginSettingsV2 { export const DEFAULT_SETTINGS_V2: PluginSettingsV2 = { version: 2, capture: { + captureEngine: "web", + systemAudioMode: "off", ffmpegPath: "", backend: "auto", microphoneDevice: "", @@ -81,7 +88,7 @@ export const DEFAULT_SETTINGS_V2: PluginSettingsV2 = { systemDevice: "", systemLabel: "", sampleRateHz: 48000, - channels: 2, + channels: 1, bitrateKbps: 160, segmentDurationSeconds: 20, captureProfile: "balanced", @@ -151,7 +158,11 @@ function normalizeCaptureSettings(raw: unknown): CaptureSettings { const input = (raw ?? {}) as Partial; const backend = input.backend; const profile = input.captureProfile; + const captureEngine = inferCaptureEngine(input); + const systemAudioMode = inferSystemAudioMode(input); return { + captureEngine, + systemAudioMode, ffmpegPath: asString(input.ffmpegPath), backend: backend === "avfoundation" || backend === "dshow" || backend === "pulse" || backend === "alsa" || backend === "auto" @@ -176,6 +187,30 @@ function normalizeCaptureSettings(raw: unknown): CaptureSettings { }; } +function inferCaptureEngine(input: Partial): CaptureEngine { + if (input.captureEngine === "web" || input.captureEngine === "ffmpeg") { + return input.captureEngine; + } + + return DEFAULT_SETTINGS_V2.capture.captureEngine; +} + +function inferSystemAudioMode(input: Partial): SystemAudioMode { + if (input.systemAudioMode === "off" || input.systemAudioMode === "loopback") { + return input.systemAudioMode; + } + + if (input.systemAudioMode === "share") { + return "off"; + } + + if (asString(input.systemDevice).trim() || asString(input.systemLabel).trim()) { + return "loopback"; + } + + return DEFAULT_SETTINGS_V2.capture.systemAudioMode; +} + function normalizeTranscriptionSettings(raw: unknown): TranscriptionSettings { const input = (raw ?? {}) as Partial; const preset = input.modelPreset; @@ -295,7 +330,7 @@ export function isLikelyTestWhisperModelPath(modelPath: string): boolean { export function isCoreConfigured(settings: PluginSettingsV2): boolean { const selectedModel = getSelectedSummaryModel(settings.summary); - if (!settings.capture.ffmpegPath.trim()) return false; + if (settings.capture.systemAudioMode === "loopback" && !settings.capture.systemDevice.trim()) return false; if (!settings.transcription.whisperCliPath.trim()) return false; if (!settings.transcription.modelPath.trim()) return false; if (isLikelyTestWhisperModelPath(settings.transcription.modelPath)) return false; @@ -304,3 +339,18 @@ export function isCoreConfigured(settings: PluginSettingsV2): boolean { } return !!selectedModel.trim() && !!getSelectedProviderApiKey(settings.summary).trim(); } + +export function isSystemAudioEnabled(capture: CaptureSettings): boolean { + return capture.systemAudioMode !== "off"; +} + +export function isLoopbackSystemAudioEnabled(capture: CaptureSettings): boolean { + return capture.systemAudioMode === "loopback"; +} + +export function resolveCaptureRuntime(capture: CaptureSettings): CaptureRuntimeKind { + if (capture.systemAudioMode === "loopback") { + return "web-multi-input"; + } + return "web-mic"; +} diff --git a/src/infrastructure/adapters/SharedAudioCaptureAdapter.ts b/src/infrastructure/adapters/SharedAudioCaptureAdapter.ts new file mode 100644 index 0000000..aad54a3 --- /dev/null +++ b/src/infrastructure/adapters/SharedAudioCaptureAdapter.ts @@ -0,0 +1,222 @@ +import { requireNodeModule } from "../node"; +import { + CaptureCancelledError, + getSharedAudioCaptureFailureDetail, + isDisplayShareCancellation, +} from "./mediaCaptureErrors"; +import { StreamingWavWriter, writeWavFile } from "./wavWriter"; + +export interface SharedAudioSegmentDescriptor { + index: number; + path: string; +} + +export interface SharedAudioCaptureStartOptions { + fullAudioPath: string; + segmentsDir: string; + segmentDurationSeconds: number; + onSegmentReady: (segment: SharedAudioSegmentDescriptor) => void; + onLog?: (line: string) => void; + onError?: (message: string) => void; +} + +interface WebAudioContextConstructor { + new (): AudioContext; +} + +export class SharedAudioCaptureAdapter { + private context: AudioContext | null = null; + private stream: MediaStream | null = null; + private sourceNode: MediaStreamAudioSourceNode | null = null; + private processor: ScriptProcessorNode | null = null; + private sinkNode: GainNode | null = null; + private segmentBuffers: Float32Array[] = []; + private segmentSampleCount = 0; + private segmentTargetSamples = 0; + private segmentIndex = 0; + private running = false; + private sampleRate = 48_000; + private fullRecordingWriter: StreamingWavWriter | null = null; + private options: SharedAudioCaptureStartOptions | null = null; + private readonly path = requireNodeModule<{ join: (...parts: string[]) => string }>("path"); + + async start(options: SharedAudioCaptureStartOptions): Promise { + if (this.running) { + throw new Error("Shared audio capture already running."); + } + + const AudioContextCtor = getAudioContextConstructor(); + if (!AudioContextCtor || !globalThis.navigator?.mediaDevices?.getDisplayMedia) { + throw new Error("Shared system audio capture is not available in this runtime."); + } + + this.options = options; + this.segmentBuffers = []; + this.segmentSampleCount = 0; + this.segmentIndex = 0; + + try { + this.stream = await globalThis.navigator.mediaDevices.getDisplayMedia({ + video: true, + audio: true, + }); + } catch (error) { + if (isDisplayShareCancellation(error)) { + throw new CaptureCancelledError("Shared audio capture was canceled."); + } + const detail = getSharedAudioCaptureFailureDetail(error); + if (detail) { + throw new Error(detail); + } + throw error; + } + + if (!this.stream.getAudioTracks().length) { + this.stream.getTracks().forEach((track) => track.stop()); + this.stream = null; + throw new Error("The shared surface did not provide an audio track. Choose a surface that shares audio or use Loopback device instead."); + } + + try { + this.context = new AudioContextCtor(); + this.sampleRate = this.context.sampleRate || 48_000; + this.segmentTargetSamples = Math.max(1, Math.floor(this.sampleRate * Math.max(1, options.segmentDurationSeconds))); + this.fullRecordingWriter = new StreamingWavWriter(options.fullAudioPath, this.sampleRate, 1); + this.sourceNode = this.context.createMediaStreamSource(this.stream); + this.processor = this.context.createScriptProcessor(4096, 1, 1); + this.sinkNode = this.context.createGain(); + this.sinkNode.gain.value = 0; + this.processor.onaudioprocess = (event) => { + try { + this.handleAudioProcess(event); + } catch (error) { + const message = String((error as Error)?.message ?? error); + this.options?.onError?.(message); + } + }; + + this.sourceNode.connect(this.processor); + this.processor.connect(this.sinkNode); + this.sinkNode.connect(this.context.destination); + + this.running = true; + options.onLog?.(`Shared audio capture started: sampleRate=${this.sampleRate}Hz, channelCount=1.`); + } catch (error) { + await this.dispose(); + throw error; + } + } + + isRunning(): boolean { + return this.running; + } + + async stop(): Promise { + if (!this.running) return; + this.running = false; + this.processor && (this.processor.onaudioprocess = null); + this.flushCurrentSegmentSync(); + this.fullRecordingWriter?.finalize(); + await this.dispose(); + } + + private handleAudioProcess(event: AudioProcessingEvent): void { + if (!this.running || !this.options) return; + + const monoSamples = extractMonoSamples(event.inputBuffer); + if (monoSamples.length === 0) return; + this.fullRecordingWriter?.append(monoSamples); + + let offset = 0; + while (offset < monoSamples.length) { + const remainingInSegment = this.segmentTargetSamples - this.segmentSampleCount; + const take = Math.min(remainingInSegment, monoSamples.length - offset); + const chunk = monoSamples.slice(offset, offset + take); + this.segmentBuffers.push(chunk); + this.segmentSampleCount += take; + offset += take; + + if (this.segmentSampleCount >= this.segmentTargetSamples) { + this.flushCurrentSegmentSync(); + } + } + } + + private flushCurrentSegmentSync(): void { + if (!this.options || this.segmentSampleCount === 0) return; + + const samples = concatFloat32Arrays(this.segmentBuffers, this.segmentSampleCount); + const segmentPath = this.path.join(this.options.segmentsDir, `segment-${String(this.segmentIndex).padStart(4, "0")}.wav`); + writeWavFile(segmentPath, samples, this.sampleRate, 1); + this.options.onSegmentReady({ index: this.segmentIndex, path: segmentPath }); + this.segmentIndex += 1; + this.segmentBuffers = []; + this.segmentSampleCount = 0; + } + + private async dispose(): Promise { + try { + this.sourceNode?.disconnect(); + } catch {} + try { + this.processor?.disconnect(); + } catch {} + try { + this.sinkNode?.disconnect(); + } catch {} + try { + this.stream?.getTracks().forEach((track) => track.stop()); + } catch {} + try { + if (this.context && this.context.state !== "closed") { + await this.context.close(); + } + } catch {} + + this.context = null; + this.stream = null; + this.sourceNode = null; + this.processor = null; + this.sinkNode = null; + this.fullRecordingWriter = null; + this.options = null; + this.running = false; + } +} + +function getAudioContextConstructor(): WebAudioContextConstructor | null { + const candidate = + globalThis.AudioContext ?? + ((globalThis as unknown as { webkitAudioContext?: WebAudioContextConstructor }).webkitAudioContext ?? null); + return candidate ?? null; +} + +function extractMonoSamples(buffer: AudioBuffer): Float32Array { + const channels = Math.max(1, buffer.numberOfChannels); + const output = new Float32Array(buffer.length); + + if (channels === 1) { + output.set(buffer.getChannelData(0)); + return output; + } + + for (let sampleIndex = 0; sampleIndex < buffer.length; sampleIndex += 1) { + let sum = 0; + for (let channelIndex = 0; channelIndex < channels; channelIndex += 1) { + sum += buffer.getChannelData(channelIndex)[sampleIndex] ?? 0; + } + output[sampleIndex] = sum / channels; + } + + return output; +} + +function concatFloat32Arrays(chunks: Float32Array[], totalLength: number): Float32Array { + const output = new Float32Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + output.set(chunk, offset); + offset += chunk.length; + } + return output; +} diff --git a/src/infrastructure/adapters/TranscriptionAdapter.ts b/src/infrastructure/adapters/TranscriptionAdapter.ts index f6d1e05..d932a37 100644 --- a/src/infrastructure/adapters/TranscriptionAdapter.ts +++ b/src/infrastructure/adapters/TranscriptionAdapter.ts @@ -95,11 +95,38 @@ export class WhisperTranscriptionAdapter { "whisper.cpp" ); + let rawTranscript = ""; if (fs.existsSync(outputTextPath)) { - return fs.readFileSync(outputTextPath, { encoding: "utf8" }).trim(); + rawTranscript = fs.readFileSync(outputTextPath, { encoding: "utf8" }).trim(); + } else { + rawTranscript = result.stdout.trim(); } - return result.stdout.trim(); + return this.cleanTranscript(rawTranscript); + } + + private cleanTranscript(text: string): string { + let cleaned = text + // Rimuove tag di speaker o suoni ambientali [Suono], [Speaker 1], [PROFESSORI] + .replace(/\[.*?\]/g, "") + // Rimuove caratteri ripetuti in modo anomalo + .replace(/[-_]{2,}/g, " ") + .replace(/\.{4,}/g, "...") + // Rimuove alcune allucinazioni note (in italiano e inglese) + .replace(/(Sottotitoli|Subtitles) (creati|a cura|di).*$/gim, "") + .replace(/Traduzione di.*$/gim, "") + .replace(/Iscriviti.*$/gim, "") + .replace(/Amara\.org/gim, "") + // Normalizza gli spazi + .replace(/\s+/g, " ") + .trim(); + + // Se la stringa pulita contiene solo punteggiatura o spazi, la consideriamo vuota + if (/^[.,!?\-;: ]*$/.test(cleaned)) { + return ""; + } + + return cleaned; } private buildWhisperArgs(inputAudioPath: string, outputPrefix: string, relaxed: boolean): string[] { diff --git a/src/infrastructure/adapters/WebCaptureAdapter.ts b/src/infrastructure/adapters/WebCaptureAdapter.ts new file mode 100644 index 0000000..18fb1ba --- /dev/null +++ b/src/infrastructure/adapters/WebCaptureAdapter.ts @@ -0,0 +1,297 @@ +import { resolvePreferredWebAudioInput, resolveWebAudioInputById } from "../system/webAudio"; +import { requireNodeModule } from "../node"; +import { StreamingWavWriter, writeWavFile } from "./wavWriter"; + +export interface WebCaptureSegmentDescriptor { + index: number; + path: string; +} + +export interface WebCaptureStartOptions { + fullAudioPath: string; + segmentsDir: string; + segmentDurationSeconds: number; + microphoneDevice?: string; + additionalSources?: WebCaptureInputSource[]; + onSegmentReady: (segment: WebCaptureSegmentDescriptor) => void; + onLog?: (line: string) => void; + onError?: (message: string) => void; +} + +export interface WebCaptureInputSource { + deviceId: string; + label?: string; + gain?: number; +} + +interface WebAudioContextConstructor { + new (): AudioContext; +} + +export class WebCaptureAdapter { + private context: AudioContext | null = null; + private streams: MediaStream[] = []; + private sourceNodes: MediaStreamAudioSourceNode[] = []; + private gainNodes: GainNode[] = []; + private processor: ScriptProcessorNode | null = null; + private sinkNode: GainNode | null = null; + private mixBus: GainNode | null = null; + private segmentBuffers: Float32Array[] = []; + private segmentSampleCount = 0; + private segmentTargetSamples = 0; + private segmentIndex = 0; + private running = false; + private sampleRate = 48_000; + private fullRecordingWriter: StreamingWavWriter | null = null; + private options: WebCaptureStartOptions | null = null; + private readonly path = requireNodeModule<{ join: (...parts: string[]) => string }>("path"); + + async start(options: WebCaptureStartOptions): Promise { + if (this.running) { + throw new Error("Web audio capture already running."); + } + + const AudioContextCtor = getAudioContextConstructor(); + if (!AudioContextCtor || !globalThis.navigator?.mediaDevices?.getUserMedia) { + throw new Error("Web Audio capture is not available in this runtime."); + } + + this.options = options; + this.segmentBuffers = []; + this.segmentSampleCount = 0; + this.segmentIndex = 0; + + const preferredDevice = await resolvePreferredWebAudioInput(options.microphoneDevice); + const requestedDeviceId = preferredDevice?.deviceId && preferredDevice.deviceId !== "default" ? preferredDevice.deviceId : undefined; + if (options.microphoneDevice?.trim() && requestedDeviceId !== options.microphoneDevice.trim()) { + options.onLog?.(`Web Audio fallback: microphone ${options.microphoneDevice.trim()} is unavailable. Using the system default microphone.`); + } + + const additionalSources = options.additionalSources ?? []; + const resolvedAdditionalSources = await Promise.all( + additionalSources.map(async (source) => { + const resolved = await resolveWebAudioInputById(source.deviceId); + if (!resolved) { + throw new Error( + source.label?.trim() + ? `Additional audio source "${source.label}" is unavailable. Refresh devices or turn it off.` + : "The selected additional audio source is unavailable. Refresh devices or turn it off." + ); + } + return { + requested: source, + resolved, + }; + }) + ); + + if ( + requestedDeviceId && + resolvedAdditionalSources.some(({ resolved }) => resolved.deviceId === requestedDeviceId) + ) { + throw new Error("The additional audio source must be different from the microphone."); + } + + try { + this.streams = []; + this.streams.push(await globalThis.navigator.mediaDevices.getUserMedia(buildAudioConstraints(requestedDeviceId))); + for (const source of resolvedAdditionalSources) { + this.streams.push(await globalThis.navigator.mediaDevices.getUserMedia(buildAudioConstraints(source.resolved.deviceId))); + } + + this.context = new AudioContextCtor(); + this.sampleRate = this.context.sampleRate || 48_000; + this.segmentTargetSamples = Math.max(1, Math.floor(this.sampleRate * Math.max(1, options.segmentDurationSeconds))); + this.fullRecordingWriter = new StreamingWavWriter(options.fullAudioPath, this.sampleRate, 1); + this.processor = this.context.createScriptProcessor(4096, 2, 1); + this.mixBus = this.context.createGain(); + this.mixBus.gain.value = 1; + this.sinkNode = this.context.createGain(); + this.sinkNode.gain.value = 0; + this.processor.onaudioprocess = (event) => { + try { + this.handleAudioProcess(event); + } catch (error) { + const message = String((error as Error)?.message ?? error); + this.options?.onError?.(message); + } + }; + + const sourceLabels: string[] = []; + const sourceConfigs = [ + { + stream: this.streams[0], + label: preferredDevice?.label || "System default input", + gain: 1, + }, + ...resolvedAdditionalSources.map((source, index) => ({ + stream: this.streams[index + 1], + label: source.resolved.label || source.requested.label || "Additional source", + gain: source.requested.gain ?? 1, + })), + ]; + + for (const config of sourceConfigs) { + const sourceNode = this.context.createMediaStreamSource(config.stream); + const gainNode = this.context.createGain(); + gainNode.gain.value = config.gain; + sourceNode.connect(gainNode); + gainNode.connect(this.mixBus); + this.sourceNodes.push(sourceNode); + this.gainNodes.push(gainNode); + sourceLabels.push(config.label); + } + + this.mixBus.connect(this.processor); + this.processor.connect(this.sinkNode); + this.sinkNode.connect(this.context.destination); + + this.running = true; + options.onLog?.( + `Web Audio capture started: sampleRate=${this.sampleRate}Hz, mixedSources=${sourceConfigs.length}, inputs=${sourceLabels.join(", ")}.` + ); + } catch (error) { + await this.dispose(); + throw error; + } + } + + isRunning(): boolean { + return this.running; + } + + async stop(): Promise { + if (!this.running) return; + this.running = false; + this.processor && (this.processor.onaudioprocess = null); + await this.flushCurrentSegment(); + this.fullRecordingWriter?.finalize(); + await this.dispose(); + } + + private handleAudioProcess(event: AudioProcessingEvent): void { + if (!this.running || !this.options) return; + + const monoSamples = extractMonoSamples(event.inputBuffer); + if (monoSamples.length === 0) return; + this.fullRecordingWriter?.append(monoSamples); + + let offset = 0; + while (offset < monoSamples.length) { + const remainingInSegment = this.segmentTargetSamples - this.segmentSampleCount; + const take = Math.min(remainingInSegment, monoSamples.length - offset); + const chunk = monoSamples.slice(offset, offset + take); + this.segmentBuffers.push(chunk); + this.segmentSampleCount += take; + offset += take; + + if (this.segmentSampleCount >= this.segmentTargetSamples) { + this.flushCurrentSegmentSync(); + } + } + } + + private async flushCurrentSegment(): Promise { + this.flushCurrentSegmentSync(); + } + + private flushCurrentSegmentSync(): void { + if (!this.options || this.segmentSampleCount === 0) return; + + const samples = concatFloat32Arrays(this.segmentBuffers, this.segmentSampleCount); + const segmentPath = this.path.join(this.options.segmentsDir, `segment-${String(this.segmentIndex).padStart(4, "0")}.wav`); + writeWavFile(segmentPath, samples, this.sampleRate, 1); + this.options.onSegmentReady({ index: this.segmentIndex, path: segmentPath }); + this.segmentIndex += 1; + this.segmentBuffers = []; + this.segmentSampleCount = 0; + } + + private async dispose(): Promise { + try { + this.sourceNodes.forEach((node) => node.disconnect()); + } catch {} + try { + this.gainNodes.forEach((node) => node.disconnect()); + } catch {} + try { + this.mixBus?.disconnect(); + } catch {} + try { + this.processor?.disconnect(); + } catch {} + try { + this.sinkNode?.disconnect(); + } catch {} + try { + this.streams.forEach((stream) => stream.getTracks().forEach((track) => track.stop())); + } catch {} + try { + if (this.context && this.context.state !== "closed") { + await this.context.close(); + } + } catch {} + + this.context = null; + this.streams = []; + this.sourceNodes = []; + this.gainNodes = []; + this.processor = null; + this.sinkNode = null; + this.mixBus = null; + this.fullRecordingWriter = null; + this.options = null; + this.running = false; + } +} + +function getAudioContextConstructor(): WebAudioContextConstructor | null { + const candidate = globalThis.AudioContext ?? ((globalThis as unknown as { webkitAudioContext?: WebAudioContextConstructor }).webkitAudioContext ?? null); + return candidate ?? null; +} + +function extractMonoSamples(buffer: AudioBuffer): Float32Array { + const channels = Math.max(1, buffer.numberOfChannels); + const output = new Float32Array(buffer.length); + + if (channels === 1) { + output.set(buffer.getChannelData(0)); + return output; + } + + for (let sampleIndex = 0; sampleIndex < buffer.length; sampleIndex += 1) { + let sum = 0; + for (let channelIndex = 0; channelIndex < channels; channelIndex += 1) { + sum += buffer.getChannelData(channelIndex)[sampleIndex] ?? 0; + } + output[sampleIndex] = sum / channels; + } + + return output; +} + +function concatFloat32Arrays(chunks: Float32Array[], totalLength: number): Float32Array { + const output = new Float32Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + output.set(chunk, offset); + offset += chunk.length; + } + return output; +} + +function buildAudioConstraints(deviceId?: string): MediaStreamConstraints { + if (deviceId?.trim()) { + return { + audio: { + deviceId: { exact: deviceId.trim() }, + }, + video: false, + }; + } + + return { + audio: true, + video: false, + }; +} diff --git a/src/infrastructure/adapters/captureUtils.ts b/src/infrastructure/adapters/captureUtils.ts index ab7f397..69704f4 100644 --- a/src/infrastructure/adapters/captureUtils.ts +++ b/src/infrastructure/adapters/captureUtils.ts @@ -1,5 +1,5 @@ import type { CaptureSettings } from "../../domain/settings"; -import { resolveCaptureBackend } from "../../domain/settings"; +import { isLoopbackSystemAudioEnabled, resolveCaptureBackend } from "../../domain/settings"; import { scanDevices, type ListedDevice } from "../system/deviceScanner"; export interface ResolvedCaptureInputs { @@ -39,6 +39,7 @@ function findAvfoundationDevice(devices: ListedDevice[], name: string, label: st export async function resolveCaptureInputs(capture: CaptureSettings): Promise { const backend = resolveCaptureBackend(capture.backend); + const includeSystemAudio = isLoopbackSystemAudioEnabled(capture); if (backend === "avfoundation") { let devices: ListedDevice[] = []; try { @@ -48,7 +49,7 @@ export async function resolveCaptureInputs(capture: CaptureSettings): Promise void; + openSync: (path: string, flags: string) => number; + writeFileSync: (path: string, data: Buffer) => void; + writeSync: (fd: number, buffer: Buffer, offset?: number, length?: number, position?: number) => number; +} + +const WAV_HEADER_BYTES = 44; +const PCM_SAMPLE_BYTES = 2; + +export function createWavHeader(dataSize: number, sampleRate: number, channels: number): Buffer { + const blockAlign = channels * PCM_SAMPLE_BYTES; + const byteRate = sampleRate * blockAlign; + const header = Buffer.alloc(WAV_HEADER_BYTES); + + header.write("RIFF", 0, "ascii"); + header.writeUInt32LE(36 + dataSize, 4); + header.write("WAVE", 8, "ascii"); + header.write("fmt ", 12, "ascii"); + header.writeUInt32LE(16, 16); + header.writeUInt16LE(1, 20); + header.writeUInt16LE(channels, 22); + header.writeUInt32LE(sampleRate, 24); + header.writeUInt32LE(byteRate, 28); + header.writeUInt16LE(blockAlign, 32); + header.writeUInt16LE(16, 34); + header.write("data", 36, "ascii"); + header.writeUInt32LE(dataSize, 40); + + return header; +} + +export function writeWavFile(filePath: string, samples: Float32Array, sampleRate: number, channels: number): void { + const fs = requireNodeModule("fs"); + const pcm = encodePcm16(samples); + const header = createWavHeader(pcm.length, sampleRate, channels); + fs.writeFileSync(filePath, Buffer.concat([header, pcm])); +} + +export class StreamingWavWriter { + private readonly fs = requireNodeModule("fs"); + private readonly fd: number; + private dataSize = 0; + private finalized = false; + + constructor( + private readonly filePath: string, + private readonly sampleRate: number, + private readonly channels: number + ) { + this.fd = this.fs.openSync(filePath, "w"); + this.fs.writeSync(this.fd, createWavHeader(0, sampleRate, channels)); + } + + append(samples: Float32Array): void { + if (this.finalized || samples.length === 0) return; + const pcm = encodePcm16(samples); + if (pcm.length === 0) return; + this.fs.writeSync(this.fd, pcm); + this.dataSize += pcm.length; + } + + finalize(): void { + if (this.finalized) return; + this.finalized = true; + const header = createWavHeader(this.dataSize, this.sampleRate, this.channels); + this.fs.writeSync(this.fd, header, 0, WAV_HEADER_BYTES, 0); + this.fs.closeSync(this.fd); + } + + getPath(): string { + return this.filePath; + } +} + +function encodePcm16(samples: Float32Array): Buffer { + const buffer = Buffer.alloc(samples.length * PCM_SAMPLE_BYTES); + for (let index = 0; index < samples.length; index += 1) { + const sample = Math.max(-1, Math.min(1, samples[index] ?? 0)); + const value = sample < 0 ? Math.round(sample * 0x8000) : Math.round(sample * 0x7fff); + buffer.writeInt16LE(value, index * PCM_SAMPLE_BYTES); + } + return buffer; +} diff --git a/src/infrastructure/storage/SessionStore.ts b/src/infrastructure/storage/SessionStore.ts index c8e42b3..b5ba0fd 100644 --- a/src/infrastructure/storage/SessionStore.ts +++ b/src/infrastructure/storage/SessionStore.ts @@ -1,6 +1,10 @@ import type { App } from "obsidian"; import { getSelectedSummaryModel } from "../../domain/providers"; -import type { PluginSettingsV2 } from "../../domain/settings"; +import { + type CaptureEngine, + type PluginSettingsV2, + type SystemAudioMode, +} from "../../domain/settings"; import type { DiagnosticsSummary, RecordingSessionManifest } from "../../domain/session"; import type { ScenarioTemplate } from "../../domain/scenarios"; import { getVaultBasePath, getVaultConfigDir, requireNodeModule } from "../node"; @@ -21,6 +25,7 @@ interface FsModule { interface PathModule { join: (...parts: string[]) => string; + extname: (path: string) => string; } export class SessionStore { @@ -51,7 +56,9 @@ export class SessionStore { const diagnosticsLogPath = path.join(rootDir, "diagnostics.log"); const transcriptTextPath = path.join(transcriptDir, "live-transcript.txt"); const summaryMarkdownPath = path.join(summaryDir, "summary.md"); - const fullAudioPath = path.join(audioDir, "recording.mp3"); + const captureEngine: CaptureEngine = "web"; + const fullAudioFilename = "recording.wav"; + const fullAudioPath = path.join(audioDir, fullAudioFilename); for (const dir of [this.getSessionsRootDir(), rootDir, audioDir, segmentsDir, transcriptDir, summaryDir]) { if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); @@ -61,13 +68,15 @@ export class SessionStore { fs.writeFileSync(diagnosticsLogPath, "", { encoding: "utf8" }); const manifest: RecordingSessionManifest = { - schemaVersion: 1, + schemaVersion: 3, sessionId, createdAt, updatedAt: createdAt, scenarioKey: scenario.key, scenarioLabel: scenario.label, - captureMode: settings.capture.systemDevice.trim() ? "microphone+system" : "microphone", + captureEngine, + systemAudioMode: settings.capture.systemAudioMode, + captureMode: deriveCaptureMode(settings.capture.systemAudioMode), status: "preflight", paths: { rootDir, @@ -238,8 +247,13 @@ export class SessionStore { private normalizeManifest(manifest: RecordingSessionManifest): RecordingSessionManifest { const fallbackTimestamp = manifest.updatedAt || manifest.createdAt || new Date().toISOString(); + const captureEngine = manifest.captureEngine ?? inferLegacyCaptureEngine(manifest); + const systemAudioMode = manifest.systemAudioMode ?? inferLegacySystemAudioMode(manifest); const normalized: RecordingSessionManifest = { ...manifest, + schemaVersion: 3, + captureEngine, + systemAudioMode, notes: manifest.notes ?? {}, diagnosticsSummary: manifest.diagnosticsSummary ?? { checkedAt: fallbackTimestamp, @@ -298,3 +312,27 @@ export class SessionStore { return String(fs.readFileSync(path, { encoding: "utf8" })); } } + +function inferLegacyCaptureEngine(manifest: RecordingSessionManifest): CaptureEngine { + return manifest.paths.fullAudioPath.toLowerCase().endsWith(".wav") ? "web" : "ffmpeg"; +} + +function inferLegacySystemAudioMode(manifest: RecordingSessionManifest): SystemAudioMode { + if (manifest.captureMode === "microphone+system") { + return "loopback"; + } + if (manifest.captureMode === "system") { + return "share"; + } + return "off"; +} + +function deriveCaptureMode(systemAudioMode: SystemAudioMode): RecordingSessionManifest["captureMode"] { + if (systemAudioMode === "share") { + return "system"; + } + if (systemAudioMode === "loopback") { + return "microphone+system"; + } + return "microphone"; +} diff --git a/src/infrastructure/system/DiagnosticsService.ts b/src/infrastructure/system/DiagnosticsService.ts index da98396..beebf53 100644 --- a/src/infrastructure/system/DiagnosticsService.ts +++ b/src/infrastructure/system/DiagnosticsService.ts @@ -4,19 +4,23 @@ import { getSelectedProviderApiKey, isLikelyTestWhisperModelPath, isCoreConfigured, - resolveCaptureBackend, + isLoopbackSystemAudioEnabled, type PluginSettingsV2, } from "../../domain/settings"; import type { DiagnosticCheck, DiagnosticsReport } from "../../domain/diagnostics"; -import { CAPTURE_PROFILE_LABELS } from "../../domain/captureProfiles"; import { requireNodeModule } from "../node"; -import { resolveCaptureInputs } from "../adapters/captureUtils"; +import { WebCaptureAdapter } from "../adapters/WebCaptureAdapter"; import { WhisperTranscriptionAdapter } from "../adapters/TranscriptionAdapter"; -import { scanDevices } from "./deviceScanner"; +import { + getMicrophonePermissionState, + getWebAudioCapability, + listWebAudioInputDevices, +} from "./webAudio"; export interface SmokeTestResult { ok: boolean; detail: string; + cancelled?: boolean; } export class DiagnosticsService { @@ -30,7 +34,8 @@ export class DiagnosticsService { statSync: (path: string) => { size: number }; }>("fs"); const checks: DiagnosticCheck[] = []; - const backend = resolveCaptureBackend(settings.capture.backend); + const loopbackEnabled = isLoopbackSystemAudioEnabled(settings.capture); + const backend = "web"; const addCheck = (check: DiagnosticCheck) => checks.push(check); const fileMode = process.platform === "win32" ? fs.constants.F_OK : fs.constants.X_OK; @@ -49,20 +54,90 @@ export class DiagnosticsService { severity: isCoreConfigured(settings) ? "ok" : "error", detail: isCoreConfigured(settings) ? "Core local-first path is configured." - : "FFmpeg, whisper.cpp, model, or summary provider configuration is incomplete.", + : loopbackEnabled + ? "Web Audio capture is ready, but the additional source, transcription, or summary configuration is still incomplete." + : "Web Audio capture is ready, but transcription or summary configuration is still incomplete.", remediation: "Open Setup & Settings in the plugin settings tab and complete the missing dependency step.", }); + const capability = getWebAudioCapability(); + const permissionState = await getMicrophonePermissionState(); + const deviceSnapshot = await listWebAudioInputDevices().catch(async () => ({ + devices: [], + permissionState, + labelsAvailable: false, + })); + addCheck({ - id: "ffmpeg", - label: "FFmpeg binary", - severity: testPath(settings.capture.ffmpegPath, fileMode) ? "ok" : "error", - detail: settings.capture.ffmpegPath - ? `Configured path: ${settings.capture.ffmpegPath}` - : "FFmpeg path is empty.", - remediation: "Set the FFmpeg executable path or use auto-detect.", + id: "web-audio", + label: "Web Audio capture", + severity: capability.hasGetUserMedia && capability.hasEnumerateDevices ? "ok" : "error", + detail: capability.hasGetUserMedia && capability.hasEnumerateDevices + ? `Web Audio APIs are available${permissionState === "granted" ? " and microphone permission is granted." : permissionState === "prompt" ? ". Microphone permission will be requested on first recording." : permissionState === "denied" ? ", but microphone permission is denied." : "."}` + : "Required browser audio APIs are unavailable in this runtime.", + remediation: + permissionState === "denied" + ? "Re-enable microphone permission for Obsidian in the operating system settings." + : "Use Obsidian desktop on a Chromium-based runtime that exposes getUserMedia and enumerateDevices.", }); + addCheck({ + id: "device-scan", + label: "Audio input devices", + severity: deviceSnapshot.devices.length > 0 ? "ok" : "warning", + detail: deviceSnapshot.devices.length > 0 + ? `${deviceSnapshot.devices.length} audio input${deviceSnapshot.devices.length === 1 ? "" : "s"} discovered via Web Audio.${deviceSnapshot.labelsAvailable ? "" : " Device labels will become more descriptive after microphone permission is granted."}` + : "No audio inputs were discovered via Web Audio.", + remediation: "Grant microphone permission and check the OS input devices if the list stays empty.", + }); + + const selectedMic = settings.capture.microphoneDevice.trim(); + const selectedMicPresent = !selectedMic || deviceSnapshot.devices.some((device) => device.deviceId === selectedMic); + addCheck({ + id: "selected-microphone", + label: "Selected microphone", + severity: selectedMicPresent ? "ok" : "warning", + detail: selectedMic + ? selectedMicPresent + ? "Selected microphone is available." + : "Selected microphone is unavailable. Recording will fall back to the system default input." + : "No specific microphone selected. Recording will use the system default input.", + remediation: "Choose a microphone device if you want to pin one instead of following the OS default.", + }); + + if (loopbackEnabled) { + const selectedAdditional = settings.capture.systemDevice.trim(); + const selectedAdditionalPresent = Boolean( + selectedAdditional && deviceSnapshot.devices.some((device) => device.deviceId === selectedAdditional) + ); + const duplicateSource = + Boolean(selectedAdditional) && + selectedAdditionalPresent && + selectedMic && + selectedAdditional === selectedMic; + + addCheck({ + id: "selected-additional-source", + label: "Additional source", + severity: !selectedAdditional + ? "error" + : duplicateSource + ? "error" + : selectedAdditionalPresent + ? "ok" + : "warning", + detail: !selectedAdditional + ? "Additional source is enabled, but no second input device is selected." + : duplicateSource + ? "The additional source matches the microphone. Pick a different input such as BlackHole or VB-Cable." + : selectedAdditionalPresent + ? "Selected additional source is available." + : "Selected additional source is unavailable. Refresh devices or disable it.", + remediation: + "Select a second audioinput device such as BlackHole, VB-Cable, or a monitor source if you want multiple sources in the same recording.", + }); + } + addCheck({ id: "whisper-cli", label: "whisper.cpp CLI", @@ -81,54 +156,6 @@ export class DiagnosticsService { remediation: getWhisperModelRemediation(settings.transcription.modelPath, fs), }); - if (!settings.capture.microphoneDevice.trim()) { - addCheck({ - id: "microphone", - label: "Microphone selection", - severity: "warning", - detail: "No microphone device selected yet.", - remediation: "Refresh devices and select the main microphone input.", - }); - } - - try { - if (settings.capture.ffmpegPath.trim()) { - const devices = await scanDevices(settings.capture.ffmpegPath, backend); - const audioDevices = devices.filter((device) => device.type === "audio"); - const micPresent = !settings.capture.microphoneDevice.trim() - ? false - : audioDevices.some((device) => device.name === settings.capture.microphoneDevice || device.label === settings.capture.microphoneLabel); - addCheck({ - id: "device-scan", - label: "Audio devices", - severity: audioDevices.length > 0 ? "ok" : "warning", - detail: audioDevices.length > 0 - ? `${audioDevices.length} audio devices discovered for backend ${backend}.` - : `No audio devices discovered for backend ${backend}.`, - remediation: "Check OS permissions, FFmpeg backend, and input devices.", - }); - if (settings.capture.microphoneDevice.trim()) { - addCheck({ - id: "selected-microphone", - label: "Selected microphone", - severity: micPresent ? "ok" : "warning", - detail: micPresent - ? "Selected microphone is present in the current FFmpeg scan." - : "Selected microphone was not found in the current FFmpeg scan.", - remediation: "Refresh devices and reselect the microphone if indexes changed.", - }); - } - } - } catch (error) { - addCheck({ - id: "device-scan", - label: "Audio devices", - severity: "warning", - detail: `Device scan failed: ${String((error as Error)?.message ?? error)}`, - remediation: "Verify FFmpeg path and backend compatibility.", - }); - } - const providerCapabilities = getProviderCapabilities(settings.summary.provider); if (providerCapabilities.kind === "local") { const ollamaHealthy = await this.checkOllamaHealth(settings.summary.ollamaEndpoint); @@ -161,19 +188,6 @@ export class DiagnosticsService { remediation: "Set a dedicated output folder if you want generated notes grouped together.", }); - const profileLabel = CAPTURE_PROFILE_LABELS[settings.capture.captureProfile]; - const transcriptionWithNonStandardRate = - settings.capture.captureProfile === "transcription" && settings.capture.sampleRateHz !== 48000; - addCheck({ - id: "capture-profile", - label: "Audio processing profile", - severity: transcriptionWithNonStandardRate ? "warning" : "ok", - detail: transcriptionWithNonStandardRate - ? `Profile: ${profileLabel}. Sample rate ${settings.capture.sampleRateHz} Hz with the Transcription profile can cause extra resampling artifacts. Consider switching to 48000 Hz.` - : `Profile: ${profileLabel}. Active on next recording.`, - remediation: "Switch to Balanced or Natural profile if audio sounds robotic, or set sample rate to 48000 Hz.", - }); - const blockingIssueIds = checks.filter((check) => check.severity === "error").map((check) => check.id); const warningIds = checks.filter((check) => check.severity === "warning").map((check) => check.id); return { @@ -192,64 +206,58 @@ export class DiagnosticsService { } async runSmokeTest(settings: PluginSettingsV2): Promise { - if (!settings.capture.ffmpegPath.trim()) { - return { ok: false, detail: "FFmpeg path is missing." }; + const capability = getWebAudioCapability(); + const loopbackEnabled = isLoopbackSystemAudioEnabled(settings.capture); + if (!capability.hasGetUserMedia || !capability.hasEnumerateDevices) { + return { ok: false, detail: "Web Audio capture is unavailable in this runtime." }; } - const resolvedInputs = await resolveCaptureInputs(settings.capture); - if (!resolvedInputs.micSpec) { - return { ok: false, detail: "No microphone input is configured." }; + + if (loopbackEnabled && !settings.capture.systemDevice.trim()) { + return { ok: false, detail: "Additional source is enabled, but no second input device is selected." }; } const fs = requireNodeModule<{ - mkdtempSync: (prefix: string) => string; existsSync: (path: string) => boolean; - unlinkSync: (path: string) => void; + mkdirSync: (path: string, options: { recursive: boolean }) => void; + mkdtempSync: (prefix: string) => string; rmSync: (path: string, options: { recursive: boolean; force: boolean }) => void; }>("fs"); const os = requireNodeModule<{ tmpdir: () => string }>("os"); const path = requireNodeModule<{ join: (...parts: string[]) => string }>("path"); - const { spawn } = requireNodeModule<{ spawn: Function }>("child_process"); - const tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), "resonance-next-smoke-")); - const audioPath = path.join(tmpRoot, "smoke.mp3"); - const args = [ - "-y", - "-f", - resolvedInputs.backend, - "-thread_queue_size", - "1024", - "-i", - resolvedInputs.micSpec, - "-t", - String(settings.diagnostics.quickTestDurationSeconds), - "-vn", - "-ar", - String(settings.capture.sampleRateHz), - "-ac", - String(settings.capture.channels), - "-c:a", - "libmp3lame", - "-b:a", - `${settings.capture.bitrateKbps}k`, - audioPath, - ]; + const tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), "resonance-next-web-smoke-")); + const segmentsDir = path.join(tmpRoot, "segments"); + const audioPath = path.join(tmpRoot, "smoke.wav"); + if (!fs.existsSync(segmentsDir)) { + fs.mkdirSync(segmentsDir, { recursive: true }); + } + const adapter = new WebCaptureAdapter(); try { - await new Promise((resolve, reject) => { - const child = spawn(settings.capture.ffmpegPath, args); - let stderr = ""; - child.stderr?.on("data", (chunk: Buffer) => { - stderr += chunk.toString(); - }); - child.on("error", (error: Error) => reject(error)); - child.on("close", (code: number) => { - if (code === 0) resolve(); - else reject(new Error(stderr || `FFmpeg exited with code ${code}`)); - }); + await adapter.start({ + fullAudioPath: audioPath, + segmentsDir, + segmentDurationSeconds: Math.max(1, settings.diagnostics.quickTestDurationSeconds), + microphoneDevice: settings.capture.microphoneDevice, + additionalSources: + loopbackEnabled && settings.capture.systemDevice.trim() + ? [ + { + deviceId: settings.capture.systemDevice.trim(), + label: settings.capture.systemLabel.trim() || "Additional source", + }, + ] + : [], + onSegmentReady: () => {}, }); + await new Promise((resolve) => { + window.setTimeout(resolve, Math.max(300, settings.diagnostics.quickTestDurationSeconds * 1_000)); + }); + await adapter.stop(); + if (settings.transcription.whisperCliPath.trim() && settings.transcription.modelPath.trim() && fs.existsSync(audioPath)) { - const adapter = new WhisperTranscriptionAdapter(settings.transcription, settings.capture.ffmpegPath); - const transcript = await adapter.transcribeFile(audioPath); + const transcriptionAdapter = new WhisperTranscriptionAdapter(settings.transcription, settings.capture.ffmpegPath); + const transcript = await transcriptionAdapter.transcribeFile(audioPath); if (!transcript.trim()) { return { ok: false, detail: "Quick test recorded audio, but whisper.cpp returned an empty transcript." }; } @@ -264,6 +272,11 @@ export class DiagnosticsService { } catch (error) { return { ok: false, detail: String((error as Error)?.message ?? error) }; } finally { + try { + if (adapter.isRunning()) { + await adapter.stop(); + } + } catch {} try { fs.rmSync(tmpRoot, { recursive: true, force: true }); } catch {} diff --git a/src/infrastructure/system/webAudio.ts b/src/infrastructure/system/webAudio.ts new file mode 100644 index 0000000..45d69be --- /dev/null +++ b/src/infrastructure/system/webAudio.ts @@ -0,0 +1,102 @@ +export type WebAudioPermissionState = PermissionState | "unsupported" | "unknown"; + +export interface WebAudioInputDevice { + deviceId: string; + label: string; + groupId: string; + isDefault: boolean; +} + +export interface WebAudioDeviceSnapshot { + devices: WebAudioInputDevice[]; + permissionState: WebAudioPermissionState; + labelsAvailable: boolean; +} + +export interface WebAudioCapability { + supported: boolean; + hasGetUserMedia: boolean; + hasEnumerateDevices: boolean; +} + +export interface WebShareAudioCapability { + supported: boolean; + hasGetDisplayMedia: boolean; +} + +export function getWebAudioCapability(): WebAudioCapability { + const mediaDevices = globalThis.navigator?.mediaDevices; + return { + supported: Boolean(mediaDevices), + hasGetUserMedia: typeof mediaDevices?.getUserMedia === "function", + hasEnumerateDevices: typeof mediaDevices?.enumerateDevices === "function", + }; +} + +export function getWebShareAudioCapability(): WebShareAudioCapability { + const mediaDevices = globalThis.navigator?.mediaDevices; + return { + supported: Boolean(mediaDevices), + hasGetDisplayMedia: typeof mediaDevices?.getDisplayMedia === "function", + }; +} + +export async function getMicrophonePermissionState(): Promise { + if (!globalThis.navigator) return "unsupported"; + + try { + const permissions = globalThis.navigator.permissions; + if (!permissions?.query) return "unknown"; + const status = await permissions.query({ name: "microphone" as PermissionName }); + return status.state; + } catch { + return "unknown"; + } +} + +export async function listWebAudioInputDevices(): Promise { + const capability = getWebAudioCapability(); + if (!capability.hasEnumerateDevices || !globalThis.navigator?.mediaDevices) { + return { + devices: [], + permissionState: capability.supported ? "unknown" : "unsupported", + labelsAvailable: false, + }; + } + + const permissionState = await getMicrophonePermissionState(); + const devices = await globalThis.navigator.mediaDevices.enumerateDevices(); + const inputs = devices + .filter((device) => device.kind === "audioinput") + .map((device) => ({ + deviceId: device.deviceId, + label: device.label || (device.deviceId === "default" ? "System default input" : "Audio input"), + groupId: device.groupId, + isDefault: device.deviceId === "default", + })); + + return { + devices: inputs, + permissionState, + labelsAvailable: inputs.some((device) => { + const normalized = device.label.trim().toLowerCase(); + return normalized !== "" && normalized !== "microphone" && normalized !== "system default microphone"; + }), + }; +} + +export async function resolvePreferredWebAudioInput(selectedDeviceId?: string): Promise { + const snapshot = await listWebAudioInputDevices(); + if (!snapshot.devices.length) return undefined; + if (selectedDeviceId?.trim()) { + const exact = snapshot.devices.find((device) => device.deviceId === selectedDeviceId.trim()); + if (exact) return exact; + } + return snapshot.devices.find((device) => device.isDefault) ?? snapshot.devices[0]; +} + +export async function resolveWebAudioInputById(selectedDeviceId?: string): Promise { + if (!selectedDeviceId?.trim()) return undefined; + const snapshot = await listWebAudioInputDevices(); + return snapshot.devices.find((device) => device.deviceId === selectedDeviceId.trim()); +} diff --git a/src/ui/SettingsTab.ts b/src/ui/SettingsTab.ts index d0fa828..71d9ecc 100644 --- a/src/ui/SettingsTab.ts +++ b/src/ui/SettingsTab.ts @@ -3,20 +3,22 @@ import { buildDashboardSnapshot, deriveSessionListItem } from "../application/da import type { SessionController } from "../application/SessionController"; import type { DashboardSnapshot } from "../domain/dashboard"; import type { SessionListItem } from "../domain/session"; -import { isCoreConfigured, resolveCaptureBackend, type PluginSettingsV2 } from "../domain/settings"; -import { CAPTURE_PROFILE_LABELS, getCaptureProfileDescription, resolveAudioFilterChain } from "../domain/captureProfiles"; +import { isCoreConfigured, type PluginSettingsV2 } from "../domain/settings"; import { VaultAdapter } from "../infrastructure/adapters/VaultAdapter"; import { requireNodeModule } from "../infrastructure/node"; import { - autoDetectFfmpeg, autoDetectWhisperCli, autoDetectWhisperModel, autoDetectWhisperRepo, inferWhisperRepoPath, } from "../infrastructure/system/autoDetect"; import { getPluginInstance } from "../infrastructure/obsidianDesktop"; -import { scanDevices } from "../infrastructure/system/deviceScanner"; import { SessionStore } from "../infrastructure/storage/SessionStore"; +import { + listWebAudioInputDevices, + type WebAudioDeviceSnapshot, + type WebAudioPermissionState, +} from "../infrastructure/system/webAudio"; import { formatBytes, formatDuration } from "../utils/format"; import { uiCopy } from "./copy"; import { TextPreviewModal } from "./modals/TextPreviewModal"; @@ -74,7 +76,7 @@ const SETUP_TABS: SettingsTabDefinition[] = [ { key: "capture", label: "Capture", - subtitle: "FFmpeg, devices, quick test.", + subtitle: "Microphone and additional audio sources.", }, { key: "transcription", @@ -173,7 +175,28 @@ export class ResonanceNextSettingTab extends PluginSettingTab { private renderHero(container: HTMLElement) { const hero = container.createEl("div", { cls: "rxn-card rxn-hero" }); - hero.createEl("h2", { text: uiCopy.appName, cls: "rxn-hero-brand" }); + + const headerRow = hero.createEl("div"); + headerRow.style.display = "flex"; + headerRow.style.justifyContent = "space-between"; + headerRow.style.alignItems = "center"; + + headerRow.createEl("h2", { text: uiCopy.appName, cls: "rxn-hero-brand" }); + + const sponsor = headerRow.createEl("a", { + href: "https://buymeacoffee.com/michaelgorini", + }); + sponsor.target = "_blank"; + sponsor.style.display = "flex"; + sponsor.style.alignItems = "center"; + sponsor.style.gap = "6px"; + sponsor.style.fontSize = "13px"; + sponsor.style.color = "var(--text-muted)"; + sponsor.style.textDecoration = "none"; + + sponsor.createEl("span", { text: "Support Resonance" }); + sponsor.createEl("span", { text: "🤍" }); + hero.createEl("p", { text: uiCopy.settings.title, cls: "rxn-muted rxn-hero-subtitle", @@ -266,9 +289,8 @@ export class ResonanceNextSettingTab extends PluginSettingTab { meta.createEl("span", { text: `${snapshot.health.blockingCount} blocking`, cls: "rxn-pill" }); meta.createEl("span", { text: `${snapshot.health.warningCount} warnings`, cls: "rxn-pill" }); meta.createEl("span", { - text: `Quick test: ${ - this.isQuickTestRunning ? "running" : this.smokeMessage ? (this.smokeMessage.includes("failed") ? "failed" : "passed") : "not run" - }`, + text: `Quick test: ${this.isQuickTestRunning ? "running" : this.smokeMessage ? (this.smokeMessage.includes("failed") ? "failed" : "passed") : "not run" + }`, cls: "rxn-pill", }); meta.createEl("span", { text: `Backend: ${snapshot.health.report?.backend ?? "n/a"}`, cls: "rxn-pill" }); @@ -344,185 +366,109 @@ export class ResonanceNextSettingTab extends PluginSettingTab { } private async renderCaptureTab(container: HTMLElement) { - const settings = this.options.getSettings(); - const ffmpeg = this.createGuideSection(container, { - badge: "Recorder", - title: "FFmpeg", - intro: "FFmpeg is required for recording.", - platformGuides: { - macos: { - text: "Install FFmpeg, then paste the path to ffmpeg below.", - code: "brew install ffmpeg", + let settings = this.options.getSettings(); + if (settings.capture.captureEngine !== "web" || settings.capture.systemAudioMode === "share") { + await this.options.saveSettings((current) => ({ + ...current, + capture: { + ...current.capture, + captureEngine: "web", + systemAudioMode: current.capture.systemAudioMode === "share" ? "off" : current.capture.systemAudioMode, }, - linux: { - text: "Install FFmpeg with your package manager, then paste the path below.", - code: "sudo apt install ffmpeg", - }, - windows: { - text: "Install FFmpeg, then paste the full path to ffmpeg.exe below.", - code: "winget install Gyan.FFmpeg", - }, - }, - }); + })); + settings = this.options.getSettings(); + } - new Setting(ffmpeg) - .setName("FFmpeg path") - .setDesc("Path to the FFmpeg executable.") - .addText((text) => - text.setPlaceholder("/opt/homebrew/bin/ffmpeg").setValue(settings.capture.ffmpegPath).onChange(async (value) => { - await this.options.saveSettings((current) => ({ - ...current, - capture: { ...current.capture, ffmpegPath: value.trim() }, - })); - }) - ) - .addButton((button) => - button.setButtonText(uiCopy.actions.detectFfmpeg).onClick(async () => { - const detected = await autoDetectFfmpeg(); - if (!detected) { - new Notice(uiCopy.notices.ffmpegNotDetected); - return; - } - await this.options.saveSettings((current) => ({ - ...current, - capture: { ...current.capture, ffmpegPath: detected }, - })); - new Notice(uiCopy.notices.ffmpegDetected); - await this.display(); - }) - ); + await this.renderWebCaptureTab(container, settings); + } + + private async renderWebCaptureTab(container: HTMLElement, settings: PluginSettingsV2) { + const deviceSnapshot = await listWebAudioInputDevices().catch(() => ({ + devices: [], + permissionState: "unknown", + labelsAvailable: false, + })); + const additionalSourceEnabled = + settings.capture.systemAudioMode === "loopback" && Boolean(settings.capture.systemDevice.trim()); const capture = this.createGuideSection(container, { badge: "Devices", - title: "Inputs", - intro: "Pick a microphone. Add system audio only if you really need it.", - platformGuides: { - macos: { - bullets: ["For system audio, install a loopback device such as BlackHole and select it below."], - }, - linux: { - bullets: ["For system audio, use a PulseAudio or PipeWire monitor source if available."], - }, - windows: { - bullets: ["For system audio, use a loopback device such as VB-Cable and select it below."], - }, - }, + title: "Audio sources", + intro: "Resonance now records through one Web Audio graph. Choose the microphone first, then optionally add a second input such as BlackHole, VB-Cable, or a monitor source.", }); - new Setting(capture) - .setName("Backend") - .setDesc("Use Automatic unless device scan fails.") - .addDropdown((dropdown) => { - dropdown.addOption("auto", "Automatic"); - dropdown.addOption("avfoundation", "avfoundation (macOS)"); - dropdown.addOption("dshow", "dshow (Windows)"); - dropdown.addOption("pulse", "pulse (Linux)"); - dropdown.addOption("alsa", "alsa (Linux)"); - dropdown.setValue(settings.capture.backend); - dropdown.onChange(async (value) => { - await this.options.saveSettings((current) => ({ - ...current, - capture: { ...current.capture, backend: value as PluginSettingsV2["capture"]["backend"] }, - })); - }); + const meta = capture.createDiv({ cls: "rxn-pill-row rxn-diagnostics-meta" }); + meta.createEl("span", { + text: `Permission: ${this.getWebPermissionLabel(deviceSnapshot.permissionState)}`, + cls: `rxn-status-pill is-${this.getWebPermissionTone(deviceSnapshot.permissionState)}`, + }); + meta.createEl("span", { + text: deviceSnapshot.devices.length > 0 ? `${deviceSnapshot.devices.length} mic inputs` : "No mic inputs found", + cls: "rxn-pill", + }); + if (!deviceSnapshot.labelsAvailable) { + meta.createEl("span", { + text: "Labels improve after granting mic permission", + cls: "rxn-pill", }); + } const micSetting = new Setting(capture) .setName("Microphone device") - .setDesc("Pick the microphone to record."); + .setDesc( + deviceSnapshot.labelsAvailable + ? "Pick the main voice input or stay on the system default input." + : "Use the system default input or grant permission once to reveal clearer device labels." + ); const micSelect = micSetting.controlEl.createEl("select"); micSelect.addClass("rxn-inline-select"); - - const systemSetting = new Setting(capture) - .setName("System audio device") - .setDesc("Optional loopback or monitor input."); - const systemSelect = systemSetting.controlEl.createEl("select"); - systemSelect.addClass("rxn-inline-select"); - - const captureActions = capture.createDiv({ cls: "rxn-action-bar" }); - this.createActionButton(captureActions, uiCopy.actions.refreshDevices, async () => { - await this.populateDevices(micSelect, systemSelect, true); - }, "rxn-btn-secondary"); - this.createActionButton( - captureActions, - this.isQuickTestRunning ? "Running quick test..." : uiCopy.actions.runQuickTest, - async () => { - await this.runQuickTest(); - await this.switchTab("control-room"); - }, - "rxn-btn-secondary", - this.isQuickTestRunning - ); - - const captureMeta = capture.createDiv({ cls: "rxn-pill-row rxn-diagnostics-meta" }); - captureMeta.createEl("span", { - text: `Backend: ${resolveCaptureBackend(settings.capture.backend)}`, - cls: "rxn-pill", + this.populateWebAudioInputs(micSelect, deviceSnapshot, settings.capture.microphoneDevice, { + defaultLabel: "System default input", }); - - await this.populateDevices(micSelect, systemSelect); micSelect.addEventListener("change", async () => { const selected = micSelect.options[micSelect.selectedIndex]; - if (!selected?.value) return; await this.options.saveSettings((current) => ({ ...current, capture: { ...current.capture, - microphoneDevice: selected.value, - microphoneLabel: selected.text, + captureEngine: "web", + microphoneDevice: selected?.value === "default" ? "" : selected?.value ?? "", + microphoneLabel: selected?.text ?? "", }, })); }); + + const systemSetting = new Setting(capture) + .setName("Additional source") + .setDesc( + deviceSnapshot.labelsAvailable + ? "Optional second input. Select a loopback device here if you want Teams, Meet, browser tabs, or desktop audio in the same recording." + : "Optional second input. Grant permission once to reveal clearer device labels for loopback devices too." + ); + const systemSelect = systemSetting.controlEl.createEl("select"); + systemSelect.addClass("rxn-inline-select"); + this.populateWebAudioInputs(systemSelect, deviceSnapshot, additionalSourceEnabled ? settings.capture.systemDevice : "", { + includeOff: true, + offLabel: "Off (microphone only)", + }); systemSelect.addEventListener("change", async () => { const selected = systemSelect.options[systemSelect.selectedIndex]; + const enabled = Boolean(selected?.value); await this.options.saveSettings((current) => ({ ...current, capture: { ...current.capture, - systemDevice: selected?.value ?? "", - systemLabel: selected?.value ? selected.text : "", + captureEngine: "web", + systemAudioMode: enabled ? "loopback" : "off", + systemDevice: enabled ? selected.value : "", + systemLabel: enabled ? selected.text : "", }, })); }); new Setting(capture) - .setName("Sample rate") - .setDesc("Usually 48000.") - .addText((text) => - text.setValue(String(settings.capture.sampleRateHz)).onChange(async (value) => { - await this.options.saveSettings((current) => ({ - ...current, - capture: { ...current.capture, sampleRateHz: Math.max(8_000, Math.min(192_000, Number(value) || 48_000)) }, - })); - }) - ); - - new Setting(capture) - .setName("Channels") - .setDesc("Mono is lighter. Stereo keeps more room context.") - .addDropdown((dropdown) => { - dropdown.addOption("1", "Mono"); - dropdown.addOption("2", "Stereo"); - dropdown.setValue(String(settings.capture.channels)); - dropdown.onChange(async (value) => { - await this.options.saveSettings((current) => ({ - ...current, - capture: { ...current.capture, channels: value === "1" ? 1 : 2 }, - })); - }); - }); - - new Setting(capture) - .setName("Bitrate / segment seconds") - .setDesc("MP3 quality and live chunk length.") - .addText((text) => - text.setPlaceholder("160").setValue(String(settings.capture.bitrateKbps)).onChange(async (value) => { - await this.options.saveSettings((current) => ({ - ...current, - capture: { ...current.capture, bitrateKbps: Math.max(64, Math.min(320, Number(value) || 160)) }, - })); - }) - ) + .setName("Segment seconds") + .setDesc("How often live transcript chunks are committed.") .addText((text) => text.setPlaceholder("20").setValue(String(settings.capture.segmentDurationSeconds)).onChange(async (value) => { await this.options.saveSettings((current) => ({ @@ -532,118 +478,10 @@ export class ResonanceNextSettingTab extends PluginSettingTab { }) ); - // ── Audio processing ─────────────────────────────────────────────────── - const audioSection = this.createGuideSection(container, { - badge: "Audio", - title: uiCopy.capture.profileSection, - intro: uiCopy.capture.profileIntro, - }); - - // Troubleshooting callout - const tsBox = audioSection.createDiv({ cls: "rxn-inline-note is-info" }); - tsBox.createEl("strong", { text: uiCopy.capture.troubleshooting.title }); - const tsList = tsBox.createEl("ul", { cls: "rxn-guide-list" }); - tsList.createEl("li", { text: uiCopy.capture.troubleshooting.robotic }); - tsList.createEl("li", { text: uiCopy.capture.troubleshooting.inaccurate }); - tsList.createEl("li", { text: uiCopy.capture.troubleshooting.noisy }); - - // Profile dropdown - const profileDesc = getCaptureProfileDescription(settings.capture.captureProfile); - new Setting(audioSection) - .setName(uiCopy.capture.profileLabel) - .setDesc(profileDesc) - .addDropdown((dropdown) => { - for (const [value, label] of Object.entries(CAPTURE_PROFILE_LABELS)) { - dropdown.addOption(value, label); - } - dropdown.setValue(settings.capture.captureProfile); - dropdown.onChange(async (value) => { - await this.options.saveSettings((current) => ({ - ...current, - capture: { ...current.capture, captureProfile: value as PluginSettingsV2["capture"]["captureProfile"] }, - })); - await this.display(); - }); - }); - - // Custom profile controls (only shown when profile = custom) - if (settings.capture.captureProfile === "custom") { - const customSection = this.createGuideSection(container, { - badge: "Custom", - title: uiCopy.capture.customSection, - intro: uiCopy.capture.customIntro, - }); - - new Setting(customSection) - .setName(uiCopy.capture.micGainLabel) - .setDesc(uiCopy.capture.micGainDesc) - .addText((text) => - text - .setPlaceholder("0") - .setValue(String(settings.capture.micGainDb)) - .onChange(async (value) => { - const parsed = parseFloat(value); - if (!Number.isFinite(parsed)) return; - await this.options.saveSettings((current) => ({ - ...current, - capture: { ...current.capture, micGainDb: Math.max(-12, Math.min(12, parsed)) }, - })); - }) - ); - - new Setting(customSection) - .setName(uiCopy.capture.systemGainLabel) - .setDesc(uiCopy.capture.systemGainDesc) - .addText((text) => - text - .setPlaceholder("0") - .setValue(String(settings.capture.systemGainDb)) - .onChange(async (value) => { - const parsed = parseFloat(value); - if (!Number.isFinite(parsed)) return; - await this.options.saveSettings((current) => ({ - ...current, - capture: { ...current.capture, systemGainDb: Math.max(-12, Math.min(12, parsed)) }, - })); - }) - ); - - new Setting(customSection) - .setName(uiCopy.capture.noiseSuppressionLabel) - .setDesc(uiCopy.capture.noiseSuppressionDesc) - .addToggle((toggle) => - toggle.setValue(settings.capture.noiseSuppression).onChange(async (value) => { - await this.options.saveSettings((current) => ({ - ...current, - capture: { ...current.capture, noiseSuppression: value }, - })); - }) - ); - - new Setting(customSection) - .setName(uiCopy.capture.limiterLabel) - .setDesc(uiCopy.capture.limiterDesc) - .addToggle((toggle) => - toggle.setValue(settings.capture.limiter).onChange(async (value) => { - await this.options.saveSettings((current) => ({ - ...current, - capture: { ...current.capture, limiter: value }, - })); - }) - ); - } - - // Filter preview — always visible - const hasDual = Boolean(settings.capture.systemDevice || settings.capture.systemLabel); - const { filterPreview } = resolveAudioFilterChain(settings.capture, hasDual); - const previewSetting = new Setting(audioSection) - .setName(uiCopy.capture.filterPreviewLabel) - .setDesc(uiCopy.capture.filterPreviewDesc); - const previewEl = previewSetting.controlEl.createEl("code", { - text: filterPreview, - cls: "rxn-filter-preview", - }); - previewEl.setAttribute("title", filterPreview); + const captureActions = capture.createDiv({ cls: "rxn-action-bar" }); + this.createActionButton(captureActions, uiCopy.actions.refreshDevices, async () => { + await this.display(); + }, "rxn-btn-secondary"); } private async renderTranscriptionTab(container: HTMLElement) { @@ -1109,6 +947,7 @@ export class ResonanceNextSettingTab extends PluginSettingTab { }); const meta = card.createDiv({ cls: "rxn-session-meta-row" }); + this.renderSessionMeta(meta, "Source", item.sourceLabel); this.renderSessionMeta(meta, "Started", new Date(item.createdAt).toLocaleString()); this.renderSessionMeta(meta, "Duration", formatDuration(item.elapsedSeconds)); this.renderSessionMeta(meta, "Segments", String(item.committedSegments)); @@ -1281,7 +1120,7 @@ export class ResonanceNextSettingTab extends PluginSettingTab { const electron = requireNodeModule<{ shell?: { showItemInFolder?: (path: string) => void } }>("electron"); const sessionsRoot = this.store.getSessionsRootDir(); electron.shell?.showItemInFolder?.(sessionsRoot); - } catch {} + } catch { } } private createGuideSection(container: HTMLElement, options: GuideSectionOptions): HTMLElement { @@ -1494,7 +1333,11 @@ export class ResonanceNextSettingTab extends PluginSettingTab { try { new Notice("Quick test started..."); const result = await this.options.controller.runSmokeTest(); - this.smokeMessage = result.ok ? uiCopy.diagnostics.smokePassed : `${uiCopy.notices.quickTestFailedPrefix}: ${result.detail}`; + this.smokeMessage = result.ok + ? uiCopy.diagnostics.smokePassed + : result.cancelled + ? result.detail + : `${uiCopy.notices.quickTestFailedPrefix}: ${result.detail}`; new Notice(this.smokeMessage); await this.refreshControlRoomData(false); } finally { @@ -1509,19 +1352,21 @@ export class ResonanceNextSettingTab extends PluginSettingTab { const fs = requireNodeModule<{ readFileSync: (path: string) => Buffer }>("fs"); const buffer = fs.readFileSync(path); const bytes = Uint8Array.from(buffer); - const url = URL.createObjectURL(new Blob([bytes], { type: "audio/mpeg" })); + const url = URL.createObjectURL(new Blob([bytes], { type: this.getAudioMimeType(path) })); this.audioUrlCache.set(path, url); return url; } private exportAudio(item: SessionListItem) { + const pathModule = requireNodeModule<{ extname: (path: string) => string }>("path"); const fs = requireNodeModule<{ readFileSync: (path: string) => Buffer }>("fs"); const bytes = Uint8Array.from(fs.readFileSync(item.paths.fullAudioPath)); - const blob = new Blob([bytes], { type: "audio/mpeg" }); + const extension = pathModule.extname(item.paths.fullAudioPath) || ".mp3"; + const blob = new Blob([bytes], { type: this.getAudioMimeType(item.paths.fullAudioPath) }); const url = URL.createObjectURL(blob); const anchor = document.createElement("a"); anchor.href = url; - anchor.download = `${item.scenarioLabel}-${item.sessionId}.mp3`; + anchor.download = `${item.scenarioLabel}-${item.sessionId}${extension}`; anchor.click(); window.setTimeout(() => URL.revokeObjectURL(url), 1_000); } @@ -1542,68 +1387,80 @@ export class ResonanceNextSettingTab extends PluginSettingTab { return button; } - private resetDeviceSelects(micSelect: HTMLSelectElement, systemSelect: HTMLSelectElement, placeholder: string) { - micSelect.empty(); - const micPlaceholder = document.createElement("option"); - micPlaceholder.value = ""; - micPlaceholder.text = placeholder; - micSelect.appendChild(micPlaceholder); + private populateWebAudioInputs( + select: HTMLSelectElement, + snapshot: WebAudioDeviceSnapshot, + selectedDeviceId: string, + options: { + includeOff?: boolean; + offLabel?: string; + defaultLabel?: string; + } = {} + ) { + select.empty(); - systemSelect.empty(); - const none = document.createElement("option"); - none.value = ""; - none.text = "(none)"; - systemSelect.appendChild(none); - } + const seen = new Set(); + const addOption = (value: string, label: string) => { + if (seen.has(value)) return; + seen.add(value); + const option = document.createElement("option"); + option.value = value; + option.text = label; + select.appendChild(option); + }; - private async populateDevices(micSelect: HTMLSelectElement, systemSelect: HTMLSelectElement, notifyOnFailure = false) { - const settings = this.options.getSettings(); - this.resetDeviceSelects( - micSelect, - systemSelect, - settings.capture.ffmpegPath.trim() ? "Refresh devices to load microphones" : "Set FFmpeg path first" - ); - if (!settings.capture.ffmpegPath.trim()) return; + if (options.includeOff) { + addOption("", options.offLabel ?? "Off"); + } - try { - const scanned = await scanDevices(settings.capture.ffmpegPath, resolveCaptureBackend(settings.capture.backend)); - const audioDevices = scanned.filter((device) => device.type === "audio"); - if (audioDevices.length === 0) { - return; - } + addOption("default", options.defaultLabel ?? "System default input"); + for (const device of snapshot.devices) { + addOption(device.deviceId, device.label); + } - micSelect.empty(); - for (const device of audioDevices) { - const option = document.createElement("option"); - option.value = device.name; - option.text = device.label; - micSelect.appendChild(option); - - const sysOption = document.createElement("option"); - sysOption.value = device.name; - sysOption.text = device.label; - systemSelect.appendChild(sysOption); - } - - if ( - settings.capture.microphoneDevice && - Array.from(micSelect.options).some((option) => option.value === settings.capture.microphoneDevice) - ) { - micSelect.value = settings.capture.microphoneDevice; - } else if (micSelect.options.length > 0) { - micSelect.selectedIndex = 0; - } - - if ( - settings.capture.systemDevice && - Array.from(systemSelect.options).some((option) => option.value === settings.capture.systemDevice) - ) { - systemSelect.value = settings.capture.systemDevice; - } - } catch (error) { - if (notifyOnFailure) { - new Notice(`Device scan failed: ${String((error as Error)?.message ?? error)}`); - } + if (selectedDeviceId && seen.has(selectedDeviceId)) { + select.value = selectedDeviceId; + } else if (!selectedDeviceId && options.includeOff) { + select.value = ""; + } else { + select.value = "default"; } } + + private getWebPermissionLabel(permission: WebAudioPermissionState): string { + switch (permission) { + case "granted": + return "Granted"; + case "denied": + return "Denied"; + case "prompt": + return "Not requested"; + case "unsupported": + return "Unsupported"; + case "unknown": + default: + return "Unknown"; + } + } + + private getWebPermissionTone(permission: WebAudioPermissionState): "healthy" | "warning" | "failed" { + switch (permission) { + case "granted": + return "healthy"; + case "denied": + case "unsupported": + return "failed"; + case "prompt": + case "unknown": + default: + return "warning"; + } + } + + private getAudioMimeType(path: string): string { + const pathModule = requireNodeModule<{ extname: (path: string) => string }>("path"); + const extension = pathModule.extname(path).toLowerCase(); + if (extension === ".wav") return "audio/wav"; + return "audio/mpeg"; + } } diff --git a/tests/captureProfiles.test.ts b/tests/captureProfiles.test.ts index 4516ca0..f0b035c 100644 --- a/tests/captureProfiles.test.ts +++ b/tests/captureProfiles.test.ts @@ -4,6 +4,8 @@ import { resolveAudioFilterChain, resolveAudioProfile, CAPTURE_PROFILE_LABELS } import type { CaptureSettings } from "../src/domain/settings"; const baseCapture: CaptureSettings = { + captureEngine: "ffmpeg", + systemAudioMode: "off", ffmpegPath: "/usr/bin/ffmpeg", backend: "avfoundation", microphoneDevice: ":2", diff --git a/tests/dashboard.test.ts b/tests/dashboard.test.ts index 511561b..dfa4eac 100644 --- a/tests/dashboard.test.ts +++ b/tests/dashboard.test.ts @@ -20,20 +20,22 @@ const diagnosticsReport: DiagnosticsReport = { }; const manifest: RecordingSessionManifest = { - schemaVersion: 1, + schemaVersion: 3, sessionId: "session-1", createdAt: "2026-04-22T08:00:00.000Z", updatedAt: "2026-04-22T08:30:00.000Z", scenarioKey: "work_meeting", scenarioLabel: "Meeting", captureMode: "microphone", + captureEngine: "web", + systemAudioMode: "off", status: "failed", paths: { rootDir: "/tmp/session-1", manifestPath: "/tmp/session-1/session.json", diagnosticsLogPath: "/tmp/session-1/diagnostics.log", audioDir: "/tmp/session-1/audio", - fullAudioPath: "/tmp/session-1/audio/recording.mp3", + fullAudioPath: "/tmp/session-1/audio/recording.wav", segmentsDir: "/tmp/session-1/audio/segments", transcriptDir: "/tmp/session-1/transcript", transcriptTextPath: "/tmp/session-1/transcript/live-transcript.txt", @@ -87,6 +89,22 @@ test("deriveSessionListItem exposes dashboard and library metadata", () => { assert.equal(item.healthBadge, "failed"); assert.equal(item.failureSummary, "Summary provider returned an empty result."); assert.equal(item.artifactAvailability.hasTranscript, true); + assert.equal(item.sourceLabel, "Microphone"); +}); + +test("deriveSessionListItem labels multi-source web sessions clearly", () => { + const item = deriveSessionListItem( + { + ...manifest, + sessionId: "session-2", + captureMode: "microphone+system", + captureEngine: "web", + systemAudioMode: "loopback", + }, + 24_000 + ); + + assert.equal(item.sourceLabel, "Microphone + additional source"); }); test("deriveSessionHealthBadge marks finalizing sessions as warning", () => { diff --git a/tests/sessionController.test.ts b/tests/sessionController.test.ts index bddc7f9..996fecb 100644 --- a/tests/sessionController.test.ts +++ b/tests/sessionController.test.ts @@ -43,3 +43,27 @@ test("collectSegmentDescriptors includes the newest segment during stop flush", assert.deepEqual(segments, [{ index: 0, path: "/tmp/segment-0000.mp3" }]); }); + +test("collectSegmentDescriptors accepts wav segments from web capture", () => { + const now = 10_000; + const segments = collectSegmentDescriptors( + [ + { + name: "segment-0000.wav", + path: "/tmp/segment-0000.wav", + mtimeMs: now - 5_000, + isFile: true, + }, + { + name: "segment-0001.wav", + path: "/tmp/segment-0001.wav", + mtimeMs: now - 100, + isFile: true, + }, + ], + now, + false + ); + + assert.deepEqual(segments, [{ index: 0, path: "/tmp/segment-0000.wav" }]); +}); diff --git a/tests/settings.test.ts b/tests/settings.test.ts index ceadee9..fd39dd8 100644 --- a/tests/settings.test.ts +++ b/tests/settings.test.ts @@ -2,15 +2,17 @@ import test from "node:test"; import assert from "node:assert/strict"; import { DEFAULT_SETTINGS_V2, + isCoreConfigured, isLikelyTestWhisperModelPath, normalizeSettingsV2, resolveCaptureBackend, + resolveCaptureRuntime, } from "../src/domain/settings"; import { getSelectedSummaryModel } from "../src/domain/providers"; test("normalizeSettingsV2 keeps defaults and clamps invalid values", () => { const settings = normalizeSettingsV2({ - capture: { sampleRateHz: 1, channels: 7, bitrateKbps: 999 }, + capture: { captureEngine: "web", sampleRateHz: 1, channels: 7, bitrateKbps: 999 }, transcription: { beamSize: 99, language: "it" }, summary: { provider: "ollama", ollamaModel: "" }, }); @@ -21,6 +23,7 @@ test("normalizeSettingsV2 keeps defaults and clamps invalid values", () => { assert.equal(settings.transcription.beamSize, 10); assert.equal(settings.transcription.language, "it"); assert.equal(settings.output.vaultFolder, DEFAULT_SETTINGS_V2.output.vaultFolder); + assert.equal(settings.capture.captureEngine, "web"); }); test("resolveCaptureBackend maps auto to OS defaults", () => { @@ -40,3 +43,139 @@ test("isLikelyTestWhisperModelPath flags whisper.cpp CI models", () => { assert.equal(isLikelyTestWhisperModelPath("/Users/test/whisper.cpp/models/for-tests-ggml-base.bin"), true); assert.equal(isLikelyTestWhisperModelPath("/Users/test/whisper.cpp/models/ggml-base.bin"), false); }); + +test("normalizeSettingsV2 migrates fresh installs to web capture by default", () => { + const settings = normalizeSettingsV2({ + capture: { + microphoneDevice: ":2", + microphoneLabel: "Legacy mic", + }, + }); + + assert.equal(settings.capture.captureEngine, "web"); +}); + +test("normalizeSettingsV2 preserves an explicit ffmpeg setting for compatibility", () => { + const settings = normalizeSettingsV2({ + capture: { + captureEngine: "ffmpeg", + ffmpegPath: "/usr/bin/ffmpeg", + captureProfile: "transcription", + }, + }); + + assert.equal(settings.capture.captureEngine, "ffmpeg"); +}); + +test("normalizeSettingsV2 keeps system-device installs on web and enables the second source", () => { + const settings = normalizeSettingsV2({ + capture: { + systemDevice: ":1", + systemLabel: "BlackHole 2ch", + }, + }); + + assert.equal(settings.capture.captureEngine, "web"); + assert.equal(settings.capture.systemAudioMode, "loopback"); +}); + +test("normalizeSettingsV2 collapses the old share mode back to off", () => { + const settings = normalizeSettingsV2({ + capture: { + captureEngine: "web", + systemAudioMode: "share", + }, + }); + + assert.equal(settings.capture.captureEngine, "web"); + assert.equal(settings.capture.systemAudioMode, "off"); + assert.equal(resolveCaptureRuntime(settings.capture), "web-mic"); +}); + +test("normalizeSettingsV2 keeps explicit off mode even when stale loopback fields exist", () => { + const settings = normalizeSettingsV2({ + capture: { + systemAudioMode: "off", + systemDevice: ":1", + systemLabel: "BlackHole 2ch", + }, + }); + + assert.equal(settings.capture.captureEngine, "web"); + assert.equal(settings.capture.systemAudioMode, "off"); + assert.equal(resolveCaptureRuntime(settings.capture), "web-mic"); +}); + +test("resolveCaptureRuntime branches across mic-only and multi-input web capture", () => { + assert.equal(resolveCaptureRuntime({ ...DEFAULT_SETTINGS_V2.capture }), "web-mic"); + assert.equal( + resolveCaptureRuntime({ ...DEFAULT_SETTINGS_V2.capture, captureEngine: "ffmpeg" }), + "web-mic" + ); + assert.equal( + resolveCaptureRuntime({ ...DEFAULT_SETTINGS_V2.capture, systemAudioMode: "loopback" }), + "web-multi-input" + ); +}); + +test("isCoreConfigured does not require ffmpeg for microphone-only web capture", () => { + const settings = normalizeSettingsV2({ + capture: { + captureEngine: "web", + ffmpegPath: "", + }, + transcription: { + whisperCliPath: "/tmp/whisper-cli", + modelPath: "/tmp/ggml-small.bin", + }, + summary: { + provider: "ollama", + ollamaEndpoint: "http://localhost:11434", + ollamaModel: "gemma3", + }, + }); + + assert.equal(isCoreConfigured(settings), true); +}); + +test("isCoreConfigured allows a second web-audio source without ffmpeg", () => { + const settings = normalizeSettingsV2({ + capture: { + captureEngine: "web", + systemDevice: ":1", + systemLabel: "BlackHole 2ch", + }, + transcription: { + whisperCliPath: "/tmp/whisper-cli", + modelPath: "/tmp/ggml-small.bin", + }, + summary: { + provider: "ollama", + ollamaEndpoint: "http://localhost:11434", + ollamaModel: "gemma3", + }, + }); + + assert.equal(isCoreConfigured(settings), true); +}); + +test("isCoreConfigured requires an explicit second source when loopback mode is selected", () => { + const settings = normalizeSettingsV2({ + capture: { + captureEngine: "web", + systemAudioMode: "loopback", + systemDevice: "", + }, + transcription: { + whisperCliPath: "/tmp/whisper-cli", + modelPath: "/tmp/ggml-small.bin", + }, + summary: { + provider: "ollama", + ollamaEndpoint: "http://localhost:11434", + ollamaModel: "gemma3", + }, + }); + + assert.equal(isCoreConfigured(settings), false); +}); diff --git a/tests/sharedAudioCapture.test.ts b/tests/sharedAudioCapture.test.ts new file mode 100644 index 0000000..c50f109 --- /dev/null +++ b/tests/sharedAudioCapture.test.ts @@ -0,0 +1,232 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { SharedAudioCaptureAdapter } from "../src/infrastructure/adapters/SharedAudioCaptureAdapter"; +import { CaptureCancelledError } from "../src/infrastructure/adapters/mediaCaptureErrors"; + +installDesktopRuntime(); + +test("SharedAudioCaptureAdapter writes wav segments and final recording from shared audio", async () => { + installFakeSharedAudioEnvironment({ + getDisplayMedia: async () => new FakeSharedMediaStream(true), + }); + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "resonance-shared-capture-")); + const segmentsDir = path.join(tmpDir, "segments"); + fs.mkdirSync(segmentsDir, { recursive: true }); + + const ready: Array<{ index: number; path: string }> = []; + const adapter = new SharedAudioCaptureAdapter(); + await adapter.start({ + fullAudioPath: path.join(tmpDir, "recording.wav"), + segmentsDir, + segmentDurationSeconds: 1, + onSegmentReady: (segment) => ready.push(segment), + }); + + const processor = FakeAudioContext.lastProcessor; + assert.ok(processor, "expected the fake script processor to be created"); + + processor.emit([ + new Float32Array([0, 0.2, 0.4, 0.6, 0.8, 1]), + new Float32Array([1, 0.8, 0.6, 0.4, 0.2, 0]), + ]); + processor.emit([ + new Float32Array([0.5, 0.5, 0.5, 0.5, 0.5]), + new Float32Array([0.25, 0.25, 0.25, 0.25, 0.25]), + ]); + + await adapter.stop(); + + assert.deepEqual( + ready.map((segment) => segment.index), + [0, 1] + ); + assert.equal(fs.existsSync(path.join(tmpDir, "recording.wav")), true); + assert.equal(fs.existsSync(path.join(segmentsDir, "segment-0000.wav")), true); + assert.equal(fs.existsSync(path.join(segmentsDir, "segment-0001.wav")), true); + assert.equal(fs.readFileSync(path.join(segmentsDir, "segment-0000.wav")).readUInt32LE(40), 20); + assert.equal(fs.readFileSync(path.join(segmentsDir, "segment-0001.wav")).readUInt32LE(40), 2); + assert.equal(fs.readFileSync(path.join(tmpDir, "recording.wav")).readUInt32LE(40), 22); + + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +test("SharedAudioCaptureAdapter turns share cancellation into CaptureCancelledError", async () => { + installFakeSharedAudioEnvironment({ + getDisplayMedia: async () => { + const error = new Error("cancelled"); + Object.assign(error, { name: "AbortError" }); + throw error; + }, + }); + + const adapter = new SharedAudioCaptureAdapter(); + await assert.rejects( + () => + adapter.start({ + fullAudioPath: "/tmp/ignored.wav", + segmentsDir: "/tmp", + segmentDurationSeconds: 1, + onSegmentReady: () => {}, + }), + CaptureCancelledError + ); +}); + +test("SharedAudioCaptureAdapter reports a friendly message when the runtime rejects shared audio as unsupported", async () => { + installFakeSharedAudioEnvironment({ + getDisplayMedia: async () => { + const error = new Error("Not supported"); + Object.assign(error, { name: "NotSupportedError" }); + throw error; + }, + }); + + const adapter = new SharedAudioCaptureAdapter(); + await assert.rejects( + () => + adapter.start({ + fullAudioPath: "/tmp/ignored.wav", + segmentsDir: "/tmp", + segmentDurationSeconds: 1, + onSegmentReady: () => {}, + }), + /use loopback device instead/i + ); +}); + +test("SharedAudioCaptureAdapter fails clearly when the shared surface exposes no audio track", async () => { + installFakeSharedAudioEnvironment({ + getDisplayMedia: async () => new FakeSharedMediaStream(false), + }); + + const adapter = new SharedAudioCaptureAdapter(); + await assert.rejects( + () => + adapter.start({ + fullAudioPath: "/tmp/ignored.wav", + segmentsDir: "/tmp", + segmentDurationSeconds: 1, + onSegmentReady: () => {}, + }), + /did not provide an audio track/i + ); +}); + +function installDesktopRuntime(): void { + Object.defineProperty(globalThis, "window", { + value: { require }, + configurable: true, + writable: true, + }); +} + +function installFakeSharedAudioEnvironment(options: { + getDisplayMedia: (constraints: DisplayMediaStreamOptions) => Promise; +}): void { + Object.defineProperty(globalThis, "navigator", { + value: { + mediaDevices: { + getDisplayMedia: options.getDisplayMedia, + }, + }, + configurable: true, + writable: true, + }); + + Object.defineProperty(globalThis, "AudioContext", { + value: FakeAudioContext, + configurable: true, + writable: true, + }); +} + +class FakeSharedMediaStream { + private readonly audioTracks: FakeMediaStreamTrack[]; + private readonly tracks: FakeMediaStreamTrack[]; + + constructor(includeAudioTrack: boolean) { + const videoTrack = new FakeMediaStreamTrack(); + this.audioTracks = includeAudioTrack ? [new FakeMediaStreamTrack()] : []; + this.tracks = includeAudioTrack ? [this.audioTracks[0], videoTrack] : [videoTrack]; + } + + getAudioTracks(): MediaStreamTrack[] { + return this.audioTracks as unknown as MediaStreamTrack[]; + } + + getTracks(): MediaStreamTrack[] { + return this.tracks as unknown as MediaStreamTrack[]; + } +} + +class FakeMediaStreamTrack { + stop(): void {} +} + +class FakeSourceNode { + connect(): void {} + disconnect(): void {} +} + +class FakeGainNode { + gain = { value: 1 }; + + connect(): void {} + disconnect(): void {} +} + +class FakeScriptProcessorNode { + onaudioprocess: ((event: AudioProcessingEvent) => void) | null = null; + + connect(): void {} + disconnect(): void {} + + emit(channels: Float32Array[]): void { + const inputBuffer = new FakeAudioBuffer(channels); + this.onaudioprocess?.({ inputBuffer } as unknown as AudioProcessingEvent); + } +} + +class FakeAudioBuffer { + readonly numberOfChannels: number; + readonly length: number; + + constructor(private readonly channels: Float32Array[]) { + this.numberOfChannels = channels.length; + this.length = channels[0]?.length ?? 0; + } + + getChannelData(channelIndex: number): Float32Array { + return this.channels[channelIndex] ?? new Float32Array(this.length); + } +} + +class FakeAudioContext { + static lastProcessor: FakeScriptProcessorNode | null = null; + + readonly sampleRate = 10; + readonly destination = {} as AudioDestinationNode; + state: AudioContextState = "running"; + + createMediaStreamSource(): MediaStreamAudioSourceNode { + return new FakeSourceNode() as unknown as MediaStreamAudioSourceNode; + } + + createScriptProcessor(): ScriptProcessorNode { + const processor = new FakeScriptProcessorNode(); + FakeAudioContext.lastProcessor = processor; + return processor as unknown as ScriptProcessorNode; + } + + createGain(): GainNode { + return new FakeGainNode() as unknown as GainNode; + } + + async close(): Promise { + this.state = "closed"; + } +} diff --git a/tests/wavWriter.test.ts b/tests/wavWriter.test.ts new file mode 100644 index 0000000..3447bb9 --- /dev/null +++ b/tests/wavWriter.test.ts @@ -0,0 +1,64 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { StreamingWavWriter, createWavHeader, writeWavFile } from "../src/infrastructure/adapters/wavWriter"; + +installDesktopRuntime(); + +test("createWavHeader writes a valid RIFF/WAVE header", () => { + const header = createWavHeader(1_600, 16_000, 1); + + assert.equal(header.length, 44); + assert.equal(header.toString("ascii", 0, 4), "RIFF"); + assert.equal(header.toString("ascii", 8, 12), "WAVE"); + assert.equal(header.toString("ascii", 12, 16), "fmt "); + assert.equal(header.toString("ascii", 36, 40), "data"); + assert.equal(header.readUInt32LE(4), 1_636); + assert.equal(header.readUInt32LE(24), 16_000); + assert.equal(header.readUInt16LE(22), 1); + assert.equal(header.readUInt32LE(40), 1_600); +}); + +test("writeWavFile writes PCM data after the header", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "resonance-wav-file-")); + const filePath = path.join(tmpDir, "segment.wav"); + writeWavFile(filePath, new Float32Array([0, 0.5, -0.5, 1]), 16_000, 1); + + const buffer = fs.readFileSync(filePath); + assert.equal(buffer.toString("ascii", 0, 4), "RIFF"); + assert.equal(buffer.readUInt32LE(40), 8); + assert.equal(buffer.length, 52); + + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +test("StreamingWavWriter patches the header with the final data size", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "resonance-wav-stream-")); + const filePath = path.join(tmpDir, "recording.wav"); + const writer = new StreamingWavWriter(filePath, 48_000, 1); + + writer.append(new Float32Array([0.25, -0.25])); + writer.append(new Float32Array([0.5, -0.5, 0])); + writer.finalize(); + + const buffer = fs.readFileSync(filePath); + assert.equal(buffer.toString("ascii", 0, 4), "RIFF"); + assert.equal(buffer.readUInt32LE(40), 10); + assert.equal(buffer.readUInt32LE(4), 46); + assert.equal(buffer.length, 54); + + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +function installDesktopRuntime(): void { + const desktopWindow = { + require: require, + }; + Object.defineProperty(globalThis, "window", { + value: desktopWindow, + configurable: true, + writable: true, + }); +} diff --git a/tests/webAudio.test.ts b/tests/webAudio.test.ts new file mode 100644 index 0000000..44f151b --- /dev/null +++ b/tests/webAudio.test.ts @@ -0,0 +1,75 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { getWebAudioCapability, getWebShareAudioCapability, resolveWebAudioInputById } from "../src/infrastructure/system/webAudio"; + +test("getWebAudioCapability and getWebShareAudioCapability report supported browser APIs", () => { + Object.defineProperty(globalThis, "navigator", { + value: { + mediaDevices: { + getUserMedia: async () => ({}) as MediaStream, + enumerateDevices: async () => [] as MediaDeviceInfo[], + getDisplayMedia: async () => ({}) as MediaStream, + }, + }, + configurable: true, + writable: true, + }); + + assert.deepEqual(getWebAudioCapability(), { + supported: true, + hasGetUserMedia: true, + hasEnumerateDevices: true, + }); + assert.deepEqual(getWebShareAudioCapability(), { + supported: true, + hasGetDisplayMedia: true, + }); +}); + +test("getWebShareAudioCapability reports unsupported when display sharing is unavailable", () => { + Object.defineProperty(globalThis, "navigator", { + value: { + mediaDevices: { + getUserMedia: async () => ({}) as MediaStream, + enumerateDevices: async () => [] as MediaDeviceInfo[], + }, + }, + configurable: true, + writable: true, + }); + + assert.deepEqual(getWebShareAudioCapability(), { + supported: true, + hasGetDisplayMedia: false, + }); +}); + +test("resolveWebAudioInputById returns a matching audioinput device", async () => { + Object.defineProperty(globalThis, "navigator", { + value: { + mediaDevices: { + getUserMedia: async () => ({}) as MediaStream, + enumerateDevices: async () => + [ + { + kind: "audioinput", + deviceId: "default", + label: "System default input", + groupId: "default-group", + }, + { + kind: "audioinput", + deviceId: "loopback-1", + label: "BlackHole 2ch", + groupId: "group-2", + }, + ] as MediaDeviceInfo[], + }, + }, + configurable: true, + writable: true, + }); + + const device = await resolveWebAudioInputById("loopback-1"); + assert.equal(device?.label, "BlackHole 2ch"); +}); diff --git a/tests/webCapture.test.ts b/tests/webCapture.test.ts new file mode 100644 index 0000000..8c7dae5 --- /dev/null +++ b/tests/webCapture.test.ts @@ -0,0 +1,248 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { WebCaptureAdapter } from "../src/infrastructure/adapters/WebCaptureAdapter"; + +installDesktopRuntime(); + +let requestedConstraints: MediaStreamConstraints[] = []; + +test("WebCaptureAdapter writes wav segments and final recording without polling", async () => { + installFakeWebAudioEnvironment(); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "resonance-web-capture-")); + const segmentsDir = path.join(tmpDir, "segments"); + fs.mkdirSync(segmentsDir, { recursive: true }); + + const ready: Array<{ index: number; path: string }> = []; + const adapter = new WebCaptureAdapter(); + await adapter.start({ + fullAudioPath: path.join(tmpDir, "recording.wav"), + segmentsDir, + segmentDurationSeconds: 1, + microphoneDevice: "mic-1", + onSegmentReady: (segment) => ready.push(segment), + }); + + const processor = FakeAudioContext.lastProcessor; + assert.ok(processor, "expected the fake script processor to be created"); + + processor.emit([new Float32Array([0, 0.1, 0.2, 0.3, 0.4, 0.5])]); + processor.emit([new Float32Array([0.6, 0.7, 0.8, 0.9, 1])]); + + await adapter.stop(); + + assert.deepEqual( + ready.map((segment) => segment.index), + [0, 1] + ); + assert.equal(fs.existsSync(path.join(tmpDir, "recording.wav")), true); + assert.equal(fs.existsSync(path.join(segmentsDir, "segment-0000.wav")), true); + assert.equal(fs.existsSync(path.join(segmentsDir, "segment-0001.wav")), true); + assert.equal(fs.readFileSync(path.join(segmentsDir, "segment-0000.wav")).readUInt32LE(40), 20); + assert.equal(fs.readFileSync(path.join(segmentsDir, "segment-0001.wav")).readUInt32LE(40), 2); + assert.equal(fs.readFileSync(path.join(tmpDir, "recording.wav")).readUInt32LE(40), 22); + + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +test("WebCaptureAdapter falls back to the default microphone when the saved device is missing", async () => { + installFakeWebAudioEnvironment(); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "resonance-web-fallback-")); + const segmentsDir = path.join(tmpDir, "segments"); + fs.mkdirSync(segmentsDir, { recursive: true }); + const logs: string[] = []; + + const adapter = new WebCaptureAdapter(); + await adapter.start({ + fullAudioPath: path.join(tmpDir, "recording.wav"), + segmentsDir, + segmentDurationSeconds: 1, + microphoneDevice: "missing-device", + onSegmentReady: () => {}, + onLog: (line) => logs.push(line), + }); + + const lastConstraints = requestedConstraints[requestedConstraints.length - 1] ?? null; + const requestedAudio = + lastConstraints && lastConstraints.audio && lastConstraints.audio !== true ? lastConstraints.audio : undefined; + assert.equal(typeof requestedAudio?.deviceId, "undefined"); + assert.ok(logs.some((line) => line.includes("fallback")), "expected a fallback log when the saved device is missing"); + + await adapter.stop(); + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +test("WebCaptureAdapter opens a second Web Audio input when an additional source is configured", async () => { + installFakeWebAudioEnvironment(); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "resonance-web-dual-input-")); + const segmentsDir = path.join(tmpDir, "segments"); + fs.mkdirSync(segmentsDir, { recursive: true }); + + const adapter = new WebCaptureAdapter(); + await adapter.start({ + fullAudioPath: path.join(tmpDir, "recording.wav"), + segmentsDir, + segmentDurationSeconds: 1, + microphoneDevice: "mic-1", + additionalSources: [{ deviceId: "loopback-1", label: "BlackHole 2ch" }], + onSegmentReady: () => {}, + }); + + assert.equal(requestedConstraints.length, 2); + const micConstraints = requestedConstraints[0]?.audio; + const systemConstraints = requestedConstraints[1]?.audio; + assert.deepEqual(micConstraints, { deviceId: { exact: "mic-1" } }); + assert.deepEqual(systemConstraints, { deviceId: { exact: "loopback-1" } }); + + await adapter.stop(); + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +test("WebCaptureAdapter fails clearly when the additional source is missing", async () => { + installFakeWebAudioEnvironment(); + const adapter = new WebCaptureAdapter(); + await assert.rejects( + () => + adapter.start({ + fullAudioPath: "/tmp/ignored.wav", + segmentsDir: "/tmp", + segmentDurationSeconds: 1, + additionalSources: [{ deviceId: "missing-loopback", label: "BlackHole 2ch" }], + onSegmentReady: () => {}, + }), + /additional audio source/i + ); +}); + +function installDesktopRuntime(): void { + Object.defineProperty(globalThis, "window", { + value: { require }, + configurable: true, + writable: true, + }); +} + +function installFakeWebAudioEnvironment(): void { + requestedConstraints = []; + + Object.defineProperty(globalThis, "navigator", { + value: { + mediaDevices: { + getUserMedia: async (constraints: MediaStreamConstraints) => { + requestedConstraints.push(constraints); + return new FakeMediaStream(); + }, + enumerateDevices: async () => + [ + { + kind: "audioinput", + deviceId: "default", + label: "System default microphone", + groupId: "default-group", + }, + { + kind: "audioinput", + deviceId: "mic-1", + label: "USB Microphone", + groupId: "group-1", + }, + { + kind: "audioinput", + deviceId: "loopback-1", + label: "BlackHole 2ch", + groupId: "group-2", + }, + ] as MediaDeviceInfo[], + }, + permissions: { + query: async () => ({ state: "prompt" }), + }, + }, + configurable: true, + writable: true, + }); + + Object.defineProperty(globalThis, "AudioContext", { + value: FakeAudioContext, + configurable: true, + writable: true, + }); +} + +class FakeMediaStream { + private readonly tracks = [new FakeMediaStreamTrack()]; + + getTracks(): MediaStreamTrack[] { + return this.tracks as unknown as MediaStreamTrack[]; + } +} + +class FakeMediaStreamTrack { + stop(): void {} +} + +class FakeSourceNode { + connect(): void {} + disconnect(): void {} +} + +class FakeGainNode { + gain = { value: 1 }; + + connect(): void {} + disconnect(): void {} +} + +class FakeScriptProcessorNode { + onaudioprocess: ((event: AudioProcessingEvent) => void) | null = null; + + connect(): void {} + disconnect(): void {} + + emit(channels: Float32Array[]): void { + const inputBuffer = new FakeAudioBuffer(channels); + this.onaudioprocess?.({ inputBuffer } as unknown as AudioProcessingEvent); + } +} + +class FakeAudioBuffer { + readonly numberOfChannels: number; + readonly length: number; + + constructor(private readonly channels: Float32Array[]) { + this.numberOfChannels = channels.length; + this.length = channels[0]?.length ?? 0; + } + + getChannelData(channelIndex: number): Float32Array { + return this.channels[channelIndex] ?? new Float32Array(this.length); + } +} + +class FakeAudioContext { + static lastProcessor: FakeScriptProcessorNode | null = null; + + readonly sampleRate = 10; + readonly destination = {} as AudioDestinationNode; + state: AudioContextState = "running"; + + createMediaStreamSource(): MediaStreamAudioSourceNode { + return new FakeSourceNode() as unknown as MediaStreamAudioSourceNode; + } + + createScriptProcessor(): ScriptProcessorNode { + const processor = new FakeScriptProcessorNode(); + FakeAudioContext.lastProcessor = processor; + return processor as unknown as ScriptProcessorNode; + } + + createGain(): GainNode { + return new FakeGainNode() as unknown as GainNode; + } + + async close(): Promise { + this.state = "closed"; + } +}