From de018c18e83134968ce2d049d92dae6668e85c76 Mon Sep 17 00:00:00 2001 From: Charles Kelsoe Date: Sun, 14 Jun 2026 16:59:44 -0400 Subject: [PATCH] Feature: per-feed media (audio/video) download Download podcast/audio/video enclosures into a vault subfolder (vault.createBinary) or to an outside-vault folder (Node fs, desktop-only, behind a Platform.isDesktop guard) for media you do not want syncing. Global defaults plus per-feed overrides (download on/off, location, subfolder). Notes record a media-file frontmatter line and point the body link at the local file. Failures are best-effort (logged, never abort the item). +20 tests. --- __tests__/import-runner.test.ts | 86 ++++++ __tests__/media-downloader.test.ts | 353 +++++++++++++++++++++++ __tests__/note-writer.test.ts | 36 +++ import-runner.ts | 30 ++ main.ts | 42 +++ media-downloader.ts | 431 +++++++++++++++++++++++++++++ note-writer.ts | 26 +- settings-tab.ts | 75 ++++- settings.ts | 44 +++ 9 files changed, 1119 insertions(+), 4 deletions(-) create mode 100644 __tests__/media-downloader.test.ts create mode 100644 media-downloader.ts diff --git a/__tests__/import-runner.test.ts b/__tests__/import-runner.test.ts index 95753c3..3641640 100644 --- a/__tests__/import-runner.test.ts +++ b/__tests__/import-runner.test.ts @@ -288,6 +288,92 @@ describe("ImportRunner.run", () => { expect(calls[0]?.body).toBe("MD:

A

"); }); + it("passes the downloadMedia result to the writer as mediaFile", async () => { + const items = [makeItem({ id: "a", mediaUrl: "https://m.test/a.mp3" })]; + const { source } = makeSource({ a: "

A

" }); + const composeOpts: Array | undefined> = []; + const stub = { + writeNote( + _item: FeedItem, + _body: string, + opts?: Record, + ): Promise { + composeOpts.push(opts); + return Promise.resolve({ status: "created", path: "Feeds/a.md" }); + }, + }; + const writer = stub as unknown as NoteWriter; + const downloadMedia = (): Promise => + Promise.resolve("Feeds/media/a.mp3"); + + const runner = new ImportRunner({ + source, + noteWriter: writer, + convert: identityConvert, + downloadMedia, + }); + await runner.run(items, {}); + + expect(composeOpts).toHaveLength(1); + expect(composeOpts[0]?.mediaFile).toBe("Feeds/media/a.mp3"); + }); + + it("writes the note without mediaFile when downloadMedia fails (best effort)", async () => { + const items = [makeItem({ id: "a", mediaUrl: "https://m.test/a.mp3" })]; + const { source } = makeSource({ a: "

A

" }); + const composeOpts: Array | undefined> = []; + const stub = { + writeNote( + _item: FeedItem, + _body: string, + opts?: Record, + ): Promise { + composeOpts.push(opts); + return Promise.resolve({ status: "created", path: "Feeds/a.md" }); + }, + }; + const writer = stub as unknown as NoteWriter; + const downloadMedia = (): Promise => + Promise.reject(new Error("media host down")); + + const runner = new ImportRunner({ + source, + noteWriter: writer, + convert: identityConvert, + downloadMedia, + }); + const tally = await runner.run(items, {}); + + // Item still succeeds; mediaFile is undefined so the remote link is used. + expect(tally.created).toBe(1); + expect(tally.failed).toBe(0); + expect(composeOpts[0]?.mediaFile).toBeUndefined(); + expect(errorSpy).toHaveBeenCalled(); + }); + + it("skips downloadMedia when the item has no media url", async () => { + const items = [makeItem({ id: "a", mediaUrl: null })]; + const { source } = makeSource({ a: "

A

" }); + const { writer } = makeWriter(() => + Promise.resolve({ status: "created", path: "Feeds/a.md" }), + ); + let called = false; + const downloadMedia = (): Promise => { + called = true; + return Promise.resolve(null); + }; + + const runner = new ImportRunner({ + source, + noteWriter: writer, + convert: identityConvert, + downloadMedia, + }); + await runner.run(items, {}); + + expect(called).toBe(false); + }); + it("uses the processImages result when it succeeds", async () => { const items = [makeItem({ id: "a" })]; const { source } = makeSource({ a: "

A

" }); diff --git a/__tests__/media-downloader.test.ts b/__tests__/media-downloader.test.ts new file mode 100644 index 0000000..c96b121 --- /dev/null +++ b/__tests__/media-downloader.test.ts @@ -0,0 +1,353 @@ +import type { FeedItem, HttpFetcher, HttpRequest, HttpResponse } from "../feed-source"; +import { + MediaDownloader, + type BinaryFileLike, + type VaultBinaryLike, +} from "../media-downloader"; + +// ----------------------------------------------------------------------------- +// Fixtures +// ----------------------------------------------------------------------------- + +/** Turn an ASCII string into an ArrayBuffer for a fake response body. */ +function bytes(text: string): ArrayBuffer { + const arr = new Uint8Array(text.length); + for (let i = 0; i < text.length; i++) { + arr[i] = text.charCodeAt(i); + } + return arr.buffer; +} + +function makeItem(overrides: Partial = {}): FeedItem { + return { + sourceId: "feed-1", + id: "item-1", + url: "https://example.com/p/one", + title: "Episode One", + author: "Author", + publishedAt: "2026-01-02T00:00:00.000Z", + kind: "podcast", + contentHtml: null, + isTruncated: false, + audience: "free", + tags: [], + section: null, + mediaUrl: "https://media.example.com/ep/one.mp3", + mediaType: "audio/mpeg", + mediaBytes: null, + ...overrides, + }; +} + +interface FetchEntry { + status?: number; + headers?: Record; + body?: ArrayBuffer; + throws?: Error; +} + +/** Fetcher stub keyed by url; records the urls it was asked for. */ +function makeFetcher(map: Record): { + fetcher: HttpFetcher; + requested: string[]; +} { + const requested: string[] = []; + const fetcher: HttpFetcher = (req: HttpRequest) => { + requested.push(req.url); + const entry = map[req.url]; + if (entry === undefined) { + return Promise.reject(new Error(`no stub for ${req.url}`)); + } + if (entry.throws !== undefined) { + return Promise.reject(entry.throws); + } + const response: HttpResponse = { + status: entry.status ?? 200, + headers: entry.headers ?? {}, + json: null, + text: "", + arrayBuffer: entry.body ?? bytes("MEDIA"), + }; + return Promise.resolve(response); + }; + return { fetcher, requested }; +} + +/** In-memory vault that records createBinary / createFolder calls. */ +class FakeVault implements VaultBinaryLike { + files = new Set(); + folders = new Set(); + binaries: Array<{ path: string; size: number }> = []; + createdFolders: string[] = []; + + getFileByPath(path: string): BinaryFileLike | null { + return this.files.has(path) ? { path } : null; + } + getFolderByPath(path: string): BinaryFileLike | null { + return this.folders.has(path) ? { path } : null; + } + createFolder(path: string): Promise { + this.folders.add(path); + this.createdFolders.push(path); + return Promise.resolve({ path }); + } + createBinary(path: string, data: ArrayBuffer): Promise { + this.files.add(path); + this.binaries.push({ path, size: data.byteLength }); + return Promise.resolve({ path }); + } +} + +// ----------------------------------------------------------------------------- +// Tests +// ----------------------------------------------------------------------------- + +describe("MediaDownloader.downloadToVault", () => { + let errorSpy: jest.SpyInstance; + beforeEach(() => { + errorSpy = jest.spyOn(console, "error").mockImplementation(() => {}); + }); + afterEach(() => { + errorSpy.mockRestore(); + }); + + it("writes the media binary at the folder/.<ext> path and returns it", async () => { + const url = "https://media.example.com/ep/one.mp3"; + const { fetcher, requested } = makeFetcher({ [url]: { body: bytes("AUDIO") } }); + const vault = new FakeVault(); + const dl = new MediaDownloader({ fetcher, vault }); + + const item = makeItem({ title: "Episode One", mediaUrl: url, mediaType: "audio/mpeg" }); + const out = await dl.downloadToVault(item, "Feeds/Podcast/media"); + + expect(requested).toEqual([url]); + expect(vault.binaries).toHaveLength(1); + expect(vault.binaries[0]?.path).toBe("Feeds/Podcast/media/Episode-One.mp3"); + expect(vault.binaries[0]?.size).toBe("AUDIO".length); + expect(out).toBe("Feeds/Podcast/media/Episode-One.mp3"); + // Ancestors created. + expect(vault.createdFolders).toEqual(["Feeds", "Feeds/Podcast", "Feeds/Podcast/media"]); + }); + + it("derives the extension from the URL path when present", async () => { + const url = "https://media.example.com/ep/one.m4a"; + const { fetcher } = makeFetcher({ [url]: { body: bytes("A") } }); + const vault = new FakeVault(); + const dl = new MediaDownloader({ fetcher, vault }); + + const item = makeItem({ title: "Show", mediaUrl: url, mediaType: null }); + await dl.downloadToVault(item, "media"); + + expect(vault.binaries[0]?.path).toBe("media/Show.m4a"); + }); + + it("derives the extension from the content-type when the URL has none", async () => { + const url = "https://media.example.com/stream/123"; + const { fetcher } = makeFetcher({ + [url]: { headers: { "Content-Type": "audio/mp4" }, body: bytes("A") }, + }); + const vault = new FakeVault(); + const dl = new MediaDownloader({ fetcher, vault }); + + const item = makeItem({ title: "Show", mediaUrl: url, mediaType: "audio/mp4" }); + await dl.downloadToVault(item, "media"); + + expect(vault.binaries[0]?.path).toBe("media/Show.m4a"); + }); + + it("falls back to a bin extension when neither URL nor content-type gives one", async () => { + const url = "https://media.example.com/stream/123"; + const { fetcher } = makeFetcher({ [url]: { body: bytes("A") } }); + const vault = new FakeVault(); + const dl = new MediaDownloader({ fetcher, vault }); + + const item = makeItem({ title: "Show", mediaUrl: url, mediaType: null }); + await dl.downloadToVault(item, "media"); + + expect(vault.binaries[0]?.path).toBe("media/Show.bin"); + }); + + it("dedupes a collision by appending a numeric suffix before the extension", async () => { + const url = "https://media.example.com/ep/one.mp3"; + const { fetcher } = makeFetcher({ [url]: { body: bytes("A") } }); + const vault = new FakeVault(); + // A file already exists at the natural target path. + vault.files.add("media/Show.mp3"); + const dl = new MediaDownloader({ fetcher, vault }); + + const item = makeItem({ title: "Show", mediaUrl: url, mediaType: "audio/mpeg" }); + const out = await dl.downloadToVault(item, "media"); + + expect(out).toBe("media/Show-1.mp3"); + expect(vault.binaries[0]?.path).toBe("media/Show-1.mp3"); + }); + + it("returns null on a non-2xx status and writes nothing", async () => { + const url = "https://media.example.com/ep/missing.mp3"; + const { fetcher } = makeFetcher({ [url]: { status: 404, body: bytes("nope") } }); + const vault = new FakeVault(); + const dl = new MediaDownloader({ fetcher, vault }); + + const item = makeItem({ mediaUrl: url }); + const out = await dl.downloadToVault(item, "media"); + + expect(out).toBeNull(); + expect(vault.binaries).toHaveLength(0); + expect(errorSpy).toHaveBeenCalled(); + }); + + it("returns null on an empty response body and writes nothing", async () => { + const url = "https://media.example.com/ep/empty.mp3"; + const { fetcher } = makeFetcher({ [url]: { body: new ArrayBuffer(0) } }); + const vault = new FakeVault(); + const dl = new MediaDownloader({ fetcher, vault }); + + const item = makeItem({ mediaUrl: url }); + const out = await dl.downloadToVault(item, "media"); + + expect(out).toBeNull(); + expect(vault.binaries).toHaveLength(0); + }); + + it("returns null on a transport error without throwing", async () => { + const url = "https://media.example.com/ep/boom.mp3"; + const { fetcher } = makeFetcher({ [url]: { throws: new Error("connection reset") } }); + const vault = new FakeVault(); + const dl = new MediaDownloader({ fetcher, vault }); + + const item = makeItem({ mediaUrl: url }); + const out = await dl.downloadToVault(item, "media"); + + expect(out).toBeNull(); + expect(errorSpy).toHaveBeenCalled(); + }); + + it("returns null without fetching when the item has no media url", async () => { + const { fetcher, requested } = makeFetcher({}); + const vault = new FakeVault(); + const dl = new MediaDownloader({ fetcher, vault }); + + const out = await dl.downloadToVault(makeItem({ mediaUrl: null }), "media"); + + expect(out).toBeNull(); + expect(requested).toHaveLength(0); + expect(vault.binaries).toHaveLength(0); + }); + + it("returns null without fetching when the item has an empty media url", async () => { + const { fetcher, requested } = makeFetcher({}); + const vault = new FakeVault(); + const dl = new MediaDownloader({ fetcher, vault }); + + const out = await dl.downloadToVault(makeItem({ mediaUrl: "" }), "media"); + + expect(out).toBeNull(); + expect(requested).toHaveLength(0); + }); +}); + +describe("MediaDownloader.downloadToOutside", () => { + let errorSpy: jest.SpyInstance; + beforeEach(() => { + errorSpy = jest.spyOn(console, "error").mockImplementation(() => {}); + }); + afterEach(() => { + errorSpy.mockRestore(); + }); + + it("writes via the injected fileWriter and returns the absolute path", async () => { + const url = "https://media.example.com/ep/one.mp3"; + const { fetcher } = makeFetcher({ [url]: { body: bytes("AUDIO") } }); + const vault = new FakeVault(); + const written: Array<{ path: string; size: number }> = []; + const fileWriter = (absPath: string, data: ArrayBuffer): void => { + written.push({ path: absPath, size: data.byteLength }); + }; + const dl = new MediaDownloader({ fetcher, vault, fileWriter }); + + const item = makeItem({ title: "Episode One", mediaUrl: url, mediaType: "audio/mpeg" }); + const out = await dl.downloadToOutside(item, "/Users/me/Podcasts"); + + expect(out).toBe("/Users/me/Podcasts/Episode-One.mp3"); + expect(written).toHaveLength(1); + expect(written[0]?.path).toBe("/Users/me/Podcasts/Episode-One.mp3"); + expect(written[0]?.size).toBe("AUDIO".length); + // The vault is never touched for an outside write. + expect(vault.binaries).toHaveLength(0); + }); + + it("returns null when no outside folder is configured", async () => { + const url = "https://media.example.com/ep/one.mp3"; + const { fetcher, requested } = makeFetcher({ [url]: { body: bytes("A") } }); + const vault = new FakeVault(); + const fileWriter = (): void => {}; + const dl = new MediaDownloader({ fetcher, vault, fileWriter }); + + const out = await dl.downloadToOutside(makeItem({ mediaUrl: url }), ""); + + expect(out).toBeNull(); + expect(requested).toHaveLength(0); + expect(errorSpy).toHaveBeenCalled(); + }); + + it("returns null without fetching when the item has no media url", async () => { + const { fetcher, requested } = makeFetcher({}); + const vault = new FakeVault(); + const fileWriter = (): void => {}; + const dl = new MediaDownloader({ fetcher, vault, fileWriter }); + + const out = await dl.downloadToOutside(makeItem({ mediaUrl: null }), "/Users/me/Podcasts"); + + expect(out).toBeNull(); + expect(requested).toHaveLength(0); + }); +}); + +describe("MediaDownloader.download", () => { + let errorSpy: jest.SpyInstance; + beforeEach(() => { + errorSpy = jest.spyOn(console, "error").mockImplementation(() => {}); + }); + afterEach(() => { + errorSpy.mockRestore(); + }); + + it("dispatches to the vault path for location 'vault'", async () => { + const url = "https://media.example.com/ep/one.mp3"; + const { fetcher } = makeFetcher({ [url]: { body: bytes("A") } }); + const vault = new FakeVault(); + const dl = new MediaDownloader({ fetcher, vault }); + + const item = makeItem({ title: "Show", mediaUrl: url, mediaType: "audio/mpeg" }); + const out = await dl.download(item, { + location: "vault", + vaultFolder: "Feeds/media", + outsideFolder: "/abs", + }); + + expect(out).toBe("Feeds/media/Show.mp3"); + expect(vault.binaries).toHaveLength(1); + }); + + it("dispatches to the outside path for location 'outside'", async () => { + const url = "https://media.example.com/ep/one.mp3"; + const { fetcher } = makeFetcher({ [url]: { body: bytes("A") } }); + const vault = new FakeVault(); + const written: string[] = []; + const fileWriter = (absPath: string): void => { + written.push(absPath); + }; + const dl = new MediaDownloader({ fetcher, vault, fileWriter }); + + const item = makeItem({ title: "Show", mediaUrl: url, mediaType: "audio/mpeg" }); + const out = await dl.download(item, { + location: "outside", + vaultFolder: "Feeds/media", + outsideFolder: "/abs/podcasts", + }); + + expect(out).toBe("/abs/podcasts/Show.mp3"); + expect(written).toEqual(["/abs/podcasts/Show.mp3"]); + expect(vault.binaries).toHaveLength(0); + }); +}); diff --git a/__tests__/note-writer.test.ts b/__tests__/note-writer.test.ts index eae10af..9317be7 100644 --- a/__tests__/note-writer.test.ts +++ b/__tests__/note-writer.test.ts @@ -276,6 +276,42 @@ describe("composeNote", () => { expect(note).not.toContain("[Episode audio]"); expect(note).not.toContain("[Media]"); }); + + it("records media-file and links the local file when mediaFile is set", () => { + const mediaUrl = "https://media.example.com/ep/42.mp3"; + const localPath = "Feeds/Podcast/media/Episode-42.mp3"; + const item = makeItem({ + kind: "podcast", + mediaUrl, + mediaType: "audio/mpeg", + }); + const note = composeNote(item, "BODY", { mediaFile: localPath }); + // Remote url is kept in frontmatter for reference. + expect(note).toContain(`media-url: "${mediaUrl}"`); + // Local file path recorded (force-quoted) after media-url. + expect(note).toContain(`media-file: "${localPath}"`); + const urlIndex = note.indexOf("media-url:"); + const fileIndex = note.indexOf("media-file:"); + expect(urlIndex).toBeGreaterThanOrEqual(0); + expect(fileIndex).toBeGreaterThan(urlIndex); + // Body links the LOCAL file, not the remote url. + expect(note).toContain(`[Episode audio](${localPath})`); + expect(note).not.toContain(`[Episode audio](${mediaUrl})`); + }); + + it("labels a non-podcast local media link as Media", () => { + const mediaUrl = "https://media.example.com/clip.mp4"; + const localPath = "Feeds/media/clip.mp4"; + const item = makeItem({ + kind: "article", + mediaUrl, + mediaType: "video/mp4", + }); + const note = composeNote(item, "BODY", { mediaFile: localPath }); + expect(note).toContain(`media-file: "${localPath}"`); + expect(note).toContain(`[Media](${localPath})`); + expect(note).not.toContain(`[Media](${mediaUrl})`); + }); }); // ----------------------------------------------------------------------------- diff --git a/import-runner.ts b/import-runner.ts index 58f387c..727380f 100644 --- a/import-runner.ts +++ b/import-runner.ts @@ -65,6 +65,13 @@ export interface ImportRunnerDeps { convert: (html: string) => string; /** Optional best-effort image download + markdown rewrite. */ processImages?: (markdown: string, item: FeedItem) => Promise<string>; + /** + * Optional best-effort media (podcast audio/video enclosure) download. Returns + * the local path to thread into the note as composeOptions.mediaFile, or null + * when there is no media or the download failed. A failure must NOT fail the + * item: the runner catches, logs, and writes the note without a local file. + */ + downloadMedia?: (item: FeedItem) => Promise<string | null>; /** Optional debug logger; defaults to a no-op when absent. */ debugLogger?: DebugLogger; } @@ -96,6 +103,7 @@ export class ImportRunner { markdown: string, item: FeedItem, ) => Promise<string>; + private readonly downloadMedia?: (item: FeedItem) => Promise<string | null>; private readonly debug?: DebugLogger; constructor(deps: ImportRunnerDeps) { @@ -103,6 +111,7 @@ export class ImportRunner { this.noteWriter = deps.noteWriter; this.convert = deps.convert; this.processImages = deps.processImages; + this.downloadMedia = deps.downloadMedia; this.debug = deps.debugLogger; } @@ -201,8 +210,29 @@ export class ImportRunner { }); } + // Best-effort media download. A failure must NOT fail the item: catch, + // log, and write the note without a local file (the remote media-url + // stays in frontmatter and as the body link). + let mediaFile: string | undefined; + if (this.downloadMedia !== undefined && full.mediaUrl !== null && full.mediaUrl.length > 0) { + try { + const local = await this.downloadMedia(full); + if (local !== null) { + mediaFile = local; + } + } catch (err) { + console.error(err); + this.debug?.log({ + kind: "error", + message: `Media download failed for "${full.title}", keeping the remote link`, + payload: describeError(err), + }); + } + } + const outcome = await this.noteWriter.writeNote(full, markdown, { feedTags, + mediaFile, }); this.debug?.log({ kind: "note", diff --git a/main.ts b/main.ts index 66ff2c6..917ac1f 100644 --- a/main.ts +++ b/main.ts @@ -8,8 +8,12 @@ import { import type { FeedSource, HttpFetcher, HttpRequest, HttpResponse, SourceType } from "./feed-source"; import { DEFAULT_SETTINGS, + effectiveDownloadMedia, effectiveImageSubfolder, effectiveImagesMode, + effectiveMediaLocation, + effectiveMediaOutsideFolder, + effectiveMediaSubfolder, effectiveNoteNameTemplate, type FeedConfig, type RssImporterSettings, @@ -24,6 +28,7 @@ import { } from "./note-writer"; import { convertHtmlToMarkdown } from "./html-converter"; import { ImageDownloader } from "./image-downloader"; +import { MediaDownloader } from "./media-downloader"; import { DismissStore } from "./dismiss-store"; import { BufferedDebugLogger } from "./debug-logger"; import { ImportRunner } from "./import-runner"; @@ -91,6 +96,10 @@ function isDuplicatePolicy(value: unknown): value is import("./settings").Duplic return value === "skip" || value === "overwrite" || value === "prompt"; } +function isMediaLocation(value: unknown): value is import("./settings").MediaLocation { + return value === "vault" || value === "outside"; +} + // Sync classification used only to pick which source resolves a freshly entered // input for the add-feed preview. The resolved feed's own sourceType (captured // into the FeedConfig) is what drives import, so a custom-domain Substack that @@ -194,6 +203,9 @@ export default class RssImporterPlugin extends Plugin implements RssImporterPlug if (!isDuplicatePolicy(this.settings.duplicatePolicy)) { this.settings.duplicatePolicy = DEFAULT_SETTINGS.duplicatePolicy; } + if (!isMediaLocation(this.settings.mediaLocation)) { + this.settings.mediaLocation = DEFAULT_SETTINGS.mediaLocation; + } for (const feed of this.settings.feeds) { // The per-feed override fields are optional literal/string unions on // FeedConfig, but data.json is user-editable so a stored value can be a @@ -209,6 +221,18 @@ export default class RssImporterPlugin extends Plugin implements RssImporterPlug if ("imageSubfolder" in record && typeof record["imageSubfolder"] !== "string") { delete record["imageSubfolder"]; } + if ("downloadMedia" in record && typeof record["downloadMedia"] !== "boolean") { + delete record["downloadMedia"]; + } + if ("mediaLocation" in record && !isMediaLocation(record["mediaLocation"])) { + delete record["mediaLocation"]; + } + if ("mediaSubfolder" in record && typeof record["mediaSubfolder"] !== "string") { + delete record["mediaSubfolder"]; + } + if ("mediaOutsideFolder" in record && typeof record["mediaOutsideFolder"] !== "string") { + delete record["mediaOutsideFolder"]; + } } } @@ -297,11 +321,29 @@ export default class RssImporterPlugin extends Plugin implements RssImporterPlug processImages = (markdown) => downloader.downloadAndRewrite(markdown, folderPath); } + let downloadMedia: ((item: import("./feed-source").FeedItem) => Promise<string | null>) | undefined; + if (effectiveDownloadMedia(feed, this.settings)) { + const mediaDownloader = new MediaDownloader({ + fetcher: this.makeFetcher(), + vault: this.app.vault, + }); + const location = effectiveMediaLocation(feed, this.settings); + const mediaSubfolder = effectiveMediaSubfolder(feed, this.settings); + const vaultFolder = + feed.destinationFolder === "" + ? mediaSubfolder + : `${feed.destinationFolder}/${mediaSubfolder}`; + const outsideFolder = effectiveMediaOutsideFolder(feed, this.settings); + downloadMedia = (item) => + mediaDownloader.download(item, { location, vaultFolder, outsideFolder }); + } + runner = new ImportRunner({ source, noteWriter, convert: convertHtmlToMarkdown, processImages, + downloadMedia, debugLogger: this.debugLogger, }); } catch (err) { diff --git a/media-downloader.ts b/media-downloader.ts new file mode 100644 index 0000000..7a695ed --- /dev/null +++ b/media-downloader.ts @@ -0,0 +1,431 @@ +// Downloads a feed item's media enclosure (podcast audio, attached audio/video) +// either into the vault or to an absolute filesystem path outside the vault. +// +// This mirrors image-downloader's robustness contract: best effort, never throw +// out of the import. A media failure (transport error, non-2xx, empty body, +// write error, non-desktop for the outside path) logs a one-line reason and +// returns null so the note is still written with the remote media-url intact. +// +// The vault path reuses image-downloader's VaultBinaryLike surface so Obsidian's +// Vault satisfies it structurally and tests inject a plain object. The outside +// path uses Node fs/path, which on a marketplace plugin must be require()'d +// behind a Platform.isDesktop guard (see CLAUDE.md). To keep the module unit +// testable without touching the real filesystem, the constructor accepts an +// optional fileWriter the outside path prefers over the Node fallback. + +import { Platform } from "obsidian"; +import type { FeedItem, HttpFetcher } from "./feed-source"; +import type { BinaryFileLike, VaultBinaryLike } from "./image-downloader"; + +export type { BinaryFileLike, VaultBinaryLike }; + +/** Where the media for one item should be written. */ +export type MediaLocation = "vault" | "outside"; + +/** + * Writes raw bytes to an absolute filesystem path. Injected for tests so the + * outside-vault path can be exercised without touching the real disk; when + * absent, downloadToOutside falls back to Node fs behind a desktop guard. + */ +export type FileWriter = (absPath: string, data: ArrayBuffer) => void; + +export interface MediaDownloaderOptions { + readonly fetcher: HttpFetcher; + readonly vault: VaultBinaryLike; + readonly fileWriter?: FileWriter; +} + +export interface MediaDownloadOptions { + readonly location: MediaLocation; + /** Vault-relative folder for the "vault" location. */ + readonly vaultFolder: string; + /** Absolute filesystem folder for the "outside" location. */ + readonly outsideFolder: string; +} + +// Map common enclosure content-types to a file extension. Audio and video MIME +// types the URL path may not carry. Anything unmapped falls back to the URL +// extension, then to a generic default. +const CONTENT_TYPE_EXT: Record<string, string> = { + "audio/mpeg": "mp3", + "audio/mp3": "mp3", + "audio/mp4": "m4a", + "audio/x-m4a": "m4a", + "audio/aac": "aac", + "audio/ogg": "ogg", + "audio/opus": "opus", + "audio/wav": "wav", + "audio/x-wav": "wav", + "audio/flac": "flac", + "video/mp4": "mp4", + "video/webm": "webm", + "video/quicktime": "mov", + "video/x-matroska": "mkv", +}; + +// Extensions accepted from a URL path. Anything else (or none) falls back to the +// content-type, then to a generic default. +const KNOWN_MEDIA_EXTS = new Set([ + "mp3", "m4a", "aac", "ogg", "opus", "wav", "flac", + "mp4", "webm", "mov", "mkv", "m4v", "oga", +]); + +const DEFAULT_EXT = "bin"; + +export class MediaDownloader { + private readonly fetcher: HttpFetcher; + private readonly vault: VaultBinaryLike; + private readonly fileWriter?: FileWriter; + + constructor(opts: MediaDownloaderOptions) { + this.fetcher = opts.fetcher; + this.vault = opts.vault; + this.fileWriter = opts.fileWriter; + } + + /** + * Dispatch by location. Returns the written path (vault-relative or absolute) + * on success, or null when there is no media or the download/write failed. + */ + async download(item: FeedItem, opts: MediaDownloadOptions): Promise<string | null> { + if (opts.location === "outside") { + return this.downloadToOutside(item, opts.outsideFolder); + } + return this.downloadToVault(item, opts.vaultFolder); + } + + /** + * Fetch the item's media and write it under `folderPath` in the vault. Returns + * the vault-relative path on success. Returns null (never throws) when there + * is no media url, the fetch is non-2xx or empty, or any write step fails. + */ + async downloadToVault(item: FeedItem, folderPath: string): Promise<string | null> { + const mediaUrl = item.mediaUrl; + if (mediaUrl === null || mediaUrl.length === 0) { + return null; + } + + const fetched = await this.fetchMedia(mediaUrl); + if (fetched === null) { + return null; + } + + const folder = normalizeFolderPath(folderPath); + try { + await this.ensureFolder(folder); + } catch (err) { + console.error(`RSS Importer: media download skipped for ${mediaUrl}: ${describe(err)}`); + return null; + } + + const fileName = deriveFileName(item, mediaUrl, fetched.contentType); + const targetPath = folder === "" ? fileName : `${folder}/${fileName}`; + const uniquePath = await this.uniqueVaultPath(targetPath); + + try { + await this.vault.createBinary(uniquePath, fetched.bytes); + } catch (err) { + console.error(`RSS Importer: media write failed for ${mediaUrl}: ${describe(err)}`); + return null; + } + return uniquePath; + } + + /** + * Fetch the item's media and write it to an absolute path under `absFolder`. + * Desktop only. Uses the injected fileWriter when present (tests); otherwise + * Node fs/path behind a Platform.isDesktop guard. Returns the absolute path on + * success, or null (never throws) on non-desktop, no media, or any failure. + */ + async downloadToOutside(item: FeedItem, absFolder: string): Promise<string | null> { + const mediaUrl = item.mediaUrl; + if (mediaUrl === null || mediaUrl.length === 0) { + return null; + } + if (this.fileWriter === undefined && !Platform.isDesktop) { + console.error(`RSS Importer: media download skipped for ${mediaUrl}: outside-vault writes need a desktop app`); + return null; + } + if (absFolder.length === 0) { + console.error(`RSS Importer: media download skipped for ${mediaUrl}: no outside folder configured`); + return null; + } + + const fetched = await this.fetchMedia(mediaUrl); + if (fetched === null) { + return null; + } + + const fileName = deriveFileName(item, mediaUrl, fetched.contentType); + + try { + return this.writeOutside(absFolder, fileName, fetched.bytes); + } catch (err) { + console.error(`RSS Importer: media write failed for ${mediaUrl}: ${describe(err)}`); + return null; + } + } + + /** + * Write bytes to absFolder/fileName, preferring the injected fileWriter. When + * none is injected, use Node fs/path behind a desktop guard: require() rather + * than a top-level import keeps the bundle clean of Node built-ins on mobile, + * per the marketplace scorecard. Returns the absolute path written. + */ + private writeOutside(absFolder: string, fileName: string, bytes: ArrayBuffer): string { + if (this.fileWriter !== undefined) { + const joined = joinPosix(absFolder, fileName); + this.fileWriter(joined, bytes); + return joined; + } + const { fs, path } = getNodeFsPath(); + fs.mkdirSync(absFolder, { recursive: true }); + const target = path.join(absFolder, fileName); + fs.writeFileSync(target, Buffer.from(bytes)); + return target; + } + + /** + * GET the media url and validate the response. Returns the bytes and the + * content-type on success, or null (logged) on non-2xx, empty body, or a + * transport error. + */ + private async fetchMedia( + url: string, + ): Promise<{ bytes: ArrayBuffer; contentType: string } | null> { + let response; + try { + response = await this.fetcher({ url, method: "GET" }); + } catch (err) { + console.error(`RSS Importer: media download failed for ${url}: ${describe(err)}`); + return null; + } + if (response.status < 200 || response.status >= 300) { + console.error(`RSS Importer: media download skipped for ${url}: status ${response.status}`); + return null; + } + const bytes = response.arrayBuffer; + if (bytes.byteLength === 0) { + console.error(`RSS Importer: media download skipped for ${url}: empty body`); + return null; + } + return { bytes, contentType: headerValue(response.headers, "content-type") }; + } + + /** + * Create the folder and each missing ancestor. createFolder throws when the + * folder already exists, so each segment is checked first. + */ + private async ensureFolder(folder: string): Promise<void> { + if (folder === "") { + return; + } + const segments = folder.split("/"); + for (let i = 1; i <= segments.length; i++) { + const partial = segments.slice(0, i).join("/"); + if (this.vault.getFolderByPath(partial) === null) { + await this.vault.createFolder(partial); + } + } + } + + /** + * Resolve a non-colliding vault path. If the target already exists, append a + * numeric suffix before the extension (name-1.ext, name-2.ext, ...). + */ + private async uniqueVaultPath(targetPath: string): Promise<string> { + if (this.vault.getFileByPath(targetPath) === null) { + return targetPath; + } + const { dir, base, ext } = splitPath(targetPath); + for (let n = 1; ; n++) { + const candidate = dir === "" ? `${base}-${n}${ext}` : `${dir}/${base}-${n}${ext}`; + if (this.vault.getFileByPath(candidate) === null) { + return candidate; + } + } + } +} + +// ----------------------------------------------------------------------------- +// Pure helpers (kept module-private; exercised through the public methods). +// ----------------------------------------------------------------------------- + +/** + * Resolve Node's fs and path modules. Obsidian's guidelines require Node + * built-ins to be loaded via a Platform.isDesktop-guarded require() so the + * mobile bundle never references them; callers must gate on Platform.isDesktop + * before reaching here. The require, the unsafe cast, and the node-module import + * are the unavoidable consequences of that guidance. + */ +function getNodeFsPath(): { fs: typeof import("fs"); path: typeof import("path") } { + if (!Platform.isDesktop) { + throw new Error("Node fs/path modules are not available on this platform."); + } + // eslint-disable-next-line @typescript-eslint/no-require-imports -- Obsidian docs require a Platform.isDesktop-guarded require() for Node built-ins so mobile builds do not pull them in; the require is the unavoidable consequence of that guidance. + const fs = require("fs") as typeof import("fs"); + // eslint-disable-next-line @typescript-eslint/no-require-imports -- Obsidian docs require a Platform.isDesktop-guarded require() for Node built-ins so mobile builds do not pull them in; the require is the unavoidable consequence of that guidance. + const path = require("path") as typeof import("path"); + return { fs, path }; +} + +/** Read a header case-insensitively; returns "" when absent. */ +function headerValue(headers: Record<string, string>, name: string): string { + const wanted = name.toLowerCase(); + for (const key of Object.keys(headers)) { + if (key.toLowerCase() === wanted) { + return headers[key] ?? ""; + } + } + return ""; +} + +/** Render a caught unknown into a one-line message. */ +function describe(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +/** + * Derive the media filename: a sanitized item title (preferred) or the URL + * basename, plus an extension from the content-type, then the URL, then a + * generic default. + */ +function deriveFileName(item: FeedItem, url: string, contentType: string): string { + const ext = deriveExtension(url, contentType); + const base = deriveBaseName(item, url); + return `${base}.${ext}`; +} + +/** + * Derive a file extension. Prefer a known media extension from the URL path; + * otherwise map the content-type; otherwise fall back to a generic default so a + * file always gets a usable extension. + */ +function deriveExtension(url: string, contentType: string): string { + const fromUrl = extFromUrl(url); + if (fromUrl !== null) { + return fromUrl; + } + const ct = contentType.split(";")[0]?.trim().toLowerCase() ?? ""; + const mapped = CONTENT_TYPE_EXT[ct]; + if (mapped !== undefined) { + return mapped; + } + return DEFAULT_EXT; +} + +/** Extract a known media extension from a URL path, or null. */ +function extFromUrl(url: string): string | null { + const path = urlPath(url); + const lastDot = path.lastIndexOf("."); + if (lastDot === -1 || lastDot === path.length - 1) { + return null; + } + const ext = path.slice(lastDot + 1).toLowerCase(); + return KNOWN_MEDIA_EXTS.has(ext) ? ext : null; +} + +/** + * Derive a sanitized base filename (without extension). Prefer the item title; + * fall back to the URL's last path segment; fall back to a short hash of the url + * so two distinct extensionless urls do not collide on the same literal name. + */ +function deriveBaseName(item: FeedItem, url: string): string { + const fromTitle = sanitizeBaseName(item.title); + if (fromTitle.length > 0) { + return fromTitle; + } + const path = urlPath(url); + const segments = path.split("/").filter((s) => s.length > 0); + const last = segments.length > 0 ? segments[segments.length - 1] : undefined; + let base = last ?? ""; + const dot = base.lastIndexOf("."); + if (dot > 0) { + base = base.slice(0, dot); + } + base = sanitizeBaseName(base); + if (base.length === 0) { + base = `media-${shortHash(url)}`; + } + return base; +} + +/** + * The path portion of a url, stripped of query and fragment, without parsing via + * URL (not guaranteed everywhere this runs). Strips the scheme and authority so + * only the path text remains. + */ +function urlPath(url: string): string { + let rest = url.replace(/^https?:\/\//i, ""); + rest = rest.split("#")[0] ?? rest; + rest = rest.split("?")[0] ?? rest; + const slash = rest.indexOf("/"); + return slash === -1 ? "" : rest.slice(slash); +} + +/** + * Sanitize a filename base: drop characters illegal on Windows/macOS/Linux and + * Obsidian's wikilink brackets, collapse whitespace and dot/space runs, clamp + * length. Returns "" when nothing usable remains so the caller can fall back. + */ +function sanitizeBaseName(name: string): string { + let out = name.trim().replace(/\s+/g, "-"); + // eslint-disable-next-line no-control-regex -- strip NUL and other non-whitespace control codes from the filename + out = out.replace(/[<>:"/\\|?*\x00-\x1f[\]]/g, "-"); + out = out.replace(/-+/g, "-"); + out = out.replace(/^[.\- ]+/, "").replace(/[.\- ]+$/, ""); + if (out.length > 100) { + out = out.slice(0, 100).replace(/[.\- ]+$/, ""); + } + return out; +} + +/** + * Split a path into its directory, base name (no extension), and extension + * (including the leading dot, or empty). Path separator is the vault "/". + */ +function splitPath(p: string): { dir: string; base: string; ext: string } { + const slash = p.lastIndexOf("/"); + const dir = slash === -1 ? "" : p.slice(0, slash); + const file = slash === -1 ? p : p.slice(slash + 1); + const dot = file.lastIndexOf("."); + if (dot <= 0) { + return { dir, base: file, ext: "" }; + } + return { dir, base: file.slice(0, dot), ext: file.slice(dot) }; +} + +/** Join a folder and a file name with a single forward slash. */ +function joinPosix(folder: string, fileName: string): string { + const trimmed = folder.replace(/[/\\]+$/, ""); + return trimmed.length === 0 ? fileName : `${trimmed}/${fileName}`; +} + +/** + * A short, stable, filesystem-safe hash. Not cryptographic: it only needs to + * keep two distinct extensionless urls from colliding on the same fallback name. + * djb2 rendered base36. + */ +function shortHash(input: string): string { + let hash = 5381; + for (let i = 0; i < input.length; i++) { + hash = ((hash << 5) + hash + input.charCodeAt(i)) >>> 0; + } + return hash.toString(36); +} + +/** + * Normalize a vault folder path: trim, drop leading/trailing slashes, collapse + * doubled slashes, drop `.` and `..` segments. `..` is dropped (not thrown on) + * because this is an internal best-effort path, not a user-facing destination. + */ +function normalizeFolderPath(folder: string): string { + const cleaned = folder + .trim() + .replace(/^\/+|\/+$/g, "") + .replace(/\/{2,}/g, "/"); + const segments = cleaned + .split("/") + .filter((s) => s !== "" && s !== "." && s !== ".."); + return segments.join("/"); +} diff --git a/note-writer.ts b/note-writer.ts index 7eb70fe..3b9d68c 100644 --- a/note-writer.ts +++ b/note-writer.ts @@ -132,6 +132,13 @@ export interface ComposeOptions { readonly feedTags?: string[]; /** Frontmatter key the merged tags are written under. Defaults to feed-tags. */ readonly tagDestination?: TagDestination; + /** + * Local path of the downloaded media file (vault-relative or absolute). When + * set, a `media-file` frontmatter line is added and the body media link points + * at this local file instead of the remote media-url. The remote media-url + * frontmatter line is kept for reference. + */ + readonly mediaFile?: string; } // ----------------------------------------------------------------------------- @@ -413,7 +420,10 @@ const TRUNCATED_CALLOUT = [ * When the item carries media (a podcast/audio/video enclosure), a `media-url` * frontmatter line records it (force-quoted, since it is a URL) and a labelled * link to the media is appended to the body so the episode is reachable from the - * note. + * note. When opts.mediaFile is set (the enclosure was downloaded locally), a + * `media-file` frontmatter line is added after media-url and the body link points + * at the local file instead of the remote URL; the media-url line is kept for + * reference. */ export function composeNote( item: FeedItem, @@ -445,9 +455,15 @@ export function composeNote( // in frontmatter. Force-quoted because it is a URL. Placed after tags and // before the truncated marker. const hasMedia = item.mediaUrl !== null && item.mediaUrl.length > 0; + const hasMediaFile = opts.mediaFile !== undefined && opts.mediaFile.length > 0; if (hasMedia && item.mediaUrl !== null) { lines.push(`media-url: ${yamlQuoted(item.mediaUrl)}`); } + // When the enclosure was downloaded, record the local file path right after + // the remote url. Force-quoted because a path can contain spaces or colons. + if (hasMediaFile && opts.mediaFile !== undefined) { + lines.push(`media-file: ${yamlQuoted(opts.mediaFile)}`); + } if (item.isTruncated) { // Machine-readable marker alongside the visible callout below. lines.push("substack-truncated: true"); @@ -459,8 +475,12 @@ export function composeNote( : bodyMarkdown; // Link the media at the end of the body so a podcast note plays its episode - // and any other media item is reachable from the note. - if (hasMedia && item.mediaUrl !== null) { + // and any other media item is reachable from the note. Prefer the downloaded + // local file when present; otherwise fall back to the remote url. + if (hasMediaFile && opts.mediaFile !== undefined) { + const label = item.kind === "podcast" ? "Episode audio" : "Media"; + body = `${body}\n\n[${label}](${opts.mediaFile})`; + } else if (hasMedia && item.mediaUrl !== null) { const label = item.kind === "podcast" ? "Episode audio" : "Media"; body = `${body}\n\n[${label}](${item.mediaUrl})`; } diff --git a/settings-tab.ts b/settings-tab.ts index f0f379a..0141aa7 100644 --- a/settings-tab.ts +++ b/settings-tab.ts @@ -18,7 +18,13 @@ import { } from "obsidian"; import type { FeedSource } from "./feed-source"; import type { FeedConfig, RssImporterSettings } from "./settings"; -import { REQUEST_DELAY_MIN, REQUEST_DELAY_MAX } from "./settings"; +import { + REQUEST_DELAY_MIN, + REQUEST_DELAY_MAX, + effectiveDownloadMedia, + effectiveMediaLocation, + effectiveMediaSubfolder, +} from "./settings"; import { AddFeedModal } from "./add-feed-modal"; /** @@ -112,6 +118,33 @@ export class RssImporterSettingTab extends PluginSettingTab { }, }, }, + { + name: "Download media", + desc: "Save podcast audio and video enclosures locally, not just link them.", + control: { type: "toggle", key: "downloadMedia" }, + }, + { + name: "Media location", + desc: "Save media into a vault subfolder, or to a folder outside the vault.", + control: { + type: "dropdown", + key: "mediaLocation", + options: { + vault: "Vault subfolder", + outside: "Outside the vault", + }, + }, + }, + { + name: "Media subfolder", + desc: "Subfolder under each feed's folder for downloaded media.", + control: { type: "text", key: "mediaSubfolder" }, + }, + { + name: "Media folder outside vault", + desc: "Absolute folder path for media when saving outside the vault (desktop only).", + control: { type: "text", key: "mediaOutsideFolder" }, + }, { name: "Tag destination", desc: "Save feed tags as a note property, or as Obsidian tags that show in the tag pane.", @@ -259,6 +292,46 @@ class FeedEditorPage extends SettingPage { }), ); + const settings = this.plugin.settings; + + new Setting(editor) + .setName("Download media") + .setDesc("Save podcast audio and video enclosures locally for this feed.") + .addToggle((toggle) => + toggle + .setValue(effectiveDownloadMedia(feed, settings)) + .onChange(async (value) => { + feed.downloadMedia = value; + await this.plugin.saveSettings(); + }), + ); + + new Setting(editor) + .setName("Media location") + .setDesc("Save media into a vault subfolder, or to a folder outside the vault.") + .addDropdown((dropdown) => + dropdown + .addOption("vault", "Vault subfolder") + .addOption("outside", "Outside the vault") + .setValue(effectiveMediaLocation(feed, settings)) + .onChange(async (value) => { + feed.mediaLocation = value === "outside" ? "outside" : "vault"; + await this.plugin.saveSettings(); + }), + ); + + new Setting(editor) + .setName("Media subfolder") + .setDesc("Subfolder under this feed's folder for downloaded media.") + .addText((text) => + text + .setValue(effectiveMediaSubfolder(feed, settings)) + .onChange(async (value) => { + feed.mediaSubfolder = value; + await this.plugin.saveSettings(); + }), + ); + new Setting(editor) .setName("Remove feed") .setDesc("Delete this feed. Imported notes are left in place.") diff --git a/settings.ts b/settings.ts index 129e8cf..6c52afa 100644 --- a/settings.ts +++ b/settings.ts @@ -14,6 +14,14 @@ import type { TagDestination } from "./note-writer"; export type ImagesMode = "link" | "download"; export type DuplicatePolicy = "skip" | "overwrite" | "prompt"; +/** + * Where downloaded media (podcast audio/video enclosures) is written. "vault" + * writes into a subfolder of the feed's destination folder; "outside" writes to + * an absolute filesystem path (desktop only) so large media stays out of the + * synced vault. + */ +export type MediaLocation = "vault" | "outside"; + /** A configured feed and the resolved metadata captured when it was added. */ export interface FeedConfig { /** Stable id derived from the canonical host; also the note sourceId. */ @@ -41,6 +49,10 @@ export interface FeedConfig { imagesMode?: ImagesMode; imageSubfolder?: string; noteNameTemplate?: string; + downloadMedia?: boolean; + mediaLocation?: MediaLocation; + mediaSubfolder?: string; + mediaOutsideFolder?: string; } export interface RssImporterSettings { @@ -52,6 +64,14 @@ export interface RssImporterSettings { requestDelayMs: number; imagesMode: ImagesMode; imageSubfolder: string; + /** Download podcast/audio/video enclosures, not just link them. */ + downloadMedia: boolean; + /** Where downloaded media is written: a vault subfolder or an outside path. */ + mediaLocation: MediaLocation; + /** Subfolder (under the feed's destination) for media when mediaLocation is "vault". */ + mediaSubfolder: string; + /** Absolute filesystem folder for media when mediaLocation is "outside" (desktop only). */ + mediaOutsideFolder: string; /** Where feed tags are written: a plain "feed-tags" property or Obsidian "tags". */ tagDestination: TagDestination; debug: boolean; @@ -69,6 +89,10 @@ export const DEFAULT_SETTINGS: RssImporterSettings = { requestDelayMs: 1200, imagesMode: "link", imageSubfolder: "images", + downloadMedia: false, + mediaLocation: "vault", + mediaSubfolder: "media", + mediaOutsideFolder: "", tagDestination: "feed-tags", debug: false, showRibbonIcon: true, @@ -94,3 +118,23 @@ export function effectiveNoteNameTemplate(feed: FeedConfig, settings: RssImporte export function effectiveImageSubfolder(feed: FeedConfig, settings: RssImporterSettings): string { return feed.imageSubfolder ?? settings.imageSubfolder; } + +/** Resolves whether media is downloaded for a feed (per-feed override or default). */ +export function effectiveDownloadMedia(feed: FeedConfig, settings: RssImporterSettings): boolean { + return feed.downloadMedia ?? settings.downloadMedia; +} + +/** Resolves the effective media location for a feed. */ +export function effectiveMediaLocation(feed: FeedConfig, settings: RssImporterSettings): MediaLocation { + return feed.mediaLocation ?? settings.mediaLocation; +} + +/** Resolves the effective media subfolder name for a feed. */ +export function effectiveMediaSubfolder(feed: FeedConfig, settings: RssImporterSettings): string { + return feed.mediaSubfolder ?? settings.mediaSubfolder; +} + +/** Resolves the effective outside (absolute) media folder for a feed. */ +export function effectiveMediaOutsideFolder(feed: FeedConfig, settings: RssImporterSettings): string { + return feed.mediaOutsideFolder ?? settings.mediaOutsideFolder; +}