Feature: per-feed media (audio/video) download

Download podcast/audio/video enclosures into a vault subfolder (vault.createBinary)
or to an outside-vault folder (Node fs, desktop-only, behind a Platform.isDesktop
guard) for media you do not want syncing. Global defaults plus per-feed overrides
(download on/off, location, subfolder). Notes record a media-file frontmatter line
and point the body link at the local file. Failures are best-effort (logged, never
abort the item). +20 tests.
This commit is contained in:
Charles Kelsoe 2026-06-14 16:59:44 -04:00
parent 91a6f9a206
commit de018c18e8
9 changed files with 1119 additions and 4 deletions

View file

@ -288,6 +288,92 @@ describe("ImportRunner.run", () => {
expect(calls[0]?.body).toBe("MD:<p>A</p>");
});
it("passes the downloadMedia result to the writer as mediaFile", async () => {
const items = [makeItem({ id: "a", mediaUrl: "https://m.test/a.mp3" })];
const { source } = makeSource({ a: "<p>A</p>" });
const composeOpts: Array<Record<string, unknown> | undefined> = [];
const stub = {
writeNote(
_item: FeedItem,
_body: string,
opts?: Record<string, unknown>,
): Promise<WriteOutcome> {
composeOpts.push(opts);
return Promise.resolve({ status: "created", path: "Feeds/a.md" });
},
};
const writer = stub as unknown as NoteWriter;
const downloadMedia = (): Promise<string | null> =>
Promise.resolve("Feeds/media/a.mp3");
const runner = new ImportRunner({
source,
noteWriter: writer,
convert: identityConvert,
downloadMedia,
});
await runner.run(items, {});
expect(composeOpts).toHaveLength(1);
expect(composeOpts[0]?.mediaFile).toBe("Feeds/media/a.mp3");
});
it("writes the note without mediaFile when downloadMedia fails (best effort)", async () => {
const items = [makeItem({ id: "a", mediaUrl: "https://m.test/a.mp3" })];
const { source } = makeSource({ a: "<p>A</p>" });
const composeOpts: Array<Record<string, unknown> | undefined> = [];
const stub = {
writeNote(
_item: FeedItem,
_body: string,
opts?: Record<string, unknown>,
): Promise<WriteOutcome> {
composeOpts.push(opts);
return Promise.resolve({ status: "created", path: "Feeds/a.md" });
},
};
const writer = stub as unknown as NoteWriter;
const downloadMedia = (): Promise<string | null> =>
Promise.reject(new Error("media host down"));
const runner = new ImportRunner({
source,
noteWriter: writer,
convert: identityConvert,
downloadMedia,
});
const tally = await runner.run(items, {});
// Item still succeeds; mediaFile is undefined so the remote link is used.
expect(tally.created).toBe(1);
expect(tally.failed).toBe(0);
expect(composeOpts[0]?.mediaFile).toBeUndefined();
expect(errorSpy).toHaveBeenCalled();
});
it("skips downloadMedia when the item has no media url", async () => {
const items = [makeItem({ id: "a", mediaUrl: null })];
const { source } = makeSource({ a: "<p>A</p>" });
const { writer } = makeWriter(() =>
Promise.resolve({ status: "created", path: "Feeds/a.md" }),
);
let called = false;
const downloadMedia = (): Promise<string | null> => {
called = true;
return Promise.resolve(null);
};
const runner = new ImportRunner({
source,
noteWriter: writer,
convert: identityConvert,
downloadMedia,
});
await runner.run(items, {});
expect(called).toBe(false);
});
it("uses the processImages result when it succeeds", async () => {
const items = [makeItem({ id: "a" })];
const { source } = makeSource({ a: "<p>A</p>" });

View file

@ -0,0 +1,353 @@
import type { FeedItem, HttpFetcher, HttpRequest, HttpResponse } from "../feed-source";
import {
MediaDownloader,
type BinaryFileLike,
type VaultBinaryLike,
} from "../media-downloader";
// -----------------------------------------------------------------------------
// Fixtures
// -----------------------------------------------------------------------------
/** Turn an ASCII string into an ArrayBuffer for a fake response body. */
function bytes(text: string): ArrayBuffer {
const arr = new Uint8Array(text.length);
for (let i = 0; i < text.length; i++) {
arr[i] = text.charCodeAt(i);
}
return arr.buffer;
}
function makeItem(overrides: Partial<FeedItem> = {}): FeedItem {
return {
sourceId: "feed-1",
id: "item-1",
url: "https://example.com/p/one",
title: "Episode One",
author: "Author",
publishedAt: "2026-01-02T00:00:00.000Z",
kind: "podcast",
contentHtml: null,
isTruncated: false,
audience: "free",
tags: [],
section: null,
mediaUrl: "https://media.example.com/ep/one.mp3",
mediaType: "audio/mpeg",
mediaBytes: null,
...overrides,
};
}
interface FetchEntry {
status?: number;
headers?: Record<string, string>;
body?: ArrayBuffer;
throws?: Error;
}
/** Fetcher stub keyed by url; records the urls it was asked for. */
function makeFetcher(map: Record<string, FetchEntry>): {
fetcher: HttpFetcher;
requested: string[];
} {
const requested: string[] = [];
const fetcher: HttpFetcher = (req: HttpRequest) => {
requested.push(req.url);
const entry = map[req.url];
if (entry === undefined) {
return Promise.reject(new Error(`no stub for ${req.url}`));
}
if (entry.throws !== undefined) {
return Promise.reject(entry.throws);
}
const response: HttpResponse = {
status: entry.status ?? 200,
headers: entry.headers ?? {},
json: null,
text: "",
arrayBuffer: entry.body ?? bytes("MEDIA"),
};
return Promise.resolve(response);
};
return { fetcher, requested };
}
/** In-memory vault that records createBinary / createFolder calls. */
class FakeVault implements VaultBinaryLike {
files = new Set<string>();
folders = new Set<string>();
binaries: Array<{ path: string; size: number }> = [];
createdFolders: string[] = [];
getFileByPath(path: string): BinaryFileLike | null {
return this.files.has(path) ? { path } : null;
}
getFolderByPath(path: string): BinaryFileLike | null {
return this.folders.has(path) ? { path } : null;
}
createFolder(path: string): Promise<unknown> {
this.folders.add(path);
this.createdFolders.push(path);
return Promise.resolve({ path });
}
createBinary(path: string, data: ArrayBuffer): Promise<BinaryFileLike> {
this.files.add(path);
this.binaries.push({ path, size: data.byteLength });
return Promise.resolve({ path });
}
}
// -----------------------------------------------------------------------------
// Tests
// -----------------------------------------------------------------------------
describe("MediaDownloader.downloadToVault", () => {
let errorSpy: jest.SpyInstance;
beforeEach(() => {
errorSpy = jest.spyOn(console, "error").mockImplementation(() => {});
});
afterEach(() => {
errorSpy.mockRestore();
});
it("writes the media binary at the folder/<title>.<ext> path and returns it", async () => {
const url = "https://media.example.com/ep/one.mp3";
const { fetcher, requested } = makeFetcher({ [url]: { body: bytes("AUDIO") } });
const vault = new FakeVault();
const dl = new MediaDownloader({ fetcher, vault });
const item = makeItem({ title: "Episode One", mediaUrl: url, mediaType: "audio/mpeg" });
const out = await dl.downloadToVault(item, "Feeds/Podcast/media");
expect(requested).toEqual([url]);
expect(vault.binaries).toHaveLength(1);
expect(vault.binaries[0]?.path).toBe("Feeds/Podcast/media/Episode-One.mp3");
expect(vault.binaries[0]?.size).toBe("AUDIO".length);
expect(out).toBe("Feeds/Podcast/media/Episode-One.mp3");
// Ancestors created.
expect(vault.createdFolders).toEqual(["Feeds", "Feeds/Podcast", "Feeds/Podcast/media"]);
});
it("derives the extension from the URL path when present", async () => {
const url = "https://media.example.com/ep/one.m4a";
const { fetcher } = makeFetcher({ [url]: { body: bytes("A") } });
const vault = new FakeVault();
const dl = new MediaDownloader({ fetcher, vault });
const item = makeItem({ title: "Show", mediaUrl: url, mediaType: null });
await dl.downloadToVault(item, "media");
expect(vault.binaries[0]?.path).toBe("media/Show.m4a");
});
it("derives the extension from the content-type when the URL has none", async () => {
const url = "https://media.example.com/stream/123";
const { fetcher } = makeFetcher({
[url]: { headers: { "Content-Type": "audio/mp4" }, body: bytes("A") },
});
const vault = new FakeVault();
const dl = new MediaDownloader({ fetcher, vault });
const item = makeItem({ title: "Show", mediaUrl: url, mediaType: "audio/mp4" });
await dl.downloadToVault(item, "media");
expect(vault.binaries[0]?.path).toBe("media/Show.m4a");
});
it("falls back to a bin extension when neither URL nor content-type gives one", async () => {
const url = "https://media.example.com/stream/123";
const { fetcher } = makeFetcher({ [url]: { body: bytes("A") } });
const vault = new FakeVault();
const dl = new MediaDownloader({ fetcher, vault });
const item = makeItem({ title: "Show", mediaUrl: url, mediaType: null });
await dl.downloadToVault(item, "media");
expect(vault.binaries[0]?.path).toBe("media/Show.bin");
});
it("dedupes a collision by appending a numeric suffix before the extension", async () => {
const url = "https://media.example.com/ep/one.mp3";
const { fetcher } = makeFetcher({ [url]: { body: bytes("A") } });
const vault = new FakeVault();
// A file already exists at the natural target path.
vault.files.add("media/Show.mp3");
const dl = new MediaDownloader({ fetcher, vault });
const item = makeItem({ title: "Show", mediaUrl: url, mediaType: "audio/mpeg" });
const out = await dl.downloadToVault(item, "media");
expect(out).toBe("media/Show-1.mp3");
expect(vault.binaries[0]?.path).toBe("media/Show-1.mp3");
});
it("returns null on a non-2xx status and writes nothing", async () => {
const url = "https://media.example.com/ep/missing.mp3";
const { fetcher } = makeFetcher({ [url]: { status: 404, body: bytes("nope") } });
const vault = new FakeVault();
const dl = new MediaDownloader({ fetcher, vault });
const item = makeItem({ mediaUrl: url });
const out = await dl.downloadToVault(item, "media");
expect(out).toBeNull();
expect(vault.binaries).toHaveLength(0);
expect(errorSpy).toHaveBeenCalled();
});
it("returns null on an empty response body and writes nothing", async () => {
const url = "https://media.example.com/ep/empty.mp3";
const { fetcher } = makeFetcher({ [url]: { body: new ArrayBuffer(0) } });
const vault = new FakeVault();
const dl = new MediaDownloader({ fetcher, vault });
const item = makeItem({ mediaUrl: url });
const out = await dl.downloadToVault(item, "media");
expect(out).toBeNull();
expect(vault.binaries).toHaveLength(0);
});
it("returns null on a transport error without throwing", async () => {
const url = "https://media.example.com/ep/boom.mp3";
const { fetcher } = makeFetcher({ [url]: { throws: new Error("connection reset") } });
const vault = new FakeVault();
const dl = new MediaDownloader({ fetcher, vault });
const item = makeItem({ mediaUrl: url });
const out = await dl.downloadToVault(item, "media");
expect(out).toBeNull();
expect(errorSpy).toHaveBeenCalled();
});
it("returns null without fetching when the item has no media url", async () => {
const { fetcher, requested } = makeFetcher({});
const vault = new FakeVault();
const dl = new MediaDownloader({ fetcher, vault });
const out = await dl.downloadToVault(makeItem({ mediaUrl: null }), "media");
expect(out).toBeNull();
expect(requested).toHaveLength(0);
expect(vault.binaries).toHaveLength(0);
});
it("returns null without fetching when the item has an empty media url", async () => {
const { fetcher, requested } = makeFetcher({});
const vault = new FakeVault();
const dl = new MediaDownloader({ fetcher, vault });
const out = await dl.downloadToVault(makeItem({ mediaUrl: "" }), "media");
expect(out).toBeNull();
expect(requested).toHaveLength(0);
});
});
describe("MediaDownloader.downloadToOutside", () => {
let errorSpy: jest.SpyInstance;
beforeEach(() => {
errorSpy = jest.spyOn(console, "error").mockImplementation(() => {});
});
afterEach(() => {
errorSpy.mockRestore();
});
it("writes via the injected fileWriter and returns the absolute path", async () => {
const url = "https://media.example.com/ep/one.mp3";
const { fetcher } = makeFetcher({ [url]: { body: bytes("AUDIO") } });
const vault = new FakeVault();
const written: Array<{ path: string; size: number }> = [];
const fileWriter = (absPath: string, data: ArrayBuffer): void => {
written.push({ path: absPath, size: data.byteLength });
};
const dl = new MediaDownloader({ fetcher, vault, fileWriter });
const item = makeItem({ title: "Episode One", mediaUrl: url, mediaType: "audio/mpeg" });
const out = await dl.downloadToOutside(item, "/Users/me/Podcasts");
expect(out).toBe("/Users/me/Podcasts/Episode-One.mp3");
expect(written).toHaveLength(1);
expect(written[0]?.path).toBe("/Users/me/Podcasts/Episode-One.mp3");
expect(written[0]?.size).toBe("AUDIO".length);
// The vault is never touched for an outside write.
expect(vault.binaries).toHaveLength(0);
});
it("returns null when no outside folder is configured", async () => {
const url = "https://media.example.com/ep/one.mp3";
const { fetcher, requested } = makeFetcher({ [url]: { body: bytes("A") } });
const vault = new FakeVault();
const fileWriter = (): void => {};
const dl = new MediaDownloader({ fetcher, vault, fileWriter });
const out = await dl.downloadToOutside(makeItem({ mediaUrl: url }), "");
expect(out).toBeNull();
expect(requested).toHaveLength(0);
expect(errorSpy).toHaveBeenCalled();
});
it("returns null without fetching when the item has no media url", async () => {
const { fetcher, requested } = makeFetcher({});
const vault = new FakeVault();
const fileWriter = (): void => {};
const dl = new MediaDownloader({ fetcher, vault, fileWriter });
const out = await dl.downloadToOutside(makeItem({ mediaUrl: null }), "/Users/me/Podcasts");
expect(out).toBeNull();
expect(requested).toHaveLength(0);
});
});
describe("MediaDownloader.download", () => {
let errorSpy: jest.SpyInstance;
beforeEach(() => {
errorSpy = jest.spyOn(console, "error").mockImplementation(() => {});
});
afterEach(() => {
errorSpy.mockRestore();
});
it("dispatches to the vault path for location 'vault'", async () => {
const url = "https://media.example.com/ep/one.mp3";
const { fetcher } = makeFetcher({ [url]: { body: bytes("A") } });
const vault = new FakeVault();
const dl = new MediaDownloader({ fetcher, vault });
const item = makeItem({ title: "Show", mediaUrl: url, mediaType: "audio/mpeg" });
const out = await dl.download(item, {
location: "vault",
vaultFolder: "Feeds/media",
outsideFolder: "/abs",
});
expect(out).toBe("Feeds/media/Show.mp3");
expect(vault.binaries).toHaveLength(1);
});
it("dispatches to the outside path for location 'outside'", async () => {
const url = "https://media.example.com/ep/one.mp3";
const { fetcher } = makeFetcher({ [url]: { body: bytes("A") } });
const vault = new FakeVault();
const written: string[] = [];
const fileWriter = (absPath: string): void => {
written.push(absPath);
};
const dl = new MediaDownloader({ fetcher, vault, fileWriter });
const item = makeItem({ title: "Show", mediaUrl: url, mediaType: "audio/mpeg" });
const out = await dl.download(item, {
location: "outside",
vaultFolder: "Feeds/media",
outsideFolder: "/abs/podcasts",
});
expect(out).toBe("/abs/podcasts/Show.mp3");
expect(written).toEqual(["/abs/podcasts/Show.mp3"]);
expect(vault.binaries).toHaveLength(0);
});
});

View file

@ -276,6 +276,42 @@ describe("composeNote", () => {
expect(note).not.toContain("[Episode audio]");
expect(note).not.toContain("[Media]");
});
it("records media-file and links the local file when mediaFile is set", () => {
const mediaUrl = "https://media.example.com/ep/42.mp3";
const localPath = "Feeds/Podcast/media/Episode-42.mp3";
const item = makeItem({
kind: "podcast",
mediaUrl,
mediaType: "audio/mpeg",
});
const note = composeNote(item, "BODY", { mediaFile: localPath });
// Remote url is kept in frontmatter for reference.
expect(note).toContain(`media-url: "${mediaUrl}"`);
// Local file path recorded (force-quoted) after media-url.
expect(note).toContain(`media-file: "${localPath}"`);
const urlIndex = note.indexOf("media-url:");
const fileIndex = note.indexOf("media-file:");
expect(urlIndex).toBeGreaterThanOrEqual(0);
expect(fileIndex).toBeGreaterThan(urlIndex);
// Body links the LOCAL file, not the remote url.
expect(note).toContain(`[Episode audio](${localPath})`);
expect(note).not.toContain(`[Episode audio](${mediaUrl})`);
});
it("labels a non-podcast local media link as Media", () => {
const mediaUrl = "https://media.example.com/clip.mp4";
const localPath = "Feeds/media/clip.mp4";
const item = makeItem({
kind: "article",
mediaUrl,
mediaType: "video/mp4",
});
const note = composeNote(item, "BODY", { mediaFile: localPath });
expect(note).toContain(`media-file: "${localPath}"`);
expect(note).toContain(`[Media](${localPath})`);
expect(note).not.toContain(`[Media](${mediaUrl})`);
});
});
// -----------------------------------------------------------------------------

View file

@ -65,6 +65,13 @@ export interface ImportRunnerDeps {
convert: (html: string) => string;
/** Optional best-effort image download + markdown rewrite. */
processImages?: (markdown: string, item: FeedItem) => Promise<string>;
/**
* Optional best-effort media (podcast audio/video enclosure) download. Returns
* the local path to thread into the note as composeOptions.mediaFile, or null
* when there is no media or the download failed. A failure must NOT fail the
* item: the runner catches, logs, and writes the note without a local file.
*/
downloadMedia?: (item: FeedItem) => Promise<string | null>;
/** Optional debug logger; defaults to a no-op when absent. */
debugLogger?: DebugLogger;
}
@ -96,6 +103,7 @@ export class ImportRunner {
markdown: string,
item: FeedItem,
) => Promise<string>;
private readonly downloadMedia?: (item: FeedItem) => Promise<string | null>;
private readonly debug?: DebugLogger;
constructor(deps: ImportRunnerDeps) {
@ -103,6 +111,7 @@ export class ImportRunner {
this.noteWriter = deps.noteWriter;
this.convert = deps.convert;
this.processImages = deps.processImages;
this.downloadMedia = deps.downloadMedia;
this.debug = deps.debugLogger;
}
@ -201,8 +210,29 @@ export class ImportRunner {
});
}
// Best-effort media download. A failure must NOT fail the item: catch,
// log, and write the note without a local file (the remote media-url
// stays in frontmatter and as the body link).
let mediaFile: string | undefined;
if (this.downloadMedia !== undefined && full.mediaUrl !== null && full.mediaUrl.length > 0) {
try {
const local = await this.downloadMedia(full);
if (local !== null) {
mediaFile = local;
}
} catch (err) {
console.error(err);
this.debug?.log({
kind: "error",
message: `Media download failed for "${full.title}", keeping the remote link`,
payload: describeError(err),
});
}
}
const outcome = await this.noteWriter.writeNote(full, markdown, {
feedTags,
mediaFile,
});
this.debug?.log({
kind: "note",

42
main.ts
View file

@ -8,8 +8,12 @@ import {
import type { FeedSource, HttpFetcher, HttpRequest, HttpResponse, SourceType } from "./feed-source";
import {
DEFAULT_SETTINGS,
effectiveDownloadMedia,
effectiveImageSubfolder,
effectiveImagesMode,
effectiveMediaLocation,
effectiveMediaOutsideFolder,
effectiveMediaSubfolder,
effectiveNoteNameTemplate,
type FeedConfig,
type RssImporterSettings,
@ -24,6 +28,7 @@ import {
} from "./note-writer";
import { convertHtmlToMarkdown } from "./html-converter";
import { ImageDownloader } from "./image-downloader";
import { MediaDownloader } from "./media-downloader";
import { DismissStore } from "./dismiss-store";
import { BufferedDebugLogger } from "./debug-logger";
import { ImportRunner } from "./import-runner";
@ -91,6 +96,10 @@ function isDuplicatePolicy(value: unknown): value is import("./settings").Duplic
return value === "skip" || value === "overwrite" || value === "prompt";
}
function isMediaLocation(value: unknown): value is import("./settings").MediaLocation {
return value === "vault" || value === "outside";
}
// Sync classification used only to pick which source resolves a freshly entered
// input for the add-feed preview. The resolved feed's own sourceType (captured
// into the FeedConfig) is what drives import, so a custom-domain Substack that
@ -194,6 +203,9 @@ export default class RssImporterPlugin extends Plugin implements RssImporterPlug
if (!isDuplicatePolicy(this.settings.duplicatePolicy)) {
this.settings.duplicatePolicy = DEFAULT_SETTINGS.duplicatePolicy;
}
if (!isMediaLocation(this.settings.mediaLocation)) {
this.settings.mediaLocation = DEFAULT_SETTINGS.mediaLocation;
}
for (const feed of this.settings.feeds) {
// The per-feed override fields are optional literal/string unions on
// FeedConfig, but data.json is user-editable so a stored value can be a
@ -209,6 +221,18 @@ export default class RssImporterPlugin extends Plugin implements RssImporterPlug
if ("imageSubfolder" in record && typeof record["imageSubfolder"] !== "string") {
delete record["imageSubfolder"];
}
if ("downloadMedia" in record && typeof record["downloadMedia"] !== "boolean") {
delete record["downloadMedia"];
}
if ("mediaLocation" in record && !isMediaLocation(record["mediaLocation"])) {
delete record["mediaLocation"];
}
if ("mediaSubfolder" in record && typeof record["mediaSubfolder"] !== "string") {
delete record["mediaSubfolder"];
}
if ("mediaOutsideFolder" in record && typeof record["mediaOutsideFolder"] !== "string") {
delete record["mediaOutsideFolder"];
}
}
}
@ -297,11 +321,29 @@ export default class RssImporterPlugin extends Plugin implements RssImporterPlug
processImages = (markdown) => downloader.downloadAndRewrite(markdown, folderPath);
}
let downloadMedia: ((item: import("./feed-source").FeedItem) => Promise<string | null>) | undefined;
if (effectiveDownloadMedia(feed, this.settings)) {
const mediaDownloader = new MediaDownloader({
fetcher: this.makeFetcher(),
vault: this.app.vault,
});
const location = effectiveMediaLocation(feed, this.settings);
const mediaSubfolder = effectiveMediaSubfolder(feed, this.settings);
const vaultFolder =
feed.destinationFolder === ""
? mediaSubfolder
: `${feed.destinationFolder}/${mediaSubfolder}`;
const outsideFolder = effectiveMediaOutsideFolder(feed, this.settings);
downloadMedia = (item) =>
mediaDownloader.download(item, { location, vaultFolder, outsideFolder });
}
runner = new ImportRunner({
source,
noteWriter,
convert: convertHtmlToMarkdown,
processImages,
downloadMedia,
debugLogger: this.debugLogger,
});
} catch (err) {

431
media-downloader.ts Normal file
View file

@ -0,0 +1,431 @@
// Downloads a feed item's media enclosure (podcast audio, attached audio/video)
// either into the vault or to an absolute filesystem path outside the vault.
//
// This mirrors image-downloader's robustness contract: best effort, never throw
// out of the import. A media failure (transport error, non-2xx, empty body,
// write error, non-desktop for the outside path) logs a one-line reason and
// returns null so the note is still written with the remote media-url intact.
//
// The vault path reuses image-downloader's VaultBinaryLike surface so Obsidian's
// Vault satisfies it structurally and tests inject a plain object. The outside
// path uses Node fs/path, which on a marketplace plugin must be require()'d
// behind a Platform.isDesktop guard (see CLAUDE.md). To keep the module unit
// testable without touching the real filesystem, the constructor accepts an
// optional fileWriter the outside path prefers over the Node fallback.
import { Platform } from "obsidian";
import type { FeedItem, HttpFetcher } from "./feed-source";
import type { BinaryFileLike, VaultBinaryLike } from "./image-downloader";
export type { BinaryFileLike, VaultBinaryLike };
/** Where the media for one item should be written. */
export type MediaLocation = "vault" | "outside";
/**
* Writes raw bytes to an absolute filesystem path. Injected for tests so the
* outside-vault path can be exercised without touching the real disk; when
* absent, downloadToOutside falls back to Node fs behind a desktop guard.
*/
export type FileWriter = (absPath: string, data: ArrayBuffer) => void;
export interface MediaDownloaderOptions {
readonly fetcher: HttpFetcher;
readonly vault: VaultBinaryLike;
readonly fileWriter?: FileWriter;
}
export interface MediaDownloadOptions {
readonly location: MediaLocation;
/** Vault-relative folder for the "vault" location. */
readonly vaultFolder: string;
/** Absolute filesystem folder for the "outside" location. */
readonly outsideFolder: string;
}
// Map common enclosure content-types to a file extension. Audio and video MIME
// types the URL path may not carry. Anything unmapped falls back to the URL
// extension, then to a generic default.
const CONTENT_TYPE_EXT: Record<string, string> = {
"audio/mpeg": "mp3",
"audio/mp3": "mp3",
"audio/mp4": "m4a",
"audio/x-m4a": "m4a",
"audio/aac": "aac",
"audio/ogg": "ogg",
"audio/opus": "opus",
"audio/wav": "wav",
"audio/x-wav": "wav",
"audio/flac": "flac",
"video/mp4": "mp4",
"video/webm": "webm",
"video/quicktime": "mov",
"video/x-matroska": "mkv",
};
// Extensions accepted from a URL path. Anything else (or none) falls back to the
// content-type, then to a generic default.
const KNOWN_MEDIA_EXTS = new Set([
"mp3", "m4a", "aac", "ogg", "opus", "wav", "flac",
"mp4", "webm", "mov", "mkv", "m4v", "oga",
]);
const DEFAULT_EXT = "bin";
export class MediaDownloader {
private readonly fetcher: HttpFetcher;
private readonly vault: VaultBinaryLike;
private readonly fileWriter?: FileWriter;
constructor(opts: MediaDownloaderOptions) {
this.fetcher = opts.fetcher;
this.vault = opts.vault;
this.fileWriter = opts.fileWriter;
}
/**
* Dispatch by location. Returns the written path (vault-relative or absolute)
* on success, or null when there is no media or the download/write failed.
*/
async download(item: FeedItem, opts: MediaDownloadOptions): Promise<string | null> {
if (opts.location === "outside") {
return this.downloadToOutside(item, opts.outsideFolder);
}
return this.downloadToVault(item, opts.vaultFolder);
}
/**
* Fetch the item's media and write it under `folderPath` in the vault. Returns
* the vault-relative path on success. Returns null (never throws) when there
* is no media url, the fetch is non-2xx or empty, or any write step fails.
*/
async downloadToVault(item: FeedItem, folderPath: string): Promise<string | null> {
const mediaUrl = item.mediaUrl;
if (mediaUrl === null || mediaUrl.length === 0) {
return null;
}
const fetched = await this.fetchMedia(mediaUrl);
if (fetched === null) {
return null;
}
const folder = normalizeFolderPath(folderPath);
try {
await this.ensureFolder(folder);
} catch (err) {
console.error(`RSS Importer: media download skipped for ${mediaUrl}: ${describe(err)}`);
return null;
}
const fileName = deriveFileName(item, mediaUrl, fetched.contentType);
const targetPath = folder === "" ? fileName : `${folder}/${fileName}`;
const uniquePath = await this.uniqueVaultPath(targetPath);
try {
await this.vault.createBinary(uniquePath, fetched.bytes);
} catch (err) {
console.error(`RSS Importer: media write failed for ${mediaUrl}: ${describe(err)}`);
return null;
}
return uniquePath;
}
/**
* Fetch the item's media and write it to an absolute path under `absFolder`.
* Desktop only. Uses the injected fileWriter when present (tests); otherwise
* Node fs/path behind a Platform.isDesktop guard. Returns the absolute path on
* success, or null (never throws) on non-desktop, no media, or any failure.
*/
async downloadToOutside(item: FeedItem, absFolder: string): Promise<string | null> {
const mediaUrl = item.mediaUrl;
if (mediaUrl === null || mediaUrl.length === 0) {
return null;
}
if (this.fileWriter === undefined && !Platform.isDesktop) {
console.error(`RSS Importer: media download skipped for ${mediaUrl}: outside-vault writes need a desktop app`);
return null;
}
if (absFolder.length === 0) {
console.error(`RSS Importer: media download skipped for ${mediaUrl}: no outside folder configured`);
return null;
}
const fetched = await this.fetchMedia(mediaUrl);
if (fetched === null) {
return null;
}
const fileName = deriveFileName(item, mediaUrl, fetched.contentType);
try {
return this.writeOutside(absFolder, fileName, fetched.bytes);
} catch (err) {
console.error(`RSS Importer: media write failed for ${mediaUrl}: ${describe(err)}`);
return null;
}
}
/**
* Write bytes to absFolder/fileName, preferring the injected fileWriter. When
* none is injected, use Node fs/path behind a desktop guard: require() rather
* than a top-level import keeps the bundle clean of Node built-ins on mobile,
* per the marketplace scorecard. Returns the absolute path written.
*/
private writeOutside(absFolder: string, fileName: string, bytes: ArrayBuffer): string {
if (this.fileWriter !== undefined) {
const joined = joinPosix(absFolder, fileName);
this.fileWriter(joined, bytes);
return joined;
}
const { fs, path } = getNodeFsPath();
fs.mkdirSync(absFolder, { recursive: true });
const target = path.join(absFolder, fileName);
fs.writeFileSync(target, Buffer.from(bytes));
return target;
}
/**
* GET the media url and validate the response. Returns the bytes and the
* content-type on success, or null (logged) on non-2xx, empty body, or a
* transport error.
*/
private async fetchMedia(
url: string,
): Promise<{ bytes: ArrayBuffer; contentType: string } | null> {
let response;
try {
response = await this.fetcher({ url, method: "GET" });
} catch (err) {
console.error(`RSS Importer: media download failed for ${url}: ${describe(err)}`);
return null;
}
if (response.status < 200 || response.status >= 300) {
console.error(`RSS Importer: media download skipped for ${url}: status ${response.status}`);
return null;
}
const bytes = response.arrayBuffer;
if (bytes.byteLength === 0) {
console.error(`RSS Importer: media download skipped for ${url}: empty body`);
return null;
}
return { bytes, contentType: headerValue(response.headers, "content-type") };
}
/**
* Create the folder and each missing ancestor. createFolder throws when the
* folder already exists, so each segment is checked first.
*/
private async ensureFolder(folder: string): Promise<void> {
if (folder === "") {
return;
}
const segments = folder.split("/");
for (let i = 1; i <= segments.length; i++) {
const partial = segments.slice(0, i).join("/");
if (this.vault.getFolderByPath(partial) === null) {
await this.vault.createFolder(partial);
}
}
}
/**
* Resolve a non-colliding vault path. If the target already exists, append a
* numeric suffix before the extension (name-1.ext, name-2.ext, ...).
*/
private async uniqueVaultPath(targetPath: string): Promise<string> {
if (this.vault.getFileByPath(targetPath) === null) {
return targetPath;
}
const { dir, base, ext } = splitPath(targetPath);
for (let n = 1; ; n++) {
const candidate = dir === "" ? `${base}-${n}${ext}` : `${dir}/${base}-${n}${ext}`;
if (this.vault.getFileByPath(candidate) === null) {
return candidate;
}
}
}
}
// -----------------------------------------------------------------------------
// Pure helpers (kept module-private; exercised through the public methods).
// -----------------------------------------------------------------------------
/**
* Resolve Node's fs and path modules. Obsidian's guidelines require Node
* built-ins to be loaded via a Platform.isDesktop-guarded require() so the
* mobile bundle never references them; callers must gate on Platform.isDesktop
* before reaching here. The require, the unsafe cast, and the node-module import
* are the unavoidable consequences of that guidance.
*/
function getNodeFsPath(): { fs: typeof import("fs"); path: typeof import("path") } {
if (!Platform.isDesktop) {
throw new Error("Node fs/path modules are not available on this platform.");
}
// eslint-disable-next-line @typescript-eslint/no-require-imports -- Obsidian docs require a Platform.isDesktop-guarded require() for Node built-ins so mobile builds do not pull them in; the require is the unavoidable consequence of that guidance.
const fs = require("fs") as typeof import("fs");
// eslint-disable-next-line @typescript-eslint/no-require-imports -- Obsidian docs require a Platform.isDesktop-guarded require() for Node built-ins so mobile builds do not pull them in; the require is the unavoidable consequence of that guidance.
const path = require("path") as typeof import("path");
return { fs, path };
}
/** Read a header case-insensitively; returns "" when absent. */
function headerValue(headers: Record<string, string>, name: string): string {
const wanted = name.toLowerCase();
for (const key of Object.keys(headers)) {
if (key.toLowerCase() === wanted) {
return headers[key] ?? "";
}
}
return "";
}
/** Render a caught unknown into a one-line message. */
function describe(err: unknown): string {
return err instanceof Error ? err.message : String(err);
}
/**
* Derive the media filename: a sanitized item title (preferred) or the URL
* basename, plus an extension from the content-type, then the URL, then a
* generic default.
*/
function deriveFileName(item: FeedItem, url: string, contentType: string): string {
const ext = deriveExtension(url, contentType);
const base = deriveBaseName(item, url);
return `${base}.${ext}`;
}
/**
* Derive a file extension. Prefer a known media extension from the URL path;
* otherwise map the content-type; otherwise fall back to a generic default so a
* file always gets a usable extension.
*/
function deriveExtension(url: string, contentType: string): string {
const fromUrl = extFromUrl(url);
if (fromUrl !== null) {
return fromUrl;
}
const ct = contentType.split(";")[0]?.trim().toLowerCase() ?? "";
const mapped = CONTENT_TYPE_EXT[ct];
if (mapped !== undefined) {
return mapped;
}
return DEFAULT_EXT;
}
/** Extract a known media extension from a URL path, or null. */
function extFromUrl(url: string): string | null {
const path = urlPath(url);
const lastDot = path.lastIndexOf(".");
if (lastDot === -1 || lastDot === path.length - 1) {
return null;
}
const ext = path.slice(lastDot + 1).toLowerCase();
return KNOWN_MEDIA_EXTS.has(ext) ? ext : null;
}
/**
* Derive a sanitized base filename (without extension). Prefer the item title;
* fall back to the URL's last path segment; fall back to a short hash of the url
* so two distinct extensionless urls do not collide on the same literal name.
*/
function deriveBaseName(item: FeedItem, url: string): string {
const fromTitle = sanitizeBaseName(item.title);
if (fromTitle.length > 0) {
return fromTitle;
}
const path = urlPath(url);
const segments = path.split("/").filter((s) => s.length > 0);
const last = segments.length > 0 ? segments[segments.length - 1] : undefined;
let base = last ?? "";
const dot = base.lastIndexOf(".");
if (dot > 0) {
base = base.slice(0, dot);
}
base = sanitizeBaseName(base);
if (base.length === 0) {
base = `media-${shortHash(url)}`;
}
return base;
}
/**
* The path portion of a url, stripped of query and fragment, without parsing via
* URL (not guaranteed everywhere this runs). Strips the scheme and authority so
* only the path text remains.
*/
function urlPath(url: string): string {
let rest = url.replace(/^https?:\/\//i, "");
rest = rest.split("#")[0] ?? rest;
rest = rest.split("?")[0] ?? rest;
const slash = rest.indexOf("/");
return slash === -1 ? "" : rest.slice(slash);
}
/**
* Sanitize a filename base: drop characters illegal on Windows/macOS/Linux and
* Obsidian's wikilink brackets, collapse whitespace and dot/space runs, clamp
* length. Returns "" when nothing usable remains so the caller can fall back.
*/
function sanitizeBaseName(name: string): string {
let out = name.trim().replace(/\s+/g, "-");
// eslint-disable-next-line no-control-regex -- strip NUL and other non-whitespace control codes from the filename
out = out.replace(/[<>:"/\\|?*\x00-\x1f[\]]/g, "-");
out = out.replace(/-+/g, "-");
out = out.replace(/^[.\- ]+/, "").replace(/[.\- ]+$/, "");
if (out.length > 100) {
out = out.slice(0, 100).replace(/[.\- ]+$/, "");
}
return out;
}
/**
* Split a path into its directory, base name (no extension), and extension
* (including the leading dot, or empty). Path separator is the vault "/".
*/
function splitPath(p: string): { dir: string; base: string; ext: string } {
const slash = p.lastIndexOf("/");
const dir = slash === -1 ? "" : p.slice(0, slash);
const file = slash === -1 ? p : p.slice(slash + 1);
const dot = file.lastIndexOf(".");
if (dot <= 0) {
return { dir, base: file, ext: "" };
}
return { dir, base: file.slice(0, dot), ext: file.slice(dot) };
}
/** Join a folder and a file name with a single forward slash. */
function joinPosix(folder: string, fileName: string): string {
const trimmed = folder.replace(/[/\\]+$/, "");
return trimmed.length === 0 ? fileName : `${trimmed}/${fileName}`;
}
/**
* A short, stable, filesystem-safe hash. Not cryptographic: it only needs to
* keep two distinct extensionless urls from colliding on the same fallback name.
* djb2 rendered base36.
*/
function shortHash(input: string): string {
let hash = 5381;
for (let i = 0; i < input.length; i++) {
hash = ((hash << 5) + hash + input.charCodeAt(i)) >>> 0;
}
return hash.toString(36);
}
/**
* Normalize a vault folder path: trim, drop leading/trailing slashes, collapse
* doubled slashes, drop `.` and `..` segments. `..` is dropped (not thrown on)
* because this is an internal best-effort path, not a user-facing destination.
*/
function normalizeFolderPath(folder: string): string {
const cleaned = folder
.trim()
.replace(/^\/+|\/+$/g, "")
.replace(/\/{2,}/g, "/");
const segments = cleaned
.split("/")
.filter((s) => s !== "" && s !== "." && s !== "..");
return segments.join("/");
}

View file

@ -132,6 +132,13 @@ export interface ComposeOptions {
readonly feedTags?: string[];
/** Frontmatter key the merged tags are written under. Defaults to feed-tags. */
readonly tagDestination?: TagDestination;
/**
* Local path of the downloaded media file (vault-relative or absolute). When
* set, a `media-file` frontmatter line is added and the body media link points
* at this local file instead of the remote media-url. The remote media-url
* frontmatter line is kept for reference.
*/
readonly mediaFile?: string;
}
// -----------------------------------------------------------------------------
@ -413,7 +420,10 @@ const TRUNCATED_CALLOUT = [
* When the item carries media (a podcast/audio/video enclosure), a `media-url`
* frontmatter line records it (force-quoted, since it is a URL) and a labelled
* link to the media is appended to the body so the episode is reachable from the
* note.
* note. When opts.mediaFile is set (the enclosure was downloaded locally), a
* `media-file` frontmatter line is added after media-url and the body link points
* at the local file instead of the remote URL; the media-url line is kept for
* reference.
*/
export function composeNote(
item: FeedItem,
@ -445,9 +455,15 @@ export function composeNote(
// in frontmatter. Force-quoted because it is a URL. Placed after tags and
// before the truncated marker.
const hasMedia = item.mediaUrl !== null && item.mediaUrl.length > 0;
const hasMediaFile = opts.mediaFile !== undefined && opts.mediaFile.length > 0;
if (hasMedia && item.mediaUrl !== null) {
lines.push(`media-url: ${yamlQuoted(item.mediaUrl)}`);
}
// When the enclosure was downloaded, record the local file path right after
// the remote url. Force-quoted because a path can contain spaces or colons.
if (hasMediaFile && opts.mediaFile !== undefined) {
lines.push(`media-file: ${yamlQuoted(opts.mediaFile)}`);
}
if (item.isTruncated) {
// Machine-readable marker alongside the visible callout below.
lines.push("substack-truncated: true");
@ -459,8 +475,12 @@ export function composeNote(
: bodyMarkdown;
// Link the media at the end of the body so a podcast note plays its episode
// and any other media item is reachable from the note.
if (hasMedia && item.mediaUrl !== null) {
// and any other media item is reachable from the note. Prefer the downloaded
// local file when present; otherwise fall back to the remote url.
if (hasMediaFile && opts.mediaFile !== undefined) {
const label = item.kind === "podcast" ? "Episode audio" : "Media";
body = `${body}\n\n[${label}](${opts.mediaFile})`;
} else if (hasMedia && item.mediaUrl !== null) {
const label = item.kind === "podcast" ? "Episode audio" : "Media";
body = `${body}\n\n[${label}](${item.mediaUrl})`;
}

View file

@ -18,7 +18,13 @@ import {
} from "obsidian";
import type { FeedSource } from "./feed-source";
import type { FeedConfig, RssImporterSettings } from "./settings";
import { REQUEST_DELAY_MIN, REQUEST_DELAY_MAX } from "./settings";
import {
REQUEST_DELAY_MIN,
REQUEST_DELAY_MAX,
effectiveDownloadMedia,
effectiveMediaLocation,
effectiveMediaSubfolder,
} from "./settings";
import { AddFeedModal } from "./add-feed-modal";
/**
@ -112,6 +118,33 @@ export class RssImporterSettingTab extends PluginSettingTab {
},
},
},
{
name: "Download media",
desc: "Save podcast audio and video enclosures locally, not just link them.",
control: { type: "toggle", key: "downloadMedia" },
},
{
name: "Media location",
desc: "Save media into a vault subfolder, or to a folder outside the vault.",
control: {
type: "dropdown",
key: "mediaLocation",
options: {
vault: "Vault subfolder",
outside: "Outside the vault",
},
},
},
{
name: "Media subfolder",
desc: "Subfolder under each feed's folder for downloaded media.",
control: { type: "text", key: "mediaSubfolder" },
},
{
name: "Media folder outside vault",
desc: "Absolute folder path for media when saving outside the vault (desktop only).",
control: { type: "text", key: "mediaOutsideFolder" },
},
{
name: "Tag destination",
desc: "Save feed tags as a note property, or as Obsidian tags that show in the tag pane.",
@ -259,6 +292,46 @@ class FeedEditorPage extends SettingPage {
}),
);
const settings = this.plugin.settings;
new Setting(editor)
.setName("Download media")
.setDesc("Save podcast audio and video enclosures locally for this feed.")
.addToggle((toggle) =>
toggle
.setValue(effectiveDownloadMedia(feed, settings))
.onChange(async (value) => {
feed.downloadMedia = value;
await this.plugin.saveSettings();
}),
);
new Setting(editor)
.setName("Media location")
.setDesc("Save media into a vault subfolder, or to a folder outside the vault.")
.addDropdown((dropdown) =>
dropdown
.addOption("vault", "Vault subfolder")
.addOption("outside", "Outside the vault")
.setValue(effectiveMediaLocation(feed, settings))
.onChange(async (value) => {
feed.mediaLocation = value === "outside" ? "outside" : "vault";
await this.plugin.saveSettings();
}),
);
new Setting(editor)
.setName("Media subfolder")
.setDesc("Subfolder under this feed's folder for downloaded media.")
.addText((text) =>
text
.setValue(effectiveMediaSubfolder(feed, settings))
.onChange(async (value) => {
feed.mediaSubfolder = value;
await this.plugin.saveSettings();
}),
);
new Setting(editor)
.setName("Remove feed")
.setDesc("Delete this feed. Imported notes are left in place.")

View file

@ -14,6 +14,14 @@ import type { TagDestination } from "./note-writer";
export type ImagesMode = "link" | "download";
export type DuplicatePolicy = "skip" | "overwrite" | "prompt";
/**
* Where downloaded media (podcast audio/video enclosures) is written. "vault"
* writes into a subfolder of the feed's destination folder; "outside" writes to
* an absolute filesystem path (desktop only) so large media stays out of the
* synced vault.
*/
export type MediaLocation = "vault" | "outside";
/** A configured feed and the resolved metadata captured when it was added. */
export interface FeedConfig {
/** Stable id derived from the canonical host; also the note sourceId. */
@ -41,6 +49,10 @@ export interface FeedConfig {
imagesMode?: ImagesMode;
imageSubfolder?: string;
noteNameTemplate?: string;
downloadMedia?: boolean;
mediaLocation?: MediaLocation;
mediaSubfolder?: string;
mediaOutsideFolder?: string;
}
export interface RssImporterSettings {
@ -52,6 +64,14 @@ export interface RssImporterSettings {
requestDelayMs: number;
imagesMode: ImagesMode;
imageSubfolder: string;
/** Download podcast/audio/video enclosures, not just link them. */
downloadMedia: boolean;
/** Where downloaded media is written: a vault subfolder or an outside path. */
mediaLocation: MediaLocation;
/** Subfolder (under the feed's destination) for media when mediaLocation is "vault". */
mediaSubfolder: string;
/** Absolute filesystem folder for media when mediaLocation is "outside" (desktop only). */
mediaOutsideFolder: string;
/** Where feed tags are written: a plain "feed-tags" property or Obsidian "tags". */
tagDestination: TagDestination;
debug: boolean;
@ -69,6 +89,10 @@ export const DEFAULT_SETTINGS: RssImporterSettings = {
requestDelayMs: 1200,
imagesMode: "link",
imageSubfolder: "images",
downloadMedia: false,
mediaLocation: "vault",
mediaSubfolder: "media",
mediaOutsideFolder: "",
tagDestination: "feed-tags",
debug: false,
showRibbonIcon: true,
@ -94,3 +118,23 @@ export function effectiveNoteNameTemplate(feed: FeedConfig, settings: RssImporte
export function effectiveImageSubfolder(feed: FeedConfig, settings: RssImporterSettings): string {
return feed.imageSubfolder ?? settings.imageSubfolder;
}
/** Resolves whether media is downloaded for a feed (per-feed override or default). */
export function effectiveDownloadMedia(feed: FeedConfig, settings: RssImporterSettings): boolean {
return feed.downloadMedia ?? settings.downloadMedia;
}
/** Resolves the effective media location for a feed. */
export function effectiveMediaLocation(feed: FeedConfig, settings: RssImporterSettings): MediaLocation {
return feed.mediaLocation ?? settings.mediaLocation;
}
/** Resolves the effective media subfolder name for a feed. */
export function effectiveMediaSubfolder(feed: FeedConfig, settings: RssImporterSettings): string {
return feed.mediaSubfolder ?? settings.mediaSubfolder;
}
/** Resolves the effective outside (absolute) media folder for a feed. */
export function effectiveMediaOutsideFolder(feed: FeedConfig, settings: RssImporterSettings): string {
return feed.mediaOutsideFolder ?? settings.mediaOutsideFolder;
}