mirror of
https://github.com/ckelsoe/obsidian-rss-importer.git
synced 2026-07-22 07:48:56 +00:00
Feature: per-feed media (audio/video) download
Download podcast/audio/video enclosures into a vault subfolder (vault.createBinary) or to an outside-vault folder (Node fs, desktop-only, behind a Platform.isDesktop guard) for media you do not want syncing. Global defaults plus per-feed overrides (download on/off, location, subfolder). Notes record a media-file frontmatter line and point the body link at the local file. Failures are best-effort (logged, never abort the item). +20 tests.
This commit is contained in:
parent
91a6f9a206
commit
de018c18e8
9 changed files with 1119 additions and 4 deletions
|
|
@ -288,6 +288,92 @@ describe("ImportRunner.run", () => {
|
|||
expect(calls[0]?.body).toBe("MD:<p>A</p>");
|
||||
});
|
||||
|
||||
it("passes the downloadMedia result to the writer as mediaFile", async () => {
|
||||
const items = [makeItem({ id: "a", mediaUrl: "https://m.test/a.mp3" })];
|
||||
const { source } = makeSource({ a: "<p>A</p>" });
|
||||
const composeOpts: Array<Record<string, unknown> | undefined> = [];
|
||||
const stub = {
|
||||
writeNote(
|
||||
_item: FeedItem,
|
||||
_body: string,
|
||||
opts?: Record<string, unknown>,
|
||||
): Promise<WriteOutcome> {
|
||||
composeOpts.push(opts);
|
||||
return Promise.resolve({ status: "created", path: "Feeds/a.md" });
|
||||
},
|
||||
};
|
||||
const writer = stub as unknown as NoteWriter;
|
||||
const downloadMedia = (): Promise<string | null> =>
|
||||
Promise.resolve("Feeds/media/a.mp3");
|
||||
|
||||
const runner = new ImportRunner({
|
||||
source,
|
||||
noteWriter: writer,
|
||||
convert: identityConvert,
|
||||
downloadMedia,
|
||||
});
|
||||
await runner.run(items, {});
|
||||
|
||||
expect(composeOpts).toHaveLength(1);
|
||||
expect(composeOpts[0]?.mediaFile).toBe("Feeds/media/a.mp3");
|
||||
});
|
||||
|
||||
it("writes the note without mediaFile when downloadMedia fails (best effort)", async () => {
|
||||
const items = [makeItem({ id: "a", mediaUrl: "https://m.test/a.mp3" })];
|
||||
const { source } = makeSource({ a: "<p>A</p>" });
|
||||
const composeOpts: Array<Record<string, unknown> | undefined> = [];
|
||||
const stub = {
|
||||
writeNote(
|
||||
_item: FeedItem,
|
||||
_body: string,
|
||||
opts?: Record<string, unknown>,
|
||||
): Promise<WriteOutcome> {
|
||||
composeOpts.push(opts);
|
||||
return Promise.resolve({ status: "created", path: "Feeds/a.md" });
|
||||
},
|
||||
};
|
||||
const writer = stub as unknown as NoteWriter;
|
||||
const downloadMedia = (): Promise<string | null> =>
|
||||
Promise.reject(new Error("media host down"));
|
||||
|
||||
const runner = new ImportRunner({
|
||||
source,
|
||||
noteWriter: writer,
|
||||
convert: identityConvert,
|
||||
downloadMedia,
|
||||
});
|
||||
const tally = await runner.run(items, {});
|
||||
|
||||
// Item still succeeds; mediaFile is undefined so the remote link is used.
|
||||
expect(tally.created).toBe(1);
|
||||
expect(tally.failed).toBe(0);
|
||||
expect(composeOpts[0]?.mediaFile).toBeUndefined();
|
||||
expect(errorSpy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips downloadMedia when the item has no media url", async () => {
|
||||
const items = [makeItem({ id: "a", mediaUrl: null })];
|
||||
const { source } = makeSource({ a: "<p>A</p>" });
|
||||
const { writer } = makeWriter(() =>
|
||||
Promise.resolve({ status: "created", path: "Feeds/a.md" }),
|
||||
);
|
||||
let called = false;
|
||||
const downloadMedia = (): Promise<string | null> => {
|
||||
called = true;
|
||||
return Promise.resolve(null);
|
||||
};
|
||||
|
||||
const runner = new ImportRunner({
|
||||
source,
|
||||
noteWriter: writer,
|
||||
convert: identityConvert,
|
||||
downloadMedia,
|
||||
});
|
||||
await runner.run(items, {});
|
||||
|
||||
expect(called).toBe(false);
|
||||
});
|
||||
|
||||
it("uses the processImages result when it succeeds", async () => {
|
||||
const items = [makeItem({ id: "a" })];
|
||||
const { source } = makeSource({ a: "<p>A</p>" });
|
||||
|
|
|
|||
353
__tests__/media-downloader.test.ts
Normal file
353
__tests__/media-downloader.test.ts
Normal file
|
|
@ -0,0 +1,353 @@
|
|||
import type { FeedItem, HttpFetcher, HttpRequest, HttpResponse } from "../feed-source";
|
||||
import {
|
||||
MediaDownloader,
|
||||
type BinaryFileLike,
|
||||
type VaultBinaryLike,
|
||||
} from "../media-downloader";
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Fixtures
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/** Turn an ASCII string into an ArrayBuffer for a fake response body. */
|
||||
function bytes(text: string): ArrayBuffer {
|
||||
const arr = new Uint8Array(text.length);
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
arr[i] = text.charCodeAt(i);
|
||||
}
|
||||
return arr.buffer;
|
||||
}
|
||||
|
||||
function makeItem(overrides: Partial<FeedItem> = {}): FeedItem {
|
||||
return {
|
||||
sourceId: "feed-1",
|
||||
id: "item-1",
|
||||
url: "https://example.com/p/one",
|
||||
title: "Episode One",
|
||||
author: "Author",
|
||||
publishedAt: "2026-01-02T00:00:00.000Z",
|
||||
kind: "podcast",
|
||||
contentHtml: null,
|
||||
isTruncated: false,
|
||||
audience: "free",
|
||||
tags: [],
|
||||
section: null,
|
||||
mediaUrl: "https://media.example.com/ep/one.mp3",
|
||||
mediaType: "audio/mpeg",
|
||||
mediaBytes: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
interface FetchEntry {
|
||||
status?: number;
|
||||
headers?: Record<string, string>;
|
||||
body?: ArrayBuffer;
|
||||
throws?: Error;
|
||||
}
|
||||
|
||||
/** Fetcher stub keyed by url; records the urls it was asked for. */
|
||||
function makeFetcher(map: Record<string, FetchEntry>): {
|
||||
fetcher: HttpFetcher;
|
||||
requested: string[];
|
||||
} {
|
||||
const requested: string[] = [];
|
||||
const fetcher: HttpFetcher = (req: HttpRequest) => {
|
||||
requested.push(req.url);
|
||||
const entry = map[req.url];
|
||||
if (entry === undefined) {
|
||||
return Promise.reject(new Error(`no stub for ${req.url}`));
|
||||
}
|
||||
if (entry.throws !== undefined) {
|
||||
return Promise.reject(entry.throws);
|
||||
}
|
||||
const response: HttpResponse = {
|
||||
status: entry.status ?? 200,
|
||||
headers: entry.headers ?? {},
|
||||
json: null,
|
||||
text: "",
|
||||
arrayBuffer: entry.body ?? bytes("MEDIA"),
|
||||
};
|
||||
return Promise.resolve(response);
|
||||
};
|
||||
return { fetcher, requested };
|
||||
}
|
||||
|
||||
/** In-memory vault that records createBinary / createFolder calls. */
|
||||
class FakeVault implements VaultBinaryLike {
|
||||
files = new Set<string>();
|
||||
folders = new Set<string>();
|
||||
binaries: Array<{ path: string; size: number }> = [];
|
||||
createdFolders: string[] = [];
|
||||
|
||||
getFileByPath(path: string): BinaryFileLike | null {
|
||||
return this.files.has(path) ? { path } : null;
|
||||
}
|
||||
getFolderByPath(path: string): BinaryFileLike | null {
|
||||
return this.folders.has(path) ? { path } : null;
|
||||
}
|
||||
createFolder(path: string): Promise<unknown> {
|
||||
this.folders.add(path);
|
||||
this.createdFolders.push(path);
|
||||
return Promise.resolve({ path });
|
||||
}
|
||||
createBinary(path: string, data: ArrayBuffer): Promise<BinaryFileLike> {
|
||||
this.files.add(path);
|
||||
this.binaries.push({ path, size: data.byteLength });
|
||||
return Promise.resolve({ path });
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Tests
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
describe("MediaDownloader.downloadToVault", () => {
|
||||
let errorSpy: jest.SpyInstance;
|
||||
beforeEach(() => {
|
||||
errorSpy = jest.spyOn(console, "error").mockImplementation(() => {});
|
||||
});
|
||||
afterEach(() => {
|
||||
errorSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("writes the media binary at the folder/<title>.<ext> path and returns it", async () => {
|
||||
const url = "https://media.example.com/ep/one.mp3";
|
||||
const { fetcher, requested } = makeFetcher({ [url]: { body: bytes("AUDIO") } });
|
||||
const vault = new FakeVault();
|
||||
const dl = new MediaDownloader({ fetcher, vault });
|
||||
|
||||
const item = makeItem({ title: "Episode One", mediaUrl: url, mediaType: "audio/mpeg" });
|
||||
const out = await dl.downloadToVault(item, "Feeds/Podcast/media");
|
||||
|
||||
expect(requested).toEqual([url]);
|
||||
expect(vault.binaries).toHaveLength(1);
|
||||
expect(vault.binaries[0]?.path).toBe("Feeds/Podcast/media/Episode-One.mp3");
|
||||
expect(vault.binaries[0]?.size).toBe("AUDIO".length);
|
||||
expect(out).toBe("Feeds/Podcast/media/Episode-One.mp3");
|
||||
// Ancestors created.
|
||||
expect(vault.createdFolders).toEqual(["Feeds", "Feeds/Podcast", "Feeds/Podcast/media"]);
|
||||
});
|
||||
|
||||
it("derives the extension from the URL path when present", async () => {
|
||||
const url = "https://media.example.com/ep/one.m4a";
|
||||
const { fetcher } = makeFetcher({ [url]: { body: bytes("A") } });
|
||||
const vault = new FakeVault();
|
||||
const dl = new MediaDownloader({ fetcher, vault });
|
||||
|
||||
const item = makeItem({ title: "Show", mediaUrl: url, mediaType: null });
|
||||
await dl.downloadToVault(item, "media");
|
||||
|
||||
expect(vault.binaries[0]?.path).toBe("media/Show.m4a");
|
||||
});
|
||||
|
||||
it("derives the extension from the content-type when the URL has none", async () => {
|
||||
const url = "https://media.example.com/stream/123";
|
||||
const { fetcher } = makeFetcher({
|
||||
[url]: { headers: { "Content-Type": "audio/mp4" }, body: bytes("A") },
|
||||
});
|
||||
const vault = new FakeVault();
|
||||
const dl = new MediaDownloader({ fetcher, vault });
|
||||
|
||||
const item = makeItem({ title: "Show", mediaUrl: url, mediaType: "audio/mp4" });
|
||||
await dl.downloadToVault(item, "media");
|
||||
|
||||
expect(vault.binaries[0]?.path).toBe("media/Show.m4a");
|
||||
});
|
||||
|
||||
it("falls back to a bin extension when neither URL nor content-type gives one", async () => {
|
||||
const url = "https://media.example.com/stream/123";
|
||||
const { fetcher } = makeFetcher({ [url]: { body: bytes("A") } });
|
||||
const vault = new FakeVault();
|
||||
const dl = new MediaDownloader({ fetcher, vault });
|
||||
|
||||
const item = makeItem({ title: "Show", mediaUrl: url, mediaType: null });
|
||||
await dl.downloadToVault(item, "media");
|
||||
|
||||
expect(vault.binaries[0]?.path).toBe("media/Show.bin");
|
||||
});
|
||||
|
||||
it("dedupes a collision by appending a numeric suffix before the extension", async () => {
|
||||
const url = "https://media.example.com/ep/one.mp3";
|
||||
const { fetcher } = makeFetcher({ [url]: { body: bytes("A") } });
|
||||
const vault = new FakeVault();
|
||||
// A file already exists at the natural target path.
|
||||
vault.files.add("media/Show.mp3");
|
||||
const dl = new MediaDownloader({ fetcher, vault });
|
||||
|
||||
const item = makeItem({ title: "Show", mediaUrl: url, mediaType: "audio/mpeg" });
|
||||
const out = await dl.downloadToVault(item, "media");
|
||||
|
||||
expect(out).toBe("media/Show-1.mp3");
|
||||
expect(vault.binaries[0]?.path).toBe("media/Show-1.mp3");
|
||||
});
|
||||
|
||||
it("returns null on a non-2xx status and writes nothing", async () => {
|
||||
const url = "https://media.example.com/ep/missing.mp3";
|
||||
const { fetcher } = makeFetcher({ [url]: { status: 404, body: bytes("nope") } });
|
||||
const vault = new FakeVault();
|
||||
const dl = new MediaDownloader({ fetcher, vault });
|
||||
|
||||
const item = makeItem({ mediaUrl: url });
|
||||
const out = await dl.downloadToVault(item, "media");
|
||||
|
||||
expect(out).toBeNull();
|
||||
expect(vault.binaries).toHaveLength(0);
|
||||
expect(errorSpy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns null on an empty response body and writes nothing", async () => {
|
||||
const url = "https://media.example.com/ep/empty.mp3";
|
||||
const { fetcher } = makeFetcher({ [url]: { body: new ArrayBuffer(0) } });
|
||||
const vault = new FakeVault();
|
||||
const dl = new MediaDownloader({ fetcher, vault });
|
||||
|
||||
const item = makeItem({ mediaUrl: url });
|
||||
const out = await dl.downloadToVault(item, "media");
|
||||
|
||||
expect(out).toBeNull();
|
||||
expect(vault.binaries).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("returns null on a transport error without throwing", async () => {
|
||||
const url = "https://media.example.com/ep/boom.mp3";
|
||||
const { fetcher } = makeFetcher({ [url]: { throws: new Error("connection reset") } });
|
||||
const vault = new FakeVault();
|
||||
const dl = new MediaDownloader({ fetcher, vault });
|
||||
|
||||
const item = makeItem({ mediaUrl: url });
|
||||
const out = await dl.downloadToVault(item, "media");
|
||||
|
||||
expect(out).toBeNull();
|
||||
expect(errorSpy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns null without fetching when the item has no media url", async () => {
|
||||
const { fetcher, requested } = makeFetcher({});
|
||||
const vault = new FakeVault();
|
||||
const dl = new MediaDownloader({ fetcher, vault });
|
||||
|
||||
const out = await dl.downloadToVault(makeItem({ mediaUrl: null }), "media");
|
||||
|
||||
expect(out).toBeNull();
|
||||
expect(requested).toHaveLength(0);
|
||||
expect(vault.binaries).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("returns null without fetching when the item has an empty media url", async () => {
|
||||
const { fetcher, requested } = makeFetcher({});
|
||||
const vault = new FakeVault();
|
||||
const dl = new MediaDownloader({ fetcher, vault });
|
||||
|
||||
const out = await dl.downloadToVault(makeItem({ mediaUrl: "" }), "media");
|
||||
|
||||
expect(out).toBeNull();
|
||||
expect(requested).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("MediaDownloader.downloadToOutside", () => {
|
||||
let errorSpy: jest.SpyInstance;
|
||||
beforeEach(() => {
|
||||
errorSpy = jest.spyOn(console, "error").mockImplementation(() => {});
|
||||
});
|
||||
afterEach(() => {
|
||||
errorSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("writes via the injected fileWriter and returns the absolute path", async () => {
|
||||
const url = "https://media.example.com/ep/one.mp3";
|
||||
const { fetcher } = makeFetcher({ [url]: { body: bytes("AUDIO") } });
|
||||
const vault = new FakeVault();
|
||||
const written: Array<{ path: string; size: number }> = [];
|
||||
const fileWriter = (absPath: string, data: ArrayBuffer): void => {
|
||||
written.push({ path: absPath, size: data.byteLength });
|
||||
};
|
||||
const dl = new MediaDownloader({ fetcher, vault, fileWriter });
|
||||
|
||||
const item = makeItem({ title: "Episode One", mediaUrl: url, mediaType: "audio/mpeg" });
|
||||
const out = await dl.downloadToOutside(item, "/Users/me/Podcasts");
|
||||
|
||||
expect(out).toBe("/Users/me/Podcasts/Episode-One.mp3");
|
||||
expect(written).toHaveLength(1);
|
||||
expect(written[0]?.path).toBe("/Users/me/Podcasts/Episode-One.mp3");
|
||||
expect(written[0]?.size).toBe("AUDIO".length);
|
||||
// The vault is never touched for an outside write.
|
||||
expect(vault.binaries).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("returns null when no outside folder is configured", async () => {
|
||||
const url = "https://media.example.com/ep/one.mp3";
|
||||
const { fetcher, requested } = makeFetcher({ [url]: { body: bytes("A") } });
|
||||
const vault = new FakeVault();
|
||||
const fileWriter = (): void => {};
|
||||
const dl = new MediaDownloader({ fetcher, vault, fileWriter });
|
||||
|
||||
const out = await dl.downloadToOutside(makeItem({ mediaUrl: url }), "");
|
||||
|
||||
expect(out).toBeNull();
|
||||
expect(requested).toHaveLength(0);
|
||||
expect(errorSpy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns null without fetching when the item has no media url", async () => {
|
||||
const { fetcher, requested } = makeFetcher({});
|
||||
const vault = new FakeVault();
|
||||
const fileWriter = (): void => {};
|
||||
const dl = new MediaDownloader({ fetcher, vault, fileWriter });
|
||||
|
||||
const out = await dl.downloadToOutside(makeItem({ mediaUrl: null }), "/Users/me/Podcasts");
|
||||
|
||||
expect(out).toBeNull();
|
||||
expect(requested).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("MediaDownloader.download", () => {
|
||||
let errorSpy: jest.SpyInstance;
|
||||
beforeEach(() => {
|
||||
errorSpy = jest.spyOn(console, "error").mockImplementation(() => {});
|
||||
});
|
||||
afterEach(() => {
|
||||
errorSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("dispatches to the vault path for location 'vault'", async () => {
|
||||
const url = "https://media.example.com/ep/one.mp3";
|
||||
const { fetcher } = makeFetcher({ [url]: { body: bytes("A") } });
|
||||
const vault = new FakeVault();
|
||||
const dl = new MediaDownloader({ fetcher, vault });
|
||||
|
||||
const item = makeItem({ title: "Show", mediaUrl: url, mediaType: "audio/mpeg" });
|
||||
const out = await dl.download(item, {
|
||||
location: "vault",
|
||||
vaultFolder: "Feeds/media",
|
||||
outsideFolder: "/abs",
|
||||
});
|
||||
|
||||
expect(out).toBe("Feeds/media/Show.mp3");
|
||||
expect(vault.binaries).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("dispatches to the outside path for location 'outside'", async () => {
|
||||
const url = "https://media.example.com/ep/one.mp3";
|
||||
const { fetcher } = makeFetcher({ [url]: { body: bytes("A") } });
|
||||
const vault = new FakeVault();
|
||||
const written: string[] = [];
|
||||
const fileWriter = (absPath: string): void => {
|
||||
written.push(absPath);
|
||||
};
|
||||
const dl = new MediaDownloader({ fetcher, vault, fileWriter });
|
||||
|
||||
const item = makeItem({ title: "Show", mediaUrl: url, mediaType: "audio/mpeg" });
|
||||
const out = await dl.download(item, {
|
||||
location: "outside",
|
||||
vaultFolder: "Feeds/media",
|
||||
outsideFolder: "/abs/podcasts",
|
||||
});
|
||||
|
||||
expect(out).toBe("/abs/podcasts/Show.mp3");
|
||||
expect(written).toEqual(["/abs/podcasts/Show.mp3"]);
|
||||
expect(vault.binaries).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
|
@ -276,6 +276,42 @@ describe("composeNote", () => {
|
|||
expect(note).not.toContain("[Episode audio]");
|
||||
expect(note).not.toContain("[Media]");
|
||||
});
|
||||
|
||||
it("records media-file and links the local file when mediaFile is set", () => {
|
||||
const mediaUrl = "https://media.example.com/ep/42.mp3";
|
||||
const localPath = "Feeds/Podcast/media/Episode-42.mp3";
|
||||
const item = makeItem({
|
||||
kind: "podcast",
|
||||
mediaUrl,
|
||||
mediaType: "audio/mpeg",
|
||||
});
|
||||
const note = composeNote(item, "BODY", { mediaFile: localPath });
|
||||
// Remote url is kept in frontmatter for reference.
|
||||
expect(note).toContain(`media-url: "${mediaUrl}"`);
|
||||
// Local file path recorded (force-quoted) after media-url.
|
||||
expect(note).toContain(`media-file: "${localPath}"`);
|
||||
const urlIndex = note.indexOf("media-url:");
|
||||
const fileIndex = note.indexOf("media-file:");
|
||||
expect(urlIndex).toBeGreaterThanOrEqual(0);
|
||||
expect(fileIndex).toBeGreaterThan(urlIndex);
|
||||
// Body links the LOCAL file, not the remote url.
|
||||
expect(note).toContain(`[Episode audio](${localPath})`);
|
||||
expect(note).not.toContain(`[Episode audio](${mediaUrl})`);
|
||||
});
|
||||
|
||||
it("labels a non-podcast local media link as Media", () => {
|
||||
const mediaUrl = "https://media.example.com/clip.mp4";
|
||||
const localPath = "Feeds/media/clip.mp4";
|
||||
const item = makeItem({
|
||||
kind: "article",
|
||||
mediaUrl,
|
||||
mediaType: "video/mp4",
|
||||
});
|
||||
const note = composeNote(item, "BODY", { mediaFile: localPath });
|
||||
expect(note).toContain(`media-file: "${localPath}"`);
|
||||
expect(note).toContain(`[Media](${localPath})`);
|
||||
expect(note).not.toContain(`[Media](${mediaUrl})`);
|
||||
});
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -65,6 +65,13 @@ export interface ImportRunnerDeps {
|
|||
convert: (html: string) => string;
|
||||
/** Optional best-effort image download + markdown rewrite. */
|
||||
processImages?: (markdown: string, item: FeedItem) => Promise<string>;
|
||||
/**
|
||||
* Optional best-effort media (podcast audio/video enclosure) download. Returns
|
||||
* the local path to thread into the note as composeOptions.mediaFile, or null
|
||||
* when there is no media or the download failed. A failure must NOT fail the
|
||||
* item: the runner catches, logs, and writes the note without a local file.
|
||||
*/
|
||||
downloadMedia?: (item: FeedItem) => Promise<string | null>;
|
||||
/** Optional debug logger; defaults to a no-op when absent. */
|
||||
debugLogger?: DebugLogger;
|
||||
}
|
||||
|
|
@ -96,6 +103,7 @@ export class ImportRunner {
|
|||
markdown: string,
|
||||
item: FeedItem,
|
||||
) => Promise<string>;
|
||||
private readonly downloadMedia?: (item: FeedItem) => Promise<string | null>;
|
||||
private readonly debug?: DebugLogger;
|
||||
|
||||
constructor(deps: ImportRunnerDeps) {
|
||||
|
|
@ -103,6 +111,7 @@ export class ImportRunner {
|
|||
this.noteWriter = deps.noteWriter;
|
||||
this.convert = deps.convert;
|
||||
this.processImages = deps.processImages;
|
||||
this.downloadMedia = deps.downloadMedia;
|
||||
this.debug = deps.debugLogger;
|
||||
}
|
||||
|
||||
|
|
@ -201,8 +210,29 @@ export class ImportRunner {
|
|||
});
|
||||
}
|
||||
|
||||
// Best-effort media download. A failure must NOT fail the item: catch,
|
||||
// log, and write the note without a local file (the remote media-url
|
||||
// stays in frontmatter and as the body link).
|
||||
let mediaFile: string | undefined;
|
||||
if (this.downloadMedia !== undefined && full.mediaUrl !== null && full.mediaUrl.length > 0) {
|
||||
try {
|
||||
const local = await this.downloadMedia(full);
|
||||
if (local !== null) {
|
||||
mediaFile = local;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
this.debug?.log({
|
||||
kind: "error",
|
||||
message: `Media download failed for "${full.title}", keeping the remote link`,
|
||||
payload: describeError(err),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const outcome = await this.noteWriter.writeNote(full, markdown, {
|
||||
feedTags,
|
||||
mediaFile,
|
||||
});
|
||||
this.debug?.log({
|
||||
kind: "note",
|
||||
|
|
|
|||
42
main.ts
42
main.ts
|
|
@ -8,8 +8,12 @@ import {
|
|||
import type { FeedSource, HttpFetcher, HttpRequest, HttpResponse, SourceType } from "./feed-source";
|
||||
import {
|
||||
DEFAULT_SETTINGS,
|
||||
effectiveDownloadMedia,
|
||||
effectiveImageSubfolder,
|
||||
effectiveImagesMode,
|
||||
effectiveMediaLocation,
|
||||
effectiveMediaOutsideFolder,
|
||||
effectiveMediaSubfolder,
|
||||
effectiveNoteNameTemplate,
|
||||
type FeedConfig,
|
||||
type RssImporterSettings,
|
||||
|
|
@ -24,6 +28,7 @@ import {
|
|||
} from "./note-writer";
|
||||
import { convertHtmlToMarkdown } from "./html-converter";
|
||||
import { ImageDownloader } from "./image-downloader";
|
||||
import { MediaDownloader } from "./media-downloader";
|
||||
import { DismissStore } from "./dismiss-store";
|
||||
import { BufferedDebugLogger } from "./debug-logger";
|
||||
import { ImportRunner } from "./import-runner";
|
||||
|
|
@ -91,6 +96,10 @@ function isDuplicatePolicy(value: unknown): value is import("./settings").Duplic
|
|||
return value === "skip" || value === "overwrite" || value === "prompt";
|
||||
}
|
||||
|
||||
function isMediaLocation(value: unknown): value is import("./settings").MediaLocation {
|
||||
return value === "vault" || value === "outside";
|
||||
}
|
||||
|
||||
// Sync classification used only to pick which source resolves a freshly entered
|
||||
// input for the add-feed preview. The resolved feed's own sourceType (captured
|
||||
// into the FeedConfig) is what drives import, so a custom-domain Substack that
|
||||
|
|
@ -194,6 +203,9 @@ export default class RssImporterPlugin extends Plugin implements RssImporterPlug
|
|||
if (!isDuplicatePolicy(this.settings.duplicatePolicy)) {
|
||||
this.settings.duplicatePolicy = DEFAULT_SETTINGS.duplicatePolicy;
|
||||
}
|
||||
if (!isMediaLocation(this.settings.mediaLocation)) {
|
||||
this.settings.mediaLocation = DEFAULT_SETTINGS.mediaLocation;
|
||||
}
|
||||
for (const feed of this.settings.feeds) {
|
||||
// The per-feed override fields are optional literal/string unions on
|
||||
// FeedConfig, but data.json is user-editable so a stored value can be a
|
||||
|
|
@ -209,6 +221,18 @@ export default class RssImporterPlugin extends Plugin implements RssImporterPlug
|
|||
if ("imageSubfolder" in record && typeof record["imageSubfolder"] !== "string") {
|
||||
delete record["imageSubfolder"];
|
||||
}
|
||||
if ("downloadMedia" in record && typeof record["downloadMedia"] !== "boolean") {
|
||||
delete record["downloadMedia"];
|
||||
}
|
||||
if ("mediaLocation" in record && !isMediaLocation(record["mediaLocation"])) {
|
||||
delete record["mediaLocation"];
|
||||
}
|
||||
if ("mediaSubfolder" in record && typeof record["mediaSubfolder"] !== "string") {
|
||||
delete record["mediaSubfolder"];
|
||||
}
|
||||
if ("mediaOutsideFolder" in record && typeof record["mediaOutsideFolder"] !== "string") {
|
||||
delete record["mediaOutsideFolder"];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -297,11 +321,29 @@ export default class RssImporterPlugin extends Plugin implements RssImporterPlug
|
|||
processImages = (markdown) => downloader.downloadAndRewrite(markdown, folderPath);
|
||||
}
|
||||
|
||||
let downloadMedia: ((item: import("./feed-source").FeedItem) => Promise<string | null>) | undefined;
|
||||
if (effectiveDownloadMedia(feed, this.settings)) {
|
||||
const mediaDownloader = new MediaDownloader({
|
||||
fetcher: this.makeFetcher(),
|
||||
vault: this.app.vault,
|
||||
});
|
||||
const location = effectiveMediaLocation(feed, this.settings);
|
||||
const mediaSubfolder = effectiveMediaSubfolder(feed, this.settings);
|
||||
const vaultFolder =
|
||||
feed.destinationFolder === ""
|
||||
? mediaSubfolder
|
||||
: `${feed.destinationFolder}/${mediaSubfolder}`;
|
||||
const outsideFolder = effectiveMediaOutsideFolder(feed, this.settings);
|
||||
downloadMedia = (item) =>
|
||||
mediaDownloader.download(item, { location, vaultFolder, outsideFolder });
|
||||
}
|
||||
|
||||
runner = new ImportRunner({
|
||||
source,
|
||||
noteWriter,
|
||||
convert: convertHtmlToMarkdown,
|
||||
processImages,
|
||||
downloadMedia,
|
||||
debugLogger: this.debugLogger,
|
||||
});
|
||||
} catch (err) {
|
||||
|
|
|
|||
431
media-downloader.ts
Normal file
431
media-downloader.ts
Normal file
|
|
@ -0,0 +1,431 @@
|
|||
// Downloads a feed item's media enclosure (podcast audio, attached audio/video)
|
||||
// either into the vault or to an absolute filesystem path outside the vault.
|
||||
//
|
||||
// This mirrors image-downloader's robustness contract: best effort, never throw
|
||||
// out of the import. A media failure (transport error, non-2xx, empty body,
|
||||
// write error, non-desktop for the outside path) logs a one-line reason and
|
||||
// returns null so the note is still written with the remote media-url intact.
|
||||
//
|
||||
// The vault path reuses image-downloader's VaultBinaryLike surface so Obsidian's
|
||||
// Vault satisfies it structurally and tests inject a plain object. The outside
|
||||
// path uses Node fs/path, which on a marketplace plugin must be require()'d
|
||||
// behind a Platform.isDesktop guard (see CLAUDE.md). To keep the module unit
|
||||
// testable without touching the real filesystem, the constructor accepts an
|
||||
// optional fileWriter the outside path prefers over the Node fallback.
|
||||
|
||||
import { Platform } from "obsidian";
|
||||
import type { FeedItem, HttpFetcher } from "./feed-source";
|
||||
import type { BinaryFileLike, VaultBinaryLike } from "./image-downloader";
|
||||
|
||||
export type { BinaryFileLike, VaultBinaryLike };
|
||||
|
||||
/** Where the media for one item should be written. */
|
||||
export type MediaLocation = "vault" | "outside";
|
||||
|
||||
/**
|
||||
* Writes raw bytes to an absolute filesystem path. Injected for tests so the
|
||||
* outside-vault path can be exercised without touching the real disk; when
|
||||
* absent, downloadToOutside falls back to Node fs behind a desktop guard.
|
||||
*/
|
||||
export type FileWriter = (absPath: string, data: ArrayBuffer) => void;
|
||||
|
||||
export interface MediaDownloaderOptions {
|
||||
readonly fetcher: HttpFetcher;
|
||||
readonly vault: VaultBinaryLike;
|
||||
readonly fileWriter?: FileWriter;
|
||||
}
|
||||
|
||||
export interface MediaDownloadOptions {
|
||||
readonly location: MediaLocation;
|
||||
/** Vault-relative folder for the "vault" location. */
|
||||
readonly vaultFolder: string;
|
||||
/** Absolute filesystem folder for the "outside" location. */
|
||||
readonly outsideFolder: string;
|
||||
}
|
||||
|
||||
// Map common enclosure content-types to a file extension. Audio and video MIME
|
||||
// types the URL path may not carry. Anything unmapped falls back to the URL
|
||||
// extension, then to a generic default.
|
||||
const CONTENT_TYPE_EXT: Record<string, string> = {
|
||||
"audio/mpeg": "mp3",
|
||||
"audio/mp3": "mp3",
|
||||
"audio/mp4": "m4a",
|
||||
"audio/x-m4a": "m4a",
|
||||
"audio/aac": "aac",
|
||||
"audio/ogg": "ogg",
|
||||
"audio/opus": "opus",
|
||||
"audio/wav": "wav",
|
||||
"audio/x-wav": "wav",
|
||||
"audio/flac": "flac",
|
||||
"video/mp4": "mp4",
|
||||
"video/webm": "webm",
|
||||
"video/quicktime": "mov",
|
||||
"video/x-matroska": "mkv",
|
||||
};
|
||||
|
||||
// Extensions accepted from a URL path. Anything else (or none) falls back to the
|
||||
// content-type, then to a generic default.
|
||||
const KNOWN_MEDIA_EXTS = new Set([
|
||||
"mp3", "m4a", "aac", "ogg", "opus", "wav", "flac",
|
||||
"mp4", "webm", "mov", "mkv", "m4v", "oga",
|
||||
]);
|
||||
|
||||
const DEFAULT_EXT = "bin";
|
||||
|
||||
export class MediaDownloader {
|
||||
private readonly fetcher: HttpFetcher;
|
||||
private readonly vault: VaultBinaryLike;
|
||||
private readonly fileWriter?: FileWriter;
|
||||
|
||||
constructor(opts: MediaDownloaderOptions) {
|
||||
this.fetcher = opts.fetcher;
|
||||
this.vault = opts.vault;
|
||||
this.fileWriter = opts.fileWriter;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch by location. Returns the written path (vault-relative or absolute)
|
||||
* on success, or null when there is no media or the download/write failed.
|
||||
*/
|
||||
async download(item: FeedItem, opts: MediaDownloadOptions): Promise<string | null> {
|
||||
if (opts.location === "outside") {
|
||||
return this.downloadToOutside(item, opts.outsideFolder);
|
||||
}
|
||||
return this.downloadToVault(item, opts.vaultFolder);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the item's media and write it under `folderPath` in the vault. Returns
|
||||
* the vault-relative path on success. Returns null (never throws) when there
|
||||
* is no media url, the fetch is non-2xx or empty, or any write step fails.
|
||||
*/
|
||||
async downloadToVault(item: FeedItem, folderPath: string): Promise<string | null> {
|
||||
const mediaUrl = item.mediaUrl;
|
||||
if (mediaUrl === null || mediaUrl.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const fetched = await this.fetchMedia(mediaUrl);
|
||||
if (fetched === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const folder = normalizeFolderPath(folderPath);
|
||||
try {
|
||||
await this.ensureFolder(folder);
|
||||
} catch (err) {
|
||||
console.error(`RSS Importer: media download skipped for ${mediaUrl}: ${describe(err)}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const fileName = deriveFileName(item, mediaUrl, fetched.contentType);
|
||||
const targetPath = folder === "" ? fileName : `${folder}/${fileName}`;
|
||||
const uniquePath = await this.uniqueVaultPath(targetPath);
|
||||
|
||||
try {
|
||||
await this.vault.createBinary(uniquePath, fetched.bytes);
|
||||
} catch (err) {
|
||||
console.error(`RSS Importer: media write failed for ${mediaUrl}: ${describe(err)}`);
|
||||
return null;
|
||||
}
|
||||
return uniquePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the item's media and write it to an absolute path under `absFolder`.
|
||||
* Desktop only. Uses the injected fileWriter when present (tests); otherwise
|
||||
* Node fs/path behind a Platform.isDesktop guard. Returns the absolute path on
|
||||
* success, or null (never throws) on non-desktop, no media, or any failure.
|
||||
*/
|
||||
async downloadToOutside(item: FeedItem, absFolder: string): Promise<string | null> {
|
||||
const mediaUrl = item.mediaUrl;
|
||||
if (mediaUrl === null || mediaUrl.length === 0) {
|
||||
return null;
|
||||
}
|
||||
if (this.fileWriter === undefined && !Platform.isDesktop) {
|
||||
console.error(`RSS Importer: media download skipped for ${mediaUrl}: outside-vault writes need a desktop app`);
|
||||
return null;
|
||||
}
|
||||
if (absFolder.length === 0) {
|
||||
console.error(`RSS Importer: media download skipped for ${mediaUrl}: no outside folder configured`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const fetched = await this.fetchMedia(mediaUrl);
|
||||
if (fetched === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const fileName = deriveFileName(item, mediaUrl, fetched.contentType);
|
||||
|
||||
try {
|
||||
return this.writeOutside(absFolder, fileName, fetched.bytes);
|
||||
} catch (err) {
|
||||
console.error(`RSS Importer: media write failed for ${mediaUrl}: ${describe(err)}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write bytes to absFolder/fileName, preferring the injected fileWriter. When
|
||||
* none is injected, use Node fs/path behind a desktop guard: require() rather
|
||||
* than a top-level import keeps the bundle clean of Node built-ins on mobile,
|
||||
* per the marketplace scorecard. Returns the absolute path written.
|
||||
*/
|
||||
private writeOutside(absFolder: string, fileName: string, bytes: ArrayBuffer): string {
|
||||
if (this.fileWriter !== undefined) {
|
||||
const joined = joinPosix(absFolder, fileName);
|
||||
this.fileWriter(joined, bytes);
|
||||
return joined;
|
||||
}
|
||||
const { fs, path } = getNodeFsPath();
|
||||
fs.mkdirSync(absFolder, { recursive: true });
|
||||
const target = path.join(absFolder, fileName);
|
||||
fs.writeFileSync(target, Buffer.from(bytes));
|
||||
return target;
|
||||
}
|
||||
|
||||
/**
|
||||
* GET the media url and validate the response. Returns the bytes and the
|
||||
* content-type on success, or null (logged) on non-2xx, empty body, or a
|
||||
* transport error.
|
||||
*/
|
||||
private async fetchMedia(
|
||||
url: string,
|
||||
): Promise<{ bytes: ArrayBuffer; contentType: string } | null> {
|
||||
let response;
|
||||
try {
|
||||
response = await this.fetcher({ url, method: "GET" });
|
||||
} catch (err) {
|
||||
console.error(`RSS Importer: media download failed for ${url}: ${describe(err)}`);
|
||||
return null;
|
||||
}
|
||||
if (response.status < 200 || response.status >= 300) {
|
||||
console.error(`RSS Importer: media download skipped for ${url}: status ${response.status}`);
|
||||
return null;
|
||||
}
|
||||
const bytes = response.arrayBuffer;
|
||||
if (bytes.byteLength === 0) {
|
||||
console.error(`RSS Importer: media download skipped for ${url}: empty body`);
|
||||
return null;
|
||||
}
|
||||
return { bytes, contentType: headerValue(response.headers, "content-type") };
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the folder and each missing ancestor. createFolder throws when the
|
||||
* folder already exists, so each segment is checked first.
|
||||
*/
|
||||
private async ensureFolder(folder: string): Promise<void> {
|
||||
if (folder === "") {
|
||||
return;
|
||||
}
|
||||
const segments = folder.split("/");
|
||||
for (let i = 1; i <= segments.length; i++) {
|
||||
const partial = segments.slice(0, i).join("/");
|
||||
if (this.vault.getFolderByPath(partial) === null) {
|
||||
await this.vault.createFolder(partial);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a non-colliding vault path. If the target already exists, append a
|
||||
* numeric suffix before the extension (name-1.ext, name-2.ext, ...).
|
||||
*/
|
||||
private async uniqueVaultPath(targetPath: string): Promise<string> {
|
||||
if (this.vault.getFileByPath(targetPath) === null) {
|
||||
return targetPath;
|
||||
}
|
||||
const { dir, base, ext } = splitPath(targetPath);
|
||||
for (let n = 1; ; n++) {
|
||||
const candidate = dir === "" ? `${base}-${n}${ext}` : `${dir}/${base}-${n}${ext}`;
|
||||
if (this.vault.getFileByPath(candidate) === null) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Pure helpers (kept module-private; exercised through the public methods).
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Resolve Node's fs and path modules. Obsidian's guidelines require Node
|
||||
* built-ins to be loaded via a Platform.isDesktop-guarded require() so the
|
||||
* mobile bundle never references them; callers must gate on Platform.isDesktop
|
||||
* before reaching here. The require, the unsafe cast, and the node-module import
|
||||
* are the unavoidable consequences of that guidance.
|
||||
*/
|
||||
function getNodeFsPath(): { fs: typeof import("fs"); path: typeof import("path") } {
|
||||
if (!Platform.isDesktop) {
|
||||
throw new Error("Node fs/path modules are not available on this platform.");
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports -- Obsidian docs require a Platform.isDesktop-guarded require() for Node built-ins so mobile builds do not pull them in; the require is the unavoidable consequence of that guidance.
|
||||
const fs = require("fs") as typeof import("fs");
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports -- Obsidian docs require a Platform.isDesktop-guarded require() for Node built-ins so mobile builds do not pull them in; the require is the unavoidable consequence of that guidance.
|
||||
const path = require("path") as typeof import("path");
|
||||
return { fs, path };
|
||||
}
|
||||
|
||||
/** Read a header case-insensitively; returns "" when absent. */
|
||||
function headerValue(headers: Record<string, string>, name: string): string {
|
||||
const wanted = name.toLowerCase();
|
||||
for (const key of Object.keys(headers)) {
|
||||
if (key.toLowerCase() === wanted) {
|
||||
return headers[key] ?? "";
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/** Render a caught unknown into a one-line message. */
|
||||
function describe(err: unknown): string {
|
||||
return err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive the media filename: a sanitized item title (preferred) or the URL
|
||||
* basename, plus an extension from the content-type, then the URL, then a
|
||||
* generic default.
|
||||
*/
|
||||
function deriveFileName(item: FeedItem, url: string, contentType: string): string {
|
||||
const ext = deriveExtension(url, contentType);
|
||||
const base = deriveBaseName(item, url);
|
||||
return `${base}.${ext}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive a file extension. Prefer a known media extension from the URL path;
|
||||
* otherwise map the content-type; otherwise fall back to a generic default so a
|
||||
* file always gets a usable extension.
|
||||
*/
|
||||
function deriveExtension(url: string, contentType: string): string {
|
||||
const fromUrl = extFromUrl(url);
|
||||
if (fromUrl !== null) {
|
||||
return fromUrl;
|
||||
}
|
||||
const ct = contentType.split(";")[0]?.trim().toLowerCase() ?? "";
|
||||
const mapped = CONTENT_TYPE_EXT[ct];
|
||||
if (mapped !== undefined) {
|
||||
return mapped;
|
||||
}
|
||||
return DEFAULT_EXT;
|
||||
}
|
||||
|
||||
/** Extract a known media extension from a URL path, or null. */
|
||||
function extFromUrl(url: string): string | null {
|
||||
const path = urlPath(url);
|
||||
const lastDot = path.lastIndexOf(".");
|
||||
if (lastDot === -1 || lastDot === path.length - 1) {
|
||||
return null;
|
||||
}
|
||||
const ext = path.slice(lastDot + 1).toLowerCase();
|
||||
return KNOWN_MEDIA_EXTS.has(ext) ? ext : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive a sanitized base filename (without extension). Prefer the item title;
|
||||
* fall back to the URL's last path segment; fall back to a short hash of the url
|
||||
* so two distinct extensionless urls do not collide on the same literal name.
|
||||
*/
|
||||
function deriveBaseName(item: FeedItem, url: string): string {
|
||||
const fromTitle = sanitizeBaseName(item.title);
|
||||
if (fromTitle.length > 0) {
|
||||
return fromTitle;
|
||||
}
|
||||
const path = urlPath(url);
|
||||
const segments = path.split("/").filter((s) => s.length > 0);
|
||||
const last = segments.length > 0 ? segments[segments.length - 1] : undefined;
|
||||
let base = last ?? "";
|
||||
const dot = base.lastIndexOf(".");
|
||||
if (dot > 0) {
|
||||
base = base.slice(0, dot);
|
||||
}
|
||||
base = sanitizeBaseName(base);
|
||||
if (base.length === 0) {
|
||||
base = `media-${shortHash(url)}`;
|
||||
}
|
||||
return base;
|
||||
}
|
||||
|
||||
/**
|
||||
* The path portion of a url, stripped of query and fragment, without parsing via
|
||||
* URL (not guaranteed everywhere this runs). Strips the scheme and authority so
|
||||
* only the path text remains.
|
||||
*/
|
||||
function urlPath(url: string): string {
|
||||
let rest = url.replace(/^https?:\/\//i, "");
|
||||
rest = rest.split("#")[0] ?? rest;
|
||||
rest = rest.split("?")[0] ?? rest;
|
||||
const slash = rest.indexOf("/");
|
||||
return slash === -1 ? "" : rest.slice(slash);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize a filename base: drop characters illegal on Windows/macOS/Linux and
|
||||
* Obsidian's wikilink brackets, collapse whitespace and dot/space runs, clamp
|
||||
* length. Returns "" when nothing usable remains so the caller can fall back.
|
||||
*/
|
||||
function sanitizeBaseName(name: string): string {
|
||||
let out = name.trim().replace(/\s+/g, "-");
|
||||
// eslint-disable-next-line no-control-regex -- strip NUL and other non-whitespace control codes from the filename
|
||||
out = out.replace(/[<>:"/\\|?*\x00-\x1f[\]]/g, "-");
|
||||
out = out.replace(/-+/g, "-");
|
||||
out = out.replace(/^[.\- ]+/, "").replace(/[.\- ]+$/, "");
|
||||
if (out.length > 100) {
|
||||
out = out.slice(0, 100).replace(/[.\- ]+$/, "");
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a path into its directory, base name (no extension), and extension
|
||||
* (including the leading dot, or empty). Path separator is the vault "/".
|
||||
*/
|
||||
function splitPath(p: string): { dir: string; base: string; ext: string } {
|
||||
const slash = p.lastIndexOf("/");
|
||||
const dir = slash === -1 ? "" : p.slice(0, slash);
|
||||
const file = slash === -1 ? p : p.slice(slash + 1);
|
||||
const dot = file.lastIndexOf(".");
|
||||
if (dot <= 0) {
|
||||
return { dir, base: file, ext: "" };
|
||||
}
|
||||
return { dir, base: file.slice(0, dot), ext: file.slice(dot) };
|
||||
}
|
||||
|
||||
/** Join a folder and a file name with a single forward slash. */
|
||||
function joinPosix(folder: string, fileName: string): string {
|
||||
const trimmed = folder.replace(/[/\\]+$/, "");
|
||||
return trimmed.length === 0 ? fileName : `${trimmed}/${fileName}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* A short, stable, filesystem-safe hash. Not cryptographic: it only needs to
|
||||
* keep two distinct extensionless urls from colliding on the same fallback name.
|
||||
* djb2 rendered base36.
|
||||
*/
|
||||
function shortHash(input: string): string {
|
||||
let hash = 5381;
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
hash = ((hash << 5) + hash + input.charCodeAt(i)) >>> 0;
|
||||
}
|
||||
return hash.toString(36);
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a vault folder path: trim, drop leading/trailing slashes, collapse
|
||||
* doubled slashes, drop `.` and `..` segments. `..` is dropped (not thrown on)
|
||||
* because this is an internal best-effort path, not a user-facing destination.
|
||||
*/
|
||||
function normalizeFolderPath(folder: string): string {
|
||||
const cleaned = folder
|
||||
.trim()
|
||||
.replace(/^\/+|\/+$/g, "")
|
||||
.replace(/\/{2,}/g, "/");
|
||||
const segments = cleaned
|
||||
.split("/")
|
||||
.filter((s) => s !== "" && s !== "." && s !== "..");
|
||||
return segments.join("/");
|
||||
}
|
||||
|
|
@ -132,6 +132,13 @@ export interface ComposeOptions {
|
|||
readonly feedTags?: string[];
|
||||
/** Frontmatter key the merged tags are written under. Defaults to feed-tags. */
|
||||
readonly tagDestination?: TagDestination;
|
||||
/**
|
||||
* Local path of the downloaded media file (vault-relative or absolute). When
|
||||
* set, a `media-file` frontmatter line is added and the body media link points
|
||||
* at this local file instead of the remote media-url. The remote media-url
|
||||
* frontmatter line is kept for reference.
|
||||
*/
|
||||
readonly mediaFile?: string;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
|
|
@ -413,7 +420,10 @@ const TRUNCATED_CALLOUT = [
|
|||
* When the item carries media (a podcast/audio/video enclosure), a `media-url`
|
||||
* frontmatter line records it (force-quoted, since it is a URL) and a labelled
|
||||
* link to the media is appended to the body so the episode is reachable from the
|
||||
* note.
|
||||
* note. When opts.mediaFile is set (the enclosure was downloaded locally), a
|
||||
* `media-file` frontmatter line is added after media-url and the body link points
|
||||
* at the local file instead of the remote URL; the media-url line is kept for
|
||||
* reference.
|
||||
*/
|
||||
export function composeNote(
|
||||
item: FeedItem,
|
||||
|
|
@ -445,9 +455,15 @@ export function composeNote(
|
|||
// in frontmatter. Force-quoted because it is a URL. Placed after tags and
|
||||
// before the truncated marker.
|
||||
const hasMedia = item.mediaUrl !== null && item.mediaUrl.length > 0;
|
||||
const hasMediaFile = opts.mediaFile !== undefined && opts.mediaFile.length > 0;
|
||||
if (hasMedia && item.mediaUrl !== null) {
|
||||
lines.push(`media-url: ${yamlQuoted(item.mediaUrl)}`);
|
||||
}
|
||||
// When the enclosure was downloaded, record the local file path right after
|
||||
// the remote url. Force-quoted because a path can contain spaces or colons.
|
||||
if (hasMediaFile && opts.mediaFile !== undefined) {
|
||||
lines.push(`media-file: ${yamlQuoted(opts.mediaFile)}`);
|
||||
}
|
||||
if (item.isTruncated) {
|
||||
// Machine-readable marker alongside the visible callout below.
|
||||
lines.push("substack-truncated: true");
|
||||
|
|
@ -459,8 +475,12 @@ export function composeNote(
|
|||
: bodyMarkdown;
|
||||
|
||||
// Link the media at the end of the body so a podcast note plays its episode
|
||||
// and any other media item is reachable from the note.
|
||||
if (hasMedia && item.mediaUrl !== null) {
|
||||
// and any other media item is reachable from the note. Prefer the downloaded
|
||||
// local file when present; otherwise fall back to the remote url.
|
||||
if (hasMediaFile && opts.mediaFile !== undefined) {
|
||||
const label = item.kind === "podcast" ? "Episode audio" : "Media";
|
||||
body = `${body}\n\n[${label}](${opts.mediaFile})`;
|
||||
} else if (hasMedia && item.mediaUrl !== null) {
|
||||
const label = item.kind === "podcast" ? "Episode audio" : "Media";
|
||||
body = `${body}\n\n[${label}](${item.mediaUrl})`;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,13 @@ import {
|
|||
} from "obsidian";
|
||||
import type { FeedSource } from "./feed-source";
|
||||
import type { FeedConfig, RssImporterSettings } from "./settings";
|
||||
import { REQUEST_DELAY_MIN, REQUEST_DELAY_MAX } from "./settings";
|
||||
import {
|
||||
REQUEST_DELAY_MIN,
|
||||
REQUEST_DELAY_MAX,
|
||||
effectiveDownloadMedia,
|
||||
effectiveMediaLocation,
|
||||
effectiveMediaSubfolder,
|
||||
} from "./settings";
|
||||
import { AddFeedModal } from "./add-feed-modal";
|
||||
|
||||
/**
|
||||
|
|
@ -112,6 +118,33 @@ export class RssImporterSettingTab extends PluginSettingTab {
|
|||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Download media",
|
||||
desc: "Save podcast audio and video enclosures locally, not just link them.",
|
||||
control: { type: "toggle", key: "downloadMedia" },
|
||||
},
|
||||
{
|
||||
name: "Media location",
|
||||
desc: "Save media into a vault subfolder, or to a folder outside the vault.",
|
||||
control: {
|
||||
type: "dropdown",
|
||||
key: "mediaLocation",
|
||||
options: {
|
||||
vault: "Vault subfolder",
|
||||
outside: "Outside the vault",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Media subfolder",
|
||||
desc: "Subfolder under each feed's folder for downloaded media.",
|
||||
control: { type: "text", key: "mediaSubfolder" },
|
||||
},
|
||||
{
|
||||
name: "Media folder outside vault",
|
||||
desc: "Absolute folder path for media when saving outside the vault (desktop only).",
|
||||
control: { type: "text", key: "mediaOutsideFolder" },
|
||||
},
|
||||
{
|
||||
name: "Tag destination",
|
||||
desc: "Save feed tags as a note property, or as Obsidian tags that show in the tag pane.",
|
||||
|
|
@ -259,6 +292,46 @@ class FeedEditorPage extends SettingPage {
|
|||
}),
|
||||
);
|
||||
|
||||
const settings = this.plugin.settings;
|
||||
|
||||
new Setting(editor)
|
||||
.setName("Download media")
|
||||
.setDesc("Save podcast audio and video enclosures locally for this feed.")
|
||||
.addToggle((toggle) =>
|
||||
toggle
|
||||
.setValue(effectiveDownloadMedia(feed, settings))
|
||||
.onChange(async (value) => {
|
||||
feed.downloadMedia = value;
|
||||
await this.plugin.saveSettings();
|
||||
}),
|
||||
);
|
||||
|
||||
new Setting(editor)
|
||||
.setName("Media location")
|
||||
.setDesc("Save media into a vault subfolder, or to a folder outside the vault.")
|
||||
.addDropdown((dropdown) =>
|
||||
dropdown
|
||||
.addOption("vault", "Vault subfolder")
|
||||
.addOption("outside", "Outside the vault")
|
||||
.setValue(effectiveMediaLocation(feed, settings))
|
||||
.onChange(async (value) => {
|
||||
feed.mediaLocation = value === "outside" ? "outside" : "vault";
|
||||
await this.plugin.saveSettings();
|
||||
}),
|
||||
);
|
||||
|
||||
new Setting(editor)
|
||||
.setName("Media subfolder")
|
||||
.setDesc("Subfolder under this feed's folder for downloaded media.")
|
||||
.addText((text) =>
|
||||
text
|
||||
.setValue(effectiveMediaSubfolder(feed, settings))
|
||||
.onChange(async (value) => {
|
||||
feed.mediaSubfolder = value;
|
||||
await this.plugin.saveSettings();
|
||||
}),
|
||||
);
|
||||
|
||||
new Setting(editor)
|
||||
.setName("Remove feed")
|
||||
.setDesc("Delete this feed. Imported notes are left in place.")
|
||||
|
|
|
|||
44
settings.ts
44
settings.ts
|
|
@ -14,6 +14,14 @@ import type { TagDestination } from "./note-writer";
|
|||
export type ImagesMode = "link" | "download";
|
||||
export type DuplicatePolicy = "skip" | "overwrite" | "prompt";
|
||||
|
||||
/**
|
||||
* Where downloaded media (podcast audio/video enclosures) is written. "vault"
|
||||
* writes into a subfolder of the feed's destination folder; "outside" writes to
|
||||
* an absolute filesystem path (desktop only) so large media stays out of the
|
||||
* synced vault.
|
||||
*/
|
||||
export type MediaLocation = "vault" | "outside";
|
||||
|
||||
/** A configured feed and the resolved metadata captured when it was added. */
|
||||
export interface FeedConfig {
|
||||
/** Stable id derived from the canonical host; also the note sourceId. */
|
||||
|
|
@ -41,6 +49,10 @@ export interface FeedConfig {
|
|||
imagesMode?: ImagesMode;
|
||||
imageSubfolder?: string;
|
||||
noteNameTemplate?: string;
|
||||
downloadMedia?: boolean;
|
||||
mediaLocation?: MediaLocation;
|
||||
mediaSubfolder?: string;
|
||||
mediaOutsideFolder?: string;
|
||||
}
|
||||
|
||||
export interface RssImporterSettings {
|
||||
|
|
@ -52,6 +64,14 @@ export interface RssImporterSettings {
|
|||
requestDelayMs: number;
|
||||
imagesMode: ImagesMode;
|
||||
imageSubfolder: string;
|
||||
/** Download podcast/audio/video enclosures, not just link them. */
|
||||
downloadMedia: boolean;
|
||||
/** Where downloaded media is written: a vault subfolder or an outside path. */
|
||||
mediaLocation: MediaLocation;
|
||||
/** Subfolder (under the feed's destination) for media when mediaLocation is "vault". */
|
||||
mediaSubfolder: string;
|
||||
/** Absolute filesystem folder for media when mediaLocation is "outside" (desktop only). */
|
||||
mediaOutsideFolder: string;
|
||||
/** Where feed tags are written: a plain "feed-tags" property or Obsidian "tags". */
|
||||
tagDestination: TagDestination;
|
||||
debug: boolean;
|
||||
|
|
@ -69,6 +89,10 @@ export const DEFAULT_SETTINGS: RssImporterSettings = {
|
|||
requestDelayMs: 1200,
|
||||
imagesMode: "link",
|
||||
imageSubfolder: "images",
|
||||
downloadMedia: false,
|
||||
mediaLocation: "vault",
|
||||
mediaSubfolder: "media",
|
||||
mediaOutsideFolder: "",
|
||||
tagDestination: "feed-tags",
|
||||
debug: false,
|
||||
showRibbonIcon: true,
|
||||
|
|
@ -94,3 +118,23 @@ export function effectiveNoteNameTemplate(feed: FeedConfig, settings: RssImporte
|
|||
export function effectiveImageSubfolder(feed: FeedConfig, settings: RssImporterSettings): string {
|
||||
return feed.imageSubfolder ?? settings.imageSubfolder;
|
||||
}
|
||||
|
||||
/** Resolves whether media is downloaded for a feed (per-feed override or default). */
|
||||
export function effectiveDownloadMedia(feed: FeedConfig, settings: RssImporterSettings): boolean {
|
||||
return feed.downloadMedia ?? settings.downloadMedia;
|
||||
}
|
||||
|
||||
/** Resolves the effective media location for a feed. */
|
||||
export function effectiveMediaLocation(feed: FeedConfig, settings: RssImporterSettings): MediaLocation {
|
||||
return feed.mediaLocation ?? settings.mediaLocation;
|
||||
}
|
||||
|
||||
/** Resolves the effective media subfolder name for a feed. */
|
||||
export function effectiveMediaSubfolder(feed: FeedConfig, settings: RssImporterSettings): string {
|
||||
return feed.mediaSubfolder ?? settings.mediaSubfolder;
|
||||
}
|
||||
|
||||
/** Resolves the effective outside (absolute) media folder for a feed. */
|
||||
export function effectiveMediaOutsideFolder(feed: FeedConfig, settings: RssImporterSettings): string {
|
||||
return feed.mediaOutsideFolder ?? settings.mediaOutsideFolder;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue