mirror of
https://github.com/ckelsoe/obsidian-rss-importer.git
synced 2026-07-22 07:48:56 +00:00
Phase 5: apply adversarial-review fixes
Correctness: resolver classifies /p/ as Substack only on a substack.com host; redirect cap aligned to 5; extractFeedItemId unescapes in a single pass (round trips literal backslash escapes). Feature: podcast/media notes now record media-url frontmatter and append an [Episode audio]/[Media] body link. Safety: the import command guards synchronous NoteWriter construction (a bad destination folder now surfaces a Notice, not a silent throw); loadSettings validates per-feed literal fields from data.json. Diagnosability: empty-body imports and skipped image downloads are logged. Structure: FeedPickerModal and DuplicatePromptModal moved to their own files; id non-empty guard at the mapping boundary; doc + type-dedup cleanups. 244 tests; build + lint green. Deferred (low/cosmetic): shared header helper, Atom updated-vs-published date precedence, the sanctioned settings-tab cast.
This commit is contained in:
parent
eec329d526
commit
9e563d8ffe
18 changed files with 413 additions and 153 deletions
13
CHANGELOG.md
13
CHANGELOG.md
|
|
@ -7,7 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.1.0] - YYYY-MM-DD
|
||||
## [0.1.0] - 2026-06-14
|
||||
|
||||
### Added
|
||||
- Initial release.
|
||||
- Initial release: import articles and podcasts from RSS, Atom, and Substack feeds into the vault as Markdown notes.
|
||||
- Multi-source support behind one feed contract. Substack publications are first-class: add a `@handle`, subdomain, custom domain, or post URL and the plugin resolves it to the feed. Any other RSS, Atom, or podcast feed imports as a generic source.
|
||||
- One destination folder per feed, with recursive deduplication by a stable `feed-item-id` stored in each note's frontmatter, so moving or renaming notes never causes a re-import.
|
||||
- Add-feed flow with live resolve and preview (publication title, host, recent item titles, source type, free or paid hint).
|
||||
- Import window with a three-state item list (imported, dismissed, available), reversible dismiss, live progress, and a per-item result summary that never aborts the run on a single bad item.
|
||||
- HTML to Markdown conversion with feed-specific rules (figure captions, code-fence language, footnotes, subscribe and share widget stripping) and deterministic output.
|
||||
- Images: link to the original URL (default) or download into the vault.
|
||||
- Podcast items import as a note from the show notes with a link to the episode media.
|
||||
- Sequential request pacing with 429 and Retry-After backoff, and a secret-free, exportable debug log.
|
||||
- Desktop only for this release.
|
||||
|
|
|
|||
|
|
@ -160,15 +160,35 @@ describe("FeedResolver: Substack subdomain and post URLs", () => {
|
|||
expect(result.handle).toBeNull();
|
||||
});
|
||||
|
||||
it("classifies a Substack post URL (/p/<slug>) as substack with host/feed", async () => {
|
||||
it("classifies a *.substack.com post URL (/p/<slug>) as substack with host/feed", async () => {
|
||||
const { fetcher } = makeFetcher({
|
||||
"https://the.lxxscrolls.com/feed": { status: 200, headers: { "Content-Type": "text/xml" } },
|
||||
"https://kevin.substack.com/feed": { status: 200, headers: { "Content-Type": "text/xml" } },
|
||||
});
|
||||
const resolver = new FeedResolver(fetcher);
|
||||
|
||||
const result = await resolver.resolve("https://kevin.substack.com/p/daniel-6");
|
||||
|
||||
expect(result.sourceType).toBe("substack");
|
||||
expect(result.canonicalHost).toBe("kevin.substack.com");
|
||||
expect(result.feedUrl).toBe("https://kevin.substack.com/feed");
|
||||
});
|
||||
|
||||
it("classifies a custom-domain post URL (/p/<slug>) as generic, not substack", async () => {
|
||||
// A /p/<slug> path on a non-substack.com host is NOT claimed as Substack:
|
||||
// custom-domain Substacks are added via @handle. Here the host's /feed
|
||||
// probe returns XML, so it resolves as a generic feed.
|
||||
const { fetcher } = makeFetcher({
|
||||
"https://the.lxxscrolls.com/feed": {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "text/xml" },
|
||||
text: "<?xml version=\"1.0\"?><rss></rss>",
|
||||
},
|
||||
});
|
||||
const resolver = new FeedResolver(fetcher);
|
||||
|
||||
const result = await resolver.resolve("https://the.lxxscrolls.com/p/daniel-6");
|
||||
|
||||
expect(result.sourceType).toBe("substack");
|
||||
expect(result.sourceType).toBe("generic");
|
||||
expect(result.canonicalHost).toBe("the.lxxscrolls.com");
|
||||
expect(result.feedUrl).toBe("https://the.lxxscrolls.com/feed");
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import type { FeedItem } from "../feed-source";
|
||||
import type { NoteWriter, WriteOutcome } from "../note-writer";
|
||||
import { NoteWriterCancelledError } from "../note-writer";
|
||||
import type { DebugEvent, DebugEventInput, DebugLogger } from "../debug-logger";
|
||||
import {
|
||||
ImportRunner,
|
||||
formatImportNotice,
|
||||
|
|
@ -9,6 +10,26 @@ import {
|
|||
type ImportTally,
|
||||
} from "../import-runner";
|
||||
|
||||
/** Recording debug logger stub: captures every logged event for assertions. */
|
||||
function makeDebugLogger(): { logger: DebugLogger; events: DebugEventInput[] } {
|
||||
const events: DebugEventInput[] = [];
|
||||
const logger: DebugLogger = {
|
||||
enabled: true,
|
||||
setEnabled(): void {},
|
||||
log(event: DebugEventInput): void {
|
||||
events.push(event);
|
||||
},
|
||||
snapshot(): readonly DebugEvent[] {
|
||||
return [];
|
||||
},
|
||||
clear(): void {},
|
||||
format(): string {
|
||||
return "";
|
||||
},
|
||||
};
|
||||
return { logger, events };
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Fixtures
|
||||
// -----------------------------------------------------------------------------
|
||||
|
|
@ -302,6 +323,38 @@ describe("ImportRunner.run", () => {
|
|||
expect(calls[0]?.body).toBe("MD:");
|
||||
});
|
||||
|
||||
it("emits a debug note (not a failure) when the imported body is empty", async () => {
|
||||
const items = [makeItem({ id: "blank", title: "Blank Post" })];
|
||||
// fetchBody yields a null body, so the converted markdown is empty.
|
||||
const source: FeedSourceLike = {
|
||||
fetchBody: (item) => Promise.resolve({ ...item, contentHtml: null }),
|
||||
};
|
||||
const { writer } = makeWriter(() =>
|
||||
Promise.resolve({ status: "created", path: "Feeds/blank.md" }),
|
||||
);
|
||||
const { logger, events } = makeDebugLogger();
|
||||
|
||||
// A pass-through converter: an empty HTML body converts to an empty body.
|
||||
const passthroughConvert = (html: string): string => html;
|
||||
const runner = new ImportRunner({
|
||||
source,
|
||||
noteWriter: writer,
|
||||
convert: passthroughConvert,
|
||||
debugLogger: logger,
|
||||
});
|
||||
const tally = await runner.run(items, {});
|
||||
|
||||
// The empty body is still counted as created, not failed.
|
||||
expect(tally.created).toBe(1);
|
||||
expect(tally.failed).toBe(0);
|
||||
// A diagnostic note records the empty body.
|
||||
const emptyNote = events.find(
|
||||
(e) => e.kind === "note" && e.message.includes("empty body"),
|
||||
);
|
||||
expect(emptyNote).toBeDefined();
|
||||
expect(emptyNote?.message).toContain("Blank Post");
|
||||
});
|
||||
|
||||
it("returns a zeroed tally for an empty item list", async () => {
|
||||
const { source } = makeSource({});
|
||||
const { writer, calls } = makeWriter(() =>
|
||||
|
|
|
|||
|
|
@ -229,6 +229,39 @@ describe("composeNote", () => {
|
|||
expect(note).not.toContain("substack-truncated");
|
||||
expect(note).not.toContain("[!warning]");
|
||||
});
|
||||
|
||||
it("links the episode media and records media-url for a podcast item", () => {
|
||||
const mediaUrl = "https://media.example.com/ep/42.mp3";
|
||||
const item = makeItem({
|
||||
kind: "podcast",
|
||||
mediaUrl,
|
||||
mediaType: "audio/mpeg",
|
||||
});
|
||||
const note = composeNote(item, "BODY", {});
|
||||
// Frontmatter records the force-quoted media URL.
|
||||
expect(note).toContain(`media-url: "${mediaUrl}"`);
|
||||
// Body links the episode audio.
|
||||
expect(note).toContain(`[Episode audio](${mediaUrl})`);
|
||||
});
|
||||
|
||||
it("labels non-podcast media as Media in the body link", () => {
|
||||
const mediaUrl = "https://media.example.com/clip.mp4";
|
||||
const item = makeItem({
|
||||
kind: "article",
|
||||
mediaUrl,
|
||||
mediaType: "video/mp4",
|
||||
});
|
||||
const note = composeNote(item, "BODY", {});
|
||||
expect(note).toContain(`media-url: "${mediaUrl}"`);
|
||||
expect(note).toContain(`[Media](${mediaUrl})`);
|
||||
});
|
||||
|
||||
it("omits media frontmatter and body link when there is no media", () => {
|
||||
const note = composeNote(makeItem({ mediaUrl: null }), "BODY", {});
|
||||
expect(note).not.toContain("media-url:");
|
||||
expect(note).not.toContain("[Episode audio]");
|
||||
expect(note).not.toContain("[Media]");
|
||||
});
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
|
|
@ -251,6 +284,17 @@ describe("extractFeedItemId", () => {
|
|||
expect(extractFeedItemId(note)).toBe("null");
|
||||
});
|
||||
|
||||
it("round-trips an id containing a literal backslash-n (two chars)", () => {
|
||||
// The id is a backslash followed by the letter n, NOT a newline. The
|
||||
// unescape order in extractFeedItemId must turn the escaped backslash back
|
||||
// into one backslash before the \n rule runs, or this would corrupt into a
|
||||
// real newline.
|
||||
const literalBackslashN = "a\\nb";
|
||||
const item = makeItem({ id: literalBackslashN });
|
||||
const note = composeNote(item, "BODY", {});
|
||||
expect(extractFeedItemId(note)).toBe(literalBackslashN);
|
||||
});
|
||||
|
||||
it("returns null when there is no frontmatter", () => {
|
||||
expect(extractFeedItemId("just a body, no frontmatter")).toBeNull();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -204,6 +204,25 @@ describe("mapRawItemToFeedItem", () => {
|
|||
expect(mapRawItemToFeedItem(raw, "host").id.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("falls back to a derived id when guid is an empty string", () => {
|
||||
// A present-but-empty guid is not a usable identity; the item must still
|
||||
// get a non-empty deterministic id from its title.
|
||||
const raw: RawFeedItem = {
|
||||
guid: "",
|
||||
link: null,
|
||||
title: "A titled post",
|
||||
author: null,
|
||||
pubDateIso: null,
|
||||
contentHtml: null,
|
||||
description: null,
|
||||
categories: [],
|
||||
enclosure: null,
|
||||
};
|
||||
const id = mapRawItemToFeedItem(raw, "host").id;
|
||||
expect(id.length).toBeGreaterThan(0);
|
||||
expect(id).toContain("A titled post");
|
||||
});
|
||||
|
||||
it("sets the conservative defaults a source may refine", () => {
|
||||
const raw = firstRawItem("substack-item.xml");
|
||||
const item = mapRawItemToFeedItem(raw, "host");
|
||||
|
|
|
|||
|
|
@ -213,9 +213,11 @@ export class AddFeedModal extends Modal {
|
|||
return;
|
||||
}
|
||||
const resolved = this.resolved;
|
||||
const folderRaw = this.folderInputEl?.value.trim() ?? "";
|
||||
const destinationFolder =
|
||||
this.folderInputEl?.value.trim() ??
|
||||
defaultDestinationFolder(this.deps.settings.defaultParentFolder, resolved.publicationTitle);
|
||||
folderRaw.length === 0
|
||||
? defaultDestinationFolder(this.deps.settings.defaultParentFolder, resolved.publicationTitle)
|
||||
: folderRaw;
|
||||
const tags = parseTagsInput(this.tagsInputEl?.value ?? "");
|
||||
|
||||
const feed: FeedConfig = {
|
||||
|
|
@ -225,13 +227,7 @@ export class AddFeedModal extends Modal {
|
|||
canonicalHost: resolved.canonicalHost,
|
||||
publicationTitle: resolved.publicationTitle,
|
||||
author: resolved.author,
|
||||
destinationFolder:
|
||||
destinationFolder.length === 0
|
||||
? defaultDestinationFolder(
|
||||
this.deps.settings.defaultParentFolder,
|
||||
resolved.publicationTitle,
|
||||
)
|
||||
: destinationFolder,
|
||||
destinationFolder,
|
||||
tags,
|
||||
tagNamespace: "",
|
||||
importSourceTags: true,
|
||||
|
|
|
|||
66
duplicate-prompt-modal.ts
Normal file
66
duplicate-prompt-modal.ts
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
// Asks the user whether to overwrite, skip, or cancel when a same-item note
|
||||
// already exists and the duplicate policy is "prompt".
|
||||
|
||||
import { App, Modal, Setting } from "obsidian";
|
||||
import type {
|
||||
DuplicatePromptContext,
|
||||
DuplicatePromptDecision,
|
||||
} from "./note-writer";
|
||||
|
||||
export class DuplicatePromptModal extends Modal {
|
||||
private readonly context: DuplicatePromptContext;
|
||||
private readonly resolve: (decision: DuplicatePromptDecision) => void;
|
||||
private decided = false;
|
||||
|
||||
constructor(
|
||||
app: App,
|
||||
context: DuplicatePromptContext,
|
||||
resolve: (decision: DuplicatePromptDecision) => void,
|
||||
) {
|
||||
super(app);
|
||||
this.context = context;
|
||||
this.resolve = resolve;
|
||||
}
|
||||
|
||||
onOpen(): void {
|
||||
this.setTitle("Note already exists");
|
||||
const { contentEl } = this;
|
||||
contentEl.createEl("p", {
|
||||
text: `A note for "${this.context.itemTitle}" already exists at ${this.context.targetPath}. Overwrite it?`,
|
||||
});
|
||||
new Setting(contentEl)
|
||||
.addButton((btn) =>
|
||||
btn
|
||||
.setButtonText("Overwrite")
|
||||
.setDestructive()
|
||||
.onClick(() => {
|
||||
this.settle("overwrite");
|
||||
}),
|
||||
)
|
||||
.addButton((btn) =>
|
||||
btn.setButtonText("Skip").onClick(() => {
|
||||
this.settle("skip");
|
||||
}),
|
||||
)
|
||||
.addButton((btn) =>
|
||||
btn.setButtonText("Cancel import").onClick(() => {
|
||||
this.settle("cancel");
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
onClose(): void {
|
||||
// Dismissing the modal without a choice cancels the import.
|
||||
this.settle("cancel");
|
||||
this.contentEl.empty();
|
||||
}
|
||||
|
||||
private settle(decision: DuplicatePromptDecision): void {
|
||||
if (this.decided) {
|
||||
return;
|
||||
}
|
||||
this.decided = true;
|
||||
this.resolve(decision);
|
||||
this.close();
|
||||
}
|
||||
}
|
||||
28
feed-picker-modal.ts
Normal file
28
feed-picker-modal.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
// Picks which configured feed to import from when more than one exists.
|
||||
|
||||
import { App, FuzzySuggestModal } from "obsidian";
|
||||
import type { FeedConfig } from "./settings";
|
||||
|
||||
export class FeedPickerModal extends FuzzySuggestModal<FeedConfig> {
|
||||
private readonly feeds: FeedConfig[];
|
||||
private readonly onChoose: (feed: FeedConfig) => void;
|
||||
|
||||
constructor(app: App, feeds: FeedConfig[], onChoose: (feed: FeedConfig) => void) {
|
||||
super(app);
|
||||
this.feeds = feeds;
|
||||
this.onChoose = onChoose;
|
||||
this.setPlaceholder("Pick a feed to import from");
|
||||
}
|
||||
|
||||
getItems(): FeedConfig[] {
|
||||
return this.feeds;
|
||||
}
|
||||
|
||||
getItemText(feed: FeedConfig): string {
|
||||
return feed.publicationTitle;
|
||||
}
|
||||
|
||||
onChooseItem(feed: FeedConfig): void {
|
||||
this.onChoose(feed);
|
||||
}
|
||||
}
|
||||
|
|
@ -129,11 +129,6 @@ function isSubstackSubdomain(host: string): boolean {
|
|||
return host.toLowerCase().endsWith(SUBSTACK_SUFFIX);
|
||||
}
|
||||
|
||||
/** True when a path looks like a Substack post permalink (`/p/<slug>`). */
|
||||
function isSubstackPostPath(pathname: string): boolean {
|
||||
return /^\/p\/[^/]+/.test(pathname);
|
||||
}
|
||||
|
||||
/** True when a path ends in a recognized feed suffix. */
|
||||
function hasFeedPathSuffix(pathname: string): boolean {
|
||||
const lower = pathname.replace(/\/$/, "").toLowerCase();
|
||||
|
|
@ -249,11 +244,12 @@ export class FeedResolver {
|
|||
* Dispatch order:
|
||||
* 1. A Substack handle (`@x`, `substack.com/@x`) -> profile API lookup.
|
||||
* 2. A URL whose host is `*.substack.com` -> Substack, feed at host/feed.
|
||||
* 3. A Substack post URL (`/p/<slug>`) on any host -> Substack, host/feed.
|
||||
* 4. A URL whose path already names a feed (`/feed`, `/rss`, ...) -> generic.
|
||||
* 5. Any other URL -> probe `host/feed`; XML means generic.
|
||||
* A `/p/<slug>` post URL on a substack.com host falls in here; the same
|
||||
* path on a custom domain does not (those are added via @handle).
|
||||
* 3. A URL whose path already names a feed (`/feed`, `/rss`, ...) -> generic.
|
||||
* 4. Any other URL -> probe `host/feed`; XML means generic.
|
||||
*
|
||||
* In cases 2 to 5 the canonical host is derived by walking redirects from
|
||||
* In cases 2 to 4 the canonical host is derived by walking redirects from
|
||||
* the candidate feed URL, so a publication that moved hosts resolves to
|
||||
* where its feed actually lives.
|
||||
*/
|
||||
|
|
@ -276,7 +272,12 @@ export class FeedResolver {
|
|||
throw new ResolveError(`Not a resolvable Substack input: ${trimmed}`);
|
||||
}
|
||||
|
||||
if (isSubstackSubdomain(url.hostname) || isSubstackPostPath(url.pathname)) {
|
||||
// A Substack post URL is recognized as Substack only on a substack.com
|
||||
// host. A `/p/<slug>` path on a custom domain is NOT enough to claim
|
||||
// Substack: a non-Substack site can use the same path shape. Custom-domain
|
||||
// Substacks are added via the @handle path instead, which the profile API
|
||||
// canonicalizes correctly.
|
||||
if (isSubstackSubdomain(url.hostname)) {
|
||||
return this.resolveSubstackHost(url.hostname, null);
|
||||
}
|
||||
|
||||
|
|
@ -386,7 +387,7 @@ export class FeedResolver {
|
|||
*/
|
||||
private async walkRedirects(startUrl: string): Promise<{ finalUrl: URL; response: HttpResponse }> {
|
||||
let current = parseUrl(startUrl);
|
||||
for (let hop = 0; hop <= MAX_REDIRECT_HOPS; hop += 1) {
|
||||
for (let hop = 0; hop < MAX_REDIRECT_HOPS; hop += 1) {
|
||||
let response: HttpResponse;
|
||||
try {
|
||||
response = await this.fetcher({ url: current.toString() });
|
||||
|
|
|
|||
|
|
@ -43,7 +43,11 @@ export interface FeedItem {
|
|||
/** ISO 8601 timestamp, or null when the feed omits a date. */
|
||||
publishedAt: string | null;
|
||||
kind: FeedItemKind;
|
||||
/** Full body (free) or teaser (paid). Null until `fetchBody` runs. */
|
||||
/**
|
||||
* Full body (free) or teaser (paid). Null until `fetchBody` runs, and may
|
||||
* still be null afterwards when the feed provides no body for the item.
|
||||
* Callers must handle null (treat it as an empty body).
|
||||
*/
|
||||
contentHtml: string | null;
|
||||
/** True when the body is a paywalled teaser rather than a complete post. */
|
||||
isTruncated: boolean;
|
||||
|
|
@ -89,8 +93,10 @@ export interface ListItemsOptions {
|
|||
|
||||
/**
|
||||
* A feed source. `resolve` classifies and canonicalizes an input; `listItems`
|
||||
* enumerates available items (bodies may be absent); `fetchBody` guarantees the
|
||||
* item's `contentHtml` is populated (full body when entitled, teaser otherwise).
|
||||
* enumerates available items (bodies may be absent); `fetchBody` attempts to
|
||||
* populate the item's `contentHtml` (full body when entitled, teaser otherwise).
|
||||
* `contentHtml` may still be null after `fetchBody` when the feed provides no
|
||||
* body for the item, so callers must handle a null body.
|
||||
*/
|
||||
export interface FeedSource {
|
||||
readonly type: SourceType;
|
||||
|
|
|
|||
|
|
@ -149,10 +149,12 @@ export class ImageDownloader {
|
|||
): Promise<string | null> {
|
||||
const response = await this.fetcher({ url, method: "GET" });
|
||||
if (response.status < 200 || response.status >= 300) {
|
||||
console.error(`RSS Importer: image download skipped for ${url}: status ${response.status}`);
|
||||
return null;
|
||||
}
|
||||
const bytes = response.arrayBuffer;
|
||||
if (bytes.byteLength === 0) {
|
||||
console.error(`RSS Importer: image download skipped for ${url}: empty body`);
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -101,6 +101,10 @@ export class ImportModal extends Modal {
|
|||
}
|
||||
|
||||
onOpen(): void {
|
||||
// Reset the abort flag once per modal open, not per import run, so a
|
||||
// reused modal instance starts clean while a single open session keeps a
|
||||
// consistent flag across loadItems and any number of runImport calls.
|
||||
this.aborted = false;
|
||||
const { contentEl } = this;
|
||||
contentEl.empty();
|
||||
contentEl.addClass("rss-importer-import-modal");
|
||||
|
|
@ -326,7 +330,6 @@ export class ImportModal extends Modal {
|
|||
}
|
||||
|
||||
this.importing = true;
|
||||
this.aborted = false;
|
||||
this.refreshImportButton();
|
||||
this.clearSummary();
|
||||
|
||||
|
|
|
|||
|
|
@ -190,6 +190,17 @@ export class ImportRunner {
|
|||
}
|
||||
}
|
||||
|
||||
// An empty body is not a failure (the feed simply gave no content), but
|
||||
// it is worth a diagnostic note so an unexpectedly blank note is
|
||||
// traceable to a content-less feed rather than a conversion bug.
|
||||
if (markdown.trim().length === 0) {
|
||||
this.debug?.log({
|
||||
kind: "note",
|
||||
message: `Imported "${full.title}" with an empty body (feed provided no content)`,
|
||||
endpoint: full.url,
|
||||
});
|
||||
}
|
||||
|
||||
const outcome = await this.noteWriter.writeNote(full, markdown, {
|
||||
feedTags,
|
||||
});
|
||||
|
|
|
|||
190
main.ts
190
main.ts
|
|
@ -1,10 +1,6 @@
|
|||
import {
|
||||
App,
|
||||
FuzzySuggestModal,
|
||||
Modal,
|
||||
Notice,
|
||||
Plugin,
|
||||
Setting,
|
||||
requestUrl,
|
||||
type RequestUrlResponse,
|
||||
} from "obsidian";
|
||||
|
|
@ -24,7 +20,6 @@ import { FetchPacer } from "./fetch-pacer";
|
|||
import {
|
||||
NoteWriter,
|
||||
type DuplicatePromptCallback,
|
||||
type DuplicatePromptContext,
|
||||
type DuplicatePromptDecision,
|
||||
} from "./note-writer";
|
||||
import { convertHtmlToMarkdown } from "./html-converter";
|
||||
|
|
@ -34,6 +29,8 @@ import { BufferedDebugLogger } from "./debug-logger";
|
|||
import { ImportRunner } from "./import-runner";
|
||||
import { ImportModal } from "./import-modal";
|
||||
import { AddFeedModal } from "./add-feed-modal";
|
||||
import { FeedPickerModal } from "./feed-picker-modal";
|
||||
import { DuplicatePromptModal } from "./duplicate-prompt-modal";
|
||||
import { RssImporterSettingTab, type RssImporterPluginLike } from "./settings-tab";
|
||||
|
||||
// Adapt Obsidian's requestUrl to the plugin's HttpFetcher contract. requestUrl
|
||||
|
|
@ -83,6 +80,17 @@ function runGuarded(action: string, fn: () => Promise<void>): void {
|
|||
});
|
||||
}
|
||||
|
||||
// Type guards for the per-feed literal-union fields read from data.json. Stored
|
||||
// settings are user-editable JSON, so a value can be missing or a wrong literal;
|
||||
// these narrow to the known unions before the value is trusted.
|
||||
function isImagesMode(value: unknown): value is import("./settings").ImagesMode {
|
||||
return value === "link" || value === "download";
|
||||
}
|
||||
|
||||
function isDuplicatePolicy(value: unknown): value is import("./settings").DuplicatePolicy {
|
||||
return value === "skip" || value === "overwrite" || value === "prompt";
|
||||
}
|
||||
|
||||
// Sync classification used only to pick which source resolves a freshly entered
|
||||
// input for the add-feed preview. The resolved feed's own sourceType (captured
|
||||
// into the FeedConfig) is what drives import, so a custom-domain Substack that
|
||||
|
|
@ -165,6 +173,33 @@ export default class RssImporterPlugin extends Plugin implements RssImporterPlug
|
|||
if (!Array.isArray(this.settings.feeds)) {
|
||||
this.settings.feeds = [];
|
||||
}
|
||||
|
||||
// Coerce the literal-union fields. data.json is user-editable, so a stored
|
||||
// value can be a missing or invalid literal; fall back to the default for
|
||||
// the top-level fields and drop invalid per-feed overrides so the default
|
||||
// applies.
|
||||
if (!isImagesMode(this.settings.imagesMode)) {
|
||||
this.settings.imagesMode = DEFAULT_SETTINGS.imagesMode;
|
||||
}
|
||||
if (!isDuplicatePolicy(this.settings.duplicatePolicy)) {
|
||||
this.settings.duplicatePolicy = DEFAULT_SETTINGS.duplicatePolicy;
|
||||
}
|
||||
for (const feed of this.settings.feeds) {
|
||||
// The per-feed override fields are optional literal/string unions on
|
||||
// FeedConfig, but data.json is user-editable so a stored value can be a
|
||||
// wrong type. View the feed as a loose record to inspect and drop the
|
||||
// bad override; deleting it makes the global default apply.
|
||||
const record = feed as unknown as Record<string, unknown>;
|
||||
if ("imagesMode" in record && !isImagesMode(record["imagesMode"])) {
|
||||
delete record["imagesMode"];
|
||||
}
|
||||
if ("noteNameTemplate" in record && typeof record["noteNameTemplate"] !== "string") {
|
||||
delete record["noteNameTemplate"];
|
||||
}
|
||||
if ("imageSubfolder" in record && typeof record["imageSubfolder"] !== "string") {
|
||||
delete record["imageSubfolder"];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async saveSettings(): Promise<void> {
|
||||
|
|
@ -226,32 +261,46 @@ export default class RssImporterPlugin extends Plugin implements RssImporterPlug
|
|||
}
|
||||
|
||||
private openImport(feed: FeedConfig): void {
|
||||
const source = this.makeSourceForFeed(feed);
|
||||
const noteWriter = new NoteWriter({
|
||||
vault: this.app.vault,
|
||||
destinationFolder: feed.destinationFolder,
|
||||
noteNameTemplate: effectiveNoteNameTemplate(feed, this.settings),
|
||||
onDuplicate: this.settings.duplicatePolicy,
|
||||
promptOnDuplicate: this.promptOnDuplicate,
|
||||
});
|
||||
let runner: ImportRunner;
|
||||
let source: FeedSource;
|
||||
// NoteWriter construction throws when the destination folder contains a
|
||||
// ".." segment that would escape the vault. That throw must not escape the
|
||||
// command callback uncaught, so the whole synchronous wiring is guarded and
|
||||
// the modal only opens on success.
|
||||
try {
|
||||
source = this.makeSourceForFeed(feed);
|
||||
const noteWriter = new NoteWriter({
|
||||
vault: this.app.vault,
|
||||
destinationFolder: feed.destinationFolder,
|
||||
noteNameTemplate: effectiveNoteNameTemplate(feed, this.settings),
|
||||
onDuplicate: this.settings.duplicatePolicy,
|
||||
promptOnDuplicate: this.promptOnDuplicate,
|
||||
});
|
||||
|
||||
let processImages: ((markdown: string, item: import("./feed-source").FeedItem) => Promise<string>) | undefined;
|
||||
if (effectiveImagesMode(feed, this.settings) === "download") {
|
||||
const downloader = new ImageDownloader({ fetcher: this.makeFetcher(), vault: this.app.vault });
|
||||
const subfolder = effectiveImageSubfolder(feed, this.settings);
|
||||
const folderPath =
|
||||
feed.destinationFolder === "" ? subfolder : `${feed.destinationFolder}/${subfolder}`;
|
||||
processImages = (markdown) => downloader.downloadAndRewrite(markdown, folderPath);
|
||||
let processImages: ((markdown: string, item: import("./feed-source").FeedItem) => Promise<string>) | undefined;
|
||||
if (effectiveImagesMode(feed, this.settings) === "download") {
|
||||
const downloader = new ImageDownloader({ fetcher: this.makeFetcher(), vault: this.app.vault });
|
||||
const subfolder = effectiveImageSubfolder(feed, this.settings);
|
||||
const folderPath =
|
||||
feed.destinationFolder === "" ? subfolder : `${feed.destinationFolder}/${subfolder}`;
|
||||
processImages = (markdown) => downloader.downloadAndRewrite(markdown, folderPath);
|
||||
}
|
||||
|
||||
runner = new ImportRunner({
|
||||
source,
|
||||
noteWriter,
|
||||
convert: convertHtmlToMarkdown,
|
||||
processImages,
|
||||
debugLogger: this.debugLogger,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
new Notice(
|
||||
`Could not start import. ${err instanceof Error ? err.message : "See the console."}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const runner = new ImportRunner({
|
||||
source,
|
||||
noteWriter,
|
||||
convert: convertHtmlToMarkdown,
|
||||
processImages,
|
||||
debugLogger: this.debugLogger,
|
||||
});
|
||||
|
||||
new ImportModal(this.app, {
|
||||
feed,
|
||||
source,
|
||||
|
|
@ -301,88 +350,3 @@ export default class RssImporterPlugin extends Plugin implements RssImporterPlug
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Picks which configured feed to import from when more than one exists.
|
||||
class FeedPickerModal extends FuzzySuggestModal<FeedConfig> {
|
||||
private readonly feeds: FeedConfig[];
|
||||
private readonly onChoose: (feed: FeedConfig) => void;
|
||||
|
||||
constructor(app: App, feeds: FeedConfig[], onChoose: (feed: FeedConfig) => void) {
|
||||
super(app);
|
||||
this.feeds = feeds;
|
||||
this.onChoose = onChoose;
|
||||
this.setPlaceholder("Pick a feed to import from");
|
||||
}
|
||||
|
||||
getItems(): FeedConfig[] {
|
||||
return this.feeds;
|
||||
}
|
||||
|
||||
getItemText(feed: FeedConfig): string {
|
||||
return feed.publicationTitle;
|
||||
}
|
||||
|
||||
onChooseItem(feed: FeedConfig): void {
|
||||
this.onChoose(feed);
|
||||
}
|
||||
}
|
||||
|
||||
// Asks the user whether to overwrite, skip, or cancel when a same-item note
|
||||
// already exists and the duplicate policy is "prompt".
|
||||
class DuplicatePromptModal extends Modal {
|
||||
private readonly context: DuplicatePromptContext;
|
||||
private readonly resolve: (decision: DuplicatePromptDecision) => void;
|
||||
private decided = false;
|
||||
|
||||
constructor(
|
||||
app: App,
|
||||
context: DuplicatePromptContext,
|
||||
resolve: (decision: DuplicatePromptDecision) => void,
|
||||
) {
|
||||
super(app);
|
||||
this.context = context;
|
||||
this.resolve = resolve;
|
||||
}
|
||||
|
||||
onOpen(): void {
|
||||
this.setTitle("Note already exists");
|
||||
const { contentEl } = this;
|
||||
contentEl.createEl("p", {
|
||||
text: `A note for "${this.context.itemTitle}" already exists at ${this.context.targetPath}. Overwrite it?`,
|
||||
});
|
||||
new Setting(contentEl)
|
||||
.addButton((btn) =>
|
||||
btn
|
||||
.setButtonText("Overwrite")
|
||||
.setDestructive()
|
||||
.onClick(() => {
|
||||
this.settle("overwrite");
|
||||
}),
|
||||
)
|
||||
.addButton((btn) =>
|
||||
btn.setButtonText("Skip").onClick(() => {
|
||||
this.settle("skip");
|
||||
}),
|
||||
)
|
||||
.addButton((btn) =>
|
||||
btn.setButtonText("Cancel import").onClick(() => {
|
||||
this.settle("cancel");
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
onClose(): void {
|
||||
// Dismissing the modal without a choice cancels the import.
|
||||
this.settle("cancel");
|
||||
this.contentEl.empty();
|
||||
}
|
||||
|
||||
private settle(decision: DuplicatePromptDecision): void {
|
||||
if (this.decided) {
|
||||
return;
|
||||
}
|
||||
this.decided = true;
|
||||
this.resolve(decision);
|
||||
this.close();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -397,6 +397,11 @@ const TRUNCATED_CALLOUT = [
|
|||
* When the item is a truncated teaser, a visible callout warning is prepended
|
||||
* to the body and a `substack-truncated: true` line is added to frontmatter so
|
||||
* the state is both human-visible and machine-queryable.
|
||||
*
|
||||
* When the item carries media (a podcast/audio/video enclosure), a `media-url`
|
||||
* frontmatter line records it (force-quoted, since it is a URL) and a labelled
|
||||
* link to the media is appended to the body so the episode is reachable from the
|
||||
* note.
|
||||
*/
|
||||
export function composeNote(
|
||||
item: FeedItem,
|
||||
|
|
@ -421,16 +426,30 @@ export function composeNote(
|
|||
if (tags.length > 0) {
|
||||
lines.push(`${FRONTMATTER_KEYS.tags}: ${yamlTagArray(tags)}`);
|
||||
}
|
||||
// Media items (podcast episodes, attached audio/video) record their media URL
|
||||
// in frontmatter. Force-quoted because it is a URL. Placed after tags and
|
||||
// before the truncated marker.
|
||||
const hasMedia = item.mediaUrl !== null && item.mediaUrl.length > 0;
|
||||
if (hasMedia && item.mediaUrl !== null) {
|
||||
lines.push(`media-url: ${yamlQuoted(item.mediaUrl)}`);
|
||||
}
|
||||
if (item.isTruncated) {
|
||||
// Machine-readable marker alongside the visible callout below.
|
||||
lines.push("substack-truncated: true");
|
||||
}
|
||||
lines.push("---");
|
||||
|
||||
const body = item.isTruncated
|
||||
let body = item.isTruncated
|
||||
? `${TRUNCATED_CALLOUT}\n\n${bodyMarkdown}`
|
||||
: bodyMarkdown;
|
||||
|
||||
// Link the media at the end of the body so a podcast note plays its episode
|
||||
// and any other media item is reachable from the note.
|
||||
if (hasMedia && item.mediaUrl !== null) {
|
||||
const label = item.kind === "podcast" ? "Episode audio" : "Media";
|
||||
body = `${body}\n\n[${label}](${item.mediaUrl})`;
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n\n${body}`;
|
||||
}
|
||||
|
||||
|
|
@ -474,13 +493,30 @@ export function extractFeedItemId(content: string): string | null {
|
|||
(value.startsWith("'") && value.endsWith("'")))
|
||||
) {
|
||||
value = value.slice(1, -1);
|
||||
// Unescape the standard double-quoted escapes we emit.
|
||||
value = value
|
||||
.replace(/\\"/g, '"')
|
||||
.replace(/\\n/g, "\n")
|
||||
.replace(/\\r/g, "\r")
|
||||
.replace(/\\t/g, "\t")
|
||||
.replace(/\\\\/g, "\\");
|
||||
// Unescape the standard double-quoted escapes we emit, in a SINGLE pass.
|
||||
// The escaped-backslash sequence `\\` must be consumed before the letter
|
||||
// escapes, otherwise a literal backslash followed by "n" (written as `\\n`)
|
||||
// would be misread as a newline. Sequential global replaces cannot do this
|
||||
// safely: turning `\\` into `\` first leaves a `\n` that the next pass then
|
||||
// rewrites to a newline. One left-to-right pass over each escape sequence
|
||||
// matches `\\` as a unit and maps the rest by their following character.
|
||||
value = value.replace(/\\(.)/g, (_match, next: string) => {
|
||||
switch (next) {
|
||||
case "\\":
|
||||
return "\\";
|
||||
case '"':
|
||||
return '"';
|
||||
case "n":
|
||||
return "\n";
|
||||
case "r":
|
||||
return "\r";
|
||||
case "t":
|
||||
return "\t";
|
||||
default:
|
||||
// Unknown escape: drop the backslash, keep the character.
|
||||
return next;
|
||||
}
|
||||
});
|
||||
}
|
||||
return value.length > 0 ? value : null;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
*/
|
||||
|
||||
import type { SourceType } from "./feed-source";
|
||||
import type { DismissedMap } from "./dismiss-store";
|
||||
|
||||
export type ImagesMode = "link" | "download";
|
||||
export type DuplicatePolicy = "skip" | "overwrite" | "prompt";
|
||||
|
|
@ -54,7 +55,7 @@ export interface RssImporterSettings {
|
|||
showRibbonIcon: boolean;
|
||||
ribbonIcon: string;
|
||||
/** Dismissed items per feed: feedId -> array of feed-item-id. */
|
||||
dismissed: Record<string, string[]>;
|
||||
dismissed: DismissedMap;
|
||||
}
|
||||
|
||||
export const DEFAULT_SETTINGS: RssImporterSettings = {
|
||||
|
|
|
|||
BIN
source-common.ts
BIN
source-common.ts
Binary file not shown.
|
|
@ -59,8 +59,9 @@ export class GenericRssFeedSource implements FeedSource {
|
|||
|
||||
/**
|
||||
* Fetch the feed and map every item onto the normalized `FeedItem` shape,
|
||||
* honoring `opts.limit`. The base mapping already pulls the body fallback, so
|
||||
* generic items come back with their `contentHtml` populated.
|
||||
* honoring `opts.limit`. The base mapping pulls the body fallback
|
||||
* (content:encoded, else description), so generic items usually carry their
|
||||
* `contentHtml` inline, but it stays null when the feed provides no body.
|
||||
*/
|
||||
async listItems(feed: ResolvedFeed, opts?: ListItemsOptions): Promise<FeedItem[]> {
|
||||
const parsed = await fetchAndParseFeed(this.fetcher, feed.feedUrl);
|
||||
|
|
|
|||
Loading…
Reference in a new issue