ckelsoe_obsidian-rss-importer/source-common.ts

/**
 * Shared building blocks for the concrete feed sources.
 *
 * Both `GenericRssFeedSource` and `SubstackFeedSource` fetch the same way, map
 * raw parsed items onto the normalized `FeedItem` the same way, and build the
 * same `ResolvedFeed` preview. That shared work lives here so the two sources do
 * not drift apart by copy-and-tweak. Anything source-specific (Substack's
 * paywall pass) is layered on top by the source after calling these helpers.
 *
 * Nothing in this module touches Obsidian or the DOM at module scope: the only
 * DOM use is inside `parseFeed`, which constructs its DOMParser lazily. That
 * keeps the module importable in a plain Node context.
 */

import type {
	FeedAudience,
	FeedItem,
	FeedItemKind,
	HttpFetcher,
	ResolvedFeed,
} from "./feed-source";
import { parseFeed } from "./feed-xml";
import type { ParsedFeed, RawFeedItem } from "./feed-xml";
import type { ResolverResult } from "./feed-resolver";

/** How many recent item titles to surface in the add-feed preview card. */
const SAMPLE_TITLE_LIMIT = 5;

/** MIME-type prefixes that mark an enclosure as a playable podcast/media item. */
const PODCAST_MEDIA_PREFIXES = ["audio/", "video/"] as const;

/**
 * Raised when a feed URL cannot be fetched as a usable feed: a transport error
 * from the fetcher, or a non-2xx HTTP status. The originating status (or the
 * underlying error) is surfaced so callers can show a real reason instead of a
 * bare "fetch failed". A `FeedParseError` thrown by `parseFeed` propagates
 * unchanged; this error covers only the fetch/status layer above it.
 */
export class FeedFetchError extends Error {
	/** The HTTP status when the failure was a non-2xx response, else null. */
	readonly status: number | null;
	/** The underlying transport error, when this wraps one. */
	readonly cause?: unknown;

	constructor(message: string, status: number | null, cause?: unknown) {
		super(message);
		this.name = "FeedFetchError";
		this.status = status;
		this.cause = cause;
	}
}

/**
 * GET `feedUrl` through the injected fetcher and parse the response body as a
 * feed. Throws `FeedFetchError` on a transport failure or non-2xx status (the
 * status is included in the message and on the error). Parse failures surface as
 * the `FeedParseError` thrown by `parseFeed`.
 */
export async function fetchAndParseFeed(
	fetcher: HttpFetcher,
	feedUrl: string,
): Promise<ParsedFeed> {
	let status: number;
	let text: string;
	try {
		const response = await fetcher({ url: feedUrl, method: "GET" });
		status = response.status;
		text = response.text;
	} catch (err) {
		throw new FeedFetchError(`Failed to fetch feed at ${feedUrl}`, null, err);
	}

	if (status < 200 || status >= 300) {
		throw new FeedFetchError(
			`Feed request to ${feedUrl} returned status ${status}`,
			status,
		);
	}

	return parseFeed(text);
}

/** True when an enclosure MIME type names playable audio or video media. */
function isMediaType(type: string | null): boolean {
	if (type === null) {
		return false;
	}
	const lower = type.trim().toLowerCase();
	return PODCAST_MEDIA_PREFIXES.some((prefix) => lower.startsWith(prefix));
}

/**
 * Derive a stable, non-empty id for an item that has neither a guid nor a link.
 * Built from the title and pubDate so the same item produces the same id across
 * runs (dedup depends on this). Never returns an empty string: a feed item with
 * no title and no date still gets a deterministic placeholder id.
 */
function deriveFallbackId(raw: RawFeedItem): string {
	const titlePart = raw.title.trim();
	const datePart = raw.pubDateIso ?? "";
	const combined = `${titlePart}${datePart}`;
	if (combined === "") {
		return "feed-item:untitled";
	}
	return `feed-item:${combined}`;
}

/**
 * Map a raw parsed item onto the normalized `FeedItem` base shape, per the
 * feed-xml contract.
 *
 * Identity: `guid`, else `link`, else a stable title+date fallback. Body
 * fallback (content:encoded, else description) is applied HERE, not in the
 * parser, which keeps the two fields separate on purpose. Kind is `podcast` when
 * the enclosure advertises an audio/video MIME type, else `article`. Media
 * fields come straight off the enclosure, treating a `length` of 0 as a real
 * advertised value rather than absent.
 *
 * The returned item carries the conservative defaults for the fields a source
 * may refine afterwards: `audience` "unknown", `isTruncated` false, `section`
 * null. Substack overwrites `audience`/`isTruncated` via its paywall pass.
 */
export function mapRawItemToFeedItem(raw: RawFeedItem, sourceId: string): FeedItem {
	// A guid or link present but empty after trim is not a usable identity; fall
	// through to the deterministic title+date fallback rather than write an empty
	// id (which would break dedup and collide every empty-id item together).
	const guid = raw.guid?.trim();
	const link = raw.link?.trim();
	const id =
		guid !== undefined && guid.length > 0
			? guid
			: link !== undefined && link.length > 0
				? link
				: deriveFallbackId(raw);
	const enclosure = raw.enclosure;
	const kind: FeedItemKind =
		enclosure !== null && isMediaType(enclosure.type) ? "podcast" : "article";

	return {
		sourceId,
		id,
		url: raw.link ?? "",
		title: raw.title,
		author: raw.author,
		publishedAt: raw.pubDateIso,
		kind,
		contentHtml: raw.contentHtml ?? raw.description,
		isTruncated: false,
		audience: "unknown",
		tags: raw.categories,
		section: null,
		mediaUrl: enclosure === null ? null : enclosure.url,
		mediaType: enclosure === null ? null : enclosure.type,
		mediaBytes: enclosure === null ? null : enclosure.length,
	};
}

/**
 * Build the `ResolvedFeed` preview record from the resolver result plus the
 * fetched feed. The feed id is the canonical host (stable across renames of the
 * publication title). The author and audience hint are best-effort from the
 * first item; `sampleTitles` is up to the first few non-empty item titles for
 * the add-feed preview card.
 */
export function buildResolvedFeed(
	resolved: ResolverResult,
	parsed: ParsedFeed,
): ResolvedFeed {
	const firstItem = parsed.items.length > 0 ? parsed.items[0] : undefined;
	const author = firstItem !== undefined ? firstItem.author : null;

	const sampleTitles: string[] = [];
	for (const item of parsed.items) {
		const title = item.title.trim();
		if (title.length === 0) {
			continue;
		}
		sampleTitles.push(title);
		if (sampleTitles.length >= SAMPLE_TITLE_LIMIT) {
			break;
		}
	}

	return {
		sourceType: resolved.sourceType,
		feedId: resolved.canonicalHost,
		canonicalHost: resolved.canonicalHost,
		feedUrl: resolved.feedUrl,
		publicationTitle: parsed.feedTitle,
		author,
		sampleTitles,
		audienceHint: "unknown",
	};
}

/**
 * Apply `opts.limit` to a list of items: a non-positive or absent limit returns
 * the list unchanged; otherwise the first `limit` items are kept. Shared so both
 * sources cap enumeration identically.
 */
export function applyLimit<T>(items: T[], limit: number | undefined): T[] {
	if (limit === undefined || limit <= 0) {
		return items;
	}
	return items.slice(0, limit);
}

/**
 * Read a string field off an unknown record, returning null when it is absent,
 * not a string, or empty after trimming. Used to parse the undocumented Substack
 * archive JSON defensively: a missing or wrong-typed field never throws, it just
 * yields null.
 */
function readString(record: Record<string, unknown>, key: string): string | null {
	const value = record[key];
	if (typeof value !== "string") {
		return null;
	}
	const trimmed = value.trim();
	return trimmed.length > 0 ? trimmed : null;
}

/**
 * Read the numeric `id` off an archive post as a string. Substack reports the id
 * as a number; we tolerate a number or a numeric/string id and return null for
 * anything else.
 */
function readId(record: Record<string, unknown>): string | null {
	const value = record["id"];
	if (typeof value === "number" && Number.isFinite(value)) {
		return String(value);
	}
	if (typeof value === "string" && value.trim().length > 0) {
		return value.trim();
	}
	return null;
}

/**
 * Read the first byline name off an archive post: `publishedBylines[0].name`.
 * Returns null when the array is absent, empty, or the entry has no usable name.
 */
function readFirstBylineName(record: Record<string, unknown>): string | null {
	const bylines = record["publishedBylines"];
	if (!Array.isArray(bylines) || bylines.length === 0) {
		return null;
	}
	const first: unknown = bylines[0];
	if (!isRecord(first)) {
		return null;
	}
	return readString(first, "name");
}

/**
 * Read post tag names off an archive post: `postTags[].name`. Skips entries that
 * are not records or have no usable name, so a malformed tag never breaks the
 * whole post. Returns an empty array when there are no tags.
 */
function readPostTags(record: Record<string, unknown>): string[] {
	const tags = record["postTags"];
	if (!Array.isArray(tags)) {
		return [];
	}
	const names: string[] = [];
	for (const entry of tags) {
		if (!isRecord(entry)) {
			continue;
		}
		const name = readString(entry, "name");
		if (name !== null) {
			names.push(name);
		}
	}
	return names;
}

/** Narrow an unknown value to a non-null object so its keys can be read. */
function isRecord(value: unknown): value is Record<string, unknown> {
	return typeof value === "object" && value !== null;
}

/**
 * Map a Substack `audience` field onto the normalized access tier. "only_paid"
 * is paid, "everyone" is free, anything else (including absent) is unknown. This
 * is intentionally narrow to the values the archive API reports, distinct from
 * the broader token matching the paywall detector uses on body structure.
 */
function mapArchiveAudience(audience: string | null): FeedAudience {
	if (audience === "only_paid") {
		return "paid";
	}
	if (audience === "everyone") {
		return "free";
	}
	return "unknown";
}

/**
 * Map one raw Substack archive post (an entry of the `/api/v1/archive` JSON
 * array) onto the normalized `FeedItem` shape. Returns null when the post has
 * neither a numeric id nor a canonical URL, since such an entry has no usable
 * identity for dedup and no permalink to fetch its body from.
 *
 * The archive list carries no body (`body_html` is null/absent here), so
 * `contentHtml` stays null and is filled later by `fetchBody`. `kind` is
 * `podcast` only when the post `type` is "podcast"; everything else maps to
 * `article`. The post `type` "thread" still reads as an article. Parsing is
 * defensive throughout: a missing or wrong-typed field yields the conservative
 * default rather than throwing.
 */
export function mapArchivePostToFeedItem(
	post: unknown,
	feedId: string,
	canonicalHost: string,
): FeedItem | null {
	if (!isRecord(post)) {
		return null;
	}

	const id = readId(post);
	const canonicalUrl = readString(post, "canonical_url");
	const slug = readString(post, "slug");

	// An entry with neither an id nor a canonical URL has no usable identity and
	// no permalink; skip it rather than emit a broken item.
	if (id === null && canonicalUrl === null) {
		return null;
	}

	const url =
		canonicalUrl ?? (slug !== null ? `https://${canonicalHost}/p/${slug}` : "");
	const kind: FeedItemKind = readString(post, "type") === "podcast" ? "podcast" : "article";

	return {
		sourceId: feedId,
		id: id ?? (canonicalUrl ?? ""),
		url,
		title: readString(post, "title") ?? "",
		author: readFirstBylineName(post),
		publishedAt: readString(post, "post_date"),
		kind,
		// The archive list omits the body; fetchBody fills it on demand.
		contentHtml: null,
		isTruncated: false,
		audience: mapArchiveAudience(readString(post, "audience")),
		tags: readPostTags(post),
		section: readString(post, "section_name"),
		mediaUrl: null,
		mediaType: null,
		mediaBytes: null,
	};
}

/**
 * Map a whole Substack archive JSON payload (expected to be an array of posts)
 * onto normalized `FeedItem`s. A non-array payload yields an empty list rather
 * than throwing. Each entry is mapped defensively; entries with no usable
 * identity are skipped, so one malformed post never drops the rest of the page.
 */
export function mapArchivePayload(
	payload: unknown,
	feedId: string,
	canonicalHost: string,
): FeedItem[] {
	if (!Array.isArray(payload)) {
		return [];
	}
	const items: FeedItem[] = [];
	for (const entry of payload) {
		const item = mapArchivePostToFeedItem(entry, feedId, canonicalHost);
		if (item !== null) {
			items.push(item);
		}
	}
	return items;
}

/**
 * Read `body_html` off a single-post JSON payload (`/api/v1/posts/<slug>`).
 * Returns null when the payload is not a record or the field is missing/empty,
 * so a teaser-only or malformed response leaves the body untouched.
 */
export function readPostBodyHtml(payload: unknown): string | null {
	if (!isRecord(payload)) {
		return null;
	}
	const value = payload["body_html"];
	if (typeof value !== "string" || value.length === 0) {
		return null;
	}
	return value;
}

/**
 * Derive the `<host>` and `<slug>` from a Substack `/p/<slug>` permalink so
 * `fetchBody` can build the single-post API URL. Returns null when the URL does
 * not parse or carries no `/p/<slug>` segment. The slug is the path segment
 * after `/p/`; any trailing path or query is ignored.
 */
export function hostAndSlugFromPostUrl(
	url: string,
): { host: string; slug: string } | null {
	let parsed: URL;
	try {
		parsed = new URL(url);
	} catch {
		return null;
	}
	const match = /\/p\/([^/?#]+)/.exec(parsed.pathname);
	const slug = match?.[1];
	if (slug === undefined || slug.length === 0) {
		return null;
	}
	return { host: parsed.hostname, slug: decodeURIComponent(slug) };
}