diff --git a/__tests__/feed-resolver.test.ts b/__tests__/feed-resolver.test.ts new file mode 100644 index 0000000..f3bc232 --- /dev/null +++ b/__tests__/feed-resolver.test.ts @@ -0,0 +1,352 @@ +import * as fs from "fs"; +import * as path from "path"; +import { FeedResolver, ResolveError } from "../feed-resolver"; +import type { HttpFetcher, HttpRequest, HttpResponse } from "../feed-source"; + +/** A canned response, addressed by exact request URL. */ +interface Canned { + status: number; + headers?: Record; + json?: unknown; + text?: string; +} + +/** Builds a full HttpResponse from the canned partial. */ +function toResponse(c: Canned): HttpResponse { + return { + status: c.status, + headers: c.headers ?? {}, + json: c.json ?? null, + text: c.text ?? "", + arrayBuffer: new ArrayBuffer(0), + }; +} + +/** + * A stub fetcher that answers from a URL -> response map and records every URL + * it was asked for, in order. An unmapped URL is a test bug, so it throws. + */ +function makeFetcher(map: Record): { fetcher: HttpFetcher; urls: string[] } { + const urls: string[] = []; + const fetcher: HttpFetcher = (req: HttpRequest) => { + urls.push(req.url); + const canned = map[req.url]; + if (canned === undefined) { + return Promise.reject(new Error(`unexpected fetch: ${req.url}`)); + } + return Promise.resolve(toResponse(canned)); + }; + return { fetcher, urls }; +} + +/** Loads the trimmed Substack public_profile fixture as parsed JSON. */ +function loadProfileFixture(): unknown { + const file = path.join(__dirname, "fixtures", "substack-profile.json"); + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +const PROFILE_URL = "https://substack.com/api/v1/user/lxxwriter/public_profile"; + +describe("FeedResolver: Substack @handle", () => { + it("resolves @handle via the profile API to the custom-domain feed", async () => { + const profile = loadProfileFixture(); + const { fetcher, urls } = makeFetcher({ + [PROFILE_URL]: { status: 200, json: profile }, + // The custom-domain feed answers directly (already canonical). + "https://the.lxxscrolls.com/feed": { status: 200, headers: { "Content-Type": "application/rss+xml" } }, + }); + const resolver = new FeedResolver(fetcher); + + const result = await resolver.resolve("@lxxwriter"); + + expect(result.sourceType).toBe("substack"); + expect(result.canonicalHost).toBe("the.lxxscrolls.com"); + expect(result.feedUrl).toBe("https://the.lxxscrolls.com/feed"); + expect(result.handle).toBe("lxxwriter"); + // The profile API was actually called. + expect(urls[0]).toBe(PROFILE_URL); + }); + + it("accepts the substack.com/@handle form and hits the same profile API", async () => { + const profile = loadProfileFixture(); + const { fetcher, urls } = makeFetcher({ + [PROFILE_URL]: { status: 200, json: profile }, + "https://the.lxxscrolls.com/feed": { status: 200, headers: { "Content-Type": "text/xml" } }, + }); + const resolver = new FeedResolver(fetcher); + + const result = await resolver.resolve("https://substack.com/@lxxwriter"); + + expect(result.handle).toBe("lxxwriter"); + expect(result.canonicalHost).toBe("the.lxxscrolls.com"); + expect(urls).toContain(PROFILE_URL); + }); + + it("falls back to .substack.com when the profile has no custom domain", async () => { + // Inline canned profile: subdomain only, no custom_domain. + const profile = { + handle: "plainpub", + primaryPublication: { subdomain: "plainpub", custom_domain: null }, + }; + const subProfileUrl = "https://substack.com/api/v1/user/plainpub/public_profile"; + const { fetcher } = makeFetcher({ + [subProfileUrl]: { status: 200, json: profile }, + "https://plainpub.substack.com/feed": { status: 200, headers: { "Content-Type": "application/xml" } }, + }); + const resolver = new FeedResolver(fetcher); + + const result = await resolver.resolve("@plainpub"); + + expect(result.sourceType).toBe("substack"); + expect(result.canonicalHost).toBe("plainpub.substack.com"); + expect(result.feedUrl).toBe("https://plainpub.substack.com/feed"); + }); + + it("raises ResolveError when the profile has no usable publication", async () => { + const subProfileUrl = "https://substack.com/api/v1/user/ghost/public_profile"; + const { fetcher } = makeFetcher({ + [subProfileUrl]: { status: 200, json: { handle: "ghost" } }, + }); + const resolver = new FeedResolver(fetcher); + + await expect(resolver.resolve("@ghost")).rejects.toBeInstanceOf(ResolveError); + await expect(resolver.resolve("@ghost")).rejects.toThrow(/no publication/i); + }); + + it("raises ResolveError when the profile API returns a non-2xx status", async () => { + const subProfileUrl = "https://substack.com/api/v1/user/missing/public_profile"; + const { fetcher } = makeFetcher({ + [subProfileUrl]: { status: 404 }, + }); + const resolver = new FeedResolver(fetcher); + + await expect(resolver.resolve("@missing")).rejects.toThrow(/status 404/); + }); + + it("reads the publication from publicationUsers when primaryPublication is absent", async () => { + const profile = { + handle: "viauser", + publicationUsers: [ + { is_primary: false, publication: { subdomain: "secondary" } }, + { is_primary: true, publication: { subdomain: "primarypub", custom_domain: "blog.example.com" } }, + ], + }; + const url = "https://substack.com/api/v1/user/viauser/public_profile"; + const { fetcher } = makeFetcher({ + [url]: { status: 200, json: profile }, + "https://blog.example.com/feed": { status: 200, headers: { "Content-Type": "application/rss+xml" } }, + }); + const resolver = new FeedResolver(fetcher); + + const result = await resolver.resolve("@viauser"); + + // The is_primary entry's custom domain wins, not the first entry. + expect(result.canonicalHost).toBe("blog.example.com"); + }); +}); + +describe("FeedResolver: Substack subdomain and post URLs", () => { + it("resolves a *.substack.com subdomain to host/feed", async () => { + const { fetcher } = makeFetcher({ + "https://kevin.substack.com/feed": { status: 200, headers: { "Content-Type": "application/rss+xml" } }, + }); + const resolver = new FeedResolver(fetcher); + + const result = await resolver.resolve("kevin.substack.com"); + + expect(result.sourceType).toBe("substack"); + expect(result.canonicalHost).toBe("kevin.substack.com"); + expect(result.feedUrl).toBe("https://kevin.substack.com/feed"); + expect(result.handle).toBeNull(); + }); + + it("classifies a Substack post URL (/p/) as substack with host/feed", async () => { + const { fetcher } = makeFetcher({ + "https://the.lxxscrolls.com/feed": { status: 200, headers: { "Content-Type": "text/xml" } }, + }); + const resolver = new FeedResolver(fetcher); + + const result = await resolver.resolve("https://the.lxxscrolls.com/p/daniel-6"); + + expect(result.sourceType).toBe("substack"); + expect(result.canonicalHost).toBe("the.lxxscrolls.com"); + expect(result.feedUrl).toBe("https://the.lxxscrolls.com/feed"); + }); +}); + +describe("FeedResolver: generic feed URLs", () => { + it("classifies a custom-domain /feed URL as generic", async () => { + const { fetcher } = makeFetcher({ + "https://blog.example.com/feed": { status: 200, headers: { "Content-Type": "application/rss+xml" } }, + }); + const resolver = new FeedResolver(fetcher); + + const result = await resolver.resolve("https://blog.example.com/feed"); + + expect(result.sourceType).toBe("generic"); + expect(result.canonicalHost).toBe("blog.example.com"); + expect(result.feedUrl).toBe("https://blog.example.com/feed"); + expect(result.handle).toBeNull(); + }); + + it("classifies a /feed/podcast URL as generic", async () => { + const { fetcher } = makeFetcher({ + "https://godjourney.example/feed/podcast": { status: 200, headers: { "Content-Type": "application/rss+xml" } }, + }); + const resolver = new FeedResolver(fetcher); + + const result = await resolver.resolve("https://godjourney.example/feed/podcast"); + + expect(result.sourceType).toBe("generic"); + expect(result.feedUrl).toBe("https://godjourney.example/feed/podcast"); + }); + + it("probes host/feed for a bare host and accepts an XML response as generic", async () => { + const { fetcher, urls } = makeFetcher({ + "https://news.example.org/feed": { + status: 200, + headers: { "Content-Type": "text/html" }, + text: "", + }, + }); + const resolver = new FeedResolver(fetcher); + + const result = await resolver.resolve("news.example.org"); + + expect(result.sourceType).toBe("generic"); + expect(result.canonicalHost).toBe("news.example.org"); + expect(result.feedUrl).toBe("https://news.example.org/feed"); + expect(urls).toEqual(["https://news.example.org/feed"]); + }); + + it("raises ResolveError when a probed host returns non-XML", async () => { + const { fetcher } = makeFetcher({ + "https://notafeed.example/feed": { + status: 200, + headers: { "Content-Type": "text/html" }, + text: "nope", + }, + }); + const resolver = new FeedResolver(fetcher); + + // A real HTML page (not XML) at /feed must not be accepted as a feed. + await expect(resolver.resolve("notafeed.example")).rejects.toThrow(/no feed found/i); + }); +}); + +describe("FeedResolver: redirect walking", () => { + it("walks a 301 chain from a subdomain to the moved custom domain", async () => { + const { fetcher, urls } = makeFetcher({ + "https://kevin.substack.com/feed": { + status: 301, + headers: { Location: "https://the.kevin.com/feed" }, + }, + "https://the.kevin.com/feed": { + status: 200, + headers: { "Content-Type": "application/rss+xml" }, + }, + }); + const resolver = new FeedResolver(fetcher); + + const result = await resolver.resolve("kevin.substack.com"); + + // Canonical host is where the chain landed, not where it started. + expect(result.canonicalHost).toBe("the.kevin.com"); + expect(result.feedUrl).toBe("https://the.kevin.com/feed"); + expect(urls).toEqual([ + "https://kevin.substack.com/feed", + "https://the.kevin.com/feed", + ]); + }); + + it("resolves a relative Location against the current URL", async () => { + const { fetcher } = makeFetcher({ + "https://blog.example.com/feed": { + status: 302, + headers: { Location: "/rss" }, + }, + "https://blog.example.com/rss": { + status: 200, + headers: { "Content-Type": "application/rss+xml" }, + }, + }); + const resolver = new FeedResolver(fetcher); + + const result = await resolver.resolve("https://blog.example.com/feed"); + + expect(result.sourceType).toBe("generic"); + expect(result.feedUrl).toBe("https://blog.example.com/rss"); + expect(result.canonicalHost).toBe("blog.example.com"); + }); + + it("treats a direct 200 (auto-followed) as canonical at the requested host", async () => { + // No Location header to walk: the requestUrl auto-follow case. + const { fetcher, urls } = makeFetcher({ + "https://kevin.substack.com/feed": { + status: 200, + headers: { "Content-Type": "application/rss+xml" }, + }, + }); + const resolver = new FeedResolver(fetcher); + + const result = await resolver.resolve("kevin.substack.com"); + + expect(result.canonicalHost).toBe("kevin.substack.com"); + expect(urls).toEqual(["https://kevin.substack.com/feed"]); + }); + + it("raises ResolveError when the redirect hop cap is exceeded", async () => { + // Each hop points at the next; the chain is longer than the cap of 5. + const map: Record = {}; + for (let i = 0; i < 10; i += 1) { + map[`https://hop${i}.example/feed`] = { + status: 301, + headers: { Location: `https://hop${i + 1}.example/feed` }, + }; + } + const { fetcher } = makeFetcher(map); + const resolver = new FeedResolver(fetcher); + + await expect(resolver.resolve("https://hop0.example/feed")).rejects.toBeInstanceOf(ResolveError); + await expect(resolver.resolve("https://hop0.example/feed")).rejects.toThrow(/too many redirects/i); + }); + + it("raises ResolveError on a redirect with no Location header", async () => { + const { fetcher } = makeFetcher({ + "https://blog.example.com/feed": { status: 301, headers: {} }, + }); + const resolver = new FeedResolver(fetcher); + + await expect(resolver.resolve("https://blog.example.com/feed")).rejects.toThrow(/no location/i); + }); +}); + +describe("FeedResolver: input validation and classification", () => { + it("raises ResolveError on empty input", async () => { + const { fetcher } = makeFetcher({}); + const resolver = new FeedResolver(fetcher); + + await expect(resolver.resolve(" ")).rejects.toThrow(/empty input/i); + }); + + it("raises ResolveError on substack.com without a handle", async () => { + const { fetcher } = makeFetcher({}); + const resolver = new FeedResolver(fetcher); + + await expect(resolver.resolve("https://substack.com/about")).rejects.toThrow(/not a resolvable/i); + }); + + it("attaches the original error as cause when the profile fetch throws", async () => { + const boom = new Error("network down"); + const fetcher: HttpFetcher = () => Promise.reject(boom); + const resolver = new FeedResolver(fetcher); + + try { + await resolver.resolve("@anyone"); + throw new Error("should have thrown"); + } catch (err) { + expect(err).toBeInstanceOf(ResolveError); + expect((err as ResolveError).cause).toBe(boom); + } + }); +}); diff --git a/__tests__/feed-xml.test.ts b/__tests__/feed-xml.test.ts new file mode 100644 index 0000000..6a7c0f3 --- /dev/null +++ b/__tests__/feed-xml.test.ts @@ -0,0 +1,203 @@ +/** @jest-environment jsdom */ + +import { readFileSync } from "fs"; +import { join } from "path"; + +import { parseFeed, FeedParseError } from "../feed-xml"; + +const FIXTURES = join(__dirname, "fixtures"); + +function fixture(name: string): string { + return readFileSync(join(FIXTURES, name), "utf8"); +} + +describe("parseFeed", () => { + describe("RSS Substack item", () => { + const parsed = parseFeed(fixture("substack-item.xml")); + const item = parsed.items[0]; + + it("reads the channel title and link", () => { + expect(parsed.feedTitle).toBe("Coach Jon McLernon"); + expect(parsed.feedLink).toBe("https://jonathanmclernon.substack.com"); + }); + + it("yields exactly one item", () => { + expect(parsed.items).toHaveLength(1); + }); + + it("reads guid, link, and title", () => { + expect(item).toBeDefined(); + if (item === undefined) { + throw new Error("expected an item"); + } + expect(item.guid).toBe("https://jonathanmclernon.substack.com/p/if-im-not-saved-by-the-system-then"); + expect(item.link).toBe("https://jonathanmclernon.substack.com/p/if-im-not-saved-by-the-system-then"); + expect(item.title).toBe("If I am not saved by the system, then how am I saved?"); + }); + + it("reads dc:creator as the author", () => { + expect(item?.author).toBe("Jonathan McLernon"); + }); + + it("prefers content:encoded over description for contentHtml", () => { + expect(item?.contentHtml).toContain("

This is one of those deeply personal and hard to write articles.

"); + expect(item?.contentHtml).toContain("Jesus, God the Father, and the Holy Spirit"); + // description is a different, shorter string. + expect(item?.description).toBe("How I came to understand salvation apart from institutional belonging."); + expect(item?.contentHtml).not.toBe(item?.description); + }); + + it("collects the categories", () => { + expect(item?.categories).toEqual(["Faith", "Salvation"]); + }); + + it("reads the cover-image enclosure with type and length", () => { + expect(item?.enclosure).toEqual({ + url: "https://substackcdn.com/image/cover.png", + type: "image/jpeg", + length: 0, + }); + }); + + it("normalizes the RFC 822 pubDate to ISO 8601", () => { + expect(item?.pubDateIso).toBe("2026-06-14T14:24:30.000Z"); + }); + }); + + describe("RSS podcast item", () => { + const parsed = parseFeed(fixture("podcast-item.xml")); + const item = parsed.items[0]; + + it("reads the audio enclosure as url, type, and length", () => { + expect(item?.enclosure).toEqual({ + url: "https://media.blubrry.com/the_god_journey/www.thegodjourney.com/audio/2026/260612.mp3", + type: "audio/mpeg", + length: 43990159, + }); + }); + + it("reads the WordPress non-permalink guid", () => { + expect(item?.guid).toBe("https://www.thegodjourney.com/?p=31764"); + }); + + it("leaves contentHtml null when content:encoded is absent, keeping the HTML in description", () => { + // The podcast item ships its body in , not content:encoded. + // The parser keeps the two fields separate; the source layer chooses the + // fallback. So contentHtml is null and description carries the HTML. + expect(item?.contentHtml).toBeNull(); + expect(item?.description).toBe("

Wayne, Kyle, and Joni continue their conversation about patriarchy.

"); + }); + + it("collects categories including the namespaced-feed values", () => { + expect(item?.categories).toEqual(["Show content", "patriarchy"]); + }); + + it("normalizes a +0000 offset pubDate to ISO 8601", () => { + expect(item?.pubDateIso).toBe("2026-06-12T13:30:16.000Z"); + }); + }); + + describe("Atom entry", () => { + const parsed = parseFeed(fixture("atom-entry.xml")); + const item = parsed.items[0]; + + it("reads the feed title and alternate link", () => { + expect(parsed.feedTitle).toBe("Example Atom feed"); + expect(parsed.feedLink).toBe("https://example.com/"); + }); + + it("reads the entry id as guid and the alternate link as link", () => { + expect(item?.guid).toBe("urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a"); + expect(item?.link).toBe("https://example.com/posts/first"); + }); + + it("reads the author name from the nested ", () => { + expect(item?.author).toBe("Ada Lovelace"); + }); + + it("reads as contentHtml and as description", () => { + expect(item?.contentHtml).toBe("

The full Atom body content lives here.

"); + expect(item?.description).toBe("A short Atom summary line."); + }); + + it("reads categories from the term attribute", () => { + expect(item?.categories).toEqual(["essays", "history"]); + }); + + it("reads the rel=enclosure link as the enclosure", () => { + expect(item?.enclosure).toEqual({ + url: "https://example.com/audio/first.mp3", + type: "audio/mpeg", + length: 12345, + }); + }); + + it("prefers over and normalizes to ISO", () => { + expect(item?.pubDateIso).toBe("2026-06-10T09:00:00.000Z"); + }); + }); + + describe("date normalization edge cases", () => { + it("returns null for an unparseable pubDate rather than throwing", () => { + const xml = + 'T' + + "xnot a date"; + const parsed = parseFeed(xml); + expect(parsed.items[0]?.pubDateIso).toBeNull(); + }); + + it("returns null for an absent pubDate", () => { + const xml = + 'T' + + "x"; + const parsed = parseFeed(xml); + expect(parsed.items[0]?.pubDateIso).toBeNull(); + }); + }); + + describe("malformed and rootless input", () => { + it("throws FeedParseError on malformed XML", () => { + expect(() => parseFeed("oops</rss>")).toThrow(FeedParseError); + }); + + it("throws FeedParseError on empty input", () => { + expect(() => parseFeed(" ")).toThrow(FeedParseError); + }); + + it("throws FeedParseError when there is no channel or feed root", () => { + const xml = '<?xml version="1.0"?><html><body><p>not a feed</p></body></html>'; + expect(() => parseFeed(xml)).toThrow(FeedParseError); + }); + + it("surfaces a reason in the FeedParseError message", () => { + let caught: unknown; + try { + parseFeed("<rss><channel><title>oops</rss>"); + } catch (err: unknown) { + caught = err; + } + expect(caught).toBeInstanceOf(FeedParseError); + expect((caught as Error).message.length).toBeGreaterThan(0); + }); + }); + + describe("enclosure resolution", () => { + it("returns null when an enclosure has no url", () => { + const xml = + '<?xml version="1.0"?><rss version="2.0"><channel><title>T' + + 'x'; + expect(parseFeed(xml).items[0]?.enclosure).toBeNull(); + }); + + it("returns null length when the length attribute is non-numeric", () => { + const xml = + 'T' + + 'x'; + expect(parseFeed(xml).items[0]?.enclosure).toEqual({ + url: "https://a/b.mp3", + type: "audio/mpeg", + length: null, + }); + }); + }); +}); diff --git a/__tests__/fixtures/atom-entry.xml b/__tests__/fixtures/atom-entry.xml new file mode 100644 index 0000000..c7717ba --- /dev/null +++ b/__tests__/fixtures/atom-entry.xml @@ -0,0 +1,22 @@ + + + Example Atom feed + + + 2026-06-12T18:30:00Z + + An Atom entry with a body + urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a + + + 2026-06-10T09:00:00Z + 2026-06-11T10:15:00Z + + Ada Lovelace + + + + A short Atom summary line. + The full Atom body content lives here.

]]>
+
+
diff --git a/__tests__/fixtures/podcast-item.xml b/__tests__/fixtures/podcast-item.xml new file mode 100644 index 0000000..6fec473 --- /dev/null +++ b/__tests__/fixtures/podcast-item.xml @@ -0,0 +1,27 @@ + + + + The God Journey + + https://www.thegodjourney.com + The adventure of living loved + + Patriarchy diminishes us all (#1039) + https://www.thegodjourney.com/2026/06/12/patriarchy-diminishes-us-all-1039/ + Fri, 12 Jun 2026 13:30:16 +0000 + https://www.thegodjourney.com/?p=31764 + + + + Wayne, Kyle, and Joni continue their conversation about patriarchy.

]]>
+ + 22 + 23 + Patriarchy diminishes us all (#1039) +
+
+
diff --git a/__tests__/fixtures/substack-content-encoded.html b/__tests__/fixtures/substack-content-encoded.html new file mode 100644 index 0000000..0613b22 --- /dev/null +++ b/__tests__/fixtures/substack-content-encoded.html @@ -0,0 +1,27 @@ +

This is one of those deeply personal and hard to write articles.

+

In the last few articles in this series, I have been slowly working through what I believe about Jesus, God the Father, and the Holy Spirit.

+

1 - Who Jesus Actually is

+

+

How Am I Saved?

+

But underneath all of that, there was another question. A more deeply personal and vulnerable one.

+

What is my salvation actually resting on?

+

Some beliefs are never written clearly enough to challenge. They are absorbed. They hang in the air.

+
+ +
A quiet chapel at dawn
+
+
A quiet chapel at dawn, where the question first surfaced.
+
+

So when that system began to crack, the question underneath it all became impossible to avoid.

+
+
+

Thanks for reading! Subscribe at no charge to receive new articles.

+
Subscribe
+
+
+

Share

+

Here is a small snippet that shaped my thinking:

+
def grace(faith):
+    return faith is not None
+
+

The end of the matter, when all has been heard.

diff --git a/__tests__/fixtures/substack-item.xml b/__tests__/fixtures/substack-item.xml new file mode 100644 index 0000000..8a68a65 --- /dev/null +++ b/__tests__/fixtures/substack-item.xml @@ -0,0 +1,20 @@ + + + + <![CDATA[Coach Jon McLernon]]> + + https://jonathanmclernon.substack.com + + <![CDATA[If I am not saved by the system, then how am I saved?]]> + + https://jonathanmclernon.substack.com/p/if-im-not-saved-by-the-system-then + https://jonathanmclernon.substack.com/p/if-im-not-saved-by-the-system-then + + Sun, 14 Jun 2026 14:24:30 GMT + + + + This is one of those deeply personal and hard to write articles.

In the last few articles in this series, I have been slowly working through what I believe about Jesus, God the Father, and the Holy Spirit.

]]>
+
+
+
diff --git a/__tests__/fixtures/substack-profile.json b/__tests__/fixtures/substack-profile.json new file mode 100644 index 0000000..444a5d7 --- /dev/null +++ b/__tests__/fixtures/substack-profile.json @@ -0,0 +1,29 @@ +{ + "id": 143604767, + "name": "Kevin Potter", + "handle": "lxxwriter", + "primaryPublication": { + "id": 6714120, + "subdomain": "lxxscrolls", + "custom_domain": "the.lxxscrolls.com", + "custom_domain_optional": false, + "name": "The LXX Scrolls", + "author_id": 143604767 + }, + "publicationUsers": [ + { + "id": 6851817, + "user_id": 143604767, + "publication_id": 6714120, + "role": "admin", + "is_primary": true, + "publication": { + "id": 6714120, + "name": "The LXX Scrolls", + "subdomain": "lxxscrolls", + "custom_domain": "the.lxxscrolls.com" + } + } + ], + "subdomainUrl": "https://lxxwriter.substack.com" +} diff --git a/__tests__/html-converter.test.ts b/__tests__/html-converter.test.ts new file mode 100644 index 0000000..4d87099 --- /dev/null +++ b/__tests__/html-converter.test.ts @@ -0,0 +1,244 @@ +/** @jest-environment jsdom */ +import { readFileSync } from "fs"; +import { join } from "path"; +import { convertHtmlToMarkdown, createConverter } from "../html-converter"; + +const FIXTURE_DIR = join(__dirname, "fixtures"); + +function fixture(name: string): string { + return readFileSync(join(FIXTURE_DIR, name), "utf8"); +} + +describe("convertHtmlToMarkdown: core Markdown", () => { + it("renders headings as atx with the right level", () => { + const md = convertHtmlToMarkdown("

Top

Sub

Deeper

"); + expect(md).toContain("# Top"); + expect(md).toContain("## Sub"); + expect(md).toContain("### Deeper"); + // atx, not setext: no underline rows. + expect(md).not.toMatch(/^=+$/m); + expect(md).not.toMatch(/^-{3,}$/m); + }); + + it("uses '-' as the bullet list marker, never '*'", () => { + const md = convertHtmlToMarkdown("
  • one
  • two
"); + // Turndown pads the marker to a 4-char list indent: "- one". + expect(md).toMatch(/^- {1,3}one$/m); + expect(md).toMatch(/^- {1,3}two$/m); + expect(md.startsWith("-")).toBe(true); + expect(md).not.toContain("* one"); + expect(md).not.toContain("+ one"); + }); + + it("renders bold, italic, and inline links", () => { + const md = convertHtmlToMarkdown( + '

bold and italic and a link

', + ); + expect(md).toContain("**bold**"); + expect(md).toContain("_italic_"); + expect(md).toContain("[a link](https://example.com/x)"); + }); + + it("keeps the code language on a fenced block from pre > code.language-xxx", () => { + const html = '
def f():\n    return 1\n
'; + const md = convertHtmlToMarkdown(html); + expect(md).toContain("```python"); + expect(md).toContain("def f():"); + expect(md).toContain("return 1"); + // fence opens and closes exactly once. + expect(md.match(/```/g)?.length).toBe(2); + }); + + it("falls back to an untagged fence when no language class is present", () => { + const md = convertHtmlToMarkdown("
plain code\n
"); + expect(md).toContain("```\nplain code\n```"); + }); +}); + +describe("convertHtmlToMarkdown: GFM plugin", () => { + it("converts an HTML table to a GFM pipe table", () => { + const html = + "" + + "
NameScore
Ann10
Bo7
"; + const md = convertHtmlToMarkdown(html); + expect(md).toContain("| Name | Score |"); + expect(md).toMatch(/\| ?-+ ?\| ?-+ ?\|/); + expect(md).toContain("| Ann | 10 |"); + expect(md).toContain("| Bo | 7 |"); + }); + + it("converts strikethrough via the gfm plugin", () => { + const md = convertHtmlToMarkdown("

gone stays

"); + // turndown-plugin-gfm wraps strikethrough in single tildes. + expect(md).toContain("~gone~"); + expect(md).toContain("stays"); + }); + + it("converts a gfm task list", () => { + const html = + '
  • done
  • ' + + '
  • todo
'; + const md = convertHtmlToMarkdown(html); + expect(md).toContain("[x]"); + expect(md).toContain("[ ]"); + expect(md).toContain("done"); + expect(md).toContain("todo"); + }); +}); + +describe("convertHtmlToMarkdown: figure and figcaption rule", () => { + it("emits the image followed by an italic caption line", () => { + const html = + '
A photo' + + "
The caption text
"; + const md = convertHtmlToMarkdown(html); + expect(md).toContain("![A photo](https://cdn.example.com/p.png)"); + expect(md).toContain("_The caption text_"); + // caption appears after the image. + const imgIdx = md.indexOf("![A photo]"); + const capIdx = md.indexOf("_The caption text_"); + expect(imgIdx).toBeGreaterThanOrEqual(0); + expect(capIdx).toBeGreaterThan(imgIdx); + }); + + it("handles a figure with a linked image and a caption", () => { + const html = + '
alt q' + + "
Linked caption
"; + const md = convertHtmlToMarkdown(html); + expect(md).toContain("![alt q](https://cdn.example.com/q.png)"); + expect(md).toContain("_Linked caption_"); + }); +}); + +describe("convertHtmlToMarkdown: widget stripping and degradation", () => { + it("strips a subscribe widget matched by class name, not by body text", () => { + const html = + "

Real body.

" + + '' + + "

More body.

"; + const md = convertHtmlToMarkdown(html); + expect(md).toContain("Real body."); + expect(md).toContain("More body."); + // the widget's promotional copy is gone. + expect(md).not.toContain("Subscribe now to get every post"); + }); + + it("degrades an unknown link-only widget to its link rather than dropping it", () => { + const html = + ''; + const md = convertHtmlToMarkdown(html); + expect(md).toContain("[Read the full thing](https://example.com/read-more)"); + }); + + it("never matches a widget on body text alone", () => { + // A normal paragraph that happens to contain the word "subscribe" must + // survive: matching is class-based only. + const md = convertHtmlToMarkdown("

You can subscribe at the library for free.

"); + expect(md).toContain("You can subscribe at the library for free."); + }); +}); + +describe("convertHtmlToMarkdown: footnotes", () => { + it("collapses anchor refs to [^n] and appends definitions in stable order", () => { + const html = + '

First claim1 and ' + + 'second claim2.

' + + '

First source.

' + + '

Second source.

'; + const md = convertHtmlToMarkdown(html); + expect(md).toContain("First claim[^1]"); + expect(md).toContain("second claim[^2]"); + expect(md).toContain("[^1]: First source."); + expect(md).toContain("[^2]: Second source."); + // definitions sit at the very end, ref 1 before ref 2. + expect(md.indexOf("[^1]: First source.")).toBeLessThan(md.indexOf("[^2]: Second source.")); + // the definition body appears exactly once: it was not also rendered + // inline where the def container sat in the document. + expect(md.split("First source.").length - 1).toBe(1); + expect(md.split("Second source.").length - 1).toBe(1); + // definitions are the tail of the document. + expect(md.trimEnd().endsWith("[^2]: Second source.")).toBe(true); + }); + + it("orders definitions by first-seen ref, not by document position", () => { + // Ref 2 appears before ref 1 in the prose; definitions are in 1,2 order + // in the source. Output footnote order should follow ref appearance: 2,1. + const html = + '

Alpha2 then beta1.

' + + '

Def one.

' + + '

Def two.

'; + const md = convertHtmlToMarkdown(html); + expect(md.indexOf("[^2]: Def two.")).toBeLessThan(md.indexOf("[^1]: Def one.")); + }); +}); + +describe("convertHtmlToMarkdown: determinism", () => { + it("produces byte-identical output when run twice on the same input", () => { + const html = + "

Title

Bold text with a " + + '1 ref.

' + + '
i
cap
' + + '' + + '

note body

'; + const first = convertHtmlToMarkdown(html); + const second = convertHtmlToMarkdown(html); + expect(second).toBe(first); + }); + + it("does not leak footnote state between separate conversions", () => { + const a = convertHtmlToMarkdown( + '

A1

' + + '

note A

', + ); + const b = convertHtmlToMarkdown("

Plain B with no footnotes.

"); + expect(a).toContain("[^1]: note A"); + expect(b).not.toContain("[^1]"); + expect(b).toBe("Plain B with no footnotes."); + }); +}); + +describe("convertHtmlToMarkdown: edge cases", () => { + it("returns an empty string for empty or whitespace-only input", () => { + expect(convertHtmlToMarkdown("")).toBe(""); + expect(convertHtmlToMarkdown(" \n ")).toBe(""); + }); + + it("collapses runs of blank lines so output is tidy", () => { + const md = convertHtmlToMarkdown("

one

two

"); + expect(md).not.toMatch(/\n{3,}/); + }); +}); + +describe("createConverter", () => { + it("returns a working TurndownService configured with atx and '-' bullets", () => { + const service = createConverter(); + const md = service.turndown("

Hi

  • x
"); + expect(md).toContain("# Hi"); + expect(md).toMatch(/^- {1,3}x$/m); + }); +}); + +describe("convertHtmlToMarkdown: real Substack content:encoded fixture", () => { + it("converts a trimmed real Substack body to non-empty Markdown without throwing", () => { + const html = fixture("substack-content-encoded.html"); + let md = ""; + expect(() => { + md = convertHtmlToMarkdown(html); + }).not.toThrow(); + expect(md.length).toBeGreaterThan(0); + // structural spot-checks against the real content. The h2 wraps a + // , so Turndown keeps the emphasis inside the atx heading. + expect(md).toContain("## **How Am I Saved?**"); + expect(md).toContain("_A quiet chapel at dawn"); + expect(md).toContain("```python"); + // the subscribe widget chrome is stripped, link and all. + expect(md).not.toContain("Subscribe at no charge"); + expect(md).not.toContain("(https://frankviola.substack.com/subscribe?)"); + // the share button widget is stripped too. + expect(md).not.toContain("action=share"); + // fixture conversion is deterministic too. + expect(convertHtmlToMarkdown(html)).toBe(md); + }); +}); diff --git a/feed-resolver.ts b/feed-resolver.ts new file mode 100644 index 0000000..36de379 --- /dev/null +++ b/feed-resolver.ts @@ -0,0 +1,417 @@ +/** + * Classifies a user-supplied feed input and resolves it to a canonical, + * fetchable feed. + * + * The user can type a Substack handle (`@kevin` or `substack.com/@kevin`), a + * Substack subdomain (`kevin.substack.com`), a Substack post URL + * (`.../p/some-slug`), a custom domain that fronts a Substack, or a plain feed + * URL (`.../feed`, `.../rss`). This resolver figures out which of those it is + * and produces the one canonical feed URL the source should enumerate from. + * + * It does NOT parse feed items. Its only job is classification plus + * canonicalization: pick the source type, find the canonical host, and return + * the feed URL. The `FeedSource` reads items from that URL afterwards. + * + * Two host-canonicalization wrinkles it handles: + * + * 1. Substack handles. A handle does not name a host, so the resolver calls + * the public profile API to learn the publication's custom domain or + * `.substack.com`. + * 2. Redirects. A `kevin.substack.com` publication that later moved to a + * custom domain answers feed requests with a 3xx to the custom domain. The + * resolver walks those redirects manually (hop cap 5) so the canonical host + * reflects where the feed actually lives. It also tolerates the + * `requestUrl` auto-follow case, where a request returns a direct 200 with + * no redirect to walk; there the requested host is treated as canonical. + * + * The resolver depends only on the injected `HttpFetcher`, never on Obsidian, + * so it is unit-testable with a stub fetcher. + */ + +import type { HttpFetcher, HttpResponse, SourceType } from "./feed-source"; + +/** Maximum number of 3xx redirects to follow before giving up. */ +const MAX_REDIRECT_HOPS = 5; + +/** Lowest status code that counts as a redirect. */ +const HTTP_REDIRECT_MIN = 300; + +/** One past the highest status code that counts as a redirect. */ +const HTTP_REDIRECT_MAX = 400; + +/** The host all Substack handle and profile-API requests go through. */ +const SUBSTACK_HOST = "substack.com"; + +/** Suffix that marks a Substack-hosted subdomain. */ +const SUBSTACK_SUFFIX = ".substack.com"; + +/** Path suffixes that unambiguously name a feed without probing. */ +const FEED_PATH_SUFFIXES = ["/feed", "/rss", "/feed/podcast"]; + +/** The result of resolving an input to a canonical, fetchable feed. */ +export interface ResolverResult { + /** Which source implementation should handle this feed. */ + sourceType: SourceType; + /** Canonical host the feed (and any API calls) resolve to. */ + canonicalHost: string; + /** Canonical feed URL items are enumerated from. */ + feedUrl: string; + /** Substack handle when the input named one, else null. */ + handle: string | null; +} + +/** + * Raised when an input cannot be resolved to a feed: malformed input, a profile + * lookup that returned nothing usable, or a redirect chain that exceeded the + * hop cap. The message states the specific reason; the original cause (when + * there is one) is attached so callers can surface it. + */ +export class ResolveError extends Error { + /** The underlying error, when this wraps one. */ + readonly cause?: unknown; + + constructor(message: string, cause?: unknown) { + super(message); + this.name = "ResolveError"; + this.cause = cause; + } +} + +/** Shape of the Substack public profile fields the resolver reads. */ +interface ProfilePublication { + subdomain?: unknown; + custom_domain?: unknown; +} + +/** Looks up a header case-insensitively; headers may use any casing. */ +function getHeader(headers: Record, name: string): string | undefined { + const target = name.toLowerCase(); + for (const key of Object.keys(headers)) { + if (key.toLowerCase() === target) { + return headers[key]; + } + } + return undefined; +} + +/** True when a status code is in the 3xx redirect range. */ +function isRedirect(status: number): boolean { + return status >= HTTP_REDIRECT_MIN && status < HTTP_REDIRECT_MAX; +} + +/** + * Parses a URL, throwing a ResolveError (rather than a bare TypeError) when the + * input is not a usable absolute URL. + */ +function parseUrl(raw: string): URL { + try { + return new URL(raw); + } catch (err) { + throw new ResolveError(`Not a valid URL: ${raw}`, err); + } +} + +/** + * Adds a scheme to a bare host or host/path so it parses as an absolute URL. + * Leaves inputs that already carry a scheme untouched. + */ +function ensureScheme(raw: string): string { + return /^https?:\/\//i.test(raw) ? raw : `https://${raw}`; +} + +/** True when a host is `substack.com` itself (not a subdomain of it). */ +function isSubstackApex(host: string): boolean { + return host.toLowerCase() === SUBSTACK_HOST; +} + +/** True when a host is `.substack.com`. */ +function isSubstackSubdomain(host: string): boolean { + return host.toLowerCase().endsWith(SUBSTACK_SUFFIX); +} + +/** True when a path looks like a Substack post permalink (`/p/`). */ +function isSubstackPostPath(pathname: string): boolean { + return /^\/p\/[^/]+/.test(pathname); +} + +/** True when a path ends in a recognized feed suffix. */ +function hasFeedPathSuffix(pathname: string): boolean { + const lower = pathname.replace(/\/$/, "").toLowerCase(); + return FEED_PATH_SUFFIXES.some((suffix) => lower.endsWith(suffix)); +} + +/** True when a content-type names an XML/RSS/Atom payload. */ +function looksLikeXmlContentType(contentType: string | undefined): boolean { + if (contentType === undefined) { + return false; + } + const lower = contentType.toLowerCase(); + return lower.includes("xml") || lower.includes("rss") || lower.includes("atom"); +} + +/** + * True when a response body begins like a feed: an XML declaration or a + * recognized feed root element. An HTML document (``, ``) + * must not match, so a plain `<` prefix is not enough. + */ +function bodyLooksLikeFeed(text: string): boolean { + const head = text.trimStart().slice(0, 200).toLowerCase(); + return ( + head.startsWith(".substack.com`. Returns null when + * neither field is a usable string. + */ +function hostFromPublication(pub: ProfilePublication | undefined): string | null { + if (pub === undefined) { + return null; + } + if (typeof pub.custom_domain === "string" && pub.custom_domain.trim().length > 0) { + return pub.custom_domain.trim().toLowerCase(); + } + if (typeof pub.subdomain === "string" && pub.subdomain.trim().length > 0) { + return `${pub.subdomain.trim().toLowerCase()}${SUBSTACK_SUFFIX}`; + } + return null; +} + +/** + * Reads the publication object out of a public_profile JSON response. Prefers + * `primaryPublication`, then the primary entry of `publicationUsers`, then the + * first `publicationUsers` entry. Returns undefined when the JSON has none. + */ +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function publicationFromProfile(json: unknown): ProfilePublication | undefined { + if (!isRecord(json)) { + return undefined; + } + + const primary = json["primaryPublication"]; + if (isRecord(primary)) { + return primary; + } + + const users = json["publicationUsers"]; + if (Array.isArray(users)) { + const list: unknown[] = users; + const primaryUser: unknown = + list.find((u): u is Record => isRecord(u) && u["is_primary"] === true) ?? + list[0]; + if (isRecord(primaryUser)) { + const pub = primaryUser["publication"]; + if (isRecord(pub)) { + return pub; + } + } + } + + return undefined; +} + +export class FeedResolver { + private readonly fetcher: HttpFetcher; + + constructor(fetcher: HttpFetcher) { + this.fetcher = fetcher; + } + + /** + * Classifies `input` and resolves it to a canonical feed. + * + * Dispatch order: + * 1. A Substack handle (`@x`, `substack.com/@x`) -> profile API lookup. + * 2. A URL whose host is `*.substack.com` -> Substack, feed at host/feed. + * 3. A Substack post URL (`/p/`) on any host -> Substack, host/feed. + * 4. A URL whose path already names a feed (`/feed`, `/rss`, ...) -> generic. + * 5. Any other URL -> probe `host/feed`; XML means generic. + * + * In cases 2 to 5 the canonical host is derived by walking redirects from + * the candidate feed URL, so a publication that moved hosts resolves to + * where its feed actually lives. + */ + async resolve(input: string): Promise { + const trimmed = input.trim(); + if (trimmed.length === 0) { + throw new ResolveError("Empty input"); + } + + const handle = extractHandle(trimmed); + if (handle !== null) { + return this.resolveHandle(handle); + } + + const url = parseUrl(ensureScheme(trimmed)); + + if (isSubstackApex(url.hostname)) { + // `substack.com/...` that is not an `@handle` is not a feed we can + // canonicalize without more information. + throw new ResolveError(`Not a resolvable Substack input: ${trimmed}`); + } + + if (isSubstackSubdomain(url.hostname) || isSubstackPostPath(url.pathname)) { + return this.resolveSubstackHost(url.hostname, null); + } + + if (hasFeedPathSuffix(url.pathname)) { + return this.resolveGenericFeedUrl(url.toString()); + } + + return this.probeForFeed(url.hostname); + } + + /** + * Resolves a Substack handle: GET the public profile, read the publication's + * custom domain or subdomain, then canonicalize that host (walking any + * redirect to a moved custom domain). + */ + private async resolveHandle(handle: string): Promise { + const profileUrl = `https://${SUBSTACK_HOST}/api/v1/user/${encodeURIComponent(handle)}/public_profile`; + let response: HttpResponse; + try { + response = await this.fetcher({ url: profileUrl }); + } catch (err) { + throw new ResolveError(`Profile lookup failed for @${handle}`, err); + } + if (response.status < 200 || response.status >= 300) { + throw new ResolveError(`Profile lookup for @${handle} returned status ${response.status}`); + } + + const publication = publicationFromProfile(response.json); + const host = hostFromPublication(publication); + if (host === null) { + throw new ResolveError(`No publication found for @${handle}`); + } + return this.resolveSubstackHost(host, handle); + } + + /** + * Canonicalizes a Substack host by walking redirects from `host/feed`, then + * returns the canonical feed URL. The host the chain lands on is canonical. + */ + private async resolveSubstackHost(host: string, handle: string | null): Promise { + const candidate = `https://${host.toLowerCase()}/feed`; + const canonicalHost = await this.walkToCanonicalHost(candidate); + return { + sourceType: "substack", + canonicalHost, + feedUrl: `https://${canonicalHost}/feed`, + handle, + }; + } + + /** + * Resolves a plain feed URL (one whose path already names a feed) as a + * generic feed, walking redirects to find the canonical host. + */ + private async resolveGenericFeedUrl(feedUrl: string): Promise { + const finalUrl = await this.walkToCanonicalUrl(feedUrl); + return { + sourceType: "generic", + canonicalHost: finalUrl.hostname.toLowerCase(), + feedUrl: finalUrl.toString(), + handle: null, + }; + } + + /** + * Probes `host/feed` for a feed when the input gave no feed path. A response + * that reads as XML resolves as a generic feed at the (post-redirect) + * canonical host. Anything else is not a feed we can use. + */ + private async probeForFeed(host: string): Promise { + const candidate = `https://${host.toLowerCase()}/feed`; + const { finalUrl, response } = await this.walkRedirects(candidate); + const contentType = getHeader(response.headers, "Content-Type"); + const isXml = looksLikeXmlContentType(contentType) || bodyLooksLikeFeed(response.text); + if (response.status >= 200 && response.status < 300 && isXml) { + return { + sourceType: "generic", + canonicalHost: finalUrl.hostname.toLowerCase(), + feedUrl: finalUrl.toString(), + handle: null, + }; + } + throw new ResolveError(`No feed found at ${candidate}`); + } + + /** Walks redirects from a URL and returns just the canonical host. */ + private async walkToCanonicalHost(startUrl: string): Promise { + const { finalUrl } = await this.walkRedirects(startUrl); + return finalUrl.hostname.toLowerCase(); + } + + /** Walks redirects from a URL and returns just the final resolved URL. */ + private async walkToCanonicalUrl(startUrl: string): Promise { + const { finalUrl } = await this.walkRedirects(startUrl); + return finalUrl; + } + + /** + * Follows 3xx redirects manually starting from `startUrl`, resolving relative + * Location headers against the current URL, up to MAX_REDIRECT_HOPS hops. + * + * Returns the final URL and the final non-redirect response. A direct 200 + * (the `requestUrl` auto-follow case, where the redirect was already + * followed and no Location is present to walk) ends the walk at the + * requested URL. A 3xx that exceeds the hop cap, or one missing a Location + * header, raises a ResolveError. + */ + private async walkRedirects(startUrl: string): Promise<{ finalUrl: URL; response: HttpResponse }> { + let current = parseUrl(startUrl); + for (let hop = 0; hop <= MAX_REDIRECT_HOPS; hop += 1) { + let response: HttpResponse; + try { + response = await this.fetcher({ url: current.toString() }); + } catch (err) { + throw new ResolveError(`Request failed for ${current.toString()}`, err); + } + + if (!isRedirect(response.status)) { + // Either a real 200/4xx/5xx, or the auto-followed 200 case: the + // requested URL is canonical. + return { finalUrl: current, response }; + } + + const location = getHeader(response.headers, "Location"); + if (location === undefined || location.trim().length === 0) { + throw new ResolveError( + `Redirect from ${current.toString()} had no Location header`, + ); + } + // Resolve relative Locations against the current URL. + current = new URL(location.trim(), current); + } + + throw new ResolveError( + `Too many redirects resolving ${startUrl} (cap ${MAX_REDIRECT_HOPS})`, + ); + } +} diff --git a/feed-xml.ts b/feed-xml.ts new file mode 100644 index 0000000..250ddc5 --- /dev/null +++ b/feed-xml.ts @@ -0,0 +1,395 @@ +/** + * Feed XML parser. + * + * Parses RSS 2.0, Atom 1.0, and podcast (RSS + itunes:*) XML into a flat + * intermediate shape (`ParsedFeed` / `RawFeedItem`). The source layer maps each + * `RawFeedItem` onto the normalized `FeedItem` contract; this module stays + * source-agnostic and does no HTTP, no DOM rendering, and no normalization + * beyond date-to-ISO and trimming. + * + * Design notes: + * - DOMParser is constructed LAZILY inside `parseFeed`, never at module load, so + * importing this file in a plain Node context (no `document`/`window`) is safe. + * - Namespaced tags (content:encoded, dc:creator, itunes:*) are read by their + * full qualified name AND by a namespace-agnostic localName scan, because + * DOMParser namespace resolution varies between engines (browser jsdom vs + * Obsidian's Electron). Relying on `getElementsByTagNameNS` alone is fragile. + * - We validate the envelope once (a recognizable channel/feed root) and + * tolerate sloppy item values (missing dates normalize to null rather than + * throwing), matching the parser convention in this plugin. + */ + +/** A media enclosure attached to an item (podcast audio, cover image, ...). */ +export interface RawEnclosure { + url: string; + type: string | null; + length: number | null; +} + +/** + * One feed item before normalization. Every field is best-effort: a producer may + * omit any of them. `title` is always a string (empty when the feed omits it) so + * downstream code never has to guard the most-used field. + */ +export interface RawFeedItem { + guid: string | null; + link: string | null; + title: string; + author: string | null; + /** ISO 8601 timestamp, or null when the date is absent or unparseable. */ + pubDateIso: string | null; + /** content:encoded (RSS) or atom , preferred over `description`. */ + contentHtml: string | null; + /** RSS or atom . */ + description: string | null; + categories: string[]; + enclosure: RawEnclosure | null; +} + +/** A parsed feed: channel/feed metadata plus its items. */ +export interface ParsedFeed { + feedTitle: string; + feedLink: string | null; + items: RawFeedItem[]; +} + +/** + * Thrown when the XML cannot be parsed (a DOMParser ``) or when no + * recognizable RSS `` or Atom `` root is present. The original + * parser-error text, when available, is surfaced in the message so the caller + * can show or log a useful reason rather than a bare "parse failed". + */ +export class FeedParseError extends Error { + constructor(message: string) { + super(message); + this.name = "FeedParseError"; + } +} + +/** + * Read the text content of the first direct (or any-descendant) child element of + * `parent` matching one of `names`, comparing by localName so a namespace prefix + * (`dc:creator`) or a default-namespaced tag both match. Returns a trimmed + * string, or null when nothing matches or the matched element is empty. + * + * `directOnly` restricts the scan to immediate children, which matters for tags + * like and <link> that can legitimately appear deeper inside item bodies + * or nested structures. + */ +function readText( + parent: Element, + names: readonly string[], + directOnly: boolean, +): string | null { + const wanted = names.map((n) => n.toLowerCase()); + const scope = directOnly ? parent.children : parent.getElementsByTagName("*"); + for (let i = 0; i < scope.length; i += 1) { + const el = scope.item(i); + if (el === null) { + continue; + } + if (!wanted.includes(el.localName.toLowerCase())) { + continue; + } + const text = el.textContent; + if (text === null) { + continue; + } + const trimmed = text.trim(); + if (trimmed.length > 0) { + return trimmed; + } + } + return null; +} + +/** + * Find the first direct-child element of `parent` whose localName matches one of + * `names`. Used for elements whose attributes we need (enclosure, atom link). + */ +function findChild(parent: Element, names: readonly string[]): Element | null { + const wanted = names.map((n) => n.toLowerCase()); + const children = parent.children; + for (let i = 0; i < children.length; i += 1) { + const el = children.item(i); + if (el === null) { + continue; + } + if (wanted.includes(el.localName.toLowerCase())) { + return el; + } + } + return null; +} + +/** Collect all direct-child elements of `parent` matching one of `names`. */ +function findChildren(parent: Element, names: readonly string[]): Element[] { + const wanted = names.map((n) => n.toLowerCase()); + const out: Element[] = []; + const children = parent.children; + for (let i = 0; i < children.length; i += 1) { + const el = children.item(i); + if (el === null) { + continue; + } + if (wanted.includes(el.localName.toLowerCase())) { + out.push(el); + } + } + return out; +} + +/** Parse a non-empty integer attribute, returning null on absence or garbage. */ +function parseIntAttr(value: string | null): number | null { + if (value === null) { + return null; + } + const trimmed = value.trim(); + if (trimmed.length === 0) { + return null; + } + const n = Number.parseInt(trimmed, 10); + return Number.isFinite(n) ? n : null; +} + +/** + * Normalize a feed date string (RFC 822 pubDate or ISO atom updated/published) + * to ISO 8601. Returns null when the value is absent or not a parseable date, + * rather than throwing, so one bad timestamp never fails the whole feed. + */ +function toIso(raw: string | null): string | null { + if (raw === null) { + return null; + } + const trimmed = raw.trim(); + if (trimmed.length === 0) { + return null; + } + const ms = Date.parse(trimmed); + if (!Number.isFinite(ms)) { + return null; + } + return new Date(ms).toISOString(); +} + +/** + * Resolve the enclosure for an item. RSS uses <enclosure url type length>; Atom + * uses <link rel="enclosure" href type length>. Returns null when no usable URL + * is present (an enclosure without a url is not actionable). + */ +function readEnclosure(item: Element): RawEnclosure | null { + const rssEnclosure = findChild(item, ["enclosure"]); + if (rssEnclosure !== null) { + const url = rssEnclosure.getAttribute("url"); + if (url !== null && url.trim().length > 0) { + return { + url: url.trim(), + type: emptyToNull(rssEnclosure.getAttribute("type")), + length: parseIntAttr(rssEnclosure.getAttribute("length")), + }; + } + } + + for (const link of findChildren(item, ["link"])) { + if ((link.getAttribute("rel") ?? "").toLowerCase() !== "enclosure") { + continue; + } + const href = link.getAttribute("href"); + if (href !== null && href.trim().length > 0) { + return { + url: href.trim(), + type: emptyToNull(link.getAttribute("type")), + length: parseIntAttr(link.getAttribute("length")), + }; + } + } + return null; +} + +/** Trim a possibly-null attribute, collapsing empty strings to null. */ +function emptyToNull(value: string | null): string | null { + if (value === null) { + return null; + } + const trimmed = value.trim(); + return trimmed.length === 0 ? null : trimmed; +} + +/** + * Resolve an item's canonical link. RSS <link> carries the URL as text content. + * Atom <link> carries it in the `href` attribute, with `rel="alternate"` (or no + * rel) being the canonical permalink; enclosure/self links are skipped. + */ +function readItemLink(item: Element): string | null { + const links = findChildren(item, ["link"]); + // Prefer an Atom alternate/relless link via href. + for (const link of links) { + const rel = (link.getAttribute("rel") ?? "alternate").toLowerCase(); + if (rel === "enclosure" || rel === "self") { + continue; + } + const href = link.getAttribute("href"); + if (href !== null && href.trim().length > 0) { + return href.trim(); + } + } + // Fall back to RSS <link> text content. + for (const link of links) { + const text = link.textContent; + if (text !== null && text.trim().length > 0) { + return text.trim(); + } + } + return null; +} + +/** + * Resolve an item's author. RSS uses dc:creator (or <author>, which on RSS is an + * email and on Atom is a wrapper element holding <name>). We try dc:creator + * first, then an Atom <author><name>, then a plain <author> text node. + */ +function readAuthor(item: Element): string | null { + const creator = readText(item, ["creator"], true); + if (creator !== null) { + return creator; + } + const authorEl = findChild(item, ["author"]); + if (authorEl !== null) { + const name = readText(authorEl, ["name"], true); + if (name !== null) { + return name; + } + const text = authorEl.textContent; + if (text !== null && text.trim().length > 0) { + return text.trim(); + } + } + return null; +} + +/** + * Collect categories. RSS <category> carries the value as text content; Atom + * <category term="..."> carries it in the `term` attribute. Both are gathered + * and de-duplicated while preserving order. + */ +function readCategories(item: Element): string[] { + const out: string[] = []; + const seen = new Set<string>(); + for (const cat of findChildren(item, ["category"])) { + const term = emptyToNull(cat.getAttribute("term")); + const text = cat.textContent === null ? null : cat.textContent.trim(); + const value = term ?? (text !== null && text.length > 0 ? text : null); + if (value === null) { + continue; + } + const key = value.toLowerCase(); + if (seen.has(key)) { + continue; + } + seen.add(key); + out.push(value); + } + return out; +} + +/** Map a single <item> (RSS) or <entry> (Atom) element to a RawFeedItem. */ +function readItem(item: Element): RawFeedItem { + const title = readText(item, ["title"], true) ?? ""; + const contentHtml = readText(item, ["encoded", "content"], true); + const description = readText(item, ["description", "summary", "subtitle"], true); + const pubDate = readText(item, ["pubdate", "published", "updated", "date"], true); + return { + guid: readText(item, ["guid", "id"], true), + link: readItemLink(item), + title, + author: readAuthor(item), + pubDateIso: toIso(pubDate), + contentHtml, + description, + categories: readCategories(item), + enclosure: readEnclosure(item), + }; +} + +/** + * Locate the channel/feed root and its item elements, supporting both RSS + * (rss > channel > item) and Atom (feed > entry). Returns null when neither + * shape is recognizable so the caller can throw a `FeedParseError`. + */ +function locateRoot( + doc: Document, +): { root: Element; items: Element[]; isAtom: boolean } | null { + // RSS: <rss><channel>...</channel></rss> (channel may also be the doc root in + // RDF-flavored feeds, so scan by localName rather than assuming the path). + const channels = doc.getElementsByTagName("channel"); + const channel = channels.item(0); + if (channel !== null) { + const items = findChildren(channel, ["item"]); + return { root: channel, items, isAtom: false }; + } + + // Atom: <feed><entry>...</entry></feed>. + const root = doc.documentElement; + if (root !== null && root.localName.toLowerCase() === "feed") { + const entries = findChildren(root, ["entry"]); + return { root, items: entries, isAtom: true }; + } + return null; +} + +/** Resolve the feed-level canonical link for RSS vs Atom roots. */ +function readFeedLink(root: Element, isAtom: boolean): string | null { + if (!isAtom) { + return readText(root, ["link"], true); + } + // Atom feed: prefer the alternate link's href. + for (const link of findChildren(root, ["link"])) { + const rel = (link.getAttribute("rel") ?? "alternate").toLowerCase(); + if (rel === "self") { + continue; + } + const href = emptyToNull(link.getAttribute("href")); + if (href !== null) { + return href; + } + } + return null; +} + +/** + * Parse a feed document (RSS, Atom, or podcast XML) into a `ParsedFeed`. + * + * Throws `FeedParseError` when the input is not well-formed XML (DOMParser emits + * a `<parsererror>` element) or when no RSS `<channel>` / Atom `<feed>` root is + * present. Individual item quirks (missing dates, absent bodies) are tolerated. + * + * The DOMParser is constructed here, not at module scope, so this module imports + * safely in environments without a DOM. + */ +export function parseFeed(xml: string): ParsedFeed { + if (typeof xml !== "string" || xml.trim().length === 0) { + throw new FeedParseError("Feed XML was empty."); + } + + const doc = new DOMParser().parseFromString(xml, "text/xml"); + + // DOMParser does not throw on malformed XML; it returns a document whose body + // contains a <parsererror> element. Detect it explicitly. + const parserError = doc.getElementsByTagName("parsererror").item(0); + if (parserError !== null) { + const detail = parserError.textContent; + const reason = detail !== null && detail.trim().length > 0 ? detail.trim() : "unknown reason"; + throw new FeedParseError(`Feed XML could not be parsed: ${reason}`); + } + + const located = locateRoot(doc); + if (located === null) { + throw new FeedParseError("Feed XML has no RSS <channel> or Atom <feed> root."); + } + + const feedTitle = readText(located.root, ["title"], true) ?? ""; + const feedLink = readFeedLink(located.root, located.isAtom); + const items = located.items.map((item) => readItem(item)); + + return { feedTitle, feedLink, items }; +} diff --git a/html-converter.ts b/html-converter.ts new file mode 100644 index 0000000..57311af --- /dev/null +++ b/html-converter.ts @@ -0,0 +1,436 @@ +/** + * HTML to Markdown conversion for imported feed bodies. + * + * Feed bodies (Substack `content:encoded`, WordPress `description`, generic RSS + * HTML) arrive as a single HTML string. This module turns that into clean, + * deterministic Markdown using Turndown plus the GitHub-flavored-markdown + * plugin (tables, strikethrough, task lists), layered with a handful of + * feed-specific rules: + * + * - `figure` + `figcaption` becomes an image followed by an italic caption. + * - Subscribe / share / button widgets matched by CLASS NAME (never body text) + * are stripped; an unrecognized widget that is fundamentally a link degrades + * to that link rather than to raw text. + * - `pre > code.language-xxx` becomes a fenced block tagged with `xxx`. + * - Anchor-based footnote references collapse to `[^n]`, and their definitions + * are appended once at the end of the document in first-seen order. + * + * Determinism is a hard requirement: converting the same input twice yields + * byte-identical Markdown. All per-conversion state lives in a fresh context + * created for each `turndown` call, never on the module or service. + * + * DOM access is lazy. Turndown constructs its own `DOMParser` internally when + * handed a string, and this module never touches `document`/`window`/ + * `DOMParser` at import time, so it loads safely in a Node context. The + * conversion itself still needs a DOM (jsdom in tests, the renderer in the + * app). + */ + +import type TurndownService from "turndown"; +import * as turndownModule from "turndown"; +import { gfm } from "turndown-plugin-gfm"; + +/** + * The Turndown package is published as CommonJS whose `module.exports` IS the + * constructor (no `__esModule`, no `.default`). esbuild synthesizes a default + * import correctly for the production bundle, but a CommonJS transpile without + * the `esModuleInterop` runtime helper binds a default import to `undefined`. + * A namespace import binds to `require("turndown")` itself across both bundlers, + * so resolve the constructor from it: the namespace value is either the + * constructor function directly (raw CommonJS) or an object carrying it on + * `.default` (interop-wrapped). Resolved once at load with no `as any`. + */ +type TurndownCtor = new (options?: ConstructorParameters<typeof TurndownService>[0]) => TurndownService; + +function resolveTurndownCtor(): TurndownCtor { + const ns: unknown = turndownModule; + if (typeof ns === "function") { + return ns as TurndownCtor; + } + if (ns !== null && typeof ns === "object" && "default" in ns) { + const inner: unknown = ns.default; + if (typeof inner === "function") { + return inner as TurndownCtor; + } + } + throw new Error("turndown module did not export a constructor"); +} + +const Turndown: TurndownCtor = resolveTurndownCtor(); + +/** + * A Turndown plugin: a function that mutates a service in place. Matches the + * `@types/turndown` `Plugin` shape and is what `service.use` expects. + */ +type TurndownPlugin = (service: TurndownService) => void; + +/** + * Resolve the GFM plugin to a typed function. The `turndown-plugin-gfm` package + * ships no types, so the imported `gfm` binding is untyped; contain that at a + * single guarded boundary (an `unknown` round-trip plus a runtime function + * check) so the rest of the module, and `service.use`, work with a typed value. + */ +function resolveGfmPlugin(): TurndownPlugin { + const candidate: unknown = gfm; + if (typeof candidate !== "function") { + throw new Error("turndown-plugin-gfm did not export a gfm plugin function"); + } + return candidate as TurndownPlugin; +} + +const gfmPlugin: TurndownPlugin = resolveGfmPlugin(); + +/** + * Class-name fragments for known subscribe / share / button widgets, removed + * outright. Matched case-insensitively as a substring of any single class + * token, so `subscription-widget-wrap-editor`, `subscribe-widget`, and + * `button-wrapper` all match. Data-driven by design: extend this list, not the + * rule logic, to retire a new widget family. We never match on body text. + */ +const WIDGET_STRIP_KEYS: readonly string[] = [ + "subscribe", + "subscription", + "share-dialog", + "share-wrapper", + "sharewrapper", + "button-wrapper", + "button-primary", + "paywall", + "poll-embed", + "comments-button", + "footer-buttons", +]; + +/** + * Class-name fragments for generic promotional wrappers that are not in the + * explicit strip list. A node matching one of these is treated as an unknown + * widget: if it is fundamentally a single link it degrades to that link; + * otherwise its inner content is kept. Also data-driven. + */ +const WIDGET_DEGRADE_KEYS: readonly string[] = [ + "cta", + "banner", + "promo", + "callout", + "widget", + "embed", +]; + +/** A node carrying a class attribute we can read. */ +interface ClassedElement { + getAttribute(name: string): string | null; + querySelector(selectors: string): Element | null; +} + +/** Per-conversion mutable state. One instance is created per `turndown` call. */ +interface ConversionContext { + /** Footnote ref order, by the ref label as it appears (e.g. "1", "2"). */ + footnoteOrder: string[]; + /** Definition Markdown keyed by ref label. */ + footnoteDefs: Map<string, string>; +} + +/** + * Reads the `class` attribute as a lowercased token list. Returns an empty + * array when there is no class attribute, so callers never see undefined. + */ +function classTokens(node: ClassedElement): string[] { + const raw = node.getAttribute("class"); + if (raw === null || raw.length === 0) { + return []; + } + return raw.toLowerCase().split(/\s+/).filter((t) => t.length > 0); +} + +/** True when any class token contains one of the given key fragments. */ +function classMatchesAny(node: ClassedElement, keys: readonly string[]): boolean { + const tokens = classTokens(node); + if (tokens.length === 0) { + return false; + } + return tokens.some((token) => keys.some((key) => token.includes(key))); +} + +/** Known subscribe/share/button widget: removed outright. */ +function isStripWidget(node: ClassedElement): boolean { + return classMatchesAny(node, WIDGET_STRIP_KEYS); +} + +/** Generic promotional wrapper not in the strip list: degraded to its link. */ +function isDegradeWidget(node: ClassedElement): boolean { + return !isStripWidget(node) && classMatchesAny(node, WIDGET_DEGRADE_KEYS); +} + +/** + * Reads a footnote ref label from an anchor, or null if it is not a footnote + * reference. Recognizes the common shapes: an `href` of `#footnote-N` / + * `#fn-N` / `#fnref...`, or an `id` of `footnote-anchor-N`. The label is the + * trailing numeric token, falling back to the anchor's trimmed text. + */ +function footnoteRefLabel(anchor: ClassedElement): string | null { + const href = anchor.getAttribute("href"); + const id = anchor.getAttribute("id"); + const fromHref = href !== null ? /^#(?:footnote|fn|fnref)[-_]?(.+)$/i.exec(href) : null; + const fromId = id !== null ? /^(?:footnote-anchor|fnref)[-_]?(.+)$/i.exec(id) : null; + const match = fromHref ?? fromId; + if (match === null) { + return null; + } + const captured = match[1]; + if (captured === undefined) { + return null; + } + const numeric = /(\d+)\s*$/.exec(captured); + if (numeric !== null && numeric[1] !== undefined) { + return numeric[1]; + } + return captured.trim().length > 0 ? captured.trim() : null; +} + +/** + * Reads a footnote definition label from a container node, or null. Recognizes + * an `id` of `footnote-N` / `fn-N` (but not the `-anchor-` ref variant). + */ +function footnoteDefLabel(node: ClassedElement): string | null { + const id = node.getAttribute("id"); + if (id === null) { + return null; + } + const match = /^(?:footnote|fn)[-_]?(.+)$/i.exec(id); + if (match === null || match[1] === undefined) { + return null; + } + if (/^anchor/i.test(match[1])) { + return null; + } + const numeric = /(\d+)\s*$/.exec(match[1]); + if (numeric !== null && numeric[1] !== undefined) { + return numeric[1]; + } + return match[1].trim().length > 0 ? match[1].trim() : null; +} + +/** Collapses runs of whitespace to single spaces and trims. */ +function collapseWhitespace(text: string): string { + return text.replace(/\s+/g, " ").trim(); +} + +/** + * Builds a configured `TurndownService` with the GFM plugin and the + * feed-specific rules wired in. The `context` is captured by the rules so each + * service instance owns exactly one conversion's worth of footnote state. + */ +function buildService(context: ConversionContext): TurndownService { + const service = new Turndown({ + headingStyle: "atx", + bulletListMarker: "-", + codeBlockStyle: "fenced", + emDelimiter: "_", + strongDelimiter: "**", + hr: "---", + linkStyle: "inlined", + }); + + service.use(gfmPlugin); + + // Known subscribe / share / button widgets, matched purely by class name, + // are removed outright. Body text is never the match criterion. + service.addRule("stripFeedWidgets", { + filter: (node): boolean => isStripWidget(node), + replacement: (): string => "", + }); + + // Generic promotional wrappers that are not on the strip list degrade: if + // the wrapper is fundamentally a single link, emit that link; otherwise keep + // its inner content rather than dropping real prose. + service.addRule("degradeUnknownWidgets", { + filter: (node): boolean => isDegradeWidget(node), + replacement: (content, node): string => { + const link = node.querySelector("a[href]"); + if (link !== null) { + const href = link.getAttribute("href"); + const label = collapseWhitespace(link.textContent ?? ""); + if (href !== null && href.length > 0 && label.length > 0) { + return `[${label}](${href})`; + } + } + return content; + }, + }); + + // figure with a figcaption: emit the inner image markdown, then the caption + // as a standalone italic line. Without a caption, fall through to the inner + // content (Turndown's default img handling). + service.addRule("figureWithCaption", { + filter: "figure", + replacement: (content, node): string => { + const captionEl = node.querySelector("figcaption"); + const img = node.querySelector("img"); + let imageMd = ""; + if (img !== null) { + const src = img.getAttribute("src"); + if (src !== null && src.length > 0) { + const alt = collapseWhitespace(img.getAttribute("alt") ?? ""); + imageMd = `![${alt}](${src})`; + } + } + if (imageMd.length === 0) { + // No usable <img>; keep whatever the children produced (e.g. a + // linked image Turndown already rendered). + imageMd = content.trim(); + } + const caption = captionEl !== null ? collapseWhitespace(captionEl.textContent ?? "") : ""; + if (caption.length === 0) { + return imageMd.length > 0 ? `\n\n${imageMd}\n\n` : ""; + } + return `\n\n${imageMd}\n\n_${caption}_\n\n`; + }, + }); + + // pre > code.language-xxx: fenced block carrying the language tag. Falls + // back to a plain fence when no language class is present. + service.addRule("fencedCodeWithLanguage", { + filter: (node): boolean => { + if (node.nodeName !== "PRE") { + return false; + } + return node.querySelector("code") !== null; + }, + replacement: (_content, node): string => { + const code = node.querySelector("code"); + const codeText = code !== null ? code.textContent ?? "" : node.textContent ?? ""; + let language = ""; + if (code !== null) { + for (const token of classTokens(code)) { + const match = /^language-(.+)$/.exec(token); + if (match !== null && match[1] !== undefined) { + language = match[1]; + break; + } + } + } + const body = codeText.replace(/\n+$/, ""); + return `\n\n\`\`\`${language}\n${body}\n\`\`\`\n\n`; + }, + }); + + // Anchor footnote references: collapse to [^N] and record the first-seen + // order. The definition body is resolved later from the def containers. + service.addRule("footnoteRef", { + filter: (node): boolean => { + if (node.nodeName !== "A") { + return false; + } + return footnoteRefLabel(node) !== null; + }, + replacement: (_content, node): string => { + const label = footnoteRefLabel(node); + if (label === null) { + return ""; + } + if (!context.footnoteOrder.includes(label)) { + context.footnoteOrder.push(label); + } + return `[^${label}]`; + }, + }); + + // Footnote definition containers: capture the body keyed by label and emit + // nothing inline. Definitions are appended at the document end afterward. + service.addRule("footnoteDef", { + filter: (node): boolean => { + if (node.nodeType !== 1) { + return false; + } + return footnoteDefLabel(node) !== null; + }, + replacement: (content, node): string => { + const label = footnoteDefLabel(node); + if (label === null) { + return content; + } + const def = collapseWhitespace(content); + if (def.length > 0) { + context.footnoteDefs.set(label, def); + } + return ""; + }, + }); + + return service; +} + +/** + * Appends footnote definitions to the converted body in stable order: refs + * first in first-seen order, then any definitions that had no matching ref + * (sorted for determinism). A definition with no captured body is skipped. + */ +function appendFootnotes(markdown: string, context: ConversionContext): string { + const seen = new Set<string>(); + const lines: string[] = []; + for (const label of context.footnoteOrder) { + const def = context.footnoteDefs.get(label); + if (def !== undefined && def.length > 0) { + lines.push(`[^${label}]: ${def}`); + } + seen.add(label); + } + const leftover = [...context.footnoteDefs.keys()].filter((label) => !seen.has(label)).sort(); + for (const label of leftover) { + const def = context.footnoteDefs.get(label); + if (def !== undefined && def.length > 0) { + lines.push(`[^${label}]: ${def}`); + } + } + if (lines.length === 0) { + return markdown; + } + const body = markdown.trimEnd(); + return `${body}\n\n${lines.join("\n")}`; +} + +/** + * Normalizes Turndown output: collapse 3+ blank lines to a single blank line + * and trim leading/trailing whitespace. This is what makes repeated conversion + * byte-stable regardless of how rules spaced their fragments. + */ +function normalizeMarkdown(markdown: string): string { + return markdown.replace(/\r\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim(); +} + +/** + * Creates a configured converter with a fresh, empty conversion context. Useful + * when a caller wants to run several conversions and inspect the underlying + * service, but most callers want `convertHtmlToMarkdown` instead, which manages + * the context lifecycle for them. + */ +export function createConverter(): TurndownService { + const context: ConversionContext = { footnoteOrder: [], footnoteDefs: new Map() }; + return buildService(context); +} + +/** + * Converts an HTML feed body to Markdown deterministically. Same input string + * yields byte-identical output every call. An empty or whitespace-only input + * returns an empty string. The caught error from a Turndown failure is + * surfaced to the caller (rethrown) rather than swallowed. + */ +export function convertHtmlToMarkdown(html: string): string { + if (html.trim().length === 0) { + return ""; + } + const context: ConversionContext = { footnoteOrder: [], footnoteDefs: new Map() }; + const service = buildService(context); + let converted: string; + try { + converted = service.turndown(html); + } catch (err) { + // Surface the original failure rather than swallow it: rethrow with the + // underlying message embedded. (`Error`'s `cause` option needs an ES2022 + // lib this project does not target, so the detail rides in the message.) + const detail = err instanceof Error ? err.message : String(err); + throw new Error(`Failed to convert HTML to Markdown: ${detail}`); + } + const withFootnotes = appendFootnotes(converted, context); + return normalizeMarkdown(withFootnotes); +} diff --git a/turndown-plugin-gfm.d.ts b/turndown-plugin-gfm.d.ts new file mode 100644 index 0000000..88eaecb --- /dev/null +++ b/turndown-plugin-gfm.d.ts @@ -0,0 +1,16 @@ +// Ambient declaration for turndown-plugin-gfm, which ships no type definitions. +// +// This MUST stay a standalone script file with NO top-level import or export. +// A `declare module 'x'` placed inside a file that is itself a module (one with +// top-level import/export, like types.d.ts) is treated as an augmentation of an +// already-typed module and is silently ignored for an untyped JS package, so it +// fails to suppress TS7016. Keeping this file script-scoped makes the block a +// real ambient module declaration. The TurndownService type is referenced via +// an inline `import('turndown')` so this file stays script-scoped. +declare module "turndown-plugin-gfm" { + type GfmPlugin = (service: import("turndown")) => void; + export const gfm: GfmPlugin; + export const tables: GfmPlugin; + export const strikethrough: GfmPlugin; + export const taskListItems: GfmPlugin; +}