fix(markdown): protect variable-length fenced code blocks

Why:
Markdown fence handling regressed when the shared segmenter only recognized backticks and exact triple fences. That let link and URL rewriting leak into supported fenced code blocks.

What:
Add fenced-code range collection with opening/closing fence length matching, include tilde-only input in the fast path, and keep fenced blocks out of the generic protected matcher. Add regressions for tilde fences, wider backtick fences, and the URL/link consumers that rely on markdown segmentation.
This commit is contained in:
Kodai Nakamura 2026-06-22 03:05:52 +09:00
parent 9209356ee0
commit db05c2ee1f
5 changed files with 116 additions and 2 deletions

View file

@ -32,6 +32,21 @@ describe("segmentMarkdown", () => {
])
})
it("protects tilde fenced code blocks", () => {
const text = "before\n~~~ts\nTypeScript\n~~~\nafter"
const segments = segmentMarkdown(text)
expect(segments.map(segment => ({
kind: segment.kind,
protectedKind: segment.protectedKind,
text: segment.text,
}))).toEqual([
{ kind: "prose", protectedKind: undefined, text: "before\n" },
{ kind: "protected", protectedKind: "fenced-code", text: "~~~ts\nTypeScript\n~~~\n" },
{ kind: "prose", protectedKind: undefined, text: "after" },
])
})
it("protects headings, tables, and callouts when requested", () => {
const text = "# TypeScript\n| TypeScript |\n> [!note]\n> TypeScript\nTypeScript"
const segments = segmentMarkdown(text, {

View file

@ -92,10 +92,45 @@ const collectTableRowRanges = (text: string): ProtectedRange[] => {
return ranges
}
const escapeRegExp = (text: string): string =>
text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
const collectFencedCodeRanges = (text: string): ProtectedRange[] => {
if (!text.includes("```") && !text.includes("~~~")) {
return []
}
const ranges: ProtectedRange[] = []
const openingFencePattern = /^ {0,3}(`{3,}|~{3,})[^\r\n]*(?:\r?\n|$)/gm
let openingMatch: RegExpExecArray | null
while ((openingMatch = openingFencePattern.exec(text)) !== null) {
const openingFence = openingMatch[1]
const fenceChar = openingFence[0]
const fenceLength = openingFence.length
const closingFencePattern = new RegExp(
`^ {0,3}${escapeRegExp(fenceChar)}{${fenceLength},}[ \\t]*(?:\\r?\\n|$)`,
"gm",
)
closingFencePattern.lastIndex = openingMatch.index + openingMatch[0].length
const closingMatch = closingFencePattern.exec(text)
const end = closingMatch
? closingMatch.index + closingMatch[0].length
: text.length
ranges.push({
start: openingMatch.index,
end,
protectedKind: "fenced-code",
})
openingFencePattern.lastIndex = end
}
return ranges
}
const buildProtectedPattern = (protectUrls: boolean): RegExp => {
const parts = [
"```[\\s\\S]*?(?:```|$)",
"~~~[\\s\\S]*?(?:~~~|$)",
"`[^`]*`",
"\\[\\[[^\\]]+\\]\\]",
"\\[[^\\]]+\\]\\([^)]+\\)",
@ -160,6 +195,7 @@ export const segmentMarkdown = (
options: SegmentMarkdownOptions = {},
): MarkdownSegment[] => {
const mayContainProtectedMarkdown = text.includes("`")
|| text.includes("~")
|| text.includes("[")
|| (options.protectHeadings && text.includes("#"))
|| (options.protectCallouts && text.includes(">"))
@ -189,6 +225,8 @@ export const segmentMarkdown = (
ranges.push(...collectTableRowRanges(text))
}
ranges.push(...collectFencedCodeRanges(text))
const protectedPattern = buildProtectedPattern(options.protectUrls ?? false)
let match: RegExpExecArray | null

View file

@ -45,6 +45,48 @@ describe("ignore code", () => {
expect(result).toBe("```typescript\nexample\n```")
})
it("tilde code block", () => {
const settings = {
scoped: false,
baseDir: undefined,
}
const { candidateMap, trie } = buildCandidateTrieForTest({
files: [{ path: "example" }, { path: "typescript" }],
settings,
})
const result = replaceLinks({
body: "~~~typescript\nexample\n~~~",
linkResolverContext: {
filePath: "journals/2022-01-01",
trie,
candidateMap,
},
settings,
})
expect(result).toBe("~~~typescript\nexample\n~~~")
})
it("wide code fence with embedded triple backticks", () => {
const settings = {
scoped: false,
baseDir: undefined,
}
const { candidateMap, trie } = buildCandidateTrieForTest({
files: [{ path: "example" }, { path: "typescript" }],
settings,
})
const result = replaceLinks({
body: "````\nnot end ```\nexample\n````",
linkResolverContext: {
filePath: "journals/2022-01-01",
trie,
candidateMap,
},
settings,
})
expect(result).toBe("````\nnot end ```\nexample\n````")
})
it("unclosed code block", () => {
const settings = {
scoped: false,

View file

@ -63,4 +63,17 @@ describe("replaceUrlWithTitle", () => {
"Line 1: [Example Title](https://example.com)\nLine 2: [Another Title](https://another.com)",
)
})
it("should ignore URLs inside tilde fenced code blocks", () => {
const body = "~~~\nhttps://example.com\n~~~"
const result = replaceUrlWithTitle({
body,
urlTitleMap: new Map(
Object.entries({
"https://example.com": "Example Title",
}),
),
})
expect(result).toBe("~~~\nhttps://example.com\n~~~")
})
})

View file

@ -32,6 +32,12 @@ describe("listupAllUrls", () => {
expect(result).not.toContain("https://example.com")
})
it("should ignore URLs inside tilde fenced code blocks", () => {
const body = "~~~\nhttps://example.com\n~~~"
const result = listupAllUrls(body)
expect(result).not.toContain("https://example.com")
})
it("ignore domains", () => {
const body = "Check this link: https://example.com"
const result = listupAllUrls(body, ["example.com"])