diff --git a/paperforge/worker/ocr_render.py b/paperforge/worker/ocr_render.py index 8d540c37..1dd727fa 100644 --- a/paperforge/worker/ocr_render.py +++ b/paperforge/worker/ocr_render.py @@ -724,11 +724,15 @@ def render_fulltext_markdown( rfid = rf.get("reader_figure_id") if not rfid: continue - cbid = rf.get("caption_block_id") - page = _block_page_map.get(cbid) if cbid is not None else None + page = None + for item in rf.get("consumed_caption_block_ids", []): + p = item.get("page") if isinstance(item, dict) else None + if p is not None: + page = p + break if page is None: - for aid in rf.get("consumed_asset_block_ids", []): - p = _block_page_map.get(aid) + for item in rf.get("consumed_asset_block_ids", []): + p = item.get("page") if isinstance(item, dict) else None if p is not None: page = p break @@ -963,6 +967,7 @@ def render_fulltext_markdown( "frontmatter_noise", "table_html", "figure_caption", + "figure_inner_text", } if role in _SKIPPED_BODY_ROLES: continue @@ -1030,6 +1035,14 @@ def render_fulltext_markdown( continue if "published online" in text.strip().lower(): continue + _BACKMATTER_HEADING_KEYWORDS = frozenset({ + "author contributions", "data availability", "funding", "acknowledg", + "conflict of interest", "competing interests", "supplementary material", + "ethics statement", "publisher", "biographies", + }) + _heading_lower = text.strip().lower() + if any(kw in _heading_lower for kw in _BACKMATTER_HEADING_KEYWORDS): + continue if _is_bogus_heading(text): if text: lines.append(text) diff --git a/tests/test_ocr_real_paper_reader_audit.py b/tests/test_ocr_real_paper_reader_audit.py index e49e3aec..07ddcba1 100644 --- a/tests/test_ocr_real_paper_reader_audit.py +++ b/tests/test_ocr_real_paper_reader_audit.py @@ -278,6 +278,106 @@ def test_reader_audit_reader_coverage_is_not_trivially_zero(rebuilt_reader_audit assert health.get("figure_reader_coverage_total", 0) > 0, f"{key}: reader coverage total is zero despite eligible legends" +_BACKMATTER_KEYWORDS = [ + "author contributions", "data availability", "funding", "acknowledg", + "conflict of interest", "competing interests", "supplementary material", + "ethics statement", "publisher", "biographies", +] + + +def _is_panel_label(text: str) -> bool: + return bool(re.match(r"^[A-H]\s*$", text.strip())) + +def _is_panel_lower(text: str) -> bool: + return bool(re.match(r"^[a-h]\s*$", text.strip())) + + +@pytest.mark.parametrize("key", ALL_KEYS) +def test_audit_fulltext_no_panel_letter_leaks(rebuilt_reader_audit_papers: dict, real_ocr_root: Path, key: str) -> None: + lines = _fulltext_lines(real_ocr_root, key) + leaks = [] + for i, line in enumerate(lines): + s = line.strip() + if _is_panel_label(s): + if i > 0 and not lines[i - 1].strip().startswith(">") and not lines[i - 1].strip().startswith("!"): + leaks.append(f"L{i}:{s}") + elif _is_panel_lower(s): + if i > 0 and not lines[i - 1].strip().startswith("##"): + leaks.append(f"L{i}:{s}") + assert not leaks, f"{key}: {len(leaks)} panel labels leaked into fulltext: {leaks[:10]}" + + +@pytest.mark.parametrize("key", ALL_KEYS) +def test_audit_fulltext_no_backmatter_headings(rebuilt_reader_audit_papers: dict, real_ocr_root: Path, key: str) -> None: + lines = _fulltext_lines(real_ocr_root, key) + back_hds = [] + for i, line in enumerate(lines): + s = line.strip().lower() + if s.startswith("## ") and any(kw in s for kw in _BACKMATTER_KEYWORDS): + back_hds.append(f"L{i}:{line.strip()[:80]}") + assert not back_hds, f"{key}: {len(back_hds)} backmatter headings rendered as body headings: {back_hds[:8]}" + + +@pytest.mark.parametrize("key", ALL_KEYS) +def test_audit_fulltext_no_figure_cards_after_refs(rebuilt_reader_audit_papers: dict, real_ocr_root: Path, key: str) -> None: + lines = _fulltext_lines(real_ocr_root, key) + ref_idx = next((i for i, l in enumerate(lines) if l.strip().lower() == "## references"), None) + if ref_idx is None: + return + fig_cards_after = sum(1 for i in range(ref_idx, len(lines)) if lines[i].strip().startswith("> **Figure")) + assert fig_cards_after == 0, f"{key}: {fig_cards_after} figure cards placed after REFERENCES section" + + +@pytest.mark.parametrize("key", ALL_KEYS) +def test_audit_fulltext_no_body_before_introduction(rebuilt_reader_audit_papers: dict, real_ocr_root: Path, key: str) -> None: + lines = _fulltext_lines(real_ocr_root, key) + first_h2 = next((i for i, l in enumerate(lines) if re.match(r"^## \d+\.", l.strip()) or l.strip().lower().startswith("## introduction")), None) + if first_h2 is None: + return + body_before = sum( + 1 for i in range(first_h2) + if lines[i].strip() + and not lines[i].strip().startswith("#") + and not lines[i].strip().startswith("