perf: simplify pre-ref backmatter to body classification, fix 3 residual bugs

paperforge/worker/ocr_document.py

@@ -154,8 +154,6 @@ def _looks_like_tail_body(block: dict) -> bool:
         return False
     if _BACKMATTER_BODY_SIGNALS.search(text):
         return True
-    if len(words) <= 25:
-        return True
     return False
 
 
@@ -495,7 +493,10 @@ def _make_zone(
 
 
 def _canonical_section_text(block: dict) -> str:
-    return _block_text(block).strip().lower()
+    text = _block_text(block).strip().lower()
+    if re.match(r"^(?:\w\s)+\w$", text):
+        text = re.sub(r"\s+", "", text)
+    return text
 
 
 def _strip_inline_html(text: str) -> str:
@@ -1584,14 +1585,7 @@ def _detect_references_start(
     return None
 
 
-_BACKMATTER_TITLE_DENY_LIST = frozenset({
-    "generative ai statement", "acknowledgments", "acknowledgements",
-    "funding", "conflict of interest", "competing interests",
-    "data availability", "supplementary materials", "supplementary material",
-    "author contributions", "declaration of competing interest",
-    "credit authorship contribution statement", "ethical statement",
-    "ethics statement", "institutional review board",
-})
+_BACKMATTER_TITLE_DENY_LIST: frozenset[str] = frozenset()
 
 
 def _page_has_strong_body_continuation(page_blocks: list[dict]) -> bool:
@@ -2465,6 +2459,10 @@ def _apply_content_zone_fallback(blocks: list[dict], region_bus: dict[str, dict]
         page = int(block.get("page", 0) or 0)
 
         if role in {"noise", "frontmatter_noise", "media_asset", "figure_asset", "figure_inner_text"}:
+            if page <= 2 and role in {"noise", "frontmatter_noise"}:
+                block["zone"] = "frontmatter_main_zone"
+            elif page <= 2 and role in {"media_asset", "figure_asset", "figure_inner_text"}:
+                block["zone"] = "display_zone"
             continue
 
         if role in {"paper_title", "authors", "affiliation", "frontmatter_support"} and page <= 2:
@@ -2775,12 +2773,7 @@ def _looks_like_backmatter_body_text(text: str) -> bool:
         "conflict of interest",
         "declaration",
         "publisher",
-        "author contributions",
-        "funding",
-        "acknowledg",
-        "data availability",
-        "supplement",
-        "ethics",
+        "copyright",
     )
     return any(marker in lower for marker in markers)
 

paperforge/worker/ocr_roles.py

@@ -31,40 +31,9 @@ _PREPROOF_MARKER = re.compile(
     re.IGNORECASE,
 )
 
-_BACKMATTER_TITLE_DENY_LIST = {
-    "generative ai statement",
-    "acknowledgments",
-    "acknowledgements",
-    "funding",
-    "conflict of interest",
-    "competing interests",
-    "data availability",
-    "supplementary materials",
-    "supplementary material",
-    "author contributions",
-    "declaration of competing interest",
-    "credit authorship contribution statement",
-    "ethical statement",
-    "ethics statement",
-    "institutional review board",
-}
+_BACKMATTER_TITLE_DENY_LIST: set[str] = set()
 
 _BACKMATTER_HEADINGS = {
-    "author contributions",
-    "funding",
-    "acknowledgments",
-    "acknowledgements",
-    "conflict of interest",
-    "competing interests",
-    "data availability",
-    "supplementary materials",
-    "supplementary material",
-    "generative ai statement",
-    "declaration of competing interest",
-    "ethical statement",
-    "ethics statement",
-    "institutional review board",
-    "credit authorship contribution statement",
     "publisher's note",
 }
 
@@ -588,10 +557,7 @@ def resolve_final_role(
         phrase in text_lower
         for phrase in (
             "conflict of interest",
-            "publisher's note",
-            "publisher’s note",
-            "ethics statement",
-            "author contributions",
+            "publisher\u2019s note",
             "the remaining authors declare",
             "copyright",
             "published online",

paperforge/worker/ocr_structural_gate.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import re
 from dataclasses import dataclass, field
 
 VERIFY_REQUIRED = {
@@ -376,7 +377,7 @@ def build_document_abstract_span(blocks: list[dict], context: dict) -> dict:
         (
             idx
             for idx, block in enumerate(blocks)
-            if block.get("seed_role") == "abstract_heading" or str(block.get("text", "")).strip().lower() == "abstract"
+            if block.get("seed_role") == "abstract_heading" or re.sub(r"\s+", "", str(block.get("text", "")).strip().lower()) == "abstract"
         ),
         None,
     )

tests/test_ocr_roles.py

@@ -183,31 +183,10 @@ def test_stabilize_frontmatter_noise_not_section_heading() -> None:
         assert assignment.role not in ("section_heading", "subsection_heading"), f"{noise} should not be a heading"
 
 
-def test_backmatter_heading_not_paper_title() -> None:
+def test_pre_ref_headings_classified_as_section_headings() -> None:
     from paperforge.worker.ocr_roles import assign_block_role
 
-    for text in ["Generative AI statement", "Acknowledgments", "Funding", "Conflict of interest"]:
-        block = {
-            "block_label": "paragraph_title",
-            "block_content": text,
-            "page": 1,
-            "block_bbox": [100, 50, 500, 80],
-        }
-        assignment = assign_block_role(block, page_blocks=[], page_height=1000)
-        assert assignment.role != "paper_title", f"'{text}' should not be paper_title"
-
-
-def test_backmatter_heading_gets_backmatter_role() -> None:
-    from paperforge.worker.ocr_roles import assign_block_role
-
-    backmatter_phrases = [
-        "Author contributions",
-        "Funding",
-        "Acknowledgments",
-        "Conflict of interest",
-        "Generative AI statement",
-    ]
-    for text in backmatter_phrases:
+    for text in ["Author contributions", "Funding", "Acknowledgments", "Conflict of interest", "Generative AI statement"]:
         block = {
             "block_label": "paragraph_title",
             "block_content": text,
@@ -215,7 +194,9 @@ def test_backmatter_heading_gets_backmatter_role() -> None:
             "block_bbox": [100, 600, 500, 630],
         }
         assignment = assign_block_role(block, page_blocks=[], page_height=1000)
-        assert assignment.role.startswith("backmatter_"), f"'{text}' should get backmatter_ role, got {assignment.role}"
+        assert assignment.role in {"section_heading", "subsection_heading", "sub_subsection_heading"}, (
+            f"'{text}' on page 20 should get a normal heading role, got {assignment.role}"
+        )
 
 
 def test_references_heading_gets_reference_role() -> None:
@@ -494,8 +475,8 @@ def test_figure_caption_candidate_for_narrative_body() -> None:
     )
 
 
-def test_backmatter_heading_candidate_on_late_page() -> None:
-    """Funding on page 10 → backmatter_heading_candidate."""
+def test_pre_ref_heading_on_late_page_is_normal_heading() -> None:
+    """Funding on page 10 → normal heading role (no longer backmatter-specific)."""
     from paperforge.worker.ocr_roles import assign_block_role
 
     block = {
@@ -507,8 +488,8 @@ def test_backmatter_heading_candidate_on_late_page() -> None:
 
     role = assign_block_role(block, page_blocks=[], page_height=1000)
 
-    assert role.role == "backmatter_heading_candidate", (
-        f"Funding on page 10 should be backmatter_heading_candidate, got {role.role}"
+    assert role.role in {"section_heading", "subsection_heading", "sub_subsection_heading"}, (
+        f"Funding on page 10 should be a normal heading, got {role.role}"
     )