diff --git a/paperforge/worker/ocr_document.py b/paperforge/worker/ocr_document.py index 4b481c12..9fb97919 100644 --- a/paperforge/worker/ocr_document.py +++ b/paperforge/worker/ocr_document.py @@ -154,8 +154,6 @@ def _looks_like_tail_body(block: dict) -> bool: return False if _BACKMATTER_BODY_SIGNALS.search(text): return True - if len(words) <= 25: - return True return False @@ -495,7 +493,10 @@ def _make_zone( def _canonical_section_text(block: dict) -> str: - return _block_text(block).strip().lower() + text = _block_text(block).strip().lower() + if re.match(r"^(?:\w\s)+\w$", text): + text = re.sub(r"\s+", "", text) + return text def _strip_inline_html(text: str) -> str: @@ -1584,14 +1585,7 @@ def _detect_references_start( return None -_BACKMATTER_TITLE_DENY_LIST = frozenset({ - "generative ai statement", "acknowledgments", "acknowledgements", - "funding", "conflict of interest", "competing interests", - "data availability", "supplementary materials", "supplementary material", - "author contributions", "declaration of competing interest", - "credit authorship contribution statement", "ethical statement", - "ethics statement", "institutional review board", -}) +_BACKMATTER_TITLE_DENY_LIST: frozenset[str] = frozenset() def _page_has_strong_body_continuation(page_blocks: list[dict]) -> bool: @@ -2465,6 +2459,10 @@ def _apply_content_zone_fallback(blocks: list[dict], region_bus: dict[str, dict] page = int(block.get("page", 0) or 0) if role in {"noise", "frontmatter_noise", "media_asset", "figure_asset", "figure_inner_text"}: + if page <= 2 and role in {"noise", "frontmatter_noise"}: + block["zone"] = "frontmatter_main_zone" + elif page <= 2 and role in {"media_asset", "figure_asset", "figure_inner_text"}: + block["zone"] = "display_zone" continue if role in {"paper_title", "authors", "affiliation", "frontmatter_support"} and page <= 2: @@ -2775,12 +2773,7 @@ def _looks_like_backmatter_body_text(text: str) -> bool: "conflict of interest", "declaration", "publisher", - "author contributions", - "funding", - "acknowledg", - "data availability", - "supplement", - "ethics", + "copyright", ) return any(marker in lower for marker in markers) diff --git a/paperforge/worker/ocr_roles.py b/paperforge/worker/ocr_roles.py index f080b907..0d0929e3 100644 --- a/paperforge/worker/ocr_roles.py +++ b/paperforge/worker/ocr_roles.py @@ -31,40 +31,9 @@ _PREPROOF_MARKER = re.compile( re.IGNORECASE, ) -_BACKMATTER_TITLE_DENY_LIST = { - "generative ai statement", - "acknowledgments", - "acknowledgements", - "funding", - "conflict of interest", - "competing interests", - "data availability", - "supplementary materials", - "supplementary material", - "author contributions", - "declaration of competing interest", - "credit authorship contribution statement", - "ethical statement", - "ethics statement", - "institutional review board", -} +_BACKMATTER_TITLE_DENY_LIST: set[str] = set() _BACKMATTER_HEADINGS = { - "author contributions", - "funding", - "acknowledgments", - "acknowledgements", - "conflict of interest", - "competing interests", - "data availability", - "supplementary materials", - "supplementary material", - "generative ai statement", - "declaration of competing interest", - "ethical statement", - "ethics statement", - "institutional review board", - "credit authorship contribution statement", "publisher's note", } @@ -588,10 +557,7 @@ def resolve_final_role( phrase in text_lower for phrase in ( "conflict of interest", - "publisher's note", - "publisher’s note", - "ethics statement", - "author contributions", + "publisher\u2019s note", "the remaining authors declare", "copyright", "published online", diff --git a/paperforge/worker/ocr_structural_gate.py b/paperforge/worker/ocr_structural_gate.py index 1ef8238b..b7da8193 100644 --- a/paperforge/worker/ocr_structural_gate.py +++ b/paperforge/worker/ocr_structural_gate.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from dataclasses import dataclass, field VERIFY_REQUIRED = { @@ -376,7 +377,7 @@ def build_document_abstract_span(blocks: list[dict], context: dict) -> dict: ( idx for idx, block in enumerate(blocks) - if block.get("seed_role") == "abstract_heading" or str(block.get("text", "")).strip().lower() == "abstract" + if block.get("seed_role") == "abstract_heading" or re.sub(r"\s+", "", str(block.get("text", "")).strip().lower()) == "abstract" ), None, ) diff --git a/tests/test_ocr_roles.py b/tests/test_ocr_roles.py index 50fc04e8..a285c091 100644 --- a/tests/test_ocr_roles.py +++ b/tests/test_ocr_roles.py @@ -183,31 +183,10 @@ def test_stabilize_frontmatter_noise_not_section_heading() -> None: assert assignment.role not in ("section_heading", "subsection_heading"), f"{noise} should not be a heading" -def test_backmatter_heading_not_paper_title() -> None: +def test_pre_ref_headings_classified_as_section_headings() -> None: from paperforge.worker.ocr_roles import assign_block_role - for text in ["Generative AI statement", "Acknowledgments", "Funding", "Conflict of interest"]: - block = { - "block_label": "paragraph_title", - "block_content": text, - "page": 1, - "block_bbox": [100, 50, 500, 80], - } - assignment = assign_block_role(block, page_blocks=[], page_height=1000) - assert assignment.role != "paper_title", f"'{text}' should not be paper_title" - - -def test_backmatter_heading_gets_backmatter_role() -> None: - from paperforge.worker.ocr_roles import assign_block_role - - backmatter_phrases = [ - "Author contributions", - "Funding", - "Acknowledgments", - "Conflict of interest", - "Generative AI statement", - ] - for text in backmatter_phrases: + for text in ["Author contributions", "Funding", "Acknowledgments", "Conflict of interest", "Generative AI statement"]: block = { "block_label": "paragraph_title", "block_content": text, @@ -215,7 +194,9 @@ def test_backmatter_heading_gets_backmatter_role() -> None: "block_bbox": [100, 600, 500, 630], } assignment = assign_block_role(block, page_blocks=[], page_height=1000) - assert assignment.role.startswith("backmatter_"), f"'{text}' should get backmatter_ role, got {assignment.role}" + assert assignment.role in {"section_heading", "subsection_heading", "sub_subsection_heading"}, ( + f"'{text}' on page 20 should get a normal heading role, got {assignment.role}" + ) def test_references_heading_gets_reference_role() -> None: @@ -494,8 +475,8 @@ def test_figure_caption_candidate_for_narrative_body() -> None: ) -def test_backmatter_heading_candidate_on_late_page() -> None: - """Funding on page 10 → backmatter_heading_candidate.""" +def test_pre_ref_heading_on_late_page_is_normal_heading() -> None: + """Funding on page 10 → normal heading role (no longer backmatter-specific).""" from paperforge.worker.ocr_roles import assign_block_role block = { @@ -507,8 +488,8 @@ def test_backmatter_heading_candidate_on_late_page() -> None: role = assign_block_role(block, page_blocks=[], page_height=1000) - assert role.role == "backmatter_heading_candidate", ( - f"Funding on page 10 should be backmatter_heading_candidate, got {role.role}" + assert role.role in {"section_heading", "subsection_heading", "sub_subsection_heading"}, ( + f"Funding on page 10 should be a normal heading, got {role.role}" )