diff --git a/paperforge/worker/ocr_document.py b/paperforge/worker/ocr_document.py index e348f2ce..2df91921 100644 --- a/paperforge/worker/ocr_document.py +++ b/paperforge/worker/ocr_document.py @@ -1797,6 +1797,16 @@ def _normalize_backmatter_roles_after_boundary( if tail_boundary is None or tail_boundary.spread_start is None or tail_boundary.spread_end is None: return + _POST_REF_PRESERVE_ROLES = { + "figure_inner_text", + "figure_caption", + "figure_caption_candidate", + "table_caption", + "table_caption_candidate", + "media_asset", + "table_html", + } + tail_start = _effective_tail_start(tail_boundary, blocks) boundary_seen = False backmatter_started = False @@ -1804,6 +1814,8 @@ def _normalize_backmatter_roles_after_boundary( page = block.get("page") if page is None or page < tail_start or page > tail_boundary.spread_end: continue + if block.get("role") in _POST_REF_PRESERVE_ROLES: + continue role = block.get("role") if backmatter_form == "container": @@ -2676,6 +2688,22 @@ def _split_merged_heading_blocks(blocks: list[dict], role_profiles: dict | None return result +def _looks_like_backmatter_body_text(text: str) -> bool: + lower = text.lower() + markers = ( + "conflict of interest", + "declaration", + "publisher", + "author contributions", + "funding", + "acknowledg", + "data availability", + "supplement", + "ethics", + ) + return any(marker in lower for marker in markers) + + def _exclude_tail_nonref_from_body_flow(blocks: list[dict]) -> None: for i, block in enumerate(blocks): effective_role = block.get("role") @@ -2686,18 +2714,13 @@ def _exclude_tail_nonref_from_body_flow(blocks: list[dict]) -> None: if block.get("zone") != "tail_nonref_hold_zone": continue - # Veto: body-like paragraph preceded by heading or containing substantial prose + # Only convert blocks that carry explicit backmatter evidence; + # preserve ordinary body continuation or body-like prose. text = str(block.get("text") or block.get("block_content") or "") - if i > 0: - prev = blocks[i - 1] - if str(prev.get("role") or "") in {"section_heading", "subsection_heading"}: - continue - if len(text) > 120 and ". " in text: - lower = text.lower() - bp = {"conflict of interest","declaration","publisher","author contributions", - "funding","acknowledg","data availability","supplement"} - if not any(b in lower for b in bp): - continue + if len(text) > 120 and ". " in text and not _looks_like_backmatter_body_text(text): + continue + if not _looks_like_backmatter_body_text(text): + continue old_role = block.get("role") block["role"] = "backmatter_body" diff --git a/tests/test_ocr_document.py b/tests/test_ocr_document.py index 4b0e3e0d..19420acb 100644 --- a/tests/test_ocr_document.py +++ b/tests/test_ocr_document.py @@ -4371,6 +4371,23 @@ def test_same_page_post_reference_non_reference_block_enters_tail_hold_zone() -> assert by_id["ack"]["zone"] == "tail_nonref_hold_zone" +def test_tail_nonref_exclusion_does_not_convert_plain_body_prose() -> None: + """Plain body prose above same-page References should stay body_paragraph, not backmatter_body.""" + from paperforge.worker.ocr_document import normalize_document_structure + + blocks = [ + {"block_id": "h1", "page": 8, "seed_role": "section_heading", "text": "Discussion", "bbox": [80, 120, 320, 180]}, + {"block_id": "p1", "page": 8, "seed_role": "body_paragraph", "text": "This study demonstrates a reproducible increase in migration after stimulation.", "bbox": [80, 220, 980, 320]}, + {"block_id": "refs", "page": 8, "seed_role": "reference_heading", "text": "References", "bbox": [80, 1100, 320, 1160]}, + {"block_id": "r1", "page": 8, "seed_role": "reference_item", "text": "[1] Ref one", "bbox": [80, 1220, 980, 1300]}, + ] + + _doc, normalized = normalize_document_structure(blocks) + by_id = {b["block_id"]: b for b in normalized} + assert by_id["p1"]["role"] == "body_paragraph" + assert by_id["p1"]["zone"] == "body_zone" + + # --------------------------------------------------------------------------- # Backmatter zone normalization # ---------------------------------------------------------------------------