fix: isolate OCR frontmatter side zones

paperforge/worker/ocr_document.py

@@ -489,6 +489,67 @@ def _page_band(start_page: int | None, end_page: int | None) -> dict[str, int] |
     return {"start_page": start_page, "end_page": end_page}
 
 
+def _is_frontmatter_side_candidate(block: dict, body_anchor: dict | None = None) -> bool:
+    page = int(block.get("page", 0) or 0)
+    if page <= 0:
+        return False
+
+    marker_type = ((block.get("marker_signature") or {}).get("type") or "none")
+    if marker_type == "preproof_marker" or _is_reference_item_candidate(block):
+        return False
+
+    text = str(block.get("text") or block.get("block_content") or "").strip()
+    if not text:
+        return False
+
+    lower = text.lower()
+    bbox = _block_bbox(block)
+    page_width = float(block.get("page_width") or 0)
+    page_height = float(block.get("page_height") or 0)
+    block_width = (bbox[2] - bbox[0]) if bbox else 0.0
+    x_center = ((bbox[0] + bbox[2]) / 2.0) if bbox else 0.0
+    narrow = page_width > 0 and block_width > 0 and block_width <= page_width * 0.38
+    side_column = page_width > 0 and bbox is not None and (x_center <= page_width * 0.28 or x_center >= page_width * 0.72)
+    top_half = page_height > 0 and bbox is not None and bbox[1] <= page_height * 0.55
+
+    furniture_phrases = (
+        "correspondence",
+        "corresponding author",
+        "highlights",
+        "received:",
+        "accepted:",
+        "published online",
+        "copyright",
+        "edited by",
+        "reviewed by",
+        "specialty section",
+        "citation:",
+        "how to cite",
+        "to cite this article",
+        "conflict of interest",
+        "equal contribution",
+        "these authors contributed equally",
+        "orcid",
+    )
+    if any(phrase in lower for phrase in furniture_phrases):
+        if page == 1:
+            return True
+        return top_half and (narrow or side_column)
+
+    body_anchor = body_anchor or {}
+    body_width = body_anchor.get("width_bucket")
+    body_font_family = body_anchor.get("font_family_norm")
+    span_signature = block.get("span_signature") or {}
+    block_font_family = span_signature.get("font_family_norm")
+    if page <= 2 and top_half and (narrow or side_column):
+        if body_width is not None and block_width and block_width <= float(body_width) - 100:
+            return True
+        if body_font_family and block_font_family and block_font_family != body_font_family:
+            return True
+
+    return False
+
+
 def infer_zones(
     blocks: list[dict],
     anchors: dict[str, dict] | None,
@@ -548,6 +609,23 @@ def infer_zones(
         and block.get("block_id")
     ]
 
+    frontmatter_side_ids = [
+        str(block.get("block_id"))
+        for block in blocks
+        if _is_frontmatter_side_candidate(block, body_anchor=body_anchor)
+        and str(block.get("block_id")) not in frontmatter_main_ids
+        and block.get("block_id")
+    ]
+    frontmatter_side_id_set = set(frontmatter_side_ids)
+    frontmatter_side_pages = sorted(
+        {
+            int(block.get("page", 0) or 0)
+            for block in blocks
+            if str(block.get("block_id") or "") in frontmatter_side_id_set
+            and int(block.get("page", 0) or 0) > 0
+        }
+    )
+
     tail_hold_start, tail_hold_end = _infer_tail_hold_band(
         blocks,
         body_sample_pages=body_sample_pages,
@@ -569,6 +647,7 @@ def infer_zones(
         and int(block.get("page", 0) or 0) > 1
         and (body_end_page is None or int(block.get("page", 0) or 0) <= body_end_page)
         and not _is_reference_item_candidate(block)
+        and str(block.get("block_id") or "") not in frontmatter_side_id_set
         and block.get("block_id")
     ]
 
@@ -594,7 +673,11 @@ def infer_zones(
             frontmatter_main_ids,
             boundary_band=_page_band(1, 1) if frontmatter_main_ids else None,
         ),
-        "frontmatter_side_zone": _make_zone("HOLD", [], boundary_band=None),
+        "frontmatter_side_zone": _make_zone(
+            "ACCEPT" if frontmatter_side_ids else "HOLD",
+            frontmatter_side_ids,
+            boundary_band=_page_band(frontmatter_side_pages[0], frontmatter_side_pages[-1]) if frontmatter_side_pages else None,
+        ),
         "body_zone": _make_zone(
             "ACCEPT" if body_block_ids else "HOLD",
             body_block_ids,
@@ -1691,6 +1774,27 @@ def _apply_zone_labels(blocks: list[dict], region_bus: dict[str, dict] | None) -
             block["zone"] = zone_name
 
 
+def _exclude_frontmatter_side_from_body_flow(blocks: list[dict]) -> None:
+    for block in blocks:
+        if block.get("role") != "body_paragraph":
+            continue
+        if block.get("zone") != "frontmatter_side_zone":
+            continue
+        if block.get("style_family") != "support_like":
+            continue
+        old_role = block.get("role")
+        block["role"] = "frontmatter_noise"
+        if old_role != block["role"]:
+            record_decision(
+                block,
+                stage="frontmatter_side_exclusion",
+                old_role=old_role,
+                new_role=block["role"],
+                reason="frontmatter-side support block excluded from body flow",
+            )
+        block.setdefault("evidence", []).append("frontmatter_side_zone excluded from body flow")
+
+
 def _get_column(block: dict, page_width: float = 1200) -> int:
     bbox = block.get("bbox") or block.get("block_bbox")
     if bbox and len(bbox) >= 4:
@@ -3124,6 +3228,7 @@ def normalize_document_structure(blocks: list[dict]) -> tuple[DocumentStructure,
             "reference_family_anchor": reference_family_anchor,
         },
     )
+    _exclude_frontmatter_side_from_body_flow(blocks)
 
     from paperforge.worker.ocr_roles import resolve_final_role
 

paperforge/worker/ocr_families.py

@@ -243,6 +243,8 @@ def _classify_style_family(
         return "reference_like", "reference_family_anchor"
     if marker_type in _REFERENCE_MARKER_TYPES:
         return "reference_like", "reference_marker"
+    if zone == "frontmatter_side_zone":
+        return "support_like", "frontmatter_side_zone"
     if marker_type == "table_number" or text_lower.startswith("table "):
         return "table_caption_like", "table_marker"
     if marker_type in {"figure_number", "panel_label"} or text_lower.startswith("figure "):

tests/test_ocr_document.py

@@ -3568,6 +3568,79 @@ def test_normalize_promotes_mixed_sidebar_blocks_into_single_structured_insert_c
     assert normalized[4]["role"] == "body_paragraph"
 
 
+def test_frontmatter_side_candidates_are_not_left_as_body_paragraph_when_source_frontmatter_is_localized() -> None:
+    from paperforge.worker.ocr_document import normalize_document_structure
+
+    blocks = [
+        {
+            "block_id": "p1_title",
+            "page": 1,
+            "role": "paper_title",
+            "text": "Magnetoresponsive stem cell spheroid-based cartilage recovery platform",
+            "bbox": [120, 90, 980, 150],
+            "page_width": 1200,
+            "page_height": 1700,
+            "marker_signature": {"type": "none"},
+            "span_signature": {"font_family_norm": "Title", "font_size_median": 16.0, "font_size_bucket": 16.0},
+            "layout_signature": {"width": 860, "width_bucket": 850, "x_center": 550, "x_center_bucket": 550},
+        },
+        {
+            "block_id": "p2_corr",
+            "page": 2,
+            "role": "body_paragraph",
+            "text": "Correspondence: jia@example.org",
+            "bbox": [830, 180, 1110, 220],
+            "page_width": 1200,
+            "page_height": 1700,
+            "marker_signature": {"type": "none"},
+            "span_signature": {"font_family_norm": "Sidebar", "font_size_median": 8.0, "font_size_bucket": 8.0},
+            "layout_signature": {"width": 280, "width_bucket": 275, "x_center": 970, "x_center_bucket": 975},
+        },
+        {
+            "block_id": "p2_highlights",
+            "page": 2,
+            "role": "body_paragraph",
+            "text": "Highlights: electromagnetic fields improved cartilage repair outcomes.",
+            "bbox": [820, 250, 1115, 340],
+            "page_width": 1200,
+            "page_height": 1700,
+            "marker_signature": {"type": "none"},
+            "span_signature": {"font_family_norm": "Sidebar", "font_size_median": 8.0, "font_size_bucket": 8.0},
+            "layout_signature": {"width": 295, "width_bucket": 300, "x_center": 968, "x_center_bucket": 975},
+        },
+    ]
+
+    for page in range(3, 6):
+        for line in range(3):
+            blocks.append(
+                {
+                    "block_id": f"p{page}_body_{line}",
+                    "page": page,
+                    "role": "body_paragraph",
+                    "text": (
+                        "Stable body paragraph text with enough words to establish the "
+                        "main article family anchor across repeated middle pages. "
+                    )
+                    * 2,
+                    "bbox": [90, 180 + line * 90, 610, 245 + line * 90],
+                    "page_width": 1200,
+                    "page_height": 1700,
+                    "marker_signature": {"type": "none"},
+                    "span_signature": {"font_family_norm": "Body", "font_size_median": 9.0, "font_size_bucket": 9.0},
+                    "layout_signature": {"width": 520, "width_bucket": 525, "x_center": 350, "x_center_bucket": 350},
+                }
+            )
+
+    _, normalized = normalize_document_structure(blocks)
+    by_id = {str(block.get("block_id")): block for block in normalized}
+
+    for block_id in ("p2_corr", "p2_highlights"):
+        block = by_id[block_id]
+        assert block.get("zone") == "frontmatter_side_zone"
+        assert block.get("style_family") == "support_like"
+        assert block.get("role") != "body_paragraph"
+
+
 def test_full_width_heading_above_two_columns_is_not_anomaly() -> None:
     """Full-width heading spanning >55% page width is NOT an error, even if it owns body in both columns."""
     from paperforge.worker.ocr_document import _run_layout_audit

tests/test_ocr_real_paper_regressions.py

@@ -9,7 +9,7 @@ import pytest
 REAL_VAULT_ENV = "PAPERFORGE_REAL_OCR_VAULT"
 REAL_KEYS_ENV = "PAPERFORGE_REAL_OCR_KEYS"
 
-PROBLEM_KEYS = ["TSCKAVIS", "CAQNW9Q2", "A8E7SRVS", "K7R8PEKW"]
+PROBLEM_KEYS = ["TSCKAVIS", "CAQNW9Q2", "A8E7SRVS", "K7R8PEKW", "DWQQK2YB", "M36WA39N"]
 CONTROL_KEYS = ["SAN9AYVR", "2GN9LMCW", "7C8829BD"]
 
 
@@ -75,7 +75,12 @@ def _read_jsonl(path: Path) -> list[dict]:
 def _require_artifacts(ocr_root: Path, key: str) -> None:
     missing = [
         str(path)
-        for path in [_structured_path(ocr_root, key), _fulltext_path(ocr_root, key), _health_path(ocr_root, key), _metadata_path(ocr_root, key)]
+        for path in [
+            _structured_path(ocr_root, key),
+            _fulltext_path(ocr_root, key),
+            _health_path(ocr_root, key),
+            _metadata_path(ocr_root, key),
+        ]
         if not path.exists()
     ]
     if missing:
@@ -118,8 +123,12 @@ def test_real_paper_rebuild_runs(rebuilt_real_papers: dict) -> None:
 BODY_RETENTION = {
     "CAQNW9Q2": {"min_body": 35, "max_non_body_insert": 8},
     "A8E7SRVS": {"min_body": 45, "max_non_body_insert": 8},
+    # K7R8PEKW remains in the problem cohort as a generic body/reference retention guard.
+    # It does not yet have a paper-specific recovery contract in this file.
     "K7R8PEKW": {"min_body": 60, "max_non_body_insert": 8},
     "TSCKAVIS": {"min_body": 55, "max_non_body_insert": 12},
+    "DWQQK2YB": {"min_body": 25, "max_non_body_insert": 12},
+    "M36WA39N": {"min_body": 50, "max_non_body_insert": 8},
 }
 
 
@@ -128,7 +137,9 @@ def _role_texts(blocks: list[dict], role: str) -> list[str]:
 
 
 @pytest.mark.parametrize("key", sorted(BODY_RETENTION))
-def test_problem_papers_retain_body_and_avoid_mass_insert_suppression(rebuilt_real_papers: dict, _ocr_root: Path, key: str) -> None:
+def test_problem_papers_retain_body_and_avoid_mass_insert_suppression(
+    rebuilt_real_papers: dict, _ocr_root: Path, key: str
+) -> None:
     _require_artifacts(_ocr_root, key)
     blocks = _read_jsonl(_structured_path(_ocr_root, key))
     roles = [block.get("role") for block in blocks]
@@ -138,6 +149,45 @@ def test_problem_papers_retain_body_and_avoid_mass_insert_suppression(rebuilt_re
     assert roles.count("non_body_insert") <= thresholds["max_non_body_insert"], roles.count("non_body_insert")
 
 
+def test_a8e7srvs_no_frontmatter_body_before_introduction(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
+    key = "A8E7SRVS"
+    _require_artifacts(_ocr_root, key)
+    blocks = _read_jsonl(_structured_path(_ocr_root, key))
+
+    intro_idx = len(blocks)
+    for idx, b in enumerate(blocks):
+        if (
+            b.get("role") in ("section_heading", "subsection_heading")
+            and "introduction" in str(b.get("text", "")).lower()
+        ):
+            intro_idx = idx
+            break
+
+    assert intro_idx < len(blocks), "No 'Introduction' heading found in A8E7SRVS"
+
+    before_intro = blocks[:intro_idx]
+    fm_bodies = [
+        b
+        for b in before_intro
+        if b.get("role") == "body_paragraph"
+        and any(
+            s in (b.get("text") or "").lower()
+            for s in [
+                "received:",
+                "accepted:",
+                "published online",
+                "copyright",
+                "each author certifies",
+                "icmje conflict",
+            ]
+        )
+    ]
+    assert len(fm_bodies) == 0, (
+        f"Frontmatter-side text leaked into body_paragraph before Introduction "
+        f"(block_ids={[b['block_id'] for b in fm_bodies]})"
+    )
+
+
 def test_tsckavis_frontmatter_and_key_points_are_callout(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
     key = "TSCKAVIS"
     _require_artifacts(_ocr_root, key)
@@ -169,6 +219,31 @@ def test_tsckavis_frontmatter_and_key_points_are_callout(rebuilt_real_papers: di
     assert meta.get("doi", {}).get("value", ""), meta.get("doi")
 
 
+def test_tsckavis_no_table_display_as_heading(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
+    key = "TSCKAVIS"
+    _require_artifacts(_ocr_root, key)
+    blocks = _read_jsonl(_structured_path(_ocr_root, key))
+
+    heading_text = "\n".join(
+        _role_texts(blocks, "section_heading")
+        + _role_texts(blocks, "subsection_heading")
+        + _role_texts(blocks, "sub_subsection_heading")
+    ).lower()
+
+    assert "published online" not in heading_text, "'Published online' leaked into heading roles for TSCKAVIS"
+
+    body_with_fig = [
+        b
+        for b in blocks
+        if b.get("role") == "body_paragraph"
+        and b.get("text", "").strip().lower().startswith("fig. ")
+        and any(c.isdigit() for c in b.get("text", "")[:10])
+    ]
+    assert len(body_with_fig) == 0, (
+        f"Figure-like text in body_paragraph: block_ids={[b['block_id'] for b in body_with_fig]}"
+    )
+
+
 CONTROL_MIN_BODY = {
     "SAN9AYVR": 250,
     "2GN9LMCW": 25,
@@ -184,7 +259,9 @@ def test_control_papers_keep_body_and_tail_stability(rebuilt_real_papers: dict,
     fulltext = _fulltext_path(_ocr_root, key).read_text(encoding="utf-8", errors="replace")
 
     assert roles.count("body_paragraph") >= CONTROL_MIN_BODY[key]
-    assert any(w in fulltext for w in ["References", "references", "REFERENCE", "Bibliography"]), "Tail reference marker not found"
+    assert any(w in fulltext for w in ["References", "references", "REFERENCE", "Bibliography"]), (
+        "Tail reference marker not found"
+    )
 
 
 @pytest.mark.parametrize("key", PROBLEM_KEYS)
@@ -197,7 +274,68 @@ def test_problem_papers_keep_reference_roles_and_exclude_legend_family_from_body
     blocks = _read_jsonl(_structured_path(_ocr_root, key))
 
     assert any(block.get("role") == "reference_item" for block in blocks)
+    assert any(block.get("style_family") for block in blocks), f"No style_family artifacts found for {key}"
     assert not any(
-        block.get("role") == "body_paragraph" and block.get("style_family") in {"legend_like", "table_caption_like", "reference_like"}
+        block.get("role") == "body_paragraph"
+        and block.get("style_family") in {"legend_like", "table_caption_like", "reference_like"}
         for block in blocks
     )
+
+
+def test_dwqqk2yb_post_preproof_not_body(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
+    key = "DWQQK2YB"
+    _require_artifacts(_ocr_root, key)
+    blocks = _read_jsonl(_structured_path(_ocr_root, key))
+
+    corr_bodies = [
+        b
+        for b in blocks
+        if b.get("role") == "body_paragraph"
+        and any(
+            phrase in (b.get("text") or "").lower()
+            for phrase in ["corresponding author", "correspondence", "highlights"]
+        )
+    ]
+    assert len(corr_bodies) == 0, (
+        f"Post-preproof frontmatter leaked into body_paragraph: block_ids={[b['block_id'] for b in corr_bodies]}"
+    )
+
+    total_refs = sum(1 for b in blocks if b.get("role") == "reference_item")
+    body_zone_refs = sum(1 for b in blocks if b.get("zone") == "body_zone" and b.get("role") == "reference_item")
+    assert body_zone_refs < total_refs, f"All {total_refs} reference items remain in body_zone for {key}"
+
+
+def test_m36wa39n_editorial_furniture_not_body(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
+    key = "M36WA39N"
+    _require_artifacts(_ocr_root, key)
+    blocks = _read_jsonl(_structured_path(_ocr_root, key))
+    body_text = "\n".join(str(b.get("text", "")) for b in blocks if b.get("role") == "body_paragraph").lower()
+
+    forbidden = ["edited by:", "reviewed by:", "correspondence:", "specialty section:", "citation:"]
+    for phrase in forbidden:
+        assert phrase not in body_text, f"'{phrase}' found in body_paragraph for {key}"
+
+    tail_phrases = ["ethics statement", "author contributions"]
+    for phrase in tail_phrases:
+        assert phrase not in body_text, f"'{phrase}' collapsed into body_paragraph for {key}"
+
+
+def test_caqnw9q2_heading_preservation_and_ref_classification(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
+    key = "CAQNW9Q2"
+    _require_artifacts(_ocr_root, key)
+    blocks = _read_jsonl(_structured_path(_ocr_root, key))
+
+    headings = [
+        b for b in blocks if b.get("role") in ("section_heading", "subsection_heading", "sub_subsection_heading")
+    ]
+    assert len(headings) >= 5, f"Too few headings ({len(headings)}) for {key}"
+
+    heading_text = "\n".join(_role_texts(blocks, "section_heading")).lower()
+    assert "conclusion" in heading_text, f"'Conclusion' heading not found for {key}"
+
+    ref_items = [b for b in blocks if b.get("role") == "reference_item"]
+    ref_like = [b for b in ref_items if b.get("style_family") == "reference_like"]
+    assert len(ref_like) > 0, (
+        f"No reference items have reference_like style_family for {key} "
+        f"(style families present: {set(b.get('style_family') for b in ref_items)})"
+    )