From 72625f76e8b6ff455f7623920a9db0f40d138609 Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Tue, 9 Jun 2026 23:49:54 +0800 Subject: [PATCH] fix: isolate OCR frontmatter side zones --- paperforge/worker/ocr_document.py | 107 +++++++++++++++- paperforge/worker/ocr_families.py | 2 + tests/test_ocr_document.py | 73 +++++++++++ tests/test_ocr_real_paper_regressions.py | 148 ++++++++++++++++++++++- 4 files changed, 324 insertions(+), 6 deletions(-) diff --git a/paperforge/worker/ocr_document.py b/paperforge/worker/ocr_document.py index 7e5559f1..4fc06306 100644 --- a/paperforge/worker/ocr_document.py +++ b/paperforge/worker/ocr_document.py @@ -489,6 +489,67 @@ def _page_band(start_page: int | None, end_page: int | None) -> dict[str, int] | return {"start_page": start_page, "end_page": end_page} +def _is_frontmatter_side_candidate(block: dict, body_anchor: dict | None = None) -> bool: + page = int(block.get("page", 0) or 0) + if page <= 0: + return False + + marker_type = ((block.get("marker_signature") or {}).get("type") or "none") + if marker_type == "preproof_marker" or _is_reference_item_candidate(block): + return False + + text = str(block.get("text") or block.get("block_content") or "").strip() + if not text: + return False + + lower = text.lower() + bbox = _block_bbox(block) + page_width = float(block.get("page_width") or 0) + page_height = float(block.get("page_height") or 0) + block_width = (bbox[2] - bbox[0]) if bbox else 0.0 + x_center = ((bbox[0] + bbox[2]) / 2.0) if bbox else 0.0 + narrow = page_width > 0 and block_width > 0 and block_width <= page_width * 0.38 + side_column = page_width > 0 and bbox is not None and (x_center <= page_width * 0.28 or x_center >= page_width * 0.72) + top_half = page_height > 0 and bbox is not None and bbox[1] <= page_height * 0.55 + + furniture_phrases = ( + "correspondence", + "corresponding author", + "highlights", + "received:", + "accepted:", + "published online", + "copyright", + "edited by", + "reviewed by", + "specialty section", + "citation:", + "how to cite", + "to cite this article", + "conflict of interest", + "equal contribution", + "these authors contributed equally", + "orcid", + ) + if any(phrase in lower for phrase in furniture_phrases): + if page == 1: + return True + return top_half and (narrow or side_column) + + body_anchor = body_anchor or {} + body_width = body_anchor.get("width_bucket") + body_font_family = body_anchor.get("font_family_norm") + span_signature = block.get("span_signature") or {} + block_font_family = span_signature.get("font_family_norm") + if page <= 2 and top_half and (narrow or side_column): + if body_width is not None and block_width and block_width <= float(body_width) - 100: + return True + if body_font_family and block_font_family and block_font_family != body_font_family: + return True + + return False + + def infer_zones( blocks: list[dict], anchors: dict[str, dict] | None, @@ -548,6 +609,23 @@ def infer_zones( and block.get("block_id") ] + frontmatter_side_ids = [ + str(block.get("block_id")) + for block in blocks + if _is_frontmatter_side_candidate(block, body_anchor=body_anchor) + and str(block.get("block_id")) not in frontmatter_main_ids + and block.get("block_id") + ] + frontmatter_side_id_set = set(frontmatter_side_ids) + frontmatter_side_pages = sorted( + { + int(block.get("page", 0) or 0) + for block in blocks + if str(block.get("block_id") or "") in frontmatter_side_id_set + and int(block.get("page", 0) or 0) > 0 + } + ) + tail_hold_start, tail_hold_end = _infer_tail_hold_band( blocks, body_sample_pages=body_sample_pages, @@ -569,6 +647,7 @@ def infer_zones( and int(block.get("page", 0) or 0) > 1 and (body_end_page is None or int(block.get("page", 0) or 0) <= body_end_page) and not _is_reference_item_candidate(block) + and str(block.get("block_id") or "") not in frontmatter_side_id_set and block.get("block_id") ] @@ -594,7 +673,11 @@ def infer_zones( frontmatter_main_ids, boundary_band=_page_band(1, 1) if frontmatter_main_ids else None, ), - "frontmatter_side_zone": _make_zone("HOLD", [], boundary_band=None), + "frontmatter_side_zone": _make_zone( + "ACCEPT" if frontmatter_side_ids else "HOLD", + frontmatter_side_ids, + boundary_band=_page_band(frontmatter_side_pages[0], frontmatter_side_pages[-1]) if frontmatter_side_pages else None, + ), "body_zone": _make_zone( "ACCEPT" if body_block_ids else "HOLD", body_block_ids, @@ -1691,6 +1774,27 @@ def _apply_zone_labels(blocks: list[dict], region_bus: dict[str, dict] | None) - block["zone"] = zone_name +def _exclude_frontmatter_side_from_body_flow(blocks: list[dict]) -> None: + for block in blocks: + if block.get("role") != "body_paragraph": + continue + if block.get("zone") != "frontmatter_side_zone": + continue + if block.get("style_family") != "support_like": + continue + old_role = block.get("role") + block["role"] = "frontmatter_noise" + if old_role != block["role"]: + record_decision( + block, + stage="frontmatter_side_exclusion", + old_role=old_role, + new_role=block["role"], + reason="frontmatter-side support block excluded from body flow", + ) + block.setdefault("evidence", []).append("frontmatter_side_zone excluded from body flow") + + def _get_column(block: dict, page_width: float = 1200) -> int: bbox = block.get("bbox") or block.get("block_bbox") if bbox and len(bbox) >= 4: @@ -3124,6 +3228,7 @@ def normalize_document_structure(blocks: list[dict]) -> tuple[DocumentStructure, "reference_family_anchor": reference_family_anchor, }, ) + _exclude_frontmatter_side_from_body_flow(blocks) from paperforge.worker.ocr_roles import resolve_final_role diff --git a/paperforge/worker/ocr_families.py b/paperforge/worker/ocr_families.py index 7efe8c46..4d6e4f22 100644 --- a/paperforge/worker/ocr_families.py +++ b/paperforge/worker/ocr_families.py @@ -243,6 +243,8 @@ def _classify_style_family( return "reference_like", "reference_family_anchor" if marker_type in _REFERENCE_MARKER_TYPES: return "reference_like", "reference_marker" + if zone == "frontmatter_side_zone": + return "support_like", "frontmatter_side_zone" if marker_type == "table_number" or text_lower.startswith("table "): return "table_caption_like", "table_marker" if marker_type in {"figure_number", "panel_label"} or text_lower.startswith("figure "): diff --git a/tests/test_ocr_document.py b/tests/test_ocr_document.py index 44da39d9..2710e405 100644 --- a/tests/test_ocr_document.py +++ b/tests/test_ocr_document.py @@ -3568,6 +3568,79 @@ def test_normalize_promotes_mixed_sidebar_blocks_into_single_structured_insert_c assert normalized[4]["role"] == "body_paragraph" +def test_frontmatter_side_candidates_are_not_left_as_body_paragraph_when_source_frontmatter_is_localized() -> None: + from paperforge.worker.ocr_document import normalize_document_structure + + blocks = [ + { + "block_id": "p1_title", + "page": 1, + "role": "paper_title", + "text": "Magnetoresponsive stem cell spheroid-based cartilage recovery platform", + "bbox": [120, 90, 980, 150], + "page_width": 1200, + "page_height": 1700, + "marker_signature": {"type": "none"}, + "span_signature": {"font_family_norm": "Title", "font_size_median": 16.0, "font_size_bucket": 16.0}, + "layout_signature": {"width": 860, "width_bucket": 850, "x_center": 550, "x_center_bucket": 550}, + }, + { + "block_id": "p2_corr", + "page": 2, + "role": "body_paragraph", + "text": "Correspondence: jia@example.org", + "bbox": [830, 180, 1110, 220], + "page_width": 1200, + "page_height": 1700, + "marker_signature": {"type": "none"}, + "span_signature": {"font_family_norm": "Sidebar", "font_size_median": 8.0, "font_size_bucket": 8.0}, + "layout_signature": {"width": 280, "width_bucket": 275, "x_center": 970, "x_center_bucket": 975}, + }, + { + "block_id": "p2_highlights", + "page": 2, + "role": "body_paragraph", + "text": "Highlights: electromagnetic fields improved cartilage repair outcomes.", + "bbox": [820, 250, 1115, 340], + "page_width": 1200, + "page_height": 1700, + "marker_signature": {"type": "none"}, + "span_signature": {"font_family_norm": "Sidebar", "font_size_median": 8.0, "font_size_bucket": 8.0}, + "layout_signature": {"width": 295, "width_bucket": 300, "x_center": 968, "x_center_bucket": 975}, + }, + ] + + for page in range(3, 6): + for line in range(3): + blocks.append( + { + "block_id": f"p{page}_body_{line}", + "page": page, + "role": "body_paragraph", + "text": ( + "Stable body paragraph text with enough words to establish the " + "main article family anchor across repeated middle pages. " + ) + * 2, + "bbox": [90, 180 + line * 90, 610, 245 + line * 90], + "page_width": 1200, + "page_height": 1700, + "marker_signature": {"type": "none"}, + "span_signature": {"font_family_norm": "Body", "font_size_median": 9.0, "font_size_bucket": 9.0}, + "layout_signature": {"width": 520, "width_bucket": 525, "x_center": 350, "x_center_bucket": 350}, + } + ) + + _, normalized = normalize_document_structure(blocks) + by_id = {str(block.get("block_id")): block for block in normalized} + + for block_id in ("p2_corr", "p2_highlights"): + block = by_id[block_id] + assert block.get("zone") == "frontmatter_side_zone" + assert block.get("style_family") == "support_like" + assert block.get("role") != "body_paragraph" + + def test_full_width_heading_above_two_columns_is_not_anomaly() -> None: """Full-width heading spanning >55% page width is NOT an error, even if it owns body in both columns.""" from paperforge.worker.ocr_document import _run_layout_audit diff --git a/tests/test_ocr_real_paper_regressions.py b/tests/test_ocr_real_paper_regressions.py index b7fbd700..6504fc28 100644 --- a/tests/test_ocr_real_paper_regressions.py +++ b/tests/test_ocr_real_paper_regressions.py @@ -9,7 +9,7 @@ import pytest REAL_VAULT_ENV = "PAPERFORGE_REAL_OCR_VAULT" REAL_KEYS_ENV = "PAPERFORGE_REAL_OCR_KEYS" -PROBLEM_KEYS = ["TSCKAVIS", "CAQNW9Q2", "A8E7SRVS", "K7R8PEKW"] +PROBLEM_KEYS = ["TSCKAVIS", "CAQNW9Q2", "A8E7SRVS", "K7R8PEKW", "DWQQK2YB", "M36WA39N"] CONTROL_KEYS = ["SAN9AYVR", "2GN9LMCW", "7C8829BD"] @@ -75,7 +75,12 @@ def _read_jsonl(path: Path) -> list[dict]: def _require_artifacts(ocr_root: Path, key: str) -> None: missing = [ str(path) - for path in [_structured_path(ocr_root, key), _fulltext_path(ocr_root, key), _health_path(ocr_root, key), _metadata_path(ocr_root, key)] + for path in [ + _structured_path(ocr_root, key), + _fulltext_path(ocr_root, key), + _health_path(ocr_root, key), + _metadata_path(ocr_root, key), + ] if not path.exists() ] if missing: @@ -118,8 +123,12 @@ def test_real_paper_rebuild_runs(rebuilt_real_papers: dict) -> None: BODY_RETENTION = { "CAQNW9Q2": {"min_body": 35, "max_non_body_insert": 8}, "A8E7SRVS": {"min_body": 45, "max_non_body_insert": 8}, + # K7R8PEKW remains in the problem cohort as a generic body/reference retention guard. + # It does not yet have a paper-specific recovery contract in this file. "K7R8PEKW": {"min_body": 60, "max_non_body_insert": 8}, "TSCKAVIS": {"min_body": 55, "max_non_body_insert": 12}, + "DWQQK2YB": {"min_body": 25, "max_non_body_insert": 12}, + "M36WA39N": {"min_body": 50, "max_non_body_insert": 8}, } @@ -128,7 +137,9 @@ def _role_texts(blocks: list[dict], role: str) -> list[str]: @pytest.mark.parametrize("key", sorted(BODY_RETENTION)) -def test_problem_papers_retain_body_and_avoid_mass_insert_suppression(rebuilt_real_papers: dict, _ocr_root: Path, key: str) -> None: +def test_problem_papers_retain_body_and_avoid_mass_insert_suppression( + rebuilt_real_papers: dict, _ocr_root: Path, key: str +) -> None: _require_artifacts(_ocr_root, key) blocks = _read_jsonl(_structured_path(_ocr_root, key)) roles = [block.get("role") for block in blocks] @@ -138,6 +149,45 @@ def test_problem_papers_retain_body_and_avoid_mass_insert_suppression(rebuilt_re assert roles.count("non_body_insert") <= thresholds["max_non_body_insert"], roles.count("non_body_insert") +def test_a8e7srvs_no_frontmatter_body_before_introduction(rebuilt_real_papers: dict, _ocr_root: Path) -> None: + key = "A8E7SRVS" + _require_artifacts(_ocr_root, key) + blocks = _read_jsonl(_structured_path(_ocr_root, key)) + + intro_idx = len(blocks) + for idx, b in enumerate(blocks): + if ( + b.get("role") in ("section_heading", "subsection_heading") + and "introduction" in str(b.get("text", "")).lower() + ): + intro_idx = idx + break + + assert intro_idx < len(blocks), "No 'Introduction' heading found in A8E7SRVS" + + before_intro = blocks[:intro_idx] + fm_bodies = [ + b + for b in before_intro + if b.get("role") == "body_paragraph" + and any( + s in (b.get("text") or "").lower() + for s in [ + "received:", + "accepted:", + "published online", + "copyright", + "each author certifies", + "icmje conflict", + ] + ) + ] + assert len(fm_bodies) == 0, ( + f"Frontmatter-side text leaked into body_paragraph before Introduction " + f"(block_ids={[b['block_id'] for b in fm_bodies]})" + ) + + def test_tsckavis_frontmatter_and_key_points_are_callout(rebuilt_real_papers: dict, _ocr_root: Path) -> None: key = "TSCKAVIS" _require_artifacts(_ocr_root, key) @@ -169,6 +219,31 @@ def test_tsckavis_frontmatter_and_key_points_are_callout(rebuilt_real_papers: di assert meta.get("doi", {}).get("value", ""), meta.get("doi") +def test_tsckavis_no_table_display_as_heading(rebuilt_real_papers: dict, _ocr_root: Path) -> None: + key = "TSCKAVIS" + _require_artifacts(_ocr_root, key) + blocks = _read_jsonl(_structured_path(_ocr_root, key)) + + heading_text = "\n".join( + _role_texts(blocks, "section_heading") + + _role_texts(blocks, "subsection_heading") + + _role_texts(blocks, "sub_subsection_heading") + ).lower() + + assert "published online" not in heading_text, "'Published online' leaked into heading roles for TSCKAVIS" + + body_with_fig = [ + b + for b in blocks + if b.get("role") == "body_paragraph" + and b.get("text", "").strip().lower().startswith("fig. ") + and any(c.isdigit() for c in b.get("text", "")[:10]) + ] + assert len(body_with_fig) == 0, ( + f"Figure-like text in body_paragraph: block_ids={[b['block_id'] for b in body_with_fig]}" + ) + + CONTROL_MIN_BODY = { "SAN9AYVR": 250, "2GN9LMCW": 25, @@ -184,7 +259,9 @@ def test_control_papers_keep_body_and_tail_stability(rebuilt_real_papers: dict, fulltext = _fulltext_path(_ocr_root, key).read_text(encoding="utf-8", errors="replace") assert roles.count("body_paragraph") >= CONTROL_MIN_BODY[key] - assert any(w in fulltext for w in ["References", "references", "REFERENCE", "Bibliography"]), "Tail reference marker not found" + assert any(w in fulltext for w in ["References", "references", "REFERENCE", "Bibliography"]), ( + "Tail reference marker not found" + ) @pytest.mark.parametrize("key", PROBLEM_KEYS) @@ -197,7 +274,68 @@ def test_problem_papers_keep_reference_roles_and_exclude_legend_family_from_body blocks = _read_jsonl(_structured_path(_ocr_root, key)) assert any(block.get("role") == "reference_item" for block in blocks) + assert any(block.get("style_family") for block in blocks), f"No style_family artifacts found for {key}" assert not any( - block.get("role") == "body_paragraph" and block.get("style_family") in {"legend_like", "table_caption_like", "reference_like"} + block.get("role") == "body_paragraph" + and block.get("style_family") in {"legend_like", "table_caption_like", "reference_like"} for block in blocks ) + + +def test_dwqqk2yb_post_preproof_not_body(rebuilt_real_papers: dict, _ocr_root: Path) -> None: + key = "DWQQK2YB" + _require_artifacts(_ocr_root, key) + blocks = _read_jsonl(_structured_path(_ocr_root, key)) + + corr_bodies = [ + b + for b in blocks + if b.get("role") == "body_paragraph" + and any( + phrase in (b.get("text") or "").lower() + for phrase in ["corresponding author", "correspondence", "highlights"] + ) + ] + assert len(corr_bodies) == 0, ( + f"Post-preproof frontmatter leaked into body_paragraph: block_ids={[b['block_id'] for b in corr_bodies]}" + ) + + total_refs = sum(1 for b in blocks if b.get("role") == "reference_item") + body_zone_refs = sum(1 for b in blocks if b.get("zone") == "body_zone" and b.get("role") == "reference_item") + assert body_zone_refs < total_refs, f"All {total_refs} reference items remain in body_zone for {key}" + + +def test_m36wa39n_editorial_furniture_not_body(rebuilt_real_papers: dict, _ocr_root: Path) -> None: + key = "M36WA39N" + _require_artifacts(_ocr_root, key) + blocks = _read_jsonl(_structured_path(_ocr_root, key)) + body_text = "\n".join(str(b.get("text", "")) for b in blocks if b.get("role") == "body_paragraph").lower() + + forbidden = ["edited by:", "reviewed by:", "correspondence:", "specialty section:", "citation:"] + for phrase in forbidden: + assert phrase not in body_text, f"'{phrase}' found in body_paragraph for {key}" + + tail_phrases = ["ethics statement", "author contributions"] + for phrase in tail_phrases: + assert phrase not in body_text, f"'{phrase}' collapsed into body_paragraph for {key}" + + +def test_caqnw9q2_heading_preservation_and_ref_classification(rebuilt_real_papers: dict, _ocr_root: Path) -> None: + key = "CAQNW9Q2" + _require_artifacts(_ocr_root, key) + blocks = _read_jsonl(_structured_path(_ocr_root, key)) + + headings = [ + b for b in blocks if b.get("role") in ("section_heading", "subsection_heading", "sub_subsection_heading") + ] + assert len(headings) >= 5, f"Too few headings ({len(headings)}) for {key}" + + heading_text = "\n".join(_role_texts(blocks, "section_heading")).lower() + assert "conclusion" in heading_text, f"'Conclusion' heading not found for {key}" + + ref_items = [b for b in blocks if b.get("role") == "reference_item"] + ref_like = [b for b in ref_items if b.get("style_family") == "reference_like"] + assert len(ref_like) > 0, ( + f"No reference items have reference_like style_family for {key} " + f"(style families present: {set(b.get('style_family') for b in ref_items)})" + )