diff --git a/paperforge/worker/ocr_blocks.py b/paperforge/worker/ocr_blocks.py index 92b1e8e2..250e14e6 100644 --- a/paperforge/worker/ocr_blocks.py +++ b/paperforge/worker/ocr_blocks.py @@ -179,6 +179,37 @@ def build_structured_blocks( if resolved.evidence: row.setdefault("evidence", []).extend(resolved.evidence) + if doc_structure: + from paperforge.worker.ocr_document import _apply_zone_labels, infer_zones + from paperforge.worker.ocr_families import partition_zone_families + + refreshed_anchors = { + "body_family_anchor": getattr(doc_structure, "body_family_anchor", None), + "reference_family_anchor": getattr(doc_structure, "reference_family_anchor", None), + } + refreshed_region_bus = infer_zones(rows, refreshed_anchors) + _apply_zone_labels(rows, refreshed_region_bus) + partition_zone_families(rows, refreshed_anchors) + doc_structure.region_bus = refreshed_region_bus + + refreshed_reference_rows = [row for row in rows if row.get("zone") == "reference_zone"] + if refreshed_reference_rows: + reference_pages = sorted({int(row.get("page", 0) or 0) for row in refreshed_reference_rows if int(row.get("page", 0) or 0) > 0}) + if reference_pages: + doc_structure.reference_zones = [ + { + "page": page, + "column_index": 0, + "y_start": 0, + "y_end": float(next((row.get("page_height", 0) or 0) for row in refreshed_reference_rows if int(row.get("page", 0) or 0) == page) or 0), + "block_indices": [ + idx for idx, row in enumerate(rows) + if row.get("zone") == "reference_zone" and int(row.get("page", 0) or 0) == page + ], + } + for page in reference_pages + ] + # Sync render_default/index_default after role normalizations for row in rows: role = row.get("role", "") diff --git a/paperforge/worker/ocr_document.py b/paperforge/worker/ocr_document.py index 52662c42..7e5559f1 100644 --- a/paperforge/worker/ocr_document.py +++ b/paperforge/worker/ocr_document.py @@ -388,9 +388,9 @@ def _canonical_section_text(block: dict) -> str: def _is_reference_heading_candidate(block: dict) -> bool: marker_type = ((block.get("marker_signature") or {}).get("type") or "none") - if marker_type != "canonical_section_name": + if _canonical_section_text(block) not in _REFERENCE_ZONE_HEADING_TEXTS: return False - return _canonical_section_text(block) in _REFERENCE_ZONE_HEADING_TEXTS + return marker_type in {"canonical_section_name", "short_fragment", "none"} def _is_reference_item_candidate(block: dict) -> bool: @@ -3125,6 +3125,32 @@ def normalize_document_structure(blocks: list[dict]) -> tuple[DocumentStructure, }, ) + from paperforge.worker.ocr_roles import resolve_final_role + + family_context = { + str(block.get("block_id") or f"block_{idx}"): { + "zone": block.get("zone"), + "style_family": block.get("style_family"), + "style_family_authority": block.get("style_family_authority"), + } + for idx, block in enumerate(blocks) + } + anchor_context = { + "body_family_anchor": body_family_anchor, + "reference_family_anchor": reference_family_anchor, + } + for block in blocks: + resolved = resolve_final_role(block, anchors=anchor_context, families=family_context) + if resolved.role != block.get("role"): + block["role"] = resolved.role + block["role_confidence"] = resolved.confidence + if resolved.evidence: + block.setdefault("evidence", []).extend(resolved.evidence) + + region_bus = infer_zones(blocks, anchor_context) + _apply_zone_labels(blocks, region_bus) + partition_zone_families(blocks, anchor_context) + tail_spread = _reconcile_tail_spread(blocks, page_layouts) if tail_spread is not None: backmatter_form = _classify_backmatter_form(tail_spread, blocks) diff --git a/paperforge/worker/ocr_families.py b/paperforge/worker/ocr_families.py index 6de3c109..7efe8c46 100644 --- a/paperforge/worker/ocr_families.py +++ b/paperforge/worker/ocr_families.py @@ -509,7 +509,7 @@ def _has_reference_heading_near_family(blocks: list[dict], family_pages: list[in continue marker_type = ((block.get("marker_signature") or {}).get("type") or "none") text = str(block.get("text") or block.get("block_content") or "").strip().lower() - if marker_type == "canonical_section_name" and text in _REFERENCE_HEADING_TEXTS: + if marker_type in {"canonical_section_name", "short_fragment", "none"} and text in _REFERENCE_HEADING_TEXTS: return True return False @@ -527,7 +527,8 @@ def _build_reference_family_candidate( font_family, font_size_bucket, width_bucket, x_center_bucket = family_key strong_tail = tail_continuity >= 1.0 strong_marker_family = item_count >= 2 and marker_count >= 2 - can_accept = strong_marker_family and strong_tail + heading_supported_singleton = heading_present and item_count >= 1 and marker_count >= 1 + can_accept = strong_tail and (strong_marker_family or heading_supported_singleton) reason = "reference_markers_with_tail_continuity" if heading_present: reason = f"{reason}_and_heading_binding" diff --git a/paperforge/worker/ocr_figures.py b/paperforge/worker/ocr_figures.py index 10c5de2e..133cc278 100644 --- a/paperforge/worker/ocr_figures.py +++ b/paperforge/worker/ocr_figures.py @@ -122,7 +122,11 @@ def _looks_like_figure_narrative_prose(text: str) -> bool: def _is_body_mention(block: dict) -> bool: + zone = str(block.get("zone") or "") + style_family = str(block.get("style_family") or "") raw_role = block.get("raw_role", block.get("role", "")) + if zone == "display_zone" and style_family == "legend_like": + return False if raw_role == "body_paragraph": return True if raw_role == "figure_caption_candidate": diff --git a/paperforge/worker/ocr_roles.py b/paperforge/worker/ocr_roles.py index 3d8b8b76..74572e12 100644 --- a/paperforge/worker/ocr_roles.py +++ b/paperforge/worker/ocr_roles.py @@ -434,11 +434,34 @@ def resolve_final_role( current_role = str(block.get("role") or "body_paragraph") current_confidence = float(block.get("role_confidence") or 0.6) body_anchor = anchors.get("body_family_anchor") or {} + reference_anchor = anchors.get("reference_family_anchor") or {} body_anchor_accepted = str(body_anchor.get("status") or "").upper() == "ACCEPT" + reference_anchor_accepted = str(reference_anchor.get("status") or "").upper() == "ACCEPT" in_body_zone = zone == "body_zone" strong_legend_authority = style_family_authority in {"figure_marker", "figure_family_anchor"} if current_role == "body_paragraph": + if ( + zone == "reference_zone" + and reference_anchor_accepted + and style_family == "reference_like" + and marker_type in { + "reference_numeric_bracket", + "reference_numeric_dot", + "reference_numeric_parenthesis", + "reference_pattern", + "citation_line", + } + ): + return RoleAssignment( + role="reference_item", + confidence=max(current_confidence, 0.82), + evidence=[ + "late role resolution: reference_like family + reference zone", + f"style_family_authority={style_family_authority or 'none'}", + f"context_source={context_source}", + ], + ) if ( in_body_zone and body_anchor_accepted @@ -458,6 +481,22 @@ def resolve_final_role( f"context_source={context_source}", ], ) + if ( + zone == "display_zone" + and strong_legend_authority + and style_family == "legend_like" + and marker_type == "figure_number" + and not _looks_like_late_figure_narrative_prose(str(block.get("text") or "")) + ): + return RoleAssignment( + role="figure_caption_candidate", + confidence=max(current_confidence, 0.8), + evidence=[ + "late role resolution: display-zone legend candidate from figure family", + f"style_family_authority={style_family_authority or 'none'}", + f"context_source={context_source}", + ], + ) return RoleAssignment( role=current_role, diff --git a/tests/test_ocr_document.py b/tests/test_ocr_document.py index e19d3335..d978d76f 100644 --- a/tests/test_ocr_document.py +++ b/tests/test_ocr_document.py @@ -305,6 +305,56 @@ def test_normalize_document_structure_wires_style_family_artifacts_into_blocks() assert normalized_blocks[3]["style_family"] == "legend_like" +def test_normalize_document_structure_preserves_reference_zone_integrity_from_anchor_flow() -> None: + from paperforge.worker.ocr_document import normalize_document_structure + + blocks = [ + { + "block_id": "p2_b1", + "page": 2, + "role": "body_paragraph", + "text": "Stable body paragraph with repeated typography and enough prose to form the body family anchor. " * 2, + "bbox": [100, 120, 460, 280], + "page_width": 1200, + "page_height": 1600, + "marker_signature": {"type": "none"}, + "span_signature": {"font_size_median": 9.0, "font_size_bucket": 9.0, "font_family_norm": "Times"}, + "layout_signature": {"width": 260, "width_bucket": 250, "x_center": 230, "x_center_bucket": 250}, + }, + { + "block_id": "p5_b1", + "page": 5, + "role": "reference_heading", + "text": "References", + "bbox": [100, 100, 340, 140], + "page_width": 1200, + "page_height": 1600, + "marker_signature": {"type": "canonical_section_name"}, + "span_signature": {"font_size_median": 10.0, "font_family_norm": "Times"}, + "layout_signature": {"width": 120, "x_center": 220}, + }, + { + "block_id": "p5_b2", + "page": 5, + "role": "body_paragraph", + "text": "[1] Example reference with authors and journal details.", + "bbox": [110, 180, 500, 260], + "page_width": 1200, + "page_height": 1600, + "marker_signature": {"type": "reference_numeric_bracket", "number": 1}, + "span_signature": {"font_size_median": 8.5, "font_size_bucket": 8.5, "font_family_norm": "Times"}, + "layout_signature": {"width": 250, "width_bucket": 250, "x_center": 240, "x_center_bucket": 250}, + }, + ] + + doc, normalized_blocks = normalize_document_structure(blocks) + + assert doc.reference_zones + ref_row = next(block for block in normalized_blocks if block["block_id"] == "p5_b2") + assert ref_row.get("zone") == "reference_zone" + assert ref_row.get("style_family") == "reference_like" + + def test_candidate_resolution_demotes_body_spine_narrative_figure_mentions() -> None: from paperforge.worker.ocr_document import DocumentStructure, PageLayoutProfile, _resolve_ambiguous_candidates diff --git a/tests/test_ocr_figures.py b/tests/test_ocr_figures.py index f10c54c5..b4299baf 100644 --- a/tests/test_ocr_figures.py +++ b/tests/test_ocr_figures.py @@ -228,6 +228,56 @@ def test_formal_figure_count_is_based_on_legends_not_raw_images() -> None: assert len(inventory["figure_legends"]) == 1 +def test_validation_first_legend_does_not_promote_body_figure_mention() -> None: + from paperforge.worker.ocr_figures import build_figure_inventory + + structured_blocks = [ + { + "paper_id": "KEY010B", + "page": 3, + "block_id": "p3_b1", + "role": "body_paragraph", + "raw_role": "body_paragraph", + "raw_label": "text", + "text": "Figure 2 shows the comparative response over time and this sentence continues as narrative body prose for the main discussion section.", + "bbox": [90, 100, 530, 180], + "zone": "body_zone", + "style_family": "body_like", + "style_family_authority": "body_family_anchor", + "body_spine_member": True, + "marker_signature": {"type": "figure_number", "number": 2}, + }, + { + "paper_id": "KEY010B", + "page": 3, + "block_id": "p3_b2", + "role": "figure_asset", + "raw_label": "image", + "text": "", + "bbox": [620, 140, 1030, 520], + }, + { + "paper_id": "KEY010B", + "page": 3, + "block_id": "p3_b3", + "role": "body_paragraph", + "raw_label": "text", + "text": "Figure 2. Formal caption with sufficient descriptive text to support validation-first legend matching near the media asset.", + "bbox": [620, 540, 1040, 620], + "zone": "display_zone", + "style_family": "legend_like", + "style_family_authority": "figure_family_anchor", + "marker_signature": {"type": "figure_number", "number": 2}, + }, + ] + + inventory = build_figure_inventory(structured_blocks) + + assert all(legend.get("legend_block_id", legend.get("block_id")) != "p3_b1" for legend in inventory["figure_legends"]) + assert any(legend.get("legend_block_id", legend.get("block_id")) == "p3_b3" for legend in inventory["figure_legends"]) + assert inventory["official_figure_count"] == 1 + + def test_figure_inventory_includes_all_sections() -> None: from paperforge.worker.ocr_figures import build_figure_inventory diff --git a/tests/test_ocr_health.py b/tests/test_ocr_health.py index f70acd6c..a6262fee 100644 --- a/tests/test_ocr_health.py +++ b/tests/test_ocr_health.py @@ -154,6 +154,37 @@ def test_ocr_health_includes_decision_counts() -> None: assert report["role_rescue_count"] == 1 +def test_ocr_health_reports_anchor_first_authority_summaries() -> None: + from paperforge.worker.ocr_document import DocumentStructure + from paperforge.worker.ocr_health import build_ocr_health + + doc = DocumentStructure( + body_family_anchor={"status": "ACCEPT"}, + reference_family_anchor={"status": "ACCEPT"}, + region_bus={ + "body_zone": {"status": "ACCEPT", "block_ids": ["p2_b1"]}, + "reference_zone": {"status": "ACCEPT", "block_ids": ["p5_b2"]}, + }, + ) + blocks = [ + {"block_id": "p2_b1", "role": "body_paragraph", "text": "Body", "page": 2, "bbox": [0, 0, 10, 10]}, + {"block_id": "p5_b2", "role": "reference_item", "text": "[1] Ref", "page": 5, "bbox": [0, 0, 10, 10]}, + ] + + report = build_ocr_health( + page_count=5, + raw_blocks_count=2, + structured_blocks=blocks, + figure_inventory={}, + table_inventory={}, + doc_structure=doc, + ) + + assert report["anchor_summary"]["body_family_anchor"] == "ACCEPT" + assert report["anchor_summary"]["reference_family_anchor"] == "ACCEPT" + assert report["zone_summary"]["reference_zone"] == "ACCEPT" + + def test_ocr_health_includes_tail_boundary_confidence() -> None: from paperforge.worker.ocr_health import build_ocr_health diff --git a/tests/test_ocr_real_paper_regressions.py b/tests/test_ocr_real_paper_regressions.py index da2e2975..b7fbd700 100644 --- a/tests/test_ocr_real_paper_regressions.py +++ b/tests/test_ocr_real_paper_regressions.py @@ -185,3 +185,19 @@ def test_control_papers_keep_body_and_tail_stability(rebuilt_real_papers: dict, assert roles.count("body_paragraph") >= CONTROL_MIN_BODY[key] assert any(w in fulltext for w in ["References", "references", "REFERENCE", "Bibliography"]), "Tail reference marker not found" + + +@pytest.mark.parametrize("key", PROBLEM_KEYS) +def test_problem_papers_keep_reference_roles_and_exclude_legend_family_from_body( + rebuilt_real_papers: dict, + _ocr_root: Path, + key: str, +) -> None: + _require_artifacts(_ocr_root, key) + blocks = _read_jsonl(_structured_path(_ocr_root, key)) + + assert any(block.get("role") == "reference_item" for block in blocks) + assert not any( + block.get("role") == "body_paragraph" and block.get("style_family") in {"legend_like", "table_caption_like", "reference_like"} + for block in blocks + ) diff --git a/tests/test_ocr_roles.py b/tests/test_ocr_roles.py index 46077aa9..30fe8990 100644 --- a/tests/test_ocr_roles.py +++ b/tests/test_ocr_roles.py @@ -256,6 +256,92 @@ def test_body_with_noise_phrase_below_backmatter_heading() -> None: ) +def test_pipeline_keeps_reference_zone_and_legend_family_out_of_default_body() -> None: + from paperforge.worker.ocr_blocks import build_structured_blocks + + raw_blocks = [ + { + "paper_id": "KEY010B", + "page": 2, + "block_id": "p2_b1", + "raw_label": "text", + "raw_order": 0, + "bbox": [110, 120, 470, 280], + "text": "Main body paragraph with stable narrative prose and enough repeated content to establish the body family anchor for the document. " * 2, + "page_width": 1200, + "page_height": 1600, + "span_metadata": {"size": 9.0, "font": "Times", "flags": ""}, + }, + { + "paper_id": "KEY010B", + "page": 3, + "block_id": "p3_b1", + "raw_label": "text", + "raw_order": 0, + "bbox": [112, 118, 472, 282], + "text": "Another core body paragraph repeating the same typography and width so the anchor-first body family remains dominant across the middle pages. " * 2, + "page_width": 1200, + "page_height": 1600, + "span_metadata": {"size": 9.0, "font": "Times", "flags": ""}, + }, + { + "paper_id": "KEY010B", + "page": 5, + "block_id": "p5_b1", + "raw_label": "paragraph_title", + "raw_order": 0, + "bbox": [100, 80, 350, 120], + "text": "References", + "page_width": 1200, + "page_height": 1600, + "span_metadata": {"size": 11.0, "font": "Times", "flags": "bold"}, + }, + { + "paper_id": "KEY010B", + "page": 5, + "block_id": "p5_b2", + "raw_label": "text", + "raw_order": 1, + "bbox": [110, 170, 500, 260], + "text": "[1] Example reference entry with journal and year details.", + "page_width": 1200, + "page_height": 1600, + "span_metadata": {"size": 8.5, "font": "Times", "flags": ""}, + }, + { + "paper_id": "KEY010B", + "page": 4, + "block_id": "p4_b9", + "raw_label": "text", + "raw_order": 2, + "bbox": [720, 780, 1030, 860], + "text": "Figure 1. Compact legend text for the nearby display panel.", + "page_width": 1200, + "page_height": 1600, + "span_metadata": {"size": 8.0, "font": "Times", "flags": ""}, + }, + { + "paper_id": "KEY010B", + "page": 4, + "block_id": "p4_b10", + "raw_label": "figure", + "raw_order": 3, + "bbox": [700, 420, 1040, 760], + "text": "", + "page_width": 1200, + "page_height": 1600, + }, + ] + + rows, _ = build_structured_blocks(raw_blocks) + + assert any(row["role"] == "reference_item" for row in rows) + assert not any( + row["role"] == "body_paragraph" and row.get("style_family") == "legend_like" + for row in rows + ) + + def test_style_aware_unnumbered_heading_detection() -> None: """Unnumbered headings with distinct visual style are detected as headings. diff --git a/tests/test_ocr_tables.py b/tests/test_ocr_tables.py index 561eae28..f878ac66 100644 --- a/tests/test_ocr_tables.py +++ b/tests/test_ocr_tables.py @@ -62,6 +62,40 @@ def test_table_without_asset_is_tracked_as_unmatched_caption() -> None: assert len(inventory["unmatched_captions"]) == 1 +def test_validation_first_table_candidate_remains_stable_under_anchor_first_roles() -> None: + from paperforge.worker.ocr_tables import build_table_inventory + + structured_blocks = [ + { + "paper_id": "KEY010B", + "page": 6, + "block_id": "p6_a1", + "role": "table_asset", + "raw_label": "table", + "text": "parsed table image", + "bbox": [120, 140, 760, 520], + }, + { + "paper_id": "KEY010B", + "page": 6, + "block_id": "p6_c1", + "role": "body_paragraph", + "raw_label": "text", + "text": "Table 4. Quantitative outcomes for anchor-first validation.", + "bbox": [120, 540, 760, 600], + "zone": "display_zone", + "style_family": "table_caption_like", + "marker_signature": {"type": "table_number", "number": 4}, + }, + ] + + inventory = build_table_inventory(structured_blocks) + + assert inventory["official_table_count"] == 1 + assert inventory["tables"][0]["caption_block_id"] == "p6_c1" + assert inventory["tables"][0]["has_asset"] is True + + def test_continuation_table_matches_same_page_asset() -> None: from paperforge.worker.ocr_tables import build_table_inventory