From 13df7189bb8afbb639873ba4a7952754d524a0bf Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Tue, 9 Jun 2026 22:29:43 +0800 Subject: [PATCH] fix: preserve OCR reference and table boundaries --- paperforge/worker/ocr_blocks.py | 32 +++++++--------- paperforge/worker/ocr_tables.py | 38 ++++++++++++++++++- tests/test_ocr_document.py | 66 +++++++++++++++++++++++++++++++++ tests/test_ocr_tables.py | 34 +++++++++++++++++ 4 files changed, 150 insertions(+), 20 deletions(-) diff --git a/paperforge/worker/ocr_blocks.py b/paperforge/worker/ocr_blocks.py index 250e14e6..d95d1b57 100644 --- a/paperforge/worker/ocr_blocks.py +++ b/paperforge/worker/ocr_blocks.py @@ -180,7 +180,7 @@ def build_structured_blocks( row.setdefault("evidence", []).extend(resolved.evidence) if doc_structure: - from paperforge.worker.ocr_document import _apply_zone_labels, infer_zones + from paperforge.worker.ocr_document import _apply_zone_labels, _detect_reference_zones, infer_zones from paperforge.worker.ocr_families import partition_zone_families refreshed_anchors = { @@ -191,24 +191,18 @@ def build_structured_blocks( _apply_zone_labels(rows, refreshed_region_bus) partition_zone_families(rows, refreshed_anchors) doc_structure.region_bus = refreshed_region_bus - - refreshed_reference_rows = [row for row in rows if row.get("zone") == "reference_zone"] - if refreshed_reference_rows: - reference_pages = sorted({int(row.get("page", 0) or 0) for row in refreshed_reference_rows if int(row.get("page", 0) or 0) > 0}) - if reference_pages: - doc_structure.reference_zones = [ - { - "page": page, - "column_index": 0, - "y_start": 0, - "y_end": float(next((row.get("page_height", 0) or 0) for row in refreshed_reference_rows if int(row.get("page", 0) or 0) == page) or 0), - "block_indices": [ - idx for idx, row in enumerate(rows) - if row.get("zone") == "reference_zone" and int(row.get("page", 0) or 0) == page - ], - } - for page in reference_pages - ] + page_layouts = getattr(doc_structure, "page_layouts", None) or {} + refreshed_reference_zones = _detect_reference_zones(rows, page_layouts) + doc_structure.reference_zones = [ + { + "page": zone.page, + "column_index": zone.column_index, + "y_start": zone.y_start, + "y_end": zone.y_end, + "block_indices": list(zone.block_indices), + } + for zone in refreshed_reference_zones + ] or None # Sync render_default/index_default after role normalizations for row in rows: diff --git a/paperforge/worker/ocr_tables.py b/paperforge/worker/ocr_tables.py index 46787349..e113cab6 100644 --- a/paperforge/worker/ocr_tables.py +++ b/paperforge/worker/ocr_tables.py @@ -53,6 +53,13 @@ def _is_insufficient_table_caption_evidence(block: dict) -> bool: return bool(_TRUNCATED_TABLE_ONLY_PATTERN.fullmatch(text)) +def _is_weak_explicit_table_caption(block: dict) -> bool: + role = str(block.get("role", "") or "") + if role not in {"table_caption", "table_caption_candidate"}: + return False + return _is_insufficient_table_caption_evidence(block) + + def _score_candidate_assets( page_assets: list[tuple[int, dict]], caption: dict, @@ -94,6 +101,7 @@ def build_table_inventory(structured_blocks: list[dict]) -> dict[str, Any]: is_cont = _is_continuation_caption(caption_text) is_validation_first_candidate = _is_validation_first_table_candidate(caption) is_weak_truncated = _is_insufficient_table_caption_evidence(caption) + is_weak_explicit_caption = _is_weak_explicit_table_caption(caption) if is_validation_first_candidate and is_weak_truncated: held_tables.append({ @@ -109,6 +117,32 @@ def build_table_inventory(structured_blocks: list[dict]) -> dict[str, Any]: "marker_signature": caption.get("marker_signature", {}), }) continue + if is_weak_explicit_caption: + unmatched_captions.append(caption) + tables.append({ + "caption_block_id": caption.get("block_id", ""), + "page": caption_page, + "caption_text": caption_text, + "table_number": table_num, + "formal_table_number": formal_table_number, + "asset_block_id": "", + "asset_bbox": [], + "assistive_text": "", + "truth_source": "image", + "has_asset": False, + "segments": [], + "is_continuation": is_cont, + "continuation_of": None, + "match_status": "ambiguous", + "candidate_assets": [], + "match_score": { + "score": 0.0, + "matched_asset_id": "", + "decision": "ambiguous", + "evidence": ["weak_explicit_caption"], + }, + }) + continue candidate_pages = [caption_page - 1, caption_page, caption_page + 1] @@ -190,7 +224,9 @@ def build_table_inventory(structured_blocks: list[dict]) -> dict[str, Any]: } for caption in captions: if caption.get("block_id", "") not in cap_block_ids_with_asset: - unmatched_captions.append(caption) + already_listed = any(c.get("block_id", "") == caption.get("block_id", "") for c in unmatched_captions) + if not already_listed: + unmatched_captions.append(caption) for i, asset in enumerate(assets): if i not in used_asset_indices: diff --git a/tests/test_ocr_document.py b/tests/test_ocr_document.py index d978d76f..44da39d9 100644 --- a/tests/test_ocr_document.py +++ b/tests/test_ocr_document.py @@ -355,6 +355,72 @@ def test_normalize_document_structure_preserves_reference_zone_integrity_from_an assert ref_row.get("style_family") == "reference_like" +def test_reference_zones_remain_column_scoped_after_authority_refresh() -> None: + from paperforge.worker.ocr_blocks import build_structured_blocks + + raw_blocks = [ + { + "paper_id": "KEYREF01", + "page": 1, + "block_id": "p1_b1", + "raw_label": "doc_title", + "raw_order": 0, + "bbox": [80, 40, 700, 90], + "text": "Reference Zone Test", + "page_width": 1200, + "page_height": 1600, + "span_metadata": [{"font": "Times-Bold", "size": 18.0, "flags": 16, "color": 0}], + }, + { + "paper_id": "KEYREF01", + "page": 4, + "block_id": "p4_b1", + "raw_label": "paragraph_title", + "raw_order": 1, + "bbox": [620, 120, 950, 160], + "text": "References", + "page_width": 1200, + "page_height": 1600, + "span_metadata": [{"font": "Times-Bold", "size": 10.0, "flags": 16, "color": 0}], + }, + { + "paper_id": "KEYREF01", + "page": 4, + "block_id": "p4_b2", + "raw_label": "text", + "raw_order": 2, + "bbox": [620, 180, 960, 250], + "text": "[1] Example reference entry with enough tokens to be reference-like.", + "page_width": 1200, + "page_height": 1600, + "span_metadata": [{"font": "Times-Roman", "size": 8.5, "flags": 0, "color": 0}] * 6, + }, + { + "paper_id": "KEYREF01", + "page": 4, + "block_id": "p4_b3", + "raw_label": "text", + "raw_order": 3, + "bbox": [80, 180, 420, 250], + "text": "Left-column body text that should not be part of the reference zone.", + "page_width": 1200, + "page_height": 1600, + "span_metadata": [{"font": "Times-Roman", "size": 9.0, "flags": 0, "color": 0}] * 6, + }, + ] + + rows, doc = build_structured_blocks(raw_blocks) + + assert doc is not None + assert doc.reference_zones + zone = doc.reference_zones[0] + assert zone["column_index"] == 1 + assert zone["y_start"] > 0 + assert zone["y_end"] < 1600 + left_body = next(row for row in rows if row["block_id"] == "p4_b3") + assert left_body.get("zone") != "reference_zone" + + def test_candidate_resolution_demotes_body_spine_narrative_figure_mentions() -> None: from paperforge.worker.ocr_document import DocumentStructure, PageLayoutProfile, _resolve_ambiguous_candidates diff --git a/tests/test_ocr_tables.py b/tests/test_ocr_tables.py index f878ac66..7c5dfbe2 100644 --- a/tests/test_ocr_tables.py +++ b/tests/test_ocr_tables.py @@ -388,6 +388,40 @@ def test_table_matching_can_hold_when_caption_and_asset_conflict() -> None: assert held["caption_block_id"] == "p12_b1" +def test_weak_explicit_table_caption_does_not_auto_match_from_geometry() -> None: + from paperforge.worker.ocr_tables import build_table_inventory + + structured_blocks = [ + { + "paper_id": "K001", + "page": 9, + "block_id": "p9_c1", + "role": "table_caption", + "text": "Table 2.", + "bbox": [100, 100, 700, 140], + }, + { + "paper_id": "K001", + "page": 9, + "block_id": "p9_a1", + "role": "table_asset", + "raw_label": "table", + "text": "", + "bbox": [100, 160, 700, 520], + }, + ] + + inv = build_table_inventory(structured_blocks) + + assert inv["held_tables"] == [] + assert len(inv["tables"]) == 1 + table = inv["tables"][0] + assert table["match_status"] == "ambiguous" + assert table["has_asset"] is False + assert table["match_score"]["decision"] == "ambiguous" + assert "weak_explicit_caption" in table["match_score"]["evidence"] + + def test_validation_first_table_candidate_with_asset_can_still_match() -> None: from paperforge.worker.ocr_tables import build_table_inventory