From 765cdf91f2b438c5e3252495e00a5f07ff77a5a7 Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Fri, 19 Jun 2026 23:34:01 +0800 Subject: [PATCH] fix: carry owned table notes through rebuild outputs --- paperforge/worker/ocr_objects.py | 5 +++++ paperforge/worker/ocr_render.py | 7 +++++++ paperforge/worker/ocr_tables.py | 17 ++++++++++++++++- tests/test_ocr_objects.py | 16 ++++++++++++++++ tests/test_ocr_rendering.py | 26 ++++++++++++++++++++++++++ tests/test_ocr_tables.py | 17 +++++++++++++++++ 6 files changed, 87 insertions(+), 1 deletion(-) diff --git a/paperforge/worker/ocr_objects.py b/paperforge/worker/ocr_objects.py index 92abc4ad..c3046b1f 100644 --- a/paperforge/worker/ocr_objects.py +++ b/paperforge/worker/ocr_objects.py @@ -57,6 +57,11 @@ def render_table_object_markdown(table: dict[str, Any]) -> str: if caption: parts.append("## Caption") parts.append(normalize_ocr_math_text(caption)) + note_texts = [normalize_ocr_math_text(t) for t in table.get("note_texts", []) if t] + if note_texts: + parts.append("") + parts.append("## Notes") + parts.extend(note_texts) if table.get("page"): parts.append("") parts.append(f"*Page {table['page']}*") diff --git a/paperforge/worker/ocr_render.py b/paperforge/worker/ocr_render.py index 31070d9d..9bd5f54a 100644 --- a/paperforge/worker/ocr_render.py +++ b/paperforge/worker/ocr_render.py @@ -832,6 +832,11 @@ def render_fulltext_markdown( reader_figures = (reader_payload or {}).get("reader_figures", []) consumed_caption_keys: set[tuple[int | None, int | str]] = set() consumed_caption_ids_unkeyed: set[int | str] = set() + consumed_table_block_ids: set[str | int] = set() + for table in table_inventory.get("tables", []): + for block_id in table.get("consumed_block_ids", []): + if block_id: + consumed_table_block_ids.add(block_id) for item in (reader_payload or {}).get("consumed_caption_block_ids", []): if isinstance(item, dict): page = item.get("page") @@ -1242,6 +1247,8 @@ def render_fulltext_markdown( block_key in consumed_caption_keys or block_id in consumed_caption_ids_unkeyed ): continue + if block_id is not None and block_id in consumed_table_block_ids: + continue if block_key in abstract_member_keys or block_id in abstract_member_ids_unkeyed: continue diff --git a/paperforge/worker/ocr_tables.py b/paperforge/worker/ocr_tables.py index 21fcd1ae..04d3313c 100644 --- a/paperforge/worker/ocr_tables.py +++ b/paperforge/worker/ocr_tables.py @@ -8,7 +8,7 @@ from paperforge.core.io import write_json from paperforge.worker.ocr_scores import score_table_match _TABLE_PREFIX_PATTERN = re.compile( - r"^(?:Table|Supplementary\s+Table|Extended\s+Data\s+Table)\s+(\d+(?:\.\d+)?)", + r"^(?:Table|Supplementary\s+Table|Extended\s+Data\s+Table)\s+(?:S\.?\s*)?(\d+(?:\.\d+)?)", flags=re.IGNORECASE, ) @@ -94,6 +94,10 @@ def build_table_inventory(structured_blocks: list[dict]) -> dict[str, Any]: height = (bbox[3] - bbox[1]) if len(bbox) >= 4 else 0 if width < 120 or height < 60: continue + if raw_label not in ("table", "table_image"): + aspect = width / max(height, 1) + if aspect < 1.5: + continue assets.append(block) used_asset_indices: set[int] = set() @@ -201,6 +205,7 @@ def build_table_inventory(structured_blocks: list[dict]) -> dict[str, Any]: }) note_block_ids: list[str] = [] + note_texts: list[str] = [] if matched_asset: asset_page = matched_asset.get("page", 0) asset_bbox = matched_asset.get("bbox", [0, 0, 0, 0]) @@ -231,6 +236,14 @@ def build_table_inventory(structured_blocks: list[dict]) -> dict[str, Any]: note_top = bbbox[1] if asset_bottom <= note_top <= asset_bottom + 80: note_block_ids.append(block.get("block_id", "")) + if btext: + note_texts.append(btext) + + consumed_block_ids = [caption.get("block_id", "")] + if matched_asset: + consumed_block_ids.append(matched_asset.get("block_id", "")) + consumed_block_ids.extend(note_block_ids) + consumed_block_ids = [bid for bid in consumed_block_ids if bid] tables.append({ "caption_block_id": caption.get("block_id", ""), @@ -245,6 +258,8 @@ def build_table_inventory(structured_blocks: list[dict]) -> dict[str, Any]: "has_asset": matched_asset is not None, "segments": segments, "note_block_ids": note_block_ids, + "note_texts": note_texts, + "consumed_block_ids": consumed_block_ids, "is_continuation": is_cont, "continuation_of": continuation_of, "match_status": match_status, diff --git a/tests/test_ocr_objects.py b/tests/test_ocr_objects.py index 50d16c8b..ff3059b3 100644 --- a/tests/test_ocr_objects.py +++ b/tests/test_ocr_objects.py @@ -336,6 +336,22 @@ def test_figure_legend_math_normalized() -> None: assert "Expression of" in md +def test_table_object_markdown_renders_owned_notes() -> None: + from paperforge.worker.ocr_objects import render_table_object_markdown + + md = render_table_object_markdown({ + "table_id": "table_001", + "page": 5, + "caption": "Table 1. Results.", + "image_relpath": "assets/tables/table_001.jpg", + "note_texts": ["* p < 0.05", "Data are mean \u00b1 SD."], + }) + + assert "## Notes" in md + assert "* p < 0.05" in md + assert "Data are mean \u00b1 SD." in md + + def test_table_caption_math_normalized() -> None: from paperforge.worker.ocr_objects import render_table_object_markdown md = render_table_object_markdown({ diff --git a/tests/test_ocr_rendering.py b/tests/test_ocr_rendering.py index ebfa8e79..66b0a1f6 100644 --- a/tests/test_ocr_rendering.py +++ b/tests/test_ocr_rendering.py @@ -2565,3 +2565,29 @@ def test_reader_visible_statuses_are_emitted_in_markdown_contract() -> None: }, ) assert "EXACT_MATCH" not in markdown + + +def test_fulltext_skips_table_note_blocks_consumed_by_inventory() -> None: + from paperforge.worker.ocr_render import render_fulltext_markdown + + structured_blocks = [ + {"page": 5, "block_id": "p5_c1", "role": "table_caption", "text": "Table 1. Main results", "bbox": [100, 420, 600, 460], "zone": "display_zone", "style_family": "table_caption_like"}, + {"page": 5, "block_id": "p5_n1", "role": "footnote", "text": "* p < 0.05", "bbox": [100, 405, 600, 425]}, + ] + table_inventory = { + "tables": [{ + "table_id": "table_001", "page": 5, "caption_block_id": "p5_c1", + "caption_text": "Table 1. Main results", "asset_block_id": "p5_a1", + "note_block_ids": ["p5_n1"], "note_texts": ["* p < 0.05"], + "consumed_block_ids": ["p5_a1", "p5_c1", "p5_n1"], + }] + } + + md = render_fulltext_markdown( + structured_blocks=structured_blocks, resolved_metadata={}, + figure_inventory={"matched_figures": []}, + table_inventory=table_inventory, page_count=5, + document_structure=None, reader_payload={"figures": []}, + ) + + assert "* p < 0.05" not in md diff --git a/tests/test_ocr_tables.py b/tests/test_ocr_tables.py index ebd44371..d2dd77a2 100644 --- a/tests/test_ocr_tables.py +++ b/tests/test_ocr_tables.py @@ -802,6 +802,23 @@ def test_table_without_matched_asset_has_empty_note_block_ids() -> None: assert inventory["tables"][0]["note_block_ids"] == [] +def test_table_inventory_emits_note_payload_and_consumed_block_ids() -> None: + from paperforge.worker.ocr_tables import build_table_inventory + + structured_blocks = [ + {"page": 5, "block_id": "p5_a1", "role": "table_asset", "text": "table data", "bbox": [100, 100, 600, 400]}, + {"page": 5, "block_id": "p5_c1", "role": "table_caption", "text": "Table 1. Main results", "bbox": [100, 420, 600, 460]}, + {"page": 5, "block_id": "p5_n1", "role": "footnote", "raw_label": "vision_footnote", "text": "* p < 0.05", "bbox": [100, 405, 600, 425]}, + ] + + inventory = build_table_inventory(structured_blocks) + table = inventory["tables"][0] + + assert table["note_block_ids"] == ["p5_n1"] + assert table["note_texts"] == ["* p < 0.05"] + assert set(table["consumed_block_ids"]) == {"p5_a1", "p5_c1", "p5_n1"} + + def test_matched_table_asset_role_is_written_back() -> None: from paperforge.worker.ocr_tables import build_table_inventory, write_back_table_roles