diff --git a/paperforge/worker/ocr_document.py b/paperforge/worker/ocr_document.py index 87446288..3011ed91 100644 --- a/paperforge/worker/ocr_document.py +++ b/paperforge/worker/ocr_document.py @@ -590,9 +590,7 @@ def rescue_roles_with_document_context( and body_match_h["match_score"] > heading_match["match_score"] + 0.1 ): block["role"] = "body_paragraph" - block.setdefault("evidence", []).append( - "rescue_family: heading → body_paragraph" - ) + block.setdefault("evidence", []).append("rescue_family: heading → body_paragraph") heading_demoted = True if not heading_demoted: body_fam = role_profiles.get("body_paragraph", {}) @@ -953,11 +951,17 @@ def _detect_body_spine(blocks: list[dict]) -> dict[int, dict]: if p > page and per_page_spine[p] is not None: next_val = per_page_spine[p] break - filled[page] = prev_val or next_val or { - "median_width": 500, "median_x": 100, - "width_range": (350, 650), "fonts": set(), - "all_fonts": all_body_fonts, - } + filled[page] = ( + prev_val + or next_val + or { + "median_width": 500, + "median_x": 100, + "width_range": (350, 650), + "fonts": set(), + "all_fonts": all_body_fonts, + } + ) return filled @@ -1009,8 +1013,7 @@ def _detect_non_body_insert_clusters( median_width = spine.get("median_width", 500) is_narrow = block_width < 0.7 * median_width or ( - page_width > 0 and median_width < page_width * 0.4 - and block_width < page_width * 0.35 + page_width > 0 and median_width < page_width * 0.4 and block_width < page_width * 0.35 ) block_font = _first_font(block) @@ -1029,6 +1032,56 @@ def _detect_non_body_insert_clusters( return indices +def _mark_non_body_media(blocks: list[dict]) -> None: + insert_blocks_by_page: dict[int, list[dict]] = {} + for block in blocks: + if block.get("_non_body_insert"): + page = block.get("page", 1) + insert_blocks_by_page.setdefault(page, []).append(block) + + if not insert_blocks_by_page: + return + + for page, inserts in insert_blocks_by_page.items(): + bboxes = [] + for ins in inserts: + b = ins.get("bbox") or ins.get("block_bbox") + if b and len(b) >= 4: + bboxes.append(b) + if not bboxes: + continue + + cluster_min_x = min(b[0] for b in bboxes) + cluster_min_y = min(b[1] for b in bboxes) + cluster_max_x = max(b[2] for b in bboxes) + cluster_max_y = max(b[3] for b in bboxes) + + margin = 50 + cluster_min_x -= margin + cluster_min_y -= margin + cluster_max_x += margin + cluster_max_y += margin + + for block in blocks: + if block.get("page", 1) != page: + continue + if block.get("_non_body_media"): + continue + role = block.get("role", "") + if role not in ("figure_asset", "media_asset"): + continue + + b = block.get("bbox") or block.get("block_bbox") + if not b or len(b) < 4: + continue + + cx = (b[0] + b[2]) / 2 + cy = (b[1] + b[3]) / 2 + + if cluster_min_x <= cx <= cluster_max_x and cluster_min_y <= cy <= cluster_max_y: + block["_non_body_media"] = True + + def normalize_document_structure(blocks: list[dict]) -> tuple[DocumentStructure, list[dict]]: """Analyze document structure and normalize roles. @@ -1068,7 +1121,9 @@ def normalize_document_structure(blocks: list[dict]) -> tuple[DocumentStructure, body_spine = _detect_body_spine(blocks) pw = max((b.get("page_width", 0) or 0) for b in blocks) or 1200 insert_indices = _detect_non_body_insert_clusters( - blocks, body_spine, page_width=pw, + blocks, + body_spine, + page_width=pw, body_end_page=tail_spread.body_end_page if tail_spread else None, ) for idx in insert_indices: @@ -1076,4 +1131,6 @@ def normalize_document_structure(blocks: list[dict]) -> tuple[DocumentStructure, blocks[idx]["role"] = "non_body_insert" blocks[idx]["_non_body_insert"] = True + _mark_non_body_media(blocks) + return doc_structure, blocks diff --git a/paperforge/worker/ocr_figures.py b/paperforge/worker/ocr_figures.py index 56e89048..92caeb56 100644 --- a/paperforge/worker/ocr_figures.py +++ b/paperforge/worker/ocr_figures.py @@ -180,8 +180,11 @@ def _media_clusters(blocks: list[dict], page_width: float = 1200) -> list[list[d media = [ b for b in blocks - if b.get("role") == "figure_asset" - or (b.get("role") == "media_asset" and b.get("raw_label", "") in {"image", "chart", "figure"}) + if not b.get("_non_body_media") + and ( + b.get("role") == "figure_asset" + or (b.get("role") == "media_asset" and b.get("raw_label", "") in {"image", "chart", "figure"}) + ) ] media.sort(key=lambda b: (b.get("page", 0), b.get("bbox", [0, 0, 0, 0])[1], b.get("bbox", [0, 0, 0, 0])[0])) @@ -296,6 +299,8 @@ def build_figure_inventory(structured_blocks: list[dict], page_width: float = 12 for block in structured_blocks: role = block.get("role", "") + if block.get("_non_body_media") or role == "non_body_insert": + continue if role == "figure_caption": if _is_body_mention(block): continue diff --git a/tests/test_ocr_figures.py b/tests/test_ocr_figures.py index f1623a5b..6710e713 100644 --- a/tests/test_ocr_figures.py +++ b/tests/test_ocr_figures.py @@ -652,3 +652,90 @@ def test_formal_legend_precedence_over_panel_subcaption() -> None: assert len(inventory["rejected_legends"]) == 1 assert "c. Redistribution" in inventory["rejected_legends"][0].get("text", "") + + +# === non-body insert media exclusion === + +PAGE1_AUTHOR_BIO_FIXTURE = [ + { + "paper_id": "K001", + "page": 1, + "block_id": "p1_b1", + "role": "non_body_insert", + "_non_body_insert": True, + "text": "Short author biography", + "bbox": [50, 50, 230, 120], + }, + { + "paper_id": "K001", + "page": 1, + "block_id": "p1_b2", + "role": "non_body_insert", + "_non_body_insert": True, + "text": "John Smith, PhD, is a professor...", + "bbox": [50, 130, 230, 200], + }, + { + "paper_id": "K001", + "page": 1, + "block_id": "p1_b3", + "role": "media_asset", + "raw_label": "image", + "_non_body_media": True, + "text": "", + "bbox": [240, 50, 350, 180], + }, + { + "paper_id": "K001", + "page": 2, + "block_id": "p2_b1", + "role": "figure_caption", + "text": "Figure 1. Migration under DC field.", + "bbox": [50, 700, 550, 740], + }, + { + "paper_id": "K001", + "page": 2, + "block_id": "p2_b2", + "role": "figure_asset", + "text": "", + "bbox": [50, 50, 550, 680], + }, +] + + +def test_non_body_insert_media_not_in_figure_assets() -> None: + from paperforge.worker.ocr_figures import build_figure_inventory + + inventory = build_figure_inventory(PAGE1_AUTHOR_BIO_FIXTURE) + asset_ids = [a.get("block_id") for a in inventory["figure_assets"]] + assert "p1_b3" not in asset_ids, "Author bio image must be excluded from figure_assets" + + +def test_non_body_insert_text_blocks_not_in_assets() -> None: + from paperforge.worker.ocr_figures import build_figure_inventory + + inventory = build_figure_inventory(PAGE1_AUTHOR_BIO_FIXTURE) + asset_ids = [a.get("block_id") for a in inventory["figure_assets"]] + assert "p1_b1" not in asset_ids + assert "p1_b2" not in asset_ids + + +def test_non_body_insert_media_not_in_unmatched_assets() -> None: + from paperforge.worker.ocr_figures import build_figure_inventory + + inventory = build_figure_inventory(PAGE1_AUTHOR_BIO_FIXTURE) + unmatched_ids = [a.get("block_id") for a in inventory["unmatched_assets"]] + assert "p1_b3" not in unmatched_ids, ( + "Author bio image must not appear in unmatched_assets either" + ) + + +def test_author_bio_media_does_not_affect_normal_figure_matching() -> None: + from paperforge.worker.ocr_figures import build_figure_inventory + + inventory = build_figure_inventory(PAGE1_AUTHOR_BIO_FIXTURE) + assert len(inventory["matched_figures"]) == 1 + assert inventory["matched_figures"][0]["legend_block_id"] == "p2_b1" + assert len(inventory["matched_figures"][0]["matched_assets"]) == 1 + assert inventory["matched_figures"][0]["matched_assets"][0]["block_id"] == "p2_b2"