From aeb89bd626a4e3e5410bffa6794fb2048757640c Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Wed, 10 Jun 2026 13:21:05 +0800 Subject: [PATCH] fix: close OCR figure and legend object gaps Add figure legend completeness check ensuring every numbered formal legend lands in an explicit outcome bucket (matched/held/ambiguous/ unresolved_cluster/unmatched). No legend may disappear without inventory explanation. Changes: - Add compute_figure_legend_completeness() to ocr_figures.py - Integrate completeness into build_figure_inventory() return value - Add figure_legend_completeness fields to health report (ocr_health.py) - Gaps degrade overall health status and add degraded reasons - Add 9 new tests for completeness edge cases --- paperforge/worker/ocr_figures.py | 132 +++++++++++++++- paperforge/worker/ocr_health.py | 20 +++ tests/test_ocr_figures.py | 255 +++++++++++++++++++++++++++++++ tests/test_ocr_objects.py | 60 ++++++++ 4 files changed, 466 insertions(+), 1 deletion(-) diff --git a/paperforge/worker/ocr_figures.py b/paperforge/worker/ocr_figures.py index 133cc278..f70bc581 100644 --- a/paperforge/worker/ocr_figures.py +++ b/paperforge/worker/ocr_figures.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextlib import re from pathlib import Path from typing import Any @@ -724,7 +725,7 @@ def build_figure_inventory(structured_blocks: list[dict], page_width: float = 12 consumed = {bid for uc in unresolved_clusters for bid in uc["media_block_ids"]} unmatched_assets = [a for a in unmatched_assets if a.get("block_id", "") not in consumed] - return { + inventory = { "figure_legends": deduped_legends, "figure_assets": assets, "matched_figures": matched_figures, @@ -737,6 +738,135 @@ def build_figure_inventory(structured_blocks: list[dict], page_width: float = 12 "official_figure_count": len(matched_figures), } + inventory["figure_legend_completeness"] = compute_figure_legend_completeness( + structured_blocks, inventory, + ) + + return inventory + + +def compute_figure_legend_completeness( + structured_blocks: list[dict], + inventory: dict[str, Any], +) -> dict[str, Any]: + """Verify every numbered formal legend lands in an explicit outcome bucket. + + Outcomes for each numbered formal legend: + - matched (figure object emitted) + - held (insufficient evidence, waiting) + - ambiguous (no asset or close-tie, unresolved) + - unmatched (low caption score, not a formal legend) + - gap (no outcome assigned — legend was silently dropped) + + Note: unresolved_cluster is a separate inventory bucket for panel groups + without legends; it is not a legend-level outcome. + + Returns a dict with total, accounted_for, gap_count, and per-legend + details. A gap_count > 0 means a formal legend was silently dropped. + """ + figure_number_re = re.compile( + r"(?:Figure|Fig\.?|Supplementary\s+Figure|Supplementary\s+Fig\.?|" + r"Extended\s+Data\s+Figure|Extended\s+Data\s+Fig\.?)\s+" + r"(?:S)?(\d+(?:\.\d+)?)", + flags=re.IGNORECASE, + ) + + # Collect all legend block_ids that entered the pipeline + legend_ids_in_pipeline: set[str] = set() + for block in structured_blocks: + role = block.get("role", "") + if block.get("_non_body_media") or role == "non_body_insert": + continue + if _PANEL_LABEL_PATTERN.match(str(block.get("text", "")).strip()): + continue + is_vfc = _is_validation_first_legend_candidate(block) + if role in ("figure_caption", "figure_caption_candidate") or is_vfc: + bid = block.get("block_id", "") + if bid: + legend_ids_in_pipeline.add(bid) + + # Build lookup sets for each outcome + matched_ids: set[str] = set() + for mf in inventory.get("matched_figures", []): + bid = mf.get("legend_block_id", "") + if bid: + matched_ids.add(bid) + + held_ids: set[str] = set() + for hf in inventory.get("held_figures", []): + bid = hf.get("legend_block_id", "") + if bid: + held_ids.add(bid) + + ambiguous_ids: set[str] = set() + for af in inventory.get("ambiguous_figures", []): + bid = af.get("legend_block_id", "") + if bid: + ambiguous_ids.add(bid) + + unmatched_ids: set[str] = set() + for ul in inventory.get("unmatched_legends", []): + bid = ul.get("block_id", "") + if bid: + unmatched_ids.add(bid) + + # Check every numbered legend in the pipeline + details: list[dict[str, Any]] = [] + total = 0 + accounted_for = 0 + gap_count = 0 + + for block in structured_blocks: + role = block.get("role", "") + if block.get("_non_body_media") or role == "non_body_insert": + continue + if _PANEL_LABEL_PATTERN.match(str(block.get("text", "")).strip()): + continue + is_vfc = _is_validation_first_legend_candidate(block) + if role not in ("figure_caption", "figure_caption_candidate") and not is_vfc: + continue + + text = block.get("text", "") + m = figure_number_re.search(text) + if m is None: + continue + + total += 1 + bid = block.get("block_id", "") + fig_num = None + with contextlib.suppress(ValueError): + fig_num = int(float(m.group(1))) + + if bid in matched_ids: + status = "matched" + elif bid in held_ids: + status = "held" + elif bid in ambiguous_ids: + status = "ambiguous" + elif bid in unmatched_ids: + status = "unmatched" + else: + status = "gap" + + if status != "gap": + accounted_for += 1 + else: + gap_count += 1 + + details.append({ + "block_id": bid, + "figure_number": fig_num, + "status": status, + "page": block.get("page"), + }) + + return { + "total": total, + "accounted_for": accounted_for, + "gap_count": gap_count, + "details": details, + } + def write_figure_inventory(dst: Path, inventory: dict[str, Any]) -> None: write_json(dst, inventory) diff --git a/paperforge/worker/ocr_health.py b/paperforge/worker/ocr_health.py index 7a34bd06..f602ba6d 100644 --- a/paperforge/worker/ocr_health.py +++ b/paperforge/worker/ocr_health.py @@ -98,6 +98,12 @@ def build_ocr_health( frontmatter_quality = 1.0 if abstract_found and references_found else 0.5 + # Figure legend completeness: every numbered formal legend must have an outcome + completeness = figure_inventory.get("figure_legend_completeness", {}) + formal_legend_total = int(completeness.get("total", 0)) + formal_legend_accounted = int(completeness.get("accounted_for", 0)) + formal_legend_gaps = int(completeness.get("gap_count", 0)) + issues = 0 if caption_without_media > 0: issues += 1 @@ -111,6 +117,8 @@ def build_ocr_health( issues += 1 if section_heading_count < 2: issues += 1 + if formal_legend_gaps > 0: + issues += 1 if issues == 0 and frontmatter_quality >= 0.5: overall = "green" @@ -207,6 +215,14 @@ def build_ocr_health( "anchor_summary": anchor_summary, "zone_summary": zone_summary, "held_counts": held_counts, + "figure_legend_completeness_total": formal_legend_total, + "figure_legend_completeness_accounted": formal_legend_accounted, + "figure_legend_completeness_gap_count": formal_legend_gaps, + "figure_legend_completeness_ratio": ( + formal_legend_accounted / formal_legend_total + if formal_legend_total > 0 + else 1.0 + ), } report.update(decision_summary) @@ -258,6 +274,10 @@ def build_ocr_health( low_confidence_tables = [s for s in table_scores if s < 0.4] if low_confidence_tables: degraded_reasons.append(f"low table match confidence ({len(low_confidence_tables)} tables)") + if formal_legend_gaps > 0: + degraded_reasons.append( + f"figure legend completeness gap ({formal_legend_gaps} numbered legends unaccounted for)" + ) return report diff --git a/tests/test_ocr_figures.py b/tests/test_ocr_figures.py index b4299baf..816c8827 100644 --- a/tests/test_ocr_figures.py +++ b/tests/test_ocr_figures.py @@ -1485,3 +1485,258 @@ def test_as_shown_in_figure_mention_rejected(): ] inventory = build_figure_inventory(blocks) assert len(inventory["unmatched_legends"]) == 1 + + +# === figure legend completeness (Task 8) === + + +def test_completeness_present_in_empty_inventory() -> None: + from paperforge.worker.ocr_figures import build_figure_inventory + + inventory = build_figure_inventory([]) + assert "figure_legend_completeness" in inventory + c = inventory["figure_legend_completeness"] + assert c["total"] == 0 + assert c["accounted_for"] == 0 + assert c["gap_count"] == 0 + assert c["details"] == [] + + +def test_completeness_all_legends_accounted_matched() -> None: + """Every numbered formal legend is matched to an asset.""" + from paperforge.worker.ocr_figures import build_figure_inventory + + blocks = [ + { + "paper_id": "K001", "page": 1, "block_id": "p1_b1", + "role": "figure_caption", + "text": "Figure 1. Migration under DC field.", + "bbox": [50, 420, 550, 460], + }, + { + "paper_id": "K001", "page": 1, "block_id": "p1_b2", + "role": "figure_asset", "text": "", + "bbox": [50, 50, 550, 400], + }, + { + "paper_id": "K001", "page": 2, "block_id": "p2_b1", + "role": "figure_caption", + "text": "Figure 2. Expression levels.", + "bbox": [50, 420, 550, 460], + }, + { + "paper_id": "K001", "page": 2, "block_id": "p2_b2", + "role": "figure_asset", "text": "", + "bbox": [50, 50, 550, 400], + }, + ] + + inventory = build_figure_inventory(blocks) + c = inventory["figure_legend_completeness"] + assert c["total"] == 2 + assert c["accounted_for"] == 2 + assert c["gap_count"] == 0 + for d in c["details"]: + assert d["status"] == "matched" + + +def test_completeness_legend_only_no_asset_is_ambiguous() -> None: + """A numbered legend with no matching asset lands in ambiguous, not gap.""" + from paperforge.worker.ocr_figures import build_figure_inventory + + blocks = [ + { + "paper_id": "K001", "page": 3, "block_id": "p3_b1", + "role": "figure_caption", + "text": "Figure 5. Caption with no asset on this page.", + "bbox": [50, 700, 550, 750], + }, + ] + + inventory = build_figure_inventory(blocks) + c = inventory["figure_legend_completeness"] + assert c["total"] == 1 + assert c["accounted_for"] == 1 + assert c["gap_count"] == 0 + assert c["details"][0]["status"] == "ambiguous" + + +def test_completeness_held_legend_is_accounted() -> None: + """A held (truncated) legend is counted as held, not gap.""" + from paperforge.worker.ocr_figures import build_figure_inventory + + blocks = [ + { + "paper_id": "K001", "page": 10, "block_id": "p10_b1", + "zone": "body_zone", "style_family": "legend_like", + "text": "Figure 1", + "marker_signature": {"type": "figure_number", "number": 1}, + "bbox": [50, 50, 300, 90], + "page_width": 1200, "page_height": 1600, + }, + { + "paper_id": "K001", "page": 10, "block_id": "p10_b2", + "zone": "body_zone", "style_family": "body_like", + "text": "Narrative prose", + "marker_signature": {"type": "none"}, + "bbox": [50, 100, 900, 140], + "page_width": 1200, "page_height": 1600, + }, + ] + + inventory = build_figure_inventory(blocks) + c = inventory["figure_legend_completeness"] + assert c["total"] == 1 + assert c["accounted_for"] == 1 + assert c["gap_count"] == 0 + assert c["details"][0]["status"] == "held" + + +def test_completeness_low_score_legend_is_unmatched_not_gap() -> None: + """A legend with low caption score goes to rejected_legends, not gap. + Note: 'Total cells' lacks a figure number so the completeness check + does not count it as a numbered formal legend -- correct behavior.""" + from paperforge.worker.ocr_figures import build_figure_inventory + + blocks = [ + { + "paper_id": "K001", "page": 1, "block_id": "p1_b1", + "role": "figure_caption", + "text": "Total cells", + "bbox": [50, 700, 200, 720], + }, + ] + + inventory = build_figure_inventory(blocks) + c = inventory["figure_legend_completeness"] + # "Total cells" has no figure number, so completeness check skips it + assert c["total"] == 0 + assert c["gap_count"] == 0 + # It IS in rejected_legends (pipeline rejects it as not formal) + assert len(inventory["rejected_legends"]) == 1 + + +def test_completeness_rejected_legend_not_in_count() -> None: + """Rejected legends (axis labels etc.) are not formal numbered legends.""" + from paperforge.worker.ocr_figures import build_figure_inventory + + blocks = [ + { + "paper_id": "K001", "page": 9, "block_id": "p9_b8", + "role": "figure_caption", "raw_label": "figure_title", + "text": "Days post culture in osteogenic differentiation supplemented medium", + "bbox": [374, 1046, 1143, 1077], + }, + ] + + inventory = build_figure_inventory(blocks) + c = inventory["figure_legend_completeness"] + assert c["total"] == 0 + assert c["gap_count"] == 0 + + +def test_completeness_mixed_outcomes_all_accounted() -> None: + """Multiple numbered legends with different outcomes are all accounted for.""" + from paperforge.worker.ocr_figures import build_figure_inventory + + blocks = [ + # Figure 1: matched + { + "paper_id": "K001", "page": 1, "block_id": "p1_b1", + "role": "figure_caption", + "text": "Figure 1. Migration under DC field.", + "bbox": [50, 420, 550, 460], + }, + { + "paper_id": "K001", "page": 1, "block_id": "p1_b2", + "role": "figure_asset", "text": "", + "bbox": [50, 50, 550, 400], + }, + # Figure 2: no asset on same page -> ambiguous + { + "paper_id": "K001", "page": 2, "block_id": "p2_b1", + "role": "figure_caption", + "text": "Figure 2. Expression levels without asset.", + "bbox": [50, 700, 550, 750], + }, + # Non-numbered caption (axis label) -> not counted by completeness + { + "paper_id": "K001", "page": 3, "block_id": "p3_b1", + "role": "figure_caption", + "text": "Days post culture", + "bbox": [50, 700, 200, 720], + }, + ] + + inventory = build_figure_inventory(blocks) + c = inventory["figure_legend_completeness"] + # Only Figure 1 and Figure 2 have figure numbers + assert c["total"] == 2 + assert c["accounted_for"] == 2 + assert c["gap_count"] == 0 + + statuses = {d["block_id"]: d["status"] for d in c["details"]} + assert statuses["p1_b1"] == "matched" + assert statuses["p2_b1"] == "ambiguous" + + +def test_compute_figure_legend_completeness_directly() -> None: + """Test the completeness function independently with a synthetic gap.""" + from paperforge.worker.ocr_figures import compute_figure_legend_completeness + + structured_blocks = [ + { + "paper_id": "K001", "page": 1, "block_id": "leg_A", + "role": "figure_caption", + "text": "Figure 1. Test.", + "bbox": [50, 420, 550, 460], + }, + { + "paper_id": "K001", "page": 2, "block_id": "leg_B", + "role": "figure_caption", + "text": "Figure 2. Test.", + "bbox": [50, 420, 550, 460], + }, + ] + + # Inventory where leg_A is matched but leg_B is missing from all buckets + inventory = { + "matched_figures": [{"legend_block_id": "leg_A"}], + "held_figures": [], + "ambiguous_figures": [], + "unmatched_legends": [], + } + + result = compute_figure_legend_completeness(structured_blocks, inventory) + assert result["total"] == 2 + assert result["accounted_for"] == 1 + assert result["gap_count"] == 1 + statuses = {d["block_id"]: d["status"] for d in result["details"]} + assert statuses["leg_A"] == "matched" + assert statuses["leg_B"] == "gap" + + +def test_compute_figure_legend_completeness_skips_body_mentions() -> None: + """Body-paragraph figure mentions are not counted as formal legends.""" + from paperforge.worker.ocr_figures import compute_figure_legend_completeness + + structured_blocks = [ + { + "paper_id": "K001", "page": 1, "block_id": "body1", + "role": "body_paragraph", "raw_role": "body_paragraph", + "text": "Figure 2 shows the results.", + "bbox": [50, 100, 550, 140], + "marker_signature": {"type": "figure_number", "number": 2}, + }, + ] + + inventory = { + "matched_figures": [], + "held_figures": [], + "ambiguous_figures": [], + "unmatched_legends": [], + } + + result = compute_figure_legend_completeness(structured_blocks, inventory) + assert result["total"] == 0 + assert result["gap_count"] == 0 diff --git a/tests/test_ocr_objects.py b/tests/test_ocr_objects.py index aef71f23..50d16c8b 100644 --- a/tests/test_ocr_objects.py +++ b/tests/test_ocr_objects.py @@ -347,3 +347,63 @@ def test_table_caption_math_normalized() -> None: }) assert "$_{50}$" in md assert "$\\\\mu$M" in md + + +# === figure legend completeness integration (Task 8) === + + +def test_figure_inventory_completeness_fields_present() -> None: + """Completeness metadata is present even for empty inventory.""" + from paperforge.worker.ocr_figures import build_figure_inventory + + inventory = build_figure_inventory([]) + c = inventory["figure_legend_completeness"] + assert "total" in c + assert "accounted_for" in c + assert "gap_count" in c + assert "details" in c + + +def test_extract_and_write_objects_with_held_figures_and_completeness(tmp_path: Path) -> None: + """Completeness data coexists with held figures in object extraction.""" + from paperforge.worker.ocr_objects import extract_and_write_objects + + render_root = tmp_path / "render" + asset_root = tmp_path / "assets" + + figure_inventory: dict[str, Any] = { + "matched_figures": [], + "held_figures": [ + { + "figure_id": "held_figure_001", + "legend_block_id": "p10_b1", + "page": 10, + "text": "Figure 1", + "figure_number": 1, + "hold_reason": "insufficient_legend_evidence", + } + ], + "unmatched_assets": [], + "rejected_legends": [], + "figure_legends": [], + "figure_assets": [], + "official_figure_count": 0, + "unresolved_clusters": [], + "figure_legend_completeness": { + "total": 1, + "accounted_for": 1, + "gap_count": 0, + "details": [{"block_id": "p10_b1", "figure_number": 1, "status": "held", "page": 10}], + }, + } + + extract_and_write_objects( + pdf_path=None, + figure_inventory=figure_inventory, + table_inventory={"tables": [], "unmatched_assets": []}, + asset_root=asset_root, + render_root=render_root, + ) + + render_files = sorted((render_root / "figures").glob("*.md")) + assert render_files == []