mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
fix: PR1 deterministic OCR residual fixes
- Clamp backfill words to original bbox (Issue 3) - Allow validation-first bare tables to fall through (Issue 1A) - Materialize split table caption continuations (Issue 1B) - Add short-form OCR health profile (Issue 4)
This commit is contained in:
LLLin000
parent
b32ddd3b29
commit
05e75e1b29
7 changed files with 382 additions and 51 deletions
|
|
@ -106,6 +106,10 @@ def _build_held_counts(doc_structure: Any, *, held_figure_count: int, held_table
|
|||
}
|
||||
|
||||
|
||||
def _health_profile(page_count: int) -> str:
|
||||
return "short_form" if int(page_count or 0) <= 2 else "standard"
|
||||
|
||||
|
||||
def build_ocr_health(
|
||||
*,
|
||||
page_count: int,
|
||||
|
|
@ -172,17 +176,20 @@ def build_ocr_health(
|
|||
formal_legend_accounted = int(completeness.get("accounted_for", 0))
|
||||
formal_legend_gaps = int(completeness.get("gap_count", 0))
|
||||
|
||||
# Only flag structural blockers as health-color issues.
|
||||
# Caption/media gaps, empty tables, legend gaps, layout anomalies
|
||||
# are kept in the report as data but do NOT drive the overall color —
|
||||
# they are display-quality concerns, not rebuild triggers.
|
||||
# Health profile determines which structural gates apply
|
||||
profile = _health_profile(page_count)
|
||||
waived_gates: list[str] = []
|
||||
structural_blockers = 0
|
||||
if not abstract_found:
|
||||
structural_blockers += 1
|
||||
if not references_found:
|
||||
structural_blockers += 1
|
||||
if section_heading_count < 2:
|
||||
structural_blockers += 1
|
||||
|
||||
if profile == "short_form":
|
||||
waived_gates.extend(["abstract_found", "section_heading_count"])
|
||||
else:
|
||||
if not abstract_found:
|
||||
structural_blockers += 1
|
||||
if not references_found:
|
||||
structural_blockers += 1
|
||||
if section_heading_count < 2:
|
||||
structural_blockers += 1
|
||||
|
||||
# Count soft issues for the report (informational only)
|
||||
soft_issues = 0
|
||||
|
|
@ -202,8 +209,9 @@ def build_ocr_health(
|
|||
else:
|
||||
overall = "red"
|
||||
|
||||
# needs_rebuild: only when structural core is broken
|
||||
needs_rebuild = structural_blockers >= 2 or (not abstract_found and not references_found)
|
||||
# needs_rebuild: only when structural core is broken.
|
||||
# Short-form papers are never red for missing structure alone.
|
||||
needs_rebuild = structural_blockers >= 2 or (not abstract_found and not references_found and profile != "short_form")
|
||||
|
||||
# Compute structural health signals
|
||||
from paperforge.worker.ocr_document import _compute_span_coverage, _detect_body_spine, _run_layout_audit
|
||||
|
|
@ -289,6 +297,8 @@ def build_ocr_health(
|
|||
"needs_rebuild": needs_rebuild,
|
||||
"structural_blockers": structural_blockers,
|
||||
"soft_issues": soft_issues,
|
||||
"health_profile": profile,
|
||||
"waived_gates": waived_gates,
|
||||
"span_coverage": span,
|
||||
"span_coverage_quality": span.get("coverage_quality", "weak"),
|
||||
"degraded_mode_active": span.get("degraded_mode_active", True),
|
||||
|
|
@ -355,18 +365,17 @@ def build_ocr_health(
|
|||
if role_gate_summary:
|
||||
report["role_gate_summary"] = role_gate_summary
|
||||
|
||||
degraded_reasons = []
|
||||
hard_degraded_reasons = []
|
||||
warning_reasons = []
|
||||
if span.get("coverage_quality", "weak") == "weak":
|
||||
degraded_reasons.append(f"weak span coverage ({span.get('coverage_ratio', 0):.0%})")
|
||||
warning_reasons.append(f"weak span coverage ({span.get('coverage_ratio', 0):.0%})")
|
||||
if spine.get("_meta", {}).get("quality", "weak") == "weak":
|
||||
degraded_reasons.append("weak body spine")
|
||||
warning_reasons.append("weak body spine")
|
||||
if layout.get("error_count", 0) > 0:
|
||||
degraded_reasons.append(f"layout audit errors ({layout.get('error_count', 0)} errors)")
|
||||
warning_reasons.append(f"layout audit errors ({layout.get('error_count', 0)} errors)")
|
||||
|
||||
if role_gate_summary.get("status") == "degraded":
|
||||
degraded_reasons.append("OCR structural role gate degraded")
|
||||
|
||||
report["degraded_reasons"] = degraded_reasons
|
||||
warning_reasons.append("OCR structural role gate degraded")
|
||||
|
||||
def _score_distribution(scores: list[float]) -> dict:
|
||||
return {
|
||||
|
|
@ -402,16 +411,26 @@ def build_ocr_health(
|
|||
|
||||
low_confidence_figures = [s for s in fig_scores if s < 0.4]
|
||||
if low_confidence_figures:
|
||||
degraded_reasons.append(f"low figure caption confidence ({len(low_confidence_figures)} figures)")
|
||||
warning_reasons.append(f"low figure caption confidence ({len(low_confidence_figures)} figures)")
|
||||
low_confidence_tables = [s for s in table_scores if s < 0.4]
|
||||
if low_confidence_tables:
|
||||
degraded_reasons.append(f"low table match confidence ({len(low_confidence_tables)} tables)")
|
||||
warning_reasons.append(f"low table match confidence ({len(low_confidence_tables)} tables)")
|
||||
if formal_legend_gaps > 0:
|
||||
degraded_reasons.append(
|
||||
warning_reasons.append(
|
||||
f"figure legend completeness gap ({formal_legend_gaps} numbered legends unaccounted for)"
|
||||
)
|
||||
if reader_payload is not None and reader_payload.get("reader_coverage", {}).get("gap_count", 0) > 0:
|
||||
degraded_reasons.append("reader_figure_coverage_gap")
|
||||
warning_reasons.append("reader_figure_coverage_gap")
|
||||
if rendered_gap["rendered_text_gap_count"] > 0:
|
||||
hard_degraded_reasons.append(
|
||||
f"rendered text gaps ({rendered_gap['rendered_text_gap_count']} segments missing from fulltext)"
|
||||
)
|
||||
|
||||
report["hard_degraded_reasons"] = hard_degraded_reasons
|
||||
report["warning_reasons"] = warning_reasons
|
||||
report["degraded_reasons"] = hard_degraded_reasons
|
||||
if profile == "short_form":
|
||||
report["degraded_reason"] = "short_paper_format"
|
||||
|
||||
return report
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,14 @@ from typing import Any
|
|||
|
||||
import fitz
|
||||
|
||||
import os as _os
|
||||
from contextlib import redirect_stderr as _redirect_stderr
|
||||
|
||||
|
||||
def _fitz_quiet_open(path: str):
|
||||
with _redirect_stderr(_os.devnull):
|
||||
return fitz.open(path)
|
||||
|
||||
|
||||
def _map_ocr_bbox_to_pdf_rect(
|
||||
bbox: list[float],
|
||||
|
|
@ -277,6 +285,21 @@ def _bbox_overlap_ratio(container_rect: Any, block_rect: Any) -> float:
|
|||
return (overlap.width * overlap.height) / area_b
|
||||
|
||||
|
||||
def _word_center_inside_rect(word_bbox, block_rect) -> bool:
|
||||
x0, y0, x1, y1 = word_bbox
|
||||
cx = (x0 + x1) / 2.0
|
||||
cy = (y0 + y1) / 2.0
|
||||
return block_rect.x0 <= cx <= block_rect.x1 and block_rect.y0 <= cy <= block_rect.y1
|
||||
|
||||
|
||||
def _word_belongs_to_block(word_bbox, block_rect) -> bool:
|
||||
word_rect = fitz.Rect(*word_bbox)
|
||||
return (
|
||||
_word_center_inside_rect(word_bbox, block_rect)
|
||||
or _bbox_overlap_ratio(word_rect, block_rect) >= 0.30
|
||||
)
|
||||
|
||||
|
||||
def _has_container_text(
|
||||
rect: Any,
|
||||
*,
|
||||
|
|
@ -401,15 +424,12 @@ def backfill_span_metadata_from_pdf(
|
|||
"""
|
||||
if not pdf_path:
|
||||
return raw_blocks
|
||||
|
||||
pdf_path = Path(pdf_path)
|
||||
if not pdf_path.exists():
|
||||
return raw_blocks
|
||||
|
||||
import fitz
|
||||
|
||||
try:
|
||||
doc = fitz.open(str(pdf_path))
|
||||
doc = _fitz_quiet_open(str(pdf_path))
|
||||
except Exception:
|
||||
return raw_blocks
|
||||
|
||||
|
|
@ -608,10 +628,8 @@ def backfill_missing_text_from_pdf(
|
|||
if not pdf_path.exists():
|
||||
return raw_blocks
|
||||
|
||||
import fitz
|
||||
|
||||
try:
|
||||
doc = fitz.open(str(pdf_path))
|
||||
doc = _fitz_quiet_open(str(pdf_path))
|
||||
except Exception:
|
||||
return raw_blocks
|
||||
|
||||
|
|
@ -696,6 +714,11 @@ def backfill_missing_text_from_pdf(
|
|||
except Exception:
|
||||
words = []
|
||||
|
||||
words = [
|
||||
w for w in words
|
||||
if len(w) >= 4 and _word_belongs_to_block(tuple(w[:4]), rect)
|
||||
]
|
||||
|
||||
text = _words_to_text(words)
|
||||
|
||||
if text:
|
||||
|
|
|
|||
|
|
@ -60,6 +60,54 @@ def _is_weak_explicit_table_caption(block: dict) -> bool:
|
|||
return _is_insufficient_table_caption_evidence(block)
|
||||
|
||||
|
||||
def _find_table_caption_continuation(caption: dict, structured_blocks: list[dict]) -> dict | None:
|
||||
"""Find the block immediately after a weak-truncated table caption that
|
||||
looks like a continuation (stolen as figure_caption or similar)."""
|
||||
caption_page = caption.get("page", 0)
|
||||
caption_bbox = caption.get("bbox") or [0, 0, 0, 0]
|
||||
next_idx = None
|
||||
for i, block in enumerate(structured_blocks):
|
||||
if block.get("block_id") == caption.get("block_id"):
|
||||
next_idx = i + 1
|
||||
break
|
||||
if next_idx is None or next_idx >= len(structured_blocks):
|
||||
return None
|
||||
|
||||
next_block = structured_blocks[next_idx]
|
||||
bbox = next_block.get("bbox") or [0, 0, 0, 0]
|
||||
|
||||
# Trigger rules from spec
|
||||
if bbox[1] - caption_bbox[3] > 25: # y-gap > 25px
|
||||
return None
|
||||
|
||||
next_text = str(next_block.get("text", "") or "")
|
||||
# reject if starts with Fig/Figure/Scheme/Plate
|
||||
if next_text.lower().startswith(("fig", "figure", "scheme", "plate")):
|
||||
return None
|
||||
|
||||
# x-overlap check
|
||||
x_overlap = max(0, min(caption_bbox[2], bbox[2]) - max(caption_bbox[0], bbox[0]))
|
||||
if x_overlap < min(caption_bbox[2] - caption_bbox[0], bbox[2] - bbox[0]) * 0.5:
|
||||
left_edge_delta = abs(caption_bbox[0] - bbox[0])
|
||||
if left_edge_delta >= 40:
|
||||
return None
|
||||
|
||||
return next_block
|
||||
|
||||
|
||||
def _materialize_table_caption(caption: dict, continuation: dict | None) -> tuple[dict, list[str]]:
|
||||
"""Merge continuation text into caption, return (merged_caption, consumed_ids)."""
|
||||
consumed_ids = [caption.get("block_id", "")]
|
||||
if continuation is None:
|
||||
return caption, consumed_ids
|
||||
|
||||
merged = dict(caption)
|
||||
cont_text = str(continuation.get("text", "") or "")
|
||||
merged["text"] = (merged.get("text", "") or "") + " " + cont_text
|
||||
consumed_ids.append(continuation.get("block_id", ""))
|
||||
return merged, consumed_ids
|
||||
|
||||
|
||||
def _has_strong_spatial_evidence_for_bare_table(caption: dict, top_candidate_score: dict | None) -> bool:
|
||||
if top_candidate_score is None:
|
||||
return False
|
||||
|
|
@ -198,25 +246,26 @@ def build_table_inventory(structured_blocks: list[dict]) -> dict[str, Any]:
|
|||
]
|
||||
if not same_page_assets:
|
||||
held_tables.append(
|
||||
{
|
||||
"table_id": f"held_table_{len(held_tables) + 1:03d}",
|
||||
"caption_block_id": caption.get("block_id", ""),
|
||||
"page": caption_page,
|
||||
"caption_text": caption_text,
|
||||
"table_number": table_num,
|
||||
"formal_table_number": formal_table_number,
|
||||
"hold_reason": "insufficient_caption_evidence",
|
||||
"zone": caption.get("zone", ""),
|
||||
"style_family": caption.get("style_family", ""),
|
||||
"marker_signature": caption.get("marker_signature", {}),
|
||||
}
|
||||
)
|
||||
continue
|
||||
{
|
||||
"table_id": f"held_table_{len(held_tables) + 1:03d}",
|
||||
"caption_block_id": caption.get("block_id", ""),
|
||||
"page": caption_page,
|
||||
"caption_text": caption_text,
|
||||
"table_number": table_num,
|
||||
"formal_table_number": formal_table_number,
|
||||
"hold_reason": "insufficient_caption_evidence",
|
||||
"zone": caption.get("zone", ""),
|
||||
"style_family": caption.get("style_family", ""),
|
||||
"marker_signature": caption.get("marker_signature", {}),
|
||||
}
|
||||
)
|
||||
continue
|
||||
# same-page asset exists → fall through into weak-explicit matching
|
||||
continuation_ids: list[str] = []
|
||||
if is_weak_truncated:
|
||||
# Truncated table label with same-page assets: proceed
|
||||
# to weak-explicit matching (same as figure's truncated
|
||||
# legend now matching to nearby assets).
|
||||
pass
|
||||
continuation = _find_table_caption_continuation(caption, structured_blocks)
|
||||
materialized_caption, continuation_ids = _materialize_table_caption(caption, continuation)
|
||||
caption_text = materialized_caption.get("text", "")
|
||||
all_candidates: list[tuple[int, dict, dict]] = []
|
||||
|
||||
if is_weak_explicit_caption:
|
||||
|
|
@ -431,6 +480,7 @@ def build_table_inventory(structured_blocks: list[dict]) -> dict[str, Any]:
|
|||
if matched_asset:
|
||||
consumed_block_ids.append(matched_asset.get("block_id", ""))
|
||||
consumed_block_ids.extend(note_block_ids)
|
||||
consumed_block_ids.extend(continuation_ids)
|
||||
consumed_block_ids = [bid for bid in consumed_block_ids if bid]
|
||||
bridge_block_ids = [
|
||||
str(block.get("block_id") or "")
|
||||
|
|
|
|||
|
|
@ -294,7 +294,7 @@ def test_ocr_health_includes_confidence_distributions() -> None:
|
|||
assert tbl_dist["low"] == 1
|
||||
|
||||
|
||||
def test_health_report_includes_degraded_reasons() -> None:
|
||||
def test_health_report_includes_warning_or_degraded_reasons() -> None:
|
||||
from paperforge.worker.ocr_health import build_ocr_health
|
||||
|
||||
report = build_ocr_health(
|
||||
|
|
@ -306,8 +306,56 @@ def test_health_report_includes_degraded_reasons() -> None:
|
|||
)
|
||||
|
||||
assert "degraded_reasons" in report
|
||||
assert len(report["degraded_reasons"]) > 0
|
||||
assert "warning_reasons" in report
|
||||
assert len(report["degraded_reasons"]) + len(report["warning_reasons"]) > 0
|
||||
|
||||
def test_health_report_separates_warning_reasons_from_hard_degradation() -> None:
|
||||
from paperforge.worker.ocr_health import build_ocr_health
|
||||
|
||||
report = build_ocr_health(
|
||||
page_count=1,
|
||||
raw_blocks_count=1,
|
||||
structured_blocks=[
|
||||
{"role": "section_heading", "text": "1 Intro", "span_metadata": {"size": 12}},
|
||||
{"role": "section_heading", "text": "2 Methods", "span_metadata": {"size": 12}},
|
||||
{"role": "abstract_body", "text": "Abstract text", "span_metadata": {"size": 10}},
|
||||
{"role": "reference_item", "text": "[1] Ref", "span_metadata": {"size": 10}},
|
||||
{"role": "body_paragraph", "text": "Body text", "span_metadata": {"size": 10}},
|
||||
],
|
||||
figure_inventory={
|
||||
"matched_figures": [{"caption_score": {"score": 0.2}, "matched_assets": [{}]}],
|
||||
"unmatched_legends": [],
|
||||
"unmatched_assets": [],
|
||||
"figure_legend_completeness": {"total": 1, "accounted": 1, "gap_count": 0, "ratio": 1.0},
|
||||
},
|
||||
table_inventory={"tables": [], "unmatched_captions": [], "unmatched_assets": []},
|
||||
)
|
||||
|
||||
assert any("low figure caption confidence" in r for r in report["warning_reasons"])
|
||||
assert all("low figure caption confidence" not in r for r in report["degraded_reasons"])
|
||||
|
||||
|
||||
def test_health_report_marks_rendered_text_gaps_as_hard_degradation() -> None:
|
||||
from paperforge.worker.ocr_health import build_ocr_health
|
||||
|
||||
report = build_ocr_health(
|
||||
page_count=1,
|
||||
raw_blocks_count=1,
|
||||
structured_blocks=[
|
||||
{
|
||||
"role": "body_paragraph",
|
||||
"text": "Short OCR body",
|
||||
"span_metadata": {"size": 10},
|
||||
"pdf_region_text": "Important missing body segment from PDF",
|
||||
}
|
||||
],
|
||||
figure_inventory={},
|
||||
table_inventory={},
|
||||
rendered_markdown="Short OCR body",
|
||||
)
|
||||
|
||||
assert any("rendered text gaps" in r for r in report["degraded_reasons"])
|
||||
assert report["rendered_text_gap_count"] > 0
|
||||
|
||||
def test_ocr_health_reports_layout_confidence_distribution() -> None:
|
||||
from paperforge.worker.ocr_document import DocumentStructure, PageLayoutProfile
|
||||
|
|
@ -623,7 +671,7 @@ def test_ocr_health_role_gate_degraded_forces_overall_red() -> None:
|
|||
"reference_item_outside_reference_zone_count": 0,
|
||||
}
|
||||
health = build_ocr_health(
|
||||
page_count=1,
|
||||
page_count=3,
|
||||
raw_blocks_count=0,
|
||||
structured_blocks=[],
|
||||
figure_inventory={},
|
||||
|
|
@ -728,3 +776,41 @@ def test_health_emits_additive_v2_fields_without_replacing_overall() -> None:
|
|||
assert "heading_total_v2" in health
|
||||
assert "matched_figure_count_v2" in health
|
||||
assert "issue_breakdown_v2" in health
|
||||
|
||||
def test_short_form_health_does_not_go_red_for_missing_abstract_headings_and_refs():
|
||||
from paperforge.worker.ocr_health import build_ocr_health
|
||||
|
||||
report = build_ocr_health(
|
||||
page_count=2,
|
||||
raw_blocks_count=2,
|
||||
structured_blocks=[
|
||||
{"page": 1, "role": "body_paragraph", "text": "Short letter text."},
|
||||
{"page": 2, "role": "body_paragraph", "text": "More short letter text."},
|
||||
],
|
||||
figure_inventory={"matched_figures": [], "held_figures": [], "unmatched_legends": [], "unmatched_assets": [], "figure_legend_completeness": {}},
|
||||
table_inventory={"tables": [], "held_tables": [], "unmatched_captions": [], "unmatched_assets": []},
|
||||
)
|
||||
|
||||
assert report["health_profile"] == "short_form"
|
||||
assert report["overall"] in {"green", "yellow"}
|
||||
assert report["needs_rebuild"] is False
|
||||
assert "abstract_found" in report["waived_gates"]
|
||||
assert report["degraded_reason"] == "short_paper_format"
|
||||
|
||||
|
||||
def test_standard_profile_still_flags_missing_structure():
|
||||
from paperforge.worker.ocr_health import build_ocr_health
|
||||
|
||||
report = build_ocr_health(
|
||||
page_count=5,
|
||||
raw_blocks_count=2,
|
||||
structured_blocks=[
|
||||
{"page": 1, "role": "body_paragraph", "text": "Body."},
|
||||
{"page": 5, "role": "body_paragraph", "text": "Tail."},
|
||||
],
|
||||
figure_inventory={"matched_figures": [], "held_figures": [], "unmatched_legends": [], "unmatched_assets": [], "figure_legend_completeness": {}},
|
||||
table_inventory={"tables": [], "held_tables": [], "unmatched_captions": [], "unmatched_assets": []},
|
||||
)
|
||||
|
||||
assert report["health_profile"] == "standard"
|
||||
assert report["overall"] == "red"
|
||||
|
|
|
|||
|
|
@ -134,3 +134,60 @@ def test_local_reference_pdf_fallback_repairs_only_local_candidate_text() -> Non
|
|||
repaired = repair_reference_entry_from_pdf_text(block_run, page_text)
|
||||
assert "N Engl J Med" in repaired
|
||||
assert repaired.startswith("Brown T, Green S.")
|
||||
|
||||
def test_backfill_expanded_clip_filters_words_to_original_bbox(monkeypatch, tmp_path):
|
||||
from paperforge.worker import ocr_pdf_spans
|
||||
import fitz
|
||||
|
||||
class FakePage:
|
||||
rect = fitz.Rect(0, 0, 1000, 1000)
|
||||
def get_text(self, kind, clip=None):
|
||||
if kind == "words":
|
||||
return [
|
||||
(100, 100, 120, 110, "inside", 0, 0, 0),
|
||||
(100, 111, 120, 121, "neighbor", 0, 0, 1),
|
||||
]
|
||||
if kind == "text":
|
||||
return "inside neighbor"
|
||||
return []
|
||||
|
||||
class FakeDoc:
|
||||
def __len__(self): return 1
|
||||
def __getitem__(self, idx): return FakePage()
|
||||
def close(self): pass
|
||||
|
||||
monkeypatch.setattr(ocr_pdf_spans, "_fitz_quiet_open", lambda path: FakeDoc())
|
||||
monkeypatch.setattr(
|
||||
ocr_pdf_spans,
|
||||
"_map_ocr_bbox_to_pdf_rect",
|
||||
lambda bbox, pw, ph, page: fitz.Rect(100, 100, 120, 110),
|
||||
)
|
||||
|
||||
pdf_path = tmp_path / "fake.pdf"
|
||||
pdf_path.write_bytes(b"%PDF-1.7\n")
|
||||
|
||||
blocks = [{
|
||||
"page": 1,
|
||||
"bbox": [100, 100, 120, 110],
|
||||
"page_width": 1000,
|
||||
"page_height": 1000,
|
||||
"raw_label": "text",
|
||||
"text": "",
|
||||
"block_content": "",
|
||||
"span_metadata": [{"font": "Body"}],
|
||||
}]
|
||||
|
||||
ocr_pdf_spans.backfill_missing_text_from_pdf(blocks, pdf_path)
|
||||
|
||||
assert blocks[0]["text"] == "inside"
|
||||
assert blocks[0]["_ocr_raw_status"] == "missing_text_recovered"
|
||||
|
||||
|
||||
def test_backfill_keeps_slightly_misaligned_words_by_center_or_overlap():
|
||||
from paperforge.worker.ocr_pdf_spans import _word_belongs_to_block
|
||||
import fitz
|
||||
|
||||
block_rect = fitz.Rect(100, 100, 120, 110)
|
||||
|
||||
assert _word_belongs_to_block((99, 100, 109, 110), block_rect) is True
|
||||
assert _word_belongs_to_block((121, 111, 131, 121), block_rect) is False
|
||||
|
|
|
|||
|
|
@ -504,3 +504,36 @@ def test_consumed_stringified_int_note_id_matches_block_int_id() -> None:
|
|||
)
|
||||
|
||||
assert "Table note with int block_id." not in md
|
||||
|
||||
|
||||
def test_materialized_table_caption_continuation_is_skipped_by_render_when_consumed():
|
||||
from paperforge.worker.ocr_render import render_fulltext_markdown
|
||||
|
||||
blocks = [
|
||||
{"page": 1, "block_id": "cap1", "role": "table_caption", "text": "Table 2", "bbox": [100, 100, 220, 120]},
|
||||
{"page": 1, "block_id": "cap2", "role": "figure_caption", "text": "Structural parameters of nanocomposites obtained from the d", "bbox": [100, 121, 500, 145]},
|
||||
]
|
||||
table_inventory = {
|
||||
"tables": [
|
||||
{
|
||||
"page": 1,
|
||||
"caption_block_id": "cap1",
|
||||
"caption_text": "Table 2 Structural parameters of nanocomposites obtained from the d",
|
||||
"consumed_block_ids": ["cap1", "cap2"],
|
||||
"has_asset": False,
|
||||
"match_status": "unmatched_caption",
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
md = render_fulltext_markdown(
|
||||
structured_blocks=blocks,
|
||||
resolved_metadata={},
|
||||
figure_inventory={"matched_figures": []},
|
||||
table_inventory=table_inventory,
|
||||
page_count=1,
|
||||
document_structure=None,
|
||||
reader_payload={"reader_figures": [], "consumed_caption_block_ids": []},
|
||||
)
|
||||
|
||||
assert "Structural parameters of nanocomposites" not in md
|
||||
|
|
|
|||
|
|
@ -1230,3 +1230,66 @@ def test_weak_match_caption_has_empty_consumed_block_ids() -> None:
|
|||
assert table["has_asset"] is False
|
||||
assert table["match_status"] in {"unmatched_caption", "ambiguous"}
|
||||
assert "p5_caption" not in table.get("consumed_block_ids", [])
|
||||
|
||||
|
||||
def test_validation_first_bare_table_with_same_page_asset_falls_through_to_weak_explicit_matching():
|
||||
from paperforge.worker.ocr_tables import build_table_inventory
|
||||
|
||||
structured_blocks = [
|
||||
{
|
||||
"block_id": "cap",
|
||||
"page": 1,
|
||||
"role": "body_paragraph",
|
||||
"raw_label": "text",
|
||||
"text": "Table 1",
|
||||
"bbox": [100, 100, 300, 120],
|
||||
"marker_signature": {"type": "table_number"},
|
||||
"zone": "display_zone",
|
||||
"style_family": "table_caption_like",
|
||||
},
|
||||
{
|
||||
"block_id": "asset",
|
||||
"page": 1,
|
||||
"role": "media_asset",
|
||||
"raw_label": "table_image",
|
||||
"text": "",
|
||||
"bbox": [100, 130, 500, 400],
|
||||
},
|
||||
]
|
||||
|
||||
inventory = build_table_inventory(structured_blocks)
|
||||
|
||||
assert inventory["official_table_count"] == 1
|
||||
assert inventory["tables"][0]["asset_block_id"] == "asset"
|
||||
assert inventory["tables"][0]["caption_block_id"] == "cap"
|
||||
|
||||
|
||||
def test_split_table_caption_materializes_continuation_stolen_as_figure_caption():
|
||||
from paperforge.worker.ocr_tables import build_table_inventory
|
||||
|
||||
structured_blocks = [
|
||||
{"block_id": "cap1", "page": 1, "role": "table_caption", "text": "Table 2", "bbox": [100, 100, 220, 120]},
|
||||
{"block_id": "cap2", "page": 1, "role": "figure_caption", "text": "Structural parameters of nanocomposites obtained from the d", "bbox": [100, 121, 500, 145]},
|
||||
{"block_id": "asset", "page": 1, "role": "media_asset", "raw_label": "table_image", "text": "", "bbox": [100, 150, 500, 400]},
|
||||
]
|
||||
|
||||
inventory = build_table_inventory(structured_blocks)
|
||||
|
||||
assert inventory["official_table_count"] == 1
|
||||
assert inventory["tables"][0]["caption_text"].startswith("Table 2 Structural parameters")
|
||||
assert "cap2" in inventory["tables"][0]["consumed_block_ids"]
|
||||
|
||||
|
||||
def test_split_table_caption_does_not_steal_real_figure_caption():
|
||||
from paperforge.worker.ocr_tables import build_table_inventory
|
||||
|
||||
structured_blocks = [
|
||||
{"block_id": "cap1", "page": 1, "role": "table_caption", "text": "Table 2", "bbox": [100, 100, 220, 120]},
|
||||
{"block_id": "figcap", "page": 1, "role": "figure_caption", "text": "Figure 3. Histology results.", "bbox": [600, 100, 900, 125]},
|
||||
{"block_id": "asset", "page": 1, "role": "media_asset", "raw_label": "table_image", "text": "", "bbox": [100, 150, 500, 400]},
|
||||
]
|
||||
|
||||
inventory = build_table_inventory(structured_blocks)
|
||||
|
||||
assert inventory["tables"][0]["caption_text"] == "Table 2"
|
||||
assert "figcap" not in inventory["tables"][0]["consumed_block_ids"]
|
||||
|
|
|
|||
Loading…
Reference in a new issue