mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
fix: reference-zone ordering fixes (P0-P4)
P0: disambiguate table field keys from block_page_by_id in ocr_render.py P1: add _is_tail_backmatter_continuation() guard, order backmatter correctly P2: phase 4b tail nonref guard with tail_backmatter_blocks list P3: page-continuation-marker false reference gate in ocr_families.py P4: force reference heading before same-page refs, ref_item bypass usable gate
This commit is contained in:
LLLin000
parent
4afe32b6d5
commit
78b9973123
4 changed files with 465 additions and 21 deletions
|
|
@ -40,8 +40,26 @@ _EDITORIAL_TAIL_PHRASES = (
|
|||
"published online",
|
||||
)
|
||||
|
||||
_PAGE_CONTINUATION_MARKER_PATTERN = re.compile(
|
||||
r"^(?:page\s*)?\d{1,4}\s*(?:[\.\-–—]?\s+of\s+|/)\s*\d{1,4}$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _looks_like_page_continuation_marker(block: dict) -> bool:
|
||||
text = str(block.get("text") or block.get("block_content") or "").strip()
|
||||
if not _PAGE_CONTINUATION_MARKER_PATTERN.match(text):
|
||||
return False
|
||||
|
||||
bbox = block.get("bbox") or block.get("block_bbox") or []
|
||||
width = float(bbox[2]) - float(bbox[0]) if len(bbox) >= 4 else None
|
||||
layout_sig = block.get("layout_signature") or {}
|
||||
if layout_sig.get("width") is not None:
|
||||
width = float(layout_sig["width"])
|
||||
|
||||
if width is not None:
|
||||
return width < 180
|
||||
return len(text.split()) <= 3
|
||||
_TABLE_PREFIX_LIKE = re.compile(
|
||||
rf"^(?:Table|Supplementary\s+Table|Extended\s+Data\s+Table)\s*"
|
||||
rf"(?:S\.?\s*)?(?:\d+(?:\.\d+)?|[IVXLCDM]+)\b",
|
||||
|
|
@ -321,6 +339,9 @@ def _anchor_matches_block(
|
|||
|
||||
|
||||
def _reference_anchor_matches_block(anchor: dict[str, Any], block: dict) -> bool:
|
||||
if _looks_like_page_continuation_marker(block):
|
||||
return False
|
||||
|
||||
if not _anchor_matches_block(anchor, block, _reference_family_key):
|
||||
return False
|
||||
marker_type = ((block.get("marker_signature") or {}).get("type") or "none")
|
||||
|
|
@ -350,6 +371,9 @@ def _has_reference_text_structure(block: dict) -> bool:
|
|||
if not text:
|
||||
return False
|
||||
|
||||
if _looks_like_page_continuation_marker(block):
|
||||
return False
|
||||
|
||||
if re.match(r"^\[\d+\]", text):
|
||||
return True
|
||||
if re.match(r"^\d+[\.)]", text):
|
||||
|
|
@ -463,6 +487,9 @@ def _word_count(block: dict) -> int:
|
|||
|
||||
|
||||
def _is_reference_family_candidate(block: dict) -> bool:
|
||||
if _looks_like_page_continuation_marker(block):
|
||||
return False
|
||||
|
||||
marker_type = ((block.get("marker_signature") or {}).get("type") or "none")
|
||||
if marker_type not in _REFERENCE_MARKER_TYPES:
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -489,6 +489,21 @@ def _ref_number_sort_key(block: dict) -> tuple:
|
|||
return (0, int(m.group(1) or m.group(2)))
|
||||
return (1, text)
|
||||
|
||||
_TAIL_BACKMATTER_CONTINUATION_PATTERN = re.compile(
|
||||
r"^(?:address correspondence|correspondence|e-?mail|email|"
|
||||
r"received|accepted|published|available online|"
|
||||
r"author contributions?|conflicts? of interest|funding|"
|
||||
r"acknowledg(?:e)?ments?)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _is_tail_backmatter_continuation(block: dict) -> bool:
|
||||
zone = str(block.get("zone") or "")
|
||||
if zone == "tail_nonref_hold_zone":
|
||||
return True
|
||||
text = str(block.get("text") or block.get("block_content") or "").strip()
|
||||
return bool(_TAIL_BACKMATTER_CONTINUATION_PATTERN.match(text))
|
||||
|
||||
def _reorder_tail_run(
|
||||
tail_blocks: list[dict],
|
||||
|
|
@ -523,13 +538,30 @@ def _reorder_tail_run(
|
|||
# Column-sort is maintained; ref items are grouped at the end.
|
||||
if skip_section_grouping:
|
||||
ref_roles = frozenset({"reference_heading", "reference_item", "reference_body"})
|
||||
backmatter_roles = frozenset({"backmatter_body", "backmatter_heading"})
|
||||
non_ref = [b for b in tail_blocks if b.get("role") not in ref_roles and b.get("role") not in backmatter_roles and b.get("role") != "footnote"]
|
||||
backmatter_roles = frozenset({
|
||||
"backmatter_body",
|
||||
"backmatter_heading",
|
||||
"backmatter_boundary_heading",
|
||||
})
|
||||
|
||||
non_ref_all = [
|
||||
b for b in tail_blocks
|
||||
if b.get("role") not in ref_roles
|
||||
and b.get("role") not in backmatter_roles
|
||||
and b.get("role") != "footnote"
|
||||
]
|
||||
|
||||
refs = [b for b in tail_blocks if b.get("role") in ref_roles]
|
||||
refs.sort(key=_ref_number_sort_key)
|
||||
|
||||
backmatter = [b for b in tail_blocks if b.get("role") in backmatter_roles]
|
||||
fnotes = [b for b in tail_blocks if b.get("role") == "footnote"]
|
||||
return non_ref + refs + backmatter + fnotes, carried_ref, carried_backmatter
|
||||
|
||||
tail_backmatter = [b for b in non_ref_all if _is_tail_backmatter_continuation(b)]
|
||||
ordinary_non_ref = [b for b in non_ref_all if not _is_tail_backmatter_continuation(b)]
|
||||
|
||||
return (ordinary_non_ref + refs + backmatter + tail_backmatter + fnotes,
|
||||
carried_ref, carried_backmatter)
|
||||
|
||||
# Quick check: do blocks have bbox data? If not, use FIFO fallback.
|
||||
has_geo = any(
|
||||
|
|
@ -559,10 +591,7 @@ def _reorder_tail_run(
|
|||
elif role == "reference_heading":
|
||||
ref_section = {"heading": block, "bodies": []}
|
||||
elif role == "reference_item":
|
||||
if _is_in_usable_content(block, header_band, footer_band):
|
||||
ref_items.append(block)
|
||||
else:
|
||||
non_tail_pass.append(block)
|
||||
ref_items.append(block)
|
||||
elif role == "backmatter_body":
|
||||
if _is_in_usable_content(block, header_band, footer_band):
|
||||
body_pool.append(block)
|
||||
|
|
@ -573,6 +602,8 @@ def _reorder_tail_run(
|
|||
idx = _find_owning_heading(block, backmatter_sections, page_width) if backmatter_sections else None
|
||||
if idx is not None:
|
||||
backmatter_sections[idx]["bodies"].append(block)
|
||||
elif _is_tail_backmatter_continuation(block):
|
||||
body_pool.append(block)
|
||||
else:
|
||||
non_tail_pass.append(block)
|
||||
else:
|
||||
|
|
@ -618,6 +649,7 @@ def _reorder_tail_run(
|
|||
|
||||
# Phase 4b — geometric body attachment for remaining bodies
|
||||
first_local_anchor_top: float | None = None
|
||||
tail_backmatter_blocks: list[dict] = []
|
||||
local_heading_tops = []
|
||||
for sec in backmatter_sections:
|
||||
bbox = sec["heading"].get("bbox") or sec["heading"].get("block_bbox")
|
||||
|
|
@ -634,7 +666,13 @@ def _reorder_tail_run(
|
|||
idx = _find_owning_heading(body, backmatter_sections, page_width)
|
||||
if idx is not None:
|
||||
backmatter_sections[idx]["bodies"].append(body)
|
||||
elif carried_backmatter is not None:
|
||||
continue
|
||||
|
||||
if _is_tail_backmatter_continuation(body):
|
||||
tail_backmatter_blocks.append(body)
|
||||
continue
|
||||
|
||||
if carried_backmatter is not None:
|
||||
bbox = body.get("bbox") or body.get("block_bbox")
|
||||
if bbox and len(bbox) >= 4:
|
||||
body_top = bbox[1]
|
||||
|
|
@ -643,7 +681,8 @@ def _reorder_tail_run(
|
|||
):
|
||||
carried_bodies.append(body)
|
||||
continue
|
||||
elif ref_section is not None and not _needs_synthetic_ref:
|
||||
|
||||
if ref_section is not None and not _needs_synthetic_ref:
|
||||
if ref_section is carried_ref:
|
||||
ref_section["bodies"].append(body)
|
||||
orphan_blocks.append(body)
|
||||
|
|
@ -677,6 +716,7 @@ def _reorder_tail_run(
|
|||
if ref_section.get("bodies"):
|
||||
ref_section["bodies"].sort(key=_ref_number_sort_key)
|
||||
result.extend(ref_section["bodies"])
|
||||
result.extend(tail_backmatter_blocks)
|
||||
result.extend(footnote_blocks)
|
||||
result.extend(orphan_blocks)
|
||||
|
||||
|
|
@ -774,6 +814,26 @@ def _sort_blocks_by_column(blocks: list[dict], page_width: int) -> list[dict]:
|
|||
return sorted(blocks, key=_column_key)
|
||||
|
||||
|
||||
|
||||
def _force_reference_heading_before_same_page_refs(page_blocks: list[dict]) -> list[dict]:
|
||||
"""Reorder page blocks so reference_item/reference_body always follow
|
||||
the first reference_heading on the same page. Non-ref blocks are unchanged."""
|
||||
headings = [b for b in page_blocks if b.get("role") == "reference_heading"]
|
||||
if not headings:
|
||||
return page_blocks
|
||||
|
||||
ref_roles = {"reference_item", "reference_body"}
|
||||
refs = [b for b in page_blocks if b.get("role") in ref_roles]
|
||||
if not refs:
|
||||
return page_blocks
|
||||
|
||||
first_heading = headings[0]
|
||||
heading_idx = page_blocks.index(first_heading)
|
||||
|
||||
before = [b for b in page_blocks[:heading_idx] if b.get("role") not in ref_roles]
|
||||
after = [b for b in page_blocks[heading_idx + 1:] if b.get("role") not in ref_roles]
|
||||
|
||||
return before + [first_heading] + sorted(refs, key=_ref_number_sort_key) + after
|
||||
def _order_tail_blocks(
|
||||
blocks: list[dict], style_profiles: dict | None = None, *, has_verified_reference_zone: bool = False
|
||||
) -> list[dict]:
|
||||
|
|
@ -823,7 +883,17 @@ def _order_tail_blocks(
|
|||
page_widths.setdefault(page, pw)
|
||||
|
||||
if not tail_pages:
|
||||
return blocks
|
||||
result: list[dict] = []
|
||||
by_page: dict[int, list[dict]] = {}
|
||||
for block in blocks:
|
||||
p = block.get("page")
|
||||
if p is None:
|
||||
result.append(block)
|
||||
else:
|
||||
by_page.setdefault(p, []).append(block)
|
||||
for page in sorted(by_page):
|
||||
result.extend(_force_reference_heading_before_same_page_refs(by_page[page]))
|
||||
return result
|
||||
|
||||
# Group blocks by page
|
||||
by_page: dict[int, list[dict]] = {}
|
||||
|
|
@ -843,7 +913,8 @@ def _order_tail_blocks(
|
|||
if page in tail_pages:
|
||||
pw = page_widths.get(page, 1200)
|
||||
sorted_blocks = _sort_blocks_by_column(page_blocks, pw)
|
||||
_page_has_ref_items = any(b.get("role") == "reference_item" for b in sorted_blocks)
|
||||
sorted_blocks = _force_reference_heading_before_same_page_refs(sorted_blocks)
|
||||
_page_has_ref_items = any(b.get("role") in {"reference_item", "reference_body"} for b in sorted_blocks)
|
||||
_page_has_ref_heading = any(b.get("role") == "reference_heading" for b in sorted_blocks)
|
||||
ordered, carried_ref, carried_backmatter = _reorder_tail_run(
|
||||
sorted_blocks,
|
||||
|
|
@ -986,6 +1057,18 @@ def _emit_reader_figures_before_references(
|
|||
lines.append("")
|
||||
|
||||
|
||||
def _add_consumed_key(
|
||||
keys: set[tuple[int | None, str | int]],
|
||||
page: int | None,
|
||||
block_id: str | int,
|
||||
) -> None:
|
||||
"""Add a per-page consumed block key, keyed by (page, block_id)."""
|
||||
if block_id is None or block_id == "":
|
||||
return
|
||||
keys.add((page, block_id))
|
||||
keys.add((page, str(block_id)))
|
||||
|
||||
|
||||
def render_fulltext_markdown(
|
||||
*,
|
||||
structured_blocks: list[dict],
|
||||
|
|
@ -1003,23 +1086,71 @@ def render_fulltext_markdown(
|
|||
reader_figures = (reader_payload or {}).get("reader_figures", [])
|
||||
consumed_caption_keys: set[tuple[int | None, int | str]] = set()
|
||||
consumed_caption_ids_unkeyed: set[int | str] = set()
|
||||
consumed_table_block_keys: set[tuple[int | None, str | int]] = set()
|
||||
# Map both raw id and str(id) because note_block_ids are stringified in
|
||||
# build_table_inventory while structured_blocks may carry int block_ids.
|
||||
block_page_by_id: dict[str | int, int | None] = {}
|
||||
# Build page-block_id index from structured blocks for fallback resolution.
|
||||
page_block_ids: dict[int | None, set[str | int]] = {}
|
||||
for block in structured_blocks:
|
||||
bid = block.get("block_id")
|
||||
if bid is None:
|
||||
continue
|
||||
page = block.get("page")
|
||||
block_page_by_id[bid] = page
|
||||
block_page_by_id[str(bid)] = page
|
||||
page_block_ids.setdefault(page, set()).add(bid)
|
||||
page_block_ids.setdefault(page, set()).add(str(bid))
|
||||
|
||||
consumed_table_block_keys: set[tuple[int | None, str | int]] = set()
|
||||
for table in table_inventory.get("tables", []):
|
||||
for block_id in table.get("consumed_block_ids", []):
|
||||
if not block_id:
|
||||
table_page = table.get("page")
|
||||
|
||||
# Build explicit page mapping from table inventory fields.
|
||||
# These fields know each block's true page and disambiguate
|
||||
# cross-page collisions (same block_id on different pages).
|
||||
explicit_id_page: dict[str | int, int | None] = {}
|
||||
|
||||
def _map_eid(bid, page):
|
||||
if bid is not None and bid != "":
|
||||
explicit_id_page[bid] = page
|
||||
explicit_id_page[str(bid)] = page
|
||||
|
||||
_map_eid(table.get("caption_block_id"), table_page)
|
||||
|
||||
for seg in table.get("segments", []):
|
||||
_map_eid(seg.get("asset_block_id"), seg.get("page") or table_page)
|
||||
|
||||
for note_bid in table.get("note_block_ids", []):
|
||||
if note_bid is not None and note_bid != "":
|
||||
# Notes may be on a different page — resolve from structured blocks
|
||||
note_page = None
|
||||
for p, ids in page_block_ids.items():
|
||||
if note_bid in ids or str(note_bid) in ids:
|
||||
note_page = p
|
||||
break
|
||||
_map_eid(note_bid, note_page or table_page)
|
||||
|
||||
for br_bid in table.get("bridge_block_ids", []):
|
||||
_map_eid(br_bid, table_page)
|
||||
|
||||
for bid in table.get("consumed_block_ids", []):
|
||||
if bid is None or bid == "":
|
||||
continue
|
||||
page = block_page_by_id.get(block_id, table.get("page"))
|
||||
consumed_table_block_keys.add((page, block_id))
|
||||
# Try explicit mapping first (handles cross-page assets and notes)
|
||||
page = explicit_id_page.get(bid) or explicit_id_page.get(str(bid))
|
||||
if page is not None:
|
||||
_add_consumed_key(consumed_table_block_keys, page, bid)
|
||||
continue
|
||||
|
||||
# Not in explicit fields — fall back to structured block lookup
|
||||
str_bid = str(bid)
|
||||
table_ids = page_block_ids.get(table_page, set())
|
||||
if bid in table_ids or str_bid in table_ids:
|
||||
_add_consumed_key(consumed_table_block_keys, table_page, bid)
|
||||
else:
|
||||
found = False
|
||||
for p, ids in page_block_ids.items():
|
||||
if bid in ids or str_bid in ids:
|
||||
_add_consumed_key(consumed_table_block_keys, p, bid)
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
_add_consumed_key(consumed_table_block_keys, table_page, bid)
|
||||
for item in (reader_payload or {}).get("consumed_caption_block_ids", []):
|
||||
if isinstance(item, dict):
|
||||
page = item.get("page")
|
||||
|
|
|
|||
|
|
@ -780,3 +780,31 @@ def test_table_roman_prefix_family_not_reference_like():
|
|||
family, authority = _classify_style_family(block, {}, {})
|
||||
assert family == "table_caption_like"
|
||||
assert authority == "table_marker"
|
||||
|
||||
|
||||
def test_page_continuation_marker_is_not_reference():
|
||||
"""Page continuation markers like "13 of 18", "13. of 18", "Page 13 of 18", "13/18" should NOT be classified as reference family."""
|
||||
from paperforge.worker.ocr_families import _is_reference_family_candidate
|
||||
|
||||
markers = [
|
||||
{"text": "13 of 18", "bbox": [0, 0, 60, 10], "marker_signature": {"type": "reference_numeric_dot"}, "span_signature": {"font_size_median": 9.0}, "layout_signature": {"width": 60}},
|
||||
{"text": "13. of 18", "bbox": [0, 0, 65, 10], "marker_signature": {"type": "reference_numeric_dot"}, "span_signature": {"font_size_median": 9.0}, "layout_signature": {"width": 65}},
|
||||
{"text": "Page 13 of 18", "bbox": [0, 0, 90, 10], "marker_signature": {"type": "reference_numeric_dot"}, "span_signature": {"font_size_median": 9.0}, "layout_signature": {"width": 90}},
|
||||
{"text": "13/18", "bbox": [0, 0, 40, 10], "marker_signature": {"type": "reference_numeric_dot"}, "span_signature": {"font_size_median": 9.0}, "layout_signature": {"width": 40}},
|
||||
]
|
||||
for marker in markers:
|
||||
assert not _is_reference_family_candidate(marker), f"'{marker['text']}' should not be reference family"
|
||||
|
||||
|
||||
def test_real_reference_is_still_reference():
|
||||
"""Real references like "13. Smith J, Wang L. Cartilage repair in 2018." should still be classified as reference family."""
|
||||
from paperforge.worker.ocr_families import _is_reference_family_candidate
|
||||
|
||||
ref = {
|
||||
"text": "13. Smith J, Wang L. Cartilage repair in 2018. Journal of Orthopaedic Research.",
|
||||
"bbox": [100, 0, 500, 20],
|
||||
"marker_signature": {"type": "reference_numeric_dot"},
|
||||
"span_signature": {"font_size_median": 10.0},
|
||||
"layout_signature": {"width": 400},
|
||||
}
|
||||
assert _is_reference_family_candidate(ref), "Real reference should still be classified as reference family"
|
||||
|
|
|
|||
|
|
@ -537,3 +537,261 @@ def test_materialized_table_caption_continuation_is_skipped_by_render_when_consu
|
|||
)
|
||||
|
||||
assert "Structural parameters of nanocomposites" not in md
|
||||
|
||||
def test_cross_page_table_asset_id_does_not_consume_same_id_on_caption_page():
|
||||
"""
|
||||
Caption page has block_id=7 reference_item.
|
||||
Table asset on previous page also has block_id=7.
|
||||
Explicit segment consumes only previous-page asset, not caption-page ref.
|
||||
"""
|
||||
from paperforge.worker.ocr_render import render_fulltext_markdown
|
||||
|
||||
table_inventory = {
|
||||
"tables": [
|
||||
{
|
||||
"page": 5,
|
||||
"caption_block_id": 10,
|
||||
"segments": [
|
||||
{"page": 4, "asset_block_id": 7},
|
||||
],
|
||||
"note_block_ids": [11],
|
||||
"bridge_block_ids": [12],
|
||||
"consumed_block_ids": [7, 10, 11, 12],
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
structured_blocks = [
|
||||
# Table asset on page 4, block_id=7
|
||||
{"block_id": 7, "page": 4, "role": "table_asset", "text": "[Table 1]"},
|
||||
# Reference item on page 5, block_id=7 — must NOT be consumed
|
||||
{"block_id": 7, "page": 5, "role": "reference_item", "text": "7. Smith J...", "bbox": [0, 0, 500, 20]},
|
||||
]
|
||||
|
||||
result = render_fulltext_markdown(
|
||||
structured_blocks=structured_blocks,
|
||||
table_inventory=table_inventory,
|
||||
resolved_metadata={},
|
||||
figure_inventory={"matched_figures": [], "unmatched_assets": [], "unresolved_clusters": []},
|
||||
page_count=10,
|
||||
)
|
||||
|
||||
assert "7. Smith J..." in result, "Reference on caption page should not be consumed"
|
||||
|
||||
|
||||
def test_table_consumed_block_ids_do_not_drop_same_id_reference_on_later_page():
|
||||
"""Reference on later page with same block_id as earlier table must not be dropped."""
|
||||
from paperforge.worker.ocr_render import render_fulltext_markdown
|
||||
|
||||
table_inventory = {
|
||||
"tables": [
|
||||
{
|
||||
"page": 1,
|
||||
"caption_block_id": 5,
|
||||
"segments": [{"page": 1, "asset_block_id": 6}],
|
||||
"note_block_ids": [],
|
||||
"bridge_block_ids": [],
|
||||
"consumed_block_ids": [5, 6],
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
structured_blocks = [
|
||||
{"block_id": 5, "page": 1, "role": "table_caption", "text": "Table 1. Results"},
|
||||
{"block_id": 6, "page": 1, "role": "table_asset", "text": "[data]"},
|
||||
# Same block_id=5 on page 2 — must NOT be consumed
|
||||
{"block_id": 5, "page": 2, "role": "reference_item", "text": "5. Author...", "bbox": [0, 0, 500, 20]},
|
||||
]
|
||||
|
||||
result = render_fulltext_markdown(
|
||||
structured_blocks=structured_blocks,
|
||||
table_inventory=table_inventory,
|
||||
resolved_metadata={},
|
||||
figure_inventory={"matched_figures": [], "unmatched_assets": [], "unresolved_clusters": []},
|
||||
page_count=10,
|
||||
)
|
||||
|
||||
assert "5. Author..." in result, "Reference on later page with same block_id should not be consumed"
|
||||
|
||||
|
||||
def test_table_caption_still_consumed():
|
||||
"""Table caption blocks on same page as table must still be consumed."""
|
||||
from paperforge.worker.ocr_render import render_fulltext_markdown
|
||||
|
||||
table_inventory = {
|
||||
"tables": [
|
||||
{
|
||||
"page": 1,
|
||||
"caption_block_id": 5,
|
||||
"segments": [{"page": 1, "asset_block_id": 6}],
|
||||
"note_block_ids": [],
|
||||
"bridge_block_ids": [],
|
||||
"consumed_block_ids": [5, 6],
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
structured_blocks = [
|
||||
{"block_id": 5, "page": 1, "role": "table_caption", "text": "Table 1. Results"},
|
||||
{"block_id": 6, "page": 1, "role": "table_asset", "text": "[data]"},
|
||||
]
|
||||
|
||||
result = render_fulltext_markdown(
|
||||
structured_blocks=structured_blocks,
|
||||
table_inventory=table_inventory,
|
||||
resolved_metadata={},
|
||||
figure_inventory={"matched_figures": [], "unmatched_assets": [], "unresolved_clusters": []},
|
||||
page_count=10,
|
||||
)
|
||||
|
||||
assert "Table 1. Results" not in result, "Table caption should be consumed (excluded from output)"
|
||||
assert "[data]" not in result, "Table asset should be consumed"
|
||||
|
||||
def test_skip_section_grouping_places_tail_nonref_after_refs():
|
||||
"""When skip_section_grouping=True, tail backmatter (Address correspondence) must appear AFTER refs."""
|
||||
from paperforge.worker.ocr_render import _reorder_tail_run
|
||||
|
||||
ref_block = {"role": "reference_item", "text": "1. Author A. Title. Journal. 2024.", "bbox": [100, 0, 500, 20], "block_number": "1"}
|
||||
tail_block = {"role": "body_paragraph", "text": "Address correspondence to: Dr. Smith", "bbox": [100, 50, 500, 70], "zone": "tail_nonref_hold_zone"}
|
||||
heading = {"role": "backmatter_heading", "text": "Correspondence", "bbox": [100, -10, 500, 10]}
|
||||
|
||||
result, ref, bm = _reorder_tail_run(
|
||||
[ref_block, tail_block, heading],
|
||||
carried_ref=None,
|
||||
carried_backmatter=None,
|
||||
page_width=800,
|
||||
skip_section_grouping=True,
|
||||
)
|
||||
texts = [b.get("text", "") for b in result]
|
||||
ref_idx = texts.index("1. Author A. Title. Journal. 2024.")
|
||||
tail_idx = texts.index("Address correspondence to: Dr. Smith")
|
||||
assert tail_idx > ref_idx, f"Tail block at {tail_idx} should be after ref at {ref_idx}"
|
||||
|
||||
|
||||
def test_skip_section_grouping_keeps_ordinary_nonref_before_refs():
|
||||
"""Ordinary non-ref blocks (not tail backmatter) must remain BEFORE refs."""
|
||||
from paperforge.worker.ocr_render import _reorder_tail_run
|
||||
|
||||
ordinary = {"role": "body_paragraph", "text": "This is some ordinary body text.", "bbox": [100, 0, 500, 20]}
|
||||
ref_block = {"role": "reference_item", "text": "1. Author A. Title. 2024.", "bbox": [100, 50, 500, 70], "block_number": "1"}
|
||||
|
||||
result, ref, bm = _reorder_tail_run(
|
||||
[ordinary, ref_block],
|
||||
carried_ref=None,
|
||||
carried_backmatter=None,
|
||||
page_width=800,
|
||||
skip_section_grouping=True,
|
||||
)
|
||||
texts = [b.get("text", "") for b in result]
|
||||
ordinary_idx = texts.index("This is some ordinary body text.")
|
||||
ref_idx = texts.index("1. Author A. Title. 2024.")
|
||||
assert ordinary_idx < ref_idx, f"Ordinary block at {ordinary_idx} should be before ref at {ref_idx}"
|
||||
|
||||
|
||||
def test_real_backmatter_heading_still_attaches_its_own_body():
|
||||
"""A real backmatter heading like 'FUNDING' must still have its body attached (not orphaned)."""
|
||||
from paperforge.worker.ocr_render import _reorder_tail_run
|
||||
|
||||
heading = {"role": "backmatter_heading", "text": "FUNDING", "bbox": [100, 0, 500, 20]}
|
||||
body = {"role": "backmatter_body", "text": "This work was supported by NIH grant R01...", "bbox": [100, 25, 500, 45]}
|
||||
ref_block = {"role": "reference_item", "text": "1. Author A. 2024.", "bbox": [100, 100, 500, 120], "block_number": "1"}
|
||||
|
||||
result, ref, bm = _reorder_tail_run(
|
||||
[heading, body, ref_block],
|
||||
carried_ref=None,
|
||||
carried_backmatter=None,
|
||||
page_width=800,
|
||||
skip_section_grouping=False,
|
||||
)
|
||||
texts = [b.get("text", "") for b in result]
|
||||
assert "FUNDING" in texts
|
||||
assert "This work was supported by NIH grant R01..." in texts
|
||||
funding_idx = texts.index("FUNDING")
|
||||
body_idx = texts.index("This work was supported by NIH grant R01...")
|
||||
assert body_idx > funding_idx, "Funding body should be after Funding heading"
|
||||
|
||||
|
||||
def test_phase_4b_does_not_swallow_tail_nonref_into_ref_section():
|
||||
"""Phase 4b must not place tail backmatter continuations into ref_section["bodies"]."""
|
||||
from paperforge.worker.ocr_render import _reorder_tail_run
|
||||
|
||||
ref_heading = {"role": "reference_heading", "text": "References", "bbox": [100, 0, 500, 20]}
|
||||
ref_item = {"role": "reference_item", "text": "1. Author. Title. 2024.", "bbox": [100, 25, 500, 45], "block_number": "1"}
|
||||
tail_body = {"role": "body_paragraph", "text": "Address correspondence to: Dr. Smith", "bbox": [100, 100, 500, 120], "zone": "tail_nonref_hold_zone"}
|
||||
|
||||
result, ref, bm = _reorder_tail_run(
|
||||
[ref_heading, ref_item, tail_body],
|
||||
carried_ref=None,
|
||||
carried_backmatter=None,
|
||||
page_width=800,
|
||||
)
|
||||
texts = [b.get("text", "") for b in result]
|
||||
ref_idx = texts.index("1. Author. Title. 2024.")
|
||||
tail_idx = texts.index("Address correspondence to: Dr. Smith")
|
||||
assert tail_idx > ref_idx, f"Tail backmatter at {tail_idx} should be after ref at {ref_idx}"
|
||||
|
||||
|
||||
def test_funding_body_still_attaches_to_funding_heading():
|
||||
"""A real funding body must still attach to its heading even with tail_backmatter_blocks logic."""
|
||||
from paperforge.worker.ocr_render import _reorder_tail_run
|
||||
|
||||
funding_heading = {"role": "backmatter_heading", "text": "FUNDING", "bbox": [100, 0, 500, 20]}
|
||||
funding_body = {"role": "backmatter_body", "text": "This work was funded by...", "bbox": [100, 25, 500, 45]}
|
||||
ref_item = {"role": "reference_item", "text": "1. Author. 2024.", "bbox": [100, 100, 500, 120], "block_number": "1"}
|
||||
|
||||
result, ref, bm = _reorder_tail_run(
|
||||
[funding_heading, funding_body, ref_item],
|
||||
carried_ref=None,
|
||||
carried_backmatter=None,
|
||||
page_width=800,
|
||||
)
|
||||
texts = [b.get("text", "") for b in result]
|
||||
funding_idx = texts.index("FUNDING")
|
||||
body_idx = texts.index("This work was funded by...")
|
||||
assert body_idx > funding_idx, "Funding body after funding heading"
|
||||
|
||||
|
||||
def test_refs_on_heading_page_are_rendered_after_heading():
|
||||
"""When reference_item blocks appear before reference_heading on the same
|
||||
page (e.g. due to column sort or PDF reading order), the helper must
|
||||
reorder them so the heading comes first."""
|
||||
from paperforge.worker.ocr_render import _force_reference_heading_before_same_page_refs
|
||||
|
||||
ref_heading = {"role": "reference_heading", "text": "References", "bbox": [400, 100, 500, 120]}
|
||||
ref_item = {"role": "reference_item", "text": "1. Author A. Title. 2024.", "bbox": [100, 0, 300, 20], "block_number": "1"}
|
||||
body_block = {"role": "body_paragraph", "text": "Some body text.", "bbox": [100, 50, 300, 70]}
|
||||
|
||||
# Input has ref_item BEFORE ref_heading (bad ordering)
|
||||
result = _force_reference_heading_before_same_page_refs([ref_item, ref_heading, body_block])
|
||||
|
||||
texts = [b.get("text", "") for b in result]
|
||||
heading_idx = texts.index("References")
|
||||
ref_idx = texts.index("1. Author A. Title. 2024.")
|
||||
body_idx = texts.index("Some body text.")
|
||||
|
||||
assert heading_idx < ref_idx, f"Heading at {heading_idx} should be before ref at {ref_idx}"
|
||||
assert body_idx < heading_idx or body_idx > ref_idx, f"Body at {body_idx} should stay in its original position relative to heading/ref"
|
||||
|
||||
|
||||
def test_non_ref_blocks_not_moved_by_guard():
|
||||
"""Blocks without reference role should not be moved or duplicated by
|
||||
the force-reorder guard."""
|
||||
from paperforge.worker.ocr_render import _force_reference_heading_before_same_page_refs
|
||||
|
||||
body1 = {"role": "body_paragraph", "text": "First body.", "bbox": [100, 0, 300, 20]}
|
||||
body2 = {"role": "body_paragraph", "text": "Second body.", "bbox": [100, 30, 300, 50]}
|
||||
ref_heading = {"role": "reference_heading", "text": "References", "bbox": [400, 100, 500, 120]}
|
||||
ref_item = {"role": "reference_item", "text": "1. Author. 2024.", "bbox": [100, 60, 300, 80], "block_number": "1"}
|
||||
|
||||
result = _force_reference_heading_before_same_page_refs([body1, ref_item, ref_heading, body2])
|
||||
|
||||
texts = [b.get("text", "") for b in result]
|
||||
assert "First body." in texts
|
||||
assert "Second body." in texts
|
||||
assert "References" in texts
|
||||
assert "1. Author. 2024." in texts
|
||||
# Body blocks must not be duplicated
|
||||
assert texts.count("First body.") == 1
|
||||
assert texts.count("Second body.") == 1
|
||||
assert texts.count("References") == 1
|
||||
assert texts.count("1. Author. 2024.") == 1
|
||||
|
|
|
|||
Loading…
Reference in a new issue