mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
P1: backmatter boundary redesign — ref-anchored partition replaces second _reconcile_tail_spread
reference_zone is the only hard boundary. Pre-ref disclosure headings (CRediT, Ethics, Declaration) are normalized as local same-column runs — they never set global boundaries. Four contracts enforced (Contract A-D in spec). - 13 new helpers in ocr_document.py: _has_verified_reference_zone, _partition_by_reference_zone, _classify_same_page_block, _normalize_reference_roles_from_partition, _normalize_pre_ref_disclosure_runs, _build_tail_boundary_from_ref_partition, etc. - infer_zones() stores effective_end_page for trimmed reference extent - normalize_document_structure(): ref-partitioned path replaces second _reconcile_tail_spread; _promote_tail_body_candidates/_assign_tail_spread_ownership skipped when active - 20 tests in test_backmatter_boundary.py - 322/322 tests pass
This commit is contained in:
Research Assistant
parent
23e7439270
commit
1a9ad1d147
4 changed files with 2132 additions and 12 deletions
|
|
@ -0,0 +1,642 @@
|
|||
# Implementation Plan: Backmatter Boundary Redesign
|
||||
|
||||
> **Based on:** `2026-06-27-backmatter-boundary-redesign.md`
|
||||
> **Type:** Ref-anchored partition replaces `_reconcile_tail_spread()`, all in `ocr_document.py`
|
||||
> **Verification:** 25K5KZAQ (CRediT fix), NC66N4Q3 (same-page body+ref), 9TW98JH8 (regression)
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Replace the later `_reconcile_tail_spread()` call in `normalize_document_structure()`
|
||||
with a ref-anchored 3-way partition (pre-ref / reference / post-ref) when a verified
|
||||
`reference_zone` exists. Pre-ref disclosure headings (CRediT, Ethics, Declaration)
|
||||
are normalized as local same-column runs only — they NEVER set global boundaries.
|
||||
|
||||
**Does NOT affect:**
|
||||
- First `infer_zones()` call near the top (unchanged)
|
||||
- `_reconcile_tail_spread()` — kept as fallback for papers without reference zone
|
||||
- `_normalize_backmatter_roles_after_boundary()` — kept as fallback
|
||||
- `_promote_tail_body_candidates()` — SKIPPED in ref-partitioned path (Contract C)
|
||||
|
||||
---
|
||||
|
||||
## Implementation guardrails (read before coding)
|
||||
|
||||
```
|
||||
- Do NOT redefine existing helpers: _zone_block_key, _block_y_top, _block_y_bottom already exist.
|
||||
- Do NOT use boundary_band.end_page as reference end in ref-partition path.
|
||||
- Do NOT classify same-column blocks as reference only by x-column membership.
|
||||
- Do NOT let a disclosure heading consume across ANY next heading, including another disclosure heading — each heading must be handled by its own outer-loop iteration.
|
||||
- Do NOT run _promote_tail_body_candidates() or _assign_tail_spread_ownership() after ref_partition_active=True.
|
||||
- After refreshing region_bus, clear stale body/reference/tail zones and re-apply zone labels.
|
||||
- Normalize reference partition roles BEFORE _sanitize_reference_zone_boundary().
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 1: `_has_verified_reference_zone()`
|
||||
|
||||
```python
|
||||
def _has_verified_reference_zone(region_bus: dict[str, dict]) -> bool:
|
||||
"""Check if region_bus contains a verified reference zone with block IDs."""
|
||||
ref_zone = region_bus.get("reference_zone", {}) or {}
|
||||
if ref_zone.get("status") not in ("ACCEPT", "HOLD"):
|
||||
return False
|
||||
block_ids = ref_zone.get("block_ids") or []
|
||||
return len(block_ids) > 0
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 2: Constants
|
||||
|
||||
```python
|
||||
_KNOWN_PRE_REFERENCE_DISCLOSURE_HEADINGS: dict[str, list[str]] = {
|
||||
"credit authorship contribution statement": [],
|
||||
"ethics approval and consent to participate": ["ethics statement", "ethical approval"],
|
||||
"declaration of competing interest": [],
|
||||
"competing interests": ["conflict of interest"],
|
||||
"data availability": ["data availability statement"],
|
||||
"acknowledgements": ["acknowledgments", "acknowledgement", "acknowledgment"],
|
||||
"author contributions": [],
|
||||
}
|
||||
|
||||
_STRONG_BODY_HEADINGS: frozenset[str] = frozenset({
|
||||
"discussion", "conclusion", "conclusions", "results", "summary",
|
||||
"limitations", "future perspectives", "materials and methods",
|
||||
"methods", "introduction",
|
||||
})
|
||||
|
||||
_HEADING_ROLES: frozenset[str] = frozenset({
|
||||
"section_heading", "subsection_heading", "sub_subsection_heading",
|
||||
"backmatter_heading", "backmatter_boundary_heading", "backmatter_heading_candidate",
|
||||
})
|
||||
|
||||
_MARKER_HEADING_TYPES: frozenset[str] = frozenset({
|
||||
"heading_numbered", "heading_arabic", "heading_decimal",
|
||||
})
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 3: `_resolve_reference_zone_extent()`
|
||||
|
||||
**P0-2 fix: Do NOT redefine `_zone_block_key` — reuse existing one.**
|
||||
|
||||
```python
|
||||
def _derive_effective_end_from_block_ids(
|
||||
blocks: list[dict], block_ids: list[str]
|
||||
) -> int | None:
|
||||
"""Scan blocks for max page among reference_zone.block_ids.
|
||||
block_ids may be bare block_id or `p{page}:{block_id}` artifact keys.
|
||||
"""
|
||||
id_set = {str(x) for x in block_ids} # P0-6b: str conversion
|
||||
max_page = None
|
||||
for b in blocks:
|
||||
bid = str(b.get("block_id") or "")
|
||||
key = _zone_block_key(b) # reuse existing helper
|
||||
if bid in id_set or key in id_set:
|
||||
p = int(b.get("page", 0) or 0)
|
||||
if p > 0:
|
||||
max_page = max(max_page, p) if max_page is not None else p
|
||||
return max_page
|
||||
|
||||
|
||||
def _resolve_reference_zone_extent(
|
||||
blocks: list[dict],
|
||||
region_bus: dict[str, dict],
|
||||
) -> tuple[int | None, int | None]:
|
||||
"""Return (ref_start_page, ref_end_page) from region_bus.
|
||||
Uses effective_end_page or derives from trimmed block_ids.
|
||||
NEVER falls back to untrimmed boundary_band.end_page.
|
||||
"""
|
||||
ref_zone = region_bus.get("reference_zone", {}) or {}
|
||||
block_ids: list[str] = ref_zone.get("block_ids") or []
|
||||
band = ref_zone.get("boundary_band") or {}
|
||||
ref_start = band.get("start_page")
|
||||
if not block_ids or ref_start is None:
|
||||
return None, None
|
||||
|
||||
effective_end = ref_zone.get("effective_end_page")
|
||||
if effective_end is None:
|
||||
effective_end = _derive_effective_end_from_block_ids(blocks, block_ids)
|
||||
if effective_end is None:
|
||||
return None, None # P0-6a: handle both-None case
|
||||
return ref_start, effective_end
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 4: `_classify_same_page_block()`
|
||||
|
||||
**P0-2: Reuse existing `_block_y_top()` / `_block_y_bottom()`. Do NOT redefine.**
|
||||
**P0-3: Use correct marker type set, not invented names.**
|
||||
**P0-4: Remove `_compute_ref_heading_pages()` — fallback conservatively returns `pre_ref`.**
|
||||
|
||||
Reference marker types from existing code:
|
||||
```python
|
||||
_REFERENCE_ZONE_MARKER_TYPES = {
|
||||
"reference_numeric_bracket", "reference_numeric_dot",
|
||||
"reference_numeric_parenthesis", "reference_pattern", "citation_line",
|
||||
}
|
||||
```
|
||||
|
||||
```python
|
||||
def _block_x_center(block: dict) -> float:
|
||||
bbox = block.get("bbox") or [0, 0, 0, 0]
|
||||
return (bbox[0] + bbox[2]) / 2.0 if len(bbox) >= 4 else 0.0
|
||||
|
||||
|
||||
def _classify_same_page_block( # P0-4: no _compute_ref_heading_pages
|
||||
block: dict,
|
||||
region_bus: dict,
|
||||
page: int,
|
||||
ref_zones: list[ReferenceZone] | None = None,
|
||||
page_layouts: dict[int, PageLayoutProfile] | None = None,
|
||||
) -> str:
|
||||
"""Classify a block on a shared body/ref page.
|
||||
|
||||
A block is "reference" only if it explicitly belongs to the reference zone.
|
||||
Conservative: when uncertain, return "pre_ref" (keeps body in body).
|
||||
"""
|
||||
role = str(block.get("role") or "")
|
||||
seed_role = str(block.get("seed_role") or "")
|
||||
if role in {"reference_heading", "reference_item"}:
|
||||
return "reference"
|
||||
if seed_role in {"reference_heading", "reference_item"}:
|
||||
return "reference"
|
||||
|
||||
# Check block_ids / composite_block_ids membership
|
||||
ref_zone = region_bus.get("reference_zone", {}) or {}
|
||||
ref_ids = {str(x) for x in (ref_zone.get("block_ids") or [])}
|
||||
ref_comp_ids = {str(x) for x in (ref_zone.get("composite_block_ids") or [])}
|
||||
block_key = str(block.get("block_id") or "")
|
||||
art_key = _zone_block_key(block)
|
||||
if block_key in ref_ids or art_key in ref_ids:
|
||||
return "reference"
|
||||
if block_key in ref_comp_ids or art_key in ref_comp_ids:
|
||||
return "reference"
|
||||
|
||||
# Column-aware: below ref heading, same column, with reference marker
|
||||
if ref_zones and page_layouts:
|
||||
block_x = _block_x_center(block)
|
||||
layout = page_layouts.get(page)
|
||||
col_boundaries = layout.column_boundaries if layout else []
|
||||
for rz in ref_zones:
|
||||
if rz.page != page:
|
||||
continue
|
||||
if not col_boundaries or rz.column_index >= len(col_boundaries):
|
||||
continue
|
||||
col_range = _column_x_range(rz.column_index, col_boundaries)
|
||||
if col_range[0] <= block_x <= col_range[1]:
|
||||
if _block_y_top(block) >= rz.y_start:
|
||||
marker = str((block.get("marker_signature") or {}).get("type") or "")
|
||||
if marker in _REFERENCE_ZONE_MARKER_TYPES:
|
||||
return "reference"
|
||||
# In ref column but not a reference item — still body flow
|
||||
# (conservative: don't force into ref zone without marker)
|
||||
return "pre_ref"
|
||||
return "pre_ref" # above ref heading in ref column
|
||||
return "pre_ref" # different column
|
||||
|
||||
# Fallback: no column data — return pre_ref conservatively
|
||||
return "pre_ref"
|
||||
```
|
||||
|
||||
Helper for column x-range:
|
||||
```python
|
||||
def _column_x_range(
|
||||
column_index: int,
|
||||
col_boundaries: list[float],
|
||||
page_width: float = 1200.0,
|
||||
) -> tuple[float, float]:
|
||||
"""Return (x_start, x_end) for a column index."""
|
||||
if not col_boundaries:
|
||||
return (0.0, page_width)
|
||||
if column_index == 0:
|
||||
right = (col_boundaries[0] + col_boundaries[1]) / 2 if len(col_boundaries) > 1 else col_boundaries[0]
|
||||
return (0.0, right)
|
||||
if column_index < len(col_boundaries):
|
||||
left = (col_boundaries[column_index - 1] + col_boundaries[column_index]) / 2 if column_index > 0 else 0.0
|
||||
right = (col_boundaries[column_index] + col_boundaries[column_index + 1]) / 2 if column_index + 1 < len(col_boundaries) else page_width
|
||||
return (left, right)
|
||||
mid = (col_boundaries[-1] + page_width) / 2
|
||||
return (mid, page_width)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 5: `_partition_by_reference_zone()`
|
||||
|
||||
```python
|
||||
def _partition_by_reference_zone(
|
||||
blocks: list[dict],
|
||||
region_bus: dict[str, dict],
|
||||
ref_zones: list[ReferenceZone] | None = None,
|
||||
page_layouts: dict[int, PageLayoutProfile] | None = None,
|
||||
) -> dict[str, list[dict]]:
|
||||
"""Partition blocks into pre-ref / reference / post-ref.
|
||||
reference_zone is the ONLY boundary. Pre-ref disclosure headings do not
|
||||
participate in partition (normalized later).
|
||||
"""
|
||||
ref_start, ref_end = _resolve_reference_zone_extent(blocks, region_bus)
|
||||
if ref_start is None or ref_end is None: # P0-6a: both required
|
||||
return {"fallback": True}
|
||||
|
||||
pre_ref, reference, post_ref = [], [], []
|
||||
for block in blocks:
|
||||
page = int(block.get("page", 0) or 0)
|
||||
if page < ref_start:
|
||||
pre_ref.append(block)
|
||||
elif page > ref_end:
|
||||
post_ref.append(block)
|
||||
else:
|
||||
zone = _classify_same_page_block(
|
||||
block, region_bus, page,
|
||||
ref_zones=ref_zones, page_layouts=page_layouts,
|
||||
)
|
||||
if zone == "pre_ref":
|
||||
pre_ref.append(block)
|
||||
elif zone == "reference":
|
||||
reference.append(block)
|
||||
else:
|
||||
post_ref.append(block)
|
||||
|
||||
return {"pre_ref": pre_ref, "reference": reference, "post_ref": post_ref}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 6a: `_normalize_reference_roles_from_partition()` **P0-1 — CRITICAL ADDITION**
|
||||
|
||||
**Must run BEFORE `_sanitize_reference_zone_boundary()` or the sanitizer will strip
|
||||
untyped blocks from the reference zone.**
|
||||
|
||||
```python
|
||||
def _normalize_reference_roles_from_partition(
|
||||
blocks: list[dict],
|
||||
partition: dict[str, list[dict]],
|
||||
) -> None:
|
||||
"""Ensure blocks in the reference partition have reference roles.
|
||||
|
||||
Without this, _sanitize_reference_zone_boundary() strips any block in
|
||||
reference_zone whose role isn't reference_heading/reference_item.
|
||||
"""
|
||||
for block in partition.get("reference", []):
|
||||
role = str(block.get("role") or "")
|
||||
seed = str(block.get("seed_role") or "")
|
||||
if role in {"reference_heading", "reference_item"}:
|
||||
continue
|
||||
if seed in {"reference_heading", "reference_item"}:
|
||||
block["role"] = seed
|
||||
continue
|
||||
# Fallback: check marker
|
||||
marker = str((block.get("marker_signature") or {}).get("type") or "")
|
||||
if marker in _REFERENCE_ZONE_MARKER_TYPES:
|
||||
block["role"] = "reference_item"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 6b: `_normalize_pre_ref_disclosure_runs()`
|
||||
|
||||
**P0-5 fix: Disclosure run must break at ANY next heading, including another disclosure heading.**
|
||||
Each disclosure heading is handled by its own outer-loop iteration.
|
||||
|
||||
```python
|
||||
def _is_known_disclosure(text: str) -> bool:
|
||||
text_norm = re.sub(r"\s+", " ", text).strip().lower()
|
||||
for key, aliases in _KNOWN_PRE_REFERENCE_DISCLOSURE_HEADINGS.items():
|
||||
if text_norm == key or text_norm in aliases:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _get_column_index(block: dict, col_boundaries: list[float]) -> int | None:
|
||||
"""Return column index (0=L, 1=R) or None if undetermined."""
|
||||
if not col_boundaries:
|
||||
return None
|
||||
bbox = block.get("bbox") or [0, 0, 0, 0]
|
||||
if len(bbox) < 4:
|
||||
return None
|
||||
cx = (bbox[0] + bbox[2]) / 2.0
|
||||
for i in range(len(col_boundaries) - 1):
|
||||
mid = (col_boundaries[i] + col_boundaries[i + 1]) / 2
|
||||
if cx <= mid:
|
||||
return i
|
||||
return len(col_boundaries) - 1
|
||||
|
||||
|
||||
def _normalize_pre_ref_disclosure_runs(
|
||||
partition: dict[str, list[dict]],
|
||||
ref_start_page: int | None,
|
||||
total_pages: int,
|
||||
page_layouts: dict[int, PageLayoutProfile] | None = None,
|
||||
) -> None:
|
||||
"""Local role normalization for known disclosure sections before ref.
|
||||
|
||||
Contract B rules enforced:
|
||||
- Local same-column runs only
|
||||
- Does NOT cross ANY next heading (including another disclosure heading) — P0-5
|
||||
- Does NOT cross STRONG_BODY_HEADING
|
||||
- Does NOT cross numbered heading
|
||||
- Does NOT cross column boundary
|
||||
- Does NOT cross ref_start_page
|
||||
"""
|
||||
pre_ref = partition.get("pre_ref", [])
|
||||
if ref_start_page is None or not pre_ref:
|
||||
return
|
||||
|
||||
for i, block in enumerate(pre_ref):
|
||||
text = str(block.get("text", "") or "").strip().lower()
|
||||
text_norm = re.sub(r"\s+", " ", text)
|
||||
|
||||
matched_key = None
|
||||
for key, aliases in _KNOWN_PRE_REFERENCE_DISCLOSURE_HEADINGS.items():
|
||||
if text_norm == key or text_norm in aliases:
|
||||
matched_key = key
|
||||
break
|
||||
if matched_key is None:
|
||||
continue
|
||||
|
||||
# Heading-like gate
|
||||
role = str(block.get("role") or "")
|
||||
seed_role = str(block.get("seed_role") or "")
|
||||
raw_label = str(block.get("raw_label") or "")
|
||||
marker_type = str((block.get("marker_signature") or {}).get("type") or "")
|
||||
heading_like = (
|
||||
role in _HEADING_ROLES
|
||||
or seed_role in _HEADING_ROLES
|
||||
or raw_label == "paragraph_title"
|
||||
or marker_type in _MARKER_HEADING_TYPES
|
||||
)
|
||||
if not heading_like:
|
||||
span = block.get("span_signature") or {}
|
||||
if not span.get("bold") and not span.get("font_size", 0) > 10:
|
||||
continue
|
||||
if len(text.split()) > 20:
|
||||
continue
|
||||
|
||||
# Proximity gate (relative to ref_start, not total_pages ratio)
|
||||
block_page = int(block.get("page", 0) or 0)
|
||||
if ref_start_page is not None:
|
||||
max_before_ref = max(2, int(total_pages * 0.25))
|
||||
if block_page > ref_start_page or block_page < max(1, ref_start_page - max_before_ref):
|
||||
continue
|
||||
|
||||
# Determine column for run — use page_layouts if available
|
||||
block_col = None
|
||||
if page_layouts is not None:
|
||||
layout = page_layouts.get(block_page)
|
||||
if layout is not None:
|
||||
block_col = _get_column_index(block, layout.column_boundaries)
|
||||
|
||||
# Normalize run
|
||||
block["role"] = "backmatter_heading"
|
||||
block["_pre_ref_disclosure_run"] = True
|
||||
|
||||
for j in range(i + 1, len(pre_ref)):
|
||||
nb = pre_ref[j]
|
||||
np_ = int(nb.get("page", 0) or 0)
|
||||
if np_ > ref_start_page:
|
||||
break
|
||||
|
||||
# Column guard — P0-5: if column data available, enforce same column
|
||||
if block_col is not None and page_layouts is not None:
|
||||
nlayout = page_layouts.get(np_)
|
||||
if nlayout is not None:
|
||||
nc = _get_column_index(nb, nlayout.column_boundaries)
|
||||
if nc is not None and nc != block_col:
|
||||
break
|
||||
|
||||
nt = str(nb.get("text", "") or "").strip().lower()
|
||||
nr = str(nb.get("role") or "")
|
||||
nrw = str(nb.get("raw_label") or "")
|
||||
nmt = str((nb.get("marker_signature") or {}).get("type") or "")
|
||||
next_heading = (
|
||||
nr in _HEADING_ROLES
|
||||
or nrw == "paragraph_title"
|
||||
or nmt in _MARKER_HEADING_TYPES
|
||||
)
|
||||
|
||||
# P0-5: Stop at ANY next heading (including another disclosure).
|
||||
# Each disclosure heading gets its own outer-loop iteration.
|
||||
if next_heading:
|
||||
break
|
||||
|
||||
# Body heading guards
|
||||
if any(nt.startswith(h) for h in _STRONG_BODY_HEADINGS):
|
||||
break
|
||||
if re.match(r"^\d+(?:\.\d+)*\.?\s", nt):
|
||||
break
|
||||
|
||||
nr_final = str(nb.get("role") or "")
|
||||
if nr_final.endswith("_heading"):
|
||||
nb["role"] = "backmatter_heading"
|
||||
elif nr_final == "body_paragraph":
|
||||
nb["role"] = "backmatter_body"
|
||||
elif nr_final == "frontmatter_noise":
|
||||
nb["role"] = "backmatter_body"
|
||||
nb["_pre_ref_disclosure_run"] = True
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 7: `_build_tail_boundary_from_ref_partition()`
|
||||
|
||||
```python
|
||||
def _build_tail_boundary_from_ref_partition(
|
||||
partition: dict[str, list[dict]],
|
||||
region_bus: dict[str, dict],
|
||||
) -> TailBoundary | None:
|
||||
"""Derive TailBoundary from the ref-anchored partition.
|
||||
body_end_page from actual body blocks. spread_start = references_start.
|
||||
"""
|
||||
pre_ref = partition.get("pre_ref", [])
|
||||
reference = partition.get("reference", [])
|
||||
post_ref = partition.get("post_ref", [])
|
||||
|
||||
body_like_pages = [
|
||||
int(b.get("page", 0) or 0) for b in pre_ref
|
||||
if b.get("role") in {"body_paragraph", "section_heading",
|
||||
"subsection_heading", "sub_subsection_heading"}
|
||||
]
|
||||
body_end_page = max(body_like_pages) if body_like_pages else None
|
||||
|
||||
ref_pages = [int(b.get("page", 0) or 0) for b in reference if b.get("page")]
|
||||
post_ref_pages = [int(b.get("page", 0) or 0) for b in post_ref if b.get("page")]
|
||||
|
||||
if not ref_pages:
|
||||
return None
|
||||
|
||||
references_start = min(ref_pages)
|
||||
all_tail = ref_pages + post_ref_pages
|
||||
|
||||
return TailBoundary(
|
||||
body_end_page=body_end_page,
|
||||
backmatter_start=references_start,
|
||||
references_start=references_start,
|
||||
spread_start=references_start, # NOT body_end_page — Contract D
|
||||
spread_end=max(all_tail),
|
||||
is_clean_separated=True,
|
||||
reason="ref_zone_partition",
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 8: Wire call site
|
||||
|
||||
**P0-6: After refreshing region_bus, clear stale zones and re-apply.**
|
||||
**P0-1: Call `_normalize_reference_roles_from_partition()` before sanitizer.**
|
||||
|
||||
Find the second `page_layouts` recomputation block in `normalize_document_structure()`.
|
||||
|
||||
```python
|
||||
# --- Ref-anchored partition (replaces second _reconcile_tail_spread) ---
|
||||
page_layouts = _build_page_layout_profiles(blocks)
|
||||
|
||||
region_bus = infer_zones(blocks, ..., tail_spread=None) # refresh for current roles
|
||||
ref_zones = _detect_reference_zones(blocks, page_layouts)
|
||||
|
||||
ref_partition_active = False
|
||||
if _has_verified_reference_zone(region_bus):
|
||||
partition = _partition_by_reference_zone(
|
||||
blocks, region_bus, ref_zones=ref_zones, page_layouts=page_layouts,
|
||||
)
|
||||
if "fallback" not in partition:
|
||||
_normalize_reference_roles_from_partition(blocks, partition) # P0-1
|
||||
_normalize_pre_ref_disclosure_runs(
|
||||
partition,
|
||||
ref_start_page=_resolve_reference_zone_extent(blocks, region_bus)[0],
|
||||
total_pages=len({b.get("page") for b in blocks if b.get("page")}),
|
||||
page_layouts=page_layouts,
|
||||
)
|
||||
tail_spread = _build_tail_boundary_from_ref_partition(partition, region_bus)
|
||||
ref_partition_active = tail_spread is not None
|
||||
|
||||
if ref_partition_active:
|
||||
# P0-6: clear stale zones and re-apply with new roles
|
||||
_clear_partition_zones(blocks)
|
||||
_apply_zone_labels(blocks, region_bus)
|
||||
_apply_content_zone_fallback(blocks, region_bus)
|
||||
_sanitize_reference_zone_boundary(blocks, region_bus)
|
||||
|
||||
if not ref_partition_active:
|
||||
tail_spread = _reconcile_tail_spread(blocks, page_layouts)
|
||||
if tail_spread is not None:
|
||||
backmatter_form = ...
|
||||
_normalize_backmatter_roles_after_boundary(tail_spread, backmatter_form, blocks)
|
||||
```
|
||||
|
||||
The stale-zone clearing helper:
|
||||
```python
|
||||
def _clear_partition_zones(blocks: list[dict]) -> None:
|
||||
"""Clear zones that will be reassigned after ref-partition role normalization."""
|
||||
_STALE_ZONES = frozenset({
|
||||
"body_zone", "reference_zone", "tail_nonref_hold_zone",
|
||||
"post_reference_backmatter_zone", "display_zone",
|
||||
})
|
||||
for b in blocks:
|
||||
if b.get("zone") in _STALE_ZONES:
|
||||
b["zone"] = ""
|
||||
```
|
||||
|
||||
Then the Contract C skip further down:
|
||||
```python
|
||||
# --- CONDITIONAL: skip old promotion in ref-partitioned path ---
|
||||
if not ref_partition_active:
|
||||
blocks = _promote_tail_body_candidates(blocks, doc_structure, ...)
|
||||
blocks = _assign_tail_spread_ownership(blocks, doc_structure)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 9: Modify `infer_zones()` — store `effective_end_page`
|
||||
|
||||
Inside `infer_zones()`, after the reference_blocks trimming step (~lines 1067–1081):
|
||||
|
||||
```python
|
||||
# After trimming reference_blocks for post-ref backmatter:
|
||||
if post_ref_backmatter_start is not None:
|
||||
trimmed_pages = [
|
||||
int(b.get("page", 0) or 0) for b in reference_blocks
|
||||
if int(b.get("page", 0) or 0) > 0
|
||||
]
|
||||
effective_reference_end_page = max(trimmed_pages) if trimmed_pages else first_reference_page
|
||||
else:
|
||||
effective_reference_end_page = reference_end_page # same as current behavior
|
||||
|
||||
# Pass into _make_zone() — NOT as ref_zone["..."] direct assignment:
|
||||
"reference_zone": _make_zone( # or simpler: pass as kwargs if _make_zone supports it
|
||||
...,
|
||||
boundary_band=_page_band(first_reference_page, reference_end_page),
|
||||
effective_end_page=effective_reference_end_page,
|
||||
),
|
||||
```
|
||||
|
||||
If `_make_zone()` does not accept arbitrary kwargs, set it after the call:
|
||||
```python
|
||||
ref_zone_entry = _make_zone(...) # existing call
|
||||
ref_zone_entry["effective_end_page"] = effective_reference_end_page # add after
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 10: Tests
|
||||
|
||||
**File:** `tests/unit/worker/test_backmatter_boundary.py`
|
||||
|
||||
| # | Test | Verifies |
|
||||
|---|------|----------|
|
||||
| 1 | `test_ref_zone_partition_simple` | Single-col body p1-8, ref p9-10, post-ref p11 → correct 3-way split |
|
||||
| 2 | `test_ref_zone_partition_corrects_credit` | 25K5KZAQ: CRediT before ref → backmatter_heading; Discussion after → unchanged |
|
||||
| 3 | `test_pre_ref_disclosure_does_not_change_ref_start` | CRediT found → ref_start_page unchanged (Contract A) |
|
||||
| 4 | `test_disclosure_does_not_span_strong_body_heading` | "CRediT ... Conclusion ... References" → Conclusion NOT consumed |
|
||||
| 5 | `test_disclosure_does_not_span_numbered_heading` | "CRediT ... 5. Conclusion ... References" → section NOT consumed |
|
||||
| 6 | `test_disclosure_stops_at_another_disclosure` | **P0-5**: "CRediT ... Ethics ... References" → CRediT does NOT consume Ethics body |
|
||||
| 7 | `test_disclosure_stops_at_ref_start` | Blocks after CRediT but before ref, past another heading → NOT consumed |
|
||||
| 8 | `test_same_page_body_ref_column_aware` | Left-col body stays body_flow, right-col ref starts correctly |
|
||||
| 9 | `test_no_ref_zone_falls_back` | No reference_zone → old path, no crash |
|
||||
| 10 | `test_known_disclosure_exact_match` | Exact match triggers; substring "funding" in body heading doesn't |
|
||||
| 11 | `test_disclosure_proximity_gate` | page 2 of 15, ref at page 12 → NOT triggered (too far) |
|
||||
| 12 | `test_disclosure_same_column_only` | Two-column: right-col CRediT does NOT consume left-col text |
|
||||
| 13 | `test_reference_partition_normalizes_reference_roles` | **P0-1**: partition reference block with role body_paragraph → reference_item before sanitizer |
|
||||
| 14 | `test_ref_partition_clears_stale_tail_zone` | **P0-6**: initial zone=tail_nonref_hold_zone, after partition → re-assigned correctly |
|
||||
| 15 | `test_same_page_ref_column_non_reference_not_forced` | Block in ref column, below ref heading, no marker → NOT forced to "reference" |
|
||||
| 16 | `test_body_end_page_from_actual_blocks` | Pre-ref body max page determines body_end_page, not mechanical `ref_start - 1` |
|
||||
| 17 | `test_spread_start_is_references_start` | spread_start == references_start, not body_end_page |
|
||||
| 18 | `test_promote_tail_body_skipped_on_ref_path` | ref_partition_active → _promote_tail_body_candidates not called |
|
||||
| 19 | `test_reference_end_page_trimming` | Post-ref backmatter page excluded from effective_end_page |
|
||||
| 20 | `test_infer_zones_stores_effective_end_page` | After trimming, effective_end_page < raw end_page |
|
||||
|
||||
---
|
||||
|
||||
## Step 11: Verify
|
||||
|
||||
**Audit papers:**
|
||||
- **25K5KZAQ:** CRediT/Ethics → `backmatter_heading`/`backmatter_body`. Discussion/Conclusion unchanged. Fulltext renders correctly.
|
||||
- **NC66N4Q3:** page 56 left-col Discussion → body_flow, right-col refs → reference_zone. No degraded tail mode.
|
||||
- **9TW98JH8 (regression):** 20/20 blocks correct, unchanged.
|
||||
|
||||
**Test run:**
|
||||
```bash
|
||||
python -m pytest tests/unit/ tests/cli/ -v --tb=short
|
||||
ruff check --fix paperforge/ && ruff format paperforge/
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Summary of P0 fixes vs original plan
|
||||
|
||||
| P0 | Issue | Fix |
|
||||
|----|-------|-----|
|
||||
| P0-1 | Missing `_normalize_reference_roles_from_partition()` | Added Step 6a — runs before sanitizer |
|
||||
| P0-2 | Redefined existing helpers | Plan now says "reuse existing `_zone_block_key`, `_block_y_top`, `_block_y_bottom`" |
|
||||
| P0-3 | Wrong marker type names | Uses existing `_REFERENCE_ZONE_MARKER_TYPES` set |
|
||||
| P0-4 | `_compute_ref_heading_pages()` doesn't exist | Removed this fallback; `_classify_same_page_block` conservatively returns `pre_ref` without column data |
|
||||
| P0-5 | Disclosure run continues past another disclosure | Loop now `break`s at ANY next heading (P0-5 §6b) |
|
||||
| P0-6 | Stale zones after region_bus refresh | Added `_clear_partition_zones()` + zone re-apply (Step 8) |
|
||||
|
||||
**Total diff:** ~320 lines new code + ~100 lines tests, all in `ocr_document.py` + new test file
|
||||
|
|
@ -0,0 +1,752 @@
|
|||
# Backmatter Boundary Redesign
|
||||
|
||||
**Date:** 2026-06-27
|
||||
**Status:** Spec — for implementation
|
||||
**Audit evidence:** 25K5KZAQ, NC66N4Q3
|
||||
**Previous attempt:** `2026-06-27-figure-containment-and-backmatter-boundary-design.md` §4 (superseded)
|
||||
|
||||
---
|
||||
|
||||
## 0. Core contracts — READ BEFORE IMPLEMENTING
|
||||
|
||||
### Contract A — reference_zone is the ONLY hard boundary
|
||||
|
||||
Pre-ref backmatter headings (CRediT, Ethics, Declaration, etc.) may only normalize
|
||||
**local same-column runs**. They must NEVER determine:
|
||||
|
||||
- `reference_zone.start` / `reference_zone.end`
|
||||
- `body_end_page`
|
||||
- `spread_start` / `spread_end`
|
||||
- Body spine anchor page selection
|
||||
- Whether the global tail region has begun
|
||||
|
||||
Only a **verified reference_zone** (status ACCEPT or HOLD with block_ids) may define
|
||||
the global partition into pre-ref / reference / post-ref.
|
||||
|
||||
### Contract B — Pre-ref disclosure runs are local only
|
||||
|
||||
When a known disclosure heading is found before `reference_zone.start`, its role
|
||||
normalization scope is limited to:
|
||||
|
||||
- The heading block itself
|
||||
- Same-column blocks between it and the NEXT heading (any heading)
|
||||
- Blocks up to `reference_zone.start` (whichever comes first)
|
||||
|
||||
It must NOT consume across:
|
||||
- A numbered body section heading
|
||||
- A canonical unnumbered body heading (Discussion, Conclusion, Results, Methods, Limitations, Future perspectives, Materials and methods)
|
||||
- A column boundary
|
||||
- `reference_zone.start`
|
||||
|
||||
### Contract C — New path skips old tail promotion
|
||||
|
||||
When the ref-partitioned path is active (`ref_partition_active = True`):
|
||||
- `_promote_tail_body_candidates()` must be SKIPPED
|
||||
- `_assign_tail_spread_ownership()` must be SKIPPED (or limited to only post-ref blocks)
|
||||
- Only the partition's own role normalization may assign pre-ref tail roles
|
||||
|
||||
### Contract D — Derive body_end_page from actual body blocks
|
||||
|
||||
`body_end_page` must be the max page of blocks classified as pre-ref body_flow,
|
||||
NOT a mechanical `ref_start_page - 1`. Same-page body+ref is valid.
|
||||
|
||||
### Implementation placement — CRITICAL: replace the LATER `_reconcile_tail_spread()` call
|
||||
|
||||
Do NOT insert the ref-partition path at the first `infer_zones()` call near the
|
||||
top of `normalize_document_structure()`. The current pipeline computes `region_bus`
|
||||
early, then resolves roles, recomputes `page_layouts`, and only THEN calls
|
||||
`_reconcile_tail_spread()`.
|
||||
|
||||
The new ref-partition path must replace that **later** `_reconcile_tail_spread()`
|
||||
decision point. At that point, recompute or refresh `region_bus`, compute `ref_zones`
|
||||
from the recomputed `page_layouts`, and then choose:
|
||||
|
||||
- Verified `reference_zone` → ref-partition path
|
||||
- No verified `reference_zone` → old `_reconcile_tail_spread()` fallback
|
||||
|
||||
---
|
||||
|
||||
## 1. Problem
|
||||
|
||||
### 1.1 Evidence (25K5KZAQ, NC66N4Q3)
|
||||
|
||||
**25K5KZAQ (Bioactive Materials, 11p):**
|
||||
- Pages 9-10: CRediT/Ethics headings at font 7.97pt bold → `subsection_heading` (not `backmatter_heading`)
|
||||
- "Declaration of competing interest" at y=829 → correctly `backmatter_heading` (contains "DECLARATION")
|
||||
- Result: CRediT text renders as body prose above Discussion, not as backmatter
|
||||
|
||||
**NC66N4Q3 (Radiographic Atlas, 56p):**
|
||||
- Page 56: two-column layout, left=Discussion, right=References
|
||||
- `body_end_page=None`, `ref_start=None` — boundary detection fails because body+ref share final page
|
||||
- `is_clean_separated = False` → tail spread logic enters degraded mode
|
||||
|
||||
### 1.2 Root cause
|
||||
|
||||
`normalize_document_structure()` computes `region_bus.reference_zone` early (first
|
||||
`infer_zones()` call), but LATER recomputes `tail_spread` through `_reconcile_tail_spread()`
|
||||
after role resolution and page-layout recomputation. The actual order is:
|
||||
|
||||
1. First `infer_zones()` → `region_bus.reference_zone` is set ✓
|
||||
2. Role resolution / family partition / page_layouts recomputation
|
||||
3. `_reconcile_tail_spread()` LATER OVERWRITES the boundary using old heuristic
|
||||
|
||||
The later heuristic becomes the authoritative `DocumentStructure` boundary and can
|
||||
contradict the already-known `reference_zone`, especially on same-page body+ref layouts.
|
||||
|
||||
On column-mixed pages (body+ref same page), the forward scan also fails because
|
||||
the left column has body without tail → `any_body_without_tail=True` → scan continues.
|
||||
|
||||
**Result:** `reference_zone` is already computed but never used to set the tail boundary.
|
||||
The heuristic (forward/backward) wins over data-driven reference_zone.
|
||||
|
||||
### 1.3 Scope of impact
|
||||
|
||||
| Paper | Effect | Frequency |
|
||||
|-------|--------|-----------|
|
||||
| 25K5KZAQ | CRediT/Ethics in body flow, not backmatter | ~30-50% of papers with CRediT |
|
||||
| Any with small-font disclosure headings | Disclosure → subsection_heading | Likely >10% |
|
||||
| Same-page body+ref | tail_spread degraded, ref_start=None | ≤5% |
|
||||
|
||||
---
|
||||
|
||||
## 2. What reference_zone data is already available
|
||||
|
||||
The current pipeline already computes reference_zone extent in `region_bus`:
|
||||
|
||||
| Data | Source | Current issue |
|
||||
|------|--------|---------------|
|
||||
| `reference_zone.block_ids` | `infer_zones()` | Trimmed list — blocks that belong to reference zone. Already excludes post-ref backmatter blocks. |
|
||||
| `reference_zone.boundary_band.start_page` | `infer_zones()` | First reference heading/item page. Reliable. |
|
||||
| `reference_zone.boundary_band.end_page` | `infer_zones()` | Uses raw `reference_item_blocks` max page — **NOT trimmed** if post-ref backmatter exists. Overstates. |
|
||||
| `region_bus` status (ACCEPT/HOLD) | `infer_zones()` | Whether reference zone is verified |
|
||||
| `ReferenceZone` per-page, per-column | `_detect_reference_zones()` | Column-scoped vertical bands. Stored in `doc.reference_zones` AFTER `infer_zones()`. |
|
||||
| `ref_heading_pages` dict | `_apply_content_zone_fallback()` | y_top of reference heading per page — used for same-page vertical split |
|
||||
|
||||
**Key insight:** `region_bus.reference_zone` is computed by `infer_zones()`, which
|
||||
runs INSIDE `normalize_document_structure()`. But `_reconcile_tail_spread()` runs
|
||||
BEFORE `infer_zones()` — so tail spread operates without the reference anchor,
|
||||
then `infer_zones()` re-derives reference zone independently but its output is
|
||||
never used to correct the tail boundary.
|
||||
|
||||
---
|
||||
|
||||
## 3. Algorithm
|
||||
|
||||
### 3.1 Overall flow
|
||||
|
||||
The new partition replaces the **later** `_reconcile_tail_spread()` call point,
|
||||
AFTER role resolution and page_layouts recomputation. Current approximate position
|
||||
in `normalize_document_structure()`:
|
||||
|
||||
```python
|
||||
# ... early: first infer_zones(), role resolution, page_layouts recomputation ...
|
||||
|
||||
# --- REPLACE THIS POINT ---
|
||||
# OLD:
|
||||
# tail_spread = _reconcile_tail_spread(blocks, page_layouts)
|
||||
# _normalize_backmatter_roles_after_boundary(tail_spread, backmatter_form, blocks)
|
||||
|
||||
# NEW:
|
||||
region_bus = infer_zones(blocks, ..., tail_spread=None) # refresh region_bus
|
||||
ref_zones = _detect_reference_zones(blocks, page_layouts)
|
||||
|
||||
if _has_verified_reference_zone(region_bus):
|
||||
partition = _partition_by_reference_zone(blocks, region_bus, ref_zones=ref_zones)
|
||||
_normalize_pre_ref_disclosure_runs(partition, region_bus)
|
||||
tail_spread = _build_tail_boundary_from_ref_partition(partition, region_bus)
|
||||
ref_partition_active = True
|
||||
else:
|
||||
tail_spread = _reconcile_tail_spread(blocks, page_layouts)
|
||||
_normalize_backmatter_roles_after_boundary(tail_spread, backmatter_form, blocks)
|
||||
ref_partition_active = False
|
||||
|
||||
# --- existing: zone labels (unchanged) ---
|
||||
_apply_zone_labels(blocks, region_bus) # includes _apply_content_zone_fallback internally
|
||||
_sanitize_reference_zone_boundary(blocks) # signature: only blocks
|
||||
|
||||
# --- CONDITIONAL (Contract C) ---
|
||||
if not ref_partition_active:
|
||||
blocks = _promote_tail_body_candidates(blocks, doc_structure, ...)
|
||||
blocks = _assign_tail_spread_ownership(blocks, doc_structure)
|
||||
# else: SKIP — old promotion would re-classify body_paragraph as tail_candidate_body
|
||||
# using spread_start/spread_end heuristics. The ref-anchored partition
|
||||
# has already assigned correct roles.
|
||||
```
|
||||
|
||||
### 3.2 Phase A — Determine reference_zone extent
|
||||
|
||||
```python
|
||||
def _resolve_reference_zone_extent(
|
||||
blocks: list[dict],
|
||||
region_bus: dict[str, dict],
|
||||
) -> tuple[int | None, int | None]:
|
||||
"""Return (ref_start_page, ref_end_page) from region_bus data.
|
||||
|
||||
Uses block_ids (which are already trimmed for post-ref backmatter).
|
||||
NEVER uses untrimmed boundary_band.end_page for the end.
|
||||
"""
|
||||
ref_zone = region_bus.get("reference_zone", {}) or {}
|
||||
block_ids: list[str] = ref_zone.get("block_ids") or []
|
||||
band = ref_zone.get("boundary_band") or {}
|
||||
ref_start = band.get("start_page")
|
||||
|
||||
if not block_ids or ref_start is None:
|
||||
return None, None
|
||||
|
||||
# Use effective_end_page stored during infer_zones (see §3.2a).
|
||||
# Do NOT fallback to band.get("end_page") — that is the untrimmed raw max.
|
||||
effective_end = ref_zone.get("effective_end_page")
|
||||
if effective_end is None:
|
||||
effective_end = _derive_effective_end_from_block_ids(blocks, block_ids)
|
||||
|
||||
return ref_start, effective_end
|
||||
|
||||
|
||||
def _derive_effective_end_from_block_ids(
|
||||
blocks: list[dict], block_ids: list[str]
|
||||
) -> int | None:
|
||||
"""Scan blocks for max page among reference_zone.block_ids.
|
||||
|
||||
block_ids may be bare block_id strings or `p{page}:{block_id}` artifact keys.
|
||||
"""
|
||||
id_set = set(block_ids)
|
||||
max_page = None
|
||||
for b in blocks:
|
||||
bid = str(b.get("block_id") or "")
|
||||
key = _zone_block_key(b)
|
||||
if bid in id_set or key in id_set:
|
||||
p = int(b.get("page", 0) or 0)
|
||||
if p > 0:
|
||||
max_page = max(max_page, p) if max_page is not None else p
|
||||
return max_page
|
||||
```
|
||||
|
||||
**§3.2a — Store effective_end_page in infer_zones()**
|
||||
|
||||
In `infer_zones()`, compute `effective_reference_end_page` from the **trimmed**
|
||||
`reference_blocks` list (NOT the raw `reference_item_blocks`). Store it inside
|
||||
`_make_zone()` for the `reference_zone` entry:
|
||||
|
||||
```python
|
||||
# Current: reference_end_page = max(page of ALL reference_item_blocks) ← overstates
|
||||
# Replace with:
|
||||
if post_ref_backmatter_start is not None:
|
||||
trimmed_pages = [
|
||||
int(b.get("page", 0) or 0) for b in reference_blocks
|
||||
if int(b.get("page", 0) or 0) > 0
|
||||
]
|
||||
effective_reference_end_page = max(trimmed_pages) if trimmed_pages else first_reference_page
|
||||
else:
|
||||
effective_reference_end_page = reference_end_page # same as current
|
||||
|
||||
# Then in _make_zone(): pass effective_end_page alongside boundary_band
|
||||
"reference_zone": _make_zone(
|
||||
status,
|
||||
block_ids,
|
||||
composite_block_ids=reference_composite_ids,
|
||||
anchor_family=...,
|
||||
boundary_band=_page_band(first_reference_page, reference_end_page),
|
||||
effective_end_page=effective_reference_end_page, # NEW
|
||||
),
|
||||
```
|
||||
|
||||
### 3.3 Phase B — Partition blocks by reference_zone
|
||||
|
||||
```python
|
||||
def _partition_by_reference_zone(
|
||||
blocks: list[dict],
|
||||
region_bus: dict[str, dict],
|
||||
ref_zones: list[ReferenceZone] | None = None,
|
||||
) -> dict[str, list[dict]]:
|
||||
"""Partition all blocks into pre-ref / reference / post-ref zones.
|
||||
|
||||
reference_zone is the ONLY boundary. Pre-ref backmatter headings
|
||||
do not participate in partition (they are normalized later in §3.4).
|
||||
|
||||
Returns:
|
||||
"pre_ref": blocks strictly before reference_zone start
|
||||
"reference": blocks within reference_zone extent
|
||||
"post_ref": blocks strictly after reference_zone end
|
||||
"""
|
||||
ref_start, ref_end = _resolve_reference_zone_extent(blocks, region_bus)
|
||||
if ref_start is None:
|
||||
return _fallback_tail_spread_partition(blocks, region_bus) # old path
|
||||
|
||||
pre_ref, reference, post_ref = [], [], []
|
||||
|
||||
for block in blocks:
|
||||
page = int(block.get("page", 0) or 0)
|
||||
role = str(block.get("role") or "")
|
||||
|
||||
if page < ref_start:
|
||||
pre_ref.append(block)
|
||||
elif page > ref_end:
|
||||
post_ref.append(block)
|
||||
else:
|
||||
# page is within [ref_start, ref_end] — same-page split needed
|
||||
zone = _classify_same_page_block(block, region_bus, page, ref_zones=ref_zones)
|
||||
if zone == "pre_ref":
|
||||
pre_ref.append(block)
|
||||
elif zone == "reference":
|
||||
reference.append(block)
|
||||
else:
|
||||
post_ref.append(block)
|
||||
|
||||
return {"pre_ref": pre_ref, "reference": reference, "post_ref": post_ref}
|
||||
```
|
||||
|
||||
### 3.4 Phase C — Normalize pre-ref disclosure runs (LOCAL ONLY)
|
||||
|
||||
Contract A+B: this function must NEVER change global boundaries.
|
||||
|
||||
```python
|
||||
_KNOWN_PRE_REFERENCE_DISCLOSURE_HEADINGS: dict[str, list[str]] = {
|
||||
"credit authorship contribution statement": [],
|
||||
"ethics approval and consent to participate": ["ethics statement", "ethical approval"],
|
||||
"declaration of competing interest": [],
|
||||
"competing interests": ["conflict of interest"],
|
||||
"data availability": ["data availability statement"],
|
||||
"acknowledgements": ["acknowledgments", "acknowledgement", "acknowledgment"],
|
||||
"author contributions": [],
|
||||
}
|
||||
|
||||
# NOT in the list: "funding", "supplementary", "appendix" — too generic,
|
||||
# risk of false positive on unnumbered body headings.
|
||||
|
||||
_STRONG_BODY_HEADINGS: frozenset[str] = frozenset({
|
||||
"discussion", "conclusion", "conclusions", "results", "summary",
|
||||
"limitations", "future perspectives", "materials and methods",
|
||||
"methods", "introduction",
|
||||
})
|
||||
|
||||
|
||||
_HEADING_ROLES = frozenset({
|
||||
"section_heading", "subsection_heading", "sub_subsection_heading",
|
||||
"backmatter_heading", "backmatter_boundary_heading", "backmatter_heading_candidate",
|
||||
})
|
||||
|
||||
_MARKER_HEADING_TYPES = frozenset({
|
||||
"heading_numbered", "heading_arabic", "heading_decimal",
|
||||
})
|
||||
|
||||
|
||||
def _normalize_pre_ref_disclosure_runs(
|
||||
blocks: list[dict],
|
||||
partition: dict[str, list[dict]],
|
||||
ref_start_page: int | None,
|
||||
total_pages: int,
|
||||
page_layouts: dict[int, PageLayoutProfile] | None = None,
|
||||
page_width: int = 0,
|
||||
) -> None:
|
||||
"""Local role normalization for known disclosure sections before ref.
|
||||
|
||||
Hard rules:
|
||||
1. Only modifies roles in the pre_ref partition.
|
||||
2. Does not change ref_start_page, body_end_page, spread_start, or any boundary field.
|
||||
3. Each disclosure heading only owns blocks from itself to the next heading
|
||||
(any heading, including another disclosure heading).
|
||||
4. Does not cross a STRONG_BODY_HEADING.
|
||||
5. Does not cross reference_zone.start.
|
||||
6. Does NOT cross a column boundary.
|
||||
7. Does not affect numbered body section headings.
|
||||
"""
|
||||
pre_ref_blocks = partition.get("pre_ref", [])
|
||||
if ref_start_page is None:
|
||||
return
|
||||
|
||||
for i, block in enumerate(pre_ref_blocks):
|
||||
text = str(block.get("text", "") or "").strip().lower()
|
||||
text_norm = re.sub(r"\s+", " ", text)
|
||||
|
||||
# Check exact match against known disclosure headings
|
||||
matched_key = None
|
||||
for key, aliases in _KNOWN_PRE_REFERENCE_DISCLOSURE_HEADINGS.items():
|
||||
if text_norm == key or text_norm in aliases:
|
||||
matched_key = key
|
||||
break
|
||||
if matched_key is None:
|
||||
continue
|
||||
|
||||
# Gate: heading must be heading-like
|
||||
role = str(block.get("role") or "")
|
||||
seed_role = str(block.get("seed_role") or "")
|
||||
raw_label = str(block.get("raw_label") or "")
|
||||
marker_type = str((block.get("marker_signature") or {}).get("type") or "")
|
||||
heading_like = (
|
||||
role in _HEADING_ROLES
|
||||
or seed_role in _HEADING_ROLES
|
||||
or raw_label == "paragraph_title"
|
||||
or marker_type in _MARKER_HEADING_TYPES
|
||||
)
|
||||
if not heading_like:
|
||||
# Not heading-like — could be a body paragraph matching disclosure text.
|
||||
# Only promote if bold + short.
|
||||
span = block.get("span_signature") or {}
|
||||
if not span.get("bold") and not span.get("font_size", 0) > 10:
|
||||
continue
|
||||
if len(text.split()) > 20:
|
||||
continue
|
||||
|
||||
# Gate: proximity to ref_start (not total_pages ratio)
|
||||
block_page = int(block.get("page", 0) or 0)
|
||||
if ref_start_page is not None:
|
||||
max_before_ref = max(2, int(total_pages * 0.25))
|
||||
if block_page > ref_start_page or block_page < max(1, ref_start_page - max_before_ref):
|
||||
continue
|
||||
|
||||
# Determine block's column once (for same-column guard below)
|
||||
block_col = _get_column_index(block, page_width) if page_width > 0 else None
|
||||
|
||||
# Normalize the local run: same column only, to next heading or ref_start
|
||||
block["role"] = "backmatter_heading"
|
||||
heading_like_count = 1 # track subsequent headings in the run
|
||||
for j in range(i + 1, len(pre_ref_blocks)):
|
||||
next_block = pre_ref_blocks[j]
|
||||
next_page = int(next_block.get("page", 0) or 0)
|
||||
if next_page > ref_start_page:
|
||||
break
|
||||
|
||||
# Column guard: do not cross column boundary
|
||||
if block_col is not None and page_width > 0:
|
||||
next_col = _get_column_index(next_block, page_width)
|
||||
if next_col is not None and next_col != block_col:
|
||||
break
|
||||
|
||||
next_text = str(next_block.get("text", "") or "").strip().lower()
|
||||
next_role = str(next_block.get("role") or "")
|
||||
next_raw_label = str(next_block.get("raw_label") or "")
|
||||
next_marker_type = str((next_block.get("marker_signature") or {}).get("type") or "")
|
||||
next_heading_like = (
|
||||
next_role in _HEADING_ROLES
|
||||
or next_raw_label == "paragraph_title"
|
||||
or next_marker_type in _MARKER_HEADING_TYPES
|
||||
)
|
||||
|
||||
if next_heading_like:
|
||||
# Another disclosure heading in sequence (e.g. CRediT → Ethics)
|
||||
if _is_known_disclosure(next_text):
|
||||
continue # will be handled by its own loop iteration
|
||||
else:
|
||||
break # non-disclosure heading → stop run
|
||||
|
||||
# Stop at strong body section start
|
||||
if any(next_text.startswith(h) for h in _STRONG_BODY_HEADINGS):
|
||||
break
|
||||
# Stop at numbered heading
|
||||
if re.match(r"^\d+(?:\.\d+)*\.?\s", next_text):
|
||||
break
|
||||
|
||||
# Normalize this block
|
||||
if next_role.endswith("_heading"):
|
||||
next_block["role"] = "backmatter_heading"
|
||||
elif next_role == "body_paragraph":
|
||||
next_block["role"] = "backmatter_body"
|
||||
elif next_role == "frontmatter_noise":
|
||||
next_block["role"] = "backmatter_body"
|
||||
|
||||
|
||||
def _is_known_disclosure(text: str) -> bool:
|
||||
text_norm = re.sub(r"\s+", " ", text).strip().lower()
|
||||
for key, aliases in _KNOWN_PRE_REFERENCE_DISCLOSURE_HEADINGS.items():
|
||||
if text_norm == key or text_norm in aliases:
|
||||
return True
|
||||
return False
|
||||
```
|
||||
|
||||
### 3.5 Phase D — Same-page body+ref column-aware split
|
||||
|
||||
```python
|
||||
def _classify_same_page_block(
|
||||
block: dict,
|
||||
region_bus: dict,
|
||||
page: int,
|
||||
ref_zones: list[ReferenceZone] | None = None,
|
||||
page_layouts: dict[int, PageLayoutProfile] | None = None,
|
||||
) -> str:
|
||||
"""Classify a block on a shared body/ref page.
|
||||
|
||||
A block is "reference" only if it explicitly belongs to the reference zone.
|
||||
Same column as a reference heading is NOT sufficient.
|
||||
|
||||
Priority:
|
||||
1. Block key in reference_zone.block_ids/composite_block_ids → "reference"
|
||||
2. role/seed_role is reference_heading/reference_item → "reference"
|
||||
3. Column-aware: below reference heading in ref column WITH reference evidence
|
||||
4. Vertical split by y (fallback)
|
||||
5. "pre_ref" (conservative — keeps body in body)
|
||||
"""
|
||||
role = str(block.get("role") or "")
|
||||
seed_role = str(block.get("seed_role") or "")
|
||||
if role in {"reference_heading", "reference_item"}:
|
||||
return "reference"
|
||||
if seed_role in {"reference_heading", "reference_item"}:
|
||||
return "reference"
|
||||
|
||||
# Check if block is explicitly in reference_zone block_ids
|
||||
ref_zone = region_bus.get("reference_zone", {}) or {}
|
||||
ref_block_ids = set(ref_zone.get("block_ids") or [])
|
||||
ref_composite_ids = set(ref_zone.get("composite_block_ids") or [])
|
||||
block_key = str(block.get("block_id") or "")
|
||||
if block_key in ref_block_ids or block_key in ref_composite_ids:
|
||||
return "reference"
|
||||
art_key = _zone_block_key(block)
|
||||
if art_key in ref_block_ids or art_key in ref_composite_ids:
|
||||
return "reference"
|
||||
|
||||
# Column-aware split: below ref heading in a ref column
|
||||
if ref_zones and page_layouts:
|
||||
block_x = _block_x_center(block)
|
||||
page_zones = [rz for rz in ref_zones if rz.page == page]
|
||||
layout = page_layouts.get(page)
|
||||
if layout:
|
||||
col_boundaries = layout.column_boundaries
|
||||
for rz in page_zones:
|
||||
col_x_range = _column_x_range(rz.column_index, col_boundaries, page_width)
|
||||
if col_x_range and col_x_range[0] <= block_x <= col_x_range[1]:
|
||||
# Same reference column — check y and evidence
|
||||
if _block_y_top(block) >= rz.y_start:
|
||||
# Below ref heading in ref column — check for reference evidence
|
||||
marker = (block.get("marker_signature") or {}).get("type")
|
||||
if marker in {"reference_bracket", "reference_sup", "reference_numeric"}:
|
||||
return "reference"
|
||||
return "post_ref" # in ref column but not a ref → post-ref backmatter
|
||||
return "pre_ref" # above ref heading in same column → body flow
|
||||
return "pre_ref" # in a different column → body flow
|
||||
|
||||
# Fallback: vertical split using ref_heading_pages
|
||||
ref_heading_pages = _compute_ref_heading_pages(region_bus)
|
||||
if page in ref_heading_pages:
|
||||
ref_top = ref_heading_pages[page]
|
||||
if _block_y_bottom(block) <= ref_top:
|
||||
return "pre_ref"
|
||||
return "reference"
|
||||
|
||||
return "pre_ref"
|
||||
```
|
||||
|
||||
### 3.6 Phase E — Build tail boundary from partition (Contract D)
|
||||
|
||||
```python
|
||||
def _build_tail_boundary_from_ref_partition(
|
||||
partition: dict[str, list[dict]],
|
||||
region_bus: dict[str, dict],
|
||||
) -> TailBoundary:
|
||||
"""Derive TailBoundary from the ref-anchored partition.
|
||||
|
||||
body_end_page = max page of pre_ref blocks that are body_flow.
|
||||
spread_start = references_start (NOT body_end_page — body_end is body metadata,
|
||||
not tail-spread start).
|
||||
"""
|
||||
pre_ref = partition.get("pre_ref", [])
|
||||
reference = partition.get("reference", [])
|
||||
post_ref = partition.get("post_ref", [])
|
||||
|
||||
# body_end_page: max page of body-like blocks in pre_ref
|
||||
body_like_pages = [
|
||||
int(b.get("page", 0) or 0) for b in pre_ref
|
||||
if b.get("role") in {"body_paragraph", "section_heading",
|
||||
"subsection_heading", "sub_subsection_heading"}
|
||||
]
|
||||
body_end_page = max(body_like_pages) if body_like_pages else None
|
||||
|
||||
# ref_start_page from region_bus
|
||||
ref_zone = region_bus.get("reference_zone", {}) or {}
|
||||
band = ref_zone.get("boundary_band") or {}
|
||||
ref_start_page = band.get("start_page")
|
||||
|
||||
ref_pages = [int(b.get("page", 0) or 0) for b in reference if b.get("page")]
|
||||
references_start = min(ref_pages) if ref_pages else ref_start_page
|
||||
references_end = max(ref_pages) if ref_pages else references_start
|
||||
|
||||
post_ref_pages = [int(b.get("page", 0) or 0) for b in post_ref if b.get("page")]
|
||||
|
||||
# spread_start = references_start (NOT body_end_page)
|
||||
# spread_end = max of reference + post-ref pages
|
||||
all_tail_pages = ref_pages + post_ref_pages
|
||||
spread_start = min(all_tail_pages) if all_tail_pages else references_start
|
||||
spread_end = max(all_tail_pages) if all_tail_pages else references_start
|
||||
|
||||
return TailBoundary(
|
||||
body_end_page=body_end_page,
|
||||
backmatter_start=references_start,
|
||||
references_start=references_start,
|
||||
spread_start=spread_start,
|
||||
spread_end=spread_end,
|
||||
is_clean_separated=True,
|
||||
reason="ref_zone_partition",
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Call site and affected code
|
||||
|
||||
### 4.1 normalize_document_structure() changes
|
||||
|
||||
```python
|
||||
def normalize_document_structure(blocks, ...):
|
||||
# ... existing preamble (role assignment, family partition, etc.) ...
|
||||
|
||||
# --- existing ---
|
||||
page_layouts = _build_page_layout_profiles(blocks)
|
||||
|
||||
# --- existing: region bus (infer_zones) ---
|
||||
region_bus = infer_zones(blocks, ..., tail_spread=tail_spread, ...)
|
||||
|
||||
# --- NEW: detect per-column reference zones (for column-aware same-page split) ---
|
||||
ref_zones = _detect_reference_zones(blocks, page_layouts)
|
||||
|
||||
# --- NEW: ref-anchored partition (replaces tail_spread when ref zone verified) ---
|
||||
ref_partition_active = False
|
||||
if _has_verified_reference_zone(region_bus):
|
||||
partition = _partition_by_reference_zone(blocks, region_bus, ref_zones=ref_zones)
|
||||
_normalize_reference_roles_from_partition(partition)
|
||||
_normalize_pre_ref_disclosure_runs(partition, region_bus, ref_start_page, total_pages)
|
||||
tail_spread = _build_tail_boundary_from_ref_partition(partition, region_bus)
|
||||
ref_partition_active = True
|
||||
else:
|
||||
# --- existing fallback ---
|
||||
tail_spread = _reconcile_tail_spread(blocks, page_layouts)
|
||||
_normalize_backmatter_roles_after_boundary(tail_spread, backmatter_form, blocks)
|
||||
ref_partition_active = False
|
||||
|
||||
# --- existing: zone labels (unchanged) ---
|
||||
_apply_zone_labels(blocks, region_bus) # includes _apply_content_zone_fallback internally
|
||||
_sanitize_reference_zone_boundary(blocks) # signature: blocks only
|
||||
|
||||
# --- CONDITIONAL: skip old promotion in ref-partitioned path (Contract C) ---
|
||||
if not ref_partition_active:
|
||||
blocks = _promote_tail_body_candidates(blocks, doc_structure, ...)
|
||||
blocks = _assign_tail_spread_ownership(blocks, doc_structure)
|
||||
|
||||
# ... rest of normalize_document_structure ...
|
||||
```
|
||||
|
||||
### 4.2 infer_zones() additions
|
||||
|
||||
Store `effective_end_page` after trimming (lines 1067–1081):
|
||||
```python
|
||||
if post_ref_backmatter_start is not None:
|
||||
trimmed_pages = [int(b.get("page", 0) or 0) for b in reference_blocks]
|
||||
effective_end_page = max(trimmed_pages) if trimmed_pages else None
|
||||
else:
|
||||
effective_end_page = reference_end_page
|
||||
ref_zone["effective_end_page"] = effective_end_page
|
||||
```
|
||||
|
||||
### 4.3 Affected files
|
||||
|
||||
| File | Change | Scope |
|
||||
|------|--------|-------|
|
||||
| `ocr_document.py` | New `_partition_by_reference_zone()` + helpers | ~80 lines |
|
||||
| `ocr_document.py` | New `_normalize_pre_ref_disclosure_runs()` | ~70 lines |
|
||||
| `ocr_document.py` | New `_classify_same_page_block()` | ~40 lines |
|
||||
| `ocr_document.py` | New `_build_tail_boundary_from_ref_partition()` | ~40 lines |
|
||||
| `ocr_document.py` | New `_has_verified_reference_zone()` | ~10 lines |
|
||||
| `ocr_document.py` | Modify `infer_zones()` to store `effective_end_page` | ~5 lines |
|
||||
| `ocr_document.py` | Modify `normalize_document_structure()` | ~30 lines |
|
||||
| `ocr_document.py` | `_detect_reference_zones()` — move before partition call | import only |
|
||||
| `ocr_document.py` | `_normalize_backmatter_roles_after_boundary()` — unchanged, kept as fallback | 0 |
|
||||
| `ocr_document.py` | `_promote_tail_body_candidates()` — SKIPPED in ref-partitioned path | 0 |
|
||||
|
||||
---
|
||||
|
||||
## 5. Test strategy
|
||||
|
||||
### 5.1 Test papers
|
||||
|
||||
| Paper | Pages | Layout | Issue | Expected after fix |
|
||||
|-------|-------|--------|-------|-------------------|
|
||||
| 25K5KZAQ | 11 | two-col | CRediT/Ethics/Declaration → body_paragraph | CRediT → backmatter_heading, text → backmatter_body. Discussion/Conclusion unchanged. |
|
||||
| NC66N4Q3 | 56 | two-col | body_end_page=None, ref_start=None | ref items → reference_zone, Discussion → body_flow, no degraded mode |
|
||||
| 9TW98JH8 | 5 | mixed-tail | 20/20 correct | No change (regression) |
|
||||
| Standard paper | any | single-col | Clean body/ref separation | No change |
|
||||
|
||||
### 5.2 Unit tests
|
||||
|
||||
| Test | Verifies |
|
||||
|------|----------|
|
||||
| `test_ref_zone_partition_simple` | Single-col body p1-8, ref p9-10, post-ref p11 → correct 3-way split |
|
||||
| `test_ref_zone_partition_credit` | 25K5KZAQ: CRediT before ref → backmatter_heading, body → backmatter_body; Discussion after CRediT → unchanged |
|
||||
| `test_pre_ref_disclosure_does_not_change_ref_start` | CRediT heading found → ref_start_page unchanged (Contract A) |
|
||||
| `test_pre_ref_disclosure_does_not_span_strong_body_heading` | "CRediT ... Conclusion ... References" → Conclusion NOT consumed (Contract B) |
|
||||
| `test_pre_ref_disclosure_does_not_span_numbered_heading` | "CRediT ... 5. Conclusion ... References" → 5. Conclusion NOT consumed |
|
||||
| `test_pre_ref_disclosure_stops_at_ref_start` | Blocks after candidate, before ref_start, past another heading → NOT consumed |
|
||||
| `test_ref_zone_partition_same_page_body_ref` | NC66N4Q3: column-aware split, left-col body stays body_flow |
|
||||
| `test_ref_zone_partition_no_ref` | No reference_zone → fallback to old algorithm, no crash |
|
||||
| `test_known_disclosure_exact_match` | Exact match triggers, substring "funding" in body heading doesn't |
|
||||
| `test_disclosure_proximity_gate` | Candidate at page 2 of 15, ref at page 12 → NOT triggered (too far from ref) |
|
||||
| `test_body_end_page_from_actual_blocks` | Same-page body+ref → body_end_page == ref_start_page, not ref_start - 1 |
|
||||
| `test_tail_boundary_derived_from_partition` | body_end_page = max pre-ref body_flow page |
|
||||
| `test_promote_tail_body_skipped_on_ref_path` | ref_partition_active=True → _promote_tail_body_candidates not called |
|
||||
| `test_reference_end_page_trimming` | Post-ref backmatter on page 12 doesn't extend reference_zone |
|
||||
| `test_infer_zones_stores_effective_end_page` | After trim, effective_end_page < raw end_page |
|
||||
|
||||
### 5.3 Regression tests
|
||||
|
||||
All existing tests pass unchanged:
|
||||
- `test_ocr_layout_zones.py` — single-col, two-col, column-major ordering, region_bus output
|
||||
- `test_pipeline_keeps_reference_zone_and_legend_family_out_of_default_body`
|
||||
- `test_reference_zone_adapter_accepts_without_tail_end_boundary`
|
||||
- Full suite: `python -m pytest tests/unit/ tests/cli/ -v --tb=short`
|
||||
|
||||
---
|
||||
|
||||
## 6. Risks and unknowns
|
||||
|
||||
### 6.1 `_detect_reference_zones()` availability
|
||||
|
||||
Currently `_detect_reference_zones()` is called inside `infer_zones()` (for `references_start` detection) but the result (`ReferenceZone` list) is stored in `DocumentStructure.reference_zones` AFTER `infer_zones()` returns. The partition runs inside `normalize_document_structure()`, which has access to `doc_structure` mid-construction. Verify that `doc_structure.reference_zones` is populated before the partition call.
|
||||
|
||||
**Mitigation:** Move the `_detect_reference_zones()` call to just before the partition, not relying on `infer_zones()` to have stored it.
|
||||
|
||||
### 6.2 Verified reference_zone ≠ perfect reference_zone
|
||||
|
||||
`_has_verified_reference_zone()` must check for actual block_ids, not just ACCEPT status. A zone can be ACCEPT with zero block_ids (if the anchor family is accepted but no items are in range). In that case, fall back to old tail_spread logic.
|
||||
|
||||
```python
|
||||
def _has_verified_reference_zone(region_bus):
|
||||
ref_zone = region_bus.get("reference_zone", {}) or {}
|
||||
if ref_zone.get("status") in ("ACCEPT", "HOLD"):
|
||||
block_ids = ref_zone.get("block_ids") or []
|
||||
if block_ids:
|
||||
return True
|
||||
return False
|
||||
```
|
||||
|
||||
### 6.3 Downstream zone assignment
|
||||
|
||||
`_apply_zone_labels()` and `_apply_content_zone_fallback()` run AFTER the partition and role normalization. Since zone labels depend on roles, and roles have been updated by the partition, zones should be assigned correctly on first pass. No stale behavior expected — but verify on 25K5KZAQ that CRediT blocks get `tail_nonref_hold_zone` or `body_zone` consistently with their new `backmatter_body` role.
|
||||
|
||||
### 6.4 Renderer compatibility
|
||||
|
||||
`_reorder_tail_run()` in `ocr_render.py` reads `tail_spread` from document structure. Since the new `_build_tail_boundary_from_ref_partition()` produces a compatible `TailBoundary`, the renderer should work unchanged. Verify with 25K5KZAQ that CRediT text renders as backmatter, not between Discussion paragraphs.
|
||||
|
||||
### 6.5 Body spine contamination
|
||||
|
||||
`_build_page_layout_profiles()` excludes pages with >50% media from body spine sampling. This is unchanged. The new path does not modify body spine selection.
|
||||
|
||||
### 6.6 Papers with post-ref backmatter only (no pre-ref)
|
||||
|
||||
Papers where all backmatter (Acknowledgements, Funding, etc.) appears after the reference section are handled by the existing `post_reference_backmatter_zone` in `infer_zones()`. No change needed for these — they are not affected by the new partition.
|
||||
|
||||
---
|
||||
|
||||
## 7. Implementation order
|
||||
|
||||
```
|
||||
Step 1: Modify infer_zones() — store effective_end_page in reference_zone entry
|
||||
Step 2: Add _has_verified_reference_zone() helper
|
||||
Step 3: Add _KNOWN_PRE_REFERENCE_DISCLOSURE_HEADINGS and _STRONG_BODY_HEADINGS constants
|
||||
Step 4: Add _resolve_reference_zone_extent()
|
||||
Step 5: Add _classify_same_page_block() using ref_zones column data
|
||||
Step 6: Add _normalize_pre_ref_disclosure_runs() with local-only scope
|
||||
Step 7: Add _partition_by_reference_zone()
|
||||
Step 8: Add _build_tail_boundary_from_ref_partition()
|
||||
Step 9: Wire into normalize_document_structure() with ref_partition_active gate
|
||||
Step 10: Ensure _promote_tail_body_candidates is skipped in ref-partitioned path
|
||||
Step 11: Tests — 15 unit tests + 25K5KZAQ/NC66N4Q3/9TW98JH8 regression
|
||||
```
|
||||
|
||||
**Estimated scope:** ~280 lines new code, all in `ocr_document.py`
|
||||
**Dependencies:** None on P0/P1
|
||||
|
|
@ -548,13 +548,13 @@ def _block_y_bottom(block: dict) -> float:
|
|||
return bbox[3] if bbox else 0.0
|
||||
|
||||
|
||||
_REFERENCE_ZONE_MARKER_TYPES = {
|
||||
_REFERENCE_ZONE_MARKER_TYPES: frozenset[str] = frozenset({
|
||||
"reference_numeric_bracket",
|
||||
"reference_numeric_dot",
|
||||
"reference_numeric_parenthesis",
|
||||
"reference_pattern",
|
||||
"citation_line",
|
||||
}
|
||||
})
|
||||
|
||||
_REFERENCE_ZONE_HEADING_TEXTS = {"references", "bibliography"}
|
||||
|
||||
|
|
@ -567,6 +567,31 @@ _TAIL_NONREF_HEADING_DENY_TYPES = {
|
|||
"preproof_marker",
|
||||
}
|
||||
|
||||
_KNOWN_PRE_REFERENCE_DISCLOSURE_HEADINGS: dict[str, list[str]] = {
|
||||
"credit authorship contribution statement": [],
|
||||
"ethics approval and consent to participate": ["ethics statement", "ethical approval"],
|
||||
"declaration of competing interest": [],
|
||||
"competing interests": ["conflict of interest"],
|
||||
"data availability": ["data availability statement"],
|
||||
"acknowledgements": ["acknowledgments", "acknowledgement", "acknowledgment"],
|
||||
"author contributions": [],
|
||||
}
|
||||
|
||||
_STRONG_BODY_HEADINGS: frozenset[str] = frozenset({
|
||||
"discussion", "conclusion", "conclusions", "results", "summary",
|
||||
"limitations", "future perspectives", "materials and methods",
|
||||
"methods", "introduction",
|
||||
})
|
||||
|
||||
_HEADING_ROLES: frozenset[str] = frozenset({
|
||||
"section_heading", "subsection_heading", "sub_subsection_heading",
|
||||
"backmatter_heading", "backmatter_boundary_heading", "backmatter_heading_candidate",
|
||||
})
|
||||
|
||||
_MARKER_HEADING_TYPES: frozenset[str] = frozenset({
|
||||
"heading_numbered", "heading_arabic", "heading_decimal",
|
||||
})
|
||||
|
||||
|
||||
def _make_zone(
|
||||
status: str,
|
||||
|
|
@ -772,6 +797,362 @@ def _zone_block_key(block: dict) -> str:
|
|||
return f"p{page}:{block_id}"
|
||||
|
||||
|
||||
def _has_verified_reference_zone(region_bus: dict) -> bool:
|
||||
ref_zone = region_bus.get("reference_zone") or {}
|
||||
status = ref_zone.get("status")
|
||||
block_ids = ref_zone.get("block_ids") or []
|
||||
return status in ("ACCEPT", "HOLD") and bool(block_ids)
|
||||
|
||||
|
||||
def _block_x_center(block: dict) -> float:
|
||||
bbox = _block_bbox(block)
|
||||
if bbox:
|
||||
return (bbox[0] + bbox[2]) / 2.0
|
||||
return 0.0
|
||||
|
||||
|
||||
def _column_x_range(column_index: int, col_boundaries: list[float], page_width: float = 1200.0) -> tuple[float, float]:
|
||||
if not col_boundaries or column_index < 0 or column_index >= len(col_boundaries) - 1:
|
||||
return 0.0, page_width
|
||||
return col_boundaries[column_index], col_boundaries[column_index + 1]
|
||||
|
||||
|
||||
def _get_column_index(block: dict, col_boundaries: list[float]) -> int | None:
|
||||
if not col_boundaries or len(col_boundaries) <= 1:
|
||||
return None
|
||||
x_center = _block_x_center(block)
|
||||
for col in range(len(col_boundaries) - 1):
|
||||
if col_boundaries[col] <= x_center <= col_boundaries[col + 1]:
|
||||
return col
|
||||
return None
|
||||
|
||||
|
||||
def _derive_effective_end_from_block_ids(blocks: list[dict], block_ids: list[str]) -> int | None:
|
||||
max_page = None
|
||||
zone_keys = set(block_ids)
|
||||
for b in blocks:
|
||||
in_zone = _zone_block_key(b) in zone_keys or str(b.get("block_id")) in zone_keys
|
||||
if in_zone and (int(b.get("page", 0) or 0) > 0):
|
||||
p = int(b.get("page", 0) or 0)
|
||||
if max_page is None or p > max_page:
|
||||
max_page = p
|
||||
return max_page
|
||||
|
||||
|
||||
def _resolve_reference_zone_extent(blocks: list[dict], region_bus: dict) -> tuple[int | None, int | None]:
|
||||
ref_zone = region_bus.get("reference_zone") or {}
|
||||
boundary_band = ref_zone.get("boundary_band") or {}
|
||||
start_page = boundary_band.get("start_page")
|
||||
effective_end = ref_zone.get("effective_end_page")
|
||||
if effective_end is None:
|
||||
effective_end = _derive_effective_end_from_block_ids(blocks, ref_zone.get("block_ids") or [])
|
||||
return start_page, effective_end
|
||||
|
||||
|
||||
def _classify_same_page_block(
|
||||
block: dict,
|
||||
region_bus: dict,
|
||||
page: int,
|
||||
ref_zones: list[ReferenceZone] | None = None,
|
||||
page_layouts: dict[int, PageLayoutProfile] | None = None,
|
||||
) -> str:
|
||||
role = block.get("role") or ""
|
||||
seed_role = block.get("seed_role") or ""
|
||||
if role in {"reference_heading", "reference_item"} or seed_role in {"reference_heading", "reference_item"}:
|
||||
return "reference"
|
||||
|
||||
ref_zone = region_bus.get("reference_zone") or {}
|
||||
block_id = str(block.get("block_id") or "")
|
||||
artifact_key = _zone_block_key(block)
|
||||
if block_id in (ref_zone.get("block_ids") or []) or artifact_key in (ref_zone.get("composite_block_ids") or []):
|
||||
return "reference"
|
||||
|
||||
if ref_zones:
|
||||
for z in ref_zones:
|
||||
if z.page != page:
|
||||
continue
|
||||
col_x_start, col_x_end = _column_x_range(z.column_index, _col_boundaries_for_page(page, page_layouts))
|
||||
x_center = _block_x_center(block)
|
||||
if not (col_x_start <= x_center <= col_x_end):
|
||||
continue
|
||||
if _block_y_top(block) < z.y_start:
|
||||
continue
|
||||
marker_type = (block.get("marker_signature") or {}).get("type") or ""
|
||||
if marker_type in _REFERENCE_ZONE_MARKER_TYPES:
|
||||
return "reference"
|
||||
return "pre_ref"
|
||||
|
||||
return "pre_ref"
|
||||
|
||||
|
||||
def _col_boundaries_for_page(page: int, page_layouts: dict[int, PageLayoutProfile] | None) -> list[float]:
|
||||
if page_layouts and page in page_layouts:
|
||||
return page_layouts[page].column_boundaries
|
||||
return []
|
||||
|
||||
|
||||
def _partition_by_reference_zone(
|
||||
blocks: list[dict],
|
||||
region_bus: dict,
|
||||
ref_zones: list[ReferenceZone] | None = None,
|
||||
page_layouts: dict[int, PageLayoutProfile] | None = None,
|
||||
) -> dict:
|
||||
if not _has_verified_reference_zone(region_bus):
|
||||
return {"fallback": True}
|
||||
|
||||
ref_start_page, ref_end_page = _resolve_reference_zone_extent(blocks, region_bus)
|
||||
if ref_start_page is None:
|
||||
return {"fallback": True}
|
||||
|
||||
ref_zone = region_bus.get("reference_zone") or {}
|
||||
ref_block_ids = set(str(bid) for bid in (ref_zone.get("block_ids") or []))
|
||||
ref_composite_ids = set(ref_zone.get("composite_block_ids") or [])
|
||||
|
||||
pre_ref: list[dict] = []
|
||||
reference: list[dict] = []
|
||||
post_ref: list[dict] = []
|
||||
|
||||
for block in blocks:
|
||||
page = int(block.get("page", 0) or 0)
|
||||
if page < ref_start_page:
|
||||
pre_ref.append(block)
|
||||
continue
|
||||
if ref_end_page is not None and page > ref_end_page:
|
||||
post_ref.append(block)
|
||||
continue
|
||||
if page == ref_start_page:
|
||||
# On the reference start page, classify each block
|
||||
classification = _classify_same_page_block(block, region_bus, page, ref_zones, page_layouts)
|
||||
if classification == "reference":
|
||||
reference.append(block)
|
||||
elif classification == "pre_ref":
|
||||
pre_ref.append(block)
|
||||
else:
|
||||
reference.append(block)
|
||||
continue
|
||||
# Pages strictly between start and end
|
||||
bid = str(block.get("block_id") or "")
|
||||
if bid in ref_block_ids or _zone_block_key(block) in ref_composite_ids:
|
||||
reference.append(block)
|
||||
continue
|
||||
role = block.get("role") or block.get("seed_role") or ""
|
||||
if role in {"reference_heading", "reference_item"}:
|
||||
reference.append(block)
|
||||
continue
|
||||
if ref_zones:
|
||||
in_any_zone = False
|
||||
for z in ref_zones:
|
||||
if z.page != page:
|
||||
continue
|
||||
bbox = _block_bbox(block)
|
||||
if not bbox:
|
||||
continue
|
||||
col_x_start, col_x_end = _column_x_range(z.column_index, _col_boundaries_for_page(page, page_layouts))
|
||||
x_center = _block_x_center(block)
|
||||
if not (col_x_start <= x_center <= col_x_end):
|
||||
continue
|
||||
if bbox[1] < z.y_start:
|
||||
in_any_zone = False
|
||||
break
|
||||
marker_type = (block.get("marker_signature") or {}).get("type") or ""
|
||||
if marker_type in _REFERENCE_ZONE_MARKER_TYPES:
|
||||
reference.append(block)
|
||||
in_any_zone = True
|
||||
break
|
||||
pre_ref.append(block)
|
||||
in_any_zone = True
|
||||
break
|
||||
if in_any_zone:
|
||||
continue
|
||||
pre_ref.append(block)
|
||||
|
||||
return {"pre_ref": pre_ref, "reference": reference, "post_ref": post_ref}
|
||||
|
||||
|
||||
def _normalize_reference_roles_from_partition(blocks: list[dict], partition: dict) -> None:
|
||||
ref_blocks = partition.get("reference", [])
|
||||
for block in ref_blocks:
|
||||
role = block.get("role") or ""
|
||||
seed_role = block.get("seed_role") or ""
|
||||
if role in {"reference_heading", "reference_item"}:
|
||||
continue
|
||||
if seed_role in {"reference_heading", "reference_item"}:
|
||||
block["role"] = seed_role
|
||||
continue
|
||||
marker_type = (block.get("marker_signature") or {}).get("type") or ""
|
||||
if marker_type in _REFERENCE_ZONE_MARKER_TYPES:
|
||||
old_role = block.get("role")
|
||||
block["role"] = "reference_item"
|
||||
if old_role != block["role"]:
|
||||
record_decision(
|
||||
block,
|
||||
stage="ref_partition_normalization",
|
||||
old_role=old_role,
|
||||
new_role="reference_item",
|
||||
reason="reference partition block normalized to reference_item",
|
||||
)
|
||||
elif seed_role in {"backmatter_heading_candidate", "backmatter_heading", "backmatter_boundary_heading"}:
|
||||
continue
|
||||
else:
|
||||
old_role = block.get("role")
|
||||
block["role"] = "reference_item"
|
||||
if old_role != block["role"]:
|
||||
record_decision(
|
||||
block,
|
||||
stage="ref_partition_normalization",
|
||||
old_role=old_role,
|
||||
new_role="reference_item",
|
||||
reason="reference partition block defaulted to reference_item",
|
||||
)
|
||||
|
||||
|
||||
def _is_known_disclosure(text: str) -> bool:
|
||||
if not text:
|
||||
return False
|
||||
lower = text.strip().lower()
|
||||
if lower in _KNOWN_PRE_REFERENCE_DISCLOSURE_HEADINGS:
|
||||
return True
|
||||
return any(lower in aliases for aliases in _KNOWN_PRE_REFERENCE_DISCLOSURE_HEADINGS.values())
|
||||
|
||||
|
||||
def _normalize_pre_ref_disclosure_runs(
|
||||
partition: dict,
|
||||
ref_start_page: int,
|
||||
total_pages: int,
|
||||
page_layouts: dict[int, PageLayoutProfile] | None = None,
|
||||
) -> None:
|
||||
pre_ref = partition.get("pre_ref", [])
|
||||
by_page: dict[int, list[dict]] = {}
|
||||
for b in pre_ref:
|
||||
p = int(b.get("page", 0) or 0)
|
||||
if p > 0:
|
||||
by_page.setdefault(p, []).append(b)
|
||||
|
||||
for page, page_blocks in by_page.items():
|
||||
if page >= ref_start_page:
|
||||
continue
|
||||
col_groups: dict[int, list[dict]] = {}
|
||||
for b in page_blocks:
|
||||
role = b.get("role") or b.get("seed_role") or ""
|
||||
if role in _STRONG_BODY_HEADINGS:
|
||||
continue
|
||||
marker_type = (b.get("marker_signature") or {}).get("type") or ""
|
||||
if marker_type in _MARKER_HEADING_TYPES:
|
||||
continue
|
||||
col_boundaries = _col_boundaries_for_page(page, page_layouts)
|
||||
col = _get_column_index(b, col_boundaries) or 0
|
||||
col_groups.setdefault(col, []).append(b)
|
||||
|
||||
for _col, col_blocks in col_groups.items():
|
||||
in_disclosure = False
|
||||
for b in col_blocks:
|
||||
text = str(b.get("text") or "").strip()
|
||||
role = b.get("role") or b.get("seed_role") or ""
|
||||
marker_type = (b.get("marker_signature") or {}).get("type") or ""
|
||||
|
||||
# Break at ANY next heading (including another disclosure heading)
|
||||
if role in _STRONG_BODY_HEADINGS or marker_type in _MARKER_HEADING_TYPES:
|
||||
in_disclosure = False
|
||||
continue
|
||||
if role in _HEADING_ROLES or _is_known_disclosure(text):
|
||||
in_disclosure = False
|
||||
|
||||
if _is_known_disclosure(text):
|
||||
in_disclosure = True
|
||||
if role not in _HEADING_ROLES:
|
||||
old_role = b.get("role")
|
||||
b["role"] = "backmatter_heading"
|
||||
if old_role != b["role"]:
|
||||
record_decision(
|
||||
b,
|
||||
stage="pre_ref_disclosure_heading",
|
||||
old_role=old_role,
|
||||
new_role="backmatter_heading",
|
||||
reason=f"known disclosure heading: {text[:50]}",
|
||||
)
|
||||
continue
|
||||
|
||||
if in_disclosure and role not in _HEADING_ROLES and role != "reference_item":
|
||||
old_role = b.get("role")
|
||||
b["role"] = "backmatter_body"
|
||||
if old_role != b["role"]:
|
||||
record_decision(
|
||||
b,
|
||||
stage="pre_ref_disclosure_body",
|
||||
old_role=old_role,
|
||||
new_role="backmatter_body",
|
||||
reason=f"body under disclosure heading: {text[:50]}",
|
||||
)
|
||||
|
||||
|
||||
def _build_tail_boundary_from_ref_partition(
|
||||
partition: dict,
|
||||
region_bus: dict,
|
||||
) -> TailBoundary | None:
|
||||
pre_ref = partition.get("pre_ref", [])
|
||||
reference = partition.get("reference", [])
|
||||
|
||||
if not reference:
|
||||
return None
|
||||
|
||||
body_end_page = 0
|
||||
for b in pre_ref:
|
||||
p = int(b.get("page", 0) or 0)
|
||||
if p > body_end_page:
|
||||
body_end_page = p
|
||||
|
||||
ref_start_page = None
|
||||
for b in reference:
|
||||
p = int(b.get("page", 0) or 0)
|
||||
if ref_start_page is None or p < ref_start_page:
|
||||
ref_start_page = p
|
||||
|
||||
ref_end_page = None
|
||||
for b in reference:
|
||||
p = int(b.get("page", 0) or 0)
|
||||
if ref_end_page is None or p > ref_end_page:
|
||||
ref_end_page = p
|
||||
|
||||
ref_zone = region_bus.get("reference_zone") or {}
|
||||
boundary_band = ref_zone.get("boundary_band") or {}
|
||||
band_ref_start = boundary_band.get("start_page")
|
||||
|
||||
references_start = band_ref_start or ref_start_page or 0
|
||||
spread_start = references_start
|
||||
spread_end = ref_end_page or references_start
|
||||
|
||||
# Find first backmatter heading in pre_ref for backmatter_start
|
||||
backmatter_start = None
|
||||
for b in pre_ref:
|
||||
role = b.get("role") or ""
|
||||
if role in {"backmatter_heading", "backmatter_boundary_heading", "backmatter_heading_candidate"}:
|
||||
p = int(b.get("page", 0) or 0)
|
||||
if backmatter_start is None or p < backmatter_start:
|
||||
backmatter_start = p
|
||||
if backmatter_start is None:
|
||||
backmatter_start = body_end_page if body_end_page > 0 else 0
|
||||
|
||||
return TailBoundary(
|
||||
body_end_page=body_end_page,
|
||||
backmatter_start=backmatter_start,
|
||||
references_start=references_start,
|
||||
spread_start=spread_start,
|
||||
spread_end=spread_end,
|
||||
is_clean_separated=True,
|
||||
reason="ref_zone_partition",
|
||||
)
|
||||
|
||||
|
||||
def _clear_partition_zones(blocks: list[dict]) -> None:
|
||||
stale_zones = {
|
||||
"body_zone", "reference_zone", "tail_nonref_hold_zone",
|
||||
"post_reference_backmatter_zone", "display_zone",
|
||||
}
|
||||
for block in blocks:
|
||||
if block.get("zone") in stale_zones:
|
||||
block["zone"] = ""
|
||||
|
||||
|
||||
def _canonical_block_id(block: dict) -> str | int | None:
|
||||
return block.get("block_id")
|
||||
|
||||
|
|
@ -1285,7 +1666,7 @@ def infer_zones(
|
|||
default=first_reference_page,
|
||||
)
|
||||
|
||||
return {
|
||||
result = {
|
||||
"frontmatter_main_zone": _make_zone(
|
||||
"ACCEPT" if frontmatter_main_ids else "HOLD",
|
||||
frontmatter_main_ids,
|
||||
|
|
@ -1349,6 +1730,22 @@ def infer_zones(
|
|||
),
|
||||
}
|
||||
|
||||
# Store effective_end_page on reference_zone for ref-partition consumers
|
||||
if post_ref_backmatter_start is not None:
|
||||
trimmed_pages = [
|
||||
int(b.get("page", 0) or 0) for b in reference_blocks
|
||||
if int(b.get("page", 0) or 0) > 0
|
||||
]
|
||||
effective_reference_end_page = max(trimmed_pages) if trimmed_pages else first_reference_page
|
||||
else:
|
||||
effective_reference_end_page = reference_end_page
|
||||
|
||||
ref_zone_entry = result.get("reference_zone")
|
||||
if ref_zone_entry and effective_reference_end_page is not None:
|
||||
ref_zone_entry["effective_end_page"] = effective_reference_end_page
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _classify_segment_hint(blocks: list[dict]) -> str:
|
||||
roles = {b.get("role") for b in blocks}
|
||||
|
|
@ -5114,13 +5511,48 @@ def normalize_document_structure(
|
|||
# are not layout-eligible, whereas resolved roles (reference_item, body_paragraph) are.
|
||||
page_layouts = _build_page_layout_profiles(blocks)
|
||||
|
||||
tail_spread = _reconcile_tail_spread(blocks, page_layouts)
|
||||
if tail_spread is not None:
|
||||
backmatter_form = _classify_backmatter_form(tail_spread, blocks)
|
||||
_label_backmatter_regime(tail_spread, backmatter_form, blocks)
|
||||
_normalize_backmatter_roles_after_boundary(tail_spread, backmatter_form, blocks)
|
||||
else:
|
||||
backmatter_form = "flat"
|
||||
# --- Ref-anchored partition (replaces second _reconcile_tail_spread) ---
|
||||
region_bus = infer_zones(
|
||||
blocks,
|
||||
{
|
||||
"body_family_anchor": body_family_anchor,
|
||||
"reference_family_anchor": reference_family_anchor,
|
||||
},
|
||||
tail_spread=None,
|
||||
)
|
||||
|
||||
ref_zones = _detect_reference_zones(blocks, page_layouts)
|
||||
|
||||
ref_partition_active = False
|
||||
if _has_verified_reference_zone(region_bus):
|
||||
partition = _partition_by_reference_zone(
|
||||
blocks, region_bus, ref_zones=ref_zones, page_layouts=page_layouts,
|
||||
)
|
||||
if "fallback" not in partition:
|
||||
_normalize_reference_roles_from_partition(blocks, partition)
|
||||
_normalize_pre_ref_disclosure_runs(
|
||||
partition,
|
||||
ref_start_page=_resolve_reference_zone_extent(blocks, region_bus)[0],
|
||||
total_pages=len({b.get("page") for b in blocks if b.get("page")}),
|
||||
page_layouts=page_layouts,
|
||||
)
|
||||
tail_spread = _build_tail_boundary_from_ref_partition(partition, region_bus)
|
||||
ref_partition_active = tail_spread is not None
|
||||
|
||||
if ref_partition_active:
|
||||
_clear_partition_zones(blocks)
|
||||
_apply_zone_labels(blocks, region_bus)
|
||||
_apply_content_zone_fallback(blocks, region_bus)
|
||||
_sanitize_reference_zone_boundary(blocks)
|
||||
|
||||
if not ref_partition_active:
|
||||
tail_spread = _reconcile_tail_spread(blocks, page_layouts)
|
||||
if tail_spread is not None:
|
||||
backmatter_form = _classify_backmatter_form(tail_spread, blocks)
|
||||
_label_backmatter_regime(tail_spread, backmatter_form, blocks)
|
||||
_normalize_backmatter_roles_after_boundary(tail_spread, backmatter_form, blocks)
|
||||
else:
|
||||
backmatter_form = "flat"
|
||||
|
||||
# Backmatter zone normalization: in post_reference_backmatter_zone,
|
||||
# headings become backmatter_heading, everything else becomes backmatter_body.
|
||||
|
|
@ -5225,8 +5657,11 @@ def normalize_document_structure(
|
|||
ref_zones = _detect_reference_zones(blocks, page_layouts)
|
||||
doc_structure.reference_zones = [dataclasses.asdict(z) for z in ref_zones] if ref_zones else None
|
||||
|
||||
blocks = _promote_tail_body_candidates(blocks, doc_structure, header_band=header_band, footer_band=footer_band)
|
||||
blocks = _assign_tail_spread_ownership(blocks, doc_structure)
|
||||
# In ref-partitioned path, role normalization already handled in partition;
|
||||
# skip old promotion to avoid double-normalizing reference blocks.
|
||||
if not ref_partition_active:
|
||||
blocks = _promote_tail_body_candidates(blocks, doc_structure, header_band=header_band, footer_band=footer_band)
|
||||
blocks = _assign_tail_spread_ownership(blocks, doc_structure)
|
||||
|
||||
# Compute span coverage for degraded mode detection
|
||||
doc_structure.span_coverage = _compute_span_coverage(blocks)
|
||||
|
|
|
|||
291
tests/unit/worker/test_backmatter_boundary.py
Normal file
291
tests/unit/worker/test_backmatter_boundary.py
Normal file
|
|
@ -0,0 +1,291 @@
|
|||
"""Tests for backmatter boundary redesign."""
|
||||
from __future__ import annotations
|
||||
|
||||
from paperforge.worker.ocr_document import (
|
||||
TailBoundary,
|
||||
_build_tail_boundary_from_ref_partition,
|
||||
_classify_same_page_block,
|
||||
_clear_partition_zones,
|
||||
_has_verified_reference_zone,
|
||||
_normalize_pre_ref_disclosure_runs,
|
||||
_normalize_reference_roles_from_partition,
|
||||
_partition_by_reference_zone,
|
||||
_resolve_reference_zone_extent,
|
||||
)
|
||||
|
||||
|
||||
class TestHasVerifiedReferenceZone:
|
||||
def test_has_verified_ref_zone_true(self):
|
||||
region_bus = {
|
||||
"reference_zone": {"status": "ACCEPT", "block_ids": ["b1"]}
|
||||
}
|
||||
assert _has_verified_reference_zone(region_bus)
|
||||
|
||||
def test_has_verified_ref_zone_empty_ids(self):
|
||||
region_bus = {
|
||||
"reference_zone": {"status": "ACCEPT", "block_ids": []}
|
||||
}
|
||||
assert not _has_verified_reference_zone(region_bus)
|
||||
|
||||
def test_has_verified_ref_zone_reject(self):
|
||||
region_bus = {
|
||||
"reference_zone": {"status": "REJECT", "block_ids": ["b1"]}
|
||||
}
|
||||
assert not _has_verified_reference_zone(region_bus)
|
||||
|
||||
|
||||
class TestResolveReferenceZoneExtent:
|
||||
def test_resolve_extent_simple(self):
|
||||
region_bus = {
|
||||
"reference_zone": {
|
||||
"boundary_band": {"start_page": 5},
|
||||
"effective_end_page": 10,
|
||||
}
|
||||
}
|
||||
start, end = _resolve_reference_zone_extent([], region_bus)
|
||||
assert start == 5
|
||||
assert end == 10
|
||||
|
||||
def test_resolve_extent_derives_from_block_ids(self):
|
||||
blocks = [
|
||||
{"block_id": "r1", "page": 1},
|
||||
{"block_id": "r2", "page": 2},
|
||||
{"block_id": "r3", "page": 3},
|
||||
]
|
||||
region_bus = {
|
||||
"reference_zone": {
|
||||
"boundary_band": {"start_page": 2},
|
||||
"block_ids": ["r1", "r2", "r3"],
|
||||
}
|
||||
}
|
||||
start, end = _resolve_reference_zone_extent(blocks, region_bus)
|
||||
assert start == 2
|
||||
assert end == 3
|
||||
|
||||
def test_resolve_extent_no_block_ids(self):
|
||||
region_bus = {
|
||||
"reference_zone": {
|
||||
"boundary_band": {"start_page": 5},
|
||||
"block_ids": [],
|
||||
}
|
||||
}
|
||||
start, end = _resolve_reference_zone_extent([], region_bus)
|
||||
assert start == 5
|
||||
assert end is None
|
||||
|
||||
|
||||
class TestClassifySamePageBlock:
|
||||
def test_classify_same_page_reference_role(self):
|
||||
block = {
|
||||
"role": "reference_heading",
|
||||
"block_id": "b1",
|
||||
"page": 1,
|
||||
"bbox": [0, 0, 100, 100],
|
||||
}
|
||||
region_bus = {"reference_zone": {"block_ids": []}}
|
||||
result = _classify_same_page_block(block, region_bus, page=1)
|
||||
assert result == "reference"
|
||||
|
||||
def test_classify_same_page_block_ids_membership(self):
|
||||
block = {
|
||||
"role": "body_paragraph",
|
||||
"block_id": "ref1",
|
||||
"page": 1,
|
||||
"bbox": [0, 0, 100, 100],
|
||||
}
|
||||
region_bus = {"reference_zone": {"block_ids": ["ref1"]}}
|
||||
result = _classify_same_page_block(block, region_bus, page=1)
|
||||
assert result == "reference"
|
||||
|
||||
def test_classify_same_page_conservative_fallback(self):
|
||||
block = {
|
||||
"role": "body_paragraph",
|
||||
"block_id": "b1",
|
||||
"page": 1,
|
||||
"bbox": [0, 0, 100, 100],
|
||||
}
|
||||
region_bus = {"reference_zone": {"block_ids": []}}
|
||||
result = _classify_same_page_block(block, region_bus, page=1)
|
||||
assert result == "pre_ref"
|
||||
|
||||
|
||||
class TestPartitionByReferenceZone:
|
||||
def test_partition_simple_3_way(self):
|
||||
blocks = []
|
||||
for i in range(1, 6):
|
||||
blocks.append({
|
||||
"block_id": f"b{i}", "page": i,
|
||||
"role": "body_paragraph",
|
||||
"bbox": [0, 0, 100, 100],
|
||||
})
|
||||
blocks.append({
|
||||
"block_id": "r1", "page": 6,
|
||||
"role": "reference_item",
|
||||
"bbox": [0, 0, 100, 100],
|
||||
})
|
||||
blocks.append({
|
||||
"block_id": "r2", "page": 7,
|
||||
"role": "reference_item",
|
||||
"bbox": [0, 0, 100, 100],
|
||||
})
|
||||
blocks.append({
|
||||
"block_id": "a1", "page": 8,
|
||||
"role": "body_paragraph",
|
||||
"bbox": [0, 0, 100, 100],
|
||||
})
|
||||
region_bus = {
|
||||
"reference_zone": {
|
||||
"status": "ACCEPT",
|
||||
"block_ids": ["r1", "r2"],
|
||||
"boundary_band": {"start_page": 6},
|
||||
"effective_end_page": 7,
|
||||
}
|
||||
}
|
||||
result = _partition_by_reference_zone(blocks, region_bus)
|
||||
assert len(result["pre_ref"]) == 5
|
||||
assert len(result["reference"]) == 2
|
||||
assert len(result["post_ref"]) == 1
|
||||
|
||||
def test_partition_fallback(self):
|
||||
region_bus = {
|
||||
"reference_zone": {"status": "REJECT", "block_ids": ["b1"]}
|
||||
}
|
||||
result = _partition_by_reference_zone([], region_bus)
|
||||
assert result == {"fallback": True}
|
||||
|
||||
|
||||
class TestNormalizeReferenceRoles:
|
||||
def test_ref_role_normalization(self):
|
||||
block = {"role": "body_paragraph", "block_id": "b1", "page": 6}
|
||||
_normalize_reference_roles_from_partition(
|
||||
[block], {"reference": [block]}
|
||||
)
|
||||
assert block["role"] == "reference_item"
|
||||
|
||||
def test_ref_role_normalization_skips_correct_roles(self):
|
||||
block = {"role": "reference_heading", "block_id": "b1"}
|
||||
_normalize_reference_roles_from_partition(
|
||||
[block], {"reference": [block]}
|
||||
)
|
||||
assert block["role"] == "reference_heading"
|
||||
|
||||
|
||||
class TestNormalizePreRefDisclosureRuns:
|
||||
def test_disclosure_exact_match(self):
|
||||
block = {
|
||||
"role": "body_paragraph", "page": 2, "block_id": "b1",
|
||||
"text": "credit authorship contribution statement",
|
||||
"bbox": [0, 0, 100, 100],
|
||||
}
|
||||
partition = {"pre_ref": [block]}
|
||||
_normalize_pre_ref_disclosure_runs(
|
||||
partition, ref_start_page=5, total_pages=10
|
||||
)
|
||||
assert block["role"] == "backmatter_heading"
|
||||
|
||||
def test_disclosure_no_substring_match(self):
|
||||
block = {
|
||||
"role": "body_paragraph", "page": 2, "block_id": "b1",
|
||||
"text": "funding mechanism",
|
||||
"bbox": [0, 0, 100, 100],
|
||||
}
|
||||
partition = {"pre_ref": [block]}
|
||||
_normalize_pre_ref_disclosure_runs(
|
||||
partition, ref_start_page=5, total_pages=10
|
||||
)
|
||||
assert block["role"] == "body_paragraph"
|
||||
|
||||
def test_disclosure_stops_at_next_heading(self):
|
||||
disclosure = {
|
||||
"role": "body_paragraph", "page": 2,
|
||||
"block_id": "b1",
|
||||
"text": "credit authorship contribution statement",
|
||||
"bbox": [0, 0, 100, 100],
|
||||
}
|
||||
heading = {
|
||||
"role": "section_heading", "page": 2,
|
||||
"block_id": "b2", "text": "something else",
|
||||
"bbox": [0, 0, 100, 100],
|
||||
}
|
||||
body = {
|
||||
"role": "body_paragraph", "page": 2,
|
||||
"block_id": "b3", "text": "after heading text",
|
||||
"bbox": [0, 0, 100, 100],
|
||||
}
|
||||
partition = {"pre_ref": [disclosure, heading, body]}
|
||||
_normalize_pre_ref_disclosure_runs(
|
||||
partition, ref_start_page=5, total_pages=10
|
||||
)
|
||||
assert disclosure["role"] == "backmatter_heading"
|
||||
assert body["role"] == "body_paragraph"
|
||||
|
||||
def test_disclosure_no_proximity_gate(self):
|
||||
block = {
|
||||
"role": "body_paragraph", "page": 2, "block_id": "b1",
|
||||
"text": "credit authorship contribution statement",
|
||||
"bbox": [0, 0, 100, 100],
|
||||
}
|
||||
partition = {"pre_ref": [block]}
|
||||
_normalize_pre_ref_disclosure_runs(
|
||||
partition, ref_start_page=12, total_pages=15
|
||||
)
|
||||
assert block["role"] == "backmatter_heading"
|
||||
|
||||
|
||||
class TestClearPartitionZones:
|
||||
def test_clear_partition_zones(self):
|
||||
blocks = [
|
||||
{"block_id": "b1", "zone": "reference_zone"},
|
||||
{"block_id": "b2", "zone": "body_zone"},
|
||||
{"block_id": "b3", "zone": "display_zone"},
|
||||
{"block_id": "b4", "zone": "other_zone"},
|
||||
]
|
||||
_clear_partition_zones(blocks)
|
||||
assert blocks[0]["zone"] == ""
|
||||
assert blocks[1]["zone"] == ""
|
||||
assert blocks[2]["zone"] == ""
|
||||
assert blocks[3]["zone"] == "other_zone"
|
||||
|
||||
|
||||
class TestBuildTailBoundary:
|
||||
def test_build_tail_boundary_from_partition(self):
|
||||
pre_ref = [
|
||||
{"block_id": "b1", "page": 1},
|
||||
{"block_id": "b2", "page": 2},
|
||||
]
|
||||
reference = [
|
||||
{"block_id": "r1", "page": 6},
|
||||
{"block_id": "r2", "page": 7},
|
||||
]
|
||||
partition = {
|
||||
"pre_ref": pre_ref,
|
||||
"reference": reference,
|
||||
"post_ref": [],
|
||||
}
|
||||
region_bus = {
|
||||
"reference_zone": {"boundary_band": {"start_page": 6}}
|
||||
}
|
||||
tb = _build_tail_boundary_from_ref_partition(partition, region_bus)
|
||||
assert tb.body_end_page == 2
|
||||
assert tb.backmatter_start == 2
|
||||
assert tb.references_start == 6
|
||||
assert tb.spread_start == 6
|
||||
assert tb.spread_end == 7
|
||||
assert tb.is_clean_separated
|
||||
assert tb.reason == "ref_zone_partition"
|
||||
|
||||
def test_tail_spread_start_is_references_start(self):
|
||||
pre_ref = [
|
||||
{"block_id": "b1", "page": 1},
|
||||
{"block_id": "b2", "page": 2},
|
||||
]
|
||||
reference = [{"block_id": "r1", "page": 5}]
|
||||
partition = {"pre_ref": pre_ref, "reference": reference}
|
||||
region_bus = {
|
||||
"reference_zone": {"boundary_band": {"start_page": 5}}
|
||||
}
|
||||
tb = _build_tail_boundary_from_ref_partition(partition, region_bus)
|
||||
assert tb.body_end_page == 2
|
||||
assert tb.references_start == 5
|
||||
assert tb.spread_start == tb.references_start
|
||||
assert tb.spread_start != tb.body_end_page
|
||||
Loading…
Reference in a new issue