mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
fix: reject weak isolated layout clusters
This commit is contained in:
Research Assistant
parent
ba68910cf3
commit
5bbe40d08b
5 changed files with 246 additions and 56 deletions
|
|
@ -17,7 +17,7 @@
|
|||
|
||||
- [ ] **Step 1: Insert `_cluster_page_column_groups()` before `_cluster_page_columns()`, then replace `_cluster_page_columns()` body with wrapper**
|
||||
|
||||
Open `paperforge/worker/ocr_document.py`. Insert the new function before `_cluster_page_columns()` (before line 321), then replace the body of `_cluster_page_columns()` (lines 321-350) with a one-line wrapper:
|
||||
Open `paperforge/worker/ocr_document.py`. Insert the new function before `_cluster_page_columns()` (currently at line 278), then replace the body of `_cluster_page_columns()` with a one-line wrapper:
|
||||
|
||||
```python
|
||||
def _cluster_page_column_groups(page_blocks: list[dict], page_width: float) -> list[dict]:
|
||||
|
|
@ -125,11 +125,11 @@ git commit -m "feat: add _cluster_page_column_groups() returning cluster metadat
|
|||
### Task 2: Add `_is_weak_isolated_column_cluster()` and modify `_classify_page_layout()`
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/worker/ocr_document.py` — insert helper, modify `_classify_page_layout()` (lines 353-418)
|
||||
- Modify: `paperforge/worker/ocr_document.py` — insert helper before `_classify_page_layout()`, replace `_classify_page_layout()` body
|
||||
|
||||
- [ ] **Step 1: Read current `_classify_page_layout()` to confirm boundaries**
|
||||
- [ ] **Step 1: Read current functions to confirm boundaries**
|
||||
|
||||
Read `paperforge/worker/ocr_document.py` lines 321-418 to confirm no changes since the spec was written.
|
||||
Read `paperforge/worker/ocr_document.py` from `_cluster_page_columns` through `_classify_page_layout` to confirm no conflicting changes since this plan was written.
|
||||
|
||||
- [ ] **Step 2: Insert `_is_weak_isolated_column_cluster()` before `_classify_page_layout()`**
|
||||
|
||||
|
|
@ -147,7 +147,7 @@ def _is_weak_isolated_column_cluster(cluster: dict, page_height: float) -> bool:
|
|||
return True
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Replace `_classify_page_layout()` (lines 353-418)**
|
||||
- [ ] **Step 3: Replace `_classify_page_layout()` body**
|
||||
|
||||
Replace the entire function with:
|
||||
|
||||
|
|
|
|||
|
|
@ -318,12 +318,14 @@ def _is_layout_eligible_block(block: dict) -> bool:
|
|||
return True
|
||||
|
||||
|
||||
def _cluster_page_columns(page_blocks: list[dict], page_width: float) -> list[float]:
|
||||
"""Cluster block x-centers by column using a gap-based approach.
|
||||
def _cluster_page_column_groups(page_blocks: list[dict], page_width: float) -> list[dict]:
|
||||
"""Cluster block x-centers by column and return per-cluster metadata.
|
||||
|
||||
Returns one representative x-center per column cluster.
|
||||
Returns list of cluster dicts sorted by center ascending. Each dict:
|
||||
center, count, center_min, center_max, y_min, y_max, y_coverage,
|
||||
word_count, width_median, block_ids, blocks
|
||||
"""
|
||||
centers: list[float] = []
|
||||
items: list[tuple[float, dict]] = []
|
||||
for block in page_blocks:
|
||||
bbox = block.get("bbox") or block.get("block_bbox")
|
||||
if not bbox or len(bbox) < 4:
|
||||
|
|
@ -332,47 +334,135 @@ def _cluster_page_columns(page_blocks: list[dict], page_width: float) -> list[fl
|
|||
if block_width <= 50:
|
||||
continue
|
||||
x_center = (bbox[0] + bbox[2]) / 2
|
||||
centers.append(x_center)
|
||||
items.append((x_center, block))
|
||||
|
||||
if not centers:
|
||||
return [page_width / 2]
|
||||
if not items:
|
||||
return [{
|
||||
"center": page_width / 2,
|
||||
"count": 0,
|
||||
"center_min": page_width / 2,
|
||||
"center_max": page_width / 2,
|
||||
"y_min": 0.0,
|
||||
"y_max": 0.0,
|
||||
"y_coverage": 0.0,
|
||||
"word_count": 0,
|
||||
"width_median": 0.0,
|
||||
"block_ids": [],
|
||||
"blocks": [],
|
||||
}]
|
||||
|
||||
centers.sort()
|
||||
items.sort(key=lambda x: x[0])
|
||||
gap_threshold = page_width * 0.15
|
||||
clusters: list[list[float]] = [[centers[0]]]
|
||||
raw_clusters: list[list[dict]] = [[items[0][1]]]
|
||||
last_center = items[0][0]
|
||||
|
||||
for c in centers[1:]:
|
||||
if c - clusters[-1][-1] > gap_threshold:
|
||||
clusters.append([c])
|
||||
for center, block in items[1:]:
|
||||
if center - last_center > gap_threshold:
|
||||
raw_clusters.append([block])
|
||||
else:
|
||||
clusters[-1].append(c)
|
||||
raw_clusters[-1].append(block)
|
||||
last_center = center
|
||||
|
||||
return [sum(cluster) / len(cluster) for cluster in clusters]
|
||||
result: list[dict] = []
|
||||
for cluster_blocks in raw_clusters:
|
||||
valid_bboxes = [
|
||||
bb for b in cluster_blocks
|
||||
for bb in [b.get("bbox") or b.get("block_bbox")]
|
||||
if bb and len(bb) >= 4
|
||||
]
|
||||
centers_for_compute = [(bb[0] + bb[2]) / 2 for bb in valid_bboxes]
|
||||
|
||||
widths = sorted(bb[2] - bb[0] for bb in valid_bboxes)
|
||||
|
||||
y_vals = [(bb[1], bb[3]) for bb in valid_bboxes]
|
||||
|
||||
center_val = sum(centers_for_compute) / len(centers_for_compute) if centers_for_compute else page_width / 2
|
||||
y_min_val = min(yv[0] for yv in y_vals) if y_vals else 0.0
|
||||
y_max_val = max(yv[1] for yv in y_vals) if y_vals else 0.0
|
||||
|
||||
result.append({
|
||||
"center": center_val,
|
||||
"count": len(cluster_blocks),
|
||||
"center_min": min(centers_for_compute) if centers_for_compute else page_width / 2,
|
||||
"center_max": max(centers_for_compute) if centers_for_compute else page_width / 2,
|
||||
"y_min": y_min_val,
|
||||
"y_max": y_max_val,
|
||||
"y_coverage": y_max_val - y_min_val,
|
||||
"word_count": sum(len(_block_text(b).split()) for b in cluster_blocks),
|
||||
"width_median": widths[len(widths) // 2] if widths else 0.0,
|
||||
"block_ids": [b.get("block_id", "") for b in cluster_blocks],
|
||||
"blocks": cluster_blocks,
|
||||
})
|
||||
|
||||
result.sort(key=lambda c: c["center"])
|
||||
return result
|
||||
|
||||
|
||||
# ponytail: Replace _cluster_page_columns() body with wrapper delegating to _cluster_page_column_groups()
|
||||
def _cluster_page_columns(page_blocks: list[dict], page_width: float) -> list[float]:
|
||||
return [c["center"] for c in _cluster_page_column_groups(page_blocks, page_width)]
|
||||
|
||||
|
||||
def _is_weak_isolated_column_cluster(cluster: dict, page_height: float) -> bool:
|
||||
"""Return True if this cluster has insufficient evidence to be a real column."""
|
||||
if cluster["count"] >= 3:
|
||||
return False
|
||||
if cluster["y_coverage"] >= page_height * 0.10:
|
||||
return False
|
||||
if cluster["word_count"] >= 25:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _classify_page_layout(page_blocks: list[dict], page_width: float, page_height: float) -> PageLayoutProfile:
|
||||
"""Classify a page's layout based on column clusters and role distribution."""
|
||||
centers = _cluster_page_columns(page_blocks, page_width)
|
||||
column_count = len(centers)
|
||||
clusters = _cluster_page_column_groups(page_blocks, page_width)
|
||||
|
||||
if column_count == 1:
|
||||
# ponytail: fallback empty cluster (count=0) should not be penalized
|
||||
if len(clusters) == 1 and clusters[0]["count"] == 0:
|
||||
return PageLayoutProfile(
|
||||
column_count=1,
|
||||
column_boundaries=centers,
|
||||
column_boundaries=[page_width / 2],
|
||||
layout_type="single_column",
|
||||
confidence=0.7,
|
||||
evidence=["eligible_body_blocks"],
|
||||
)
|
||||
|
||||
real_clusters = [
|
||||
c for c in clusters
|
||||
if not _is_weak_isolated_column_cluster(c, page_height)
|
||||
]
|
||||
|
||||
if not real_clusters:
|
||||
return PageLayoutProfile(
|
||||
column_count=1,
|
||||
column_boundaries=[page_width / 2],
|
||||
layout_type="single_column",
|
||||
confidence=0.35,
|
||||
evidence=["eligible_body_blocks", "all_column_clusters_weak"],
|
||||
)
|
||||
|
||||
evidence_extra: list[str] = []
|
||||
if len(real_clusters) < len(clusters):
|
||||
evidence_extra.append("weak_isolated_column_cluster_ignored")
|
||||
|
||||
centers = [c["center"] for c in real_clusters]
|
||||
column_count = len(real_clusters)
|
||||
|
||||
if column_count == 1:
|
||||
return PageLayoutProfile(
|
||||
column_count=1,
|
||||
column_boundaries=[real_clusters[0]["center"]],
|
||||
layout_type="single_column",
|
||||
confidence=0.55 if evidence_extra else 0.7,
|
||||
evidence=["eligible_body_blocks"] + evidence_extra,
|
||||
)
|
||||
|
||||
if column_count == 2:
|
||||
col_blocks: dict[int, list[str]] = {0: [], 1: []}
|
||||
for block in page_blocks:
|
||||
bbox = block.get("bbox") or block.get("block_bbox")
|
||||
if not bbox or len(bbox) < 4:
|
||||
continue
|
||||
x_center = (bbox[0] + bbox[2]) / 2
|
||||
col = 0 if x_center < page_width / 2 else 1
|
||||
col_blocks[col].append(block.get("role", ""))
|
||||
for i in (0, 1):
|
||||
for block in real_clusters[i]["blocks"]:
|
||||
col_blocks[i].append(block.get("role", ""))
|
||||
|
||||
body_roles = {
|
||||
"body_paragraph",
|
||||
|
|
@ -398,7 +488,7 @@ def _classify_page_layout(page_blocks: list[dict], page_width: float, page_heigh
|
|||
column_boundaries=centers,
|
||||
layout_type="mixed_tail",
|
||||
confidence=0.6,
|
||||
evidence=["eligible_body_blocks", "two_center_clusters"],
|
||||
evidence=["eligible_body_blocks", "two_center_clusters"] + evidence_extra,
|
||||
)
|
||||
|
||||
return PageLayoutProfile(
|
||||
|
|
@ -406,7 +496,7 @@ def _classify_page_layout(page_blocks: list[dict], page_width: float, page_heigh
|
|||
column_boundaries=centers,
|
||||
layout_type="two_column",
|
||||
confidence=0.7,
|
||||
evidence=["eligible_body_blocks"],
|
||||
evidence=["eligible_body_blocks"] + evidence_extra,
|
||||
)
|
||||
|
||||
return PageLayoutProfile(
|
||||
|
|
|
|||
|
|
@ -37,7 +37,6 @@ _EDITORIAL_TAIL_PHRASES = (
|
|||
"ethics statement",
|
||||
"author contributions",
|
||||
"the remaining authors declare",
|
||||
"copyright",
|
||||
"published online",
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -52,7 +52,6 @@ _PAGE1_ARTICLE_TYPE_LABELS = frozenset(
|
|||
|
||||
FRONTMATTER_NOISE = {
|
||||
"open access",
|
||||
"copyright",
|
||||
"citation",
|
||||
"keywords",
|
||||
"edited by",
|
||||
|
|
@ -335,7 +334,7 @@ def _is_backmatter_boundary_heading(block: dict, page_num: int, total_pages: int
|
|||
|
||||
upper = text.upper()
|
||||
has_container_words = (
|
||||
"ADDITIONAL" in upper or "SUPPLEMENTARY" in upper or "DECLARATION" in upper or "INFORMATION" in upper
|
||||
"ADDITIONAL" in upper or "DECLARATION" in upper or "INFORMATION" in upper
|
||||
)
|
||||
|
||||
if not is_visually_heading and not has_container_words:
|
||||
|
|
@ -575,7 +574,6 @@ def resolve_final_role(
|
|||
"publisher's note",
|
||||
"publisher\u2019s note",
|
||||
"the remaining authors declare",
|
||||
"copyright",
|
||||
"published online",
|
||||
)
|
||||
)
|
||||
|
|
@ -1119,7 +1117,7 @@ def assign_block_role(
|
|||
confidence = 0.7
|
||||
lower_txt = text.lower()
|
||||
has_container_words = any(
|
||||
w in lower_txt for w in ["additional information", "declaration", "supplementary"]
|
||||
w in lower_txt for w in ["additional information", "declaration"]
|
||||
)
|
||||
span_meta = block.get("span_metadata", {}) or {}
|
||||
is_bold = False
|
||||
|
|
@ -1293,7 +1291,7 @@ def assign_block_role(
|
|||
# Page-1 DOI / received / accepted / published footnotes → frontmatter_noise
|
||||
if raw_label in {"footnote", "vision_footnote"} and page_num == 1:
|
||||
lower_txt = text.lower()
|
||||
if lower_txt.startswith(("doi", "received", "accepted", "published", "copyright")):
|
||||
if lower_txt.startswith(("doi", "received", "accepted", "published")):
|
||||
return RoleAssignment(
|
||||
role="frontmatter_noise",
|
||||
confidence=0.75,
|
||||
|
|
@ -1348,13 +1346,7 @@ def assign_block_role(
|
|||
evidence=[f"abstract heading from text block: {text[:40]}"],
|
||||
)
|
||||
|
||||
# Check for copyright (page 1 only)
|
||||
if still_frontmatter and ("copyright" in lower_txt or "©" in text):
|
||||
return RoleAssignment(
|
||||
role="frontmatter_noise",
|
||||
confidence=0.85,
|
||||
evidence=[f"copyright text: {text[:60]}"],
|
||||
)
|
||||
# Check for copyright — removed: position-based checks are sufficient
|
||||
|
||||
# Check for email / ORCID / DOI patterns (page 1 only)
|
||||
if (
|
||||
|
|
@ -1505,7 +1497,7 @@ def assign_block_role(
|
|||
confidence = 0.7
|
||||
lower_txt = text.lower()
|
||||
has_container_words = any(
|
||||
w in lower_txt for w in ["additional information", "declaration", "supplementary"]
|
||||
w in lower_txt for w in ["additional information", "declaration"]
|
||||
)
|
||||
span_meta = block.get("span_metadata", {}) or {}
|
||||
is_bold = False
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ def test_page_layout_profile_includes_confidence_and_evidence() -> None:
|
|||
from paperforge.worker.ocr_document import _build_page_layout_profiles
|
||||
|
||||
blocks = [
|
||||
{"role": "body_paragraph", "page": 1, "bbox": [100, 100, 500, 160], "page_width": 1200, "page_height": 1700},
|
||||
{"role": "body_paragraph", "page": 1, "bbox": [700, 100, 1100, 160], "page_width": 1200, "page_height": 1700},
|
||||
{"role": "body_paragraph", "page": 1, "bbox": [100, 100, 500, 280], "page_width": 1200, "page_height": 1700},
|
||||
{"role": "body_paragraph", "page": 1, "bbox": [700, 100, 1100, 280], "page_width": 1200, "page_height": 1700},
|
||||
]
|
||||
|
||||
profile = _build_page_layout_profiles(blocks)[1]
|
||||
|
|
@ -23,8 +23,8 @@ def test_layout_profiles_ignore_wide_headings_and_media() -> None:
|
|||
blocks = [
|
||||
{"role": "section_heading", "page": 1, "bbox": [50, 50, 1150, 90], "page_width": 1200, "page_height": 1700},
|
||||
{"role": "figure_asset", "page": 1, "bbox": [400, 120, 800, 500], "page_width": 1200, "page_height": 1700},
|
||||
{"role": "body_paragraph", "page": 1, "bbox": [100, 600, 500, 660], "page_width": 1200, "page_height": 1700},
|
||||
{"role": "body_paragraph", "page": 1, "bbox": [700, 600, 1100, 660], "page_width": 1200, "page_height": 1700},
|
||||
{"role": "body_paragraph", "page": 1, "bbox": [100, 600, 500, 780], "page_width": 1200, "page_height": 1700},
|
||||
{"role": "body_paragraph", "page": 1, "bbox": [700, 600, 1100, 780], "page_width": 1200, "page_height": 1700},
|
||||
]
|
||||
|
||||
profile = _build_page_layout_profiles(blocks)[1]
|
||||
|
|
@ -607,7 +607,7 @@ def test_reference_zones_remain_column_scoped_after_authority_refresh() -> None:
|
|||
"block_id": "p4_b2",
|
||||
"raw_label": "text",
|
||||
"raw_order": 2,
|
||||
"bbox": [620, 180, 960, 250],
|
||||
"bbox": [620, 180, 960, 360],
|
||||
"text": "[1] Example reference entry with enough tokens to be reference-like.",
|
||||
"page_width": 1200,
|
||||
"page_height": 1600,
|
||||
|
|
@ -619,7 +619,7 @@ def test_reference_zones_remain_column_scoped_after_authority_refresh() -> None:
|
|||
"block_id": "p4_b3",
|
||||
"raw_label": "text",
|
||||
"raw_order": 3,
|
||||
"bbox": [80, 180, 420, 250],
|
||||
"bbox": [80, 180, 420, 360],
|
||||
"text": "Left-column body text that should not be part of the reference zone.",
|
||||
"page_width": 1200,
|
||||
"page_height": 1600,
|
||||
|
|
@ -2608,9 +2608,9 @@ def test_layout_profile_build_profiles() -> None:
|
|||
{"page": 1, "bbox": [100, 100, 700, 140], "role": "body_paragraph", "page_width": 800, "page_height": 1000},
|
||||
{"page": 1, "bbox": [100, 160, 700, 200], "role": "body_paragraph", "page_width": 800, "page_height": 1000},
|
||||
# Page 2: two-column, body left + tail right
|
||||
{"page": 2, "bbox": [50, 100, 380, 140], "role": "body_paragraph", "page_width": 800, "page_height": 1000},
|
||||
{"page": 2, "bbox": [50, 100, 380, 210], "role": "body_paragraph", "page_width": 800, "page_height": 1000},
|
||||
{"page": 2, "bbox": [420, 100, 750, 140], "role": "backmatter_heading", "page_width": 800, "page_height": 1000},
|
||||
{"page": 2, "bbox": [420, 160, 750, 200], "role": "reference_item", "page_width": 800, "page_height": 1000},
|
||||
{"page": 2, "bbox": [420, 160, 750, 270], "role": "reference_item", "page_width": 800, "page_height": 1000},
|
||||
]
|
||||
|
||||
profiles = _build_page_layout_profiles(blocks)
|
||||
|
|
@ -2833,9 +2833,9 @@ def test_forward_body_end_mixed_page() -> None:
|
|||
{"page": 2, "bbox": [100, 100, 700, 140], "role": "body_paragraph", "page_width": 800, "page_height": 1000},
|
||||
{"page": 3, "bbox": [100, 100, 700, 140], "role": "body_paragraph", "page_width": 800, "page_height": 1000},
|
||||
# Page 4: left body (clean column), right tail
|
||||
{"page": 4, "bbox": [50, 100, 380, 140], "role": "body_paragraph", "page_width": 800, "page_height": 1000},
|
||||
{"page": 4, "bbox": [50, 100, 380, 210], "role": "body_paragraph", "page_width": 800, "page_height": 1000},
|
||||
{"page": 4, "bbox": [420, 100, 750, 140], "role": "backmatter_heading", "page_width": 800, "page_height": 1000},
|
||||
{"page": 4, "bbox": [420, 160, 750, 200], "role": "backmatter_body", "page_width": 800, "page_height": 1000},
|
||||
{"page": 4, "bbox": [420, 160, 750, 260], "role": "backmatter_body", "page_width": 800, "page_height": 1000},
|
||||
# Page 5: tail only
|
||||
{"page": 5, "bbox": [420, 100, 750, 140], "role": "reference_heading", "page_width": 800, "page_height": 1000},
|
||||
{"page": 5, "bbox": [420, 160, 750, 200], "role": "reference_item", "page_width": 800, "page_height": 1000},
|
||||
|
|
@ -4943,3 +4943,112 @@ def test_reference_hold_does_not_promote_final_reference_membership() -> None:
|
|||
]
|
||||
zone = build_verified_reference_zone_from_artifacts(blocks, {"region_bus": {"reference_zone_ids": set()}})
|
||||
assert zone.get("status") != "ACCEPT"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Weak cluster rejection tests (Task 3-7)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_short_isolated_body_line_does_not_create_two_column_layout() -> None:
|
||||
"""Replicates page 12 of paper 2E4EPHN2: 6 full-width body blocks +
|
||||
1 short offset line should NOT be classified as two_column."""
|
||||
from paperforge.worker.ocr_document import _classify_page_layout
|
||||
|
||||
page_width = 1191.0
|
||||
page_height = 1684.0
|
||||
blocks = [
|
||||
{"role": "body_paragraph", "bbox": [325, 224, 1123, 298], "text": "Long paragraph " * 40, "page_width": page_width},
|
||||
{"role": "body_paragraph", "bbox": [325, 299, 1124, 377], "text": "Long paragraph " * 40, "page_width": page_width},
|
||||
{"role": "body_paragraph", "bbox": [325, 398, 1125, 633], "text": "Long paragraph " * 40, "page_width": page_width},
|
||||
{"role": "body_paragraph", "bbox": [326, 643, 1125, 762], "text": "Long paragraph " * 40, "page_width": page_width},
|
||||
{"role": "body_paragraph", "bbox": [325, 772, 1124, 820], "text": "Long paragraph " * 40, "page_width": page_width},
|
||||
{"role": "body_paragraph", "bbox": [328, 832, 704, 854], "text": "Informed Consent Statement: Not applicable.", "page_width": page_width},
|
||||
{"role": "body_paragraph", "bbox": [327, 867, 1122, 914], "text": "Long paragraph " * 40, "page_width": page_width},
|
||||
]
|
||||
|
||||
profile = _classify_page_layout(blocks, page_width, page_height)
|
||||
|
||||
assert profile.layout_type == "single_column", f"Expected single_column, got {profile.layout_type}"
|
||||
assert profile.column_count == 1, f"Expected column_count=1, got {profile.column_count}"
|
||||
assert "weak_isolated_column_cluster_ignored" in profile.evidence, (
|
||||
f"Expected weak_isolated_column_cluster_ignored in evidence, got {profile.evidence}"
|
||||
)
|
||||
|
||||
|
||||
def test_balanced_two_column_layout_still_detected() -> None:
|
||||
"""True two-column page with multiple blocks per column must remain two_column."""
|
||||
from paperforge.worker.ocr_document import _classify_page_layout
|
||||
|
||||
page_width = 800.0
|
||||
page_height = 1000.0
|
||||
blocks = [
|
||||
{"role": "body_paragraph", "bbox": [50, 100, 380, 250], "text": "Left body " * 30, "page_width": page_width},
|
||||
{"role": "body_paragraph", "bbox": [50, 270, 380, 420], "text": "Left body " * 30, "page_width": page_width},
|
||||
{"role": "body_paragraph", "bbox": [420, 100, 750, 250], "text": "Right body " * 30, "page_width": page_width},
|
||||
{"role": "body_paragraph", "bbox": [420, 270, 750, 420], "text": "Right body " * 30, "page_width": page_width},
|
||||
]
|
||||
|
||||
profile = _classify_page_layout(blocks, page_width, page_height)
|
||||
assert profile.layout_type == "two_column", f"Expected two_column, got {profile.layout_type}"
|
||||
assert profile.column_count == 2, f"Expected column_count=2, got {profile.column_count}"
|
||||
|
||||
|
||||
def test_single_large_block_per_column_still_two_column() -> None:
|
||||
"""Each column has only 1 block, but large y_coverage and word_count
|
||||
must still classify as two_column (not killed by count=1 guard)."""
|
||||
from paperforge.worker.ocr_document import _classify_page_layout
|
||||
|
||||
page_width = 800.0
|
||||
page_height = 1000.0
|
||||
blocks = [
|
||||
{"role": "body_paragraph", "bbox": [50, 100, 380, 800], "text": "Left body " * 100, "page_width": page_width},
|
||||
{"role": "body_paragraph", "bbox": [420, 100, 750, 800], "text": "Right body " * 100, "page_width": page_width},
|
||||
]
|
||||
|
||||
profile = _classify_page_layout(blocks, page_width, page_height)
|
||||
assert profile.layout_type == "two_column", f"Expected two_column, got {profile.layout_type}"
|
||||
assert profile.column_count == 2, f"Expected column_count=2, got {profile.column_count}"
|
||||
|
||||
|
||||
def test_multiple_weak_offset_lines_do_not_create_wide_dispersion() -> None:
|
||||
"""3 raw clusters (2 weak, 1 real) must produce single_column, not two_column/wide_dispersion."""
|
||||
from paperforge.worker.ocr_document import _classify_page_layout
|
||||
|
||||
page_width = 1191.0
|
||||
page_height = 1684.0
|
||||
blocks = [
|
||||
{"role": "body_paragraph", "bbox": [325, 224, 1123, 298], "text": "Long paragraph " * 40, "page_width": page_width},
|
||||
{"role": "body_paragraph", "bbox": [325, 299, 1124, 377], "text": "Long paragraph " * 40, "page_width": page_width},
|
||||
{"role": "body_paragraph", "bbox": [328, 832, 704, 854], "text": "IC Statement: Not applicable.", "page_width": page_width},
|
||||
{"role": "body_paragraph", "bbox": [328, 870, 550, 890], "text": "Short note.", "page_width": page_width},
|
||||
{"role": "body_paragraph", "bbox": [327, 900, 1122, 950], "text": "Long paragraph " * 40, "page_width": page_width},
|
||||
]
|
||||
|
||||
profile = _classify_page_layout(blocks, page_width, page_height)
|
||||
|
||||
assert profile.layout_type == "single_column", f"Expected single_column, got {profile.layout_type}"
|
||||
assert profile.column_count == 1, f"Expected column_count=1, got {profile.column_count}"
|
||||
assert "weak_isolated_column_cluster_ignored" in profile.evidence, (
|
||||
f"Expected weak_isolated_column_cluster_ignored in evidence, got {profile.evidence}"
|
||||
)
|
||||
|
||||
|
||||
def test_all_clusters_weak_fallback_to_single_column() -> None:
|
||||
"""When all clusters are weak (e.g. only short offset lines), fallback to low-confidence single_column."""
|
||||
from paperforge.worker.ocr_document import _classify_page_layout
|
||||
|
||||
page_width = 1191.0
|
||||
page_height = 1684.0
|
||||
blocks = [
|
||||
{"role": "body_paragraph", "bbox": [328, 832, 704, 854], "text": "IC Statement: Not applicable.", "page_width": page_width},
|
||||
{"role": "body_paragraph", "bbox": [60, 870, 300, 890], "text": "Footnote text.", "page_width": page_width},
|
||||
]
|
||||
|
||||
profile = _classify_page_layout(blocks, page_width, page_height)
|
||||
|
||||
assert profile.layout_type == "single_column", f"Expected single_column, got {profile.layout_type}"
|
||||
assert profile.confidence <= 0.35, f"Expected confidence <= 0.35, got {profile.confidence}"
|
||||
assert "all_column_clusters_weak" in profile.evidence, (
|
||||
f"Expected all_column_clusters_weak in evidence, got {profile.evidence}"
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in a new issue