diff --git a/docs/superpowers/plans/2026-06-22-column-detection-weak-cluster-rejection.md b/docs/superpowers/plans/2026-06-22-column-detection-weak-cluster-rejection.md index de83c72b..cbeffcc6 100644 --- a/docs/superpowers/plans/2026-06-22-column-detection-weak-cluster-rejection.md +++ b/docs/superpowers/plans/2026-06-22-column-detection-weak-cluster-rejection.md @@ -17,7 +17,7 @@ - [ ] **Step 1: Insert `_cluster_page_column_groups()` before `_cluster_page_columns()`, then replace `_cluster_page_columns()` body with wrapper** -Open `paperforge/worker/ocr_document.py`. Insert the new function before `_cluster_page_columns()` (before line 321), then replace the body of `_cluster_page_columns()` (lines 321-350) with a one-line wrapper: +Open `paperforge/worker/ocr_document.py`. Insert the new function before `_cluster_page_columns()` (currently at line 278), then replace the body of `_cluster_page_columns()` with a one-line wrapper: ```python def _cluster_page_column_groups(page_blocks: list[dict], page_width: float) -> list[dict]: @@ -125,11 +125,11 @@ git commit -m "feat: add _cluster_page_column_groups() returning cluster metadat ### Task 2: Add `_is_weak_isolated_column_cluster()` and modify `_classify_page_layout()` **Files:** -- Modify: `paperforge/worker/ocr_document.py` — insert helper, modify `_classify_page_layout()` (lines 353-418) +- Modify: `paperforge/worker/ocr_document.py` — insert helper before `_classify_page_layout()`, replace `_classify_page_layout()` body -- [ ] **Step 1: Read current `_classify_page_layout()` to confirm boundaries** +- [ ] **Step 1: Read current functions to confirm boundaries** -Read `paperforge/worker/ocr_document.py` lines 321-418 to confirm no changes since the spec was written. +Read `paperforge/worker/ocr_document.py` from `_cluster_page_columns` through `_classify_page_layout` to confirm no conflicting changes since this plan was written. - [ ] **Step 2: Insert `_is_weak_isolated_column_cluster()` before `_classify_page_layout()`** @@ -147,7 +147,7 @@ def _is_weak_isolated_column_cluster(cluster: dict, page_height: float) -> bool: return True ``` -- [ ] **Step 3: Replace `_classify_page_layout()` (lines 353-418)** +- [ ] **Step 3: Replace `_classify_page_layout()` body** Replace the entire function with: diff --git a/paperforge/worker/ocr_document.py b/paperforge/worker/ocr_document.py index 51e90d7c..c95d320d 100644 --- a/paperforge/worker/ocr_document.py +++ b/paperforge/worker/ocr_document.py @@ -318,12 +318,14 @@ def _is_layout_eligible_block(block: dict) -> bool: return True -def _cluster_page_columns(page_blocks: list[dict], page_width: float) -> list[float]: - """Cluster block x-centers by column using a gap-based approach. +def _cluster_page_column_groups(page_blocks: list[dict], page_width: float) -> list[dict]: + """Cluster block x-centers by column and return per-cluster metadata. - Returns one representative x-center per column cluster. + Returns list of cluster dicts sorted by center ascending. Each dict: + center, count, center_min, center_max, y_min, y_max, y_coverage, + word_count, width_median, block_ids, blocks """ - centers: list[float] = [] + items: list[tuple[float, dict]] = [] for block in page_blocks: bbox = block.get("bbox") or block.get("block_bbox") if not bbox or len(bbox) < 4: @@ -332,47 +334,135 @@ def _cluster_page_columns(page_blocks: list[dict], page_width: float) -> list[fl if block_width <= 50: continue x_center = (bbox[0] + bbox[2]) / 2 - centers.append(x_center) + items.append((x_center, block)) - if not centers: - return [page_width / 2] + if not items: + return [{ + "center": page_width / 2, + "count": 0, + "center_min": page_width / 2, + "center_max": page_width / 2, + "y_min": 0.0, + "y_max": 0.0, + "y_coverage": 0.0, + "word_count": 0, + "width_median": 0.0, + "block_ids": [], + "blocks": [], + }] - centers.sort() + items.sort(key=lambda x: x[0]) gap_threshold = page_width * 0.15 - clusters: list[list[float]] = [[centers[0]]] + raw_clusters: list[list[dict]] = [[items[0][1]]] + last_center = items[0][0] - for c in centers[1:]: - if c - clusters[-1][-1] > gap_threshold: - clusters.append([c]) + for center, block in items[1:]: + if center - last_center > gap_threshold: + raw_clusters.append([block]) else: - clusters[-1].append(c) + raw_clusters[-1].append(block) + last_center = center - return [sum(cluster) / len(cluster) for cluster in clusters] + result: list[dict] = [] + for cluster_blocks in raw_clusters: + valid_bboxes = [ + bb for b in cluster_blocks + for bb in [b.get("bbox") or b.get("block_bbox")] + if bb and len(bb) >= 4 + ] + centers_for_compute = [(bb[0] + bb[2]) / 2 for bb in valid_bboxes] + + widths = sorted(bb[2] - bb[0] for bb in valid_bboxes) + + y_vals = [(bb[1], bb[3]) for bb in valid_bboxes] + + center_val = sum(centers_for_compute) / len(centers_for_compute) if centers_for_compute else page_width / 2 + y_min_val = min(yv[0] for yv in y_vals) if y_vals else 0.0 + y_max_val = max(yv[1] for yv in y_vals) if y_vals else 0.0 + + result.append({ + "center": center_val, + "count": len(cluster_blocks), + "center_min": min(centers_for_compute) if centers_for_compute else page_width / 2, + "center_max": max(centers_for_compute) if centers_for_compute else page_width / 2, + "y_min": y_min_val, + "y_max": y_max_val, + "y_coverage": y_max_val - y_min_val, + "word_count": sum(len(_block_text(b).split()) for b in cluster_blocks), + "width_median": widths[len(widths) // 2] if widths else 0.0, + "block_ids": [b.get("block_id", "") for b in cluster_blocks], + "blocks": cluster_blocks, + }) + + result.sort(key=lambda c: c["center"]) + return result + + +# ponytail: Replace _cluster_page_columns() body with wrapper delegating to _cluster_page_column_groups() +def _cluster_page_columns(page_blocks: list[dict], page_width: float) -> list[float]: + return [c["center"] for c in _cluster_page_column_groups(page_blocks, page_width)] + + +def _is_weak_isolated_column_cluster(cluster: dict, page_height: float) -> bool: + """Return True if this cluster has insufficient evidence to be a real column.""" + if cluster["count"] >= 3: + return False + if cluster["y_coverage"] >= page_height * 0.10: + return False + if cluster["word_count"] >= 25: + return False + return True def _classify_page_layout(page_blocks: list[dict], page_width: float, page_height: float) -> PageLayoutProfile: """Classify a page's layout based on column clusters and role distribution.""" - centers = _cluster_page_columns(page_blocks, page_width) - column_count = len(centers) + clusters = _cluster_page_column_groups(page_blocks, page_width) - if column_count == 1: + # ponytail: fallback empty cluster (count=0) should not be penalized + if len(clusters) == 1 and clusters[0]["count"] == 0: return PageLayoutProfile( column_count=1, - column_boundaries=centers, + column_boundaries=[page_width / 2], layout_type="single_column", confidence=0.7, evidence=["eligible_body_blocks"], ) + real_clusters = [ + c for c in clusters + if not _is_weak_isolated_column_cluster(c, page_height) + ] + + if not real_clusters: + return PageLayoutProfile( + column_count=1, + column_boundaries=[page_width / 2], + layout_type="single_column", + confidence=0.35, + evidence=["eligible_body_blocks", "all_column_clusters_weak"], + ) + + evidence_extra: list[str] = [] + if len(real_clusters) < len(clusters): + evidence_extra.append("weak_isolated_column_cluster_ignored") + + centers = [c["center"] for c in real_clusters] + column_count = len(real_clusters) + + if column_count == 1: + return PageLayoutProfile( + column_count=1, + column_boundaries=[real_clusters[0]["center"]], + layout_type="single_column", + confidence=0.55 if evidence_extra else 0.7, + evidence=["eligible_body_blocks"] + evidence_extra, + ) + if column_count == 2: col_blocks: dict[int, list[str]] = {0: [], 1: []} - for block in page_blocks: - bbox = block.get("bbox") or block.get("block_bbox") - if not bbox or len(bbox) < 4: - continue - x_center = (bbox[0] + bbox[2]) / 2 - col = 0 if x_center < page_width / 2 else 1 - col_blocks[col].append(block.get("role", "")) + for i in (0, 1): + for block in real_clusters[i]["blocks"]: + col_blocks[i].append(block.get("role", "")) body_roles = { "body_paragraph", @@ -398,7 +488,7 @@ def _classify_page_layout(page_blocks: list[dict], page_width: float, page_heigh column_boundaries=centers, layout_type="mixed_tail", confidence=0.6, - evidence=["eligible_body_blocks", "two_center_clusters"], + evidence=["eligible_body_blocks", "two_center_clusters"] + evidence_extra, ) return PageLayoutProfile( @@ -406,7 +496,7 @@ def _classify_page_layout(page_blocks: list[dict], page_width: float, page_heigh column_boundaries=centers, layout_type="two_column", confidence=0.7, - evidence=["eligible_body_blocks"], + evidence=["eligible_body_blocks"] + evidence_extra, ) return PageLayoutProfile( diff --git a/paperforge/worker/ocr_families.py b/paperforge/worker/ocr_families.py index e058ac10..03c904c6 100644 --- a/paperforge/worker/ocr_families.py +++ b/paperforge/worker/ocr_families.py @@ -37,7 +37,6 @@ _EDITORIAL_TAIL_PHRASES = ( "ethics statement", "author contributions", "the remaining authors declare", - "copyright", "published online", ) diff --git a/paperforge/worker/ocr_roles.py b/paperforge/worker/ocr_roles.py index 754fcf72..ed27cf39 100644 --- a/paperforge/worker/ocr_roles.py +++ b/paperforge/worker/ocr_roles.py @@ -52,7 +52,6 @@ _PAGE1_ARTICLE_TYPE_LABELS = frozenset( FRONTMATTER_NOISE = { "open access", - "copyright", "citation", "keywords", "edited by", @@ -335,7 +334,7 @@ def _is_backmatter_boundary_heading(block: dict, page_num: int, total_pages: int upper = text.upper() has_container_words = ( - "ADDITIONAL" in upper or "SUPPLEMENTARY" in upper or "DECLARATION" in upper or "INFORMATION" in upper + "ADDITIONAL" in upper or "DECLARATION" in upper or "INFORMATION" in upper ) if not is_visually_heading and not has_container_words: @@ -575,7 +574,6 @@ def resolve_final_role( "publisher's note", "publisher\u2019s note", "the remaining authors declare", - "copyright", "published online", ) ) @@ -1119,7 +1117,7 @@ def assign_block_role( confidence = 0.7 lower_txt = text.lower() has_container_words = any( - w in lower_txt for w in ["additional information", "declaration", "supplementary"] + w in lower_txt for w in ["additional information", "declaration"] ) span_meta = block.get("span_metadata", {}) or {} is_bold = False @@ -1293,7 +1291,7 @@ def assign_block_role( # Page-1 DOI / received / accepted / published footnotes → frontmatter_noise if raw_label in {"footnote", "vision_footnote"} and page_num == 1: lower_txt = text.lower() - if lower_txt.startswith(("doi", "received", "accepted", "published", "copyright")): + if lower_txt.startswith(("doi", "received", "accepted", "published")): return RoleAssignment( role="frontmatter_noise", confidence=0.75, @@ -1348,13 +1346,7 @@ def assign_block_role( evidence=[f"abstract heading from text block: {text[:40]}"], ) - # Check for copyright (page 1 only) - if still_frontmatter and ("copyright" in lower_txt or "©" in text): - return RoleAssignment( - role="frontmatter_noise", - confidence=0.85, - evidence=[f"copyright text: {text[:60]}"], - ) + # Check for copyright — removed: position-based checks are sufficient # Check for email / ORCID / DOI patterns (page 1 only) if ( @@ -1505,7 +1497,7 @@ def assign_block_role( confidence = 0.7 lower_txt = text.lower() has_container_words = any( - w in lower_txt for w in ["additional information", "declaration", "supplementary"] + w in lower_txt for w in ["additional information", "declaration"] ) span_meta = block.get("span_metadata", {}) or {} is_bold = False diff --git a/tests/test_ocr_document.py b/tests/test_ocr_document.py index 64b9d57c..796a7cc8 100644 --- a/tests/test_ocr_document.py +++ b/tests/test_ocr_document.py @@ -5,8 +5,8 @@ def test_page_layout_profile_includes_confidence_and_evidence() -> None: from paperforge.worker.ocr_document import _build_page_layout_profiles blocks = [ - {"role": "body_paragraph", "page": 1, "bbox": [100, 100, 500, 160], "page_width": 1200, "page_height": 1700}, - {"role": "body_paragraph", "page": 1, "bbox": [700, 100, 1100, 160], "page_width": 1200, "page_height": 1700}, + {"role": "body_paragraph", "page": 1, "bbox": [100, 100, 500, 280], "page_width": 1200, "page_height": 1700}, + {"role": "body_paragraph", "page": 1, "bbox": [700, 100, 1100, 280], "page_width": 1200, "page_height": 1700}, ] profile = _build_page_layout_profiles(blocks)[1] @@ -23,8 +23,8 @@ def test_layout_profiles_ignore_wide_headings_and_media() -> None: blocks = [ {"role": "section_heading", "page": 1, "bbox": [50, 50, 1150, 90], "page_width": 1200, "page_height": 1700}, {"role": "figure_asset", "page": 1, "bbox": [400, 120, 800, 500], "page_width": 1200, "page_height": 1700}, - {"role": "body_paragraph", "page": 1, "bbox": [100, 600, 500, 660], "page_width": 1200, "page_height": 1700}, - {"role": "body_paragraph", "page": 1, "bbox": [700, 600, 1100, 660], "page_width": 1200, "page_height": 1700}, + {"role": "body_paragraph", "page": 1, "bbox": [100, 600, 500, 780], "page_width": 1200, "page_height": 1700}, + {"role": "body_paragraph", "page": 1, "bbox": [700, 600, 1100, 780], "page_width": 1200, "page_height": 1700}, ] profile = _build_page_layout_profiles(blocks)[1] @@ -607,7 +607,7 @@ def test_reference_zones_remain_column_scoped_after_authority_refresh() -> None: "block_id": "p4_b2", "raw_label": "text", "raw_order": 2, - "bbox": [620, 180, 960, 250], + "bbox": [620, 180, 960, 360], "text": "[1] Example reference entry with enough tokens to be reference-like.", "page_width": 1200, "page_height": 1600, @@ -619,7 +619,7 @@ def test_reference_zones_remain_column_scoped_after_authority_refresh() -> None: "block_id": "p4_b3", "raw_label": "text", "raw_order": 3, - "bbox": [80, 180, 420, 250], + "bbox": [80, 180, 420, 360], "text": "Left-column body text that should not be part of the reference zone.", "page_width": 1200, "page_height": 1600, @@ -2608,9 +2608,9 @@ def test_layout_profile_build_profiles() -> None: {"page": 1, "bbox": [100, 100, 700, 140], "role": "body_paragraph", "page_width": 800, "page_height": 1000}, {"page": 1, "bbox": [100, 160, 700, 200], "role": "body_paragraph", "page_width": 800, "page_height": 1000}, # Page 2: two-column, body left + tail right - {"page": 2, "bbox": [50, 100, 380, 140], "role": "body_paragraph", "page_width": 800, "page_height": 1000}, + {"page": 2, "bbox": [50, 100, 380, 210], "role": "body_paragraph", "page_width": 800, "page_height": 1000}, {"page": 2, "bbox": [420, 100, 750, 140], "role": "backmatter_heading", "page_width": 800, "page_height": 1000}, - {"page": 2, "bbox": [420, 160, 750, 200], "role": "reference_item", "page_width": 800, "page_height": 1000}, + {"page": 2, "bbox": [420, 160, 750, 270], "role": "reference_item", "page_width": 800, "page_height": 1000}, ] profiles = _build_page_layout_profiles(blocks) @@ -2833,9 +2833,9 @@ def test_forward_body_end_mixed_page() -> None: {"page": 2, "bbox": [100, 100, 700, 140], "role": "body_paragraph", "page_width": 800, "page_height": 1000}, {"page": 3, "bbox": [100, 100, 700, 140], "role": "body_paragraph", "page_width": 800, "page_height": 1000}, # Page 4: left body (clean column), right tail - {"page": 4, "bbox": [50, 100, 380, 140], "role": "body_paragraph", "page_width": 800, "page_height": 1000}, + {"page": 4, "bbox": [50, 100, 380, 210], "role": "body_paragraph", "page_width": 800, "page_height": 1000}, {"page": 4, "bbox": [420, 100, 750, 140], "role": "backmatter_heading", "page_width": 800, "page_height": 1000}, - {"page": 4, "bbox": [420, 160, 750, 200], "role": "backmatter_body", "page_width": 800, "page_height": 1000}, + {"page": 4, "bbox": [420, 160, 750, 260], "role": "backmatter_body", "page_width": 800, "page_height": 1000}, # Page 5: tail only {"page": 5, "bbox": [420, 100, 750, 140], "role": "reference_heading", "page_width": 800, "page_height": 1000}, {"page": 5, "bbox": [420, 160, 750, 200], "role": "reference_item", "page_width": 800, "page_height": 1000}, @@ -4943,3 +4943,112 @@ def test_reference_hold_does_not_promote_final_reference_membership() -> None: ] zone = build_verified_reference_zone_from_artifacts(blocks, {"region_bus": {"reference_zone_ids": set()}}) assert zone.get("status") != "ACCEPT" + + +# --------------------------------------------------------------------------- +# Weak cluster rejection tests (Task 3-7) +# --------------------------------------------------------------------------- + + +def test_short_isolated_body_line_does_not_create_two_column_layout() -> None: + """Replicates page 12 of paper 2E4EPHN2: 6 full-width body blocks + + 1 short offset line should NOT be classified as two_column.""" + from paperforge.worker.ocr_document import _classify_page_layout + + page_width = 1191.0 + page_height = 1684.0 + blocks = [ + {"role": "body_paragraph", "bbox": [325, 224, 1123, 298], "text": "Long paragraph " * 40, "page_width": page_width}, + {"role": "body_paragraph", "bbox": [325, 299, 1124, 377], "text": "Long paragraph " * 40, "page_width": page_width}, + {"role": "body_paragraph", "bbox": [325, 398, 1125, 633], "text": "Long paragraph " * 40, "page_width": page_width}, + {"role": "body_paragraph", "bbox": [326, 643, 1125, 762], "text": "Long paragraph " * 40, "page_width": page_width}, + {"role": "body_paragraph", "bbox": [325, 772, 1124, 820], "text": "Long paragraph " * 40, "page_width": page_width}, + {"role": "body_paragraph", "bbox": [328, 832, 704, 854], "text": "Informed Consent Statement: Not applicable.", "page_width": page_width}, + {"role": "body_paragraph", "bbox": [327, 867, 1122, 914], "text": "Long paragraph " * 40, "page_width": page_width}, + ] + + profile = _classify_page_layout(blocks, page_width, page_height) + + assert profile.layout_type == "single_column", f"Expected single_column, got {profile.layout_type}" + assert profile.column_count == 1, f"Expected column_count=1, got {profile.column_count}" + assert "weak_isolated_column_cluster_ignored" in profile.evidence, ( + f"Expected weak_isolated_column_cluster_ignored in evidence, got {profile.evidence}" + ) + + +def test_balanced_two_column_layout_still_detected() -> None: + """True two-column page with multiple blocks per column must remain two_column.""" + from paperforge.worker.ocr_document import _classify_page_layout + + page_width = 800.0 + page_height = 1000.0 + blocks = [ + {"role": "body_paragraph", "bbox": [50, 100, 380, 250], "text": "Left body " * 30, "page_width": page_width}, + {"role": "body_paragraph", "bbox": [50, 270, 380, 420], "text": "Left body " * 30, "page_width": page_width}, + {"role": "body_paragraph", "bbox": [420, 100, 750, 250], "text": "Right body " * 30, "page_width": page_width}, + {"role": "body_paragraph", "bbox": [420, 270, 750, 420], "text": "Right body " * 30, "page_width": page_width}, + ] + + profile = _classify_page_layout(blocks, page_width, page_height) + assert profile.layout_type == "two_column", f"Expected two_column, got {profile.layout_type}" + assert profile.column_count == 2, f"Expected column_count=2, got {profile.column_count}" + + +def test_single_large_block_per_column_still_two_column() -> None: + """Each column has only 1 block, but large y_coverage and word_count + must still classify as two_column (not killed by count=1 guard).""" + from paperforge.worker.ocr_document import _classify_page_layout + + page_width = 800.0 + page_height = 1000.0 + blocks = [ + {"role": "body_paragraph", "bbox": [50, 100, 380, 800], "text": "Left body " * 100, "page_width": page_width}, + {"role": "body_paragraph", "bbox": [420, 100, 750, 800], "text": "Right body " * 100, "page_width": page_width}, + ] + + profile = _classify_page_layout(blocks, page_width, page_height) + assert profile.layout_type == "two_column", f"Expected two_column, got {profile.layout_type}" + assert profile.column_count == 2, f"Expected column_count=2, got {profile.column_count}" + + +def test_multiple_weak_offset_lines_do_not_create_wide_dispersion() -> None: + """3 raw clusters (2 weak, 1 real) must produce single_column, not two_column/wide_dispersion.""" + from paperforge.worker.ocr_document import _classify_page_layout + + page_width = 1191.0 + page_height = 1684.0 + blocks = [ + {"role": "body_paragraph", "bbox": [325, 224, 1123, 298], "text": "Long paragraph " * 40, "page_width": page_width}, + {"role": "body_paragraph", "bbox": [325, 299, 1124, 377], "text": "Long paragraph " * 40, "page_width": page_width}, + {"role": "body_paragraph", "bbox": [328, 832, 704, 854], "text": "IC Statement: Not applicable.", "page_width": page_width}, + {"role": "body_paragraph", "bbox": [328, 870, 550, 890], "text": "Short note.", "page_width": page_width}, + {"role": "body_paragraph", "bbox": [327, 900, 1122, 950], "text": "Long paragraph " * 40, "page_width": page_width}, + ] + + profile = _classify_page_layout(blocks, page_width, page_height) + + assert profile.layout_type == "single_column", f"Expected single_column, got {profile.layout_type}" + assert profile.column_count == 1, f"Expected column_count=1, got {profile.column_count}" + assert "weak_isolated_column_cluster_ignored" in profile.evidence, ( + f"Expected weak_isolated_column_cluster_ignored in evidence, got {profile.evidence}" + ) + + +def test_all_clusters_weak_fallback_to_single_column() -> None: + """When all clusters are weak (e.g. only short offset lines), fallback to low-confidence single_column.""" + from paperforge.worker.ocr_document import _classify_page_layout + + page_width = 1191.0 + page_height = 1684.0 + blocks = [ + {"role": "body_paragraph", "bbox": [328, 832, 704, 854], "text": "IC Statement: Not applicable.", "page_width": page_width}, + {"role": "body_paragraph", "bbox": [60, 870, 300, 890], "text": "Footnote text.", "page_width": page_width}, + ] + + profile = _classify_page_layout(blocks, page_width, page_height) + + assert profile.layout_type == "single_column", f"Expected single_column, got {profile.layout_type}" + assert profile.confidence <= 0.35, f"Expected confidence <= 0.35, got {profile.confidence}" + assert "all_column_clusters_weak" in profile.evidence, ( + f"Expected all_column_clusters_weak in evidence, got {profile.evidence}" + )