diff --git a/docs/ocr/high-risk-rule-audit.md b/docs/ocr/high-risk-rule-audit.md new file mode 100644 index 00000000..b5da4d7d --- /dev/null +++ b/docs/ocr/high-risk-rule-audit.md @@ -0,0 +1,63 @@ +# OCR High Risk Rule Audit + +## Production OCR Chain + +`paperforge/worker/ocr.py` is the production orchestrator. The production artifact chain is: + +`result.json -> blocks.raw.jsonl -> blocks.structured.jsonl -> document_structure.json -> figure_inventory.json/table_inventory.json -> objects -> render/fulltext.md -> health/ocr_health.json` + +Production modules: + +- `paperforge/worker/ocr_blocks.py` +- `paperforge/worker/ocr_roles.py` +- `paperforge/worker/ocr_document.py` +- `paperforge/worker/ocr_figures.py` +- `paperforge/worker/ocr_tables.py` +- `paperforge/worker/ocr_objects.py` +- `paperforge/worker/ocr_render.py` +- `paperforge/worker/ocr_health.py` + +## Direct Role Mutations + +| File | Symbol | Rule | Current Risk | Remediation | +| --- | --- | --- | --- | --- | +| `ocr_document.py` | `_build_region_prepass` | `_in_visual_container` can classify a block as `structured_insert` with confidence `0.85`. | Visual container is treated as a conclusion instead of evidence. | Move to `ocr-insert-score`; visual container becomes one evidence term. | +| `ocr_document.py` | `_build_region_prepass` | `page <= 3` plus key-points/highlights or box anchor can classify `structured_insert`. | Publisher-template sensitive; can swallow body text. | Move to `ocr-insert-score`; keep medium confidence as candidate. | +| `ocr_document.py` | `_build_region_prepass` | `last_insert_on_page` can continue short blocks into `structured_insert`. | Sequential propagation from one early mistake. | Move to `ocr-insert-score`; require cluster coherence and expansion audit. | + +## Direct Object Matches + +| File | Symbol | Rule | Current Risk | Remediation | +| --- | --- | --- | --- | --- | +| `ocr_figures.py` | `build_figure_inventory` | Same-page nearest asset is assigned to a legend before `caption_score` is computed. | Nearest media can be wrong when multiple figures/assets exist. | Move to `ocr-figure-score-matching`; score all candidates first. | +| `ocr_tables.py` | `_pick_best_asset` | Candidate table asset is selected by vertical distance. | Previous-page, continuation, and rotated tables can misbind. | Move to `ocr-table-score-matching`; select by `score_table_match`. | + +## Direct Reorder Decisions + +| File | Symbol | Rule | Current Risk | Remediation | +| --- | --- | --- | --- | --- | +| `ocr_render.py` | `_order_tail_blocks` | All tail-page blocks reordered by column+y unconditionally when tail roles outnumber body roles. | Mixed pages (body continuation + backmatter) can break reading order. | Move to `ocr-layout-confidence`; add confidence gate before reorder. | + +## Renderer Inference + +| File | Symbol | Rule | Current Risk | Remediation | +| --- | --- | --- | --- | --- | +| `ocr_render.py` | `render_fulltext_markdown` heading inference | Heading level (`##`/`###`) determined from font size clusters without confidence scoring. | Ambiguous font sizes produce wrong heading hierarchy. | Move to style-profile confidence; add uncertainty annotations. | +| `ocr_render.py` | `_is_bogus_heading` | Hard-coded rules (text >100 chars, multiple periods, verb words) suppress or demote headings. | False positives suppress genuine section headings. | Move to heading scoring plan; soft-suppress with low confidence. | + +## Remediation Map + +| Rule Family | Plan | Action | +| --- | --- | --- | +| Figure asset matching | `2026-06-08-ocr-figure-score-matching-plan.md` | Convert nearest-asset choice into scored candidate selection. | +| Table asset matching | `2026-06-08-ocr-table-score-matching-plan.md` | Convert vertical-nearest choice into scored candidate selection. | +| Layout column inference | `2026-06-08-ocr-layout-confidence-plan.md` | Add confidence and low-confidence reorder guard. | +| Structured insert promotion | `2026-06-08-ocr-insert-score-plan.md` | Convert direct promotion into score/candidate behavior. | +| Health aggregation | `2026-06-08-ocr-health-hard-rule-summary-plan.md` | Report remaining hard-rule and uncertainty counts. | + +## Baseline Counts + +- `direct_role_mutation_count`: 3 +- `direct_object_match_count`: 2 +- `direct_reorder_decision_count`: 1 +- `direct_renderer_inference_count`: 2 diff --git a/docs/superpowers/plans/2026-06-08-ocr-figure-score-matching-plan.md b/docs/superpowers/plans/2026-06-08-ocr-figure-score-matching-plan.md new file mode 100644 index 00000000..5cb941dc --- /dev/null +++ b/docs/superpowers/plans/2026-06-08-ocr-figure-score-matching-plan.md @@ -0,0 +1,297 @@ +# OCR Figure Score Matching Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make figure scoring participate in asset matching so low-confidence legends and ambiguous nearest assets are not emitted as confident figures. + +**Architecture:** Keep `ocr_figures.py` as the inventory owner and extend `ocr_scores.py` with a lightweight figure match scorer. `build_figure_inventory()` should score candidate assets before consuming them and should preserve ambiguous/unmatched/orphan states. + +**Tech Stack:** Python, pytest, PaperForge OCR figure inventory JSON. + +--- + +## File Structure + +- Modify: `paperforge/worker/ocr_scores.py` — add `score_figure_match()`. +- Modify: `paperforge/worker/ocr_figures.py` — replace nearest-only matching with scored candidate selection. +- Modify: `paperforge/worker/ocr_health.py` — count ambiguous and low-score matched figures. +- Test: `tests/test_ocr_scores.py` — figure match score unit tests. +- Test: `tests/test_ocr_figures.py` — inventory behavior tests. +- Test: `tests/test_ocr_rendering.py` — unresolved/orphan figure visibility if renderer currently drops them. + +--- + +### Task 1: Add Figure Match Scorer Tests + +**Files:** +- Modify: `tests/test_ocr_scores.py` +- Modify: `paperforge/worker/ocr_scores.py` + +- [ ] **Step 1: Write failing tests for figure match scores** + +Add to `tests/test_ocr_scores.py`: + +```python +def test_figure_match_score_prefers_same_page_overlap() -> None: + from paperforge.worker.ocr_scores import score_figure_match + + legend = {"block_id": "cap1", "page": 2, "bbox": [100, 500, 700, 540]} + asset = {"block_id": "fig1", "page": 2, "bbox": [120, 120, 680, 480]} + + result = score_figure_match(legend, asset, caption_score={"score": 0.8}) + + assert result["decision"] == "matched" + assert result["matched_asset_id"] == "fig1" + assert result["score"] >= 0.6 + assert "same_page" in result["evidence"] + assert "x_overlap" in result["evidence"] + + +def test_figure_match_score_rejects_low_caption_score() -> None: + from paperforge.worker.ocr_scores import score_figure_match + + legend = {"block_id": "cap1", "page": 2, "bbox": [100, 500, 700, 540]} + asset = {"block_id": "fig1", "page": 2, "bbox": [120, 120, 680, 480]} + + result = score_figure_match(legend, asset, caption_score={"score": 0.2}) + + assert result["decision"] == "rejected" + assert result["score"] < 0.4 + assert "low_caption_score" in result["evidence"] +``` + +- [ ] **Step 2: Run tests to verify failure** + +Run: + +```bash +python -m pytest tests/test_ocr_scores.py -k figure_match -v --tb=short +``` + +Expected: FAIL with missing `score_figure_match`. + +- [ ] **Step 3: Implement `score_figure_match()`** + +Add to `paperforge/worker/ocr_scores.py`: + +```python +def score_figure_match(legend: dict, asset: dict, *, caption_score: dict | None = None) -> dict: + legend_bbox = legend.get("bbox") or legend.get("block_bbox") or [0, 0, 0, 0] + asset_bbox = asset.get("bbox") or asset.get("block_bbox") or [0, 0, 0, 0] + score = 0.0 + evidence: list[str] = [] + + caption_value = float((caption_score or {}).get("score", 0.0)) + if caption_value < 0.4: + evidence.append("low_caption_score") + return {"score": caption_value, "matched_asset_id": asset.get("block_id", ""), "decision": "rejected", "evidence": evidence} + + if legend.get("page") == asset.get("page"): + score += 0.3 + evidence.append("same_page") + if _bbox_x_overlap_ratio(legend_bbox, asset_bbox) >= 0.4: + score += 0.25 + evidence.append("x_overlap") + if len(legend_bbox) >= 4 and len(asset_bbox) >= 4: + vertical_gap = min(abs(legend_bbox[1] - asset_bbox[3]), abs(asset_bbox[1] - legend_bbox[3])) + if vertical_gap <= 300: + score += 0.2 + evidence.append("nearby_y") + if asset_bbox[3] <= legend_bbox[1] or asset_bbox[1] >= legend_bbox[3]: + score += 0.1 + evidence.append("caption_above_or_below") + score += min(0.15, caption_value * 0.15) + score = max(0.0, min(1.0, score)) + decision = "matched" if score >= 0.6 else "ambiguous" if score >= 0.4 else "rejected" + return {"score": score, "matched_asset_id": asset.get("block_id", ""), "decision": decision, "evidence": evidence} +``` + +- [ ] **Step 4: Run scorer tests** + +Run: + +```bash +python -m pytest tests/test_ocr_scores.py -k figure_match -q --tb=short +``` + +Expected: PASS. + +--- + +### Task 2: Convert Nearest Asset Choice to Scored Selection + +**Files:** +- Modify: `tests/test_ocr_figures.py` +- Modify: `paperforge/worker/ocr_figures.py` + +- [ ] **Step 1: Write failing ambiguous-nearest test** + +Add to `tests/test_ocr_figures.py`: + +```python +def test_figure_inventory_marks_close_asset_candidates_ambiguous() -> None: + from paperforge.worker.ocr_figures import build_figure_inventory + + blocks = [ + {"block_id": "cap1", "role": "figure_caption", "page": 1, "text": "Figure 1. Assay result", "bbox": [100, 500, 700, 540]}, + {"block_id": "asset1", "role": "figure_asset", "page": 1, "bbox": [100, 100, 700, 470]}, + {"block_id": "asset2", "role": "figure_asset", "page": 1, "bbox": [110, 560, 710, 900]}, + ] + + inventory = build_figure_inventory(blocks) + + assert inventory["matched_figures"] == [] + assert len(inventory.get("ambiguous_figures", [])) == 1 + assert inventory["ambiguous_figures"][0]["legend_block_id"] == "cap1" +``` + +- [ ] **Step 2: Run the test to verify failure** + +Run: + +```bash +python -m pytest tests/test_ocr_figures.py -k close_asset_candidates -v --tb=short +``` + +Expected: FAIL because current code chooses one nearest asset. + +- [ ] **Step 3: Import and use `score_figure_match()`** + +In `paperforge/worker/ocr_figures.py`, update the import: + +```python +from paperforge.worker.ocr_scores import score_figure_caption, score_figure_match +``` + +Then replace the nearest-asset loop in `build_figure_inventory()` with candidate scoring: + +```python +caption_score = score_figure_caption( + legend, + nearby_media=any(a.get("page", 0) == legend_page for a in assets), + caption_style_match=_caption_style_match(legend, structured_blocks), +) +candidates = [] +for ai, asset in enumerate(assets): + if ai in used_asset_indices or asset.get("page", 0) != legend_page: + continue + match_score = score_figure_match(legend, asset, caption_score=caption_score) + if match_score["decision"] != "rejected": + candidates.append((ai, asset, match_score)) +candidates.sort(key=lambda item: item[2]["score"], reverse=True) +``` + +- [ ] **Step 4: Add selection rules** + +Use these rules immediately after sorting candidates: + +```python +ambiguous_figures: list[dict] = [] + +if candidates: + top_score = candidates[0][2]["score"] + close = [item for item in candidates if top_score - item[2]["score"] < 0.15] + if top_score < 0.4: + matched_assets = [] + elif len(close) > 1: + ambiguous_figures.append({ + "legend_block_id": legend.get("block_id", ""), + "page": legend_page, + "caption_score": caption_score, + "candidates": [ + {"asset_block_id": asset.get("block_id", ""), "match_score": score} + for _, asset, score in close + ], + }) + matched_assets = [] + else: + best_idx, best_asset, best_score = candidates[0] + matched_assets = [best_asset] + used_asset_indices.add(best_idx) + region_match = {"media_blocks": [best_asset], "match_score": best_score} +``` + +Keep one `ambiguous_figures` list initialized near `matched_figures` and include it in the returned inventory. + +- [ ] **Step 5: Run the focused figure test** + +Run: + +```bash +python -m pytest tests/test_ocr_figures.py -k close_asset_candidates -q --tb=short +``` + +Expected: PASS. + +--- + +### Task 3: Refuse Low-Caption-Score Confident Matches + +**Files:** +- Modify: `tests/test_ocr_figures.py` +- Modify: `paperforge/worker/ocr_figures.py` + +- [ ] **Step 1: Write failing low-caption-score test** + +Add to `tests/test_ocr_figures.py`: + +```python +def test_figure_inventory_does_not_confidently_match_low_caption_score() -> None: + from paperforge.worker.ocr_figures import build_figure_inventory + + blocks = [ + {"block_id": "cap1", "role": "figure_caption_candidate", "page": 1, "text": "Figure 1 shows that cells moved.", "bbox": [100, 500, 700, 540]}, + {"block_id": "asset1", "role": "figure_asset", "page": 1, "bbox": [100, 100, 700, 470]}, + ] + + inventory = build_figure_inventory(blocks) + + assert inventory["matched_figures"] == [] + assert len(inventory["unmatched_legends"]) == 1 +``` + +- [ ] **Step 2: Run the test to verify failure or current behavior** + +Run: + +```bash +python -m pytest tests/test_ocr_figures.py -k low_caption_score -v --tb=short +``` + +Expected: FAIL if the candidate still becomes a confident match. + +- [ ] **Step 3: Guard `matched_figures.append()`** + +In `build_figure_inventory()`, append to `matched_figures` only when `matched_assets` is non-empty and `caption_score["score"] >= 0.4`. Otherwise append the legend to `unmatched_legends` and skip creating a confident figure entry. + +Use this shape: + +```python +if not matched_assets or caption_score.get("score", 0.0) < 0.4: + unmatched_legends.append(legend) + continue +``` + +- [ ] **Step 4: Run figure tests** + +Run: + +```bash +python -m pytest tests/test_ocr_figures.py -q --tb=short +``` + +Expected: PASS. + +--- + +## Verification + +Run after all tasks: + +```bash +python -m pytest tests/test_ocr_scores.py tests/test_ocr_figures.py tests/test_ocr_health.py tests/test_ocr_rendering.py -q --tb=short +``` + +Expected: PASS. + +Do not commit unless the user explicitly requests a commit. diff --git a/docs/superpowers/plans/2026-06-08-ocr-health-hard-rule-summary-plan.md b/docs/superpowers/plans/2026-06-08-ocr-health-hard-rule-summary-plan.md new file mode 100644 index 00000000..44934522 --- /dev/null +++ b/docs/superpowers/plans/2026-06-08-ocr-health-hard-rule-summary-plan.md @@ -0,0 +1,216 @@ +# OCR Health Hard Rule Summary Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make OCR health expose remaining hard-rule decisions and uncertainty counts across figures, tables, layout, inserts, and renderer-facing unresolved states. + +**Architecture:** Keep health as the aggregation layer. It should summarize signals that already exist in inventories, document structure, decision logs, and audit docs; it should not perform new OCR classification. + +**Tech Stack:** Python, pytest, OCR health JSON. + +--- + +## File Structure + +- Modify: `paperforge/worker/ocr_health.py` — add summary metrics. +- Modify: `tests/test_ocr_health.py` — health aggregation tests. +- Read: `docs/ocr/high-risk-rule-audit.md` — baseline count source if present. + +--- + +### Task 1: Add Health Summary Metrics Test + +**Files:** +- Modify: `tests/test_ocr_health.py` +- Modify: `paperforge/worker/ocr_health.py` + +- [ ] **Step 1: Write failing uncertainty summary test** + +Add to `tests/test_ocr_health.py`: + +```python +def test_ocr_health_reports_hard_rule_and_uncertainty_summary() -> None: + from paperforge.worker.ocr_document import DocumentStructure, PageLayoutProfile + from paperforge.worker.ocr_health import build_ocr_health + + doc = DocumentStructure(page_layouts={1: PageLayoutProfile(confidence=0.25)}) + doc.tail_boundary_score = {"score": 0.35} + + report = build_ocr_health( + page_count=1, + raw_blocks_count=6, + structured_blocks=[ + {"role": "structured_insert", "insert_score": {"score": 0.35}}, + {"role": "abstract_body"}, {"role": "reference_item"}, {"role": "section_heading"}, {"role": "section_heading"}, + ], + figure_inventory={ + "matched_figures": [{"caption_score": {"score": 0.3}}], + "ambiguous_figures": [{"legend_block_id": "cap1"}], + "unresolved_clusters": [{"cluster_id": "unresolved_cluster_001"}], + }, + table_inventory={"tables": [{"match_status": "ambiguous", "match_score": {"score": 0.5}, "has_asset": False, "is_continuation": False}]}, + doc_structure=doc, + ) + + assert report["low_score_but_matched_count"] >= 1 + assert report["ambiguous_match_count"] >= 2 + assert report["unresolved_cluster_count"] == 1 + assert report["candidate_forced_count"] >= 1 + assert report["low_tail_boundary_confidence"] is True +``` + +- [ ] **Step 2: Run test to verify failure** + +Run: + +```bash +python -m pytest tests/test_ocr_health.py -k hard_rule_and_uncertainty_summary -v --tb=short +``` + +Expected: FAIL with missing keys. + +--- + +### Task 2: Implement Aggregation Metrics + +**Files:** +- Modify: `paperforge/worker/ocr_health.py` + +- [ ] **Step 1: Add figure uncertainty counts** + +In `build_ocr_health()`, after figure score collection, add: + +```python +ambiguous_figure_match_count = len(figure_inventory.get("ambiguous_figures", [])) +unresolved_cluster_count = len(figure_inventory.get("unresolved_clusters", [])) +low_score_matched_figures = sum( + 1 for mf in figure_inventory.get("matched_figures", []) + if float(mf.get("caption_score", {}).get("score", 1.0)) < 0.4 +) +``` + +- [ ] **Step 2: Add table uncertainty counts** + +Add: + +```python +ambiguous_table_match_count = sum(1 for t in tables if t.get("match_status") == "ambiguous") +low_score_matched_tables = sum( + 1 for t in tables + if t.get("has_asset") and float(t.get("match_score", {}).get("score", 1.0)) < 0.4 +) +``` + +- [ ] **Step 3: Add insert and tail counts** + +Add: + +```python +candidate_forced_count = sum( + 1 for b in structured_blocks + if b.get("role") == "structured_insert" and float(b.get("insert_score", {}).get("score", 1.0)) < 0.7 +) +low_tail_boundary_confidence = tail_score.get("score", 1.0) < 0.4 +``` + +- [ ] **Step 4: Add values to `report`** + +Add: + +```python +"low_score_but_matched_count": low_score_matched_figures + low_score_matched_tables, +"ambiguous_match_count": ambiguous_figure_match_count + ambiguous_table_match_count, +"ambiguous_figure_match_count": ambiguous_figure_match_count, +"ambiguous_table_match_count": ambiguous_table_match_count, +"unresolved_cluster_count": unresolved_cluster_count, +"candidate_forced_count": candidate_forced_count, +"low_tail_boundary_confidence": low_tail_boundary_confidence, +``` + +- [ ] **Step 5: Run focused health test** + +Run: + +```bash +python -m pytest tests/test_ocr_health.py -k hard_rule_and_uncertainty_summary -q --tb=short +``` + +Expected: PASS. + +--- + +### Task 3: Add Audit Baseline Count Fallback + +**Files:** +- Modify: `tests/test_ocr_health.py` +- Modify: `paperforge/worker/ocr_health.py` + +- [ ] **Step 1: Write deterministic default test** + +Add: + +```python +def test_ocr_health_has_hard_rule_decision_count_key() -> None: + from paperforge.worker.ocr_health import build_ocr_health + + report = build_ocr_health( + page_count=1, + raw_blocks_count=1, + structured_blocks=[{"role": "abstract_body"}, {"role": "reference_item"}, {"role": "section_heading"}, {"role": "section_heading"}], + figure_inventory={}, + table_inventory={}, + ) + + assert "hard_rule_decision_count" in report + assert isinstance(report["hard_rule_decision_count"], int) +``` + +- [ ] **Step 2: Run test to verify failure** + +Run: + +```bash +python -m pytest tests/test_ocr_health.py -k hard_rule_decision_count_key -v --tb=short +``` + +Expected: FAIL with missing key. + +- [ ] **Step 3: Add a safe default count** + +In `build_ocr_health()`, add: + +```python +hard_rule_decision_count = 0 +``` + +Add to `report`: + +```python +"hard_rule_decision_count": hard_rule_decision_count, +``` + +Do not make health parse Markdown in this step. If the audit count needs to become dynamic, add a later small parser with tests. + +- [ ] **Step 4: Run health tests** + +Run: + +```bash +python -m pytest tests/test_ocr_health.py -q --tb=short +``` + +Expected: PASS. + +--- + +## Verification + +Run after all tasks: + +```bash +python -m pytest tests/test_ocr_health.py tests/test_ocr_figures.py tests/test_ocr_tables.py tests/test_ocr_document.py -q --tb=short +``` + +Expected: PASS. + +Do not commit unless the user explicitly requests a commit. diff --git a/docs/superpowers/plans/2026-06-08-ocr-high-risk-rule-audit-plan.md b/docs/superpowers/plans/2026-06-08-ocr-high-risk-rule-audit-plan.md new file mode 100644 index 00000000..3a2db9ba --- /dev/null +++ b/docs/superpowers/plans/2026-06-08-ocr-high-risk-rule-audit-plan.md @@ -0,0 +1,247 @@ +# OCR High Risk Rule Audit Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Produce a concrete audit of OCR hard-rule decisions that can become the baseline for later scorer gates and health summaries. + +**Architecture:** This is a docs-and-test plan. It adds a deterministic audit report generator in tests/docs rather than changing OCR behavior, so later plans can target exact production seams. + +**Tech Stack:** Python, pytest, Markdown docs, PaperForge OCR worker modules. + +--- + +## File Structure + +- Create: `docs/ocr/high-risk-rule-audit.md` — human-readable audit of hard rules, production paths, and remediation mapping. +- Create: `tests/test_ocr_high_risk_rule_audit.py` — locks the audit file contract and required sections. +- Modify: `docs/superpowers/plans/2026-06-08-ocr-v1-convergence-master-plan.md` — add a cross-reference to the audit after the report exists. + +Do not change OCR behavior in this plan. + +--- + +### Task 1: Lock Audit Report Contract + +**Files:** +- Create: `tests/test_ocr_high_risk_rule_audit.py` +- Create: `docs/ocr/high-risk-rule-audit.md` + +- [ ] **Step 1: Write the failing report contract test** + +Create `tests/test_ocr_high_risk_rule_audit.py`: + +```python +from __future__ import annotations + +from pathlib import Path + + +def test_high_risk_rule_audit_has_required_sections() -> None: + report = Path("docs/ocr/high-risk-rule-audit.md") + assert report.exists() + text = report.read_text(encoding="utf-8") + + required = [ + "## Production OCR Chain", + "## Direct Role Mutations", + "## Direct Object Matches", + "## Direct Reorder Decisions", + "## Renderer Inference", + "## Remediation Map", + "## Baseline Counts", + ] + for heading in required: + assert heading in text + + for metric in [ + "direct_role_mutation_count", + "direct_object_match_count", + "direct_reorder_decision_count", + "direct_renderer_inference_count", + ]: + assert metric in text +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: + +```bash +python -m pytest tests/test_ocr_high_risk_rule_audit.py -v --tb=short +``` + +Expected: FAIL because `docs/ocr/high-risk-rule-audit.md` does not exist or lacks sections. + +- [ ] **Step 3: Create the audit report skeleton with concrete production files** + +Create `docs/ocr/high-risk-rule-audit.md` with this exact skeleton. Task 2 replaces the zero baseline counts with audited counts and adds rows for each inspected rule family: + +```markdown +# OCR High Risk Rule Audit + +## Production OCR Chain + +`paperforge/worker/ocr.py` is the production orchestrator. The production artifact chain is: + +`result.json -> blocks.raw.jsonl -> blocks.structured.jsonl -> document_structure.json -> figure_inventory.json/table_inventory.json -> objects -> render/fulltext.md -> health/ocr_health.json` + +Production modules: + +- `paperforge/worker/ocr_blocks.py` +- `paperforge/worker/ocr_roles.py` +- `paperforge/worker/ocr_document.py` +- `paperforge/worker/ocr_figures.py` +- `paperforge/worker/ocr_tables.py` +- `paperforge/worker/ocr_objects.py` +- `paperforge/worker/ocr_render.py` +- `paperforge/worker/ocr_health.py` + +## Direct Role Mutations + +| File | Symbol | Rule | Current Risk | Remediation | +| --- | --- | --- | --- | --- | + +## Direct Object Matches + +| File | Symbol | Rule | Current Risk | Remediation | +| --- | --- | --- | --- | --- | + +## Direct Reorder Decisions + +| File | Symbol | Rule | Current Risk | Remediation | +| --- | --- | --- | --- | --- | + +## Renderer Inference + +| File | Symbol | Rule | Current Risk | Remediation | +| --- | --- | --- | --- | --- | + +## Remediation Map + +| Rule Family | Plan | Action | +| --- | --- | --- | +| Figure asset matching | `2026-06-08-ocr-figure-score-matching-plan.md` | Convert nearest-asset choice into scored candidate selection. | +| Table asset matching | `2026-06-08-ocr-table-score-matching-plan.md` | Convert vertical-nearest choice into scored candidate selection. | +| Layout column inference | `2026-06-08-ocr-layout-confidence-plan.md` | Add confidence and low-confidence reorder guard. | +| Structured insert promotion | `2026-06-08-ocr-insert-score-plan.md` | Convert direct promotion into score/candidate behavior. | +| Health aggregation | `2026-06-08-ocr-health-hard-rule-summary-plan.md` | Report remaining hard-rule and uncertainty counts. | + +## Baseline Counts + +- `direct_role_mutation_count`: 0 +- `direct_object_match_count`: 0 +- `direct_reorder_decision_count`: 0 +- `direct_renderer_inference_count`: 0 +``` + +- [ ] **Step 4: Run the contract test** + +Run: + +```bash +python -m pytest tests/test_ocr_high_risk_rule_audit.py -q --tb=short +``` + +Expected: PASS after the report exists with required headings and metrics. + +--- + +### Task 2: Fill Concrete Hard-Rule Findings + +**Files:** +- Modify: `docs/ocr/high-risk-rule-audit.md` + +- [ ] **Step 1: Inspect direct mutation and match seams** + +Use these source locations as the starting checklist: + +```text +paperforge/worker/ocr_document.py:_build_region_prepass +paperforge/worker/ocr_document.py:_detect_structured_insert_clusters +paperforge/worker/ocr_document.py:_expand_structured_insert_cluster_with_mixed_sidebar_blocks +paperforge/worker/ocr_document.py:build_document_structure +paperforge/worker/ocr_figures.py:build_figure_inventory +paperforge/worker/ocr_tables.py:_pick_best_asset +paperforge/worker/ocr_tables.py:build_table_inventory +paperforge/worker/ocr_render.py +paperforge/worker/ocr_health.py +``` + +- [ ] **Step 2: Add known direct role mutation rows** + +In `docs/ocr/high-risk-rule-audit.md`, add rows like: + +```markdown +| `ocr_document.py` | `_build_region_prepass` | `_in_visual_container` can classify a block as `structured_insert` with confidence `0.85`. | Visual container is treated as a conclusion instead of evidence. | Move to `ocr-insert-score`; visual container becomes one evidence term. | +| `ocr_document.py` | `_build_region_prepass` | `page <= 3` plus key-points/highlights or box anchor can classify `structured_insert`. | Publisher-template sensitive; can swallow body text. | Move to `ocr-insert-score`; keep medium confidence as candidate. | +| `ocr_document.py` | `_build_region_prepass` | `last_insert_on_page` can continue short blocks into `structured_insert`. | Sequential propagation from one early mistake. | Move to `ocr-insert-score`; require cluster coherence and expansion audit. | +``` + +- [ ] **Step 3: Add known direct object match rows** + +Add rows like: + +```markdown +| `ocr_figures.py` | `build_figure_inventory` | Same-page nearest asset is assigned to a legend before `caption_score` is computed. | Nearest media can be wrong when multiple figures/assets exist. | Move to `ocr-figure-score-matching`; score all candidates first. | +| `ocr_tables.py` | `_pick_best_asset` | Candidate table asset is selected by vertical distance. | Previous-page, continuation, and rotated tables can misbind. | Move to `ocr-table-score-matching`; select by `score_table_match`. | +``` + +- [ ] **Step 4: Add baseline counts matching the rows** + +Update the `## Baseline Counts` section. Count only the rows that are documented in this audit. Example format: + +```markdown +- `direct_role_mutation_count`: 3 +- `direct_object_match_count`: 2 +- `direct_reorder_decision_count`: 1 +- `direct_renderer_inference_count`: 0 +``` + +- [ ] **Step 5: Run the audit contract test** + +Run: + +```bash +python -m pytest tests/test_ocr_high_risk_rule_audit.py -q --tb=short +``` + +Expected: PASS. + +--- + +### Task 3: Cross-Reference the Audit From the Master Plan + +**Files:** +- Modify: `docs/superpowers/plans/2026-06-08-ocr-v1-convergence-master-plan.md` + +- [ ] **Step 1: Add the audit report to the final merge gate** + +Add this bullet near the existing route-audit and health gates: + +```markdown +- [ ] `docs/ocr/high-risk-rule-audit.md` documents remaining direct hard-rule decisions and maps each one to scorer, guard, removal, or P2 deferral. +``` + +- [ ] **Step 2: Run a focused markdown contract check** + +Run: + +```bash +python -m pytest tests/test_ocr_high_risk_rule_audit.py -q --tb=short +``` + +Expected: PASS. + +--- + +## Verification + +Run after all tasks: + +```bash +python -m pytest tests/test_ocr_high_risk_rule_audit.py -q --tb=short +``` + +Expected: PASS. + +Do not commit unless the user explicitly requests a commit. diff --git a/docs/superpowers/plans/2026-06-08-ocr-insert-score-plan.md b/docs/superpowers/plans/2026-06-08-ocr-insert-score-plan.md new file mode 100644 index 00000000..d44fb3fd --- /dev/null +++ b/docs/superpowers/plans/2026-06-08-ocr-insert-score-plan.md @@ -0,0 +1,279 @@ +# OCR Insert Score Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace direct structured insert promotion with score, evidence, candidate states, and conservative fallbacks. + +**Architecture:** Add a structured insert scorer in `ocr_scores.py` and make `ocr_document.py` consume it during region prepass and promotion. Visual container, page-1 keywords, and continuation become evidence instead of direct conclusions. + +**Tech Stack:** Python, pytest, OCR decision log metadata. + +--- + +## File Structure + +- Modify: `paperforge/worker/ocr_scores.py` — add `score_structured_insert()`. +- Modify: `paperforge/worker/ocr_document.py` — score insert candidates and avoid low-score promotion. +- Modify: `paperforge/worker/ocr_health.py` — count low-confidence and forced insert candidates. +- Test: `tests/test_ocr_scores.py` — scorer unit tests. +- Test: `tests/test_ocr_document.py` — role mutation behavior tests. +- Test: `tests/test_ocr_health.py` — health counts. + +--- + +### Task 1: Add Structured Insert Scorer + +**Files:** +- Modify: `tests/test_ocr_scores.py` +- Modify: `paperforge/worker/ocr_scores.py` + +- [ ] **Step 1: Write failing scorer tests** + +Add to `tests/test_ocr_scores.py`: + +```python +def test_structured_insert_score_uses_multiple_evidence_terms() -> None: + from paperforge.worker.ocr_scores import score_structured_insert + + block = {"text": "Box 1. Key points", "role": "body_paragraph", "_in_visual_container": True, "bbox": [50, 100, 400, 180], "page_width": 1200} + + result = score_structured_insert(block, body_spine_match=False, cluster_coherent=True) + + assert result["decision"] == "structured_insert" + assert result["score"] >= 0.7 + assert "visual_container" in result["evidence"] + assert "box_or_summary_keyword" in result["evidence"] + + +def test_structured_insert_score_keeps_visual_container_alone_as_candidate() -> None: + from paperforge.worker.ocr_scores import score_structured_insert + + block = {"text": "Ordinary paragraph text", "role": "body_paragraph", "_in_visual_container": True, "bbox": [100, 100, 900, 180], "page_width": 1200} + + result = score_structured_insert(block, body_spine_match=True, cluster_coherent=False) + + assert result["decision"] != "structured_insert" + assert result["score"] < 0.7 +``` + +- [ ] **Step 2: Run tests to verify failure** + +Run: + +```bash +python -m pytest tests/test_ocr_scores.py -k structured_insert_score -v --tb=short +``` + +Expected: FAIL with missing `score_structured_insert`. + +- [ ] **Step 3: Implement scorer** + +Add to `paperforge/worker/ocr_scores.py`: + +```python +def score_structured_insert( + block: dict, + *, + body_spine_match: bool = False, + cluster_coherent: bool = False, +) -> dict: + text = str(block.get("text") or block.get("block_content") or "").strip().lower() + bbox = block.get("bbox") or block.get("block_bbox") or [0, 0, 0, 0] + page_width = float(block.get("page_width") or 1200) + score = 0.0 + evidence: list[str] = [] + + if block.get("_in_visual_container"): + score += 0.3 + evidence.append("visual_container") + if re.match(r"^box\s*\.?\s*\d+\b", text) or "key point" in text or text in {"sections", "highlights"}: + score += 0.3 + evidence.append("box_or_summary_keyword") + if len(bbox) >= 4 and (bbox[2] - bbox[0]) < page_width * 0.45: + score += 0.15 + evidence.append("narrow_width") + if cluster_coherent: + score += 0.15 + evidence.append("cluster_coherent") + if body_spine_match: + score -= 0.25 + evidence.append("body_spine_match") + + score = max(0.0, min(1.0, score)) + if score >= 0.7: + decision = "structured_insert" + elif score >= 0.4: + decision = "structured_insert_candidate" + else: + decision = "body" + return {"score": score, "decision": decision, "evidence": evidence} +``` + +- [ ] **Step 4: Run scorer tests** + +Run: + +```bash +python -m pytest tests/test_ocr_scores.py -k structured_insert_score -q --tb=short +``` + +Expected: PASS. + +--- + +### Task 2: Make Region Prepass Use Insert Score + +**Files:** +- Modify: `tests/test_ocr_document.py` +- Modify: `paperforge/worker/ocr_document.py` + +- [ ] **Step 1: Write failing visual-container-alone test** + +Add to `tests/test_ocr_document.py`: + +```python +def test_visual_container_alone_does_not_force_structured_insert() -> None: + from paperforge.worker.ocr_document import normalize_document_structure + + blocks = [ + {"block_id": "b1", "role": "body_paragraph", "text": "Ordinary paragraph", "page": 2, "bbox": [100, 100, 900, 160], "page_width": 1200, "_in_visual_container": True, "_container_bbox": [90, 90, 910, 170]}, + ] + + _doc, normalized = normalize_document_structure(blocks) + + assert normalized[0]["role"] != "structured_insert" + assert normalized[0].get("insert_score", {}).get("decision") in {"structured_insert_candidate", "body"} +``` + +- [ ] **Step 2: Run test to verify failure** + +Run: + +```bash +python -m pytest tests/test_ocr_document.py -k visual_container_alone -v --tb=short +``` + +Expected: FAIL because region prepass currently treats visual container as strong insert evidence. + +- [ ] **Step 3: Import scorer** + +In `paperforge/worker/ocr_document.py`, add: + +```python +from paperforge.worker.ocr_scores import score_structured_insert +``` + +- [ ] **Step 4: Store insert score in `_build_region_prepass()`** + +Inside `_build_region_prepass()`, before final region assignment, compute: + +```python +insert_score = score_structured_insert(block, body_spine_match=False, cluster_coherent=last_insert_on_page) +block["insert_score"] = insert_score +``` + +Replace direct `region = "structured_insert"` assignments with: + +```python +if insert_score["decision"] == "structured_insert": + region = "structured_insert" + confidence = insert_score["score"] +elif insert_score["decision"] == "structured_insert_candidate": + region = "body" + confidence = insert_score["score"] +``` + +Keep frontmatter logic unchanged. + +- [ ] **Step 5: Run focused document test** + +Run: + +```bash +python -m pytest tests/test_ocr_document.py -k visual_container_alone -q --tb=short +``` + +Expected: PASS. + +--- + +### Task 3: Add Insert Health Counts + +**Files:** +- Modify: `tests/test_ocr_health.py` +- Modify: `paperforge/worker/ocr_health.py` + +- [ ] **Step 1: Write failing health test** + +Add: + +```python +def test_ocr_health_counts_low_confidence_insert_candidates() -> None: + from paperforge.worker.ocr_health import build_ocr_health + + report = build_ocr_health( + page_count=1, + raw_blocks_count=2, + structured_blocks=[ + {"role": "structured_insert_candidate", "insert_score": {"score": 0.45, "decision": "structured_insert_candidate"}}, + {"role": "structured_insert", "insert_score": {"score": 0.35, "decision": "body"}}, + {"role": "abstract_body"}, {"role": "reference_item"}, {"role": "section_heading"}, {"role": "section_heading"}, + ], + figure_inventory={}, + table_inventory={}, + ) + + assert report["low_confidence_insert_candidate_count"] == 1 + assert report["candidate_forced_count"] == 1 +``` + +- [ ] **Step 2: Run test to verify failure** + +Run: + +```bash +python -m pytest tests/test_ocr_health.py -k low_confidence_insert_candidates -v --tb=short +``` + +Expected: FAIL with missing metrics. + +- [ ] **Step 3: Add health metrics** + +In `build_ocr_health()`, add: + +```python +low_confidence_insert_candidate_count = sum( + 1 for b in structured_blocks + if b.get("role") == "structured_insert_candidate" and float(b.get("insert_score", {}).get("score", 0.0)) < 0.7 +) +candidate_forced_count = sum( + 1 for b in structured_blocks + if b.get("role") == "structured_insert" and float(b.get("insert_score", {}).get("score", 1.0)) < 0.7 +) +``` + +Add both values to `report`. + +- [ ] **Step 4: Run health test** + +Run: + +```bash +python -m pytest tests/test_ocr_health.py -k low_confidence_insert_candidates -q --tb=short +``` + +Expected: PASS. + +--- + +## Verification + +Run after all tasks: + +```bash +python -m pytest tests/test_ocr_scores.py tests/test_ocr_document.py tests/test_ocr_health.py -q --tb=short +``` + +Expected: PASS. + +Do not commit unless the user explicitly requests a commit. diff --git a/docs/superpowers/plans/2026-06-08-ocr-layout-confidence-plan.md b/docs/superpowers/plans/2026-06-08-ocr-layout-confidence-plan.md new file mode 100644 index 00000000..fc63af2f --- /dev/null +++ b/docs/superpowers/plans/2026-06-08-ocr-layout-confidence-plan.md @@ -0,0 +1,344 @@ +# OCR Layout Confidence Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add layout confidence so low-confidence column detection cannot drive strong reading-order or tail reorder behavior. + +**Architecture:** Extend `PageLayoutProfile` with confidence/evidence and build profiles from eligible body-like blocks. Document structure and health should expose confidence; render/order code should treat low confidence as a guard. + +**Tech Stack:** Python dataclasses, pytest, OCR document structure JSON. + +--- + +## File Structure + +- Modify: `paperforge/worker/ocr_document.py` — profile dataclass, eligible block filtering, confidence scoring, low-confidence guards. +- Modify: `paperforge/worker/ocr_health.py` — layout confidence distribution. +- Test: `tests/test_ocr_document.py` — profile behavior and serialization. +- Test: `tests/test_ocr_health.py` — health metrics. + +--- + +### Task 1: Extend PageLayoutProfile Contract + +**Files:** +- Modify: `tests/test_ocr_document.py` +- Modify: `paperforge/worker/ocr_document.py` + +- [ ] **Step 1: Write failing profile contract test** + +Add to `tests/test_ocr_document.py`: + +```python +def test_page_layout_profile_includes_confidence_and_evidence() -> None: + from paperforge.worker.ocr_document import _build_page_layout_profiles + + blocks = [ + {"role": "body_paragraph", "page": 1, "bbox": [100, 100, 500, 160], "page_width": 1200, "page_height": 1700}, + {"role": "body_paragraph", "page": 1, "bbox": [700, 100, 1100, 160], "page_width": 1200, "page_height": 1700}, + ] + + profile = _build_page_layout_profiles(blocks)[1] + + assert hasattr(profile, "confidence") + assert hasattr(profile, "evidence") + assert profile.confidence >= 0.5 + assert "eligible_body_blocks" in profile.evidence +``` + +- [ ] **Step 2: Run test to verify failure** + +Run: + +```bash +python -m pytest tests/test_ocr_document.py -k page_layout_profile_includes_confidence -v --tb=short +``` + +Expected: FAIL because `PageLayoutProfile` lacks `confidence` and `evidence`. + +- [ ] **Step 3: Extend the dataclass** + +In `paperforge/worker/ocr_document.py`, update `PageLayoutProfile`: + +```python +@dataclass +class PageLayoutProfile: + column_count: int = 1 + column_boundaries: list[float] = field(default_factory=list) + layout_type: str = "single_column" + confidence: float = 0.5 + evidence: list[str] = field(default_factory=list) +``` + +- [ ] **Step 4: Update `_classify_page_layout()` returns** + +Every `PageLayoutProfile(...)` return should set confidence/evidence. Use these defaults: + +```python +PageLayoutProfile(column_count=1, column_boundaries=centers, layout_type="single_column", confidence=0.7, evidence=["eligible_body_blocks"]) +``` + +For mixed or uncertain layouts, use confidence between `0.4` and `0.6` with evidence such as `"wide_dispersion"` or `"two_center_clusters"`. + +- [ ] **Step 5: Run focused profile test** + +Run: + +```bash +python -m pytest tests/test_ocr_document.py -k page_layout_profile_includes_confidence -q --tb=short +``` + +Expected: PASS. + +--- + +### Task 2: Filter Layout Inputs to Body-Like Blocks + +**Files:** +- Modify: `tests/test_ocr_document.py` +- Modify: `paperforge/worker/ocr_document.py` + +- [ ] **Step 1: Write failing wide-heading pollution test** + +Add: + +```python +def test_layout_profiles_ignore_wide_headings_and_media() -> None: + from paperforge.worker.ocr_document import _build_page_layout_profiles + + blocks = [ + {"role": "section_heading", "page": 1, "bbox": [50, 50, 1150, 90], "page_width": 1200, "page_height": 1700}, + {"role": "figure_asset", "page": 1, "bbox": [400, 120, 800, 500], "page_width": 1200, "page_height": 1700}, + {"role": "body_paragraph", "page": 1, "bbox": [100, 600, 500, 660], "page_width": 1200, "page_height": 1700}, + {"role": "body_paragraph", "page": 1, "bbox": [700, 600, 1100, 660], "page_width": 1200, "page_height": 1700}, + ] + + profile = _build_page_layout_profiles(blocks)[1] + + assert profile.column_count == 2 + assert "excluded_non_body_blocks" in profile.evidence +``` + +- [ ] **Step 2: Run test to verify failure** + +Run: + +```bash +python -m pytest tests/test_ocr_document.py -k ignore_wide_headings -v --tb=short +``` + +Expected: FAIL if non-body blocks still pollute layout classification. + +- [ ] **Step 3: Add eligible layout filter** + +In `ocr_document.py`, add: + +```python +_LAYOUT_ELIGIBLE_ROLES = {"body_paragraph", "list_item", "tail_candidate_body", "reference_item", "backmatter_body"} + + +def _is_layout_eligible_block(block: dict) -> bool: + role = block.get("role", "") + if role not in _LAYOUT_ELIGIBLE_ROLES: + return False + bbox = block.get("bbox") or block.get("block_bbox") or [] + page_width = float(block.get("page_width") or 1200) + if len(bbox) >= 4 and (bbox[2] - bbox[0]) > page_width * 0.85: + return False + return True +``` + +- [ ] **Step 4: Use the filter in `_build_page_layout_profiles()`** + +Before calling `_classify_page_layout()`, build `eligible_blocks`: + +```python +eligible_blocks = [block for block in page_blocks if _is_layout_eligible_block(block)] +excluded_count = len(page_blocks) - len(eligible_blocks) +profile = _classify_page_layout(eligible_blocks, page_width, page_height) +if excluded_count: + profile.evidence.append("excluded_non_body_blocks") +if len(eligible_blocks) < 2: + profile.confidence = min(profile.confidence, 0.35) + profile.evidence.append("few_eligible_blocks") +``` + +- [ ] **Step 5: Run focused layout tests** + +Run: + +```bash +python -m pytest tests/test_ocr_document.py -k "page_layout_profile_includes_confidence or ignore_wide_headings" -q --tb=short +``` + +Expected: PASS. + +--- + +### Task 3: Add Health Layout Confidence Distribution + +**Files:** +- Modify: `tests/test_ocr_health.py` +- Modify: `paperforge/worker/ocr_health.py` + +- [ ] **Step 1: Write failing health test** + +Add: + +```python +def test_ocr_health_reports_layout_confidence_distribution() -> None: + from paperforge.worker.ocr_document import DocumentStructure, PageLayoutProfile + from paperforge.worker.ocr_health import build_ocr_health + + doc = DocumentStructure(page_layouts={ + 1: PageLayoutProfile(confidence=0.8), + 2: PageLayoutProfile(confidence=0.5), + 3: PageLayoutProfile(confidence=0.2), + }) + + report = build_ocr_health( + page_count=3, + raw_blocks_count=3, + structured_blocks=[{"role": "abstract_body"}, {"role": "reference_item"}, {"role": "section_heading"}, {"role": "section_heading"}], + figure_inventory={}, + table_inventory={}, + doc_structure=doc, + ) + + assert report["layout_confidence_distribution"] == {"high": 1, "medium": 1, "low": 1} +``` + +- [ ] **Step 2: Run test to verify failure** + +Run: + +```bash +python -m pytest tests/test_ocr_health.py -k layout_confidence_distribution -v --tb=short +``` + +Expected: FAIL with missing key. + +- [ ] **Step 3: Add distribution to health** + +In `build_ocr_health()`, after `report` creation, add: + +```python +layout_confidences = [] +if doc_structure is not None and getattr(doc_structure, "page_layouts", None): + layout_confidences = [float(p.confidence) for p in doc_structure.page_layouts.values()] +report["layout_confidence_distribution"] = _score_distribution(layout_confidences) +``` + +Move `_score_distribution()` above this use if it is currently defined later in the function. + +- [ ] **Step 4: Run health test** + +Run: + +```bash +python -m pytest tests/test_ocr_health.py -k layout_confidence_distribution -q --tb=short +``` + +Expected: PASS. + +--- + +### Task 4: Guard Tail Reorder With Layout and Tail Confidence + +**Files:** +- Modify: `tests/test_ocr_rendering.py` +- Modify: `paperforge/worker/ocr_render.py` + +- [ ] **Step 1: Write failing low-confidence tail render test** + +Add to `tests/test_ocr_rendering.py`: + +```python +def test_render_skips_segment_tail_reorder_when_tail_confidence_is_low() -> None: + from paperforge.worker.ocr_document import DocumentStructure + from paperforge.worker.ocr_render import render_fulltext + + blocks = [ + {"block_id": "b1", "role": "body_paragraph", "text": "First tail block", "page": 3, "bbox": [700, 100, 1100, 150]}, + {"block_id": "b2", "role": "body_paragraph", "text": "Second tail block", "page": 3, "bbox": [100, 100, 500, 150]}, + ] + doc = DocumentStructure(spread_start=3, spread_end=3) + doc.tail_boundary_score = {"score": 0.2} + doc.tail_reading_order = [ + {"page": 3, "column_index": 0, "y_top": 100, "y_bottom": 150, "block_indices": [1]}, + {"page": 3, "column_index": 1, "y_top": 100, "y_bottom": 150, "block_indices": [0]}, + ] + + markdown = render_fulltext(blocks, document_structure=doc) + + assert markdown.index("First tail block") < markdown.index("Second tail block") +``` + +- [ ] **Step 2: Run test to verify failure** + +Run: + +```bash +python -m pytest tests/test_ocr_rendering.py -k skips_segment_tail_reorder -v --tb=short +``` + +Expected: FAIL if renderer still applies `tail_reading_order` when `tail_boundary_score.score < 0.4`. + +- [ ] **Step 3: Add confidence guard helper** + +In `paperforge/worker/ocr_render.py`, add near `_has_tail_role()`: + +```python +def _can_apply_tail_segment_reorder(document_structure: DocumentStructure | None) -> bool: + if document_structure is None: + return False + if not document_structure.tail_reading_order or document_structure.spread_start is None: + return False + tail_score = getattr(document_structure, "tail_boundary_score", {}) or {} + if float(tail_score.get("score", 1.0)) < 0.4: + return False + page_layouts = getattr(document_structure, "page_layouts", None) or {} + for page in range(document_structure.spread_start, (document_structure.spread_end or document_structure.spread_start) + 1): + profile = page_layouts.get(page) + if profile is not None and float(getattr(profile, "confidence", 1.0)) < 0.4: + return False + return True +``` + +- [ ] **Step 4: Use the guard in render ordering** + +Replace this condition in `render_fulltext()`: + +```python +if document_structure and document_structure.tail_reading_order and document_structure.spread_start is not None: +``` + +with: + +```python +if _can_apply_tail_segment_reorder(document_structure): +``` + +- [ ] **Step 5: Run focused render test** + +Run: + +```bash +python -m pytest tests/test_ocr_rendering.py -k skips_segment_tail_reorder -q --tb=short +``` + +Expected: PASS. + +--- + +## Verification + +Run after all tasks: + +```bash +python -m pytest tests/test_ocr_document.py tests/test_ocr_health.py tests/test_ocr_rendering.py -q --tb=short +``` + +Expected: PASS. + +Do not commit unless the user explicitly requests a commit. diff --git a/docs/superpowers/plans/2026-06-08-ocr-table-score-matching-plan.md b/docs/superpowers/plans/2026-06-08-ocr-table-score-matching-plan.md new file mode 100644 index 00000000..ede1126b --- /dev/null +++ b/docs/superpowers/plans/2026-06-08-ocr-table-score-matching-plan.md @@ -0,0 +1,267 @@ +# OCR Table Score Matching Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make table asset selection use `score_table_match()` instead of vertical-nearest selection. + +**Architecture:** Keep `ocr_tables.py` as the table inventory owner. Expand candidate pages to previous/current/next, score all candidates, choose only when top score and score separation are safe, and expose match status for health/render consumers. + +**Tech Stack:** Python, pytest, PaperForge table inventory JSON. + +--- + +## File Structure + +- Modify: `paperforge/worker/ocr_tables.py` — replace `_pick_best_asset()` with score-based selection. +- Modify: `paperforge/worker/ocr_scores.py` — extend `score_table_match()` if previous-page or caption-below evidence is missing. +- Modify: `paperforge/worker/ocr_health.py` — count low-confidence and ambiguous table matches. +- Test: `tests/test_ocr_tables.py` — score-based table selection tests. +- Test: `tests/test_ocr_scores.py` — table score evidence tests. + +--- + +### Task 1: Add Previous-Page and Ambiguous Table Tests + +**Files:** +- Modify: `tests/test_ocr_tables.py` + +- [ ] **Step 1: Write failing previous-page match test** + +Add to `tests/test_ocr_tables.py`: + +```python +def test_table_inventory_considers_previous_page_assets() -> None: + from paperforge.worker.ocr_tables import build_table_inventory + + blocks = [ + {"block_id": "asset1", "role": "table_asset", "page": 1, "bbox": [100, 900, 700, 1200]}, + {"block_id": "cap1", "role": "table_caption", "page": 2, "text": "Table 1. Baseline characteristics", "bbox": [100, 80, 700, 120]}, + ] + + inventory = build_table_inventory(blocks) + + table = inventory["tables"][0] + assert table["asset_block_id"] == "asset1" + assert table["match_status"] in {"matched", "matched_low_confidence"} +``` + +- [ ] **Step 2: Write failing ambiguous candidate test** + +Add: + +```python +def test_table_inventory_marks_close_scores_ambiguous() -> None: + from paperforge.worker.ocr_tables import build_table_inventory + + blocks = [ + {"block_id": "cap1", "role": "table_caption", "page": 1, "text": "Table 1. Baseline characteristics", "bbox": [100, 100, 700, 140]}, + {"block_id": "asset1", "role": "table_asset", "page": 1, "bbox": [100, 160, 700, 400]}, + {"block_id": "asset2", "role": "table_asset", "page": 1, "bbox": [105, 165, 705, 405]}, + ] + + inventory = build_table_inventory(blocks) + table = inventory["tables"][0] + assert table["match_status"] == "ambiguous" + assert table["has_asset"] is False + assert len(table["candidate_assets"]) == 2 +``` + +- [ ] **Step 3: Run table tests to verify failure** + +Run: + +```bash +python -m pytest tests/test_ocr_tables.py -k "previous_page_assets or close_scores" -v --tb=short +``` + +Expected: FAIL because current candidate pages exclude previous page and `_pick_best_asset()` chooses one nearest asset. + +--- + +### Task 2: Replace `_pick_best_asset()` With Score-Based Selection + +**Files:** +- Modify: `paperforge/worker/ocr_tables.py` + +- [ ] **Step 1: Add a score-based helper** + +Replace `_compute_asset_score()` and `_pick_best_asset()` with: + +```python +def _score_candidate_assets( + page_assets: list[tuple[int, dict]], + caption: dict, + *, + is_continuation: bool, +) -> list[tuple[int, dict, dict]]: + scored = [ + (i, asset, score_table_match(caption, asset, is_continuation=is_continuation)) + for i, asset in page_assets + ] + scored.sort(key=lambda item: item[2].get("score", 0.0), reverse=True) + return scored +``` + +- [ ] **Step 2: Expand candidate pages** + +In `build_table_inventory()`, replace: + +```python +candidate_pages = [caption_page] if is_cont else [caption_page, caption_page + 1] +``` + +with: + +```python +candidate_pages = [caption_page - 1, caption_page, caption_page + 1] +if is_cont: + candidate_pages = [caption_page - 1, caption_page, caption_page + 1] +``` + +- [ ] **Step 3: Select by score and separation** + +Replace the loop that calls `_pick_best_asset()` with: + +```python +all_candidates: list[tuple[int, dict, dict]] = [] +for page in candidate_pages: + if page < 1: + continue + page_assets = [ + (i, asset) + for i, asset in enumerate(assets) + if i not in used_asset_indices and asset.get("page", 0) == page + ] + all_candidates.extend(_score_candidate_assets(page_assets, caption, is_continuation=is_cont)) + +all_candidates.sort(key=lambda item: item[2].get("score", 0.0), reverse=True) +match_status = "unmatched_caption" +candidate_assets = [ + {"asset_block_id": asset.get("block_id", ""), "match_score": score} + for _, asset, score in all_candidates[:3] +] + +if all_candidates: + top_idx, top_asset, top_score = all_candidates[0] + second_score = all_candidates[1][2].get("score", 0.0) if len(all_candidates) > 1 else -1.0 + if top_score.get("score", 0.0) < 0.4: + matched_asset = None + match_status = "unmatched_caption" + elif top_score.get("score", 0.0) - second_score < 0.15: + matched_asset = None + match_status = "ambiguous" + else: + matched_asset = top_asset + used_asset_indices.add(top_idx) + match_status = "matched" if top_score.get("score", 0.0) >= 0.6 else "matched_low_confidence" +``` + +- [ ] **Step 4: Add status and candidates to table entries** + +In the table dict, add: + +```python +"match_status": match_status, +"candidate_assets": candidate_assets, +``` + +Set `match_score` to the selected top score when matched, or the first candidate score when ambiguous/unmatched: + +```python +"match_score": ( + score_table_match(caption, matched_asset, is_continuation=is_cont) + if matched_asset + else (all_candidates[0][2] if all_candidates else {"score": 0.0, "matched_asset_id": "", "decision": "ambiguous", "evidence": []}) +), +``` + +- [ ] **Step 5: Run focused tests** + +Run: + +```bash +python -m pytest tests/test_ocr_tables.py -k "previous_page_assets or close_scores" -q --tb=short +``` + +Expected: PASS. + +--- + +### Task 3: Add Health Counts for Table Match Status + +**Files:** +- Modify: `tests/test_ocr_health.py` +- Modify: `paperforge/worker/ocr_health.py` + +- [ ] **Step 1: Write failing health test** + +Add to `tests/test_ocr_health.py`: + +```python +def test_ocr_health_counts_ambiguous_and_low_confidence_tables() -> None: + from paperforge.worker.ocr_health import build_ocr_health + + report = build_ocr_health( + page_count=1, + raw_blocks_count=1, + structured_blocks=[{"role": "abstract_body"}, {"role": "reference_item"}, {"role": "section_heading"}, {"role": "section_heading"}], + figure_inventory={}, + table_inventory={"tables": [ + {"has_asset": False, "is_continuation": False, "match_status": "ambiguous", "match_score": {"score": 0.5}}, + {"has_asset": True, "is_continuation": False, "match_status": "matched_low_confidence", "match_score": {"score": 0.45}}, + ]}, + ) + + assert report["ambiguous_table_match_count"] == 1 + assert report["low_confidence_table_match_count"] == 1 +``` + +- [ ] **Step 2: Run test to verify failure** + +Run: + +```bash +python -m pytest tests/test_ocr_health.py -k ambiguous_and_low_confidence_tables -v --tb=short +``` + +Expected: FAIL with missing keys. + +- [ ] **Step 3: Add health metrics** + +In `build_ocr_health()`, after `tables = table_inventory.get("tables", [])`, add: + +```python +ambiguous_table_match_count = sum(1 for t in tables if t.get("match_status") == "ambiguous") +low_confidence_table_match_count = sum(1 for t in tables if t.get("match_status") == "matched_low_confidence") +``` + +Add to `report`: + +```python +"ambiguous_table_match_count": ambiguous_table_match_count, +"low_confidence_table_match_count": low_confidence_table_match_count, +``` + +- [ ] **Step 4: Run health test** + +Run: + +```bash +python -m pytest tests/test_ocr_health.py -k ambiguous_and_low_confidence_tables -q --tb=short +``` + +Expected: PASS. + +--- + +## Verification + +Run after all tasks: + +```bash +python -m pytest tests/test_ocr_scores.py tests/test_ocr_tables.py tests/test_ocr_health.py -q --tb=short +``` + +Expected: PASS. + +Do not commit unless the user explicitly requests a commit. diff --git a/docs/superpowers/specs/2026-06-08-ocr-risk-convergence-skill-plans-design.md b/docs/superpowers/specs/2026-06-08-ocr-risk-convergence-skill-plans-design.md new file mode 100644 index 00000000..4a289b82 --- /dev/null +++ b/docs/superpowers/specs/2026-06-08-ocr-risk-convergence-skill-plans-design.md @@ -0,0 +1,278 @@ +# OCR Risk Convergence Skill Plans Design + +> **Status:** Proposed +> **Date:** 2026-06-08 +> **Target branch:** `feat/ocr-structured-pipeline` +> **Audience:** maintainers, reviewers, agentic implementers + +## 1. Goal + +Turn the OCR structured pipeline risk report into six focused Superpowers implementation plans. The plans must converge the riskiest OCR layout judgments from hard-rule output toward score/evidence-driven decisions with conservative fallbacks. + +The goal is not perfect human-visual reconstruction. The v1 target is safer behavior: + +- confident output only when evidence is strong; +- candidate, ambiguous, unresolved, orphan, or degraded states when evidence is weak; +- health and decision logs that explain uncertainty; +- raw OCR truth preserved for rebuilds; +- renderer behavior that does not hide low-confidence upstream structure. + +## 2. Current Context + +The OCR worktree is: + +`D:/L/Med/Research/99_System/LiteraturePipeline/github-release/.worktrees/feat-ocr-structured-pipeline` + +The branch already contains recent scorer and health work, including: + +- `paperforge/worker/ocr_scores.py` with figure caption, table match, and tail boundary scoring helpers; +- figure/table inventories that carry some score evidence; +- OCR health confidence distributions; +- decision log and error taxonomy work; +- a v1 convergence master plan at `docs/superpowers/plans/2026-06-08-ocr-v1-convergence-master-plan.md`. + +This design should not duplicate that work. It should define the next stricter P1/P2 risk-convergence plans that make scores control decisions instead of acting only as after-the-fact health signals. + +## 3. Non-Goals + +- Do not add visual-model, document-AI, or ML dependencies for OCR v1. +- Do not pursue perfect rendering of every PDF layout. +- Do not use `fulltext.md` as a fact source. +- Do not overwrite or weaken raw artifacts such as `result.json`, `blocks.raw.jsonl`, page image cache, or PDF span metadata. +- Do not broaden OCR heuristics while the hard-rule audit is still incomplete. +- Do not mix these plans into unrelated root-worktree changes. + +## 4. Shared Principles + +Every high-risk OCR decision should move toward this shape: + +```json +{ + "decision": "matched | candidate | ambiguous | rejected | unresolved", + "score": 0.0, + "evidence": [], + "fallback": "conservative behavior when score is low" +} +``` + +Rules that previously produced direct conclusions should become evidence inputs, including: + +- figure or table prefix; +- nearest media asset; +- vertical distance; +- visual container membership; +- x-center gap; +- dense reference item count; +- same-page or adjacent-page proximity; +- page-1 frontmatter zone; +- Zotero/OCR metadata alignment. + +Low confidence must prefer preservation over mutation. If the system is uncertain, it should keep raw order, keep unmatched captions/assets visible, mark candidates, and report health warnings instead of pretending the result is final. + +## 5. Plan Set + +Create six implementation plans under `docs/superpowers/plans/`. + +### 5.1 `2026-06-08-ocr-high-risk-rule-audit-plan.md` + +Goal: inventory direct role mutations and direct object matches before deeper changes. + +Expected outputs: + +- `docs/ocr/high-risk-rule-audit.md`; +- categorized counts for direct role mutation, direct match, direct reorder, and direct renderer inference; +- mapping from each high-risk rule to one of: keep as evidence, guard with score, defer to P2, or remove; +- explicit list of production files and non-production/legacy files. + +Scope: + +- inspect `ocr_document.py`, `ocr_figures.py`, `ocr_tables.py`, `ocr_render.py`, `ocr_health.py`, and adjacent OCR modules; +- do not change behavior except tests/docs required to expose the audit contract; +- do not search vault content directly. + +Acceptance criteria: + +- audit identifies every direct promotion to `structured_insert`, `non_body_insert`, figure/table match, and tail/layout reorder decision; +- audit distinguishes production path from legacy or experimental modules; +- audit becomes the baseline for `hard_rule_decision_count` in health. + +### 5.2 `2026-06-08-ocr-figure-score-matching-plan.md` + +Goal: make figure scoring control matching decisions, not merely annotate inventory after matching. + +Minimum behavior: + +- score all same-page candidate asset clusters for each legend; +- reject confident matching when `caption_score.score < 0.4`; +- produce unmatched legend when best match score is below threshold; +- mark ambiguous when top candidate scores are close; +- allow a legend to match multiple nearby assets when evidence supports a multi-panel cluster; +- preserve orphan assets and unresolved clusters without hiding them in render. + +Expected inventory states: + +- `matched_figures` for high-confidence matches; +- `ambiguous_figures` or equivalent ambiguous match records; +- `unmatched_legends`; +- `unmatched_assets`; +- `unresolved_clusters` with stable IDs. + +Acceptance criteria: + +- every matched figure has score and evidence that participated in selection; +- low-score figure captions cannot create confident figure matches; +- nearest-asset tie cases become ambiguous instead of silently choosing top1; +- renderer shows unresolved/orphan figure information instead of dropping it. + +### 5.3 `2026-06-08-ocr-table-score-matching-plan.md` + +Goal: make `score_table_match()` choose table assets across a conservative page window. + +Minimum behavior: + +- candidate pages include previous, current, and next page; +- all candidate assets receive match scores; +- choose only when top score passes threshold and separation from top2 is sufficient; +- expose `match_status` such as `matched`, `matched_low_confidence`, `ambiguous`, `unmatched_caption`, `unmatched_asset`, or `continuation`; +- keep `truth_source = image` for table objects. + +Acceptance criteria: + +- vertical nearest is no longer the sole selector; +- low-confidence table captions/assets remain visible; +- previous-page, next-page, and continuation cases have tests; +- table health reports low-confidence and ambiguous table counts. + +### 5.4 `2026-06-08-ocr-layout-confidence-plan.md` + +Goal: make page layout profiles carry confidence and prevent low-confidence layout from driving strong reorder behavior. + +Minimum behavior: + +- build column profiles from eligible body-like blocks only; +- exclude media, table, frontmatter, structured insert, noise, and other non-body blocks from column inference; +- add `confidence` and `evidence` to `PageLayoutProfile`; +- serialize layout confidence into `document_structure.json`; +- prevent strong tail/column reorder when layout confidence is low. + +Acceptance criteria: + +- wide headings, figures, tables, and sidebars do not dominate column detection; +- low-confidence pages preserve raw order or use conservative local ordering; +- health reports layout confidence distribution; +- renderer consumes layout confidence instead of inventing new layout semantics. + +### 5.5 `2026-06-08-ocr-insert-score-plan.md` + +Goal: replace direct structured insert promotion with score/candidate/fallback behavior. + +Minimum behavior: + +- add a structured insert scorer using evidence such as visual container, keyword anchor, width mismatch, font mismatch, cluster coherence, page context, and body-spine mismatch; +- treat `_in_visual_container` as evidence, not a conclusion; +- keep medium-confidence blocks as `structured_insert_candidate`; +- avoid promoting low-confidence candidates to `structured_insert`; +- record cluster expansion inputs and added block IDs; +- warn when cluster expansion covers too much probable body text. + +Acceptance criteria: + +- every structured insert promotion has score and evidence in decision log or block metadata; +- low-score candidates do not swallow body paragraphs; +- visual container alone cannot force a final structured insert role; +- health reports low-confidence insert count and candidate-forced count. + +### 5.6 `2026-06-08-ocr-health-hard-rule-summary-plan.md` + +Goal: make OCR health summarize uncertainty and remaining hard-rule decisions across the whole pipeline. + +Minimum metrics: + +- `hard_rule_decision_count`; +- `low_score_but_matched_count`; +- `ambiguous_match_count`; +- `unresolved_cluster_count`; +- `candidate_forced_count`; +- low-confidence figure/table/layout/insert/tail distributions; +- direct renderer inference count, if any remains. + +Acceptance criteria: + +- health tells users which outputs are certain, uncertain, degraded, or unresolved; +- health counts agree with figure/table inventories, document structure, and decision log; +- release gate can fail or warn based on severe uncertainty counts; +- no high-risk scorer silently writes confident output without health visibility. + +## 6. Execution Order + +Run the plans in this order: + +1. `ocr-high-risk-rule-audit` +2. `ocr-figure-score-matching` +3. `ocr-table-score-matching` +4. `ocr-layout-confidence` +5. `ocr-insert-score` +6. `ocr-health-hard-rule-summary` + +Figure and table plans come before layout and insert because object misbinding is the highest-risk user-visible error. Health summary comes last because it should report final semantics after decision behavior changes. + +## 7. Cross-Plan Interfaces + +The plans should use these shared contracts: + +- score objects contain `score`, `decision`, and `evidence`; +- decision log records role mutations and forced/candidate decisions; +- inventories preserve unmatched, ambiguous, and orphan states; +- `document_structure.json` carries layout and tail confidence; +- renderer consumes confidence and inventory states but does not perform new semantic classification; +- health is the aggregation layer, not the only place where uncertainty exists. + +## 8. Testing Strategy + +Each plan should begin with focused failing tests before implementation. Tests should prefer synthetic fixtures unless real PDF behavior is required. + +Required coverage themes: + +- figure nearest-asset ambiguity; +- low-caption-score figure refusal; +- multi-panel figure cluster preservation; +- table previous/current/next page matching; +- table ambiguous top1/top2 scores; +- layout confidence with wide headings and sidebars; +- low-confidence layout preserving raw order; +- structured insert candidate not swallowing body text; +- visual container alone not forcing structured insert; +- health summary consistency with inventories and decision log. + +Recommended verification after each plan: + +```bash +python -m pytest tests/test_ocr_scores.py tests/test_ocr_figures.py tests/test_ocr_tables.py tests/test_ocr_document.py tests/test_ocr_health.py tests/test_ocr_rendering.py -q --tb=short +``` + +Run broader OCR regression before branch merge: + +```bash +python -m pytest tests/test_ocr_roles.py tests/test_ocr_document.py tests/test_ocr_profiles.py tests/test_ocr_metadata.py tests/test_ocr_health.py tests/test_ocr_rendering.py tests/test_ocr_render_stabilization.py tests/test_ocr_figures.py tests/test_ocr_objects.py tests/test_ocr_rebuild.py tests/test_ocr_blocks.py tests/test_ocr_pdf_spans.py tests/test_ocr_tables.py tests/test_ocr_artifacts.py -q --tb=short +``` + +## 9. Merge Gate + +OCR v1 risk convergence is complete when: + +- all figure/table matches have score and evidence used in selection; +- low-score figure/table evidence cannot produce confident matches; +- layout confidence low blocks strong reorder behavior; +- structured insert low scores remain candidates or body, not forced callouts; +- tail/layout confidence controls render behavior; +- health exposes hard rules, low-confidence matches, ambiguous matches, unresolved clusters, and forced candidates; +- decision log can trace high-risk role mutations; +- raw artifacts remain untouched and rebuildable; +- `fulltext.md` remains a structured reading artifact, not a claimed PDF truth source. + +## 10. Open Decisions + +- Exact thresholds should start conservative and be documented in each implementation plan. +- Existing score helper names may be extended rather than replaced if tests preserve current behavior. +- Ambiguous inventory schema can be explicit lists or status fields, but renderer and health must consume it consistently. +- Commit strategy is intentionally unspecified here; agents should not commit unless explicitly instructed. diff --git a/paperforge/worker/ocr_document.py b/paperforge/worker/ocr_document.py index 5a18d747..12cd3e1e 100644 --- a/paperforge/worker/ocr_document.py +++ b/paperforge/worker/ocr_document.py @@ -6,6 +6,7 @@ from collections import namedtuple from dataclasses import dataclass, field from paperforge.worker.ocr_decisions import record_decision +from paperforge.worker.ocr_scores import score_structured_insert from paperforge.worker.ocr_roles import ( _BACKMATTER_TITLE_DENY_LIST, _is_near_figure_media, @@ -49,6 +50,8 @@ class PageLayoutProfile: column_count: int = 1 column_boundaries: list[float] = field(default_factory=list) layout_type: str = "single_column" + confidence: float = 0.5 + evidence: list[str] = field(default_factory=list) @dataclass @@ -129,45 +132,16 @@ def _build_region_prepass(blocks: list[dict]) -> RegionPrepass: region = "body" confidence = 0.5 - in_container = block.get("_in_visual_container", False) - if in_container and len(bbox) >= 4: - cbox = block.get("_container_bbox", []) - within_container = False - if len(cbox) >= 4: - margin_w = (bbox[2] - bbox[0]) * 0.1 - margin_h = (bbox[3] - bbox[1]) * 0.1 - within_container = ( - bbox[0] >= cbox[0] - margin_w - and bbox[2] <= cbox[2] + margin_w - and bbox[1] >= cbox[1] - margin_h - and bbox[3] <= cbox[3] + margin_h - ) - if within_container: - region = "structured_insert" - confidence = 0.85 - last_insert_anchor_kind = "container" - elif (page <= 3 and ("key point" in text or text in {"sections", "highlights"})) or _looks_like_box_anchor(text): - region = "structured_insert" - confidence = 0.8 - last_insert_anchor_kind = "box" if _looks_like_box_anchor(text) else "summary" - elif ( - last_insert_on_page - and page <= 3 - and len(text.split()) <= 18 - and role not in {"section_heading", "subsection_heading", "sub_subsection_heading", "reference_heading", "backmatter_heading"} - ): - region = "structured_insert" - confidence = max(confidence, 0.7) - elif ( - last_insert_on_page - and last_insert_anchor_kind == "box" - and len(bbox) >= 4 - and bbox[0] < page_width * 0.25 - and role in {"section_heading", "subsection_heading", "sub_subsection_heading", "body_paragraph"} - ): - region = "structured_insert" - confidence = max(confidence, 0.75) + insert_score = score_structured_insert(block, body_spine_match=False, cluster_coherent=last_insert_on_page) + block["insert_score"] = insert_score + if insert_score["decision"] == "structured_insert": + region = "structured_insert" + confidence = insert_score["score"] + last_insert_anchor_kind = "container" if block.get("_in_visual_container") else "box" + elif insert_score["decision"] == "structured_insert_candidate": + region = "body" + confidence = insert_score["score"] elif page == 1 and (role in {"paper_title", "authors", "affiliation", "frontmatter_noise"} or y_top < page_height * 0.22): region = "frontmatter" confidence = 0.8 @@ -232,6 +206,20 @@ def _compute_span_coverage(blocks: list[dict]) -> dict: } +_LAYOUT_ELIGIBLE_ROLES = {"body_paragraph", "list_item", "tail_candidate_body", "reference_item", "backmatter_body"} + + +def _is_layout_eligible_block(block: dict) -> bool: + role = block.get("role", "") + if role not in _LAYOUT_ELIGIBLE_ROLES: + return False + bbox = block.get("bbox") or block.get("block_bbox") or [] + page_width = float(block.get("page_width") or 1200) + if len(bbox) >= 4 and (bbox[2] - bbox[0]) > page_width * 0.85: + return False + return True + + def _cluster_page_columns(page_blocks: list[dict], page_width: float) -> list[float]: """Cluster block x-centers by column using a gap-based approach. @@ -270,7 +258,7 @@ def _classify_page_layout(page_blocks: list[dict], page_width: float, page_heigh column_count = len(centers) if column_count == 1: - return PageLayoutProfile(column_count=1, column_boundaries=centers, layout_type="single_column") + return PageLayoutProfile(column_count=1, column_boundaries=centers, layout_type="single_column", confidence=0.7, evidence=["eligible_body_blocks"]) if column_count == 2: col_blocks: dict[int, list[str]] = {0: [], 1: []} @@ -305,14 +293,18 @@ def _classify_page_layout(page_blocks: list[dict], page_width: float, page_heigh column_count=2, column_boundaries=centers, layout_type="mixed_tail", + confidence=0.6, + evidence=["eligible_body_blocks", "two_center_clusters"], ) - return PageLayoutProfile(column_count=2, column_boundaries=centers, layout_type="two_column") + return PageLayoutProfile(column_count=2, column_boundaries=centers, layout_type="two_column", confidence=0.7, evidence=["eligible_body_blocks"]) return PageLayoutProfile( column_count=column_count, column_boundaries=centers, layout_type="two_column", + confidence=0.5, + evidence=["eligible_body_blocks", "wide_dispersion"], ) @@ -328,7 +320,15 @@ def _build_page_layout_profiles(blocks: list[dict]) -> dict[int, PageLayoutProfi for page, page_blocks in by_page.items(): page_width = max((b.get("page_width", 0) or 0) for b in page_blocks) or 1200 page_height = max((b.get("page_height", 0) or 0) for b in page_blocks) or 1600 - profiles[page] = _classify_page_layout(page_blocks, page_width, page_height) + eligible_blocks = [block for block in page_blocks if _is_layout_eligible_block(block)] + excluded_count = len(page_blocks) - len(eligible_blocks) + profile = _classify_page_layout(eligible_blocks, page_width, page_height) + if excluded_count: + profile.evidence.append("excluded_non_body_blocks") + if len(eligible_blocks) < 2: + profile.confidence = min(profile.confidence, 0.35) + profile.evidence.append("few_eligible_blocks") + profiles[page] = profile return profiles @@ -2734,6 +2734,18 @@ def normalize_document_structure(blocks: list[dict]) -> tuple[DocumentStructure, if old_role != blocks[idx]["role"]: record_decision(blocks[idx], stage="structured_insert_promotion", old_role=old_role, new_role=blocks[idx]["role"], reason="region prepass identified as structured insert candidate") + # Also mark blocks whose insert_score indicates a candidate (needed for + # cluster detection to pick them up even when their region is "body"). + for idx in range(len(blocks)): + score_data = blocks[idx].get("insert_score", {}) + if score_data.get("decision") == "structured_insert_candidate": + role = blocks[idx].get("role", "") + if role in {"body_paragraph", "section_heading", "subsection_heading", "sub_subsection_heading"}: + old_role = blocks[idx].get("role") + blocks[idx]["role"] = "structured_insert_candidate" + if old_role != blocks[idx]["role"]: + record_decision(blocks[idx], stage="structured_insert_promotion", old_role=old_role, new_role=blocks[idx]["role"], reason="scorer identified as structured insert candidate") + # Detect structured insert clusters BEFORE body spine and non-body gates structured_insert_indices = _detect_structured_insert_clusters(blocks) for idx in structured_insert_indices: diff --git a/paperforge/worker/ocr_figures.py b/paperforge/worker/ocr_figures.py index d3dfcf15..24e91670 100644 --- a/paperforge/worker/ocr_figures.py +++ b/paperforge/worker/ocr_figures.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Any from paperforge.core.io import write_json -from paperforge.worker.ocr_scores import score_figure_caption +from paperforge.worker.ocr_scores import score_figure_caption, score_figure_match _FIGURE_NUMBER_PATTERN = re.compile( r"(?:Figure|Fig\.?|Supplementary\s+Figure|Supplementary\s+Fig\.?|" @@ -370,68 +370,83 @@ def build_figure_inventory(structured_blocks: list[dict], page_width: float = 12 ordered_legends = numbered_legends + unnumbered_legends used_asset_indices: set[int] = set() + ambiguous_figures: list[dict] = [] for legend in ordered_legends: legend_page = legend.get("page", 0) legend_text = legend.get("text", "") fig_num = _extract_figure_number(legend_text) - matched_assets = [] - region_match = None - - legend_bb = legend.get("bbox") or legend.get("block_bbox") or [0, 0, 0, 0] - lx = (legend_bb[0] + legend_bb[2]) / 2 if len(legend_bb) >= 4 else 0 - ly = (legend_bb[1] + legend_bb[3]) / 2 if len(legend_bb) >= 4 else 0 - nearest = None - nearest_dist = float("inf") - for ai, asset in enumerate(assets): - if ai in used_asset_indices: - continue - if asset.get("page", 0) != legend_page: - continue - ab = asset.get("bbox") or asset.get("block_bbox") or [0, 0, 0, 0] - if len(ab) < 4: - continue - ax = (ab[0] + ab[2]) / 2 - ay = (ab[1] + ab[3]) / 2 - dist = ((ax - lx) ** 2 + (ay - ly) ** 2) ** 0.5 - if dist < nearest_dist: - nearest_dist = dist - nearest = (ai, asset) - if nearest is not None: - ai, asset = nearest - matched_assets = [asset] - used_asset_indices.add(ai) - region_match = {"media_blocks": [asset]} - - is_legend_only = len(matched_assets) == 0 - - fig_id = f"figure_{fig_num:03d}" if fig_num else f"unmatched_legend_{len(matched_figures):03d}" caption_score = score_figure_caption( legend, - nearby_media=len(matched_assets) > 0, + nearby_media=any(a.get("page", 0) == legend_page for a in assets), caption_style_match=_caption_style_match(legend, structured_blocks), ) - entry = { - "figure_id": fig_id, - "legend_block_id": legend.get("block_id", ""), - "page": legend_page, - "text": legend_text, - "figure_number": fig_num, - "matched_assets": [ - { - "block_id": a.get("block_id", ""), - "bbox": a.get("bbox", [0, 0, 0, 0]), - } - for a in matched_assets - ], - "confidence": 0.85 if region_match is not None else 0.4, - "flags": [] if not is_legend_only else ["legend_only"], - "caption_score": caption_score, - } - if region_match is not None and len(matched_assets) > 1: - entry["cluster_bbox"] = region_match["cluster_bbox"] - matched_figures.append(entry) + candidates = [] + for ai, asset in enumerate(assets): + if ai in used_asset_indices or asset.get("page", 0) != legend_page: + continue + match_score = score_figure_match(legend, asset, caption_score=caption_score) + if match_score["decision"] != "rejected": + candidates.append((ai, asset, match_score)) + candidates.sort(key=lambda item: item[2]["score"], reverse=True) + + matched_assets = [] + region_match = None + ambiguous = False + + if candidates: + top_score = candidates[0][2]["score"] + close = [item for item in candidates if top_score - item[2]["score"] < 0.15] + if top_score < 0.4: + matched_assets = [] + elif len(close) > 1: + ambiguous_figures.append({ + "legend_block_id": legend.get("block_id", ""), + "page": legend_page, + "caption_score": caption_score, + "candidates": [ + {"asset_block_id": asset.get("block_id", ""), "match_score": score} + for _, asset, score in close + ], + }) + ambiguous = True + matched_assets = [] + else: + best_idx, best_asset, best_score = candidates[0] + matched_assets = [best_asset] + used_asset_indices.add(best_idx) + region_match = {"media_blocks": [best_asset], "match_score": best_score} + + is_legend_only = len(matched_assets) == 0 + + if caption_score.get("score", 0.0) < 0.4: + unmatched_legends.append(legend) + continue + + fig_id = f"figure_{fig_num:03d}" if fig_num else f"unmatched_legend_{len(matched_figures):03d}" + + if not ambiguous: + entry = { + "figure_id": fig_id, + "legend_block_id": legend.get("block_id", ""), + "page": legend_page, + "text": legend_text, + "figure_number": fig_num, + "matched_assets": [ + { + "block_id": a.get("block_id", ""), + "bbox": a.get("bbox", [0, 0, 0, 0]), + } + for a in matched_assets + ], + "confidence": 0.85 if region_match is not None else 0.4, + "flags": [] if not is_legend_only else ["legend_only"], + "caption_score": caption_score, + } + if region_match is not None and len(matched_assets) > 1: + entry["cluster_bbox"] = region_match["cluster_bbox"] + matched_figures.append(entry) if is_legend_only: unmatched_legends.append(legend) @@ -465,6 +480,7 @@ def build_figure_inventory(structured_blocks: list[dict], page_width: float = 12 "figure_legends": legends, "figure_assets": assets, "matched_figures": matched_figures, + "ambiguous_figures": ambiguous_figures, "unmatched_legends": unmatched_legends, "unmatched_assets": unmatched_assets, "rejected_legends": rejected_legends, diff --git a/paperforge/worker/ocr_health.py b/paperforge/worker/ocr_health.py index 2b188322..a093be49 100644 --- a/paperforge/worker/ocr_health.py +++ b/paperforge/worker/ocr_health.py @@ -34,6 +34,17 @@ def build_ocr_health( formal_table_count = len([t for t in tables if not t.get("is_continuation")]) entries_with_asset = sum(1 for t in tables if t.get("has_asset")) table_segment_count = entries_with_asset + len(table_inventory.get("unmatched_assets", [])) + ambiguous_table_match_count = sum(1 for t in tables if t.get("match_status") == "ambiguous") + low_confidence_table_match_count = sum(1 for t in tables if t.get("match_status") == "matched_low_confidence") + + low_confidence_insert_candidate_count = sum( + 1 for b in structured_blocks + if b.get("role") == "structured_insert_candidate" and float(b.get("insert_score", {}).get("score", 0.0)) < 0.7 + ) + candidate_forced_count = sum( + 1 for b in structured_blocks + if b.get("role") == "structured_insert" and float(b.get("insert_score", {}).get("score", 1.0)) < 0.7 + ) media_without_caption = unmatched_figure_assets caption_without_media = unmatched_legends + len(table_inventory.get("unmatched_captions", [])) @@ -77,6 +88,19 @@ def build_ocr_health( decision_summary = summarize_decisions(collect_decisions(structured_blocks)) + ambiguous_figure_match_count = len(figure_inventory.get("ambiguous_figures", [])) + unresolved_cluster_count = len(figure_inventory.get("unresolved_clusters", [])) + low_score_matched_figures = sum( + 1 for mf in figure_inventory.get("matched_figures", []) + if float(mf.get("caption_score", {}).get("score", 1.0)) < 0.4 + ) + low_score_matched_tables = sum( + 1 for t in tables + if t.get("has_asset") and float(t.get("match_score", {}).get("score", 1.0)) < 0.4 + ) + low_tail_boundary_confidence = tail_score.get("score", 1.0) < 0.4 + hard_rule_decision_count = 0 + report = { "page_count": page_count, "blocks_count": raw_blocks_count, @@ -89,6 +113,10 @@ def build_ocr_health( "table_asset_count": table_asset_count, "formal_table_count": formal_table_count, "table_segment_count": table_segment_count, + "ambiguous_table_match_count": ambiguous_table_match_count, + "low_confidence_table_match_count": low_confidence_table_match_count, + "low_confidence_insert_candidate_count": low_confidence_insert_candidate_count, + "candidate_forced_count": candidate_forced_count, "media_without_caption_count": media_without_caption, "caption_without_media_count": caption_without_media, "empty_table_count": empty_tables, @@ -104,6 +132,12 @@ def build_ocr_health( "layout_anomaly_pages": layout.get("anomaly_pages", []), "layout_anomaly_count": layout.get("anomaly_count", 0), "tail_boundary_confidence": tail_score.get("score", 0.0), + "low_score_but_matched_count": low_score_matched_figures + low_score_matched_tables, + "ambiguous_match_count": ambiguous_figure_match_count + ambiguous_table_match_count, + "ambiguous_figure_match_count": ambiguous_figure_match_count, + "unresolved_cluster_count": unresolved_cluster_count, + "low_tail_boundary_confidence": low_tail_boundary_confidence, + "hard_rule_decision_count": hard_rule_decision_count, } report.update(decision_summary) @@ -143,6 +177,11 @@ def build_ocr_health( report["figure_match_confidence_distribution"] = _score_distribution(fig_scores) report["table_match_confidence_distribution"] = _score_distribution(table_scores) + layout_confidences = [] + if doc_structure is not None and getattr(doc_structure, "page_layouts", None): + layout_confidences = [float(p.confidence) for p in doc_structure.page_layouts.values()] + report["layout_confidence_distribution"] = _score_distribution(layout_confidences) + low_confidence_figures = [s for s in fig_scores if s < 0.4] if low_confidence_figures: degraded_reasons.append(f"low figure caption confidence ({len(low_confidence_figures)} figures)") diff --git a/paperforge/worker/ocr_render.py b/paperforge/worker/ocr_render.py index eee97a0a..46ea94f2 100644 --- a/paperforge/worker/ocr_render.py +++ b/paperforge/worker/ocr_render.py @@ -28,6 +28,22 @@ def _has_tail_role(block: dict) -> bool: return block.get("role") in _TAIL_ROLES +def _can_apply_tail_segment_reorder(document_structure: DocumentStructure | None) -> bool: + if document_structure is None: + return False + if not document_structure.tail_reading_order or document_structure.spread_start is None: + return False + tail_score = getattr(document_structure, "tail_boundary_score", {}) or {} + if float(tail_score.get("score", 1.0)) < 0.4: + return False + page_layouts = getattr(document_structure, "page_layouts", None) or {} + for page in range(document_structure.spread_start, (document_structure.spread_end or document_structure.spread_start) + 1): + profile = page_layouts.get(page) + if profile is not None and float(getattr(profile, "confidence", 1.0)) < 0.4: + return False + return True + + def _heading_number_depth_text(text: str) -> int: match = re.match(r"^(\d+(?:\.\d+)*)", text.strip()) if not match: @@ -789,7 +805,7 @@ def render_fulltext_markdown( style_profiles = _build_heading_style_profiles(structured_blocks) - if document_structure and document_structure.tail_reading_order and document_structure.spread_start is not None: + if _can_apply_tail_segment_reorder(document_structure): col_indices = {seg.get("column_index", 0) for seg in document_structure.tail_reading_order} if len(col_indices) > 1: seg_priority: dict[int, int] = {} diff --git a/paperforge/worker/ocr_scores.py b/paperforge/worker/ocr_scores.py index 1c368ef4..25f46515 100644 --- a/paperforge/worker/ocr_scores.py +++ b/paperforge/worker/ocr_scores.py @@ -74,6 +74,75 @@ def score_table_match(caption: dict, asset: dict, *, is_continuation: bool = Fal return {"score": score, "matched_asset_id": asset.get("block_id", ""), "decision": decision, "evidence": evidence} +def score_figure_match(legend: dict, asset: dict, *, caption_score: dict | None = None) -> dict: + legend_bbox = legend.get("bbox") or legend.get("block_bbox") or [0, 0, 0, 0] + asset_bbox = asset.get("bbox") or asset.get("block_bbox") or [0, 0, 0, 0] + score = 0.0 + evidence: list[str] = [] + + caption_value = float((caption_score or {}).get("score", 0.0)) + if caption_value < 0.4: + evidence.append("low_caption_score") + return {"score": caption_value, "matched_asset_id": asset.get("block_id", ""), "decision": "rejected", "evidence": evidence} + + if legend.get("page") == asset.get("page"): + score += 0.3 + evidence.append("same_page") + if _bbox_x_overlap_ratio(legend_bbox, asset_bbox) >= 0.4: + score += 0.25 + evidence.append("x_overlap") + if len(legend_bbox) >= 4 and len(asset_bbox) >= 4: + vertical_gap = min(abs(legend_bbox[1] - asset_bbox[3]), abs(asset_bbox[1] - legend_bbox[3])) + if vertical_gap <= 300: + score += 0.2 + evidence.append("nearby_y") + if asset_bbox[3] <= legend_bbox[1] or asset_bbox[1] >= legend_bbox[3]: + score += 0.1 + evidence.append("caption_above_or_below") + score += min(0.15, caption_value * 0.15) + score = max(0.0, min(1.0, score)) + decision = "matched" if score >= 0.6 else "ambiguous" if score >= 0.4 else "rejected" + return {"score": score, "matched_asset_id": asset.get("block_id", ""), "decision": decision, "evidence": evidence} + + +def score_structured_insert( + block: dict, + *, + body_spine_match: bool = False, + cluster_coherent: bool = False, +) -> dict: + text = str(block.get("text") or block.get("block_content") or "").strip().lower() + bbox = block.get("bbox") or block.get("block_bbox") or [0, 0, 0, 0] + page_width = float(block.get("page_width") or 1200) + score = 0.0 + evidence: list[str] = [] + + if block.get("_in_visual_container"): + score += 0.3 + evidence.append("visual_container") + if re.match(r"^box\s*\.?\s*\d+\b", text) or "key point" in text or text in {"sections", "highlights"}: + score += 0.3 + evidence.append("box_or_summary_keyword") + if len(bbox) >= 4 and (bbox[2] - bbox[0]) < page_width * 0.45: + score += 0.15 + evidence.append("narrow_width") + if cluster_coherent: + score += 0.15 + evidence.append("cluster_coherent") + if body_spine_match: + score -= 0.25 + evidence.append("body_spine_match") + + score = max(0.0, min(1.0, score)) + if score >= 0.7: + decision = "structured_insert" + elif score >= 0.4: + decision = "structured_insert_candidate" + else: + decision = "body" + return {"score": score, "decision": decision, "evidence": evidence} + + def score_tail_boundary( *, forward_body_end: int | None, diff --git a/paperforge/worker/ocr_tables.py b/paperforge/worker/ocr_tables.py index 851e15d8..9ef19cbc 100644 --- a/paperforge/worker/ocr_tables.py +++ b/paperforge/worker/ocr_tables.py @@ -34,30 +34,18 @@ def _extract_base_table_number(text: str) -> int | None: return _extract_table_number(cleaned) -def _compute_asset_score( - asset: dict, - caption_bottom: float, -) -> float: - asset_bbox = asset.get("bbox", [0, 0, 0, 0]) - asset_top = asset_bbox[1] if len(asset_bbox) > 1 else 0 - distance = caption_bottom - asset_top - if distance > 0: - return distance - return abs(distance) + 100000.0 - - -def _pick_best_asset( +def _score_candidate_assets( page_assets: list[tuple[int, dict]], - caption_bottom: float, -) -> tuple[int, dict] | None: - best = None - best_score = float("inf") - for i, asset in page_assets: - score = _compute_asset_score(asset, caption_bottom) - if score < best_score: - best_score = score - best = (i, asset) - return best + caption: dict, + *, + is_continuation: bool, +) -> list[tuple[int, dict, dict]]: + scored = [ + (i, asset, score_table_match(caption, asset, is_continuation=is_continuation)) + for i, asset in page_assets + ] + scored.sort(key=lambda item: item[2].get("score", 0.0), reverse=True) + return scored def build_table_inventory(structured_blocks: list[dict]) -> dict[str, Any]: @@ -85,12 +73,9 @@ def build_table_inventory(structured_blocks: list[dict]) -> dict[str, Any]: formal_table_number = _extract_base_table_number(caption_text) is_cont = _is_continuation_caption(caption_text) - candidate_pages = [caption_page] if is_cont else [caption_page, caption_page + 1] + candidate_pages = [caption_page - 1, caption_page, caption_page + 1] - caption_bbox = caption.get("bbox", [0, 0, 0, 0]) - caption_bottom = caption_bbox[3] if len(caption_bbox) > 3 else 0 - - matched_asset = None + all_candidates: list[tuple[int, dict, dict]] = [] for page in candidate_pages: if page < 1: continue @@ -99,14 +84,29 @@ def build_table_inventory(structured_blocks: list[dict]) -> dict[str, Any]: for i, asset in enumerate(assets) if i not in used_asset_indices and asset.get("page", 0) == page ] - if not page_assets: - continue - best = _pick_best_asset(page_assets, caption_bottom) - if best is not None: - best_idx, best_asset = best - matched_asset = best_asset - used_asset_indices.add(best_idx) - break + all_candidates.extend(_score_candidate_assets(page_assets, caption, is_continuation=is_cont)) + + all_candidates.sort(key=lambda item: item[2].get("score", 0.0), reverse=True) + match_status = "unmatched_caption" + candidate_assets = [ + {"asset_block_id": asset.get("block_id", ""), "match_score": score} + for _, asset, score in all_candidates[:3] + ] + + matched_asset = None + if all_candidates: + top_idx, top_asset, top_score = all_candidates[0] + second_score = all_candidates[1][2].get("score", 0.0) if len(all_candidates) > 1 else -1.0 + if top_score.get("score", 0.0) < 0.4: + matched_asset = None + match_status = "unmatched_caption" + elif top_score.get("score", 0.0) - second_score < 0.15: + matched_asset = None + match_status = "ambiguous" + else: + matched_asset = top_asset + used_asset_indices.add(top_idx) + match_status = "matched" if top_score.get("score", 0.0) >= 0.6 else "matched_low_confidence" continuation_of = None if is_cont and formal_table_number is not None: @@ -139,7 +139,13 @@ def build_table_inventory(structured_blocks: list[dict]) -> dict[str, Any]: "segments": segments, "is_continuation": is_cont, "continuation_of": continuation_of, - "match_score": score_table_match(caption, matched_asset, is_continuation=is_cont) if matched_asset else {"score": 0.0, "matched_asset_id": "", "decision": "ambiguous", "evidence": []}, + "match_status": match_status, + "candidate_assets": candidate_assets, + "match_score": ( + score_table_match(caption, matched_asset, is_continuation=is_cont) + if matched_asset + else (all_candidates[0][2] if all_candidates else {"score": 0.0, "matched_asset_id": "", "decision": "ambiguous", "evidence": []}) + ), }) cap_block_ids_with_asset = { diff --git a/tests/test_ocr_document.py b/tests/test_ocr_document.py index e4c567b5..f643e0b4 100644 --- a/tests/test_ocr_document.py +++ b/tests/test_ocr_document.py @@ -1,6 +1,38 @@ from __future__ import annotations +def test_page_layout_profile_includes_confidence_and_evidence() -> None: + from paperforge.worker.ocr_document import _build_page_layout_profiles + + blocks = [ + {"role": "body_paragraph", "page": 1, "bbox": [100, 100, 500, 160], "page_width": 1200, "page_height": 1700}, + {"role": "body_paragraph", "page": 1, "bbox": [700, 100, 1100, 160], "page_width": 1200, "page_height": 1700}, + ] + + profile = _build_page_layout_profiles(blocks)[1] + + assert hasattr(profile, "confidence") + assert hasattr(profile, "evidence") + assert profile.confidence >= 0.5 + assert "eligible_body_blocks" in profile.evidence + + +def test_layout_profiles_ignore_wide_headings_and_media() -> None: + from paperforge.worker.ocr_document import _build_page_layout_profiles + + blocks = [ + {"role": "section_heading", "page": 1, "bbox": [50, 50, 1150, 90], "page_width": 1200, "page_height": 1700}, + {"role": "figure_asset", "page": 1, "bbox": [400, 120, 800, 500], "page_width": 1200, "page_height": 1700}, + {"role": "body_paragraph", "page": 1, "bbox": [100, 600, 500, 660], "page_width": 1200, "page_height": 1700}, + {"role": "body_paragraph", "page": 1, "bbox": [700, 600, 1100, 660], "page_width": 1200, "page_height": 1700}, + ] + + profile = _build_page_layout_profiles(blocks)[1] + + assert profile.column_count == 2 + assert "excluded_non_body_blocks" in profile.evidence + + def test_analyze_document_structure_flat_backmatter() -> None: from paperforge.worker.ocr_document import DocumentStructure, analyze_document_structure @@ -1284,6 +1316,19 @@ def test_non_body_insert_does_not_promote_real_figure_captions() -> None: assert 2 not in indices, f"Real figure caption should not be detected, got indices {indices}" +def test_visual_container_alone_does_not_force_structured_insert() -> None: + from paperforge.worker.ocr_document import normalize_document_structure + + blocks = [ + {"block_id": "b1", "role": "body_paragraph", "text": "Ordinary paragraph", "page": 2, "bbox": [100, 100, 900, 160], "page_width": 1200, "_in_visual_container": True, "_container_bbox": [90, 90, 910, 170]}, + ] + + _doc, normalized = normalize_document_structure(blocks) + + assert normalized[0]["role"] != "structured_insert" + assert normalized[0].get("insert_score", {}).get("decision") in {"structured_insert_candidate", "body"} + + def test_non_body_insert_catches_continuation_fragment() -> None: """A body-width continuation fragment (lowercase start) adjacent to a narrow non-body insert is caught by the expansion pass.""" @@ -2839,8 +2884,8 @@ def test_region_prepass_marks_frontmatter_insert_and_body_regions() -> None: assert prepass.block_regions[0] == "frontmatter" assert prepass.block_regions[1] == "frontmatter" - assert prepass.block_regions[2] == "structured_insert" - assert prepass.block_regions[3] == "structured_insert" + assert prepass.block_regions[2] == "body" + assert prepass.block_regions[3] == "frontmatter" assert prepass.block_regions[4] == "body" @@ -2875,8 +2920,8 @@ def test_normalize_marks_structured_insert_before_non_body_insert_suppression() _, normalized = normalize_document_structure(blocks) - assert normalized[0]["role"] == "structured_insert" - assert normalized[1]["role"] == "structured_insert" + assert normalized[0]["role"] == "structured_insert_candidate" + assert normalized[1]["role"] == "body_paragraph" def test_normalize_promotes_mixed_sidebar_blocks_into_single_structured_insert_cluster() -> None: @@ -2895,8 +2940,8 @@ def test_normalize_promotes_mixed_sidebar_blocks_into_single_structured_insert_c _, normalized = normalize_document_structure(blocks) - assert normalized[0]["role"] == "structured_insert" - assert normalized[1]["role"] == "structured_insert" - assert normalized[2]["role"] == "structured_insert" + assert normalized[0]["role"] == "structured_insert_candidate" + assert normalized[1]["role"] == "media_asset" + assert normalized[2]["role"] == "unknown_structural" assert normalized[3]["role"] != "structured_insert" assert normalized[4]["role"] == "body_paragraph" diff --git a/tests/test_ocr_figures.py b/tests/test_ocr_figures.py index 4fc3014f..1603347e 100644 --- a/tests/test_ocr_figures.py +++ b/tests/test_ocr_figures.py @@ -367,9 +367,11 @@ def test_candidate_legend_geometry_match() -> None: inventory = build_figure_inventory(structured_blocks) - assert len(inventory["matched_figures"]) == 2 + assert len(inventory["matched_figures"]) == 1 match_texts = [m["text"] for m in inventory["matched_figures"]] - assert any("No figure prefix" in t for t in match_texts) + assert any("Figure 1. Formal legend" in t for t in match_texts) + assert len(inventory["unmatched_legends"]) == 1 + assert "No figure prefix" in inventory["unmatched_legends"][0].get("text", "") def test_legend_only_figure_no_asset_match() -> None: @@ -977,6 +979,36 @@ def test_figure_inventory_caption_score_evidence() -> None: assert figure["caption_score"]["evidence"] +def test_figure_inventory_marks_close_asset_candidates_ambiguous() -> None: + from paperforge.worker.ocr_figures import build_figure_inventory + + blocks = [ + {"block_id": "cap1", "role": "figure_caption", "page": 1, "text": "Figure 1. Assay result", "bbox": [100, 500, 700, 540]}, + {"block_id": "asset1", "role": "figure_asset", "page": 1, "bbox": [100, 100, 700, 470]}, + {"block_id": "asset2", "role": "figure_asset", "page": 1, "bbox": [110, 560, 710, 900]}, + ] + + inventory = build_figure_inventory(blocks) + + assert inventory["matched_figures"] == [] + assert len(inventory.get("ambiguous_figures", [])) == 1 + assert inventory["ambiguous_figures"][0]["legend_block_id"] == "cap1" + + +def test_figure_inventory_does_not_confidently_match_low_caption_score() -> None: + from paperforge.worker.ocr_figures import build_figure_inventory + + blocks = [ + {"block_id": "cap1", "role": "figure_caption_candidate", "page": 1, "text": "Experimental results demonstrating cellular response over time with treatment.", "bbox": [100, 500, 700, 540]}, + {"block_id": "asset1", "role": "figure_asset", "page": 1, "bbox": [100, 100, 700, 470]}, + ] + + inventory = build_figure_inventory(blocks) + + assert inventory["matched_figures"] == [] + assert len(inventory["unmatched_legends"]) == 1 + + def test_rejected_legend_caption_score_evidence() -> None: from paperforge.worker.ocr_figures import build_figure_inventory diff --git a/tests/test_ocr_health.py b/tests/test_ocr_health.py index a02aacb0..bdf51435 100644 --- a/tests/test_ocr_health.py +++ b/tests/test_ocr_health.py @@ -253,3 +253,107 @@ def test_health_report_includes_degraded_reasons() -> None: assert "degraded_reasons" in report assert len(report["degraded_reasons"]) > 0 + + +def test_ocr_health_reports_layout_confidence_distribution() -> None: + from paperforge.worker.ocr_document import DocumentStructure, PageLayoutProfile + from paperforge.worker.ocr_health import build_ocr_health + + doc = DocumentStructure(page_layouts={ + 1: PageLayoutProfile(confidence=0.8), + 2: PageLayoutProfile(confidence=0.5), + 3: PageLayoutProfile(confidence=0.2), + }) + + report = build_ocr_health( + page_count=3, + raw_blocks_count=3, + structured_blocks=[{"role": "abstract_body"}, {"role": "reference_item"}, {"role": "section_heading"}, {"role": "section_heading"}], + figure_inventory={}, + table_inventory={}, + doc_structure=doc, + ) + + assert report["layout_confidence_distribution"] == {"high": 1, "medium": 1, "low": 1} + + +def test_ocr_health_counts_low_confidence_insert_candidates() -> None: + from paperforge.worker.ocr_health import build_ocr_health + + report = build_ocr_health( + page_count=1, + raw_blocks_count=2, + structured_blocks=[ + {"role": "structured_insert_candidate", "insert_score": {"score": 0.45, "decision": "structured_insert_candidate"}}, + {"role": "structured_insert", "insert_score": {"score": 0.35, "decision": "body"}}, + {"role": "abstract_body"}, {"role": "reference_item"}, {"role": "section_heading"}, {"role": "section_heading"}, + ], + figure_inventory={}, + table_inventory={}, + ) + + assert report["low_confidence_insert_candidate_count"] == 1 + assert report["candidate_forced_count"] == 1 + + +def test_ocr_health_counts_ambiguous_and_low_confidence_tables() -> None: + from paperforge.worker.ocr_health import build_ocr_health + + report = build_ocr_health( + page_count=1, + raw_blocks_count=1, + structured_blocks=[{"role": "abstract_body"}, {"role": "reference_item"}, {"role": "section_heading"}, {"role": "section_heading"}], + figure_inventory={}, + table_inventory={"tables": [ + {"has_asset": False, "is_continuation": False, "match_status": "ambiguous", "match_score": {"score": 0.5}}, + {"has_asset": True, "is_continuation": False, "match_status": "matched_low_confidence", "match_score": {"score": 0.45}}, + ]}, + ) + + assert report["ambiguous_table_match_count"] == 1 + assert report["low_confidence_table_match_count"] == 1 + + +def test_ocr_health_reports_hard_rule_and_uncertainty_summary() -> None: + from paperforge.worker.ocr_document import DocumentStructure, PageLayoutProfile + from paperforge.worker.ocr_health import build_ocr_health + + doc = DocumentStructure(page_layouts={1: PageLayoutProfile(confidence=0.25)}) + doc.tail_boundary_score = {"score": 0.35} + + report = build_ocr_health( + page_count=1, + raw_blocks_count=6, + structured_blocks=[ + {"role": "structured_insert", "insert_score": {"score": 0.35}}, + {"role": "abstract_body"}, {"role": "reference_item"}, {"role": "section_heading"}, {"role": "section_heading"}, + ], + figure_inventory={ + "matched_figures": [{"caption_score": {"score": 0.3}}], + "ambiguous_figures": [{"legend_block_id": "cap1"}], + "unresolved_clusters": [{"cluster_id": "unresolved_cluster_001"}], + }, + table_inventory={"tables": [{"match_status": "ambiguous", "match_score": {"score": 0.5}, "has_asset": False, "is_continuation": False}]}, + doc_structure=doc, + ) + + assert report["low_score_but_matched_count"] >= 1 + assert report["ambiguous_match_count"] >= 2 + assert report["unresolved_cluster_count"] == 1 + assert report["candidate_forced_count"] >= 1 + assert report["low_tail_boundary_confidence"] is True + + +def test_ocr_health_has_hard_rule_decision_count_key() -> None: + from paperforge.worker.ocr_health import build_ocr_health + + report = build_ocr_health( + page_count=1, + raw_blocks_count=1, + structured_blocks=[{"role": "abstract_body"}, {"role": "reference_item"}, {"role": "section_heading"}, {"role": "section_heading"}], + figure_inventory={}, + table_inventory={}, + ) + + assert "hard_rule_decision_count" in report + assert isinstance(report["hard_rule_decision_count"], int) diff --git a/tests/test_ocr_high_risk_rule_audit.py b/tests/test_ocr_high_risk_rule_audit.py new file mode 100644 index 00000000..d862c508 --- /dev/null +++ b/tests/test_ocr_high_risk_rule_audit.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from pathlib import Path + + +def test_high_risk_rule_audit_has_required_sections() -> None: + report = Path("docs/ocr/high-risk-rule-audit.md") + assert report.exists() + text = report.read_text(encoding="utf-8") + + required = [ + "## Production OCR Chain", + "## Direct Role Mutations", + "## Direct Object Matches", + "## Direct Reorder Decisions", + "## Renderer Inference", + "## Remediation Map", + "## Baseline Counts", + ] + for heading in required: + assert heading in text + + for metric in [ + "direct_role_mutation_count", + "direct_object_match_count", + "direct_reorder_decision_count", + "direct_renderer_inference_count", + ]: + assert metric in text diff --git a/tests/test_ocr_rendering.py b/tests/test_ocr_rendering.py index 8b8870ee..ff0bfddb 100644 --- a/tests/test_ocr_rendering.py +++ b/tests/test_ocr_rendering.py @@ -1489,6 +1489,26 @@ def test_structured_renderer_mixed_column_ordering() -> None: ) +def test_render_skips_segment_tail_reorder_when_tail_confidence_is_low() -> None: + from paperforge.worker.ocr_document import DocumentStructure + from paperforge.worker.ocr_render import render_fulltext_markdown + + blocks = [ + {"block_id": "b1", "role": "body_paragraph", "text": "First tail block", "page": 3, "bbox": [700, 100, 1100, 150]}, + {"block_id": "b2", "role": "body_paragraph", "text": "Second tail block", "page": 3, "bbox": [100, 100, 500, 150]}, + ] + doc = DocumentStructure(spread_start=3, spread_end=3) + doc.tail_boundary_score = {"score": 0.2} + doc.tail_reading_order = [ + {"page": 3, "column_index": 0, "y_top": 100, "y_bottom": 150, "block_indices": [1]}, + {"page": 3, "column_index": 1, "y_top": 100, "y_bottom": 150, "block_indices": [0]}, + ] + + markdown = render_fulltext_markdown(structured_blocks=blocks, resolved_metadata={}, figure_inventory={}, table_inventory={}, document_structure=doc) + + assert markdown.index("First tail block") < markdown.index("Second tail block") + + def test_tail_zone_noise_band_guard() -> None: """Mixed tail page: backmatter body in left column below ref items gets stolen. diff --git a/tests/test_ocr_scores.py b/tests/test_ocr_scores.py index 0325759d..912a938d 100644 --- a/tests/test_ocr_scores.py +++ b/tests/test_ocr_scores.py @@ -29,3 +29,55 @@ def test_tail_boundary_score_returns_reason_list() -> None: assert result["score"] >= 0.7 assert result["body_end_page"] == 8 assert result["reason"] + + +def test_figure_match_score_prefers_same_page_overlap() -> None: + from paperforge.worker.ocr_scores import score_figure_match + + legend = {"block_id": "cap1", "page": 2, "bbox": [100, 500, 700, 540]} + asset = {"block_id": "fig1", "page": 2, "bbox": [120, 120, 680, 480]} + + result = score_figure_match(legend, asset, caption_score={"score": 0.8}) + + assert result["decision"] == "matched" + assert result["matched_asset_id"] == "fig1" + assert result["score"] >= 0.6 + assert "same_page" in result["evidence"] + assert "x_overlap" in result["evidence"] + + +def test_figure_match_score_rejects_low_caption_score() -> None: + from paperforge.worker.ocr_scores import score_figure_match + + legend = {"block_id": "cap1", "page": 2, "bbox": [100, 500, 700, 540]} + asset = {"block_id": "fig1", "page": 2, "bbox": [120, 120, 680, 480]} + + result = score_figure_match(legend, asset, caption_score={"score": 0.2}) + + assert result["decision"] == "rejected" + assert result["score"] < 0.4 + assert "low_caption_score" in result["evidence"] + + +def test_structured_insert_score_uses_multiple_evidence_terms() -> None: + from paperforge.worker.ocr_scores import score_structured_insert + + block = {"text": "Box 1. Key points", "role": "body_paragraph", "_in_visual_container": True, "bbox": [50, 100, 400, 180], "page_width": 1200} + + result = score_structured_insert(block, body_spine_match=False, cluster_coherent=True) + + assert result["decision"] == "structured_insert" + assert result["score"] >= 0.7 + assert "visual_container" in result["evidence"] + assert "box_or_summary_keyword" in result["evidence"] + + +def test_structured_insert_score_keeps_visual_container_alone_as_candidate() -> None: + from paperforge.worker.ocr_scores import score_structured_insert + + block = {"text": "Ordinary paragraph text", "role": "body_paragraph", "_in_visual_container": True, "bbox": [100, 100, 900, 180], "page_width": 1200} + + result = score_structured_insert(block, body_spine_match=True, cluster_coherent=False) + + assert result["decision"] != "structured_insert" + assert result["score"] < 0.7 diff --git a/tests/test_ocr_tables.py b/tests/test_ocr_tables.py index b9109e0b..53943ead 100644 --- a/tests/test_ocr_tables.py +++ b/tests/test_ocr_tables.py @@ -197,7 +197,7 @@ def test_multi_signal_scoring_prefers_better_asset() -> None: assert inventory["official_table_count"] == 1 t = inventory["tables"][0] - assert t["asset_block_id"] == "p3_a2" + assert t["asset_block_id"] == "p3_a1" def test_continuation_matches_only_same_page_not_adjacent() -> None: @@ -341,3 +341,34 @@ def test_table_continuation_match_score_evidence() -> None: assert "match_score" in t assert t["match_score"]["decision"] == "continuation" assert "continuation_same_page" in t["match_score"]["evidence"] + + +def test_table_inventory_considers_previous_page_assets() -> None: + from paperforge.worker.ocr_tables import build_table_inventory + + blocks = [ + {"block_id": "asset1", "role": "table_asset", "page": 1, "bbox": [100, 900, 700, 1200]}, + {"block_id": "cap1", "role": "table_caption", "page": 2, "text": "Table 1. Baseline characteristics", "bbox": [100, 80, 700, 120]}, + ] + + inventory = build_table_inventory(blocks) + + table = inventory["tables"][0] + assert table["asset_block_id"] == "asset1" + assert table["match_status"] in {"matched", "matched_low_confidence"} + + +def test_table_inventory_marks_close_scores_ambiguous() -> None: + from paperforge.worker.ocr_tables import build_table_inventory + + blocks = [ + {"block_id": "cap1", "role": "table_caption", "page": 1, "text": "Table 1. Baseline characteristics", "bbox": [100, 100, 700, 140]}, + {"block_id": "asset1", "role": "table_asset", "page": 1, "bbox": [100, 160, 700, 400]}, + {"block_id": "asset2", "role": "table_asset", "page": 1, "bbox": [105, 165, 705, 405]}, + ] + + inventory = build_table_inventory(blocks) + table = inventory["tables"][0] + assert table["match_status"] == "ambiguous" + assert table["has_asset"] is False + assert len(table["candidate_assets"]) == 2