mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
- Add RegionPrepass data model for frontmatter/structured_insert/body classification - Add PDF visual container detection (filled rects) for sidebar detection - Add Box N and keyword-based sidebar anchor detection - Add column-aware block expansion for mixed sidebar clusters - Add _container_text from PDF text blocks for cross-column truncation - Add degraded-mode heading level normalization with font size clustering - Merge adjacent structured_insert blocks into single callout - Fix metadata enrichment from Literature-hub note frontmatter for old papers - Fix heading hierarchy using span font-size clusters when available - Add real-paper regression harness (TSCKAVIS, CAQNW9Q2, A8E7SRVS, K7R8PEKW) - Add body retention, heading, metadata, and callout acceptance tests - 219 passed, 4 control guards passed
143 lines
4.5 KiB
Python
143 lines
4.5 KiB
Python
from __future__ import annotations
|
|
|
|
|
|
def test_health_report_is_independent_from_ocr_status() -> None:
|
|
from paperforge.worker.ocr_health import build_ocr_health
|
|
|
|
structured_blocks = [
|
|
{"role": "section_heading", "text": "1 Introduction"},
|
|
{"role": "body_paragraph", "text": "Body"},
|
|
{"role": "figure_caption", "text": "Figure 1. Example"},
|
|
]
|
|
figure_inventory = {
|
|
"matched_figures": [],
|
|
"unmatched_legends": [{"text": "Figure 1. Example"}],
|
|
"unmatched_assets": [],
|
|
}
|
|
table_inventory = {
|
|
"tables": [],
|
|
"unmatched_captions": [],
|
|
"unmatched_assets": [],
|
|
}
|
|
|
|
report = build_ocr_health(
|
|
page_count=3,
|
|
raw_blocks_count=20,
|
|
structured_blocks=structured_blocks,
|
|
figure_inventory=figure_inventory,
|
|
table_inventory=table_inventory,
|
|
)
|
|
|
|
assert report["page_count"] == 3
|
|
assert report["figure_caption_count"] == 1
|
|
assert report["overall"] in {"yellow", "red"}
|
|
|
|
|
|
def test_health_report_distinguishes_formal_tables_from_segments() -> None:
|
|
from paperforge.worker.ocr_health import build_ocr_health
|
|
|
|
table_inventory = {
|
|
"tables": [
|
|
{"table_id": "tbl_001", "has_asset": True},
|
|
{"table_id": "tbl_002", "has_asset": True},
|
|
],
|
|
"unmatched_captions": [],
|
|
"unmatched_assets": [
|
|
{"asset_id": "seg_001"},
|
|
{"asset_id": "seg_002"},
|
|
{"asset_id": "seg_003"},
|
|
],
|
|
}
|
|
|
|
report = build_ocr_health(
|
|
page_count=5,
|
|
raw_blocks_count=50,
|
|
structured_blocks=[],
|
|
figure_inventory={},
|
|
table_inventory=table_inventory,
|
|
)
|
|
|
|
assert report.get("formal_table_count", 0) == 2
|
|
assert report.get("table_segment_count", 0) == 5
|
|
|
|
|
|
def test_build_spine_health() -> None:
|
|
from paperforge.worker.ocr_health import build_spine_health
|
|
|
|
body_spine = {
|
|
"_meta": {
|
|
"quality": "strong",
|
|
"anchor_pages": [3, 4, 5],
|
|
"sample_count": 12,
|
|
"font_coherence": 0.92,
|
|
"width_dispersion": 0.15,
|
|
},
|
|
}
|
|
result = build_spine_health(body_spine)
|
|
assert result["body_spine_quality"] == "strong"
|
|
assert result["body_anchor_pages"] == [3, 4, 5]
|
|
assert result["body_spine_sample_count"] == 12
|
|
|
|
# Missing _meta defaults
|
|
empty = build_spine_health({})
|
|
assert empty["body_spine_quality"] == "weak"
|
|
assert empty["body_anchor_pages"] == []
|
|
assert empty["body_spine_sample_count"] == 0
|
|
|
|
|
|
def test_build_span_coverage_health() -> None:
|
|
from paperforge.worker.ocr_health import build_span_coverage_health
|
|
|
|
blocks = [
|
|
{"span_metadata": {"size": 10}},
|
|
{"span_metadata": {"size": 10}},
|
|
{"span_metadata": None},
|
|
]
|
|
result = build_span_coverage_health(blocks)
|
|
assert result["coverage_ratio"] == 2 / 3
|
|
assert result["coverage_quality"] == "moderate"
|
|
assert result["blocks_with_span"] == 2
|
|
assert result["blocks_without_span"] == 1
|
|
assert result["degraded_mode_active"] is False
|
|
|
|
# Empty blocks
|
|
empty = build_span_coverage_health([])
|
|
assert empty["degraded_mode_active"] is True
|
|
assert empty["coverage_quality"] == "weak"
|
|
|
|
|
|
def test_layout_audit_health_surface() -> None:
|
|
from paperforge.worker.ocr_health import build_layout_audit_health
|
|
|
|
audit = {
|
|
"status": "warn",
|
|
"page_warnings": {"3": ["heading owns body in different column above it"]},
|
|
"anomaly_count": 1,
|
|
"anomaly_pages": [3],
|
|
}
|
|
result = build_layout_audit_health(audit)
|
|
assert result["layout_audit_status"] == "warn"
|
|
assert result["layout_anomaly_pages"] == [3]
|
|
assert result["layout_anomaly_count"] == 1
|
|
|
|
empty = build_layout_audit_health({})
|
|
assert empty["layout_audit_status"] == "unknown"
|
|
assert empty["layout_anomaly_pages"] == []
|
|
assert empty["layout_anomaly_count"] == 0
|
|
|
|
|
|
def test_ocr_health_includes_span_spine_and_layout_signals() -> None:
|
|
from paperforge.worker.ocr_health import build_ocr_health
|
|
|
|
blocks = [
|
|
{"role": "section_heading", "span_metadata": [{"font": "Body"}]},
|
|
{"role": "section_heading", "span_metadata": [{"font": "Body"}]},
|
|
{"role": "abstract_body", "span_metadata": [{"font": "Body"}]},
|
|
{"role": "reference_item", "span_metadata": [{"font": "Body"}]},
|
|
]
|
|
|
|
health = build_ocr_health(page_count=2, raw_blocks_count=4, structured_blocks=blocks, figure_inventory={}, table_inventory={})
|
|
|
|
assert "span_coverage_quality" in health
|
|
assert "body_spine_quality" in health
|
|
assert "layout_audit_status" in health
|