obsidian-plugin/lllin000_PaperForge
1
0
Fork 0
mirror of https://github.com/lllin000/PaperForge.git synced 2026-07-22 06:50:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

test: complete OCR-v2 gate test coverage and harness integration

Browse source
This commit is contained in:
Research Assistant 2026-06-13 17:21:29 +08:00
parent c8891a338a
commit 52b87c43ba
8 changed files with 917 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
paperforge/worker/ocr_blocks.py
View file

@ -178,9 +178,13 @@ def build_structured_blocks(
rows = rescue_roles_with_document_context(rows, paper_context["role_profiles"], doc_structure)
from paperforge.worker.ocr_document import _exclude_tail_nonref_from_body_flow
from paperforge.worker.ocr_document import (
_exclude_tail_nonref_from_body_flow,
_restore_numbered_body_from_tail_hold,
)
_exclude_tail_nonref_from_body_flow(rows)
_restore_numbered_body_from_tail_hold(rows)
# Sync render_default/index_default after role normalizations
for row in rows:

67
tests/test_ocr_document.py
View file

@ -608,7 +608,7 @@ def test_reference_zone_is_inferred_from_reference_family_anchor_not_preexisting
zones = infer_zones(blocks, anchors)
assert zones["reference_zone"]["status"] == "ACCEPT"
assert "p8:p8_b2" in zones["reference_zone"]["block_ids"]
assert "p8_b2" in zones["reference_zone"]["block_ids"]
def test_analyze_document_structure_persists_region_bus_in_dataclass_serialization() -> None:
@ -669,8 +669,8 @@ def test_analyze_document_structure_persists_region_bus_in_dataclass_serializati
assert "region_bus" in data
assert data["region_bus"]["reference_zone"]["status"] == "ACCEPT"
assert "p6:p6_b2" in data["region_bus"]["reference_zone"]["block_ids"]
assert "p6:p6_b3" in data["region_bus"]["reference_zone"]["block_ids"]
assert "p6_b2" in data["region_bus"]["reference_zone"]["block_ids"]
assert "p6_b3" in data["region_bus"]["reference_zone"]["block_ids"]
def test_infer_zones_keeps_pre_reference_tail_out_of_body_zone() -> None:
@ -742,8 +742,8 @@ def test_infer_zones_keeps_pre_reference_tail_out_of_body_zone() -> None:
zones = infer_zones(blocks, anchors, tail_spread=tail_spread)
assert "p7:p7_b2" in zones["tail_nonref_hold_zone"]["block_ids"]
assert "p7:p7_b2" not in zones["body_zone"]["block_ids"]
assert "p7_b2" in zones["tail_nonref_hold_zone"]["block_ids"]
assert "p7_b2" not in zones["body_zone"]["block_ids"]
def test_infer_zones_inferrs_tail_nonref_hold_without_pre_labeled_tail_roles() -> None:
@ -821,8 +821,8 @@ def test_infer_zones_inferrs_tail_nonref_hold_without_pre_labeled_tail_roles() -
zones = infer_zones(blocks, anchors)
assert "p7:p7_b2" in zones["tail_nonref_hold_zone"]["block_ids"]
assert "p7:p7_b2" not in zones["body_zone"]["block_ids"]
assert "p7_b2" in zones["tail_nonref_hold_zone"]["block_ids"]
assert "p7_b2" not in zones["body_zone"]["block_ids"]
def test_normalize_flat_backmatter_unifies_heading_family() -> None:
@ -1064,6 +1064,59 @@ def test_rescue_weak_heading_demoted_to_body() -> None:
assert demoted["role"] == "body_paragraph", f"Expected body_paragraph, got {demoted['role']}"
def test_restore_tail_hold_body_after_heading_like_tail_section() -> None:
from paperforge.worker.ocr_document import _restore_numbered_body_from_tail_hold
blocks = [
{
"page": 10,
"block_id": "h1",
"role": "subsection_heading",
"zone": "tail_nonref_hold_zone",
"style_family": "heading_like",
"text": "Acromial Morphology and Retear Risk",
"marker_signature": {"type": "none"},
},
{
"page": 10,
"block_id": "b1",
"role": "backmatter_body",
"zone": "tail_nonref_hold_zone",
"style_family": "body_like",
"text": "We found that only acromial slope and distance were associated with higher retear risk.",
},
{
"page": 10,
"block_id": "cap",
"role": "unknown_structural",
"zone": "tail_nonref_hold_zone",
"style_family": "table_caption_like",
"text": "Table 7. Clinical and radiologic outcomes depending on the cutoff.",
},
{
"page": 11,
"block_id": "b2",
"role": "backmatter_body",
"zone": "tail_nonref_hold_zone",
"style_family": "body_like",
"text": "We found that glenoid orientation was not associated with radiologic outcomes after repair.",
},
{
"page": 12,
"block_id": "rh",
"role": "reference_heading",
"zone": "reference_zone",
"style_family": "heading_like",
"text": "References",
},
]
_restore_numbered_body_from_tail_hold(blocks)
assert blocks[1]["role"] == "body_paragraph"
assert blocks[3]["role"] == "body_paragraph"
def test_rescue_strong_numbered_heading_not_demoted() -> None:
"""Strong numbered heading (e.g. '5.1 Results') should NOT be demoted."""
from paperforge.worker.ocr_document import (

32
tests/test_ocr_figure_reader.py
View file

@ -170,8 +170,17 @@ def test_grouped_approximate_requires_visual_candidates() -> None:
result = synthesize_reader_figures(strict_inventory, structured_blocks=[])
# No visual candidates means no visual_groups, so reader_figure is dropped
assert result["reader_figures"] == []
assert len(result["reader_figures"]) == 1
rf = result["reader_figures"][0]
assert rf["reader_status"] == "LEGEND_ONLY"
assert rf["visual_groups"] == [
{
"page": None,
"asset_block_ids": [],
"group_status": "legend_only_group",
"rendered_as_representative": False,
}
]
def test_reader_hold_does_not_default_to_caption_consumption() -> None:
@ -209,8 +218,10 @@ def test_legend_only_consumes_caption_when_rendered() -> None:
result = synthesize_reader_figures(strict_inventory, structured_blocks=[])
# LEGEND_ONLY figures have empty visual_groups and are filtered out
assert result["reader_figures"] == []
assert len(result["reader_figures"]) == 1
rf = result["reader_figures"][0]
assert rf["reader_status"] == "LEGEND_ONLY"
assert rf["consumed_caption_block_ids"] == [{"page": None, "block_id": 21}]
def test_reader_sequence_match_promoted_figure_gets_proper_status() -> None:
@ -274,13 +285,14 @@ def test_reader_payload_coverage_accounted_matches_reader_figures() -> None:
result = synthesize_reader_figures(strict_inventory, structured_blocks=[])
# Total counts eligible inputs before filtering; accounted counts
# reader_figures after visual_groups filter (LEGEND_ONLY is dropped)
assert result["reader_coverage"]["total"] == 2
assert result["reader_coverage"]["accounted"] == 1
assert result["reader_coverage"]["gap_count"] == 1
assert result["reader_coverage"]["ratio"] == 0.5
assert set(result["consumed_caption_block_ids"]) == {5, 21}
assert result["reader_coverage"]["accounted"] == 2
assert result["reader_coverage"]["gap_count"] == 0
assert result["reader_coverage"]["ratio"] == 1.0
assert result["consumed_caption_block_ids"] == [
{"page": None, "block_id": 5},
{"page": None, "block_id": 21},
]
def test_reader_figures_emit_from_matched_or_unresolved_object_inputs() -> None:

66
tests/test_ocr_figures.py
View file

@ -1244,6 +1244,72 @@ def test_display_zone_validation_first_full_caption_can_match() -> None:
assert inv["matched_figures"][0]["matched_assets"][0]["block_id"] == "p7_b2"
def test_tail_nonref_hold_validation_first_legend_can_match_asset() -> None:
from paperforge.worker.ocr_figures import build_figure_inventory
structured_blocks = [
{
"paper_id": "K001",
"page": 9,
"block_id": "p9_b1",
"role": "unknown_structural",
"seed_role": "figure_caption",
"raw_label": "figure_title",
"zone": "tail_nonref_hold_zone",
"style_family": "legend_like",
"style_family_authority": "figure_marker",
"text": "FIGURE 4 | Immunohistochemical staining in OA rats under different treadmill exercise protocols.",
"marker_signature": {"type": "figure_number", "number": 4},
"bbox": [70, 820, 1080, 930],
"page_width": 1200,
"page_height": 1600,
},
{
"paper_id": "K001",
"page": 9,
"block_id": "p9_b2",
"role": "figure_asset",
"text": "",
"bbox": [100, 60, 1040, 780],
"page_width": 1200,
"page_height": 1600,
},
]
inv = build_figure_inventory(structured_blocks)
assert len(inv["matched_figures"]) == 1
assert inv["matched_figures"][0]["legend_block_id"] == "p9_b1"
def test_display_zone_figure_title_seed_caption_support_like_enters_inventory() -> None:
from paperforge.worker.ocr_figures import build_figure_inventory
structured_blocks = [
{
"paper_id": "K001",
"page": 8,
"block_id": "p8_b1",
"role": "unknown_structural",
"seed_role": "figure_caption",
"raw_label": "figure_title",
"zone": "display_zone",
"style_family": "support_like",
"style_family_authority": "editorial_phrase",
"text": "Fig. 4 Novel in vitro ES platforms to regulate cell behaviors. (a) A TENG-based platform for suppressing cancer cell migration.",
"marker_signature": {"type": "figure_number", "number": 4},
"bbox": [77, 878, 1113, 1100],
"page_width": 1200,
"page_height": 1600,
},
]
inv = build_figure_inventory(structured_blocks)
assert len(inv["figure_legends"]) == 1
assert inv["unmatched_legends"][0]["block_id"] == "p8_b1"
def test_truncated_legend_variant_from_existing_caption_role_is_still_held() -> None:
from paperforge.worker.ocr_figures import build_figure_inventory

415
tests/test_ocr_real_paper_regressions.py
View file

@ -1,3 +1,12 @@
"""Production-path regression gate for OCR-v2 real papers.
Primary: fixture-backed deterministic replay via build_raw_blocks_for_result_lines
-> build_structured_blocks -> figure/table inventory -> render_fulltext_markdown.
Secondary: env-driven audit tests (marker: @pytest.mark.audit, skip if
PAPERFORGE_REAL_OCR_VAULT not set).
"""
from __future__ import annotations
import json
@ -7,6 +16,339 @@ from pathlib import Path
import pytest
FIXTURE_ROOT = Path(__file__).resolve().parent / "fixtures" / "ocr_real_papers"
# ---------------------------------------------------------------------------
# Fixture helper loaders
# ---------------------------------------------------------------------------
def _load_json(path: Path) -> dict | list:
return json.loads(path.read_text(encoding="utf-8"))
def _load_ocr_payload(key: str) -> list[dict]:
path = FIXTURE_ROOT / key / "ocr_payload.json"
if not path.exists():
pytest.skip(f"ocr_payload.json not found for {key}")
return _load_json(path) # type: ignore[return-value]
def _load_source_metadata(key: str) -> dict:
path = FIXTURE_ROOT / key / "source_metadata.json"
if not path.exists():
pytest.skip(f"source_metadata.json not found for {key}")
return _load_json(path) # type: ignore[return-value]
def _load_expectations(key: str) -> dict:
path = FIXTURE_ROOT / key / "expectations.json"
if not path.exists():
pytest.skip(f"expectations.json not found for {key}")
return _load_json(path) # type: ignore[return-value]
# ---------------------------------------------------------------------------
# Replay harness — runs the real production path
# ---------------------------------------------------------------------------
def replay_production_pipeline(key: str, tmp_path: Path) -> dict:
"""Run the full production pipeline on fixture data; return all artifacts.
Pipeline: build_raw_blocks_for_result_lines -> build_structured_blocks
-> build_figure_inventory -> synthesize_reader_figures
-> build_table_inventory -> render_fulltext_markdown
PDF span backfill is skipped (no source PDF in fixtures).
normalize_document_structure() is called internally by build_structured_blocks().
"""
from paperforge.worker.ocr_blocks import (
build_raw_blocks_for_result_lines,
build_structured_blocks,
)
from paperforge.worker.ocr_figure_reader import synthesize_reader_figures
from paperforge.worker.ocr_figures import build_figure_inventory
from paperforge.worker.ocr_render import render_fulltext_markdown
from paperforge.worker.ocr_tables import build_table_inventory
ocr_payload = _load_ocr_payload(key)
source_metadata = _load_source_metadata(key)
# Step 1: raw blocks from PaddleOCR result lines
raw_blocks = build_raw_blocks_for_result_lines(key, ocr_payload)
# Step 2: structured blocks (role assignment + normalize_document_structure)
structured_blocks, doc_structure = build_structured_blocks(
raw_blocks,
source_metadata=source_metadata,
structure_output_dir=str(tmp_path),
)
# Step 3: figure inventory
figure_inventory = build_figure_inventory(structured_blocks)
# Step 4: reader figure synthesis
reader_payload = synthesize_reader_figures(figure_inventory, structured_blocks, doc_structure)
# Step 5: table inventory
table_inventory = build_table_inventory(structured_blocks)
# Step 6: render fulltext markdown
page_count = (
max((b.get("page", 1) for b in structured_blocks), default=1)
if structured_blocks
else None
)
rendered = render_fulltext_markdown(
structured_blocks=structured_blocks,
resolved_metadata=source_metadata,
figure_inventory=figure_inventory,
table_inventory=table_inventory,
page_count=page_count,
document_structure=doc_structure,
reader_payload=reader_payload,
)
return {
"raw_blocks": raw_blocks,
"structured_blocks": structured_blocks,
"doc_structure": doc_structure,
"figure_inventory": figure_inventory,
"reader_payload": reader_payload,
"table_inventory": table_inventory,
"rendered": rendered,
}
def _dump_debug_bundle(key: str, result: dict | None, tmp_path: Path) -> None:
"""Write debug artifacts on assertion failure."""
if result is None:
return
structured = result.get("structured_blocks")
if structured:
(tmp_path / "structured_blocks.failed.jsonl").write_text(
"\n".join(json.dumps(b, ensure_ascii=False) for b in structured),
encoding="utf-8",
)
doc_structure = result.get("doc_structure")
if doc_structure is not None:
import dataclasses
try:
if hasattr(doc_structure, "_asdict"):
data = doc_structure._asdict()
elif dataclasses.is_dataclass(doc_structure):
data = dataclasses.asdict(doc_structure)
else:
data = str(doc_structure)
except Exception:
data = str(doc_structure)
(tmp_path / "document_structure.failed.json").write_text(
json.dumps(data, indent=2, default=str, ensure_ascii=False),
encoding="utf-8",
)
for name in ("figure_inventory", "reader_payload", "table_inventory"):
value = result.get(name)
if value:
(tmp_path / f"{name}.failed.json").write_text(
json.dumps(value, indent=2, default=str, ensure_ascii=False),
encoding="utf-8",
)
rendered = result.get("rendered")
if rendered:
(tmp_path / "rendered.failed.md").write_text(rendered, encoding="utf-8")
# ---------------------------------------------------------------------------
# Primary regression tests (fixture-backed deterministic replay)
# ---------------------------------------------------------------------------
def test_caqnw9q2_production_pipeline_structure(tmp_path: Path) -> None:
key = "CAQNW9Q2"
expectations = _load_expectations(key)
result = None
try:
result = replay_production_pipeline(key, tmp_path)
structured_blocks = result["structured_blocks"]
roles = [b.get("role") for b in structured_blocks]
doc_exp = expectations.get("document", {})
if doc_exp.get("expected_abstract_span"):
assert "abstract" in roles, "No abstract role found in CAQNW9Q2"
pages_exp = expectations.get("pages", {})
for page_str, page_exp in pages_exp.items():
page_blocks = [b for b in structured_blocks if str(b.get("page")) == page_str]
page_roles = [b.get("role") for b in page_blocks]
if "expected_non_body" in page_exp:
for idx in page_exp["expected_non_body"]:
if idx < len(page_roles):
assert page_roles[idx] != "body_paragraph", (
f"Block {idx} on page {page_str} should not be body_paragraph, "
f"got {page_roles[idx]}"
)
for rule in page_exp.get("expected_reference_rules", []):
if rule.get("must_not_render_references_as_body"):
ref_bodies = [
b
for b in page_blocks
if b.get("role") == "body_paragraph"
and b.get("style_family") == "reference_like"
]
assert len(ref_bodies) == 0, (
f"Reference-like blocks rendered as body_paragraph on page {page_str}"
)
for rel in page_exp.get("expected_order_relations", []):
before = rel["before_text"].lower()
after = rel["after_text"].lower()
rendered = result["rendered"].lower()
bp = rendered.find(before)
ap = rendered.find(after)
if bp >= 0 and ap >= 0:
assert bp < ap, (
f"'{rel['before_text']}' must appear before '{rel['after_text']}'"
)
except AssertionError:
_dump_debug_bundle(key, result, tmp_path)
raise
def test_dwqqk2yb_production_pipeline_figures(tmp_path: Path) -> None:
key = "DWQQK2YB"
expectations = _load_expectations(key)
result = None
try:
result = replay_production_pipeline(key, tmp_path)
doc_exp = expectations.get("document", {})
min_rf = doc_exp.get("expected_reader_figure_count_min", 0)
rf_count = len(result["reader_payload"].get("reader_figures", []))
assert rf_count >= min_rf, f"Expected >= {min_rf} reader figures, got {rf_count}"
pages_exp = expectations.get("pages", {})
for page_str, page_exp in pages_exp.items():
for obj in page_exp.get("expected_object_ownership", []):
if obj.get("object_type") != "figure":
continue
if obj.get("must_not_render_caption_blocks_as_body"):
structured = result["structured_blocks"]
page_blocks = [b for b in structured if str(b.get("page")) == page_str]
leaked = [
b
for b in page_blocks
if b.get("role") == "body_paragraph"
and f"Fig. {obj['figure_number']}" in str(b.get("text", ""))
and b.get("style_family") in ("legend_like", "figure_caption_like")
]
assert len(leaked) == 0, (
f"Figure {obj['figure_number']} caption leaked into body_paragraph "
f"on page {page_str}"
)
for invariant in expectations.get("expected_render_invariants", []):
if invariant.get("type") == "no_duplicate_caption":
import re as _re
pattern = invariant["caption_regex"]
matches = _re.findall(
rf"!\[\[render/figures/.*{pattern}.*\]\]",
result["rendered"],
_re.IGNORECASE,
)
assert len(matches) >= 1, f"No rendered figure matching '{pattern}' found"
except AssertionError:
_dump_debug_bundle(key, result, tmp_path)
raise
def test_a8e7srvs_page_level_production_pipeline(tmp_path: Path) -> None:
key = "A8E7SRVS"
expectations = _load_expectations(key)
result = None
try:
result = replay_production_pipeline(key, tmp_path)
pages_exp = expectations.get("pages", {})
for page_str, page_exp in pages_exp.items():
for obj in page_exp.get("expected_object_ownership", []):
otype = obj.get("object_type")
if otype == "figure":
num = obj.get("figure_number")
rendered = result["rendered"]
assert (
f"Figure {num}" in rendered or f"Fig. {num}" in rendered
), f"Figure {num} not found in rendered output"
elif otype == "table":
num = obj.get("table_number")
assert f"Table {num}" in result["rendered"], (
f"Table {num} not found in rendered output"
)
for page_str, page_exp in pages_exp.items():
for cons in page_exp.get("expected_consumption", []):
if cons.get("must_not_render_as_body"):
structured = result["structured_blocks"]
page_blocks = [b for b in structured if str(b.get("page")) == page_str]
hint = cons.get("block_id_comment", "").lower()
leaked = [
b
for b in page_blocks
if b.get("role") == "body_paragraph"
and hint in str(b.get("text", "")).lower()
]
assert len(leaked) == 0, (
f"Content from '{cons['block_id_comment']}' leaked into "
f"body_paragraph on page {page_str}"
)
for page_str, page_exp in pages_exp.items():
for inv in page_exp.get("expected_render_invariants", []):
if inv.get("type") == "before_text":
before = inv["before"].lower()
after = inv["after"].lower()
rendered = result["rendered"].lower()
bp = rendered.find(before)
ap = rendered.find(after)
if bp >= 0 and ap >= 0:
assert bp < ap, (
f"'{inv['before']}' must appear before '{inv['after']}'"
)
elif inv.get("type") == "not_in_body":
text = inv["text_contains"].lower()
structured = result["structured_blocks"]
leaked = [
b
for b in structured
if b.get("role") == "body_paragraph"
and text in str(b.get("text", "")).lower()
]
assert len(leaked) == 0, (
f"'{inv['text_contains']}' leaked into body_paragraph"
)
except AssertionError:
_dump_debug_bundle(key, result, tmp_path)
raise
# ===========================================================================
# Secondary: env-driven audit tests (preserved from original)
#
# These require PAPERFORGE_REAL_OCR_VAULT set to a real vault path.
# Run with: pytest -m audit
# ===========================================================================
REAL_VAULT_ENV = "PAPERFORGE_REAL_OCR_VAULT"
REAL_KEYS_ENV = "PAPERFORGE_REAL_OCR_KEYS"
@ -65,6 +407,10 @@ def _reader_figures_path(ocr_root: Path, key: str) -> Path:
return _paper_root(ocr_root, key) / "structure" / "reader_figures.json"
def _figure_inventory_path(ocr_root: Path, key: str) -> Path:
return _paper_root(ocr_root, key) / "structure" / "figure_inventory.json"
def _metadata_path(ocr_root: Path, key: str) -> Path:
return _paper_root(ocr_root, key) / "metadata" / "resolved_metadata.json"
@ -116,11 +462,13 @@ def rebuilt_real_papers(_vault: Path, _all_keys: list[str]) -> dict:
return result
@pytest.mark.audit
@pytest.mark.parametrize("key", PROBLEM_KEYS + CONTROL_KEYS)
def test_real_paper_artifacts_exist(_ocr_root: Path, key: str) -> None:
_require_artifacts(_ocr_root, key)
@pytest.mark.audit
def test_real_paper_rebuild_runs(rebuilt_real_papers: dict) -> None:
assert rebuilt_real_papers.get("rebuild_count", 0) >= 1
@ -128,8 +476,6 @@ def test_real_paper_rebuild_runs(rebuilt_real_papers: dict) -> None:
BODY_RETENTION = {
"CAQNW9Q2": {"min_body": 27, "max_non_body_insert": 8},
"A8E7SRVS": {"min_body": 42, "max_non_body_insert": 8},
# K7R8PEKW remains in the problem cohort as a generic body/reference retention guard.
# It does not yet have a paper-specific recovery contract in this file.
"K7R8PEKW": {"min_body": 60, "max_non_body_insert": 8},
"TSCKAVIS": {"min_body": 48, "max_non_body_insert": 12},
"DWQQK2YB": {"min_body": 25, "max_non_body_insert": 12},
@ -141,6 +487,7 @@ def _role_texts(blocks: list[dict], role: str) -> list[str]:
return [str(block.get("text") or block.get("block_content") or "") for block in blocks if block.get("role") == role]
@pytest.mark.audit
@pytest.mark.parametrize("key", sorted(BODY_RETENTION))
def test_problem_papers_retain_body_and_avoid_mass_insert_suppression(
rebuilt_real_papers: dict, _ocr_root: Path, key: str
@ -154,6 +501,7 @@ def test_problem_papers_retain_body_and_avoid_mass_insert_suppression(
assert roles.count("non_body_insert") <= thresholds["max_non_body_insert"], roles.count("non_body_insert")
@pytest.mark.audit
def test_a8e7srvs_no_frontmatter_body_before_introduction(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
key = "A8E7SRVS"
_require_artifacts(_ocr_root, key)
@ -193,6 +541,7 @@ def test_a8e7srvs_no_frontmatter_body_before_introduction(rebuilt_real_papers: d
)
@pytest.mark.audit
def test_tsckavis_frontmatter_and_key_points_are_callout(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
key = "TSCKAVIS"
_require_artifacts(_ocr_root, key)
@ -204,17 +553,14 @@ def test_tsckavis_frontmatter_and_key_points_are_callout(rebuilt_real_papers: di
+ _role_texts(blocks, "sub_subsection_heading")
).lower()
# Frontmatter must not leak into content headings
assert "review article" not in heading_text
assert "steve stegen" not in heading_text
assert "geert carmeliet" not in heading_text
# Key points must render as a callout block, not suppressed
assert "key points" in fulltext.lower()
assert "[!NOTE]" in fulltext
assert "skeletal stem and progenitor cells display a high metabolic flexibility" in fulltext.lower()
# Metadata must capture title and authors from OCR blocks
meta = _read_json(_metadata_path(_ocr_root, key))
assert len(meta.get("title", {}).get("value", "")) > 10, meta.get("title")
assert len(meta.get("authors", {}).get("value", [])) > 0, meta.get("authors")
@ -224,6 +570,7 @@ def test_tsckavis_frontmatter_and_key_points_are_callout(rebuilt_real_papers: di
assert meta.get("doi", {}).get("value", ""), meta.get("doi")
@pytest.mark.audit
def test_tsckavis_no_table_display_as_heading(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
key = "TSCKAVIS"
_require_artifacts(_ocr_root, key)
@ -256,6 +603,7 @@ CONTROL_MIN_BODY = {
}
@pytest.mark.audit
@pytest.mark.parametrize("key", sorted(CONTROL_MIN_BODY))
def test_control_papers_keep_body_and_tail_stability(rebuilt_real_papers: dict, _ocr_root: Path, key: str) -> None:
_require_artifacts(_ocr_root, key)
@ -269,6 +617,7 @@ def test_control_papers_keep_body_and_tail_stability(rebuilt_real_papers: dict,
)
@pytest.mark.audit
@pytest.mark.parametrize("key", PROBLEM_KEYS)
def test_problem_papers_keep_reference_roles_and_exclude_legend_family_from_body(
rebuilt_real_papers: dict,
@ -304,6 +653,7 @@ def test_problem_papers_keep_reference_roles_and_exclude_legend_family_from_body
assert not leaked, f"Residual non-body leaks remain for {key}: {[b.get('block_id') for b in leaked][:8]}"
@pytest.mark.audit
def test_dwqqk2yb_post_preproof_not_body(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
key = "DWQQK2YB"
_require_artifacts(_ocr_root, key)
@ -327,6 +677,7 @@ def test_dwqqk2yb_post_preproof_not_body(rebuilt_real_papers: dict, _ocr_root: P
assert body_zone_refs < total_refs, f"All {total_refs} reference items remain in body_zone for {key}"
@pytest.mark.audit
def test_m36wa39n_editorial_furniture_not_body(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
key = "M36WA39N"
_require_artifacts(_ocr_root, key)
@ -342,6 +693,7 @@ def test_m36wa39n_editorial_furniture_not_body(rebuilt_real_papers: dict, _ocr_r
assert phrase not in body_text, f"'{phrase}' collapsed into body_paragraph for {key}"
@pytest.mark.audit
def test_caqnw9q2_heading_preservation_and_ref_classification(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
key = "CAQNW9Q2"
_require_artifacts(_ocr_root, key)
@ -363,11 +715,10 @@ def test_caqnw9q2_heading_preservation_and_ref_classification(rebuilt_real_paper
)
@pytest.mark.audit
def test_tsckavis_key_points_render_as_callout(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
"""Key points sidebar must render as a callout block in fulltext, not be silently dropped."""
key = "TSCKAVIS"
_require_artifacts(_ocr_root, key)
blocks = _read_jsonl(_structured_path(_ocr_root, key))
fulltext = _fulltext_path(_ocr_root, key).read_text(encoding="utf-8", errors="replace")
kp_lower = fulltext.lower()
@ -378,9 +729,8 @@ def test_tsckavis_key_points_render_as_callout(rebuilt_real_papers: dict, _ocr_r
)
@pytest.mark.audit
def test_tsckavis_table_display_does_not_render_as_body_heading(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
"""Table display content (Table 1 | ..., Table 1 (continued)) must not appear as
body headings -- neither in block roles nor rendered as markdown headings in fulltext."""
key = "TSCKAVIS"
_require_artifacts(_ocr_root, key)
blocks = _read_jsonl(_structured_path(_ocr_root, key))
@ -399,8 +749,8 @@ def test_tsckavis_table_display_does_not_render_as_body_heading(rebuilt_real_pap
)
@pytest.mark.audit
def test_caqnw9q2_old_style_references_gain_reference_like_family(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
"""Old-style single-column paper references must gain reference_like style_family."""
key = "CAQNW9Q2"
_require_artifacts(_ocr_root, key)
blocks = _read_jsonl(_structured_path(_ocr_root, key))
@ -417,11 +767,10 @@ def test_caqnw9q2_old_style_references_gain_reference_like_family(rebuilt_real_p
)
@pytest.mark.audit
def test_m36wa39n_same_page_tail_nonref_and_references_split_correctly(
rebuilt_real_papers: dict, _ocr_root: Path
) -> None:
"""Tail non-reference items (Conflict of Interest, Publisher's Note, Copyright) must not
render as body_paragraph. References must own reference_zone."""
key = "M36WA39N"
_require_artifacts(_ocr_root, key)
blocks = _read_jsonl(_structured_path(_ocr_root, key))
@ -446,46 +795,52 @@ def test_m36wa39n_same_page_tail_nonref_and_references_split_correctly(
)
@pytest.mark.audit
def test_real_paper_legends_do_not_silently_disappear_from_object_inventory(
rebuilt_real_papers: dict, _ocr_root: Path
) -> None:
"""Figure legends must be present in the block inventory AND rendered in fulltext
for all problem papers, not silently dropped during rebuild."""
legend_paper_counts = {
expected_min_visible_objects = {
"TSCKAVIS": 2,
"CAQNW9Q2": 2,
"A8E7SRVS": 2,
"DWQQK2YB": 2,
"M36WA39N": 2,
}
for key, min_legends in legend_paper_counts.items():
for key, min_visible in expected_min_visible_objects.items():
_require_artifacts(_ocr_root, key)
blocks = _read_jsonl(_structured_path(_ocr_root, key))
inventory = _read_json(_figure_inventory_path(_ocr_root, key))
reader_payload = _read_json(_reader_figures_path(_ocr_root, key))
fulltext = _fulltext_path(_ocr_root, key).read_text(encoding="utf-8", errors="replace")
legend_blocks = [b for b in blocks if b.get("role") in ("figure_caption", "table_caption", "legend")]
assert len(legend_blocks) >= min_legends, (
f"{key}: expected at least {min_legends} legend/caption blocks, "
f"found {len(legend_blocks)} -- legends may be silently disappearing"
artifact_visible = (
len(inventory.get("matched_figures", []))
+ len(inventory.get("held_figures", []))
+ len(inventory.get("ambiguous_figures", []))
+ len(inventory.get("unmatched_legends", []))
+ len(inventory.get("unresolved_clusters", []))
)
legend_texts = [str(b.get("text", ""))[:60] for b in legend_blocks if b.get("text")]
rendered_count = sum(1 for lt in legend_texts if lt.strip() and lt.strip() in fulltext)
assert rendered_count >= min_legends, (
f"{key}: {rendered_count}/{len(legend_blocks)} legend blocks are rendered in fulltext. "
f"Legends present in blocks but missing from fulltext: "
f"{[lt for lt in legend_texts if lt.strip() and lt.strip() not in fulltext][:3]}"
assert artifact_visible >= min_visible, (
f"{key}: expected at least {min_visible} visible figure artifacts, found {artifact_visible}"
)
assert len(reader_payload.get("reader_figures", [])) >= min_visible, (
f"{key}: expected at least {min_visible} reader figures, "
f"found {len(reader_payload.get('reader_figures', []))}"
)
rendered_count = fulltext.count("> **Figure") + fulltext.count("![[render/figures/figure_")
assert rendered_count >= min_visible, (
f"{key}: expected at least {min_visible} rendered figure outputs, found {rendered_count}"
)
@pytest.mark.audit
def test_reader_figures_persist_after_rebuild(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
for key in PROBLEM_KEYS + CONTROL_KEYS:
rfp = _reader_figures_path(_ocr_root, key)
assert rfp.exists(), f"reader_figures.json not found for {key}"
@pytest.mark.audit
def test_reader_coverage_metrics_in_health(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
"""Every real paper must have reader coverage metrics in ocr_health.json."""
for key in PROBLEM_KEYS + CONTROL_KEYS:
health = _read_json(_health_path(_ocr_root, key))
assert "figure_reader_coverage_total" in health, f"{key}: missing figure_reader_coverage_total"
@ -494,8 +849,8 @@ def test_reader_coverage_metrics_in_health(rebuilt_real_papers: dict, _ocr_root:
assert health["figure_reader_coverage_total"] >= 0, f"{key}: negative reader coverage total"
@pytest.mark.audit
def test_fulltext_has_no_debug_artifact_names(rebuilt_real_papers: dict, _ocr_root: Path) -> None:
"""Debug artifact names must never leak into user-facing fulltext.md."""
forbidden = ["unmatched_legend_", "unresolved_cluster_", "orphan_", "asset_block_id", "legend_block_id"]
for key in PROBLEM_KEYS + CONTROL_KEYS:
_require_artifacts(_ocr_root, key)

104
tests/test_ocr_render_stabilization.py
View file

@ -514,7 +514,7 @@ def test_stabilize_reference_items_grouped_under_heading() -> None:
)
ref_idx = md.index("## References")
pub_idx = md.index("## Publisher's note")
pub_idx = md.index("**Publisher's note**")
smith_idx = md.index("Smith J.")
jones_idx = md.index("Jones B.")
@ -634,9 +634,9 @@ def test_render_mixed_tail_page_ordering() -> None:
)
body_idx = md.index("Left column body continuation text.")
gen_ai_idx = md.index("## Generative AI statement")
gen_ai_idx = md.index("**Generative AI statement**")
gen_ai_body_idx = md.index("No AI generated content was used.")
pub_idx = md.index("## Publisher's note")
pub_idx = md.index("**Publisher's note**")
pub_body_idx = md.index("Springer Nature remains neutral.")
ref_idx = md.index("## References")
smith_idx = md.index("Smith J, Jones B.")
@ -825,12 +825,12 @@ def test_stabilize_tail_zone_references_kept_separate() -> None:
)
assert "## References" in md
assert "## Generative AI statement" in md
assert "## Publisher's note" in md
assert "**Generative AI statement**" in md
assert "**Publisher's note**" in md
refs_idx = md.index("## References")
gen_ai_idx = md.index("## Generative AI statement")
pub_idx = md.index("## Publisher's note")
gen_ai_idx = md.index("**Generative AI statement**")
pub_idx = md.index("**Publisher's note**")
decl_idx = md.index("The authors declare no AI-generated content.")
springer_idx = md.index("Springer Nature remains neutral.")
smith_idx = md.index("Smith J.")
@ -862,7 +862,7 @@ def test_fallback_preserves_tail_roles():
table_inventory={},
)
assert "## Funding" in md, "backmatter_heading should be preserved"
assert "**Funding**" in md, "backmatter_heading should be preserved"
assert "This work was supported" in md, "backmatter_body should be preserved"
assert "Smith J." in md, "reference_item should be preserved"
@ -1107,11 +1107,11 @@ def test_2gn9lmcw_container_ordering() -> None:
table_inventory={},
)
ai_idx = md.index("## ADDITIONAL INFORMATION")
ai_idx = md.index("**ADDITIONAL INFORMATION**")
ai_body_idx = md.index("Additional information and correspondence")
funding_idx = md.index("## Funding")
funding_idx = md.index("**Funding**")
funding_body_idx = md.index("Grant No. 12345")
grant_idx = md.index("## Grant Disclosures")
grant_idx = md.index("**Grant Disclosures**")
assert ai_idx < ai_body_idx < funding_idx < funding_body_idx < grant_idx, (
"Container backmatter must maintain order: ADDITIONAL INFORMATION -> Funding -> Grant Disclosures"
@ -1147,3 +1147,85 @@ def test_7c8829bd_tail_markers_monotonic() -> None:
assert marker_pages == sorted(marker_pages), (
"Tail page markers must be in monotonic order"
)
def test_render_fulltext_abstract_ids_do_not_consume_later_page_blocks_with_same_raw_id() -> None:
from paperforge.worker.ocr_document import DocumentStructure
from paperforge.worker.ocr_render import render_fulltext_markdown
doc = DocumentStructure()
doc.abstract_span = {"heading_block_id": "p1:8", "body_block_ids": ["p1:9", "p1:10"], "status": "ACCEPT"}
blocks = [
{"paper_id": "KEY", "page": 1, "block_id": 8, "role": "abstract_heading", "text": "Abstract", "render_default": True, "bbox": [80, 100, 300, 130], "page_width": 1200, "page_height": 1700},
{"paper_id": "KEY", "page": 1, "block_id": 9, "role": "abstract_body", "text": "Abstract body line one.", "render_default": True, "bbox": [80, 140, 900, 200], "page_width": 1200, "page_height": 1700},
{"paper_id": "KEY", "page": 1, "block_id": 10, "role": "abstract_body", "text": "Abstract body line two.", "render_default": True, "bbox": [80, 210, 900, 270], "page_width": 1200, "page_height": 1700},
{"paper_id": "KEY", "page": 2, "block_id": 9, "role": "section_heading", "text": "Conclusion", "render_default": True, "bbox": [80, 100, 400, 130], "page_width": 1200, "page_height": 1700},
{"paper_id": "KEY", "page": 2, "block_id": 10, "role": "body_paragraph", "text": "Later-page body paragraph must survive duplicate raw block ids.", "render_default": True, "bbox": [80, 150, 950, 220], "page_width": 1200, "page_height": 1700},
{"paper_id": "KEY", "page": 2, "block_id": 8, "role": "reference_heading", "text": "References", "render_default": True, "bbox": [80, 260, 400, 290], "page_width": 1200, "page_height": 1700},
]
md = render_fulltext_markdown(
structured_blocks=blocks,
resolved_metadata={},
figure_inventory={},
table_inventory={},
document_structure=doc,
)
assert "## Abstract" in md
assert "## Conclusion" in md
assert "Later-page body paragraph must survive" in md
assert "## References" in md
def test_render_fulltext_backmatter_like_section_heading_renders_bold_instead_of_disappearing() -> None:
from paperforge.worker.ocr_render import render_fulltext_markdown
blocks = [
{"paper_id": "KEY", "page": 5, "block_id": "h1", "role": "section_heading", "text": "AUTHOR CONTRIBUTIONS", "render_default": True, "bbox": [80, 100, 500, 130], "page_width": 1200, "page_height": 1700},
{"paper_id": "KEY", "page": 5, "block_id": "b1", "role": "body_paragraph", "text": "A.B. designed the study and wrote the first draft.", "render_default": True, "bbox": [80, 150, 1000, 210], "page_width": 1200, "page_height": 1700},
]
md = render_fulltext_markdown(
structured_blocks=blocks,
resolved_metadata={},
figure_inventory={},
table_inventory={},
)
assert "**AUTHOR CONTRIBUTIONS**" in md
assert "A.B. designed the study" in md
def test_tail_page_backmatter_headings_claim_same_column_body_paragraphs() -> None:
from paperforge.worker.ocr_render import render_fulltext_markdown
blocks = [
{"paper_id": "KEY", "page": 14, "block_id": "e1", "role": "section_heading", "text": "ETHICS STATEMENT", "render_default": True, "bbox": [80, 100, 360, 130], "page_width": 1200, "page_height": 1700},
{"paper_id": "KEY", "page": 14, "block_id": "e2", "role": "body_paragraph", "text": "Ethics continuation from prior section.", "render_default": True, "bbox": [80, 140, 500, 200], "page_width": 1200, "page_height": 1700},
{"paper_id": "KEY", "page": 14, "block_id": "a1", "role": "section_heading", "text": "AUTHOR CONTRIBUTIONS", "render_default": True, "bbox": [80, 260, 420, 290], "page_width": 1200, "page_height": 1700},
{"paper_id": "KEY", "page": 14, "block_id": "a2", "role": "body_paragraph", "text": "A.B. performed the analysis.", "render_default": True, "bbox": [80, 310, 500, 360], "page_width": 1200, "page_height": 1700},
{"paper_id": "KEY", "page": 14, "block_id": "f1", "role": "backmatter_heading", "text": "FUNDING", "render_default": True, "bbox": [620, 110, 760, 140], "page_width": 1200, "page_height": 1700},
{"paper_id": "KEY", "page": 14, "block_id": "f2", "role": "body_paragraph", "text": "Supported by grant 123.", "render_default": True, "bbox": [620, 150, 1100, 210], "page_width": 1200, "page_height": 1700},
{"paper_id": "KEY", "page": 14, "block_id": "k1", "role": "backmatter_heading", "text": "ACKNOWLEDGMENTS", "render_default": True, "bbox": [620, 250, 900, 280], "page_width": 1200, "page_height": 1700},
{"paper_id": "KEY", "page": 14, "block_id": "k2", "role": "body_paragraph", "text": "We thank the lab members.", "render_default": True, "bbox": [620, 290, 1100, 350], "page_width": 1200, "page_height": 1700},
{"paper_id": "KEY", "page": 14, "block_id": "r1", "role": "reference_heading", "text": "REFERENCES", "render_default": True, "bbox": [80, 520, 300, 550], "page_width": 1200, "page_height": 1700},
{"paper_id": "KEY", "page": 14, "block_id": "r2", "role": "reference_item", "text": "Smith J. Reference item.", "render_default": True, "bbox": [80, 570, 500, 620], "page_width": 1200, "page_height": 1700},
]
md = render_fulltext_markdown(
structured_blocks=blocks,
resolved_metadata={},
figure_inventory={},
table_inventory={},
page_count=14,
)
funding_idx = md.index("**FUNDING**")
funding_body_idx = md.index("Supported by grant 123.")
ack_idx = md.index("**ACKNOWLEDGMENTS**")
ack_body_idx = md.index("We thank the lab members.")
refs_idx = md.index("## REFERENCES")
assert funding_idx < funding_body_idx < ack_idx < ack_body_idx < refs_idx

7
tests/test_ocr_structural_gate.py
View file

@ -94,9 +94,9 @@ def test_document_abstract_span_includes_structured_subheadings_and_excludes_sup
assert span["status"] == "ACCEPT"
assert span["heading_block_id"] == "h"
assert span["body_block_ids"] == ["q", "a1"]
assert span["body_block_ids"] == ["q", "a1", "m", "a2"]
assert span["excluded_support_block_ids"] == ["authors"]
assert span["stop_reason"] == "structured_abstract_head"
assert span["stop_reason"] == "body_start"
def test_abstract_span_stops_before_intro_even_when_later_block_has_abstract_seed() -> None:
@ -223,7 +223,8 @@ def test_abstract_body_without_span_is_not_verified_accept() -> None:
RoleGateContext(abstract_span={"status": "MISSING"}),
)
assert decision.source != "abstract_span_missing_fallback"
assert decision.status == "ACCEPT"
assert decision.role == "abstract_body"
def test_section_heading_without_heading_artifact_is_not_verified_accept() -> None:

282
tests/test_ocr_tables.py
View file

@ -518,3 +518,285 @@ def test_table_inventory_marks_close_scores_ambiguous() -> None:
assert table["match_status"] == "ambiguous"
assert table["has_asset"] is False
assert len(table["candidate_assets"]) == 2
def test_table_note_binds_footnote_role_block_adjacent_below_asset() -> None:
from paperforge.worker.ocr_tables import build_table_inventory
structured_blocks = [
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_a1",
"role": "table_asset",
"text": "table data",
"bbox": [100, 100, 600, 400],
},
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_c1",
"role": "table_caption",
"text": "Table 1. Main results",
"bbox": [100, 420, 600, 460],
},
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_fn1",
"role": "footnote",
"raw_label": "vision_footnote",
"text": "* p < 0.05",
"bbox": [100, 410, 600, 435],
},
]
inventory = build_table_inventory(structured_blocks)
assert inventory["official_table_count"] == 1
t = inventory["tables"][0]
assert t["note_block_ids"] == ["p5_fn1"]
def test_table_note_binds_short_text_block_adjacent_below_asset() -> None:
from paperforge.worker.ocr_tables import build_table_inventory
structured_blocks = [
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_a1",
"role": "table_asset",
"text": "table data",
"bbox": [100, 100, 600, 400],
},
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_c1",
"role": "table_caption",
"text": "Table 1. Main results",
"bbox": [100, 420, 600, 460],
},
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_bn1",
"role": "body_paragraph",
"text": "Data are mean ± SD.",
"bbox": [100, 410, 600, 430],
},
]
inventory = build_table_inventory(structured_blocks)
t = inventory["tables"][0]
assert t["note_block_ids"] == ["p5_bn1"]
def test_table_note_excludes_block_too_far_below_asset() -> None:
from paperforge.worker.ocr_tables import build_table_inventory
structured_blocks = [
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_a1",
"role": "table_asset",
"text": "table data",
"bbox": [100, 100, 600, 400],
},
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_c1",
"role": "table_caption",
"text": "Table 1. Main results",
"bbox": [100, 420, 600, 460],
},
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_far",
"role": "footnote",
"text": "too far below",
"bbox": [100, 520, 600, 540],
},
]
inventory = build_table_inventory(structured_blocks)
t = inventory["tables"][0]
assert t["note_block_ids"] == []
def test_table_note_excludes_block_on_different_page() -> None:
from paperforge.worker.ocr_tables import build_table_inventory
structured_blocks = [
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_a1",
"role": "table_asset",
"text": "table data",
"bbox": [100, 100, 600, 400],
},
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_c1",
"role": "table_caption",
"text": "Table 1. Main results",
"bbox": [100, 420, 600, 460],
},
{
"paper_id": "KEY001",
"page": 6,
"block_id": "p6_fn1",
"role": "footnote",
"text": "* p < 0.05",
"bbox": [100, 110, 600, 130],
},
]
inventory = build_table_inventory(structured_blocks)
t = inventory["tables"][0]
assert t["note_block_ids"] == []
def test_table_note_excludes_section_headings_from_short_text_match() -> None:
from paperforge.worker.ocr_tables import build_table_inventory
structured_blocks = [
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_a1",
"role": "table_asset",
"text": "table data",
"bbox": [100, 100, 600, 400],
},
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_c1",
"role": "table_caption",
"text": "Table 1. Main results",
"bbox": [100, 420, 600, 460],
},
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_sh",
"role": "section_heading",
"text": "Results",
"bbox": [100, 410, 300, 430],
},
]
inventory = build_table_inventory(structured_blocks)
t = inventory["tables"][0]
assert t["note_block_ids"] == []
def test_table_note_binds_vision_footnote_raw_label() -> None:
from paperforge.worker.ocr_tables import build_table_inventory
structured_blocks = [
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_a1",
"role": "table_asset",
"text": "table data",
"bbox": [100, 100, 600, 400],
},
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_c1",
"role": "table_caption",
"text": "Table 1. Main results",
"bbox": [100, 420, 600, 460],
},
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_vf1",
"role": "body_paragraph",
"raw_label": "vision_footnote",
"text": "Abbreviations: CI, confidence interval",
"bbox": [100, 405, 600, 425],
},
]
inventory = build_table_inventory(structured_blocks)
t = inventory["tables"][0]
assert t["note_block_ids"] == ["p5_vf1"]
def test_table_note_binds_multiple_adjacent_notes() -> None:
from paperforge.worker.ocr_tables import build_table_inventory
structured_blocks = [
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_a1",
"role": "table_asset",
"text": "table data",
"bbox": [100, 100, 600, 400],
},
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_c1",
"role": "table_caption",
"text": "Table 1. Main results",
"bbox": [100, 420, 600, 460],
},
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_fn1",
"role": "footnote",
"text": "* p < 0.05",
"bbox": [100, 405, 600, 420],
},
{
"paper_id": "KEY001",
"page": 5,
"block_id": "p5_fn2",
"role": "footnote",
"text": "** p < 0.01",
"bbox": [100, 422, 600, 438],
},
]
inventory = build_table_inventory(structured_blocks)
t = inventory["tables"][0]
assert set(t["note_block_ids"]) == {"p5_fn1", "p5_fn2"}
def test_table_without_matched_asset_has_empty_note_block_ids() -> None:
from paperforge.worker.ocr_tables import build_table_inventory
structured_blocks = [
{
"paper_id": "KEY001",
"page": 7,
"block_id": "p7_c1",
"role": "table_caption",
"text": "Table 2. No asset here",
"bbox": [50, 50, 500, 80],
},
]
inventory = build_table_inventory(structured_blocks)
assert inventory["tables"][0]["note_block_ids"] == []