diff --git a/tests/test_ocr_metadata.py b/tests/test_ocr_metadata.py index a4266ed4..e442f78c 100644 --- a/tests/test_ocr_metadata.py +++ b/tests/test_ocr_metadata.py @@ -1,5 +1,7 @@ from __future__ import annotations +from pathlib import Path + def test_resolved_metadata_prefers_zotero_but_preserves_ocr_candidates() -> None: from paperforge.worker.ocr_metadata import resolve_metadata @@ -68,3 +70,21 @@ def test_resolved_metadata_preserves_alternatives() -> None: alt["source"] == "ocr_frontmatter" for alt in resolved["title"]["alternatives"] ) + + +def test_legacy_frontmatter_recovers_title_from_first_page(tmp_path: Path) -> None: + from paperforge.worker.ocr_metadata import extract_frontmatter_candidates + import json + + blocks_path = tmp_path / "blocks.structured.jsonl" + blocks_path.write_text( + json.dumps({"role": "paper_title", "text": "Test Paper Title"}) + "\n" + + json.dumps({"role": "authors", "text": "Author A, Author B"}) + "\n" + + json.dumps({"role": "doi", "text": "10.1000/xyz"}) + "\n", + encoding="utf-8", + ) + + candidates = extract_frontmatter_candidates(blocks_path) + assert candidates["title"] == "Test Paper Title" + assert candidates["authors_text"] == "Author A, Author B" + assert "10.1000/xyz" in candidates.get("doi_candidates", []) diff --git a/tests/test_ocr_objects.py b/tests/test_ocr_objects.py index 526f36e6..365e89b1 100644 --- a/tests/test_ocr_objects.py +++ b/tests/test_ocr_objects.py @@ -47,3 +47,17 @@ def test_orphan_object_markdown() -> None: assert "# Orphan Media" in md assert "![[../assets/orphans/orphan_001.jpg]]" in md + + +def test_stabilize_object_wikilink_uses_correct_relative_path() -> None: + from paperforge.worker.ocr_objects import render_figure_object_markdown + + md = render_figure_object_markdown({ + "figure_id": "figure_001", + "page": 5, + "caption": "Figure 1. Results.", + "image_relpath": "assets/figures/figure_001.jpg", + "confidence": 0.9, + }) + + assert "![[../../assets/figures/figure_001.jpg]]" in md diff --git a/tests/test_ocr_render_stabilization.py b/tests/test_ocr_render_stabilization.py new file mode 100644 index 00000000..33965cc1 --- /dev/null +++ b/tests/test_ocr_render_stabilization.py @@ -0,0 +1,97 @@ +from __future__ import annotations + + +def test_stabilize_render_suppresses_frontmatter_noise() -> None: + from paperforge.worker.ocr_render import render_fulltext_markdown + + structured_blocks = [ + {"paper_id": "KEY001", "page": 1, "block_id": "b1", "role": "section_heading", "text": "OPEN ACCESS", "render_default": True}, + {"paper_id": "KEY001", "page": 1, "block_id": "b2", "role": "section_heading", "text": "CITATION", "render_default": True}, + {"paper_id": "KEY001", "page": 1, "block_id": "b3", "role": "body_paragraph", "text": "Real body text should render.", "render_default": True}, + ] + + md = render_fulltext_markdown( + structured_blocks=structured_blocks, + resolved_metadata={}, + figure_inventory={}, + table_inventory={}, + ) + + assert "OPEN ACCESS" not in md + assert "CITATION" not in md + assert "Real body text should render." in md + + +def test_stabilize_render_output_starts_with_metadata_and_abstract() -> None: + from paperforge.worker.ocr_render import render_fulltext_markdown + + resolved_metadata = { + "title": {"value": "Test Paper Title"}, + "authors": {"value": ["Author A", "Author B"]}, + "journal": {"value": "Test Journal"}, + "year": {"value": 2025}, + "doi": {"value": "10.1000/xyz"}, + } + structured_blocks = [ + {"paper_id": "KEY001", "page": 1, "block_id": "b1", "role": "abstract_heading", "text": "Abstract", "render_default": True}, + {"paper_id": "KEY001", "page": 1, "block_id": "b2", "role": "abstract_body", "text": "This is the abstract text.", "render_default": True}, + {"paper_id": "KEY001", "page": 2, "block_id": "b3", "role": "section_heading", "text": "1 Introduction", "render_default": True}, + ] + + md = render_fulltext_markdown( + structured_blocks=structured_blocks, + resolved_metadata=resolved_metadata, + figure_inventory={}, + table_inventory={}, + ) + + lines = md.strip().split("\n") + assert lines[0].startswith("# Test Paper Title") + title_idx = next(i for i, l in enumerate(lines) if l.startswith("# Test")) + abstract_idx = next(i for i, l in enumerate(lines) if "Abstract" in l and l.startswith("##")) + intro_idx = next(i for i, l in enumerate(lines) if "Introduction" in l) + assert title_idx < abstract_idx < intro_idx + + +def test_stabilize_figure_not_appended_at_end() -> None: + from paperforge.worker.ocr_render import render_fulltext_markdown + + structured_blocks = [ + {"paper_id": "KEY001", "page": 1, "block_id": "b1", "role": "body_paragraph", "text": "Body text.", "render_default": True}, + {"paper_id": "KEY001", "page": 2, "block_id": "b2", "role": "figure_caption", "text": "Figure 1. Results.", "render_default": True}, + {"paper_id": "KEY001", "page": 3, "block_id": "b3", "role": "section_heading", "text": "2 Discussion", "render_default": True}, + ] + figure_inventory = { + "matched_figures": [{"figure_id": "fig_001", "page": 2}], + } + + md = render_fulltext_markdown( + structured_blocks=structured_blocks, + resolved_metadata={}, + figure_inventory=figure_inventory, + table_inventory={}, + ) + + discussion_idx = md.index("2 Discussion") + fig_link_idx = md.index("![[render/figures/fig_001.md]]") + assert fig_link_idx < discussion_idx + + +def test_stabilize_latex_normalization() -> None: + from paperforge.worker.ocr_render import render_fulltext_markdown + + structured_blocks = [ + {"paper_id": "KEY001", "page": 1, "block_id": "b1", "role": "body_paragraph", + "text": "Expression $ ^{1} $ and $ ^{\\u2020} $ should be compact.", "render_default": True}, + ] + + md = render_fulltext_markdown( + structured_blocks=structured_blocks, + resolved_metadata={}, + figure_inventory={}, + table_inventory={}, + ) + + assert "$^{1}$" in md + assert "$ ^{1} $" not in md + assert "$^{\\u2020}$" in md or "$^{†}$" in md diff --git a/tests/test_ocr_rendering.py b/tests/test_ocr_rendering.py index 86ebd64f..77858ba1 100644 --- a/tests/test_ocr_rendering.py +++ b/tests/test_ocr_rendering.py @@ -839,3 +839,18 @@ def test_structured_renderer_respects_render_default_false() -> None: assert "ALSO_VISIBLE" in output assert "HIDDEN_BODY" not in output assert "Hidden Heading" not in output + + +def test_stabilize_renderer_output_starts_with_title() -> None: + from paperforge.worker.ocr_render import render_fulltext_markdown + + md = render_fulltext_markdown( + structured_blocks=[ + {"paper_id": "KEY001", "page": 1, "block_id": "b1", "role": "body_paragraph", "text": "Body.", "render_default": True}, + ], + resolved_metadata={"title": {"value": "The Paper Title"}}, + figure_inventory={}, + table_inventory={}, + ) + + assert md.startswith("# The Paper Title") diff --git a/tests/test_ocr_roles.py b/tests/test_ocr_roles.py index 083ee968..40999d32 100644 --- a/tests/test_ocr_roles.py +++ b/tests/test_ocr_roles.py @@ -117,3 +117,36 @@ def test_unknown_structural_for_unrecognized() -> None: assert role.role in {"body_paragraph", "unknown_structural"} assert role.confidence < 0.8 + + +def test_stabilize_role_classifies_paper_title() -> None: + from paperforge.worker.ocr_roles import assign_block_role + + assignment = assign_block_role( + block={"block_label": "paragraph_title", "block_content": "Test Paper About Science"}, + page_blocks=[], + ) + + assert assignment.role == "paper_title" + + +def test_stabilize_abstract_heading_is_not_body() -> None: + from paperforge.worker.ocr_roles import assign_block_role + + assignment = assign_block_role( + block={"block_label": "paragraph_title", "block_content": "Abstract"}, + page_blocks=[], + ) + + assert assignment.role == "abstract_heading" + + +def test_stabilize_frontmatter_noise_not_section_heading() -> None: + from paperforge.worker.ocr_roles import assign_block_role + + for noise in ("OPEN ACCESS", "CITATION", "COPYRIGHT", "KEYWORDS", "EDITED BY", "REVIEWED BY"): + assignment = assign_block_role( + block={"block_label": "paragraph_title", "block_content": noise}, + page_blocks=[], + ) + assert assignment.role not in ("section_heading", "subsection_heading"), f"{noise} should not be a heading"