diff --git a/paperforge/worker/ocr.py b/paperforge/worker/ocr.py index 44d0e9eb..76d064ff 100644 --- a/paperforge/worker/ocr.py +++ b/paperforge/worker/ocr.py @@ -1847,6 +1847,10 @@ def postprocess_ocr_result(vault: Path, key: str, all_results: list[dict]) -> tu meta["raw_upgradable"] = state["raw_upgradable"] meta["derived_stale"] = state["derived_stale"] meta["version_state_updated_at"] = __import__("datetime").datetime.now().isoformat() + try: + meta["legacy_images_path"] = str(images_dir.relative_to(vault)).replace("\\", "/") + except ValueError: + meta["legacy_images_path"] = str(images_dir) write_json(meta_path, meta) fulltext_path = ocr_root / "fulltext.md" diff --git a/paperforge/worker/ocr_health.py b/paperforge/worker/ocr_health.py index 16678689..8c6f6aed 100644 --- a/paperforge/worker/ocr_health.py +++ b/paperforge/worker/ocr_health.py @@ -13,8 +13,14 @@ def build_ocr_health( table_inventory: dict, ) -> dict[str, Any]: section_heading_count = sum(1 for b in structured_blocks if b.get("role") == "section_heading") - abstract_found = any(b.get("role") == "abstract_body" for b in structured_blocks) - references_found = any(b.get("role") == "reference_item" for b in structured_blocks) + abstract_found = any( + b.get("role") in ("abstract_heading", "abstract_body") or b.get("raw_label") == "abstract" + for b in structured_blocks + ) + references_found = any( + b.get("role") in ("reference_heading", "reference_item") or b.get("raw_label") == "reference_content" + for b in structured_blocks + ) figure_caption_count = sum(1 for b in structured_blocks if b.get("role") == "figure_caption") table_caption_count = sum(1 for b in structured_blocks if b.get("role") == "table_caption") diff --git a/paperforge/worker/ocr_objects.py b/paperforge/worker/ocr_objects.py index 8ea422d2..b330d4b8 100644 --- a/paperforge/worker/ocr_objects.py +++ b/paperforge/worker/ocr_objects.py @@ -20,7 +20,7 @@ def render_figure_object_markdown(figure: dict[str, Any]) -> str: parts = [f"# {label}", "", f"![](../../{image_relpath})", ""] if caption: - parts.append("**Legend:**") + parts.append("## Legend") parts.append(caption) if figure.get("page"): parts.append("") @@ -42,7 +42,7 @@ def render_table_object_markdown(table: dict[str, Any]) -> str: parts = [f"# {label}", "", f"![](../../{image_relpath})", ""] if caption: - parts.append("**Legend:**") + parts.append("## Caption") parts.append(caption) if table.get("page"): parts.append("") diff --git a/paperforge/worker/ocr_render.py b/paperforge/worker/ocr_render.py index ea981ab7..307170f6 100644 --- a/paperforge/worker/ocr_render.py +++ b/paperforge/worker/ocr_render.py @@ -14,6 +14,18 @@ def _normalize_latex(text: str) -> str: return text +def _is_bogus_heading(text: str) -> bool: + t = text.strip() + if len(t) > 100: + return True + if t.count(". ") > 1: + return True + if any(v in t.lower().split() for v in ["is", "are", "was", "were", "have", "has", "been"]): + if len(t) > 50: + return True + return False + + def render_fulltext_markdown( *, structured_blocks: list[dict], @@ -121,8 +133,13 @@ def render_fulltext_markdown( if role == "section_heading": if text.strip().lower() in FRONTMATTER_NOISE: continue - lines.append(f"## {text}") - lines.append("") + if _is_bogus_heading(text): + if text: + lines.append(text) + lines.append("") + else: + lines.append(f"## {text}") + lines.append("") elif role == "subsection_heading": lines.append(f"### {text}") lines.append("") diff --git a/tests/test_ocr_objects.py b/tests/test_ocr_objects.py index e7c28d22..887e2f89 100644 --- a/tests/test_ocr_objects.py +++ b/tests/test_ocr_objects.py @@ -15,7 +15,7 @@ def test_figure_object_markdown_links_image_and_legend() -> None: assert "# Figure 1" in md assert "![](../../assets/figures/figure_001.jpg)" in md - assert "**Legend:**" in md + assert "## Legend" in md assert "Figure 1. Example." in md @@ -32,6 +32,7 @@ def test_table_object_markdown_includes_image_and_caption() -> None: assert "# Table 1" in md assert "![](../../assets/tables/table_001.jpg)" in md + assert "## Caption" in md assert "Table 1. Results." in md diff --git a/tests/test_ocr_render_stabilization.py b/tests/test_ocr_render_stabilization.py index 4e0c996f..640371fd 100644 --- a/tests/test_ocr_render_stabilization.py +++ b/tests/test_ocr_render_stabilization.py @@ -140,6 +140,114 @@ def test_stabilize_author_recovery_from_ocr() -> None: tmppath.unlink() +def test_stabilize_heading_sanity_downgrades_long_heading() -> None: + from paperforge.worker.ocr_render import render_fulltext_markdown + + # A very long section_heading (>100 chars) should be downgraded to body text + long_text = "This is a very long heading that exceeds one hundred characters and should definitely be downgraded to a body paragraph instead of being rendered as a heading" + structured_blocks = [ + {"paper_id": "KEY001", "page": 1, "block_id": "b1", "role": "section_heading", + "text": long_text, "render_default": True}, + {"paper_id": "KEY001", "page": 1, "block_id": "b2", "role": "body_paragraph", + "text": "Normal body text.", "render_default": True}, + ] + + md = render_fulltext_markdown( + structured_blocks=structured_blocks, + resolved_metadata={}, + figure_inventory={}, + table_inventory={}, + ) + + # The long heading should NOT appear as a markdown heading + assert "## " + long_text not in md + # But the text itself should still be present (downgraded) + assert long_text in md + + +def test_stabilize_heading_sanity_downgrades_multi_sentence_heading() -> None: + from paperforge.worker.ocr_render import render_fulltext_markdown + + # A heading with multiple sentence-ending periods should be downgraded + structured_blocks = [ + {"paper_id": "KEY001", "page": 1, "block_id": "b1", "role": "section_heading", + "text": "This is a heading. It has multiple sentences. This is the third one.", + "render_default": True}, + {"paper_id": "KEY001", "page": 1, "block_id": "b2", "role": "body_paragraph", + "text": "Normal body text.", "render_default": True}, + ] + + md = render_fulltext_markdown( + structured_blocks=structured_blocks, + resolved_metadata={}, + figure_inventory={}, + table_inventory={}, + ) + + assert "## This is a heading" not in md + assert "This is a heading. It has multiple sentences." in md + + +def test_stabilize_heading_sanity_allows_valid_short_heading() -> None: + from paperforge.worker.ocr_render import render_fulltext_markdown + + structured_blocks = [ + {"paper_id": "KEY001", "page": 1, "block_id": "b1", "role": "section_heading", + "text": "1 Introduction", "render_default": True}, + ] + + md = render_fulltext_markdown( + structured_blocks=structured_blocks, + resolved_metadata={}, + figure_inventory={}, + table_inventory={}, + ) + + assert "## 1 Introduction" in md + + +def test_stabilize_heading_sanity_downgrades_verb_heavy_heading() -> None: + from paperforge.worker.ocr_render import render_fulltext_markdown + + # A heading longer than 50 chars with common sentence verbs should be downgraded + structured_blocks = [ + {"paper_id": "KEY001", "page": 1, "block_id": "b1", "role": "section_heading", + "text": "This is a method that was used for the experiment and has many words", + "render_default": True}, + {"paper_id": "KEY001", "page": 1, "block_id": "b2", "role": "body_paragraph", + "text": "Normal body text.", "render_default": True}, + ] + + md = render_fulltext_markdown( + structured_blocks=structured_blocks, + resolved_metadata={}, + figure_inventory={}, + table_inventory={}, + ) + + assert "## This is a method" not in md + assert "This is a method that was used" in md + + +def test_stabilize_heading_sanity_allows_verb_short_heading() -> None: + from paperforge.worker.ocr_render import render_fulltext_markdown + + # A short heading with verbs is OK (under 50 chars) + structured_blocks = [ + {"paper_id": "KEY001", "page": 1, "block_id": "b1", "role": "section_heading", + "text": "Results are shown", "render_default": True}, + ] + + md = render_fulltext_markdown( + structured_blocks=structured_blocks, + resolved_metadata={}, + figure_inventory={}, + table_inventory={}, + ) + + assert "## Results are shown" in md + + def test_stabilize_reference_content_mapped() -> None: from paperforge.worker.ocr_roles import assign_block_role