fix: stabilize object notes, render heading sanity, health/index detection

paperforge/worker/ocr.py

@@ -1847,6 +1847,10 @@ def postprocess_ocr_result(vault: Path, key: str, all_results: list[dict]) -> tu
     meta["raw_upgradable"] = state["raw_upgradable"]
     meta["derived_stale"] = state["derived_stale"]
     meta["version_state_updated_at"] = __import__("datetime").datetime.now().isoformat()
+    try:
+        meta["legacy_images_path"] = str(images_dir.relative_to(vault)).replace("\\", "/")
+    except ValueError:
+        meta["legacy_images_path"] = str(images_dir)
     write_json(meta_path, meta)
 
     fulltext_path = ocr_root / "fulltext.md"

paperforge/worker/ocr_health.py

@@ -13,8 +13,14 @@ def build_ocr_health(
     table_inventory: dict,
 ) -> dict[str, Any]:
     section_heading_count = sum(1 for b in structured_blocks if b.get("role") == "section_heading")
-    abstract_found = any(b.get("role") == "abstract_body" for b in structured_blocks)
-    references_found = any(b.get("role") == "reference_item" for b in structured_blocks)
+    abstract_found = any(
+        b.get("role") in ("abstract_heading", "abstract_body") or b.get("raw_label") == "abstract"
+        for b in structured_blocks
+    )
+    references_found = any(
+        b.get("role") in ("reference_heading", "reference_item") or b.get("raw_label") == "reference_content"
+        for b in structured_blocks
+    )
     figure_caption_count = sum(1 for b in structured_blocks if b.get("role") == "figure_caption")
     table_caption_count = sum(1 for b in structured_blocks if b.get("role") == "table_caption")
 

paperforge/worker/ocr_objects.py

@@ -20,7 +20,7 @@ def render_figure_object_markdown(figure: dict[str, Any]) -> str:
 
     parts = [f"# {label}", "", f"![](../../{image_relpath})", ""]
     if caption:
-        parts.append("**Legend:**")
+        parts.append("## Legend")
         parts.append(caption)
     if figure.get("page"):
         parts.append("")
@@ -42,7 +42,7 @@ def render_table_object_markdown(table: dict[str, Any]) -> str:
 
     parts = [f"# {label}", "", f"![](../../{image_relpath})", ""]
     if caption:
-        parts.append("**Legend:**")
+        parts.append("## Caption")
         parts.append(caption)
     if table.get("page"):
         parts.append("")

paperforge/worker/ocr_render.py

@@ -14,6 +14,18 @@ def _normalize_latex(text: str) -> str:
     return text
 
 
+def _is_bogus_heading(text: str) -> bool:
+    t = text.strip()
+    if len(t) > 100:
+        return True
+    if t.count(". ") > 1:
+        return True
+    if any(v in t.lower().split() for v in ["is", "are", "was", "were", "have", "has", "been"]):
+        if len(t) > 50:
+            return True
+    return False
+
+
 def render_fulltext_markdown(
     *,
     structured_blocks: list[dict],
@@ -121,8 +133,13 @@ def render_fulltext_markdown(
         if role == "section_heading":
             if text.strip().lower() in FRONTMATTER_NOISE:
                 continue
-            lines.append(f"## {text}")
-            lines.append("")
+            if _is_bogus_heading(text):
+                if text:
+                    lines.append(text)
+                    lines.append("")
+            else:
+                lines.append(f"## {text}")
+                lines.append("")
         elif role == "subsection_heading":
             lines.append(f"### {text}")
             lines.append("")

tests/test_ocr_objects.py

@@ -15,7 +15,7 @@ def test_figure_object_markdown_links_image_and_legend() -> None:
 
     assert "# Figure 1" in md
     assert "![](../../assets/figures/figure_001.jpg)" in md
-    assert "**Legend:**" in md
+    assert "## Legend" in md
     assert "Figure 1. Example." in md
 
 
@@ -32,6 +32,7 @@ def test_table_object_markdown_includes_image_and_caption() -> None:
 
     assert "# Table 1" in md
     assert "![](../../assets/tables/table_001.jpg)" in md
+    assert "## Caption" in md
     assert "Table 1. Results." in md
 
 

tests/test_ocr_render_stabilization.py

@@ -140,6 +140,114 @@ def test_stabilize_author_recovery_from_ocr() -> None:
         tmppath.unlink()
 
 
+def test_stabilize_heading_sanity_downgrades_long_heading() -> None:
+    from paperforge.worker.ocr_render import render_fulltext_markdown
+
+    # A very long section_heading (>100 chars) should be downgraded to body text
+    long_text = "This is a very long heading that exceeds one hundred characters and should definitely be downgraded to a body paragraph instead of being rendered as a heading"
+    structured_blocks = [
+        {"paper_id": "KEY001", "page": 1, "block_id": "b1", "role": "section_heading",
+         "text": long_text, "render_default": True},
+        {"paper_id": "KEY001", "page": 1, "block_id": "b2", "role": "body_paragraph",
+         "text": "Normal body text.", "render_default": True},
+    ]
+
+    md = render_fulltext_markdown(
+        structured_blocks=structured_blocks,
+        resolved_metadata={},
+        figure_inventory={},
+        table_inventory={},
+    )
+
+    # The long heading should NOT appear as a markdown heading
+    assert "## " + long_text not in md
+    # But the text itself should still be present (downgraded)
+    assert long_text in md
+
+
+def test_stabilize_heading_sanity_downgrades_multi_sentence_heading() -> None:
+    from paperforge.worker.ocr_render import render_fulltext_markdown
+
+    # A heading with multiple sentence-ending periods should be downgraded
+    structured_blocks = [
+        {"paper_id": "KEY001", "page": 1, "block_id": "b1", "role": "section_heading",
+         "text": "This is a heading. It has multiple sentences. This is the third one.",
+         "render_default": True},
+        {"paper_id": "KEY001", "page": 1, "block_id": "b2", "role": "body_paragraph",
+         "text": "Normal body text.", "render_default": True},
+    ]
+
+    md = render_fulltext_markdown(
+        structured_blocks=structured_blocks,
+        resolved_metadata={},
+        figure_inventory={},
+        table_inventory={},
+    )
+
+    assert "## This is a heading" not in md
+    assert "This is a heading. It has multiple sentences." in md
+
+
+def test_stabilize_heading_sanity_allows_valid_short_heading() -> None:
+    from paperforge.worker.ocr_render import render_fulltext_markdown
+
+    structured_blocks = [
+        {"paper_id": "KEY001", "page": 1, "block_id": "b1", "role": "section_heading",
+         "text": "1 Introduction", "render_default": True},
+    ]
+
+    md = render_fulltext_markdown(
+        structured_blocks=structured_blocks,
+        resolved_metadata={},
+        figure_inventory={},
+        table_inventory={},
+    )
+
+    assert "## 1 Introduction" in md
+
+
+def test_stabilize_heading_sanity_downgrades_verb_heavy_heading() -> None:
+    from paperforge.worker.ocr_render import render_fulltext_markdown
+
+    # A heading longer than 50 chars with common sentence verbs should be downgraded
+    structured_blocks = [
+        {"paper_id": "KEY001", "page": 1, "block_id": "b1", "role": "section_heading",
+         "text": "This is a method that was used for the experiment and has many words",
+         "render_default": True},
+        {"paper_id": "KEY001", "page": 1, "block_id": "b2", "role": "body_paragraph",
+         "text": "Normal body text.", "render_default": True},
+    ]
+
+    md = render_fulltext_markdown(
+        structured_blocks=structured_blocks,
+        resolved_metadata={},
+        figure_inventory={},
+        table_inventory={},
+    )
+
+    assert "## This is a method" not in md
+    assert "This is a method that was used" in md
+
+
+def test_stabilize_heading_sanity_allows_verb_short_heading() -> None:
+    from paperforge.worker.ocr_render import render_fulltext_markdown
+
+    # A short heading with verbs is OK (under 50 chars)
+    structured_blocks = [
+        {"paper_id": "KEY001", "page": 1, "block_id": "b1", "role": "section_heading",
+         "text": "Results are shown", "render_default": True},
+    ]
+
+    md = render_fulltext_markdown(
+        structured_blocks=structured_blocks,
+        resolved_metadata={},
+        figure_inventory={},
+        table_inventory={},
+    )
+
+    assert "## Results are shown" in md
+
+
 def test_stabilize_reference_content_mapped() -> None:
     from paperforge.worker.ocr_roles import assign_block_role