fix: final hardening — policy revision + Layer A all green

- _body_unit_role_kind now excludes only reference_item/reference_heading
  (per policy: everything in fulltext except ref zone)
- backmatter_body unit_kind removed; all units are 'body'
- bounds_map in structure_tree uses page-qualified key (fixes collision
  when same block_id appears on different pages)
- Layer A test plan: emitted_ids uses page-qualified keys + handles
  block_id=0 (falsy value)
- Layer A: 1609 passed, 0 failed on real vault (22 papers)
- Unit tests: 152 passed, 1 pre-existing skip

%TEMP%test_import.py

@@ -0,0 +1 @@
+import paperforge

docs/plans/memory-layer-functional-test-plan.md

@@ -113,7 +113,7 @@ for key in KEYS:
         tree = read_json(OCR_ROOT / key / "index" / "structure-tree.json")
         nodes = all_nodes(tree)
         render_map = read_json(OCR_ROOT / key / "render" / "render-map.json")
-        emitted_ids = {str(e["block_id"]) for e in render_map["emitted_blocks"] if e.get("block_id")}
+        emitted_ids = {f"p{e["page"]}:{str(e["block_id"])}" for e in render_map["emitted_blocks"] if e.get("page") is not None and e.get("block_id") is not None}
 
         check(f"{key}: tree has nodes", len(nodes) > 0)
 

paperforge/retrieval/structure_tree.py

@@ -65,14 +65,14 @@ def build_structure_tree(
             if h2["markdown_level"] <= h["markdown_level"]:
                 next_sibling = h2["emitted_order"]
                 break
-        bounds_map[h["block_id"]] = {
+        bounds_map[f"p{h['page']}:{h['block_id']}"] = {
             "start": h["emitted_order"],
             "end": next_sibling or float("inf"),
         }
 
     # ── interval assignment (recursive) ──
     def _assign_intervals(node: dict[str, Any]) -> None:
-        bounds = bounds_map.get(node["block_id"])
+        bounds = bounds_map.get(f"p{node['page']}:{node['block_id']}")
         if bounds is None:
             return
 

paperforge/retrieval/units.py

@@ -15,12 +15,12 @@ from typing import Any
 # ── Role helper ──
 
 def _body_unit_role_kind(role: str) -> str | None:
-    """Map a block's role to body unit kind, or None if excluded."""
-    if role == "body_paragraph":
-        return "body"
-    if role in {"structured_insert", "non_body_insert", "backmatter_body"}:
-        return "backmatter_body"
-    return None  # reference_item, reference_heading, heading, caption, asset, etc.
+    """Map a block's role to body unit kind, or None if excluded.
+    Only reference-zone blocks are excluded; everything that appears
+    in the rendered fulltext is fair game."""
+    if role in {"reference_item", "reference_heading"}:
+        return None
+    return "body"
 
 
 def _split_if_oversized(text: str, max_tokens: int = 1000) -> list[str]:

paperforge/worker/ocr_versions.py

@@ -9,7 +9,7 @@ EXPECTED_CANONICAL_BLOCK_VERSION = "1.0.0"
 EXPECTED_STRUCTURE_VERSION = "1.0.0"
 EXPECTED_METADATA_RESOLVER_VERSION = "1.0.0"
 EXPECTED_ASSET_EXTRACTOR_VERSION = "1.0.0"
-EXPECTED_RENDERER_VERSION = "2.0.0"
+EXPECTED_RENDERER_VERSION = "2.1.0"
 EXPECTED_DOCTOR_VERSION = "1.0.0"
 
 

tests/test_layer4_units_and_fts.py

@@ -139,7 +139,7 @@ def test_backmatter_body_creates_separate_unit():
     ]
     units = build_body_units(tree=tree, structured_blocks=blocks)
     assert len(units) == 1
-    assert units[0]["unit_kind"] == "backmatter_body"
+    assert units[0]["unit_kind"] == "body"
 
 
 def test_mixed_body_and_backmatter_split():
@@ -155,10 +155,8 @@ def test_mixed_body_and_backmatter_split():
     ]
     units = build_body_units(tree=tree, structured_blocks=blocks)
     kinds = {u["unit_kind"] for u in units}
-    assert "body" in kinds
-    assert "backmatter_body" in kinds
-    assert len(units) == 2
-    assert len({u["unit_id"] for u in units}) == len(units)
+    assert kinds == {"body"}
+    assert len(units) == 1  # merged since same kind now
 
 
 def test_token_cap_splits_into_parts():