fix: token cap recursive split + policy version check + test plan update

- _split_if_oversized: recursive _halve_text until every part <= 1000 tokens
- _incremental_units_only: also checks retrieval_policy_version match
- _rebuild_paper_units: use RETRIEVAL_POLICY_VERSION constant
- Functional test plan: Layer A (artifact) + Layer B (API), all 8 review
  fixes incorporated (persisted artifacts, no jump-assertion, stronger
  object/backmatter/abstract tests, proper embed flow)

paperforge/memory/builder.py

@@ -19,7 +19,7 @@ from paperforge.memory.schema import (
     ensure_schema,
     get_schema_version,
 )
-from paperforge.retrieval.manifest import build_paper_manifest
+from paperforge.retrieval.manifest import RETRIEVAL_POLICY_VERSION, build_paper_manifest
 from paperforge.retrieval.units import build_body_units, build_object_units
 from paperforge.worker.asset_index import read_index
 from paperforge.worker.asset_state import (
@@ -489,7 +489,8 @@ def _incremental_units_only(conn: sqlite3.Connection, items: list[dict], ocr_roo
         ).fetchone()
         if row:
             stored = json.loads(row[0])
-            if stored.get("ocr_result_hash") == current_hash:
+            if (stored.get("ocr_result_hash") == current_hash
+                and stored.get("retrieval_policy_version") == RETRIEVAL_POLICY_VERSION):
                 continue
         _rebuild_paper_units(conn, key, paper_dir, tree_path, blocks_path)
         built_count += 1
@@ -520,7 +521,7 @@ def _rebuild_paper_units(conn: sqlite3.Connection, key: str, paper_dir: Path,
         paper_id=key,
         ocr_result_hash=current_hash,
         structure_tree_bytes=tree_path.read_bytes(),
-        retrieval_policy_version="l4.body.v1",
+        retrieval_policy_version=RETRIEVAL_POLICY_VERSION,
         body_units=body_units,
         object_units=object_units,
         source_paths={

paperforge/retrieval/units.py

@@ -24,9 +24,12 @@ def _body_unit_role_kind(role: str) -> str | None:
 
 
 def _split_if_oversized(text: str, max_tokens: int = 1000) -> list[str]:
-    """Split text by paragraph if token estimate exceeds max_tokens."""
+    """Split text by paragraph if token estimate exceeds max_tokens.
+    Recursively splits oversized paragraphs until each part fits."""
     if len(text) // 4 <= max_tokens:
         return [text]
+
+    # Try paragraph boundaries first
     paragraphs = text.split("\n\n")
     parts: list[str] = []
     current: list[str] = []
@@ -42,15 +45,33 @@ def _split_if_oversized(text: str, max_tokens: int = 1000) -> list[str]:
             current_tokens += para_tokens
     if current:
         parts.append("\n\n".join(current))
-    # Fallback: if a single oversized paragraph wasn't split, halve it
-    if len(parts) == 1 and len(parts[0]) // 4 > max_tokens:
-        mid = len(parts[0]) // 2
-        break_at = parts[0].rfind(". ", 0, mid)
-        if break_at < mid // 2:
-            break_at = parts[0].rfind(" ", 0, mid)
-        if break_at > 0:
-            return [parts[0][:break_at + 1].strip(), parts[0][break_at + 1:].strip()]
-    return parts if parts else [text]
+
+    # Recursively split any part that's still oversized
+    result: list[str] = []
+    for part in parts:
+        if len(part) // 4 > max_tokens:
+            result.extend(_halve_text(part, max_tokens))
+        else:
+            result.append(part)
+    return result if result else [text]
+
+
+def _halve_text(text: str, max_tokens: int) -> list[str]:
+    """Recursively split oversized text by sentence/word boundary."""
+    if len(text) // 4 <= max_tokens:
+        return [text]
+    mid = len(text) // 2
+    break_at = text.rfind(". ", 0, mid)
+    if break_at < mid // 2:
+        break_at = text.rfind(" ", 0, mid)
+    if break_at <= 0:
+        break_at = mid
+    left = text[:break_at + 1].strip()
+    right = text[break_at + 1:].strip()
+    if not left or not right:
+        # Can't split further, return as-is even if oversized
+        return [text]
+    return _halve_text(left, max_tokens) + _halve_text(right, max_tokens)
 
 
 def build_unit_id(