From 1e6edad7baa057188082d1c701fddbf825748d03 Mon Sep 17 00:00:00 2001 From: LLLin000 <809867916@qq.com> Date: Sun, 5 Jul 2026 14:39:19 +0800 Subject: [PATCH] feat: route Layer 4 gateway through body units and structure tree --- paperforge/commands/paper_navigation.py | 62 +-- paperforge/commands/retrieve.py | 10 +- paperforge/commands/scoped_fetch.py | 101 +---- paperforge/commands/search.py | 4 + paperforge/retrieval/gateway.py | 508 +++++++++++++++++++++++- tests/test_layer4_gateway_commands.py | 451 +++++++++++++++++++-- 6 files changed, 941 insertions(+), 195 deletions(-) diff --git a/paperforge/commands/paper_navigation.py b/paperforge/commands/paper_navigation.py index 1a4c5790..9cc3a90f 100644 --- a/paperforge/commands/paper_navigation.py +++ b/paperforge/commands/paper_navigation.py @@ -1,60 +1,24 @@ +"""paperforge.commands.paper_navigation — ``paperforge paper-navigation`` gateway command.""" + from __future__ import annotations -import argparse import json from paperforge import __version__ as PF_VERSION from paperforge.core.result import PFResult -from paperforge.core.io import read_json -from paperforge.retrieval.structure_tree import summarize_role_index +from paperforge.retrieval import gateway -def _resolve_paper_root(vault_path, query: str): - """Resolve a paper query to its OCR root directory.""" - from paperforge.memory.db import get_connection, get_memory_db_path - from paperforge.memory.query import lookup_paper - from paperforge.worker._utils import pipeline_paths - - db_path = get_memory_db_path(vault_path) - if not db_path or not db_path.exists(): - return None - conn = get_connection(db_path, read_only=True) - try: - entries = lookup_paper(conn, query) - if entries: - ocr_root = pipeline_paths(vault_path)["ocr"] - return ocr_root / entries[0]["zotero_key"] - finally: - conn.close() - return None - - -def run(args: argparse.Namespace) -> int: - paper_root = _resolve_paper_root(args.vault_path, args.query) - if paper_root is None: - data = {"mode": "not_found", "paper_id": args.query, "error": "Paper not found in vault"} - if args.json: - print(PFResult(ok=False, command="paper-navigation", version=PF_VERSION, data=data).to_json()) - else: - print(json.dumps(data, ensure_ascii=False, indent=2)) - return 1 - - tree_path = paper_root / "index" / "structure-tree.json" - if tree_path.exists(): - tree = read_json(tree_path) - payload = {"mode": "structure_tree", "paper_id": tree.get("paper_id", ""), "nodes": tree.get("nodes", [])} - else: - role_index_path = paper_root / "index" / "role-index.json" - if role_index_path.exists(): - role_index = read_json(role_index_path) - summary = summarize_role_index(role_index) - payload = {"mode": "role_index_summary", "paper_id": args.query, "summary": summary} - else: - payload = {"mode": "no_index", "paper_id": args.query} - - result = PFResult(ok=True, command="paper-navigation", version=PF_VERSION, data=payload) +def run(args) -> int: + """Execute ``paper-navigation`` via the Layer 4 gateway.""" + result = gateway.route_gateway( + args.vault_path, + "paper-navigation", + args.query, + json_mode=bool(args.json), + ) if args.json: print(result.to_json()) else: - print(json.dumps(payload, ensure_ascii=False, indent=2)) - return 0 + print(json.dumps(result.data, ensure_ascii=False, indent=2)) + return 0 if result.ok else 1 diff --git a/paperforge/commands/retrieve.py b/paperforge/commands/retrieve.py index 63bcd058..64fb861e 100644 --- a/paperforge/commands/retrieve.py +++ b/paperforge/commands/retrieve.py @@ -126,7 +126,15 @@ def run(args: argparse.Namespace) -> int: finally: conn.close() - data = {"query": query, "chunks": chunks, "count": len(chunks)} + data = { + "query": query, + "chunks": chunks, + "count": len(chunks), + "route_explanation": { + "primary_arm": "vector_retrieve", + "compatibility_mode": False, + }, + } warnings: list[str] = [] next_actions: list[dict] = [] if len(chunks) == 0: diff --git a/paperforge/commands/scoped_fetch.py b/paperforge/commands/scoped_fetch.py index db8620b9..add1467d 100644 --- a/paperforge/commands/scoped_fetch.py +++ b/paperforge/commands/scoped_fetch.py @@ -5,115 +5,18 @@ from __future__ import annotations import json from pathlib import Path -from paperforge.core.io import read_json -from paperforge.core.result import PFResult -from paperforge.memory.db import get_connection, get_memory_db_path from paperforge.retrieval import gateway -from paperforge.retrieval.manifest import build_paper_manifest -from paperforge.retrieval.units import build_body_units, build_object_units -def _find_ocr_dir(vault: Path, paper_id: str) -> Path | None: - """Look up the OCR output directory for a paper by scanning the ocr root. - - Returns None if no OCR output exists. - """ - ocr_root = vault / "System" / "PaperForge" / "ocr" - if not ocr_root.exists(): - return None - for d in ocr_root.iterdir(): - if d.is_dir() and d.name == paper_id: - return d - # also check metadata inside - index_path = d / "index" - tree_path = index_path / "structure-tree.json" - if tree_path.exists(): - try: - tree = read_json(tree_path) - if tree.get("paper_id") == paper_id: - return d - except Exception: - continue - return None - - -def _build_units_for_paper( - vault: Path, - paper_id: str, -) -> dict: - """Build retrieval units and manifest for a paper from its OCR output. - - Returns a dict with keys ``body_units``, ``object_units``, ``manifest`` - (or empty lists / None when OCR data is unavailable). - """ - ocr_dir = _find_ocr_dir(vault, paper_id) - if ocr_dir is None: - return {"body_units": [], "object_units": [], "manifest": None} - - index_root = ocr_dir / "index" - tree_path = index_root / "structure-tree.json" - structured_path = ocr_dir / "structured-blocks.json" - if not tree_path.exists() or not structured_path.exists(): - return {"body_units": [], "object_units": [], "manifest": None} - - tree = read_json(tree_path) - structured_blocks = read_json(structured_path) - role_index_path = index_root / "role-index.json" - role_index = read_json(role_index_path) if role_index_path.exists() else {} - - body_units = build_body_units(tree=tree, structured_blocks=structured_blocks) - object_units = build_object_units( - tree=tree, structured_blocks=structured_blocks, role_index=role_index - ) - - # Try to read result hash - result_hash_path = index_root / "result-hash.txt" - ocr_result_hash = "" - if result_hash_path.exists(): - ocr_result_hash = result_hash_path.read_text(encoding="utf-8").strip() - - manifest = build_paper_manifest( - paper_id=paper_id, - ocr_result_hash=ocr_result_hash, - structure_tree_bytes=tree_path.read_bytes(), - retrieval_policy_version="l4.body.v1", - body_units=body_units, - object_units=object_units, - source_paths={ - "structured_blocks": str(structured_path), - "role_index": str(role_index_path), - "fulltext": str(ocr_dir / "fulltext.md"), - }, - ) - return {"body_units": body_units, "object_units": object_units, "manifest": manifest} - - -def run(args): +def run(args) -> int: """Execute ``scoped-fetch`` via the Layer 4 gateway.""" vault = Path(args.vault_path) result = gateway.route_gateway( vault, "scoped-fetch", args.query, - json_mode=args.json, + json_mode=bool(args.json), limit=getattr(args, "limit", 5), ) - - # If we have a valid route result, enrich it with retrieval units - if result.ok and result.data: - # Extract paper_id from the route plan - plan = result.data.get("route_plan", {}) - paper_id = ( - plan.get("paper_id") - or plan.get("primary_paper_id") - or plan.get("target_id") - or "" - ) - if paper_id: - units = _build_units_for_paper(vault, paper_id) - result.data["body_units"] = units["body_units"] - result.data["object_units"] = units["object_units"] - result.data["manifest"] = units["manifest"] - print(result.to_json() if args.json else result.data) return 0 if result.ok else 1 diff --git a/paperforge/commands/search.py b/paperforge/commands/search.py index d3897d7c..f7e0fb0e 100644 --- a/paperforge/commands/search.py +++ b/paperforge/commands/search.py @@ -59,6 +59,10 @@ def run(args: argparse.Namespace) -> int: "lifecycle": args.lifecycle, "next_step": args.next_step, }, + "route_explanation": { + "primary_arm": "paper_fts", + "compatibility_mode": False, + }, } warnings: list[str] = [] next_actions: list[dict] = [] diff --git a/paperforge/retrieval/gateway.py b/paperforge/retrieval/gateway.py index 191211e7..cf0c494c 100644 --- a/paperforge/retrieval/gateway.py +++ b/paperforge/retrieval/gateway.py @@ -1,17 +1,22 @@ """paperforge.retrieval.gateway — Layer 4 gateway command routing. Provides the core `route_gateway()` function that all gateway commands -route through. Currently delegates to existing capabilities; later tasks -will upgrade routing to use real Layer 4 artifacts. +route through. Each intent is routed to the correct real data source. """ from __future__ import annotations +import json +import logging +import re from pathlib import Path from paperforge import __version__ as PF_VERSION +from paperforge.core.io import read_json from paperforge.core.result import PFResult -from paperforge.query_planning import build_query_plan, enrich_query_plan_with_runtime +from paperforge.memory.db import get_connection, get_memory_db_path + +logger = logging.getLogger(__name__) # Map gateway intent names to build_query_plan intent strings INTENTS: dict[str, str] = { @@ -27,41 +32,510 @@ def route_gateway( intent: str, query: str, *, - json_mode: bool, + json_mode: bool, # noqa: ARG001 limit: int = 5, ) -> PFResult: - """Route a gateway command through the query planning pipeline. + """Route a gateway command to the real Layer 4 data source. Parameters ---------- vault : Path PaperForge vault root. intent : str - Gateway intent name (e.g. ``"paper-lookup"``). + Gateway intent name (``"paper-lookup"``, ``"content-discovery"``, + ``"paper-navigation"``, or ``"scoped-fetch"``). query : str Free-text or structured query. json_mode : bool - Whether the caller expects JSON output. + Whether the caller expects JSON output (passed through to callers). limit : int, optional Maximum result count (default 5). Returns ------- PFResult - Result packet with the route plan and intent metadata. + Result packet with data from the real source of truth for the intent. """ - plan = enrich_query_plan_with_runtime( - build_query_plan(query, INTENTS[intent]), - vault, - ) + if intent == "paper-lookup": + return _run_paper_lookup(vault, query, limit=limit) + if intent == "content-discovery": + if _body_units_fts_exists(vault): + return _run_body_unit_discovery(vault, query, limit=limit) + return _run_compat_content_discovery(vault, query, limit=limit) + if intent == "paper-navigation": + return _run_paper_navigation(vault, query) + if intent == "scoped-fetch": + return _run_scoped_fetch(vault, query) + raise ValueError(f"Unsupported Layer 4 intent: {intent}") + + +# --------------------------------------------------------------------------- +# FTS helpers +# --------------------------------------------------------------------------- + + +def _tokenize_for_fts(q: str) -> str: + """Tokenize a query string into safe FTS5 match terms.""" + tokens = re.findall(r"[\w\u4e00-\u9fff]+", q) + if not tokens: + return q + return " OR ".join(f'"{t}"' for t in tokens) + + +def _body_units_fts_exists(vault: Path) -> bool: + """Check whether the body_units_fts table exists and has rows.""" + db_path = get_memory_db_path(vault) + if not db_path.exists(): + return False + conn = get_connection(db_path, read_only=True) + try: + row = conn.execute("SELECT COUNT(*) AS cnt FROM body_units_fts").fetchone() + return row["cnt"] > 0 + except Exception: + return False + finally: + conn.close() + + +# --------------------------------------------------------------------------- +# Paper root resolution (shared by navigation & scoped-fetch) +# --------------------------------------------------------------------------- + + +def _resolve_paper_root(vault: Path, query: str) -> Path | None: + """Resolve a paper query to its OCR root directory.""" + from paperforge.config import paperforge_paths + from paperforge.memory.query import lookup_paper + + db_path = get_memory_db_path(vault) + if not db_path or not db_path.exists(): + return None + conn = get_connection(db_path, read_only=True) + try: + entries = lookup_paper(conn, query) + if entries: + ocr_root = paperforge_paths(vault)["ocr"] + return ocr_root / entries[0]["zotero_key"] + finally: + conn.close() + return None + + +# --------------------------------------------------------------------------- +# Intent: paper-lookup +# --------------------------------------------------------------------------- + + +def _run_paper_lookup(vault: Path, query: str, *, limit: int = 5) -> PFResult: + """Route paper-lookup intent through ``lookup_paper()``.""" + from paperforge.memory.query import lookup_paper + + db_path = get_memory_db_path(vault) + if not db_path.exists(): + return PFResult( + ok=False, + command="paper-lookup", + version=PF_VERSION, + data={ + "intent": "paper-lookup", + "query": query, + "results": [], + "route_explanation": { + "primary_arm": "lookup_paper", + "error": "database_not_found", + }, + }, + ) + conn = get_connection(db_path, read_only=True) + try: + entries = lookup_paper(conn, query) + limited = entries[:limit] + return PFResult( + ok=True, + command="paper-lookup", + version=PF_VERSION, + data={ + "intent": "paper-lookup", + "query": query, + "results": limited, + "count": len(limited), + "route_explanation": { + "primary_arm": "lookup_paper", + "matched": len(limited) > 0, + "compatibility_mode": False, + }, + }, + ) + except Exception as exc: + logger.exception("paper-lookup failed") + return PFResult( + ok=False, + command="paper-lookup", + version=PF_VERSION, + data={ + "intent": "paper-lookup", + "query": query, + "results": [], + "route_explanation": { + "primary_arm": "lookup_paper", + "error": str(exc), + }, + }, + ) + finally: + conn.close() + + +# --------------------------------------------------------------------------- +# Intent: content-discovery +# --------------------------------------------------------------------------- + + +def _run_body_unit_discovery( + vault: Path, query: str, *, limit: int = 5 +) -> PFResult: + """Search ``body_units_fts`` for content-level discovery.""" + db_path = get_memory_db_path(vault) + if not db_path.exists(): + return PFResult( + ok=False, + command="content-discovery", + version=PF_VERSION, + data={ + "intent": "content-discovery", + "query": query, + "results": [], + "route_explanation": { + "primary_arm": "body_units_fts", + "error": "database_not_found", + }, + }, + ) + conn = get_connection(db_path, read_only=True) + try: + fts_query = _tokenize_for_fts(query) + rows = conn.execute( + """SELECT unit_id, paper_id, section_path, unit_text, rank + FROM body_units_fts + WHERE body_units_fts MATCH ? + ORDER BY rank + LIMIT ?""", + (fts_query, limit), + ).fetchall() + results = [dict(r) for r in rows] + return PFResult( + ok=True, + command="content-discovery", + version=PF_VERSION, + data={ + "intent": "content-discovery", + "query": query, + "results": results, + "count": len(results), + "route_explanation": { + "primary_arm": "body_units_fts", + "fallback_arms": ["vector_retrieve"], + "compatibility_mode": False, + }, + }, + ) + except Exception as exc: + logger.exception("body_units_fts query failed, falling back") + return _run_compat_content_discovery( + vault, query, limit=limit, explanation_note=str(exc) + ) + finally: + conn.close() + + +def _run_compat_content_discovery( + vault: Path, + query: str, + *, + limit: int = 5, + explanation_note: str = "", +) -> PFResult: + """Fallback content discovery using ``paper_fts`` (metadata-only).""" + from paperforge.memory.fts import search_papers + + db_path = get_memory_db_path(vault) + if not db_path.exists(): + return PFResult( + ok=False, + command="content-discovery", + version=PF_VERSION, + data={ + "intent": "content-discovery", + "query": query, + "results": [], + "route_explanation": { + "primary_arm": "paper_fts", + "error": "database_not_found", + }, + }, + ) + conn = get_connection(db_path, read_only=True) + try: + results = search_papers(conn, query, limit=limit) + return PFResult( + ok=True, + command="content-discovery", + version=PF_VERSION, + data={ + "intent": "content-discovery", + "query": query, + "results": results, + "count": len(results), + "route_explanation": { + "primary_arm": "paper_fts", + "compatibility_mode": True, + "note": ( + explanation_note + or "body_units_fts unavailable, using metadata FTS" + ), + }, + }, + ) + except Exception as exc: + return PFResult( + ok=False, + command="content-discovery", + version=PF_VERSION, + data={ + "intent": "content-discovery", + "query": query, + "results": [], + "route_explanation": { + "primary_arm": "paper_fts", + "error": str(exc), + }, + }, + ) + finally: + conn.close() + + +# --------------------------------------------------------------------------- +# Intent: paper-navigation +# --------------------------------------------------------------------------- + + +def _run_paper_navigation(vault: Path, query: str) -> PFResult: + """Read structure-tree.json (or fall back to role-index) for navigation.""" + paper_root = _resolve_paper_root(vault, query) + if paper_root is None: + return PFResult( + ok=False, + command="paper-navigation", + version=PF_VERSION, + data={ + "intent": "paper-navigation", + "query": query, + "mode": "not_found", + "route_explanation": { + "primary_arm": "structure_tree", + "note": "paper not found", + }, + }, + ) + + tree_path = paper_root / "index" / "structure-tree.json" + if tree_path.exists(): + tree = read_json(tree_path) + return PFResult( + ok=True, + command="paper-navigation", + version=PF_VERSION, + data={ + "intent": "paper-navigation", + "query": query, + "mode": "structure_tree", + "paper_id": tree.get("paper_id", ""), + "nodes": tree.get("nodes", []), + "route_explanation": { + "primary_arm": "structure_tree", + "fallback": False, + }, + }, + ) + + # Fallback: role-index summary + role_index_path = paper_root / "index" / "role-index.json" + if role_index_path.exists(): + from paperforge.retrieval.structure_tree import summarize_role_index + + role_index = read_json(role_index_path) + summary = summarize_role_index(role_index) + return PFResult( + ok=True, + command="paper-navigation", + version=PF_VERSION, + data={ + "intent": "paper-navigation", + "query": query, + "mode": "role_index_summary", + "paper_id": query, + "summary": summary, + "route_explanation": { + "primary_arm": "structure_tree", + "fallback_arm": "role_index", + "note": "structure-tree.json not found, using role-index", + }, + }, + ) + return PFResult( - ok=True, - command=intent, + ok=False, + command="paper-navigation", version=PF_VERSION, data={ - "intent": intent, + "intent": "paper-navigation", "query": query, - "route_plan": plan, - "limit": limit, + "mode": "no_index", + "route_explanation": { + "primary_arm": "structure_tree", + "note": "no index found for paper", + }, }, ) + + +# --------------------------------------------------------------------------- +# Intent: scoped-fetch +# --------------------------------------------------------------------------- + + +def _run_scoped_fetch(vault: Path, query: str) -> PFResult: + """Fetch body units scoped by paper, optionally filtered by section or node. + + The *query* is treated as a paper identifier (author, title, DOI, etc.). + An optional ``:section_path`` or ``#node_id`` suffix narrows the results. + """ + from paperforge.memory.query import lookup_paper + + # Parse optional section / node filter from query suffix + base_query: str = query + section_filter: str | None = None + node_filter: str | None = None + if ":" in query and not query.startswith(":"): + parts = query.split(":", 1) + base_query = parts[0] + section_filter = parts[1].strip() + elif "#" in query: + parts = query.split("#", 1) + base_query = parts[0] + node_filter = parts[1].strip() + + db_path = get_memory_db_path(vault) + if not db_path.exists(): + return PFResult( + ok=False, + command="scoped-fetch", + version=PF_VERSION, + data={ + "intent": "scoped-fetch", + "query": query, + "body_units": [], + "route_explanation": { + "primary_arm": "body_units", + "error": "database_not_found", + }, + }, + ) + + conn = get_connection(db_path, read_only=True) + try: + entries = lookup_paper(conn, base_query) + if not entries: + return PFResult( + ok=False, + command="scoped-fetch", + version=PF_VERSION, + data={ + "intent": "scoped-fetch", + "query": query, + "body_units": [], + "route_explanation": { + "primary_arm": "body_units", + "note": "paper not found", + }, + }, + ) + + paper_id = entries[0].get("zotero_key", "") + + if section_filter: + rows = conn.execute( + """SELECT unit_id, paper_id, section_path, unit_text, + page_span_json, block_span_json + FROM body_units + WHERE paper_id = ? AND section_path LIKE ? + ORDER BY unit_id""", + (paper_id, f"%{section_filter}%"), + ).fetchall() + elif node_filter: + rows = conn.execute( + """SELECT unit_id, paper_id, section_path, unit_text, + page_span_json, block_span_json + FROM body_units + WHERE paper_id = ? AND unit_id LIKE ? + ORDER BY unit_id""", + (paper_id, f"%{node_filter}%"), + ).fetchall() + else: + rows = conn.execute( + """SELECT unit_id, paper_id, section_path, unit_text, + page_span_json, block_span_json + FROM body_units + WHERE paper_id = ? + ORDER BY unit_id""", + (paper_id,), + ).fetchall() + + units = [dict(r) for r in rows] + + # Attach manifest if available + manifest_row = conn.execute( + "SELECT value FROM meta WHERE key = ?", + (f"manifest:{paper_id}",), + ).fetchone() + manifest_data: dict | None = ( + json.loads(manifest_row["value"]) if manifest_row else None + ) + + return PFResult( + ok=True, + command="scoped-fetch", + version=PF_VERSION, + data={ + "intent": "scoped-fetch", + "query": query, + "paper_id": paper_id, + "body_units": units, + "count": len(units), + "manifest": manifest_data, + "route_explanation": { + "primary_arm": "body_units", + "section_filter": bool(section_filter), + "node_filter": bool(node_filter), + }, + }, + ) + except Exception as exc: + logger.exception("scoped-fetch failed") + return PFResult( + ok=False, + command="scoped-fetch", + version=PF_VERSION, + data={ + "intent": "scoped-fetch", + "query": query, + "body_units": [], + "route_explanation": { + "primary_arm": "body_units", + "error": str(exc), + }, + }, + ) + finally: + conn.close() diff --git a/tests/test_layer4_gateway_commands.py b/tests/test_layer4_gateway_commands.py index 9eca89a9..333f11ee 100644 --- a/tests/test_layer4_gateway_commands.py +++ b/tests/test_layer4_gateway_commands.py @@ -1,8 +1,370 @@ -"""Tests for Layer 4 gateway commands (Task 1).""" +"""Tests for Layer 4 gateway command routing (Task 5). +Validates that ``route_gateway()`` dispatches each intent to the correct +real data source and surfaces ``route_explanation`` in the result data. +""" + +from __future__ import annotations + +import json +import sqlite3 from argparse import Namespace +from pathlib import Path -from paperforge.commands import paper_lookup, content_discovery, paper_navigation, scoped_fetch +from paperforge.commands import ( + content_discovery, + paper_lookup, + paper_navigation, + scoped_fetch, +) +from paperforge.memory.db import get_connection, get_memory_db_path +from paperforge.memory.schema import ensure_schema +from paperforge.retrieval.gateway import ( + _body_units_fts_exists, + route_gateway, +) + +# --------------------------------------------------------------------------- +# Helpers — populate test databases matching the real schema +# --------------------------------------------------------------------------- + + +def _make_populated_db(tmp_path: Path) -> Path: + """Create a minimal paperforge.db with papers + body_unit FTS content.""" + db_path = get_memory_db_path(tmp_path) + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(str(db_path)) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL;") + conn.execute("PRAGMA foreign_keys=ON;") + ensure_schema(conn) + + # Insert a paper row + conn.execute( + """INSERT INTO papers + (zotero_key, citation_key, title, year, doi, journal, first_author, + authors_json, abstract, domain, collection_path, collections_json, + has_pdf, ocr_status, deep_reading_status, lifecycle, next_step, + impact_factor) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + "TST001", + "Smith2024", + "Delirium Prevention in ICU Patients", + "2024", + "10.1234/delirium", + "Critical Care Medicine", + "Smith", + '["Smith", "Jones"]', + "Delirium is a common complication in ICU patients...", + "ICU", + "", + "[]", + 1, + "done", + "pending", + "active", + "deep-read", + 5.2, + ), + ) + # Insert an alias for DOI lookup + conn.execute( + "INSERT INTO paper_aliases (paper_id, alias, alias_norm, alias_type) VALUES (?, ?, ?, ?)", + ("TST001", "10.1234/delirium", "101234delirium", "doi"), + ) + # Insert body units + conn.execute( + """INSERT INTO body_units + (unit_id, paper_id, section_path, unit_text, + page_span_json, block_span_json, token_estimate, + indexable, veto_reason, quality_hints_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + "TST001:body:sec:intro:1-b1:2-b3", + "TST001", + "Introduction", + "Delirium prevention strategies include early mobilization and reducing sedation.", + json.dumps([1, 2]), + json.dumps([[1, "b1"], [2, "b3"]]), + 50, + 1, + "", + "[]", + ), + ) + conn.execute( + """INSERT INTO body_units + (unit_id, paper_id, section_path, unit_text, + page_span_json, block_span_json, token_estimate, + indexable, veto_reason, quality_hints_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + "TST001:body:sec:methods:3-b4:5-b7", + "TST001", + "Methods", + "We randomized 200 patients to early mobilization vs standard care.", + json.dumps([3, 5]), + json.dumps([[3, "b4"], [5, "b7"]]), + 30, + 1, + "", + "[]", + ), + ) + conn.commit() + + # Populate paper_fts + conn.execute( + """INSERT INTO paper_fts(rowid, zotero_key, citation_key, title, first_author, + authors_json, abstract, journal, domain, collection_path, collections_json) + SELECT rowid, zotero_key, citation_key, title, first_author, + authors_json, abstract, journal, domain, collection_path, collections_json + FROM papers""" + ) + conn.commit() + + # Populate body_units_fts + conn.execute( + """INSERT INTO body_units_fts(rowid, unit_id, paper_id, section_path, unit_text) + SELECT rowid, unit_id, paper_id, section_path, unit_text FROM body_units""" + ) + conn.commit() + + # Store manifest + manifest = { + "paper_id": "TST001", + "retrieval_policy_version": "l4.body.v1", + "body_unit_count": 2, + "built_at": "2025-01-01T00:00:00Z", + } + conn.execute( + "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", + ("manifest:TST001", json.dumps(manifest)), + ) + conn.commit() + conn.close() + return db_path + + +def _make_paper_only_db(tmp_path: Path) -> Path: + """Create a DB with only paper_fts (no body_units_fts).""" + db_path = get_memory_db_path(tmp_path) + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(str(db_path)) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL;") + conn.execute("PRAGMA foreign_keys=ON;") + ensure_schema(conn) + + conn.execute( + """INSERT INTO papers + (zotero_key, citation_key, title, year, doi, journal, first_author, + authors_json, abstract, domain, collection_path, collections_json, + has_pdf, ocr_status, deep_reading_status, lifecycle, next_step, + impact_factor) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + "TST002", + "Johnson2023", + "Sedation Protocols in the ICU", + "2023", + "10.1234/sedation", + "Intensive Care Medicine", + "Johnson", + '["Johnson"]', + "Sedation is critical in ICU management...", + "ICU", + "", + "[]", + 1, + "pending", + "pending", + "active", + "deep-read", + 4.1, + ), + ) + conn.commit() + conn.execute( + """INSERT INTO paper_fts(rowid, zotero_key, citation_key, title, first_author, + authors_json, abstract, journal, domain, collection_path, collections_json) + SELECT rowid, zotero_key, citation_key, title, first_author, + authors_json, abstract, journal, domain, collection_path, collections_json + FROM papers""" + ) + conn.commit() + conn.close() + return db_path + + +def _make_structure_tree(tmp_path: Path, paper_id: str) -> Path: + """Create a mock structure-tree.json for a paper.""" + ocr_root = tmp_path / "System" / "PaperForge" / "ocr" / paper_id / "index" + ocr_root.mkdir(parents=True, exist_ok=True) + tree = { + "paper_id": paper_id, + "nodes": [ + { + "node_id": "sec:intro", + "kind": "section", + "title": "Introduction", + "level": 1, + "section_path": ["Introduction"], + "page_span": [1, 2], + }, + { + "node_id": "sec:methods", + "kind": "section", + "title": "Methods", + "level": 1, + "section_path": ["Methods"], + "page_span": [3, 5], + }, + ], + } + (ocr_root / "structure-tree.json").write_text( + json.dumps(tree, ensure_ascii=False), encoding="utf-8" + ) + return ocr_root + + +# --------------------------------------------------------------------------- +# paper-lookup intent +# --------------------------------------------------------------------------- + + +def test_paper_lookup_returns_results_when_db_has_match(tmp_path): + _make_populated_db(tmp_path) + result = route_gateway(tmp_path, "paper-lookup", "Smith 2024", json_mode=True) + assert result.ok + assert result.data["intent"] == "paper-lookup" + assert len(result.data["results"]) > 0 + assert "route_explanation" in result.data + assert result.data["route_explanation"]["primary_arm"] == "lookup_paper" + + +def test_paper_lookup_returns_empty_when_no_match(tmp_path): + _make_populated_db(tmp_path) + result = route_gateway( + tmp_path, "paper-lookup", "Nonexistent Author 2099", json_mode=True + ) + assert result.ok + assert len(result.data["results"]) == 0 + assert result.data["route_explanation"]["matched"] is False + + +def test_paper_lookup_fails_gracefully_without_db(tmp_path): + result = route_gateway(tmp_path, "paper-lookup", "Smith 2024", json_mode=True) + assert not result.ok + assert "route_explanation" in result.data + + +# --------------------------------------------------------------------------- +# content-discovery intent +# --------------------------------------------------------------------------- + + +def test_content_discovery_prefers_body_units_fts_when_present(tmp_path): + _make_populated_db(tmp_path) + result = route_gateway( + tmp_path, "content-discovery", "delirium prevention", json_mode=True, limit=5 + ) + assert result.ok + assert result.data["intent"] == "content-discovery" + assert result.data["route_explanation"]["primary_arm"] == "body_units_fts" + assert result.data["route_explanation"]["compatibility_mode"] is False + assert "results" in result.data + + +def test_paper_navigation_reads_structure_tree_when_present(tmp_path): + _make_populated_db(tmp_path) + _make_structure_tree(tmp_path, "TST001") + result = route_gateway(tmp_path, "paper-navigation", "10.1234/delirium", json_mode=True) + assert result.ok + assert result.data["intent"] == "paper-navigation" + assert result.data["mode"] == "structure_tree" + assert "nodes" in result.data + assert result.data["route_explanation"]["primary_arm"] == "structure_tree" + assert result.data["route_explanation"]["fallback"] is False + + +def test_content_discovery_returns_body_unit_matches(tmp_path): + _make_populated_db(tmp_path) + result = route_gateway( + tmp_path, "content-discovery", "delirium", json_mode=True, limit=5 + ) + assert result.ok + assert len(result.data["results"]) > 0 + for r in result.data["results"]: + assert "unit_id" in r + assert "unit_text" in r + assert "section_path" in r + + +def test_content_discovery_falls_back_to_paper_fts(tmp_path): + _make_paper_only_db(tmp_path) + result = route_gateway( + tmp_path, "content-discovery", "sedation", json_mode=True, limit=5 + ) + assert result.ok + assert result.data["route_explanation"]["primary_arm"] == "paper_fts" + assert result.data["route_explanation"]["compatibility_mode"] is True + assert len(result.data["results"]) > 0 + + +def test_content_discovery_fails_gracefully_without_db(tmp_path): + result = route_gateway( + tmp_path, "content-discovery", "delirium", json_mode=True, limit=5 + ) + assert not result.ok + assert "route_explanation" in result.data + + +# --------------------------------------------------------------------------- +# paper-navigation intent + + +def test_paper_navigation_returns_not_found_without_db(tmp_path): + result = route_gateway(tmp_path, "paper-navigation", "ABCD1234", json_mode=True) + assert not result.ok + assert result.data["mode"] == "not_found" + + +# --------------------------------------------------------------------------- +# scoped-fetch intent +# --------------------------------------------------------------------------- + + +def test_scoped_fetch_returns_body_units_for_paper(tmp_path): + _make_populated_db(tmp_path) + result = route_gateway(tmp_path, "scoped-fetch", "Smith 2024", json_mode=True) + assert result.ok + assert result.data["intent"] == "scoped-fetch" + assert len(result.data["body_units"]) > 0 + assert "manifest" in result.data + assert result.data["route_explanation"]["primary_arm"] == "body_units" + + +def test_scoped_fetch_by_doi(tmp_path): + _make_populated_db(tmp_path) + result = route_gateway(tmp_path, "scoped-fetch", "10.1234/delirium", json_mode=True) + assert result.ok + assert len(result.data["body_units"]) > 0 + + +def test_scoped_fetch_empty_for_unknown_paper(tmp_path): + _make_populated_db(tmp_path) + result = route_gateway( + tmp_path, "scoped-fetch", "Unknown Author 1999", json_mode=True + ) + assert not result.ok + assert len(result.data["body_units"]) == 0 + + +# --------------------------------------------------------------------------- +# Command integration tests (exercise the CLI entry points) +# --------------------------------------------------------------------------- def test_paper_lookup_command_registered_and_json(tmp_path): @@ -11,43 +373,74 @@ def test_paper_lookup_command_registered_and_json(tmp_path): assert exit_code in {0, 1} -def test_content_discovery_warns_when_only_metadata_fts_exists(tmp_path, monkeypatch): - from paperforge.core.result import PFResult - from paperforge.retrieval import gateway - - def fake_route_gateway(*args, **kwargs): - return PFResult( - ok=True, - command="content-discovery", - version="x", - data={"mode": "metadata_only"}, - warnings=["body_units_fts missing"], - ) - - monkeypatch.setattr(gateway, "route_gateway", fake_route_gateway) - args = Namespace(vault_path=tmp_path, query="delirium prevention", json=True, limit=5) - assert content_discovery.run(args) == 0 +def test_content_discovery_command_runs(tmp_path): + args = Namespace( + vault_path=tmp_path, query="delirium prevention", json=True, limit=5 + ) + exit_code = content_discovery.run(args) + assert exit_code in {0, 1} -def test_paper_navigation_runs(tmp_path): - args = Namespace(vault_path=tmp_path, query="10.1234/test-doi-here", json=True, limit=5) +def test_paper_navigation_command_runs(tmp_path): + args = Namespace( + vault_path=tmp_path, query="10.1234/test-doi-here", json=True + ) exit_code = paper_navigation.run(args) assert exit_code in {0, 1} -def test_scoped_fetch_runs(tmp_path): +def test_scoped_fetch_command_runs(tmp_path): args = Namespace(vault_path=tmp_path, query="Smith 2021", json=False, limit=3) exit_code = scoped_fetch.run(args) assert exit_code in {0, 1} -def test_content_discovery_non_json_output(tmp_path, monkeypatch): - from paperforge.core.result import PFResult - from paperforge.retrieval import gateway +# --------------------------------------------------------------------------- +# Helper unit tests +# --------------------------------------------------------------------------- - def fake_route_gateway(*args, **kwargs): - return PFResult(ok=True, command="content-discovery", version="x", data={"mode": "metadata_only"}) - monkeypatch.setattr(gateway, "route_gateway", fake_route_gateway) - args = Namespace(vault_path=tmp_path, query="delirium prevention", json=False, limit=5) - assert content_discovery.run(args) == 0 +def test_body_units_fts_exists_returns_true_when_populated(tmp_path): + _make_populated_db(tmp_path) + assert _body_units_fts_exists(tmp_path) is True + + +def test_body_units_fts_exists_returns_false_without_db(tmp_path): + assert _body_units_fts_exists(tmp_path) is False + + +def test_body_units_fts_exists_returns_false_no_body_units(tmp_path): + _make_paper_only_db(tmp_path) + assert _body_units_fts_exists(tmp_path) is False + + +def test_unsupported_intent_raises(tmp_path): + import pytest + + with pytest.raises(ValueError, match="Unsupported Layer 4 intent"): + route_gateway(tmp_path, "unknown-intent", "query", json_mode=True) + + +# --------------------------------------------------------------------------- +# route_explanation contract +# --------------------------------------------------------------------------- + + +def test_all_intents_surface_route_explanation(tmp_path): + _make_populated_db(tmp_path) + _make_structure_tree(tmp_path, "TST001") + + for intent, query in [ + ("paper-lookup", "Smith 2024"), + ("content-discovery", "delirium"), + ("paper-navigation", "10.1234/delirium"), + ("scoped-fetch", "Smith 2024"), + ]: + result = route_gateway(tmp_path, intent, query, json_mode=True) + # Even failing results should have route_explanation + assert "route_explanation" in result.data, ( + f"{intent} missing route_explanation" + ) + assert "primary_arm" in result.data["route_explanation"], ( + f"{intent} missing primary_arm" + )