feat: route Layer 4 gateway through body units and structure tree

This commit is contained in:
LLLin000 2026-07-05 14:39:19 +08:00
parent 2ab3aa4219
commit 1e6edad7ba
6 changed files with 941 additions and 195 deletions

View file

@ -1,60 +1,24 @@
"""paperforge.commands.paper_navigation — ``paperforge paper-navigation`` gateway command."""
from __future__ import annotations
import argparse
import json
from paperforge import __version__ as PF_VERSION
from paperforge.core.result import PFResult
from paperforge.core.io import read_json
from paperforge.retrieval.structure_tree import summarize_role_index
from paperforge.retrieval import gateway
def _resolve_paper_root(vault_path, query: str):
"""Resolve a paper query to its OCR root directory."""
from paperforge.memory.db import get_connection, get_memory_db_path
from paperforge.memory.query import lookup_paper
from paperforge.worker._utils import pipeline_paths
db_path = get_memory_db_path(vault_path)
if not db_path or not db_path.exists():
return None
conn = get_connection(db_path, read_only=True)
try:
entries = lookup_paper(conn, query)
if entries:
ocr_root = pipeline_paths(vault_path)["ocr"]
return ocr_root / entries[0]["zotero_key"]
finally:
conn.close()
return None
def run(args: argparse.Namespace) -> int:
paper_root = _resolve_paper_root(args.vault_path, args.query)
if paper_root is None:
data = {"mode": "not_found", "paper_id": args.query, "error": "Paper not found in vault"}
if args.json:
print(PFResult(ok=False, command="paper-navigation", version=PF_VERSION, data=data).to_json())
else:
print(json.dumps(data, ensure_ascii=False, indent=2))
return 1
tree_path = paper_root / "index" / "structure-tree.json"
if tree_path.exists():
tree = read_json(tree_path)
payload = {"mode": "structure_tree", "paper_id": tree.get("paper_id", ""), "nodes": tree.get("nodes", [])}
else:
role_index_path = paper_root / "index" / "role-index.json"
if role_index_path.exists():
role_index = read_json(role_index_path)
summary = summarize_role_index(role_index)
payload = {"mode": "role_index_summary", "paper_id": args.query, "summary": summary}
else:
payload = {"mode": "no_index", "paper_id": args.query}
result = PFResult(ok=True, command="paper-navigation", version=PF_VERSION, data=payload)
def run(args) -> int:
"""Execute ``paper-navigation`` via the Layer 4 gateway."""
result = gateway.route_gateway(
args.vault_path,
"paper-navigation",
args.query,
json_mode=bool(args.json),
)
if args.json:
print(result.to_json())
else:
print(json.dumps(payload, ensure_ascii=False, indent=2))
return 0
print(json.dumps(result.data, ensure_ascii=False, indent=2))
return 0 if result.ok else 1

View file

@ -126,7 +126,15 @@ def run(args: argparse.Namespace) -> int:
finally:
conn.close()
data = {"query": query, "chunks": chunks, "count": len(chunks)}
data = {
"query": query,
"chunks": chunks,
"count": len(chunks),
"route_explanation": {
"primary_arm": "vector_retrieve",
"compatibility_mode": False,
},
}
warnings: list[str] = []
next_actions: list[dict] = []
if len(chunks) == 0:

View file

@ -5,115 +5,18 @@ from __future__ import annotations
import json
from pathlib import Path
from paperforge.core.io import read_json
from paperforge.core.result import PFResult
from paperforge.memory.db import get_connection, get_memory_db_path
from paperforge.retrieval import gateway
from paperforge.retrieval.manifest import build_paper_manifest
from paperforge.retrieval.units import build_body_units, build_object_units
def _find_ocr_dir(vault: Path, paper_id: str) -> Path | None:
"""Look up the OCR output directory for a paper by scanning the ocr root.
Returns None if no OCR output exists.
"""
ocr_root = vault / "System" / "PaperForge" / "ocr"
if not ocr_root.exists():
return None
for d in ocr_root.iterdir():
if d.is_dir() and d.name == paper_id:
return d
# also check metadata inside
index_path = d / "index"
tree_path = index_path / "structure-tree.json"
if tree_path.exists():
try:
tree = read_json(tree_path)
if tree.get("paper_id") == paper_id:
return d
except Exception:
continue
return None
def _build_units_for_paper(
vault: Path,
paper_id: str,
) -> dict:
"""Build retrieval units and manifest for a paper from its OCR output.
Returns a dict with keys ``body_units``, ``object_units``, ``manifest``
(or empty lists / None when OCR data is unavailable).
"""
ocr_dir = _find_ocr_dir(vault, paper_id)
if ocr_dir is None:
return {"body_units": [], "object_units": [], "manifest": None}
index_root = ocr_dir / "index"
tree_path = index_root / "structure-tree.json"
structured_path = ocr_dir / "structured-blocks.json"
if not tree_path.exists() or not structured_path.exists():
return {"body_units": [], "object_units": [], "manifest": None}
tree = read_json(tree_path)
structured_blocks = read_json(structured_path)
role_index_path = index_root / "role-index.json"
role_index = read_json(role_index_path) if role_index_path.exists() else {}
body_units = build_body_units(tree=tree, structured_blocks=structured_blocks)
object_units = build_object_units(
tree=tree, structured_blocks=structured_blocks, role_index=role_index
)
# Try to read result hash
result_hash_path = index_root / "result-hash.txt"
ocr_result_hash = ""
if result_hash_path.exists():
ocr_result_hash = result_hash_path.read_text(encoding="utf-8").strip()
manifest = build_paper_manifest(
paper_id=paper_id,
ocr_result_hash=ocr_result_hash,
structure_tree_bytes=tree_path.read_bytes(),
retrieval_policy_version="l4.body.v1",
body_units=body_units,
object_units=object_units,
source_paths={
"structured_blocks": str(structured_path),
"role_index": str(role_index_path),
"fulltext": str(ocr_dir / "fulltext.md"),
},
)
return {"body_units": body_units, "object_units": object_units, "manifest": manifest}
def run(args):
def run(args) -> int:
"""Execute ``scoped-fetch`` via the Layer 4 gateway."""
vault = Path(args.vault_path)
result = gateway.route_gateway(
vault,
"scoped-fetch",
args.query,
json_mode=args.json,
json_mode=bool(args.json),
limit=getattr(args, "limit", 5),
)
# If we have a valid route result, enrich it with retrieval units
if result.ok and result.data:
# Extract paper_id from the route plan
plan = result.data.get("route_plan", {})
paper_id = (
plan.get("paper_id")
or plan.get("primary_paper_id")
or plan.get("target_id")
or ""
)
if paper_id:
units = _build_units_for_paper(vault, paper_id)
result.data["body_units"] = units["body_units"]
result.data["object_units"] = units["object_units"]
result.data["manifest"] = units["manifest"]
print(result.to_json() if args.json else result.data)
return 0 if result.ok else 1

View file

@ -59,6 +59,10 @@ def run(args: argparse.Namespace) -> int:
"lifecycle": args.lifecycle,
"next_step": args.next_step,
},
"route_explanation": {
"primary_arm": "paper_fts",
"compatibility_mode": False,
},
}
warnings: list[str] = []
next_actions: list[dict] = []

View file

@ -1,17 +1,22 @@
"""paperforge.retrieval.gateway — Layer 4 gateway command routing.
Provides the core `route_gateway()` function that all gateway commands
route through. Currently delegates to existing capabilities; later tasks
will upgrade routing to use real Layer 4 artifacts.
route through. Each intent is routed to the correct real data source.
"""
from __future__ import annotations
import json
import logging
import re
from pathlib import Path
from paperforge import __version__ as PF_VERSION
from paperforge.core.io import read_json
from paperforge.core.result import PFResult
from paperforge.query_planning import build_query_plan, enrich_query_plan_with_runtime
from paperforge.memory.db import get_connection, get_memory_db_path
logger = logging.getLogger(__name__)
# Map gateway intent names to build_query_plan intent strings
INTENTS: dict[str, str] = {
@ -27,41 +32,510 @@ def route_gateway(
intent: str,
query: str,
*,
json_mode: bool,
json_mode: bool, # noqa: ARG001
limit: int = 5,
) -> PFResult:
"""Route a gateway command through the query planning pipeline.
"""Route a gateway command to the real Layer 4 data source.
Parameters
----------
vault : Path
PaperForge vault root.
intent : str
Gateway intent name (e.g. ``"paper-lookup"``).
Gateway intent name (``"paper-lookup"``, ``"content-discovery"``,
``"paper-navigation"``, or ``"scoped-fetch"``).
query : str
Free-text or structured query.
json_mode : bool
Whether the caller expects JSON output.
Whether the caller expects JSON output (passed through to callers).
limit : int, optional
Maximum result count (default 5).
Returns
-------
PFResult
Result packet with the route plan and intent metadata.
Result packet with data from the real source of truth for the intent.
"""
plan = enrich_query_plan_with_runtime(
build_query_plan(query, INTENTS[intent]),
vault,
)
if intent == "paper-lookup":
return _run_paper_lookup(vault, query, limit=limit)
if intent == "content-discovery":
if _body_units_fts_exists(vault):
return _run_body_unit_discovery(vault, query, limit=limit)
return _run_compat_content_discovery(vault, query, limit=limit)
if intent == "paper-navigation":
return _run_paper_navigation(vault, query)
if intent == "scoped-fetch":
return _run_scoped_fetch(vault, query)
raise ValueError(f"Unsupported Layer 4 intent: {intent}")
# ---------------------------------------------------------------------------
# FTS helpers
# ---------------------------------------------------------------------------
def _tokenize_for_fts(q: str) -> str:
"""Tokenize a query string into safe FTS5 match terms."""
tokens = re.findall(r"[\w\u4e00-\u9fff]+", q)
if not tokens:
return q
return " OR ".join(f'"{t}"' for t in tokens)
def _body_units_fts_exists(vault: Path) -> bool:
"""Check whether the body_units_fts table exists and has rows."""
db_path = get_memory_db_path(vault)
if not db_path.exists():
return False
conn = get_connection(db_path, read_only=True)
try:
row = conn.execute("SELECT COUNT(*) AS cnt FROM body_units_fts").fetchone()
return row["cnt"] > 0
except Exception:
return False
finally:
conn.close()
# ---------------------------------------------------------------------------
# Paper root resolution (shared by navigation & scoped-fetch)
# ---------------------------------------------------------------------------
def _resolve_paper_root(vault: Path, query: str) -> Path | None:
"""Resolve a paper query to its OCR root directory."""
from paperforge.config import paperforge_paths
from paperforge.memory.query import lookup_paper
db_path = get_memory_db_path(vault)
if not db_path or not db_path.exists():
return None
conn = get_connection(db_path, read_only=True)
try:
entries = lookup_paper(conn, query)
if entries:
ocr_root = paperforge_paths(vault)["ocr"]
return ocr_root / entries[0]["zotero_key"]
finally:
conn.close()
return None
# ---------------------------------------------------------------------------
# Intent: paper-lookup
# ---------------------------------------------------------------------------
def _run_paper_lookup(vault: Path, query: str, *, limit: int = 5) -> PFResult:
"""Route paper-lookup intent through ``lookup_paper()``."""
from paperforge.memory.query import lookup_paper
db_path = get_memory_db_path(vault)
if not db_path.exists():
return PFResult(
ok=False,
command="paper-lookup",
version=PF_VERSION,
data={
"intent": "paper-lookup",
"query": query,
"results": [],
"route_explanation": {
"primary_arm": "lookup_paper",
"error": "database_not_found",
},
},
)
conn = get_connection(db_path, read_only=True)
try:
entries = lookup_paper(conn, query)
limited = entries[:limit]
return PFResult(
ok=True,
command="paper-lookup",
version=PF_VERSION,
data={
"intent": "paper-lookup",
"query": query,
"results": limited,
"count": len(limited),
"route_explanation": {
"primary_arm": "lookup_paper",
"matched": len(limited) > 0,
"compatibility_mode": False,
},
},
)
except Exception as exc:
logger.exception("paper-lookup failed")
return PFResult(
ok=False,
command="paper-lookup",
version=PF_VERSION,
data={
"intent": "paper-lookup",
"query": query,
"results": [],
"route_explanation": {
"primary_arm": "lookup_paper",
"error": str(exc),
},
},
)
finally:
conn.close()
# ---------------------------------------------------------------------------
# Intent: content-discovery
# ---------------------------------------------------------------------------
def _run_body_unit_discovery(
vault: Path, query: str, *, limit: int = 5
) -> PFResult:
"""Search ``body_units_fts`` for content-level discovery."""
db_path = get_memory_db_path(vault)
if not db_path.exists():
return PFResult(
ok=False,
command="content-discovery",
version=PF_VERSION,
data={
"intent": "content-discovery",
"query": query,
"results": [],
"route_explanation": {
"primary_arm": "body_units_fts",
"error": "database_not_found",
},
},
)
conn = get_connection(db_path, read_only=True)
try:
fts_query = _tokenize_for_fts(query)
rows = conn.execute(
"""SELECT unit_id, paper_id, section_path, unit_text, rank
FROM body_units_fts
WHERE body_units_fts MATCH ?
ORDER BY rank
LIMIT ?""",
(fts_query, limit),
).fetchall()
results = [dict(r) for r in rows]
return PFResult(
ok=True,
command="content-discovery",
version=PF_VERSION,
data={
"intent": "content-discovery",
"query": query,
"results": results,
"count": len(results),
"route_explanation": {
"primary_arm": "body_units_fts",
"fallback_arms": ["vector_retrieve"],
"compatibility_mode": False,
},
},
)
except Exception as exc:
logger.exception("body_units_fts query failed, falling back")
return _run_compat_content_discovery(
vault, query, limit=limit, explanation_note=str(exc)
)
finally:
conn.close()
def _run_compat_content_discovery(
vault: Path,
query: str,
*,
limit: int = 5,
explanation_note: str = "",
) -> PFResult:
"""Fallback content discovery using ``paper_fts`` (metadata-only)."""
from paperforge.memory.fts import search_papers
db_path = get_memory_db_path(vault)
if not db_path.exists():
return PFResult(
ok=False,
command="content-discovery",
version=PF_VERSION,
data={
"intent": "content-discovery",
"query": query,
"results": [],
"route_explanation": {
"primary_arm": "paper_fts",
"error": "database_not_found",
},
},
)
conn = get_connection(db_path, read_only=True)
try:
results = search_papers(conn, query, limit=limit)
return PFResult(
ok=True,
command="content-discovery",
version=PF_VERSION,
data={
"intent": "content-discovery",
"query": query,
"results": results,
"count": len(results),
"route_explanation": {
"primary_arm": "paper_fts",
"compatibility_mode": True,
"note": (
explanation_note
or "body_units_fts unavailable, using metadata FTS"
),
},
},
)
except Exception as exc:
return PFResult(
ok=False,
command="content-discovery",
version=PF_VERSION,
data={
"intent": "content-discovery",
"query": query,
"results": [],
"route_explanation": {
"primary_arm": "paper_fts",
"error": str(exc),
},
},
)
finally:
conn.close()
# ---------------------------------------------------------------------------
# Intent: paper-navigation
# ---------------------------------------------------------------------------
def _run_paper_navigation(vault: Path, query: str) -> PFResult:
"""Read structure-tree.json (or fall back to role-index) for navigation."""
paper_root = _resolve_paper_root(vault, query)
if paper_root is None:
return PFResult(
ok=False,
command="paper-navigation",
version=PF_VERSION,
data={
"intent": "paper-navigation",
"query": query,
"mode": "not_found",
"route_explanation": {
"primary_arm": "structure_tree",
"note": "paper not found",
},
},
)
tree_path = paper_root / "index" / "structure-tree.json"
if tree_path.exists():
tree = read_json(tree_path)
return PFResult(
ok=True,
command="paper-navigation",
version=PF_VERSION,
data={
"intent": "paper-navigation",
"query": query,
"mode": "structure_tree",
"paper_id": tree.get("paper_id", ""),
"nodes": tree.get("nodes", []),
"route_explanation": {
"primary_arm": "structure_tree",
"fallback": False,
},
},
)
# Fallback: role-index summary
role_index_path = paper_root / "index" / "role-index.json"
if role_index_path.exists():
from paperforge.retrieval.structure_tree import summarize_role_index
role_index = read_json(role_index_path)
summary = summarize_role_index(role_index)
return PFResult(
ok=True,
command="paper-navigation",
version=PF_VERSION,
data={
"intent": "paper-navigation",
"query": query,
"mode": "role_index_summary",
"paper_id": query,
"summary": summary,
"route_explanation": {
"primary_arm": "structure_tree",
"fallback_arm": "role_index",
"note": "structure-tree.json not found, using role-index",
},
},
)
return PFResult(
ok=True,
command=intent,
ok=False,
command="paper-navigation",
version=PF_VERSION,
data={
"intent": intent,
"intent": "paper-navigation",
"query": query,
"route_plan": plan,
"limit": limit,
"mode": "no_index",
"route_explanation": {
"primary_arm": "structure_tree",
"note": "no index found for paper",
},
},
)
# ---------------------------------------------------------------------------
# Intent: scoped-fetch
# ---------------------------------------------------------------------------
def _run_scoped_fetch(vault: Path, query: str) -> PFResult:
"""Fetch body units scoped by paper, optionally filtered by section or node.
The *query* is treated as a paper identifier (author, title, DOI, etc.).
An optional ``:section_path`` or ``#node_id`` suffix narrows the results.
"""
from paperforge.memory.query import lookup_paper
# Parse optional section / node filter from query suffix
base_query: str = query
section_filter: str | None = None
node_filter: str | None = None
if ":" in query and not query.startswith(":"):
parts = query.split(":", 1)
base_query = parts[0]
section_filter = parts[1].strip()
elif "#" in query:
parts = query.split("#", 1)
base_query = parts[0]
node_filter = parts[1].strip()
db_path = get_memory_db_path(vault)
if not db_path.exists():
return PFResult(
ok=False,
command="scoped-fetch",
version=PF_VERSION,
data={
"intent": "scoped-fetch",
"query": query,
"body_units": [],
"route_explanation": {
"primary_arm": "body_units",
"error": "database_not_found",
},
},
)
conn = get_connection(db_path, read_only=True)
try:
entries = lookup_paper(conn, base_query)
if not entries:
return PFResult(
ok=False,
command="scoped-fetch",
version=PF_VERSION,
data={
"intent": "scoped-fetch",
"query": query,
"body_units": [],
"route_explanation": {
"primary_arm": "body_units",
"note": "paper not found",
},
},
)
paper_id = entries[0].get("zotero_key", "")
if section_filter:
rows = conn.execute(
"""SELECT unit_id, paper_id, section_path, unit_text,
page_span_json, block_span_json
FROM body_units
WHERE paper_id = ? AND section_path LIKE ?
ORDER BY unit_id""",
(paper_id, f"%{section_filter}%"),
).fetchall()
elif node_filter:
rows = conn.execute(
"""SELECT unit_id, paper_id, section_path, unit_text,
page_span_json, block_span_json
FROM body_units
WHERE paper_id = ? AND unit_id LIKE ?
ORDER BY unit_id""",
(paper_id, f"%{node_filter}%"),
).fetchall()
else:
rows = conn.execute(
"""SELECT unit_id, paper_id, section_path, unit_text,
page_span_json, block_span_json
FROM body_units
WHERE paper_id = ?
ORDER BY unit_id""",
(paper_id,),
).fetchall()
units = [dict(r) for r in rows]
# Attach manifest if available
manifest_row = conn.execute(
"SELECT value FROM meta WHERE key = ?",
(f"manifest:{paper_id}",),
).fetchone()
manifest_data: dict | None = (
json.loads(manifest_row["value"]) if manifest_row else None
)
return PFResult(
ok=True,
command="scoped-fetch",
version=PF_VERSION,
data={
"intent": "scoped-fetch",
"query": query,
"paper_id": paper_id,
"body_units": units,
"count": len(units),
"manifest": manifest_data,
"route_explanation": {
"primary_arm": "body_units",
"section_filter": bool(section_filter),
"node_filter": bool(node_filter),
},
},
)
except Exception as exc:
logger.exception("scoped-fetch failed")
return PFResult(
ok=False,
command="scoped-fetch",
version=PF_VERSION,
data={
"intent": "scoped-fetch",
"query": query,
"body_units": [],
"route_explanation": {
"primary_arm": "body_units",
"error": str(exc),
},
},
)
finally:
conn.close()

View file

@ -1,8 +1,370 @@
"""Tests for Layer 4 gateway commands (Task 1)."""
"""Tests for Layer 4 gateway command routing (Task 5).
Validates that ``route_gateway()`` dispatches each intent to the correct
real data source and surfaces ``route_explanation`` in the result data.
"""
from __future__ import annotations
import json
import sqlite3
from argparse import Namespace
from pathlib import Path
from paperforge.commands import paper_lookup, content_discovery, paper_navigation, scoped_fetch
from paperforge.commands import (
content_discovery,
paper_lookup,
paper_navigation,
scoped_fetch,
)
from paperforge.memory.db import get_connection, get_memory_db_path
from paperforge.memory.schema import ensure_schema
from paperforge.retrieval.gateway import (
_body_units_fts_exists,
route_gateway,
)
# ---------------------------------------------------------------------------
# Helpers — populate test databases matching the real schema
# ---------------------------------------------------------------------------
def _make_populated_db(tmp_path: Path) -> Path:
"""Create a minimal paperforge.db with papers + body_unit FTS content."""
db_path = get_memory_db_path(tmp_path)
db_path.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(str(db_path))
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA journal_mode=WAL;")
conn.execute("PRAGMA foreign_keys=ON;")
ensure_schema(conn)
# Insert a paper row
conn.execute(
"""INSERT INTO papers
(zotero_key, citation_key, title, year, doi, journal, first_author,
authors_json, abstract, domain, collection_path, collections_json,
has_pdf, ocr_status, deep_reading_status, lifecycle, next_step,
impact_factor)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(
"TST001",
"Smith2024",
"Delirium Prevention in ICU Patients",
"2024",
"10.1234/delirium",
"Critical Care Medicine",
"Smith",
'["Smith", "Jones"]',
"Delirium is a common complication in ICU patients...",
"ICU",
"",
"[]",
1,
"done",
"pending",
"active",
"deep-read",
5.2,
),
)
# Insert an alias for DOI lookup
conn.execute(
"INSERT INTO paper_aliases (paper_id, alias, alias_norm, alias_type) VALUES (?, ?, ?, ?)",
("TST001", "10.1234/delirium", "101234delirium", "doi"),
)
# Insert body units
conn.execute(
"""INSERT INTO body_units
(unit_id, paper_id, section_path, unit_text,
page_span_json, block_span_json, token_estimate,
indexable, veto_reason, quality_hints_json)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(
"TST001:body:sec:intro:1-b1:2-b3",
"TST001",
"Introduction",
"Delirium prevention strategies include early mobilization and reducing sedation.",
json.dumps([1, 2]),
json.dumps([[1, "b1"], [2, "b3"]]),
50,
1,
"",
"[]",
),
)
conn.execute(
"""INSERT INTO body_units
(unit_id, paper_id, section_path, unit_text,
page_span_json, block_span_json, token_estimate,
indexable, veto_reason, quality_hints_json)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(
"TST001:body:sec:methods:3-b4:5-b7",
"TST001",
"Methods",
"We randomized 200 patients to early mobilization vs standard care.",
json.dumps([3, 5]),
json.dumps([[3, "b4"], [5, "b7"]]),
30,
1,
"",
"[]",
),
)
conn.commit()
# Populate paper_fts
conn.execute(
"""INSERT INTO paper_fts(rowid, zotero_key, citation_key, title, first_author,
authors_json, abstract, journal, domain, collection_path, collections_json)
SELECT rowid, zotero_key, citation_key, title, first_author,
authors_json, abstract, journal, domain, collection_path, collections_json
FROM papers"""
)
conn.commit()
# Populate body_units_fts
conn.execute(
"""INSERT INTO body_units_fts(rowid, unit_id, paper_id, section_path, unit_text)
SELECT rowid, unit_id, paper_id, section_path, unit_text FROM body_units"""
)
conn.commit()
# Store manifest
manifest = {
"paper_id": "TST001",
"retrieval_policy_version": "l4.body.v1",
"body_unit_count": 2,
"built_at": "2025-01-01T00:00:00Z",
}
conn.execute(
"INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)",
("manifest:TST001", json.dumps(manifest)),
)
conn.commit()
conn.close()
return db_path
def _make_paper_only_db(tmp_path: Path) -> Path:
"""Create a DB with only paper_fts (no body_units_fts)."""
db_path = get_memory_db_path(tmp_path)
db_path.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(str(db_path))
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA journal_mode=WAL;")
conn.execute("PRAGMA foreign_keys=ON;")
ensure_schema(conn)
conn.execute(
"""INSERT INTO papers
(zotero_key, citation_key, title, year, doi, journal, first_author,
authors_json, abstract, domain, collection_path, collections_json,
has_pdf, ocr_status, deep_reading_status, lifecycle, next_step,
impact_factor)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(
"TST002",
"Johnson2023",
"Sedation Protocols in the ICU",
"2023",
"10.1234/sedation",
"Intensive Care Medicine",
"Johnson",
'["Johnson"]',
"Sedation is critical in ICU management...",
"ICU",
"",
"[]",
1,
"pending",
"pending",
"active",
"deep-read",
4.1,
),
)
conn.commit()
conn.execute(
"""INSERT INTO paper_fts(rowid, zotero_key, citation_key, title, first_author,
authors_json, abstract, journal, domain, collection_path, collections_json)
SELECT rowid, zotero_key, citation_key, title, first_author,
authors_json, abstract, journal, domain, collection_path, collections_json
FROM papers"""
)
conn.commit()
conn.close()
return db_path
def _make_structure_tree(tmp_path: Path, paper_id: str) -> Path:
"""Create a mock structure-tree.json for a paper."""
ocr_root = tmp_path / "System" / "PaperForge" / "ocr" / paper_id / "index"
ocr_root.mkdir(parents=True, exist_ok=True)
tree = {
"paper_id": paper_id,
"nodes": [
{
"node_id": "sec:intro",
"kind": "section",
"title": "Introduction",
"level": 1,
"section_path": ["Introduction"],
"page_span": [1, 2],
},
{
"node_id": "sec:methods",
"kind": "section",
"title": "Methods",
"level": 1,
"section_path": ["Methods"],
"page_span": [3, 5],
},
],
}
(ocr_root / "structure-tree.json").write_text(
json.dumps(tree, ensure_ascii=False), encoding="utf-8"
)
return ocr_root
# ---------------------------------------------------------------------------
# paper-lookup intent
# ---------------------------------------------------------------------------
def test_paper_lookup_returns_results_when_db_has_match(tmp_path):
_make_populated_db(tmp_path)
result = route_gateway(tmp_path, "paper-lookup", "Smith 2024", json_mode=True)
assert result.ok
assert result.data["intent"] == "paper-lookup"
assert len(result.data["results"]) > 0
assert "route_explanation" in result.data
assert result.data["route_explanation"]["primary_arm"] == "lookup_paper"
def test_paper_lookup_returns_empty_when_no_match(tmp_path):
_make_populated_db(tmp_path)
result = route_gateway(
tmp_path, "paper-lookup", "Nonexistent Author 2099", json_mode=True
)
assert result.ok
assert len(result.data["results"]) == 0
assert result.data["route_explanation"]["matched"] is False
def test_paper_lookup_fails_gracefully_without_db(tmp_path):
result = route_gateway(tmp_path, "paper-lookup", "Smith 2024", json_mode=True)
assert not result.ok
assert "route_explanation" in result.data
# ---------------------------------------------------------------------------
# content-discovery intent
# ---------------------------------------------------------------------------
def test_content_discovery_prefers_body_units_fts_when_present(tmp_path):
_make_populated_db(tmp_path)
result = route_gateway(
tmp_path, "content-discovery", "delirium prevention", json_mode=True, limit=5
)
assert result.ok
assert result.data["intent"] == "content-discovery"
assert result.data["route_explanation"]["primary_arm"] == "body_units_fts"
assert result.data["route_explanation"]["compatibility_mode"] is False
assert "results" in result.data
def test_paper_navigation_reads_structure_tree_when_present(tmp_path):
_make_populated_db(tmp_path)
_make_structure_tree(tmp_path, "TST001")
result = route_gateway(tmp_path, "paper-navigation", "10.1234/delirium", json_mode=True)
assert result.ok
assert result.data["intent"] == "paper-navigation"
assert result.data["mode"] == "structure_tree"
assert "nodes" in result.data
assert result.data["route_explanation"]["primary_arm"] == "structure_tree"
assert result.data["route_explanation"]["fallback"] is False
def test_content_discovery_returns_body_unit_matches(tmp_path):
_make_populated_db(tmp_path)
result = route_gateway(
tmp_path, "content-discovery", "delirium", json_mode=True, limit=5
)
assert result.ok
assert len(result.data["results"]) > 0
for r in result.data["results"]:
assert "unit_id" in r
assert "unit_text" in r
assert "section_path" in r
def test_content_discovery_falls_back_to_paper_fts(tmp_path):
_make_paper_only_db(tmp_path)
result = route_gateway(
tmp_path, "content-discovery", "sedation", json_mode=True, limit=5
)
assert result.ok
assert result.data["route_explanation"]["primary_arm"] == "paper_fts"
assert result.data["route_explanation"]["compatibility_mode"] is True
assert len(result.data["results"]) > 0
def test_content_discovery_fails_gracefully_without_db(tmp_path):
result = route_gateway(
tmp_path, "content-discovery", "delirium", json_mode=True, limit=5
)
assert not result.ok
assert "route_explanation" in result.data
# ---------------------------------------------------------------------------
# paper-navigation intent
def test_paper_navigation_returns_not_found_without_db(tmp_path):
result = route_gateway(tmp_path, "paper-navigation", "ABCD1234", json_mode=True)
assert not result.ok
assert result.data["mode"] == "not_found"
# ---------------------------------------------------------------------------
# scoped-fetch intent
# ---------------------------------------------------------------------------
def test_scoped_fetch_returns_body_units_for_paper(tmp_path):
_make_populated_db(tmp_path)
result = route_gateway(tmp_path, "scoped-fetch", "Smith 2024", json_mode=True)
assert result.ok
assert result.data["intent"] == "scoped-fetch"
assert len(result.data["body_units"]) > 0
assert "manifest" in result.data
assert result.data["route_explanation"]["primary_arm"] == "body_units"
def test_scoped_fetch_by_doi(tmp_path):
_make_populated_db(tmp_path)
result = route_gateway(tmp_path, "scoped-fetch", "10.1234/delirium", json_mode=True)
assert result.ok
assert len(result.data["body_units"]) > 0
def test_scoped_fetch_empty_for_unknown_paper(tmp_path):
_make_populated_db(tmp_path)
result = route_gateway(
tmp_path, "scoped-fetch", "Unknown Author 1999", json_mode=True
)
assert not result.ok
assert len(result.data["body_units"]) == 0
# ---------------------------------------------------------------------------
# Command integration tests (exercise the CLI entry points)
# ---------------------------------------------------------------------------
def test_paper_lookup_command_registered_and_json(tmp_path):
@ -11,43 +373,74 @@ def test_paper_lookup_command_registered_and_json(tmp_path):
assert exit_code in {0, 1}
def test_content_discovery_warns_when_only_metadata_fts_exists(tmp_path, monkeypatch):
from paperforge.core.result import PFResult
from paperforge.retrieval import gateway
def fake_route_gateway(*args, **kwargs):
return PFResult(
ok=True,
command="content-discovery",
version="x",
data={"mode": "metadata_only"},
warnings=["body_units_fts missing"],
)
monkeypatch.setattr(gateway, "route_gateway", fake_route_gateway)
args = Namespace(vault_path=tmp_path, query="delirium prevention", json=True, limit=5)
assert content_discovery.run(args) == 0
def test_content_discovery_command_runs(tmp_path):
args = Namespace(
vault_path=tmp_path, query="delirium prevention", json=True, limit=5
)
exit_code = content_discovery.run(args)
assert exit_code in {0, 1}
def test_paper_navigation_runs(tmp_path):
args = Namespace(vault_path=tmp_path, query="10.1234/test-doi-here", json=True, limit=5)
def test_paper_navigation_command_runs(tmp_path):
args = Namespace(
vault_path=tmp_path, query="10.1234/test-doi-here", json=True
)
exit_code = paper_navigation.run(args)
assert exit_code in {0, 1}
def test_scoped_fetch_runs(tmp_path):
def test_scoped_fetch_command_runs(tmp_path):
args = Namespace(vault_path=tmp_path, query="Smith 2021", json=False, limit=3)
exit_code = scoped_fetch.run(args)
assert exit_code in {0, 1}
def test_content_discovery_non_json_output(tmp_path, monkeypatch):
from paperforge.core.result import PFResult
from paperforge.retrieval import gateway
# ---------------------------------------------------------------------------
# Helper unit tests
# ---------------------------------------------------------------------------
def fake_route_gateway(*args, **kwargs):
return PFResult(ok=True, command="content-discovery", version="x", data={"mode": "metadata_only"})
monkeypatch.setattr(gateway, "route_gateway", fake_route_gateway)
args = Namespace(vault_path=tmp_path, query="delirium prevention", json=False, limit=5)
assert content_discovery.run(args) == 0
def test_body_units_fts_exists_returns_true_when_populated(tmp_path):
_make_populated_db(tmp_path)
assert _body_units_fts_exists(tmp_path) is True
def test_body_units_fts_exists_returns_false_without_db(tmp_path):
assert _body_units_fts_exists(tmp_path) is False
def test_body_units_fts_exists_returns_false_no_body_units(tmp_path):
_make_paper_only_db(tmp_path)
assert _body_units_fts_exists(tmp_path) is False
def test_unsupported_intent_raises(tmp_path):
import pytest
with pytest.raises(ValueError, match="Unsupported Layer 4 intent"):
route_gateway(tmp_path, "unknown-intent", "query", json_mode=True)
# ---------------------------------------------------------------------------
# route_explanation contract
# ---------------------------------------------------------------------------
def test_all_intents_surface_route_explanation(tmp_path):
_make_populated_db(tmp_path)
_make_structure_tree(tmp_path, "TST001")
for intent, query in [
("paper-lookup", "Smith 2024"),
("content-discovery", "delirium"),
("paper-navigation", "10.1234/delirium"),
("scoped-fetch", "Smith 2024"),
]:
result = route_gateway(tmp_path, intent, query, json_mode=True)
# Even failing results should have route_explanation
assert "route_explanation" in result.data, (
f"{intent} missing route_explanation"
)
assert "primary_arm" in result.data["route_explanation"], (
f"{intent} missing primary_arm"
)