mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
refactor: v2.1 contract hardening — PFResult unification, ErrorCode expansion, adapter cleanup, SyncService orchestration
Core contract: - ErrorCode: 8→26 with _missing_() graceful degradation for unknown codes - PFResult: +warnings/next_actions, PFError: +suggestions - OcrStatus: 4→9 granular states (NONE/QUEUED/BLOCKED/NO_PDF/DONE_INCOMPLETE restored 1:1) - field_registry.yaml: +owner/deprecated/replacement/enum values/default Adapter cleanup: - new core/io.py (read_json/write_json), core/date_utils.py (extract_year) - new adapters/collections.py (build_collection_lookup) - adapters/bbt.py: cut worker dependency, _private→public+alias - worker/_utils.py + _domain.py: re-export from core instead of duplicate defs Command unification: - All 6 commands (sync/status/ocr/deep/repair/dashboard) → PFResult - cli.py --json dest unified to 'json' (was json_output on status/doctor) - commands/sync.py → SyncService (no more int/dict/PFResult mixing) Service hardening: - SyncService.run() orchestrates full sync lifecycle (select→index→clean) - cleanup loops (orphaned records, flat notes) migrated from worker - worker/sync.py: freeze line + services→worker one-way dependency - plugin/main.js: formal-library.json fallback deprecation warning Verification: 181/181 tests, 0 new lint errors
This commit is contained in:
parent
b52046d209
commit
05629bce82
22 changed files with 905 additions and 321 deletions
|
|
@ -3,10 +3,14 @@ from __future__ import annotations
|
|||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from paperforge.adapters.collections import build_collection_lookup
|
||||
from paperforge.core.date_utils import extract_year
|
||||
from paperforge.core.io import read_json
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _normalize_attachment_path(path: str, zotero_dir: Path | None = None) -> tuple[str, str, str]:
|
||||
def normalize_attachment_path(path: str, zotero_dir: Path | None = None) -> tuple[str, str, str]:
|
||||
"""Normalize a BBT attachment path to a consistent storage: format.
|
||||
|
||||
Handles three real-world BBT export formats:
|
||||
|
|
@ -42,7 +46,7 @@ def _normalize_attachment_path(path: str, zotero_dir: Path | None = None) -> tup
|
|||
|
||||
# Format 1: Absolute Windows path pointing to Zotero storage
|
||||
candidate = Path(raw)
|
||||
_looks_absolute = candidate.is_absolute() or (len(raw) >= 2 and raw[0].isalpha() and raw[1] == ':')
|
||||
_looks_absolute = candidate.is_absolute() or (len(raw) >= 2 and raw[0].isalpha() and raw[1] == ":")
|
||||
if _looks_absolute:
|
||||
norm_path = raw.replace("\\", "/")
|
||||
# Detect Zotero storage pattern: .../storage/8CHARKEY/...
|
||||
|
|
@ -63,7 +67,7 @@ def _normalize_attachment_path(path: str, zotero_dir: Path | None = None) -> tup
|
|||
return (f"storage:{norm}", bbt_path_raw, zotero_storage_key)
|
||||
|
||||
|
||||
def _identify_main_pdf(attachments: list[dict]) -> tuple[dict | None, list[dict]]:
|
||||
def identify_main_pdf(attachments: list[dict]) -> tuple[dict | None, list[dict]]:
|
||||
"""Identify the main PDF and supplementary materials from attachments.
|
||||
|
||||
Uses a hybrid three-priority strategy (Decision D-02):
|
||||
|
|
@ -151,9 +155,6 @@ def resolve_item_collection_paths(item: dict, collection_lookup: dict) -> list[s
|
|||
|
||||
|
||||
def load_export_rows(path: Path) -> list[dict]:
|
||||
from paperforge.worker._domain import build_collection_lookup
|
||||
from paperforge.worker._utils import _extract_year, read_json
|
||||
|
||||
data = read_json(path)
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
|
|
@ -168,7 +169,7 @@ def load_export_rows(path: Path) -> list[dict]:
|
|||
if not isinstance(attachment, dict):
|
||||
continue
|
||||
raw_path = attachment.get("path", "")
|
||||
normalized_path, bbt_path_raw, zotero_storage_key = _normalize_attachment_path(raw_path)
|
||||
normalized_path, bbt_path_raw, zotero_storage_key = normalize_attachment_path(raw_path)
|
||||
# Preserve contentType from BBT if present; fallback to file extension
|
||||
content_type = attachment.get("contentType", "")
|
||||
if not content_type and str(normalized_path).lower().endswith(".pdf"):
|
||||
|
|
@ -183,7 +184,7 @@ def load_export_rows(path: Path) -> list[dict]:
|
|||
"size": attachment.get("size", 0) or 0,
|
||||
}
|
||||
)
|
||||
main_pdf, supplementary_pdfs = _identify_main_pdf(attachments)
|
||||
main_pdf, supplementary_pdfs = identify_main_pdf(attachments)
|
||||
pdf_path = main_pdf["path"] if main_pdf else ""
|
||||
bbt_path_raw = main_pdf["bbt_path_raw"] if main_pdf else ""
|
||||
zotero_storage_key = main_pdf["zotero_storage_key"] if main_pdf else ""
|
||||
|
|
@ -199,7 +200,7 @@ def load_export_rows(path: Path) -> list[dict]:
|
|||
"abstract": item.get("abstractNote", ""),
|
||||
"journal": item.get("publicationTitle", ""),
|
||||
"extra": item.get("extra", ""),
|
||||
"year": _extract_year(item.get("date", "")),
|
||||
"year": extract_year(item.get("date", "")),
|
||||
"date": item.get("date", ""),
|
||||
"doi": item.get("DOI", ""),
|
||||
"pmid": item.get("PMID", ""),
|
||||
|
|
@ -215,3 +216,9 @@ def load_export_rows(path: Path) -> list[dict]:
|
|||
)
|
||||
return rows
|
||||
raise ValueError(f"Unsupported export format: {path}")
|
||||
|
||||
|
||||
# ── Backward-compat aliases (v2.1 → 2.2 migration) ──
|
||||
_normalize_attachment_path = normalize_attachment_path
|
||||
_identify_main_pdf = identify_main_pdf
|
||||
_extract_year = extract_year
|
||||
|
|
|
|||
30
paperforge/adapters/collections.py
Normal file
30
paperforge/adapters/collections.py
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
from __future__ import annotations
|
||||
|
||||
|
||||
def build_collection_lookup(collections: dict) -> dict:
|
||||
"""Build parent-resolved path cache from a Zotero collection tree.
|
||||
|
||||
Returns a dict with:
|
||||
path_by_key: {collection_key: "Parent/Sub/Name", ...}
|
||||
paths_by_item_id: {item_id: [collection_path, ...], ...}
|
||||
"""
|
||||
path_cache: dict[str, str] = {}
|
||||
item_paths: dict[str, list[str]] = {}
|
||||
|
||||
def path_for(key: str) -> str:
|
||||
if key in path_cache:
|
||||
return path_cache[key]
|
||||
node = collections.get(key, {})
|
||||
parent = node.get("parent") or ""
|
||||
name = node.get("name", "")
|
||||
parent_path = path_for(parent) if parent else ""
|
||||
full_path = f"{parent_path}/{name}" if parent_path else name
|
||||
path_cache[key] = full_path
|
||||
return full_path
|
||||
|
||||
for key, node in collections.items():
|
||||
full_path = path_for(key)
|
||||
for item_id in node.get("items", []):
|
||||
item_paths.setdefault(item_id, []).append(full_path)
|
||||
|
||||
return {"path_by_key": path_cache, "paths_by_item_id": item_paths}
|
||||
|
|
@ -152,7 +152,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
|
||||
# status
|
||||
status_p = sub.add_parser("status", help="Run the literature pipeline status check")
|
||||
status_p.add_argument("--json", action="store_true", dest="json_output", help="Output JSON")
|
||||
status_p.add_argument("--json", action="store_true", help="Output JSON")
|
||||
|
||||
# sync (new unified command)
|
||||
p_sync = sub.add_parser("sync", help="Sync Zotero selection and refresh literature index")
|
||||
|
|
@ -194,12 +194,14 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
sub.add_parser("index-refresh", help="Refresh formal literature notes from library records")
|
||||
|
||||
# deep-reading
|
||||
sub.add_parser("deep-reading", help="Check deep-reading queue status")
|
||||
p_dr = sub.add_parser("deep-reading", help="Check deep-reading queue status")
|
||||
p_dr.add_argument("--json", action="store_true", help="Output as PFResult JSON")
|
||||
|
||||
# repair
|
||||
p_repair = sub.add_parser("repair", help="Repair divergent literature notes")
|
||||
p_repair.add_argument("--fix", action="store_true", help="Actually apply repairs instead of dry-run")
|
||||
p_repair.add_argument("--fix-paths", action="store_true", help="Re-resolve PDF paths for items with path_error")
|
||||
p_repair.add_argument("--json", action="store_true", help="Output result as JSON (PFResult envelope)")
|
||||
|
||||
# ocr (unified)
|
||||
p_ocr = sub.add_parser("ocr", help="OCR operations")
|
||||
|
|
@ -262,7 +264,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
|
||||
# doctor
|
||||
doctor_p = sub.add_parser("doctor", help="Validate PaperForge setup and configuration")
|
||||
doctor_p.add_argument("--json", action="store_true", dest="json_output", help="Output JSON")
|
||||
doctor_p.add_argument("--json", action="store_true", help="Output JSON")
|
||||
|
||||
# update
|
||||
sub.add_parser("update", help="Update PaperForge to the latest version")
|
||||
|
|
@ -478,7 +480,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||
kw = {}
|
||||
if getattr(args, "verbose", False):
|
||||
kw["verbose"] = True
|
||||
if getattr(args, "json_output", False):
|
||||
if getattr(args, "json", False):
|
||||
kw["json_output"] = True
|
||||
return run_doctor(vault, **kw)
|
||||
|
||||
|
|
@ -511,11 +513,10 @@ def main(argv: list[str] | None = None) -> int:
|
|||
vault=vault,
|
||||
agent_key=args.agent,
|
||||
paddleocr_key=getattr(args, "paddleocr_key", None),
|
||||
paddleocr_url=getattr(args, "paddleocr_url",
|
||||
"https://paddleocr.aistudio-app.com/api/v2/ocr/jobs"),
|
||||
system_dir=getattr(args, "system_dir", None) or "System",
|
||||
resources_dir=getattr(args, "resources_dir", None) or "Resources",
|
||||
base_dir=getattr(args, "base_dir", None) or "Bases",
|
||||
paddleocr_url=getattr(args, "paddleocr_url", "https://paddleocr.aistudio-app.com/api/v2/ocr/jobs"),
|
||||
system_dir=getattr(args, "system_dir", None) or "System",
|
||||
resources_dir=getattr(args, "resources_dir", None) or "Resources",
|
||||
base_dir=getattr(args, "base_dir", None) or "Bases",
|
||||
zotero_data=getattr(args, "zotero_data", None),
|
||||
skip_checks=getattr(args, "skip_checks", False),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,9 @@ import argparse
|
|||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from paperforge import __version__
|
||||
from paperforge.core.result import PFResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -28,12 +31,47 @@ def _get_run_deep_reading():
|
|||
|
||||
|
||||
def run(args: argparse.Namespace) -> int:
|
||||
"""Run deep-reading queue check."""
|
||||
"""Run deep-reading queue check. Supports --json for PFResult output."""
|
||||
vault = getattr(args, "vault_path", None)
|
||||
if vault is None:
|
||||
from paperforge.config import resolve_vault
|
||||
|
||||
vault = resolve_vault(cli_vault=getattr(args, "vault", None))
|
||||
|
||||
json_output = getattr(args, "json", False)
|
||||
run_deep_reading = _get_run_deep_reading()
|
||||
return run_deep_reading(vault, verbose=getattr(args, "verbose", False))
|
||||
exit_code = run_deep_reading(vault, verbose=getattr(args, "verbose", False))
|
||||
|
||||
if json_output:
|
||||
from paperforge.worker._utils import get_analyze_queue
|
||||
|
||||
queue = get_analyze_queue(vault)
|
||||
ready = [r for r in queue if r.get("ocr_status") == "done"]
|
||||
blocked = [r for r in queue if r.get("ocr_status") != "done"]
|
||||
|
||||
pf = PFResult(
|
||||
ok=True,
|
||||
command="deep-reading",
|
||||
version=__version__,
|
||||
data={
|
||||
"queue": [
|
||||
{
|
||||
"zotero_key": r["zotero_key"],
|
||||
"domain": r["domain"],
|
||||
"title": r["title"],
|
||||
"ocr_status": r.get("ocr_status", "pending"),
|
||||
"ready": r.get("ocr_status") == "done",
|
||||
}
|
||||
for r in queue
|
||||
],
|
||||
"summary": {
|
||||
"total": len(queue),
|
||||
"ready": len(ready),
|
||||
"blocked": len(blocked),
|
||||
},
|
||||
},
|
||||
)
|
||||
print(pf.to_json())
|
||||
return 0
|
||||
|
||||
return exit_code
|
||||
|
|
|
|||
|
|
@ -63,8 +63,16 @@ def _diagnose(vault: Path, live: bool = False, json_output: bool = False) -> int
|
|||
queue_data = _collect_ocr_queue_data(vault)
|
||||
pf_error = None
|
||||
if not passed:
|
||||
if level == 1:
|
||||
ec = ErrorCode.OCR_TOKEN_MISSING
|
||||
elif level == 2:
|
||||
ec = ErrorCode.OCR_UPLOAD_FAILED
|
||||
elif level == 3:
|
||||
ec = ErrorCode.OCR_RESULT_INVALID
|
||||
else:
|
||||
ec = ErrorCode.INTERNAL_ERROR
|
||||
pf_error = PFError(
|
||||
code=ErrorCode.OCR_TOKEN_MISSING if level == 1 else ErrorCode.INTERNAL_ERROR,
|
||||
code=ec,
|
||||
message=result.get("error", "OCR diagnosis failed"),
|
||||
details={"level": level, "fix": result.get("fix", "")},
|
||||
)
|
||||
|
|
@ -124,6 +132,7 @@ def run(args: argparse.Namespace) -> int:
|
|||
Default behavior: run OCR queue.
|
||||
--diagnose: diagnose only (no upload).
|
||||
--key KEY: process specific item (passed through if supported).
|
||||
Supports --json for PFResult output in both diagnose and normal modes.
|
||||
"""
|
||||
vault = getattr(args, "vault_path", None)
|
||||
if vault is None:
|
||||
|
|
@ -147,6 +156,17 @@ def run(args: argparse.Namespace) -> int:
|
|||
run_ocr = _get_run_ocr()
|
||||
exit_code = run_ocr(vault, verbose=getattr(args, "verbose", False), no_progress=getattr(args, "no_progress", False))
|
||||
|
||||
if json_output:
|
||||
queue_data = _collect_ocr_queue_data(vault)
|
||||
pf = PFResult(
|
||||
ok=exit_code == 0,
|
||||
command="ocr",
|
||||
version=__version__,
|
||||
data=queue_data,
|
||||
)
|
||||
print(pf.to_json())
|
||||
return 0 if pf.ok else 1
|
||||
|
||||
# Auto-diagnose after successful run (new unified behavior)
|
||||
if exit_code == 0 and ocr_action is None and not diagnose_only and not key:
|
||||
logger.info("Running post-OCR diagnostic...")
|
||||
|
|
|
|||
|
|
@ -4,6 +4,10 @@ import argparse
|
|||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from paperforge import __version__
|
||||
from paperforge.core.errors import ErrorCode
|
||||
from paperforge.core.result import PFError, PFResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -28,7 +32,7 @@ def _get_run_repair():
|
|||
|
||||
|
||||
def run(args: argparse.Namespace) -> int:
|
||||
"""Run repair command."""
|
||||
"""Run repair command. Supports --json for PFResult output."""
|
||||
vault = getattr(args, "vault_path", None)
|
||||
paths = getattr(args, "paths", None)
|
||||
if vault is None:
|
||||
|
|
@ -43,6 +47,7 @@ def run(args: argparse.Namespace) -> int:
|
|||
paths = pipeline_paths(vault)
|
||||
|
||||
run_repair = _get_run_repair()
|
||||
json_output = getattr(args, "json", False)
|
||||
result = run_repair(
|
||||
vault,
|
||||
paths,
|
||||
|
|
@ -50,11 +55,43 @@ def run(args: argparse.Namespace) -> int:
|
|||
fix=getattr(args, "fix", False),
|
||||
fix_paths=getattr(args, "fix_paths", False),
|
||||
)
|
||||
# Report path_error summary from repair scan
|
||||
|
||||
# ── Build PFResult ──
|
||||
divergent = result.get("divergent", 0)
|
||||
path_errors = result.get("path_errors", {})
|
||||
path_error_total = path_errors.get("total", 0)
|
||||
has_issues = bool(divergent) or path_error_total > 0
|
||||
|
||||
pf_error = None
|
||||
if has_issues:
|
||||
pf_error = PFError(
|
||||
code=ErrorCode.VALIDATION_ERROR,
|
||||
message=f"Repair found {divergent} divergences and {path_error_total} path errors",
|
||||
details=result,
|
||||
)
|
||||
|
||||
pf = PFResult(
|
||||
ok=not has_issues,
|
||||
command="repair",
|
||||
version=__version__,
|
||||
data={
|
||||
"scanned": result.get("scanned", 0),
|
||||
"divergent": divergent,
|
||||
"fixed": result.get("fixed", 0),
|
||||
"errors": result.get("errors", []),
|
||||
"rebuilt": result.get("rebuilt", 0),
|
||||
"path_errors": result.get("path_errors", {}),
|
||||
},
|
||||
error=pf_error,
|
||||
)
|
||||
|
||||
if json_output:
|
||||
print(pf.to_json())
|
||||
return 0 if pf.ok else 1
|
||||
|
||||
# Human-readable output
|
||||
path_errors = result.get("path_errors", {})
|
||||
if path_errors.get("total", 0) > 0:
|
||||
error_summary = ", ".join(f"{count} {err}" for err, count in sorted(path_errors.get("by_type", {}).items()))
|
||||
print(f"repair: Found {path_errors['total']} items with path_error: {error_summary}")
|
||||
# Return non-zero if any divergences or path_errors remain
|
||||
has_issues = bool(result.get("divergent")) or path_errors.get("total", 0) > 0
|
||||
return 1 if has_issues else 0
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
"""Status command.
|
||||
|
||||
Reports vault statistics including library record counts, OCR progress,
|
||||
and path_error counts (displayed by run_status in literature_pipeline).
|
||||
and path_error counts. Uses PFResult contract for JSON output.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from paperforge.core.result import PFResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -32,7 +34,7 @@ def _get_run_status():
|
|||
|
||||
|
||||
def run(args: argparse.Namespace) -> int:
|
||||
"""Run status check."""
|
||||
"""Run status check. Worker handles PFResult JSON output when json_output=True."""
|
||||
vault = getattr(args, "vault_path", None)
|
||||
if vault is None:
|
||||
from paperforge.config import resolve_vault
|
||||
|
|
@ -40,4 +42,6 @@ def run(args: argparse.Namespace) -> int:
|
|||
vault = resolve_vault(cli_vault=getattr(args, "vault", None))
|
||||
|
||||
run_status = _get_run_status()
|
||||
return run_status(vault, verbose=getattr(args, "verbose", False), json_output=getattr(args, "json_output", False))
|
||||
exit_code = run_status(vault, verbose=getattr(args, "verbose", False), json_output=getattr(args, "json", False))
|
||||
# Worker already prints PFResult when json_output=True; propagate exit code
|
||||
return exit_code
|
||||
|
|
|
|||
|
|
@ -1,63 +1,21 @@
|
|||
"""Sync command — unifies selection-sync and index-refresh."""
|
||||
"""Sync command — unified sync through SyncService."""
|
||||
|
||||
import argparse
|
||||
import json as _json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from paperforge import __version__
|
||||
from paperforge.config import migrate_paperforge_json
|
||||
from paperforge.core.errors import ErrorCode
|
||||
from paperforge.core.result import PFError, PFResult
|
||||
from paperforge.core.result import PFResult
|
||||
from paperforge import __version__
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_run_selection_sync():
|
||||
"""Get run_selection_sync, preferring cli patches if available."""
|
||||
try:
|
||||
from paperforge.cli import run_selection_sync
|
||||
|
||||
if run_selection_sync is not None:
|
||||
return run_selection_sync
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
import sys
|
||||
|
||||
repo_root = Path(__file__).resolve().parent.parent.parent
|
||||
if str(repo_root) not in sys.path:
|
||||
sys.path.insert(0, str(repo_root))
|
||||
from paperforge.worker.sync import run_selection_sync
|
||||
|
||||
return run_selection_sync
|
||||
|
||||
|
||||
def _get_run_index_refresh():
|
||||
"""Get run_index_refresh, preferring cli patches if available."""
|
||||
try:
|
||||
from paperforge.cli import run_index_refresh
|
||||
|
||||
if run_index_refresh is not None:
|
||||
return run_index_refresh
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
import sys
|
||||
|
||||
repo_root = Path(__file__).resolve().parent.parent.parent
|
||||
if str(repo_root) not in sys.path:
|
||||
sys.path.insert(0, str(repo_root))
|
||||
from paperforge.worker.sync import run_index_refresh
|
||||
|
||||
return run_index_refresh
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> int:
|
||||
"""Run sync command.
|
||||
"""Run sync command through SyncService.
|
||||
|
||||
By default runs both selection-sync and index-refresh.
|
||||
By default runs both selection-sync and index-refresh + cleanup.
|
||||
Use --selection or --index to run only one phase.
|
||||
SyncService is the canonical entry point for all sync operations.
|
||||
"""
|
||||
vault = getattr(args, "vault_path", None)
|
||||
if vault is None:
|
||||
|
|
@ -66,7 +24,6 @@ def run(args: argparse.Namespace) -> int:
|
|||
vault = resolve_vault(cli_vault=getattr(args, "vault", None))
|
||||
|
||||
verbose = getattr(args, "verbose", False)
|
||||
# Auto-migrate paperforge.json from legacy top-level keys to vault_config block
|
||||
migrated = migrate_paperforge_json(vault)
|
||||
if migrated:
|
||||
logger.info("Migrated paperforge.json to vault_config canonical format")
|
||||
|
|
@ -76,8 +33,6 @@ def run(args: argparse.Namespace) -> int:
|
|||
dry_run = getattr(args, "dry_run", False)
|
||||
selection_only = getattr(args, "selection", False)
|
||||
index_only = getattr(args, "index", False)
|
||||
rebuild_index = getattr(args, "rebuild_index", False)
|
||||
domain = getattr(args, "domain", None)
|
||||
json_output = getattr(args, "json", False)
|
||||
|
||||
if dry_run:
|
||||
|
|
@ -99,44 +54,15 @@ def run(args: argparse.Namespace) -> int:
|
|||
print(" - selection-sync")
|
||||
if index_only:
|
||||
print(" - index-refresh")
|
||||
if domain:
|
||||
print(f" Filtered by domain: {domain}")
|
||||
return 0
|
||||
|
||||
exit_code = 0
|
||||
sync_counts = {"new": 0, "updated": 0, "skipped": 0, "failed": 0, "errors": []}
|
||||
from paperforge.services.sync_service import SyncService
|
||||
|
||||
if not index_only:
|
||||
run_selection_sync = _get_run_selection_sync()
|
||||
result = run_selection_sync(vault, verbose=getattr(args, "verbose", False), json_output=json_output)
|
||||
if json_output:
|
||||
sync_counts = result
|
||||
elif result != 0:
|
||||
exit_code = result
|
||||
|
||||
if not selection_only:
|
||||
run_index_refresh = _get_run_index_refresh()
|
||||
code = run_index_refresh(vault, verbose=getattr(args, "verbose", False), rebuild_index=rebuild_index, json_output=json_output)
|
||||
if code != 0 and exit_code == 0:
|
||||
exit_code = code
|
||||
svc = SyncService(vault)
|
||||
result = svc.run(verbose=verbose, json_output=json_output)
|
||||
|
||||
if json_output:
|
||||
errors = sync_counts.get("errors", [])
|
||||
pf_error = None
|
||||
if errors or exit_code != 0:
|
||||
pf_error = PFError(
|
||||
code=ErrorCode.SYNC_FAILED,
|
||||
message=f"Sync completed with {len(errors)} error(s)",
|
||||
details={"errors": errors, "exit_code": exit_code},
|
||||
)
|
||||
result = PFResult(
|
||||
ok=exit_code == 0 and not errors,
|
||||
command="sync",
|
||||
version=__version__,
|
||||
data=sync_counts,
|
||||
error=pf_error,
|
||||
)
|
||||
print(result.to_json())
|
||||
return 0
|
||||
|
||||
return exit_code
|
||||
return 0 if result.ok else 1
|
||||
|
|
|
|||
17
paperforge/core/date_utils.py
Normal file
17
paperforge/core/date_utils.py
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def extract_year(value: str) -> str:
|
||||
"""Extract a 4-digit year (19xx or 20xx) from a date string.
|
||||
|
||||
Returns empty string if no year found.
|
||||
|
||||
Examples:
|
||||
extract_year("2024-03-15") -> "2024"
|
||||
extract_year("March 2023") -> "2023"
|
||||
extract_year("n.d.") -> ""
|
||||
"""
|
||||
match = re.search(r"(19|20)\d{2}", value or "")
|
||||
return match.group(0) if match else ""
|
||||
|
|
@ -4,16 +4,57 @@ from enum import Enum
|
|||
|
||||
|
||||
class ErrorCode(str, Enum):
|
||||
"""Centralized error codes for PFResult contracts."""
|
||||
"""Centralized error codes for PFResult contracts.
|
||||
|
||||
Codes are grouped by subsystem. When adding a new code, prefer a
|
||||
narrow, actionable code over a catch-all so that plugin UI can
|
||||
generate targeted fix-it guidance.
|
||||
"""
|
||||
|
||||
# ── Runtime ──
|
||||
PYTHON_NOT_FOUND = "PYTHON_NOT_FOUND"
|
||||
PYTHON_VERSION_TOO_OLD = "PYTHON_VERSION_TOO_OLD"
|
||||
PAPERFORGE_NOT_INSTALLED = "PAPERFORGE_NOT_INSTALLED"
|
||||
VERSION_MISMATCH = "VERSION_MISMATCH"
|
||||
|
||||
# ── Config / Vault ──
|
||||
VAULT_NOT_FOUND = "VAULT_NOT_FOUND"
|
||||
CONFIG_NOT_FOUND = "CONFIG_NOT_FOUND"
|
||||
CONFIG_INVALID = "CONFIG_INVALID"
|
||||
PATH_NOT_FOUND = "PATH_NOT_FOUND"
|
||||
|
||||
# ── BBT / Zotero ──
|
||||
BBT_EXPORT_NOT_FOUND = "BBT_EXPORT_NOT_FOUND"
|
||||
BBT_EXPORT_INVALID = "BBT_EXPORT_INVALID"
|
||||
BBT_CITATION_KEY_MISSING = "BBT_CITATION_KEY_MISSING"
|
||||
ZOTERO_DATA_NOT_FOUND = "ZOTERO_DATA_NOT_FOUND"
|
||||
PDF_PATH_UNRESOLVED = "PDF_PATH_UNRESOLVED"
|
||||
|
||||
# ── OCR ──
|
||||
OCR_TOKEN_MISSING = "OCR_TOKEN_MISSING"
|
||||
OCR_UPLOAD_FAILED = "OCR_UPLOAD_FAILED"
|
||||
OCR_POLL_TIMEOUT = "OCR_POLL_TIMEOUT"
|
||||
OCR_RESULT_INVALID = "OCR_RESULT_INVALID"
|
||||
|
||||
# ── Sync ──
|
||||
SYNC_FAILED = "SYNC_FAILED"
|
||||
CANDIDATE_BUILD_FAILED = "CANDIDATE_BUILD_FAILED"
|
||||
NOTE_WRITE_FAILED = "NOTE_WRITE_FAILED"
|
||||
|
||||
# ── Schema ──
|
||||
FIELD_MISSING = "FIELD_MISSING"
|
||||
FIELD_TYPE_INVALID = "FIELD_TYPE_INVALID"
|
||||
INDEX_SCHEMA_INVALID = "INDEX_SCHEMA_INVALID"
|
||||
|
||||
# ── Generic ──
|
||||
VALIDATION_ERROR = "VALIDATION_ERROR"
|
||||
INTERNAL_ERROR = "INTERNAL_ERROR"
|
||||
UNKNOWN = "UNKNOWN"
|
||||
|
||||
@classmethod
|
||||
def _missing_(cls, value):
|
||||
"""Gracefully handle unknown codes from newer plugin versions."""
|
||||
return cls.UNKNOWN
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.value
|
||||
|
|
|
|||
15
paperforge/core/io.py
Normal file
15
paperforge/core/io.py
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def read_json(path: Path):
|
||||
"""Read and parse a JSON file."""
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def write_json(path: Path, data) -> None:
|
||||
"""Write data as JSON, creating parent directories as needed."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
|
@ -12,6 +12,7 @@ class PFError:
|
|||
code: ErrorCode
|
||||
message: str
|
||||
details: dict = field(default_factory=dict)
|
||||
suggestions: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -21,6 +22,8 @@ class PFResult:
|
|||
version: str
|
||||
data: Any = None
|
||||
error: PFError | None = None
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
next_actions: list[dict] = field(default_factory=list)
|
||||
|
||||
def __bool__(self) -> bool:
|
||||
return self.ok
|
||||
|
|
@ -41,8 +44,14 @@ class PFResult:
|
|||
"message": self.error.message,
|
||||
"details": self.error.details,
|
||||
}
|
||||
if self.error.suggestions:
|
||||
raw["error"]["suggestions"] = self.error.suggestions
|
||||
else:
|
||||
raw["error"] = None
|
||||
if self.warnings:
|
||||
raw["warnings"] = self.warnings
|
||||
if self.next_actions:
|
||||
raw["next_actions"] = self.next_actions
|
||||
return raw
|
||||
|
||||
def to_json(self) -> str:
|
||||
|
|
@ -57,6 +66,7 @@ class PFResult:
|
|||
code=ErrorCode(err_data["code"]),
|
||||
message=err_data["message"],
|
||||
details=err_data.get("details", {}),
|
||||
suggestions=err_data.get("suggestions", []),
|
||||
)
|
||||
return cls(
|
||||
ok=data["ok"],
|
||||
|
|
@ -64,4 +74,6 @@ class PFResult:
|
|||
version=data["version"],
|
||||
data=data.get("data"),
|
||||
error=error,
|
||||
warnings=data.get("warnings", []),
|
||||
next_actions=data.get("next_actions", []),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,26 +7,45 @@ from typing import ClassVar
|
|||
|
||||
|
||||
class OcrStatus(str, Enum):
|
||||
"""OCR processing state for a paper."""
|
||||
"""OCR processing state for a paper.
|
||||
|
||||
States are granular so that plugin UI can display targeted next actions:
|
||||
none – OCR not yet initiated
|
||||
pending – do_ocr=true, not yet queued
|
||||
queued – in queue, waiting for worker
|
||||
processing – currently being OCR'd by PaddleOCR
|
||||
done – full OCR completed successfully
|
||||
done_incomplete – OCR completed but some pages/images missing
|
||||
failed – API returned error or task failed
|
||||
blocked – can't proceed (missing PDF, invalid key, etc.)
|
||||
nopdf – no PDF file available to OCR
|
||||
"""
|
||||
|
||||
NONE = "none"
|
||||
PENDING = "pending"
|
||||
QUEUED = "queued"
|
||||
PROCESSING = "processing"
|
||||
DONE = "done"
|
||||
DONE_INCOMPLETE = "done_incomplete"
|
||||
FAILED = "failed"
|
||||
BLOCKED = "blocked"
|
||||
NO_PDF = "nopdf"
|
||||
|
||||
@classmethod
|
||||
def from_legacy(cls, value: str) -> OcrStatus:
|
||||
"""Map legacy OCR state strings to canonical enum."""
|
||||
"""Map legacy OCR state strings to canonical enum (1:1, no compression)."""
|
||||
mapping = {
|
||||
"none": cls.NONE,
|
||||
"pending": cls.PENDING,
|
||||
"processing": cls.PROCESSING,
|
||||
"queued": cls.PENDING,
|
||||
"queued": cls.QUEUED,
|
||||
"running": cls.PROCESSING,
|
||||
"processing": cls.PROCESSING,
|
||||
"done": cls.DONE,
|
||||
"done_incomplete": cls.DONE_INCOMPLETE,
|
||||
"failed": cls.FAILED,
|
||||
"error": cls.FAILED,
|
||||
"blocked": cls.FAILED,
|
||||
"nopdf": cls.FAILED,
|
||||
"done_incomplete": cls.DONE,
|
||||
"blocked": cls.BLOCKED,
|
||||
"nopdf": cls.NO_PDF,
|
||||
}
|
||||
return mapping.get(value.strip().lower(), cls.PENDING)
|
||||
|
||||
|
|
@ -36,6 +55,7 @@ class OcrStatus(str, Enum):
|
|||
|
||||
class PdfStatus(str, Enum):
|
||||
"""PDF attachment health state."""
|
||||
|
||||
HEALTHY = "healthy"
|
||||
BROKEN = "broken"
|
||||
MISSING = "missing"
|
||||
|
|
@ -46,6 +66,7 @@ class PdfStatus(str, Enum):
|
|||
|
||||
class Lifecycle(str, Enum):
|
||||
"""Derived lifecycle stage summarizing the paper's overall progress."""
|
||||
|
||||
PDF_READY = "pdf_ready"
|
||||
OCR_READY = "ocr_ready"
|
||||
ANALYZE_READY = "analyze_ready"
|
||||
|
|
@ -63,10 +84,15 @@ class ALLOWED_TRANSITIONS:
|
|||
"""
|
||||
|
||||
OCR_STATUS: ClassVar[dict[OcrStatus, list[OcrStatus]]] = {
|
||||
OcrStatus.PENDING: [OcrStatus.PROCESSING, OcrStatus.FAILED],
|
||||
OcrStatus.PROCESSING: [OcrStatus.DONE, OcrStatus.FAILED],
|
||||
OcrStatus.NONE: [OcrStatus.PENDING, OcrStatus.QUEUED, OcrStatus.NO_PDF],
|
||||
OcrStatus.PENDING: [OcrStatus.QUEUED, OcrStatus.PROCESSING, OcrStatus.NO_PDF, OcrStatus.BLOCKED],
|
||||
OcrStatus.QUEUED: [OcrStatus.PROCESSING, OcrStatus.FAILED],
|
||||
OcrStatus.PROCESSING: [OcrStatus.DONE, OcrStatus.DONE_INCOMPLETE, OcrStatus.FAILED],
|
||||
OcrStatus.DONE: [OcrStatus.PENDING], # re-run
|
||||
OcrStatus.DONE_INCOMPLETE: [OcrStatus.PENDING], # re-run full or retry partial
|
||||
OcrStatus.FAILED: [OcrStatus.PENDING], # retry
|
||||
OcrStatus.BLOCKED: [OcrStatus.PENDING], # fix preconditions then retry
|
||||
OcrStatus.NO_PDF: [OcrStatus.PENDING], # add PDF then retry
|
||||
}
|
||||
|
||||
DEEP_READING_STATUS: ClassVar[dict[str, list[str]]] = {
|
||||
|
|
|
|||
|
|
@ -386,6 +386,11 @@ class PaperForgeStatusView extends ItemView {
|
|||
}
|
||||
|
||||
_fallbackFetchStats(quiet, vp, plugin) {
|
||||
if (!this._fallbackWarned) {
|
||||
console.warn('[PaperForge] formal-library.json fallback is deprecated. ' +
|
||||
'Ensure `paperforge status --json` is available for accurate dashboard data.');
|
||||
this._fallbackWarned = true;
|
||||
}
|
||||
const systemDir = plugin?.settings?.system_dir || 'System';
|
||||
const indexPath = path.join(vp, systemDir, 'PaperForge', 'indexes', 'formal-library.json');
|
||||
|
||||
|
|
@ -494,6 +499,11 @@ class PaperForgeStatusView extends ItemView {
|
|||
|
||||
/* ── Index Loading (D-11, D-17, D-19) ── */
|
||||
_loadIndex() {
|
||||
if (!this._fallbackWarned) {
|
||||
console.warn('[PaperForge] formal-library.json fallback is deprecated. ' +
|
||||
'Ensure `paperforge status --json` is available for accurate dashboard data.');
|
||||
this._fallbackWarned = true;
|
||||
}
|
||||
const vp = this.app.vault.adapter.basePath;
|
||||
const plugin = this.app.plugins.plugins['paperforge'];
|
||||
const systemDir = plugin?.settings?.system_dir || 'System';
|
||||
|
|
|
|||
|
|
@ -4,81 +4,134 @@ frontmatter:
|
|||
required: true
|
||||
public: true
|
||||
description: "Zotero citation key"
|
||||
owner: sync
|
||||
introduced_in: "1.0"
|
||||
domain:
|
||||
type: str
|
||||
required: true
|
||||
public: true
|
||||
description: "Classification domain"
|
||||
owner: sync
|
||||
introduced_in: "1.0"
|
||||
title:
|
||||
type: str
|
||||
required: true
|
||||
public: true
|
||||
description: "Paper title"
|
||||
owner: sync
|
||||
introduced_in: "1.0"
|
||||
year:
|
||||
type: str
|
||||
required: false
|
||||
public: true
|
||||
description: "Publication year"
|
||||
owner: sync
|
||||
introduced_in: "1.0"
|
||||
doi:
|
||||
type: str
|
||||
required: false
|
||||
public: true
|
||||
description: "Digital Object Identifier"
|
||||
owner: sync
|
||||
introduced_in: "1.0"
|
||||
collection_path:
|
||||
type: str
|
||||
required: false
|
||||
public: true
|
||||
description: "Zotero collection path"
|
||||
owner: sync
|
||||
introduced_in: "1.0"
|
||||
has_pdf:
|
||||
type: bool
|
||||
required: true
|
||||
public: true
|
||||
description: "Has PDF attachment"
|
||||
description: "Has PDF attachment (DEPRECATED)"
|
||||
owner: sync
|
||||
deprecated: true
|
||||
replacement: lifecycle
|
||||
introduced_in: "1.0"
|
||||
pdf_path:
|
||||
type: str
|
||||
required: false
|
||||
public: true
|
||||
description: "Wikilink to main PDF"
|
||||
owner: sync
|
||||
introduced_in: "1.0"
|
||||
supplementary:
|
||||
type: list
|
||||
required: false
|
||||
public: true
|
||||
description: "Wikilinks to supplementary PDFs"
|
||||
owner: sync
|
||||
introduced_in: "1.0"
|
||||
fulltext_md_path:
|
||||
type: str
|
||||
required: false
|
||||
public: true
|
||||
description: "Wikilink to OCR fulltext"
|
||||
owner: ocr
|
||||
introduced_in: "1.0"
|
||||
recommend_analyze:
|
||||
type: bool
|
||||
required: false
|
||||
public: true
|
||||
description: "System recommends analysis"
|
||||
description: "System recommends analysis (DEPRECATED)"
|
||||
owner: sync
|
||||
deprecated: true
|
||||
replacement: lifecycle
|
||||
introduced_in: "1.0"
|
||||
analyze:
|
||||
type: bool
|
||||
required: false
|
||||
public: true
|
||||
description: "User requests deep reading"
|
||||
owner: user
|
||||
introduced_in: "1.0"
|
||||
do_ocr:
|
||||
type: bool
|
||||
required: false
|
||||
public: true
|
||||
description: "User requests OCR"
|
||||
description: "User requests OCR (DEPRECATED)"
|
||||
owner: user
|
||||
deprecated: true
|
||||
replacement: ocr_status
|
||||
introduced_in: "1.0"
|
||||
ocr_status:
|
||||
type: str
|
||||
type: enum
|
||||
values:
|
||||
- none
|
||||
- pending
|
||||
- queued
|
||||
- processing
|
||||
- done
|
||||
- done_incomplete
|
||||
- failed
|
||||
- blocked
|
||||
- nopdf
|
||||
required: false
|
||||
public: true
|
||||
description: "OCR processing status"
|
||||
owner: ocr
|
||||
default: "none"
|
||||
introduced_in: "2.1"
|
||||
deep_reading_status:
|
||||
type: str
|
||||
type: enum
|
||||
values:
|
||||
- pending
|
||||
- done
|
||||
required: false
|
||||
public: true
|
||||
description: "Deep reading status"
|
||||
owner: deep_reading
|
||||
default: "pending"
|
||||
introduced_in: "1.0"
|
||||
path_error:
|
||||
type: str
|
||||
required: false
|
||||
public: true
|
||||
description: "Path resolution error"
|
||||
owner: sync
|
||||
introduced_in: "1.0"
|
||||
|
||||
index_entry:
|
||||
file:
|
||||
|
|
@ -86,130 +139,217 @@ index_entry:
|
|||
required: true
|
||||
public: true
|
||||
description: "Path to formal note"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
zotero_key:
|
||||
type: str
|
||||
required: true
|
||||
public: true
|
||||
description: "Zotero citation key"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
domain:
|
||||
type: str
|
||||
required: true
|
||||
public: true
|
||||
description: "Classification domain"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
title:
|
||||
type: str
|
||||
required: true
|
||||
public: true
|
||||
description: "Paper title"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
year:
|
||||
type: str
|
||||
required: false
|
||||
public: true
|
||||
description: "Publication year"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
doi:
|
||||
type: str
|
||||
required: false
|
||||
public: true
|
||||
description: "DOI"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
collection_path:
|
||||
type: str
|
||||
required: false
|
||||
public: true
|
||||
description: "Zotero collection path"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
has_pdf:
|
||||
type: bool
|
||||
required: true
|
||||
public: true
|
||||
description: "Has PDF"
|
||||
description: "Has PDF (DEPRECATED)"
|
||||
owner: index
|
||||
deprecated: true
|
||||
replacement: lifecycle
|
||||
introduced_in: "2.1"
|
||||
pdf_path:
|
||||
type: str
|
||||
required: false
|
||||
public: true
|
||||
description: "PDF path"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
supplementary:
|
||||
type: list
|
||||
required: false
|
||||
public: true
|
||||
description: "Supplementary PDFs"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
fulltext_md_path:
|
||||
type: str
|
||||
required: false
|
||||
public: true
|
||||
description: "Fulltext path"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
recommend_analyze:
|
||||
type: bool
|
||||
required: false
|
||||
public: true
|
||||
description: "Recommend analysis"
|
||||
description: "Recommend analysis (DEPRECATED)"
|
||||
owner: index
|
||||
deprecated: true
|
||||
replacement: lifecycle
|
||||
introduced_in: "2.1"
|
||||
analyze:
|
||||
type: bool
|
||||
required: false
|
||||
public: true
|
||||
description: "Analysis flag"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
do_ocr:
|
||||
type: bool
|
||||
required: false
|
||||
public: true
|
||||
description: "OCR flag"
|
||||
description: "OCR flag (DEPRECATED)"
|
||||
owner: index
|
||||
deprecated: true
|
||||
replacement: ocr_status
|
||||
introduced_in: "2.1"
|
||||
ocr_status:
|
||||
type: str
|
||||
type: enum
|
||||
values:
|
||||
- none
|
||||
- pending
|
||||
- queued
|
||||
- processing
|
||||
- done
|
||||
- done_incomplete
|
||||
- failed
|
||||
- blocked
|
||||
- nopdf
|
||||
required: false
|
||||
public: true
|
||||
description: "OCR status"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
deep_reading_status:
|
||||
type: str
|
||||
type: enum
|
||||
values:
|
||||
- pending
|
||||
- done
|
||||
required: false
|
||||
public: true
|
||||
description: "Deep reading status"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
path_error:
|
||||
type: str
|
||||
required: false
|
||||
public: true
|
||||
description: "Path error"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
lifecycle:
|
||||
type: str
|
||||
type: enum
|
||||
values:
|
||||
- pdf_ready
|
||||
- ocr_ready
|
||||
- analyze_ready
|
||||
- deep_read_done
|
||||
- error_state
|
||||
required: false
|
||||
public: true
|
||||
description: "Derived lifecycle stage"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
health:
|
||||
type: str
|
||||
type: dict
|
||||
required: false
|
||||
public: true
|
||||
description: "Derived health dimension scores"
|
||||
description: "Derived health dimension scores (dict: pdf_health, ocr_health, note_health, asset_health)"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
maturity:
|
||||
type: str
|
||||
required: false
|
||||
public: true
|
||||
description: "Derived maturity score"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
next_step:
|
||||
type: str
|
||||
required: false
|
||||
public: true
|
||||
description: "Recommended next action"
|
||||
owner: index
|
||||
introduced_in: "2.1"
|
||||
|
||||
ocr_meta:
|
||||
ocr_status:
|
||||
type: str
|
||||
type: enum
|
||||
values:
|
||||
- none
|
||||
- pending
|
||||
- queued
|
||||
- processing
|
||||
- done
|
||||
- done_incomplete
|
||||
- failed
|
||||
- blocked
|
||||
- nopdf
|
||||
required: true
|
||||
public: true
|
||||
description: "OCR processing status"
|
||||
owner: ocr
|
||||
introduced_in: "2.1"
|
||||
error:
|
||||
type: str
|
||||
required: false
|
||||
public: true
|
||||
description: "Error message if failed"
|
||||
owner: ocr
|
||||
introduced_in: "1.0"
|
||||
start_time:
|
||||
type: str
|
||||
required: false
|
||||
public: false
|
||||
description: "OCR start timestamp"
|
||||
owner: ocr
|
||||
introduced_in: "1.0"
|
||||
completion_time:
|
||||
type: str
|
||||
required: false
|
||||
public: false
|
||||
description: "OCR completion timestamp"
|
||||
owner: ocr
|
||||
introduced_in: "1.0"
|
||||
page_count:
|
||||
type: int
|
||||
required: false
|
||||
public: false
|
||||
description: "Number of pages processed"
|
||||
owner: ocr
|
||||
introduced_in: "1.0"
|
||||
|
|
|
|||
|
|
@ -1,14 +1,15 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from paperforge import __version__
|
||||
from paperforge.adapters.bbt import (
|
||||
extract_authors,
|
||||
_identify_main_pdf,
|
||||
_normalize_attachment_path,
|
||||
identify_main_pdf,
|
||||
load_export_rows,
|
||||
normalize_attachment_path,
|
||||
resolve_item_collection_paths,
|
||||
)
|
||||
from paperforge.adapters.obsidian_frontmatter import (
|
||||
|
|
@ -21,12 +22,20 @@ from paperforge.adapters.zotero_paths import (
|
|||
obsidian_wikilink_for_path,
|
||||
obsidian_wikilink_for_pdf,
|
||||
)
|
||||
from paperforge.core.errors import ErrorCode
|
||||
from paperforge.core.result import PFError, PFResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SyncService:
|
||||
"""Orchestrates the sync lifecycle: load BBT exports, match entries, generate notes."""
|
||||
"""Orchestrates the sync lifecycle: load BBT exports, match entries, generate notes.
|
||||
|
||||
v2.1: This service currently delegates heavy lifting to worker/sync.py.
|
||||
The load_exports / build_candidates methods are functional; run() wraps
|
||||
the legacy worker and enforces the PFResult contract. Full migration
|
||||
of note-writing logic is planned for v2.2.
|
||||
"""
|
||||
|
||||
def __init__(self, vault: Path):
|
||||
self.vault = vault
|
||||
|
|
@ -34,15 +43,17 @@ class SyncService:
|
|||
self._resolved = False
|
||||
|
||||
def resolve_paths(self) -> dict[str, Path]:
|
||||
"""Resolve vault paths using pipeline_paths."""
|
||||
from paperforge.worker._utils import pipeline_paths
|
||||
self.paths = pipeline_paths(self.vault)
|
||||
"""Resolve vault paths using paperforge_paths."""
|
||||
from paperforge.config import paperforge_paths
|
||||
|
||||
self.paths = paperforge_paths(self.vault)
|
||||
self._resolved = True
|
||||
return self.paths
|
||||
|
||||
def load_exports(self) -> list[dict]:
|
||||
"""Load all BBT export JSON files."""
|
||||
from paperforge.config import load_vault_config
|
||||
|
||||
cfg = load_vault_config(self.vault)
|
||||
system_dir = self.vault / cfg.get("system_dir", "99_System")
|
||||
exports_dir = system_dir / "PaperForge" / "exports"
|
||||
|
|
@ -58,6 +69,135 @@ class SyncService:
|
|||
logger.error("Failed to load %s: %s", f.name, e)
|
||||
return rows
|
||||
|
||||
def clean_orphaned_records(
|
||||
self, exports: dict[str, dict[str, dict]], paths: dict[str, Path], json_output: bool = False
|
||||
) -> int:
|
||||
"""Remove formal notes whose title duplicates an exported item but lacks PDF.
|
||||
|
||||
Migrated from worker/sync.py run_index_refresh (v2.2).
|
||||
Returns number of orphaned records deleted.
|
||||
"""
|
||||
from paperforge.config import paperforge_paths
|
||||
|
||||
_paths = paths if paths else paperforge_paths(self.vault)
|
||||
lit_dir = _paths.get("literature")
|
||||
if not lit_dir or not lit_dir.exists():
|
||||
return 0
|
||||
|
||||
total_deleted = 0
|
||||
for domain_dir in lit_dir.iterdir():
|
||||
if not domain_dir.is_dir():
|
||||
continue
|
||||
domain = domain_dir.name
|
||||
domain_export_keys = set(exports.get(domain, {}).keys())
|
||||
records_by_title: dict[str, list[str]] = {}
|
||||
records_info: dict[str, dict] = {}
|
||||
|
||||
for record_file in domain_dir.rglob("*.md"):
|
||||
if record_file.name in ("fulltext.md", "deep-reading.md", "discussion.md"):
|
||||
continue
|
||||
try:
|
||||
content = record_file.read_text(encoding="utf-8")
|
||||
key_match = re.search(r"^zotero_key:\s*(.+)$", content, re.MULTILINE)
|
||||
if not key_match:
|
||||
continue
|
||||
key = key_match.group(1).strip()
|
||||
title_match = re.search(r"^title:\s*[\"']?(.+)[\"']?\s*$", content, re.MULTILINE)
|
||||
title = title_match.group(1) if title_match else ""
|
||||
has_pdf = "has_pdf: true" in content
|
||||
normalized = re.sub(r"[^a-z0-9]", "", title.lower())[:20]
|
||||
records_info[key] = {
|
||||
"file": record_file,
|
||||
"title": title,
|
||||
"has_pdf": has_pdf,
|
||||
"normalized": normalized,
|
||||
}
|
||||
if normalized not in records_by_title:
|
||||
records_by_title[normalized] = []
|
||||
records_by_title[normalized].append(key)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
to_delete: list[str] = []
|
||||
for normalized, keys in records_by_title.items():
|
||||
keys_in_export = [k for k in keys if k in domain_export_keys]
|
||||
keys_not_in_export = [k for k in keys if k not in domain_export_keys]
|
||||
if keys_in_export and keys_not_in_export:
|
||||
for k in keys_not_in_export:
|
||||
if not records_info[k]["has_pdf"]:
|
||||
to_delete.append(k)
|
||||
|
||||
deleted = 0
|
||||
for key in to_delete:
|
||||
try:
|
||||
records_info[key]["file"].unlink()
|
||||
deleted += 1
|
||||
except Exception:
|
||||
pass
|
||||
if deleted > 0 and not json_output:
|
||||
print(f"index-refresh: cleaned {deleted} orphaned records in {domain}")
|
||||
total_deleted += deleted
|
||||
|
||||
return total_deleted
|
||||
|
||||
def clean_flat_notes(self, paths: dict[str, Path], json_output: bool = False) -> int:
|
||||
"""Delete flat formal notes whose workspace equivalent exists.
|
||||
|
||||
Migrated from worker/sync.py run_index_refresh (v2.2).
|
||||
Returns number of flat notes deleted.
|
||||
"""
|
||||
from paperforge.config import paperforge_paths
|
||||
from paperforge.worker._utils import slugify_filename
|
||||
|
||||
try:
|
||||
from paperforge.worker.asset_index import read_index as _read_idx
|
||||
|
||||
index_data = _read_idx(self.vault)
|
||||
except Exception:
|
||||
index_data = None
|
||||
|
||||
_paths = paths if paths else paperforge_paths(self.vault)
|
||||
lit_dir = _paths.get("literature")
|
||||
if not lit_dir or not lit_dir.exists():
|
||||
return 0
|
||||
|
||||
ws_keys: set[str] = set()
|
||||
if isinstance(index_data, dict):
|
||||
for item in index_data.get("items", []):
|
||||
ws_dir = (
|
||||
lit_dir
|
||||
/ item.get("domain", "")
|
||||
/ (item.get("zotero_key", "") + " - " + slugify_filename(item.get("title", "")))
|
||||
)
|
||||
if ws_dir.is_dir():
|
||||
ws_keys.add(item.get("zotero_key"))
|
||||
|
||||
if not ws_keys:
|
||||
return 0
|
||||
|
||||
cleaned = 0
|
||||
for domain_dir in sorted(lit_dir.iterdir()):
|
||||
if not domain_dir.is_dir():
|
||||
continue
|
||||
for flat_note in list(domain_dir.glob("*.md")):
|
||||
try:
|
||||
text = flat_note.read_text(encoding="utf-8")
|
||||
m = re.search(r"^zotero_key:\s*\"?(\S+?)\"?\s*$", text, re.MULTILINE)
|
||||
key = m.group(1) if m else ""
|
||||
except Exception:
|
||||
continue
|
||||
if key and key in ws_keys:
|
||||
try:
|
||||
flat_note.unlink()
|
||||
cleaned += 1
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if cleaned > 0 and not json_output:
|
||||
print(f"index-refresh: cleaned {cleaned} flat note(s) (migrated to workspace)")
|
||||
|
||||
return cleaned
|
||||
|
||||
def build_candidates(self, rows: list[dict]) -> list[dict]:
|
||||
"""Generate candidate markdown entries from BBT rows."""
|
||||
candidates = []
|
||||
|
|
@ -69,10 +209,97 @@ class SyncService:
|
|||
logger.error("Failed to build candidate: %s", e)
|
||||
return candidates
|
||||
|
||||
def run_sync(self, verbose: bool = False, json_output: bool = False) -> int | dict:
|
||||
"""Full sync orchestration. Delegates to run_selection_sync logic.
|
||||
def run(self, verbose: bool = False, json_output: bool = False) -> PFResult:
|
||||
"""Full sync orchestration. Returns PFResult contract.
|
||||
|
||||
Returns int (exit code) for CLI mode, dict for json_output mode.
|
||||
v2.2: Service now orchestrates the full sync lifecycle:
|
||||
1. Load BBT exports
|
||||
2. Selection sync (count items from worker)
|
||||
3. Build canonical index (asset_index)
|
||||
4. Clean orphaned records + flat notes
|
||||
5. Return PFResult
|
||||
"""
|
||||
# ── Phase 1: Select ──
|
||||
from paperforge.worker.sync import run_selection_sync
|
||||
return run_selection_sync(self.vault, verbose=verbose, json_output=json_output)
|
||||
|
||||
_export_code = "ok"
|
||||
try:
|
||||
rows = self.load_exports()
|
||||
if not rows:
|
||||
_export_code = "BBT_EXPORT_NOT_FOUND"
|
||||
except Exception:
|
||||
_export_code = "BBT_EXPORT_INVALID"
|
||||
|
||||
try:
|
||||
selection_result = run_selection_sync(self.vault, verbose=verbose, json_output=json_output)
|
||||
except Exception as exc:
|
||||
return PFResult(
|
||||
ok=False,
|
||||
command="sync",
|
||||
version=__version__,
|
||||
error=PFError(
|
||||
code=ErrorCode.SYNC_FAILED,
|
||||
message=str(exc),
|
||||
details={"phase": "selection", "exception_type": type(exc).__name__},
|
||||
),
|
||||
)
|
||||
|
||||
# ── Phase 2: Index ──
|
||||
from paperforge.worker._utils import pipeline_paths
|
||||
from paperforge.worker._domain import load_domain_config
|
||||
from paperforge.worker.base_views import ensure_base_views
|
||||
|
||||
paths = pipeline_paths(self.vault)
|
||||
config = load_domain_config(paths)
|
||||
ensure_base_views(self.vault, paths, config)
|
||||
domain_lookup = {entry["export_file"]: entry["domain"] for entry in config["domains"]}
|
||||
|
||||
from paperforge.config import load_vault_config
|
||||
|
||||
cfg = load_vault_config(self.vault)
|
||||
exports: dict[str, dict[str, dict]] = {}
|
||||
for export_path in sorted(paths["exports"].glob("*.json")):
|
||||
domain = domain_lookup.get(export_path.name, export_path.stem)
|
||||
export_rows = load_export_rows(export_path)
|
||||
exports[domain] = {row["key"]: row for row in export_rows}
|
||||
|
||||
from paperforge.worker.sync import migrate_to_workspace
|
||||
|
||||
migrate_to_workspace(self.vault, paths)
|
||||
|
||||
import paperforge.worker.asset_index as asset_index
|
||||
|
||||
index_count = asset_index.build_index(self.vault, verbose)
|
||||
|
||||
# ── Phase 3: Clean ──
|
||||
orphaned = self.clean_orphaned_records(exports, paths, json_output=json_output)
|
||||
flat_cleaned = self.clean_flat_notes(paths, json_output=json_output)
|
||||
|
||||
if not json_output:
|
||||
print(f"index-refresh: {index_count} entries in index")
|
||||
total = sum(1 for _ in paths["literature"].rglob("*.md")) if paths["literature"].exists() else 0
|
||||
print(f"index-refresh: {total} formal note(s) in literature")
|
||||
|
||||
all_errors = list(selection_result.get("errors", []))
|
||||
is_ok = selection_result.get("failed", 0) == 0 and len(all_errors) == 0
|
||||
|
||||
result = PFResult(
|
||||
ok=is_ok,
|
||||
command="sync",
|
||||
version=__version__,
|
||||
data={
|
||||
"selection": selection_result,
|
||||
"index": {"updated": index_count, "orphaned_cleaned": orphaned, "flat_cleaned": flat_cleaned},
|
||||
},
|
||||
)
|
||||
|
||||
if _export_code != "ok":
|
||||
result.warnings.append(f"BBT exports: {_export_code}")
|
||||
|
||||
return result
|
||||
|
||||
# ── Legacy passthrough (backward-compat for commands/sync.py) ──
|
||||
|
||||
def run_sync(self, verbose: bool = False, json_output: bool = False) -> PFResult:
|
||||
"""Alias for run(). Returns PFResult (not int|dict)."""
|
||||
return self.run(verbose=verbose, json_output=json_output)
|
||||
|
|
|
|||
|
|
@ -14,39 +14,13 @@ import logging
|
|||
from pathlib import Path
|
||||
|
||||
from paperforge.worker._utils import read_json, write_json
|
||||
from paperforge.adapters.collections import build_collection_lookup
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ── Collection tree utilities ────────────────────────────────────────────────
|
||||
|
||||
|
||||
def build_collection_lookup(collections: dict) -> dict:
|
||||
"""Build parent-resolved path cache from a Zotero collection tree.
|
||||
|
||||
Returns a dict with:
|
||||
path_by_key: {collection_key: "Parent/Sub/Name", ...}
|
||||
paths_by_item_id: {item_id: [collection_path, ...], ...}
|
||||
"""
|
||||
path_cache = {}
|
||||
item_paths = {}
|
||||
|
||||
def path_for(key: str) -> str:
|
||||
if key in path_cache:
|
||||
return path_cache[key]
|
||||
node = collections.get(key, {})
|
||||
parent = node.get("parent") or ""
|
||||
name = node.get("name", "")
|
||||
parent_path = path_for(parent) if parent else ""
|
||||
full_path = f"{parent_path}/{name}" if parent_path else name
|
||||
path_cache[key] = full_path
|
||||
return full_path
|
||||
|
||||
for key, node in collections.items():
|
||||
full_path = path_for(key)
|
||||
for item_id in node.get("items", []):
|
||||
item_paths.setdefault(item_id, []).append(full_path)
|
||||
return {"path_by_key": path_cache, "paths_by_item_id": item_paths}
|
||||
# build_collection_lookup re-exported from paperforge.adapters.collections
|
||||
|
||||
|
||||
def export_collection_paths(export_path: Path) -> list[str]:
|
||||
|
|
@ -122,4 +96,8 @@ def load_domain_collections(paths: dict[str, Path]) -> dict[str, list[str]]:
|
|||
Used by candidate collection resolution in sync.py.
|
||||
"""
|
||||
config = load_domain_config(paths)
|
||||
return {entry["domain"]: entry.get("allowed_collections", []) for entry in config.get("domains", []) if entry.get("domain")}
|
||||
return {
|
||||
entry["domain"]: entry.get("allowed_collections", [])
|
||||
for entry in config.get("domains", [])
|
||||
if entry.get("domain")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,8 +20,7 @@ STANDARD_VIEW_NAMES = frozenset(
|
|||
# --- Journal Database ---
|
||||
|
||||
|
||||
def read_json(path: Path):
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
from paperforge.core.io import read_json, write_json
|
||||
|
||||
|
||||
_JOURNAL_DB: dict[str, dict] | None = None
|
||||
|
|
@ -64,11 +63,7 @@ def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str:
|
|||
|
||||
|
||||
# --- JSON I/O ---
|
||||
|
||||
|
||||
def write_json(path: Path, data) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
# write_json re-exported from paperforge.core.io
|
||||
|
||||
|
||||
def read_jsonl(path: Path):
|
||||
|
|
@ -128,9 +123,7 @@ def slugify_filename(text: str) -> str:
|
|||
return cleaned or "untitled"
|
||||
|
||||
|
||||
def _extract_year(value: str) -> str:
|
||||
match = re.search("(19|20)\\d{2}", value or "")
|
||||
return match.group(0) if match else ""
|
||||
from paperforge.core.date_utils import extract_year as _extract_year
|
||||
|
||||
|
||||
# --- Deep-Reading Queue ---
|
||||
|
|
@ -191,7 +184,9 @@ def get_analyze_queue(vault: Path) -> list[dict]:
|
|||
continue
|
||||
|
||||
# Quick exit: check analyze before extracting other fields
|
||||
analyze_match = re.search(r"^analyze:\s*(?:[\"'])?(true|false)(?:[\"'])?\s*$", text, re.MULTILINE | re.IGNORECASE)
|
||||
analyze_match = re.search(
|
||||
r"^analyze:\s*(?:[\"'])?(true|false)(?:[\"'])?\s*$", text, re.MULTILINE | re.IGNORECASE
|
||||
)
|
||||
if not analyze_match or analyze_match.group(1).lower() != "true":
|
||||
continue
|
||||
|
||||
|
|
@ -225,16 +220,18 @@ def get_analyze_queue(vault: Path) -> list[dict]:
|
|||
if dr_match:
|
||||
dr_status = dr_match.group(1).strip()
|
||||
|
||||
results.append({
|
||||
"zotero_key": zotero_key,
|
||||
"domain": domain,
|
||||
"title": title,
|
||||
"analyze": True,
|
||||
"do_ocr": do_ocr,
|
||||
"ocr_status": ocr_status,
|
||||
"deep_reading_status": dr_status,
|
||||
"note_path": note_file,
|
||||
})
|
||||
results.append(
|
||||
{
|
||||
"zotero_key": zotero_key,
|
||||
"domain": domain,
|
||||
"title": title,
|
||||
"analyze": True,
|
||||
"do_ocr": do_ocr,
|
||||
"ocr_status": ocr_status,
|
||||
"deep_reading_status": dr_status,
|
||||
"note_path": note_file,
|
||||
}
|
||||
)
|
||||
|
||||
results.sort(key=lambda r: (r["domain"], r["zotero_key"]))
|
||||
return results
|
||||
|
|
@ -278,6 +275,7 @@ def install_obsidian_plugin(vault: Path) -> bool:
|
|||
plugin_src = vault / "paperforge" / "plugin"
|
||||
if not plugin_src.is_dir():
|
||||
import paperforge
|
||||
|
||||
plugin_src = Path(paperforge.__file__).parent.resolve() / "plugin"
|
||||
if not plugin_src.is_dir():
|
||||
logger.warning("Plugin source not found: %s", plugin_src)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,12 @@
|
|||
from __future__ import annotations
|
||||
|
||||
# =============================================================================
|
||||
# FREEZE LINE (v2.1-contract-hardened)
|
||||
# No new business logic allowed in this file.
|
||||
# New logic → services/ / adapters/ / core/.
|
||||
# This file: deletion, migration, legacy wrappers only.
|
||||
# =============================================================================
|
||||
|
||||
import html
|
||||
import logging
|
||||
import os
|
||||
|
|
@ -58,8 +65,8 @@ from paperforge.adapters.bbt import (
|
|||
|
||||
|
||||
import paperforge.worker.asset_index as asset_index
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_export_inventory(paths: dict[str, Path]) -> dict[str, dict]:
|
||||
|
|
@ -179,7 +186,6 @@ def apply_existing_library_match(row: dict, inventory: dict[str, dict]) -> dict:
|
|||
return resolved
|
||||
|
||||
|
||||
|
||||
def load_control_actions(paths: dict[str, Path]) -> dict[str, dict]:
|
||||
actions = {}
|
||||
lit_root = paths.get("literature")
|
||||
|
|
@ -201,14 +207,16 @@ def load_control_actions(paths: dict[str, Path]) -> dict[str, dict]:
|
|||
if do_ocr_match:
|
||||
do_ocr = do_ocr_match.group(1).lower() == "true"
|
||||
analyze = False
|
||||
analyze_match = re.search(r"^analyze:\s*(?:[\"'])?(true|false)(?:[\"'])?\s*$", text, re.MULTILINE | re.IGNORECASE)
|
||||
analyze_match = re.search(
|
||||
r"^analyze:\s*(?:[\"'])?(true|false)(?:[\"'])?\s*$", text, re.MULTILINE | re.IGNORECASE
|
||||
)
|
||||
if analyze_match:
|
||||
analyze = analyze_match.group(1).lower() == "true"
|
||||
actions[zotero_key] = {"analyze": analyze, "do_ocr": do_ocr}
|
||||
return actions
|
||||
|
||||
|
||||
def run_selection_sync(vault: Path, verbose: bool = False, json_output: bool = False) -> int | dict:
|
||||
def run_selection_sync(vault: Path, verbose: bool = False, json_output: bool = False) -> dict:
|
||||
from paperforge.worker.base_views import ensure_base_views
|
||||
from paperforge.worker.ocr import validate_ocr_meta
|
||||
|
||||
|
|
@ -267,10 +275,10 @@ def run_selection_sync(vault: Path, verbose: bool = False, json_output: bool = F
|
|||
# Formal notes now carry workflow flags (do_ocr, analyze) directly.
|
||||
# Existing library-records are migrated via Phase 40 logic.
|
||||
updated += 1
|
||||
if json_output:
|
||||
return {"new": written, "updated": updated, "skipped": 0, "failed": 0, "errors": []}
|
||||
print(f"selection-sync: wrote {written} records, updated {updated} records")
|
||||
return 0
|
||||
result = {"new": written, "updated": updated, "skipped": 0, "failed": 0, "errors": []}
|
||||
if not json_output:
|
||||
print(f"selection-sync: wrote {written} records, updated {updated} records")
|
||||
return result
|
||||
|
||||
|
||||
def load_candidates_by_id(paths: dict[str, Path]) -> dict[str, dict]:
|
||||
|
|
@ -1118,7 +1126,9 @@ def migrate_to_workspace(vault: Path, paths: dict) -> int:
|
|||
entry = indexed_entries.get(key, {})
|
||||
title = str(entry.get("title", "") or "").strip()
|
||||
if not title:
|
||||
title_match = re.search(r'^title:\s*["\']?(.+?)["\']?\s*$', flat_note_path.read_text(encoding="utf-8"), re.MULTILINE)
|
||||
title_match = re.search(
|
||||
r'^title:\s*["\']?(.+?)["\']?\s*$', flat_note_path.read_text(encoding="utf-8"), re.MULTILINE
|
||||
)
|
||||
title = title_match.group(1).strip() if title_match else ""
|
||||
stem = flat_note_path.stem
|
||||
if title:
|
||||
|
|
@ -1182,6 +1192,7 @@ def migrate_to_workspace(vault: Path, paths: dict) -> int:
|
|||
target_fulltext = workspace_dir / "fulltext.md"
|
||||
if source_fulltext.exists() and not target_fulltext.exists():
|
||||
import shutil
|
||||
|
||||
shutil.copy2(str(source_fulltext), str(target_fulltext))
|
||||
except Exception:
|
||||
pass
|
||||
|
|
@ -1194,7 +1205,9 @@ def migrate_to_workspace(vault: Path, paths: dict) -> int:
|
|||
return migrated
|
||||
|
||||
|
||||
def run_index_refresh(vault: Path, verbose: bool = False, rebuild_index: bool = False, json_output: bool = False) -> int:
|
||||
def run_index_refresh(
|
||||
vault: Path, verbose: bool = False, rebuild_index: bool = False, json_output: bool = False
|
||||
) -> dict:
|
||||
"""Refresh the canonical asset index.
|
||||
|
||||
Default behavior: full rebuild. This is the safe default because
|
||||
|
|
@ -1227,90 +1240,9 @@ def run_index_refresh(vault: Path, verbose: bool = False, rebuild_index: bool =
|
|||
migrate_to_workspace(vault, paths)
|
||||
# Delegate to asset_index.build_index() for the core build loop
|
||||
count = asset_index.build_index(vault, verbose)
|
||||
control_records_dir = paths["literature"]
|
||||
if control_records_dir.exists():
|
||||
for domain_dir in control_records_dir.iterdir():
|
||||
if not domain_dir.is_dir():
|
||||
continue
|
||||
domain = domain_dir.name
|
||||
domain_export_keys = set(exports.get(domain, {}).keys())
|
||||
records_by_title = {}
|
||||
records_info = {}
|
||||
for record_file in domain_dir.rglob("*.md"):
|
||||
if record_file.name in ("fulltext.md", "deep-reading.md", "discussion.md"):
|
||||
continue
|
||||
try:
|
||||
content = record_file.read_text(encoding="utf-8")
|
||||
key_match = re.search(r"^zotero_key:\s*(.+)$", content, re.MULTILINE)
|
||||
if not key_match:
|
||||
continue
|
||||
key = key_match.group(1).strip()
|
||||
title_match = re.search("^title:\\s*[\"\\']?(.+)[\"\\']?\\s*$", content, re.MULTILINE)
|
||||
title = title_match.group(1) if title_match else ""
|
||||
has_pdf = "has_pdf: true" in content
|
||||
normalized = re.sub("[^a-z0-9]", "", title.lower())[:20]
|
||||
records_info[key] = {
|
||||
"file": record_file,
|
||||
"title": title,
|
||||
"has_pdf": has_pdf,
|
||||
"normalized": normalized,
|
||||
}
|
||||
if normalized not in records_by_title:
|
||||
records_by_title[normalized] = []
|
||||
records_by_title[normalized].append(key)
|
||||
except Exception:
|
||||
continue
|
||||
to_delete = []
|
||||
for normalized, keys in records_by_title.items():
|
||||
keys_in_export = [k for k in keys if k in domain_export_keys]
|
||||
keys_not_in_export = [k for k in keys if k not in domain_export_keys]
|
||||
if keys_in_export and keys_not_in_export:
|
||||
for k in keys_not_in_export:
|
||||
if not records_info[k]["has_pdf"]:
|
||||
to_delete.append(k)
|
||||
deleted_count = 0
|
||||
for key in to_delete:
|
||||
try:
|
||||
records_info[key]["file"].unlink()
|
||||
deleted_count += 1
|
||||
except Exception:
|
||||
pass
|
||||
if deleted_count > 0 and not json_output:
|
||||
print(f"index-refresh: cleaned {deleted_count} orphaned records in {domain}")
|
||||
|
||||
# Clean up flat notes: delete only if confirmed by canonical index + workspace
|
||||
index_data = asset_index.read_index(vault)
|
||||
ws_keys = set()
|
||||
if isinstance(index_data, dict):
|
||||
for item in index_data.get("items", []):
|
||||
ws_dir = paths["literature"] / item.get("domain", "") / (item.get("zotero_key", "") + " - " + slugify_filename(item.get("title", "")))
|
||||
if ws_dir.is_dir():
|
||||
ws_keys.add(item.get("zotero_key"))
|
||||
|
||||
lit_dir = paths["literature"]
|
||||
cleaned_flat = 0
|
||||
if lit_dir.exists() and ws_keys:
|
||||
for domain_dir in sorted(lit_dir.iterdir()):
|
||||
if not domain_dir.is_dir():
|
||||
continue
|
||||
for flat_note in list(domain_dir.glob("*.md")):
|
||||
try:
|
||||
text = flat_note.read_text(encoding="utf-8")
|
||||
m = re.search(r"^zotero_key:\s*\"?(\S+?)\"?\s*$", text, re.MULTILINE)
|
||||
key = m.group(1) if m else ""
|
||||
except Exception:
|
||||
continue
|
||||
if key and key in ws_keys:
|
||||
try:
|
||||
flat_note.unlink()
|
||||
cleaned_flat += 1
|
||||
except Exception:
|
||||
pass
|
||||
if cleaned_flat > 0 and not json_output:
|
||||
print(f"index-refresh: cleaned {cleaned_flat} flat note(s) (migrated to workspace)")
|
||||
|
||||
if control_records_dir.exists() and not json_output:
|
||||
total = sum(1 for _ in control_records_dir.rglob("*.md"))
|
||||
if paths["literature"].exists() and not json_output:
|
||||
total = sum(1 for _ in paths["literature"].rglob("*.md"))
|
||||
print(f"index-refresh: {total} formal note(s) in literature")
|
||||
|
||||
return 0
|
||||
return {"updated": count, "failed": 0, "errors": []}
|
||||
|
|
|
|||
|
|
@ -21,8 +21,16 @@ class TestErrorCodeMembers:
|
|||
assert str(ErrorCode.PYTHON_NOT_FOUND) == "PYTHON_NOT_FOUND"
|
||||
assert str(ErrorCode.UNKNOWN) == "UNKNOWN"
|
||||
|
||||
def test_member_count(self) -> None:
|
||||
assert len(ErrorCode) == 8
|
||||
def test_member_count_minimum(self) -> None:
|
||||
assert len(ErrorCode) >= 8 # expanded from 8 to 26 in v2.1 contract hardening
|
||||
|
||||
def test_is_str_enum(self) -> None:
|
||||
assert isinstance(ErrorCode.PYTHON_NOT_FOUND, str)
|
||||
|
||||
def test_unknown_code_returns_unknown(self) -> None:
|
||||
"""Newer plugin versions may send codes not yet in this runtime."""
|
||||
assert ErrorCode("NONEXISTENT_CODE") is ErrorCode.UNKNOWN
|
||||
|
||||
def test_future_code_returns_unknown(self) -> None:
|
||||
assert ErrorCode("PAPERFORGE_NOT_INSTALLED") is ErrorCode.PAPERFORGE_NOT_INSTALLED
|
||||
assert ErrorCode("OCR_UPLOAD_FAILED") is ErrorCode.OCR_UPLOAD_FAILED
|
||||
|
|
|
|||
|
|
@ -103,3 +103,103 @@ class TestPFResultRoundTrip:
|
|||
result = PFResult.from_dict(data)
|
||||
assert result.error is None
|
||||
assert result.ok is True
|
||||
|
||||
|
||||
class TestPFResultV21Fields:
|
||||
"""v2.1 contract hardening: warnings, next_actions, suggestions round-trip."""
|
||||
|
||||
def test_warnings_round_trip(self) -> None:
|
||||
original = PFResult(
|
||||
ok=True,
|
||||
command="sync",
|
||||
version="2.1.0",
|
||||
warnings=["BBT export is stale; re-export recommended"],
|
||||
)
|
||||
d = original.to_dict()
|
||||
assert d["warnings"] == ["BBT export is stale; re-export recommended"]
|
||||
reconstructed = PFResult.from_dict(d)
|
||||
assert reconstructed.warnings == original.warnings
|
||||
assert reconstructed == original
|
||||
|
||||
def test_warnings_empty_omitted_from_dict(self) -> None:
|
||||
result = PFResult(ok=True, command="sync", version="2.1.0")
|
||||
d = result.to_dict()
|
||||
assert "warnings" not in d
|
||||
|
||||
def test_warnings_from_dict_missing_defaults_empty(self) -> None:
|
||||
data = {"ok": True, "command": "test", "version": "1.0.0"}
|
||||
result = PFResult.from_dict(data)
|
||||
assert result.warnings == []
|
||||
|
||||
def test_next_actions_round_trip(self) -> None:
|
||||
original = PFResult(
|
||||
ok=False,
|
||||
command="sync",
|
||||
version="2.1.0",
|
||||
next_actions=[
|
||||
{"id": "open_exports_dir", "label": "Open exports folder"},
|
||||
{"id": "run_zotero", "label": "Launch Zotero"},
|
||||
],
|
||||
)
|
||||
d = original.to_dict()
|
||||
assert len(d["next_actions"]) == 2
|
||||
assert d["next_actions"][0]["id"] == "open_exports_dir"
|
||||
reconstructed = PFResult.from_dict(d)
|
||||
assert reconstructed.next_actions == original.next_actions
|
||||
|
||||
def test_next_actions_empty_omitted_from_dict(self) -> None:
|
||||
result = PFResult(ok=True, command="sync", version="2.1.0")
|
||||
d = result.to_dict()
|
||||
assert "next_actions" not in d
|
||||
|
||||
def test_error_suggestions_round_trip(self) -> None:
|
||||
error = PFError(
|
||||
code=ErrorCode.BBT_EXPORT_NOT_FOUND,
|
||||
message="No BBT export found",
|
||||
suggestions=[
|
||||
"In Zotero, right-click collection → Export Collection",
|
||||
"Format: Better BibTeX JSON",
|
||||
"Tick 'Keep updated'",
|
||||
],
|
||||
)
|
||||
result = PFResult(ok=False, command="sync", version="2.1.0", error=error)
|
||||
d = result.to_dict()
|
||||
assert "suggestions" in d["error"]
|
||||
assert len(d["error"]["suggestions"]) == 3
|
||||
reconstructed = PFResult.from_dict(d)
|
||||
assert reconstructed.error is not None
|
||||
assert reconstructed.error.suggestions == error.suggestions
|
||||
|
||||
def test_error_suggestions_empty_omitted(self) -> None:
|
||||
error = PFError(code=ErrorCode.SYNC_FAILED, message="fail")
|
||||
result = PFResult(ok=False, command="sync", version="2.1.0", error=error)
|
||||
d = result.to_dict()
|
||||
assert "suggestions" not in d["error"]
|
||||
|
||||
def test_from_dict_missing_suggestions_defaults_empty(self) -> None:
|
||||
data = {
|
||||
"ok": False,
|
||||
"command": "test",
|
||||
"version": "1.0.0",
|
||||
"error": {"code": "SYNC_FAILED", "message": "fail", "details": {}},
|
||||
}
|
||||
result = PFResult.from_dict(data)
|
||||
assert result.error is not None
|
||||
assert result.error.suggestions == []
|
||||
|
||||
def test_from_dict_unknown_error_code_graceful(self) -> None:
|
||||
"""Newer plugin may send error codes not yet in this runtime."""
|
||||
data = {
|
||||
"ok": False,
|
||||
"command": "sync",
|
||||
"version": "2.1.0",
|
||||
"error": {
|
||||
"code": "SOME_FUTURE_ERROR_CODE",
|
||||
"message": "Something new happened",
|
||||
"details": {},
|
||||
},
|
||||
}
|
||||
result = PFResult.from_dict(data)
|
||||
assert result.error is not None
|
||||
assert result.error.code is ErrorCode.UNKNOWN
|
||||
assert result.error.message == "Something new happened"
|
||||
|
|
|
|||
|
|
@ -10,54 +10,51 @@ from paperforge.core.state import (
|
|||
|
||||
|
||||
class TestOcrStatus:
|
||||
"""OcrStatus enum values and legacy mapping."""
|
||||
"""OcrStatus enum values and legacy mapping (1:1, no compression)."""
|
||||
|
||||
def test_values_match_expected_strings(self) -> None:
|
||||
def test_all_canonical_values(self) -> None:
|
||||
assert OcrStatus.NONE.value == "none"
|
||||
assert OcrStatus.PENDING.value == "pending"
|
||||
assert OcrStatus.QUEUED.value == "queued"
|
||||
assert OcrStatus.PROCESSING.value == "processing"
|
||||
assert OcrStatus.DONE.value == "done"
|
||||
assert OcrStatus.DONE_INCOMPLETE.value == "done_incomplete"
|
||||
assert OcrStatus.FAILED.value == "failed"
|
||||
assert OcrStatus.BLOCKED.value == "blocked"
|
||||
assert OcrStatus.NO_PDF.value == "nopdf"
|
||||
|
||||
def test_from_legacy_canonical_pending(self) -> None:
|
||||
def test_from_legacy_identity_mappings(self) -> None:
|
||||
assert OcrStatus.from_legacy("none") is OcrStatus.NONE
|
||||
assert OcrStatus.from_legacy("pending") is OcrStatus.PENDING
|
||||
|
||||
def test_from_legacy_canonical_processing(self) -> None:
|
||||
assert OcrStatus.from_legacy("queued") is OcrStatus.QUEUED
|
||||
assert OcrStatus.from_legacy("processing") is OcrStatus.PROCESSING
|
||||
|
||||
def test_from_legacy_canonical_done(self) -> None:
|
||||
assert OcrStatus.from_legacy("done") is OcrStatus.DONE
|
||||
|
||||
def test_from_legacy_canonical_failed(self) -> None:
|
||||
assert OcrStatus.from_legacy("done_incomplete") is OcrStatus.DONE_INCOMPLETE
|
||||
assert OcrStatus.from_legacy("failed") is OcrStatus.FAILED
|
||||
assert OcrStatus.from_legacy("blocked") is OcrStatus.BLOCKED
|
||||
assert OcrStatus.from_legacy("nopdf") is OcrStatus.NO_PDF
|
||||
|
||||
def test_from_legacy_queued_maps_to_pending(self) -> None:
|
||||
assert OcrStatus.from_legacy("queued") is OcrStatus.PENDING
|
||||
|
||||
def test_from_legacy_running_maps_to_processing(self) -> None:
|
||||
def test_from_legacy_aliases(self) -> None:
|
||||
assert OcrStatus.from_legacy("running") is OcrStatus.PROCESSING
|
||||
|
||||
def test_from_legacy_error_maps_to_failed(self) -> None:
|
||||
assert OcrStatus.from_legacy("error") is OcrStatus.FAILED
|
||||
|
||||
def test_from_legacy_blocked_maps_to_failed(self) -> None:
|
||||
assert OcrStatus.from_legacy("blocked") is OcrStatus.FAILED
|
||||
|
||||
def test_from_legacy_nopdf_maps_to_failed(self) -> None:
|
||||
assert OcrStatus.from_legacy("nopdf") is OcrStatus.FAILED
|
||||
|
||||
def test_from_legacy_done_incomplete_maps_to_done(self) -> None:
|
||||
assert OcrStatus.from_legacy("done_incomplete") is OcrStatus.DONE
|
||||
|
||||
def test_from_legacy_unknown_maps_to_pending(self) -> None:
|
||||
assert OcrStatus.from_legacy("nonexistent") is OcrStatus.PENDING
|
||||
|
||||
def test_from_legacy_case_and_whitespace_insensitive(self) -> None:
|
||||
assert OcrStatus.from_legacy(" DONE ") is OcrStatus.DONE
|
||||
assert OcrStatus.from_legacy("Queued") is OcrStatus.PENDING
|
||||
assert OcrStatus.from_legacy("Queued") is OcrStatus.QUEUED
|
||||
assert OcrStatus.from_legacy(" NoPdf ") is OcrStatus.NO_PDF
|
||||
|
||||
def test_str_returns_value(self) -> None:
|
||||
assert str(OcrStatus.PENDING) == "pending"
|
||||
assert str(OcrStatus.DONE) == "done"
|
||||
assert str(OcrStatus.NONE) == "none"
|
||||
|
||||
def test_enum_equals_string_value(self) -> None:
|
||||
assert OcrStatus.DONE == "done"
|
||||
assert OcrStatus.BLOCKED == "blocked"
|
||||
assert OcrStatus.NO_PDF == "nopdf"
|
||||
|
||||
|
||||
class TestPdfStatus:
|
||||
|
|
@ -101,8 +98,13 @@ class TestAllowedTransitionsOcr:
|
|||
assert ok is True
|
||||
assert msg == ""
|
||||
|
||||
def test_pending_to_failed_valid(self) -> None:
|
||||
def test_pending_to_failed_invalid(self) -> None:
|
||||
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.PENDING, OcrStatus.FAILED)
|
||||
assert ok is False
|
||||
assert "Illegal OCR transition" in msg
|
||||
|
||||
def test_pending_to_queued_valid(self) -> None:
|
||||
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.PENDING, OcrStatus.QUEUED)
|
||||
assert ok is True
|
||||
assert msg == ""
|
||||
|
||||
|
|
@ -111,6 +113,11 @@ class TestAllowedTransitionsOcr:
|
|||
assert ok is True
|
||||
assert msg == ""
|
||||
|
||||
def test_processing_to_done_incomplete_valid(self) -> None:
|
||||
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.PROCESSING, OcrStatus.DONE_INCOMPLETE)
|
||||
assert ok is True
|
||||
assert msg == ""
|
||||
|
||||
def test_processing_to_failed_valid(self) -> None:
|
||||
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.PROCESSING, OcrStatus.FAILED)
|
||||
assert ok is True
|
||||
|
|
@ -126,6 +133,16 @@ class TestAllowedTransitionsOcr:
|
|||
assert ok is True
|
||||
assert msg == ""
|
||||
|
||||
def test_blocked_to_pending_valid(self) -> None:
|
||||
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.BLOCKED, OcrStatus.PENDING)
|
||||
assert ok is True
|
||||
assert msg == ""
|
||||
|
||||
def test_nopdf_to_pending_valid(self) -> None:
|
||||
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.NO_PDF, OcrStatus.PENDING)
|
||||
assert ok is True
|
||||
assert msg == ""
|
||||
|
||||
def test_pending_to_done_invalid(self) -> None:
|
||||
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.PENDING, OcrStatus.DONE)
|
||||
assert ok is False
|
||||
|
|
@ -144,8 +161,8 @@ class TestAllowedTransitionsOcr:
|
|||
def test_invalid_transition_includes_allowed_list(self) -> None:
|
||||
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.PENDING, OcrStatus.DONE)
|
||||
assert ok is False
|
||||
assert "processing" in msg
|
||||
assert "failed" in msg
|
||||
assert "queued" in msg or "Queued" in msg
|
||||
assert "processing" in msg or "Processing" in msg
|
||||
|
||||
|
||||
class TestAllowedTransitionsLifecycle:
|
||||
|
|
|
|||
Loading…
Reference in a new issue