From 19c28a8b9183e42a306469963d276af813bf4de0 Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Mon, 18 May 2026 22:47:30 +0800 Subject: [PATCH] chore: commit session work before hotfix switch --- .../plans/2026-05-18-orphan-cleanup-prune.md | 711 ++++++++++++++++++ .../specs/2026-05-18-orphan-cleanup-prune.md | 127 ++++ paperforge/adapters/obsidian_frontmatter.py | 8 +- paperforge/commands/agent_context.py | 2 +- paperforge/commands/dashboard.py | 1 - paperforge/commands/embed.py | 9 +- paperforge/commands/memory.py | 4 +- paperforge/commands/reading_log.py | 1 - paperforge/commands/retrieve.py | 5 +- paperforge/commands/runtime_health.py | 1 - paperforge/commands/search.py | 2 +- paperforge/commands/status.py | 2 - paperforge/doctor/field_validator.py | 1 - paperforge/embedding/_config.py | 5 +- paperforge/memory/__init__.py | 2 +- paperforge/memory/_columns.py | 1 - paperforge/memory/permanent.py | 2 +- paperforge/memory/runtime_health.py | 2 +- paperforge/memory/schema.py | 3 +- paperforge/memory/state_snapshot.py | 1 - paperforge/memory/vector_db.py | 20 +- paperforge/plugin/main.js | 25 +- paperforge/plugin/styles.css | 51 +- paperforge/schema/__init__.py | 2 +- paperforge/setup/checker.py | 1 - paperforge/setup/config_writer.py | 2 +- paperforge/setup/plan.py | 7 +- paperforge/setup/runtime.py | 7 +- paperforge/worker/_domain.py | 2 +- paperforge/worker/base_views.py | 4 - paperforge/worker/discussion.py | 3 +- paperforge/worker/ocr.py | 2 +- paperforge/worker/paper_meta.py | 2 +- paperforge/worker/paper_resolver.py | 15 +- paperforge/worker/repair.py | 4 +- paperforge/worker/status.py | 16 +- paperforge/worker/sync.py | 47 +- paperforge/worker/update.py | 11 +- paperforge/worker/vector_db.py | 2 +- 39 files changed, 978 insertions(+), 135 deletions(-) create mode 100644 docs/superpowers/plans/2026-05-18-orphan-cleanup-prune.md create mode 100644 docs/superpowers/specs/2026-05-18-orphan-cleanup-prune.md diff --git a/docs/superpowers/plans/2026-05-18-orphan-cleanup-prune.md b/docs/superpowers/plans/2026-05-18-orphan-cleanup-prune.md new file mode 100644 index 00000000..06df6c36 --- /dev/null +++ b/docs/superpowers/plans/2026-05-18-orphan-cleanup-prune.md @@ -0,0 +1,711 @@ +# Orphan Paper Cleanup (Prune) — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add `paperforge sync --prune` and `paperforge prune` commands to delete orphaned workspace/OCR/vector artifacts for papers removed from Zotero. + +**Architecture:** New standalone `worker/prune.py` module — testable without SyncService. Consumed by both `SyncService.run()` (for `sync --prune`) and a standalone `commands/prune.py` (for `paperforge prune`). Three-tier safety: dry-run default, `--force` to execute, explicit key-match filtering. + +**Tech Stack:** Python 3.11+, `shutil.rmtree`, ChromaDB `collection.delete(ids=...)` (single-key, safe). No new dependencies. + +--- + +## Files Changed + +| Type | Path | Responsibility | +|---|---|---| +| Create | `paperforge/worker/prune.py` | Core prune logic: collect orphan keys, delete workspace/OCR/vectors | +| Modify | `paperforge/services/sync_service.py` | Add `prune(vault, paths, fresh_index, dry_run)` method; hook into `run()` after index rebuild | +| Modify | `paperforge/commands/sync.py` | Add `--prune` / `--force` CLI args; pass to SyncService | +| Modify | `paperforge/cli.py` | Add `prune` subcommand parser | +| Create | `paperforge/commands/prune.py` | Standalone `paperforge prune [--force]` CLI module | +| Modify | `paperforge/commands/__init__.py` | Register `prune` in command registry | +| Create | `tests/unit/worker/test_prune.py` | Unit tests for core prune logic | + +### Unchanged + +| File | Why unchanged | +|---|---| +| `paperforge/embedding/_chroma.py` | `delete_paper_vectors()` already exists and is safe | +| `paperforge/commands/embed.py` | Not involved — prune calls `delete_paper_vectors` directly | +| `paperforge/worker/asset_index.py` | Not involved — prune reads fresh index after build | +| `paperforge/memory/builder.py` | Not involved — memory DB is rebuilt from fresh index before prune runs | + +--- + +## Task Breakdown + +### Task 1: Core prune module (`worker/prune.py`) + +**Files:** +- Create: `paperforge/worker/prune.py` +- Test: `tests/unit/worker/test_prune.py` + +Logic: + +``` +def prune_orphan_papers(vault: Path, *, fresh_index: dict, dry_run: bool = True) -> dict: + 1. Build key_set from fresh_index["items"][*]["zotero_key"] + 2. Scan literature/ subdirectories for {key} - {slug}/ pattern + 3. For each key on filesystem but not in key_set: + a. collect: ocr_dir, workspace_dir + b. if dry_run: accumulate in preview list + c. if not dry_run: rmtree each, then delete_paper_vectors(key) + 4. Return {"deleted": [...], "counts": {...}} +``` + +- [ ] **Step 1: Write failing tests** + +```python +"""Tests for worker/prune.py — orphan detection and cleanup.""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from paperforge.worker.prune import ( + _collect_orphan_candidates, + prune_orphan_papers, +) + + +class TestCollectOrphanCandidates: + """_collect_orphan_candidates(lit_dir, fresh_keys) -> list[dict]""" + + def test_returns_no_orphans_when_all_match(self, tmp_path: Path) -> None: + """All workspace dirs have keys in the fresh index.""" + lit = tmp_path / "Literature" / "CS" + (lit / "key1 - Paper One").mkdir(parents=True) + (lit / "key2 - Paper Two").mkdir(parents=True) + fresh_keys = {"key1", "key2"} + result = _collect_orphan_candidates(lit, fresh_keys) + assert result == [] + + def test_returns_orphan_for_missing_key(self, tmp_path: Path) -> None: + """Workspace with key not in fresh index is orphan.""" + lit = tmp_path / "Literature" / "CS" + ws = lit / "key1 - Orphan Paper" + ws.mkdir(parents=True) + fresh_keys = set() + result = _collect_orphan_candidates(lit, fresh_keys) + assert len(result) == 1 + assert result[0]["key"] == "key1" + assert result[0]["workspace_dir"] == ws + + def test_skips_non_workspace_dirs(self, tmp_path: Path) -> None: + """Directories not matching {key} - {slug} pattern are skipped.""" + lit = tmp_path / "Literature" / "CS" + (lit / "orphan_file.md").write_text("not a dir") + (lit / "random_dir").mkdir() + fresh_keys = set() + result = _collect_orphan_candidates(lit, fresh_keys) + assert result == [] + + def test_skips_dirs_without_dash_space_pattern(self, tmp_path: Path) -> None: + """Directory name must contain ' - ' to be a workspace.""" + lit = tmp_path / "Literature" / "CS" + (lit / "justakey").mkdir() + (lit / "key-with-dashes-no-slug").mkdir() + fresh_keys = set() + result = _collect_orphan_candidates(lit, fresh_keys) + assert result == [] + + def test_handles_multiple_domains(self, tmp_path: Path) -> None: + """Scans all domain subdirectories under literature root.""" + lit = tmp_path / "Literature" + (lit / "CS" / "key1 - Paper One").mkdir(parents=True) + (lit / "Med" / "key2 - Paper Two").mkdir(parents=True) + (lit / "Sport" / "key3 - Paper Three").mkdir(parents=True) + fresh_keys = {"key1"} + result = _collect_orphan_candidates(lit, fresh_keys) + assert len(result) == 2 + returned_keys = {c["key"] for c in result} + assert returned_keys == {"key2", "key3"} + + +class TestPruneOrphanPapers: + """prune_orphan_pairs(vault, fresh_index, dry_run)""" + + def test_dry_run_does_not_delete(self, tmp_path: Path) -> None: + """With dry_run=True, nothing is actually deleted.""" + lit = tmp_path / "Literature" / "CS" + ws = lit / "key1 - Orphan" + ws.mkdir(parents=True) + note = ws / "note.md" + note.write_text("hello") + fresh_index = {"schema_version": "3", "items": []} + + result = prune_orphan_papers(tmp_path, fresh_index=fresh_index, dry_run=True) + + assert len(result["preview"]) == 1 + assert note.exists() # not deleted + + def test_force_deletes_workspace(self, tmp_path: Path) -> None: + """With dry_run=False, workspace dir is deleted.""" + lit = tmp_path / "Literature" / "CS" + ws = lit / "key1 - Orphan" + ws.mkdir(parents=True) + (ws / "note.md").write_text("hello") + (ws / "ai" / "discussion.md").write_text("some discussion") + fresh_index = {"schema_version": "3", "items": []} + + result = prune_orphan_papers(tmp_path, fresh_index=fresh_index, dry_run=False) + + assert result["deleted"] == ["key1"] + assert not ws.exists() + + def test_force_deletes_ocr_dir(self, tmp_path: Path) -> None: + """With dry_run=False, OCR dir is deleted.""" + ocr = tmp_path / "System" / "PaperForge" / "ocr" / "key1" + ocr.mkdir(parents=True) + (ocr / "fulltext.md").write_text("fulltext") + lit = tmp_path / "Literature" / "CS" + (lit / "key1 - Orphan").mkdir(parents=True) + fresh_index = {"schema_version": "3", "items": []} + + result = prune_orphan_papers(tmp_path, fresh_index=fresh_index, dry_run=False) + + assert result["deleted"] == ["key1"] + assert not ocr.exists() + + def test_vectors_not_deleted_in_dry_run(self, tmp_path: Path, monkeypatch) -> None: + """Dry run must not call delete_paper_vectors.""" + calls = [] + + def _mock_delete(vault, key): + calls.append(key) + return 0 + + monkeypatch.setattr("paperforge.worker.prune.delete_paper_vectors", _mock_delete) + + lit = tmp_path / "Literature" / "CS" + (lit / "key1 - Orphan").mkdir(parents=True) + fresh_index = {"schema_version": "3", "items": []} + + prune_orphan_papers(tmp_path, fresh_index=fresh_index, dry_run=True) + assert calls == [] + + def test_orphan_not_in_fresh_index_is_skipped(self, tmp_path: Path) -> None: + """A key present in fresh_index must NOT be deleted.""" + lit = tmp_path / "Literature" / "CS" + ws = lit / "key1 - Active Paper" + ws.mkdir(parents=True) + fresh_index = { + "schema_version": "3", + "items": [{"zotero_key": "key1", "title": "Active Paper"}], + } + + result = prune_orphan_papers(tmp_path, fresh_index=fresh_index, dry_run=False) + assert result["deleted"] == [] + assert ws.exists() +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest tests/unit/worker/test_prune.py -v` +Expected: `ModuleNotFoundError` or `FAILED` for all tests (no `worker/prune.py` yet) + +- [ ] **Step 3: Implement `worker/prune.py`** + +```python +"""paperforge.worker.prune — orphan paper cleanup. + +Orphan detection: scan filesystem workspace dirs, find keys +not present in the fresh canonical index, and delete their +workspace/OCR/vector artifacts. + +Safety: dry_run=True by default; --force required to execute. +""" + +from __future__ import annotations + +import logging +import shutil +from pathlib import Path + +from paperforge.config import paperforge_paths + +logger = logging.getLogger(__name__) + + +def _collect_orphan_candidates( + lit_dir: Path, fresh_keys: set[str] +) -> list[dict]: + """Scan literature dir for workspace dirs whose key is not in fresh_keys. + + Returns list of dicts: {key, domain, workspace_dir, ocr_dir} + Only matches dirs named ``{key} - {slug}``. + """ + if not lit_dir.exists(): + return [] + + candidates: list[dict] = [] + for domain_dir in sorted(lit_dir.iterdir()): + if not domain_dir.is_dir(): + continue + for sub in sorted(domain_dir.iterdir()): + if not sub.is_dir(): + continue + parts = sub.name.split(" - ", 1) + if len(parts) < 2: + continue + key = parts[0] + if not key: + continue + if key in fresh_keys: + continue + candidates.append({ + "key": key, + "domain": domain_dir.name, + "workspace_dir": sub, + "ocr_dir": None, # resolved later + }) + + return candidates + + +def _resolve_ocr_dir(vault: Path, key: str) -> Path: + """Resolve OCR directory for a given key.""" + cfg = paperforge_paths(vault) + ocr_root = cfg.get("ocr", vault / "System" / "PaperForge" / "ocr") + return ocr_root / key + + +def prune_orphan_papers( + vault: Path, + *, + fresh_index: dict, + dry_run: bool = True, +) -> dict: + """Delete orphan paper artifacts for keys not in the fresh index. + + Args: + vault: Vault root path. + fresh_index: The just-rebuilt canonical index dict. + dry_run: If True, only preview; if False, actually delete. + + Returns: + dict with keys: + - preview: list of {key, domain, paths} (always populated) + - deleted: list of keys actually deleted (only non-dry-run) + - counts: {workspace, ocr, vectors, failed} + """ + cfg = paperforge_paths(vault) + lit_dir = cfg.get("literature") + if not lit_dir: + return {"preview": [], "deleted": [], "counts": {}} + + fresh_keys = { + item["zotero_key"] + for item in fresh_index.get("items", []) + if item.get("zotero_key") + } + + candidates = _collect_orphan_candidates(lit_dir, fresh_keys) + if not candidates: + return {"preview": [], "deleted": [], "counts": {}} + + # Resolve OCR dirs for all candidates + for c in candidates: + c["ocr_dir"] = _resolve_ocr_dir(vault, c["key"]) + + preview = [ + { + "key": c["key"], + "domain": c["domain"], + "workspace": str(c["workspace_dir"]), + "ocr_dir": str(c["ocr_dir"]) if c["ocr_dir"].exists() else None, + } + for c in candidates + ] + + if dry_run: + return {"preview": preview, "deleted": [], "counts": {}} + + deleted: list[str] = [] + counts = {"workspace": 0, "ocr": 0, "vectors": 0, "failed": 0} + + for c in candidates: + key = c["key"] + try: + # 1. Delete OCR dir first (safest — easily re-OCR'd) + ocr = c["ocr_dir"] + if ocr and ocr.exists(): + shutil.rmtree(ocr, ignore_errors=True) + counts["ocr"] += 1 + + # 2. Delete workspace dir (irreversible — discussion history) + ws = c["workspace_dir"] + if ws.exists(): + shutil.rmtree(ws, ignore_errors=True) + counts["workspace"] += 1 + + # 3. Delete vectors — single-key, non-corrupting + try: + from paperforge.embedding._chroma import delete_paper_vectors + n = delete_paper_vectors(vault, key) + if n > 0: + counts["vectors"] += n + except Exception as vec_err: + logger.warning("prune: failed to delete vectors for %s: %s", key, vec_err) + counts["failed"] += 1 + + deleted.append(key) + + except Exception as exc: + logger.error("prune: failed to clean up %s: %s", key, exc) + counts["failed"] += 1 + + return {"preview": preview, "deleted": deleted, "counts": counts} +``` + +- [ ] **Step 4: Run tests again** + +Run: `pytest tests/unit/worker/test_prune.py -v` +Expected: All tests pass + +- [ ] **Step 5: Commit** + +```bash +git add paperforge/worker/prune.py tests/unit/worker/test_prune.py +git commit -m "feat(prune): core orphan paper cleanup module" +``` + +--- + +### Task 2: Standalone `prune` CLI command + +**Files:** +- Create: `paperforge/commands/prune.py` +- Modify: `paperforge/commands/__init__.py` +- Modify: `paperforge/cli.py` + +- [ ] **Step 1: Create `commands/prune.py`** + +```python +"""Prune command — standalone orphan paper cleanup.""" +from __future__ import annotations + +import argparse +import json +import logging + +from paperforge import __version__ +from paperforge.core.result import PFResult +from paperforge.worker.asset_index import read_index + +logger = logging.getLogger(__name__) + + +def run(args: argparse.Namespace) -> int: + vault = args.vault_path + dry_run = not getattr(args, "force", False) + json_output = getattr(args, "json", False) + + # Read the current canonical index + try: + fresh_index = read_index(vault) + except Exception as e: + logger.error("prune: failed to read canonical index: %s", e) + if json_output: + result = PFResult( + ok=False, command="prune", version=__version__, + data={"error": f"cannot read index: {e}"}, + ) + print(result.to_json()) + else: + print(f"[FAIL] Cannot read canonical index: {e}") + return 1 + + from paperforge.worker.prune import prune_orphan_papers + + result_data = prune_orphan_papers(vault, fresh_index=fresh_index, dry_run=dry_run) + + if json_output: + result = PFResult( + ok=True, command="prune", version=__version__, data=result_data, + ) + print(result.to_json()) + return 0 + + # Human-readable output + preview = result_data.get("preview", []) + if not preview: + print("[OK] No orphan papers found.") + return 0 + + print(f"[PRUNE] Found {len(preview)} orphan paper(s):") + for p in preview: + extras = [] + if p.get("ocr_dir"): + extras.append("OCR") + print(f" {p['key']} ({p['domain']}) — workspace + {' + '.join(extras)}") + + if dry_run: + print(f"\n--- Dry run (pass --force to actually delete) ---") + else: + counts = result_data.get("counts", {}) + print(f"\n[PRUNE] Deleted {len(result_data.get('deleted', []))} paper(s)") + print(f" workspaces: {counts.get('workspace', 0)}") + print(f" OCR dirs: {counts.get('ocr', 0)}") + print(f" vectors: {counts.get('vectors', 0)}") + if counts.get("failed", 0): + print(f" failed: {counts['failed']}") + + return 0 +``` + +- [ ] **Step 2: Register in command registry** + +Edit `paperforge/commands/__init__.py`: add `"prune": "paperforge.commands.prune"` to `_COMMAND_REGISTRY`. + +- [ ] **Step 3: Add CLI parser** + +Edit `paperforge/cli.py`: + +```python +# In build_parser(), after embed parser: +p_prune = subparsers.add_parser("prune", help="Delete orphan paper artifacts") +p_prune.add_argument("--force", action="store_true", help="Actually delete (default: dry-run)") +_shared_args(p_prune) + +# In main(), after embed dispatch: +if args.command == "prune": + from paperforge.commands.prune import run + return run(args) +``` + +- [ ] **Step 4: Test the standalone command** + +Run: `python -m paperforge prune --vault ` (or use a fixture) +Expected: Shows "No orphan papers found." or dry-run list. + +- [ ] **Step 5: Commit** + +```bash +git add paperforge/commands/prune.py paperforge/commands/__init__.py paperforge/cli.py +git commit -m "feat(prune): standalone CLI command" +``` + +--- + +### Task 3: `sync --prune` integration + +**Files:** +- Modify: `paperforge/services/sync_service.py` +- Modify: `paperforge/commands/sync.py` + +- [ ] **Step 1: Add `prune()` method to SyncService** + +```python +# In sync_service.py, add method: +def prune(self, paths: dict, fresh_index: dict | None = None, *, dry_run: bool = True) -> dict: + """Run orphan paper cleanup. Default dry_run=True.""" + if fresh_index is None: + from paperforge.worker.asset_index import read_index + fresh_index = read_index(self.vault) + from paperforge.worker.prune import prune_orphan_papers + return prune_orphan_papers(self.vault, fresh_index=fresh_index, dry_run=dry_run) +``` + +- [ ] **Step 2: Hook into `run()` method** + +In `SyncService.run()`, after Phase 3 (index + memory rebuild), add: + +```python +# In run() method, after line 298 (memory rebuild): +prune_result = None +if getattr(args, 'prune', False) or getattr(args, 'prune_force', False): + dry_run = not getattr(args, 'prune_force', False) + prune_result = self.prune(paths, fresh_index=None, dry_run=dry_run) +``` + +Update the method signature to accept `prune: bool = False, prune_force: bool = False`: +```python +def run(self, verbose=False, json_output=False, selection_only=False, + index_only=False, prune=False, prune_force=False) -> PFResult: +``` + +Also add `prune` to the `_print_summary` section if present, or print after the index summary. + +Update the `result.data` dict to include `prune` data. + +- [ ] **Step 3: Update `commands/sync.py` to pass `--prune` args** + +```python +# Add after lines 59-62: +svc = SyncService(vault) +prune_flag = getattr(args, "prune", False) +prune_force = getattr(args, "prune_force", False) +result = svc.run( + verbose=verbose, json_output=json_output, + selection_only=selection_only, index_only=index_only, + prune=prune_flag, prune_force=prune_force, +) +``` + +- [ ] **Step 4: Add CLI args for sync prune** + +Edit `paperforge/cli.py` sync parser: + +```python +p_sync = subparsers.add_parser("sync", help="Sync Zotero selection + rebuild index") +p_sync.add_argument("--prune", action="store_true", help="Dry-run orphan cleanup") +p_sync.add_argument("--prune-force", action="store_true", help="Execute orphan cleanup") +# ... existing args +``` + +- [ ] **Step 5: Test sync integration** + +Run: `python -m paperforge sync --prune --vault ` +Expected: Sync runs normally, then prints orphan preview (or "no orphans"). + +- [ ] **Step 6: Commit** + +```bash +git add paperforge/services/sync_service.py paperforge/commands/sync.py paperforge/cli.py +git commit -m "feat(prune): integrate prune into sync --prune" +``` + +--- + +### Task 4: Tests for SyncService prune integration + +**Files:** +- Create: `tests/unit/services/test_sync_service_prune.py` (or add to existing `test_sync_service.py` if it exists) + +- [ ] **Step 1: Write tests** + +```python +"""Tests for SyncService.prune() integration.""" +from __future__ import annotations +from pathlib import Path +from unittest.mock import patch, MagicMock +from paperforge.services.sync_service import SyncService + + +class TestSyncServicePrune: + + def test_prune_calls_worker_module(self, tmp_path: Path) -> None: + svc = SyncService(tmp_path) + svc.paths = {"literature": tmp_path / "Literature"} + fresh_index = {"schema_version": "3", "items": []} + + with patch("paperforge.services.sync_service.prune_orphan_papers") as mock_fn: + mock_fn.return_value = {"preview": [], "deleted": [], "counts": {}} + result = svc.prune(svc.paths, fresh_index=fresh_index, dry_run=True) + + mock_fn.assert_called_once() + assert result["deleted"] == [] + + def test_sync_run_passes_prune_true(self, tmp_path: Path) -> None: + """SyncService.run() with prune=True must call self.prune().""" + vault = tmp_path + cfg = vault / "System" / "PaperForge" + cfg.mkdir(parents=True) + cfg_exports = cfg / "exports" + cfg_exports.mkdir(parents=True) + cfg_indexes = cfg / "indexes" + cfg_indexes.mkdir(parents=True) + + # minimal paperforge.json + (vault / "paperforge.json").write_text('{"system_dir": "System", "resources_dir": "Resources", "literature_dir": "Resources/Literature"}') + + svc = SyncService(vault) + svc.paths = { + "literature": vault / "Resources" / "Literature", + "exports": cfg_exports, + "index": cfg_indexes, + "system_dir": vault / "System", + } + (svc.paths["literature"]).mkdir(parents=True, exist_ok=True) + + with patch.object(svc, "prune") as mock_prune: + mock_prune.return_value = {"preview": [], "deleted": [], "counts": {}} + # run() will call build_index etc, need broader patching + with patch("paperforge.worker.asset_index.build_index") as mock_build: + mock_build.return_value = 0 + with patch("paperforge.worker.asset_index.read_index") as mock_read: + mock_read.return_value = {"schema_version": "3", "items": []} + with patch.object(svc, "resolve_paths", return_value=svc.paths): + with patch("paperforge.services.sync_service.load_export_rows", return_value=[]): + with patch("paperforge.services.sync_service.load_domain_config"): + with patch("paperforge.services.sync_service.ensure_base_views"): + with patch("paperforge.services.sync_service.load_vault_config"): + with patch("paperforge.services.sync_service.migrate_to_workspace"): + svc.run(prune=True, prune_force=True) + + mock_prune.assert_called_once() +``` + +(Note: the `run()` integration test is complex due to many dependencies — the plan implementer may need to adjust the patching surface. The core `prune()` method test on SyncService is the essential one.) + +- [ ] **Step 2: Run tests** + +Run: `pytest tests/unit/services/test_sync_service_prune.py -v` +Expected: All pass + +- [ ] **Step 3: Commit** + +```bash +git add tests/unit/services/test_sync_service_prune.py +git commit -m "test(prune): add SyncService prune integration tests" +``` + +--- + +### Task 5: Full suite + lint + +- [ ] **Step 1: Run full test suite** + +```bash +python -m pytest tests/ -v --tb=short 2>&1 | tail -30 +``` +Expected: All existing + new tests pass. + +- [ ] **Step 2: Run lint** + +```bash +ruff check --fix paperforge/ && ruff format paperforge/ +``` +Expected: Clean. + +- [ ] **Step 3: Run type check (optional)** + +```bash +python -m mypy paperforge/worker/prune.py --ignore-missing-imports +``` +Expected: No type errors. + +- [ ] **Step 4: Version bump** + +Update version in `paperforge/__init__.py`, `manifest.json`, `plugin/manifest.json` to `1.5.6rc4` or next appropriate version. + +- [ ] **Step 5: Commit** + +```bash +git add -A +git commit -m "chore: bump version to 1.5.6rc4" +``` + +--- + +## Vector DB Safety (重中之重) + +| Concern | Mitigation | +|---|---| +| `delete_paper_vectors()` crashes | Wrapped in try/except — logs warning, continues cleanup. Corrupt HNSW index already handled by `get_collection()` (catches on access, recreates collection silently). | +| Partial delete (vector gone, file still there) | Not possible: delete order is 1-OCR 2-workspace 3-vectors. If step 3 fails, steps 1-2 already done; key not in index, no future operation touches it. | +| Resume mode confused by pruned vectors | Resume reads index keys then checks ChromaDB — a key in index that still has vectors = skip. A key in index whose vectors were pruned (impossible: prune only deletes keys NOT in index). Perfectly orthogonal. | +| Concurrent sync + embed build | Not a realistic threat vector. If somehow both run: embed build writes to ChromaDB, prune reads from index (already static). No lock contention. | +| Dry-run safety | `--prune` without `--force` (or bare `paperforge prune`) never touches disk. Only reports. User must explicitly pass `--force` or `--prune-force`. | + +## Rollback Strategy + +If prune accidentally deletes artifacts for an active paper: +1. Resync: `paperforge sync` restores `formal-library.json` entry +2. Re-OCR: `paperforge ocr run` restores OCR directory +3. Re-embed: `paperforge embed build --resume` restores vectors +4. Discussion history: **lost** (workspace + ai/ deletion is irreversible). This is the one irreversible cost, and matches the user's stated intent ("不然容易留下很多垃圾"). diff --git a/docs/superpowers/specs/2026-05-18-orphan-cleanup-prune.md b/docs/superpowers/specs/2026-05-18-orphan-cleanup-prune.md new file mode 100644 index 00000000..6b988769 --- /dev/null +++ b/docs/superpowers/specs/2026-05-18-orphan-cleanup-prune.md @@ -0,0 +1,127 @@ +# Orphan Paper Cleanup (Prune) — Design Spec + +> **Status:** Draft | **Date:** 2026-05-18 +> **Review:** Pending +> **Depends on:** embedding-package-extraction (embedding/ package exists) + +## Motivation + +Sync rebuilds `formal-library.json` from current BBT exports. Papers removed from Zotero disappear from the index, but their physical files remain permanently: + +| Artifact | After sync | Problem | +|---|---|---| +| Workspace dir (note + ai/) | Orphan | "最近讨论"卡片仍可访问已删论文 | +| OCR dir (`System/PaperForge/ocr/{key}/`) | Orphan | 占空间 | +| Note `.md` | Orphan | 用户仍能在 Obsidian 中看到 | +| ChromaDB vectors | Orphan | 除非 `embed build --force` 全量重建 | +| Formal-library index | Clean (rebuilt) | — | +| Memory DB (SQLite) | Clean (rebuilt) | — | + +这些孤儿文件会无限堆积，且 `sync` 对其完全沉默。 + +## Design + +### Approach: key-based deletion, post-index-rebuild + +核心思路：在 `run_index_refresh()` 末尾（index 已重建后），用新 index 的 key set 扫描工作区，删除不在 set 中的孤儿。 + +不需要 diff 旧 index，不需要记录历史。新 index 就是真相源。 + +### Entry point + +```python +# sync_service.py 末尾，run_index_refresh() 执行后 +prune_orphan_papers(vault, paths, fresh_index) # 可选 +``` + +### What to delete + +For each orphan key `k` (present on filesystem but absent from fresh index): + +``` +1. rm -rf {literature}/{domain}/{k} - {slug}/ # workspace + ai/ + note +2. rm -rf System/PaperForge/ocr/{k}/ # OCR fulltext + images +3. delete_paper_vectors(k) # ChromaDB 单条删除（非全量重建） +``` + +以上三者全部删除。 + +### Safety: key matching + +- 从 `{literature}/{domain}/` 下各子目录提取 key：`{子目录名}.split(" - ")[0]` +- 如果该 key 不在新鲜 index 的 key set 中 → 孤儿 +- 只扫 `{literature}/{domain}/` 下一级（`{key} - {slug}/` 结构的目录），不碰未知目录 +- 遇到无法 split 出 key 的目录 → 跳过（不是 PaperForge 管理的） + +### Deletion order + +1. 先删除 OCR 目录（最安全，可随时重新 OCR） +2. 再删除 workspace 目录 +3. 最后删除 ChromaDB 向量（ID 匹配） + +顺序保证：如果任何一步失败，上一步已经删了的东西不会回滚，但已删的部分不致命（key 不在 index 中，不会产生冲突）。 + +### `--prune` flag on sync + +```bash +paperforge sync --prune # dry-run：打印将删除的文件列表，不实际删除 +paperforge sync --prune --force # 实际执行删除 +``` + +两步确认机制： +- `--prune` 只输出 affected keys 列表 +- `--prune --force` 实际执行 `rmtree` + `delete_paper_vectors` + +### Standalone command + +```bash +paperforge prune # dry-run +paperforge prune --force # 实际删除 +``` + +作为独立 CLI 命令注册，方便手动调用。 + +### Logging + +``` +[PRUNE] Deleted OCR dir abc12345 → System/PaperForge/ocr/abc12345/ +[PRUNE] Deleted workspace abc12345 → Resources/Literature/CS/abc12345 - Old Title/ +[PRUNE] Deleted vectors abc12345 → 47 chunks removed +``` + +记录在 `sync` 的标准输出和 `project-log` 中。 + +## What is NOT covered + +- **Vector DB 的 `--resume` 模式不受影响**：prune 只删除 index 中没有的 paper 的向量，重建时 resume 遍历 index 中存在的 paper。完全正交。 +- **Memory DB 不需要额外操作**：`build_from_index()` 已用新鲜 index 重建。 +- **不是 GC/defrag**：不压缩 ChromaDB HNSW 索引，不清理 SQLite WAL。 +- **不会误删除在 Zotero 中后来以不同 key 重新添加的 paper**：新 key 不在 old key 的匹配范围内。 + +## Sequence Diagram + +``` +sync (--prune --force) + ├── run_selection_sync() # BBT 计数 + ├── asset_index.build_index() # 重建 formal-library.json + ├── build_from_index() # 重建 memory DB + ├── prune_orphan_papers() + │ ├── 读取新鲜 index → key_set + │ ├── 遍历 literature/ 目录 → 提取文件系统 key_set + │ ├── orphan_keys = filesystem - fresh + │ ├── for key in orphan_keys: + │ │ ├── rmtree(ocr_dir) # 跳过如果已不存在 + │ │ ├── rmtree(workspace_dir) # 跳过如果已不存在 + │ │ └── delete_paper_vectors(key) + │ └── 打印删除摘要 + └── 标准 status 报告 +``` + +## Risks and Mitigations + +| Risk | Mitigation | +|---|---| +| 用户意外删除 Zotero 文献后 sync 导致 note 丢失 | 两层保护：(1) `--prune` 是 dry-run；(2) `--force` 才实际删除。用户需显式确认 | +| 文件名格式不匹配导致误删非 PaperForge 目录 | 只处理 `{key} - {slug}` 格式的目录，未知格式跳过 | +| ChromaDB `delete_paper_vectors` 失败 | 打印 warning，不中断流程。OCR 和 workspace 已删，向量残留在下次 `embed build` 时会被 IndexError 发现 | +| 多进程安全（sync 和 embed 同时运行） | prune 只能在 sync 末尾且无其他子进程操作对应 key 时执行。现有 lock 机制足以防护 | diff --git a/paperforge/adapters/obsidian_frontmatter.py b/paperforge/adapters/obsidian_frontmatter.py index 69723df1..7c17d5b6 100644 --- a/paperforge/adapters/obsidian_frontmatter.py +++ b/paperforge/adapters/obsidian_frontmatter.py @@ -2,7 +2,6 @@ from __future__ import annotations import re from pathlib import Path -from typing import Optional def _yaml_quote(value: str) -> str: @@ -51,14 +50,14 @@ def read_frontmatter_bool(note_path: Path, key: str, default: bool = False) -> b return default -def _read_frontmatter_optional_bool_from_text(text: str, key: str) -> Optional[bool]: +def _read_frontmatter_optional_bool_from_text(text: str, key: str) -> bool | None: match = re.search(rf"^{re.escape(key)}:\s*(?:[\"'])?(true|false)(?:[\"'])?\s*$", text, re.MULTILINE | re.IGNORECASE) if not match: return None return match.group(1).lower() == "true" -def read_frontmatter_optional_bool(note_path: Path, key: str) -> Optional[bool]: +def read_frontmatter_optional_bool(note_path: Path, key: str) -> bool | None: """Read an optional boolean field from a formal note's YAML frontmatter (Path-based).""" if not note_path or not note_path.exists(): return None @@ -69,7 +68,7 @@ def read_frontmatter_optional_bool(note_path: Path, key: str) -> Optional[bool]: return None -def _legacy_control_flags(paths: dict[str, Path], zotero_key: str) -> dict[str, Optional[bool]]: +def _legacy_control_flags(paths: dict[str, Path], zotero_key: str) -> dict[str, bool | None]: records_root = paths.get("library_records") if not records_root or not records_root.exists(): return {"do_ocr": None, "analyze": None} @@ -333,6 +332,7 @@ def read_frontmatter_dict(text: str) -> dict: Returns empty dict if no frontmatter. """ import re + import yaml fm_match = re.match(r"^---\s*\n(.*?)\n---", text, re.DOTALL) diff --git a/paperforge/commands/agent_context.py b/paperforge/commands/agent_context.py index 453c9442..5be94222 100644 --- a/paperforge/commands/agent_context.py +++ b/paperforge/commands/agent_context.py @@ -3,10 +3,10 @@ from __future__ import annotations import argparse import sys +from paperforge import __version__ as PF_VERSION from paperforge.core.errors import ErrorCode from paperforge.core.result import PFError, PFResult from paperforge.memory.context import get_agent_context -from paperforge import __version__ as PF_VERSION COMMANDS = { "paper-status": { diff --git a/paperforge/commands/dashboard.py b/paperforge/commands/dashboard.py index bbf87717..0e8e0e63 100644 --- a/paperforge/commands/dashboard.py +++ b/paperforge/commands/dashboard.py @@ -53,7 +53,6 @@ def run(args) -> int: def _dashboard_from_db(vault: Path) -> dict | None: """Build dashboard stats from paperforge.db. Returns None if DB unavailable.""" - from pathlib import Path as _P db_path = vault / "System" / "PaperForge" / "indexes" / "paperforge.db" if not db_path.exists(): return None diff --git a/paperforge/commands/embed.py b/paperforge/commands/embed.py index e8581394..ddd9ac09 100644 --- a/paperforge/commands/embed.py +++ b/paperforge/commands/embed.py @@ -3,13 +3,10 @@ from __future__ import annotations import argparse import os import sys -from pathlib import Path +from paperforge import __version__ as PF_VERSION from paperforge.core.errors import ErrorCode from paperforge.core.result import PFError, PFResult -from paperforge.memory.chunker import chunk_fulltext -from paperforge.memory.state_snapshot import write_vector_runtime -from paperforge.worker.asset_index import read_index from paperforge.embedding import ( delete_paper_vectors, embed_paper, @@ -20,7 +17,9 @@ from paperforge.embedding import ( read_vector_build_state, ) from paperforge.embedding.preflight import _preflight_check -from paperforge import __version__ as PF_VERSION +from paperforge.memory.chunker import chunk_fulltext +from paperforge.memory.state_snapshot import write_vector_runtime +from paperforge.worker.asset_index import read_index def run(args: argparse.Namespace) -> int: diff --git a/paperforge/commands/memory.py b/paperforge/commands/memory.py index fb5b4b74..89e1941b 100644 --- a/paperforge/commands/memory.py +++ b/paperforge/commands/memory.py @@ -63,9 +63,9 @@ def run(args: argparse.Namespace) -> int: status = get_memory_status(vault) # Write memory-runtime-state.json snapshot (JS-First Memory State) try: - from paperforge.memory.state_snapshot import write_memory_runtime - from paperforge.memory.db import get_memory_db_path, get_connection + from paperforge.memory.db import get_connection, get_memory_db_path from paperforge.memory.schema import get_schema_version + from paperforge.memory.state_snapshot import write_memory_runtime _last_full_build = "" _schema_ver_db = 0 _fts_ok = False diff --git a/paperforge/commands/reading_log.py b/paperforge/commands/reading_log.py index 0e19abb6..c62e32e6 100644 --- a/paperforge/commands/reading_log.py +++ b/paperforge/commands/reading_log.py @@ -2,7 +2,6 @@ from __future__ import annotations import argparse import datetime -import json import re from pathlib import Path diff --git a/paperforge/commands/retrieve.py b/paperforge/commands/retrieve.py index 2cdacc41..e03c9beb 100644 --- a/paperforge/commands/retrieve.py +++ b/paperforge/commands/retrieve.py @@ -2,13 +2,12 @@ from __future__ import annotations import argparse import sys -import json +from paperforge import __version__ as PF_VERSION from paperforge.core.errors import ErrorCode from paperforge.core.result import PFError, PFResult -from paperforge.memory.db import get_connection, get_memory_db_path from paperforge.embedding import retrieve_chunks -from paperforge import __version__ as PF_VERSION +from paperforge.memory.db import get_connection, get_memory_db_path def run(args: argparse.Namespace) -> int: diff --git a/paperforge/commands/runtime_health.py b/paperforge/commands/runtime_health.py index 310a58ca..2cb3b944 100644 --- a/paperforge/commands/runtime_health.py +++ b/paperforge/commands/runtime_health.py @@ -1,7 +1,6 @@ from __future__ import annotations import argparse -import sys from paperforge import __version__ as PF_VERSION from paperforge.core.result import PFResult diff --git a/paperforge/commands/search.py b/paperforge/commands/search.py index 0de275ad..7edb9baf 100644 --- a/paperforge/commands/search.py +++ b/paperforge/commands/search.py @@ -3,11 +3,11 @@ from __future__ import annotations import argparse import sys +from paperforge import __version__ as PF_VERSION from paperforge.core.errors import ErrorCode from paperforge.core.result import PFError, PFResult from paperforge.memory.db import get_connection, get_memory_db_path from paperforge.memory.fts import search_papers -from paperforge import __version__ as PF_VERSION def run(args: argparse.Namespace) -> int: diff --git a/paperforge/commands/status.py b/paperforge/commands/status.py index 31d5c2bc..365abf89 100644 --- a/paperforge/commands/status.py +++ b/paperforge/commands/status.py @@ -8,8 +8,6 @@ import argparse import logging from pathlib import Path -from paperforge.core.result import PFResult - logger = logging.getLogger(__name__) diff --git a/paperforge/doctor/field_validator.py b/paperforge/doctor/field_validator.py index c4a326eb..9f703488 100644 --- a/paperforge/doctor/field_validator.py +++ b/paperforge/doctor/field_validator.py @@ -3,7 +3,6 @@ from __future__ import annotations from pathlib import Path -from typing import Any def validate_entry_fields( diff --git a/paperforge/embedding/_config.py b/paperforge/embedding/_config.py index 612674f2..9c0bda99 100644 --- a/paperforge/embedding/_config.py +++ b/paperforge/embedding/_config.py @@ -23,10 +23,7 @@ def get_api_key(vault: Path) -> str: env_file = vault / ".env" if env_file.exists(): for line in env_file.read_text(encoding="utf-8").splitlines(): - if line.startswith("VECTOR_DB_API_KEY="): - api_key = line.split("=", 1)[1].strip().strip('"').strip("'") - break - elif line.startswith("OPENAI_API_KEY="): + if line.startswith("VECTOR_DB_API_KEY=") or line.startswith("OPENAI_API_KEY="): api_key = line.split("=", 1)[1].strip().strip('"').strip("'") break return api_key diff --git a/paperforge/memory/__init__.py b/paperforge/memory/__init__.py index 5585cd62..07ce98c9 100644 --- a/paperforge/memory/__init__.py +++ b/paperforge/memory/__init__.py @@ -1,7 +1,7 @@ from __future__ import annotations from paperforge.memory.db import get_connection, get_memory_db_path -from paperforge.memory.schema import ensure_schema, drop_all_tables +from paperforge.memory.schema import drop_all_tables, ensure_schema __all__ = [ "get_connection", diff --git a/paperforge/memory/_columns.py b/paperforge/memory/_columns.py index 61c0ce98..727e42c6 100644 --- a/paperforge/memory/_columns.py +++ b/paperforge/memory/_columns.py @@ -2,7 +2,6 @@ from __future__ import annotations import json - PAPER_COLUMNS = [ "zotero_key", "citation_key", "title", "year", "doi", "pmid", "journal", "first_author", "authors_json", "abstract", "domain", diff --git a/paperforge/memory/permanent.py b/paperforge/memory/permanent.py index cf033fc0..6e5a9773 100644 --- a/paperforge/memory/permanent.py +++ b/paperforge/memory/permanent.py @@ -1,7 +1,7 @@ from __future__ import annotations -import json import datetime +import json import logging import secrets from pathlib import Path diff --git a/paperforge/memory/runtime_health.py b/paperforge/memory/runtime_health.py index 7b4329a7..5575a63e 100644 --- a/paperforge/memory/runtime_health.py +++ b/paperforge/memory/runtime_health.py @@ -53,8 +53,8 @@ def _check_bootstrap(vault: Path) -> dict: def _check_read(vault: Path) -> dict: - from paperforge.memory.db import get_memory_db_path from paperforge.config import paperforge_paths + from paperforge.memory.db import get_memory_db_path paths = paperforge_paths(vault) index_path = paths.get("index") diff --git a/paperforge/memory/schema.py b/paperforge/memory/schema.py index 65048c23..b2d29f98 100644 --- a/paperforge/memory/schema.py +++ b/paperforge/memory/schema.py @@ -1,8 +1,7 @@ from __future__ import annotations -import sqlite3 - import logging +import sqlite3 logger = logging.getLogger(__name__) diff --git a/paperforge/memory/state_snapshot.py b/paperforge/memory/state_snapshot.py index 9ab053ba..8f57cd1b 100644 --- a/paperforge/memory/state_snapshot.py +++ b/paperforge/memory/state_snapshot.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json from datetime import datetime, timezone from pathlib import Path diff --git a/paperforge/memory/vector_db.py b/paperforge/memory/vector_db.py index 9372dbef..46964403 100644 --- a/paperforge/memory/vector_db.py +++ b/paperforge/memory/vector_db.py @@ -4,16 +4,16 @@ from __future__ import annotations import warnings from paperforge.embedding import ( - delete_paper_vectors, # noqa: F401 - embed_paper, # noqa: F401 - get_collection, # noqa: F401 - get_embed_status, # noqa: F401 - get_vector_build_state_path, # noqa: F401 - get_vector_db_path, # noqa: F401 - mark_vector_build_state, # noqa: F401 - read_vector_build_state, # noqa: F401 - retrieve_chunks, # noqa: F401 - write_vector_build_state, # noqa: F401 + delete_paper_vectors, # noqa: F401 + embed_paper, # noqa: F401 + get_collection, # noqa: F401 + get_embed_status, # noqa: F401 + get_vector_build_state_path, # noqa: F401 + get_vector_db_path, # noqa: F401 + mark_vector_build_state, # noqa: F401 + read_vector_build_state, # noqa: F401 + retrieve_chunks, # noqa: F401 + write_vector_build_state, # noqa: F401 ) warnings.warn( diff --git a/paperforge/plugin/main.js b/paperforge/plugin/main.js index 4014b2ad..a69876e6 100644 --- a/paperforge/plugin/main.js +++ b/paperforge/plugin/main.js @@ -1982,21 +1982,26 @@ class PaperForgeStatusView extends ItemView { for (const qa of pairs) { const item = card.createEl('div', { cls: 'paperforge-discussion-item' }); const qEl = item.createEl('div', { cls: 'paperforge-discussion-q' }); - await MarkdownRenderer.render(this.app, '**提问：**' + qa.question, qEl, mdPath, this); + qEl.createEl('span', { cls: 'paperforge-discussion-q-label', text: '提问：' }); + qEl.createEl('span', { cls: 'paperforge-discussion-q-text', text: qa.question }); const aEl = item.createEl('div', { cls: 'paperforge-discussion-a' }); + let longAnswer = false; if (qa.answer && qa.answer.length > 500) { - aEl.style.maxHeight = '200px'; - aEl.style.overflow = 'hidden'; - const toggle = item.createEl('button', { cls: 'paperforge-expand-btn', text: '展开更多 ▾' }); - let expanded = false; - toggle.addEventListener('click', () => { - expanded = !expanded; - aEl.style.maxHeight = expanded ? '' : '200px'; - toggle.setText(expanded ? '收起 ▴' : '展开更多 ▾'); - }); + longAnswer = true; + aEl.classList.add('paperforge-discussion-a-collapsed'); } await MarkdownRenderer.render(this.app, qa.answer || '', aEl, mdPath, this); + + if (longAnswer) { + let expanded = false; + item.style.cursor = 'pointer'; + item.addEventListener('click', () => { + expanded = !expanded; + aEl.classList.toggle('paperforge-discussion-a-collapsed', !expanded); + aEl.classList.toggle('paperforge-discussion-a-expanded', expanded); + }); + } } // "查看全部" link diff --git a/paperforge/plugin/styles.css b/paperforge/plugin/styles.css index f742a844..95d1a09f 100644 --- a/paperforge/plugin/styles.css +++ b/paperforge/plugin/styles.css @@ -1926,19 +1926,62 @@ } .paperforge-discussion-q { - font-size: 14px; - font-weight: 600; - color: var(--text-normal); - margin-bottom: 4px; + padding: 8px 10px; + margin-bottom: 8px; + border-radius: 6px; + background: color-mix(in srgb, #3e5a47 15%, transparent); line-height: 1.5; } +.paperforge-discussion-q-label { + font-size: 13px; + font-weight: 500; + color: var(--text-muted); + margin-right: 4px; +} + +.paperforge-discussion-q-text { + color: var(--text-normal); + font-weight: 600; +} + .paperforge-discussion-a { font-size: 14px; color: var(--text-normal); line-height: 1.68; } +.paperforge-discussion-a-collapsed { + max-height: 200px; + overflow: hidden; + position: relative; +} + +.paperforge-discussion-a-collapsed::after { + content: ''; + position: absolute; + bottom: 0; + left: 0; + right: 0; + height: 60px; + background: linear-gradient(transparent, var(--background-primary)); + pointer-events: none; +} + +.paperforge-discussion-a-expanded { + max-height: none; + overflow: visible; +} + +.paperforge-discussion-item { + position: relative; + margin-bottom: 12px; +} + +.paperforge-discussion-item:last-child { + margin-bottom: 0; +} + .paperforge-discussion-viewall { diff --git a/paperforge/schema/__init__.py b/paperforge/schema/__init__.py index c5e9b674..d5065ff4 100644 --- a/paperforge/schema/__init__.py +++ b/paperforge/schema/__init__.py @@ -18,7 +18,7 @@ def load_field_registry(path: Path | None = None) -> dict: if not path.exists(): return {} - with open(path, "r", encoding="utf-8") as f: + with open(path, encoding="utf-8") as f: data = yaml.safe_load(f) return data if isinstance(data, dict) else {} diff --git a/paperforge/setup/checker.py b/paperforge/setup/checker.py index 5f03b7a1..75d965a0 100644 --- a/paperforge/setup/checker.py +++ b/paperforge/setup/checker.py @@ -3,7 +3,6 @@ from __future__ import annotations import shutil -import subprocess import sys from pathlib import Path from typing import Any diff --git a/paperforge/setup/config_writer.py b/paperforge/setup/config_writer.py index 38a301e6..1348c374 100644 --- a/paperforge/setup/config_writer.py +++ b/paperforge/setup/config_writer.py @@ -70,7 +70,7 @@ class ConfigWriter: if not self.config_path.exists(): return None try: - with open(self.config_path, "r", encoding="utf-8") as f: + with open(self.config_path, encoding="utf-8") as f: return json.load(f) except Exception: return None diff --git a/paperforge/setup/plan.py b/paperforge/setup/plan.py index 836492b6..a443812f 100644 --- a/paperforge/setup/plan.py +++ b/paperforge/setup/plan.py @@ -3,16 +3,15 @@ from __future__ import annotations import json -import sys +from collections.abc import Callable from pathlib import Path -from typing import Callable from paperforge.setup import SetupStepResult +from paperforge.setup.agent import AgentInstaller from paperforge.setup.checker import SetupChecker from paperforge.setup.config_writer import ConfigWriter -from paperforge.setup.vault import VaultInitializer from paperforge.setup.runtime import RuntimeInstaller -from paperforge.setup.agent import AgentInstaller +from paperforge.setup.vault import VaultInitializer ProgressCallback = Callable[[str], None] diff --git a/paperforge/setup/runtime.py b/paperforge/setup/runtime.py index 0fffab9d..6dbf3347 100644 --- a/paperforge/setup/runtime.py +++ b/paperforge/setup/runtime.py @@ -4,13 +4,12 @@ from __future__ import annotations import subprocess import sys +from collections.abc import Callable from pathlib import Path -from typing import Callable from paperforge.core.errors import ErrorCode from paperforge.setup import SetupStepResult - ProgressCallback = Callable[[str], None] @@ -74,9 +73,7 @@ class RuntimeInstaller: ) else: error_code = ErrorCode.INTERNAL_ERROR - if "pip" in stderr.lower(): - error_code = ErrorCode.INTERNAL_ERROR - elif "connection" in stderr.lower() or "timeout" in stderr.lower(): + if "pip" in stderr.lower() or "connection" in stderr.lower() or "timeout" in stderr.lower(): error_code = ErrorCode.INTERNAL_ERROR return SetupStepResult( diff --git a/paperforge/worker/_domain.py b/paperforge/worker/_domain.py index 5e4d8f4f..5524c6fb 100644 --- a/paperforge/worker/_domain.py +++ b/paperforge/worker/_domain.py @@ -13,8 +13,8 @@ from __future__ import annotations import logging from pathlib import Path -from paperforge.worker._utils import read_json, write_json from paperforge.adapters.collections import build_collection_lookup +from paperforge.worker._utils import read_json, write_json logger = logging.getLogger(__name__) diff --git a/paperforge/worker/base_views.py b/paperforge/worker/base_views.py index 3df9df84..7976b527 100644 --- a/paperforge/worker/base_views.py +++ b/paperforge/worker/base_views.py @@ -4,11 +4,8 @@ import logging import os from pathlib import Path -from paperforge.config import paperforge_paths from paperforge.worker._utils import ( - read_json, slugify_filename, - write_json, ) logger = logging.getLogger(__name__) @@ -284,7 +281,6 @@ def merge_base_views(existing_content: str | None, new_views: list[dict]) -> str Returns: Merged .base file content with PaperForge views updated, user views preserved. """ - import re PROPERTIES_YAML = """properties: zotero_key: diff --git a/paperforge/worker/discussion.py b/paperforge/worker/discussion.py index 6e3e8c0a..69daf29f 100644 --- a/paperforge/worker/discussion.py +++ b/paperforge/worker/discussion.py @@ -19,10 +19,11 @@ import os import sys import tempfile import uuid -import filelock from datetime import datetime, timezone from pathlib import Path +import filelock + from paperforge.config import paperforge_paths from paperforge.worker._utils import slugify_filename diff --git a/paperforge/worker/ocr.py b/paperforge/worker/ocr.py index 3bf54ee4..fe7cc5a4 100644 --- a/paperforge/worker/ocr.py +++ b/paperforge/worker/ocr.py @@ -35,13 +35,13 @@ def _read_dotenv(vault: Path, key: str) -> str: return "" -from paperforge.worker.asset_index import refresh_index_entry from paperforge.worker._retry import retry_with_meta from paperforge.worker._utils import ( pipeline_paths, read_json, write_json, ) +from paperforge.worker.asset_index import refresh_index_entry from paperforge.worker.sync import ( load_control_actions, load_export_rows, diff --git a/paperforge/worker/paper_meta.py b/paperforge/worker/paper_meta.py index 039b35a3..da2e44af 100644 --- a/paperforge/worker/paper_meta.py +++ b/paperforge/worker/paper_meta.py @@ -8,7 +8,7 @@ while preserving all state for tools and AI context. from __future__ import annotations import json -from datetime import datetime, timezone, timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path BEIJING = timezone(timedelta(hours=8)) diff --git a/paperforge/worker/paper_resolver.py b/paperforge/worker/paper_resolver.py index 7c860be6..f673ae5c 100644 --- a/paperforge/worker/paper_resolver.py +++ b/paperforge/worker/paper_resolver.py @@ -13,7 +13,6 @@ import re import sys from dataclasses import dataclass, field from pathlib import Path -from typing import Optional @dataclass @@ -81,7 +80,7 @@ class PaperResolver: self._items = [] self._loaded = True - def resolve_key(self, key: str) -> Optional[PaperWorkspace]: + def resolve_key(self, key: str) -> PaperWorkspace | None: """Exact match on zotero_key.""" self._ensure_loaded() for entry in self._items: @@ -89,7 +88,7 @@ class PaperResolver: return self._build_workspace(entry) return None - def resolve_doi(self, doi: str) -> Optional[PaperWorkspace]: + def resolve_doi(self, doi: str) -> PaperWorkspace | None: """Exact match on DOI (case-insensitive, normalized).""" self._ensure_loaded() normalized = self._normalize_doi(doi) @@ -101,10 +100,10 @@ class PaperResolver: def search( self, - title: Optional[str] = None, - author: Optional[str] = None, - year: Optional[int | str] = None, - domain: Optional[str] = None, + title: str | None = None, + author: str | None = None, + year: int | str | None = None, + domain: str | None = None, limit: int = 20, ) -> list[PaperWorkspace]: """Multi-field search with substring matching, sorted by relevance score. @@ -214,7 +213,7 @@ class PaperResolver: return s -def _resolve_ocr_base(paths: dict[str, Path], key: str) -> Optional[Path]: +def _resolve_ocr_base(paths: dict[str, Path], key: str) -> Path | None: """Get the OCR directory for a given zotero key.""" ocr_dir = paths.get("ocr") if ocr_dir and key: diff --git a/paperforge/worker/repair.py b/paperforge/worker/repair.py index b8a93671..b9d9d6dd 100644 --- a/paperforge/worker/repair.py +++ b/paperforge/worker/repair.py @@ -4,12 +4,12 @@ import logging import re from pathlib import Path -from paperforge.config import load_vault_config, paperforge_paths -from paperforge.worker.asset_index import refresh_index_entry +from paperforge.config import load_vault_config from paperforge.worker._utils import ( read_json, write_json, ) +from paperforge.worker.asset_index import refresh_index_entry from paperforge.worker.ocr import validate_ocr_meta from paperforge.worker.sync import ( load_export_rows, diff --git a/paperforge/worker/status.py b/paperforge/worker/status.py index 082aed5e..0eb087d6 100644 --- a/paperforge/worker/status.py +++ b/paperforge/worker/status.py @@ -10,12 +10,14 @@ from json import JSONDecodeError from pathlib import Path from paperforge.config import ( + CONFIG_PATH_KEYS, + get_paperforge_schema_version, load_vault_config, paperforge_paths, read_paperforge_json, - CONFIG_PATH_KEYS, - get_paperforge_schema_version, ) +from paperforge.core.result import PFResult +from paperforge.memory.state_snapshot import write_memory_runtime from paperforge.worker._domain import load_domain_config from paperforge.worker._utils import ( pipeline_paths, @@ -23,9 +25,6 @@ from paperforge.worker._utils import ( write_json, ) from paperforge.worker.base_views import ensure_base_views -from paperforge.memory.state_snapshot import write_memory_runtime - -from paperforge.core.result import PFResult logger = logging.getLogger(__name__) @@ -610,8 +609,8 @@ def run_doctor(vault: Path, verbose: bool = False, json_output: bool = False) -> # --- Field registry validation (Phase 59) --- try: - from paperforge.schema import load_field_registry from paperforge.doctor.field_validator import validate_frontmatter_from_file + from paperforge.schema import load_field_registry registry = load_field_registry() except Exception: @@ -673,7 +672,8 @@ def run_doctor(vault: Path, verbose: bool = False, json_output: bool = False) -> # --- Index Health section (Phase 25: derived from canonical index) --- try: - from paperforge.worker.asset_index import read_index as _read_idx, summarize_index as _summarize_idx + from paperforge.worker.asset_index import read_index as _read_idx + from paperforge.worker.asset_index import summarize_index as _summarize_idx _summary = _summarize_idx(vault) except Exception: @@ -1073,8 +1073,8 @@ def run_status(vault: Path, verbose: bool = False, json_output: bool = False) -> # Write memory-runtime-state.json snapshot (JS-First Memory State) try: + from paperforge.memory.db import get_connection, get_memory_db_path from paperforge.memory.query import get_memory_status - from paperforge.memory.db import get_memory_db_path, get_connection from paperforge.memory.schema import get_schema_version ms = get_memory_status(vault) _last_full_build = "" diff --git a/paperforge/worker/sync.py b/paperforge/worker/sync.py index e996a055..331269a2 100644 --- a/paperforge/worker/sync.py +++ b/paperforge/worker/sync.py @@ -6,7 +6,6 @@ from __future__ import annotations # New logic → services/ / adapters/ / core/. # This file: deletion, migration, legacy wrappers only. # ============================================================================= - import html import logging import os @@ -14,32 +13,29 @@ import re import urllib.parse from datetime import datetime, timezone from pathlib import Path -from typing import Optional from xml.etree import ElementTree as ET import requests -from paperforge.config import load_vault_config, paperforge_paths -from paperforge.worker._domain import build_collection_lookup, load_domain_config, load_domain_collections +import paperforge.worker.asset_index as asset_index +from paperforge.adapters.bbt import ( + collection_fields, + load_export_rows, +) +from paperforge.adapters.obsidian_frontmatter import ( + _legacy_control_flags, + _read_frontmatter_optional_bool_from_text, + canonicalize_decision, + compute_final_collection, + extract_preserved_deep_reading, + update_frontmatter_field, +) from paperforge.adapters.zotero_paths import ( - absolutize_vault_path, obsidian_wikilink_for_path, obsidian_wikilink_for_pdf, ) -from paperforge.adapters.obsidian_frontmatter import ( - _add_missing_frontmatter_fields, - _extract_section, - _legacy_control_flags, - _read_frontmatter_bool_from_text, - _read_frontmatter_optional_bool_from_text, - canonicalize_decision, - candidate_markdown, - compute_final_collection, - extract_preserved_deep_reading, - generate_review, - has_deep_reading_content, - update_frontmatter_field, -) +from paperforge.config import load_vault_config +from paperforge.worker._domain import load_domain_collections, load_domain_config from paperforge.worker._utils import ( _extract_year, lookup_impact_factor, @@ -54,18 +50,6 @@ from paperforge.worker._utils import ( yaml_quote, ) -from paperforge.adapters.bbt import ( - _identify_main_pdf, - _normalize_attachment_path, - collection_fields, - extract_authors, - load_export_rows, - resolve_item_collection_paths, -) - - -import paperforge.worker.asset_index as asset_index - logger = logging.getLogger(__name__) @@ -1231,7 +1215,6 @@ def run_index_refresh( json_output: If True, suppress human-readable print output. """ from paperforge.worker.base_views import ensure_base_views - from paperforge.worker.ocr import validate_ocr_meta paths = pipeline_paths(vault) config = load_domain_config(paths) diff --git a/paperforge/worker/update.py b/paperforge/worker/update.py index 0ac69a5e..205c5c7e 100644 --- a/paperforge/worker/update.py +++ b/paperforge/worker/update.py @@ -1,7 +1,7 @@ from __future__ import annotations -import hashlib import base64 +import hashlib import json import logging import os @@ -14,11 +14,7 @@ import zipfile from datetime import datetime from pathlib import Path -from paperforge.config import load_vault_config, paperforge_paths -from paperforge.worker._utils import ( - read_json, - write_json, -) +from paperforge.config import load_vault_config from paperforge.worker.status import GITHUB_REPO, GITHUB_ZIP, UPDATEABLE_PATHS logger = logging.getLogger(__name__) @@ -28,6 +24,7 @@ GITHUB_PIP_SOURCE = f"git+https://github.com/{GITHUB_REPO}.git" def _sync_obsidian_plugin(vault: Path) -> None: """Reload utils and sync the Obsidian plugin into the current vault.""" import importlib + import paperforge.worker._utils as _pf_utils importlib.reload(_pf_utils) @@ -245,8 +242,8 @@ def update_via_zip(vault: Path) -> bool: def _deploy_all_skills(vault: Path) -> None: """Deploy latest skills and AGENTS.md to vault after update.""" try: - from paperforge.services.skill_deploy import deploy_skills from paperforge.config import load_vault_config + from paperforge.services.skill_deploy import deploy_skills config = load_vault_config(vault) agent_key = config.get("agent_platform") or "opencode" diff --git a/paperforge/worker/vector_db.py b/paperforge/worker/vector_db.py index 75c170f4..91b3b066 100644 --- a/paperforge/worker/vector_db.py +++ b/paperforge/worker/vector_db.py @@ -4,7 +4,7 @@ from __future__ import annotations import warnings from paperforge.embedding.preflight import _preflight_check # noqa: F401 -from paperforge.embedding.status import get_embed_status # noqa: F401 +from paperforge.embedding.status import get_embed_status # noqa: F401 warnings.warn( "paperforge.worker.vector_db is deprecated, use paperforge.embedding instead",