From 2b552be9e6938abc4e02075a6fedf9409875118a Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Sat, 16 May 2026 22:38:43 +0800 Subject: [PATCH] feat: runtime contract hardening + skill/command truth alignment (Package A+B) Atomic snapshots, canonical index mutation serialization, sync post-clean truth, plugin path config-awareness, embed stop signal honesty, full snapshot bootstrap, pf_ prefix unification, workflow command/lifecycle/path corrections, mechanical/cognitive route separation with unknown-command guard. --- .../snapshots/paths_json/default_config.json | 2 +- paperforge/commands/agent_context.py | 6 +- paperforge/commands/embed.py | 23 ++++-- paperforge/config.py | 1 - paperforge/core/io.py | 11 ++- paperforge/memory/db.py | 8 +- paperforge/memory/runtime_health.py | 4 +- paperforge/memory/state_snapshot.py | 7 +- paperforge/memory/vector_db.py | 5 +- paperforge/plugin/main.js | 62 ++++++++++++--- paperforge/plugin/src/testable.js | 55 ++++++++++++- paperforge/plugin/tests/runtime.test.mjs | 47 +++++++++++ paperforge/services/sync_service.py | 12 +++ paperforge/skills/paperforge/SKILL.md | 15 +++- .../paperforge/workflows/deep-reading.md | 2 +- .../skills/paperforge/workflows/paper-qa.md | 16 +++- .../paperforge/workflows/paper-search.md | 11 +-- .../paperforge/workflows/project-log.md | 4 +- .../paperforge/workflows/reading-log.md | 2 +- paperforge/worker/asset_index.py | 77 ++++++++++++++----- paperforge/worker/repair.py | 45 ++--------- tests/cli/test_json_contracts.py | 2 +- tests/test_config.py | 12 +-- tests/unit/commands/test_embed.py | 33 ++++++++ tests/unit/core/test_io.py | 33 ++++++++ tests/unit/memory/test_refresh.py | 32 ++++++++ 26 files changed, 413 insertions(+), 114 deletions(-) create mode 100644 tests/unit/core/test_io.py diff --git a/fixtures/snapshots/paths_json/default_config.json b/fixtures/snapshots/paths_json/default_config.json index d8c3db2b..dd5e377f 100644 --- a/fixtures/snapshots/paths_json/default_config.json +++ b/fixtures/snapshots/paths_json/default_config.json @@ -1,5 +1,5 @@ { "vault": "", "worker_script": "/System/PaperForge/paperforge/worker", - "ld_deep_script": "/.opencode/skills/literature-qa/scripts/ld_deep.py" + "pf_deep_script": "/.opencode/skills/paperforge/scripts/pf_deep.py" } diff --git a/paperforge/commands/agent_context.py b/paperforge/commands/agent_context.py index 4ad294b5..453c9442 100644 --- a/paperforge/commands/agent_context.py +++ b/paperforge/commands/agent_context.py @@ -14,12 +14,12 @@ COMMANDS = { "purpose": "Look up one paper's full status and recommended next action", }, "search": { - "usage": "paperforge search --json [--collection NAME] [--domain NAME] [--ocr done|pending] [--year-from N] [--year-to N] [--limit N]", - "purpose": "Full-text search with optional collection/domain/lifecycle filters", + "usage": "paperforge search --json [--domain NAME] [--ocr done|pending|failed|processing] [--year-from N] [--year-to N] [--limit N] [--lifecycle indexed|pdf_ready|fulltext_ready|deep_read_done]", + "purpose": "Full-text search with optional domain and workflow-state filters", }, "retrieve": { "usage": "paperforge retrieve --json [--limit N]", - "purpose": "Search OCR fulltext chunks for evidence paragraphs (coming soon)", + "purpose": "Search OCR fulltext chunks for evidence paragraphs", }, "deep": { "usage": "/pf-deep ", diff --git a/paperforge/commands/embed.py b/paperforge/commands/embed.py index 560973dc..9041c4d1 100644 --- a/paperforge/commands/embed.py +++ b/paperforge/commands/embed.py @@ -1,6 +1,7 @@ from __future__ import annotations import argparse +import os import sys from pathlib import Path @@ -78,15 +79,27 @@ def run(args: argparse.Namespace) -> int: import signal try: os.kill(pid, signal.SIGTERM) - except Exception: - pass + except Exception as exc: + result = PFResult( + ok=False, + command="embed stop", + version=PF_VERSION, + error=PFError(code=ErrorCode.INTERNAL_ERROR, message=f"Failed to stop embed build: {exc}"), + data={"state": "running", "pid": pid}, + ) + if args.json: + print(result.to_json()) + else: + print(result.error.message, file=sys.stderr) + return 1 mark_vector_build_state(vault, status="stopping", message="Stop requested") - result = PFResult(ok=True, command="embed stop", version=PF_VERSION, - data={"state": "stopping" if pid else "idle"}) + result = PFResult(ok=True, command="embed stop", version=PF_VERSION, data={"state": "stopping", "pid": pid}) + else: + result = PFResult(ok=True, command="embed stop", version=PF_VERSION, data={"state": "idle"}) if args.json: print(result.to_json()) else: - print("Stop requested." if pid else "No active build.") + print("Stop requested." if result.data["state"] == "stopping" else "No active build.") return 0 # Build diff --git a/paperforge/config.py b/paperforge/config.py index 4ea35594..528ac80a 100644 --- a/paperforge/config.py +++ b/paperforge/config.py @@ -331,7 +331,6 @@ def paperforge_paths( # ── v2.2: canonical locations below paperforge/ ── "config": paperforge / "config" / "domain-collections.json", "index": paperforge / "indexes" / "formal-library.json", - "memory_db": paperforge / "indexes" / "paperforge.db", } diff --git a/paperforge/core/io.py b/paperforge/core/io.py index 80697ff7..389ac90f 100644 --- a/paperforge/core/io.py +++ b/paperforge/core/io.py @@ -9,7 +9,14 @@ def read_json(path: Path): return json.loads(path.read_text(encoding="utf-8")) +def write_json_atomic(path: Path, data) -> None: + """Write JSON atomically using a temp file and replace.""" + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(f"{path.suffix}.tmp") + tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + tmp.replace(path) + + def write_json(path: Path, data) -> None: """Write data as JSON, creating parent directories as needed.""" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + write_json_atomic(path, data) diff --git a/paperforge/memory/db.py b/paperforge/memory/db.py index 19dbf837..146df734 100644 --- a/paperforge/memory/db.py +++ b/paperforge/memory/db.py @@ -9,10 +9,10 @@ from paperforge.config import paperforge_paths def get_memory_db_path(vault: Path) -> Path: """Return the absolute path to paperforge.db.""" paths = paperforge_paths(vault) - db_path = paths.get("memory_db") - if not db_path: - raise FileNotFoundError("memory_db path not configured") - return db_path + index_path = paths.get("index") + if not index_path: + raise FileNotFoundError("index path not configured") + return index_path.parent / "paperforge.db" def get_connection(db_path: Path, read_only: bool = False) -> sqlite3.Connection: diff --git a/paperforge/memory/runtime_health.py b/paperforge/memory/runtime_health.py index 178a000e..6aa0b144 100644 --- a/paperforge/memory/runtime_health.py +++ b/paperforge/memory/runtime_health.py @@ -110,11 +110,11 @@ def _check_index(vault: Path) -> dict: if version: return _layer("ok", [f"Schema version: {version['value']}"]) return _layer("degraded", ["Schema version unknown"], - "Rebuild memory DB", "paperforge memory build --force") + "Rebuild memory DB", "paperforge memory build") except Exception as e: return _layer("blocked", [f"DB query failed: {e}"], "Rebuild memory DB", - "paperforge memory build --force") + "paperforge memory build") def _check_vector(vault: Path) -> dict: diff --git a/paperforge/memory/state_snapshot.py b/paperforge/memory/state_snapshot.py index f9308bf0..924f43ef 100644 --- a/paperforge/memory/state_snapshot.py +++ b/paperforge/memory/state_snapshot.py @@ -5,6 +5,7 @@ from datetime import datetime, timezone from pathlib import Path from paperforge.config import paperforge_paths +from paperforge.core.io import write_json_atomic def _snapshot_dir(vault: Path) -> Path: @@ -31,7 +32,7 @@ def write_memory_runtime(vault: Path, *, paper_count_db: int, "fts_ready": fts_ready, } path = _snapshot_dir(vault) / "memory-runtime-state.json" - path.write_text(json.dumps(snap, ensure_ascii=False, indent=2), encoding="utf-8") + write_json_atomic(path, snap) def write_vector_runtime(vault: Path, *, enabled: bool, mode: str, model: str, @@ -53,9 +54,9 @@ def write_vector_runtime(vault: Path, *, enabled: bool, mode: str, model: str, "build_state": build_state or {}, } path = _snapshot_dir(vault) / "vector-runtime-state.json" - path.write_text(json.dumps(snap, ensure_ascii=False, indent=2), encoding="utf-8") + write_json_atomic(path, snap) def write_runtime_health(vault: Path, health_data: dict) -> None: path = _snapshot_dir(vault) / "runtime-health.json" - path.write_text(json.dumps(health_data, ensure_ascii=False, indent=2), encoding="utf-8") + write_json_atomic(path, health_data) diff --git a/paperforge/memory/vector_db.py b/paperforge/memory/vector_db.py index d03221c6..c3be4730 100644 --- a/paperforge/memory/vector_db.py +++ b/paperforge/memory/vector_db.py @@ -40,7 +40,8 @@ def get_vector_db_path(vault: Path) -> Path: """Return the ChromaDB persistence directory.""" from paperforge.config import paperforge_paths paths = paperforge_paths(vault) - return (paths.get("memory_db", paths.get("index", vault / "System" / "PaperForge"))).parent / "vectors" + index_path = paths.get("index", vault / "System" / "PaperForge" / "indexes" / "formal-library.json") + return index_path.parent / "vectors" def get_collection(vault: Path): @@ -291,7 +292,7 @@ def get_vector_build_state_path(vault: Path) -> Path: """Return path to vector-build-state.json.""" from paperforge.config import paperforge_paths paths = paperforge_paths(vault) - index_dir = (paths.get("memory_db", paths.get("index", vault / "System" / "PaperForge"))).parent + index_dir = paths.get("index", vault / "System" / "PaperForge" / "indexes" / "formal-library.json").parent return index_dir / "vector-build-state.json" diff --git a/paperforge/plugin/main.js b/paperforge/plugin/main.js index 4a99c23d..ce59b6cd 100644 --- a/paperforge/plugin/main.js +++ b/paperforge/plugin/main.js @@ -10,8 +10,38 @@ const memoryState = (() => { const path = require('path'); const { execFileSync } = require('node:child_process'); + function readPathConfig(vaultPath) { + const pfPath = path.join(vaultPath, 'paperforge.json'); + const defaults = { + system_dir: 'System', + resources_dir: 'Resources', + literature_dir: 'Literature', + base_dir: 'Bases', + }; + + try { + if (!fs.existsSync(pfPath)) { + return { ...defaults, _warning: 'paperforge.json not found; using defaults' }; + } + const raw = fs.readFileSync(pfPath, 'utf-8'); + const data = JSON.parse(raw); + const vc = data.vault_config || {}; + return { + system_dir: vc.system_dir || data.system_dir || defaults.system_dir, + resources_dir: vc.resources_dir || data.resources_dir || defaults.resources_dir, + literature_dir: vc.literature_dir || data.literature_dir || defaults.literature_dir, + base_dir: vc.base_dir || data.base_dir || defaults.base_dir, + _warning: null, + }; + } catch (e) { + console.warn('PaperForge: Failed to read paperforge.json, using defaults', e); + return { ...defaults, _warning: 'paperforge.json invalid; using defaults' }; + } + } + function resolveVaultPaths(vaultPath) { - const systemDir = path.join(vaultPath, 'System', 'PaperForge'); + const cfg = readPathConfig(vaultPath); + const systemDir = path.join(vaultPath, cfg.system_dir, 'PaperForge'); return { vault: vaultPath, systemDir, @@ -22,8 +52,11 @@ const memoryState = (() => { vectorStatePath: path.join(systemDir, 'indexes', 'vector-runtime-state.json'), healthStatePath: path.join(systemDir, 'indexes', 'runtime-health.json'), buildStatePath: path.join(systemDir, 'indexes', 'vector-build-state.json'), + exportsDir: path.join(systemDir, 'exports'), + ocrDir: path.join(systemDir, 'ocr'), pluginDataPath: path.join(vaultPath, '.obsidian', 'plugins', 'paperforge', 'data.json'), pfJsonPath: path.join(vaultPath, 'paperforge.json'), + configWarning: cfg._warning, }; } @@ -99,7 +132,7 @@ const memoryState = (() => { return _cachedPython; } } - var sysCandidates = [{path:'python',extraArgs:[]},{path:'python3',extraArgs:[]}]; + var sysCandidates = [{path:'py',extraArgs:['-3']},{path:'python',extraArgs:[]},{path:'python3',extraArgs:[]}]; for (var j = 0; j < sysCandidates.length; j++) { try { var c = sysCandidates[j]; @@ -170,6 +203,7 @@ function resolvePythonExecutable(vaultPath, settings, _fs, _execFileSync) { } const systemCandidates = [ + { path: "py", extraArgs: ["-3"] }, { path: "python", extraArgs: [] }, { path: "python3", extraArgs: [] }, ]; @@ -242,7 +276,7 @@ function buildRuntimeInstallCommand(pythonExe, version, extraArgs) { const gitUrl = `git+https://github.com/LLLin000/PaperForge.git@v${version}`; const pypiArgs = [...extraArgs, "-m", "pip", "install", "--upgrade", pypiPkg]; const gitArgs = [...extraArgs, "-m", "pip", "install", "--upgrade", gitUrl]; - return { cmd: pythonExe, pypiArgs, gitArgs, timeout: 120000 }; + return { cmd: pythonExe, url: gitUrl.replace('@v', '@'), args: gitArgs, pypiArgs, gitArgs, timeout: 120000 }; } function parseRuntimeStatus(err, stdout, stderr) { @@ -4618,7 +4652,7 @@ module.exports = class PaperForgePlugin extends Plugin { /* ── Auto-update PaperForge (deferred — don't slow startup) ── */ - if (this.settings.auto_update !== false && this.settings.setup_complete) { + if (this.settings.auto_update_on_startup === true && this.settings.setup_complete) { setTimeout(() => this._autoUpdate(), 3000); } this._startFilePolling(); @@ -4627,14 +4661,22 @@ module.exports = class PaperForgePlugin extends Plugin { (() => { const vp = this.app.vault.adapter.basePath; if (!vp) return; - const memSnap = require('path').join(vp, 'System', 'PaperForge', 'indexes', 'memory-runtime-state.json'); + const runtimePaths = memoryState.resolveVaultPaths(vp); + const memSnap = runtimePaths.memoryStatePath; const fs = require('fs'); if (!fs.existsSync(memSnap)) { // No snapshots — first launch or upgrade. Fire and forget. const py = resolvePythonExecutable(vp, this.settings); const { execFile } = require('node:child_process'); - const args = [...py.extraArgs, '-m', 'paperforge', '--vault', vp, 'runtime-health', '--json']; - execFile(py.path, args, { cwd: vp, timeout: 60000, windowsHide: true }, () => {}); + const commands = [ + ['runtime-health', '--json'], + ['memory', 'status', '--json'], + ['embed', 'status', '--json'], + ]; + commands.forEach((cmdArgs) => { + const args = [...py.extraArgs, '-m', 'paperforge', '--vault', vp, ...cmdArgs]; + execFile(py.path, args, { cwd: vp, timeout: 60000, windowsHide: true }, () => {}); + }); } })(); } @@ -4692,7 +4734,7 @@ module.exports = class PaperForgePlugin extends Plugin { _checkExports(vaultPath, fs, path, exec) { if (this._autoSyncRunning) return; - const exportsDir = path.join(vaultPath, 'System', 'PaperForge', 'exports'); + const exportsDir = memoryState.resolveVaultPaths(vaultPath).exportsDir; if (!fs.existsSync(exportsDir)) return; let newestMtime = 0; @@ -4728,7 +4770,7 @@ module.exports = class PaperForgePlugin extends Plugin { try { const fs = require('fs'); const path = require('path'); - const exportsDir = path.join(vaultPath, 'System', 'PaperForge', 'exports'); + const exportsDir = memoryState.resolveVaultPaths(vaultPath).exportsDir; let newest = 0; fs.readdirSync(exportsDir).forEach(f => { if (!f.endsWith('.json')) return; @@ -4741,7 +4783,7 @@ module.exports = class PaperForgePlugin extends Plugin { _checkOcr(vaultPath, fs, path, exec) { if (this._autoSyncRunning) return; - const ocrDir = path.join(vaultPath, 'System', 'PaperForge', 'ocr'); + const ocrDir = memoryState.resolveVaultPaths(vaultPath).ocrDir; if (!fs.existsSync(ocrDir)) return; try { diff --git a/paperforge/plugin/src/testable.js b/paperforge/plugin/src/testable.js index f1ee3742..addbe4e2 100644 --- a/paperforge/plugin/src/testable.js +++ b/paperforge/plugin/src/testable.js @@ -6,6 +6,56 @@ const fs = require('fs'); const path = require('path'); const { execFile, spawn } = require('node:child_process'); +function readPathConfig(vaultPath, _fs) { + const f = _fs || fs; + const pfPath = path.join(vaultPath, 'paperforge.json'); + const defaults = { + system_dir: 'System', + resources_dir: 'Resources', + literature_dir: 'Literature', + base_dir: 'Bases', + }; + + try { + if (!f.existsSync(pfPath)) { + return { ...defaults, _warning: 'paperforge.json not found; using defaults' }; + } + const raw = f.readFileSync(pfPath, 'utf-8'); + const data = JSON.parse(raw); + const vc = data.vault_config || {}; + return { + system_dir: vc.system_dir || data.system_dir || defaults.system_dir, + resources_dir: vc.resources_dir || data.resources_dir || defaults.resources_dir, + literature_dir: vc.literature_dir || data.literature_dir || defaults.literature_dir, + base_dir: vc.base_dir || data.base_dir || defaults.base_dir, + _warning: null, + }; + } catch { + return { ...defaults, _warning: 'paperforge.json invalid; using defaults' }; + } +} + +function resolveRuntimePaths(vaultPath, _fs) { + const cfg = readPathConfig(vaultPath, _fs); + const systemRoot = path.join(vaultPath, cfg.system_dir, 'PaperForge'); + return { + vault: vaultPath, + systemDir: systemRoot, + indexesDir: path.join(systemRoot, 'indexes'), + logsDir: path.join(systemRoot, 'logs'), + dbPath: path.join(systemRoot, 'indexes', 'paperforge.db'), + memoryStatePath: path.join(systemRoot, 'indexes', 'memory-runtime-state.json'), + vectorStatePath: path.join(systemRoot, 'indexes', 'vector-runtime-state.json'), + healthStatePath: path.join(systemRoot, 'indexes', 'runtime-health.json'), + buildStatePath: path.join(systemRoot, 'indexes', 'vector-build-state.json'), + exportsDir: path.join(systemRoot, 'exports'), + ocrDir: path.join(systemRoot, 'ocr'), + pluginDataPath: path.join(vaultPath, '.obsidian', 'plugins', 'paperforge', 'data.json'), + pfJsonPath: path.join(vaultPath, 'paperforge.json'), + configWarning: cfg._warning, + }; +} + // ── Runtime helpers ── function resolvePythonExecutable(vaultPath, settings, _fs, _execFileSync) { @@ -33,6 +83,7 @@ function resolvePythonExecutable(vaultPath, settings, _fs, _execFileSync) { } const systemCandidates = [ + { path: "py", extraArgs: ["-3"] }, { path: "python", extraArgs: [] }, { path: "python3", extraArgs: [] }, ]; @@ -107,7 +158,7 @@ function buildRuntimeInstallCommand(pythonExe, version, extraArgs) { const gitUrl = `git+https://github.com/LLLin000/PaperForge.git@v${version}`; const pypiArgs = [...extraArgs, "-m", "pip", "install", "--upgrade", pypiPkg]; const gitArgs = [...extraArgs, "-m", "pip", "install", "--upgrade", gitUrl]; - return { cmd: pythonExe, pypiArgs, gitArgs, timeout: 120000 }; + return { cmd: pythonExe, url: gitUrl.replace('@v', '@'), args: gitArgs, pypiArgs, gitArgs, timeout: 120000 }; } function parseRuntimeStatus(err, stdout, stderr) { @@ -211,6 +262,8 @@ function shouldRenderVectorReady(vectorDepsOk, embedStatusText) { } module.exports = { + readPathConfig, + resolveRuntimePaths, resolvePythonExecutable, getPluginVersion, checkRuntimeVersion, diff --git a/paperforge/plugin/tests/runtime.test.mjs b/paperforge/plugin/tests/runtime.test.mjs index 93e0da08..16fd964b 100644 --- a/paperforge/plugin/tests/runtime.test.mjs +++ b/paperforge/plugin/tests/runtime.test.mjs @@ -7,11 +7,58 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; const { + readPathConfig, + resolveRuntimePaths, resolvePythonExecutable, getPluginVersion, checkRuntimeVersion, } = await import('../src/testable.js'); +describe('readPathConfig', () => { + it('uses vault_config system_dir when present', () => { + const mockFs = { + existsSync: vi.fn(() => true), + readFileSync: vi.fn(() => JSON.stringify({ vault_config: { system_dir: '99_System' } })), + }; + const cfg = readPathConfig('/vault', mockFs); + expect(cfg.system_dir).toBe('99_System'); + expect(cfg._warning).toBeNull(); + }); + + it('falls back to defaults with warning when config is missing', () => { + const mockFs = { + existsSync: vi.fn(() => false), + readFileSync: vi.fn(), + }; + const cfg = readPathConfig('/vault', mockFs); + expect(cfg.system_dir).toBe('System'); + expect(cfg._warning).toContain('using defaults'); + }); + + it('falls back to defaults with warning when config is invalid', () => { + const mockFs = { + existsSync: vi.fn(() => true), + readFileSync: vi.fn(() => '{bad json'), + }; + const cfg = readPathConfig('/vault', mockFs); + expect(cfg.system_dir).toBe('System'); + expect(cfg._warning).toContain('invalid'); + }); +}); + +describe('resolveRuntimePaths', () => { + it('uses configured system_dir for runtime file paths', () => { + const mockFs = { + existsSync: vi.fn(() => true), + readFileSync: vi.fn(() => JSON.stringify({ vault_config: { system_dir: '99_System' } })), + }; + const paths = resolveRuntimePaths('/vault', mockFs); + expect(paths.memoryStatePath).toContain('99_System'); + expect(paths.exportsDir).toContain('99_System'); + expect(paths.ocrDir).toContain('99_System'); + }); +}); + describe('resolvePythonExecutable', () => { /** Create injected fs + execFileSync mocks */ function mockDeps(existsSyncImpl, execFileSyncImpl) { diff --git a/paperforge/services/sync_service.py b/paperforge/services/sync_service.py index e4c59307..1f058e28 100644 --- a/paperforge/services/sync_service.py +++ b/paperforge/services/sync_service.py @@ -288,6 +288,18 @@ class SyncService: orphaned = self.clean_orphaned_records(exports, paths, json_output=json_output) flat_cleaned = self.clean_flat_notes(paths, json_output=json_output) + if orphaned > 0 or flat_cleaned > 0: + index_count = asset_index.build_index(self.vault, verbose) + + try: + from paperforge.memory.builder import build_from_index + from paperforge.memory.db import get_memory_db_path + + if get_memory_db_path(self.vault).exists(): + build_from_index(self.vault) + except Exception: + pass + if not json_output: print(f"index-refresh: {index_count} entries in index") total = sum(1 for _ in paths["literature"].rglob("*.md")) if paths["literature"].exists() else 0 diff --git a/paperforge/skills/paperforge/SKILL.md b/paperforge/skills/paperforge/SKILL.md index d6185602..48e719a7 100644 --- a/paperforge/skills/paperforge/SKILL.md +++ b/paperforge/skills/paperforge/SKILL.md @@ -98,7 +98,17 @@ Reading-log 不是事实源。它记录的是**之前的关注点、解读和预 ## 6. 意图路由 -用户输入对应唯一一个 workflow 文件（打开并执行其完整流程）： +用户输入分两类处理： + +### A. 机械命令（直接执行并解读结果，不走研究 workflow） + +| 用户说 | 动作 | +| --- | --- | +| `/pf-sync` | 执行 `paperforge sync`，检查结果并解释 | +| `/pf-ocr` | 执行 `paperforge ocr`，检查结果并解释 | +| `/pf-status` | 执行 `paperforge status --json` 或 `paperforge runtime-health --json`，解读状态 | + +### B. 研究工作流（打开唯一 workflow 文件并执行完整流程） | 用户说 | 打开 | | -------------------------------------------------------- | -------------------------------- | @@ -108,10 +118,11 @@ Reading-log 不是事实源。它记录的是**之前的关注点、解读和预 | "记一下" "记录阅读" "reading log" "读完这段记一下" | `workflows/reading-log.md` | | "总结会话" "工作记录" "项目记录" "project log" "记决策" | `workflows/project-log.md` | | "提取方法论" "总结规律" "存档写作规律" | `workflows/methodology.md` | -| "branch" "代码审查" "feature" "dashboard" "memory layer" "用户反馈" "报错" "安装失败" "Git" "Zotero" "BetterBibTeX" "OCR" "插件" | `workflows/project-engineering.md` | +| "branch" "代码审查" "feature" "dashboard" "memory layer" "用户反馈" "报错" "安装失败" "Git" "Zotero" "BetterBibTeX" "插件" | `workflows/project-engineering.md` | | 不确定 / 空输入 | 问用户：搜文献、精读、问答、记笔记、记工作、提方法论？ | 路由后如用户切换意图，重新判断并打开对应 workflow。 +未知或拼错的 `/pf-*` 命令不要静默掉进 `project-engineering`，必须明确提示用户命令不存在或请澄清意图。 --- diff --git a/paperforge/skills/paperforge/workflows/deep-reading.md b/paperforge/skills/paperforge/workflows/deep-reading.md index 10cebc8d..771d07c2 100644 --- a/paperforge/skills/paperforge/workflows/deep-reading.md +++ b/paperforge/skills/paperforge/workflows/deep-reading.md @@ -120,7 +120,7 @@ $PYTHON "$SKILL_DIR/scripts/pf_deep.py" prepare --key --vault "$VAU ### Step 4: Postprocess（跑校验，修正问题） ```bash -$PYTHON "$SKILL_DIR/scripts/pf_deep.py" postprocess-pass2 "" --figures --vault "$VAULT" +$PYTHON "$SKILL_DIR/scripts/pf_deep.py" postprocess-pass2 "" --figures ``` - 输出 `OK` → 继续 Step 5 diff --git a/paperforge/skills/paperforge/workflows/paper-qa.md b/paperforge/skills/paperforge/workflows/paper-qa.md index 98a4f3aa..f7f349dd 100644 --- a/paperforge/skills/paperforge/workflows/paper-qa.md +++ b/paperforge/skills/paperforge/workflows/paper-qa.md @@ -21,21 +21,29 @@ 用户可能给 zotero_key、DOI、标题片段、作者+年份。按以下方式查找： -**优先用 paper-context（一次拿到全部信息）：** +**如果用户给的是 zotero_key：** ```bash -$PYTHON -m paperforge --vault "$VAULT" paper-context --json +$PYTHON -m paperforge --vault "$VAULT" paper-context --json ``` 返回 JSON 包含 paper 元数据、OCR 状态、prior_notes 等。 -**paper-context 无结果时的备选：** +**如果用户给的是 DOI、标题片段、作者+年份，先解析成候选：** + +```bash +$PYTHON -m paperforge --vault "$VAULT" paper-status "" --json +``` + +如果 `paper-status` 直接唯一命中，取返回的 `zotero_key` 再调 `paper-context`。 + +**paper-status 无法唯一定位时的备选：** ```bash $PYTHON -m paperforge --vault "$VAULT" search "" --json --limit 5 ``` -如果多候选，列出让用户选（同 paper-search 的 Step 4-5 格式）。 +如果多候选，列出让用户选（同 paper-search 的 Step 4-5 格式）；用户选定后，再用选中的 `zotero_key` 调 `paper-context`。 如果无结果，告知用户并停止。 ### Step 2: 加载文献内容 diff --git a/paperforge/skills/paperforge/workflows/paper-search.md b/paperforge/skills/paperforge/workflows/paper-search.md index 56803a04..07426a78 100644 --- a/paperforge/skills/paperforge/workflows/paper-search.md +++ b/paperforge/skills/paperforge/workflows/paper-search.md @@ -16,8 +16,8 @@ 提取以下信息（缺什么就问用户）： - **搜索词**：关键词、作者名、年份 -- **范围**：domain（如"骨科"）、collection（如"DC"）、不指定=全库 -- **过滤条件**：OCR 状态（done/pending）、年份范围（--year-from/--year-to）、lifecycle +- **范围**：domain（如"骨科"）、不指定=全库；如果用户说 collection，只把它当结果展示时的二级筛选条件 +- **过滤条件**：OCR 状态（done/pending/failed/processing）、年份范围（--year-from/--year-to）、lifecycle ### Step 2: 执行搜索 @@ -25,8 +25,8 @@ $PYTHON -m paperforge --vault "$VAULT" search --json --limit 15 \ [--domain ""] \ [--year-from ] [--year-to ] \ - [--ocr ] \ - [--lifecycle ] + [--ocr ] \ + [--lifecycle ] ``` 返回 JSON 结构： @@ -46,7 +46,7 @@ $PYTHON -m paperforge --vault "$VAULT" search --json --limit 15 \ "collection_path": "...", "ocr_status": "done", "deep_reading_status": "pending", - "lifecycle": "active", + "lifecycle": "pdf_ready", "has_pdf": true, "rank": "..." } @@ -69,6 +69,7 @@ $PYTHON -m paperforge --vault "$VAULT" paper-context --json ``` 目的：拿到 `ocr_status`、`prior_notes` 数量、`analyze` 状态，帮助用户判断哪些可以直接读。 +如果用户一开始明确要求 collection 范围，在这一步根据 `collection_path` 做结果后筛选，而不是给 `search` 传不存在的 `--collection` 参数。 ### Step 4: 展示候选清单 diff --git a/paperforge/skills/paperforge/workflows/project-log.md b/paperforge/skills/paperforge/workflows/project-log.md index 98bde2af..9106748b 100644 --- a/paperforge/skills/paperforge/workflows/project-log.md +++ b/paperforge/skills/paperforge/workflows/project-log.md @@ -79,7 +79,7 @@ Agent 按 JSON schema 写入 project-log.jsonl。 展示给用户确认后再写入： ``` -即将记录到 Project/综述写作/project-log.md: +即将记录到 Resources/Projects/综述写作/project-log.md: 日期: 2026-05-14 类型: session_summary 标题: DC 段参数窗审计完成 @@ -111,7 +111,7 @@ $PYTHON -m paperforge --vault "$VAULT" project-log --write \ $PYTHON -m paperforge --vault "$VAULT" project-log --render --project "" ``` -输出到 `Project//project-log.md`。 +输出到 `Resources/Projects//project-log.md`。 --- diff --git a/paperforge/skills/paperforge/workflows/reading-log.md b/paperforge/skills/paperforge/workflows/reading-log.md index 5fcf5ccd..8aba3fa8 100644 --- a/paperforge/skills/paperforge/workflows/reading-log.md +++ b/paperforge/skills/paperforge/workflows/reading-log.md @@ -100,7 +100,7 @@ $PYTHON -m paperforge --vault "$VAULT" reading-log --write \ $PYTHON -m paperforge --vault "$VAULT" reading-log --render --project "" ``` -输出到 `Project//reading-log.md`。 +输出到 `Resources/Projects//reading-log.md`。 --- diff --git a/paperforge/worker/asset_index.py b/paperforge/worker/asset_index.py index 1c7b9a17..ba051ccb 100644 --- a/paperforge/worker/asset_index.py +++ b/paperforge/worker/asset_index.py @@ -59,6 +59,46 @@ LOCK_TIMEOUT = 10 ``filelock.Timeout``.""" +def _index_lock(path: Path) -> filelock.FileLock: + return filelock.FileLock(path.with_suffix(".json.lock"), timeout=LOCK_TIMEOUT) + + +def mutate_index(vault: Path, mutator) -> bool: + """Read, mutate, and write the canonical index under one lock.""" + path = get_index_path(vault) + path.parent.mkdir(parents=True, exist_ok=True) + + with _index_lock(path): + if not path.exists(): + return False + try: + with open(path, encoding="utf-8") as fh: + current = json.load(fh) + except (json.JSONDecodeError, OSError) as exc: + logger.warning("Corrupt index at %s during mutation: %s", path, exc) + return False + + updated = mutator(current) + if updated is None: + return False + + tmp = tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", delete=False, dir=path.parent) + try: + json.dump(updated, tmp, ensure_ascii=False, indent=2) + tmp.flush() + os.fsync(tmp.fileno()) + tmp.close() + os.replace(tmp.name, path) + except BaseException: + tmp.close() + try: + os.unlink(tmp.name) + except OSError: + pass + raise + return True + + # --------------------------------------------------------------------------- # Path resolution # --------------------------------------------------------------------------- @@ -123,10 +163,7 @@ def atomic_write_index(path: Path, data: dict) -> None: """ path.parent.mkdir(parents=True, exist_ok=True) - lock_path = path.with_suffix(".json.lock") - lock = filelock.FileLock(lock_path, timeout=LOCK_TIMEOUT) - - with lock: + with _index_lock(path): tmp = None try: tmp = tempfile.NamedTemporaryFile( @@ -587,9 +624,6 @@ def refresh_index_entry(vault: Path, key: str) -> bool: build_index(vault) return False - # Envelope format — incremental refresh - items = existing.get("items", []) - # Find the export item for the requested key config = load_domain_config(paths) domain_lookup = {entry["export_file"]: entry["domain"] for entry in config["domains"]} @@ -623,19 +657,24 @@ def refresh_index_entry(vault: Path, key: str) -> bool: except Exception: pass # memory DB refresh is best-effort - replaced = False - for i, existing_entry in enumerate(items): - if existing_entry.get("zotero_key") == key: - items[i] = new_entry - replaced = True - break - if not replaced: - items.append(new_entry) + def _apply(current): + if is_legacy_format(current) or not isinstance(current, dict): + return None + items = list(current.get("items", [])) + replaced = False + for i, existing_entry in enumerate(items): + if existing_entry.get("zotero_key") == key: + items[i] = new_entry + replaced = True + break + if not replaced: + items.append(new_entry) + return build_envelope(items) + + if not mutate_index(vault, _apply): + build_index(vault) + return False - # Write back atomically - index_path = paths["index"] - envelope = build_envelope(items) - atomic_write_index(index_path, envelope) logger.info("refresh_index_entry: updated entry %s (%s)", key, found_domain) return True diff --git a/paperforge/worker/repair.py b/paperforge/worker/repair.py index c57dea33..b8a93671 100644 --- a/paperforge/worker/repair.py +++ b/paperforge/worker/repair.py @@ -287,25 +287,6 @@ def run_repair(vault: Path, paths: dict, verbose: bool = False, fix: bool = Fals if new_record_text != record_text: record_path.write_text(new_record_text, encoding="utf-8") fixed_formal_note_primary = True - if index_ocr_status is not None: - try: - index_data = read_json(paths["index"]) - if isinstance(index_data, dict): - idx_items = index_data.get("items", []) - elif isinstance(index_data, list): - idx_items = index_data - else: - idx_items = [] - for entry in idx_items: - if isinstance(entry, dict) and entry.get("zotero_key") == zotero_key: - entry["ocr_status"] = new_status - break - if isinstance(index_data, dict): - index_data["items"] = idx_items - write_json(paths["index"], index_data) - fixed_index_entry = True - except Exception as e: - logger.warning("Failed to update index entry for %s: %s", zotero_key, e) if meta_validated_status is not None and meta_validated_status != "done": if meta_path.exists(): try: @@ -328,25 +309,6 @@ def run_repair(vault: Path, paths: dict, verbose: bool = False, fix: bool = Fals if new_record_text != record_text: record_path.write_text(new_record_text, encoding="utf-8") fixed_formal_note_primary = True - if index_ocr_status is not None: - try: - index_data = read_json(paths["index"]) - if isinstance(index_data, dict): - idx_items = index_data.get("items", []) - elif isinstance(index_data, list): - idx_items = index_data - else: - idx_items = [] - for entry in idx_items: - if isinstance(entry, dict) and entry.get("zotero_key") == zotero_key: - entry["ocr_status"] = new_status - break - if isinstance(index_data, dict): - index_data["items"] = idx_items - write_json(paths["index"], index_data) - fixed_index_entry = True - except Exception as e: - logger.warning("Failed to update index entry for %s: %s", zotero_key, e) if meta_path.exists(): try: meta = read_json(meta_path) @@ -370,9 +332,14 @@ def run_repair(vault: Path, paths: dict, verbose: bool = False, fix: bool = Fals logger.info("fixed %d files for %s", fixed_count, zotero_key) if fixed_count > 0: try: - refresh_index_entry(vault, zotero_key) + fixed_index_entry = refresh_index_entry(vault, zotero_key) except Exception as e: logger.warning("Failed to refresh index for %s: %s", zotero_key, e) + else: + if fixed_index_entry: + result["fixed"] += 1 + else: + logger.warning("Failed to update index entry for %s via shared refresh", zotero_key) # Path error detection and repair path_errors = _detect_path_errors(paths, verbose) if path_errors["total"] > 0: diff --git a/tests/cli/test_json_contracts.py b/tests/cli/test_json_contracts.py index d4e4afc7..0381b709 100644 --- a/tests/cli/test_json_contracts.py +++ b/tests/cli/test_json_contracts.py @@ -16,7 +16,7 @@ from .test_contract_helpers import ( class TestPathsJson: """Contract: 'paperforge paths --json' returns valid JSON with correct value types.""" - REQUIRED_KEYS = {"vault", "worker_script", "ld_deep_script"} + REQUIRED_KEYS = {"vault", "worker_script", "pf_deep_script"} def test_paths_json_value_semantics(self, cli_invoker): """paths --json returns paths with correct value types (all non-empty strings).""" diff --git a/tests/test_config.py b/tests/test_config.py index 10c1b92b..04e93486 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -318,7 +318,7 @@ def test_paperforge_paths_returns_exact_keys(tmp_path: Path): "bases", "worker_script", "skill_dir", - "ld_deep_script", + "pf_deep_script", # ── v2.2: canonical locations below paperforge/ ── "config", "index", @@ -370,8 +370,8 @@ def test_paperforge_paths_includes_worker_script(tmp_path: Path): assert paths["worker_script"].name == "__init__.py" -def test_paperforge_paths_includes_ld_deep_script(tmp_path: Path): - """ld_deep_script key must point to ld_deep.py.""" +def test_paperforge_paths_includes_pf_deep_script(tmp_path: Path): + """pf_deep_script key must point to pf_deep.py.""" from paperforge.config import paperforge_paths vault = tmp_path / "vault_ld" @@ -385,8 +385,8 @@ def test_paperforge_paths_includes_ld_deep_script(tmp_path: Path): (vault / ".opencode" / "command").mkdir(parents=True) paths = paperforge_paths(vault) - assert "ld_deep_script" in paths - assert paths["ld_deep_script"].name == "ld_deep.py" + assert "pf_deep_script" in paths + assert paths["pf_deep_script"].name == "pf_deep.py" # --------------------------------------------------------------------------- @@ -413,7 +413,7 @@ def test_paths_as_strings_returns_string_values(): "bases": Path("/some/vault/05_Bases"), "worker_script": Path("/some/vault/99_System/PaperForge/worker/scripts/literature_pipeline.py"), "skill_dir": Path("/some/vault/.opencode/skills"), - "ld_deep_script": Path("/some/vault/.opencode/skills/literature-qa/scripts/ld_deep.py"), + "pf_deep_script": Path("/some/vault/.opencode/skills/paperforge/scripts/pf_deep.py"), } result = paths_as_strings(paths) diff --git a/tests/unit/commands/test_embed.py b/tests/unit/commands/test_embed.py index ab434a11..b2d0c92f 100644 --- a/tests/unit/commands/test_embed.py +++ b/tests/unit/commands/test_embed.py @@ -28,3 +28,36 @@ def test_embed_status_includes_build_state(tmp_path): mock_read.return_value = {"status": "running", "current": 3, "total": 10} result_code = run(args) assert result_code == 0 + + +def test_embed_stop_requests_signal_for_running_job(tmp_path): + from argparse import Namespace + + vault = tmp_path / "vault" + vault.mkdir() + args = Namespace(vault_path=vault, embed_subcommand="stop", json=True) + + with patch("paperforge.commands.embed.read_vector_build_state") as mock_read: + mock_read.return_value = {"status": "running", "pid": 12345} + with patch("paperforge.commands.embed.os.kill") as mock_kill: + with patch("paperforge.commands.embed.mark_vector_build_state") as mock_mark: + result_code = run(args) + + assert result_code == 0 + mock_kill.assert_called_once() + mock_mark.assert_called_once() + + +def test_embed_stop_returns_error_when_signal_fails(tmp_path): + from argparse import Namespace + + vault = tmp_path / "vault" + vault.mkdir() + args = Namespace(vault_path=vault, embed_subcommand="stop", json=True) + + with patch("paperforge.commands.embed.read_vector_build_state") as mock_read: + mock_read.return_value = {"status": "running", "pid": 12345} + with patch("paperforge.commands.embed.os.kill", side_effect=OSError("denied")): + result_code = run(args) + + assert result_code == 1 diff --git a/tests/unit/core/test_io.py b/tests/unit/core/test_io.py new file mode 100644 index 00000000..415e0857 --- /dev/null +++ b/tests/unit/core/test_io.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import json + +from paperforge.core.io import read_json, write_json, write_json_atomic + + +def test_write_json_atomic_creates_parent_dirs(tmp_path): + path = tmp_path / "nested" / "state.json" + + write_json_atomic(path, {"ok": True}) + + assert path.exists() + assert read_json(path) == {"ok": True} + + +def test_write_json_atomic_replaces_existing_file(tmp_path): + path = tmp_path / "state.json" + path.write_text(json.dumps({"old": True}), encoding="utf-8") + + write_json_atomic(path, {"new": True}) + + assert read_json(path) == {"new": True} + assert not path.with_suffix(".json.tmp").exists() + + +def test_write_json_uses_atomic_writer(tmp_path): + path = tmp_path / "state.json" + + write_json(path, {"value": 1}) + + assert read_json(path) == {"value": 1} + assert not path.with_suffix(".json.tmp").exists() diff --git a/tests/unit/memory/test_refresh.py b/tests/unit/memory/test_refresh.py index 103440fa..6c1f457e 100644 --- a/tests/unit/memory/test_refresh.py +++ b/tests/unit/memory/test_refresh.py @@ -15,3 +15,35 @@ def test_refresh_paper_returns_false_for_empty_key(): def test_refresh_paper_returns_false_for_missing_key(): assert refresh_paper(Path("/nonexistent/vault"), {"title": "No Key"}) is False + + +def test_refresh_paper_rebuild_from_index_removes_stale_rows(tmp_path): + from paperforge.memory.builder import build_from_index + from paperforge.memory.db import get_connection, get_memory_db_path + from paperforge.worker.asset_index import build_envelope, get_index_path + + vault = tmp_path / "vault" + vault.mkdir() + (vault / "paperforge.json").write_text( + '{"vault_config":{"system_dir":"System","resources_dir":"Resources","literature_dir":"Literature","base_dir":"Bases","control_dir":"LiteratureControl"}}', + encoding="utf-8", + ) + + index_path = get_index_path(vault) + index_path.parent.mkdir(parents=True, exist_ok=True) + first = [{"zotero_key": "AAA", "title": "Paper A", "domain": "test", "has_pdf": False, "ocr_status": "pending", "analyze": False, "deep_reading_status": "pending", "pdf_path": "", "note_path": "", "main_note_path": "", "fulltext_path": "", "ocr_json_path": "", "ai_path": "", "paper_root": "", "citation_key": "", "doi": "", "journal": "", "first_author": "", "collection_path": "", "collections": [], "authors": [], "abstract": "", "zotero_storage_key": "", "attachment_count": 0, "supplementary": [], "recommend_analyze": False, "path_error": "", "analysis_note": "", "lifecycle": "indexed", "maturity": {"score": 0}, "next_step": "ready"}] + index_path.write_text(__import__("json").dumps(build_envelope(first), ensure_ascii=False, indent=2), encoding="utf-8") + + build_from_index(vault) + + second = [] + index_path.write_text(__import__("json").dumps(build_envelope(second), ensure_ascii=False, indent=2), encoding="utf-8") + build_from_index(vault) + + conn = get_connection(get_memory_db_path(vault), read_only=True) + try: + count = conn.execute("SELECT COUNT(*) AS c FROM papers").fetchone()["c"] + finally: + conn.close() + + assert count == 0