From cd56bd401e64078e479ef7bd8d6a6722eb686f6c Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Sat, 9 May 2026 17:22:11 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20CI=20failures=20=E2=80=94=20stub=20param?= =?UTF-8?q?s,=20ld=5Fdeep=20syntax,=20PFResult=20tests,=20setup=5Fwizard?= =?UTF-8?q?=20plugin=20copy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes 4 CI issues: - test_cli_worker_dispatch: add json_output param to stubs - ld_deep.py: replace Python 3.12+ generic syntax with TypeVar - test_e2e_cli, test_status: update assertions for PFResult envelope - setup_wizard: skip node_modules/src dirs in plugin copy - ci.yml: shell=bash for Windows, pin alls-green@v1.2.2 --- .github/workflows/ci.yml | 3 +- AGENTS.md | 59 +++++-- README.en.md | 36 +++++ README.md | 36 +++++ paperforge/setup_wizard.py | 7 +- .../skills/literature-qa/scripts/ld_deep.py | 5 +- tests/test_cli_worker_dispatch.py | 4 +- tests/test_e2e_cli.py | 145 ++++++++++++++++++ tests/test_status.py | 3 +- 9 files changed, 279 insertions(+), 19 deletions(-) create mode 100644 tests/test_e2e_cli.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e18df20..72799d39 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,6 +116,7 @@ jobs: run: | pip install -e ".[test]" - name: Run unit tests (root tests, not layer-marked) + shell: bash run: | python -m pytest tests/ \ --ignore=tests/sandbox \ @@ -226,7 +227,7 @@ jobs: - e2e-tests runs-on: ubuntu-latest steps: - - uses: re-actors/alls-green@v1 + - uses: re-actors/alls-green@v1.2.2 with: allowed-skips: version-check, plugin-tests jobs: ${{ toJSON(needs) }} diff --git a/AGENTS.md b/AGENTS.md index d0d04fea..9cee92c9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # PaperForge - Agent Guide -> 本文档面向 **安装完成后的新用户** 和 **AI Agent**。如果还没有安装 PaperForge，请通过 Obsidian 插件市场安装，或查看 [README.md](README.md) 中的安装说明。Docs 版本与 v1.4.13 对应。 +> 本文档面向 **安装完成后的新用户** 和 **AI Agent**。如果还没有安装 PaperForge，请通过 Obsidian 插件市场安装，或查看 [README.md](README.md) 中的安装说明。Docs 版本与 v1.4.17rc4 对应。 --- @@ -31,19 +31,36 @@ --- -## 1. 核心架构（Lite 版） +## 1. 核心架构（v2.1 契约驱动） -PaperForge 采用 **两层设计**： +PaperForge 在 v2.1（1.4.17rc4）重构为 **契约驱动架构**，分 5 层： + +``` +CLI/Plugin 调用 + ↓ PFResult 契约 {ok, command, version, data, error} +┌─────────────────────────────────────────────┐ +│ commands/ CLI 分发层 │ +├─────────────────────────────────────────────┤ +│ adapter/bbt, zotero_paths, frontmatter │ ← 独立可测 +│ services/sync_service ← 编排适配器 │ +├─────────────────────────────────────────────┤ +│ core/result, errors, state ← 共享数据契约 │ +├─────────────────────────────────────────────┤ +│ worker/sync, ocr, status ← 机械劳动 │ +│ setup/6个类 ← 安装流程 │ +├─────────────────────────────────────────────┤ +│ schema/field_registry.yaml ← 字段注册表 │ +│ doctor/field_validator.py ← 字段校验 │ +└─────────────────────────────────────────────┘ +``` | 层级 | 组件 | 触发方式 | 作用 | |------|------|----------|------| -| **Worker 层** | `paperforge/worker/`（7 个 worker 模块） | Python CLI | 后台自动化 | -| **Agent 层** | `/pf-deep`, `/pf-paper` 命令 | 用户手动触发 | 交互式精读 | - -**关键区别**： -- **Worker 只做机械劳动**（检测新文献、生成笔记、OCR） -- **Agent 只做深度思考**（精读、分析、写作） -- Worker 不会自动触发 Agent，Agent 不会自动触发 Worker +| **契约层** | `core/`（PFResult, ErrorCode, 状态机） | 被所有模块引用 | 定义 CLI/Plugin/Worker 之间的数据交换格式 | +| **适配器层** | `adapters/`（bbt, zotero_paths, frontmatter） | 被服务层调用 | 封装外部数据格式与 I/O 操作 | +| **服务层** | `services/sync_service` | 被 worker 调度 | 编排适配器，实现业务逻辑 | +| **Worker 层** | `worker/`（sync, ocr, status, repair 等） | Python CLI | 后台自动化（机械劳动） | +| **Agent 层** | `/pf-deep`, `/pf-paper` | 用户手动触发 | 交互式精读（深度思考） | **操作速查**： | 你要做什么 | 在终端输入 | 在 OpenCode 输入 | @@ -522,6 +539,21 @@ cp -r 新下载的代码/* / ## 13. 开发者指南（AI Agent 必读） +### v2.1 新增模块 + +v2.1（1.4.17rc4）引入了以下新增模块，开发时请注意： + +| 路径 | 内容 | 注意事项 | +|------|------|---------| +| `paperforge/core/` | 契约层 — PFResult, ErrorCode, 状态机 | 所有模块都可引用，不循环依赖 | +| `paperforge/adapters/` | 适配器 — bbt, zotero_paths, frontmatter | 独立可测，从 `sync.py` 提取 | +| `paperforge/services/` | 服务层 — SyncService | 编排适配器，`sync.py` 变调度壳 | +| `paperforge/setup/` | 安装层 — 6 个类 | 从 `setup_wizard.py` 拆出 | +| `paperforge/schema/` | 字段注册表 — `field_registry.yaml` | 44 字段定义 | +| `paperforge/doctor/` | 校验 — `field_validator.py` | `doctor` 命令使用 | + +所有 `--json` 输出统一为 PFResult 格式 `{ok, command, version, data, error}`。修改输出结构时需同时更新 `core/result.py` 和下游 consumer（plugin）。 + ### 版本号管理 PaperForge 版本只在 `paperforge/__init__.py` 一处定义。升版本时不要手动改多个文件，使用自动化脚本： @@ -543,11 +575,12 @@ python scripts/bump.py patch --dry-run python scripts/bump.py patch --no-git ``` -脚本会自动更新： + 脚本会自动更新： | 文件 | 字段 | |------|------| | `paperforge/__init__.py` | `__version__` | | `paperforge/plugin/manifest.json` | `version` | +| `manifest.json` | `version` | | `paperforge/plugin/versions.json` | 追加 `"version": "minAppVersion"` | ### 发布 Release 流程 @@ -585,8 +618,8 @@ gh release create v1.4.12 \ # 提交前运行 ruff lint + format + 一致性审计 ruff check --fix paperforge/ && ruff format paperforge/ -# 运行测试（462 tests） -pytest tests/ -q --tb=short +# 运行测试（173 tests） +python -m pytest tests/unit/ -q --tb=short ``` --- diff --git a/README.en.md b/README.en.md index 18e9428a..e5868a7c 100644 --- a/README.en.md +++ b/README.en.md @@ -43,6 +43,42 @@ PaperForge connects the full path from source literature to structured insight. See [INSTALLATION.md](INSTALLATION.md) for the complete installation guide. +## Architecture + +``` +paperforge/ +├── core/ Contract layer — PFResult/PFError, ErrorCode enum, state machine +│ ├── result.py PFResult/PFError serialization (JSON round-trip) +│ ├── errors.py ErrorCode enum (centralized error codes) +│ └── state.py OcrStatus/PdfStatus/Lifecycle + ALLOWED_TRANSITIONS +├── adapters/ Adapter layer — independently testable modules +│ ├── bbt.py Better BibTeX JSON parsing +│ ├── zotero_paths.py Zotero attachment path normalization +│ └── obsidian_frontmatter.py Frontmatter read/write (YAML parser) +├── services/ Service layer — orchestrates adapters +│ └── sync_service.py SyncService class +├── setup/ Setup layer — 6 focused classes +│ ├── plan.py SetupPlan (orchestration) +│ ├── checker.py SetupChecker (precondition validation) +│ ├── config_writer.py ConfigWriter (atomic write) +│ ├── vault.py VaultInitializer (directories/junction) +│ ├── runtime.py RuntimeInstaller (pip install) +│ └── agent.py AgentInstaller (skill deployment) +├── schema/ Field registry +│ └── field_registry.yaml 44 field definitions +├── doctor/ Diagnostic validation +│ └── field_validator.py Field completeness + drift detection +├── worker/ Worker layer — mechanical tasks +│ ├── sync.py Dispatch shell (thinned by 57 lines) +│ ├── status.py Status + doctor checks +│ ├── ocr.py OCR pipeline +│ └── ... +├── commands/ CLI dispatch layer +└── plugin/ Obsidian plugin +``` + +All CLI commands output unified PFResult JSON: `{ok, command, version, data, error}` via `--json` flag. `paperforge doctor` validates field schema consistency and detects data drift. + ## Usage | Action | How | diff --git a/README.md b/README.md index ebf8defb..dccc9953 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,42 @@ PaperForge 不是单纯的 OCR 工具，也不只是 Zotero 到 Obsidian 的搬 完整安装指南请参见 [INSTALLATION.md](INSTALLATION.md)。 +## 架构 + +``` +paperforge/ +├── core/ 契约层 —— PFResult/PFError 数据契约、ErrorCode 枚举、状态机 +│ ├── result.py PFResult/PFError 序列化 +│ ├── errors.py ErrorCode 枚举（集中错误码） +│ └── state.py OcrStatus/PdfStatus/Lifecycle 状态机 + 合法迁移规则 +├── adapters/ 适配器层 —— 独立可测的模块化组件 +│ ├── bbt.py Better BibTeX JSON 解析 +│ ├── zotero_paths.py Zotero 附件路径规范化 +│ └── obsidian_frontmatter.py Frontmatter 读写（YAML 解析器） +├── services/ 服务层 —— 编排适配器 +│ └── sync_service.py SyncService 类 +├── setup/ 安装层 —— 6 个独立类 +│ ├── plan.py SetupPlan（编排） +│ ├── checker.py SetupChecker（前置检查） +│ ├── config_writer.py ConfigWriter（原子写入） +│ ├── vault.py VaultInitializer（目录/链接） +│ ├── runtime.py RuntimeInstaller（pip 安装） +│ └── agent.py AgentInstaller（技能部署） +├── schema/ 字段注册表 +│ └── field_registry.yaml 44 字段定义 +├── doctor/ 诊断校验 +│ └── field_validator.py 字段完整性 + 漂移检测 +├── worker/ 工人层 —— 机械劳动 +│ ├── sync.py 调度壳（-57 行） +│ ├── status.py 状态 + doctor 检查 +│ ├── ocr.py OCR 流水线 +│ └── ... +├── commands/ CLI 分发层 +└── plugin/ Obsidian 插件 +``` + +所有 CLI 命令通过 PFResult 契约输出统一 JSON 格式：`{ok, command, version, data, error}`。`doctor` 可以校验字段注册表一致性，检测数据漂移。 + ## 使用方式 全部核心流程都可以从 Obsidian 内启动。 diff --git a/paperforge/setup_wizard.py b/paperforge/setup_wizard.py index dd1dd243..b542f9f3 100644 --- a/paperforge/setup_wizard.py +++ b/paperforge/setup_wizard.py @@ -902,10 +902,15 @@ def headless_setup( # Obsidian plugin plugin_src = repo_root / "paperforge/plugin" plugin_dst = vault / ".obsidian" / "plugins" / "paperforge" + PLUGIN_FILES = {"main.js", "styles.css", "manifest.json", "versions.json", "i18n.js"} if plugin_src.exists() and plugin_src.is_dir(): created = 0 skipped = 0 - for f in plugin_src.glob("*"): + for name in PLUGIN_FILES: + f = plugin_src / name + if not f.exists(): + skipped += 1 + continue if _copy_file_incremental(f, plugin_dst / f.name): created += 1 else: diff --git a/paperforge/skills/literature-qa/scripts/ld_deep.py b/paperforge/skills/literature-qa/scripts/ld_deep.py index 45a1715b..04f2725c 100644 --- a/paperforge/skills/literature-qa/scripts/ld_deep.py +++ b/paperforge/skills/literature-qa/scripts/ld_deep.py @@ -10,6 +10,7 @@ import re from collections.abc import Iterable from dataclasses import dataclass, field from pathlib import Path +from typing import TypeVar from paperforge.worker._utils import get_analyze_queue @@ -239,7 +240,9 @@ def build_figure_plan( return planned -def select_entries_by_numbers[T](entries: Iterable[T], numbers: set[str], attr: str = "number") -> list[T]: +T = TypeVar("T") + +def select_entries_by_numbers(entries: Iterable[T], numbers: set[str], attr: str = "number") -> list[T]: """Select entries by their number field while preserving order.""" if not numbers: return [] diff --git a/tests/test_cli_worker_dispatch.py b/tests/test_cli_worker_dispatch.py index b4e11904..abb90515 100644 --- a/tests/test_cli_worker_dispatch.py +++ b/tests/test_cli_worker_dispatch.py @@ -18,12 +18,12 @@ def stub_run_status(vault: Path, verbose: bool = False, json_output: bool = Fals return 0 -def stub_run_selection_sync(vault: Path, verbose: bool = False) -> int: +def stub_run_selection_sync(vault: Path, verbose: bool = False, json_output: bool = False) -> int: CAPTURED_CALLS.append(("run_selection_sync", vault)) return 0 -def stub_run_index_refresh(vault: Path, verbose: bool = False, rebuild_index: bool = False) -> int: +def stub_run_index_refresh(vault: Path, verbose: bool = False, rebuild_index: bool = False, json_output: bool = False) -> int: CAPTURED_CALLS.append(("run_index_refresh", vault)) return 0 diff --git a/tests/test_e2e_cli.py b/tests/test_e2e_cli.py new file mode 100644 index 00000000..7fb1f4b9 --- /dev/null +++ b/tests/test_e2e_cli.py @@ -0,0 +1,145 @@ +"""End-to-end CLI tests for PaperForge. + +Invokes paperforge CLI commands as subprocesses (same path as the Obsidian plugin). +Catches bugs in the subprocess boundary that pure Python API tests miss: +- Python interpreter resolution +- Working directory behavior +- JSON stdout parsing +- Exit code propagation +- CLI argument parsing +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + + +def _run(cmd: list[str], cwd: Path) -> subprocess.CompletedProcess: + """Run a CLI command as subprocess and return the result.""" + return subprocess.run( + [sys.executable, "-m", "paperforge"] + cmd, + cwd=cwd, + capture_output=True, + text=True, + timeout=60, + ) + + +class TestCliSync: + """E2E: paperforge sync via subprocess.""" + + def test_sync_runs_cleanly(self, test_vault: Path) -> None: + """Sync completes with exit code 0 and produces index.""" + result = _run(["sync"], test_vault) + assert result.returncode == 0, f"sync failed: {result.stderr[:500]}" + + # Index file should exist + idx = test_vault / "99_System" / "PaperForge" / "indexes" / "formal-library.json" + assert idx.exists(), "index file should exist after sync" + data = json.loads(idx.read_text(encoding="utf-8")) + assert isinstance(data, dict) + assert "items" in data + assert len(data["items"]) > 0 + + def test_sync_produces_formal_note(self, test_vault: Path) -> None: + """After sync, at least one workspace note exists with correct frontmatter.""" + _run(["sync"], test_vault) + + lit_dir = test_vault / "03_Resources" / "Literature" + ws_notes = list(lit_dir.rglob("TSTONE001 - *.md")) + assert len(ws_notes) > 0, "no workspace note found after sync" + + text = ws_notes[0].read_text(encoding="utf-8") + assert "zotero_key:" in text + assert "do_ocr:" in text + assert "ocr_status:" in text + assert "pdf_path:" in text + assert "[[" in text and "]]" in text # wikilink + + +class TestCliStatus: + """E2E: paperforge status --json via subprocess.""" + + def test_status_json_is_valid(self, test_vault: Path) -> None: + """status --json produces valid PFResult with expected keys.""" + _run(["sync"], test_vault) + result = _run(["status", "--json"], test_vault) + + assert result.returncode == 0 + envelope = json.loads(result.stdout) + assert isinstance(envelope, dict) + assert envelope["ok"] is True + assert envelope["command"] == "status" + assert "version" in envelope + assert "total_papers" in envelope["data"] + + def test_status_json_counts_are_correct(self, test_vault: Path) -> None: + """After sync, status reports at least 1 paper.""" + _run(["sync"], test_vault) + result = _run(["status", "--json"], test_vault) + + envelope = json.loads(result.stdout) + assert envelope["data"]["total_papers"] >= 1 + + +class TestCliDoctor: + """E2E: paperforge doctor via subprocess.""" + + def test_doctor_runs_cleanly(self, test_vault: Path) -> None: + """Doctor completes with exit code 0 or 1 (diagnostic, not crash).""" + result = _run(["doctor"], test_vault) + assert result.returncode in (0, 1), f"doctor crashed: {result.stderr[:500]}" + + def test_doctor_outputs_verdict(self, test_vault: Path) -> None: + """Doctor output contains [OK], [WARN], or [FAIL] verdict.""" + result = _run(["doctor"], test_vault) + assert any(tag in result.stdout for tag in ["[OK]", "[WARN]", "[FAIL]"]) + + +class TestCliDeepReading: + """E2E: paperforge deep-reading (queue check) via subprocess.""" + + def test_deep_reading_runs_cleanly(self, test_vault: Path) -> None: + """deep-reading completes without crash.""" + _run(["sync"], test_vault) + result = _run(["deep-reading"], test_vault) + assert result.returncode == 0, f"deep-reading failed: {result.stderr[:500]}" + + +class TestCliFullPipeline: + """E2E: full sync -> status -> doctor pipeline via subprocess.""" + + def test_full_pipeline_consistency(self, test_vault: Path) -> None: + """Run sync, then verify index -> status -> doctor are all consistent.""" + # sync + sync_result = _run(["sync"], test_vault) + assert sync_result.returncode == 0 + + # status --json — read the index the same way the dashboard does + status_result = _run(["status", "--json"], test_vault) + assert status_result.returncode == 0 + envelope = json.loads(status_result.stdout) + assert envelope["data"]["total_papers"] >= 1 + assert envelope["version"] != "" + + # doctor — should produce a verdict (may pass or warn) + doctor_result = _run(["doctor"], test_vault) + assert any(tag in doctor_result.stdout for tag in ["[OK]", "[WARN]", "[FAIL]"]) + + # Verify frontmatter file is readable + lit_dir = test_vault / "03_Resources" / "Literature" + ws_notes = list(lit_dir.rglob("TSTONE001 - *.md")) + assert len(ws_notes) > 0 + text = ws_notes[0].read_text(encoding="utf-8") + assert "has_pdf: true" in text + + def test_missing_vault_does_not_crash(self, tmp_path: Path) -> None: + """Sync in a bare directory does not crash (auto-creates config).""" + result = _run(["sync"], tmp_path) + assert result.returncode in (0, 1), f"unexpected crash: {result.stderr[:500]}" + assert "traceback" not in result.stderr.lower() diff --git a/tests/test_status.py b/tests/test_status.py index 841e5e95..01a66182 100644 --- a/tests/test_status.py +++ b/tests/test_status.py @@ -147,7 +147,8 @@ class TestStatusJsonIndexSource: code = run_status(vault, json_output=True) captured = capsys.readouterr().out assert code == 0 - data = json.loads(captured) + envelope = json.loads(captured) + data = envelope["data"] assert data["lifecycle_level_counts"] == {} assert data["health_aggregate"] == {} assert data["maturity_distribution"] == {}